TDvorak/Trackeep
1
0
Fork 0
mirror of https://github.com/Dvorinka/Trackeep.git synced 2026-06-03 20:12:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

first test

Browse Source
This commit is contained in:
Tomas Dvorak
2026-02-08 14:14:55 +01:00
parent 18aa702174
commit d27cf14110
372 changed files with 98089 additions and 2585 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/ai_recommendation_service.go
+553
View File
@@ -0,0 +1,553 @@
package services
import (
"fmt"
"sort"
"strings"
"time"
"github.com/trackeep/backend/models"
"gorm.io/gorm"
)
// AIRecommendationService provides AI-powered recommendations
type AIRecommendationService struct {
db *gorm.DB
}
// NewAIRecommendationService creates a new AI recommendation service
func NewAIRecommendationService(db *gorm.DB) *AIRecommendationService {
return &AIRecommendationService{db: db}
}
// RecommendationRequest represents a request for recommendations
type RecommendationRequest struct {
UserID uint `json:"user_id"`
RecommendationType string `json:"recommendation_type"` // content, task, learning, connection
Limit int `json:"limit"` // Max recommendations to return
MinConfidence float64 `json:"min_confidence"` // Minimum confidence threshold
IncludeDismissed bool `json:"include_dismissed"` // Include previously dismissed items
Context string `json:"context"` // Current user context
}
// RecommendationScore represents a scored recommendation
type RecommendationScore struct {
Recommendation models.AIRecommendation
Score float64
Reason string
}
// GetRecommendations generates personalized recommendations for a user
func (s *AIRecommendationService) GetRecommendations(req RecommendationRequest) ([]models.AIRecommendation, error) {
// Get user preferences
var prefs models.UserPreference
if err := s.db.Where("user_id = ?", req.UserID).First(&prefs).Error; err != nil {
// Create default preferences if not found
prefs = models.UserPreference{
UserID: req.UserID,
EnableRecommendations: true,
MinConfidenceThreshold: 0.6,
MaxRecommendationsPerDay: 5,
MaxAgeHours: 168,
}
s.db.Create(&prefs)
}
if !prefs.EnableRecommendations {
return []models.AIRecommendation{}, nil
}
// Check daily limit
today := time.Now().Format("2006-01-02")
var todayCount int64
s.db.Model(&models.AIRecommendation{}).
Where("user_id = ? AND DATE(created_at) = ?", req.UserID, today).
Count(&todayCount)
if int(todayCount) >= prefs.MaxRecommendationsPerDay {
return []models.AIRecommendation{}, nil
}
// Generate recommendations based on type
var scoredRecommendations []RecommendationScore
switch req.RecommendationType {
case "content":
scoredRecommendations = s.generateContentRecommendations(req.UserID, &prefs)
case "task":
scoredRecommendations = s.generateTaskRecommendations(req.UserID, &prefs)
case "learning":
scoredRecommendations = s.generateLearningRecommendations(req.UserID, &prefs)
case "connection":
scoredRecommendations = s.generateConnectionRecommendations(req.UserID, &prefs)
default:
// Generate mixed recommendations
scoredRecommendations = s.generateMixedRecommendations(req.UserID, &prefs)
}
// Filter by confidence and dismissed status
minConf := req.MinConfidence
if minConf == 0 {
minConf = prefs.MinConfidenceThreshold
}
var filtered []RecommendationScore
for _, rec := range scoredRecommendations {
if rec.Score >= minConf && (req.IncludeDismissed || !rec.Recommendation.Dismissed) {
// Check if not expired
if rec.Recommendation.ExpiresAt == nil || rec.Recommendation.ExpiresAt.After(time.Now()) {
filtered = append(filtered, rec)
}
}
}
// Sort by score
sort.Slice(filtered, func(i, j int) bool {
return filtered[i].Score > filtered[j].Score
})
// Apply limit
limit := req.Limit
if limit == 0 {
limit = 5
}
if limit > len(filtered) {
limit = len(filtered)
}
// Save recommendations to database
var recommendations []models.AIRecommendation
for i := 0; i < limit; i++ {
rec := filtered[i].Recommendation
rec.Confidence = filtered[i].Score
// Check if already exists
var existing models.AIRecommendation
err := s.db.Where("user_id = ? AND content_type = ? AND content_id = ?",
rec.UserID, rec.ContentType, rec.ContentID).First(&existing).Error
if err == gorm.ErrRecordNotFound {
// Create new recommendation
if err := s.db.Create(&rec).Error; err != nil {
continue
}
recommendations = append(recommendations, rec)
} else {
// Update existing recommendation
existing.Confidence = rec.Confidence
existing.Reasoning = filtered[i].Reason
existing.UpdatedAt = time.Now()
s.db.Save(&existing)
recommendations = append(recommendations, existing)
}
}
return recommendations, nil
}
// generateContentRecommendations generates content-based recommendations
func (s *AIRecommendationService) generateContentRecommendations(userID uint, prefs *models.UserPreference) []RecommendationScore {
var recommendations []RecommendationScore
// Get user's recent activity and interests
userTags := s.getUserInterests(userID)
userCategories := s.getUserCategories(userID)
// Find similar content from other users
var similarContent []struct {
ContentType string
ContentID uint
Title string
URL string
Preview string
Author string
Tags string
Score float64
}
// Query for popular content among similar users
query := `
SELECT
b.content_type,
b.content_id,
b.title,
b.url,
SUBSTRING(b.content, 1, 200) as preview,
u.full_name as author,
b.tags,
COUNT(*) as interaction_count
FROM bookmarks b
INNER JOIN users u ON b.user_id = u.id
WHERE b.user_id != ?
AND b.created_at > ?
AND (b.tags ?| array[?] OR b.content_type = ANY(?))
GROUP BY b.content_type, b.content_id, b.title, b.url, b.content, u.full_name, b.tags
HAVING COUNT(*) > 1
ORDER BY interaction_count DESC
LIMIT 20
`
// This is a simplified version - in practice you'd use more sophisticated similarity algorithms
s.db.Raw(query, userID, time.Now().AddDate(0, -3, 0), userTags, prefs.PreferredContentTypes).Scan(&similarContent)
for _, content := range similarContent {
score := s.calculateContentScore(content, userTags, userCategories, prefs)
expiresAt := time.Now().Add(time.Hour * 24 * 7)
recommendation := models.AIRecommendation{
UserID: userID,
RecommendationType: "content",
ContentType: content.ContentType,
ContentID: &content.ContentID,
Title: content.Title,
Description: fmt.Sprintf("Recommended based on your interests in %s", strings.Join(userTags, ", ")),
ContentTitle: content.Title,
ContentURL: content.URL,
ContentPreview: content.Preview,
AuthorName: content.Author,
Tags: content.Tags,
Priority: s.getPriorityFromScore(score),
ExpiresAt: &expiresAt,
SourceModel: "collaborative_filtering_v1",
}
recommendations = append(recommendations, RecommendationScore{
Recommendation: recommendation,
Score: score,
Reason: fmt.Sprintf("Similar users with interests in %s also liked this", strings.Join(userTags[:2], ", ")),
})
}
return recommendations
}
// generateTaskRecommendations generates task-based recommendations
func (s *AIRecommendationService) generateTaskRecommendations(userID uint, prefs *models.UserPreference) []RecommendationScore {
var recommendations []RecommendationScore
// Get user's task patterns and upcoming deadlines
var userTasks []models.Task
s.db.Where("user_id = ? AND status != 'completed'", userID).Find(&userTasks)
// Get calendar events for context
var upcomingEvents []models.CalendarEvent
s.db.Where("user_id = ? AND start_time BETWEEN ? AND ?",
userID, time.Now(), time.Now().AddDate(0, 0, 7)).Find(&upcomingEvents)
// Analyze patterns and suggest tasks
for _, task := range userTasks {
if string(task.Priority) == "high" && task.DueDate != nil && task.DueDate.Before(time.Now().AddDate(0, 0, 3)) {
// High priority task due soon
score := 0.9
// Convert tags to string
var tagStr string
for i, tag := range task.Tags {
if i > 0 {
tagStr += ","
}
tagStr += tag.Name
}
expiresAt := task.DueDate
recommendation := models.AIRecommendation{
UserID: userID,
RecommendationType: "task",
ContentType: "task",
ContentID: &task.ID,
Title: fmt.Sprintf("Focus on: %s", task.Title),
Description: fmt.Sprintf("This high-priority task is due on %s", task.DueDate.Format("Jan 2")),
ContentTitle: task.Title,
ContentPreview: task.Description,
Tags: tagStr,
Priority: "high",
Confidence: score,
ExpiresAt: expiresAt,
SourceModel: "deadline_priority_v1",
}
recommendations = append(recommendations, RecommendationScore{
Recommendation: recommendation,
Score: score,
})
}
}
return recommendations
}
// generateLearningRecommendations generates learning-based recommendations
func (s *AIRecommendationService) generateLearningRecommendations(userID uint, prefs *models.UserPreference) []RecommendationScore {
var recommendations []RecommendationScore
// Get user's learning progress and interests
var enrollments []models.Enrollment
s.db.Preload("Course").Preload("LearningPath").Where("user_id = ?", userID).Find(&enrollments)
var completedCategories []string
var inProgressCategories []string
for _, enrollment := range enrollments {
if enrollment.Progress >= 100 {
if enrollment.CourseID != nil && enrollment.Course != nil {
completedCategories = append(completedCategories, enrollment.Course.Category)
} else if enrollment.LearningPathID != 0 {
completedCategories = append(completedCategories, enrollment.LearningPath.Category)
}
} else {
if enrollment.CourseID != nil && enrollment.Course != nil {
inProgressCategories = append(inProgressCategories, enrollment.Course.Category)
} else if enrollment.LearningPathID != 0 {
inProgressCategories = append(inProgressCategories, enrollment.LearningPath.Category)
}
}
}
// Recommend next courses based on completed ones
for _, category := range completedCategories {
var nextCourses []models.Course
s.db.Where("category = ? AND level != ?", category, "beginner").Limit(3).Find(&nextCourses)
for _, course := range nextCourses {
score := 0.8
// Convert topics to tags string
var tagsStr string
for i, topic := range course.Topics {
if i > 0 {
tagsStr += ","
}
tagsStr += topic
}
expiresAt := time.Now().Add(time.Hour * 24 * 14)
recommendation := models.AIRecommendation{
UserID: userID,
RecommendationType: "learning",
ContentType: "course",
ContentID: &course.ID,
Title: fmt.Sprintf("Continue learning: %s", course.Title),
Description: fmt.Sprintf("Based on your completion of %s courses", category),
ContentTitle: course.Title,
ContentPreview: course.Description,
Tags: tagsStr,
Priority: "medium",
Confidence: score,
ExpiresAt: &expiresAt,
SourceModel: "learning_path_v1",
}
recommendations = append(recommendations, RecommendationScore{
Recommendation: recommendation,
Score: score,
})
}
}
return recommendations
}
// generateConnectionRecommendations generates user connection recommendations
func (s *AIRecommendationService) generateConnectionRecommendations(userID uint, prefs *models.UserPreference) []RecommendationScore {
var recommendations []RecommendationScore
if !prefs.ConnectionRecommendations {
return recommendations
}
// Get user's skills and interests
var user models.User
s.db.Preload("Skills").Where("id = ?", userID).First(&user)
// Find similar users
var similarUsers []models.User
s.db.Where("id != ? AND (skills ?| array[?] OR location = ?)",
userID, s.getSkillNames(user.Skills), user.Location).Limit(5).Find(&similarUsers)
for _, similarUser := range similarUsers {
score := s.calculateUserSimilarity(&user, &similarUser)
if score > 0.6 {
expiresAt := time.Now().Add(time.Hour * 24 * 30)
recommendation := models.AIRecommendation{
UserID: userID,
RecommendationType: "connection",
ContentType: "user",
ContentID: &similarUser.ID,
Title: fmt.Sprintf("Connect with %s", similarUser.FullName),
Description: fmt.Sprintf("Similar interests in %s", s.getSharedInterests(&user, &similarUser)),
ContentTitle: similarUser.FullName,
ContentPreview: similarUser.Bio,
AuthorName: similarUser.JobTitle,
Priority: "low",
Confidence: score,
ExpiresAt: &expiresAt,
SourceModel: "user_similarity_v1",
}
recommendations = append(recommendations, RecommendationScore{
Recommendation: recommendation,
Score: score,
})
}
}
return recommendations
}
// generateMixedRecommendations generates a mix of all recommendation types
func (s *AIRecommendationService) generateMixedRecommendations(userID uint, prefs *models.UserPreference) []RecommendationScore {
var allRecommendations []RecommendationScore
// Get recommendations from all types
contentRecs := s.generateContentRecommendations(userID, prefs)
taskRecs := s.generateTaskRecommendations(userID, prefs)
learningRecs := s.generateLearningRecommendations(userID, prefs)
connectionRecs := s.generateConnectionRecommendations(userID, prefs)
allRecommendations = append(allRecommendations, contentRecs...)
allRecommendations = append(allRecommendations, taskRecs...)
allRecommendations = append(allRecommendations, learningRecs...)
allRecommendations = append(allRecommendations, connectionRecs...)
// Ensure diverse mix by limiting each type
maxPerType := 2
typeCounts := make(map[string]int)
var mixedRecs []RecommendationScore
for _, rec := range allRecommendations {
if typeCounts[rec.Recommendation.ContentType] < maxPerType {
mixedRecs = append(mixedRecs, rec)
typeCounts[rec.Recommendation.ContentType]++
}
}
return mixedRecs
}
// Helper functions
func (s *AIRecommendationService) getUserInterests(userID uint) []string {
var tags []string
s.db.Model(&models.Bookmark{}).
Select("DISTINCT unnest(string_to_array(tags, ',')) as tag").
Where("user_id = ?", userID).
Pluck("tag", &tags)
return tags
}
func (s *AIRecommendationService) getUserCategories(userID uint) []string {
var categories []string
s.db.Model(&models.Course{}).
Select("DISTINCT category").
Joins("JOIN enrollments ON courses.id = enrollments.course_id").
Where("enrollments.user_id = ?", userID).
Pluck("category", &categories)
return categories
}
func (s *AIRecommendationService) calculateContentScore(content interface{}, userTags, userCategories []string, prefs *models.UserPreference) float64 {
// Simplified scoring algorithm
score := 0.5 // Base score
// Add points for tag matches
// In practice, this would be more sophisticated
score += float64(len(userTags)) * 0.1
// Ensure score is within bounds
if score > 1.0 {
score = 1.0
}
if score < 0.0 {
score = 0.0
}
return score
}
func (s *AIRecommendationService) getPriorityFromScore(score float64) string {
if score >= 0.8 {
return "high"
} else if score >= 0.6 {
return "medium"
}
return "low"
}
func (s *AIRecommendationService) getSkillNames(skills []models.Skill) []string {
var names []string
for _, skill := range skills {
names = append(names, skill.Name)
}
return names
}
func (s *AIRecommendationService) calculateUserSimilarity(user1, user2 *models.User) float64 {
// Simplified similarity calculation
score := 0.0
// Location match
if user1.Location == user2.Location && user1.Location != "" {
score += 0.3
}
// Skills overlap (simplified)
if len(user1.Skills) > 0 && len(user2.Skills) > 0 {
score += 0.4
}
// Random factor for demo
score += 0.2
if score > 1.0 {
score = 1.0
}
return score
}
func (s *AIRecommendationService) getSharedInterests(user1, user2 *models.User) string {
// Simplified - in practice would analyze actual shared interests
interests := []string{"technology", "productivity", "learning"}
if len(interests) > 2 {
return strings.Join(interests[:2], ", ")
}
return strings.Join(interests, ", ")
}
// RecordInteraction records user interaction with recommendations
func (s *AIRecommendationService) RecordInteraction(userID, recommendationID uint, interactionType, context string) error {
interaction := models.RecommendationInteraction{
UserID: userID,
RecommendationID: recommendationID,
InteractionType: interactionType,
Context: context,
SessionID: fmt.Sprintf("session_%d_%d", userID, time.Now().Unix()),
DeviceType: "web", // Would be detected from request
}
// Update recommendation based on interaction
var recommendation models.AIRecommendation
if err := s.db.First(&recommendation, recommendationID).Error; err != nil {
return err
}
switch interactionType {
case "click":
recommendation.Clicked = true
now := time.Now()
recommendation.ClickedAt = &now
case "dismiss":
recommendation.Dismissed = true
now := time.Now()
recommendation.DismissedAt = &now
case "feedback":
// Additional feedback handling would go here
}
s.db.Save(&recommendation)
return s.db.Create(&interaction).Error
}
backend/services/ai_service.go
+532
View File
@@ -0,0 +1,532 @@
package services
import (
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"time"
)
// AIProvider represents different AI providers
type AIProvider string
const (
ProviderMistral AIProvider = "mistral"
ProviderLongCat AIProvider = "longcat"
ProviderGrok AIProvider = "grok"
ProviderDeepSeek AIProvider = "deepseek"
ProviderOllama AIProvider = "ollama"
ProviderOpenRouter AIProvider = "openrouter"
)
// AIRequest represents a generic AI request
type AIRequest struct {
Messages []Message `json:"messages"`
Model string `json:"model"`
MaxTokens int `json:"max_tokens,omitempty"`
Temperature float64 `json:"temperature,omitempty"`
ModelType string `json:"model_type,omitempty"` // "standard", "thinking", "upgraded_thinking"
}
// Message represents a chat message
type Message struct {
Role string `json:"role"`
Content string `json:"content"`
ReasoningContent string `json:"reasoning_content,omitempty"`
}
// AIResponse represents a generic AI response
type AIResponse struct {
ID string `json:"id"`
Object string `json:"object"`
Created int64 `json:"created"`
Model string `json:"model"`
Choices []Choice `json:"choices"`
Usage Usage `json:"usage"`
}
// Choice represents a choice in AI response
type Choice struct {
Index int `json:"index"`
Message Message `json:"message"`
FinishReason string `json:"finish_reason"`
}
// Usage represents token usage
type Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
}
// AIService handles multiple AI providers
type AIService struct {
provider AIProvider
}
// NewAIService creates a new AI service with the specified provider
func NewAIService(provider AIProvider) *AIService {
return &AIService{provider: provider}
}
// GetAvailableProviders returns available AI providers
func GetAvailableProviders() []AIProvider {
// Return all known providers so the frontend can show them in settings
// regardless of current environment configuration. Environment flags
// and API keys still control whether requests actually succeed.
return []AIProvider{
ProviderMistral,
ProviderLongCat,
ProviderGrok,
ProviderDeepSeek,
ProviderOllama,
ProviderOpenRouter,
}
}
// ChatCompletion sends a chat completion request to the configured provider
func (s *AIService) ChatCompletion(req AIRequest) (*AIResponse, error) {
switch s.provider {
case ProviderMistral:
return s.callMistral(req)
case ProviderLongCat:
return s.callLongCat(req)
case ProviderGrok:
return s.callGrok(req)
case ProviderDeepSeek:
return s.callDeepSeek(req)
case ProviderOllama:
return s.callOllama(req)
case ProviderOpenRouter:
return s.callOpenRouter(req)
default:
return nil, fmt.Errorf("unsupported AI provider: %s", s.provider)
}
}
// ChatCompletionWithThinking sends a chat completion request with thinking model
func (s *AIService) ChatCompletionWithThinking(req AIRequest) (*AIResponse, error) {
// Override model with thinking model
thinkingModel := s.getThinkingModel()
if thinkingModel != "" {
req.Model = thinkingModel
}
return s.ChatCompletion(req)
}
// ChatCompletionWithUpgradedThinking sends a chat completion request with upgraded thinking model (LongCat only)
func (s *AIService) ChatCompletionWithUpgradedThinking(req AIRequest) (*AIResponse, error) {
if s.provider != ProviderLongCat {
return nil, fmt.Errorf("upgraded thinking model only available for LongCat provider")
}
// Override model with upgraded thinking model
upgradedModel := os.Getenv("LONGCAT_MODEL_THINKING_UPGRADED")
if upgradedModel != "" {
req.Model = upgradedModel
}
return s.ChatCompletion(req)
}
// ParseThinkingResponse extracts the actual content from thinking model responses
func ParseThinkingResponse(resp *AIResponse, provider AIProvider, modelType string) string {
if provider == ProviderLongCat {
// Handle LongCat thinking models
if resp.Choices[0].Message.Content != "" {
content := resp.Choices[0].Message.Content
// For LongCat-Flash-Thinking, remove thinking tags
if strings.Contains(content, "<longcat_think>") {
// Extract content after thinking tags
parts := strings.Split(content, "</longcat_think>")
if len(parts) > 1 {
return strings.TrimSpace(parts[1])
}
// If no closing tag, try to extract after the thinking content
lines := strings.Split(content, "\n")
for i, line := range lines {
if strings.Contains(line, "</longcat_think>") {
return strings.TrimSpace(strings.Join(lines[i+1:], "\n"))
}
}
}
return content
} else if resp.Choices[0].Message.ReasoningContent != "" {
// For LongCat-Flash-Thinking-2601, check if there's actual content
// This model puts reasoning in reasoning_content and final answer in content
// If content is null, we might need to extract from reasoning or return the reasoning itself
return resp.Choices[0].Message.ReasoningContent
}
}
// For Grok, DeepSeek, Mistral and other providers, or if no special handling needed
return resp.Choices[0].Message.Content
}
// getThinkingModel returns the appropriate thinking model for the provider
func (s *AIService) getThinkingModel() string {
switch s.provider {
case ProviderMistral:
return os.Getenv("MISTRAL_MODEL_THINKING")
case ProviderLongCat:
return os.Getenv("LONGCAT_MODEL_THINKING")
case ProviderGrok:
return os.Getenv("GROK_MODEL_THINKING")
case ProviderDeepSeek:
return os.Getenv("DEEPSEEK_MODEL_THINKING")
case ProviderOllama:
return os.Getenv("OLLAMA_MODEL_THINKING")
case ProviderOpenRouter:
return os.Getenv("OPENROUTER_MODEL_THINKING")
default:
return ""
}
}
// callOpenRouter calls the OpenRouter API (OpenAI-compatible)
func (s *AIService) callOpenRouter(req AIRequest) (*AIResponse, error) {
apiKey := os.Getenv("OPENROUTER_API_KEY")
baseURL := os.Getenv("OPENROUTER_BASE_URL")
if baseURL == "" {
baseURL = "https://openrouter.ai/api"
}
model := os.Getenv("OPENROUTER_MODEL")
if model == "" {
model = "openrouter/auto"
}
if req.Model == "" {
req.Model = model
}
jsonData, err := json.Marshal(req)
if err != nil {
return nil, err
}
httpReq, err := http.NewRequest("POST", baseURL+"/v1/chat/completions", strings.NewReader(string(jsonData)))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/json")
if apiKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("OpenRouter API returned status %d", resp.StatusCode)
}
var orResp AIResponse
if err := json.NewDecoder(resp.Body).Decode(&orResp); err != nil {
return nil, err
}
return &orResp, nil
}
// callMistral calls Mistral AI API
func (s *AIService) callMistral(req AIRequest) (*AIResponse, error) {
apiKey := os.Getenv("MISTRAL_API_KEY")
baseURL := "https://api.mistral.ai/v1"
model := os.Getenv("MISTRAL_MODEL")
if model == "" {
model = "mistral-small-latest"
}
if req.Model == "" {
req.Model = model
}
jsonData, err := json.Marshal(req)
if err != nil {
return nil, err
}
httpReq, err := http.NewRequest("POST", baseURL+"/chat/completions", strings.NewReader(string(jsonData)))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Mistral API returned status %d", resp.StatusCode)
}
var mistralResp AIResponse
if err := json.NewDecoder(resp.Body).Decode(&mistralResp); err != nil {
return nil, err
}
return &mistralResp, nil
}
// callLongCat calls LongCat AI API
func (s *AIService) callLongCat(req AIRequest) (*AIResponse, error) {
apiKey := os.Getenv("LONGCAT_API_KEY")
// Determine format and endpoint
format := os.Getenv("LONGCAT_FORMAT")
if format == "" {
format = "openai" // Default to OpenAI format
}
var baseURL string
switch format {
case "openai":
baseURL = "https://api.longcat.chat/openai"
case "anthropic":
baseURL = "https://api.longcat.chat/anthropic"
default:
baseURL = "https://api.longcat.chat/openai"
}
model := os.Getenv("LONGCAT_MODEL")
if model == "" {
model = "LongCat-Flash-Chat"
}
if req.Model == "" {
req.Model = model
}
var jsonBody []byte
var httpReq *http.Request
var err error
if format == "anthropic" {
// Convert to Anthropic format
anthropicReq := map[string]interface{}{
"model": req.Model,
"max_tokens": req.MaxTokens,
"messages": req.Messages,
}
if req.Temperature > 0 {
anthropicReq["temperature"] = req.Temperature
}
jsonBody, err = json.Marshal(anthropicReq)
if err != nil {
return nil, err
}
httpReq, err = http.NewRequest("POST", baseURL+"/v1/messages", strings.NewReader(string(jsonBody)))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
httpReq.Header.Set("anthropic-version", "2023-06-01")
} else {
// OpenAI format
jsonBody, err = json.Marshal(req)
if err != nil {
return nil, err
}
httpReq, err = http.NewRequest("POST", baseURL+"/v1/chat/completions", strings.NewReader(string(jsonBody)))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
}
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("LongCat API returned status %d", resp.StatusCode)
}
var longcatResp AIResponse
if err := json.NewDecoder(resp.Body).Decode(&longcatResp); err != nil {
return nil, err
}
return &longcatResp, nil
}
// callGrok calls Grok AI API
func (s *AIService) callGrok(req AIRequest) (*AIResponse, error) {
apiKey := os.Getenv("GROK_API_KEY")
baseURL := os.Getenv("GROK_BASE_URL")
if baseURL == "" {
baseURL = "https://api.x.ai/v1"
}
model := os.Getenv("GROK_MODEL")
if model == "" {
model = "grok-beta"
}
if req.Model == "" {
req.Model = model
}
jsonData, err := json.Marshal(req)
if err != nil {
return nil, err
}
httpReq, err := http.NewRequest("POST", baseURL+"/chat/completions", strings.NewReader(string(jsonData)))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Grok API returned status %d", resp.StatusCode)
}
var grokResp AIResponse
if err := json.NewDecoder(resp.Body).Decode(&grokResp); err != nil {
return nil, err
}
return &grokResp, nil
}
// callDeepSeek calls DeepSeek API
func (s *AIService) callDeepSeek(req AIRequest) (*AIResponse, error) {
apiKey := os.Getenv("DEEPSEEK_API_KEY")
baseURL := os.Getenv("DEEPSEEK_BASE_URL")
if baseURL == "" {
baseURL = "https://api.deepseek.com"
}
model := os.Getenv("DEEPSEEK_MODEL")
if model == "" {
model = "deepseek-chat"
}
if req.Model == "" {
req.Model = model
}
jsonData, err := json.Marshal(req)
if err != nil {
return nil, err
}
httpReq, err := http.NewRequest("POST", baseURL+"/chat/completions", strings.NewReader(string(jsonData)))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("DeepSeek API returned status %d", resp.StatusCode)
}
var deepseekResp AIResponse
if err := json.NewDecoder(resp.Body).Decode(&deepseekResp); err != nil {
return nil, err
}
return &deepseekResp, nil
}
// callOllama calls Ollama API
func (s *AIService) callOllama(req AIRequest) (*AIResponse, error) {
baseURL := os.Getenv("OLLAMA_BASE_URL")
if baseURL == "" {
baseURL = "http://localhost:11434"
}
model := os.Getenv("OLLAMA_MODEL")
if model == "" {
model = "llama3.1"
}
if req.Model == "" {
req.Model = model
}
jsonData, err := json.Marshal(req)
if err != nil {
return nil, err
}
httpReq, err := http.NewRequest("POST", baseURL+"/api/chat", strings.NewReader(string(jsonData)))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/json")
client := &http.Client{Timeout: 60 * time.Second} // Ollama can be slower
resp, err := client.Do(httpReq)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Ollama API returned status %d", resp.StatusCode)
}
var ollamaResp AIResponse
if err := json.NewDecoder(resp.Body).Decode(&ollamaResp); err != nil {
return nil, err
}
return &ollamaResp, nil
}
// SetProvider changes the AI provider
func (s *AIService) SetProvider(provider AIProvider) {
s.provider = provider
}
// GetProvider returns the current AI provider
func (s *AIService) GetProvider() AIProvider {
return s.provider
}
backend/services/computer_vision_service.go
+461
View File
@@ -0,0 +1,461 @@
package services
import (
"bytes"
"encoding/base64"
"fmt"
"image"
"regexp"
"strings"
"time"
"github.com/trackeep/backend/models"
"gorm.io/gorm"
)
// ComputerVisionService provides computer vision capabilities
type ComputerVisionService struct {
db *gorm.DB
}
// NewComputerVisionService creates a new computer vision service
func NewComputerVisionService(db *gorm.DB) *ComputerVisionService {
return &ComputerVisionService{db: db}
}
// ImageAnalysisRequest represents a request for image analysis
type ImageAnalysisRequest struct {
ImageData string `json:"image_data" binding:"required"` // Base64 encoded image
AnalysisType string `json:"analysis_type" binding:"required"` // ocr, objects, text, faces, all
FileID *uint `json:"file_id,omitempty"`
}
// ImageAnalysisResponse represents the result of image analysis
type ImageAnalysisResponse struct {
Success bool `json:"success"`
Analysis map[string]interface{} `json:"analysis"`
Text string `json:"text,omitempty"`
Objects []ObjectDetection `json:"objects,omitempty"`
Faces []FaceDetection `json:"faces,omitempty"`
Metadata ImageMetadata `json:"metadata"`
}
// ObjectDetection represents a detected object
type ObjectDetection struct {
Name string `json:"name"`
Confidence float64 `json:"confidence"`
BoundingBox BoundingBox `json:"bounding_box"`
}
// FaceDetection represents a detected face
type FaceDetection struct {
Confidence float64 `json:"confidence"`
BoundingBox BoundingBox `json:"bounding_box"`
Age *int `json:"age,omitempty"`
Gender *string `json:"gender,omitempty"`
Emotion *string `json:"emotion,omitempty"`
}
// BoundingBox represents coordinates of a detected object
type BoundingBox struct {
X int `json:"x"`
Y int `json:"y"`
Width int `json:"width"`
Height int `json:"height"`
}
// ImageMetadata represents metadata about the analyzed image
type ImageMetadata struct {
Width int `json:"width"`
Height int `json:"height"`
Format string `json:"format"`
SizeBytes int `json:"size_bytes"`
ColorSpace string `json:"color_space"`
DominantColors []string `json:"dominant_colors"`
TextDensity float64 `json:"text_density"`
}
// AnalyzeImage performs computer vision analysis on an image
func (s *ComputerVisionService) AnalyzeImage(req ImageAnalysisRequest) (*ImageAnalysisResponse, error) {
// Decode base64 image
imageData, err := base64.StdEncoding.DecodeString(req.ImageData)
if err != nil {
return nil, fmt.Errorf("invalid base64 image data: %v", err)
}
// Parse image to get metadata
img, format, err := image.Decode(bytes.NewReader(imageData))
if err != nil {
return nil, fmt.Errorf("failed to decode image: %v", err)
}
bounds := img.Bounds()
response := &ImageAnalysisResponse{
Success: true,
Analysis: make(map[string]interface{}),
Metadata: ImageMetadata{
Width: bounds.Dx(),
Height: bounds.Dy(),
Format: format,
SizeBytes: len(imageData),
ColorSpace: "RGB", // Simplified
},
}
// Perform requested analysis types
if req.AnalysisType == "ocr" || req.AnalysisType == "all" {
text, err := s.extractText(imageData)
if err == nil {
response.Text = text
response.Analysis["text"] = text
response.Analysis["word_count"] = len(strings.Fields(text))
response.Metadata.TextDensity = float64(len(text)) / float64(bounds.Dx()*bounds.Dy()) * 1000
}
}
if req.AnalysisType == "objects" || req.AnalysisType == "all" {
objects := s.detectObjects(imageData)
response.Objects = objects
response.Analysis["objects"] = objects
response.Analysis["object_count"] = len(objects)
}
if req.AnalysisType == "faces" || req.AnalysisType == "all" {
faces := s.detectFaces(imageData)
response.Faces = faces
response.Analysis["faces"] = faces
response.Analysis["face_count"] = len(faces)
}
if req.AnalysisType == "text" || req.AnalysisType == "all" {
// Extract readable text from image
text, err := s.extractText(imageData)
if err == nil {
response.Analysis["readable_text"] = text
response.Analysis["has_text"] = len(strings.TrimSpace(text)) > 0
}
}
// Extract dominant colors
colors := s.extractDominantColors(imageData)
response.Metadata.DominantColors = colors
// Save analysis to database if file ID is provided
if req.FileID != nil {
s.saveImageAnalysis(*req.FileID, response)
}
return response, nil
}
// extractText performs OCR on the image (simplified implementation)
func (s *ComputerVisionService) extractText(imageData []byte) (string, error) {
// This is a simplified OCR implementation
// In a real implementation, you would use:
// - Tesseract OCR
// - Google Cloud Vision API
// - Azure Computer Vision
// - AWS Textract
// For demo purposes, we'll extract text from common patterns
// This is just a placeholder implementation
// Try to detect common text patterns in the image
// In reality, this would require actual OCR processing
// Simulate OCR by returning sample text based on image analysis
text := `
This is sample OCR text extracted from the image.
In a real implementation, this would contain the actual
text content found in the image using OCR technology.
Common use cases:
- Document scanning
- Receipt processing
- Business card reading
- Screenshot text extraction
`
return strings.TrimSpace(text), nil
}
// detectObjects performs object detection on the image
func (s *ComputerVisionService) detectObjects(imageData []byte) []ObjectDetection {
// This is a simplified object detection implementation
// In a real implementation, you would use:
// - YOLO (You Only Look Once)
// - TensorFlow Object Detection API
// - OpenCV DNN
// - Cloud vision services
// Simulate object detection with common objects
objects := []ObjectDetection{
{
Name: "document",
Confidence: 0.95,
BoundingBox: BoundingBox{X: 10, Y: 10, Width: 300, Height: 400},
},
{
Name: "text",
Confidence: 0.88,
BoundingBox: BoundingBox{X: 20, Y: 30, Width: 280, Height: 200},
},
{
Name: "logo",
Confidence: 0.72,
BoundingBox: BoundingBox{X: 250, Y: 20, Width: 50, Height: 50},
},
}
return objects
}
// detectFaces performs face detection on the image
func (s *ComputerVisionService) detectFaces(imageData []byte) []FaceDetection {
// This is a simplified face detection implementation
// In a real implementation, you would use:
// - OpenCV Face Detection
// - Dlib
// - FaceNet
// - Cloud face detection services
// Simulate face detection
faces := []FaceDetection{
{
Confidence: 0.92,
BoundingBox: BoundingBox{X: 100, Y: 80, Width: 120, Height: 150},
Age: func() *int { age := 28; return &age }(),
Gender: func() *string { gender := "male"; return &gender }(),
Emotion: func() *string { emotion := "happy"; return &emotion }(),
},
}
return faces
}
// extractDominantColors extracts the dominant colors from the image
func (s *ComputerVisionService) extractDominantColors(imageData []byte) []string {
// This is a simplified color extraction
// In a real implementation, you would use:
// - K-means clustering
// - Color histogram analysis
// - Median cut algorithm
// Simulate dominant colors
colors := []string{
"#FFFFFF", // White
"#333333", // Dark gray
"#0066CC", // Blue
"#FF6600", // Orange
"#00CC66", // Green
}
return colors
}
// saveImageAnalysis saves the analysis results to the database
func (s *ComputerVisionService) saveImageAnalysis(fileID uint, analysis *ImageAnalysisResponse) error {
// Convert analysis to JSON for storage
analysisJSON := fmt.Sprintf(`{
"text": "%s",
"object_count": %d,
"face_count": %d,
"metadata": %+v
}`, analysis.Text, len(analysis.Objects), len(analysis.Faces), analysis.Metadata)
// Create or update file analysis record
var fileAnalysis models.FileAnalysis
err := s.db.Where("file_id = ?", fileID).First(&fileAnalysis).Error
if err == gorm.ErrRecordNotFound {
// Create new analysis record
now := time.Now()
fileAnalysis = models.FileAnalysis{
FileID: fileID,
AnalysisType: "computer_vision",
Results: analysisJSON,
Confidence: 0.85,
ProcessedAt: &now,
}
return s.db.Create(&fileAnalysis).Error
} else if err == nil {
// Update existing record
fileAnalysis.Results = analysisJSON
now := time.Now()
fileAnalysis.ProcessedAt = &now
return s.db.Save(&fileAnalysis).Error
}
return err
}
// ProcessDocumentImage processes a document image for text extraction and structure
func (s *ComputerVisionService) ProcessDocumentImage(imageData []byte) (*DocumentAnalysis, error) {
// Extract text using OCR
text, err := s.extractText(imageData)
if err != nil {
return nil, err
}
// Analyze document structure
analysis := &DocumentAnalysis{
Text: text,
WordCount: len(strings.Fields(text)),
LineCount: len(strings.Split(text, "\n")),
Language: s.detectLanguage(text),
DocumentType: s.detectDocumentType(text),
Sections: s.extractSections(text),
Tables: s.extractTables(text),
Links: s.extractLinks(text),
Emails: s.extractEmails(text),
PhoneNumbers: s.extractPhoneNumbers(text),
}
return analysis, nil
}
// DocumentAnalysis represents the analysis of a document image
type DocumentAnalysis struct {
Text string `json:"text"`
WordCount int `json:"word_count"`
LineCount int `json:"line_count"`
Language string `json:"language"`
DocumentType string `json:"document_type"`
Sections []DocumentSection `json:"sections"`
Tables []DocumentTable `json:"tables"`
Links []string `json:"links"`
Emails []string `json:"emails"`
PhoneNumbers []string `json:"phone_numbers"`
}
// DocumentSection represents a section in a document
type DocumentSection struct {
Title string `json:"title"`
Content string `json:"content"`
Level int `json:"level"`
}
// DocumentTable represents a table in a document
type DocumentTable struct {
Headers []string `json:"headers"`
Rows [][]string `json:"rows"`
}
// detectLanguage detects the language of the text
func (s *ComputerVisionService) detectLanguage(text string) string {
// Simplified language detection
// In a real implementation, you would use:
// - Language detection libraries
// - Machine learning models
// - Cloud language detection services
if strings.Contains(strings.ToLower(text), "the") && strings.Contains(strings.ToLower(text), "and") {
return "en"
} else if strings.Contains(text, "est") && strings.Contains(text, "que") {
return "es"
} else if strings.Contains(text, "und") && strings.Contains(text, "der") {
return "de"
}
return "unknown"
}
// detectDocumentType detects the type of document
func (s *ComputerVisionService) detectDocumentType(text string) string {
text = strings.ToLower(text)
if strings.Contains(text, "invoice") || strings.Contains(text, "bill") {
return "invoice"
} else if strings.Contains(text, "receipt") || strings.Contains(text, "purchase") {
return "receipt"
} else if strings.Contains(text, "resume") || strings.Contains(text, "curriculum") {
return "resume"
} else if strings.Contains(text, "contract") || strings.Contains(text, "agreement") {
return "contract"
} else if strings.Contains(text, "report") || strings.Contains(text, "analysis") {
return "report"
}
return "general"
}
// extractSections extracts document sections
func (s *ComputerVisionService) extractSections(text string) []DocumentSection {
var sections []DocumentSection
lines := strings.Split(text, "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
// Simple section detection (headers followed by content)
if len(line) < 100 && (strings.HasSuffix(line, ":") || strings.ToUpper(line) == line) {
sections = append(sections, DocumentSection{
Title: line,
Content: "",
Level: 1,
})
}
}
return sections
}
// extractTables extracts tables from the text
func (s *ComputerVisionService) extractTables(text string) []DocumentTable {
// Simplified table extraction
// In a real implementation, this would be much more sophisticated
var tables []DocumentTable
// Look for tabular data patterns
lines := strings.Split(text, "\n")
for i, line := range lines {
if strings.Contains(line, "\t") || strings.Contains(line, " ") {
// Potential table row
if i > 0 && strings.Contains(lines[i-1], "\t") {
// Multiple consecutive rows with tabs - likely a table
table := DocumentTable{
Headers: strings.Split(lines[i-1], "\t"),
Rows: [][]string{strings.Split(line, "\t")},
}
tables = append(tables, table)
}
}
}
return tables
}
// extractLinks extracts URLs from the text
func (s *ComputerVisionService) extractLinks(text string) []string {
urlRegex := regexp.MustCompile(`https?://[^\s]+`)
return urlRegex.FindAllString(text, -1)
}
// extractEmails extracts email addresses from the text
func (s *ComputerVisionService) extractEmails(text string) []string {
emailRegex := regexp.MustCompile(`[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}`)
return emailRegex.FindAllString(text, -1)
}
// extractPhoneNumbers extracts phone numbers from the text
func (s *ComputerVisionService) extractPhoneNumbers(text string) []string {
phoneRegex := regexp.MustCompile(`\b\d{3}[-.]?\d{3}[-.]?\d{4}\b`)
return phoneRegex.FindAllString(text, -1)
}
// CreateFileAnalysis creates a file analysis record
func (s *ComputerVisionService) CreateFileAnalysis(fileID uint, analysisType, results string, confidence float64) error {
now := time.Now()
fileAnalysis := models.FileAnalysis{
FileID: fileID,
AnalysisType: analysisType,
Results: results,
Confidence: confidence,
ProcessedAt: &now,
}
return s.db.Create(&fileAnalysis).Error
}
backend/services/metadata.go
+328
View File
@@ -0,0 +1,328 @@
package services
import (
"crypto/md5"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
)
// WebsiteMetadata represents extracted website information
type WebsiteMetadata struct {
Title string `json:"title"`
Description string `json:"description"`
Favicon string `json:"favicon"`
SiteName string `json:"site_name"`
Image string `json:"image"`
Author string `json:"author"`
PublishedAt string `json:"published_at"`
}
// FetchWebsiteMetadata extracts metadata from a URL
func FetchWebsiteMetadata(targetURL string) (*WebsiteMetadata, error) {
// Parse URL to ensure it's valid
parsedURL, err := url.Parse(targetURL)
if err != nil {
return nil, fmt.Errorf("invalid URL: %w", err)
}
// Create HTTP client with timeout
client := &http.Client{
Timeout: 10 * time.Second,
}
// Make request
req, err := http.NewRequest("GET", targetURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
// Set user agent to avoid being blocked
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch URL: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status)
}
// Read response body
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body: %w", err)
}
content := string(body)
metadata := &WebsiteMetadata{}
// Extract Open Graph and Twitter Card metadata
metadata = extractOpenGraphMetadata(content, metadata)
metadata = extractTwitterMetadata(content, metadata)
metadata = extractBasicHTMLMetadata(content, metadata)
// Extract favicon
if metadata.Favicon == "" {
metadata.Favicon = extractFavicon(content, parsedURL)
}
// If still no favicon, try default locations
if metadata.Favicon == "" {
metadata.Favicon = getDefaultFavicon(parsedURL)
}
return metadata, nil
}
// extractOpenGraphMetadata extracts Open Graph meta tags
func extractOpenGraphMetadata(content string, metadata *WebsiteMetadata) *WebsiteMetadata {
// This is a simple implementation - in production, you might want to use a proper HTML parser
ogPatterns := map[string]string{
`<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']`: "Title",
`<meta[^>]+property=["']og:description["'][^>]+content=["']([^"']+)["']`: "Description",
`<meta[^>]+property=["']og:image["'][^>]+content=["']([^"']+)["']`: "Image",
`<meta[^>]+property=["']og:site_name["'][^>]+content=["']([^"']+)["']`: "SiteName",
`<meta[^>]+property=["']article:author["'][^>]+content=["']([^"']+)["']`: "Author",
`<meta[^>]+property=["']article:published_time["'][^>]+content=["']([^"']+)["']`: "PublishedAt",
}
for pattern, field := range ogPatterns {
if re := regexp.MustCompile(pattern); re != nil {
if matches := re.FindStringSubmatch(content); len(matches) > 1 {
switch field {
case "Title":
metadata.Title = matches[1]
case "Description":
metadata.Description = matches[1]
case "Image":
metadata.Image = matches[1]
case "SiteName":
metadata.SiteName = matches[1]
case "Author":
metadata.Author = matches[1]
case "PublishedAt":
metadata.PublishedAt = matches[1]
}
}
}
}
return metadata
}
// extractTwitterMetadata extracts Twitter Card meta tags
func extractTwitterMetadata(content string, metadata *WebsiteMetadata) *WebsiteMetadata {
twitterPatterns := map[string]string{
`<meta[^>]+name=["']twitter:title["'][^>]+content=["']([^"']+)["']`: "Title",
`<meta[^>]+name=["']twitter:description["'][^>]+content=["']([^"']+)["']`: "Description",
`<meta[^>]+name=["']twitter:image["'][^>]+content=["']([^"']+)["']`: "Image",
`<meta[^>]+name=["']twitter:site["'][^>]+content=["']([^"']+)["']`: "SiteName",
`<meta[^>]+name=["']twitter:creator["'][^>]+content=["']([^"']+)["']`: "Author",
}
for pattern, field := range twitterPatterns {
if re := regexp.MustCompile(pattern); re != nil {
if matches := re.FindStringSubmatch(content); len(matches) > 1 {
// Only set if not already set by Open Graph
switch field {
case "Title":
if metadata.Title == "" {
metadata.Title = matches[1]
}
case "Description":
if metadata.Description == "" {
metadata.Description = matches[1]
}
case "Image":
if metadata.Image == "" {
metadata.Image = matches[1]
}
case "SiteName":
if metadata.SiteName == "" {
metadata.SiteName = matches[1]
}
case "Author":
if metadata.Author == "" {
metadata.Author = matches[1]
}
}
}
}
}
return metadata
}
// extractBasicHTMLMetadata extracts basic HTML title and description
func extractBasicHTMLMetadata(content string, metadata *WebsiteMetadata) *WebsiteMetadata {
// Extract title
if metadata.Title == "" {
if re := regexp.MustCompile(`<title[^>]*>([^<]+)</title>`); re != nil {
if matches := re.FindStringSubmatch(content); len(matches) > 1 {
metadata.Title = strings.TrimSpace(matches[1])
}
}
}
// Extract description meta tag
if metadata.Description == "" {
if re := regexp.MustCompile(`<meta[^>]+name=["']description["'][^>]+content=["']([^"']+)["']`); re != nil {
if matches := re.FindStringSubmatch(content); len(matches) > 1 {
metadata.Description = matches[1]
}
}
}
return metadata
}
// extractFavicon extracts favicon from HTML with enhanced detection
func extractFavicon(content string, baseURL *url.URL) string {
// Enhanced patterns for favicon detection
patterns := []string{
// Standard favicon link tags
`<link[^>]+rel=["'](?:icon|shortcut icon)["'][^>]+href=["']([^"']+)["']`,
`<link[^>]+href=["']([^"']+)["'][^>]+rel=["'](?:icon|shortcut icon)["']`,
// Apple touch icons
`<link[^>]+rel=["']apple-touch-icon["'][^>]+href=["']([^"']+)["']`,
`<link[^>]+href=["']([^"']+)["'][^>]+rel=["']apple-touch-icon["']`,
// Apple touch icon precomposed
`<link[^>]+rel=["']apple-touch-icon-precomposed["'][^>]+href=["']([^"']+)["']`,
`<link[^>]+href=["']([^"']+)["'][^>]+rel=["']apple-touch-icon-precomposed["']`,
// Android icons
`<link[^>]+rel=["']android-chrome-[\w\-\d]+["'][^>]+href=["']([^"']+)["']`,
`<link[^>]+href=["']([^"']+)["'][^>]+rel=["']android-chrome-[\w\-\d]+["']`,
// Microsoft tiles
`<meta[^>]+name=["']msapplication-TileImage["'][^>]+content=["']([^"']+)["']`,
// Open Graph image (can be used as logo)
`<meta[^>]+property=["']og:image["'][^>]+content=["']([^"']+)["']`,
// Twitter image
`<meta[^>]+name=["']twitter:image["'][^>]+content=["']([^"']+)["']`,
// Logo patterns
`<link[^>]+rel=["']logo["'][^>]+href=["']([^"']+)["']`,
`<link[^>]+href=["']([^"']+)["'][^>]+rel=["']logo["']`,
}
for _, pattern := range patterns {
if re := regexp.MustCompile(pattern); re != nil {
if matches := re.FindStringSubmatch(content); len(matches) > 1 {
href := matches[1]
// Convert relative URL to absolute
if strings.HasPrefix(href, "/") {
return baseURL.Scheme + "://" + baseURL.Host + href
} else if !strings.HasPrefix(href, "http") {
return baseURL.Scheme + "://" + baseURL.Host + "/" + href
}
return href
}
}
}
return ""
}
// getDefaultFavicon tries common favicon locations with enhanced detection
func getDefaultFavicon(baseURL *url.URL) string {
commonPaths := []string{
"/favicon.ico",
"/favicon.png",
"/favicon.svg",
"/apple-touch-icon.png",
"/apple-touch-icon-precomposed.png",
"/android-chrome-192x192.png",
"/icon-192x192.png",
"/touch-icon-192x192.png",
"/logo.png",
"/logo.svg",
"/assets/favicon.ico",
"/assets/favicon.png",
"/static/favicon.ico",
"/static/favicon.png",
"/images/favicon.ico",
"/images/favicon.png",
}
for _, path := range commonPaths {
faviconURL := baseURL.Scheme + "://" + baseURL.Host + path
// Check if favicon exists with a quick HEAD request
if resp, err := http.Head(faviconURL); err == nil && resp.StatusCode == http.StatusOK {
// Check content type to ensure it's an image
contentType := resp.Header.Get("Content-Type")
if strings.HasPrefix(contentType, "image/") {
return faviconURL
}
}
}
// Try to find high-resolution favicons from common CDNs
host := baseURL.Host
if !strings.Contains(host, "www.") {
host = "www." + host
}
// Try Google's favicon service with higher resolution
return fmt.Sprintf("https://www.google.com/s2/favicons?domain=%s&sz=128", baseURL.Host)
}
// CacheService handles caching of metadata
type CacheService struct {
cache map[string]*WebsiteMetadata
}
func NewCacheService() *CacheService {
return &CacheService{
cache: make(map[string]*WebsiteMetadata),
}
}
func (cs *CacheService) Get(key string) (*WebsiteMetadata, bool) {
if metadata, exists := cs.cache[key]; exists {
return metadata, true
}
return nil, false
}
func (cs *CacheService) Set(key string, metadata *WebsiteMetadata) {
cs.cache[key] = metadata
}
// Global cache instance
var metadataCache = NewCacheService()
// GetCachedMetadata fetches metadata with caching
func GetCachedMetadata(url string) (*WebsiteMetadata, error) {
// Create cache key
cacheKey := fmt.Sprintf("%x", md5.Sum([]byte(url)))
// Try to get from cache
if metadata, exists := metadataCache.Get(cacheKey); exists {
return metadata, nil
}
// Fetch fresh metadata
metadata, err := FetchWebsiteMetadata(url)
if err != nil {
return nil, err
}
// Cache the result
metadataCache.Set(cacheKey, metadata)
return metadata, nil
}
backend/services/performance.go
+228
View File
@@ -0,0 +1,228 @@
package services
import (
"fmt"
"time"
"gorm.io/gorm"
)
type PerformanceService struct {
db *gorm.DB
}
func NewPerformanceService(db *gorm.DB) *PerformanceService {
return &PerformanceService{
db: db,
}
}
// OptimizeDatabase performs database optimizations
func (s *PerformanceService) OptimizeDatabase() error {
// Create indexes for frequently queried fields
indexes := []string{
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_email ON users(email)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_username ON users(username)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_bookmarks_user_id ON bookmarks(user_id)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_bookmarks_created_at ON bookmarks(created_at)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tasks_user_id ON tasks(user_id)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tasks_status ON tasks(status)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_tasks_created_at ON tasks(created_at)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_notes_user_id ON notes(user_id)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_notes_is_public ON notes(is_public)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_files_user_id ON files(user_id)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_files_created_at ON files(created_at)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_time_entries_user_id ON time_entries(user_id)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_time_entries_created_at ON time_entries(created_at)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_audit_logs_user_id ON audit_logs(user_id)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_audit_logs_created_at ON audit_logs(created_at)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_audit_logs_action ON audit_logs(action)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_audit_logs_resource ON audit_logs(resource)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_marketplace_items_status ON marketplace_items(status)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_marketplace_items_category ON marketplace_items(category)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_challenges_status ON challenges(status)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_challenges_start_date ON challenges(start_date)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_challenge_participants_user_id ON challenge_participants(user_id)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_challenge_participants_challenge_id ON challenge_participants(challenge_id)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mentorships_status ON mentorships(status)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mentorships_mentor_id ON mentorships(mentor_id)",
"CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_mentorships_mentee_id ON mentorships(mentee_id)",
}
for _, indexSQL := range indexes {
if err := s.db.Exec(indexSQL).Error; err != nil {
// Log error but continue with other indexes
fmt.Printf("Failed to create index: %s, error: %v\n", indexSQL, err)
}
}
// Analyze tables to update statistics
tables := []string{
"users", "bookmarks", "tasks", "notes", "files",
"time_entries", "audit_logs", "marketplace_items",
"challenges", "challenge_participants", "mentorships",
}
for _, table := range tables {
if err := s.db.Exec(fmt.Sprintf("ANALYZE %s", table)).Error; err != nil {
fmt.Printf("Failed to analyze table %s: %v\n", table, err)
}
}
return nil
}
// CleanupOldAuditLogs removes old audit logs to maintain performance
func (s *PerformanceService) CleanupOldAuditLogs(retentionDays int) error {
cutoffDate := time.Now().AddDate(0, 0, -retentionDays)
result := s.db.Where("created_at < ?", cutoffDate).Delete(&struct{}{})
if result.Error != nil {
return result.Error
}
fmt.Printf("Cleaned up %d old audit log entries\n", result.RowsAffected)
return nil
}
// GetDatabaseStats returns database performance statistics
func (s *PerformanceService) GetDatabaseStats() (map[string]interface{}, error) {
stats := make(map[string]interface{})
// Get table sizes
var tableStats []struct {
TableName string `json:"table_name"`
RowCount int64 `json:"row_count"`
}
query := `
SELECT
table_name as table_name,
n_tup_ins - n_tup_del as row_count
FROM pg_stat_user_tables
ORDER BY n_tup_ins - n_tup_del DESC
LIMIT 10
`
if err := s.db.Raw(query).Scan(&tableStats).Error; err != nil {
return nil, err
}
stats["table_sizes"] = tableStats
// Get slow queries (if pg_stat_statements is available)
var slowQueries []struct {
Query string `json:"query"`
Calls int64 `json:"calls"`
TotalTime float64 `json:"total_time"`
MeanTime float64 `json:"mean_time"`
}
slowQuerySQL := `
SELECT
query,
calls,
total_time,
mean_time
FROM pg_stat_statements
ORDER BY mean_time DESC
LIMIT 10
`
if err := s.db.Raw(slowQuerySQL).Scan(&slowQueries).Error; err == nil {
stats["slow_queries"] = slowQueries
}
// Get cache hit ratio
var cacheStats struct {
HitRatio float64 `json:"hit_ratio"`
}
cacheSQL := `
SELECT
ROUND(sum(heap_blks_hit)::numeric /
(sum(heap_blks_hit) + sum(heap_blks_read)), 4) as hit_ratio
FROM pg_statio_user_tables
`
if err := s.db.Raw(cacheSQL).Scan(&cacheStats).Error; err == nil {
stats["cache_hit_ratio"] = cacheStats.HitRatio
}
return stats, nil
}
// OptimizeQueries optimizes common query patterns
func (s *PerformanceService) OptimizeQueries() error {
// Enable query plan caching
if err := s.db.Exec("SET plan_cache_mode = force_generic_plan").Error; err != nil {
fmt.Printf("Failed to set plan cache mode: %v\n", err)
}
// Set appropriate work_mem for sorting operations
if err := s.db.Exec("SET work_mem = '16MB'").Error; err != nil {
fmt.Printf("Failed to set work_mem: %v\n", err)
}
// Set maintenance_work_mem for index creation
if err := s.db.Exec("SET maintenance_work_mem = '64MB'").Error; err != nil {
fmt.Printf("Failed to set maintenance_work_mem: %v\n", err)
}
// Enable parallel query processing
if err := s.db.Exec("SET max_parallel_workers_per_gather = 2").Error; err != nil {
fmt.Printf("Failed to set max_parallel_workers_per_gather: %v\n", err)
}
return nil
}
// MonitorPerformance monitors system performance
func (s *PerformanceService) MonitorPerformance() (map[string]interface{}, error) {
stats := make(map[string]interface{})
// Database connections
var dbStats struct {
ActiveConnections int `json:"active_connections"`
MaxConnections int `json:"max_connections"`
}
if err := s.db.Raw("SELECT count(*) as active_connections FROM pg_stat_activity WHERE state = 'active'").Scan(&dbStats.ActiveConnections).Error; err == nil {
s.db.Raw("SHOW max_connections").Scan(&dbStats.MaxConnections)
stats["database_connections"] = dbStats
}
// Redis stats (if available) - currently not implemented
stats["redis_info"] = "Redis not configured"
// Memory usage
var memoryStats struct {
TotalMemory int64 `json:"total_memory"`
UsedMemory int64 `json:"used_memory"`
}
if err := s.db.Raw("SELECT setting::int * 1024 * 1024 as total_memory FROM pg_settings WHERE name = 'shared_buffers'").Scan(&memoryStats.TotalMemory).Error; err == nil {
stats["memory_usage"] = memoryStats
}
return stats, nil
}
// WarmupCache preloads frequently accessed data into cache
// Currently not implemented - requires Redis or other caching solution
func (s *PerformanceService) WarmupCache() error {
// TODO: Implement caching when Redis is added
return nil
}
// ClearCache clears all cache entries
// Currently not implemented - requires Redis or other caching solution
func (s *PerformanceService) ClearCache() error {
// TODO: Implement cache clearing when Redis is added
return nil
}
// GetCacheStats returns cache performance statistics
// Currently not implemented - requires Redis or other caching solution
func (s *PerformanceService) GetCacheStats() (map[string]interface{}, error) {
return map[string]interface{}{"status": "cache_not_implemented"}, nil
}
backend/services/video_bookmark.go
+277
View File
@@ -0,0 +1,277 @@
package services
import (
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"time"
"github.com/trackeep/backend/models"
"gorm.io/gorm"
)
// VideoBookmarkService handles video bookmark operations
type VideoBookmarkService struct {
db *gorm.DB
}
// NewVideoBookmarkService creates a new video bookmark service
func NewVideoBookmarkService(db *gorm.DB) *VideoBookmarkService {
return &VideoBookmarkService{db: db}
}
// VideoInfo represents video information from scraper
type VideoInfo struct {
VideoID string `json:"video_id"`
Title string `json:"title"`
Channel string `json:"channel"`
Thumbnail string `json:"thumbnail_url"`
Success bool `json:"success"`
Error string `json:"error,omitempty"`
}
// SaveVideoRequest represents the request to save a video
type SaveVideoRequest struct {
URL string `json:"url" binding:"required"`
Description string `json:"description"`
Tags string `json:"tags"`
IsFavorite bool `json:"is_favorite"`
}
// SaveVideoBookmark saves a video bookmark
func (vbs *VideoBookmarkService) SaveVideoBookmark(userID uint, req SaveVideoRequest) (*models.VideoBookmark, error) {
// Extract video info using scraper
videoInfo, err := vbs.extractVideoInfo(req.URL)
if err != nil {
return nil, fmt.Errorf("failed to extract video info: %w", err)
}
if !videoInfo.Success {
return nil, fmt.Errorf("scraper error: %s", videoInfo.Error)
}
// Check if video already bookmarked by this user
var existingBookmark models.VideoBookmark
if err := vbs.db.Where("user_id = ? AND video_id = ?", userID, videoInfo.VideoID).First(&existingBookmark).Error; err == nil {
return nil, fmt.Errorf("video already bookmarked")
}
// Create bookmark
bookmark := models.VideoBookmark{
VideoID: videoInfo.VideoID,
Title: videoInfo.Title,
Channel: videoInfo.Channel,
Thumbnail: videoInfo.Thumbnail,
URL: req.URL,
UserID: userID,
Description: req.Description,
Tags: req.Tags,
IsFavorite: req.IsFavorite,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
if err := vbs.db.Create(&bookmark).Error; err != nil {
return nil, fmt.Errorf("failed to save bookmark: %w", err)
}
return &bookmark, nil
}
// GetUserBookmarks gets all bookmarks for a user
func (vbs *VideoBookmarkService) GetUserBookmarks(userID uint, limit int, offset int) ([]models.VideoBookmark, error) {
var bookmarks []models.VideoBookmark
query := vbs.db.Where("user_id = ?", userID).Order("created_at DESC")
if limit > 0 {
query = query.Limit(limit)
}
if offset > 0 {
query = query.Offset(offset)
}
if err := query.Find(&bookmarks).Error; err != nil {
return nil, fmt.Errorf("failed to get bookmarks: %w", err)
}
return bookmarks, nil
}
// GetBookmarkByID gets a bookmark by ID
func (vbs *VideoBookmarkService) GetBookmarkByID(userID uint, bookmarkID uint) (*models.VideoBookmark, error) {
var bookmark models.VideoBookmark
if err := vbs.db.Where("id = ? AND user_id = ?", bookmarkID, userID).First(&bookmark).Error; err != nil {
return nil, fmt.Errorf("bookmark not found: %w", err)
}
return &bookmark, nil
}
// UpdateBookmark updates a bookmark
func (vbs *VideoBookmarkService) UpdateBookmark(userID uint, bookmarkID uint, req SaveVideoRequest) (*models.VideoBookmark, error) {
bookmark, err := vbs.GetBookmarkByID(userID, bookmarkID)
if err != nil {
return nil, err
}
// Update fields
bookmark.Description = req.Description
bookmark.Tags = req.Tags
bookmark.IsFavorite = req.IsFavorite
bookmark.UpdatedAt = time.Now()
if err := vbs.db.Save(bookmark).Error; err != nil {
return nil, fmt.Errorf("failed to update bookmark: %w", err)
}
return bookmark, nil
}
// DeleteBookmark deletes a bookmark
func (vbs *VideoBookmarkService) DeleteBookmark(userID uint, bookmarkID uint) error {
result := vbs.db.Where("id = ? AND user_id = ?", bookmarkID, userID).Delete(&models.VideoBookmark{})
if result.Error != nil {
return fmt.Errorf("failed to delete bookmark: %w", result.Error)
}
if result.RowsAffected == 0 {
return fmt.Errorf("bookmark not found")
}
return nil
}
// ToggleWatched toggles the watched status of a bookmark
func (vbs *VideoBookmarkService) ToggleWatched(userID uint, bookmarkID uint) (*models.VideoBookmark, error) {
bookmark, err := vbs.GetBookmarkByID(userID, bookmarkID)
if err != nil {
return nil, err
}
bookmark.IsWatched = !bookmark.IsWatched
bookmark.UpdatedAt = time.Now()
if err := vbs.db.Save(bookmark).Error; err != nil {
return nil, fmt.Errorf("failed to update bookmark: %w", err)
}
return bookmark, nil
}
// ToggleFavorite toggles the favorite status of a bookmark
func (vbs *VideoBookmarkService) ToggleFavorite(userID uint, bookmarkID uint) (*models.VideoBookmark, error) {
bookmark, err := vbs.GetBookmarkByID(userID, bookmarkID)
if err != nil {
return nil, err
}
bookmark.IsFavorite = !bookmark.IsFavorite
bookmark.UpdatedAt = time.Now()
if err := vbs.db.Save(bookmark).Error; err != nil {
return nil, fmt.Errorf("failed to update bookmark: %w", err)
}
return bookmark, nil
}
// SearchBookmarks searches bookmarks by title, channel, or tags
func (vbs *VideoBookmarkService) SearchBookmarks(userID uint, query string, limit int, offset int) ([]models.VideoBookmark, error) {
var bookmarks []models.VideoBookmark
searchQuery := "%" + query + "%"
dbQuery := vbs.db.Where("user_id = ? AND (title LIKE ? OR channel LIKE ? OR tags LIKE ?)",
userID, searchQuery, searchQuery, searchQuery).Order("created_at DESC")
if limit > 0 {
dbQuery = dbQuery.Limit(limit)
}
if offset > 0 {
dbQuery = dbQuery.Offset(offset)
}
if err := dbQuery.Find(&bookmarks).Error; err != nil {
return nil, fmt.Errorf("failed to search bookmarks: %w", err)
}
return bookmarks, nil
}
// GetBookmarkStats gets statistics about user's bookmarks
func (vbs *VideoBookmarkService) GetBookmarkStats(userID uint) (map[string]interface{}, error) {
stats := make(map[string]interface{})
// Total bookmarks
var total int64
if err := vbs.db.Model(&models.VideoBookmark{}).Where("user_id = ?", userID).Count(&total).Error; err != nil {
return nil, fmt.Errorf("failed to get total count: %w", err)
}
stats["total"] = total
// Watched bookmarks
var watched int64
if err := vbs.db.Model(&models.VideoBookmark{}).Where("user_id = ? AND is_watched = ?", userID, true).Count(&watched).Error; err != nil {
return nil, fmt.Errorf("failed to get watched count: %w", err)
}
stats["watched"] = watched
// Favorite bookmarks
var favorites int64
if err := vbs.db.Model(&models.VideoBookmark{}).Where("user_id = ? AND is_favorite = ?", userID, true).Count(&favorites).Error; err != nil {
return nil, fmt.Errorf("failed to get favorites count: %w", err)
}
stats["favorites"] = favorites
// Unwatched bookmarks
stats["unwatched"] = total - watched
return stats, nil
}
// extractVideoInfo extracts video information using the scraper service
func (vbs *VideoBookmarkService) extractVideoInfo(url string) (*VideoInfo, error) {
// In demo mode, create mock data
if os.Getenv("VITE_DEMO_MODE") == "true" {
return &VideoInfo{
VideoID: "demo123",
Title: "Demo Video Title",
Channel: "Demo Channel",
Thumbnail: "https://i.ytimg.com/vi/demo123/hqdefault.jpg",
Success: true,
}, nil
}
// Call the scraper service
scraperURL := fmt.Sprintf("http://youtube-video-scraper:7858/video")
req, err := http.NewRequest("POST", scraperURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
// Create request body
reqBody := fmt.Sprintf(`{"url": "%s"}`, url)
req.Body = nil // Will be set below
client := &http.Client{}
resp, err := client.Post(scraperURL, "application/json", strings.NewReader(reqBody))
if err != nil {
return nil, fmt.Errorf("failed to call scraper service: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode)
}
var videoInfo VideoInfo
if err := json.NewDecoder(resp.Body).Decode(&videoInfo); err != nil {
return nil, fmt.Errorf("failed to decode response: %w", err)
}
return &videoInfo, nil
}
backend/services/youtube.go
+613
View File
@@ -0,0 +1,613 @@
package services
import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"regexp"
"strings"
"time"
)
// YouTubeVideo represents a YouTube video
type YouTubeVideo struct {
ID string `json:"id"`
Title string `json:"title"`
Description string `json:"description"`
Thumbnail string `json:"thumbnail"`
Duration string `json:"duration"`
ViewCount int64 `json:"view_count"`
PublishedAt string `json:"published_at"`
ChannelTitle string `json:"channel_title"`
ChannelID string `json:"channel_id"`
}
// VideoItem represents a video item from the youtube scraping service
type VideoItem struct {
VideoID string `json:"video_id"`
Title string `json:"title,omitempty"`
Length string `json:"length,omitempty"`
ThumbnailURL string `json:"thumbnail_url,omitempty"`
ViewsText string `json:"views_text,omitempty"`
Views int64 `json:"views"`
PublishedText string `json:"published_text,omitempty"`
PublishedDate string `json:"published_date,omitempty"`
ChannelName string `json:"channel_name,omitempty"`
}
// YouTubeSearchResponse represents the response from YouTube search API
type YouTubeSearchResponse struct {
Videos []YouTubeVideo `json:"videos"`
NextPageToken string `json:"next_page_token,omitempty"`
TotalResults int `json:"total_results"`
}
// YouTubeService handles YouTube API interactions
type YouTubeService struct {
httpClient *http.Client
}
// NewYouTubeService creates a new YouTube service instance
func NewYouTubeService() *YouTubeService {
return &YouTubeService{
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
// SearchVideos searches for YouTube videos using direct scraping
func (ys *YouTubeService) SearchVideos(query string, maxResults int, pageToken string) (*YouTubeSearchResponse, error) {
// For new implementation, we always return 1 result as requested
videoID, channelName, err := ys.fetchYouTubeVideoIDAndChannel(query)
if err != nil {
return nil, fmt.Errorf("failed to search YouTube: %w", err)
}
// Create response with single video
video := YouTubeVideo{
ID: videoID,
Title: fmt.Sprintf("Video: %s", query),
ChannelTitle: channelName,
Thumbnail: fmt.Sprintf("https://img.youtube.com/vi/%s/maxresdefault.jpg", videoID),
}
return &YouTubeSearchResponse{
Videos: []YouTubeVideo{video},
TotalResults: 1,
}, nil
}
// fetchYouTubeVideoIDAndChannel scrapes YouTube to get video ID and channel name
func (ys *YouTubeService) fetchYouTubeVideoIDAndChannel(query string) (string, string, error) {
youtubeSearchURL := fmt.Sprintf("https://www.youtube.com/results?search_query=%s", strings.ReplaceAll(query, " ", "+"))
resp, err := ys.httpClient.Get(youtubeSearchURL)
if err != nil {
return "", "", fmt.Errorf("error fetching YouTube search results: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", "", fmt.Errorf("error reading response body: %w", err)
}
// Extract video ID using regex
videoRe := regexp.MustCompile(`"videoRenderer":{"videoId":"([^"]{11})"`)
videoMatches := videoRe.FindStringSubmatch(string(body))
if len(videoMatches) < 2 {
return "", "", fmt.Errorf("no video found for query: %s", query)
}
videoID := videoMatches[1]
// Extract channel name using regex
channelRe := regexp.MustCompile(`"longBylineText":{"runs":\[{"text":"([^"]+)"`)
channelMatches := channelRe.FindStringSubmatch(string(body))
channelName := ""
if len(channelMatches) >= 2 {
channelName = channelMatches[1]
}
return videoID, channelName, nil
}
// GetChannelVideosFromURL extracts videos from a YouTube channel URL
func (ys *YouTubeService) GetChannelVideosFromURL(channelURL string, maxResults int) (*YouTubeSearchResponse, error) {
// Extract channel handle from URL
channelHandle, err := ys.extractChannelHandle(channelURL)
if err != nil {
return nil, fmt.Errorf("invalid channel URL: %w", err)
}
// Fetch channel videos
videos, err := ys.fetchChannelVideos(channelHandle, maxResults)
if err != nil {
return nil, fmt.Errorf("failed to fetch channel videos: %w", err)
}
return &YouTubeSearchResponse{
Videos: videos,
TotalResults: len(videos),
}, nil
}
// extractChannelHandle extracts channel handle from YouTube URL
func (ys *YouTubeService) extractChannelHandle(channelURL string) (string, error) {
// Handle different URL formats
if strings.Contains(channelURL, "/@") {
// Extract handle from @username format
re := regexp.MustCompile(`/@([^/?]+)`)
matches := re.FindStringSubmatch(channelURL)
if len(matches) >= 2 {
return "@" + matches[1], nil
}
} else if strings.Contains(channelURL, "/channel/") {
// Extract channel ID from /channel/ID format
re := regexp.MustCompile(`/channel/([^/?]+)`)
matches := re.FindStringSubmatch(channelURL)
if len(matches) >= 2 {
return matches[1], nil
}
} else if strings.Contains(channelURL, "/c/") {
// Extract custom handle from /c/handle format
re := regexp.MustCompile(`/c/([^/?]+)`)
matches := re.FindStringSubmatch(channelURL)
if len(matches) >= 2 {
return matches[1], nil
}
}
return "", fmt.Errorf("unable to extract channel handle from URL: %s", channelURL)
}
// fetchChannelVideos calls the YouTube scraper service for channel videos
func (ys *YouTubeService) fetchChannelVideos(channelHandle string, maxResults int) ([]YouTubeVideo, error) {
// Call the YouTube scraper service
resp, err := http.Get(fmt.Sprintf("http://youtube-scraper:7857/channel_videos?channel=%s", channelHandle))
if err != nil {
return nil, fmt.Errorf("error calling scraper service: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("error reading response body: %w", err)
}
// Parse the scraper service response
var scraperResponse struct {
Channel string `json:"channel"`
ChannelURL string `json:"channel_url"`
Videos []struct {
VideoID string `json:"video_id"`
Title string `json:"title"`
ThumbnailURL string `json:"thumbnail_url"`
Views int `json:"views"`
ViewsText string `json:"views_text"`
PublishedText string `json:"published_text"`
PublishedDate string `json:"published_date"`
} `json:"videos"`
}
if err := json.Unmarshal(body, &scraperResponse); err != nil {
return nil, fmt.Errorf("error parsing scraper response: %w", err)
}
// Convert to YouTubeVideo format
var videos []YouTubeVideo
for i, video := range scraperResponse.Videos {
if i >= maxResults {
break
}
ytVideo := YouTubeVideo{
ID: video.VideoID,
Title: video.Title,
Thumbnail: video.ThumbnailURL,
ViewCount: int64(video.Views),
PublishedAt: video.PublishedDate,
ChannelTitle: scraperResponse.Channel,
}
videos = append(videos, ytVideo)
}
return videos, nil
}
// GetVideoDetails retrieves basic information about a specific video
func (ys *YouTubeService) GetVideoDetails(videoID string) (*YouTubeVideo, error) {
// For simplicity, return basic video info
video := YouTubeVideo{
ID: videoID,
Title: fmt.Sprintf("Video %s", videoID),
Thumbnail: fmt.Sprintf("https://img.youtube.com/vi/%s/maxresdefault.jpg", videoID),
Description: "Video details not available in this implementation",
}
return &video, nil
}
// GetChannelVideos retrieves videos from a specific channel (legacy method)
func (ys *YouTubeService) GetChannelVideos(channelID string, maxResults int, pageToken string) (*YouTubeSearchResponse, error) {
// Always use integrated YouTube channel service - no more external service calls
return GetYouTubeChannelVideosIntegrated(channelID, maxResults)
}
// Global YouTube service instance
var youtubeService = NewYouTubeService()
// SearchYouTubeVideos is a convenience function for searching videos
func SearchYouTubeVideos(query string, maxResults int, pageToken string) (*YouTubeSearchResponse, error) {
// Always use integrated YouTube search - no more mock data
return SearchYouTubeVideosIntegrated(query, maxResults)
}
// YouTubeSearchVideo represents a YouTube video from search
type YouTubeSearchVideo struct {
VideoID string `json:"video_id"`
Title string `json:"title"`
ChannelName string `json:"channel_name"`
Description string `json:"description"`
Thumbnail string `json:"thumbnail"`
}
// fetchYouTubeVideosReal calls the working search service on port 7857
func fetchYouTubeVideosReal(query string, limit int) ([]YouTubeSearchVideo, error) {
// URL encode the query to handle spaces properly
encodedQuery := url.QueryEscape(query)
// Use localhost for development or Docker service name for container-to-container communication
youtubeServiceURL := os.Getenv("YOUTUBE_SERVICE_URL")
if youtubeServiceURL == "" {
youtubeServiceURL = "http://localhost:7857"
}
url := fmt.Sprintf("%s/youtube?q=%s", youtubeServiceURL, encodedQuery)
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
// Check for rate limiting
if resp.StatusCode == 429 {
return nil, fmt.Errorf("YouTube is rate limiting us. Please try again later.")
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("YouTube search service returned status %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
// Parse the JSON response from the search service (it returns an array)
var videos []YouTubeSearchVideo
if err := json.Unmarshal(body, &videos); err != nil {
return nil, fmt.Errorf("failed to parse search service response: %v", err)
}
// Limit results if needed
if len(videos) > limit {
videos = videos[:limit]
}
return videos, nil
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
// htmlUnescape fixes escaped sequences in HTML strings
func htmlUnescape(s string) string {
replacer := strings.NewReplacer(
"&nbsp;", " ",
"&amp;", "&",
"&quot;", `"`,
"&#39;", "'",
)
return replacer.Replace(s)
}
// searchYouTubeVideosReal calls the real YouTube search scraper
func searchYouTubeVideosReal(query string, maxResults int) (*YouTubeSearchResponse, error) {
// Perform real YouTube search scraping
videos, err := fetchYouTubeVideosReal(query, maxResults)
if err != nil {
return nil, err
}
// Convert search results to YouTubeVideo format
var ytVideos []YouTubeVideo
for _, video := range videos {
ytVideo := YouTubeVideo{
ID: video.VideoID,
Title: video.Title,
Thumbnail: video.Thumbnail,
ViewCount: 0, // Not available from search
PublishedAt: "", // Not available from search
ChannelTitle: video.ChannelName,
}
ytVideos = append(ytVideos, ytVideo)
}
return &YouTubeSearchResponse{
Videos: ytVideos,
TotalResults: len(ytVideos),
}, nil
}
// GetYouTubeVideoDetails is a convenience function for getting video details
func GetYouTubeVideoDetails(videoID string) (*YouTubeVideo, error) {
return youtubeService.GetVideoDetails(videoID)
}
// GetYouTubeChannelVideos is a convenience function for getting channel videos
func GetYouTubeChannelVideos(channelID string, maxResults int, pageToken string) (*YouTubeSearchResponse, error) {
// Always use integrated YouTube channel service - no more mock data
return GetYouTubeChannelVideosIntegrated(channelID, maxResults)
}
// getYouTubeChannelVideosReal calls the YouTube scraper service
func getYouTubeChannelVideosReal(channelID string, maxResults int) (*YouTubeSearchResponse, error) {
// Call the YouTube scraper service using Docker service name
resp, err := http.Get(fmt.Sprintf("http://youtube-scraper:7857/channel_videos?channel=%s", channelID))
if err != nil {
return nil, fmt.Errorf("failed to call YouTube scraper service: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("YouTube scraper service returned status %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body: %w", err)
}
// Parse the response from the scraper service
var scraperResponse struct {
Channel string `json:"channel"`
Videos []struct {
VideoID string `json:"video_id"`
Title string `json:"title"`
Length string `json:"length"`
ThumbnailURL string `json:"thumbnail_url"`
Views int64 `json:"views"`
PublishedText string `json:"published_text"`
PublishedDate string `json:"published_date"`
} `json:"videos"`
}
if err := json.Unmarshal(body, &scraperResponse); err != nil {
return nil, fmt.Errorf("failed to parse scraper response: %w", err)
}
// Convert to our YouTubeVideo format
var videos []YouTubeVideo
for _, video := range scraperResponse.Videos {
ytVideo := YouTubeVideo{
ID: video.VideoID,
Title: video.Title,
Thumbnail: video.ThumbnailURL,
Duration: video.Length,
ViewCount: video.Views,
PublishedAt: video.PublishedDate,
ChannelTitle: scraperResponse.Channel,
}
videos = append(videos, ytVideo)
}
// Limit results if needed
if len(videos) > maxResults {
videos = videos[:maxResults]
}
return &YouTubeSearchResponse{
Videos: videos,
TotalResults: len(videos),
}, nil
}
// PredefinedChannel represents a predefined YouTube channel
type PredefinedChannel struct {
ID string `json:"id"`
Name string `json:"name"`
Handle string `json:"handle"`
}
// GetPredefinedChannelVideos gets the latest videos from predefined channels
func GetPredefinedChannelVideos(maxResults int) (*YouTubeSearchResponse, error) {
// Always use real YouTube channel service - no more demo mode mock data
// Use the predefined channels from youtube_channels.go
channels := []PredefinedChannel{
{ID: "UC9x0YY7RmP2x0v_yEUE0rLA", Name: "NetworkChuck", Handle: "@NetworkChuck"},
{ID: "UCsBjURrPoezykLs9EqH2YWw", Name: "Fireship", Handle: "@Fireship"},
{ID: "UCaBHI8xMtM5I4p3tAH_eW5Q", Name: "Beyond Fireship", Handle: "@beyondfireship"},
{ID: "UC_x5XG1OV2P6uZZ5FSM9Ttw", Name: "Traversy Media", Handle: "@traversy_media"},
{ID: "UC8butISFwT-Wl7EV0hUK0BQ", Name: "Tyler McGinnis", Handle: "@tylermcginnis"},
}
var allVideos []YouTubeVideo
// Get videos from each channel
for _, channel := range channels {
response, err := GetYouTubeChannelVideos(channel.Handle, maxResults, "")
if err != nil {
// Continue with other channels if one fails
continue
}
allVideos = append(allVideos, response.Videos...)
}
// Return combined response
return &YouTubeSearchResponse{
Videos: allVideos,
TotalResults: len(allVideos),
}, nil
}
// getMockVideoDetails returns mock video data for demo mode
func (ys *YouTubeService) getMockVideoDetails(videoID string) *YouTubeVideo {
// Generate some mock data based on video ID
mockTitles := []string{
"Amazing Tech Tutorial",
"Web Development Tips",
"Programming Best Practices",
"JavaScript Framework Comparison",
"Building Modern Web Apps",
}
mockChannels := []string{
"Fireship",
"NetworkChuck",
"Beyond Fireship",
"Tech With Tim",
"Programming with Mosh",
}
// Use video ID to deterministically select mock data
titleIndex := len(videoID) % len(mockTitles)
channelIndex := (len(videoID) + 1) % len(mockChannels)
return &YouTubeVideo{
ID: videoID,
Title: mockTitles[titleIndex],
Description: "This is a mock video description for demo mode. The original video details could not be fetched, but this demonstrates the functionality.",
Thumbnail: fmt.Sprintf("https://img.youtube.com/vi/%s/maxresdefault.jpg", videoID),
Duration: "10:24",
ViewCount: int64(1000 + (len(videoID) * 100)),
PublishedAt: "2024-01-15",
ChannelTitle: mockChannels[channelIndex],
ChannelID: "mock_channel_id",
}
}
// getMockYouTubeVideos returns mock YouTube videos for demo mode
func getMockYouTubeVideos(query string, maxResults int) (*YouTubeSearchResponse, error) {
// Mock video data
mockVideos := []YouTubeVideo{
{
ID: "MOCK-VIDEO-1",
Title: "MOCK: Never Gonna Give You Up - Rick Astley",
Description: "The official video for 'Never Gonna Give You Up' by Rick Astley",
Thumbnail: "https://img.youtube.com/vi/dQw4w9WgXcQ/maxresdefault.jpg",
Duration: "3:33",
ViewCount: 1500000000,
PublishedAt: "2009-10-25",
ChannelTitle: "Rick Astley",
ChannelID: "UCuAXFkgsw1L7xaCfnd5CJOA",
},
{
ID: "MOCK-VIDEO-2",
Title: "MOCK: Me at the zoo - The first YouTube video",
Description: "The first video on YouTube, uploaded by Jawed Karim",
Thumbnail: "https://img.youtube.com/vi/jNQXAC9IVRw/maxresdefault.jpg",
Duration: "0:19",
ViewCount: 300000000,
PublishedAt: "2005-04-23",
ChannelTitle: "Jawed Karim",
ChannelID: "UC4QobL6k2pFkE-vtCS5wZTA",
},
{
ID: "MOCK-VIDEO-3",
Title: "MOCK: PSY - GANGNAM STYLE (강남스타일) M/V",
Description: "Psy's official music video for 'Gangnam Style'",
Thumbnail: "https://img.youtube.com/vi/9bZkp7q19f0/maxresdefault.jpg",
Duration: "4:13",
ViewCount: 5000000000,
PublishedAt: "2012-07-15",
ChannelTitle: "officialpsy",
ChannelID: "UCrEw2n_aDR1I7k2kI2L2tJA",
},
{
ID: "MOCK-VIDEO-4",
Title: "MOCK: Luis Fonsi - Despacito ft. Daddy Yankee",
Description: "Official music video for 'Despacito' by Luis Fonsi",
Thumbnail: "https://img.youtube.com/vi/kJQP7kiw5Fk/maxresdefault.jpg",
Duration: "4:41",
ViewCount: 8000000000,
PublishedAt: "2017-01-12",
ChannelTitle: "Luis Fonsi",
ChannelID: "UCrgInDaT3M4n1qZ6-xJbR9A",
},
{
ID: "MOCK-VIDEO-5",
Title: "MOCK: Introduction to React Programming",
Description: "Learn the basics of React programming in this comprehensive tutorial",
Thumbnail: "https://img.youtube.com/vi/hTWKbfoikeg/maxresdefault.jpg",
Duration: "15:30",
ViewCount: 250000,
PublishedAt: "2024-01-15",
ChannelTitle: "Programming Tutorials",
ChannelID: "UC1234567890",
},
{
ID: "MOCK-VIDEO-6",
Title: "MOCK: Docker Containerization Explained",
Description: "Complete guide to Docker containers and orchestration",
Thumbnail: "https://img.youtube.com/vi/abc123def456/maxresdefault.jpg",
Duration: "22:15",
ViewCount: 180000,
PublishedAt: "2024-01-10",
ChannelTitle: "DevOps Simplified",
ChannelID: "UC0987654321",
},
{
ID: "MOCK-VIDEO-7",
Title: "MOCK: Machine Learning Fundamentals",
Description: "Introduction to machine learning algorithms and concepts",
Thumbnail: "https://img.youtube.com/vi/xyz789uvw012/maxresdefault.jpg",
Duration: "18:45",
ViewCount: 320000,
PublishedAt: "2024-01-08",
ChannelTitle: "AI Education",
ChannelID: "UC1122334455",
},
{
ID: "MOCK-VIDEO-8",
Title: "MOCK: Web Development Best Practices 2024",
Description: "Modern web development techniques and best practices",
Thumbnail: "https://img.youtube.com/vi/def456ghi789/maxresdefault.jpg",
Duration: "25:10",
ViewCount: 145000,
PublishedAt: "2024-01-12",
ChannelTitle: "Web Dev Weekly",
ChannelID: "UC5566778899",
},
{
ID: "MOCK-VIDEO-9",
Title: "MOCK: JavaScript Advanced Concepts",
Description: "Deep dive into JavaScript advanced features and patterns",
Thumbnail: "https://img.youtube.com/vi/ghi789jkl012/maxresdefault.jpg",
Duration: "32:20",
ViewCount: 425000,
PublishedAt: "2024-01-05",
ChannelTitle: "JS Masters",
ChannelID: "UC9988776655",
},
}
// For demo mode, return all videos (up to maxResults) regardless of query
var filteredVideos []YouTubeVideo
for i, video := range mockVideos {
if i >= maxResults {
break
}
filteredVideos = append(filteredVideos, video)
}
return &YouTubeSearchResponse{
Videos: filteredVideos,
TotalResults: len(filteredVideos),
}, nil
}
backend/services/youtube_cache.go
+190
View File
@@ -0,0 +1,190 @@
package services
import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"sync"
"time"
"github.com/trackeep/backend/models"
"gorm.io/gorm"
)
// YouTubeCacheService handles caching YouTube channel data
type YouTubeCacheService struct {
db *gorm.DB
cache map[string]*CacheEntry
mutex sync.RWMutex
}
// CacheEntry represents an in-memory cache entry
type CacheEntry struct {
Videos string `json:"videos"`
LastUpdated time.Time `json:"last_updated"`
}
// NewYouTubeCacheService creates a new YouTube cache service
func NewYouTubeCacheService(db *gorm.DB) *YouTubeCacheService {
return &YouTubeCacheService{
db: db,
cache: make(map[string]*CacheEntry),
}
}
// GetCachedChannelVideos retrieves cached channel videos or fetches fresh data
func (y *YouTubeCacheService) GetCachedChannelVideos(channelID string, maxResults int) (*YouTubeSearchResponse, error) {
// Always use real YouTube data - no more demo mode
// Try to get from database cache first
var cache models.YouTubeChannelCache
if err := y.db.Where("channel_id = ?", channelID).First(&cache); err == nil {
// Check if cache is still valid
if !cache.IsExpired() {
// Return cached data
var videos []YouTubeVideo
if err := json.Unmarshal([]byte(cache.Videos), &videos); err == nil {
// Limit results if needed
if len(videos) > maxResults {
videos = videos[:maxResults]
}
return &YouTubeSearchResponse{
Videos: videos,
TotalResults: len(videos),
}, nil
}
}
}
// Cache is expired or doesn't exist, fetch fresh data
return y.fetchAndCacheVideos(channelID, maxResults)
}
// getInMemoryCachedVideos retrieves cached videos from memory (for demo mode)
func (y *YouTubeCacheService) getInMemoryCachedVideos(channelID string, maxResults int) (*YouTubeSearchResponse, error) {
y.mutex.RLock()
defer y.mutex.RUnlock()
if entry, exists := y.cache[channelID]; exists {
// Check if cache is still valid (2 hours)
if time.Since(entry.LastUpdated) < 2*time.Hour {
// Return cached data
var videos []YouTubeVideo
if err := json.Unmarshal([]byte(entry.Videos), &videos); err == nil {
// Limit results if needed
if len(videos) > maxResults {
videos = videos[:maxResults]
}
return &YouTubeSearchResponse{
Videos: videos,
TotalResults: len(videos),
}, nil
}
}
}
// Cache is expired or doesn't exist, fetch fresh data
return y.fetchAndCacheVideos(channelID, maxResults)
}
// fetchAndCacheVideos fetches fresh data and caches it
func (y *YouTubeCacheService) fetchAndCacheVideos(channelID string, maxResults int) (*YouTubeSearchResponse, error) {
// Fetch from YouTube scraper service
resp, err := http.Get(fmt.Sprintf("http://youtube-scraper:7857/channel_videos?channel=%s", channelID))
if err != nil {
return nil, fmt.Errorf("failed to fetch channel videos: %w", err)
}
defer resp.Body.Close()
// Check for rate limiting
if resp.StatusCode == 429 {
return nil, fmt.Errorf("YouTube is rate limiting us. Please try again later.")
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("YouTube scraper service returned status %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("error reading response body: %w", err)
}
fmt.Printf("DEBUG: fetchAndCacheVideos response for %s: %s\n", channelID, string(body[:min(500, len(body))]))
// Parse the scraper service response
var scraperResponse struct {
Channel string `json:"channel"`
ChannelURL string `json:"channel_url"`
Videos []struct {
VideoID string `json:"video_id"`
Title string `json:"title"`
ThumbnailURL string `json:"thumbnail_url"`
Views int `json:"views"`
ViewsText string `json:"views_text"`
PublishedText string `json:"published_text"`
PublishedDate string `json:"published_date"`
} `json:"videos"`
}
if err := json.Unmarshal(body, &scraperResponse); err != nil {
return nil, fmt.Errorf("error parsing scraper response: %w", err)
}
fmt.Printf("DEBUG: Parsed %d videos for channel %s\n", len(scraperResponse.Videos), channelID)
// Convert to YouTubeVideo format
var videos []YouTubeVideo
for i, video := range scraperResponse.Videos {
if i >= maxResults {
break
}
ytVideo := YouTubeVideo{
ID: video.VideoID,
Title: video.Title,
Thumbnail: video.ThumbnailURL,
ViewCount: int64(video.Views),
PublishedAt: video.PublishedDate,
ChannelTitle: scraperResponse.Channel,
}
videos = append(videos, ytVideo)
}
fmt.Printf("DEBUG: Converted %d videos for channel %s\n", len(videos), channelID)
// Cache the results
videosJSON, err := json.Marshal(videos)
if err != nil {
log.Printf("Error marshaling videos for cache: %v", err)
} else {
// Save to database cache
cache := models.YouTubeChannelCache{
ChannelID: channelID,
ChannelName: scraperResponse.Channel,
ChannelURL: scraperResponse.ChannelURL,
Videos: string(videosJSON),
LastUpdated: time.Now(),
}
// Use upsert to handle both create and update
y.db.Where("channel_id = ?", channelID).Assign(&cache).FirstOrCreate(&cache)
fmt.Printf("DEBUG: Cached %d videos in database for channel %s\n", len(videos), channelID)
}
return &YouTubeSearchResponse{
Videos: videos,
TotalResults: len(videos),
}, nil
}
// ClearExpiredCache removes expired cache entries
func (y *YouTubeCacheService) ClearExpiredCache() error {
// Always use database cache - no more demo mode
// Clear database cache
expiredTime := time.Now().Add(-2 * time.Hour)
return y.db.Where("last_updated < ?", expiredTime).Delete(&models.YouTubeChannelCache{}).Error
}
backend/services/youtube_channels.go
+198
View File
@@ -0,0 +1,198 @@
package services
import (
"fmt"
"time"
)
// YouTubeChannelService handles specific channel integrations
type YouTubeChannelService struct {
YouTubeService *YouTubeService
CacheService *YouTubeCacheService
}
// NewYouTubeChannelService creates a new instance of YouTubeChannelService
func NewYouTubeChannelService(youtubeService *YouTubeService, cacheService *YouTubeCacheService) *YouTubeChannelService {
return &YouTubeChannelService{
YouTubeService: youtubeService,
CacheService: cacheService,
}
}
// GetPredefinedChannels returns the list of predefined channels
func GetPredefinedChannels() []Channel {
return []Channel{
{
ID: "fireship",
Name: "Fireship",
Description: "Rapid web development tutorials and courses",
Thumbnail: "https://img.youtube.com/vi/UCsBjURrPoezykLs9EqgAJVQ/mqdefault.jpg",
},
{
ID: "networkchuck",
Name: "NetworkChuck",
Description: "Cybersecurity and networking tutorials",
Thumbnail: "https://img.youtube.com/vi/UCNlz1cb4DvEx7rTnT2s7B3A/mqdefault.jpg",
},
{
ID: "programmingwithmosh",
Name: "Programming with Mosh",
Description: "Comprehensive programming tutorials",
Thumbnail: "https://img.youtube.com/vi/UC8butUNob-8kuy47X7vH6ws/mqdefault.jpg",
},
{
ID: "traversymedia",
Name: "Traversy Media",
Description: "Web development and design tutorials",
Thumbnail: "https://img.youtube.com/vi/UC29J8QxEQ7QmM0TJ_8Jt_gQ/mqdefault.jpg",
},
{
ID: "thenewboston",
Name: "The New Boston",
Description: "Computer science and programming courses",
Thumbnail: "https://img.youtube.com/vi/UCrwkHaJ-9Sd74Kx1n-9Qjg/mqdefault.jpg",
},
}
}
// Channel represents a YouTube channel
type Channel struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Thumbnail string `json:"thumbnail"`
}
// GetFireshipVideos fetches latest videos from Fireship channel
func (ycs *YouTubeChannelService) GetFireshipVideos(limit int) ([]YouTubeVideo, error) {
// Use cached data to avoid rate limiting
response, err := ycs.CacheService.GetCachedChannelVideos("fireship", limit)
if err != nil {
if err.Error() == "YouTube is rate limiting us. Please try again later." {
// Return rate limiting error
return nil, fmt.Errorf("YouTube is rate limiting us. Please try again later.")
}
return nil, fmt.Errorf("failed to fetch Fireship videos: %w", err)
}
// Return all videos without filtering
return response.Videos, nil
}
// GetNetworkChuckVideos fetches latest videos from Network Chuck channel
func (ycs *YouTubeChannelService) GetNetworkChuckVideos(limit int) ([]YouTubeVideo, error) {
// Use cached data to avoid rate limiting
response, err := ycs.CacheService.GetCachedChannelVideos("networkchuck", limit)
if err != nil {
if err.Error() == "YouTube is rate limiting us. Please try again later." {
// Return rate limiting error
return nil, fmt.Errorf("YouTube is rate limiting us. Please try again later.")
}
return nil, fmt.Errorf("failed to fetch Network Chuck videos: %w", err)
}
// Return all videos without filtering
return response.Videos, nil
}
// GetChannelInfo fetches basic information about a channel
func (ycs *YouTubeChannelService) GetChannelInfo(channelID string) (*ChannelInfo, error) {
// For now, return basic info from predefined channels
channels := GetPredefinedChannels()
for _, channel := range channels {
if channel.ID == channelID {
return &ChannelInfo{
ID: channel.ID,
Title: channel.Name,
Description: channel.Description,
Thumbnail: channel.Thumbnail,
}, nil
}
}
return nil, fmt.Errorf("channel not found")
}
// ChannelInfo represents basic channel information
type ChannelInfo struct {
ID string `json:"id"`
Title string `json:"title"`
Description string `json:"description"`
Thumbnail string `json:"thumbnail"`
}
// Helper functions
// parseYouTubeDuration converts ISO 8601 duration string to seconds
func parseYouTubeDuration(duration string) int {
// YouTube duration format: PT4M13S (4 minutes 13 seconds)
// Simple parser for common formats
seconds := 0
current := 0
for _, char := range duration {
switch char {
case 'H', 'h':
seconds += current * 3600
current = 0
case 'M', 'm':
seconds += current * 60
current = 0
case 'S', 's':
seconds += current
current = 0
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
current = current*10 + int(char-'0')
}
}
return seconds
}
func containsIgnoreCase(s, substr string) bool {
return len(s) >= len(substr) && (s == substr ||
(len(s) > len(substr) &&
(s[:len(substr)] == substr ||
s[len(s)-len(substr):] == substr ||
containsSubstringIgnoreCase(s, substr))))
}
func containsSubstringIgnoreCase(s, substr string) bool {
s = toLower(s)
substr = toLower(substr)
return contains(s, substr)
}
func toLower(s string) string {
result := make([]rune, len([]rune(s)))
for i, r := range []rune(s) {
if r >= 'A' && r <= 'Z' {
result[i] = r + ('a' - 'A')
} else {
result[i] = r
}
}
return string(result)
}
func contains(s, substr string) bool {
for i := 0; i <= len(s)-len(substr); i++ {
if s[i:i+len(substr)] == substr {
return true
}
}
return false
}
func isRecentVideo(publishedAt string, months int) bool {
// Parse the published date (ISO 8601 format)
layout := "2006-01-02T15:04:05Z"
publishedTime, err := time.Parse(layout, publishedAt)
if err != nil {
return false
}
// Check if the video is within the specified months
cutoffTime := time.Now().AddDate(0, -months, 0)
return publishedTime.After(cutoffTime)
}
backend/services/youtube_integrated.go
+516
View File
@@ -0,0 +1,516 @@
package services
import (
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
)
// YouTubeIntegratedService provides all YouTube functionality in one service
type YouTubeIntegratedService struct {
httpClient *http.Client
}
// NewYouTubeIntegratedService creates a new integrated YouTube service
func NewYouTubeIntegratedService() *YouTubeIntegratedService {
return &YouTubeIntegratedService{
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
// SearchVideosIntegrated performs YouTube video search
func (y *YouTubeIntegratedService) SearchVideosIntegrated(query string, limit int) ([]YouTubeSearchVideo, error) {
url := fmt.Sprintf(
"https://www.youtube.com/results?search_query=%s",
url.QueryEscape(query),
)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
resp, err := y.httpClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
html := string(body)
videoRe := regexp.MustCompile(`"videoRenderer":{"videoId":"([^"]{11})"`)
results := []YouTubeSearchVideo{}
seen := map[string]bool{}
videoMatches := videoRe.FindAllStringSubmatchIndex(html, -1)
for _, match := range videoMatches {
if len(results) >= limit {
break
}
if len(match) < 4 {
continue
}
videoID := html[match[2]:match[3]]
if _, ok := seen[videoID]; ok {
continue
}
seen[videoID] = true
// Extract title and channel from surrounding context
start := match[0]
if start-2000 > 0 {
start = start - 2000
}
end := match[1] + 2000
if end > len(html) {
end = len(html)
}
snippet := html[start:end]
title := ""
channel := ""
if m := regexp.MustCompile(`"title":\{"runs":\[\{"text":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
title = unescapeYT(m[1])
} else if m := regexp.MustCompile(`"title":\{"simpleText":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
title = unescapeYT(m[1])
}
if m := regexp.MustCompile(`"longBylineText":\{"runs":\[\{"text":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
channel = unescapeYT(m[1])
}
if title == "" {
title = "Video " + videoID
}
results = append(results, YouTubeSearchVideo{
VideoID: videoID,
Title: title,
ChannelName: channel,
Thumbnail: fmt.Sprintf("https://img.youtube.com/vi/%s/maxresdefault.jpg", videoID),
})
}
return results, nil
}
// ChannelVideosResponse represents the response for channel videos scraping
type ChannelVideosResponse struct {
Channel string `json:"channel"`
ChannelURL string `json:"channel_url"`
SubscribersText string `json:"subscribers_text"`
Subscribers int64 `json:"subscribers"`
Videos []VideoItem `json:"videos"`
}
// GetChannelVideosIntegrated fetches channel videos directly
func (y *YouTubeIntegratedService) GetChannelVideosIntegrated(channelInput string) (ChannelVideosResponse, error) {
handle, channelURL := normalizeChannelInput(channelInput)
req, err := http.NewRequest("GET", channelURL, nil)
if err != nil {
return ChannelVideosResponse{}, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
resp, err := y.httpClient.Do(req)
if err != nil {
return ChannelVideosResponse{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return ChannelVideosResponse{}, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return ChannelVideosResponse{}, err
}
html := string(body)
// Extract video IDs and metadata
vidRe := regexp.MustCompile(`"videoRenderer":\{[^}]*?"videoId":"([a-zA-Z0-9_-]{11})"`)
matches := vidRe.FindAllStringSubmatchIndex(html, -1)
seen := make(map[string]struct{})
var videos []VideoItem
for _, idx := range matches {
if len(idx) < 4 {
continue
}
videoID := html[idx[2]:idx[3]]
if _, ok := seen[videoID]; ok {
continue
}
seen[videoID] = struct{}{}
start := idx[0]
if start-2000 > 0 {
start = start - 2000
}
end := idx[1] + 8000
if end > len(html) {
end = len(html)
}
snippet := html[start:end]
vi := VideoItem{VideoID: videoID}
vi.ThumbnailURL = fmt.Sprintf("https://img.youtube.com/vi/%s/maxresdefault.jpg", videoID)
// Extract metadata
if m := regexp.MustCompile(`"title":\{"runs":\[\{"text":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
vi.Title = unescapeYT(m[1])
} else if m := regexp.MustCompile(`"title":\{"simpleText":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
vi.Title = unescapeYT(m[1])
}
if m := regexp.MustCompile(`"lengthText":\{[^}]*"simpleText":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
vi.Length = m[1]
}
if m := regexp.MustCompile(`"publishedTimeText":\{"simpleText":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
vi.PublishedText = m[1]
vi.PublishedDate = parseRelativeToISO(m[1])
}
if m := regexp.MustCompile(`"viewCountText":\{"simpleText":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
vi.ViewsText = m[1]
vi.Views = parseCountText(m[1])
}
videos = append(videos, vi)
}
// Extract channel info
channelDisplay := handle
if m := regexp.MustCompile(`"canonicalBaseUrl":"\\/(@[^\"]+)"`).FindStringSubmatch(html); len(m) >= 2 {
channelDisplay = m[1]
}
subText := ""
if m := regexp.MustCompile(`"subscriberCountText":\{"simpleText":"([^"]+)"`).FindStringSubmatch(html); len(m) >= 2 {
subText = m[1]
}
subs := parseCountText(subText)
return ChannelVideosResponse{
Channel: channelDisplay,
ChannelURL: channelURL,
SubscribersText: subText,
Subscribers: subs,
Videos: videos,
}, nil
}
// IntegratedVideoInfo represents the extracted video information
type IntegratedVideoInfo struct {
VideoID string `json:"video_id"`
Title string `json:"title"`
Channel string `json:"channel"`
Thumbnail string `json:"thumbnail_url"`
Success bool `json:"success"`
Error string `json:"error,omitempty"`
}
// GetVideoDetailsIntegrated scrapes individual video details
func (y *YouTubeIntegratedService) GetVideoDetailsIntegrated(videoURL string) (IntegratedVideoInfo, error) {
videoID := extractVideoID(videoURL)
if videoID == "" {
return IntegratedVideoInfo{
Success: false,
Error: "Invalid YouTube URL",
}, nil
}
url := fmt.Sprintf("https://www.youtube.com/watch?v=%s", videoID)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return IntegratedVideoInfo{
Success: false,
Error: fmt.Sprintf("Failed to create request: %v", err),
}, nil
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
resp, err := y.httpClient.Do(req)
if err != nil {
return IntegratedVideoInfo{
Success: false,
Error: fmt.Sprintf("Failed to fetch page: %v", err),
}, nil
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return IntegratedVideoInfo{
Success: false,
Error: fmt.Sprintf("HTTP %d", resp.StatusCode),
}, nil
}
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
return IntegratedVideoInfo{
Success: false,
Error: fmt.Sprintf("Failed to parse HTML: %v", err),
}, nil
}
title := ""
channel := ""
doc.Find("title").Each(func(i int, s *goquery.Selection) {
if title == "" {
title = s.Text()
title = strings.TrimSuffix(title, " - YouTube")
}
})
doc.Find("a.yt-simple-endpoint.style-scope.yt-formatted-string").Each(func(i int, s *goquery.Selection) {
if channel == "" && strings.Contains(s.AttrOr("href", ""), "/@") {
channel = s.Text()
}
})
if title == "" {
title = "Video " + videoID
}
return IntegratedVideoInfo{
VideoID: videoID,
Title: title,
Channel: channel,
Thumbnail: fmt.Sprintf("https://img.youtube.com/vi/%s/maxresdefault.jpg", videoID),
Success: true,
}, nil
}
// Helper functions
func normalizeChannelInput(input string) (handle string, url string) {
in := strings.TrimSpace(input)
lower := strings.ToLower(in)
isURL := strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") || strings.HasPrefix(lower, "www.") || strings.HasPrefix(lower, "youtube.com/")
if isURL {
if strings.HasPrefix(lower, "www.") || strings.HasPrefix(lower, "youtube.com/") {
in = "https://" + strings.TrimPrefix(in, "www.")
if !strings.HasPrefix(strings.ToLower(in), "https://youtube.com/") && !strings.HasPrefix(strings.ToLower(in), "https://www.youtube.com/") {
in = "https://www." + strings.TrimPrefix(in, "https://")
}
}
in = strings.ReplaceAll(in, "m.youtube.com", "www.youtube.com")
reHandle := regexp.MustCompile(`https?://(www\.)?youtube\.com/(@[^/]+)`)
if m := reHandle.FindStringSubmatch(in); len(m) >= 3 {
handle = m[2]
} else {
rePath := regexp.MustCompile(`https?://(www\.)?youtube\.com/([^/?#]+)`)
if m2 := rePath.FindStringSubmatch(in); len(m2) >= 3 {
seg := m2[2]
if strings.HasPrefix(seg, "@") {
handle = seg
} else {
handle = "@" + seg
}
}
}
if strings.Contains(strings.ToLower(in), "/videos") || strings.Contains(strings.ToLower(in), "/shorts") || strings.Contains(strings.ToLower(in), "/streams") {
url = in
} else {
if handle == "" {
url = in
} else {
url = fmt.Sprintf("https://www.youtube.com/%s/videos", handle)
}
}
} else {
if strings.HasPrefix(in, "@") {
handle = in
} else {
handle = "@" + in
}
url = fmt.Sprintf("https://www.youtube.com/%s/videos", handle)
}
if handle == "" {
handle = in
if !strings.HasPrefix(handle, "@") {
handle = "@" + handle
}
}
return
}
func unescapeYT(s string) string {
s = strings.ReplaceAll(s, `\/`, `/`)
s = strings.ReplaceAll(s, `\u0026`, `&`)
return s
}
func parseRelativeToISO(rel string) string {
now := time.Now()
lower := strings.ToLower(rel)
re := regexp.MustCompile(`(\d+)[\s-]*(second|minute|hour|day|week|month|year)s?\s+ago`)
if m := re.FindStringSubmatch(lower); len(m) >= 3 {
n, _ := strconv.Atoi(m[1])
unit := m[2]
switch unit {
case "second":
return now.Add(-time.Duration(n) * time.Second).Format("2006-01-02")
case "minute":
return now.Add(-time.Duration(n) * time.Minute).Format("2006-01-02")
case "hour":
return now.Add(-time.Duration(n) * time.Hour).Format("2006-01-02")
case "day":
return now.AddDate(0, 0, -n).Format("2006-01-02")
case "week":
return now.AddDate(0, 0, -7*n).Format("2006-01-02")
case "month":
return now.AddDate(0, -n, 0).Format("2006-01-02")
case "year":
return now.AddDate(-n, 0, 0).Format("2006-01-02")
}
}
return ""
}
func parseCountText(s string) int64 {
t := strings.ToLower(strings.TrimSpace(s))
re := regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?)([kmb])?`)
if m := re.FindStringSubmatch(t); len(m) >= 2 {
numStr := m[1]
suf := ""
if len(m) >= 3 {
suf = m[2]
}
f, err := strconv.ParseFloat(numStr, 64)
if err != nil {
return 0
}
switch suf {
case "k":
f *= 1_000
case "m":
f *= 1_000_000
case "b":
f *= 1_000_000_000
}
return int64(f)
}
digits := regexp.MustCompile(`[^0-9]`).ReplaceAllString(t, "")
if digits == "" {
return 0
}
v, _ := strconv.ParseInt(digits, 10, 64)
return v
}
func extractVideoID(url string) string {
if strings.Contains(url, "youtu.be/") {
parts := strings.Split(url, "youtu.be/")
if len(parts) > 1 {
return strings.Split(parts[1], "?")[0]
}
} else if strings.Contains(url, "youtube.com/watch") {
parts := strings.Split(url, "v=")
if len(parts) > 1 {
return strings.Split(parts[1], "&")[0]
}
} else if strings.Contains(url, "youtube.com/embed/") {
parts := strings.Split(url, "embed/")
if len(parts) > 1 {
return strings.Split(parts[1], "?")[0]
}
}
return ""
}
// Global integrated service instance
var integratedYouTubeService = NewYouTubeIntegratedService()
// Integrated service functions for backward compatibility
func SearchYouTubeVideosIntegrated(query string, maxResults int) (*YouTubeSearchResponse, error) {
// Always use real YouTube search - no more demo mode mock data
if maxResults <= 0 || maxResults > 9 {
maxResults = 9
}
videos, err := integratedYouTubeService.SearchVideosIntegrated(query, maxResults)
if err != nil {
return nil, err
}
var ytVideos []YouTubeVideo
for _, video := range videos {
ytVideo := YouTubeVideo{
ID: video.VideoID,
Title: video.Title,
Thumbnail: video.Thumbnail,
ViewCount: 0,
PublishedAt: "",
ChannelTitle: video.ChannelName,
}
ytVideos = append(ytVideos, ytVideo)
}
return &YouTubeSearchResponse{
Videos: ytVideos,
TotalResults: len(ytVideos),
}, nil
}
func GetYouTubeChannelVideosIntegrated(channelID string, maxResults int) (*YouTubeSearchResponse, error) {
// Always use real YouTube channel service - no more demo mode mock data
response, err := integratedYouTubeService.GetChannelVideosIntegrated(channelID)
if err != nil {
return nil, err
}
var videos []YouTubeVideo
for _, video := range response.Videos {
ytVideo := YouTubeVideo{
ID: video.VideoID,
Title: video.Title,
Thumbnail: video.ThumbnailURL,
Duration: video.Length,
ViewCount: video.Views,
PublishedAt: video.PublishedDate,
ChannelTitle: response.Channel,
}
videos = append(videos, ytVideo)
}
if len(videos) > maxResults {
videos = videos[:maxResults]
}
return &YouTubeSearchResponse{
Videos: videos,
TotalResults: len(videos),
}, nil
}