Devour/cmd/verify.go

package cmd

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/spf13/cobra"
	"github.com/yourorg/devour/internal/scraper"
)

var (
	verifyFormat  string
	verifyTimeout int
)

var verifyCmd = &cobra.Command{
	Use:   "verify",
	Short: "Run Devour verification suites",
	Long:  `Run deterministic and live verification suites for Devour commands and scrapers.`,
}

var verifySmokeCmd = &cobra.Command{
	Use:   "smoke",
	Short: "Run live docs scraping smoke checks",
	Long:  `Run an opt-in live network smoke suite and persist a machine-readable report under devour_data/verify/.`,
	RunE:  runVerifySmoke,
}

func init() {
	verifyCmd.AddCommand(verifySmokeCmd)
	verifySmokeCmd.Flags().StringVar(&verifyFormat, "format", "text", "output format (text, json)")
	verifySmokeCmd.Flags().IntVar(&verifyTimeout, "timeout", 90, "timeout per smoke case in seconds")
}

type verifyCase struct {
	Name   string             `json:"name"`
	Type   scraper.SourceType `json:"type"`
	URL    string             `json:"url"`
	Passed bool               `json:"passed"`
	Docs   int                `json:"docs"`
	Error  string             `json:"error,omitempty"`
	TookMs int64              `json:"took_ms"`
}

type verifyReport struct {
	CreatedAt time.Time    `json:"created_at"`
	Duration  string       `json:"duration"`
	Passed    int          `json:"passed"`
	Failed    int          `json:"failed"`
	Cases     []verifyCase `json:"cases"`
}

func runVerifySmoke(cmd *cobra.Command, args []string) error {
	cfg, err := loadAppConfig()
	if err != nil {
		return err
	}

	if verifyTimeout <= 0 {
		verifyTimeout = 90
	}

	cases := []verifyCase{
		{Name: "Go net/http", Type: scraper.SourceTypeGoDocs, URL: "https://pkg.go.dev/net/http"},
		{Name: "Python asyncio", Type: scraper.SourceTypePythonDocs, URL: "https://docs.python.org/3/library/asyncio.html"},
		{Name: "React reference", Type: scraper.SourceTypeReactDocs, URL: "https://react.dev/reference/react"},
		{Name: "TypeScript handbook", Type: scraper.SourceTypeTSDocs, URL: "https://www.typescriptlang.org/docs/handbook/2/basic-types.html"},
		{Name: "Next.js docs", Type: scraper.SourceTypeWeb, URL: "https://nextjs.org/docs"},
		{Name: "Svelte docs", Type: scraper.SourceTypeWeb, URL: "https://svelte.dev/docs/kit"},
		{Name: "Angular guide", Type: scraper.SourceTypeWeb, URL: "https://angular.dev/guide/http"},
		{Name: "Remix docs", Type: scraper.SourceTypeWeb, URL: "https://v2.remix.run/docs"},
		{Name: "Solid docs repo", Type: scraper.SourceTypeGitHub, URL: "https://github.com/solidjs/solid-docs"},
		{Name: "Express guide", Type: scraper.SourceTypeWeb, URL: "https://expressjs.com/en/guide/routing.html"},
	}

	startAll := time.Now()
	passed := 0
	failed := 0

	for i := range cases {
		c := &cases[i]
		caseStart := time.Now()
		s := scraper.NewScraper(c.Type, toScraperConfig(cfg, 4))
		if s == nil {
			c.Error = "scraper not available"
			c.Passed = false
			failed++
			continue
		}

		ctx, cancel := context.WithTimeout(context.Background(), time.Duration(verifyTimeout)*time.Second)
		docs, err := s.Scrape(ctx, &scraper.Source{Name: c.Name, Type: c.Type, URL: c.URL})
		cancel()
		c.TookMs = time.Since(caseStart).Milliseconds()

		if err != nil {
			c.Error = err.Error()
			c.Passed = false
			failed++
			continue
		}
		c.Docs = len(docs)
		if len(docs) == 0 {
			c.Error = "0 documents"
			c.Passed = false
			failed++
			continue
		}
		c.Passed = true
		passed++
	}

	report := verifyReport{
		CreatedAt: time.Now(),
		Duration:  time.Since(startAll).String(),
		Passed:    passed,
		Failed:    failed,
		Cases:     cases,
	}

	rootDataDir := filepath.Dir(cfg.Storage.DocsDir)
	verifyDir := filepath.Join(rootDataDir, "verify")
	if err := os.MkdirAll(verifyDir, 0o755); err != nil {
		return err
	}
	filename := fmt.Sprintf("smoke-%s.json", time.Now().Format("20060102-150405"))
	reportPath := filepath.Join(verifyDir, filename)
	b, err := json.MarshalIndent(report, "", "  ")
	if err != nil {
		return err
	}
	if err := os.WriteFile(reportPath, b, 0o644); err != nil {
		return err
	}

	switch verifyFormat {
	case "json":
		enc := json.NewEncoder(cmd.OutOrStdout())
		enc.SetIndent("", "  ")
		if err := enc.Encode(report); err != nil {
			return err
		}
	default:
		fmt.Fprintf(cmd.OutOrStdout(), "Smoke verification complete\n")
		fmt.Fprintf(cmd.OutOrStdout(), "  Passed: %d\n", report.Passed)
		fmt.Fprintf(cmd.OutOrStdout(), "  Failed: %d\n", report.Failed)
		fmt.Fprintf(cmd.OutOrStdout(), "  Report: %s\n", reportPath)
		for _, c := range report.Cases {
			status := "PASS"
			if !c.Passed {
				status = "FAIL"
			}
			fmt.Fprintf(cmd.OutOrStdout(), "  - [%s] %s (%d docs, %dms)", status, c.Name, c.Docs, c.TookMs)
			if c.Error != "" {
				fmt.Fprintf(cmd.OutOrStdout(), " error=%s", c.Error)
			}
			fmt.Fprintln(cmd.OutOrStdout())
		}
	}

	if report.Failed > 0 {
		return fmt.Errorf("smoke verification completed with failures")
	}
	return nil
}