mirror of
https://github.com/Dvorinka/Devour.git
synced 2026-06-03 20:13:03 +00:00
update
This commit is contained in:
Tomas Dvorak
+226
-58
@@ -2,6 +2,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
@@ -11,112 +12,210 @@ var getCmd = &cobra.Command{
|
||||
Use: "get <language> <keyword>",
|
||||
Short: "Get documentation for a language/framework",
|
||||
Long: `Quickly fetch documentation for popular languages and frameworks.
|
||||
This command automatically maps language+keyword combinations to their official documentation sites.
|
||||
|
||||
Supported languages:
|
||||
go, golang - Go documentation (pkg.go.dev)
|
||||
rust - Rust documentation (docs.rs)
|
||||
python, py - Python documentation (docs.python.org)
|
||||
java - Java documentation (docs.oracle.com)
|
||||
spring - Spring Boot documentation (docs.spring.io)
|
||||
typescript, ts - TypeScript documentation (typescriptlang.org)
|
||||
react - React documentation (react.dev)
|
||||
vue - Vue.js documentation (vuejs.org)
|
||||
nuxt - Nuxt documentation (nuxt.com)
|
||||
docker - Docker documentation (docs.docker.com)
|
||||
cloudflare, cf - Cloudflare documentation (developers.cloudflare.com)
|
||||
astro - Astro documentation (docs.astro.build)
|
||||
This command maps language+keyword combinations to official documentation sources.
|
||||
|
||||
Examples:
|
||||
devour get go http # Go HTTP package documentation
|
||||
devour get python asyncio # Python asyncio module
|
||||
devour get react hooks # React Hooks documentation
|
||||
devour get docker compose # Docker Compose docs
|
||||
devour get rust tokio # Rust Tokio crate`,
|
||||
devour get go http
|
||||
devour get python asyncio
|
||||
devour get react hooks
|
||||
devour get nextjs routing
|
||||
devour get express middleware`,
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: runGet,
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Add flags that can override defaults
|
||||
getCmd.Flags().StringVarP(&scrapeFormat, "format", "f", "json", "output format (json, markdown)")
|
||||
getCmd.Flags().StringVarP(&scrapeOutput, "output", "o", "", "output directory (default: devour_data/docs)")
|
||||
getCmd.Flags().StringVarP(&scrapeOutput, "output", "o", "", "output directory (default: configured docs dir)")
|
||||
getCmd.Flags().IntVar(&scrapeConcurrency, "concurrency", 10, "parallel scraping workers")
|
||||
}
|
||||
|
||||
func runGet(cmd *cobra.Command, args []string) error {
|
||||
language := strings.ToLower(args[0])
|
||||
keyword := strings.ToLower(args[1])
|
||||
langIn := strings.ToLower(strings.TrimSpace(args[0]))
|
||||
keyword := strings.TrimSpace(args[1])
|
||||
if keyword == "" {
|
||||
return fmt.Errorf("keyword is required")
|
||||
}
|
||||
|
||||
language, ok := normalizeLanguage(langIn)
|
||||
if !ok {
|
||||
return fmt.Errorf("unsupported language: %s. Supported: %s", langIn, strings.Join(supportedLanguages(), ", "))
|
||||
}
|
||||
|
||||
// Map language to base URL and construct full URL
|
||||
url, err := constructDocURL(language, keyword)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set the scrape type based on language
|
||||
sourceType := mapLanguageToType(language)
|
||||
|
||||
// Reuse the existing scrape logic with pre-determined values
|
||||
scrapeType = string(sourceType)
|
||||
sourceURL := url
|
||||
scrapeType = sourceType
|
||||
|
||||
fmt.Printf("Getting docs for: %s %s\n", language, keyword)
|
||||
fmt.Printf("URL: %s\n", sourceURL)
|
||||
fmt.Printf("Type: %s\n", sourceType)
|
||||
fmt.Println()
|
||||
fmt.Printf("URL: %s\n", url)
|
||||
fmt.Printf("Type: %s\n\n", sourceType)
|
||||
|
||||
// Call the existing scrape logic
|
||||
return runScrape(cmd, []string{sourceURL})
|
||||
return runScrape(cmd, []string{url})
|
||||
}
|
||||
|
||||
func constructDocURL(language, keyword string) (string, error) {
|
||||
language = strings.ToLower(strings.TrimSpace(language))
|
||||
keyword = strings.TrimSpace(keyword)
|
||||
lowerKeyword := strings.ToLower(keyword)
|
||||
|
||||
switch language {
|
||||
case "go", "golang":
|
||||
return fmt.Sprintf("https://pkg.go.dev/%s", keyword), nil
|
||||
case "go":
|
||||
return fmt.Sprintf("https://pkg.go.dev/%s", lowerKeyword), nil
|
||||
case "rust":
|
||||
return fmt.Sprintf("https://docs.rs/%s/latest/%s/", keyword, keyword), nil
|
||||
case "python", "py":
|
||||
if keyword == "stdlib" || keyword == "standard" {
|
||||
return fmt.Sprintf("https://docs.rs/%s/latest/%s/", lowerKeyword, lowerKeyword), nil
|
||||
case "python":
|
||||
if lowerKeyword == "stdlib" || lowerKeyword == "standard" {
|
||||
return "https://docs.python.org/3/library/", nil
|
||||
}
|
||||
return fmt.Sprintf("https://docs.python.org/3/library/%s.html", keyword), nil
|
||||
return fmt.Sprintf("https://docs.python.org/3/library/%s.html", lowerKeyword), nil
|
||||
case "java":
|
||||
return fmt.Sprintf("https://docs.oracle.com/javase/8/docs/api/%s.html", keyword), nil
|
||||
return fmt.Sprintf("https://docs.oracle.com/javase/8/docs/api/%s.html", lowerKeyword), nil
|
||||
case "spring":
|
||||
return fmt.Sprintf("https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#%s", keyword), nil
|
||||
case "typescript", "ts":
|
||||
return fmt.Sprintf("https://www.typescriptlang.org/docs/handbook/%s.html", keyword), nil
|
||||
if lowerKeyword == "mcp" || lowerKeyword == "mcp-overview" {
|
||||
return "https://docs.spring.io/spring-ai/reference/api/mcp/mcp-overview.html", nil
|
||||
}
|
||||
return fmt.Sprintf("https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#%s", lowerKeyword), nil
|
||||
case "typescript":
|
||||
return fmt.Sprintf("https://www.typescriptlang.org/docs/handbook/%s.html", lowerKeyword), nil
|
||||
case "react":
|
||||
return fmt.Sprintf("https://react.dev/reference/react/%s", keyword), nil
|
||||
if lowerKeyword == "hooks" {
|
||||
return "https://react.dev/reference/react", nil
|
||||
}
|
||||
return fmt.Sprintf("https://react.dev/reference/react/%s", lowerKeyword), nil
|
||||
case "vue":
|
||||
return fmt.Sprintf("https://vuejs.org/guide/%s.html", keyword), nil
|
||||
if strings.Contains(lowerKeyword, "api") {
|
||||
return "https://vuejs.org/api/", nil
|
||||
}
|
||||
return fmt.Sprintf("https://vuejs.org/guide/%s.html", lowerKeyword), nil
|
||||
case "nuxt":
|
||||
return fmt.Sprintf("https://nuxt.com/docs/guide/%s", keyword), nil
|
||||
return fmt.Sprintf("https://nuxt.com/docs/guide/%s", lowerKeyword), nil
|
||||
case "docker":
|
||||
return fmt.Sprintf("https://docs.docker.com/%s", keyword), nil
|
||||
case "cloudflare", "cf":
|
||||
return fmt.Sprintf("https://developers.cloudflare.com/%s", keyword), nil
|
||||
return fmt.Sprintf("https://docs.docker.com/%s", lowerKeyword), nil
|
||||
case "cloudflare":
|
||||
return fmt.Sprintf("https://developers.cloudflare.com/%s", lowerKeyword), nil
|
||||
case "astro":
|
||||
return fmt.Sprintf("https://docs.astro.build/en/guides/%s", keyword), nil
|
||||
path := lowerKeyword
|
||||
switch lowerKeyword {
|
||||
case "components":
|
||||
path = "basics/astro-components"
|
||||
case "api":
|
||||
path = "reference/api-reference"
|
||||
case "install", "setup", "getting-started":
|
||||
path = "install-and-setup"
|
||||
default:
|
||||
if !strings.Contains(lowerKeyword, "/") {
|
||||
path = "guides/" + lowerKeyword
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("https://docs.astro.build/en/%s/", path), nil
|
||||
case "csharp":
|
||||
lowerKeyword = strings.TrimPrefix(lowerKeyword, "/")
|
||||
if strings.Contains(lowerKeyword, "regex") || strings.Contains(lowerKeyword, "regular-expression") {
|
||||
return "https://learn.microsoft.com/en-us/dotnet/standard/base-types/regular-expressions", nil
|
||||
}
|
||||
return fmt.Sprintf("https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/%s", lowerKeyword), nil
|
||||
case "kotlin":
|
||||
lowerKeyword = strings.TrimPrefix(lowerKeyword, "/")
|
||||
if lowerKeyword == "regex" || lowerKeyword == "regexp" {
|
||||
lowerKeyword = "strings"
|
||||
}
|
||||
if strings.HasSuffix(lowerKeyword, ".html") {
|
||||
return fmt.Sprintf("https://kotlinlang.org/docs/%s", lowerKeyword), nil
|
||||
}
|
||||
return fmt.Sprintf("https://kotlinlang.org/docs/%s.html", lowerKeyword), nil
|
||||
case "php":
|
||||
lowerKeyword = strings.TrimPrefix(lowerKeyword, "/")
|
||||
if strings.HasSuffix(lowerKeyword, ".php") || strings.Contains(lowerKeyword, "function.") || strings.Contains(lowerKeyword, "book.") {
|
||||
return fmt.Sprintf("https://www.php.net/manual/en/%s", lowerKeyword), nil
|
||||
}
|
||||
return fmt.Sprintf("https://www.php.net/manual/en/book.%s.php", lowerKeyword), nil
|
||||
case "ruby":
|
||||
keyword = strings.TrimPrefix(keyword, "/")
|
||||
switch strings.ToLower(keyword) {
|
||||
case "regex", "regexp":
|
||||
keyword = "Regexp"
|
||||
case "string":
|
||||
keyword = "String"
|
||||
case "array":
|
||||
keyword = "Array"
|
||||
default:
|
||||
if !strings.Contains(keyword, "::") && len(keyword) > 0 {
|
||||
keyword = strings.ToUpper(keyword[:1]) + strings.ToLower(keyword[1:])
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("https://ruby-doc.org/core/%s.html", keyword), nil
|
||||
case "elixir":
|
||||
keyword = strings.TrimPrefix(keyword, "/")
|
||||
switch strings.ToLower(keyword) {
|
||||
case "regex":
|
||||
keyword = "Regex"
|
||||
case "string":
|
||||
keyword = "String"
|
||||
case "enum":
|
||||
keyword = "Enum"
|
||||
default:
|
||||
if len(keyword) > 0 {
|
||||
keyword = strings.ToUpper(keyword[:1]) + strings.ToLower(keyword[1:])
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("https://hexdocs.pm/elixir/%s.html", keyword), nil
|
||||
case "nextjs":
|
||||
if strings.Contains(lowerKeyword, "routing") {
|
||||
return "https://nextjs.org/docs/app/building-your-application/routing", nil
|
||||
}
|
||||
if strings.Contains(lowerKeyword, "data") || strings.Contains(lowerKeyword, "fetch") {
|
||||
return "https://nextjs.org/docs/app/building-your-application/data-fetching", nil
|
||||
}
|
||||
return "https://nextjs.org/docs", nil
|
||||
case "svelte":
|
||||
if strings.Contains(lowerKeyword, "kit") {
|
||||
return "https://svelte.dev/docs/kit", nil
|
||||
}
|
||||
return "https://svelte.dev/docs/svelte/overview", nil
|
||||
case "angular":
|
||||
if strings.Contains(lowerKeyword, "http") {
|
||||
return "https://angular.dev/guide/http", nil
|
||||
}
|
||||
return "https://angular.dev/guide/components", nil
|
||||
case "remix":
|
||||
if strings.Contains(lowerKeyword, "route") {
|
||||
return "https://v2.remix.run/docs/file-conventions/routes", nil
|
||||
}
|
||||
return "https://v2.remix.run/docs", nil
|
||||
case "solid":
|
||||
// Solid docs are published from this repository and include solid-router content.
|
||||
return "https://github.com/solidjs/solid-docs", nil
|
||||
case "express":
|
||||
if strings.Contains(lowerKeyword, "routing") {
|
||||
return "https://expressjs.com/en/guide/routing.html", nil
|
||||
}
|
||||
if strings.Contains(lowerKeyword, "middleware") {
|
||||
return "https://expressjs.com/en/guide/using-middleware.html", nil
|
||||
}
|
||||
return "https://expressjs.com/en/guide/writing-middleware.html", nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported language: %s. Supported languages: go, rust, python, java, spring, typescript, react, vue, nuxt, docker, cloudflare, astro", language)
|
||||
return "", fmt.Errorf("unsupported language: %s. Supported: %s", language, strings.Join(supportedLanguages(), ", "))
|
||||
}
|
||||
}
|
||||
|
||||
func mapLanguageToType(language string) string {
|
||||
language, _ = normalizeLanguage(language)
|
||||
switch language {
|
||||
case "go", "golang":
|
||||
case "go":
|
||||
return "godocs"
|
||||
case "rust":
|
||||
return "rustdocs"
|
||||
case "python", "py":
|
||||
case "python":
|
||||
return "pythondocs"
|
||||
case "java":
|
||||
return "javadocs"
|
||||
case "spring":
|
||||
return "springdocs"
|
||||
case "typescript", "ts":
|
||||
case "typescript":
|
||||
return "tsdocs"
|
||||
case "react":
|
||||
return "reactdocs"
|
||||
@@ -126,11 +225,80 @@ func mapLanguageToType(language string) string {
|
||||
return "nuxtdocs"
|
||||
case "docker":
|
||||
return "dockerdocs"
|
||||
case "cloudflare", "cf":
|
||||
case "cloudflare":
|
||||
return "cloudflaredocs"
|
||||
case "astro":
|
||||
return "astrodocs"
|
||||
case "csharp", "kotlin", "php", "ruby", "elixir", "nextjs", "svelte", "angular", "remix", "express":
|
||||
return "url"
|
||||
case "solid":
|
||||
return "github"
|
||||
default:
|
||||
return "web"
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeLanguage(language string) (string, bool) {
|
||||
language = strings.ToLower(strings.TrimSpace(language))
|
||||
if language == "" {
|
||||
return "", false
|
||||
}
|
||||
if canonical, ok := languageAliases()[language]; ok {
|
||||
return canonical, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func languageAliases() map[string]string {
|
||||
return map[string]string{
|
||||
"go": "go",
|
||||
"golang": "go",
|
||||
"rust": "rust",
|
||||
"python": "python",
|
||||
"py": "python",
|
||||
"java": "java",
|
||||
"spring": "spring",
|
||||
"typescript": "typescript",
|
||||
"ts": "typescript",
|
||||
"react": "react",
|
||||
"vue": "vue",
|
||||
"nuxt": "nuxt",
|
||||
"docker": "docker",
|
||||
"cloudflare": "cloudflare",
|
||||
"cf": "cloudflare",
|
||||
"astro": "astro",
|
||||
"csharp": "csharp",
|
||||
"cs": "csharp",
|
||||
"kotlin": "kotlin",
|
||||
"kt": "kotlin",
|
||||
"php": "php",
|
||||
"ruby": "ruby",
|
||||
"rb": "ruby",
|
||||
"elixir": "elixir",
|
||||
"ex": "elixir",
|
||||
"next": "nextjs",
|
||||
"nextjs": "nextjs",
|
||||
"svelte": "svelte",
|
||||
"angular": "angular",
|
||||
"ng": "angular",
|
||||
"remix": "remix",
|
||||
"solid": "solid",
|
||||
"solidjs": "solid",
|
||||
"express": "express",
|
||||
"expressjs": "express",
|
||||
}
|
||||
}
|
||||
|
||||
func supportedLanguages() []string {
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0)
|
||||
for _, canonical := range languageAliases() {
|
||||
if seen[canonical] {
|
||||
continue
|
||||
}
|
||||
seen[canonical] = true
|
||||
out = append(out, canonical)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user