feat: phase 4 integrations — connectors, rybbit analytics, webhooks

- core.integrations (migration 00005) with AES-256-GCM-encrypted
  credentials (PRIMORA_ENCRYPTION_KEY, hex/base64, dev fallback);
  has_credentials serialization only — secrets never leave the server
- /projects/:id/integrations list/create/delete/test; health check hits
  the stored base URL with a 5s timeout and records status
- Rybbit connector: GET …/integrations/:id/analytics proxies
  /api/organizations + /api/sites/* server-side into a normalized shape
- core.webhooks + core.webhook_deliveries; CRUD, deliveries, test, and
  POST …/deploy-markers. Events: issue.created (new fingerprint on
  ingest), deploy.marker, webhook.test. X-Primora-Signature HMAC-SHA256
  over the raw body; in-process dispatcher, 3 attempts, quadratic
  backoff, 10s timeout, pending-scan on boot
- HTTPS enforced for public targets; HTTP allowed for loopback/private/
  self-hosted sinks
- Integrations page (connectors + webhooks tabs, inline analytics,
  delivery log, secret-shown-once), demo-mode stubs
- 12 new primora_* MCP tools
- workspace 0.4.0

Authored By: TDvorak <[email protected]>
This commit is contained in:
Tomas Dvorak committed 2026-09-16 18:47:31 +02:00
1 parent 8cfddf9184
commit e25eb92339
82 files changed
+5710 -61

No files matched your search

@@ -0,0 +1,269 @@
package services
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
db "github.com/tdvorak/primora/apps/backend/internal/database/db"
"github.com/tdvorak/primora/apps/backend/internal/models"
)
// IntegrationCredentials is the secret payload stored AES-256-GCM encrypted in
// core.integrations.credentials. It is never serialized to the client — see
// IntegrationSummary.
type IntegrationCredentials struct {
APIKey string `json:"api_key,omitempty"`
}
// IntegrationConfig holds non-secret connector settings (core.integrations.config).
type IntegrationConfig struct {
SiteID string `json:"site_id,omitempty"`
}
type CreateIntegrationInput struct {
Name string `json:"name" validate:"required,min=1"`
Type string `json:"type" validate:"required,oneof=rybbit"`
BaseURL string `json:"base_url" validate:"required,url"`
APIKey string `json:"api_key"`
SiteID string `json:"site_id"`
}
// IntegrationSummary is the client-facing shape — credentials reduced to a flag.
type IntegrationSummary struct {
ID uuid.UUID `json:"id"`
ProjectID uuid.UUID `json:"project_id"`
Type string `json:"type"`
Name string `json:"name"`
BaseURL string `json:"base_url"`
SiteID string `json:"site_id"`
Status string `json:"status"`
LastHealthAt *time.Time `json:"last_health_at"`
HasCredentials bool `json:"has_credentials"`
CreatedAt time.Time `json:"created_at"`
}
var integrationTypes = map[string]bool{"rybbit": true}
func parseIntegrationConfig(raw []byte) IntegrationConfig {
var cfg IntegrationConfig
_ = json.Unmarshal(raw, &cfg)
return cfg
}
func toIntegrationSummary(row db.CoreIntegration) IntegrationSummary {
cfg := parseIntegrationConfig(row.Config)
out := IntegrationSummary{
ID: row.ID,
ProjectID: row.ProjectID,
Type: row.Type,
Name: row.Name,
BaseURL: row.BaseUrl,
SiteID: cfg.SiteID,
Status: row.Status,
HasCredentials: len(row.Credentials) > 0,
CreatedAt: row.CreatedAt.Time,
}
if row.LastHealthAt.Valid {
t := row.LastHealthAt.Time
out.LastHealthAt = &t
}
return out
}
func normalizeBaseURL(raw string) (string, error) {
trimmed := strings.TrimSpace(strings.TrimRight(raw, "/"))
u, err := url.Parse(trimmed)
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" {
return "", errors.New("base_url must be an http(s) URL")
}
return trimmed, nil
}
func (s *PlatformService) decryptCredentials(row db.CoreIntegration) (IntegrationCredentials, error) {
var creds IntegrationCredentials
if len(row.Credentials) == 0 {
return creds, nil
}
if s.enc == nil {
return creds, errors.New("encryption not configured")
}
plain, err := s.enc.Decrypt(row.Credentials)
if err != nil {
return creds, fmt.Errorf("credentials unreadable: %w", err)
}
if err := json.Unmarshal(plain, &creds); err != nil {
return creds, fmt.Errorf("credentials malformed: %w", err)
}
return creds, nil
}
func (s *PlatformService) integrationFor(ctx context.Context, actor *models.Actor, projectID, integrationID uuid.UUID, write bool) (db.CoreIntegration, error) {
roles := []string{"admin", "developer", "viewer"}
if write {
roles = []string{"admin", "developer"}
}
if err := s.requireProjectRole(ctx, actor, projectID, roles...); err != nil {
return db.CoreIntegration{}, err
}
row, err := s.repo.Queries().GetIntegrationByID(ctx, integrationID)
if err != nil {
return db.CoreIntegration{}, err
}
if row.ProjectID != projectID {
return db.CoreIntegration{}, fmt.Errorf("integration access denied")
}
return row, nil
}
func (s *PlatformService) ListIntegrations(ctx context.Context, actor *models.Actor, projectID uuid.UUID) ([]IntegrationSummary, error) {
if err := s.requireProjectRole(ctx, actor, projectID, "admin", "developer", "viewer"); err != nil {
return nil, err
}
rows, err := s.repo.Queries().ListIntegrations(ctx, projectID)
if err != nil {
return nil, err
}
out := make([]IntegrationSummary, 0, len(rows))
for _, row := range rows {
out = append(out, toIntegrationSummary(row))
}
return out, nil
}
func (s *PlatformService) CreateIntegration(ctx context.Context, actor *models.Actor, projectID uuid.UUID, input CreateIntegrationInput, requestID string) (IntegrationSummary, error) {
if err := s.requireProjectRole(ctx, actor, projectID, "admin", "developer"); err != nil {
return IntegrationSummary{}, err
}
if !integrationTypes[input.Type] {
return IntegrationSummary{}, fmt.Errorf("unsupported integration type %q", input.Type)
}
name := normalizeSlug(input.Name)
if name == "" {
return IntegrationSummary{}, errors.New("name must contain letters or digits")
}
baseURL, err := normalizeBaseURL(input.BaseURL)
if err != nil {
return IntegrationSummary{}, err
}
var credBytes []byte
if strings.TrimSpace(input.APIKey) != "" {
raw, _ := json.Marshal(IntegrationCredentials{APIKey: strings.TrimSpace(input.APIKey)})
credBytes, err = s.enc.Encrypt(raw)
if err != nil {
return IntegrationSummary{}, err
}
}
cfgBytes, _ := json.Marshal(IntegrationConfig{SiteID: strings.TrimSpace(input.SiteID)})
var createdBy pgtype.UUID
if actor.UserID != nil {
createdBy = pgtype.UUID{Bytes: *actor.UserID, Valid: true}
}
row, err := s.repo.Queries().CreateIntegration(ctx, db.CreateIntegrationParams{
ProjectID: projectID,
Type: input.Type,
Name: name,
BaseUrl: baseURL,
Config: cfgBytes,
Credentials: credBytes,
CreatedByUserID: createdBy,
})
if err != nil {
if strings.Contains(err.Error(), "integrations_project_id_name_key") {
return IntegrationSummary{}, fmt.Errorf("an integration named %s already exists", name)
}
return IntegrationSummary{}, err
}
project, err := s.repo.Queries().GetProjectByID(ctx, projectID)
if err == nil {
_, _ = s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "integration.created", "integration", row.ID.String(), map[string]any{
"name": row.Name, "type": row.Type, "base_url": row.BaseUrl,
}))
}
return toIntegrationSummary(row), nil
}
func (s *PlatformService) DeleteIntegration(ctx context.Context, actor *models.Actor, projectID, integrationID uuid.UUID, requestID string) error {
row, err := s.integrationFor(ctx, actor, projectID, integrationID, true)
if err != nil {
return err
}
if _, err := s.repo.Queries().DeleteIntegration(ctx, db.DeleteIntegrationParams{
ID: integrationID,
ProjectID: projectID,
}); err != nil {
return err
}
project, _ := s.repo.Queries().GetProjectByID(ctx, projectID)
_, _ = s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "integration.deleted", "integration", integrationID.String(), map[string]any{
"name": row.Name, "type": row.Type,
}))
return nil
}
// integrationProbePath is the lightest authenticated endpoint per type; it
// verifies reachability and credentials in one request.
func integrationProbePath(integrationType string) string {
if integrationType == "rybbit" {
return "/api/organizations"
}
return "/"
}
func (s *PlatformService) TestIntegration(ctx context.Context, actor *models.Actor, projectID, integrationID uuid.UUID, requestID string) (map[string]any, error) {
row, err := s.integrationFor(ctx, actor, projectID, integrationID, true)
if err != nil {
return nil, err
}
creds, err := s.decryptCredentials(row)
if err != nil {
return nil, err
}
status := "error"
result := map[string]any{"ok": false}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, row.BaseUrl+integrationProbePath(row.Type), nil)
if err == nil {
if creds.APIKey != "" {
req.Header.Set("Authorization", "Bearer "+creds.APIKey)
}
client := &http.Client{Timeout: 5 * time.Second}
resp, reqErr := client.Do(req)
if reqErr != nil {
result["error"] = reqErr.Error()
} else {
defer resp.Body.Close()
result["status_code"] = resp.StatusCode
if resp.StatusCode >= 200 && resp.StatusCode < 400 {
result["ok"] = true
status = "ok"
} else {
result["error"] = fmt.Sprintf("unexpected status %d", resp.StatusCode)
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
result["error"] = "credentials rejected"
}
}
}
} else {
result["error"] = err.Error()
}
_ = s.repo.Queries().UpdateIntegrationHealth(ctx, db.UpdateIntegrationHealthParams{
ID: row.ID,
Status: status,
})
if project, err := s.repo.Queries().GetProjectByID(ctx, projectID); err == nil {
_, _ = s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "integration.tested", "integration", integrationID.String(), map[string]any{
"name": row.Name, "status": status,
}))
}
return result, nil
}
@@ -0,0 +1,107 @@
package services
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"strings"
"testing"
"github.com/google/uuid"
db "github.com/tdvorak/primora/apps/backend/internal/database/db"
)
func TestSignBodyMatchesReceiverVerification(t *testing.T) {
secret := "whsec_test_123"
body := []byte(`{"event":"issue.created","data":{"fingerprint":"abc"}}`)
got := signBody(secret, body)
mac := hmac.New(sha256.New, []byte(secret))
mac.Write(body)
want := hex.EncodeToString(mac.Sum(nil))
if got != want {
t.Fatalf("signature mismatch: got %s want %s", got, want)
}
if len(got) != 64 {
t.Fatalf("expected 64-char hex digest, got %d", len(got))
}
if signBody(secret, []byte(`{"event":"issue.created"}`)) == got {
t.Error("different body must produce a different signature")
}
if signBody("other-secret", body) == got {
t.Error("different secret must produce a different signature")
}
}
func TestIntegrationSummaryNeverExposesCredentials(t *testing.T) {
ciphertext := []byte("nonce|aes-gcm-ciphertext|contains|api_key=super-secret")
row := db.CoreIntegration{
ID: uuid.New(),
ProjectID: uuid.New(),
Type: "rybbit",
Name: "prod",
BaseUrl: "https://analytics.example.com",
Config: []byte(`{"site_id":"s1"}`),
Credentials: ciphertext,
Status: "ok",
}
summary := toIntegrationSummary(row)
if !summary.HasCredentials {
t.Error("expected has_credentials=true")
}
raw, err := json.Marshal(summary)
if err != nil {
t.Fatalf("marshal: %v", err)
}
for _, leak := range []string{"super-secret", "ciphertext", "api_key", "\"credentials\"", "aes-gcm"} {
if strings.Contains(string(raw), leak) {
t.Errorf("serialized summary leaks %q: %s", leak, raw)
}
}
}
func TestNormalizeWebhookURL(t *testing.T) {
ok := []string{
"https://hooks.example.com/x",
"http://localhost:9000/sink",
"http://127.0.0.1:9000/sink",
"http://primora-sink:9000/",
"http://192.168.1.40:8080/hook",
"http://nas.internal/events",
}
for _, u := range ok {
if _, err := normalizeWebhookURL(u); err != nil {
t.Errorf("expected %q accepted: %v", u, err)
}
}
bad := []string{
"",
"not-a-url",
"ftp://example.com",
"http://example.com/plain-remote",
"http://8.8.8.8/x",
"https://",
}
for _, u := range bad {
if _, err := normalizeWebhookURL(u); err == nil {
t.Errorf("expected %q rejected", u)
}
}
}
func TestValidateWebhookEvents(t *testing.T) {
got, err := validateWebhookEvents([]string{"issue.created", "deploy.marker", "issue.created", " "})
if err != nil {
t.Fatalf("validate: %v", err)
}
if len(got) != 2 {
t.Fatalf("expected deduped 2 events, got %v", got)
}
if _, err := validateWebhookEvents([]string{"bogus.event"}); err == nil {
t.Error("expected unknown event rejected")
}
}
@@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"io"
"log/slog"
"net/url"
"path/filepath"
"strings"
@@ -22,6 +23,7 @@ import (
"github.com/tdvorak/primora/apps/backend/internal/dbx"
"github.com/tdvorak/primora/apps/backend/internal/models"
"github.com/tdvorak/primora/apps/backend/internal/repositories"
"github.com/tdvorak/primora/apps/backend/internal/secrets"
"github.com/tdvorak/primora/apps/backend/internal/storage"
)
@@ -33,6 +35,8 @@ type PlatformService struct {
dbx *dbx.Client
managedDBSeeds []ManagedDBSeed
hub *EventHub
enc *secrets.Encryptor
dispatcher *WebhookDispatcher
}
type BootstrapInput struct {
@@ -120,8 +124,8 @@ type CreateCollectionInput struct {
}
type UpdateCollectionInput struct {
Name *string `json:"name" validate:"omitempty,min=2"`
Description *string `json:"description"`
Name *string `json:"name" validate:"omitempty,min=2"`
Description *string `json:"description"`
Schema map[string]any `json:"schema"`
}
@@ -210,8 +214,20 @@ type InvitationSummary struct {
Status string `json:"status"`
}
func NewPlatformService(repo *repositories.CoreRepository, store *storage.LocalStore, mailer *Mailer, publicURL string, dbxClient *dbx.Client, managedDBSeeds []ManagedDBSeed) *PlatformService {
return &PlatformService{repo: repo, store: store, mailer: mailer, publicURL: publicURL, dbx: dbxClient, managedDBSeeds: managedDBSeeds, hub: NewEventHub()}
func NewPlatformService(repo *repositories.CoreRepository, store *storage.LocalStore, mailer *Mailer, publicURL string, dbxClient *dbx.Client, managedDBSeeds []ManagedDBSeed, enc *secrets.Encryptor, logger *slog.Logger) *PlatformService {
s := &PlatformService{repo: repo, store: store, mailer: mailer, publicURL: publicURL, dbx: dbxClient, managedDBSeeds: managedDBSeeds, hub: NewEventHub(), enc: enc}
if enc != nil {
s.dispatcher = NewWebhookDispatcher(repo.Queries(), enc, logger)
s.dispatcher.Start(context.Background())
}
return s
}
// Close stops the webhook dispatcher. Idempotent.
func (s *PlatformService) Close() {
if s.dispatcher != nil {
s.dispatcher.Stop()
}
}
func (s *PlatformService) Me(ctx context.Context, actor *models.Actor) (PlatformSummary, error) {
+274
View File
@@ -0,0 +1,274 @@
package services
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"time"
"github.com/google/uuid"
"github.com/tdvorak/primora/apps/backend/internal/models"
)
// rybbitClient is a read-only client for a self-hosted Rybbit instance's
// documented stats API (/api/*). The Bearer key stays server-side.
type rybbitClient struct {
base string
apiKey string
http *http.Client
}
func newRybbitClient(base, apiKey string) *rybbitClient {
return &rybbitClient{base: base, apiKey: apiKey, http: &http.Client{Timeout: 8 * time.Second}}
}
// get fetches path and unmarshals the body. Non-2xx surfaces the status code.
func (c *rybbitClient) get(ctx context.Context, path string, query url.Values, out any) error {
u := c.base + path
if len(query) > 0 {
u += "?" + query.Encode()
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return err
}
if c.apiKey != "" {
req.Header.Set("Authorization", "Bearer "+c.apiKey)
}
resp, err := c.http.Do(req)
if err != nil {
return fmt.Errorf("rybbit unreachable: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if err != nil {
return err
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("rybbit returned status %d", resp.StatusCode)
}
if len(body) == 0 {
return nil
}
return json.Unmarshal(body, out)
}
type RybbitSite struct {
SiteID string `json:"site_id"`
Name string `json:"name"`
Domain string `json:"domain,omitempty"`
}
type RybbitOverview struct {
Sessions *float64 `json:"sessions,omitempty"`
Pageviews *float64 `json:"pageviews,omitempty"`
Users *float64 `json:"users,omitempty"`
PagesPerSession *float64 `json:"pages_per_session,omitempty"`
BounceRate *float64 `json:"bounce_rate,omitempty"`
SessionDuration *float64 `json:"session_duration,omitempty"`
}
type RybbitSeriesPoint struct {
Time string `json:"time"`
Sessions *float64 `json:"sessions,omitempty"`
Pageviews *float64 `json:"pageviews,omitempty"`
Users *float64 `json:"users,omitempty"`
BounceRate *float64 `json:"bounce_rate,omitempty"`
}
type RybbitMetricItem struct {
Value string `json:"value"`
Count float64 `json:"count"`
Percentage *float64 `json:"percentage,omitempty"`
}
type IntegrationAnalyticsResult struct {
IntegrationID uuid.UUID `json:"integration_id"`
Site *RybbitSite `json:"site,omitempty"`
Sites []RybbitSite `json:"sites,omitempty"`
Overview *RybbitOverview `json:"overview,omitempty"`
Series []RybbitSeriesPoint `json:"series,omitempty"`
TopPages []RybbitMetricItem `json:"top_pages,omitempty"`
TopReferrers []RybbitMetricItem `json:"top_referrers,omitempty"`
WindowDays int `json:"window_days"`
}
// rybbitOrg normalizes the few shapes GET /api/organizations returns across
// versions ([]org, {data:[...]}, {organizations:[...]}).
type rybbitOrg struct {
ID string `json:"id"`
Name string `json:"name"`
Slug string `json:"slug"`
}
func (c *rybbitClient) organizations(ctx context.Context) ([]rybbitOrg, error) {
var raw json.RawMessage
if err := c.get(ctx, "/api/organizations", nil, &raw); err != nil {
return nil, err
}
var list []rybbitOrg
if err := json.Unmarshal(raw, &list); err == nil {
return list, nil
}
for _, key := range []string{"data", "organizations"} {
var wrapped map[string]json.RawMessage
if err := json.Unmarshal(raw, &wrapped); err != nil {
break
}
if inner, ok := wrapped[key]; ok {
if err := json.Unmarshal(inner, &list); err == nil {
return list, nil
}
}
}
return nil, errors.New("unexpected /api/organizations shape")
}
func (c *rybbitClient) orgSites(ctx context.Context, orgID string) ([]RybbitSite, error) {
var resp struct {
Sites []struct {
SiteID json.Number `json:"siteId"`
Name string `json:"name"`
Domain string `json:"domain"`
} `json:"sites"`
}
if err := c.get(ctx, "/api/organizations/"+url.PathEscape(orgID)+"/sites", nil, &resp); err != nil {
return nil, err
}
out := make([]RybbitSite, 0, len(resp.Sites))
for _, site := range resp.Sites {
out = append(out, RybbitSite{SiteID: site.SiteID.String(), Name: site.Name, Domain: site.Domain})
}
return out, nil
}
func (c *rybbitClient) siteDetail(ctx context.Context, siteID string) (*RybbitSite, error) {
var resp struct {
SiteID json.Number `json:"siteId"`
Name string `json:"name"`
Domain string `json:"domain"`
}
if err := c.get(ctx, "/api/sites/"+url.PathEscape(siteID), nil, &resp); err != nil {
return nil, err
}
return &RybbitSite{SiteID: resp.SiteID.String(), Name: resp.Name, Domain: resp.Domain}, nil
}
func rybbitRangeQuery(days int) url.Values {
end := time.Now().UTC()
start := end.AddDate(0, 0, -days)
return url.Values{
"start_date": {start.Format("2006-01-02")},
"end_date": {end.Format("2006-01-02")},
"time_zone": {"UTC"},
}
}
// IntegrationAnalytics pulls normalized stats from a rybbit-type integration.
// Site resolution: ?site= override → configured site_id → first accessible site.
func (s *PlatformService) IntegrationAnalytics(ctx context.Context, actor *models.Actor, projectID, integrationID uuid.UUID, siteOverride string, days int) (*IntegrationAnalyticsResult, error) {
row, err := s.integrationFor(ctx, actor, projectID, integrationID, false)
if err != nil {
return nil, err
}
if row.Type != "rybbit" {
return nil, fmt.Errorf("integration %s does not provide analytics", row.Type)
}
creds, err := s.decryptCredentials(row)
if err != nil {
return nil, err
}
if creds.APIKey == "" {
return nil, errors.New("integration has no credentials configured")
}
if days <= 0 || days > 365 {
days = 30
}
client := newRybbitClient(row.BaseUrl, creds.APIKey)
cfg := parseIntegrationConfig(row.Config)
result := &IntegrationAnalyticsResult{IntegrationID: row.ID, WindowDays: days}
siteID := siteOverride
if siteID == "" {
siteID = cfg.SiteID
}
// Site discovery only runs when no site is pinned — it needs an
// org-scope endpoint personal keys may not expose.
if siteID == "" {
orgs, err := client.organizations(ctx)
if err != nil {
return nil, fmt.Errorf("site discovery failed: %w — set site_id on the integration or pass ?site=", err)
}
for _, org := range orgs {
orgID := org.ID
if orgID == "" {
orgID = org.Slug
}
if orgID == "" {
continue
}
sites, err := client.orgSites(ctx, orgID)
if err != nil {
continue
}
result.Sites = append(result.Sites, sites...)
}
if len(result.Sites) == 0 {
return nil, errors.New("no rybbit sites found — set site_id on the integration")
}
siteID = result.Sites[0].SiteID
}
site, err := client.siteDetail(ctx, siteID)
if err == nil {
result.Site = site
} else {
result.Site = &RybbitSite{SiteID: siteID}
}
query := rybbitRangeQuery(days)
var overview struct {
Data RybbitOverview `json:"data"`
}
if err := client.get(ctx, "/api/sites/"+url.PathEscape(siteID)+"/overview", query, &overview); err != nil {
return nil, err
}
result.Overview = &overview.Data
seriesQuery := rybbitRangeQuery(days)
seriesQuery.Set("bucket", "day")
var series struct {
Data []RybbitSeriesPoint `json:"data"`
}
if err := client.get(ctx, "/api/sites/"+url.PathEscape(siteID)+"/overview/time-series", seriesQuery, &series); err == nil {
result.Series = series.Data
}
var metric struct {
Data struct {
Data []RybbitMetricItem `json:"data"`
} `json:"data"`
}
pagesQuery := rybbitRangeQuery(days)
pagesQuery.Set("parameter", "pathname")
pagesQuery.Set("limit", "10")
if err := client.get(ctx, "/api/sites/"+url.PathEscape(siteID)+"/metric", pagesQuery, &metric); err == nil {
result.TopPages = metric.Data.Data
}
referrerQuery := rybbitRangeQuery(days)
referrerQuery.Set("parameter", "referrer")
referrerQuery.Set("limit", "10")
if err := client.get(ctx, "/api/sites/"+url.PathEscape(siteID)+"/metric", referrerQuery, &metric); err == nil {
result.TopReferrers = metric.Data.Data
}
return result, nil
}
@@ -89,6 +89,7 @@ func (s *PlatformService) IngestEvents(ctx context.Context, actor *models.Actor,
projectID := *actor.ProjectID
var inserted []TelemetryEvent
var newIssues []TelemetryEvent
err := s.repo.WithTx(ctx, func(q *db.Queries) error {
compCache := map[string]*uuid.UUID{}
for _, ev := range events {
@@ -136,6 +137,16 @@ func (s *PlatformService) IngestEvents(ctx context.Context, actor *models.Actor,
if len(payload) == 0 {
payload = json.RawMessage(`{}`)
}
newFingerprint := false
if ev.Type == "error" && fp != "" {
seen, err := q.HasFingerprintSeen(ctx, db.HasFingerprintSeenParams{
ProjectID: projectID,
Fingerprint: fp,
})
if err == nil && !seen {
newFingerprint = true
}
}
var cid pgtype.UUID
if compID != nil {
cid = pgtype.UUID{Bytes: *compID, Valid: true}
@@ -168,6 +179,9 @@ func (s *PlatformService) IngestEvents(ctx context.Context, actor *models.Actor,
out.ComponentName = ev.Component
}
inserted = append(inserted, out)
if newFingerprint {
newIssues = append(newIssues, out)
}
}
return nil
})
@@ -178,6 +192,18 @@ func (s *PlatformService) IngestEvents(ctx context.Context, actor *models.Actor,
for _, ev := range inserted {
s.hub.Broadcast(ev.ProjectID.String(), ev)
}
// A fresh error fingerprint is a new issue group — notify webhooks.
for _, issue := range newIssues {
s.dispatchEvent(ctx, projectID, WebhookEventIssueCreated, map[string]any{
"fingerprint": issue.Fingerprint,
"message": issue.Message,
"severity": issue.Severity,
"component_id": issue.ComponentID,
"component_name": issue.ComponentName,
"event_id": issue.ID,
"first_seen": issue.TS,
})
}
return len(inserted), nil
}
+664
View File
@@ -0,0 +1,664 @@
package services
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
db "github.com/tdvorak/primora/apps/backend/internal/database/db"
"github.com/tdvorak/primora/apps/backend/internal/models"
"github.com/tdvorak/primora/apps/backend/internal/secrets"
)
const (
WebhookEventIssueCreated = "issue.created"
WebhookEventDeployMarker = "deploy.marker"
WebhookEventTest = "webhook.test"
)
var webhookEventTypes = map[string]bool{
WebhookEventIssueCreated: true,
WebhookEventDeployMarker: true,
WebhookEventTest: true,
}
const webhookMaxAttempts = 3
type CreateWebhookInput struct {
URL string `json:"url" validate:"required,url"`
Secret string `json:"secret"`
Events []string `json:"events"`
Enabled *bool `json:"enabled"`
}
type UpdateWebhookInput struct {
URL *string `json:"url" validate:"omitempty,url"`
Secret *string `json:"secret"`
Events []string `json:"events"`
Enabled *bool `json:"enabled"`
}
type WebhookSummary struct {
ID uuid.UUID `json:"id"`
ProjectID uuid.UUID `json:"project_id"`
URL string `json:"url"`
Events []string `json:"events"`
Enabled bool `json:"enabled"`
HasSecret bool `json:"has_secret"`
CreatedAt time.Time `json:"created_at"`
// Secret is populated only when Primora generated the secret — it is the
// one chance to see it. Not part of the stored shape.
Secret string `json:"secret,omitempty"`
}
type WebhookDeliverySummary struct {
ID uuid.UUID `json:"id"`
WebhookID uuid.UUID `json:"webhook_id"`
EventType string `json:"event_type"`
Payload json.RawMessage `json:"payload"`
Status string `json:"status"`
Attempts int32 `json:"attempts"`
LastStatusCode *int32 `json:"last_status_code"`
LastError string `json:"last_error"`
DeliveredAt *time.Time `json:"delivered_at"`
NextRetryAt *time.Time `json:"next_retry_at"`
CreatedAt time.Time `json:"created_at"`
}
type CreateDeployMarkerInput struct {
Version string `json:"version"`
Ref string `json:"ref"`
Environment string `json:"environment"`
Note string `json:"note"`
}
type DeployMarker struct {
Version string `json:"version,omitempty"`
Ref string `json:"ref,omitempty"`
Environment string `json:"environment,omitempty"`
Note string `json:"note,omitempty"`
RecordedAt time.Time `json:"recorded_at"`
}
func toWebhookSummary(row db.CoreWebhook) WebhookSummary {
events := row.Events
if events == nil {
events = []string{}
}
return WebhookSummary{
ID: row.ID,
ProjectID: row.ProjectID,
URL: row.Url,
Events: events,
Enabled: row.Enabled,
HasSecret: len(row.Secret) > 0,
CreatedAt: row.CreatedAt.Time,
}
}
func toDeliverySummary(row db.CoreWebhookDelivery) WebhookDeliverySummary {
out := WebhookDeliverySummary{
ID: row.ID,
WebhookID: row.WebhookID,
EventType: row.EventType,
Payload: row.Payload,
Status: row.Status,
Attempts: row.Attempts,
LastError: row.LastError,
CreatedAt: row.CreatedAt.Time,
}
if row.LastStatusCode != nil {
code := *row.LastStatusCode
out.LastStatusCode = &code
}
if row.DeliveredAt.Valid {
t := row.DeliveredAt.Time
out.DeliveredAt = &t
}
if row.NextRetryAt.Valid {
t := row.NextRetryAt.Time
out.NextRetryAt = &t
}
return out
}
// normalizeWebhookURL requires https for public targets. Plain http is allowed
// for private/self-hosted sinks: loopback, RFC1918/link-local IPs, .local /
// .internal / .lan names, host.docker.internal, and bare hostnames (Docker
// service names on the internal network).
func normalizeWebhookURL(raw string) (string, error) {
trimmed := strings.TrimSpace(raw)
u, err := url.Parse(trimmed)
if err != nil || u.Host == "" {
return "", errors.New("webhook url must be an https URL")
}
if u.Scheme == "https" {
return trimmed, nil
}
if u.Scheme == "http" && isPrivateWebhookHost(u.Hostname()) {
return trimmed, nil
}
return "", errors.New("webhook url must be https (http allowed only for private/self-hosted targets)")
}
func isPrivateWebhookHost(host string) bool {
h := strings.ToLower(strings.Trim(host, "[]"))
if h == "localhost" || h == "host.docker.internal" ||
strings.HasSuffix(h, ".local") || strings.HasSuffix(h, ".internal") || strings.HasSuffix(h, ".lan") {
return true
}
if !strings.Contains(h, ".") && !strings.Contains(h, ":") {
return true // bare hostname — docker service name, unqualified LAN host
}
ip := net.ParseIP(h)
if ip == nil {
return false
}
return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast()
}
func validateWebhookEvents(events []string) ([]string, error) {
out := make([]string, 0, len(events))
seen := map[string]bool{}
for _, e := range events {
e = strings.TrimSpace(e)
if e == "" {
continue
}
if !webhookEventTypes[e] {
return nil, fmt.Errorf("unknown event type %q", e)
}
if !seen[e] {
seen[e] = true
out = append(out, e)
}
}
return out, nil
}
func (s *PlatformService) webhookFor(ctx context.Context, actor *models.Actor, projectID, webhookID uuid.UUID, write bool) (db.CoreWebhook, error) {
roles := []string{"admin", "developer", "viewer"}
if write {
roles = []string{"admin", "developer"}
}
if err := s.requireProjectRole(ctx, actor, projectID, roles...); err != nil {
return db.CoreWebhook{}, err
}
row, err := s.repo.Queries().GetWebhookByID(ctx, webhookID)
if err != nil {
return db.CoreWebhook{}, err
}
if row.ProjectID != projectID {
return db.CoreWebhook{}, fmt.Errorf("webhook access denied")
}
return row, nil
}
func (s *PlatformService) ListWebhooks(ctx context.Context, actor *models.Actor, projectID uuid.UUID) ([]WebhookSummary, error) {
if err := s.requireProjectRole(ctx, actor, projectID, "admin", "developer", "viewer"); err != nil {
return nil, err
}
rows, err := s.repo.Queries().ListWebhooks(ctx, projectID)
if err != nil {
return nil, err
}
out := make([]WebhookSummary, 0, len(rows))
for _, row := range rows {
out = append(out, toWebhookSummary(row))
}
return out, nil
}
func (s *PlatformService) CreateWebhook(ctx context.Context, actor *models.Actor, projectID uuid.UUID, input CreateWebhookInput, requestID string) (WebhookSummary, error) {
if err := s.requireProjectRole(ctx, actor, projectID, "admin", "developer"); err != nil {
return WebhookSummary{}, err
}
hookURL, err := normalizeWebhookURL(input.URL)
if err != nil {
return WebhookSummary{}, err
}
events, err := validateWebhookEvents(input.Events)
if err != nil {
return WebhookSummary{}, err
}
secret := strings.TrimSpace(input.Secret)
generated := false
if secret == "" {
secret, err = randomToken(16)
if err != nil {
return WebhookSummary{}, err
}
generated = true
}
secretBytes, err := s.enc.Encrypt([]byte(secret))
if err != nil {
return WebhookSummary{}, err
}
enabled := true
if input.Enabled != nil {
enabled = *input.Enabled
}
var createdBy pgtype.UUID
if actor.UserID != nil {
createdBy = pgtype.UUID{Bytes: *actor.UserID, Valid: true}
}
row, err := s.repo.Queries().CreateWebhook(ctx, db.CreateWebhookParams{
ProjectID: projectID,
Url: hookURL,
Secret: secretBytes,
Events: events,
Enabled: enabled,
CreatedByUserID: createdBy,
})
if err != nil {
if strings.Contains(err.Error(), "webhooks_project_id_url_key") {
return WebhookSummary{}, fmt.Errorf("a webhook for %s already exists", hookURL)
}
return WebhookSummary{}, err
}
project, err := s.repo.Queries().GetProjectByID(ctx, projectID)
if err == nil {
_, _ = s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "webhook.created", "webhook", row.ID.String(), map[string]any{
"url": row.Url, "events": events,
}))
}
summary := toWebhookSummary(row)
if generated {
summary.Secret = secret
}
return summary, nil
}
func (s *PlatformService) UpdateWebhook(ctx context.Context, actor *models.Actor, projectID, webhookID uuid.UUID, input UpdateWebhookInput, requestID string) (WebhookSummary, error) {
row, err := s.webhookFor(ctx, actor, projectID, webhookID, true)
if err != nil {
return WebhookSummary{}, err
}
hookURL := row.Url
if input.URL != nil {
if hookURL, err = normalizeWebhookURL(*input.URL); err != nil {
return WebhookSummary{}, err
}
}
events := row.Events
if input.Events != nil {
if events, err = validateWebhookEvents(input.Events); err != nil {
return WebhookSummary{}, err
}
}
enabled := row.Enabled
if input.Enabled != nil {
enabled = *input.Enabled
}
rotateSecret := input.Secret != nil
var secretBytes []byte
if rotateSecret {
secret := strings.TrimSpace(*input.Secret)
if secret == "" {
if secret, err = randomToken(16); err != nil {
return WebhookSummary{}, err
}
}
if secretBytes, err = s.enc.Encrypt([]byte(secret)); err != nil {
return WebhookSummary{}, err
}
}
updated, err := s.repo.Queries().UpdateWebhook(ctx, db.UpdateWebhookParams{
ID: webhookID,
ProjectID: projectID,
Url: hookURL,
Events: events,
Enabled: enabled,
Column6: rotateSecret,
Secret: secretBytes,
})
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return WebhookSummary{}, err
}
if strings.Contains(err.Error(), "webhooks_project_id_url_key") {
return WebhookSummary{}, fmt.Errorf("a webhook for %s already exists", hookURL)
}
return WebhookSummary{}, err
}
project, _ := s.repo.Queries().GetProjectByID(ctx, projectID)
_, _ = s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "webhook.updated", "webhook", webhookID.String(), map[string]any{
"url": updated.Url, "events": updated.Events, "enabled": updated.Enabled, "secret_rotated": rotateSecret,
}))
summary := toWebhookSummary(updated)
if rotateSecret && input.Secret != nil && strings.TrimSpace(*input.Secret) == "" {
// generated on rotation — return once
plain, decErr := s.enc.Decrypt(updated.Secret)
if decErr == nil {
summary.Secret = string(plain)
}
}
return summary, nil
}
func (s *PlatformService) DeleteWebhook(ctx context.Context, actor *models.Actor, projectID, webhookID uuid.UUID, requestID string) error {
row, err := s.webhookFor(ctx, actor, projectID, webhookID, true)
if err != nil {
return err
}
if _, err := s.repo.Queries().DeleteWebhook(ctx, db.DeleteWebhookParams{
ID: webhookID,
ProjectID: projectID,
}); err != nil {
return err
}
project, _ := s.repo.Queries().GetProjectByID(ctx, projectID)
_, _ = s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "webhook.deleted", "webhook", webhookID.String(), map[string]any{
"url": row.Url,
}))
return nil
}
func (s *PlatformService) ListWebhookDeliveries(ctx context.Context, actor *models.Actor, projectID, webhookID uuid.UUID, limit int32) ([]WebhookDeliverySummary, error) {
if _, err := s.webhookFor(ctx, actor, projectID, webhookID, false); err != nil {
return nil, err
}
if limit <= 0 || limit > 200 {
limit = 50
}
rows, err := s.repo.Queries().ListWebhookDeliveries(ctx, db.ListWebhookDeliveriesParams{
WebhookID: webhookID,
Limit: limit,
})
if err != nil {
return nil, err
}
out := make([]WebhookDeliverySummary, 0, len(rows))
for _, row := range rows {
out = append(out, toDeliverySummary(row))
}
return out, nil
}
// TestWebhook queues a webhook.test delivery to one endpoint.
func (s *PlatformService) TestWebhook(ctx context.Context, actor *models.Actor, projectID, webhookID uuid.UUID, requestID string) (WebhookDeliverySummary, error) {
row, err := s.webhookFor(ctx, actor, projectID, webhookID, true)
if err != nil {
return WebhookDeliverySummary{}, err
}
delivery, err := s.enqueueDelivery(ctx, row, WebhookEventTest, map[string]any{
"message": "primora webhook test",
})
if err != nil {
return WebhookDeliverySummary{}, err
}
if project, err := s.repo.Queries().GetProjectByID(ctx, projectID); err == nil {
_, _ = s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "webhook.tested", "webhook", webhookID.String(), map[string]any{
"url": row.Url,
}))
}
return toDeliverySummary(delivery), nil
}
// CreateDeployMarker records the marker in the audit log and fans out a
// deploy.marker event to matching webhooks.
func (s *PlatformService) CreateDeployMarker(ctx context.Context, actor *models.Actor, projectID uuid.UUID, input CreateDeployMarkerInput, requestID string) (DeployMarker, error) {
if err := s.requireProjectRole(ctx, actor, projectID, "admin", "developer"); err != nil {
return DeployMarker{}, err
}
marker := DeployMarker{
Version: strings.TrimSpace(input.Version),
Ref: strings.TrimSpace(input.Ref),
Environment: strings.TrimSpace(input.Environment),
Note: strings.TrimSpace(input.Note),
RecordedAt: time.Now().UTC(),
}
if marker.Version == "" && marker.Ref == "" {
return DeployMarker{}, errors.New("version or ref required")
}
project, err := s.repo.Queries().GetProjectByID(ctx, projectID)
if err != nil {
return DeployMarker{}, err
}
meta := map[string]any{"version": marker.Version, "ref": marker.Ref, "environment": marker.Environment, "note": marker.Note}
if _, err := s.repo.Queries().CreateAuditLog(ctx, newAuditParams(project.OrganizationID, projectID, actor, requestID, "deploy.marker", "deploy_marker", marker.Ref+marker.Version, meta)); err != nil {
return DeployMarker{}, err
}
s.dispatchEvent(ctx, projectID, WebhookEventDeployMarker, map[string]any{
"version": marker.Version,
"ref": marker.Ref,
"environment": marker.Environment,
"note": marker.Note,
"recorded_at": marker.RecordedAt,
})
return marker, nil
}
// dispatchEvent fans an event out to matching enabled webhooks. Deliveries are
// persisted first, then queued in-process — the DB row is the source of truth.
func (s *PlatformService) dispatchEvent(ctx context.Context, projectID uuid.UUID, eventType string, data map[string]any) {
hooks, err := s.repo.Queries().ListWebhooksForEvent(ctx, db.ListWebhooksForEventParams{
ProjectID: projectID,
Column2: eventType,
})
if err != nil {
return
}
for _, hook := range hooks {
_, _ = s.enqueueDelivery(ctx, hook, eventType, data)
}
}
func (s *PlatformService) enqueueDelivery(ctx context.Context, hook db.CoreWebhook, eventType string, data map[string]any) (db.CoreWebhookDelivery, error) {
payload, _ := json.Marshal(map[string]any{
"event": eventType,
"project_id": hook.ProjectID.String(),
"occurred_at": time.Now().UTC().Format(time.RFC3339),
"data": data,
})
delivery, err := s.repo.Queries().InsertWebhookDelivery(ctx, db.InsertWebhookDeliveryParams{
WebhookID: hook.ID,
EventType: eventType,
Payload: payload,
})
if err != nil {
return db.CoreWebhookDelivery{}, err
}
if s.dispatcher != nil {
s.dispatcher.push(delivery.ID)
}
return delivery, nil
}
// ---------------------------------------------------------------------------
// dispatcher — in-process worker, buffered queue, backoff retry
// ---------------------------------------------------------------------------
type WebhookDispatcher struct {
repo queries
enc *secrets.Encryptor
logger *slog.Logger
http *http.Client
queue chan uuid.UUID
quit chan struct{}
wg sync.WaitGroup
timerMu sync.Mutex
timers map[uuid.UUID]*time.Timer
started sync.Once
}
// queries is the slice of db.Queries the dispatcher needs — kept narrow so
// tests can stub it without a database.
type queries interface {
GetWebhookDeliveryByID(ctx context.Context, id uuid.UUID) (db.CoreWebhookDelivery, error)
GetWebhookByID(ctx context.Context, id uuid.UUID) (db.CoreWebhook, error)
ListPendingWebhookDeliveries(ctx context.Context, limit int32) ([]db.CoreWebhookDelivery, error)
MarkWebhookDelivery(ctx context.Context, params db.MarkWebhookDeliveryParams) error
}
func NewWebhookDispatcher(q queries, enc *secrets.Encryptor, logger *slog.Logger) *WebhookDispatcher {
if logger == nil {
logger = slog.Default()
}
return &WebhookDispatcher{
repo: q,
enc: enc,
logger: logger,
http: &http.Client{Timeout: 10 * time.Second},
queue: make(chan uuid.UUID, 1024),
quit: make(chan struct{}),
timers: map[uuid.UUID]*time.Timer{},
}
}
// Start launches the worker and re-queues deliveries left pending by a crash.
func (d *WebhookDispatcher) Start(ctx context.Context) {
d.started.Do(func() {
d.wg.Add(1)
go d.run()
if pending, err := d.repo.ListPendingWebhookDeliveries(ctx, 200); err == nil {
for _, row := range pending {
d.push(row.ID)
}
}
})
}
func (d *WebhookDispatcher) Stop() {
close(d.quit)
d.timerMu.Lock()
for _, t := range d.timers {
t.Stop()
}
d.timerMu.Unlock()
d.wg.Wait()
}
func (d *WebhookDispatcher) push(id uuid.UUID) {
select {
case d.queue <- id:
case <-d.quit:
default:
// queue full — drop; the row stays pending and is retried on restart
}
}
func (d *WebhookDispatcher) run() {
defer d.wg.Done()
for {
select {
case id := <-d.queue:
d.deliver(context.Background(), id)
case <-d.quit:
return
}
}
}
// signBody is the canonical HMAC-SHA256 signature: hex of the keyed digest of
// the exact bytes sent. Receivers verify against the raw body.
func signBody(secret string, body []byte) string {
mac := hmac.New(sha256.New, []byte(secret))
mac.Write(body)
return hex.EncodeToString(mac.Sum(nil))
}
func (d *WebhookDispatcher) deliver(ctx context.Context, deliveryID uuid.UUID) {
delivery, err := d.repo.GetWebhookDeliveryByID(ctx, deliveryID)
if err != nil || delivery.Status != "pending" {
return
}
hook, err := d.repo.GetWebhookByID(ctx, delivery.WebhookID)
if err != nil {
d.mark(delivery, "failed", nil, "webhook deleted", nil)
return
}
var statusCode *int32
var sendErr error
secret := ""
if len(hook.Secret) > 0 && d.enc != nil {
if plain, err := d.enc.Decrypt(hook.Secret); err == nil {
secret = string(plain)
}
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, hook.Url, bytes.NewReader(delivery.Payload))
if err == nil {
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Primora-Event", delivery.EventType)
req.Header.Set("X-Primora-Delivery", delivery.ID.String())
req.Header.Set("X-Primora-Signature", "sha256="+signBody(secret, delivery.Payload))
resp, doErr := d.http.Do(req)
if doErr != nil {
sendErr = doErr
} else {
code := int32(resp.StatusCode)
statusCode = &code
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16))
resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
now := time.Now()
d.mark(delivery, "delivered", statusCode, "", &now)
return
}
sendErr = fmt.Errorf("endpoint returned %d", resp.StatusCode)
}
} else {
sendErr = err
}
attempts := delivery.Attempts + 1
if attempts >= webhookMaxAttempts {
d.mark(delivery, "failed", statusCode, errString(sendErr), nil)
return
}
retryAt := time.Now().Add(time.Duration(attempts*attempts) * 15 * time.Second)
d.mark(delivery, "pending", statusCode, errString(sendErr), nil, retryAt)
d.timerMu.Lock()
d.timers[delivery.ID] = time.AfterFunc(time.Duration(attempts*attempts)*15*time.Second, func() {
d.timerMu.Lock()
delete(d.timers, delivery.ID)
d.timerMu.Unlock()
d.push(delivery.ID)
})
d.timerMu.Unlock()
}
func (d *WebhookDispatcher) mark(delivery db.CoreWebhookDelivery, status string, statusCode *int32, lastError string, deliveredAt *time.Time, nextRetry ...time.Time) {
params := db.MarkWebhookDeliveryParams{
ID: delivery.ID,
Status: status,
LastError: lastError,
}
params.LastStatusCode = statusCode
if deliveredAt != nil {
params.DeliveredAt = pgtype.Timestamptz{Time: *deliveredAt, Valid: true}
}
if len(nextRetry) > 0 {
params.NextRetryAt = pgtype.Timestamptz{Time: nextRetry[0], Valid: true}
} else {
params.NextRetryAt = pgtype.Timestamptz{}
}
if err := d.repo.MarkWebhookDelivery(context.Background(), params); err != nil {
d.logger.Warn("webhook delivery update failed", "delivery", delivery.ID, "error", err)
}
}
func errString(err error) string {
if err == nil {
return ""
}
return err.Error()
}