mirror of
https://github.com/Dvorinka/Containr.git
synced 2026-10-28 07:41:27 +00:00
feat: build-on-node — git checkouts, remote image builds, image transport (#60)
Git-sourced deploys never materialized a checkout: sourcePath was build_path resolved against the server cwd, so builds consumed the backend's own directory. Remote placement could only run registry-pulled images. - internal/source: per-deployment checkout (init/fetch/checkout) at branch or commit; GIT_ASKPASS keeps tokens out of argv, .git/config, and the build context; dumb-transport and SHA-refusal fallbacks; build_path validated inside the checkout. - BuildManager.PackageContext: produces the same tar context Build ships to the daemon — generated Dockerfile at tar root, .git excluded. Fixes nixpacks passing the Dockerfile path as content. - Agent artifact endpoint: GET /api/agents/artifacts/:id streams build contexts and docker-save tarballs behind agent-token auth; hex-validated ids, per-deployment cleanup, boot-time purge. - Agent commands: build_image (docker build -, --build-arg values via process env, not argv) and load_image (docker load for containr-* rollback tags no registry serves). - Deployment flow: remote git builds package context → artifact → node docker-builds the deployment tag; remote rollbacks of local tags export via SaveImage → artifact → node docker-loads. Verified end-to-end: agent-pinned git service built on the node (build_image → create_container → deployed) and rollback shipped a docker-save tarball (load_image → deployed). Artifact auth returns 401/404 correctly; workspaces and blobs are removed after deploy. Authored By: TDvorak <[email protected]>
This commit is contained in:
1 parent
d57ac72aa5
commit
3cc84553f4
18 files changed
+1020
-51
No files matched your search
@@ -30,7 +30,7 @@ COPY auth/src ./src
|
||||
# Final runtime image (Go API + Better Auth sidecar in one container)
|
||||
FROM node:22-alpine
|
||||
|
||||
RUN apk --no-cache add ca-certificates tzdata wget
|
||||
RUN apk --no-cache add ca-certificates tzdata wget git
|
||||
|
||||
RUN addgroup -g 1001 -S appgroup && \
|
||||
adduser -u 1001 -S appuser -G appgroup
|
||||
|
||||
@@ -316,7 +316,7 @@ func processCommands(ctx context.Context, client *http.Client, cfg agentConfig,
|
||||
if item.Type == "self_upgrade" {
|
||||
result, reexecPath, runErr = selfUpgrade(cfg, item.Payload)
|
||||
} else {
|
||||
result, runErr = executeCommand(item)
|
||||
result, runErr = executeCommand(cfg, item)
|
||||
}
|
||||
status := "completed"
|
||||
errText := ""
|
||||
@@ -343,7 +343,7 @@ func processCommands(ctx context.Context, client *http.Client, cfg agentConfig,
|
||||
return nil
|
||||
}
|
||||
|
||||
func executeCommand(item command) (string, error) {
|
||||
func executeCommand(cfg agentConfig, item command) (string, error) {
|
||||
switch item.Type {
|
||||
case "create_container":
|
||||
return createContainer(item.Payload)
|
||||
@@ -360,6 +360,10 @@ func executeCommand(item command) (string, error) {
|
||||
return out, nil
|
||||
}
|
||||
return out, err
|
||||
case "build_image":
|
||||
return buildImage(cfg, item.Payload)
|
||||
case "load_image":
|
||||
return loadImage(cfg, item.Payload)
|
||||
case "prune":
|
||||
return dockerPrune(item.Payload)
|
||||
case "system_df":
|
||||
@@ -568,6 +572,120 @@ func createContainer(payload map[string]interface{}) (string, error) {
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// fetchArtifact downloads a server-side deploy artifact (build context or
|
||||
// docker-save tarball) into a temp file the docker CLI consumes.
|
||||
func fetchArtifact(cfg agentConfig, artifactID string) (*os.File, error) {
|
||||
if len(artifactID) != 32 {
|
||||
return nil, fmt.Errorf("invalid artifact id")
|
||||
}
|
||||
url := fmt.Sprintf("%s/api/agents/artifacts/%s", cfg.APIURL, artifactID)
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("X-Containr-Agent-Token", cfg.Token)
|
||||
resp, err := (&http.Client{Timeout: 10 * time.Minute}).Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download artifact: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("download artifact: server returned %d", resp.StatusCode)
|
||||
}
|
||||
f, err := os.CreateTemp("", "containr-artifact-*.tar")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := io.Copy(f, resp.Body); err != nil {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
return nil, fmt.Errorf("write artifact: %w", err)
|
||||
}
|
||||
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
return nil, err
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// buildImage docker-builds a packaged context tar on this node. The context
|
||||
// already carries its Dockerfile at the tar root, so `docker build -` can
|
||||
// stream it straight from stdin. Result reports the image digest so the
|
||||
// server records what each node actually produced.
|
||||
func buildImage(cfg agentConfig, payload map[string]interface{}) (string, error) {
|
||||
image := stringValue(payload["image"])
|
||||
artifactID := stringValue(payload["artifact_id"])
|
||||
if image == "" || artifactID == "" {
|
||||
return "", errors.New("build_image requires image and artifact_id")
|
||||
}
|
||||
f, err := fetchArtifact(cfg, artifactID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
args := []string{"build", "-t", image}
|
||||
if boolValue(payload["no_cache"]) {
|
||||
args = append(args, "--no-cache")
|
||||
}
|
||||
// Bare --build-arg keys read their values from the process env — keeps
|
||||
// token-bearing args out of the node's process list.
|
||||
env := os.Environ()
|
||||
if buildArgs, ok := payload["build_args"].(map[string]interface{}); ok {
|
||||
for k, v := range buildArgs {
|
||||
args = append(args, "--build-arg", k)
|
||||
env = append(env, fmt.Sprintf("%s=%v", k, v))
|
||||
}
|
||||
}
|
||||
args = append(args, "-")
|
||||
cmd := exec.Command("docker", args...)
|
||||
cmd.Stdin = f
|
||||
cmd.Env = env
|
||||
outBytes, err := cmd.CombinedOutput()
|
||||
out := strings.TrimSpace(string(outBytes))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("docker build failed: %s", tailOutput(out, 4096))
|
||||
}
|
||||
digest, _ := runDocker("inspect", "-f", "{{.Id}}", image)
|
||||
size, _ := runDocker("inspect", "-f", "{{.Size}}", image)
|
||||
result, _ := json.Marshal(map[string]string{
|
||||
"image": image, "id": digest, "size": size,
|
||||
"log_tail": tailOutput(out, 4096),
|
||||
})
|
||||
return string(result), nil
|
||||
}
|
||||
|
||||
// loadImage imports a docker-save tarball — rollbacks of locally-built tags
|
||||
// reach remote nodes this way since no registry serves containr-* images.
|
||||
func loadImage(cfg agentConfig, payload map[string]interface{}) (string, error) {
|
||||
artifactID := stringValue(payload["artifact_id"])
|
||||
if artifactID == "" {
|
||||
return "", errors.New("load_image requires artifact_id")
|
||||
}
|
||||
f, err := fetchArtifact(cfg, artifactID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
defer os.Remove(f.Name())
|
||||
out, err := runDockerStdin(f, "load")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("docker load failed: %s", tailOutput(out, 4096))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// tailOutput bounds command results — full docker build logs would swamp
|
||||
// the agent_commands row.
|
||||
func tailOutput(s string, max int) string {
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[len(s)-max:]
|
||||
}
|
||||
|
||||
// registryHostOf mirrors the server's registryHost: first path segment with
|
||||
// a "." or ":" is a registry host; bare names mean Docker Hub.
|
||||
func registryHostOf(imageRef string) string {
|
||||
@@ -779,6 +897,15 @@ func runDocker(args ...string) (string, error) {
|
||||
return strings.TrimSpace(string(output)), err
|
||||
}
|
||||
|
||||
// runDockerStdin pipes stdin into docker — build contexts and saved image
|
||||
// tarballs stream in without a second copy on disk.
|
||||
func runDockerStdin(stdin io.Reader, args ...string) (string, error) {
|
||||
cmd := exec.Command("docker", args...)
|
||||
cmd.Stdin = stdin
|
||||
output, err := cmd.CombinedOutput()
|
||||
return strings.TrimSpace(string(output)), err
|
||||
}
|
||||
|
||||
func dockerAvailable() bool {
|
||||
_, err := exec.LookPath("docker")
|
||||
if err != nil {
|
||||
|
||||
@@ -769,7 +769,9 @@ func (h *NodeAgentHandler) CompleteCommand(c *gin.Context) {
|
||||
}
|
||||
|
||||
// scrubCommandSecrets rewrites a completed command's payload with any
|
||||
// registry password blanked. Best-effort — a failed scrub leaves the row.
|
||||
// credential-bearing fields blanked — registry passwords and build-time
|
||||
// args (tokens like NPM_TOKEN ride along as build args). Best-effort — a
|
||||
// failed scrub leaves the row.
|
||||
func scrubCommandSecrets(ctx context.Context, q *sqlcdb.Queries, commandID string, payload pqtype.NullRawMessage) {
|
||||
if !payload.Valid || len(payload.RawMessage) == 0 {
|
||||
return
|
||||
@@ -778,15 +780,24 @@ func scrubCommandSecrets(ctx context.Context, q *sqlcdb.Queries, commandID strin
|
||||
if err := json.Unmarshal(payload.RawMessage, &doc); err != nil {
|
||||
return
|
||||
}
|
||||
container, ok := doc["container"].(map[string]interface{})
|
||||
if !ok {
|
||||
dirty := false
|
||||
if container, ok := doc["container"].(map[string]interface{}); ok {
|
||||
if reg, ok := container["registry"].(map[string]interface{}); ok && reg["password"] != nil {
|
||||
reg["password"] = "***"
|
||||
dirty = true
|
||||
}
|
||||
}
|
||||
for _, key := range []string{"build_args", "environment"} {
|
||||
if args, ok := doc[key].(map[string]interface{}); ok && len(args) > 0 {
|
||||
for k := range args {
|
||||
args[k] = "***"
|
||||
}
|
||||
dirty = true
|
||||
}
|
||||
}
|
||||
if !dirty {
|
||||
return
|
||||
}
|
||||
reg, ok := container["registry"].(map[string]interface{})
|
||||
if !ok || reg["password"] == nil {
|
||||
return
|
||||
}
|
||||
reg["password"] = "***"
|
||||
if err := q.ScrubCommandPayload(ctx, sqlcdb.ScrubCommandPayloadParams{
|
||||
ID: commandID,
|
||||
Payload: rawJSON(doc),
|
||||
@@ -1458,6 +1469,10 @@ func (h *NodeAgentHandler) SetupPublicRoutes(router *gin.RouterGroup) {
|
||||
// public; the enroll token is the actual secret.
|
||||
agents.GET("/install.sh", h.ServeInstallScript)
|
||||
agents.GET("/download/:platform", h.ServeAgentBinary)
|
||||
|
||||
// Deploy artifacts (build contexts, image tarballs) — token-gated;
|
||||
// the blobs are per-deployment source code, not public bootstrap.
|
||||
agents.GET("/artifacts/:id", h.ServeArtifact)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// containrWorkDir is the build workspace root — same dir the BuildManager
|
||||
// was constructed with; checkouts and agent artifacts live beneath it.
|
||||
const containrWorkDir = "/tmp/containr-builds"
|
||||
|
||||
// artifacts is a one-shot blob store for node agents: build contexts and
|
||||
// docker-save tarballs the server produced on behalf of a deployment. Files
|
||||
// live under workdir/artifacts/<id> and are removed when the deployment
|
||||
// finishes or after maxArtifactAge.
|
||||
var artifacts = &artifactStore{files: map[string]string{}}
|
||||
|
||||
type artifactStore struct {
|
||||
mu sync.Mutex
|
||||
dir string
|
||||
files map[string]string
|
||||
}
|
||||
|
||||
const maxArtifactAge = 2 // hours — a deployment can never outlive this
|
||||
|
||||
// initArtifacts points the store at dir (created on demand).
|
||||
func initArtifacts(dir string) {
|
||||
artifacts.mu.Lock()
|
||||
defer artifacts.mu.Unlock()
|
||||
artifacts.dir = dir
|
||||
_ = os.MkdirAll(dir, 0o755)
|
||||
}
|
||||
|
||||
// artifactDir returns the store root, defaulting under the build workdir.
|
||||
func artifactDir() string {
|
||||
artifacts.mu.Lock()
|
||||
defer artifacts.mu.Unlock()
|
||||
if artifacts.dir != "" {
|
||||
return artifacts.dir
|
||||
}
|
||||
return filepath.Join(containrWorkDir, "artifacts")
|
||||
}
|
||||
|
||||
// putArtifact streams rc into the store and returns the artifact id.
|
||||
func putArtifact(rc io.Reader) (string, error) {
|
||||
buf := make([]byte, 16)
|
||||
if _, err := rand.Read(buf); err != nil {
|
||||
return "", err
|
||||
}
|
||||
id := hex.EncodeToString(buf)
|
||||
dir := artifactDir()
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
path := filepath.Join(dir, id+".tar")
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := io.Copy(f, rc); err != nil {
|
||||
f.Close()
|
||||
os.Remove(path)
|
||||
return "", err
|
||||
}
|
||||
f.Close()
|
||||
|
||||
artifacts.mu.Lock()
|
||||
artifacts.files[id] = path
|
||||
artifacts.mu.Unlock()
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// removeArtifact deletes the file and forgets the id. Safe on missing ids.
|
||||
func removeArtifact(id string) {
|
||||
artifacts.mu.Lock()
|
||||
path, ok := artifacts.files[id]
|
||||
delete(artifacts.files, id)
|
||||
artifacts.mu.Unlock()
|
||||
if ok {
|
||||
_ = os.Remove(path)
|
||||
return
|
||||
}
|
||||
// Registered before a restart — remove best-effort by name.
|
||||
if id != "" {
|
||||
_ = os.Remove(filepath.Join(artifactDir(), id+".tar"))
|
||||
}
|
||||
}
|
||||
|
||||
// ServeArtifact streams a stored blob to a token-authenticated agent. Files
|
||||
// persist across downloads — spread builds share one context — and are
|
||||
// deleted by the deployment that created them.
|
||||
func (h *NodeAgentHandler) ServeArtifact(c *gin.Context) {
|
||||
if !h.isValidAgentAuthToken(c.Request.Context(), agentAuthTokenFromRequest(c)) {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "Invalid auth token"})
|
||||
return
|
||||
}
|
||||
id := c.Param("id")
|
||||
raw, err := hex.DecodeString(id)
|
||||
if len(id) != 32 || err != nil || len(raw) != 16 {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Artifact not found"})
|
||||
return
|
||||
}
|
||||
path := filepath.Join(artifactDir(), id+".tar")
|
||||
info, err := os.Stat(path)
|
||||
if err != nil || info.IsDir() {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "Artifact not found"})
|
||||
return
|
||||
}
|
||||
c.Header("Content-Type", "application/x-tar")
|
||||
c.File(path)
|
||||
}
|
||||
|
||||
// purgeStaleArtifacts drops files older than maxArtifactAge. Called on boot.
|
||||
func purgeStaleArtifacts() {
|
||||
entries, err := os.ReadDir(artifactDir())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if info.ModTime().Before(time.Now().Add(-maxArtifactAge * time.Hour)) {
|
||||
_ = os.Remove(filepath.Join(artifactDir(), e.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,11 +5,15 @@ import (
|
||||
"containr/internal/deployment"
|
||||
"containr/internal/deployqueue"
|
||||
"containr/internal/docker"
|
||||
"containr/internal/source"
|
||||
"containr/internal/types"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -377,6 +381,23 @@ func runDeploymentAndSyncWithImage(
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// The checkout and any shipped artifacts die with this deployment.
|
||||
var artifactIDs []string
|
||||
checkoutRoot := filepath.Join(containrWorkDir, "repos", dbDeployment.ID.String())
|
||||
defer func() {
|
||||
_ = os.RemoveAll(checkoutRoot)
|
||||
for _, id := range artifactIDs {
|
||||
removeArtifact(id)
|
||||
}
|
||||
}()
|
||||
failDeploy := func(msg string) {
|
||||
failedAt := time.Now()
|
||||
_, _ = db.Exec(
|
||||
`UPDATE deployments SET status = 'failed', error = $1, completed_at = $2, updated_at = $2 WHERE id = $3`,
|
||||
msg, failedAt, dbDeployment.ID,
|
||||
)
|
||||
}
|
||||
|
||||
sourcePath := strings.TrimSpace(service.BuildPath)
|
||||
if sourcePath == "" {
|
||||
sourcePath = "."
|
||||
@@ -421,19 +442,31 @@ func runDeploymentAndSyncWithImage(
|
||||
maintURL = maintenanceURL(db)
|
||||
}
|
||||
|
||||
// Node-pinned services dispatch to the agent's command queue. Builds run
|
||||
// on the local host, so remote placement only works for registry-pulled
|
||||
// images — git-sourced and rollback deploys fail fast with that message.
|
||||
// Node-pinned and spread services dispatch to agent command queues.
|
||||
nodeID := serviceNodeID(db, service.ID)
|
||||
spread := serviceSpread(db, service.ID)
|
||||
if (nodeID != "" || spread) && (service.GitRepo != "" || imageOverride != "") {
|
||||
failedAt := time.Now()
|
||||
msg := "service uses remote node placement but remote nodes only run registry-pulled images (git builds and rollbacks stay local — build-on-node is not yet supported)"
|
||||
_, _ = db.Exec(
|
||||
`UPDATE deployments SET status = 'failed', error = $1, completed_at = $2, updated_at = $2 WHERE id = $3`,
|
||||
msg, failedAt, dbDeployment.ID,
|
||||
)
|
||||
return
|
||||
remote := nodeID != "" || spread
|
||||
|
||||
// Materialize the git checkout — local builds consume it directly;
|
||||
// remote builds package it into a context artifact the node builds.
|
||||
if service.GitRepo != "" && imageOverride == "" {
|
||||
var ownerID string
|
||||
_ = db.QueryRow(`SELECT owner_id FROM projects WHERE id = $1`, service.ProjectID).Scan(&ownerID)
|
||||
branch := req.Branch
|
||||
if branch == "" {
|
||||
branch = service.GitBranch
|
||||
}
|
||||
cloneSpec, err := resolveGitCloneSpec(db, ownerID, service.GitRepo, branch, req.CommitHash, service.BuildPath)
|
||||
if err != nil {
|
||||
failDeploy("resolve git source: " + err.Error())
|
||||
return
|
||||
}
|
||||
src, err := source.Checkout(ctx, checkoutRoot, cloneSpec)
|
||||
if err != nil {
|
||||
failDeploy("git checkout: " + err.Error())
|
||||
return
|
||||
}
|
||||
sourcePath = src
|
||||
}
|
||||
|
||||
deployReq := &deployment.DeploymentRequest{
|
||||
@@ -480,14 +513,36 @@ func runDeploymentAndSyncWithImage(
|
||||
if image == "" {
|
||||
image = service.Image
|
||||
}
|
||||
var ownerID string
|
||||
_ = db.QueryRow(`SELECT owner_id FROM projects WHERE id = $1`, service.ProjectID).Scan(&ownerID)
|
||||
auth := registryAuthFor(db, ownerID, image)
|
||||
deployReq.BuildConfig = &deployment.BuildConfig{
|
||||
BuildType: "prebuilt",
|
||||
PrebuiltImage: image,
|
||||
PullUsername: auth.Username,
|
||||
PullPassword: auth.Password,
|
||||
if remote && strings.HasPrefix(image, "containr-") {
|
||||
// Locally-built tag — no registry serves it. Ship the image
|
||||
// itself as a docker-save artifact; the node loads it.
|
||||
rc, err := engine.DockerClient().SaveImage(ctx, image)
|
||||
if err != nil {
|
||||
failDeploy("export rollback image: " + err.Error())
|
||||
return
|
||||
}
|
||||
artifactID, err := putArtifact(rc)
|
||||
_ = rc.Close()
|
||||
if err != nil {
|
||||
failDeploy("store rollback image artifact: " + err.Error())
|
||||
return
|
||||
}
|
||||
artifactIDs = append(artifactIDs, artifactID)
|
||||
deployReq.Config.RemoteLoad = &deployment.RemoteLoadSpec{ArtifactID: artifactID}
|
||||
deployReq.BuildConfig = &deployment.BuildConfig{
|
||||
BuildType: "remote",
|
||||
PrebuiltImage: image,
|
||||
}
|
||||
} else {
|
||||
var ownerID string
|
||||
_ = db.QueryRow(`SELECT owner_id FROM projects WHERE id = $1`, service.ProjectID).Scan(&ownerID)
|
||||
auth := registryAuthFor(db, ownerID, image)
|
||||
deployReq.BuildConfig = &deployment.BuildConfig{
|
||||
BuildType: "prebuilt",
|
||||
PrebuiltImage: image,
|
||||
PullUsername: auth.Username,
|
||||
PullPassword: auth.Password,
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// "auto" leaves BuildType empty so the manager detects it; the static
|
||||
@@ -497,16 +552,54 @@ func runDeploymentAndSyncWithImage(
|
||||
if service.Builder != "" && service.Builder != "auto" {
|
||||
buildType = service.Builder
|
||||
}
|
||||
deployReq.BuildConfig = &deployment.BuildConfig{
|
||||
BuildType: buildType,
|
||||
SourcePath: sourcePath,
|
||||
Branch: req.Branch,
|
||||
Commit: req.CommitHash,
|
||||
NoCache: req.NoCache,
|
||||
}
|
||||
buildArgs := map[string]string{}
|
||||
buildCommand := ""
|
||||
if buildType == "static" {
|
||||
deployReq.BuildConfig.BuildCommand = firstNonEmpty(service.StaticBuildCmd, "npm ci && npm run build")
|
||||
deployReq.BuildConfig.BuildArgs = map[string]string{"STATIC_DIR": firstNonEmpty(service.StaticDir, "dist")}
|
||||
buildCommand = firstNonEmpty(service.StaticBuildCmd, "npm ci && npm run build")
|
||||
buildArgs["STATIC_DIR"] = firstNonEmpty(service.StaticDir, "dist")
|
||||
}
|
||||
if remote {
|
||||
// Package the context — the node docker-builds it itself.
|
||||
packReq := &types.BuildRequest{
|
||||
BuildType: buildType,
|
||||
SourcePath: sourcePath,
|
||||
BuildCommand: buildCommand,
|
||||
BuildArgs: buildArgs,
|
||||
Environment: env,
|
||||
}
|
||||
rc, err := engine.BuildManager().PackageContext(ctx, packReq)
|
||||
if err != nil {
|
||||
failDeploy("package build context: " + err.Error())
|
||||
return
|
||||
}
|
||||
artifactID, err := putArtifact(rc)
|
||||
_ = rc.Close()
|
||||
if err != nil {
|
||||
failDeploy("store build context artifact: " + err.Error())
|
||||
return
|
||||
}
|
||||
artifactIDs = append(artifactIDs, artifactID)
|
||||
deployReq.Config.RemoteBuild = &deployment.RemoteBuildSpec{
|
||||
ArtifactID: artifactID,
|
||||
BuildArgs: buildArgs,
|
||||
NoCache: req.NoCache,
|
||||
}
|
||||
deployReq.BuildConfig = &deployment.BuildConfig{
|
||||
BuildType: "remote",
|
||||
Branch: req.Branch,
|
||||
Commit: req.CommitHash,
|
||||
NoCache: req.NoCache,
|
||||
}
|
||||
} else {
|
||||
deployReq.BuildConfig = &deployment.BuildConfig{
|
||||
BuildType: buildType,
|
||||
SourcePath: sourcePath,
|
||||
BuildCommand: buildCommand,
|
||||
BuildArgs: buildArgs,
|
||||
Branch: req.Branch,
|
||||
Commit: req.CommitHash,
|
||||
NoCache: req.NoCache,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"containr/internal/database"
|
||||
"containr/internal/source"
|
||||
)
|
||||
|
||||
// resolveGitCloneSpec turns a service's git_repo (clone URL, ssh remote, or
|
||||
// owner/repo) into a source.Spec with the project owner's provider token.
|
||||
// Public repos get no token — the clone proceeds anonymously. github_app
|
||||
// providers store the installation id, not a usable token; a fresh
|
||||
// installation token is minted per deploy.
|
||||
func resolveGitCloneSpec(db *database.DB, ownerID, gitRepo, branch, commit, buildPath string) (source.Spec, error) {
|
||||
raw := strings.TrimSpace(gitRepo)
|
||||
cloneURL, err := source.NormalizeCloneURL(raw)
|
||||
if err != nil {
|
||||
return source.Spec{}, err
|
||||
}
|
||||
|
||||
spec := source.Spec{Branch: branch, Commit: commit, BuildPath: buildPath}
|
||||
|
||||
var providerID string
|
||||
if !strings.Contains(cloneURL, "://") {
|
||||
// Bare owner/repo — the connected repository row knows the host.
|
||||
err = db.QueryRow(
|
||||
`SELECT clone_url, provider_id FROM git_repositories
|
||||
WHERE full_name = $1 AND user_id = $2 LIMIT 1`,
|
||||
cloneURL, ownerID,
|
||||
).Scan(&cloneURL, &providerID)
|
||||
if err != nil {
|
||||
return source.Spec{}, fmt.Errorf("repository %s is not connected — connect it under Git settings or store a clone URL", raw)
|
||||
}
|
||||
} else {
|
||||
// URL form — match a connected repo for its provider when possible;
|
||||
// an unmatched public URL still clones, just without credentials.
|
||||
_ = db.QueryRow(
|
||||
`SELECT provider_id FROM git_repositories
|
||||
WHERE clone_url = $1 AND user_id = $2 LIMIT 1`,
|
||||
cloneURL, ownerID,
|
||||
).Scan(&providerID)
|
||||
}
|
||||
|
||||
if err := source.ValidateRemoteURL(cloneURL); err != nil {
|
||||
return source.Spec{}, err
|
||||
}
|
||||
spec.CloneURL = cloneURL
|
||||
|
||||
if providerID == "" {
|
||||
return spec, nil
|
||||
}
|
||||
var name, apiURL, token string
|
||||
if err := db.QueryRow(
|
||||
`SELECT name, api_url, access_token FROM git_providers
|
||||
WHERE id = $1 AND user_id = $2`,
|
||||
providerID, ownerID,
|
||||
).Scan(&name, &apiURL, &token); err != nil {
|
||||
return spec, nil // provider row gone — anonymous clone still works for public repos
|
||||
}
|
||||
|
||||
switch strings.ToLower(name) {
|
||||
case "github_app":
|
||||
installationID, err := strconv.ParseInt(strings.TrimSpace(token), 10, 64)
|
||||
if err == nil && installationID > 0 {
|
||||
if minted, err := getGitHubAppInstallationToken(apiURL, installationID); err == nil {
|
||||
spec.Token = minted
|
||||
spec.User = "x-access-token"
|
||||
}
|
||||
}
|
||||
case "gitlab":
|
||||
spec.Token = token
|
||||
spec.User = "oauth2"
|
||||
case "bitbucket":
|
||||
spec.Token = token
|
||||
spec.User = "x-token-auth"
|
||||
default:
|
||||
spec.Token = token
|
||||
spec.User = "x-access-token"
|
||||
}
|
||||
return spec, nil
|
||||
}
|
||||
@@ -157,6 +157,45 @@ func (r *agentNodeRunner) reconcileOnAgents(ctx context.Context, spec deployment
|
||||
})
|
||||
}
|
||||
|
||||
// Ship the image to every distinct target agent before replicas start.
|
||||
// RemoteBuild nodes docker-build from the packaged context artifact;
|
||||
// RemoteLoad nodes docker-load a saved tarball (local-tag rollbacks).
|
||||
// Registry-pullable images skip this — docker run pulls on demand.
|
||||
if spec.RemoteBuild != nil || spec.RemoteLoad != nil {
|
||||
seen := map[string]bool{}
|
||||
for i := 0; i < replicas; i++ {
|
||||
agentID := targetFor(i)
|
||||
if seen[agentID] {
|
||||
continue
|
||||
}
|
||||
seen[agentID] = true
|
||||
if spec.RemoteBuild != nil {
|
||||
buildArgs := map[string]interface{}{}
|
||||
for k, v := range spec.RemoteBuild.BuildArgs {
|
||||
buildArgs[k] = v
|
||||
}
|
||||
_, err := r.enqueueAndWaitTimeout(ctx, agentID, "", "build_image",
|
||||
map[string]interface{}{
|
||||
"artifact_id": spec.RemoteBuild.ArtifactID,
|
||||
"image": spec.Image,
|
||||
"build_args": buildArgs,
|
||||
"no_cache": spec.RemoteBuild.NoCache,
|
||||
}, 25*time.Minute)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build image on node %s: %w", agentID, err)
|
||||
}
|
||||
} else if spec.RemoteLoad != nil {
|
||||
_, err := r.enqueueAndWaitTimeout(ctx, agentID, "", "load_image",
|
||||
map[string]interface{}{
|
||||
"artifact_id": spec.RemoteLoad.ArtifactID,
|
||||
}, 10*time.Minute)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load image on node %s: %w", agentID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
state := &deployment.RuntimeState{Desired: replicas, Status: "running"}
|
||||
for i := 0; i < replicas; i++ {
|
||||
agentID := targetFor(i)
|
||||
@@ -375,6 +414,10 @@ func (r *agentNodeRunner) RetireService(ctx context.Context, serviceID string) e
|
||||
// poll on their own interval (≥5s), so this blocks — it runs inside the
|
||||
// deploy queue worker, never on a request path.
|
||||
func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, containerID, cmdType string, payload map[string]interface{}) (string, error) {
|
||||
return r.enqueueAndWaitTimeout(ctx, agentID, containerID, cmdType, payload, 8*time.Minute)
|
||||
}
|
||||
|
||||
func (r *agentNodeRunner) enqueueAndWaitTimeout(ctx context.Context, agentID, containerID, cmdType string, payload map[string]interface{}, timeout time.Duration) (string, error) {
|
||||
var cid sql.NullString
|
||||
if containerID != "" {
|
||||
cid = sql.NullString{String: containerID, Valid: true}
|
||||
@@ -390,7 +433,7 @@ func (r *agentNodeRunner) enqueueAndWait(ctx context.Context, agentID, container
|
||||
return "", err
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(8 * time.Minute) // image pulls on a cold node
|
||||
deadline := time.Now().Add(timeout) // image pulls/builds on a cold node
|
||||
for {
|
||||
row, err := r.q.GetCommandForAgent(ctx, sqlcdb.GetCommandForAgentParams{
|
||||
ID: cmd.ID,
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"containr/internal/build"
|
||||
@@ -43,6 +44,8 @@ func SetupRoutes(router *gin.Engine, db *database.DB, redis *database.Redis, cfg
|
||||
dockerClient = client
|
||||
buildManager = build.NewBuildManager("/tmp/containr-builds", dockerClient)
|
||||
deploymentEngine = deployment.NewDeploymentEngine(buildManager, dockerClient)
|
||||
initArtifacts(filepath.Join("/tmp/containr-builds", "artifacts"))
|
||||
purgeStaleArtifacts()
|
||||
// Node-pinned services dispatch container lifecycle to the agent's
|
||||
// command queue instead of the local Docker socket.
|
||||
deploymentEngine.SetNodeRunner(newAgentNodeRunner(db))
|
||||
|
||||
@@ -539,12 +539,16 @@ func createBuildContext(sourcePath, dockerfileContent string) (io.ReadCloser, er
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add source files to tar
|
||||
// Add source files to tar. The checkout's .git metadata is never build
|
||||
// context — it bloats the tar and ships clone URLs to remote daemons.
|
||||
err = filepath.Walk(sourcePath, func(file string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if fi.IsDir() {
|
||||
if fi.Name() == ".git" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ package build
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
@@ -92,6 +94,75 @@ func (bm *BuildManager) Build(ctx context.Context, req *types.BuildRequest) (*ty
|
||||
}
|
||||
}
|
||||
|
||||
// PackageContext produces the same build-context tar Build would hand the
|
||||
// local daemon, without building it. Node agents download the tar and run
|
||||
// docker build themselves — the Dockerfile lands at the tar root.
|
||||
func (bm *BuildManager) PackageContext(ctx context.Context, req *types.BuildRequest) (io.ReadCloser, error) {
|
||||
if req.BuildType == "" {
|
||||
detectedType, err := bm.DetectBuildType(ctx, req.SourcePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to detect build type: %w", err)
|
||||
}
|
||||
req.BuildType = string(detectedType)
|
||||
}
|
||||
|
||||
var dockerfileContent string
|
||||
switch BuildType(req.BuildType) {
|
||||
case BuildTypeDockerfile:
|
||||
name, err := bm.dockerfileBuilder.DetectDockerfile(ctx, req.SourcePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to detect Dockerfile: %w", err)
|
||||
}
|
||||
path := filepath.Join(req.SourcePath, name)
|
||||
if optimized, err := bm.dockerfileBuilder.OptimizeDockerfile(ctx, path); err == nil {
|
||||
path = optimized
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read Dockerfile: %w", err)
|
||||
}
|
||||
dockerfileContent = string(data)
|
||||
case BuildTypeRailpack:
|
||||
if err := bm.railpackBuilder.DetectRailpack(ctx, req.SourcePath); err != nil {
|
||||
return nil, fmt.Errorf("Railpack cannot build this project: %w", err)
|
||||
}
|
||||
content, err := bm.railpackBuilder.generateDockerfile(ctx, req.SourcePath, &RailpackConfig{
|
||||
BuildCmd: req.BuildCommand,
|
||||
StartCmd: req.StartCommand,
|
||||
Env: req.Environment,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate Dockerfile with Railpack: %w", err)
|
||||
}
|
||||
dockerfileContent = content
|
||||
case BuildTypeNixpacks:
|
||||
plan, err := bm.nixpacksBuilder.GeneratePlan(ctx, req.SourcePath, &NixpacksConfig{
|
||||
BuildCmd: req.BuildCommand,
|
||||
StartCmd: req.StartCommand,
|
||||
Env: req.Environment,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate build plan: %w", err)
|
||||
}
|
||||
path, err := bm.nixpacksBuilder.generateDockerfile(plan)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate Dockerfile: %w", err)
|
||||
}
|
||||
defer os.Remove(path)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read generated Dockerfile: %w", err)
|
||||
}
|
||||
dockerfileContent = string(data)
|
||||
case BuildTypeStatic:
|
||||
dockerfileContent = staticDockerfile
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported build type for remote packaging: %s", req.BuildType)
|
||||
}
|
||||
|
||||
return createBuildContext(req.SourcePath, dockerfileContent)
|
||||
}
|
||||
|
||||
// buildPrebuilt handles prebuilt image deployments
|
||||
func (bm *BuildManager) buildPrebuilt(ctx context.Context, req *types.BuildRequest) (*types.BuildResponse, error) {
|
||||
if req.PrebuiltImage == "" {
|
||||
|
||||
@@ -476,17 +476,22 @@ func (n *NixpacksBuilder) Build(ctx context.Context, req *types.BuildRequest) (*
|
||||
}
|
||||
|
||||
// Create temporary Dockerfile
|
||||
dockerfile, err := n.generateDockerfile(plan)
|
||||
dockerfilePath, err := n.generateDockerfile(plan)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate Dockerfile: %w", err)
|
||||
}
|
||||
defer os.Remove(dockerfile)
|
||||
defer os.Remove(dockerfilePath)
|
||||
|
||||
// Build Docker image
|
||||
imageName := fmt.Sprintf("%s:%s", req.ImageName, req.ImageTag)
|
||||
|
||||
// Create build context tar
|
||||
buildCtx, err := createBuildContext(req.SourcePath, dockerfile)
|
||||
// Create build context tar — createBuildContext takes the Dockerfile
|
||||
// contents, not its path.
|
||||
dockerfileData, err := os.ReadFile(dockerfilePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read generated Dockerfile: %w", err)
|
||||
}
|
||||
buildCtx, err := createBuildContext(req.SourcePath, string(dockerfileData))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create build context: %w", err)
|
||||
}
|
||||
|
||||
@@ -95,12 +95,29 @@ type ServiceConfig struct {
|
||||
// reuses it so public URLs survive restarts. 0 = pick ephemeral.
|
||||
PublishedPort int32 `json:"published_port,omitempty"`
|
||||
// NodeID pins the service to a registered node agent; "" = local Docker
|
||||
// host. Remote placement requires a registry-pullable image — builds stay
|
||||
// on the local host (build-on-node is not yet supported).
|
||||
// host.
|
||||
NodeID string `json:"node_id,omitempty"`
|
||||
// Spread distributes replicas across every online, schedulable agent.
|
||||
// Same registry-image constraint as NodeID.
|
||||
Spread bool `json:"spread,omitempty"`
|
||||
// RemoteBuild, when set, makes each target node docker-build the image
|
||||
// itself from a packaged context artifact instead of pulling a registry
|
||||
// tag. The image name comes from the deployment, not the spec.
|
||||
RemoteBuild *RemoteBuildSpec `json:"remote_build,omitempty"`
|
||||
// RemoteLoad ships a docker-save tarball to each target node — used for
|
||||
// rollbacks of locally-built tags that no registry can serve.
|
||||
RemoteLoad *RemoteLoadSpec `json:"remote_load,omitempty"`
|
||||
}
|
||||
|
||||
// RemoteBuildSpec is the agent-side docker build request.
|
||||
type RemoteBuildSpec struct {
|
||||
ArtifactID string `json:"artifact_id"`
|
||||
BuildArgs map[string]string `json:"build_args,omitempty"`
|
||||
NoCache bool `json:"no_cache,omitempty"`
|
||||
}
|
||||
|
||||
// RemoteLoadSpec asks the agent to docker-load an image tarball artifact.
|
||||
type RemoteLoadSpec struct {
|
||||
ArtifactID string `json:"artifact_id"`
|
||||
}
|
||||
|
||||
type PortMapping struct {
|
||||
@@ -211,6 +228,12 @@ func (de *DeploymentEngine) DockerClient() *docker.Client {
|
||||
return de.dockerClient
|
||||
}
|
||||
|
||||
// BuildManager exposes the engine's builders for context packaging — remote
|
||||
// nodes build the tar this produces themselves.
|
||||
func (de *DeploymentEngine) BuildManager() *build.BuildManager {
|
||||
return de.buildManager
|
||||
}
|
||||
|
||||
// SetNodeRunner wires remote-node dispatch for services pinned to an agent.
|
||||
func (de *DeploymentEngine) SetNodeRunner(r NodeRunner) {
|
||||
de.nodeRunner = r
|
||||
@@ -326,11 +349,19 @@ func (de *DeploymentEngine) executeDeployment(ctx context.Context, deployment *D
|
||||
})
|
||||
}
|
||||
|
||||
// buildImage builds the container image
|
||||
// buildImage builds the container image. "remote" build types produce no
|
||||
// local image — the target node builds or loads it from an artifact; the
|
||||
// deployment still needs a deterministic tag so every node names it alike.
|
||||
func (de *DeploymentEngine) buildImage(ctx context.Context, deployment *Deployment, buildConfig *BuildConfig) (string, error) {
|
||||
if buildConfig == nil {
|
||||
return "", fmt.Errorf("build config is required")
|
||||
}
|
||||
if buildConfig.BuildType == "remote" {
|
||||
if buildConfig.PrebuiltImage != "" {
|
||||
return buildConfig.PrebuiltImage, nil
|
||||
}
|
||||
return fmt.Sprintf("containr-%s-%s:%s", deployment.ServiceID, deployment.Environment, deployment.ID), nil
|
||||
}
|
||||
|
||||
buildReq := &types.BuildRequest{
|
||||
BuildType: buildConfig.BuildType,
|
||||
@@ -393,6 +424,8 @@ func (de *DeploymentEngine) deployService(ctx context.Context, deployment *Deplo
|
||||
BasicAuthUsers: cfg.BasicAuthUsers,
|
||||
NodeID: cfg.NodeID,
|
||||
Spread: cfg.Spread,
|
||||
RemoteBuild: cfg.RemoteBuild,
|
||||
RemoteLoad: cfg.RemoteLoad,
|
||||
}
|
||||
for _, pm := range cfg.PortMappings {
|
||||
if spec.Port == 0 {
|
||||
|
||||
@@ -63,6 +63,12 @@ type RuntimeSpec struct {
|
||||
// PlacementTags restricts auto/spread candidates to agents carrying
|
||||
// every listed tag. An explicit NodeID pin bypasses tag filtering.
|
||||
PlacementTags []string
|
||||
// RemoteBuild makes target nodes docker-build Image from a packaged
|
||||
// context artifact before replicas start (git-sourced remote deploys).
|
||||
RemoteBuild *RemoteBuildSpec
|
||||
// RemoteLoad makes target nodes docker-load a saved image tarball —
|
||||
// remote rollbacks of locally-built tags.
|
||||
RemoteLoad *RemoteLoadSpec
|
||||
}
|
||||
|
||||
// RuntimeContainer describes one live replica.
|
||||
|
||||
@@ -336,6 +336,13 @@ func (c *Client) BuildImage(ctx context.Context, buildContext io.Reader, options
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// SaveImage streams refs as a docker-save tarball. Remote nodes receive
|
||||
// these through the artifact endpoint and docker load them — that is how
|
||||
// locally-built rollback tags reach nodes without a registry.
|
||||
func (c *Client) SaveImage(ctx context.Context, refs ...string) (io.ReadCloser, error) {
|
||||
return c.cli.ImageSave(ctx, refs)
|
||||
}
|
||||
|
||||
// RemoveImage removes an image
|
||||
func (c *Client) RemoveImage(ctx context.Context, imageID string, force bool) ([]image.DeleteResponse, error) {
|
||||
return c.cli.ImageRemove(ctx, imageID, image.RemoveOptions{
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
// Package source materializes git checkouts for builds. Services reference
|
||||
// a repository by URL (or owner/repo); deployments fetch the exact branch or
|
||||
// commit into an isolated workdir, then point the builders at build_path
|
||||
// inside it. Credentials travel through GIT_ASKPASS so tokens never land in
|
||||
// argv, .git/config, logs, or the build context.
|
||||
package source
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Spec describes one checkout.
|
||||
type Spec struct {
|
||||
CloneURL string // https URL (NormalizeCloneURL output); local paths allowed for tests
|
||||
Branch string // ref to fetch when Commit is empty
|
||||
Commit string // optional SHA pinned over Branch
|
||||
BuildPath string // subdirectory inside the repo; "" or "." = repo root
|
||||
Token string // optional http credential
|
||||
User string // credential username; defaults per provider in ResolveUser
|
||||
}
|
||||
|
||||
// NormalizeCloneURL converts the forms services store into an https clone
|
||||
// URL: ssh remotes (git@host:org/repo, ssh://git@host/org/repo) map onto
|
||||
// https so token auth works uniformly. A bare "owner/repo" is returned
|
||||
// unchanged — the caller resolves the host through git_repositories.
|
||||
func NormalizeCloneURL(raw string) (string, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return "", fmt.Errorf("empty repository reference")
|
||||
}
|
||||
// scp-style ssh remote: [email protected]:org/repo(.git)
|
||||
if strings.HasPrefix(raw, "git@") && strings.Contains(raw, ":") {
|
||||
host := raw[len("git@"):strings.Index(raw, ":")]
|
||||
path := strings.TrimPrefix(raw[strings.Index(raw, ":")+1:], "/")
|
||||
return "https://" + host + "/" + path, nil
|
||||
}
|
||||
if strings.HasPrefix(raw, "ssh://") {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid ssh remote %q", raw)
|
||||
}
|
||||
return "https://" + u.Host + u.Path, nil
|
||||
}
|
||||
if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {
|
||||
return raw, nil
|
||||
}
|
||||
if strings.Contains(raw, "://") {
|
||||
return "", fmt.Errorf("unsupported repository scheme in %q", raw)
|
||||
}
|
||||
if strings.Count(raw, "/") == 1 && !strings.ContainsAny(raw, " @:") {
|
||||
return raw, nil // owner/repo — resolved against git_repositories upstream
|
||||
}
|
||||
return "", fmt.Errorf("unsupported repository reference %q", raw)
|
||||
}
|
||||
|
||||
// ValidateRemoteURL rejects non-http(s) clones before a deploy accepts them.
|
||||
// Local checkouts (tests, mounted workspaces) bypass this via Checkout.
|
||||
func ValidateRemoteURL(cloneURL string) error {
|
||||
if strings.HasPrefix(cloneURL, "http://") || strings.HasPrefix(cloneURL, "https://") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("clone URL must be http(s), got %q", cloneURL)
|
||||
}
|
||||
|
||||
// Checkout fetches spec into destDir/repo and returns the directory the
|
||||
// builders should consume (checkout root + build_path). Callers
|
||||
// RemoveAll(destDir) — the checkout is a per-deployment workspace, never
|
||||
// shared. The askpass helper lives outside repo so it can't leak into the
|
||||
// build context.
|
||||
func Checkout(ctx context.Context, destDir string, spec Spec) (string, error) {
|
||||
repoDir := filepath.Join(destDir, "repo")
|
||||
if err := os.MkdirAll(repoDir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
env := append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
|
||||
if spec.Token != "" {
|
||||
// Askpass keeps the token out of argv and .git/config.
|
||||
askpass := filepath.Join(destDir, ".git-askpass")
|
||||
script := "#!/bin/sh\ncase \"$1\" in\n*sername*) printf '%s' \"$CONTAINR_GIT_USER\" ;;\n*) printf '%s' \"$CONTAINR_GIT_TOKEN\" ;;\nesac\n"
|
||||
if err := os.WriteFile(askpass, []byte(script), 0o700); err != nil {
|
||||
return "", fmt.Errorf("write askpass: %w", err)
|
||||
}
|
||||
user := spec.User
|
||||
if user == "" {
|
||||
user = "x-access-token"
|
||||
}
|
||||
env = append(env,
|
||||
"GIT_ASKPASS="+askpass,
|
||||
"CONTAINR_GIT_USER="+user,
|
||||
"CONTAINR_GIT_TOKEN="+spec.Token,
|
||||
)
|
||||
}
|
||||
|
||||
run := func(args ...string) error {
|
||||
cmd := exec.CommandContext(ctx, "git", args...)
|
||||
cmd.Dir = repoDir
|
||||
cmd.Env = env
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("git %s: %s", args[0], strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := run("init", "-q"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := run("remote", "add", "origin", spec.CloneURL); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
ref := spec.Commit
|
||||
if ref == "" {
|
||||
ref = spec.Branch
|
||||
}
|
||||
if ref == "" {
|
||||
ref = "HEAD"
|
||||
}
|
||||
// Cheap path first, widening on failure: dumb transports reject
|
||||
// --depth outright, and some hosts refuse direct SHA fetches so a
|
||||
// pinned commit may only be reachable through its branch.
|
||||
attempts := [][]string{
|
||||
{"fetch", "--depth", "1", "origin", ref},
|
||||
{"fetch", "origin", ref},
|
||||
}
|
||||
if spec.Commit != "" {
|
||||
branchRef := spec.Branch
|
||||
if branchRef == "" {
|
||||
branchRef = "HEAD"
|
||||
}
|
||||
attempts = append(attempts,
|
||||
[]string{"fetch", "--depth", "100", "origin", branchRef},
|
||||
[]string{"fetch", "origin", branchRef},
|
||||
)
|
||||
}
|
||||
var fetchErr error
|
||||
for _, args := range attempts {
|
||||
if fetchErr = run(args...); fetchErr == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if fetchErr != nil {
|
||||
return "", fetchErr
|
||||
}
|
||||
if err := run("checkout", "-q", spec.CommitOrFetchHead()); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return ResolveBuildPath(repoDir, spec.BuildPath)
|
||||
}
|
||||
|
||||
// CommitOrFetchHead is the checkout target: the pinned commit or FETCH_HEAD.
|
||||
func (s Spec) CommitOrFetchHead() string {
|
||||
if s.Commit != "" {
|
||||
return s.Commit
|
||||
}
|
||||
return "FETCH_HEAD"
|
||||
}
|
||||
|
||||
// ResolveBuildPath maps build_path onto the checkout and rejects escapes —
|
||||
// a path like ../../etc must never leave the workspace.
|
||||
func ResolveBuildPath(checkoutDir, buildPath string) (string, error) {
|
||||
bp := strings.TrimSpace(buildPath)
|
||||
if bp == "" || bp == "." {
|
||||
return checkoutDir, nil
|
||||
}
|
||||
if filepath.IsAbs(bp) {
|
||||
return "", fmt.Errorf("build_path must be relative, got %q", bp)
|
||||
}
|
||||
clean := filepath.Clean(bp)
|
||||
if clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("build_path escapes the checkout: %q", bp)
|
||||
}
|
||||
resolved := filepath.Join(checkoutDir, clean)
|
||||
info, err := os.Stat(resolved)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("build_path %q not found in checkout", bp)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", fmt.Errorf("build_path %q is not a directory", bp)
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeCloneURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
in, want string
|
||||
err bool
|
||||
}{
|
||||
{"https://github.com/org/repo.git", "https://github.com/org/repo.git", false},
|
||||
{"[email protected]:org/repo.git", "https://github.com/org/repo.git", false},
|
||||
{"ssh://[email protected]/org/repo.git", "https://gitlab.example.com/org/repo.git", false},
|
||||
{"org/repo", "org/repo", false},
|
||||
{" https://gitlab.com/a/b.git ", "https://gitlab.com/a/b.git", false},
|
||||
{"", "", true},
|
||||
{"ftp://example.com/repo", "", true},
|
||||
{"weird string", "", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, err := NormalizeCloneURL(c.in)
|
||||
if c.err {
|
||||
if err == nil {
|
||||
t.Errorf("%q: expected error, got %q", c.in, got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("%q: unexpected error %v", c.in, err)
|
||||
continue
|
||||
}
|
||||
if got != c.want {
|
||||
t.Errorf("%q: got %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRemoteURL(t *testing.T) {
|
||||
if err := ValidateRemoteURL("https://github.com/o/r.git"); err != nil {
|
||||
t.Errorf("https should pass: %v", err)
|
||||
}
|
||||
if err := ValidateRemoteURL("file:///tmp/repo"); err == nil {
|
||||
t.Error("file:// should be rejected")
|
||||
}
|
||||
if err := ValidateRemoteURL("/tmp/repo"); err == nil {
|
||||
t.Error("local path should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
// initRepo creates a small git repo with a branch and returns its path.
|
||||
func initRepo(t *testing.T) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
run := func(args ...string) {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = append(os.Environ(),
|
||||
"GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t",
|
||||
"GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t")
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %s", args, out)
|
||||
}
|
||||
}
|
||||
run("init", "-q", "-b", "main")
|
||||
if err := os.WriteFile(filepath.Join(dir, "app.txt"), []byte("v1"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "sub"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "sub", "inner.txt"), []byte("sub"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run("add", ".")
|
||||
run("commit", "-q", "-m", "first")
|
||||
return dir
|
||||
}
|
||||
|
||||
func TestCheckoutBranch(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("git not available")
|
||||
}
|
||||
repo := initRepo(t)
|
||||
dest := t.TempDir()
|
||||
dir, err := Checkout(context.Background(), dest, Spec{CloneURL: repo, Branch: "main"})
|
||||
if err != nil {
|
||||
t.Fatalf("checkout: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "app.txt")); err != nil {
|
||||
t.Errorf("expected app.txt in checkout: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckoutBuildPath(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("git not available")
|
||||
}
|
||||
repo := initRepo(t)
|
||||
dest := t.TempDir()
|
||||
dir, err := Checkout(context.Background(), dest, Spec{CloneURL: repo, Branch: "main", BuildPath: "sub"})
|
||||
if err != nil {
|
||||
t.Fatalf("checkout: %v", err)
|
||||
}
|
||||
if filepath.Base(dir) != "sub" {
|
||||
t.Errorf("expected sub build path, got %s", dir)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "inner.txt")); err != nil {
|
||||
t.Errorf("expected inner.txt under build path: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckoutCommit(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("git not available")
|
||||
}
|
||||
repo := initRepo(t)
|
||||
out, err := exec.Command("git", "-C", repo, "rev-parse", "HEAD").Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sha := string(out[:len(out)-1])
|
||||
dest := t.TempDir()
|
||||
dir, err := Checkout(context.Background(), dest, Spec{CloneURL: repo, Commit: sha, Branch: "main"})
|
||||
if err != nil {
|
||||
t.Fatalf("checkout commit: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "app.txt")); err != nil {
|
||||
t.Errorf("expected app.txt in pinned checkout: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBuildPathEscapes(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if _, err := ResolveBuildPath(root, "../../etc"); err == nil {
|
||||
t.Error("escape should be rejected")
|
||||
}
|
||||
if _, err := ResolveBuildPath(root, "/abs/path"); err == nil {
|
||||
t.Error("absolute path should be rejected")
|
||||
}
|
||||
if _, err := ResolveBuildPath(root, "missing"); err == nil {
|
||||
t.Error("missing dir should be rejected")
|
||||
}
|
||||
if got, err := ResolveBuildPath(root, "."); err != nil || got != root {
|
||||
t.Errorf("dot should resolve to root, got %q err=%v", got, err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user