Fix image lookup with remote registry

Author: akclace

internal/app/container_handler.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"html/template"
 	"io/fs"
+	"math"
 	"net/http"
 	"net/url"
 	"os"
@@ -294,6 +295,8 @@ func (h *ContainerHandler) idleAppShutdown(ctx context.Context) {
 }
 
 func (h *ContainerHandler) healthChecker(ctx context.Context) {
+	time.Sleep(60 * time.Second) // wait for 1 minute to let the app start up
+	h.Debug().Msgf("Health checker started for app %s", h.app.Id)
 	for range h.healthCheckTicker.C {
 		err := h.WaitForHealth(h.containerConfig.StatusHealthAttempts)
 		if err == nil {
@@ -707,21 +710,18 @@ func (h *ContainerHandler) WaitForHealth(attempts int) error {
 
 	var err error
 	var resp *http.Response
-	proxyUrl, err := url.Parse(h.GetProxyUrl())
-	if err != nil {
-		return err
-	}
-	if !h.stripAppPath {
-		// Apps like Streamlit require the app path to be present
-		proxyUrl = proxyUrl.JoinPath(h.app.Path)
-	}
+	for attempt := 1; attempt <= attempts; attempt++ {
+		proxyUrl, err := url.Parse(h.GetProxyUrl())
+		if err != nil {
+			return err
+		}
+		if !h.stripAppPath {
+			// Apps like Streamlit require the app path to be present
+			proxyUrl = proxyUrl.JoinPath(h.app.Path)
+		}
 
-	proxyUrl = proxyUrl.JoinPath(h.health)
-	if err != nil {
-		return err
-	}
+		proxyUrl = proxyUrl.JoinPath(h.health)
 
-	for attempt := 1; attempt <= attempts; attempt++ {
 		resp, err = client.Get(proxyUrl.String())
 		statusCode := "N/A"
 		if err == nil {
@@ -736,7 +736,8 @@ func (h *ContainerHandler) WaitForHealth(attempts int) error {
 		}
 
 		h.Debug().Msgf("Attempt %d failed on %s : status %s err %s", attempt, proxyUrl, statusCode, err)
-		time.Sleep(1 * time.Second)
+		sleepSecs := math.Min(float64(attempt), 5)
+		time.Sleep(time.Duration(sleepSecs) * time.Second)
 	}
 	return err
 }

internal/container/command.go

@@ -147,7 +147,7 @@ func (c *ContainerCommand) RemoveContainer(ctx context.Context, name ContainerNa
 
 // GetContainerState returns the host:port of the running container, "" if not running. running is true if the container is running.
 func (c *ContainerCommand) GetContainerState(ctx context.Context, name ContainerName) (string, bool, error) {
-	containers, err := c.getContainers(ctx, name, false)
+	containers, err := c.getContainers(ctx, name, true)
 	if err != nil {
 		return "", false, fmt.Errorf("error getting containers: %w", err)
 	}
@@ -345,6 +345,10 @@ func (c *ContainerCommand) RunContainer(ctx context.Context, appEntry *types.App
 }
 
 func (c *ContainerCommand) ImageExists(ctx context.Context, name ImageName) (bool, error) {
+	if c.config.Registry.URL != "" {
+		return ImageExists(ctx, c.Logger, string(name), &c.config.Registry)
+	}
+
 	c.Debug().Msgf("Getting images with name %s", name)
 	args := []string{"images", string(name)}
 	cmd := exec.CommandContext(ctx, c.config.System.ContainerCommand, args...)

internal/container/delegate.go

@@ -276,7 +276,8 @@ func delegateBuild(ctx context.Context, logger *types.Logger, config *types.Serv
 		return fmt.Errorf("read image from docker daemon: %w", err)
 	}
 
-	remoteRef, remoteOpts, err := GetDockerConfig(ctx, remoteTag, data.RegistryConfig)
+	logger.Debug().Msgf("Getting remote registry config for %s", data.ImageTag)
+	remoteRef, remoteOpts, err := GetDockerConfig(ctx, data.ImageTag, data.RegistryConfig)
 	if err != nil {
 		return fmt.Errorf("get remote registry config: %w", err)
 	}

internal/container/kubernetes.go

@@ -52,12 +52,16 @@ func loadConfig() (*rest.Config, error) {
 }
 
 func (k *KubernetesContainerManager) ImageExists(ctx context.Context, name ImageName) (bool, error) {
+	if k.config.Registry.URL == "" {
+		return false, fmt.Errorf("registry url is required for kubernetes container manager")
+	}
 	return ImageExists(ctx, k.Logger, string(name), &k.config.Registry)
 }
 
 func (k *KubernetesContainerManager) BuildImage(ctx context.Context, imgName ImageName, sourceUrl, containerFile string, containerArgs map[string]string) error {
 	targetUrl, found := strings.CutPrefix(k.config.Builder.Mode, "delegate:")
 	if found {
+		// delegated build
 		err := sendDelegateBuild(targetUrl, DelegateRequest{
 			ImageTag:       string(imgName),
 			ContainerFile:  containerFile,

internal/container/registry.go

@@ -214,6 +214,7 @@ func GetDockerConfig(ctx context.Context, imageRef string, registryConfig *types
 		parseOpts = append(parseOpts, name.Insecure)
 	}
 
+	imageRef = registryConfig.URL + "/" + imageRef
 	ref, err := name.ParseReference(imageRef, parseOpts...)
 	if err != nil {
 		return nil, nil, fmt.Errorf("parse ref: %w", err)

internal/system/config_test.go

@@ -70,12 +70,12 @@ func TestServerConfig(t *testing.T) {
 
 	// Container Settings
 	testutil.AssertEqualsString(t, "health", "/", c.AppConfig.Container.HealthUrl)
-	testutil.AssertEqualsInt(t, "attempts", 30, c.AppConfig.Container.HealthAttemptsAfterStartup)
+	testutil.AssertEqualsInt(t, "attempts", 10, c.AppConfig.Container.HealthAttemptsAfterStartup)
 	testutil.AssertEqualsInt(t, "timeout", 5, c.AppConfig.Container.HealthTimeoutSecs)
 	testutil.AssertEqualsInt(t, "idle", 180, c.AppConfig.Container.IdleShutdownSecs)
 	testutil.AssertEqualsInt(t, "idle bytes high watermark", 1500, c.AppConfig.Container.IdleBytesHighWatermark)
-	testutil.AssertEqualsInt(t, "status interval", 5, c.AppConfig.Container.StatusCheckIntervalSecs)
-	testutil.AssertEqualsInt(t, "status attempts", 3, c.AppConfig.Container.StatusHealthAttempts)
+	testutil.AssertEqualsInt(t, "status interval", 20, c.AppConfig.Container.StatusCheckIntervalSecs)
+	testutil.AssertEqualsInt(t, "status attempts", 10, c.AppConfig.Container.StatusHealthAttempts)
 
 	testutil.AssertEqualsInt(t, "proxy max idle", 250, c.AppConfig.Proxy.MaxIdleConns)
 	testutil.AssertEqualsInt(t, "proxy idle timeout", 15, c.AppConfig.Proxy.IdleConnTimeoutSecs)

internal/system/openrun.default.toml

@@ -108,7 +108,7 @@ cors.max_age = "2678400"
 
 # Health check Config
 container.health_url = "/"
-container.health_attempts_after_startup = 30
+container.health_attempts_after_startup = 10
 container.health_timeout_secs = 5
 
 # Idle Shutdown Config
@@ -118,8 +118,8 @@ container.idle_bytes_high_watermark = 1500 # bytes high watermark for idle shutd
                                            # (1500 bytes sent and recv over 180 seconds)
 
 # Status check Config
-container.status_check_interval_secs = 5
-container.status_health_attempts = 3
+container.status_check_interval_secs = 20
+container.status_health_attempts = 10
 
 # Proxy related settings
 proxy.max_idle_conns = 250