Merge pull request #1743 from roboflow/proxy-sam3-endpoint

Proxy sam3 endpoint

Author: grzegorz-roboflow

inference/core/interfaces/http/http_api.py

@@ -9,6 +9,7 @@
 from uuid import uuid4
 
 import asgi_correlation_id
+import requests
 import uvicorn
 from fastapi import (
     BackgroundTasks,
@@ -125,6 +126,7 @@
 )
 from inference.core.env import (
     ALLOW_ORIGINS,
+    API_BASE_URL,
     API_KEY,
     API_LOGGING_ENABLED,
     BUILDER_ORIGIN,
@@ -165,7 +167,10 @@
     NOTEBOOK_PORT,
     PRELOAD_MODELS,
     PROFILE,
+    ROBOFLOW_INTERNAL_SERVICE_NAME,
+    ROBOFLOW_INTERNAL_SERVICE_SECRET,
     ROBOFLOW_SERVICE_SECRET,
+    SAM3_EXEC_MODE,
     WEBRTC_WORKER_ENABLED,
     WORKFLOWS_MAX_CONCURRENT_STEPS,
     WORKFLOWS_PROFILER_BUFFER_SIZE,
@@ -228,6 +233,7 @@
 from inference.core.managers.metrics import get_container_stats
 from inference.core.managers.prometheus import InferenceInstrumentator
 from inference.core.roboflow_api import (
+    build_roboflow_api_headers,
     get_roboflow_workspace,
     get_roboflow_workspace_async,
     get_workflow_specification,
@@ -2379,7 +2385,7 @@ def sam2_segment_image(
                         )
                     return model_response
 
-            if CORE_MODEL_SAM3_ENABLED:
+            if CORE_MODEL_SAM3_ENABLED and not GCP_SERVERLESS:
 
                 @app.post(
                     "/sam3/embed_image",
@@ -2401,6 +2407,12 @@ def sam3_embed_image(
                 ):
                     logger.debug(f"Reached /sam3/embed_image")
 
+                    if SAM3_EXEC_MODE == "remote":
+                        raise HTTPException(
+                            status_code=501,
+                            detail="SAM3 embedding is not supported in remote execution mode.",
+                        )
+
                     self.model_manager.add_model(
                         "sam3/sam3_interactive",
                         api_key=api_key,
@@ -2434,6 +2446,12 @@ def sam3_visual_segment(
                 ):
                     logger.debug(f"Reached /sam3/visual_segment")
 
+                    if SAM3_EXEC_MODE == "remote":
+                        raise HTTPException(
+                            status_code=501,
+                            detail="SAM3 visual segmentation is not supported in remote execution mode.",
+                        )
+
                     self.model_manager.add_model(
                         "sam3/sam3_interactive",
                         api_key=api_key,
@@ -2447,6 +2465,8 @@ def sam3_visual_segment(
                     )
                     return model_response
 
+            if CORE_MODEL_SAM3_ENABLED:
+
                 @app.post(
                     "/sam3/concept_segment",
                     response_model=Sam3SegmentationResponse,
@@ -2465,6 +2485,99 @@ def sam3_segment_image(
                     countinference: Optional[bool] = None,
                     service_secret: Optional[str] = None,
                 ):
+                    if SAM3_EXEC_MODE == "remote":
+                        if not inference_request.model_id.startswith("sam3/"):
+                            raise HTTPException(
+                                status_code=501,
+                                detail="Fine-tuned SAM3 models are not supported in remote execution mode yet. Please use a workflow or self-host the server.",
+                            )
+                        endpoint = f"{API_BASE_URL}/inferenceproxy/seg-preview"
+
+                        # Construct payload for remote API
+                        # The remote API expects:
+                        # {
+                        #     "image": {"type": "base64", "value": ...},
+                        #     "prompts": [{"type": "text", "text": ...}, ...],
+                        #     "output_prob_thresh": ...
+                        # }
+
+                        # Extract prompts from request
+                        http_prompts = []
+                        for prompt in inference_request.prompts:
+                            p_dict = prompt.dict(exclude_none=True)
+                            # Ensure type is set if missing (default to text if text is present)
+                            if "type" not in p_dict:
+                                if "text" in p_dict:
+                                    p_dict["type"] = "text"
+                            http_prompts.append(p_dict)
+
+                        # Prepare image
+                        # inference_request.image is InferenceRequestImage
+                        if inference_request.image.type == "base64":
+                            http_image = {
+                                "type": "base64",
+                                "value": inference_request.image.value,
+                            }
+                        elif inference_request.image.type == "url":
+                            http_image = {
+                                "type": "url",
+                                "value": inference_request.image.value,
+                            }
+                        elif inference_request.image.type == "numpy":
+                            # Numpy not supported for remote proxy easily without serialization,
+                            # but InferenceRequestImage usually comes as base64/url in HTTP API.
+                            # If it is numpy, we might need to handle it, but for now assume base64/url.
+                            # If it's numpy, it's likely from internal call, but this is HTTP API.
+                            http_image = {
+                                "type": "numpy",
+                                "value": inference_request.image.value,
+                            }
+                        else:
+                            http_image = {
+                                "type": inference_request.image.type,
+                                "value": inference_request.image.value,
+                            }
+
+                        payload = {
+                            "image": http_image,
+                            "prompts": http_prompts,
+                            "output_prob_thresh": inference_request.output_prob_thresh,
+                        }
+
+                        try:
+                            headers = {"Content-Type": "application/json"}
+                            if ROBOFLOW_INTERNAL_SERVICE_NAME:
+                                headers["X-Roboflow-Internal-Service-Name"] = (
+                                    ROBOFLOW_INTERNAL_SERVICE_NAME
+                                )
+                            if ROBOFLOW_INTERNAL_SERVICE_SECRET:
+                                headers["X-Roboflow-Internal-Service-Secret"] = (
+                                    ROBOFLOW_INTERNAL_SERVICE_SECRET
+                                )
+
+                            headers = build_roboflow_api_headers(
+                                explicit_headers=headers
+                            )
+
+                            response = requests.post(
+                                f"{endpoint}?api_key={api_key}",
+                                json=payload,
+                                headers=headers,
+                                timeout=60,
+                            )
+                            response.raise_for_status()
+                            resp_json = response.json()
+
+                            # The remote API returns the same structure as Sam3SegmentationResponse
+                            return Sam3SegmentationResponse(**resp_json)
+
+                        except Exception as e:
+                            logger.error(f"SAM3 remote request failed: {e}")
+                            raise HTTPException(
+                                status_code=500,
+                                detail=f"SAM3 remote request failed: {str(e)}",
+                            )
+
                     if inference_request.model_id.startswith("sam3/"):
                         self.model_manager.add_model(
                             inference_request.model_id,

inference/core/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.62.1"
+__version__ = "0.62.2"
 
 
 if __name__ == "__main__":