Merge pull request #1762 from roboflow/feat/openai-block-v4

OpenAI workflow block v4

Author: brunopicinin

inference/core/workflows/core_steps/formatters/vlm_as_detector/v2.py

@@ -125,9 +125,11 @@ class BlockManifest(WorkflowBlockManifest):
             }
         },
     )
-    model_type: Literal["google-gemini", "anthropic-claude", "florence-2"] = Field(
-        description="Type of the model that generated prediction",
-        examples=[["google-gemini", "anthropic-claude", "florence-2"]],
+    model_type: Literal["openai", "google-gemini", "anthropic-claude", "florence-2"] = (
+        Field(
+            description="Type of the model that generated prediction",
+            examples=[["google-gemini", "anthropic-claude", "florence-2"]],
+        )
     )
     task_type: Literal[tuple(SUPPORTED_TASKS)] = Field(
         description="Task type to performed by model.",
@@ -234,7 +236,7 @@ def try_parse_json(content: str) -> Tuple[bool, dict]:
         return True, {}
 
 
-def parse_gemini_object_detection_response(
+def parse_llm_object_detection_response(
     image: WorkflowImageData,
     parsed_data: dict,
     classes: List[str],
@@ -353,8 +355,11 @@ def get_4digit_from_md5(input_string):
 
 
 REGISTERED_PARSERS = {
-    ("google-gemini", "object-detection"): parse_gemini_object_detection_response,
-    ("anthropic-claude", "object-detection"): parse_gemini_object_detection_response,
+    # LLMs
+    ("openai", "object-detection"): parse_llm_object_detection_response,
+    ("google-gemini", "object-detection"): parse_llm_object_detection_response,
+    ("anthropic-claude", "object-detection"): parse_llm_object_detection_response,
+    # Florence 2
     ("florence-2", "object-detection"): partial(
         parse_florence2_object_detection_response, florence_task_type="<OD>"
     ),

inference/core/workflows/core_steps/loader.py

@@ -221,6 +221,9 @@
 from inference.core.workflows.core_steps.models.foundation.openai.v3 import (
     OpenAIBlockV3,
 )
+from inference.core.workflows.core_steps.models.foundation.openai.v4 import (
+    OpenAIBlockV4,
+)
 from inference.core.workflows.core_steps.models.foundation.perception_encoder.v1 import (
     PerceptionEncoderModelBlockV1,
 )
@@ -642,6 +645,7 @@ def load_blocks() -> List[Type[WorkflowBlock]]:
         OpenAIBlockV1,
         OpenAIBlockV2,
         OpenAIBlockV3,
+        OpenAIBlockV4,
         PathDeviationAnalyticsBlockV1,
         PathDeviationAnalyticsBlockV2,
         PixelateVisualizationBlockV1,