regenerate compiled workflows

Signed-off-by: Humair Khan <[email protected]>

Author: HumairAK

test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml

@@ -2,25 +2,47 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   creationTimestamp: null
-  generateName: pipeline-with-semaphore-
+  generateName: pipeline-with-custom-path-artifact-
 spec:
   arguments:
     parameters:
-    - name: components-8c43378018a8d9ed1a17d6cd02a579db5fa506abf47189a0385306301b7fa2c2
-      value: '{"executorLabel":"exec-comp"}'
-    - name: implementations-8c43378018a8d9ed1a17d6cd02a579db5fa506abf47189a0385306301b7fa2c2
-      value: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif
+    - name: components-0692d431e617fbe942c422f90e6bc3e379106797bac2421b5f1889953b36c4bd
+      value: '{"executorLabel":"exec-component-output-artifact","outputDefinitions":{"artifacts":{"out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}'
+    - name: implementations-0692d431e617fbe942c422f90e6bc3e379106797bac2421b5f1889953b36c4bd
+      value: '{"args":["--executor_input","{{$}}","--function_to_execute","component_output_artifact"],"command":["sh","-c","\nif
         ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3
         -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
-        python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.3'' ''--no-deps''
+        python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.6'' ''--no-deps''
         ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026
         \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\"
         \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3
         -m kfp.dsl.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n","\nimport
         kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef
-        comp():\n    pass\n\n"],"image":"python:3.11"}'
+        component_output_artifact(out_dataset: Output[Dataset]):\n    out_dataset.set_path(''/tmp/out_dataset'')\n\n    with
+        open(out_dataset.path, ''w'') as f:\n        f.write(''Hello, World!'')\n\n    if
+        out_dataset.path != ''/tmp/out_dataset'':\n        raise ValueError(f\"File
+        path is {out_dataset.path} but should be ''/tmp/out_dataset''.\")\n    if
+        out_dataset.custom_path != ''/tmp/out_dataset'':\n        raise ValueError(f\"File
+        uri is {out_dataset.custom_path} but should be ''/tmp/out_dataset''.\")\n\n"],"image":"python:3.11"}'
+    - name: components-0641839dce68d0225ee497406363513592d841c856891f7313f5368a99e8a50d
+      value: '{"executorLabel":"exec-validate-input-artifact","inputDefinitions":{"artifacts":{"in_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}},"outputDefinitions":{"parameters":{"Output":{"parameterType":"BOOLEAN"}}}}'
+    - name: implementations-0641839dce68d0225ee497406363513592d841c856891f7313f5368a99e8a50d
+      value: '{"args":["--executor_input","{{$}}","--function_to_execute","validate_input_artifact"],"command":["sh","-c","\nif
+        ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip || python3
+        -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
+        python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.6'' ''--no-deps''
+        ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026
+        \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\"
+        \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3
+        -m kfp.dsl.executor_main                         --component_module_path                         \"$program_path/ephemeral_component.py\"                         \"$@\"\n","\nimport
+        kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef
+        validate_input_artifact(in_dataset: Input[Dataset]) -\u003e bool:\n    if
+        in_dataset is None:\n        raise ValueError(\"Input artifact is None.\")\n    with
+        open(in_dataset.path, ''r'') as f:\n        for line in f:\n            if
+        line != ''Hello, World!'':\n                raise ValueError(f\"File content
+        is {line} but should be ''Hello, World!''.\")\n    return True\n\n"],"image":"python:3.11"}'
     - name: components-root
-      value: '{"dag":{"tasks":{"comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"taskInfo":{"name":"comp"}}}}}'
+      value: '{"dag":{"tasks":{"component-output-artifact":{"cachingOptions":{},"componentRef":{"name":"comp-component-output-artifact"},"taskInfo":{"name":"component-output-artifact"}},"validate-input-artifact":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-validate-input-artifact"},"dependentTasks":["component-output-artifact"],"inputs":{"artifacts":{"in_dataset":{"taskOutputArtifact":{"outputArtifactKey":"out_dataset","producerTask":"component-output-artifact"}}}},"taskInfo":{"name":"validate-input-artifact"}}}}}'
   entrypoint: entrypoint
   podMetadata:
     annotations:
@@ -34,7 +56,7 @@ spec:
       - --type
       - CONTAINER
       - --pipeline_name
-      - pipeline-with-semaphore
+      - pipeline-with-custom-path-artifact
       - --run_id
       - '{{workflow.uid}}'
       - --run_name
@@ -240,26 +262,51 @@ spec:
       - arguments:
           parameters:
           - name: component
-            value: '{{workflow.parameters.components-8c43378018a8d9ed1a17d6cd02a579db5fa506abf47189a0385306301b7fa2c2}}'
+            value: '{{workflow.parameters.components-0692d431e617fbe942c422f90e6bc3e379106797bac2421b5f1889953b36c4bd}}'
+          - name: task
+            value: '{"cachingOptions":{},"componentRef":{"name":"comp-component-output-artifact"},"taskInfo":{"name":"component-output-artifact"}}'
+          - name: container
+            value: '{{workflow.parameters.implementations-0692d431e617fbe942c422f90e6bc3e379106797bac2421b5f1889953b36c4bd}}'
+          - name: task-name
+            value: component-output-artifact
+          - name: parent-dag-task-id
+            value: '{{inputs.parameters.parent-dag-task-id}}'
+        name: component-output-artifact-driver
+        template: system-container-driver
+      - arguments:
+          parameters:
+          - name: pod-spec-patch
+            value: '{{tasks.component-output-artifact-driver.outputs.parameters.pod-spec-patch}}'
+          - default: "false"
+            name: cached-decision
+            value: '{{tasks.component-output-artifact-driver.outputs.parameters.cached-decision}}'
+        depends: component-output-artifact-driver.Succeeded
+        name: component-output-artifact
+        template: system-container-executor
+      - arguments:
+          parameters:
+          - name: component
+            value: '{{workflow.parameters.components-0641839dce68d0225ee497406363513592d841c856891f7313f5368a99e8a50d}}'
           - name: task
-            value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"taskInfo":{"name":"comp"}}'
+            value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-validate-input-artifact"},"dependentTasks":["component-output-artifact"],"inputs":{"artifacts":{"in_dataset":{"taskOutputArtifact":{"outputArtifactKey":"out_dataset","producerTask":"component-output-artifact"}}}},"taskInfo":{"name":"validate-input-artifact"}}'
           - name: container
-            value: '{{workflow.parameters.implementations-8c43378018a8d9ed1a17d6cd02a579db5fa506abf47189a0385306301b7fa2c2}}'
+            value: '{{workflow.parameters.implementations-0641839dce68d0225ee497406363513592d841c856891f7313f5368a99e8a50d}}'
           - name: task-name
-            value: comp
+            value: validate-input-artifact
           - name: parent-dag-task-id
             value: '{{inputs.parameters.parent-dag-task-id}}'
-        name: comp-driver
+        depends: component-output-artifact.Succeeded
+        name: validate-input-artifact-driver
         template: system-container-driver
       - arguments:
           parameters:
           - name: pod-spec-patch
-            value: '{{tasks.comp-driver.outputs.parameters.pod-spec-patch}}'
+            value: '{{tasks.validate-input-artifact-driver.outputs.parameters.pod-spec-patch}}'
           - default: "false"
             name: cached-decision
-            value: '{{tasks.comp-driver.outputs.parameters.cached-decision}}'
-        depends: comp-driver.Succeeded
-        name: comp
+            value: '{{tasks.validate-input-artifact-driver.outputs.parameters.cached-decision}}'
+        depends: validate-input-artifact-driver.Succeeded
+        name: validate-input-artifact
         template: system-container-executor
     inputs:
       parameters:
@@ -272,7 +319,7 @@ spec:
       - --type
       - '{{inputs.parameters.driver-type}}'
       - --pipeline_name
-      - pipeline-with-semaphore
+      - pipeline-with-custom-path-artifact
       - --run_id
       - '{{workflow.uid}}'
       - --run_name

test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml

@@ -72,7 +72,7 @@ spec:
         kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef
         sum_numbers(a: int, b: int) -\u003e int:\n    return a + b\n\n"],"image":"python:3.11","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}'
     - name: components-root
-      value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","memory-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}'
+      value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}'
   entrypoint: entrypoint
   podMetadata:
     annotations:
@@ -390,15 +390,15 @@ spec:
           - name: component
             value: '{{workflow.parameters.components-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}'
           - name: task
-            value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","memory-limit","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}'
+            value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","accelerator-type","cpu-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}'
           - name: container
             value: '{{workflow.parameters.implementations-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}'
           - name: task-name
             value: sum-numbers
           - name: parent-dag-task-id
             value: '{{inputs.parameters.parent-dag-task-id}}'
-        depends: accelerator-limit.Succeeded && accelerator-type.Succeeded && memory-limit.Succeeded
-          && cpu-limit.Succeeded
+        depends: accelerator-limit.Succeeded && accelerator-type.Succeeded && cpu-limit.Succeeded
+          && memory-limit.Succeeded
         name: sum-numbers-driver
         template: system-container-driver
       - arguments: