feat(backend): Replace MLMD with KFP Server APIs

.github/actions/deploy/action.yml

@@ -64,7 +64,7 @@ runs:
     - name: Load Docker Images
       shell: bash
       run: |
-        APPS=("apiserver" "driver" "launcher" "scheduledworkflow" "persistenceagent" "frontend" "metadata-writer")
+        APPS=("apiserver" "driver" "launcher" "scheduledworkflow" "persistenceagent" "frontend")
         for app in "${APPS[@]}"; do
           docker image load -i ${{ inputs.image_path }}/$app/$app.tar
           docker push ${{ inputs.image_registry }}/$app:${{ inputs.image_tag }}

.github/actions/kfp-k8s/action.yml

@@ -1,5 +1,5 @@
-name: "Install kfp & kfp-kubernetes"
-description: "Install kfp & kfp-kubernetes"
+name: "Install kfp-server-api, kfp & kfp-kubernetes"
+description: "Install kfp-server-api, kfp & kfp-kubernetes from source"
 inputs:
   build_version:
     required: true
@@ -16,6 +16,13 @@ runs:
       shell: bash
       run: pip install build==${{inputs.build_version}}
 
+    - name: Build kfp-server-api dist
+      id: build-kfp-server-api
+      shell: bash
+      working-directory: backend/api/v2beta1/python_http_client
+      run: |
+        python -m build .
+
     - name: Build kfp dist
       id: install-kfp
       shell: bash
@@ -37,15 +44,15 @@ runs:
       working-directory: ./kubernetes_platform
       run: make golang
 
-    # kfp is installed transitively
-    # --find-links ensures pip first looks in the sdk/python/dist folder
-    # outputted from generate-kfp-kubernetes-proto-files step before looking at pypi
+    # kfp and kfp-server-api are installed transitively
+    # --find-links ensures pip first looks in the dist folders before looking at pypi
+    # for kfp-server-api and kfp packages
     - name: Install kfp & kfp-kubernetes from source
       id: install-kfp-kubernetes
       shell: bash
       if: ${{ steps.generate-kfp-kubernetes-proto-files.outcome == 'success' }}
       run: |
-        pip install -e ./kubernetes_platform/python[dev] --find-links=sdk/python/dist
+        pip install -e ./kubernetes_platform/python[dev] --find-links=backend/api/v2beta1/python_http_client/dist --find-links=sdk/python/dist
 
     # testing reinstalling kfp package from source with no deps
     - name: Reinstall kfp from source with no deps

.github/resources/manifests/kubernetes-native/default/kustomization.yaml

@@ -18,19 +18,12 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -47,11 +40,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml

@@ -18,19 +18,12 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -63,11 +56,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/multiuser/cache-disabled/kustomization.yaml

@@ -18,10 +18,6 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
-
 patches:
   - path: ../../base/apiserver-env.yaml
     target:
@@ -31,10 +27,6 @@ patches:
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -51,11 +43,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/multiuser/default/kustomization.yaml

@@ -18,19 +18,12 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -47,11 +40,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/multiuser/minio/kustomization.yaml

@@ -18,19 +18,12 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -47,11 +40,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/standalone/cache-disabled-proxy-minio/apiserver-env.yaml

@@ -15,4 +15,4 @@ spec:
             - name: HTTPS_PROXY
               value: "http://squid.squid.svc.cluster.local:3128"
             - name: NO_PROXY
-              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,minio-service.kubeflow,metadata-grpc-service,metadata-grpc-service.kubeflow,ml-pipeline.kubeflow"
+              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,minio-service.kubeflow,ml-pipeline.kubeflow"

.github/resources/manifests/standalone/cache-disabled-proxy/apiserver-env.yaml

@@ -15,6 +15,6 @@ spec:
             - name: HTTPS_PROXY
               value: "http://squid.squid.svc.cluster.local:3128"
             - name: NO_PROXY
-              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,minio-service.kubeflow,metadata-grpc-service,metadata-grpc-service.kubeflow,ml-pipeline.kubeflow"
+              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,minio-service.kubeflow,ml-pipeline.kubeflow"
             - name: OBJECTSTORECONFIG_HOST
               value: "minio-service.kubeflow.svc.cluster.local"

.github/resources/manifests/standalone/cache-disabled/kustomization.yaml

@@ -18,9 +18,6 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml
@@ -31,10 +28,6 @@ patches:
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -51,11 +44,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/standalone/default/kustomization.yaml

@@ -18,19 +18,12 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -47,11 +40,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/standalone/minio/kustomization.yaml

@@ -18,9 +18,6 @@ images:
   - name: ghcr.io/kubeflow/kfp-frontend
     newName: kind-registry:5000/frontend
     newTag: latest
-  - name: ghcr.io/kubeflow/kfp-metadata-writer
-    newName: kind-registry:5000/metadata-writer
-    newTag: latest
 
 patches:
   - path: ../../base/apiserver-env.yaml
@@ -31,10 +28,6 @@ patches:
     target:
       kind: Deployment
       name: ml-pipeline
-  - path: ../../base/grpc-specs.yaml
-    target:
-      kind: Deployment
-      name: metadata-grpc-deployment
   - path: ../../base/cache-specs.yaml
     target:
       kind: Deployment
@@ -51,11 +44,6 @@ replacements:
           name: ml-pipeline
         fieldPaths:
           - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
-      - select:
-          kind: Deployment
-          name: metadata-grpc-deployment
-        fieldPaths:
-          - spec.template.spec.dnsConfig.searches.[=NAMESPACE.svc.cluster.local]
       - select:
           kind: Deployment
           name: cache-server

.github/resources/manifests/standalone/proxy-minio/apiserver-env.yaml

@@ -13,6 +13,6 @@ spec:
             - name: HTTPS_PROXY
               value: "http://squid.squid.svc.cluster.local:3128"
             - name: NO_PROXY
-              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,minio-service.kubeflow,metadata-grpc-service,metadata-grpc-service.kubeflow,ml-pipeline.kubeflow"
+              value: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc,minio-service.kubeflow,ml-pipeline.kubeflow"
             - name: OBJECTSTORECONFIG_HOST
               value: "minio-service.kubeflow.svc.cluster.local"