feat(ci): add support for spark and hive engine integration tests (#5287)

Author: kazutoiris

.github/workflows/integration-test.yml

@@ -152,6 +152,8 @@ jobs:
             sleep 20
           done
 
+          bash ./linkis-dist/helm/scripts/prepare-for-spark.sh
+
           #show linkis pod logs
           #POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-cg-linkismanager -o jsonpath='{.items[0].metadata.name}'`
           #kubectl logs -n linkis  ${POD_NAME} -f --tail=10000
@@ -170,16 +172,15 @@ jobs:
 
           # Execute test by linkis-cli
           POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-mg-gateway -o jsonpath='{.items[0].metadata.name}'`
-          kubectl exec -it -n linkis  ${POD_NAME} -- bash -c " \
+          kubectl exec -n linkis  ${POD_NAME} -- bash -c " \
           sh /opt/linkis/bin/linkis-cli -engineType shell-1 -codeType shell -code \"pwd\" ";
 
-          kubectl exec -it -n linkis  ${POD_NAME} -- bash -c " \
+          kubectl exec -n linkis  ${POD_NAME} -- bash -c " \
           sh /opt/linkis/bin/linkis-cli -engineType python-python2 -codeType python -code   'print(\"hello\")' "
 
-          #todo
-          #kubectl exec -it -n linkis  ${POD_NAME} -- bash -c " \
-          #sh /opt/linkis/bin/linkis-cli -engineType hive-3.1.3 -codeType hql -code   'show databases' "
+          kubectl exec -n linkis  ${POD_NAME} -- bash -c " \
+          sh /opt/linkis/bin/linkis-cli -engineType hive-3.1.3 -codeType hql -code   'show databases' "
 
-          #kubectl exec -it -n linkis  ${POD_NAME} -- bash -c " \
-          #sh /opt/linkis/bin/linkis-cli -engineType spark-3.2.1 -codeType sql -code   'show databases' "
+          kubectl exec -n linkis  ${POD_NAME} -- bash -c " \
+          sh /opt/linkis/bin/linkis-cli -engineType spark-3.2.1 -codeType sql -code   'show databases' "
         shell: bash

linkis-dist/bin/install-linkis-to-kubernetes.sh

@@ -86,6 +86,8 @@ tag(){
 make_linkis_image_with_mysql_jdbc(){
     ${ROOT_DIR}/docker/scripts/make-linkis-image-with-mysql-jdbc.sh
     docker tag linkis:with-jdbc linkis:dev
+    ${ROOT_DIR}/docker/scripts/make-ldh-image-with-mysql-jdbc.sh
+    docker tag linkis-ldh:with-jdbc linkis-ldh:dev
 }
 #creating a kind cluster
 create_kind_cluster(){

linkis-dist/docker/ldh-with-mysql-jdbc.Dockerfile

@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ARG LINKIS_IMAGE=linkis-ldh:dev
+
+######################################################################
+# linkis-ldh image with mysql jdbc
+######################################################################
+FROM ${LINKIS_IMAGE}
+
+ARG LDH_HOME=/opt/ldh/current
+ARG MYSQL_JDBC_VERSION=8.0.28
+
+COPY mysql-connector-java-${MYSQL_JDBC_VERSION}.jar ${LDH_HOME}/hive/lib/
+COPY mysql-connector-java-${MYSQL_JDBC_VERSION}.jar ${LDH_HOME}/spark/lib/

linkis-dist/docker/ldh.Dockerfile

@@ -75,6 +75,10 @@ ADD ldh-tars/spark-${SPARK_VERSION}-bin-hadoop${SPARK_HADOOP_VERSION}.tgz /opt/l
 ADD ldh-tars/flink-${FLINK_VERSION}-bin-scala_2.11.tgz /opt/ldh/${LINKIS_VERSION}/
 ADD ldh-tars/apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz /opt/ldh/${LINKIS_VERSION}/
 
+RUN ln -s /opt/ldh/${LINKIS_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${SPARK_HADOOP_VERSION} /opt/ldh/current/spark \
+    && ln -s /opt/ldh/${LINKIS_VERSION}/hadoop-${HADOOP_VERSION} /opt/ldh/current/hadoop \
+    && ln -s /opt/ldh/${LINKIS_VERSION}/apache-hive-${HIVE_VERSION}-bin /opt/ldh/current/hive
+
 RUN mkdir -p /etc/ldh \
     && mkdir -p /var/log/hadoop && chmod 777 -R /var/log/hadoop \
     && mkdir -p /var/log/hive && chmod 777 -R /var/log/hive \
@@ -91,9 +95,10 @@ RUN mkdir -p /etc/ldh \
 #ADD ldh-tars/mysql-connector-java-${MYSQL_JDBC_VERSION}.jar /opt/ldh/current/hive/lib/
 #ADD ldh-tars/mysql-connector-java-${MYSQL_JDBC_VERSION}.jar /opt/ldh/current/spark/jars/
 
-ENV JAVA_HOME /etc/alternatives/jre
-ENV PATH /opt/ldh/current/hadoop/bin:/opt/ldh/current/hive/bin:/opt/ldh/current/spark/bin:/opt/ldh/current/flink/bin:/opt/ldh/current/zookeeper/bin:$PATH
+ENV JAVA_HOME=/etc/alternatives/jre
+ENV PATH=/opt/ldh/current/hadoop/bin:/opt/ldh/current/hive/bin:/opt/ldh/current/spark/bin:/opt/ldh/current/flink/bin:/opt/ldh/current/zookeeper/bin:$PATH
 ENV HADOOP_CONF_DIR=/etc/ldh/hadoop
+ENV YARN_CONF_DIR=/etc/ldh/hadoop
 ENV HIVE_CONF_DIR=/etc/ldh/hive
 ENV SPARK_CONF_DIR=/etc/ldh/spark
 ENV FLINK_CONF_DIR=/etc/ldh/flink

linkis-dist/docker/scripts/make-ldh-image-with-mysql-jdbc.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+WORK_DIR=`cd $(dirname $0); pwd -P`
+
+. ${WORK_DIR}/utils.sh
+
+IMAGE_NAME=${IMAGE_NAME:-linkis-ldh:with-jdbc}
+LINKIS_IMAGE=${LINKIS_IMAGE:-linkis-ldh:dev}
+LINKIS_HOME=${LINKIS_HOME:-/opt/ldh/current}
+MYSQL_JDBC_VERSION=${MYSQL_JDBC_VERSION:-8.0.28}
+MYSQL_JDBC_FILENAME=mysql-connector-java-${MYSQL_JDBC_VERSION}.jar
+MYSQL_JDBC_URL="https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_JDBC_VERSION}/${MYSQL_JDBC_FILENAME}"
+
+BUILD_DIR=`mktemp -d -t linkis-build-XXXXX`
+
+echo "#          build dir: ${BUILD_DIR}"
+echo "#         base image: ${LINKIS_IMAGE}"
+echo "# mysql jdbc version: ${MYSQL_JDBC_VERSION}"
+
+download ${MYSQL_JDBC_URL} ${MYSQL_JDBC_FILENAME} ${BUILD_DIR}
+
+echo "try to exec: docker build -f ${WORK_DIR}/../ldh-with-mysql-jdbc.Dockerfile \
+  -t ${IMAGE_NAME} \
+  --build-arg LINKIS_IMAGE=${LINKIS_IMAGE} \
+  --build-arg LINKIS_HOME=${LINKIS_HOME} \
+  --build-arg MYSQL_JDBC_VERSION=${MYSQL_JDBC_VERSION} \
+  ${BUILD_DIR}"
+
+docker build -f ${WORK_DIR}/../ldh-with-mysql-jdbc.Dockerfile \
+  -t ${IMAGE_NAME} \
+  --build-arg LINKIS_IMAGE=${LINKIS_IMAGE} \
+  --build-arg LINKIS_HOME=${LINKIS_HOME} \
+  --build-arg MYSQL_JDBC_VERSION=${MYSQL_JDBC_VERSION} \
+  ${BUILD_DIR}
+
+echo "# done, image: ${IMAGE_NAME}"

linkis-dist/helm/charts/linkis/templates/jobs.yaml

@@ -28,7 +28,16 @@ spec:
           command:
             - /bin/bash
             - -ecx
-            - >-
+            - |-
+              sed -i 's#@YARN_RESTFUL_URL#{{ .Values.linkis.deps.yarn.restfulUrl }}#g' {{ .Values.linkis.locations.homeDir }}/db/linkis_dml.sql
+              sed -i 's#@HADOOP_VERSION#{{ .Values.linkis.deps.hadoop.version }}#g' {{ .Values.linkis.locations.homeDir }}/db/linkis_dml.sql
+              sed -i 's#@YARN_AUTH_ENABLE#{{ .Values.linkis.deps.yarn.authEnable }}#g' {{ .Values.linkis.locations.homeDir }}/db/linkis_dml.sql
+              sed -i 's#@YARN_AUTH_USER#{{ .Values.linkis.deps.yarn.authUser }}#g' {{ .Values.linkis.locations.homeDir }}/db/linkis_dml.sql
+              sed -i 's#@YARN_AUTH_PWD#{{ .Values.linkis.deps.yarn.authPassword }}#g' {{ .Values.linkis.locations.homeDir }}/db/linkis_dml.sql
+              sed -i 's#@YARN_KERBEROS_ENABLE#{{ .Values.linkis.deps.yarn.kerberosEnable }}#g' {{ .Values.linkis.locations.homeDir }}/db/linkis_dml.sql
+              sed -i 's#@YARN_PRINCIPAL_NAME#{{ .Values.linkis.deps.yarn.principal }}#g' {{ .Values.linkis.locations.homeDir }}/db/linkis_dml.sql
+              sed -i 's#@YARN_KEYTAB_PATH#{{ .Values.linkis.deps.yarn.keytab }}#g' {{ .Values.linkis.locations.homeDir }}/db/linkis_dml.sql
+              sed -i 's#@YARN_KRB5_PATH#{{ .Values.linkis.deps.yarn.krb5 }}#g' {{ .Values.linkis.locations.homeDir }}/db/linkis_dml.sql
               mysql -h{{ .Values.linkis.datasource.host }} -P{{ .Values.linkis.datasource.port }} -u{{ .Values.linkis.datasource.username }} -p{{ .Values.linkis.datasource.password }} --default-character-set=utf8 -e "CREATE DATABASE IF NOT EXISTS {{ .Values.linkis.datasource.database }} DEFAULT CHARSET utf8 COLLATE utf8_general_ci";
               mysql -h{{ .Values.linkis.datasource.host }} -P{{ .Values.linkis.datasource.port }} -u{{ .Values.linkis.datasource.username }} -p{{ .Values.linkis.datasource.password }} -D{{ .Values.linkis.datasource.database }}  --default-character-set=utf8 -e "source {{ .Values.linkis.locations.homeDir }}/db//linkis_ddl.sql";
               mysql -h{{ .Values.linkis.datasource.host }} -P{{ .Values.linkis.datasource.port }} -u{{ .Values.linkis.datasource.username }} -p{{ .Values.linkis.datasource.password }} -D{{ .Values.linkis.datasource.database }}  --default-character-set=utf8 -e "source {{ .Values.linkis.locations.homeDir }}/db//linkis_dml.sql"

linkis-dist/helm/scripts/install-ldh.sh

@@ -16,7 +16,7 @@
 #
 
 WORK_DIR=`cd $(dirname $0); pwd -P`
-
+ROOT_DIR=${WORK_DIR}/../..
 . ${WORK_DIR}/common.sh
 
 set -e
@@ -27,6 +27,9 @@ echo "# LDH version: ${LINKIS_IMAGE_TAG}"
 
 # load image
 if [[ "X${USING_KIND}" == "Xtrue" ]]; then
+  echo "# Preparing LDH image ..."
+  ${ROOT_DIR}/docker/scripts/make-ldh-image-with-mysql-jdbc.sh
+  docker tag linkis-ldh:with-jdbc linkis-ldh:dev
   echo "# Loading LDH image ..."
   kind load docker-image linkis-ldh:${LINKIS_IMAGE_TAG} --name ${KIND_CLUSTER_NAME}
 fi

linkis-dist/helm/scripts/prepare-for-spark.sh

@@ -21,22 +21,16 @@ WORK_DIR=`cd $(dirname $0); pwd -P`
 
 ## copy spark resource from ldh to linkis-cg-engineconnmanager
 
-LDH_POD_NAME=`kubectl get pods -n ldh -l app=ldh    -o jsonpath='{.items[0].metadata.name}'`
-kubectl cp -n ldh ${LDH_POD_NAME}:/opt/ldh/ ./ldh
-
+LDH_POD_NAME=`kubectl get pods -n ldh -l app=ldh -o jsonpath='{.items[0].metadata.name}'`
 ECM_POD_NAME=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-cg-engineconnmanager -o jsonpath='{.items[0].metadata.name}'`
-kubectl cp ./ldh  -n linkis ${ECM_POD_NAME}:/opt/ ;
-
-
-kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "chmod +x /opt/ldh/1.3.0/spark-3.2.1-bin-hadoop3.2/bin/*"
-kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s /opt/ldh/1.3.0/spark-3.2.1-bin-hadoop3.2 /opt/ldh/current/spark"
-kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s /opt/ldh/1.3.0/hadoop-3.3.4 /opt/ldh/current/hadoop"
-kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "ln -s /opt/ldh/1.3.0/apache-hive-3.1.3-bin /opt/ldh/current/hive"
 
+kubectl exec -n ldh ${LDH_POD_NAME} -- tar -C /opt -cf - ldh | \
+kubectl exec -i -n linkis ${ECM_POD_NAME} -- tar -C /opt -xf - --no-same-owner
 
-kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "echo 'export SPARK_HOME=/opt/ldh/current/spark' |sudo tee --append /etc/profile"
-kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "echo 'export PATH=\$SPARK_HOME/bin:\$PATH' |sudo tee --append  /etc/profile"
-kubectl exec -it -n linkis ${ECM_POD_NAME} -- bash -c "source /etc/profile"
+kubectl exec -n linkis ${ECM_POD_NAME} -- bash -c "sudo mkdir -p /appcom/Install && sudo chmod 0777 /appcom/Install && ln -s /opt/ldh/current/spark /appcom/Install/spark"
+kubectl exec -n linkis ${ECM_POD_NAME} -- bash -c "echo 'export SPARK_HOME=/opt/ldh/current/spark' |sudo tee --append /etc/profile"
+kubectl exec -n linkis ${ECM_POD_NAME} -- bash -c "echo 'export PATH=\$SPARK_HOME/bin:\$PATH' |sudo tee --append  /etc/profile"
+kubectl exec -n linkis ${ECM_POD_NAME} -- bash -c "source /etc/profile"
 
 # add ecm dns for ldh pod
 ECM_POD_IP=`kubectl get pods -n linkis -l app.kubernetes.io/instance=linkis-demo-cg-engineconnmanager -o jsonpath='{.items[0].status.podIP}'`
@@ -45,7 +39,4 @@ ECM_POD_SUBDOMAIN=`kubectl get pods -n linkis -l app.kubernetes.io/instance=link
 
 ECM_DNS="${ECM_POD_IP}   ${ECM_POD_NAME}.${ECM_POD_SUBDOMAIN}.linkis.svc.cluster.local"
 
-kubectl exec -it -n ldh ${LDH_POD_NAME} -- bash -c "echo ${ECM_DNS} |sudo tee --append  /etc/hosts"
-
-
-rm -rf ldh;
+kubectl exec -n ldh ${LDH_POD_NAME} -- bash -c "echo ${ECM_DNS} |sudo tee --append  /etc/hosts"

linkis-dist/helm/scripts/resources/ldh/configmaps/configmap-hadoop.yaml

@@ -591,4 +591,127 @@ data:
           <aclAdministerApps>*</aclAdministerApps>
         </queue>
       </queue>
-    </allocations>
+    </allocations>
+  capacity-scheduler.xml: |
+    <!--
+      Licensed under the Apache License, Version 2.0 (the "License");
+      you may not use this file except in compliance with the License.
+      You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+      See the License for the specific language governing permissions and
+      limitations under the License. See accompanying LICENSE file.
+    -->
+    <configuration>
+
+      <property>
+        <name>yarn.scheduler.capacity.maximum-applications</name>
+        <value>4</value>
+        <description>
+          Maximum number of applications that can be pending and running.
+        </description>
+      </property>
+
+      <property>
+        <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
+        <value>0.5</value>
+        <description>
+          Maximum percent of resources in the cluster which can be used to run
+          application masters i.e. controls number of concurrent running
+          applications.
+        </description>
+      </property>
+
+      <property>
+        <name>yarn.scheduler.capacity.resource-calculator</name>
+        <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
+        <description>
+          The ResourceCalculator implementation to be used to compare
+          Resources in the scheduler.
+          The default i.e. DefaultResourceCalculator only uses Memory while
+          DominantResourceCalculator uses dominant-resource to compare
+          multi-dimensional resources such as Memory, CPU etc.
+        </description>
+      </property>
+
+      <property>
+        <name>yarn.scheduler.capacity.root.queues</name>
+        <value>default</value>
+        <description>
+          The queues at the this level (root is the root queue).
+        </description>
+      </property>
+
+      <property>
+        <name>yarn.scheduler.capacity.root.default.capacity</name>
+        <value>100</value>
+        <description>Default queue target capacity.</description>
+      </property>
+
+      <property>
+        <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
+        <value>1</value>
+        <description>
+          Default queue user limit a percentage from 0.0 to 1.0.
+        </description>
+      </property>
+
+      <property>
+        <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
+        <value>100</value>
+        <description>
+          The maximum capacity of the default queue.
+        </description>
+      </property>
+
+      <property>
+        <name>yarn.scheduler.capacity.root.default.state</name>
+        <value>RUNNING</value>
+        <description>
+          The state of the default queue. State can be one of RUNNING or STOPPED.
+        </description>
+      </property>
+
+      <property>
+        <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
+        <value>*</value>
+        <description>
+          The ACL of who can submit jobs to the default queue.
+        </description>
+      </property>
+
+      <property>
+        <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
+        <value>*</value>
+        <description>
+          The ACL of who can administer jobs on the default queue.
+        </description>
+      </property>
+
+      <property>
+        <name>yarn.scheduler.capacity.node-locality-delay</name>
+        <value>40</value>
+        <description>
+          Number of missed scheduling opportunities after which the CapacityScheduler
+          attempts to schedule rack-local containers.
+          Typically this should be set to number of nodes in the cluster, By default is setting
+          approximately number of nodes in one rack which is 40.
+        </description>
+      </property>
+
+      <property>
+        <name>yarn.scheduler.capacity.queue-mappings-override.enable</name>
+        <value>false</value>
+        <description>
+          If a queue mapping is present, will it override the value specified
+          by the user? This can be used by administrators to place jobs in queues
+          that are different than the one specified by the user.
+          The default is false.
+        </description>
+      </property>
+
+    </configuration>