Skip to content

Commit ff742c4

Browse files
github-actions[bot]CalvinKirs
github-actions[bot]
and
CalvinKirs
authored
branch-4.0: [feat](web)Adapted for Spring Boot 3 and Jetty 12 #57710 (#57782)
Cherry-picked from #57710 #57979 #57810 #57784 #57202 #57629 --------- Co-authored-by: Calvin Kirs <[email protected]>
1 parent 6dce942 commit ff742c4

File tree

209 files changed

+663
-1470
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

209 files changed

+663
-1470
lines changed

‎bin/start_be.sh‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -238,16 +238,10 @@ done
238238

239239
if [[ -d "${DORIS_HOME}/lib/hadoop_hdfs/" ]]; then
240240
# add hadoop libs
241-
for f in "${DORIS_HOME}/lib/hadoop_hdfs/common"/*.jar; do
241+
for f in "${DORIS_HOME}/lib/hadoop_hdfs"/*.jar; do
242242
DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}"
243243
done
244-
for f in "${DORIS_HOME}/lib/hadoop_hdfs/common/lib"/*.jar; do
245-
DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}"
246-
done
247-
for f in "${DORIS_HOME}/lib/hadoop_hdfs/hdfs"/*.jar; do
248-
DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}"
249-
done
250-
for f in "${DORIS_HOME}/lib/hadoop_hdfs/hdfs/lib"/*.jar; do
244+
for f in "${DORIS_HOME}/lib/hadoop_hdfs/lib"/*.jar; do
251245
DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}"
252246
done
253247
fi

‎build.sh‎

Lines changed: 38 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ if [[ -z "${DORIS_THIRDPARTY}" ]]; then
3535
fi
3636
export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include"
3737
export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib"
38-
38+
HADOOP_DEPS_NAME="hadoop-deps"
3939
. "${DORIS_HOME}/env.sh"
4040

4141
# Check args
@@ -204,6 +204,7 @@ else
204204
;;
205205
--cloud)
206206
BUILD_CLOUD=1
207+
BUILD_BE_JAVA_EXTENSIONS=1
207208
shift
208209
;;
209210
--broker)
@@ -553,6 +554,7 @@ if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then
553554
# lakesoul-scanner has been deprecated
554555
# modules+=("be-java-extensions/lakesoul-scanner")
555556
modules+=("be-java-extensions/preload-extensions")
557+
modules+=("be-java-extensions/${HADOOP_DEPS_NAME}")
556558

557559
# If the BE_EXTENSION_IGNORE variable is not empty, remove the modules that need to be ignored from FE_MODULES
558560
if [[ -n "${BE_EXTENSION_IGNORE}" ]]; then
@@ -811,10 +813,6 @@ if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then
811813
cp -r -p "${DORIS_HOME}/be/output/conf"/* "${DORIS_OUTPUT}/be/conf"/
812814
cp -r -p "${DORIS_HOME}/be/output/dict" "${DORIS_OUTPUT}/be/"
813815

814-
if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then
815-
cp -r -p "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${DORIS_OUTPUT}/be/lib/"
816-
fi
817-
818816
if [[ -f "${DORIS_THIRDPARTY}/installed/lib/libz.so" ]]; then
819817
cp -r -p "${DORIS_THIRDPARTY}/installed/lib/libz.so"* "${DORIS_OUTPUT}/be/lib/"
820818
fi
@@ -884,6 +882,7 @@ EOF
884882
# extensions_modules+=("lakesoul-scanner")
885883
extensions_modules+=("preload-extensions")
886884
extensions_modules+=("iceberg-metadata-scanner")
885+
extensions_modules+=("${HADOOP_DEPS_NAME}")
887886

888887
if [[ -n "${BE_EXTENSION_IGNORE}" ]]; then
889888
IFS=',' read -r -a ignore_modules <<<"${BE_EXTENSION_IGNORE}"
@@ -913,13 +912,34 @@ EOF
913912
module_jar="${DORIS_HOME}/fe/be-java-extensions/${extensions_module}/target/${extensions_module}-jar-with-dependencies.jar"
914913
module_proj_jar="${DORIS_HOME}/fe/be-java-extensions/${extensions_module}/target/${extensions_module}-project.jar"
915914
mkdir "${BE_JAVA_EXTENSIONS_DIR}"/"${extensions_module}"
916-
if [[ -f "${module_jar}" ]]; then
917-
cp "${module_jar}" "${BE_JAVA_EXTENSIONS_DIR}"/"${extensions_module}"
918-
fi
919-
if [[ -f "${module_proj_jar}" ]]; then
920-
cp "${module_proj_jar}" "${BE_JAVA_EXTENSIONS_DIR}"/"${extensions_module}"
915+
echo "Copy Be Extensions ${extensions_module} jar to ${BE_JAVA_EXTENSIONS_DIR}/${extensions_module}"
916+
if [[ "${extensions_module}" == "${HADOOP_DEPS_NAME}" ]]; then
917+
918+
BE_HADOOP_HDFS_DIR="${DORIS_OUTPUT}/be/lib/hadoop_hdfs/"
919+
echo "Copy Be Extensions hadoop deps jars to ${BE_HADOOP_HDFS_DIR}"
920+
rm -rf "${BE_HADOOP_HDFS_DIR}"
921+
mkdir "${BE_HADOOP_HDFS_DIR}"
922+
HADOOP_DEPS_JAR_DIR="${DORIS_HOME}/fe/be-java-extensions/${HADOOP_DEPS_NAME}/target"
923+
echo "HADOOP_DEPS_JAR_DIR: ${HADOOP_DEPS_JAR_DIR}"
924+
if [[ -f "${HADOOP_DEPS_JAR_DIR}/${HADOOP_DEPS_NAME}.jar" ]]; then
925+
echo "Copy Be Extensions hadoop deps jar to ${BE_HADOOP_HDFS_DIR}"
926+
cp "${HADOOP_DEPS_JAR_DIR}/${HADOOP_DEPS_NAME}.jar" "${BE_HADOOP_HDFS_DIR}"
927+
fi
928+
if [[ -d "${HADOOP_DEPS_JAR_DIR}/lib" ]]; then
929+
cp -r "${HADOOP_DEPS_JAR_DIR}/lib" "${BE_HADOOP_HDFS_DIR}/"
930+
fi
931+
else
932+
if [[ -f "${module_jar}" ]]; then
933+
cp "${module_jar}" "${BE_JAVA_EXTENSIONS_DIR}"/"${extensions_module}"
934+
fi
935+
if [[ -f "${module_proj_jar}" ]]; then
936+
cp "${module_proj_jar}" "${BE_JAVA_EXTENSIONS_DIR}"/"${extensions_module}"
937+
fi
938+
if [[ -d "${DORIS_HOME}/fe/be-java-extensions/${extensions_module}/target/lib" ]]; then
939+
cp -r "${DORIS_HOME}/fe/be-java-extensions/${extensions_module}/target/lib" "${BE_JAVA_EXTENSIONS_DIR}/${extensions_module}/"
940+
fi
921941
fi
922-
done
942+
done
923943

924944
# copy jindofs jars, only support for Linux x64 or arm
925945
install -d "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
@@ -958,8 +978,13 @@ fi
958978
if [[ ${BUILD_CLOUD} -eq 1 ]]; then
959979
rm -rf "${DORIS_HOME}/output/ms"
960980
rm -rf "${DORIS_HOME}/cloud/output/lib/hadoop_hdfs"
961-
if [[ -d "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" ]]; then
962-
cp -r -p "${DORIS_THIRDPARTY}/installed/lib/hadoop_hdfs/" "${DORIS_HOME}/cloud/output/lib"
981+
# If hadoop dependencies are required, building cloud module must be done after building be-java-extensions first
982+
# so when running ./build.sh --cloud,we also build be-java-extensions automatically.
983+
# If hadoop-depencies are not needed, you can disable it explicitly, by setting DISABLE_BE_JAVA_EXTENSIONS during the build.
984+
HADOOP_DEPS_JAR_DIR="${DORIS_HOME}/fe/be-java-extensions/${HADOOP_DEPS_NAME}/target"
985+
if [[ -d "${HADOOP_DEPS_JAR_DIR}/lib" ]]; then
986+
mkdir -p "${DORIS_HOME}/cloud/output/lib/hadoop_hdfs"
987+
cp -r "${HADOOP_DEPS_JAR_DIR}/lib/"* "${DORIS_HOME}/cloud/output/lib/hadoop_hdfs/"
963988
fi
964989
cp -r -p "${DORIS_HOME}/cloud/output" "${DORIS_HOME}/output/ms"
965990
fi

‎cloud/script/start.sh‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -117,16 +117,7 @@ if [[ ${enable_hdfs} -eq 1 ]]; then
117117

118118
if [[ -d "${DORIS_HOME}/lib/hadoop_hdfs/" ]]; then
119119
# add hadoop libs
120-
for f in "${DORIS_HOME}/lib/hadoop_hdfs/common"/*.jar; do
121-
DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}"
122-
done
123-
for f in "${DORIS_HOME}/lib/hadoop_hdfs/common/lib"/*.jar; do
124-
DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}"
125-
done
126-
for f in "${DORIS_HOME}/lib/hadoop_hdfs/hdfs"/*.jar; do
127-
DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}"
128-
done
129-
for f in "${DORIS_HOME}/lib/hadoop_hdfs/hdfs/lib"/*.jar; do
120+
for f in "${DORIS_HOME}/lib/hadoop_hdfs"/*.jar; do
130121
DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}"
131122
done
132123
fi

‎dist/LICENSE-dist.txt‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -773,10 +773,6 @@ The Apache Software License, Version 2.0
773773
- commons-digester:commons-digester:1.8 (http://jakarta.apache.org/commons/digester/)
774774
* EL:
775775
- commons-el:commons-el:1.0 (http://jakarta.apache.org/commons/el/)
776-
* Fabric8 :: Kubernetes :: Java Client:
777-
- io.fabric8:kubernetes-client:5.12.2 (http://fabric8.io/kubernetes-client/)
778-
* Fabric8 :: Kubernetes Model:
779-
- io.fabric8:kubernetes-model:5.12.2 (http://fabric8.io/kubernetes-model/)
780776
* FindBugs-jsr305:
781777
- com.google.code.findbugs:jsr305:3.0.0 (http://findbugs.sourceforge.net/)
782778
- com.google.code.findbugs:jsr305:3.0.2 (http://findbugs.sourceforge.net/)

‎fe/be-java-extensions/avro-scanner/pom.xml‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,6 @@ under the License.
4141
<artifactId>avro-mapred</artifactId>
4242
<version>${avro.version}</version>
4343
</dependency>
44-
<dependency>
45-
<groupId>org.apache.doris</groupId>
46-
<artifactId>java-common</artifactId>
47-
<version>${project.version}</version>
48-
</dependency>
4944
<dependency>
5045
<groupId>org.apache.hadoop</groupId>
5146
<artifactId>hadoop-client</artifactId>
@@ -63,7 +58,8 @@ under the License.
6358
</dependency>
6459
<dependency>
6560
<groupId>org.apache.doris</groupId>
66-
<artifactId>hive-catalog-shade</artifactId>
61+
<artifactId>preload-extensions</artifactId>
62+
<version>${project.version}</version>
6763
<scope>provided</scope>
6864
</dependency>
6965
</dependencies>

‎fe/be-java-extensions/hadoop-deps/pom.xml‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing,
14+
software distributed under the License is distributed on an
15+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
KIND, either express or implied. See the License for the
17+
specific language governing permissions and limitations
18+
under the License.
19+
-->
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
<parent>
25+
<artifactId>be-java-extensions</artifactId>
26+
<groupId>org.apache.doris</groupId>
27+
<version>${revision}</version>
28+
</parent>
29+
30+
<artifactId>hadoop-deps</artifactId>
31+
32+
<dependencies>
33+
<dependency>
34+
<groupId>com.fasterxml.woodstox</groupId>
35+
<artifactId>woodstox-core</artifactId>
36+
</dependency>
37+
<dependency>
38+
<groupId>org.apache.commons</groupId>
39+
<artifactId>commons-collections4</artifactId>
40+
</dependency>
41+
<dependency>
42+
<groupId>org.apache.commons</groupId>
43+
<artifactId>commons-configuration2</artifactId>
44+
</dependency>
45+
46+
<dependency>
47+
<groupId>org.apache.hadoop</groupId>
48+
<artifactId>hadoop-hdfs</artifactId>
49+
</dependency>
50+
<dependency>
51+
<groupId>org.apache.hadoop</groupId>
52+
<artifactId>hadoop-common</artifactId>
53+
</dependency>
54+
<dependency>
55+
<groupId>org.apache.hadoop</groupId>
56+
<artifactId>hadoop-azure</artifactId>
57+
</dependency>
58+
<dependency>
59+
<groupId>org.apache.hadoop</groupId>
60+
<artifactId>hadoop-auth</artifactId>
61+
</dependency>
62+
<dependency>
63+
<groupId>org.apache.hadoop.thirdparty</groupId>
64+
<artifactId>hadoop-shaded-protobuf_3_25</artifactId>
65+
</dependency>
66+
<dependency>
67+
<groupId>org.apache.hadoop</groupId>
68+
<artifactId>hadoop-client</artifactId>
69+
<exclusions>
70+
<exclusion>
71+
<groupId>org.apache.hadoop</groupId>
72+
<artifactId>hadoop-yarn-api</artifactId>
73+
</exclusion>
74+
<exclusion>
75+
<groupId>org.apache.hadoop</groupId>
76+
<artifactId>hadoop-yarn-client</artifactId>
77+
</exclusion>
78+
<exclusion>
79+
<groupId>org.apache.hadoop</groupId>
80+
<artifactId>hadoop-mapreduce-client-core</artifactId>
81+
</exclusion>
82+
<exclusion>
83+
<groupId>org.apache.hadoop</groupId>
84+
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
85+
</exclusion>
86+
</exclusions>
87+
</dependency>
88+
</dependencies>
89+
90+
<build>
91+
<plugins>
92+
<plugin>
93+
<artifactId>maven-jar-plugin</artifactId>
94+
<configuration>
95+
<finalName>${project.artifactId}</finalName>
96+
</configuration>
97+
</plugin>
98+
<plugin>
99+
<artifactId>maven-dependency-plugin</artifactId>
100+
<version>3.6.0</version>
101+
<executions>
102+
<execution>
103+
<id>copy-dependencies</id>
104+
<phase>package</phase>
105+
<goals>
106+
<goal>copy-dependencies</goal>
107+
</goals>
108+
<configuration>
109+
<outputDirectory>${project.build.directory}/lib</outputDirectory>
110+
<includeScope>runtime</includeScope>
111+
</configuration>
112+
</execution>
113+
</executions>
114+
</plugin>
115+
</plugins>
116+
</build>
117+
118+
</project>

‎fe/be-java-extensions/hadoop-hudi-scanner/pom.xml‎

Lines changed: 6 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -39,42 +39,35 @@ under the License.
3939
<dependencies>
4040
<dependency>
4141
<groupId>org.apache.doris</groupId>
42-
<artifactId>java-common</artifactId>
42+
<artifactId>preload-extensions</artifactId>
4343
<version>${project.version}</version>
44-
<exclusions>
45-
<exclusion>
46-
<groupId>org.apache.thrift</groupId>
47-
<artifactId>libthrift</artifactId>
48-
</exclusion>
49-
</exclusions>
44+
<scope>provided</scope>
5045
</dependency>
51-
5246
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
5347
<dependency>
5448
<groupId>org.apache.hadoop</groupId>
5549
<artifactId>hadoop-hdfs-client</artifactId>
5650
<version>${hadoop.version}</version>
51+
<scope>provided</scope>
5752
</dependency>
5853

5954
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
6055
<dependency>
6156
<groupId>org.apache.hadoop</groupId>
6257
<artifactId>hadoop-common</artifactId>
58+
<scope>provided</scope>
6359
</dependency>
6460

6561
<dependency>
6662
<groupId>org.apache.hadoop</groupId>
6763
<artifactId>hadoop-aws</artifactId>
64+
<scope>provided</scope>
6865
</dependency>
6966

7067
<dependency>
7168
<groupId>org.apache.hadoop</groupId>
7269
<artifactId>hadoop-mapreduce-client-core</artifactId>
73-
</dependency>
74-
75-
<dependency>
76-
<groupId>org.junit.jupiter</groupId>
77-
<artifactId>junit-jupiter</artifactId>
70+
<scope>provided</scope>
7871
</dependency>
7972

8073
<!-- https://mvnrepository.com/artifact/org.apache.hudi/hudi-common -->
@@ -123,26 +116,6 @@ under the License.
123116
<version>${parquet.version}</version>
124117
</dependency>
125118

126-
<!-- https://mvnrepository.com/artifact/org.apache.parquet/parquet-avro -->
127-
<dependency>
128-
<groupId>org.apache.parquet</groupId>
129-
<artifactId>parquet-avro</artifactId>
130-
<version>${parquet.version}</version>
131-
</dependency>
132-
133-
<!-- https://mvnrepository.com/artifact/org.apache.avro/avro -->
134-
<dependency>
135-
<groupId>org.apache.avro</groupId>
136-
<artifactId>avro</artifactId>
137-
<version>${avro.version}</version>
138-
<exclusions>
139-
<exclusion>
140-
<groupId>org.apache.commons</groupId>
141-
<artifactId>commons-compress</artifactId>
142-
</exclusion>
143-
</exclusions>
144-
</dependency>
145-
146119
<dependency>
147120
<groupId>io.airlift</groupId>
148121
<artifactId>concurrent</artifactId>

‎fe/be-java-extensions/iceberg-metadata-scanner/pom.xml‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,16 +37,10 @@ under the License.
3737
<dependencies>
3838
<dependency>
3939
<groupId>org.apache.doris</groupId>
40-
<artifactId>java-common</artifactId>
40+
<artifactId>preload-extensions</artifactId>
4141
<version>${project.version}</version>
42-
<exclusions>
43-
<exclusion>
44-
<groupId>org.apache.thrift</groupId>
45-
<artifactId>libthrift</artifactId>
46-
</exclusion>
47-
</exclusions>
42+
<scope>provided</scope>
4843
</dependency>
49-
5044
<dependency>
5145
<groupId>org.apache.iceberg</groupId>
5246
<artifactId>iceberg-core</artifactId>
@@ -60,7 +54,6 @@ under the License.
6054
<dependency>
6155
<groupId>software.amazon.awssdk</groupId>
6256
<artifactId>auth</artifactId>
63-
<version>${awssdk.version}</version>
6457
<scope>provided</scope>
6558
</dependency>
6659
</dependencies>

0 commit comments

Comments
 (0)