Merge pull request #893 from Altinity/867-potential-connection-leak-in-createconnection-method

Use Hikari Connection Pool for handling Clickhouse connections

Author: subkanthi

.github/workflows/pull-request.yml

@@ -30,6 +30,15 @@ jobs:
     with:
       SINK_CONNECTOR_IMAGE: altinityinfra/clickhouse-sink-connector:${{ github.event.number }}-${{ github.sha }}-lt
 
+  testflows-lightweight-hikari-pool:
+    needs: [build-kafka-lightweight]
+    uses: ./.github/workflows/testflows-sink-connector-lightweight.yml
+    secrets: inherit
+    with:
+      SINK_CONNECTOR_IMAGE: altinityinfra/clickhouse-sink-connector:${{ github.event.number }}-${{ github.sha }}-lt
+      extra_args: --hikari-pool
+      artifact_suffix: hikari-pool
+
   java-tests-kafka:
     needs: [build-kafka-lightweight]
     uses: ./.github/workflows/sink-connector-kafka-tests.yml

.github/workflows/testflows-sink-connector-lightweight.yml

@@ -11,11 +11,21 @@ on:
       package:
         description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'"
         type: string
-        default: docker://clickhouse/clickhouse-server:23.3
+        default: docker://clickhouse/clickhouse-server:23.8
       output_format:
         description: "Testflows output style."
         type: string
         default: new-fails
+      extra_args:
+        description: "Extra arguments for the tests."
+        required: false
+        type: string
+        default: ""
+      artifact_suffix:
+        description: "Artifact sufix"
+        required: false
+        type: string
+        default: ""
     secrets:
       DOCKERHUB_USERNAME:
         required: false
@@ -34,11 +44,16 @@ on:
       package:
         description: "Package either 'docker://' or 'https://'. Example: 'https://s3.amazonaws.com/clickhouse-builds/23.3/.../package_release/clickhouse-common-static_23.3.1.64_amd64.deb', or 'docker://altinity/clickhouse-server:23.8.8'"
         type: string
-        default: docker://clickhouse/clickhouse-server:23.3
-      extra_args:
+        default: docker://clickhouse/clickhouse-server:23.8
+      suite:
         description: "Specific Suite To Run (Default * to run everything)."
         required: false
         type: string
+      extra_args:
+        description: "Extra arguments for the tests."
+        required: false
+        type: string
+        default: ""
       custom_run_name:
         description: 'Custom run name (optional)'
         required: false
@@ -115,7 +130,7 @@ jobs:
 
       - name: Run testflows tests
         working-directory: sink-connector-lightweight/tests/integration
-        run: python3 -u regression.py --only "/mysql to clickhouse replication/auto table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
+        run: python3 -u regression.py --only "/mysql to clickhouse replication/auto table creation/${{ inputs.suite != '' && inputs.suite || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end ${{ inputs.extra_args != '' && inputs.extra_args || '' }} --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
 
       - name: Create tfs results report
         if: always()
@@ -139,7 +154,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: testflows-sink-connector-lightweight-artefacts
+          name: testflows-sink-connector-lightweight-artefacts${{ inputs.artifact_suffix != '' && '-' || '' }}${{ inputs.artifact_suffix }}
           path: |
             sink-connector-lightweight/tests/integration/logs/*.log
             sink-connector-lightweight/tests/integration/env/auto/configs/*.yml
@@ -193,7 +208,7 @@ jobs:
 
       - name: Run testflows tests
         working-directory: sink-connector-lightweight/tests/integration
-        run: python3 -u regression.py --only "/mysql to clickhouse replication/auto replicated table creation/${{ inputs.extra_args != '' && inputs.extra_args || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
+        run: python3 -u regression.py --only "/mysql to clickhouse replication/auto replicated table creation/${{ inputs.suite != '' && inputs.suite || '*' }}" --clickhouse-binary-path="${{inputs.package}}" --test-to-end ${{ inputs.extra_args != '' && inputs.extra_args || '' }} --output ${{ inputs.output_format }} --collect-service-logs --attr project="${GITHUB_REPOSITORY}" project.id="$GITHUB_RUN_NUMBER" user.name="$GITHUB_ACTOR" github_actions_run="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" sink_version="registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:latest" s3_url="https://altinity-test-reports.s3.amazonaws.com/index.html#altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_${{github.run.number}}/" --log logs/raw.log
 
       - name: Create tfs results report
         if: always()
@@ -217,7 +232,7 @@ jobs:
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: testflows-sink-connector-lightweight-replicated-artefacts
+          name: testflows-sink-connector-lightweight-replicated-artefacts${{ inputs.artifact_suffix != '' && '-' || '' }}${{ inputs.artifact_suffix }}
           path: |
             sink-connector-lightweight/tests/integration/logs/*.log
             sink-connector-lightweight/tests/integration/env/auto_replicated/configs/*.yml

doc/Troubleshooting.md

@@ -33,6 +33,11 @@ This can be limited by passing this configuration parameter to the JVM:
 #N can be 50.
 -XX:ParallelGCThreads=<N>.
 ```
+### Caused by: java.sql.BatchUpdateException: Connect to timed out
+Set `keepalive.timeout=3` in clickhouse server and make sure
+the following configuration 
+`clickhouse.jdbc.params: "keepalive.timeout=3,max_buffer_size=1000000,socket_timeout=30000,connection_timeout=30000"`
+is added to `config.yml`
 
 ### Slow startup when replicating Database with high number of tables.
 Use the following configuration to limit the number of tables that Debezium would capture in the schema_only mode.
@@ -57,4 +62,4 @@ https://stackoverflow.com/questions/63523998/multiple-debezium-connector-for-one
 Make sure to add `slot.name` to the configuration(config.yml) and change it to a unique name.
 
 ### PostgreSQL (WAL size growing)
-[Handling PostgreSQL WAL Growth with Debezium Connectors](doc/postgres_wal_growth.md)
+[Handling PostgreSQL WAL Growth with Debezium Connectors](doc/postgres_wal_growth.md)

sink-connector-lightweight/dependency-reduced-pom.xml

@@ -121,12 +121,6 @@
       <artifactId>javalin-testtools</artifactId>
       <version>5.5.0</version>
       <scope>test</scope>
-      <exclusions>
-        <exclusion>
-          <artifactId>okhttp</artifactId>
-          <groupId>com.squareup.okhttp3</groupId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>io.debezium</groupId>
@@ -162,10 +156,6 @@
           <artifactId>oracle-xe</artifactId>
           <groupId>org.testcontainers</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>junit-platform-launcher</artifactId>
-          <groupId>org.junit.platform</groupId>
-        </exclusion>
         <exclusion>
           <artifactId>junit-jupiter</artifactId>
           <groupId>org.junit.jupiter</groupId>
@@ -174,10 +164,6 @@
           <artifactId>quarkus-test-common</artifactId>
           <groupId>io.quarkus</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>okhttp</artifactId>
-          <groupId>com.squareup.okhttp3</groupId>
-        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -207,7 +193,7 @@
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>testcontainers</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
       <exclusions>
         <exclusion>
@@ -231,7 +217,7 @@
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>jdbc</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
       <exclusions>
         <exclusion>
@@ -243,37 +229,37 @@
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>junit-jupiter</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>clickhouse</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>mysql</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>postgresql</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>mariadb</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
       <artifactId>mongodb</artifactId>
-      <version>1.19.1</version>
+      <version>1.20.4</version>
       <scope>test</scope>
     </dependency>
     <dependency>
@@ -289,10 +275,6 @@
       <version>5.8.1</version>
       <scope>test</scope>
       <exclusions>
-        <exclusion>
-          <artifactId>junit-platform-engine</artifactId>
-          <groupId>org.junit.platform</groupId>
-        </exclusion>
         <exclusion>
           <artifactId>junit-jupiter-api</artifactId>
           <groupId>org.junit.jupiter</groupId>
@@ -317,10 +299,38 @@
           <artifactId>junit-jupiter-api</artifactId>
           <groupId>org.junit.jupiter</groupId>
         </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <version>1.8.2</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>apiguardian-api</artifactId>
+          <groupId>org.apiguardian</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-engine</artifactId>
+      <version>1.8.2</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>opentest4j</artifactId>
+          <groupId>org.opentest4j</groupId>
+        </exclusion>
         <exclusion>
-          <artifactId>junit-platform-launcher</artifactId>
+          <artifactId>junit-platform-commons</artifactId>
           <groupId>org.junit.platform</groupId>
         </exclusion>
+        <exclusion>
+          <artifactId>apiguardian-api</artifactId>
+          <groupId>org.apiguardian</groupId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -356,7 +366,7 @@
   <properties>
     <quarkus.platform.version>2.14.0.Final</quarkus.platform.version>
     <quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
-    <version.testcontainers>1.19.1</version.testcontainers>
+    <version.testcontainers>1.20.4</version.testcontainers>
     <surefire-plugin.version>3.0.0-M7</surefire-plugin.version>
     <version.kafka>3.8.0</version.kafka>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

sink-connector-lightweight/docker/clickhouse.sh

@@ -0,0 +1 @@
+docker exec -it clickhouse clickhouse-client --database public --final 1

sink-connector-lightweight/docker/config.yml

@@ -29,7 +29,7 @@ database.server.name: "ER54"
 
 # database.include.list An optional list of regular expressions that match database names to be monitored;
 # any database name not included in the whitelist will be excluded from monitoring. By default all databases will be monitored.
-database.include.list: test
+database.include.list: sbtest
 
 # table.include.list An optional list of regular expressions that match fully-qualified table identifiers for tables to be monitored;
 table.include.list: ""
@@ -130,7 +130,7 @@ database.connectionTimeZone: "UTC"
 
 # Configuration to override the clickhouse database name for a given MySQL database name. If this configuration is not
 # provided, the MySQL database name will be used as the ClickHouse database name.
-clickhouse.database.override.map: "test:ch_test"
+#clickhouse.database.override.map: "test:ch_test"
 
 # clickhouse.datetime.timezone: This timezone will override the default timezone of ClickHouse server. Timezone columns will be set to this timezone.
 #clickhouse.datetime.timezone: "UTC"
@@ -168,7 +168,7 @@ ignore.ddl.regex: "(?i)(ANALYZE PARTITION).*"
 #buffer.max.records: "10000"
 
 # ClickHouse JDBC configuration parameters, as a list of key-value pairs separated by commas.
-#clickhouse.jdbc.params: "max_buffer_size=1000000,socket_timeout=10000"
+clickhouse.jdbc.params: "keepalive.timeout=3,max_buffer_size=1000000,socket_timeout=30000,connection_timeout=30000"
 
 # Maximum number of threads in the thread pool for processing CDC records.
 #thread.pool.size: 10
@@ -179,6 +179,9 @@ ignore.ddl.regex: "(?i)(ANALYZE PARTITION).*"
 #Metrics (Prometheus target), required for Grafana Dashboard
 metrics.enable: "true"
 
+#connection.pool.disable: "true"
+#connection.pool.max.size: 500
+
 # Skip schema history capturing, use the following configuration
 # to reduce slow startup when replicating dbs with large number of tables
 #schema.history.internal.store.only.captured.tables.ddl: "true"
@@ -189,4 +192,4 @@ use.nongraceful.disconnect: "true"
 database.keep.alive.interval.`ms: "30000"  #Send keepalive every 30 seconds
 database.connection.reconnect.backoff.ms: "1000"
 database.connection.reconnect.backoff.max.ms: "10000"
-database.ssl.mode: "disabled"
+database.ssl.mode: "disabled"

sink-connector-lightweight/docker/config_postgres.yml

@@ -93,6 +93,8 @@ auto.create.tables: "true"
 # database.dbname: The name of the PostgreSQL database from which events are to be captured when not using snapshot mode.
 database.dbname: "public"
 
+clickhouse.jdbc.params: "keepalive.timeout=3,max_buffer_size=1000000,socket_timeout=30000,connection_timeout=30000"
+
 # clickhouse.datetime.timezone: This timezone will override the default timezone of ClickHouse server. Timezone columns will be set to this timezone.
 #clickhouse.datetime.timezone: "UTC"
 

sink-connector-lightweight/docker/postgres.sh

@@ -0,0 +1,2 @@
+docker exec -it docker_postgres_1 psql -U root public
+

sink-connector-lightweight/docker/startMySQLCHLocalSinkDev.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+docker network create docker_default
+docker-compose create mysql-master
+docker-compose create clickhouse
+docker-compose start mysql-master
+docker-compose start clickhouse

sink-connector-lightweight/pom.xml

@@ -15,7 +15,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <version.debezium>3.0.0.Final</version.debezium>
         <version.junit>5.9.1</version.junit>
-        <version.testcontainers>1.19.1</version.testcontainers>
+        <version.testcontainers>1.20.4</version.testcontainers>
         <version.checkstyle.plugin>3.1.1</version.checkstyle.plugin>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -233,6 +233,11 @@
             <!-- <scope>test</scope>-->
         </dependency>
 
+        <dependency>
+            <groupId>com.zaxxer</groupId>
+            <artifactId>HikariCP</artifactId>
+            <version>6.0.0</version>
+        </dependency>
         <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
@@ -334,6 +339,18 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.junit.platform</groupId>
+            <artifactId>junit-platform-launcher</artifactId>
+            <version>1.8.2</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.platform</groupId>
+            <artifactId>junit-platform-engine</artifactId>
+            <version>1.8.2</version>
+            <scope>test</scope>
+        </dependency>
         <dependency> <!-- necessary for Java 9+ -->
             <groupId>org.apache.tomcat</groupId>
             <artifactId>annotations-api</artifactId>