Avoid preferences lock file contention for tests due to spawned child processes (#10083)

* fix: Try using a test task specific folder for user preference to avoid locks
* fix: Use a temporary preference folder for gradle smoke tests
* chore: Emit a stacktrace for gradle jobs
* chore: Patch smoke test forked processes to use a custom preference file
* fix: Don't use the the shared tempdir as it is shared for every test
* fix: Previous code, was generating a single gradle properties file per test.

Co-authored-by: Alexey Kuznetsov <[email protected]>

Author: bric3

buildSrc/src/main/kotlin/dd-trace-java.configure-tests.gradle.kts

@@ -42,6 +42,10 @@ tasks.withType<Test>().configureEach {
     !rootProject.providers.gradleProperty("rerun.tests.${project.name}").isPresent
   }
 
+  // Trick to avoid on CI: "Couldn't flush user prefs: java.util.prefs.BackingStoreException: Couldn't get file lock."
+  // Use a task-specific user prefs directory
+  systemProperty("java.util.prefs.userRoot", "$buildDir/tmp/userPrefs/${name}")
+
   // Split up tests that want to run forked in their own separate JVM for generated tasks
   if (name.startsWith("forkedTest") || name.endsWith("ForkedTest")) {
     setExcludes(emptyList())

dd-java-agent/agent-ci-visibility/civisibility-test-fixtures/src/main/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy

@@ -6,9 +6,13 @@ import datadog.trace.api.config.CiVisibilityConfig
 import datadog.trace.api.config.GeneralConfig
 import datadog.trace.api.config.TraceInstrumentationConfig
 import datadog.trace.api.config.TracerConfig
+import java.nio.file.Paths
 import spock.lang.Specification
+import spock.lang.TempDir
 import spock.util.environment.Jvm
 
+import java.nio.file.Path
+
 import static datadog.trace.util.ConfigStrings.propertyNameToSystemPropertyName
 
 abstract class CiVisibilitySmokeTest extends Specification {
@@ -21,6 +25,9 @@ abstract class CiVisibilitySmokeTest extends Specification {
 
   private static final Map<String,String> DEFAULT_TRACER_CONFIG = defaultJvmArguments()
 
+  @TempDir
+  protected Path prefsDir
+
   protected static String buildJavaHome() {
     if (Jvm.current.isJava8()) {
       return System.getenv("JAVA_8_HOME")
@@ -69,6 +76,9 @@ abstract class CiVisibilitySmokeTest extends Specification {
 
   protected List<String> buildJvmArguments(String mockBackendIntakeUrl, String serviceName, Map<String, String> additionalArgs) {
     List<String> arguments = []
+
+    arguments += preventJulPrefsFileLock()
+
     Map<String, String> argMap = buildJvmArgMap(mockBackendIntakeUrl, serviceName, additionalArgs)
 
     // for convenience when debugging locally
@@ -85,6 +95,28 @@ abstract class CiVisibilitySmokeTest extends Specification {
     return arguments
   }
 
+  /**
+   * Trick to prevent jul Prefs file lock issue on forked processes, in particular in CI which
+   * runs on Linux and have competing processes trying to write to it, including the Gradle daemon.
+   *
+   * <pre><code>
+   * Couldn't flush user prefs: java.util.prefs.BackingStoreException: Couldn't get file lock.
+   * </code></pre>
+   *
+   * Note, some tests can setup arguments on spec level, so `prefsDir` will be `null` during
+   * `setupSpec()`.
+   */
+  protected String preventJulPrefsFileLock() {
+    String prefsPath = (prefsDir ?: tempUserPrefsPath()).toAbsolutePath()
+    return "-Djava.util.prefs.userRoot=$prefsPath".toString()
+  }
+
+  private static Path tempUserPrefsPath() {
+    String uniqueId = "${System.currentTimeMillis()}_${System.nanoTime()}_${Thread.currentThread().id}"
+    Path prefsPath = Paths.get(System.getProperty("java.io.tmpdir"), "gradle-test-userPrefs", uniqueId)
+    return prefsPath
+  }
+
   protected verifyEventsAndCoverages(String projectName, String toolchain, String toolchainVersion, List<Map<String, Object>> events, List<Map<String, Object>> coverages, List<String> additionalDynamicTags = []) {
     def additionalReplacements = ["content.meta.['test.toolchain']": "$toolchain:$toolchainVersion"]
 

dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProcessBuilderHelper.java

@@ -28,6 +28,20 @@ public static ProcessBuilder createProcessBuilder(
       String mainClassName,
       String... params) {
 
+    // Trick to prevent jul preferences file lock issue on forked processes, in particular in CI
+    // which runs on Linux and have competing processes trying to write to it, including the
+    // Gradle daemon.
+    //
+    //   Couldn't flush user prefs: java.util.prefs.BackingStoreException: Couldn't get file lock.
+    String prefsDir =
+        System.getProperty("java.io.tmpdir")
+            + File.separator
+            + "userPrefs"
+            + File.separator
+            + mainClassName
+            + "_"
+            + System.nanoTime();
+
     List<String> baseCommand =
         Arrays.asList(
             javaPath(),
@@ -37,7 +51,8 @@ public static ProcessBuilder createProcessBuilder(
             "-javaagent:" + agentShadowJar(),
             "-XX:ErrorFile=/tmp/hs_err_pid%p.log",
             "-Ddd.env=smoketest",
-            "-Ddd.version=99");
+            "-Ddd.version=99",
+            "-Djava.util.prefs.userRoot=" + prefsDir);
 
     List<String> command = new ArrayList<>();
     command.addAll(baseCommand);

dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy

@@ -34,10 +34,6 @@ class GradleDaemonSmokeTest extends AbstractGradleTest {
   @TempDir
   Path testKitFolder
 
-  def setupSpec() {
-    givenGradleProperties()
-  }
-
   @IgnoreIf(reason = "Jacoco plugin does not work with OpenJ9 in older Gradle versions", value = {
     JavaVirtualMachine.isJ9()
   })
@@ -79,6 +75,7 @@ class GradleDaemonSmokeTest extends AbstractGradleTest {
   def "test junit4 class ordering v#gradleVersion"() {
     givenGradleVersionIsCompatibleWithCurrentJvm(gradleVersion)
     givenGradleProjectFiles(projectName)
+    givenGradleProjectProperties()
     ensureDependenciesDownloaded(gradleVersion)
 
     mockBackend.givenKnownTests(true)
@@ -124,6 +121,7 @@ class GradleDaemonSmokeTest extends AbstractGradleTest {
     givenGradleVersionIsCompatibleWithCurrentJvm(gradleVersion)
     givenConfigurationCacheIsCompatibleWithCurrentPlatform(configurationCache)
     givenGradleProjectFiles(projectName)
+    givenGradleProjectProperties()
     ensureDependenciesDownloaded(gradleVersion)
 
     mockBackend.givenFlakyRetries(flakyRetries)
@@ -150,7 +148,7 @@ class GradleDaemonSmokeTest extends AbstractGradleTest {
     }
   }
 
-  private void givenGradleProperties() {
+  private void givenGradleProjectProperties() {
     assert new File(AGENT_JAR).isFile()
 
     def ddApiKeyPath = testKitFolder.resolve(".dd.api.key")
@@ -173,7 +171,9 @@ class GradleDaemonSmokeTest extends AbstractGradleTest {
     def arguments = buildJvmArguments(mockBackend.intakeUrl, TEST_SERVICE_NAME, additionalArgs)
 
     def gradleProperties = "org.gradle.jvmargs=${arguments.join(" ")}".toString()
-    Files.write(testKitFolder.resolve("gradle.properties"), gradleProperties.getBytes())
+    // Write to projectFolder (per-test) instead of testKitFolder (shared), so each
+    // Gradle daemon gets its own unique preference directory
+    Files.write(projectFolder.resolve("gradle.properties"), gradleProperties.getBytes())
   }
 
   private BuildResult runGradleTests(String gradleVersion, boolean successExpected = true, boolean configurationCache = false) {

dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy

@@ -188,6 +188,12 @@ abstract class AbstractSmokeTest extends ProcessManager {
   def javaProperties() {
     def tmpDir = "/tmp"
 
+    // Trick to prevent jul preferences file lock issue on forked processes, in particular in CI which
+    // runs on Linux and have competing processes trying to write to it, including the Gradle daemon.
+    //
+    //   Couldn't flush user prefs: java.util.prefs.BackingStoreException: Couldn't get file lock.
+    def prefsDir = "${tmpDir}/userPrefs/${this.getClass().simpleName}_${System.nanoTime()}"
+
     def ret = [
       "${getMaxMemoryArgumentForFork()}",
       "${getMinMemoryArgumentForFork()}",
@@ -204,7 +210,8 @@ abstract class AbstractSmokeTest extends ProcessManager {
       "-Ddd.profiling.ddprof.alloc.enabled=${isDdprofSafe()}",
       "-Ddatadog.slf4j.simpleLogger.defaultLogLevel=${logLevel()}",
       "-Dorg.slf4j.simpleLogger.defaultLogLevel=${logLevel()}",
-      "-Ddd.site="
+      "-Ddd.site=",
+      "-Djava.util.prefs.userRoot=${prefsDir}"
     ]
     if (inferServiceName())  {
       ret += "-Ddd.service.name=${SERVICE_NAME}"