[bazel] Rewrite overlay handling to starlark (#170000)

Starlark is perfectly capable of doing what we need and this avoids the
dependency on a host Python

Author: dzbarsky

utils/bazel/configure.bzl

@@ -4,9 +4,6 @@
 
 """Helper macros to configure the LLVM overlay project."""
 
-# Directory of overlay files relative to WORKSPACE
-DEFAULT_OVERLAY_PATH = "llvm-project-overlay"
-
 DEFAULT_TARGETS = [
     "AArch64",
     "AMDGPU",
@@ -30,43 +27,54 @@ DEFAULT_TARGETS = [
     "XCore",
 ]
 
+
+MAX_TRAVERSAL_STEPS = 1000000  # "big number" upper bound on total visited dirs
+
 def _overlay_directories(repository_ctx):
-    src_path = repository_ctx.path(Label("@llvm-raw//:WORKSPACE")).dirname
-    bazel_path = src_path.get_child("utils").get_child("bazel")
-    overlay_path = bazel_path.get_child("llvm-project-overlay")
-    script_path = bazel_path.get_child("overlay_directories.py")
-
-    python_bin = repository_ctx.which("python3")
-    if not python_bin:
-        # Windows typically just defines "python" as python3. The script itself
-        # contains a check to ensure python3.
-        python_bin = repository_ctx.which("python")
-
-    if not python_bin:
-        fail("Failed to find python3 binary")
-
-    cmd = [
-        python_bin,
-        script_path,
-        "--src",
-        src_path,
-        "--overlay",
-        overlay_path,
-        "--target",
-        ".",
-    ]
-    exec_result = repository_ctx.execute(cmd, timeout = 20)
-
-    if exec_result.return_code != 0:
-        fail(("Failed to execute overlay script: '{cmd}'\n" +
-              "Exited with code {return_code}\n" +
-              "stdout:\n{stdout}\n" +
-              "stderr:\n{stderr}\n").format(
-            cmd = " ".join([str(arg) for arg in cmd]),
-            return_code = exec_result.return_code,
-            stdout = exec_result.stdout,
-            stderr = exec_result.stderr,
-        ))
+    src_root = repository_ctx.path(Label("@llvm-raw//:WORKSPACE")).dirname
+    overlay_root = src_root.get_child("utils/bazel/llvm-project-overlay")
+    target_root = repository_ctx.path(".")
+
+    # Tries to minimize the number of symlinks created (that is, does not symlink
+    # every single file). Symlinks every file in the overlay directory. Only symlinks
+    # individual files in the source directory if their parent directory is also
+    # contained in the overlay directory tree.
+
+    stack = ["."]
+    for _ in range(MAX_TRAVERSAL_STEPS):
+        rel_dir = stack.pop()
+
+        overlay_dirs = set()
+
+        # Symlink overlay files, overlay dirs will be handled in future iterations.
+        for entry in overlay_root.get_child(rel_dir).readdir():
+            name = entry.basename
+            full_rel_path = rel_dir + "/" + name
+
+            if entry.is_dir:
+                stack.append(full_rel_path)
+                overlay_dirs.add(name)
+            else:
+                src_path = overlay_root.get_child(full_rel_path)
+                dst_path = target_root.get_child(full_rel_path)
+                repository_ctx.symlink(src_path, dst_path)
+
+        # Symlink source dirs (if not themselves overlaid) and files.
+        for src_entry in src_root.get_child(rel_dir).readdir():
+            name = src_entry.basename
+            if name in overlay_dirs:
+                # Skip: overlay has a directory with this name
+                continue
+
+            repository_ctx.symlink(src_entry, target_root.get_child(rel_dir + "/" + name))
+
+        if not stack:
+            return
+
+    fail("overlay_directories: exceeded MAX_TRAVERSAL_STEPS ({}). " +
+         "Tree too large or a cycle in the filesystem?".format(
+             MAX_TRAVERSAL_STEPS,
+         ))
 
 def _extract_cmake_settings(repository_ctx, llvm_cmake):
     # The list to be written to vars.bzl