fix: Replace invalid UTF-8 characters in doc comments (#453)

The types for doc comments in our Protobuf code are strings,
which means the the contents must be valid UTF-8. However,
we were not doing validation before storing the contents.

This PR adds a validation step, and if the validation fails,
then we substitute invalid characters with the standard
unicode replacement character.

The utfcpp library was chosen as it is pretty decent at
[benchmarks](https://thephd.dev/the-c-c++-rust-string-text-encoding-api-landscape)
and has a very easy to use API for our purposes.

Author: varungandhi-src

fetch_deps.bzl

@@ -12,6 +12,7 @@ _WYHASH_COMMIT = "ea3b25e1aef55d90f707c3a292eeb9162e2615d8"
 _SPDLOG_COMMIT = "edc51df1bdad8667b628999394a1e7c4dc6f3658"
 _PROTOBUF_VERSION = "3.21.12"
 _SCIP_COMMIT = "aa0e511dcfefbacc3b96dcc2fe2abd9894416b1e"
+_UTFCPP_VERSION = "4.0.5"
 # ^ When bumping this version, check if any new fields are introduced
 # in the types for which we implement hashing and comparison in
 # indexer/ScipExtras.{h,cc}
@@ -234,3 +235,11 @@ def fetch_direct_dependencies():
         strip_prefix = "rules_python-%s" % _RULES_PYTHON_VERSION,
         url = "https://github.com/bazelbuild/rules_python/releases/download/{0}/rules_python-{0}.tar.gz".format(_RULES_PYTHON_VERSION),
     )
+
+    http_archive(
+        name = "utfcpp",
+        sha256 = "ffc668a310e77607d393f3c18b32715f223da1eac4c4d6e0579a11df8e6b59cf",
+        build_file = "@scip_clang//third_party:utfcpp.BUILD",
+        strip_prefix = "utfcpp-%s" % _UTFCPP_VERSION,
+        url = "https://github.com/nemtrif/utfcpp/archive/refs/tags/v{0}.tar.gz".format(_UTFCPP_VERSION),
+    )

indexer/BUILD

@@ -37,6 +37,7 @@ cc_library(
         "@llvm-project//clang:frontend",
         "@llvm-project//clang:tooling",
         "@scip",
+        "@utfcpp",
     ],
 )
 

indexer/Indexer.cc

@@ -10,6 +10,7 @@
 #include "absl/strings/strip.h"
 #include "perfetto/perfetto.h"
 #include "spdlog/fmt/fmt.h"
+#include "utfcpp/utf8.h"
 
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
@@ -315,6 +316,13 @@ void DocComment::addTo(std::string &slot) {
   if (stripped.empty()) {
     return;
   }
+  if (!utf8::is_valid(stripped)) {
+    slot.clear();
+    utf8::replace_invalid(stripped.begin(), stripped.end(),
+                          std::back_inserter(slot));
+    this->contents.clear();
+    return;
+  }
   if (stripped.size() == this->contents.size()) {
     slot = std::move(this->contents);
     return;

indexer/ScipExtras.cc

@@ -9,6 +9,7 @@
 #include "absl/algorithm/container.h"
 #include "absl/functional/function_ref.h"
 #include "perfetto/perfetto.h"
+#include "utfcpp/utf8.h"
 
 #include "llvm/Support/Path.h"
 #include "llvm/Support/StringSaver.h"

third_party/utfcpp.BUILD

@@ -0,0 +1,8 @@
+cc_library(
+    name = "utfcpp",
+    includes = ["source"],
+    include_prefix = "utfcpp",
+    strip_include_prefix = "source",
+    hdrs = glob(["source/**/*.h"], allow_empty=False),
+    visibility = ["//visibility:public"],
+)