Refactor: Split out schemafiltering into its own package (#3220)

Alternative approach for
#3210.
[Maintaining a whole repo for a single
function](pulumi/provider-schema-filter#1) comes
with some overhead.
Consider also that the logic in the filter package is tightly coupled
with expectations set elsewhere in the bridge.

- **Create schemafilter package**
- **Use schemafilter package function**
- **Add README**
- **cleanups**

Author: guineveresaenger

pkg/tfgen/generate.go

@@ -26,7 +26,6 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
-	"regexp"
 	"sort"
 	"strings"
 	"unicode"
@@ -48,6 +47,7 @@ import (
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tf2pulumi/il"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfgen/internal/paths"
+	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfgen/schemafilter"
 	shim "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/schema"
 	"github.com/pulumi/pulumi-terraform-bridge/v3/unstable/metadata"
@@ -1043,64 +1043,6 @@ type GenerateOptions struct {
 	ModuleFormat string
 }
 
-func (g *Generator) FilterSchemaByLanguage(schemaBytes []byte) []byte {
-	// The span string stems from g.fixUpPropertyReference in docsgen and looks as follows:
-	// <span pulumi-lang-nodejs="firstProperty" pulumi-lang-go="FirstProperty" ...>first_property</span>
-	// When rendered in schema it uses escapes and unicode chars for the angle brackets:
-	// \u003cspan pulumi-lang-nodejs=\"`random.RandomBytes`\" pulumi-lang-dotnet=\"`random.RandomBytes`\" ... \u003e ...
-	spanRegex := regexp.MustCompile(`\\u003cspan pulumi-lang-nodejs=.*?\\u003c/span\\u003e`)
-
-	// Extract the language-specific inflection for the found inflection span
-	schemaBytes = spanRegex.ReplaceAllFunc(schemaBytes, func(match []byte) []byte {
-		languageKey := []byte(fmt.Sprintf(`pulumi-lang-%s=\"`, g.language))
-		_, startLanguageValue, _ := bytes.Cut(match, languageKey)
-		var languageValue []byte
-
-		// Sometimes we have double quotes in our language span. Handle this case so that we return the quotes.
-		doubleEscapedQuotes := []byte(`\"\"`)
-		singleEscapedQuotes := []byte(`\"`)
-		if loc := bytes.Index(startLanguageValue, doubleEscapedQuotes); loc > 0 {
-			// Cut after the first quote to include it in the result
-			languageValue = startLanguageValue[:loc+(len(singleEscapedQuotes))]
-		} else {
-			languageValue, _, _ = bytes.Cut(startLanguageValue, singleEscapedQuotes)
-		}
-		return languageValue
-	})
-
-	// Find code chooser blocks and filter to only keep the current language
-	codeChooserRegex := regexp.MustCompile(
-		`\\u003c!--Start PulumiCodeChooser --\\u003e.*?\\u003c!--End PulumiCodeChooser --\\u003e`,
-	)
-
-	schemaBytes = codeChooserRegex.ReplaceAllFunc(schemaBytes, func(match []byte) []byte {
-		content := string(match)
-
-		// In code choosers for registry docsgen, "nodejs" is "typescript"
-		codeLang := g.language
-		if g.language == "nodejs" {
-			codeLang = "typescript"
-		}
-		// In code choosers, "dotnet" is "csharp"
-		if g.language == "dotnet" {
-			codeLang = "csharp"
-		}
-		// Extract language-specific example only
-		_, after, found := strings.Cut(content, fmt.Sprintf("```%s", codeLang))
-		if !found {
-			return []byte("")
-		}
-		codeForLanguage, _, found := strings.Cut(after, "```")
-		if !found {
-			return []byte("")
-		}
-		codeForLanguage = fmt.Sprintf("```%s", codeLang) + codeForLanguage + "```"
-
-		return []byte(codeForLanguage)
-	})
-	return schemaBytes
-}
-
 // Generate creates Pulumi packages from the information it was initialized with.
 func (g *Generator) Generate() (*GenerateSchemaResult, error) {
 	if g.language == "schema" || g.language == "registry-docs" || g.language == "pulumi" {
@@ -1117,7 +1059,7 @@ func (g *Generator) Generate() (*GenerateSchemaResult, error) {
 		return nil, err
 	}
 	// Generate the language-specific bytes
-	languageSchemaBytes := g.FilterSchemaByLanguage(schemaBytes)
+	languageSchemaBytes := schemafilter.FilterSchemaByLanguage(schemaBytes, string(g.language))
 
 	// Parse the filtered schema bytes back into PackageSpec
 	var languagePackageSpec pschema.PackageSpec

pkg/tfgen/generate_test.go

@@ -15,7 +15,6 @@
 package tfgen
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"os"
@@ -24,7 +23,6 @@ import (
 	"testing"
 
 	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
-	"github.com/hexops/autogold/v2"
 	pschema "github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors"
@@ -891,89 +889,3 @@ func TestExtraMappingError(t *testing.T) {
 		})
 	}
 }
-
-func TestFilterSchemaByLanguage(t *testing.T) {
-	t.Parallel()
-	testCases := []struct {
-		name                        string
-		inputSchema                 []byte
-		expectedLanguageSchemaBytes []byte
-		generator                   *Generator
-	}{
-		{
-			name:        "Generates nodejs schema",
-			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
-			generator: &Generator{
-				version:  "1.2.3-test",
-				language: "nodejs",
-			},
-		},
-		{
-			name:        "Generates python schema",
-			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
-			generator: &Generator{
-				version:  "1.2.3-test",
-				language: "python",
-			},
-		},
-		{
-			name:        "Generates dotnet schema",
-			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
-			generator: &Generator{
-				version:  "1.2.3-test",
-				language: "dotnet",
-			},
-		},
-		{
-			name:        "Generates go schema",
-			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
-			generator: &Generator{
-				version:  "1.2.3-test",
-				language: "go",
-			},
-		},
-		{
-			name:        "Generates yaml schema",
-			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
-			generator: &Generator{
-				version:  "1.2.3-test",
-				language: "yaml",
-			},
-		},
-		{
-			name:        "Generates java schema",
-			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
-			generator: &Generator{
-				version:  "1.2.3-test",
-				language: "java",
-			},
-		},
-		{
-			name:        "Handles property names that are not surrounded by back ticks",
-			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema-no-backticks.json")),
-			generator: &Generator{
-				version:  "1.2.3-test",
-				language: "nodejs",
-			},
-		},
-		{
-			name:        "Handles property names that are surrounded by back ticks AND double quotes",
-			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema-backticks-and-quotes.json")),
-			generator: &Generator{
-				version:  "1.2.3-test",
-				language: "nodejs",
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			actual := tc.generator.FilterSchemaByLanguage(tc.inputSchema)
-			hasSpan := bytes.Contains(actual, []byte("span"))
-			require.False(t, hasSpan, "there should be no spans in the filtered schema")
-			hasCodeChoosers := bytes.Contains(actual, []byte("PulumiCodeChooser"))
-			require.False(t, hasCodeChoosers)
-			autogold.ExpectFile(t, autogold.Raw(actual))
-		})
-	}
-}

pkg/tfgen/schemafilter/README.md

@@ -0,0 +1,37 @@
+# provider-schema-filter
+A small library for provider schema processing
+
+> [NOTE] This library is meant for Pulumi-internal use. It is experimental and subject to change.
+
+## Purpose
+
+Prepare Pulumi provider schemas to be passed to the pulumi package gen-sdk command.
+
+Pulumi provider schemas (traditionally located at `provider/cmd/pulumi-resource-foo/schema.json`) are our source for generating registry documentation, provider binaries, and language SDKs, including in-line documentation with examples.
+The schema contains language-specific translations of examples, as well as language-specific inflections of code strings.
+
+Because the schema contains translations for _all_ Pulumi-supported languages, we need to filter the schema by its language before we pass it along to the Pulumi CLI's SDK generator.
+This library is meant for that purpose.
+The filter expects a certain schema format, which is consistent for all bridged providers, detailed below.
+
+## Use
+
+`import "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfgen/schemafilter"`
+
+## Schema markups
+
+The Pulumi schema may contain the following markups:
+
+### Pulumi Code Chooser
+
+The content between the code chooser tags contains the examples that we are rendering in the registry via the language tab selector.
+The outline is as follows:
+`<!--Start PulumiCodeChooser -->```typescript {example code}```\n```python {example code}```\n```csharp {example code}```\n```go {example code}```/n```java {example code}```\n```yaml {example code}```\n<!--End PulumiCodeChooser -->`
+For each SDK, we want to display only the example relevant to that SDK's language.
+
+### Variable inflection
+
+The documentation contains property or resource names that should be inflected by language.
+The precise inflection format depends on whether we are inflecting a resource, a function, or a property name, but the basic format loks like this:
+`<span pulumi-lang-nodejs="exampleProperty" pulumi-lang-dotnet="ExampleProperty" pulumi-lang-go="exampleProperty" pulumi-lang-python="example_property" pulumi-lang-yaml="exampleProperty" pulumi-lang-java="exampleProperty">%s</span>`
+

pkg/tfgen/schemafilter/filter.go

@@ -0,0 +1,67 @@
+package schemafilter
+
+import (
+	"bytes"
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+func FilterSchemaByLanguage(schemaBytes []byte, language string) []byte {
+	// The span string stems from the Terraform bridge's generator's fixUpPropertyReference method in docsgen.
+	// It looks as follows:
+	// <span pulumi-lang-nodejs="firstProperty" pulumi-lang-go="FirstProperty" ...>first_property</span>
+	// When rendered in schema it uses escapes and unicode chars for the angle brackets:
+	// \u003cspan pulumi-lang-nodejs=\"`random.RandomBytes`\" pulumi-lang-dotnet=\"`random.RandomBytes`\" ... \u003e ...
+	spanRegex := regexp.MustCompile(`\\u003cspan pulumi-lang-nodejs=.*?\\u003c/span\\u003e`)
+
+	// Extract the language-specific inflection for the found inflection span
+	schemaBytes = spanRegex.ReplaceAllFunc(schemaBytes, func(match []byte) []byte {
+		languageKey := []byte(fmt.Sprintf(`pulumi-lang-%s=\"`, language))
+		_, startLanguageValue, _ := bytes.Cut(match, languageKey)
+		var languageValue []byte
+
+		// Sometimes we have double quotes in our language span. Handle this case so that we return the quotes.
+		doubleEscapedQuotes := []byte(`\"\"`)
+		singleEscapedQuotes := []byte(`\"`)
+		if loc := bytes.Index(startLanguageValue, doubleEscapedQuotes); loc > 0 {
+			// Cut after the first quote to include it in the result
+			languageValue = startLanguageValue[:loc+(len(singleEscapedQuotes))]
+		} else {
+			languageValue, _, _ = bytes.Cut(startLanguageValue, singleEscapedQuotes)
+		}
+		return languageValue
+	})
+
+	// Find code chooser blocks and filter to only keep the current language
+	codeChooserRegex := regexp.MustCompile(
+		`\\u003c!--Start PulumiCodeChooser --\\u003e.*?\\u003c!--End PulumiCodeChooser --\\u003e`,
+	)
+
+	schemaBytes = codeChooserRegex.ReplaceAllFunc(schemaBytes, func(match []byte) []byte {
+		content := string(match)
+
+		// In code choosers for registry docsgen, "nodejs" is "typescript"
+		codeLang := language
+		if language == "nodejs" {
+			codeLang = "typescript"
+		}
+		// In code choosers, "dotnet" is "csharp"
+		if language == "dotnet" {
+			codeLang = "csharp"
+		}
+		// Extract language-specific example only
+		_, after, found := strings.Cut(content, fmt.Sprintf("```%s", codeLang))
+		if !found {
+			return []byte("")
+		}
+		codeForLanguage, _, found := strings.Cut(after, "```")
+		if !found {
+			return []byte("")
+		}
+		codeForLanguage = fmt.Sprintf("```%s", codeLang) + codeForLanguage + "```"
+
+		return []byte(codeForLanguage)
+	})
+	return schemaBytes
+}

pkg/tfgen/schemafilter/filter_test.go

@@ -0,0 +1,81 @@
+package schemafilter
+
+import (
+	"bytes"
+	"os"
+	"testing"
+
+	"github.com/hexops/autogold/v2"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFilterSchemaByLanguage(t *testing.T) {
+	t.Parallel()
+	testCases := []struct {
+		name                        string
+		inputSchema                 []byte
+		expectedLanguageSchemaBytes []byte
+		language                    string
+		// generator                   *Generator
+	}{
+		{
+			name:        "Generates nodejs schema",
+			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
+			language:    "nodejs",
+		},
+		{
+			name:        "Generates python schema",
+			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
+			language:    "python",
+		},
+		{
+			name:        "Generates dotnet schema",
+			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
+			language:    "dotnet",
+		},
+		{
+			name:        "Generates go schema",
+			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
+			language:    "go",
+		},
+		{
+			name:        "Generates yaml schema",
+			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
+			language:    "yaml",
+		},
+		{
+			name:        "Generates java schema",
+			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema.json")),
+			language:    "java",
+		},
+		{
+			name:        "Handles property names that are not surrounded by back ticks",
+			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema-no-backticks.json")),
+			language:    "nodejs",
+		},
+		{
+			name:        "Handles property names that are surrounded by back ticks AND double quotes",
+			inputSchema: []byte(readfile(t, "testdata/TestFilterSchemaByLanguage/schema-backticks-and-quotes.json")),
+			language:    "nodejs",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			actual := FilterSchemaByLanguage(tc.inputSchema, tc.language)
+			hasSpan := bytes.Contains(actual, []byte("span"))
+			require.False(t, hasSpan, "there should be no spans in the filtered schema")
+			hasCodeChoosers := bytes.Contains(actual, []byte("PulumiCodeChooser"))
+			require.False(t, hasCodeChoosers)
+
+			autogold.ExpectFile(t, autogold.Raw(actual))
+		})
+	}
+}
+
+func readfile(t *testing.T, file string) string {
+	t.Helper()
+	bytes, err := os.ReadFile(file)
+	require.NoError(t, err)
+	return string(bytes)
+}