Allow Unicode characters in Selectors (#510)

* Allow Unicode characters in `Selector`s

Resolves #454

* Refactor internal `ParserSettings` to use static or const members

Refactored internal `ParserSettings` to convert instance-level properties and methods to static or const members.

* feat: Filter `Selector` chars by allowlist or blocklilst (#511)

* Change enum `FilterType` to `SelectorFilterType`

Implement proposals from review:

* SelectorFilterType.Alphanumeric: alphanumeric characters (upper and lower case), plus '_' and '-'
* SelectorFilterType.VisualUnicodeChars: All Unicode characters are allowed in a selector, except 68 non-visual characters: Control Characters (U+0000–U+001F, U+007F), Format Characters (Category: Cf), Directional Formatting (Category: Cf), Invisible Separator, Common Combining Marks (Category: Mn), Whitespace Characters (non-glyph spacing).

* Make `NonVisualUnicodeCharacters` read-only

Author: axunonb

src/SmartFormat.Tests/Core/CharSetTests.cs

@@ -0,0 +1,54 @@
+using System;
+using System.Linq;
+using NUnit.Framework;
+using SmartFormat.Core.Parsing;
+
+namespace SmartFormat.Tests.Core;
+
+[TestFixture]
+internal class CharSetTests
+{
+    [Test]
+    public void CharSet_Add_Remove()
+    {
+        char[] asciiChars = ['A', 'B', 'C'];
+        char[] nonAsciiChars = ['Ā', 'Б', '中'];
+        var charSet = new CharSet();
+        charSet.AddRange(asciiChars.AsEnumerable());
+        charSet.AddRange(nonAsciiChars.AsSpan());
+        var countBeforeRemoval = charSet.Count;
+        var existingRemoved = charSet.Remove('C');
+        charSet.Remove('中');
+        // trying to remove a not existing char returns false
+        var nonExistingRemoved = charSet.Remove('?');
+        var count = charSet.Count;
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(countBeforeRemoval, Is.EqualTo(asciiChars.Length + nonAsciiChars.Length));
+            Assert.That(count, Is.EqualTo(countBeforeRemoval - 2));
+            Assert.That(existingRemoved, Is.True);
+            Assert.That(nonExistingRemoved, Is.False);   
+        });
+    }
+
+    [Test]
+    public void CharSet_CreateFromSpan_GetCharacters_Contains()
+    {
+        char[] asciiAndNonAscii = ['\0', 'A', 'B', 'C', 'Ā', 'Б', '中'];
+        var charSet = new CharSet(asciiAndNonAscii.AsSpan());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(charSet, Has.Count.EqualTo(7));
+            Assert.That(charSet.Contains('A'), Is.True); // ASCII
+            Assert.That(charSet.Contains('\0'), Is.True); // control character
+            Assert.That(charSet.Contains('中'), Is.True); // non-ASCII
+            Assert.That(charSet.Contains('?'), Is.False);
+            Assert.That(charSet.GetCharacters(), Is.EquivalentTo(asciiAndNonAscii));
+            charSet.Clear();
+            Assert.That(charSet, Has.Count.EqualTo(0));
+            Assert.That(charSet.GetCharacters(), Is.Empty);
+        });
+    }
+}

src/SmartFormat.Tests/Core/ParserTests.cs

@@ -1,11 +1,12 @@
-using NUnit.Framework;
-using SmartFormat.Core.Parsing;
-using SmartFormat.Core.Settings;
-using SmartFormat.Tests.TestUtils;
-using System;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
+using NUnit.Framework;
+using SmartFormat.Core.Parsing;
+using SmartFormat.Core.Settings;
+using SmartFormat.Tests.TestUtils;
 
 namespace SmartFormat.Tests.Core;
 
@@ -66,9 +67,9 @@ public void Parser_Throws_Exceptions(string format)
         Assert.Throws<ParsingErrors>(() => formatter.Test(format, args, "Error"));
     }
 
-    [TestCase("{V(LU)}")] // braces are illegal
-    [TestCase("{V LU }")] // blanks are illegal
-    [TestCase("{VĀLUĒ}")] // 0x100 and 0x112 are illegal chars
+    [TestCase("{V(LU)}")] // braces are not allowed
+    [TestCase("{V LU\\}")] // escape char is not allowed
+    [TestCase("{V?LU,}")] // ? and , are allowed chars
     public void Parser_Throws_On_Illegal_Selector_Chars(string format)
     {
         var parser = GetRegularParser();
@@ -81,9 +82,9 @@ public void Parser_Throws_On_Illegal_Selector_Chars(string format)
         {
             Assert.Multiple(() =>
             {
-                // Throws, because selector contains 2 illegal characters
+                // Throws, because selector contains disallowed characters
                 Assert.That(e, Is.InstanceOf<ParsingErrors>());
-                Assert.That(((ParsingErrors) e).Issues, Has.Count.EqualTo(2));
+                Assert.That(((ParsingErrors) e).Issues, Has.Count.GreaterThanOrEqualTo(1));
             });
         }
     }
@@ -154,6 +155,7 @@ public void Parser_Error_Action_Ignore()
         //                     | Literal  | Erroneous     | | Okay |  
         var invalidTemplate = "Hello, I'm {Name from {City} {Street}";
 
+        // settings must be set before parser instantiation
         var parser = GetRegularParser(new SmartSettings {Parser = new ParserSettings {ErrorAction = ParseErrorAction.Ignore}});
         using var parsed = parser.ParseFormat(invalidTemplate);
 
@@ -176,6 +178,7 @@ public void Parser_Error_Action_Ignore()
     [TestCase("Hello, I'm {Name from {City} {Street", false)]
     public void Parser_Error_Action_MaintainTokens(string invalidTemplate, bool lastItemIsPlaceholder)
     {
+        // settings must be set before parser instantiation
         var parser = GetRegularParser(new SmartSettings {Parser = new ParserSettings {ErrorAction = ParseErrorAction.MaintainTokens}});
         using var parsed = parser.ParseFormat(invalidTemplate);
 
@@ -203,8 +206,16 @@ public void Parser_Error_Action_OutputErrorInResult()
     {
         //                     | Literal  | Erroneous     |
         var invalidTemplate = "Hello, I'm {Name from {City}";
-            
-        var parser = GetRegularParser(new SmartSettings {Parser = new ParserSettings {ErrorAction = ParseErrorAction.OutputErrorInResult}});
+
+        var parser = GetRegularParser(new SmartSettings
+        {
+            Parser = new ParserSettings
+                {
+                    SelectorCharFilter = SelectorFilterType.Alphanumeric, // default
+                    ErrorAction = ParseErrorAction.OutputErrorInResult
+                }
+        });
+
         using var parsed = parser.ParseFormat(invalidTemplate);
 
         Assert.That(parsed.Items, Has.Count.EqualTo(1));
@@ -412,11 +423,11 @@ public void Parser_NotifyParsingError()
         });
 
         formatter.Parser.OnParsingFailure += (o, args) => parsingError = args.Errors;
-        var res = formatter.Format("{NoName {Other} {Same", default(object)!);
+        var res = formatter.Format("{NoName {Other} {Same");
         Assert.Multiple(() =>
         {
             Assert.That(parsingError!.Issues, Has.Count.EqualTo(3));
-            Assert.That(parsingError.Issues[2].Issue, Is.EqualTo(new Parser.ParsingErrorText()[SmartFormat.Core.Parsing.Parser.ParsingError.MissingClosingBrace]));
+            Assert.That(parsingError.Issues[2].Issue, Is.EqualTo(new Parser.ParsingErrorText()[Parser.ParsingError.MissingClosingBrace]));
         });
     }
 
@@ -457,6 +468,18 @@ public void Escaping_TheEscapingCharacter_ShouldWork()
         Assert.That(result, Is.EqualTo(@"\\aaa\{}bbb ccc\x{}ddd\\"));
     }
 
+    [Test]
+    public void Parsing_Selector_With_CharFromBlocklist_ShouldThrow()
+    {
+        var settings = new SmartSettings { Parser = new ParserSettings { SelectorCharFilter = SelectorFilterType.VisualUnicodeChars } };
+        var parser = GetRegularParser(settings);
+
+        // The newline character is in the default blocklist of disallowed characters
+        Assert.That(() => parser.ParseFormat("{A\nB}"),
+            Throws.Exception.InstanceOf<ParsingErrors>().And.Message
+                .Contains(new Parser.ParsingErrorText()[Parser.ParsingError.InvalidCharactersInSelector]));
+    }
+
     [Test]
     public void StringFormat_Escaping_In_Literal()
     {
@@ -534,8 +557,10 @@ public void Parse_Unicode(string formatString, string unicodeLiteral, int itemIn
     [TestCase("{%C}", '%')]
     public void Selector_With_Custom_Selector_Character(string formatString, char customChar)
     {
+        // settings must be set before parser instantiation
         var settings = new SmartSettings();
-        settings.Parser.AddCustomSelectorChars(new[]{customChar});
+        settings.Parser.AddCustomSelectorChars([customChar]);
+        var x = settings.Parser.GetSelectorChars();
         var parser = GetRegularParser(settings);
         var result = parser.ParseFormat(formatString);
 
@@ -544,7 +569,7 @@ public void Selector_With_Custom_Selector_Character(string formatString, char cu
         Assert.That(placeholder!.Selectors, Has.Count.EqualTo(1));
         Assert.Multiple(() =>
         {
-            Assert.That(placeholder!.Selectors, Has.Count.EqualTo(placeholder!.GetSelectors().Count));
+            Assert.That(placeholder.Selectors, Has.Count.EqualTo(placeholder.GetSelectors().Count));
             Assert.That(placeholder.Selectors[0].ToString(), Is.EqualTo(formatString.Substring(1, 2)));
         });
     }
@@ -553,8 +578,10 @@ public void Selector_With_Custom_Selector_Character(string formatString, char cu
     [TestCase("{a°b}", '°')]
     public void Selectors_With_Custom_Operator_Character(string formatString, char customChar)
     {
-        var parser = GetRegularParser();
-        parser.Settings.Parser.AddCustomOperatorChars(new[]{customChar});
+        // settings must be set before parser instantiation
+        var settings = new SmartSettings();
+        settings.Parser.AddCustomOperatorChars([customChar]);
+        var parser = GetRegularParser(settings);
         var result = parser.ParseFormat(formatString);
 
         var placeholder = result.Items[0] as Placeholder;
@@ -568,6 +595,31 @@ public void Selectors_With_Custom_Operator_Character(string formatString, char c
         });
     }
 
+    [TestCase("German |öäüßÖÄÜ!")]
+    [TestCase("Russian абвгдеёжзийклмн")]
+    [TestCase("French >éèêëçàùâîô")]
+    [TestCase("Spanish <áéíóúñü¡¿")]
+    [TestCase("Portuguese !ãõáâêéíóúç")]
+    [TestCase("Chinese 汉字测试")]
+    [TestCase("Arabic مرحبا بالعالم")]
+    [TestCase("Turkish çğöşüİı")]
+    [TestCase("Hindi नमस्ते दुनिया")]
+    public void Selector_WorksWithAllUnicodeChars(string selector)
+    {
+        // See https://github.com/axuno/SmartFormat/issues/454
+
+        // settings must be set before parser instantiation
+        var settings = new SmartSettings { Parser = { SelectorCharFilter = SelectorFilterType.VisualUnicodeChars } };
+        const string expected = "The Value";
+        // The default formatter with default settings should be able to handle any
+        // Unicode characters in selectors except the "magic" disallowed ones
+        var formatter = Smart.CreateDefaultSmartFormat(settings);
+        // Use the Unicode string as a selector of the placeholder
+        var template = $"{{{selector}}}";
+        var result = formatter.Format(template, new Dictionary<string, string> { { selector, expected } });
+        Assert.That(result, Is.EqualTo(expected));
+    }
+
     [TestCase("{A?.B}")]
     [TestCase("{Selector0?.Selector1}")]
     [TestCase("{A?[1].B}")]
@@ -622,10 +674,11 @@ public void Selector_With_Nullable_Operator_Character(string formatString)
     public void Selector_With_Other_Contiguous_Operator_Characters(string formatString, char customChar)
     {
         // contiguous operator characters are parsed as "ONE operator string"
-
-        var parser = GetRegularParser();
+        var settings = new SmartSettings();
+        settings.Parser.AddCustomOperatorChars([customChar]);
+        var parser = GetRegularParser(settings);
         // adding '.' is ignored, as it's a standard operator
-        parser.Settings.Parser.AddCustomOperatorChars(new[]{customChar});
+        parser.Settings.Parser.AddCustomOperatorChars([customChar]);
         var result = parser.ParseFormat(formatString);
 
         var placeholder = result.Items[0] as Placeholder;
@@ -681,6 +734,41 @@ public void ParseInputAsHtml(string input)
         Assert.That(literalText!.RawText, Is.EqualTo(input));
     }
 
+    #region * Parse HTML input without ParserSetting 'IsHtml'
+
+    /// <summary>
+    /// <see cref="ParserSettings.SelectorCharFilter"/> is <see cref="FilterType.Blocklist"/>:
+    /// all characters are allowed in selectors
+    /// </summary>
+    [TestCase("<script>{Placeholder}</script>", "{Placeholder}")]
+    [TestCase("<style>{Placeholder}</style>", "{Placeholder}")]
+    [TestCase("Something <style>h1 { color : #000; }</style>! nice", "{ color : #000; }")]
+    [TestCase("Something <script>{const a = '</script>';}</script>! nice", "{const a = '</script>';}")]
+    public void ParseHtmlInput_Without_ParserSetting_IsHtml(string input, string selector)
+    {
+        var parser = GetRegularParser(new SmartSettings
+        {
+            StringFormatCompatibility = false,
+            Parser = new ParserSettings
+            {
+                SelectorCharFilter = SelectorFilterType.VisualUnicodeChars,
+                ErrorAction = ParseErrorAction.ThrowError,
+                ParseInputAsHtml = false
+            }
+        });
+
+        var result = parser.ParseFormat(input);
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Items, Has.Count.EqualTo(3));
+            Assert.That(((Placeholder) result.Items[1]).RawText, Is.EqualTo(selector));
+        });
+    }
+
+    /// <summary>
+    /// <see cref="ParserSettings.SelectorCharFilter"/> is <see cref="FilterType.Allowlist"/>:
+    /// Predefined set of allowed characters in selectors
+    /// </summary>
     [TestCase("<script>{Placeholder}</script>", false)] // should parse a placeholder
     [TestCase("<style>{Placeholder}</style>", false)] // should parse a placeholder
     [TestCase("Something <style>h1 { color : #000; }</style>! nice", true)] // illegal selector chars
@@ -690,7 +778,12 @@ public void ParseHtmlInput_Without_ParserSetting_IsHtml(string input, bool shoul
         var parser = GetRegularParser(new SmartSettings
         {
             StringFormatCompatibility = false,
-            Parser = new ParserSettings { ErrorAction = ParseErrorAction.ThrowError, ParseInputAsHtml = false }
+            Parser = new ParserSettings
+            {
+                SelectorCharFilter = SelectorFilterType.Alphanumeric,
+                ErrorAction = ParseErrorAction.ThrowError,
+                ParseInputAsHtml = false
+            }
         });
 
         switch (shouldThrow)
@@ -707,6 +800,8 @@ public void ParseHtmlInput_Without_ParserSetting_IsHtml(string input, bool shoul
         }
     }
 
+    #endregion
+
     /// <summary>
     /// SmartFormat is able to parse script tags, if <see cref="ParserSettings.ParseInputAsHtml"/> is <see langword="true"/>
     /// </summary>
@@ -807,29 +902,31 @@ function interpolationSearch(sortedArray, seekIndex) {
     [TestCase(true, false)]
     public void StyleTags_Can_Be_Parsed_Without_Failure(bool inputIsHtml, bool shouldFail)
     {
-        var styles = @"
-<style type='text/css'>
-.media {
-  display: grid;
-  grid-template-columns: 1fr 3fr;
-}
+        var styles = """
 
-.media .content {
-  font-size: .8rem;
-}
+                     <style type='text/css'>
+                     .media {
+                       display: grid;
+                       grid-template-columns: 1fr 3fr;
+                     }
 
-.comment img {
-  border: 1px solid grey;
-  anything: 'xyz'
-}
+                     .media .content {
+                       font-size: .8rem;
+                     }
 
-.list-item {
-  border-bottom: 1px solid grey;
-}
-/* Comment: { which mixes up the parser without ParserSettings.ParseInputAsHtml = true */
-</style>
-<p>############### {TheVariable} ###############</p>
-";
+                     .comment img {
+                       border: 1px solid grey;
+                       anything: 'xyz'
+                     }
+
+                     .list-item {
+                       border-bottom: 1px solid grey;
+                     }
+                     /* Comment: { which mixes up the parser without ParserSettings.ParseInputAsHtml = true */
+                     </style>
+                     <p>############### {TheVariable} ###############</p>
+
+                     """;
         var parsingFailures = 0;
         var parser = GetRegularParser(new SmartSettings
         {