fix for issues with char duplication (#309)

* fix for issues with char duplication

* fixes and tests

Author: Zemurin

.github/workflows/linux_coverage.yml

@@ -56,7 +56,7 @@ jobs:
   analyze:
     name: Coverage Check Foreign
     if: github.repository_owner != 'ParadoxGameConverters'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false

Parser.cpp

@@ -270,12 +270,9 @@ std::string commonItems::getNextLexeme(std::istream& theStream)
 	auto inLiteralQuote = false;
 	unsigned char previousCharacter = '\0';
 
-	while (true)
+	char inputChar;
+	while (theStream >> inputChar)
 	{
-		char inputChar;
-		theStream >> inputChar;
-		if (theStream.eof())
-			break;
 		if (!inQuotes && inputChar == '#')
 		{
 			std::string bitBucket;

ParserHelpers.cpp

@@ -451,19 +451,13 @@ stringOfItem::stringOfItem(std::istream& theStream)
 	{
 		bool inQuotes = false;
 		auto braceDepth = 1;
-		while (true)
+		unsigned char previousCharacter = '\0';
+		char inputChar;
+		while (theStream >> inputChar)
 		{
-			if (theStream.eof())
-			{
-				return;
-			}
-
-			char inputChar;
-			theStream >> inputChar;
-
 			theString += inputChar;
 
-			if (inputChar == '\"')
+			if (inputChar == '\"' && previousCharacter != '\\')
 			{
 				if (!inQuotes)
 				{
@@ -486,6 +480,7 @@ stringOfItem::stringOfItem(std::istream& theStream)
 					return;
 				}
 			}
+			previousCharacter = inputChar;
 		}
 	}
 }

tests/ParserHelperTests.cpp

@@ -52,6 +52,28 @@ TEST(ParserHelper_Tests, IgnoreItemIgnoresAssignedBracedItem)
 	ASSERT_EQ(" More text", std::string{buffer});
 }
 
+TEST(ParserHelper_Tests, IgnoreItemIgnoresMisquotedItem)
+{
+	std::stringstream input{"= \\\"ignore_me\" More text"};
+	input >> std::noskipws;
+	commonItems::ignoreItem("unused", input);
+
+	char buffer[256];
+	input.getline(buffer, sizeof buffer);
+	ASSERT_EQ("More text", std::string{buffer});
+}
+
+TEST(ParserHelper_Tests, IgnoreItemIgnoresMisquotedBracedItem)
+{
+	std::stringstream input{"= { \\\"ignore_me\" } More text"};
+	input >> std::noskipws;
+	commonItems::ignoreItem("unused", input);
+
+	char buffer[256];
+	input.getline(buffer, sizeof buffer);
+	ASSERT_EQ(" More text", std::string{buffer});
+}
+
 TEST(ParserHelper_Tests, IgnoreItemIgnoresAssignedBracedItemOnExistsEquals)
 {
 	std::stringstream input{"?= { { ignore_me } } More text"};
@@ -867,6 +889,21 @@ TEST(ParserHelper_Tests, StringOfItemGetsStringAfterEquals)
 }
 
 
+TEST(ParserHelper_Tests, StringOfItemHandlesMismatchedQuotes)
+{
+	std::stringstream input;
+	input >> std::noskipws;
+	input << "= {\n";
+	input << "\tfoo = \"some junk\\\"\" \n";
+	input << "\tbar = baz\n";
+	input << "}";
+
+	const commonItems::stringOfItem theItem(input);
+
+	ASSERT_EQ(input.str(), theItem.getString());
+}
+
+
 TEST(ParserHelper_Tests, StringOfItemsConvertsItemsWithinBracesToStrings)
 {
 	std::stringstream input;

tests/ParserTests.cpp

@@ -387,6 +387,29 @@ TEST(Parser_Tests, QuotedRegexesAreQuotedlyMatched)
 	EXPECT_EQ("value", test.value);
 }
 
+TEST(Parser_Tests, MisQuotedRegexesAreQuotedlyMatched)
+{
+	std::stringstream input{R"("\"key" = value)"};
+	class Test: commonItems::parser
+	{
+	  public:
+		explicit Test(std::istream& stream)
+		{
+			registerRegex(R"([k\"\\ey]+)", [this](const std::string& keyword, std::istream& theStream) {
+				key = keyword;
+				value = commonItems::getString(theStream);
+			});
+			parseStream(stream);
+		}
+		std::string key;
+		std::string value;
+	};
+	const auto test = Test(input);
+
+	EXPECT_EQ("\"\\\"key\"", test.key);
+	EXPECT_EQ("value", test.value);
+}
+
 TEST(Parser_Tests, CatchAllCatchesQuotedKeys)
 {
 	std::stringstream input{"\"key\" = value"};