Adds multiple single line delimiters to assembly file + updates file types

-Adds multiple delimiters for single line comments for assembly files
- Adds a more assembly file types to Lang selector dict
- Adds testing to assert behaviour change is correct
Signed-off-by: McGuire, Kevin <[email protected]>

Author: sbomming-possibly

comment_filter/language.py

@@ -3,7 +3,11 @@
 
 class Lang:
     def __init__(self, line_comment, comment_bookends, nested_comments):
-        self.line_comment = line_comment
+        # Support both single string and list of strings for line_comment
+        if isinstance(line_comment, str):
+            self.line_comment = [line_comment]
+        else:
+            self.line_comment = line_comment
         self.comment_bookends = comment_bookends
         self.nested_comments = nested_comments
         self.string_literal_start = '"'
@@ -15,7 +19,7 @@ def __init__(self, line_comment, comment_bookends, nested_comments):
     nested_comments=False)
 
 assembly = Lang(
-    line_comment=';',
+    line_comment=[';', '#'], 
     comment_bookends=[('/*', '*/')],
     nested_comments=False)
 
@@ -62,7 +66,9 @@ def __init__(self, line_comment, comment_bookends, nested_comments):
     '.cxx': c,
     '.cpp': c,
     '.h': c,
-    '.S': c,
+    '.S': assembly,
+    '.asm': assembly,
+    '.nasm': assembly,
     '.java': java,
     '.go': go,
     '.hs': haskell,

comment_filter/rfc.py

@@ -160,15 +160,16 @@ def parse_code(lang, state):
     while True:
         line = state.line
         multi_start_tokens = [start for start, end in lang.comment_bookends]
-        tokens = multi_start_tokens + [
-            lang.line_comment,
+        tokens = multi_start_tokens + lang.line_comment + [
             lang.string_literal_start,
             lang.string_literal2_start]
         i = index_of_first_found(line, tokens)
         if i != -1:
             state.line = line[i:]
             code += line[:i]
-            if line.startswith(lang.line_comment, i) or \
+            # Check if any line comment delimiter starts at position i
+            line_comment_found = line.startswith(tuple(lang.line_comment), i)
+            if line_comment_found or \
                     index_of_first_found(line, multi_start_tokens) == i:
                 return code, state
             elif line.startswith(lang.string_literal_start, i):
@@ -274,15 +275,15 @@ def parse_line_comment(lang, state, keep_tokens=True):
       (string, State)
     """
     line = state.line
-    line_comment = lang.line_comment
-    if line.startswith(line_comment):
-        state.line = ''
-        i = len(line_comment)
-        if not keep_tokens:
-            line_comment = ' ' * i
-        return line_comment + line[i:], state
-    else:
-        return '', state
+    # Check each possible line comment delimiter
+    for line_comment in lang.line_comment:
+        if line.startswith(line_comment):
+            state.line = ''
+            i = len(line_comment)
+            if not keep_tokens:
+                line_comment = ' ' * i
+            return line_comment + line[i:], state
+    return '', state
 
 
 def parse_multiline_comment(lang, state, keep_tokens=True):

comment_filter/rfc_test.py

@@ -263,6 +263,18 @@ def test_parse_file():
     assert assembly_comments('/* line1\nline2\n*/line3\n') == ['/* line1\n', 'line2\n', '*/     \n']
 
 
+def test_assembly_hash_comments():
+    """Test that # works as line comment delimiter for assembly"""
+    assert assembly_comments('# comment\ncode\n') == ['# comment\n', '    \n']
+    assert assembly_comments('# hash comment\n') == ['# hash comment\n']
+
+
+def test_assembly_mixed_comments():
+    """Test that both ; and # work together in assembly"""
+    assert assembly_comments('; semicolon comment\n# hash comment\ncode\n') == ['; semicolon comment\n', '# hash comment\n', '    \n']
+    assert assembly_comments('code ; inline semicolon\ncode # inline hash\n') == ['     ; inline semicolon\n', '     # inline hash\n']
+
+
 def test_parse_comments_via_reduce():
     def f(st, x):
         st.line = x