Accept single-line prologue declarations

lib/lrama/lexer.rb

@@ -159,7 +159,6 @@ def lex_c_code
       raise "end_symbol is not set" unless end_symbol
 
       end_symbol_pattern = Regexp.new(end_symbol)
-      escaped_end_symbol_pattern = Regexp.new(Regexp.escape(end_symbol))
       reset_first_position
 
       until @scanner.eos? do
@@ -180,17 +179,33 @@ def lex_c_code
         when @scanner.scan(/\n/)
           code << @scanner.matched
           newline
-        when @scanner.scan(/".*?"/)
-          code << %Q(#{@scanner.matched})
-          @line += @scanner.matched.count("\n")
-        when @scanner.scan(/'.*?'/)
-          code << %Q(#{@scanner.matched})
-        when @scanner.scan(/[^\"'\{\}\n]+/)
-          code << @scanner.matched
-        when @scanner.scan(escaped_end_symbol_pattern)
-          code << @scanner.matched
+        when (fragment = @scanner.scan(%r{/\*|//[^\n]*|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'}m))
+          if fragment == '/*'
+            comment = @scanner.scan_until(%r{\*/})
+            raise ParseError, location.generate_error_message("Unterminated comment") unless comment # steep:ignore UnknownConstant
+
+            fragment << comment
+          end
+          code << fragment
+          @line += fragment.count("\n")
+          if (last_newline = fragment.b.rindex("\n"))
+            @head = @scanner.pos - fragment.bytesize + last_newline + 1
+          end
         else
-          code << @scanner.getch
+          start = @scanner.pos
+          # Search for delimiters without retaining backtracking positions for ordinary C text.
+          if (length = @scanner.skip_until(%r{["'{}\n]|/[/*]|%\}}))
+            delimiter_size = @scanner.matched_size
+            @scanner.pos -= delimiter_size
+            if length > delimiter_size
+              code << @scanner.string.byteslice(start, length - delimiter_size)
+            else
+              code << @scanner.getch
+            end
+          else
+            code << @scanner.rest
+            @scanner.terminate
+          end
         end
       end
 

spec/lrama/lexer_spec.rb

@@ -431,6 +431,76 @@
     MSG
   end
 
+  [
+    ' int x; ',
+    ' int x = 5 % 2; ',
+    ' int x = 12 / 3 % 2;/* %} */',
+    ' int x = 12/3; const char *s = "%}"; ',
+    %q( const char *s = "%}"; char c = '%'; ),
+    %q( const char *s = "\" a%}"; char c = '\''; ),
+    ' /* %} */ int x; ',
+  ].each do |code|
+    it "lexes a single-line prologue containing #{code.inspect}" do
+      grammar_file = Lrama::Lexer::GrammarFile.new("prologue.y", "%{#{code}%}")
+      lexer = Lrama::Lexer.new(grammar_file)
+
+      expect(lexer.next_token).to eq(['%{', token_class::Token.new(s_value: '%{')])
+      lexer.status = :c_declaration
+      lexer.end_symbol = '%}'
+      token = lexer.next_token
+
+      expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: code)])
+      expect(token[1].location).to eq Lrama::Lexer::Location.new(grammar_file: grammar_file, first_line: 1, first_column: 2, last_line: 1, last_column: 2 + code.length)
+      lexer.status = :initial
+
+      expect(lexer.next_token).to eq(['%}', token_class::Token.new(s_value: '%}')])
+      expect(lexer.next_token).to be_nil
+    end
+  end
+
+  it 'preserves prologue terminators inside line and multiline comments' do
+    code = "\n// %}\n/*\n%} */ int x; "
+    grammar_file = Lrama::Lexer::GrammarFile.new("prologue.y", "%{#{code}%}")
+    lexer = Lrama::Lexer.new(grammar_file)
+
+    expect(lexer.next_token).to eq(['%{', token_class::Token.new(s_value: '%{')])
+    lexer.status = :c_declaration
+    lexer.end_symbol = '%}'
+    token = lexer.next_token
+
+    expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: code)])
+    expect(token[1].location).to eq Lrama::Lexer::Location.new(grammar_file: grammar_file, first_line: 1, first_column: 2, last_line: 4, last_column: 13)
+    lexer.status = :initial
+
+    expect(lexer.next_token).to eq(['%}', token_class::Token.new(s_value: '%}')])
+    expect(lexer.next_token).to be_nil
+  end
+
+  [
+    [%q( printf("\"}"); ), 1, 17],
+    [%q( printf("\"{"); ), 1, 17],
+    [%q( printf("\\\\"); { int x; } ), 1, 27],
+    [" printf(\"あ\\\n\\\"}\"); ", 2, 7],
+  ].each do |code, last_line, last_column|
+    it "lexes an action containing an escaped C string in #{code.inspect}" do
+      grammar_file = Lrama::Lexer::GrammarFile.new("escaped.y", "{#{code}};")
+      lexer = Lrama::Lexer.new(grammar_file)
+
+      expect(lexer.next_token).to eq(['{', token_class::Token.new(s_value: '{')])
+      lexer.status = :c_declaration
+      lexer.end_symbol = '}'
+      token = lexer.next_token
+
+      expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: code)])
+      expect(token[1].location).to eq Lrama::Lexer::Location.new(grammar_file: grammar_file, first_line: 1, first_column: 1, last_line: last_line, last_column: last_column)
+      lexer.status = :initial
+
+      expect(lexer.next_token).to eq(['}', token_class::Token.new(s_value: '}')])
+      expect(lexer.next_token).to eq([';', token_class::Token.new(s_value: ';')])
+      expect(lexer.next_token).to be_nil
+    end
+  end
+
   context 'unexpected_c_code.y' do
     it do
       grammar_file = Lrama::Lexer::GrammarFile.new("invalid.y", "@invalid")
@@ -462,6 +532,63 @@
     end
   end
 
+  ['%}', '}', '\Z'].each do |end_symbol|
+    ['/*', '/*/', '/* unterminated } %}', "/* unterminated\n/* still unterminated"].each do |code|
+      it "rejects an unterminated C comment #{code.inspect} before #{end_symbol.inspect}" do
+        grammar_file = Lrama::Lexer::GrammarFile.new("comment.y", code)
+        lexer = Lrama::Lexer.new(grammar_file)
+        lexer.status = :c_declaration
+        lexer.end_symbol = end_symbol
+
+        expect { lexer.next_token }.to raise_error(ParseError, /Unterminated comment/)
+      end
+    end
+  end
+
+  [
+    'value / 2 % 3',
+    '/**/value/2;/***/',
+    'value /= 2; value %= 3;',
+    'value / 2/* } */%3;',
+    "value/2// }\n%3;",
+    %q(value/2; const char *s = "/* } */";),
+    'value/',
+    'value%',
+    'value\Z',
+    '%}',
+    '/%}',
+    '"',
+    '"unterminated\\',
+  ].each do |code|
+    it "preserves C operators and comments in #{code.inspect}" do
+      grammar_file = Lrama::Lexer::GrammarFile.new("operators.y", code)
+      lexer = Lrama::Lexer.new(grammar_file)
+      lexer.status = :c_declaration
+      lexer.end_symbol = '\Z'
+
+      expect(lexer.next_token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: code)])
+    end
+  end
+
+  it 'tracks action locations after multibyte comments and operators' do
+    code = " value/2; /* あ\n } */value%3; "
+    grammar_file = Lrama::Lexer::GrammarFile.new("operators.y", "{#{code}};")
+    lexer = Lrama::Lexer.new(grammar_file)
+
+    expect(lexer.next_token).to eq(['{', token_class::Token.new(s_value: '{')])
+    lexer.status = :c_declaration
+    lexer.end_symbol = '}'
+    token = lexer.next_token
+
+    expect(token).to eq([:C_DECLARATION, token_class::UserCode.new(s_value: code)])
+    expect(token[1].location).to eq Lrama::Lexer::Location.new(grammar_file: grammar_file, first_line: 1, first_column: 1, last_line: 2, last_column: 14)
+    lexer.status = :initial
+
+    expect(lexer.next_token).to eq(['}', token_class::Token.new(s_value: '}')])
+    expect(lexer.next_token).to eq([';', token_class::Token.new(s_value: ';')])
+    expect(lexer.next_token).to be_nil
+  end
+
   it 'lex a trailing space with newline' do
     grammar_file = Lrama::Lexer::GrammarFile.new("trailing_space.y", " \n%require")
     lexer = Lrama::Lexer.new(grammar_file)