diff options
| author | Kevin Newton <[email protected]> | 2024-03-04 09:27:18 -0500 |
|---|---|---|
| committer | git <[email protected]> | 2024-03-04 14:39:52 +0000 |
| commit | 5856ea3fd16437279cb0a2e137e4a2b58fc655cb () | |
| tree | f2da33982d983ea8a616fbf8569bf72beed725f0 /lib/prism | |
| parent | 2c787bf90ff4176970f6dea331401e8246fdde95 (diff) | |
[ruby/prism] Fix up some minor parser incompatibilities
https://.com/ruby/prism/commit/c6c771d1fa
| -rw-r--r-- | lib/prism/translation.rb | 2 | ||||
| -rw-r--r-- | lib/prism/translation/parser.rb | 2 | ||||
| -rw-r--r-- | lib/prism/translation/parser/lexer.rb | 31 | ||||
| -rw-r--r-- | lib/prism/translation/parser33.rb | 2 | ||||
| -rw-r--r-- | lib/prism/translation/parser34.rb | 2 |
5 files changed, 27 insertions, 12 deletions
@@ -5,6 +5,8 @@ module Prism # syntax trees. module Translation # steep:ignore autoload :Parser, "prism/translation/parser" autoload :Ripper, "prism/translation/ripper" autoload :RubyParser, "prism/translation/ruby_parser" end @@ -168,7 +168,7 @@ module Prism # Build the parser gem tokens from the prism tokens. def build_tokens(tokens, offset_cache) - Lexer.new(source_buffer, tokens.map(&:first), offset_cache).to_a end # Build a range from a prism location. @@ -177,12 +177,23 @@ module Prism WORDS_SEP: :tSPACE } - private_constant :TYPES # The Parser::Source::Buffer that the tokens were lexed from. attr_reader :source_buffer - # An array of prism tokens that we lexed. attr_reader :lexed # A hash that maps offsets in bytes to offsets in characters. @@ -205,9 +216,9 @@ module Prism index = 0 while index < lexed.length - token, = lexed[index] index += 1 - next if token.type == :IGNORED_NEWLINE || token.type == :EOF type = TYPES.fetch(token.type) value = token.value @@ -218,13 +229,13 @@ module Prism value.delete_prefix!("?") when :tCOMMENT if token.type == :EMBDOC_BEGIN - until (next_token = lexed[index]) && next_token.type == :EMBDOC_END value += next_token.value index += 1 end value += next_token.value - location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[lexed[index].location.end_offset]) index += 1 else value.chomp! @@ -247,6 +258,8 @@ module Prism value.chomp!(":") when :tLABEL_END value.chomp!(":") when :tNTH_REF value = Integer(value.delete_prefix("$")) when :tOP_ASGN @@ -256,13 +269,13 @@ module Prism when :tSPACE value = nil when :tSTRING_BEG - if ["\"", "'"].include?(value) && (next_token = lexed[index]) && next_token.type == :STRING_END next_location = token.location.join(next_token.location) type = :tSTRING value = "" location = Range.new(source_buffer, offset_cache[next_location.start_offset], offset_cache[next_location.end_offset]) index += 1 - elsif ["\"", "'"].include?(value) && (next_token = lexed[index]) && next_token.type == :STRING_CONTENT && (next_next_token = lexed[index + 1]) && next_next_token.type == :STRING_END next_location = token.location.join(next_next_token.location) type = :tSTRING value = next_token.value @@ -280,7 +293,7 @@ module Prism location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.start_offset + 1]) end when :tSYMBEG - if (next_token = lexed[index]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR next_location = token.location.join(next_token.location) type = :tSYMBOL value = next_token.value @@ -1,4 +1,4 @@ -require_relative "parser" module Prism module Translation @@ -1,4 +1,4 @@ -require_relative "parser" module Prism module Translation |