5 files changed, 27 insertions, 12 deletions
@@ -5,6 +5,8 @@ module Prism
 # syntax trees.
 module Translation # steep:ignore
 autoload :Parser, "prism/translation/parser"
 autoload :Ripper, "prism/translation/ripper"
 autoload :RubyParser, "prism/translation/ruby_parser"
 end
@@ -168,7 +168,7 @@ module Prism
 # Build the parser gem tokens from the prism tokens.
 def build_tokens(tokens, offset_cache)
- Lexer.new(source_buffer, tokens.map(&:first), offset_cache).to_a
 end
 # Build a range from a prism location.
@@ -177,12 +177,23 @@ module Prism
 WORDS_SEP: :tSPACE
 }
- private_constant :TYPES
 # The Parser::Source::Buffer that the tokens were lexed from.
 attr_reader :source_buffer
- # An array of prism tokens that we lexed.
 attr_reader :lexed
 # A hash that maps offsets in bytes to offsets in characters.
@@ -205,9 +216,9 @@ module Prism
 index = 0
 while index < lexed.length
- token, = lexed[index]
 index += 1
- next if token.type == :IGNORED_NEWLINE || token.type == :EOF
 type = TYPES.fetch(token.type)
 value = token.value
@@ -218,13 +229,13 @@ module Prism
 value.delete_prefix!("?")
 when :tCOMMENT
 if token.type == :EMBDOC_BEGIN
- until (next_token = lexed[index]) && next_token.type == :EMBDOC_END
 value += next_token.value
 index += 1
 end
 value += next_token.value
- location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[lexed[index].location.end_offset])
 index += 1
 else
 value.chomp!
@@ -247,6 +258,8 @@ module Prism
 value.chomp!(":")
 when :tLABEL_END
 value.chomp!(":")
 when :tNTH_REF
 value = Integer(value.delete_prefix("$"))
 when :tOP_ASGN
@@ -256,13 +269,13 @@ module Prism
 when :tSPACE
 value = nil
 when :tSTRING_BEG
- if ["\"", "'"].include?(value) && (next_token = lexed[index]) && next_token.type == :STRING_END
 next_location = token.location.join(next_token.location)
 type = :tSTRING
 value = ""
 location = Range.new(source_buffer, offset_cache[next_location.start_offset], offset_cache[next_location.end_offset])
 index += 1
- elsif ["\"", "'"].include?(value) && (next_token = lexed[index]) && next_token.type == :STRING_CONTENT && (next_next_token = lexed[index + 1]) && next_next_token.type == :STRING_END
 next_location = token.location.join(next_next_token.location)
 type = :tSTRING
 value = next_token.value
@@ -280,7 +293,7 @@ module Prism
 location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.start_offset + 1])
 end
 when :tSYMBEG
- if (next_token = lexed[index]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR
 next_location = token.location.join(next_token.location)
 type = :tSYMBOL
 value = next_token.value
@@ -1,4 +1,4 @@
-require_relative "parser"
 module Prism
 module Translation
@@ -1,4 +1,4 @@
-require_relative "parser"
 module Prism
 module Translation