Merge pull request #3390 from Earlopain/parser-translator-binary-enco…

…ding

Fix binary encoding for parser translator

lib/prism/translation/parser.rb

@@ -51,7 +51,7 @@ def parse(source_buffer)
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
+        result = unwrap(Prism.parse(source, **prism_options), offset_cache)
 
         build_ast(result.value, offset_cache)
       ensure
@@ -64,7 +64,7 @@ def parse_with_comments(source_buffer)
         source = source_buffer.source
 
         offset_cache = build_offset_cache(source)
-        result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
+        result = unwrap(Prism.parse(source, **prism_options), offset_cache)
 
         [
           build_ast(result.value, offset_cache),
@@ -83,7 +83,7 @@ def tokenize(source_buffer, recover = false)
         offset_cache = build_offset_cache(source)
         result =
           begin
-            unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), partial_script: true, encoding: false), offset_cache)
+            unwrap(Prism.parse_lex(source, **prism_options), offset_cache)
           rescue ::Parser::SyntaxError
             raise if !recover
           end
@@ -285,6 +285,20 @@ def build_range(location, offset_cache)
         )
       end
 
+      # Options for how prism should parse/lex the source.
+      def prism_options
+        options = {
+          filepath: @source_buffer.name,
+          version: convert_for_prism(version),
+          partial_script: true,
+        }
+        # The parser gem always encodes to UTF-8, unless it is binary.
+        # https://github.com/whitequark/parser/blob/v3.3.6.0/lib/parser/source/buffer.rb#L80-L107
+        options[:encoding] = false if @source_buffer.source.encoding != Encoding::BINARY
+
+        options
+      end
+
       # Converts the version format handled by Parser to the format handled by Prism.
       def convert_for_prism(version)
         case version

test/prism/fixtures/encoding_binary.txt

@@ -0,0 +1,9 @@
+# encoding: binary
+
+"\xcd"
+
+:"\xcd"
+
+/#{"\xcd"}/
+
+%W[\xC0]

test/prism/fixtures/encoding_euc_jp.txt

@@ -0,0 +1,6 @@
+# encoding: euc-jp
+
+# \x8E indicates a double-byte character, \x01 is not a valid second byte in euc-jp
+"\x8E\x01"
+
+%W["\x8E\x01"]

test/prism/ruby/parser_test.rb

@@ -17,6 +17,18 @@
 # First, opt in to every AST feature.
 Parser::Builders::Default.modernize
 
+# The parser gem rejects some strings that would most likely lead to errors
+# in consumers due to encoding problems. RuboCop however monkey-patches this
+# method out in order to accept such code.
+# https://github.com/whitequark/parser/blob/v3.3.6.0/lib/parser/builders/default.rb#L2289-L2295
+Parser::Builders::Default.prepend(
+  Module.new {
+    def string_value(token)
+      value(token)
+    end
+  }
+)
+
 # Modify the source map == check so that it doesn't check against the node
 # itself so we don't get into a recursive loop.
 Parser::Source::Map.prepend(

test/prism/ruby/ruby_parser_test.rb

@@ -26,6 +26,7 @@ def ==(other)
 module Prism
   class RubyParserTest < TestCase
     todos = [
+      "encoding_euc_jp.txt",
       "newline_terminated.txt",
       "regex_char_width.txt",
       "seattlerb/bug169.txt",

test/prism/snippets_test.rb

@@ -5,6 +5,7 @@
 module Prism
   class SnippetsTest < TestCase
     except = [
+      "encoding_binary.txt",
       "newline_terminated.txt",
       "seattlerb/begin_rescue_else_ensure_no_bodies.txt",
       "seattlerb/case_in.txt",