From 7b4eb2963f9b1ec75e64d5899381807b861c082b Mon Sep 17 00:00:00 2001 From: David Wessman Date: Sat, 14 Sep 2024 12:49:12 +0200 Subject: [PATCH] Google Translator: Improves newline handling - The newline replacement only worked when extra spaces were added by Google Translate API, which is not always happening. - Adds some more tests to catch this and changes the replacement to be closer to replacing of interpolations. - Fixes #595 This is a frustating bug that has been open for many years: https://issuetracker.google.com/issues/119256504?pli=1 Maybe one of the other APIs are more consistent. --- .../tasks/translators/google_translator.rb | 10 +++- spec/google_translate_spec.rb | 59 ++++++++++++------- 2 files changed, 46 insertions(+), 23 deletions(-) diff --git a/lib/i18n/tasks/translators/google_translator.rb b/lib/i18n/tasks/translators/google_translator.rb index 0890e922..686544ef 100644 --- a/lib/i18n/tasks/translators/google_translator.rb +++ b/lib/i18n/tasks/translators/google_translator.rb @@ -21,7 +21,7 @@ def translate_values(list, **options) EasyTranslate.translate( replace_newlines_with_placeholder(list, options[:html]), options, - format: :text + format: options[:html] ? :html : :text ), options[:html] ) @@ -69,7 +69,9 @@ def replace_newlines_with_placeholder(list, html) return list unless html list.map do |value| - value.gsub("\n", NEWLINE_PLACEHOLDER) + value.gsub(/\n(\s*)/) do + "" + end end end @@ -77,7 +79,9 @@ def restore_newlines(translations, html) return translations unless html translations.map do |translation| - translation.gsub("#{NEWLINE_PLACEHOLDER} ", "\n") + translation.gsub(//) do + "\n#{' ' * ::Regexp.last_match(1).to_i}" + end end end end diff --git a/spec/google_translate_spec.rb b/spec/google_translate_spec.rb index 4ecb867e..e06c43bd 100644 --- a/spec/google_translate_spec.rb +++ b/spec/google_translate_spec.rb @@ -7,10 +7,26 @@ nil_value_test = ['nil-value-key', nil, nil] empty_value_test = ['empty-value-key', '', ''] text_test = ['hello', "Hello, %{user} O'Neill!", "¡Hola, %{user} O'Neill!"] - text_test_multiline = ['hello_multiline', "Hello,\n%{user}\nO'Neill!", "Hola,\n%{user}\n¡O'Neill!"] + text_test_multiline = [ + 'hello_multiline', + "Hello,\n%{user}\nO'Neill!", + "Hola,\n%{user}\nO'Neill!" + ] html_test = ['html-key.html', "Hello, %{user} O'neill", "Hola, %{user} O'neill"] html_test_plrl = ['html-key.html.one', 'Hello %{count}', 'Hola %{count}'] - html_test_multiline = ['html-key.html.multiline', "Hello\n%{user}", "Hola\n%{user}"] + html_test_multiline = [ + 'html-key.html.multiline_html', + "Hello\n%{user}", + "Hola\n %{user}" + ] + # Google Translate API adds extra spaces before some characters + # https://issuetracker.google.com/issues/119256504?pli=1 + # Atleast it should be valid HTML + html_test_multiline_indentation = [ + 'html-key.html.multiline_indentation_html', + "

Hello

\n
    \n
  • %{user}
    • \n
  • \n %{user2}\n
    • \n
  • Dog
    • \n
      \n", + "

      Hola

      \n
        \n
      • %{user}
        • \n
      • \n %{user2}\n
        • \n
      • Perro
        • \n
          \n" + ] array_test = ['array-key', ['Hello.', nil, '', 'Goodbye.'], ['Hola.', nil, '', 'Adiós.']] fixnum_test = ['numeric-key', 1, 1] ref_key_test = ['ref-key', :reference, :reference] @@ -30,27 +46,29 @@ let(:task) { i18n_task } it 'works' do # rubocop:disable RSpec/MultipleExpectations - skip 'temporarily disabled on JRuby due to https://github.com/jruby/jruby/issues/4802' if RUBY_ENGINE == 'jruby' skip 'GOOGLE_TRANSLATE_API_KEY env var not set' unless ENV['GOOGLE_TRANSLATE_API_KEY'] skip 'GOOGLE_TRANSLATE_API_KEY env var is empty' if ENV['GOOGLE_TRANSLATE_API_KEY'].empty? in_test_app_dir do - task.data[:en] = build_tree('en' => { - 'common' => { - 'a' => 'λ', - 'hello' => text_test[1], - 'hello_multiline' => text_test_multiline[1], - 'hello_html' => html_test[1], - 'hello_plural_html' => { - 'one' => html_test_plrl[1] - }, - 'hello_multiline_html' => html_test_multiline[1], - 'array_key' => array_test[1], - 'nil-value-key' => nil_value_test[1], - 'empty-value-key' => empty_value_test[1], - 'fixnum-key' => fixnum_test[1], - 'ref-key' => ref_key_test[1] - } - }) + task.data[:en] = build_tree( + 'en' => { + 'common' => { + 'a' => 'λ', + 'hello' => text_test[1], + 'hello_multiline' => text_test_multiline[1], + 'hello_html' => html_test[1], + 'hello_plural_html' => { + 'one' => html_test_plrl[1] + }, + 'hello_multiline_html' => html_test_multiline[1], + 'multiline_indentation_html' => html_test_multiline_indentation[1], + 'array_key' => array_test[1], + 'nil-value-key' => nil_value_test[1], + 'empty-value-key' => empty_value_test[1], + 'fixnum-key' => fixnum_test[1], + 'ref-key' => ref_key_test[1] + } + } + ) task.data[:es] = build_tree('es' => { 'common' => { 'a' => 'λ' @@ -63,6 +81,7 @@ expect(task.t('common.hello_html', 'es')).to eq(html_test[2]) expect(task.t('common.hello_plural_html.one', 'es')).to eq(html_test_plrl[2]) expect(task.t('common.hello_multiline_html', 'es')).to eq(html_test_multiline[2]) + expect(task.t('common.multiline_indentation_html', 'es')).to eq(html_test_multiline_indentation[2]) expect(task.t('common.array_key', 'es')).to eq(array_test[2]) expect(task.t('common.nil-value-key', 'es')).to eq(nil_value_test[2]) expect(task.t('common.empty-value-key', 'es')).to eq(empty_value_test[2])