scm: replace invalid utf-8 sequences in comments instead of stripping on Ruby 1.8.

git-svn-id: svn+ssh://rubyforge.org/var/svn/redmine/trunk@5373 e93f8b46-1217-0410-a6f0-8f06a7374b81

scm: replace invalid utf-8 sequences in comments instead of stripping on Ruby 1.8.
git-svn-id: svn+ssh://rubyforge.org/var/svn/redmine/trunk@5373 e93f8b46-1217-0410-a6f0-8f06a7374b81
e19c56d6 · Toshi MARUYAMA · Eric Davis · 0adf1a87 · e19c56d6 · e19c56d6
Commit e19c56d6 authored Apr 09, 2011 by Toshi MARUYAMA Committed by Eric Davis Jun 03, 2011
Hide whitespace changes
Inline Side-by-side

Showing with 25 additions and 31 deletions

changeset.rb app/models/changeset.rb +11 -12

changeset_test.rb test/unit/changeset_test.rb +14 -19

No files found.
--- a/app/models/changeset.rb
+++ b/app/models/changeset.rb
@@ -245,8 +245,8 @@ class Changeset < ActiveRecord::Base
      str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
      return str
    end
+    enc = encoding.blank? ? "UTF-8" : encoding
    if str.respond_to?(:force_encoding)
-      enc = encoding.blank? ? "UTF-8" : encoding
      if enc != "UTF-8"
        str.force_encoding(enc)
        str = str.encode("UTF-8", :invalid => :replace,
@@ -259,19 +259,18 @@ class Changeset < ActiveRecord::Base
        end
      end
    else
-      unless encoding.blank? || encoding == 'UTF-8'
+      ic = Iconv.new('UTF-8', enc)
-        begin
+      txtar = ""
-          str = Iconv.conv('UTF-8', encoding, str)
-        rescue Iconv::Failure
-          # do nothing here
-        end
-      end
-      # removes invalid UTF8 sequences
      begin
-        str = Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + '  ')[0..-3]
+        txtar += ic.iconv(str)
-      rescue Iconv::InvalidEncoding
+      rescue Iconv::IllegalSequence
-        # "UTF-8//IGNORE" is not supported on some OS
+        txtar += $!.success
+        str = '?' + $!.failed[1,$!.failed.length]
+        retry
+      rescue
+        txtar += $!.success
      end
+      str = txtar
    end
    str
  end

--- a/test/unit/changeset_test.rb
+++ b/test/unit/changeset_test.rb
@@ -14,7 +14,9 @@
 require File.expand_path('../../test_helper', __FILE__)
 class ChangesetTest < ActiveSupport::TestCase
-  fixtures :projects, :repositories, :issues, :issue_statuses, :changesets, :changes, :issue_categories, :enumerations, :custom_fields, :custom_values, :users, :members, :member_roles, :trackers
+  fixtures :projects, :repositories, :issues, :issue_statuses,
+           :changesets, :changes, :issue_categories, :enumerations,
+           :custom_fields, :custom_values, :users, :members, :member_roles, :trackers
  def setup
  end
@@ -235,27 +237,24 @@ class ChangesetTest < ActiveSupport::TestCase
      assert_equal "Texte encodé en ISO-8859-1.", c.comments
  end
-  def test_invalid_utf8_sequences_in_comments_should_be_stripped
+  def test_invalid_utf8_sequences_in_comments_should_be_replaced_latin1
      proj = Project.find(3)
      str = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
      r = Repository::Bazaar.create!(
-            :project => proj, :url => '/tmp/test/bazaar',
+            :project => proj,
+            :url => '/tmp/test/bazaar',
            :log_encoding => 'UTF-8' )
      assert r
-      c = Changeset.new(:repository => r,
+      c = Changeset.new(:repository   => r,
                        :committed_on => Time.now,
-                        :revision => '123',
+                        :revision     => '123',
-                        :scmid => '12345',
+                        :scmid        => '12345',
-                        :comments => str)
+                        :comments     => str)
      assert( c.save )
-      if str.respond_to?(:force_encoding)
+      assert_equal "Texte encod? en ISO-8859-1.", c.comments
-        assert_equal "Texte encod? en ISO-8859-1.", c.comments
-      else
-        assert_equal "Texte encod en ISO-8859-1.", c.comments
-      end
  end
-  def test_invalid_utf8_sequences_in_comments_should_be_stripped_ja_jis
+  def test_invalid_utf8_sequences_in_comments_should_be_replaced_ja_jis
      proj = Project.find(3)
      str = "test\xb5\xfetest\xb5\xfe"
      if str.respond_to?(:force_encoding)
@@ -263,7 +262,7 @@ class ChangesetTest < ActiveSupport::TestCase
      end
      r = Repository::Bazaar.create!(
            :project => proj,
-            :url => '/tmp/test/bazaar',
+            :url     => '/tmp/test/bazaar',
            :log_encoding => 'ISO-2022-JP' )
      assert r
      c = Changeset.new(:repository   => r,
@@ -272,11 +271,7 @@ class ChangesetTest < ActiveSupport::TestCase
                        :scmid        => '12345',
                        :comments     => str)
      assert( c.save )
-      if str.respond_to?(:force_encoding)
+      assert_equal "test??test??", c.comments
-        assert_equal "test??test??", c.comments
-      else
-        assert_equal "testtest", c.comments
-      end
  end
  def test_comments_should_be_converted_all_latin1_to_utf8