Merge branch 'master' of http://code.zemanta.com/tsolc/git/unidecode

Author: iki

README

@@ -36,6 +36,11 @@ DESCRIPTION
 REQUIREMENTS
 
     Nothing except Python itself.
+    
+    You will need a Python build with "wide" Unicode characters in order
+    for unidecode to work correctly with characters outside of Basic
+    Multilingual Plane. Surrogate pair encoding of "narrow" builds is not
+    supported.
 
 
 INSTALLATION
@@ -58,20 +63,20 @@ AVAILABILITY
     The latest version of Unidecode is available from the GIT
     repository at
 
-	http://code.zemanta.com/tsolc/git/wikiprep
+	http://code.zemanta.com/tsolc/git/unidecode
 
     You can get it by running:
 
-	git clone http://code.zemanta.com/tsolc/git/wikiprep
+	git clone http://code.zemanta.com/tsolc/git/unidecode
 
 
 COPYRIGHT
 
-Character transliteration tables:
+Original character transliteration tables:
 
 Copyright 2001, Sean M. Burke <[email protected]>, all rights reserved.
 
-Python code:
+Python code and later additions:
 
 Copyright 2011, Tomaz Solc <[email protected]>
 

tests/basic_2.py

@@ -1,7 +1,21 @@
 # -*- coding: utf-8 -*-
 import unittest
+import sys
 from unidecode import unidecode
 
+# workaround for Python < 2.7
+if not hasattr(unittest, 'skipIf'):
+	def skipIf(condition, reason):
+		def d(f):
+			def df(*args):
+				if condition:
+					print "skipped %r" % (reason,)
+				else:
+					return f(*args)
+			return df
+		return d
+	unittest.skipIf = skipIf
+
 class TestUnidecode(unittest.TestCase):
 	def test_ascii(self):
 		for n in xrange(0,128):
@@ -23,6 +37,7 @@ def test_circled_latin(self):
 
 			self.failUnlessEqual(b, a)
 
+	@unittest.skipIf(sys.maxunicode < 0x10000, "narrow build")
 	def test_mathematical_latin(self):
 		# 13 consecutive sequences of A-Z, a-z with some codepoints
 		# undefined. We just count the undefined ones and don't check
@@ -42,6 +57,7 @@ def test_mathematical_latin(self):
 
 		self.failUnlessEqual(empty, 24)
 
+	@unittest.skipIf(sys.maxunicode < 0x10000, "narrow build")
 	def test_mathematical_digits(self):
 		# 5 consecutive sequences of 0-9
 		for n in xrange(0x1d7ce, 0x1d800):
@@ -97,7 +113,15 @@ def test_specific(self):
 				# Table that has less than 256 entriees
 				(u'\u1eff',
 				''),
+			]
 
+		for input, output in TESTS:
+			self.failUnlessEqual(unidecode(input), output)
+
+	@unittest.skipIf(sys.maxunicode < 0x10000, "narrow build")
+	def test_specific_wide(self):
+
+		TESTS = [
 				# Non-BMP character
 				(u'\U0001d5a0',
 				'A'),

tests/basic_3.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 import unittest
+import sys
 from unidecode import unidecode
 
 class TestUnidecode(unittest.TestCase):
@@ -16,12 +17,13 @@ def test_bmp(self):
 
 	def test_circled_latin(self):
 		# 1 sequence of a-z
-		for n in xrange(0, 26):
+		for n in range(0, 26):
 			a = chr(ord('a') + n)
 			b = unidecode(chr(0x24d0 + n))
 
 			self.failUnlessEqual(b, a)
 
+	@unittest.skipIf(sys.maxunicode < 0x10000, "narrow build")
 	def test_mathematical_latin(self):
 		# 13 consecutive sequences of A-Z, a-z with some codepoints
 		# undefined. We just count the undefined ones and don't check
@@ -41,6 +43,7 @@ def test_mathematical_latin(self):
 
 		self.failUnlessEqual(empty, 24)
 
+	@unittest.skipIf(sys.maxunicode < 0x10000, "narrow build")
 	def test_mathematical_digits(self):
 		# 5 consecutive sequences of 0-9
 		for n in range(0x1d7ce, 0x1d800):
@@ -96,7 +99,15 @@ def test_specific(self):
 				# Table that has less than 256 entriees
 				('\u1eff',
 				''),
+			]
+
+		for instr, output in TESTS:
+			self.failUnlessEqual(unidecode(instr), output)
+
+	@unittest.skipIf(sys.maxunicode < 0x10000, "narrow build")
+	def test_specific_wide(self):
 
+		TESTS = [
 				# Non-BMP character
 				('\U0001d5a0',
 				'A'),