Fix Non-ASCII exceptions UnicodeDecodeError (#33)

* Fix Non-ASCII exceptions UnicodeDecodeError

Fix some code includes Non-ASCII characters will cause `UnicodeDecodeError`

* Add test encoding & regenerate test data

better_exceptions/__init__.py

@@ -23,6 +23,7 @@ import os
 import re
 import sys
 import traceback
+import codecs
 
 from better_exceptions.color import STREAM, SUPPORTS_COLOR
 from better_exceptions.log import BetExcLogger, patch as patch_logging
@@ -62,6 +63,32 @@ THEME = {
 
 MAX_LENGTH = 128
 
+PY3 = sys.version_info[0] >= 3
+
+
+def _byte(val):
+    unicode_type = str if PY3 else unicode
+    if isinstance(val, unicode_type):
+        try:
+            return val.encode(ENCODING)
+        except UnicodeEncodeError:
+            if PY3:
+                return codecs.escape_decode(val)[0]
+            else:
+                return val.encode("unicode-escape").decode("string-escape")
+
+    return val
+
+
+def _unicode(val):
+    if isinstance(val, bytes):
+        try:
+            return val.decode(ENCODING)
+        except UnicodeDecodeError:
+            return val.decode("unicode-escape")
+
+    return val
+
 
 def colorize_comment(source):
     match = COMMENT_REGXP.match(source)
@@ -248,10 +275,15 @@ def format_traceback_frame(tb):
         for pc in pipe_cols:
             line += (' ' * (pc - index)) + PIPE_CHAR
             index = pc + 1
+
+        if not PY3 and isinstance(val, str):
+            # In Python2 the Non-ASCII value will be the escaped string,
+            # use string-escape to decode the string to show the text in human way.
+            val = _unicode(val.decode("string-escape"))
+
         line += u'{}{} {}'.format((' ' * (col - index)), CAP_CHAR, val)
         lines.append(THEME['inspect'](line))
-
+    formatted = u'\n    '.join([_unicode(x) for x in lines])
-    formatted = '\n    '.join(lines)
 
     return (filename, lineno, function, formatted), color_source
 
@@ -287,7 +319,7 @@ def format_traceback(tb=None):
 
 
 def write_stream(data):
-    data = data.encode(ENCODING)
+    data = _byte(data)
 
     if sys.version_info[0] < 3:
         STREAM.write(data)

test/output/python2-dumb-UTF-8-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python2-dumb-UTF-8-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python2-dumb-ascii-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python2-dumb-ascii-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python2-vt100-UTF-8-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python2-vt100-UTF-8-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python2-vt100-ascii-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python2-vt100-ascii-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python2-xterm-UTF-8-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python2-xterm-UTF-8-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python2-xterm-ascii-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python2-xterm-ascii-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python2 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-dumb-UTF-8-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-dumb-UTF-8-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-dumb-ascii-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-dumb-ascii-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-vt100-UTF-8-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-vt100-UTF-8-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-vt100-ascii-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-vt100-ascii-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-xterm-UTF-8-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-xterm-UTF-8-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    └ <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           └ <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               └ '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-xterm-ascii-color.out

@@ -26,6 +26,23 @@ True
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep('天')
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/output/python3-xterm-ascii-nocolor.out

@@ -26,6 +26,23 @@ False
 
 
 
+python3 test/test_encoding.py
+
+
+Traceback (most recent call last):
+  File "test/test_encoding.py", line 13, in <module>
+    div()
+    -> <function div at 0xDEADBEEF>
+  File "test/test_encoding.py", line 10, in div
+    return _deep("天")
+           -> <function _deep at 0xDEADBEEF>
+  File "test/test_encoding.py", line 7, in _deep
+    return 1 / val
+               -> '天'
+TypeError: unsupported operand type(s) for /: 'int' and 'str'
+
+
+
 ./test/test_interactive.sh
 
 

test/test_encoding.py

@@ -0,0 +1,13 @@
+# -*- coding:utf-8 -*-
+
+import better_exceptions
+
+
+def _deep(val):
+    return 1 / val
+
+def div():
+    return _deep("天")
+
+
+div()

test_all.sh

@@ -34,6 +34,7 @@ function test_case {
 function test_all {
 	test_case "$BETEXC_PYTHON" "test/test.py"
 	test_case "$BETEXC_PYTHON" "test/test_color.py"
+	test_case "$BETEXC_PYTHON" "test/test_encoding.py"
 	test_case "./test/test_interactive.sh"
 	# test_case "./test/test_interactive_raw.sh"
 	test_case "./test/test_string.sh"