Fix Non-ASCII exceptions UnicodeDecodeError (#33)

* Fix Non-ASCII exceptions UnicodeDecodeError

Fix some code includes Non-ASCII characters will cause `UnicodeDecodeError`

* Add test encoding & regenerate test data
This commit is contained in:
cold 2017-04-25 06:48:26 +08:00 committed by Josh Junon
parent 74eb5373aa
commit cd378ffe7d
27 changed files with 457 additions and 3 deletions

View File

@ -23,6 +23,7 @@ import os
import re
import sys
import traceback
import codecs
from better_exceptions.color import STREAM, SUPPORTS_COLOR
from better_exceptions.log import BetExcLogger, patch as patch_logging
@ -62,6 +63,32 @@ THEME = {
MAX_LENGTH = 128
PY3 = sys.version_info[0] >= 3
def _byte(val):
unicode_type = str if PY3 else unicode
if isinstance(val, unicode_type):
try:
return val.encode(ENCODING)
except UnicodeEncodeError:
if PY3:
return codecs.escape_decode(val)[0]
else:
return val.encode("unicode-escape").decode("string-escape")
return val
def _unicode(val):
if isinstance(val, bytes):
try:
return val.decode(ENCODING)
except UnicodeDecodeError:
return val.decode("unicode-escape")
return val
def colorize_comment(source):
match = COMMENT_REGXP.match(source)
@ -248,10 +275,15 @@ def format_traceback_frame(tb):
for pc in pipe_cols:
line += (' ' * (pc - index)) + PIPE_CHAR
index = pc + 1
if not PY3 and isinstance(val, str):
# In Python2 the Non-ASCII value will be the escaped string,
# use string-escape to decode the string to show the text in human way.
val = _unicode(val.decode("string-escape"))
line += u'{}{} {}'.format((' ' * (col - index)), CAP_CHAR, val)
lines.append(THEME['inspect'](line))
formatted = '\n '.join(lines)
formatted = u'\n '.join([_unicode(x) for x in lines])
return (filename, lineno, function, formatted), color_source
@ -287,7 +319,7 @@ def format_traceback(tb=None):
def write_stream(data):
data = data.encode(ENCODING)
data = _byte(data)
if sys.version_info[0] < 3:
STREAM.write(data)

View File

@ -26,6 +26,23 @@ True
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 └ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 └ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
└ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
└ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ True
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 -> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 -> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
-> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
-> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ True
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 └ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 └ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
└ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
└ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ True
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 -> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 -> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
-> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
-> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ True
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 └ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 └ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
└ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
└ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ True
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 -> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 -> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python2 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
-> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
-> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ True
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 └ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 └ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
└ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
└ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ True
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 -> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 -> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
-> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
-> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ True
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 └ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 └ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
└ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
└ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ True
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 -> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 -> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
-> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
-> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ True
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 └ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 └ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
└ <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
└ <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
└ '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ True
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep('天')
 -> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
 -> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

View File

@ -26,6 +26,23 @@ False
python3 test/test_encoding.py
Traceback (most recent call last):
File "test/test_encoding.py", line 13, in <module>
div()
-> <function div at 0xDEADBEEF>
File "test/test_encoding.py", line 10, in div
return _deep("天")
-> <function _deep at 0xDEADBEEF>
File "test/test_encoding.py", line 7, in _deep
return 1 / val
-> '天'
TypeError: unsupported operand type(s) for /: 'int' and 'str'
./test/test_interactive.sh

13
test/test_encoding.py Normal file
View File

@ -0,0 +1,13 @@
# -*- coding:utf-8 -*-
import better_exceptions
def _deep(val):
return 1 / val
def div():
return _deep("")
div()

View File

@ -34,6 +34,7 @@ function test_case {
function test_all {
test_case "$BETEXC_PYTHON" "test/test.py"
test_case "$BETEXC_PYTHON" "test/test_color.py"
test_case "$BETEXC_PYTHON" "test/test_encoding.py"
test_case "./test/test_interactive.sh"
# test_case "./test/test_interactive_raw.sh"
test_case "./test/test_string.sh"