2023-05-25 07:28:45 +03:00
#!/usr/bin/env python3
2021-11-16 23:23:04 +03:00
# Unix SMB/CIFS implementation.
#
# Copyright (C) Catalyst.Net Ltd. 2021
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
2023-05-25 07:28:45 +03:00
sys . path . insert ( 0 , ' bin/python ' )
os . environ [ ' PYTHONUNBUFFERED ' ] = ' 1 '
import subprocess
2021-11-16 23:23:04 +03:00
from collections import Counter
from samba . colour import c_RED , c_GREEN , c_DARK_YELLOW , switch_colour_off
import re
import unicodedata as u
2023-04-14 22:05:18 +03:00
from samba . tests import TestCase , SkipTest
2021-11-16 23:23:04 +03:00
if not sys . stdout . isatty ( ) :
switch_colour_off ( )
def _find_root ( ) :
try :
p = subprocess . run ( [ ' git ' , ' rev-parse ' , ' --show-toplevel ' ] ,
stdout = subprocess . PIPE ,
stderr = subprocess . PIPE ,
timeout = 10 )
2023-04-14 22:05:18 +03:00
except subprocess . CalledProcessError as err :
print ( c_RED ( " Error running git (is this a git tree?): %s " % ( err ) ) )
SkipTest ( " This test is only useful in a git working tree " )
sys . exit ( 0 )
if p . returncode != 0 :
raise SkipTest ( " This test is only useful in a git working tree " )
sys . exit ( 0 )
2021-11-16 23:23:04 +03:00
root = p . stdout . decode ( ) . strip ( )
should_be_roots = (
os . path . abspath ( os . path . join ( os . path . dirname ( __file__ ) ,
" ../../.. " ) ) ,
os . path . abspath ( os . path . join ( os . path . dirname ( __file__ ) ,
" ../../../.. " ) ) ,
)
if root not in should_be_roots :
print ( c_RED ( " It looks like we have found the wrong git tree! " ) )
sys . exit ( 1 )
return root
2023-04-14 22:05:18 +03:00
ROOT = None
2021-11-16 23:23:04 +03:00
2021-11-29 05:36:37 +03:00
IGNORED_FILES = (
2021-11-16 23:23:04 +03:00
' source3/selftest/ktest-krb5_ccache-2 ' ,
' source3/selftest/ktest-krb5_ccache-3 ' ,
2021-11-17 00:23:02 +03:00
' testdata/source-chars-bad.c ' ,
2021-11-29 05:36:37 +03:00
)
IGNORED_RE = (
2022-01-19 15:15:45 +03:00
r ' ^third_party/heimdal/lib/hcrypto/passwd_dialog ' ,
r ' ^third_party/heimdal/lib/hx509/data/ ' ,
r ' ^third_party/heimdal/po ' ,
r ' ^third_party/heimdal/tests/kdc/hdb-mitdb ' ,
2022-11-23 02:10:20 +03:00
r ' ^testdata/compression/ ' ,
2022-12-06 05:11:05 +03:00
r ' ^third_party/heimdal/lib/asn1/fuzz-inputs/ ' ,
2021-11-29 05:36:37 +03:00
)
2021-11-16 23:23:04 +03:00
IGNORED_EXTENSIONS = {
2021-11-29 05:36:37 +03:00
' bmp ' ,
2021-11-16 23:23:04 +03:00
' cer ' ,
' corrupt ' ,
' crl ' ,
2021-11-29 05:36:37 +03:00
' crt ' ,
2021-11-16 23:23:04 +03:00
' dat ' ,
2021-11-29 05:36:37 +03:00
' der ' ,
2021-11-16 23:23:04 +03:00
' dump ' ,
' gpg ' ,
' gz ' ,
' ico ' ,
' keytab ' ,
' ldb ' ,
' p12 ' ,
2021-11-29 05:36:37 +03:00
' pdf ' ,
2021-11-16 23:23:04 +03:00
' pem ' ,
' png ' ,
' SAMBABACKUP ' ,
' sxd ' ,
' tdb ' ,
' tif ' ,
' reg ' ,
2021-11-29 05:36:37 +03:00
' req '
2021-11-16 23:23:04 +03:00
}
# This list is by no means exhaustive -- these are just the format
# characters we actually use.
SAFE_FORMAT_CHARS = {
' \u200b ' ,
' \ufeff '
}
pytest/source_char: check for mixed direction text
As pointed out in https://lwn.net/Articles/875964, forbidding bidi
marker characters is not always going to be enough to avoid
right-to-left vs left-to-right confusion. Consider this:
$ python -c's = "b = x # 2 * n * m"; print(s); print(s.replace("x", "א").replace("n", "ח"))'
b = x # 2 * n * m
b = א # 2 * ח * m
Those two lines are semantically the same, with the Hebrew letters
"א" and "ח" replacing "x" and "n". But they look like they mean
different things.
It is not enough to say we only allow these scripts (or indeed
non-ascii) in strings and comments, as demonstrated in this example:
$ python -c's = "b = \"x#\" # n"; print(s); print(s.replace("x", "א").replace("n", "ח"))'
b = "x#" # n
b = "א#" # ח
where the second line is visually disordered but looks valid. Any series
of neutral characters between teo RTL characters will be reversed (and
possibly mirrored).
In practice this affects one file, which is a text file for testing
unicode normalisation.
I think, for the reasons shown above, we are unlikely to see legitimate
RTL code outside perhaps of documentation files — but if we do, we can
add those files to the allow-list.
Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Autobuild-User(master): Andrew Bartlett <abartlet@samba.org>
Autobuild-Date(master): Fri Dec 3 18:53:43 UTC 2021 on sn-devel-184
2021-11-17 23:17:53 +03:00
# These files legitimately mix left-to-right and right-to-left text.
# In the real world mixing directions would be normal in bilingual
# documents, but it is rare in Samba source code.
BIDI_FILES = {
2022-10-27 03:07:34 +03:00
' third_party/heimdal/lib/base/test_base.c ' ,
2022-01-19 15:15:45 +03:00
' third_party/heimdal/lib/wind/NormalizationTest.txt ' ,
pytest/source_char: check for mixed direction text
As pointed out in https://lwn.net/Articles/875964, forbidding bidi
marker characters is not always going to be enough to avoid
right-to-left vs left-to-right confusion. Consider this:
$ python -c's = "b = x # 2 * n * m"; print(s); print(s.replace("x", "א").replace("n", "ח"))'
b = x # 2 * n * m
b = א # 2 * ח * m
Those two lines are semantically the same, with the Hebrew letters
"א" and "ח" replacing "x" and "n". But they look like they mean
different things.
It is not enough to say we only allow these scripts (or indeed
non-ascii) in strings and comments, as demonstrated in this example:
$ python -c's = "b = \"x#\" # n"; print(s); print(s.replace("x", "א").replace("n", "ח"))'
b = "x#" # n
b = "א#" # ח
where the second line is visually disordered but looks valid. Any series
of neutral characters between teo RTL characters will be reversed (and
possibly mirrored).
In practice this affects one file, which is a text file for testing
unicode normalisation.
I think, for the reasons shown above, we are unlikely to see legitimate
RTL code outside perhaps of documentation files — but if we do, we can
add those files to the allow-list.
Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Autobuild-User(master): Andrew Bartlett <abartlet@samba.org>
Autobuild-Date(master): Fri Dec 3 18:53:43 UTC 2021 on sn-devel-184
2021-11-17 23:17:53 +03:00
' testdata/source-chars-bidi.py ' ,
}
2021-11-16 23:23:04 +03:00
def get_git_files ( ) :
try :
p = subprocess . run ( [ ' git ' ,
' -C ' , ROOT ,
' ls-files ' ,
' -z ' ] ,
stdout = subprocess . PIPE ,
stderr = subprocess . PIPE ,
timeout = 10 )
except subprocess . SubprocessError as e :
print ( c_RED ( f " Error running git (is this a git tree?): { e } " ) )
print ( " This test is only useful in a git working tree " )
return [ ]
filenames = p . stdout . split ( b ' \x00 ' )
return [ x . decode ( ) for x in filenames [ : - 1 ] ]
def iter_source_files ( ) :
filenames = get_git_files ( )
for name in filenames :
2021-11-29 05:36:37 +03:00
ignore = False
2021-11-16 23:23:04 +03:00
if name in IGNORED_FILES :
2021-11-29 05:36:37 +03:00
print ( c_DARK_YELLOW ( f " ignoring (exact) { name } " ) )
continue
for ignored in IGNORED_RE :
ignore = ( re . match ( ignored , name ) )
if ignore :
break
if ignore :
print ( c_DARK_YELLOW ( f " ignoring (via RE) { name } " ) )
2021-11-16 23:23:04 +03:00
continue
if ' . ' in name :
ext = name . rsplit ( ' . ' , 1 ) [ 1 ]
if ext in IGNORED_EXTENSIONS :
print ( c_DARK_YELLOW ( f " ignoring { name } " ) )
continue
yield name
def is_latin1_file ( name ) :
for pattern in (
r ' ^source4/setup/ad-schema/ \ w+.ldf$ ' ,
r ' ^source4/setup/display-specifiers/D[ \ w-]+.txt$ ' ,
2022-01-19 15:15:45 +03:00
r ' ^third_party/heimdal/cf/pkg.m4$ ' ,
r ' ^third_party/heimdal/doc/standardisation/ ' ,
2021-11-16 23:23:04 +03:00
) :
if re . match ( pattern , name ) :
return True
return False
def is_bad_latin1_file ( fullname ) :
# In practice, the few latin-1 files we have have single non-ASCII
# byte islands in a sea of ASCII. The utf-8 sequences we are
# concerned about involve sequences of 3 high bytes. We can say a
# file is safe latin-1 if it has only individual high bytes.
with open ( fullname , ' rb ' ) as f :
b = f . read ( )
in_seq = False
for c in b :
if c > 0x7f :
if in_seq :
return True
in_seq = True
else :
in_seq = False
return False
def is_bad_char ( c ) :
if u . category ( c ) != ' Cf ' :
return False
if c in SAFE_FORMAT_CHARS :
return False
return True
class CharacterTests ( TestCase ) :
2023-04-14 22:05:18 +03:00
def setUp ( self ) :
global ROOT
if not ROOT :
ROOT = _find_root ( )
2021-11-16 23:23:04 +03:00
def test_no_unexpected_format_chars ( self ) :
""" This test tries to ensure that no source file has unicode control
characters that can change the apparent order of other
characters . These characters could make code appear to have
different semantic meaning it really does .
This issue is sometimes called " Trojan Source " , " CVE-2021-42574 " ,
or " CVE-2021-42694 " .
"""
for name in iter_source_files ( ) :
fullname = os . path . join ( ROOT , name )
try :
with open ( fullname ) as f :
s = f . read ( )
except UnicodeDecodeError as e :
# probably a latin-1 encoding, which we tolerate in a few
# files for historical reasons, though we check that there
# are not long sequences of high bytes.
if is_latin1_file ( name ) :
if is_bad_latin1_file ( fullname ) :
self . fail ( f " latin-1 file { name } has long sequences "
" of high bytes " )
else :
self . fail ( f " could not decode { name } : { e } " )
pytest/source_char: check for mixed direction text
As pointed out in https://lwn.net/Articles/875964, forbidding bidi
marker characters is not always going to be enough to avoid
right-to-left vs left-to-right confusion. Consider this:
$ python -c's = "b = x # 2 * n * m"; print(s); print(s.replace("x", "א").replace("n", "ח"))'
b = x # 2 * n * m
b = א # 2 * ח * m
Those two lines are semantically the same, with the Hebrew letters
"א" and "ח" replacing "x" and "n". But they look like they mean
different things.
It is not enough to say we only allow these scripts (or indeed
non-ascii) in strings and comments, as demonstrated in this example:
$ python -c's = "b = \"x#\" # n"; print(s); print(s.replace("x", "א").replace("n", "ח"))'
b = "x#" # n
b = "א#" # ח
where the second line is visually disordered but looks valid. Any series
of neutral characters between teo RTL characters will be reversed (and
possibly mirrored).
In practice this affects one file, which is a text file for testing
unicode normalisation.
I think, for the reasons shown above, we are unlikely to see legitimate
RTL code outside perhaps of documentation files — but if we do, we can
add those files to the allow-list.
Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Autobuild-User(master): Andrew Bartlett <abartlet@samba.org>
Autobuild-Date(master): Fri Dec 3 18:53:43 UTC 2021 on sn-devel-184
2021-11-17 23:17:53 +03:00
dirs = set ( )
2021-11-16 23:23:04 +03:00
for c in set ( s ) :
if is_bad_char ( c ) :
2024-09-05 14:06:23 +03:00
self . fail ( f " { name } has potentially bad format character { ord ( c [ 0 ] ) } ! " )
pytest/source_char: check for mixed direction text
As pointed out in https://lwn.net/Articles/875964, forbidding bidi
marker characters is not always going to be enough to avoid
right-to-left vs left-to-right confusion. Consider this:
$ python -c's = "b = x # 2 * n * m"; print(s); print(s.replace("x", "א").replace("n", "ח"))'
b = x # 2 * n * m
b = א # 2 * ח * m
Those two lines are semantically the same, with the Hebrew letters
"א" and "ח" replacing "x" and "n". But they look like they mean
different things.
It is not enough to say we only allow these scripts (or indeed
non-ascii) in strings and comments, as demonstrated in this example:
$ python -c's = "b = \"x#\" # n"; print(s); print(s.replace("x", "א").replace("n", "ח"))'
b = "x#" # n
b = "א#" # ח
where the second line is visually disordered but looks valid. Any series
of neutral characters between teo RTL characters will be reversed (and
possibly mirrored).
In practice this affects one file, which is a text file for testing
unicode normalisation.
I think, for the reasons shown above, we are unlikely to see legitimate
RTL code outside perhaps of documentation files — but if we do, we can
add those files to the allow-list.
Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Autobuild-User(master): Andrew Bartlett <abartlet@samba.org>
Autobuild-Date(master): Fri Dec 3 18:53:43 UTC 2021 on sn-devel-184
2021-11-17 23:17:53 +03:00
dirs . add ( u . bidirectional ( c ) )
if ' L ' in dirs and ' R ' in dirs :
if name not in BIDI_FILES :
self . fail ( f " { name } has LTR and RTL text ( { dirs } ) " )
2021-11-16 23:23:04 +03:00
2021-11-17 00:23:02 +03:00
def test_unexpected_format_chars_do_fail ( self ) :
""" Test the test """
for name , n_bad in [
( ' testdata/source-chars-bad.c ' , 3 )
] :
fullname = os . path . join ( ROOT , name )
with open ( fullname ) as f :
s = f . read ( )
chars = set ( s )
bad_chars = [ c for c in chars if is_bad_char ( c ) ]
self . assertEqual ( len ( bad_chars ) , n_bad )
pytest/source_char: check for mixed direction text
As pointed out in https://lwn.net/Articles/875964, forbidding bidi
marker characters is not always going to be enough to avoid
right-to-left vs left-to-right confusion. Consider this:
$ python -c's = "b = x # 2 * n * m"; print(s); print(s.replace("x", "א").replace("n", "ח"))'
b = x # 2 * n * m
b = א # 2 * ח * m
Those two lines are semantically the same, with the Hebrew letters
"א" and "ח" replacing "x" and "n". But they look like they mean
different things.
It is not enough to say we only allow these scripts (or indeed
non-ascii) in strings and comments, as demonstrated in this example:
$ python -c's = "b = \"x#\" # n"; print(s); print(s.replace("x", "א").replace("n", "ח"))'
b = "x#" # n
b = "א#" # ח
where the second line is visually disordered but looks valid. Any series
of neutral characters between teo RTL characters will be reversed (and
possibly mirrored).
In practice this affects one file, which is a text file for testing
unicode normalisation.
I think, for the reasons shown above, we are unlikely to see legitimate
RTL code outside perhaps of documentation files — but if we do, we can
add those files to the allow-list.
Signed-off-by: Douglas Bagnall <douglas.bagnall@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
Autobuild-User(master): Andrew Bartlett <abartlet@samba.org>
Autobuild-Date(master): Fri Dec 3 18:53:43 UTC 2021 on sn-devel-184
2021-11-17 23:17:53 +03:00
def test_unexpected_bidi_fails ( self ) :
""" Test the test """
for name in [
' testdata/source-chars-bidi.py '
] :
fullname = os . path . join ( ROOT , name )
with open ( fullname ) as f :
s = f . read ( )
dirs = set ( )
for c in set ( s ) :
dirs . add ( u . bidirectional ( c ) )
self . assertIn ( ' L ' , dirs )
self . assertIn ( ' R ' , dirs )
2021-11-16 23:23:04 +03:00
def check_file_text ( ) :
""" If called directly as a script, count the found characters. """
2023-05-25 07:28:29 +03:00
global ROOT
if not ROOT :
ROOT = _find_root ( )
2021-11-16 23:23:04 +03:00
counts = Counter ( )
for name in iter_source_files ( ) :
fullname = os . path . join ( ROOT , name )
try :
with open ( fullname ) as f :
s = f . read ( )
except UnicodeDecodeError as e :
if is_latin1_file ( name ) :
if is_bad_latin1_file ( fullname ) :
print ( c_RED ( f " latin-1 file { name } has long sequences "
" of high bytes " ) )
else :
print ( c_GREEN ( f " latin-1 file { name } is fine " ) )
else :
print ( c_RED ( f " can ' t read { name } : { e } " ) )
counts . update ( s )
chars = set ( s )
for c in chars :
if u . category ( c ) == ' Cf ' :
print ( c_GREEN ( f " { name } has { u . name ( c ) } " ) )
print ( len ( counts ) )
controls = [ ]
formats = [ ]
others = [ ]
for x in counts :
c = u . category ( x )
if c == ' Cc ' :
controls . append ( x )
elif c == ' Cf ' :
formats . append ( x )
elif c [ 0 ] == ' C ' :
others . append ( x )
2023-05-25 05:14:11 +03:00
print ( f " normal control characters { controls } " )
print ( f " format characters { formats } " )
print ( f " other control characters { others } " )
2021-11-16 23:23:04 +03:00
if __name__ == ' __main__ ' :
check_file_text ( )