diff --git a/tools/license-audit/TowerLicenses.csv b/tools/license-audit/TowerLicenses.csv index be8843361c..a7d6ff055e 100644 --- a/tools/license-audit/TowerLicenses.csv +++ b/tools/license-audit/TowerLicenses.csv @@ -1,7 +1,8 @@ Babel,BSD,http://babel.pocoo.org/,pip -IPy,BSD,https://github.com/autocracy/python-ipy,pip +Django,BSD,http://www.djangoproject.com/,pip Markdown,BSD,http://packages.python.org/Markdown/,pip PrettyTable,BSD,http://code.google.com/p/prettytable/,pip +PyYAML,MIT,http://pyyaml.org/wiki/PyYAML,pip South,Apache 2.0,http://south.aeracode.org/,pip amqp,LGPL 2.1,http://github.com/celery/py-amqp,pip angular,MIT,https://github.com/angular/angular.js.git,js @@ -27,8 +28,12 @@ bootstrap,MIT,https://github.com/twbs/bootstrap.git,js bootstrap-datepicker,Apache 2.0,https://github.com/eternicode/bootstrap-datepicker,js boto,MIT,https://github.com/boto/boto/,pip celery,BSD,http://celeryproject.org,pip +cffi,MIT,http://cffi.readthedocs.org,pip +cliff,Apache 2.0,https://launchpad.net/python-cliff,pip +cmd2,MIT,http://packages.python.org/cmd2/,pip codemirror,MIT,https://github.com/codemirror/CodeMirror.git,js components-font-awesome,SIL Open Font License and MIT,http://fortawesome.github.io/Font-Awesome/,js +cryptography,BSD or Apache 2.0,https://github.com/pyca/cryptography,pip d2to1,BSD,http://pypi.python.org/pypi/d2to1,pip d3,BSD,https://github.com/mbostock/d3.git,js distribute,PSF or ZPL,http://packages.python.org/distribute,pip @@ -37,6 +42,7 @@ django-celery,BSD,http://celeryproject.org,pip django-crum,BSD,https://projects.ninemoreminutes.com/projects/django-crum/,pip django-extensions,MIT,http://github.com/django-extensions/django-extensions,pip django-jsonfield,BSD,http://bitbucket.org/schinckel/django-jsonfield/,pip +django-polymorphic,BSD,https://github.com/chrisglass/django_polymorphic,pip django-qsstats-magic,MIT,http://bitbucket.org/kmike/django-qsstats-magic/,pip django-rest-framework-mongoengine,MIT,https://github.com/umutbozkurt/django-rest-framework-mongoengine,pip django-split-settings,BSD,http://github.com/2general/django-split-settings,pip @@ -46,10 +52,16 @@ djangorestframework,BSD,http://www.django-rest-framework.org,pip dogpile.cache,BSD,http://bitbucket.org/zzzeek/dogpile.cache,pip dogpile.core,BSD,http://bitbucket.org/zzzeek/dogpile.core,pip ember-cli-test-loader,MIT,https://github.com/rjackson/ember-cli-test-loader,js +enum34,BSD,https://pypi.python.org/pypi/enum34,pip +gevent,MIT,http://www.gevent.org/,pip gevent-socketio,BSD,https://github.com/abourget/gevent-socketio,pip gevent-websocket,Apache 2.0,https://bitbucket.org/Jeffrey/gevent-websocket,pip +greenlet,MIT,https://github.com/python-greenlet/greenlet,pip httplib2,MIT,https://github.com/jcgregorio/httplib2,pip +idna,BSD,https://github.com/kjd/idna,pip importlib,PSF,https://pypi.python.org/pypi/importlib,pip +ipaddress,PSF,https://github.com/phihag/ipaddress,pip +ipy,BSD,https://github.com/autocracy/python-ipy,pip iso8601,MIT,https://bitbucket.org/micktwomey/pyiso8601,pip isodate,BSD,http://cheeseshop.python.org/pypi/isodate,pip jQuery.dotdotdot,MIT and GPL (*Ansible licenses via MIT),https://github.com/BeSite/jQuery.dotdotdot,js @@ -59,19 +71,28 @@ jquery-ui,MIT,http://jqueryui.com/,js jqueryui,MIT,http://jqueryui.com/,js js-yaml,MIT,https://github.com/nodeca/js-yaml,js jsonlint,MIT,https://github.com/zaach/jsonlint.git,js +jsonpatch,BSD,https://github.com/stefankoegl/python-json-patch,pip +jsonpointer,BSD,https://github.com/stefankoegl/python-json-pointer,pip +jsonschema,MIT,http://github.com/Julian/jsonschema,pip kapusta-jquery.sparkline,BSD,http://omnipotent.net/jquery.sparkline/,js keyring,PSF and MIT,http://bitbucket.org/kang/python-keyring-lib,pip kombu,BSD,http://kombu.readthedocs.org,pip loader.js,MIT,https://github.com/stefanpenner/loader.js,js lodash,MIT,https://github.com/lodash/lodash,js lrInfiniteScroll,MIT,https://github.com/lorenzofox3/lrInfiniteScroll,js +lxml,BSD,http://lxml.de/,pip mock,BSD,http://www.voidspace.org.uk/python/mock/,pip moment,MIT,http://momentjs.com/,js mongoengine,MIT,http://mongoengine.org/,pip +msgpack-python,Apache 2.0,http://msgpack.org/,pip netaddr,BSD,https://github.com/drkjam/netaddr/,pip +netifaces,MIT,https://bitbucket.org/al45tair/netifaces,pip nvd3,Apache 2.0,http://www.nvd3.org,js ordereddict,MIT,https://pypi.python.org/pypi/ordereddict,pip os-client-config,Apache 2.0,http://www.openstack.org/,pip +os-diskconfig-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_diskconfig_python_novaclient_ext,pip +os-networksv2-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_networksv2_python_novaclient_ext,pip +os-virtual-interfacesv2-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_virtual_interfacesv2_ext,pip os_diskconfig_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_diskconfig_python_novaclient_ext,pip os_networksv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_networksv2_python_novaclient_ext,pip os_virtual_interfacesv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_virtual_interfacesv2_ext,pip @@ -82,21 +103,33 @@ oslo.utils,Apache 2.0,http://launchpad.net/oslo,pip pbr,Apache 2.0,http://pypi.python.org/pypi/pbr,pip pexpect,ISC,http://pexpect.readthedocs.org/,pip pip,MIT,http://www.pip-installer.org,pip +prettytable,BSD,http://code.google.com/p/prettytable/,pip psphere,Apache 2.0,https://github.com/jkinred/psphere,pip +psycopg2,LGPL with exceptions or ZPL,http://initd.org/psycopg/,pip +pyOpenSSL,Apache 2.0,https://github.com/pyca/pyopenssl,pip +pyasn1,BSD,http://sourceforge.net/projects/pyasn1/,pip +pycparser,BSD,https://github.com/eliben/pycparser,pip +pycrypto,Public domain,http://www.pycrypto.org/,pip +pymongo,Apache 2.0,http://github.com/mongodb/mongo-python-driver,pip +pyparsing,MIT,http://pyparsing.wikispaces.com/,pip pyrax,Apache 2.0,https://github.com/rackspace/pyrax,pip python-cinderclient,Apache 2.0,http://www.openstack.org/,pip python-dateutil,BSD,https://dateutil.readthedocs.org,pip python-glanceclient,Apache 2.0,http://www.openstack.org/,pip python-ironicclient,Apache 2.0,http://www.openstack.org/,pip python-keystoneclient,Apache 2.0,http://www.openstack.org/,pip +python-ldap,Python style,http://www.python-ldap.org/,pip python-neutronclient,Apache 2.0,http://www.openstack.org/,pip python-novaclient,Apache 2.0,https://git.openstack.org/cgit/openstack/python-novaclient,pip python-swiftclient,Apache 2.0,http://www.openstack.org/,pip python-troveclient,Apache 2.0,http://www.openstack.org/,pip pytz,MIT,http://pythonhosted.org/pytz,pip pywinrm,MIT,http://github.com/diyan/pywinrm/,pip +pyzmq,LGPL+BSD,http://github.com/zeromq/pyzmq,pip rackspace-auth-openstack,Apache 2.0,https://github.com/rackerlabs/rackspace-auth-openstack,pip rackspace-novaclient,Apache 2.0,https://github.com/rackerlabs/rackspace-novaclient,pip +rax-default-network-flags-python-novaclient-ext,Apache 2.0,https://github.com/rackspace/rax_default_network_flags_python_novaclient_ext,pip +rax-scheduled-images-python-novaclient-ext,Apache 2.0,https://github.com/rackspace-titan/rax_scheduled_images_python_novaclient_ext,pip rax_default_network_flags_python_novaclient_ext,Apache 2.0,https://github.com/rackspace/rax_default_network_flags_python_novaclient_ext,pip rax_scheduled_images_python_novaclient_ext,Apache 2.0,https://github.com/rackspace-titan/rax_scheduled_images_python_novaclient_ext,pip redis,MIT,http://github.com/andymccurdy/redis-py,pip @@ -112,7 +145,11 @@ sizzle,MIT,https://github.com/jquery/sizzle.git,js socket.io-client,MIT,https://github.com/Automattic/socket.io-client.git,js stevedore,Apache 2.0,https://github.com/dreamhost/stevedore,pip suds,LGPL 3,https://fedorahosted.org/suds,pip +superlance,BSD,http://supervisord.org,pip timezone-js,Apache 2.0,https://github.com/mde/timezone-js.git,js twitter,MIT,http://getbootstrap.com,js underscore,MIT,https://github.com/jashkenas/underscore,js +warlock,Apache 2.0,http://github.com/bcwaldon/warlock,pip +wheel,MIT,http://bitbucket.org/pypa/wheel/,pip +wsgiref,PSF or ZPL,http://cheeseshop.python.org/pypi/wsgiref,pip xmltodict,MIT,https://github.com/martinblech/xmltodict,pip diff --git a/tools/license-audit/license-audit.py b/tools/license-audit/license-audit.py index c33092a6d7..e5a71ec8e2 100755 --- a/tools/license-audit/license-audit.py +++ b/tools/license-audit/license-audit.py @@ -17,52 +17,55 @@ def usage(): sys.exit(1) def read_requirements(towerpath): - filename = '%s/awx/lib/site-packages/README' % (towerpath,) + filename = '%s/requirements/requirements.txt' % (towerpath,) ret = {} - f = open(filename) - if not f: + try: + f = open(filename) + except: print "failed to open %s" %(filename,) return None data = f.readlines() f.close() for line in data: + if line[0] == '#': + continue if '==' in line: - m = re.match(r"(\S+)==(\S+) \((\S+)",line) + m = re.match(r"(\S+)==(\S+)",line) if m: name = m.group(1) version = m.group(2) - pathname = m.group(3) - if pathname.endswith(',') or pathname.endswith(')'): - pathname = pathname[:-1] - if pathname.endswith('/*'): - pathname = pathname[:-2] item = {} item['name'] = name item['version'] = version - item['path'] = pathname ret[name] = item - return ret - -def get_python(towerpath): - excludes = [ - 'README*', - '*.dist-info', - 'funtests', - 'easy_install.py', - 'oslo', - 'pkg_resources', - '_markerlib' - ] - directory = '%s/awx/lib/site-packages' % (towerpath,) - dirlist = os.listdir(directory) - ret = [] - for item in dirlist: - use = True - for exclude in excludes: - if fnmatch.fnmatch(item, exclude): - use = False - if use: - ret.append(item) + continue + elif line.startswith("git+https"): + l = line.rsplit('/',1) + m = re.match(r"(\S+).git@(\S+)#",l[1]) + if m: + name = m.group(1) + version = m.group(2) + if version.startswith('tower_'): + version = version[6:] + item = {} + if name == 'python-ipy': + name='ipy' + item['name'] = name + item['version'] = version + if len(version) > 20: + # it's a sha1sum, read it off the egg spec + lver = l[1].rsplit('-',1) + if lver[1] == l[1]: + lver = l[1].rsplit('_',1) + item['version'] = lver[1][:-1] + ret[name] = item + continue + else: + item = {} + item['name'] = line[:-1] + item['version'] = '' + ret[name] = item + continue return ret def get_js(towerpath): @@ -116,13 +119,13 @@ def normalize_license(license): license = license.replace('"','') if license == 'None': return 'UNKNOWN' - if license in ['Apache License, Version 2.0', 'Apache License (2.0)', 'Apache License 2.0', 'Apache-2.0', 'Apache License, v2.0']: + if license in ['Apache License, Version 2.0', 'Apache License (2.0)', 'Apache License 2.0', 'Apache-2.0', 'Apache License, v2.0', 'APL2']: return 'Apache 2.0' if license == 'ISC license': return 'ISC' if license == 'MIT License' or license == 'MIT license': return 'MIT' - if license == 'BSD License' or license == 'Simplified BSD': + if license in ['BSD License', 'Simplified BSD', 'BSD-derived (http://www.repoze.org/LICENSE.txt)', 'BSD-like', 'Modified BSD License']: return 'BSD' if license == 'LGPL': return 'LGPL 2.1' @@ -131,6 +134,10 @@ def normalize_license(license): return 'Apache 2.0' if license.find('https://github.com/umutbozkurt/django-rest-framework-mongoengine/blob/master/LICENSE') != -1: return 'MIT' + if license == '"BSD or Apache License, Version 2.0"': + return 'BSD or Apache 2.0' + if license == 'Modified BSD License': + return 'BSD' if license == 'Python Software Foundation License': return 'PSF' return license @@ -183,24 +190,11 @@ if not olddata or not requirements: print "No starting data" sys.exit(1) -# Get directory of vendored things from site-packages... -python_packages = get_python(tower_path) - -# ... and ensure they're noted in the requirements file -ok = True -for package in python_packages: - if not search_requirements(requirements, package): - print "%s not in requirements!" % (package,) - ok = False -if not ok: - sys.exit(1) - - # See if there's pip things in our current license list that we don't have now reqs = requirements.keys() for item in olddata.values(): if item['source'] == 'pip' and item['name'] not in reqs: - print "No longer vendoring %s" %(item['name'],) + print "Potentially no longer vendoring %s" %(item['name'],) # Get directory of vendored JS things from the js dir js_packages = get_js(tower_path) @@ -214,10 +208,12 @@ for item in olddata.values(): # Take the requirements file, and get license information where necessary cs = yolk.pypi.CheeseShop() for req in requirements.values(): - cs_info = cs.release_data(req['name'],req['version']) + # name sanitization + (pname, pvers) = cs.query_versions_pypi(req['name']) + cs_info = cs.release_data(pname,req['version']) if not cs_info: - print "Couldn't find '%s-%s'" %(req['name'],req['version']) - if 'name' not in olddata: + print "Couldn't find '%s==%s'" %(req['name'],req['version']) + if req['name'] not in olddata: print "... and it's not in the current data. This needs fixed!" sys.exit(1) continue @@ -241,7 +237,7 @@ for req in requirements.values(): # Update JS package info for pkg in js: - if 'pkg' in olddata: + if pkg in olddata: data = olddata[pkg] new = js_packages[pkg] if new['license'] != 'UNKNOWN' and new['license'] != data['license']: