2013-03-18 01:06:52 +04:00
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301 USA.
import unittest
import time
import logging
import re
import platform
import sys
2013-09-26 02:52:41 +04:00
import urlgrabber . progress
2013-04-11 03:48:07 +04:00
from tests import utils
2013-09-26 02:52:41 +04:00
from virtinst import Guest
2013-08-09 05:42:44 +04:00
from virtinst import urlfetcher
from virtinst . urlfetcher import FedoraDistro
from virtinst . urlfetcher import SuseDistro
from virtinst . urlfetcher import DebianDistro
from virtinst . urlfetcher import CentOSDistro
from virtinst . urlfetcher import SLDistro
from virtinst . urlfetcher import UbuntuDistro
from virtinst . urlfetcher import MandrivaDistro
2013-03-18 01:06:52 +04:00
2013-04-12 00:32:00 +04:00
# pylint: disable=W0212
# Access to protected member, needed to unittest stuff
2013-03-18 01:06:52 +04:00
# Variable used to store a local iso or dir path to check for a distro
# Specified via 'python setup.py test_urls --path"
LOCAL_MEDIA = [ ]
2013-09-26 18:24:28 +04:00
OLD_FEDORA_URL = " https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/ %s /Fedora/ %s /os/ "
DEVFEDORA_URL = " http://download.fedoraproject.org/pub/fedora/linux/development/ %s / %s /os/ "
FEDORA_URL = " http://download.fedoraproject.org/pub/fedora/linux/releases/ %s /Fedora/ %s /os/ "
2013-03-18 01:06:52 +04:00
2013-09-26 18:24:28 +04:00
OLD_CENTOS_URL = " http://vault.centos.org/ %s /os/ %s "
CENTOS_URL = " http://ftp.linux.ncsu.edu/pub/CentOS/ %s /os/ %s / "
SCIENTIFIC_URL = " http://ftp.scientificlinux.org/linux/scientific/ %s / %s / "
2013-03-18 01:06:52 +04:00
2013-09-26 18:24:28 +04:00
OPENSUSE10 = " http://ftp.hosteurope.de/mirror/ftp.opensuse.org/discontinued/10.0 "
OLD_OPENSUSE_URL = " http://ftp5.gwdg.de/pub/opensuse/discontinued/distribution/ %s /repo/oss "
OPENSUSE_URL = " http://download.opensuse.org/distribution/ %s /repo/oss/ "
2013-03-18 01:06:52 +04:00
2013-09-26 18:24:28 +04:00
OLD_UBUNTU_URL = " http://old-releases.ubuntu.com/ubuntu/dists/ %s /main/installer- %s "
UBUNTU_URL = " http://us.archive.ubuntu.com/ubuntu/dists/ %s /main/installer- %s "
OLD_DEBIAN_URL = " http://archive.debian.org/debian/dists/ %s /main/installer- %s / "
DAILY_DEBIAN_URL = " http://d-i.debian.org/daily-images/ %s / "
DEBIAN_URL = " http://ftp.us.debian.org/debian/dists/ %s /main/installer- %s / "
MANDRIVA_URL = " http://ftp.uwsg.indiana.edu/linux/mandrake/official/ %s / %s / "
2013-03-18 01:06:52 +04:00
2013-04-13 22:34:52 +04:00
2013-09-26 22:40:52 +04:00
urls = { }
_distro = None
2013-09-26 02:52:41 +04:00
2013-09-26 18:24:28 +04:00
class _DistroURL ( object ) :
def __init__ ( self , x86_64 , detectdistro = " linux " , i686 = None ,
hasxen = True , hasbootiso = True , name = None ) :
self . x86_64 = x86_64
self . i686 = i686
self . detectdistro = detectdistro
self . hasxen = hasxen
self . hasbootiso = hasbootiso
self . name = name or self . detectdistro
2013-09-26 22:40:52 +04:00
self . distroclass = _distro
2013-09-26 18:24:28 +04:00
2013-09-26 22:40:52 +04:00
def _set_distro ( _d ) :
# Saves us from having to pass distro class to ever _add invocation
global _distro
_distro = _d
2013-09-26 18:24:28 +04:00
def _add ( * args , * * kwargs ) :
_d = _DistroURL ( * args , * * kwargs )
if _d . name in urls :
raise RuntimeError ( " distro= %s url= %s collides with entry in urls, "
" set a unique name " % ( _d . name , _d . x86_64 ) )
urls [ _d . name ] = _d
# Goal here is generally to cover all tree variants for each distro,
# where feasible. Don't exhaustively test i686 trees since most people
# aren't using it and it slows down the test, only use it in a couple
# places. Follow the comments for what trees to keep around
2013-09-26 22:40:52 +04:00
_set_distro ( FedoraDistro )
2013-09-26 18:24:28 +04:00
# One old Fedora
_add ( OLD_FEDORA_URL % ( " 14 " , " x86_64 " ) , " fedora14 " ,
i686 = OLD_FEDORA_URL % ( " 14 " , " i386 " ) )
# 2 Latest releases
_add ( FEDORA_URL % ( " 18 " , " x86_64 " ) , " fedora18 " )
_add ( FEDORA_URL % ( " 19 " , " x86_64 " ) , " fedora19 " )
# Any Dev release
_add ( DEVFEDORA_URL % ( " 20 " , " x86_64 " ) , " fedora20 " )
# Rawhide w/ i686 test
_add ( DEVFEDORA_URL % ( " rawhide " , " x86_64 " ) , " fedora20 " ,
i686 = DEVFEDORA_URL % ( " rawhide " , " i386 " ) ,
name = " fedora-rawhide " )
2013-09-26 22:40:52 +04:00
_set_distro ( CentOSDistro )
2013-09-26 18:24:28 +04:00
# One old and new centos 4. No distro detection since there's no treeinfo
_add ( OLD_CENTOS_URL % ( " 4.0 " , " x86_64 " ) , hasxen = False , name = " centos-4.0 " )
_add ( OLD_CENTOS_URL % ( " 4.9 " , " x86_64 " ) , name = " centos-4.9 " )
# One old centos 5
_add ( OLD_CENTOS_URL % ( " 5.0 " , " x86_64 " ) , name = " centos-5.0 " )
# Latest centos 5 w/ i686
_add ( CENTOS_URL % ( " 5 " , " x86_64 " ) , " rhel5.4 " , name = " centos-5-latest " ,
i686 = CENTOS_URL % ( " 5 " , " i386 " ) )
# Latest centos 6 w/ i686
_add ( CENTOS_URL % ( " 6 " , " x86_64 " ) , " rhel6 " , name = " centos-6-latest " ,
i686 = CENTOS_URL % ( " 6 " , " i386 " ) )
2013-09-26 22:40:52 +04:00
_set_distro ( SLDistro )
2013-09-26 18:24:28 +04:00
# Early scientific 5
_add ( SCIENTIFIC_URL % ( " 50 " , " x86_64 " ) , name = " sl-5.0 " )
# Pre-5.4 w/ treeinfo for distro detection
_add ( SCIENTIFIC_URL % ( " 52 " , " x86_64 " ) , " rhel5 " , name = " sl-5.2 " )
# Latest scientific 5
_add ( SCIENTIFIC_URL % ( " 55 " , " x86_64 " ) , " rhel5.4 " , name = " sl-5latest " )
# Latest scientific 6
_add ( SCIENTIFIC_URL % ( " 6 " , " x86_64 " ) , " rhel6 " , name = " sl-6latest " )
2013-09-26 22:40:52 +04:00
_set_distro ( SuseDistro )
2013-09-26 18:24:28 +04:00
# opensuse 10.0 uses different paths, so keep this around
_add ( OPENSUSE10 , i686 = OPENSUSE10 , hasxen = False , hasbootiso = False ,
name = " opensuse-10.0 " )
# Latest 10 series
_add ( OLD_OPENSUSE_URL % ( " 10.3 " ) , hasbootiso = False , name = " opensuse-10.3 " )
# Latest 11 series
_add ( OLD_OPENSUSE_URL % ( " 11.4 " ) , hasbootiso = False , name = " opensuse-11.4 " )
# Latest 12 series
# Only keep i686 for the latest opensuse
_add ( OPENSUSE_URL % ( " 12.3 " ) , i686 = OPENSUSE_URL % ( " 12.3 " ) , hasbootiso = False ,
name = " opensuse-12.3 " )
2013-09-26 22:40:52 +04:00
_set_distro ( DebianDistro )
2013-09-26 18:24:28 +04:00
# Debian releases rarely enough that we can just do every release since lenny
_add ( OLD_DEBIAN_URL % ( " lenny " , " amd64 " ) , hasxen = False , name = " debian-lenny " )
_add ( DEBIAN_URL % ( " squeeze " , " amd64 " ) , name = " debian-squeeze " )
_add ( DEBIAN_URL % ( " wheezy " , " amd64 " ) , name = " debian-wheezy " )
# And daily builds, since we specially handle that URL
_add ( DAILY_DEBIAN_URL % ( " amd64 " ) , name = " debian-daily " )
2013-09-26 22:40:52 +04:00
_set_distro ( UbuntuDistro )
2013-09-26 18:24:28 +04:00
# One old ubuntu
_add ( OLD_UBUNTU_URL % ( " hardy " , " amd64 " ) ,
i686 = OLD_UBUNTU_URL % ( " hardy " , " i386 " ) ,
hasxen = False , name = " ubuntu-hardy " )
# Latest LTS
_add ( UBUNTU_URL % ( " precise " , " amd64 " ) , name = " ubuntu-precise " )
# Latest release
_add ( UBUNTU_URL % ( " raring " , " amd64 " ) , name = " ubuntu-raring " )
2013-09-26 22:40:52 +04:00
_set_distro ( MandrivaDistro )
2013-09-26 18:24:28 +04:00
# One old mandriva
_add ( MANDRIVA_URL % ( " 2010.2 " , " x86_64 " ) ,
i686 = MANDRIVA_URL % ( " 2010.2 " , " i586 " ) ,
hasxen = False , name = " mandriva-2010.2 " )
2013-03-18 01:06:52 +04:00
2013-07-05 16:59:58 +04:00
testconn = utils . open_testdefault ( )
2013-09-26 21:04:28 +04:00
hvmguest = Guest ( testconn )
hvmguest . os . os_type = " hvm "
xenguest = Guest ( testconn )
xenguest . os . os_type = " xen "
2013-09-26 19:49:16 +04:00
meter = urlgrabber . progress . BaseMeter ( )
if utils . get_debug ( ) :
meter = urlgrabber . progress . TextMeter ( fo = sys . stdout )
2013-03-18 01:06:52 +04:00
2013-04-13 22:34:52 +04:00
2013-09-26 21:04:28 +04:00
def _storeForDistro ( fetcher , guest ) :
2013-09-26 19:49:16 +04:00
"""
Helper to lookup the Distro store object , basically detecting the
URL . Handle occasional proxy errors
"""
for ignore in range ( 0 , 10 ) :
2013-03-18 01:06:52 +04:00
try :
2013-09-26 21:04:28 +04:00
return urlfetcher . getDistroStore ( guest , fetcher )
2013-03-18 01:06:52 +04:00
except Exception , e :
2013-09-26 19:49:16 +04:00
if str ( e ) . count ( " 502 " ) :
logging . debug ( " Caught proxy error: %s " , str ( e ) )
time . sleep ( .5 )
2013-03-18 01:06:52 +04:00
continue
2013-09-26 19:49:16 +04:00
raise
raise
2013-03-18 01:06:52 +04:00
2013-09-26 19:49:16 +04:00
def _testLocalMedia ( fetcher , path ) :
"""
Test a local path explicitly requested by the user
"""
print " \n Checking local path: %s " % path
arch = platform . machine ( )
2013-03-18 01:06:52 +04:00
2013-09-26 19:49:16 +04:00
# Make sure we detect _a_ distro
2013-09-26 21:04:28 +04:00
hvmguest . os . arch = arch
hvmstore = _storeForDistro ( fetcher , hvmguest )
2013-09-26 19:49:16 +04:00
logging . debug ( " Local distro detected as: %s " , hvmstore )
2013-03-18 01:06:52 +04:00
2013-09-26 21:04:28 +04:00
def _testURL ( fetcher , distname , arch , distroobj ) :
2013-09-26 19:49:16 +04:00
"""
Test that our URL detection logic works for grabbing kernel , xen
kernel , and boot . iso
"""
print " \n Testing %s - %s " % ( distname , arch )
2013-09-26 21:04:28 +04:00
hvmguest . os . arch = arch
xenguest . os . arch = arch
2013-03-18 01:06:52 +04:00
2013-09-26 21:04:28 +04:00
hvmstore = _storeForDistro ( fetcher , hvmguest )
2013-09-26 19:49:16 +04:00
xenstore = None
2013-09-26 21:04:28 +04:00
if distroobj . hasxen :
xenstore = _storeForDistro ( fetcher , xenguest )
2013-03-18 01:06:52 +04:00
2013-09-26 19:49:16 +04:00
for s in [ hvmstore , xenstore ] :
2013-09-26 22:40:52 +04:00
if s and not isinstance ( s , distroobj . distroclass ) :
2013-09-26 19:49:16 +04:00
raise AssertionError ( " ( %s ): expected store %s , was %s " %
2013-09-26 22:40:52 +04:00
( distname , distroobj . distroclass , s ) )
2013-03-18 01:06:52 +04:00
# Make sure the stores are reporting correct distro name/variant
2013-09-26 18:24:28 +04:00
if s and distroobj . detectdistro != s . os_variant :
2013-09-26 19:49:16 +04:00
raise AssertionError ( " Store distro/variant did not match "
" expected values: store= %s , found= %s expect= %s " %
2013-09-26 18:24:28 +04:00
( s , s . os_variant , distroobj . detectdistro ) )
2013-09-26 19:49:16 +04:00
# Do this only after the distro detection, since we actually need
# to fetch files for that part
2013-09-26 21:04:28 +04:00
def fakeAcquireFile ( filename ) :
2013-09-26 19:49:16 +04:00
logging . debug ( " Fake acquiring %s " , filename )
return fetcher . hasFile ( filename )
fetcher . acquireFile = fakeAcquireFile
# Fetch boot iso
2013-09-26 18:24:28 +04:00
if not distroobj . hasbootiso :
2013-09-26 19:49:16 +04:00
logging . debug ( " Known lack of boot.iso in %s tree. Skipping. " ,
distname )
else :
2013-09-26 21:04:28 +04:00
boot = hvmstore . acquireBootDisk ( hvmguest )
2013-09-26 19:49:16 +04:00
logging . debug ( " acquireBootDisk: %s " , str ( boot ) )
if boot is not True :
raise AssertionError ( " %s - %s : bootiso fetching failed " %
( distname , arch ) )
# Fetch regular kernel
2013-09-26 21:04:28 +04:00
kern = hvmstore . acquireKernel ( hvmguest )
2013-09-26 19:49:16 +04:00
logging . debug ( " acquireKernel (hvm): %s " , str ( kern ) )
if kern [ 0 ] is not True or kern [ 1 ] is not True :
AssertionError ( " %s - %s : hvm kernel fetching failed " %
( distname , arch ) )
# Fetch xen kernel
2013-09-26 18:24:28 +04:00
if not xenstore :
logging . debug ( " acquireKernel (xen): Hardcoded skipping. " )
else :
2013-09-26 21:04:28 +04:00
kern = xenstore . acquireKernel ( xenguest )
2013-09-26 19:49:16 +04:00
logging . debug ( " acquireKernel (xen): %s " , str ( kern ) )
if kern [ 0 ] is not True or kern [ 1 ] is not True :
raise AssertionError ( " %s - %s : xen kernel fetching " %
( distname , arch ) )
def _fetch_wrapper ( url , cb , * args ) :
2013-09-26 21:04:28 +04:00
fetcher = urlfetcher . fetcherForURI ( url , " /tmp " , meter )
2013-09-26 19:49:16 +04:00
try :
fetcher . prepareLocation ( )
return cb ( fetcher , * args )
finally :
fetcher . cleanupLocation ( )
def _make_test_wrapper ( url , cb , args ) :
def cmdtemplate ( ) :
return _fetch_wrapper ( url , cb , * args )
return lambda _self : cmdtemplate ( )
# Register tests to be picked up by unittest
# If local ISO tests requested, skip all other URL tests
class URLTests ( unittest . TestCase ) :
pass
def _make_tests ( ) :
if LOCAL_MEDIA :
newidx = 0
for p in LOCAL_MEDIA :
newidx + = 1
args = ( p , )
testfunc = _make_test_wrapper ( p , _testLocalMedia , args )
setattr ( URLTests , " testLocalMedia %s " % newidx , testfunc )
else :
2013-03-18 01:06:52 +04:00
keys = urls . keys ( )
keys . sort ( )
2013-09-26 19:49:16 +04:00
for key in keys :
2013-09-26 18:24:28 +04:00
distroobj = urls [ key ]
2013-09-26 19:49:16 +04:00
2013-09-26 18:24:28 +04:00
for arch , url in [ ( " i686 " , distroobj . i686 ) ,
( " x86_64 " , distroobj . x86_64 ) ] :
if not url :
continue
2013-09-26 21:04:28 +04:00
args = ( key , arch , distroobj )
2013-09-26 19:49:16 +04:00
testfunc = _make_test_wrapper ( url , _testURL , args )
setattr ( URLTests , " testURL %s %s " % ( key , arch ) , testfunc )
2013-03-18 01:06:52 +04:00
2013-09-26 19:49:16 +04:00
_make_tests ( )