2013-03-18 01:06:52 +04:00
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301 USA.
import unittest
import time
import logging
import re
import platform
import sys
2013-09-26 02:52:41 +04:00
import urlgrabber . progress
2013-04-11 03:48:07 +04:00
from tests import utils
2013-09-26 02:52:41 +04:00
from virtinst import Guest
2013-08-09 05:42:44 +04:00
from virtinst import urlfetcher
from virtinst . urlfetcher import FedoraDistro
from virtinst . urlfetcher import SuseDistro
from virtinst . urlfetcher import DebianDistro
from virtinst . urlfetcher import CentOSDistro
from virtinst . urlfetcher import SLDistro
from virtinst . urlfetcher import UbuntuDistro
from virtinst . urlfetcher import MandrivaDistro
2013-03-18 01:06:52 +04:00
2013-04-12 00:32:00 +04:00
# pylint: disable=W0212
# Access to protected member, needed to unittest stuff
2013-03-18 01:06:52 +04:00
# Filters for including/excluding certain distros.
MATCH_FILTER = " .* "
# Variable used to store a local iso or dir path to check for a distro
# Specified via 'python setup.py test_urls --path"
LOCAL_MEDIA = [ ]
FEDORA_BASEURL = " http://download.fedoraproject.org/pub/fedora/linux/releases/ %s /Fedora/ %s /os/ "
OPENSUSE_BASEURL = " http://download.opensuse.org/distribution/ %s /repo/oss/ "
OLD_OPENSUSE_BASEURL = " http://ftp5.gwdg.de/pub/opensuse/discontinued/distribution/ %s /repo/oss "
2013-09-26 02:52:41 +04:00
OLDUBUNTU_BASEURL = " http://old-releases.ubuntu.com/ubuntu/dists/ %s /main/installer- %s "
2013-03-18 01:06:52 +04:00
UBUNTU_BASEURL = " http://us.archive.ubuntu.com/ubuntu/dists/ %s /main/installer- %s "
2013-09-26 02:52:41 +04:00
OLDDEBIAN_BASEURL = " http://archive.debian.org/debian/dists/ %s /main/installer- %s / "
2013-03-18 01:06:52 +04:00
DEBIAN_BASEURL = " http://ftp.us.debian.org/debian/dists/ %s /main/installer- %s / "
CURCENTOS_BASEURL = " http://ftp.linux.ncsu.edu/pub/CentOS/ %s /os/ %s / "
OLDCENTOS_BASEURL = " http://vault.centos.org/ %s /os/ %s "
MANDRIVA_BASEURL = " http://ftp.uwsg.indiana.edu/linux/mandrake/official/ %s / %s / "
SCIENTIFIC_BASEURL = " http://ftp.scientificlinux.org/linux/scientific/ %s / %s / "
# Doesn't appear to be a simple boot iso in newer suse trees
NOBOOTISO_FILTER = " .*opensuse12.*|.*opensuse11.*|.*opensuse10.3.*|.*opensuse10.0.* "
# Opensuse < 10.3 (and some sles) require crazy rpm hacking to get a bootable
# kernel. We expect failure in this case since our test harness doesn't
# actually fetch anything
EXPECT_XEN_FAIL = " .*opensuse10.2.*|.*opensuse10.0.* "
2013-04-13 22:34:52 +04:00
2013-09-26 02:52:41 +04:00
# Return the expected Distro class for the passed distro label
2013-03-18 01:06:52 +04:00
def distroClass ( distname ) :
if re . match ( r " .*fedora.* " , distname ) :
return FedoraDistro
elif re . match ( r " .*suse.* " , distname ) :
return SuseDistro
elif re . match ( r " .*debian.* " , distname ) :
return DebianDistro
elif re . match ( r " .*centos.* " , distname ) :
return CentOSDistro
elif re . match ( r " .*ubuntu.* " , distname ) :
return UbuntuDistro
elif re . match ( r " .*mandriva.* " , distname ) :
return MandrivaDistro
elif re . match ( r " .*scientific.* " , distname ) :
return SLDistro
raise RuntimeError ( " distroClass: no distro registered for ' %s ' " % distname )
2013-09-26 02:52:41 +04:00
2013-03-18 01:06:52 +04:00
# Dictionary with all the test data
urls = {
# Fedora Distros
" fedora15 " : {
2013-09-26 02:52:41 +04:00
' x86_64 ' : FEDORA_BASEURL % ( " 18 " , " x86_64 " ) ,
' distro ' : " fedora18 "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" fedora16 " : {
2013-09-26 02:52:41 +04:00
' x86_64 ' : FEDORA_BASEURL % ( " 19 " , " x86_64 " ) ,
' distro ' : " fedora19 "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
# SUSE Distros
" opensuse10.2 " : {
' x86_64 ' : OLD_OPENSUSE_BASEURL % ( " 10.2 " )
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" opensuse10.3 " : {
' x86_64 ' : OLD_OPENSUSE_BASEURL % ( " 10.3 " )
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" opensuse11.4 " : {
' x86_64 ' : OPENSUSE_BASEURL % ( " 11.4 " )
2013-04-13 22:34:52 +04:00
} ,
2013-09-26 02:52:41 +04:00
# Only keep i686 for the latest
2013-03-18 01:06:52 +04:00
" opensuse12.1 " : {
' i386 ' : OPENSUSE_BASEURL % ( " 12.1 " ) ,
' x86_64 ' : OPENSUSE_BASEURL % ( " 12.1 " )
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
# Debian Distros
" debian-lenny-64 " : {
" noxen " : True ,
2013-09-26 02:52:41 +04:00
' x86_64 ' : OLDDEBIAN_BASEURL % ( " lenny " , " amd64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" debian-squeeze " : {
' i386 ' : DEBIAN_BASEURL % ( " squeeze " , " i386 " ) ,
' x86_64 ' : DEBIAN_BASEURL % ( " squeeze " , " amd64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" debian-wheezy " : {
' x86_64 ' : DEBIAN_BASEURL % ( " wheezy " , " amd64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" debian-sid " : {
' x86_64 ' : DEBIAN_BASEURL % ( " sid " , " amd64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" debian-daily " : {
' i386 ' : " http://d-i.debian.org/daily-images/amd64/ " ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
# CentOS Distros
2013-09-26 02:52:41 +04:00
" centos-6-latest " : {
' i386 ' : CURCENTOS_BASEURL % ( " 6 " , " i386 " ) ,
' x86_64 ' : CURCENTOS_BASEURL % ( " 6 " , " x86_64 " ) ,
' distro ' : " rhel6 "
} ,
2013-03-18 01:06:52 +04:00
" centos-5-latest " : {
' i386 ' : CURCENTOS_BASEURL % ( " 5 " , " i386 " ) ,
2013-09-26 02:52:41 +04:00
' x86_64 ' : CURCENTOS_BASEURL % ( " 5 " , " x86_64 " ) ,
' distro ' : " rhel5.4 "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" centos-5.0 " : {
' x86_64 ' : OLDCENTOS_BASEURL % ( " 5.0 " , " x86_64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" centos-4.0 " : {
" noxen " : True ,
' x86_64 ' : OLDCENTOS_BASEURL % ( " 4.0 " , " x86_64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-09-26 02:52:41 +04:00
" centos-4.9 " : {
' x86_64 ' : OLDCENTOS_BASEURL % ( " 4.9 " , " x86_64 " ) ,
' distro ' : " linux "
} ,
2013-03-18 01:06:52 +04:00
# Scientific Linux
" scientific-5.4 " : {
' x86_64 ' : SCIENTIFIC_BASEURL % ( " 54 " , " x86_64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " rhel5.4 "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" scientific-5.2 " : {
' x86_64 ' : SCIENTIFIC_BASEURL % ( " 52 " , " x86_64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " rhel5 "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" scientific-5.0 " : {
' x86_64 ' : SCIENTIFIC_BASEURL % ( " 50 " , " x86_64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
# Ubuntu
" ubuntu-hardy " : {
" noxen " : True ,
2013-09-26 02:52:41 +04:00
' i386 ' : OLDUBUNTU_BASEURL % ( " hardy " , " i386 " ) ,
' x86_64 ' : OLDUBUNTU_BASEURL % ( " hardy " , " amd64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" ubuntu-maverick " : {
2013-09-26 02:52:41 +04:00
' i386 ' : OLDUBUNTU_BASEURL % ( " maverick " , " i386 " ) ,
' x86_64 ' : OLDUBUNTU_BASEURL % ( " maverick " , " amd64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" ubuntu-natty " : {
2013-09-26 02:52:41 +04:00
' i386 ' : OLDUBUNTU_BASEURL % ( " natty " , " i386 " ) ,
' x86_64 ' : OLDUBUNTU_BASEURL % ( " natty " , " amd64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" ubuntu-oneiric " : {
' i386 ' : UBUNTU_BASEURL % ( " oneiric " , " i386 " ) ,
' x86_64 ' : UBUNTU_BASEURL % ( " oneiric " , " amd64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" ubuntu-precise " : {
' i386 ' : UBUNTU_BASEURL % ( " precise " , " i386 " ) ,
' x86_64 ' : UBUNTU_BASEURL % ( " precise " , " amd64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
# Mandriva
" mandriva-2009.1 " : {
" noxen " : True ,
' i586 ' : MANDRIVA_BASEURL % ( " 2009.1 " , " i586 " ) ,
' x86_64 ' : MANDRIVA_BASEURL % ( " 2009.1 " , " x86_64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
" mandriva-2010.2 " : {
" noxen " : True ,
' i586 ' : MANDRIVA_BASEURL % ( " 2010.2 " , " i586 " ) ,
' x86_64 ' : MANDRIVA_BASEURL % ( " 2010.2 " , " x86_64 " ) ,
2013-08-11 02:48:43 +04:00
' distro ' : " linux "
2013-04-13 22:34:52 +04:00
} ,
2013-03-18 01:06:52 +04:00
}
2013-07-05 16:59:58 +04:00
testconn = utils . open_testdefault ( )
2013-09-26 02:52:41 +04:00
testguest = Guest ( testconn )
2013-03-18 01:06:52 +04:00
2013-04-13 22:34:52 +04:00
2013-03-18 01:06:52 +04:00
class TestURLFetch ( unittest . TestCase ) :
def setUp ( self ) :
self . meter = urlgrabber . progress . BaseMeter ( )
if utils . get_debug ( ) :
self . meter = urlgrabber . progress . TextMeter ( fo = sys . stdout )
def _fetchLocalMedia ( self , mediapath ) :
arch = platform . machine ( )
2013-08-09 05:42:44 +04:00
fetcher = urlfetcher . _fetcherForURI ( mediapath , " /tmp " )
2013-03-18 01:06:52 +04:00
try :
fetcher . prepareLocation ( )
# Make sure we detect _a_ distro
hvmstore = self . _getStore ( fetcher , mediapath , " hvm " , arch )
logging . debug ( " Local distro detected as: %s " , hvmstore )
finally :
fetcher . cleanupLocation ( )
def _fetchFromURLDict ( self , distname , url , arch , distro_info , check_xen ) :
logging . debug ( " \n Distro= ' %s ' arch= ' %s ' url= %s " ,
distname , arch , url )
2013-08-09 05:42:44 +04:00
fetcher = urlfetcher . _fetcherForURI ( url , " /tmp " )
2013-03-18 01:06:52 +04:00
try :
fetcher . prepareLocation ( )
except Exception , e :
# Don't raise an error here: the site might be down atm
logging . error ( " %s - %s : Couldn ' t access url %s : %s . Skipping. " ,
distname , arch , fetcher . location , str ( e ) )
fetcher . cleanupLocation ( )
2013-09-26 02:52:41 +04:00
self . skipTest ( " " )
2013-03-18 01:06:52 +04:00
return
try :
self . _grabURLMedia ( fetcher , distname , url , arch , distro_info ,
check_xen )
finally :
fetcher . cleanupLocation ( )
def _checkDistroReporting ( self , stores , distro_info ) :
if distro_info is None :
return
2013-08-11 02:48:43 +04:00
dvariant = distro_info
2013-03-18 01:06:52 +04:00
for store in stores :
if not store :
continue
2013-08-11 02:48:43 +04:00
v = store . os_variant
2013-03-18 01:06:52 +04:00
2013-08-11 02:48:43 +04:00
if dvariant != v :
2013-03-18 01:06:52 +04:00
raise RuntimeError ( " Store distro/variant did not match "
2013-08-11 02:48:43 +04:00
" expected values: %s %s != %s "
% ( store , v , dvariant ) )
2013-03-18 01:06:52 +04:00
# Verify the values are valid
if v :
testguest . os_variant = v
def _grabURLMedia ( self , fetcher , distname , url , arch , distro_info ,
check_xen ) :
hvmstore = self . _getStore ( fetcher , url , " hvm " , arch )
if check_xen :
xenstore = self . _getStore ( fetcher , url , " xen " , arch )
else :
xenstore = None
exp_store = distroClass ( distname )
for s in [ hvmstore , xenstore ] :
if s and not isinstance ( s , exp_store ) :
logging . error ( " ( %s ): expected store %s , was %s " ,
distname , exp_store , s )
self . fail ( )
# Make sure the stores are reporting correct distro name/variant
try :
self . _checkDistroReporting ( [ hvmstore , xenstore ] , distro_info )
except :
2013-09-26 02:52:41 +04:00
logging . exception ( " \n \n Distro detection failed. " )
2013-03-18 01:06:52 +04:00
self . fail ( )
def fakeAcquireFile ( filename , meter ) :
if not isinstance ( meter , urlgrabber . progress . BaseMeter ) :
raise ValueError ( " passed meter is ' %s ' not an "
" actual meter. " % meter )
logging . debug ( " Fake acquiring %s " , filename )
return fetcher . hasFile ( filename )
# Replace acquireFile with hasFile, so we don't actually have to fetch
# 1000 kernels
fetcher . acquireFile = fakeAcquireFile
# Fetch boot iso
try :
if re . match ( r " %s " % NOBOOTISO_FILTER , distname ) :
logging . debug ( " Known lack of boot.iso in %s tree. Skipping. " ,
distname )
else :
boot = hvmstore . acquireBootDisk ( testguest , fetcher , self . meter )
logging . debug ( " acquireBootDisk: %s " , str ( boot ) )
2013-04-13 22:34:52 +04:00
if boot is not True :
2013-03-18 01:06:52 +04:00
raise RuntimeError ( " Didn ' t fetch any boot iso. " )
except Exception , e :
2013-09-26 02:52:41 +04:00
logging . exception ( " \n \n %s - %s : bootdisk fetching: %s " ,
2013-03-18 01:06:52 +04:00
distname , arch , str ( e ) )
self . fail ( )
# Fetch regular kernel
try :
kern = hvmstore . acquireKernel ( testguest , fetcher , self . meter )
logging . debug ( " acquireKernel (hvm): %s " , str ( kern ) )
if kern [ 0 ] is not True or kern [ 1 ] is not True :
raise RuntimeError ( " Didn ' t fetch any hvm kernel. " )
except Exception , e :
2013-09-26 02:52:41 +04:00
logging . exception ( " \n \n %s - %s : hvm kernel fetching: %s " ,
2013-03-18 01:06:52 +04:00
distname , arch , str ( e ) )
self . fail ( )
# Fetch xen kernel
try :
if xenstore and check_xen :
kern = xenstore . acquireKernel ( testguest , fetcher , self . meter )
logging . debug ( " acquireKernel (xen): %s " , str ( kern ) )
if kern [ 0 ] is not True or kern [ 1 ] is not True :
raise RuntimeError ( " Didn ' t fetch any xen kernel. " )
else :
logging . debug ( " acquireKernel (xen): Hardcoded skipping. " )
except Exception , e :
if re . match ( r " %s " % EXPECT_XEN_FAIL , distname ) :
logging . debug ( " %s : anticipated xen failure. " , distname )
else :
2013-09-26 02:52:41 +04:00
logging . exception ( " \n \n %s - %s : xen kernel fetching: %s " ,
2013-03-18 01:06:52 +04:00
distname , arch , str ( e ) )
self . fail ( )
def _getStore ( self , fetcher , url , _type , arch ) :
for ignore in range ( 0 , 10 ) :
try :
2013-08-09 05:42:44 +04:00
return urlfetcher . _storeForDistro ( fetcher = fetcher , baseuri = url ,
2013-03-18 01:06:52 +04:00
progresscb = self . meter ,
arch = arch , typ = _type )
except Exception , e :
if str ( e ) . count ( " 502 " ) :
logging . debug ( " Caught proxy error: %s " , str ( e ) )
time . sleep ( .5 )
continue
raise
raise
def testURLFetch ( self ) :
if LOCAL_MEDIA :
logging . debug ( " Skipping URL tests since local path is specified. " )
return
keys = urls . keys ( )
keys . sort ( )
assertions = 0
for label in keys :
distro_info = None
if MATCH_FILTER and not re . match ( r " %s " % MATCH_FILTER , label ) :
logging . debug ( " Excluding ' %s ' from exclude filter. " , label )
continue
check_xen = not bool ( urls [ label ] . get ( " noxen " ) )
if " distro " in urls [ label ] :
distro_info = urls [ label ] [ " distro " ]
for arch , url in urls [ label ] . items ( ) :
if arch == " distro " or arch == " noxen " :
continue
try :
print " Testing %s - %s : %s " % ( label , arch , url )
self . _fetchFromURLDict ( label , url , arch , distro_info ,
check_xen )
except AssertionError :
print " %s - %s FAILED. " % ( label , arch )
assertions + = 1
except Exception :
print " %s - %s ERROR. " % ( label , arch )
assertions + = 1
if assertions != 0 :
raise AssertionError ( " Found %d errors in URL suite. " % assertions )
def testLocalMedia ( self ) :
assertions = 0
if LOCAL_MEDIA :
for p in LOCAL_MEDIA :
print " Checking local path: %s " % p
try :
self . _fetchLocalMedia ( p )
except Exception , e :
2013-09-26 02:52:41 +04:00
logging . exception ( " \n \n Local path ' %s ' failed: %s " , p , e )
2013-03-18 01:06:52 +04:00
print " Local path FAILED. "
assertions + = 1
if assertions != 0 :
raise AssertionError ( " Found %d errors in local fetch utils. " %
assertions )