andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

object-storage: final removal of ufo code

Browse Source 
See https://git.gluster.org/gluster-swift.git for the new location of
the Gluster-Swift code.

With this patch, no OpenStack Swift related RPMs are constructed.

This patch also removes the unused code that references the
user.ufo-test xattr key in the DHT translator.

Change-Id: I2da32642cbd777737a41c5f9f6d33f059c85a2c1
BUG: 961902 (https://bugzilla.redhat.com/show_bug.cgi?id=961902)
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/4970
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Reviewed-by: Luis Pabon <lpabon@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>
This commit is contained in:
Peter Portante 2013-05-08 21:51:58 -04:00 committed by Anand Avati
parent fef1270fc2
commit 40d026e100
60 changed files with 2 additions and 5907 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile.am
View File

@ -22,4 +22,3 @@ gitclean: distclean
dist-hook:
(cd $(srcdir) && git diff && echo ===== git log ==== && git log) > $(distdir)/ChangeLog
tar czf gluster-swift-ufo-$(VERSION).tar.gz ufo

25
extras/LinuxRPM/Makefile.am
View File

@ -1,6 +1,5 @@
GFS_TAR = ../../glusterfs-*git.tar.gz
UFO_TAR = ../../gluster-swift-ufo-*git.tar.gz
GFS_SPEC = ../../glusterfs.spec
.PHONY: all
@ -13,11 +12,8 @@ all:
glusterrpms: prep srcrpm rpms
-rm -rf rpmbuild
glusterrpmswithoutufo: prep srcrpm rpmswithoutufo
-rm -rf rpmbuild
prep::
if [ ! -e $(GFS_TAR) -a ! -e $(UFO_TAR) -a ! -e $(GFS_SPEC) ]; then \
if [ ! -e $(GFS_TAR) -a ! -e $(GFS_SPEC) ]; then \
$(MAKE) -c ../.. dist; \
fi
-mkdir -p rpmbuild/SPECS
@ -38,21 +34,6 @@ prep::
cp ../../*.tar.gz ./rpmbuild/SOURCES
cp ../../glusterfs.spec ./rpmbuild/SPECS
GRIZZLY_TAR = swift-1.8.0.tar.gz
GRIZZLY_URL = https://launchpad.net/swift/grizzly/1.8.0/+download/$(GRIZZLY_TAR)
prep::
@if [ -d /d/cache -a -f /d/cache/$(GRIZZLY_TAR) -a -d ./rpmbuild/SOURCES ]; then \
echo "copying swift source tarball from local cache..." ; \
cp /d/cache/$(GRIZZLY_TAR) ./rpmbuild/SOURCES/ ; \
elif [ -x /usr/bin/curl -a -d ./rpmbuild/SOURCES ]; then \
echo "fetching swift from launchpad.net..." ; \
cd ./rpmbuild/SOURCES && /usr/bin/curl -sOL $(GRIZZLY_URL) ; \
else \
echo "swift source not fetched, you don't have curl installed!" ; \
exit 1 ; \
fi
srcrpm:
rpmbuild --define '_topdir $(shell pwd)/rpmbuild' -bs rpmbuild/SPECS/glusterfs.spec
mv rpmbuild/SRPMS/* .
@ -61,10 +42,6 @@ rpms:
rpmbuild --define '_topdir $(shell pwd)/rpmbuild' -bb rpmbuild/SPECS/glusterfs.spec
mv rpmbuild/RPMS/*/* .
rpmswithoutufo:
rpmbuild --define '_topdir $(shell pwd)/rpmbuild' -bb rpmbuild/SPECS/glusterfs.spec --without ufo
mv rpmbuild/RPMS/*/* .
# EPEL-5 does not like new versions of rpmbuild and requires some
# _source_* defines

718
glusterfs.spec.in
View File

@ -26,22 +26,6 @@
# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz --without ocf
%{?_without_ocf:%global _without_ocf --without-ocf}
# if you wish to build rpms without UFO, compile like this
# rpmbuild -ta @PACKAGE_NAME@-@PACKAGE_VERSION@tar.gz --without ufo
%{?_without_ufo:%global _without_ufo true}
%if ( 0%{?fedora} ) || ( 0%{?rhel} && 0%{?rhel} >= 6 )
%global SWIFTVER 1.8.0
%if 0%{_for_fedora_koji_builds}
%global UFOVER 1.1
%else
%global UFOVER @PACKAGE_VERSION@
%endif
%if ! 0%{?_without_ufo:1}
%global _with_ufo true
%endif
%endif
%if ( 0%{?fedora} && 0%{?fedora} > 16 ) || ( 0%{?rhel} && 0%{?rhel} > 6 )
%global _with_systemd true
%endif
@ -98,10 +82,6 @@ Requires(postun): systemd-units
# can't seem to make a generic macro that works
%define _init_glusterd %{_unitdir}/glusterd.service
%define _init_glusterfsd %{_unitdir}/glusterfsd.service
%define _init_gluster_swift_account %{_unitdir}/gluster-swift-account.service
%define _init_gluster_swift_container %{_unitdir}/gluster-swift-container.service
%define _init_gluster_swift_object %{_unitdir}/gluster-swift-object.service
%define _init_gluster_swift_proxy %{_unitdir}/gluster-swift-proxy.service
%else
%if 0%{_for_fedora_koji_builds}
%global glusterd_service %{S:%{SOURCE12}}
@ -119,10 +99,6 @@ Requires(postun): /sbin/service
# can't seem to make a generic macro that works
%define _init_glusterd %{_sysconfdir}/init.d/glusterd
%define _init_glusterfsd %{_sysconfdir}/init.d/glusterfsd
%define _init_gluster_swift_account %{_sysconfdir}/init.d/gluster-swift-account
%define _init_gluster_swift_container %{_sysconfdir}/init.d/gluster-swift-container
%define _init_gluster_swift_object %{_sysconfdir}/init.d/gluster-swift-object
%define _init_gluster_swift_proxy %{_sysconfdir}/init.d/gluster-swift-proxy
%endif
BuildRequires: bison flex
@ -306,269 +282,8 @@ is in user space and easily manageable.
This package provides the development libraries.
%if 0%{?_with_ufo:1}
%if ( 0%{?fedora} && 0%{?fedora} < 19 ) || ( 0%{?rhel} && 0%{?rhel} < 7 )
%package swift
Summary: GlusterFS OpenStack Object Storage
Group: Applications/System
License: ASL 2.0
BuildArch: noarch
%if 0%{_for_fedora_koji_builds}
Source20: http://launchpad.net/swift/folsom/%{SWIFTVER}/+download/swift-%{SWIFTVER}.tar.gz
%else
Source20: swift-%{SWIFTVER}.tar.gz
%endif
Source30: gluster-swift-account.service
Source31: gluster-swift-container.service
Source32: gluster-swift-object.service
Source33: gluster-swift-proxy.service
Source34: gluster-swift-account@.service
Source35: gluster-swift-container@.service
Source36: gluster-swift-object@.service
Source37: gluster-swift.tmpfs
Source40: gluster-swift-account.init
Source41: gluster-swift-container.init
Source42: gluster-swift-object.init
Source43: gluster-swift-proxy.init
Source44: gluster-swift-functions
# these first appeared in openstack-swift-1.7.4-2.fc19
Source50: gluster-swift-account-replicator.service
Source51: gluster-swift-account-replicator@.service
Source52: gluster-swift-account-auditor.service
Source53: gluster-swift-account-auditor@.service
Source54: gluster-swift-account-reaper.service
Source55: gluster-swift-account-reaper@.service
Source56: gluster-swift-container-replicator.service
Source57: gluster-swift-container-replicator@.service
Source58: gluster-swift-container-auditor.service
Source59: gluster-swift-container-auditor@.service
Source60: gluster-swift-container-updater.service
Source61: gluster-swift-container-updater@.service
Source62: gluster-swift-object-replicator.service
Source63: gluster-swift-object-replicator@.service
Source64: gluster-swift-object-auditor.service
Source65: gluster-swift-object-auditor@.service
Source66: gluster-swift-object-updater.service
Source67: gluster-swift-object-updater@.service
Source68: gluster-swift-object-expirer.service
Source69: gluster-swift-object-expirer@.service
# these first appeared in openstack-swift-1.7.4-1.fc18 and -1.7.4-2.el6
Source70: account-server.conf
Source71: container-server.conf
Source72: object-server.conf
Source73: proxy-server.conf
Source74: swift.conf
Source75: object-expirer.conf
Patch20: 0001-Use-updated-parallel-install-versions-of-epel-packag.patch
Patch21: 0002-Add-fixes-for-building-the-doc-package.patch
Patch22: glusterfs-3.3.1.swift.constraints.backport-1.7.4.patch
Patch23: glusterfs-3.4.0.swift.egginfo-grizzly.patch
Patch24: 0002-Add-fixes-for-building-the-doc-package.patch.180
#BuildRoot: %(mktemp -ud %{_tmppath}/swift-%{SWIFTVER}-%{release}-XXXXXX)
%if 0%{?_with_systemd:1}
%global glusterswiftaccount_service %{S:%{SOURCE30}}
%global glusterswiftcontainer_service %{S:%{SOURCE31}}
%global glusterswiftobject_service %{S:%{SOURCE32}}
%global glusterswiftproxy_service %{S:%{SOURCE33}}
%else
%global glusterswiftaccount_service %{S:%{SOURCE40}}
%global glusterswiftcontainer_service %{S:%{SOURCE41}}
%global glusterswiftobject_service %{S:%{SOURCE42}}
%global glusterswiftproxy_service %{S:%{SOURCE43}}
%endif
BuildRequires: python-devel
BuildRequires: python-setuptools
BuildRequires: python-netifaces
%if ( 0%{?rhel} && 0%{?rhel} < 7 )
BuildRequires: python-webob1.0
BuildRequires: python-paste-deploy1.5
Requires: python-webob1.0
Requires: python-paste-deploy1.5
%else
BuildRequires: python-webob
BuildRequires: python-paste-deploy
Requires: python-webob
Requires: python-paste-deploy
%endif
Requires: %{name} = %{version}-%{release}
Requires: python-configobj
Requires: python-eventlet >= 0.9.8
Requires: python-greenlet >= 0.3.1
Requires: python-simplejson
Requires: pyxattr
Requires: python-setuptools
Requires: python-netifaces
%if "%{SWIFTVER}" != "1.7.4"
Requires: python-swiftclient
%endif
%if ( 0%{?fedora} && 0%{?fedora} < 19 ) || ( 0%{?rhel} && 0%{?rhel} < 7 )
Conflicts: openstack-swift
%endif
%description swift
OpenStack Object Storage (swift) aggregates commodity servers to work together
in clusters for reliable, redundant, and large-scale storage of static objects.
Objects are written to multiple hardware devices in the data center, with the
OpenStack software responsible for ensuring data replication and integrity
across the cluster. Storage clusters can scale horizontally by adding new nodes,
which are automatically configured. Should a node fail, OpenStack works to
replicate its content from other active nodes. Because OpenStack uses software
logic to ensure data replication and distribution across different devices,
inexpensive commodity hard drives and servers can be used in lieu of more
expensive equipment.
%package swift-account
Summary: A swift account server
Group: Applications/System
License: ASL 2.0
BuildArch: noarch
Requires: %{name}-swift = %{version}-%{release}
%description swift-account
OpenStack Object Storage (swift) aggregates commodity servers to work together
in clusters for reliable, redundant, and large-scale storage of static objects.
This package contains the %{name}-swift account server.
%package swift-container
Summary: A swift container server
Group: Applications/System
License: ASL 2.0
BuildArch: noarch
Requires: %{name}-swift = %{version}-%{release}
%description swift-container
OpenStack Object Storage (swift) aggregates commodity servers to work together
in clusters for reliable, redundant, and large-scale storage of static objects.
This package contains the %{name}-swift container server.
%package swift-object
Summary: A swift object server
Group: Applications/System
License: ASL 2.0
BuildArch: noarch
Requires: %{name}-swift = %{version}-%{release}
Requires: rsync >= 3.0
%description swift-object
OpenStack Object Storage (swift) aggregates commodity servers to work together
in clusters for reliable, redundant, and large-scale storage of static objects.
This package contains the %{name}-swift object server.
%package swift-proxy
Summary: A swift proxy server
Group: Applications/System
License: ASL 2.0
BuildArch: noarch
Requires: %{name}-swift = %{version}-%{release}
%description swift-proxy
OpenStack Object Storage (swift) aggregates commodity servers to work together
in clusters for reliable, redundant, and large-scale storage of static objects.
This package contains the %{name}-swift proxy server.
%package swift-doc
Summary: Documentation for %{name}
Group: Documentation
BuildArch: noarch
# Required for generating docs
BuildRequires: python-eventlet
BuildRequires: python-simplejson
%if ( 0%{?rhel} && 0%{?rhel} < 7 )
BuildRequires: python-webob1.0
BuildRequires: python-sphinx10
%else
BuildRequires: python-webob
BuildRequires: python-sphinx
%endif
BuildRequires: pyxattr
%description swift-doc
OpenStack Object Storage (swift) aggregates commodity servers to work together
in clusters for reliable, redundant, and large-scale storage of static objects.
This package contains documentation files for %{name}-swift.
%endif
%package ufo
Summary: GlusterFS Unified File and Object Storage.
Group: Applications/System
License: ASL 2.0
BuildArch: noarch
%if ( 0%{?fedora} && 0%{?fedora} < 19 ) || ( 0%{?rhel} && 0%{?rhel} < 7 )
Requires: %{name}-swift = %{version}-%{release}
%else
Requires: openstack-swift = %{SWIFTVER}
%endif
Requires: memcached
Requires: openssl
Requires: python
Obsoletes: glusterfs-swift-plugin < 3.3.1-4
Obsoletes: glusterfs-swift-ufo <= 3.3.1-4
%if 0%{_for_fedora_koji_builds}
Source15: http://download.gluster.org/pub/gluster/glusterfs/3.3/3.3.1/UFO/gluster-swift-ufo-%{UFOVER}.tar.gz
%else
Source15: gluster-swift-ufo-@PACKAGE_VERSION@.tar.gz
%endif
Patch15: %{name}-3.3.1.ufo.gluster.swift.common.DiskFile-1.7.4.patch
%description ufo
Gluster Unified File and Object Storage unifies NAS and object storage
technology. This provides a system for data storage that enables users to access
the same data as an object and as a file, simplifying management and controlling
storage costs.
%endif
%prep
%setup -q -n %{name}-%{version}
%if 0%{?_with_ufo:1}
# unpack swift-1.x.y
%setup -q -T -D -n %{name}-%{version} -a 20
# unpack gluster ufo
%setup -q -T -D -n %{name}-%{version} -a 15
%if 0%{_for_fedora_koji_builds}
#%patch0 -p0
%patch1 -p0
%endif
cd swift-%{SWIFTVER}
%if ( 0%{?rhel} && 0%{?rhel} < 7 )
%patch20 -p1
%if "%{SWIFTVER}" == "1.7.4"
%patch21 -p1
%else
%patch24 -p1
%endif
%endif
%if "%{SWIFTVER}" == "1.7.4"
%patch22 -p1
%else
%patch23 -p1
%endif
%if 0%{_for_fedora_koji_builds}
%if "%{UFOVER}" == "1.1"
cd ../ufo
%patch15 -p1
%endif
%endif
%endif
%build
./autogen.sh
@ -580,18 +295,6 @@ sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool
%{__make} %{?_smp_mflags}
%if 0%{?_with_ufo:1}
cd swift-%{SWIFTVER}
%{__python} setup.py build
%{__mkdir_p} doc/build
%if ( 0%{?fedora} )
%{__python} setup.py build_sphinx
%endif
cd ..
cd ufo
%{__python} setup.py build
cd ..
%endif
%install
%{__rm} -rf %{buildroot}
@ -714,98 +417,6 @@ touch %{buildroot}%{_sharedstatedir}/glusterd/options
touch %{buildroot}%{_sharedstatedir}/glusterd/nfs/nfs-server.vol
touch %{buildroot}%{_sharedstatedir}/glusterd/nfs/run/nfs.pid
%if 0%{?_with_ufo:1}
cd swift-%{SWIFTVER}
%{__python} setup.py install -O1 --skip-build --root %{buildroot}
# common swift .service or .init files
%_init_install %{glusterswiftaccount_service} gluster-swift-account
%_init_install %{glusterswiftcontainer_service} gluster-swift-container
%_init_install %{glusterswiftobject_service} gluster-swift-object
%_init_install %{glusterswiftproxy_service} gluster-swift-proxy
%if 0%{?_with_systemd:1}
# extra systemd .service files
%_init_install %{SOURCE34} gluster-swift-account@
%_init_install %{SOURCE35} gluster-swift-container@
%_init_install %{SOURCE36} gluster-swift-object@
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
# more extra systemd .service files in f19
%_init_install %{SOURCE50} gluster-swift-account-replicator
%_init_install %{SOURCE51} gluster-swift-account-replicator@
%_init_install %{SOURCE52} gluster-swift-account-auditor
%_init_install %{SOURCE53} gluster-swift-account-auditor@
%_init_install %{SOURCE54} gluster-swift-account-reaper
%_init_install %{SOURCE55} gluster-swift-account-reaper@
%_init_install %{SOURCE56} gluster-swift-container-replicator
%_init_install %{SOURCE57} gluster-swift-container-replicator@
%_init_install %{SOURCE58} gluster-swift-container-auditor
%_init_install %{SOURCE59} gluster-swift-container-auditor@
%_init_install %{SOURCE60} gluster-swift-container-updater
%_init_install %{SOURCE61} gluster-swift-container-updater@
%_init_install %{SOURCE62} gluster-swift-object-replicator
%_init_install %{SOURCE63} gluster-swift-object-replicator@
%_init_install %{SOURCE64} gluster-swift-object-auditor
%_init_install %{SOURCE65} gluster-swift-object-auditor@
%_init_install %{SOURCE66} gluster-swift-object-updater
%_init_install %{SOURCE67} gluster-swift-object-updater@
%_init_install %{SOURCE68} gluster-swift-object-expirer
%_init_install %{SOURCE69} gluster-swift-object-expirer@
%endif
%else
# Init helper functions
%{__install} -p -D -m 644 %{SOURCE44} %{buildroot}%{_datarootdir}/gluster-swift/functions
# Init scripts
%_init_install %{glusterswiftaccount_service} gluster-swift-account
%_init_install %{glusterswiftcontainer_service} gluster-swift-container
%_init_install %{glusterswiftobject_service} gluster-swift-object
%_init_install %{glusterswiftproxy_service} gluster-swift-proxy
%endif
# Misc other
%{__install} -d -m 755 %{buildroot}%{_sysconfdir}/swift
%{__install} -d -m 755 %{buildroot}%{_sysconfdir}/swift/account-server
%{__install} -d -m 755 %{buildroot}%{_sysconfdir}/swift/container-server
%{__install} -d -m 755 %{buildroot}%{_sysconfdir}/swift/object-server
%{__install} -d -m 755 %{buildroot}%{_sysconfdir}/swift/proxy-server
# Config files
#%if ( 0%{?fedora} && 0%{?fedora} > 17 )
# these first appeared in openstack-swift-1.7.4-1.fc18
#install -p -D -m 660 %{SOURCE70} %{buildroot}%{_sysconfdir}/swift/account-server.conf
#install -p -D -m 660 %{SOURCE71} %{buildroot}%{_sysconfdir}/swift/container-server.conf
#install -p -D -m 660 %{SOURCE72} %{buildroot}%{_sysconfdir}/swift/object-server.conf
#install -p -D -m 660 %{SOURCE73} %{buildroot}%{_sysconfdir}/swift/proxy-server.conf
#install -p -D -m 660 %{SOURCE74} %{buildroot}%{_sysconfdir}/swift/swift.conf
#install -p -D -m 660 %{SOURCE75} %{buildroot}%{_sysconfdir}/swift/object-expirer.conf
#%endif
# Install pid directory
%{__install} -d -m 755 %{buildroot}%{_localstatedir}/run/swift
%{__install} -d -m 755 %{buildroot}%{_localstatedir}/run/swift/account-server
%{__install} -d -m 755 %{buildroot}%{_localstatedir}/run/swift/container-server
%{__install} -d -m 755 %{buildroot}%{_localstatedir}/run/swift/object-server
%{__install} -d -m 755 %{buildroot}%{_localstatedir}/run/swift/proxy-server
%if 0%{?_with_systemd:1}
# Swift run directories
%{__mkdir_p} %{buildroot}%{_sysconfdir}/tmpfiles.d
install -p -m 0644 %{SOURCE37} %{buildroot}%{_sysconfdir}/tmpfiles.d/gluster-swift.conf
%endif
# Install recon directory
install -d -m 755 %{buildroot}%{_localstatedir}/cache/swift
# man pages
install -d -m 755 %{buildroot}%{_mandir}/man5
for m in doc/manpages/*.5; do
install -p -m 0644 $m %{buildroot}%{_mandir}/man5
done
install -d -m 755 %{buildroot}%{_mandir}/man1
for m in doc/manpages/*.1; do
install -p -m 0644 $m %{buildroot}%{_mandir}/man1
done
cd ..
cd ufo
%{__python} setup.py install -O1 --skip-build --root %{buildroot}
cd ..
%{__mkdir_p} %{buildroot}%{_sysconfdir}/swift
cp -r ufo/etc/* %{buildroot}%{_sysconfdir}/swift/
%{__mkdir_p} %{buildroot}%{_bindir}
cp ufo/bin/gluster-swift-gen-builders %{buildroot}%{_bindir}/
%endif
# Remove tests
%{__rm} -rf %{buildroot}/%{python_sitelib}/test
%clean
@ -953,157 +564,6 @@ fi
%exclude %{_includedir}/glusterfs/y.tab.h
%{_libdir}/*.so
%if 0%{?_with_ufo:1}
%files swift
%defattr(-,root,root,-)
%doc swift-%{SWIFTVER}/AUTHORS
%doc swift-%{SWIFTVER}/LICENSE
%doc swift-%{SWIFTVER}/README*
%doc swift-%{SWIFTVER}/etc/dispersion.conf-sample
%doc swift-%{SWIFTVER}/etc/drive-audit.conf-sample
%doc swift-%{SWIFTVER}/etc/object-expirer.conf-sample
%doc swift-%{SWIFTVER}/etc/swift.conf-sample
%{_mandir}/man5/dispersion.conf.5*
%{_mandir}/man1/swift*.1*
%if 0%{?_with_systemd:1}
%config(noreplace) %{_sysconfdir}/tmpfiles.d/gluster-swift.conf
%else
%dir %{_datarootdir}/gluster-swift/functions
%endif
%dir %{_sysconfdir}/swift
#%if ( 0%{?fedora} && 0%{?fedora} > 17 )
#%config(noreplace) %attr(660, root, swift) %{_sysconfdir}/swift/swift.conf
#%endif
%dir %attr(0755, swift, swift) %{_localstatedir}/run/swift
%dir %{python_sitelib}/swift
%{_bindir}/swift-account-audit
%{_bindir}/swift-bench
%{_bindir}/swift-bench-client
%{_bindir}/swift-drive-audit
%{_bindir}/swift-get-nodes
%{_bindir}/swift-init
%{_bindir}/swift-ring-builder
%{_bindir}/swift-dispersion-populate
%{_bindir}/swift-dispersion-report
%{_bindir}/swift-recon*
%{_bindir}/swift-object-expirer
%{_bindir}/swift-oldies
%{_bindir}/swift-orphans
%{_bindir}/swift-form-signature
%{_bindir}/swift-temp-url
%{python_sitelib}/swift/*.py*
%{python_sitelib}/swift/common
%{python_sitelib}/swift-%{SWIFTVER}-*.egg-info
%files swift-account
%defattr(-,root,root,-)
%doc swift-%{SWIFTVER}/etc/account-server.conf-sample
%{_mandir}/man5/account-server.conf.5*
%{_mandir}/man1/swift-account-auditor.1*
%{_mandir}/man1/swift-account-reaper.1*
%{_mandir}/man1/swift-account-replicator.1*
%{_mandir}/man1/swift-account-server.1*
%_init_gluster_swift_account
%if 0%{?_with_systemd:1}
%{_unitdir}/gluster-swift-account@.service
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%{_unitdir}/gluster-swift-account-*.service
%endif
%endif
%dir %attr(0755, swift, swift) %{_localstatedir}/run/swift/account-server
%dir %{_sysconfdir}/swift/account-server
%{_bindir}/swift-account-auditor
%{_bindir}/swift-account-reaper
%{_bindir}/swift-account-replicator
%{_bindir}/swift-account-server
%{python_sitelib}/swift/account
%files swift-container
%defattr(-,root,root,-)
%doc swift-%{SWIFTVER}/etc/container-server.conf-sample
%{_mandir}/man5/container-server.conf.5*
%{_mandir}/man1/swift-container-auditor.1*
%{_mandir}/man1/swift-container-replicator.1*
%{_mandir}/man1/swift-container-server.1*
%{_mandir}/man1/swift-container-sync.1*
%{_mandir}/man1/swift-container-updater.1*
%_init_gluster_swift_container
%if 0%{?_with_systemd:1}
%{_unitdir}/gluster-swift-container@.service
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%{_unitdir}/gluster-swift-container-*.service
%endif
%endif
%dir %attr(0755, swift, swift) %{_localstatedir}/run/swift/container-server
%dir %attr(0755, swift, swift) %{_localstatedir}/cache/swift
%dir %{_sysconfdir}/swift/container-server
%{_bindir}/swift-container-auditor
%{_bindir}/swift-container-server
%{_bindir}/swift-container-replicator
%{_bindir}/swift-container-updater
%{_bindir}/swift-container-sync
%{python_sitelib}/swift/container
%files swift-object
%defattr(-,root,root,-)
%doc swift-%{SWIFTVER}/etc/object-server.conf-sample
%doc swift-%{SWIFTVER}/etc/rsyncd.conf-sample
%{_mandir}/man5/object-server.conf.5*
%{_mandir}/man1/swift-object-auditor.1*
%{_mandir}/man1/swift-object-info.1*
%{_mandir}/man1/swift-object-replicator.1*
%{_mandir}/man1/swift-object-server.1*
%{_mandir}/man1/swift-object-updater.1*
%_init_gluster_swift_object
%if 0%{?_with_systemd:1}
%{_unitdir}/gluster-swift-object@.service
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%{_unitdir}/gluster-swift-object-*.service
%exclude %{_unitdir}/gluster-swift-object-expirer*.service
%endif
%endif
%dir %attr(0755, swift, swift) %{_localstatedir}/run/swift/object-server
%dir %attr(0755, swift, swift) %{_localstatedir}/cache/swift
%dir %{_sysconfdir}/swift/object-server
%{_bindir}/swift-object-auditor
%{_bindir}/swift-object-info
%{_bindir}/swift-object-replicator
%{_bindir}/swift-object-server
%{_bindir}/swift-object-updater
%{python_sitelib}/swift/obj
%files swift-proxy
%defattr(-,root,root,-)
%doc swift-%{SWIFTVER}/etc/proxy-server.conf-sample
%doc swift-%{SWIFTVER}/etc/object-expirer.conf-sample
%{_mandir}/man5/object-expirer.conf.5*
%{_mandir}/man5/proxy-server.conf.5*
%{_mandir}/man1/swift-object-expirer.1*
%{_mandir}/man1/swift-proxy-server.1*
%_init_gluster_swift_proxy
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%{_unitdir}/gluster-swift-object-expirer*.service
%endif
%dir %attr(0755, swift, swift) %{_localstatedir}/run/swift/proxy-server
%dir %attr(0755, swift, swift) %{_localstatedir}/cache/swift
%dir %{_sysconfdir}/swift/proxy-server
%{_bindir}/swift-object-expirer
%{_bindir}/swift-proxy-server
%{python_sitelib}/swift/proxy
%files swift-doc
%defattr(-,root,root,-)
%doc swift-%{SWIFTVER}/LICENSE
%files ufo
%defattr(-,root,root,-)
%{python_sitelib}/gluster
%{python_sitelib}/gluster_swift_ufo-*-*.egg-info
%{_bindir}/gluster-swift-gen-builders
%{_sysconfdir}/swift/*-gluster
%{_sysconfdir}/swift/*/1.conf-gluster
%endif
%post server
# Legacy server
@ -1169,184 +629,6 @@ if [ $1 -ge 1 ]; then
fi
%if 0%{?_with_ufo:1}
%pre swift
getent group swift >/dev/null || groupadd -r swift -g 160
getent passwd swift >/dev/null || \
useradd -r -g swift -u 160 -d %{_sharedstatedir}/swift -s /sbin/nologin \
-c "OpenStack Swift Daemons" swift
exit 0
%pre swift-account
if [ -f /etc/swift/account-server/1.conf ]; then
echo "warning: /etc/swift/account-server/1.conf saved as /etc/swift/account-server/1.conf.rpmsave"
cp /etc/swift/account-server/1.conf /etc/swift/account-server/1.conf.rpmsave
fi
%post swift-account
%_init_enable gluster-swift-account
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_enable gluster-swift-account-replicator
%_init_enable gluster-swift-account-auditor
%_init_enable gluster-swift-account-reaper
%endif
%preun swift-account
if [ $1 = 0 ] ; then
%_init_stop gluster-swift-account
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_disable gluster-swift-account
%_init_stop gluster-swift-account-replicator
%_init_disable gluster-swift-account-replicator
%_init_stop gluster-swift-account-auditor
%_init_disable gluster-swift-account-auditor
%_init_stop gluster-swift-account-reaper
%_init_disable gluster-swift-account-reaper
%endif
fi
%postun swift-account
if [ $1 -ge 1 ] ; then
%_init_restart gluster-swift-account
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_restart gluster-swift-account-replicator
%_init_restart gluster-swift-account-auditor
%_init_restart gluster-swift-account-reaper
%endif
fi
%pre swift-container
if [ -f /etc/swift/container-server/1.conf ]; then
echo "warning: /etc/swift/container-server/1.conf saved as /etc/swift/container-server/1.conf.rpmsave"
cp /etc/swift/container-server/1.conf /etc/swift/container-server/1.conf.rpmsave
fi
%post swift-container
%_init_enable gluster-swift-container
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_enable gluster-swift-container-replicator
%_init_enable gluster-swift-container-auditor
%_init_enable gluster-swift-container-updater
%endif
%preun swift-container
if [ $1 = 0 ] ; then
%_init_stop gluster-swift-container
%_init_disable gluster-swift-container
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_stop gluster-swift-container-replicator
%_init_disable gluster-swift-container-replicator
%_init_stop gluster-swift-container-auditor
%_init_disable gluster-swift-container-auditor
%_init_stop gluster-swift-container-updater
%_init_disable gluster-swift-container-updater
%endif
fi
%postun swift-container
if [ $1 -ge 1 ] ; then
%_init_restart gluster-swift-container
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_restart gluster-swift-container-replicator
%_init_restart gluster-swift-container-auditor
%_init_restart gluster-swift-container-updator
%endif
fi
%pre swift-object
if [ -f /etc/swift/object-server/1.conf ]; then
echo "warning: /etc/swift/object-server/1.conf saved as /etc/swift/object-server/1.conf.rpmsave"
cp /etc/swift/object-server/1.conf /etc/swift/object-server/1.conf.rpmsave
fi
%post swift-object
%_init_enable gluster-swift-object
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_enable gluster-swift-object-replicator
%_init_enable gluster-swift-object-auditor
%_init_enable gluster-swift-object-updater
%endif
%preun swift-object
if [ $1 = 0 ] ; then
%_init_stop gluster-swift-object
%_init_disable gluster-swift-object
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_stop gluster-swift-object-replicator
%_init_disable gluster-swift-object-replicator
%_init_stop gluster-swift-object-auditor
%_init_disable gluster-swift-object-auditor
%_init_stop gluster-swift-object-updater
%_init_disable gluster-swift-object-updater
%endif
fi
%postun swift-object
if [ $1 -ge 1 ] ; then
%_init_restart gluster-swift-object
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_restart gluster-swift-object-replicator
%_init_restart gluster-swift-object-auditor
%_init_restart gluster-swift-object-updater
%endif
fi
%pre swift-proxy
if [ -f /etc/swift/proxy-server.conf ]; then
echo "warning: /etc/swift/proxy-server.conf saved as /etc/swift/proxy-server.conf.rpmsave"
cp /etc/swift/proxy-server.conf /etc/swift/proxy-server.conf.rpmsave
fi
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
if [ -f /etc/swift/object-expirer.conf ]; then
echo "warning: /etc/swift/object-expirer.conf saved as /etc/swift/object-expirer.conf.rpmsave"
cp /etc/swift/object-expirer.conf /etc/swift/object-expirer.conf.rpmsave
fi
%endif
%post swift-proxy
%_init_enable gluster-swift-proxy
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_enable gluster-swift-object-expirer
%endif
%preun swift-proxy
if [ $1 = 0 ] ; then
%_init_stop gluster-swift-proxy
%_init_disable gluster-swift-proxy
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_stop gluster-swift-object-expirer
%_init_disable gluster-swift-object-expirer
%endif
fi
%postun swift-proxy
if [ $1 -ge 1 ] ; then
%_init_restart gluster-swift-proxy
%if ( 0%{?fedora} && 0%{?fedora} > 18 )
%_init_restart gluster-swift-object-expirer
%endif
fi
%endif
%changelog
* Mon Mar 4 2013 Niels de Vos <ndevos@redhat.com>
- Package /var/run/gluster so that statedumps can be created

3
tests/basic/rpm.t
View File

@ -45,7 +45,7 @@ cd ${RESULT_DIR}/sources
git clone -q -s file://${REPO} .
git checkout -q -b rpm-test ${COMMIT}
# build the glusterfs-*.tar.gz and gluster-swift-ufo-*.tar.gz
# build the glusterfs-*.tar.gz
[ -e configure ] || ./autogen.sh 2>&1 > /dev/null
TEST ./configure --enable-fusermount
TEST make dist
@ -100,4 +100,3 @@ done
[ "${DEBUG}" = "0" ] && rm -rf ${RESULT_DIR}
cleanup

16
ufo/README
View File

@ -1,16 +0,0 @@
Gluster Unified File and Object Storage allows files and directories created
via gluster-native/nfs mount to be accessed as containers and objects. It is
a plugin for OpenStack Swift project.
Install
* TBD
Once this is done, you can access the GlusterFS volumes as Swift accounts.
Add the Volume names with the user-name and its corresponding password to the
/etc/swift/proxy-server.conf (follow the syntax used in the sample conf file).
Command to start the servers (TBD)
swift-init main start
Command to stop the servers (TBD)
swift-init main stop

44
ufo/bin/gluster-swift-gen-builders
View File

@ -1,44 +0,0 @@
#!/bin/bash
# Note that these port numbers must match the configured values for the
# various servers in their configuration files.
declare -A port=(["account.builder"]=6012 ["container.builder"]=6011 \
["object.builder"]=6010)
builder_files="account.builder container.builder object.builder"
function create {
swift-ring-builder $1 create 1 1 1 >> /tmp/out
}
function add {
swift-ring-builder $1 add z$2-127.0.0.1:$3/$4_ 100.0
}
function rebalance {
swift-ring-builder $1 rebalance
}
function build {
swift-ring-builder $1
}
if [ "$1x" = "x" ]; then
echo "Please specify the gluster volume name to use."
exit 1
fi
for builder_file in $builder_files
do
create $builder_file
zone=1
for volname in $@
do
add $builder_file $zone ${port[$builder_file]} $volname
zone=$(expr $zone + 1)
done
rebalance $builder_file
build $builder_file
done

19
ufo/etc/account-server/1.conf-gluster
View File

@ -1,19 +0,0 @@
[DEFAULT]
devices = /mnt/gluster-object
mount_check = true
bind_port = 6012
user = root
log_facility = LOG_LOCAL2
[pipeline:main]
pipeline = account-server
[app:account-server]
use = egg:gluster_swift_ufo#account
[account-replicator]
vm_test_mode = yes
[account-auditor]
[account-reaper]

21
ufo/etc/container-server/1.conf-gluster
View File

@ -1,21 +0,0 @@
[DEFAULT]
devices = /mnt/gluster-object
mount_check = true
bind_port = 6011
user = root
log_facility = LOG_LOCAL2
[pipeline:main]
pipeline = container-server
[app:container-server]
use = egg:gluster_swift_ufo#container
[container-replicator]
vm_test_mode = yes
[container-updater]
[container-auditor]
[container-sync]

17
ufo/etc/fs.conf-gluster
View File

@ -1,17 +0,0 @@
[DEFAULT]
# IP address of a GlusterFS volume server member. By default, we assume the
# local host.
mount_ip = localhost
# By default it is assumed the Gluster volumes can be accessed using other
# methods besides UFO (not object only), which disables a caching
# optimizations in order to keep in sync with file system changes.
object_only = no
# Performance optimization parameter. When turned off, the filesystem will
# see a reduced number of stat calls, resulting in substantially faster
# response time for GET and HEAD container requests on containers with large
# numbers of objects, at the expense of an accurate count of combined bytes
# used by all objects in the container. For most installations "off" works
# fine.
accurate_size_in_listing = off

36
ufo/etc/object-server/1.conf-gluster
View File

@ -1,36 +0,0 @@
[DEFAULT]
devices = /mnt/gluster-object
mount_check = true
bind_port = 6010
# If not doing the above, setting this value initially to match the number of
# CPUs is a good starting point for determining the right value.
workers = 1
[pipeline:main]
pipeline = object-server
[app:object-server]
use = egg:gluster_swift_ufo#object
user = root
log_facility = LOG_LOCAL2
# Timeout clients that don't read or write to the proxy server after 5
# seconds.
conn_timeout = 5
# For high load situations, once connected to a container server, allow for
# delays communicating with it.
node_timeout = 60
# Adjust this value to match the stripe width of the underlying storage array
# (not the stripe element size). This will provide a reasonable starting point
# for tuning this value.
disk_chunk_size = 65536
# Adjust this value match whatever is set for the disk_chunk_size
# initially. This will provide a reasonable starting point for tuning this
# value.
network_chunk_size = 65556
[object-replicator]
vm_test_mode = yes
[object-updater]
[object-auditor]

69
ufo/etc/proxy-server.conf-gluster
View File

@ -1,69 +0,0 @@
[DEFAULT]
bind_port = 8080
user = root
log_facility = LOG_LOCAL1
# Consider using 1 worker per CPU
workers = 1
[pipeline:main]
pipeline = healthcheck cache tempauth proxy-server
[app:proxy-server]
use = egg:gluster_swift_ufo#proxy
log_facility = LOG_LOCAL1
# The API allows for account creation and deletion, but since Gluster/Swift
# automounts a Gluster volume for a given account, there is no way to create
# or delete an account. So leave this off.
allow_account_management = false
account_autocreate = true
# Only need to recheck the account exists once a day
recheck_account_existence = 86400
# May want to consider bumping this up if containers are created and destroyed
# infrequently.
recheck_container_existence = 60
# Timeout clients that don't read or write to the proxy server after 5
# seconds.
client_timeout = 5
# Give more time to connect to the object, container or account servers in
# cases of high load.
conn_timeout = 5
# For high load situations, once connected to an object, container or account
# server, allow for delays communicating with them.
node_timeout = 60
# May want to consider bumping up this value to 1 - 4 MB depending on how much
# traffic is for multi-megabyte or gigabyte requests; perhaps matching the
# stripe width (not stripe element size) of your storage volume is a good
# starting point. See below for sizing information.
object_chunk_size = 65536
# If you do decide to increase the object_chunk_size, then consider lowering
# this value to one. Up to "put_queue_length" object_chunk_size'd buffers can
# be queued to the object server for processing. Given one proxy server worker
# can handle up to 1,024 connections, by default, it will consume 10 * 65,536
# * 1,024 bytes of memory in the worse case (default values). Be sure the
# amount of memory available on the system can accommodate increased values
# for object_chunk_size.
put_queue_depth = 10
[filter:tempauth]
use = egg:swift#tempauth
# Here you need to add users explicitly. See the OpenStack Swift Deployment
# Guide for more information. The user and user64 directives take the
# following form:
# user_<account>_<username> = <key> [group] [group] [...] [storage_url]
# user64_<account_b64>_<username_b64> = <key> [group] [group] [...] [storage_url]
# Where you use user64 for accounts and/or usernames that include underscores.
#
# NOTE (and WARNING): The account name must match the device name specified
# when generating the account, container, and object build rings.
#
# E.g.
# user_ufo0_admin = abc123 .admin
[filter:healthcheck]
use = egg:swift#healthcheck
[filter:cache]
use = egg:swift#memcache
# Update this line to contain a comma separated list of memcache servers
# shared by all nodes running the proxy-server service.
memcache_servers = localhost:11211

91
ufo/etc/swift.conf-gluster
View File

@ -1,91 +0,0 @@
[DEFAULT]
[swift-hash]
# random unique string that can never change (DO NOT LOSE)
swift_hash_path_suffix = gluster
# The swift-constraints section sets the basic constraints on data
# saved in the swift cluster.
[swift-constraints]
# max_file_size is the largest "normal" object that can be saved in
# the cluster. This is also the limit on the size of each segment of
# a "large" object when using the large object manifest support.
# This value is set in bytes. Setting it to lower than 1MiB will cause
# some tests to fail. It is STRONGLY recommended to leave this value at
# the default (5 * 2**30 + 2).
# FIXME: Really? Gluster can handle a 2^64 sized file? And can the fronting
# web service handle such a size? I think with UFO, we need to keep with the
# default size from Swift and encourage users to research what size their web
# services infrastructure can handle.
max_file_size = 18446744073709551616
# max_meta_name_length is the max number of bytes in the utf8 encoding
# of the name portion of a metadata header.
#max_meta_name_length = 128
# max_meta_value_length is the max number of bytes in the utf8 encoding
# of a metadata value
#max_meta_value_length = 256
# max_meta_count is the max number of metadata keys that can be stored
# on a single account, container, or object
#max_meta_count = 90
# max_meta_overall_size is the max number of bytes in the utf8 encoding
# of the metadata (keys + values)
#max_meta_overall_size = 4096
# max_object_name_length is the max number of bytes in the utf8 encoding of an
# object name: Gluster FS can handle much longer file names, but the length
# between the slashes of the URL is handled below. Remember that most web
# clients can't handle anything greater than 2048, and those that do are
# rather clumsy.
max_object_name_length = 2048
# max_object_name_component_length (GlusterFS) is the max number of bytes in
# the utf8 encoding of an object name component (the part between the
# slashes); this is a limit imposed by the underlying file system (for XFS it
# is 255 bytes).
max_object_name_component_length = 255
# container_listing_limit is the default (and max) number of items
# returned for a container listing request
#container_listing_limit = 10000
# account_listing_limit is the default (and max) number of items returned
# for an account listing request
#account_listing_limit = 10000
# max_account_name_length is the max number of bytes in the utf8 encoding of
# an account name: Gluster FS Filename limit (XFS limit?), must be the same
# size as max_object_name_component_length above.
max_account_name_length = 255
# max_container_name_length is the max number of bytes in the utf8 encoding
# of a container name: Gluster FS Filename limit (XFS limit?), must be the same
# size as max_object_name_component_length above.
max_container_name_length = 255

79
ufo/gluster-swift-ufo.spec
View File

@ -1,79 +0,0 @@
############################################################################################################
# Command to build rpms.#
# $ rpmbuild -ta %{name}-%{version}-%{release}.tar.gz #
############################################################################################################
# Setting up the environment. #
# * Create a directory %{name}-%{version} under $HOME/rpmbuild/SOURCES #
# * Copy the contents of gluster directory into $HOME/rpmbuild/SOURCES/%{name}-%{version} #
# * tar zcvf %{name}-%{version}-%{release}.tar.gz $HOME/rpmbuild/SOURCES/%{name}-%{version} %{name}.spec #
# For more information refer #
# http://fedoraproject.org/wiki/How_to_create_an_RPM_package #
############################################################################################################
%if ! (0%{?fedora} > 12 || 0%{?rhel} > 5)
%{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
%endif
%define _confdir %{_sysconfdir}/swift
%define _ufo_version 1.1
%define _ufo_release 4
Summary : GlusterFS Unified File and Object Storage.
Name : gluster-swift-ufo
Version : %{_ufo_version}
Release : %{_ufo_release}
Group : Application/File
Vendor : Red Hat Inc.
Source0 : %{name}-%{version}-%{release}.tar.gz
Packager : gluster-users@gluster.org
License : Apache
BuildArch: noarch
Requires : memcached
Requires : openssl
Requires : python
Requires : openstack-swift >= 1.4.8
Requires : openstack-swift-account >= 1.4.8
Requires : openstack-swift-container >= 1.4.8
Requires : openstack-swift-object >= 1.4.8
Requires : openstack-swift-proxy >= 1.4.8
Obsoletes: gluster-swift
Obsoletes: gluster-swift-plugin
%description
Gluster Unified File and Object Storage unifies NAS and object storage
technology. This provides a system for data storage that enables users to access
the same data as an object and as a file, simplifying management and controlling
storage costs.
%prep
%setup -q
%build
%{__python} setup.py build
%install
rm -rf %{buildroot}
%{__python} setup.py install -O1 --skip-build --root %{buildroot}
mkdir -p %{buildroot}/%{_confdir}/
cp -r etc/* %{buildroot}/%{_confdir}/
mkdir -p %{buildroot}/%{_bindir}/
cp bin/gluster-swift-gen-builders %{buildroot}/%{_bindir}/
%clean
rm -rf %{buildroot}
%files
%defattr(-,root,root)
%{python_sitelib}/gluster
%{python_sitelib}/gluster_swift_ufo-%{version}-*.egg-info
%{_bindir}/gluster-swift-gen-builders
%dir %{_confdir}
%config %{_confdir}/account-server/1.conf-gluster
%config %{_confdir}/container-server/1.conf-gluster
%config %{_confdir}/object-server/1.conf-gluster
%config %{_confdir}/swift.conf-gluster
%config %{_confdir}/proxy-server.conf-gluster
%config %{_confdir}/fs.conf-gluster

0
ufo/gluster/__init__.py
View File

18
ufo/gluster/swift/__init__.py
View File

@ -1,18 +0,0 @@
""" Gluster Swift UFO """
class Version(object):
def __init__(self, canonical_version, final):
self.canonical_version = canonical_version
self.final = final
@property
def pretty_version(self):
if self.final:
return self.canonical_version
else:
return '%s-dev' % (self.canonical_version,)
_version = Version('1.1', False)
__version__ = _version.pretty_version
__canonical_version__ = _version.canonical_version

0
ufo/gluster/swift/account/__init__.py
View File

45
ufo/gluster/swift/account/server.py
View File

@ -1,45 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Account Server for Gluster Swift UFO """
# Simply importing this monkey patches the constraint handling to fit our
# needs
import gluster.swift.common.constraints
from swift.account import server
from gluster.swift.common.DiskDir import DiskAccount
class AccountController(server.AccountController):
def _get_account_broker(self, drive, part, account):
"""
Overriden to provide the GlusterFS specific broker that talks to
Gluster for the information related to servicing a given request
instead of talking to a database.
:param drive: drive that holds the container
:param part: partition the container is in
:param account: account name
:returns: DiskDir object
"""
return DiskAccount(self.root, drive, account, self.logger)
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI account server apps."""
conf = global_conf.copy()
conf.update(local_conf)
return AccountController(conf)

496
ufo/gluster/swift/common/DiskDir.py
View File

@ -1,496 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os, errno
from gluster.swift.common.utils import clean_metadata, dir_empty, rmdirs, \
mkdirs, validate_account, validate_container, is_marker, \
get_container_details, get_account_details, get_container_metadata, \
create_container_metadata, create_account_metadata, DEFAULT_GID, \
DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \
write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \
X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \
X_CONTAINER_COUNT, CONTAINER, os_path
from gluster.swift.common import Glusterfs
from swift.common.constraints import CONTAINER_LISTING_LIMIT
from swift.common.utils import normalize_timestamp, TRUE_VALUES
DATADIR = 'containers'
# Create a dummy db_file in /etc/swift
_unittests_enabled = os.getenv('GLUSTER_UNIT_TEST_ENABLED', 'no')
if _unittests_enabled in TRUE_VALUES:
_tmp_dir = '/tmp/gluster_unit_tests'
try:
os.mkdir(_tmp_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
_db_file = os.path.join(_tmp_dir, 'db_file.db')
else:
_db_file = '/etc/swift/db_file.db'
if not os.path.exists(_db_file):
file(_db_file, 'w+')
def _read_metadata(dd):
""" Filter read metadata so that it always returns a tuple that includes
some kind of timestamp. With 1.4.8 of the Swift integration the
timestamps were not stored. Here we fabricate timestamps for volumes
where the existing data has no timestamp (that is, stored data is not
a tuple), allowing us a measure of backward compatibility.
FIXME: At this time it does not appear that the timestamps on each
metadata are used for much, so this should not hurt anything.
"""
metadata_i = read_metadata(dd)
metadata = {}
timestamp = 0
for key, value in metadata_i.iteritems():
if not isinstance(value, tuple):
value = (value, timestamp)
metadata[key] = value
return metadata
class DiskCommon(object):
def is_deleted(self):
return not os_path.exists(self.datadir)
def filter_prefix(self, objects, prefix):
"""
Accept sorted list.
"""
found = 0
filtered_objs = []
for object_name in objects:
if object_name.startswith(prefix):
filtered_objs.append(object_name)
found = 1
else:
if found:
break
return filtered_objs
def filter_delimiter(self, objects, delimiter, prefix):
"""
Accept sorted list.
Objects should start with prefix.
"""
filtered_objs=[]
for object_name in objects:
tmp_obj = object_name.replace(prefix, '', 1)
sufix = tmp_obj.split(delimiter, 1)
new_obj = prefix + sufix[0]
if new_obj and new_obj not in filtered_objs:
filtered_objs.append(new_obj)
return filtered_objs
def filter_marker(self, objects, marker):
"""
TODO: We can traverse in reverse order to optimize.
Accept sorted list.
"""
filtered_objs=[]
found = 0
if objects[-1] < marker:
return filtered_objs
for object_name in objects:
if object_name > marker:
filtered_objs.append(object_name)
return filtered_objs
def filter_end_marker(self, objects, end_marker):
"""
Accept sorted list.
"""
filtered_objs=[]
for object_name in objects:
if object_name < end_marker:
filtered_objs.append(object_name)
else:
break
return filtered_objs
def filter_limit(self, objects, limit):
filtered_objs=[]
for i in range(0, limit):
filtered_objs.append(objects[i])
return filtered_objs
class DiskDir(DiskCommon):
"""
Manage object files on disk.
:param path: path to devices on the node
:param drive: gluster volume drive name
:param account: account name for the object
:param container: container name for the object
:param logger: account or container server logging object
:param uid: user ID container object should assume
:param gid: group ID container object should assume
"""
def __init__(self, path, drive, account, container, logger,
uid=DEFAULT_UID, gid=DEFAULT_GID):
self.root = path
if container:
self.container = container
else:
self.container = None
if self.container:
self.datadir = os.path.join(path, drive, self.container)
else:
self.datadir = os.path.join(path, drive)
self.account = account
assert logger is not None
self.logger = logger
self.metadata = {}
self.container_info = None
self.object_info = None
self.uid = int(uid)
self.gid = int(gid)
self.db_file = _db_file
self.dir_exists = os_path.exists(self.datadir)
if self.dir_exists:
try:
self.metadata = _read_metadata(self.datadir)
except EOFError:
create_container_metadata(self.datadir)
else:
return
if self.container:
if not self.metadata:
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_container(self.metadata):
create_container_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not self.metadata:
create_account_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
else:
if not validate_account(self.metadata):
create_account_metadata(self.datadir)
self.metadata = _read_metadata(self.datadir)
def empty(self):
return dir_empty(self.datadir)
def delete(self):
if self.empty():
#For delete account.
if os_path.ismount(self.datadir):
clean_metadata(self.datadir)
else:
rmdirs(self.datadir)
self.dir_exists = False
def put_metadata(self, metadata):
"""
Write metadata to directory/container.
"""
write_metadata(self.datadir, metadata)
self.metadata = metadata
def put(self, metadata):
"""
Create and write metatdata to directory/container.
:param metadata: Metadata to write.
"""
if not self.dir_exists:
mkdirs(self.datadir)
os.chown(self.datadir, self.uid, self.gid)
write_metadata(self.datadir, metadata)
self.metadata = metadata
self.dir_exists = True
def put_obj(self, content_length, timestamp):
ocnt = self.metadata[X_OBJECTS_COUNT][0]
self.metadata[X_OBJECTS_COUNT] = (int(ocnt) + 1, timestamp)
self.metadata[X_PUT_TIMESTAMP] = timestamp
bused = self.metadata[X_BYTES_USED][0]
self.metadata[X_BYTES_USED] = (int(bused) + int(content_length), timestamp)
#TODO: define update_metadata instad of writing whole metadata again.
self.put_metadata(self.metadata)
def delete_obj(self, content_length):
ocnt, timestamp = self.metadata[X_OBJECTS_COUNT][0]
self.metadata[X_OBJECTS_COUNT] = (int(ocnt) - 1, timestamp)
bused, timestamp = self.metadata[X_BYTES_USED]
self.metadata[X_BYTES_USED] = (int(bused) - int(content_length), timestamp)
self.put_metadata(self.metadata)
def put_container(self, container, put_timestamp, del_timestamp, object_count, bytes_used):
"""
For account server.
"""
self.metadata[X_OBJECTS_COUNT] = (0, put_timestamp)
self.metadata[X_BYTES_USED] = (0, put_timestamp)
ccnt = self.metadata[X_CONTAINER_COUNT][0]
self.metadata[X_CONTAINER_COUNT] = (int(ccnt) + 1, put_timestamp)
self.metadata[X_PUT_TIMESTAMP] = (1, put_timestamp)
self.put_metadata(self.metadata)
def delete_container(self, object_count, bytes_used):
"""
For account server.
"""
self.metadata[X_OBJECTS_COUNT] = (0, 0)
self.metadata[X_BYTES_USED] = (0, 0)
ccnt, timestamp = self.metadata[X_CONTAINER_COUNT]
self.metadata[X_CONTAINER_COUNT] = (int(ccnt) - 1, timestamp)
self.put_metadata(self.metadata)
def unlink(self):
"""
Remove directory/container if empty.
"""
if dir_empty(self.datadir):
rmdirs(self.datadir)
def list_objects_iter(self, limit, marker, end_marker,
prefix, delimiter, path):
"""
Returns tuple of name, created_at, size, content_type, etag.
"""
if path:
prefix = path = path.rstrip('/') + '/'
delimiter = '/'
if delimiter and not prefix:
prefix = ''
self.update_object_count()
objects, object_count, bytes_used = self.object_info
if objects and prefix:
objects = self.filter_prefix(objects, prefix)
if objects and delimiter:
objects = self.filter_delimiter(objects, delimiter, prefix)
if objects and marker:
objects = self.filter_marker(objects, marker)
if objects and end_marker:
objects = self.filter_end_marker(objects, end_marker)
if objects and limit:
if len(objects) > limit:
objects = self.filter_limit(objects, limit)
container_list = []
if objects:
for obj in objects:
list_item = []
list_item.append(obj)
obj_path = os.path.join(self.datadir, obj)
metadata = read_metadata(obj_path)
if not metadata or not validate_object(metadata):
metadata = create_object_metadata(obj_path)
if metadata:
list_item.append(metadata[X_TIMESTAMP])
list_item.append(int(metadata[X_CONTENT_LENGTH]))
list_item.append(metadata[X_CONTENT_TYPE])
list_item.append(metadata[X_ETAG])
container_list.append(list_item)
return container_list
def update_object_count(self):
if not self.object_info:
self.object_info = get_container_details(self.datadir)
objects, object_count, bytes_used = self.object_info
if X_OBJECTS_COUNT not in self.metadata \
or int(self.metadata[X_OBJECTS_COUNT][0]) != object_count \
or X_BYTES_USED not in self.metadata \
or int(self.metadata[X_BYTES_USED][0]) != bytes_used:
self.metadata[X_OBJECTS_COUNT] = (object_count, 0)
self.metadata[X_BYTES_USED] = (bytes_used, 0)
write_metadata(self.datadir, self.metadata)
def update_container_count(self):
if not self.container_info:
self.container_info = get_account_details(self.datadir)
containers, container_count = self.container_info
if X_CONTAINER_COUNT not in self.metadata \
or int(self.metadata[X_CONTAINER_COUNT][0]) != container_count:
self.metadata[X_CONTAINER_COUNT] = (container_count, 0)
write_metadata(self.datadir, self.metadata)
def get_info(self, include_metadata=False):
"""
Get global data for the container.
:returns: dict with keys: account, container, object_count, bytes_used,
hash, id, created_at, put_timestamp, delete_timestamp,
reported_put_timestamp, reported_delete_timestamp,
reported_object_count, and reported_bytes_used.
If include_metadata is set, metadata is included as a key
pointing to a dict of tuples of the metadata
"""
# TODO: delete_timestamp, reported_put_timestamp
# reported_delete_timestamp, reported_object_count,
# reported_bytes_used, created_at
if not Glusterfs.OBJECT_ONLY:
# If we are not configured for object only environments, we should
# update the object counts in case they changed behind our back.
self.update_object_count()
data = {'account' : self.account, 'container' : self.container,
'object_count' : self.metadata.get(X_OBJECTS_COUNT, ('0', 0))[0],
'bytes_used' : self.metadata.get(X_BYTES_USED, ('0',0))[0],
'hash': '', 'id' : '', 'created_at' : '1',
'put_timestamp' : self.metadata.get(X_PUT_TIMESTAMP, ('0',0))[0],
'delete_timestamp' : '1',
'reported_put_timestamp' : '1', 'reported_delete_timestamp' : '1',
'reported_object_count' : '1', 'reported_bytes_used' : '1'}
if include_metadata:
data['metadata'] = self.metadata
return data
def put_object(self, name, timestamp, size, content_type,
etag, deleted=0):
# TODO: Implement the specifics of this func.
pass
def initialize(self, timestamp):
pass
def update_put_timestamp(self, timestamp):
"""
Create the container if it doesn't exist and update the timestamp
"""
if not os_path.exists(self.datadir):
self.put(self.metadata)
def delete_object(self, name, timestamp):
# TODO: Implement the delete object
pass
def delete_db(self, timestamp):
"""
Delete the container
"""
self.unlink()
def update_metadata(self, metadata):
assert self.metadata, "Valid container/account metadata should have been created by now"
if metadata:
new_metadata = self.metadata.copy()
new_metadata.update(metadata)
if new_metadata != self.metadata:
write_metadata(self.datadir, new_metadata)
self.metadata = new_metadata
class DiskAccount(DiskDir):
def __init__(self, root, drive, account, logger):
super(DiskAccount, self).__init__(root, drive, account, None, logger)
assert self.dir_exists
def list_containers_iter(self, limit, marker, end_marker,
prefix, delimiter):
"""
Return tuple of name, object_count, bytes_used, 0(is_subdir).
Used by account server.
"""
if delimiter and not prefix:
prefix = ''
self.update_container_count()
containers, container_count = self.container_info
if containers:
containers.sort()
if containers and prefix:
containers = self.filter_prefix(containers, prefix)
if containers and delimiter:
containers = self.filter_delimiter(containers, delimiter, prefix)
if containers and marker:
containers = self.filter_marker(containers, marker)
if containers and end_marker:
containers = self.filter_end_marker(containers, end_marker)
if containers and limit:
if len(containers) > limit:
containers = self.filter_limit(containers, limit)
account_list = []
if containers:
for cont in containers:
list_item = []
metadata = None
list_item.append(cont)
cont_path = os.path.join(self.datadir, cont)
metadata = _read_metadata(cont_path)
if not metadata or not validate_container(metadata):
metadata = create_container_metadata(cont_path)
if metadata:
list_item.append(metadata[X_OBJECTS_COUNT][0])
list_item.append(metadata[X_BYTES_USED][0])
list_item.append(0)
account_list.append(list_item)
return account_list
def get_info(self, include_metadata=False):
"""
Get global data for the account.
:returns: dict with keys: account, created_at, put_timestamp,
delete_timestamp, container_count, object_count,
bytes_used, hash, id
"""
if not Glusterfs.OBJECT_ONLY:
# If we are not configured for object only environments, we should
# update the container counts in case they changed behind our back.
self.update_container_count()
data = {'account' : self.account, 'created_at' : '1',
'put_timestamp' : '1', 'delete_timestamp' : '1',
'container_count' : self.metadata.get(X_CONTAINER_COUNT, (0,0))[0],
'object_count' : self.metadata.get(X_OBJECTS_COUNT, (0,0))[0],
'bytes_used' : self.metadata.get(X_BYTES_USED, (0,0))[0],
'hash' : '', 'id' : ''}
if include_metadata:
data['metadata'] = self.metadata
return data
def get_container_timestamp(self, container):
cont_path = os.path.join(self.datadir, container)
metadata = read_metadata(cont_path)
return int(metadata.get(X_PUT_TIMESTAMP, ('0',0))[0]) or None

338
ufo/gluster/swift/common/DiskFile.py
View File

@ -1,338 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import errno
import random
from hashlib import md5
from contextlib import contextmanager
from swift.common.utils import normalize_timestamp, renamer
from swift.common.exceptions import DiskFileNotExist
from gluster.swift.common.exceptions import AlreadyExistsAsDir
from gluster.swift.common.utils import mkdirs, rmdirs, validate_object, \
create_object_metadata, do_open, do_close, do_unlink, do_chown, \
do_listdir, read_metadata, write_metadata, os_path, do_fsync
from gluster.swift.common.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, \
X_TIMESTAMP, X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, \
X_BYTES_USED, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, OBJECT, DIR_TYPE, \
FILE_TYPE, DEFAULT_UID, DEFAULT_GID
import logging
from swift.obj.server import DiskFile
DEFAULT_DISK_CHUNK_SIZE = 65536
# keep these lower-case
DISALLOWED_HEADERS = set('content-length content-type deleted etag'.split())
def _adjust_metadata(metadata):
# Fix up the metadata to ensure it has a proper value for the
# Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE
# metadata values.
content_type = metadata['Content-Type']
if not content_type:
# FIXME: How can this be that our caller supplied us with metadata
# that has a content type that evaluates to False?
#
# FIXME: If the file exists, we would already know it is a
# directory. So why are we assuming it is a file object?
metadata['Content-Type'] = FILE_TYPE
x_object_type = FILE
else:
x_object_type = MARKER_DIR if content_type.lower() == DIR_TYPE else FILE
metadata[X_TYPE] = OBJECT
metadata[X_OBJECT_TYPE] = x_object_type
return metadata
class Gluster_DiskFile(DiskFile):
"""
Manage object files on disk.
:param path: path to devices on the node/mount path for UFO.
:param device: device name/account_name for UFO.
:param partition: partition on the device the object lives in
:param account: account name for the object
:param container: container name for the object
:param obj: object name for the object
:param logger: logger object for writing out log file messages
:param keep_data_fp: if True, don't close the fp, otherwise close it
:param disk_chunk_Size: size of chunks on file reads
:param uid: user ID disk object should assume (file or directory)
:param gid: group ID disk object should assume (file or directory)
"""
def __init__(self, path, device, partition, account, container, obj,
logger, keep_data_fp=False,
disk_chunk_size=DEFAULT_DISK_CHUNK_SIZE,
uid=DEFAULT_UID, gid=DEFAULT_GID, iter_hook=None):
self.disk_chunk_size = disk_chunk_size
self.iter_hook = iter_hook
# Don't support obj_name ending/begining with '/', like /a, a/, /a/b/,
# etc.
obj = obj.strip(os.path.sep)
if os.path.sep in obj:
self._obj_path, self._obj = os.path.split(obj)
else:
self._obj_path = ''
self._obj = obj
if self._obj_path:
self.name = os.path.join(container, self._obj_path)
else:
self.name = container
# Absolute path for object directory.
self.datadir = os.path.join(path, device, self.name)
self.device_path = os.path.join(path, device)
self._container_path = os.path.join(path, device, container)
self._is_dir = False
self.tmppath = None
self.logger = logger
self.metadata = {}
self.meta_file = None
self.fp = None
self.iter_etag = None
self.started_at_0 = False
self.read_to_eof = False
self.quarantined_dir = None
self.keep_cache = False
self.uid = int(uid)
self.gid = int(gid)
# Don't store a value for data_file until we know it exists.
self.data_file = None
data_file = os.path.join(self.datadir, self._obj)
if not os_path.exists(data_file):
return
self.data_file = os.path.join(data_file)
self.metadata = read_metadata(data_file)
if not self.metadata:
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
if not validate_object(self.metadata):
create_object_metadata(data_file)
self.metadata = read_metadata(data_file)
self.filter_metadata()
if os_path.isdir(data_file):
self._is_dir = True
else:
if keep_data_fp:
# The caller has an assumption that the "fp" field of this
# object is an file object if keep_data_fp is set. However,
# this implementation of the DiskFile object does not need to
# open the file for internal operations. So if the caller
# requests it, we'll just open the file for them.
self.fp = do_open(data_file, 'rb')
def close(self, verify_file=True):
"""
Close the file. Will handle quarantining file if necessary.
:param verify_file: Defaults to True. If false, will not check
file to see if it needs quarantining.
"""
#Marker directory
if self._is_dir:
return
if self.fp:
do_close(self.fp)
self.fp = None
def is_deleted(self):
"""
Check if the file is deleted.
:returns: True if the file doesn't exist or has been flagged as
deleted.
"""
return not self.data_file
def _create_dir_object(self, dir_path):
#TODO: if object already exists???
if os_path.exists(dir_path) and not os_path.isdir(dir_path):
self.logger.error("Deleting file %s", dir_path)
do_unlink(dir_path)
#If dir aleady exist just override metadata.
mkdirs(dir_path)
do_chown(dir_path, self.uid, self.gid)
create_object_metadata(dir_path)
def put_metadata(self, metadata, tombstone=False):
"""
Short hand for putting metadata to .meta and .ts files.
:param metadata: dictionary of metadata to be written
:param tombstone: whether or not we are writing a tombstone
"""
if tombstone:
# We don't write tombstone files. So do nothing.
return
assert self.data_file is not None, "put_metadata: no file to put metadata into"
metadata = _adjust_metadata(metadata)
write_metadata(self.data_file, metadata)
self.metadata = metadata
self.filter_metadata()
def put(self, fd, metadata, extension='.data'):
"""
Finalize writing the file on disk, and renames it from the temp file to
the real location. This should be called after the data has been
written to the temp file.
:param fd: file descriptor of the temp file
:param metadata: dictionary of metadata to be written
:param extension: extension to be used when making the file
"""
# Our caller will use '.data' here; we just ignore it since we map the
# URL directly to the file system.
extension = ''
metadata = _adjust_metadata(metadata)
if metadata[X_OBJECT_TYPE] == MARKER_DIR:
if not self.data_file:
self.data_file = os.path.join(self.datadir, self._obj)
self._create_dir_object(self.data_file)
self.put_metadata(metadata)
return
# Check if directory already exists.
if self._is_dir:
# FIXME: How can we have a directory and it not be marked as a
# MARKER_DIR (see above)?
msg = 'File object exists as a directory: %s' % self.data_file
raise AlreadyExistsAsDir(msg)
timestamp = normalize_timestamp(metadata[X_TIMESTAMP])
write_metadata(self.tmppath, metadata)
if X_CONTENT_LENGTH in metadata:
self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH]))
do_fsync(fd)
if self._obj_path:
dir_objs = self._obj_path.split('/')
assert len(dir_objs) >= 1
tmp_path = self._container_path
for dir_name in dir_objs:
tmp_path = os.path.join(tmp_path, dir_name)
self._create_dir_object(tmp_path)
newpath = os.path.join(self.datadir, self._obj)
renamer(self.tmppath, newpath)
do_chown(newpath, self.uid, self.gid)
self.metadata = metadata
self.data_file = newpath
self.filter_metadata()
return
def unlinkold(self, timestamp):
"""
Remove any older versions of the object file. Any file that has an
older timestamp than timestamp will be deleted.
:param timestamp: timestamp to compare with each file
"""
if not self.metadata or self.metadata['X-Timestamp'] >= timestamp:
return
assert self.data_file, \
"Have metadata, %r, but no data_file" % self.metadata
if self._is_dir:
# Marker directory object
if not rmdirs(self.data_file):
logging.error('Unable to delete dir object: %s', self.data_file)
return
else:
# File object
do_unlink(self.data_file)
self.metadata = {}
self.data_file = None
def get_data_file_size(self):
"""
Returns the os_path.getsize for the file. Raises an exception if this
file does not match the Content-Length stored in the metadata. Or if
self.data_file does not exist.
:returns: file size as an int
:raises DiskFileError: on file size mismatch.
:raises DiskFileNotExist: on file not existing (including deleted)
"""
#Marker directory.
if self._is_dir:
return 0
try:
file_size = 0
if self.data_file:
file_size = os_path.getsize(self.data_file)
if X_CONTENT_LENGTH in self.metadata:
metadata_size = int(self.metadata[X_CONTENT_LENGTH])
if file_size != metadata_size:
self.metadata[X_CONTENT_LENGTH] = file_size
write_metadata(self.data_file, self.metadata)
return file_size
except OSError as err:
if err.errno != errno.ENOENT:
raise
raise DiskFileNotExist('Data File does not exist.')
def filter_metadata(self):
if X_TYPE in self.metadata:
self.metadata.pop(X_TYPE)
if X_OBJECT_TYPE in self.metadata:
self.metadata.pop(X_OBJECT_TYPE)
@contextmanager
def mkstemp(self):
"""Contextmanager to make a temporary file."""
# Creating intermidiate directories and corresponding metadata.
# For optimization, check if the subdirectory already exists,
# if exists, then it means that it also has its metadata.
# Not checking for container, since the container should already
# exist for the call to come here.
if not os_path.exists(self.datadir):
path = self._container_path
subdir_list = self._obj_path.split(os.path.sep)
for i in range(len(subdir_list)):
path = os.path.join(path, subdir_list[i]);
if not os_path.exists(path):
self._create_dir_object(path)
tmpfile = '.' + self._obj + '.' + md5(self._obj + \
str(random.random())).hexdigest()
self.tmppath = os.path.join(self.datadir, tmpfile)
fd = do_open(self.tmppath, os.O_RDWR | os.O_CREAT | os.O_EXCL)
try:
yield fd
finally:
try:
do_close(fd)
except OSError:
pass
tmppath, self.tmppath = self.tmppath, None
try:
do_unlink(tmppath)
except OSError as err:
if err.errno != errno.ENOENT:
raise

144
ufo/gluster/swift/common/Glusterfs.py
View File

@ -1,144 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os, sys, fcntl, time, errno
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from swift.common.utils import TRUE_VALUES, search_tree
from gluster.swift.common.fs_utils import mkdirs
#
# Read the fs.conf file once at startup (module load)
#
_fs_conf = ConfigParser()
MOUNT_IP = 'localhost'
OBJECT_ONLY = False
RUN_DIR='/var/run/swift'
SWIFT_DIR = '/etc/swift'
_do_getsize = False
if _fs_conf.read(os.path.join('/etc/swift', 'fs.conf')):
try:
MOUNT_IP = _fs_conf.get('DEFAULT', 'mount_ip', 'localhost')
except (NoSectionError, NoOptionError):
pass
try:
OBJECT_ONLY = _fs_conf.get('DEFAULT', 'object_only', "no") in TRUE_VALUES
except (NoSectionError, NoOptionError):
pass
try:
RUN_DIR = _fs_conf.get('DEFAULT', 'run_dir', '/var/run/swift')
except (NoSectionError, NoOptionError):
pass
try:
_do_getsize = _fs_conf.get('DEFAULT', 'accurate_size_in_listing', \
"no") in TRUE_VALUES
except (NoSectionError, NoOptionError):
pass
NAME = 'glusterfs'
def _busy_wait(full_mount_path):
# Iterate for definite number of time over a given
# interval for successful mount
for i in range(0, 5):
if os.path.ismount(os.path.join(full_mount_path)):
return True
time.sleep(2)
logging.error('Busy wait for mount timed out for mount %s', full_mount_path)
return False
def mount(root, drive):
# FIXME: Possible thundering herd problem here
el = _get_export_list()
for export in el:
if drive == export:
break
else:
logging.error('No export found in %r matching drive, %s', el, drive)
return False
# NOTE: root is typically the default value of /mnt/gluster-object
full_mount_path = os.path.join(root, drive)
if not os.path.isdir(full_mount_path):
mkdirs(full_mount_path)
lck_file = os.path.join(RUN_DIR, '%s.lock' %drive);
if not os.path.exists(RUN_DIR):
mkdirs(RUN_DIR)
fd = os.open(lck_file, os.O_CREAT|os.O_RDWR)
with os.fdopen(fd, 'r+b') as f:
try:
fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB)
except IOError as ex:
if ex.errno in (errno.EACCES, errno.EAGAIN):
# This means that some other process is mounting the
# filesystem, so wait for the mount process to complete
return _busy_wait(full_mount_path)
else:
raise ex
mnt_cmd = 'mount -t glusterfs %s:%s %s' % (MOUNT_IP, export, \
full_mount_path)
if os.system(mnt_cmd) or not _busy_wait(full_mount_path):
logging.error('Mount failed %s: %s', NAME, mnt_cmd)
return False
return True
def unmount(full_mount_path):
# FIXME: Possible thundering herd problem here
umnt_cmd = 'umount %s 2>> /dev/null' % full_mount_path
if os.system(umnt_cmd):
logging.error('Unable to unmount %s %s' % (full_mount_path, NAME))
def _get_export_list():
cmnd = 'gluster --remote-host=%s volume info' % MOUNT_IP
export_list = []
if os.system(cmnd + ' >> /dev/null'):
logging.error('Getting volume info failed for %s', NAME)
else:
fp = os.popen(cmnd)
while True:
item = fp.readline()
if not item:
break
item = item.strip('\n').strip(' ')
if item.lower().startswith('volume name:'):
export_list.append(item.split(':')[1].strip(' '))
return export_list
def get_mnt_point(vol_name, conf_dir=SWIFT_DIR, conf_file="object-server*"):
"""Read the object-server's configuration file and return
the device value"""
mnt_dir = ''
conf_files = search_tree(conf_dir, conf_file, '.conf')
if not conf_files:
raise Exception("Config file not found")
_conf = ConfigParser()
if _conf.read(conf_files[0]):
try:
mnt_dir = _conf.get('DEFAULT', 'devices', '')
except (NoSectionError, NoOptionError):
raise
return os.path.join(mnt_dir, vol_name)

0
ufo/gluster/swift/common/__init__.py
View File

94
ufo/gluster/swift/common/constraints.py
View File

@ -1,94 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
try:
from webob.exc import HTTPBadRequest
except ImportError:
from swift.common.swob import HTTPBadRequest
import swift.common.constraints
import swift.common.ring as _ring
from gluster.swift.common import Glusterfs, ring
if hasattr(swift.common.constraints, 'constraints_conf_int'):
MAX_OBJECT_NAME_COMPONENT_LENGTH = \
swift.common.constraints.constraints_conf_int(
'max_object_name_component_length', 255)
else:
MAX_OBJECT_NAME_COMPONENT_LENGTH = 255
def validate_obj_name_component(obj):
if len(obj) > MAX_OBJECT_NAME_COMPONENT_LENGTH:
return 'too long (%d)' % len(obj)
if obj == '.' or obj == '..':
return 'cannot be . or ..'
return ''
# Save the original check object creation
__check_object_creation = swift.common.constraints.check_object_creation
# Define our new one which invokes the original
def gluster_check_object_creation(req, object_name):
"""
Check to ensure that everything is alright about an object to be created.
Monkey patches swift.common.constraints.check_object_creation, invoking
the original, and then adding an additional check for individual object
name components.
:param req: HTTP request object
:param object_name: name of object to be created
:raises HTTPRequestEntityTooLarge: the object is too large
:raises HTTPLengthRequered: missing content-length header and not
a chunked request
:raises HTTPBadRequest: missing or bad content-type header, or
bad metadata
"""
ret = __check_object_creation(req, object_name)
if ret is None:
for obj in object_name.split('/'):
reason = validate_obj_name_component(obj)
if reason:
bdy = 'Invalid object name "%s", component "%s" %s' \
% (object_name, obj, reason)
ret = HTTPBadRequest(body=bdy,
request=req,
content_type='text/plain')
return ret
# Replace the original check object creation with ours
swift.common.constraints.check_object_creation = gluster_check_object_creation
# Save the original check mount
__check_mount = swift.common.constraints.check_mount
# Define our new one which invokes the original
def gluster_check_mount(root, drive):
# FIXME: Potential performance optimization here to not call the original
# check mount which makes two stat calls. We could do what they do with
# just one.
if __check_mount(root, drive):
return True
return Glusterfs.mount(root, drive)
# Replace the original check mount with ours
swift.common.constraints.check_mount = gluster_check_mount
# Save the original Ring class
__Ring = _ring.Ring
# Replace the original Ring class
_ring.Ring = ring.Ring

27
ufo/gluster/swift/common/exceptions.py
View File

@ -1,27 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class GlusterfsException(Exception):
pass
class FileOrDirNotFoundError(GlusterfsException):
pass
class NotDirectoryError(GlusterfsException):
pass
class AlreadyExistsAsDir(GlusterfsException):
pass

179
ufo/gluster/swift/common/fs_utils.py
View File

@ -1,179 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
import errno
import os.path as os_path
from eventlet import tpool
from gluster.swift.common.exceptions import FileOrDirNotFoundError, \
NotDirectoryError
def do_walk(*args, **kwargs):
return os.walk(*args, **kwargs)
def do_write(fd, msg):
try:
cnt = os.write(fd, msg)
except OSError as err:
logging.exception("Write failed, err: %s", str(err))
raise
return cnt
def do_mkdir(path):
try:
os.mkdir(path)
except OSError as err:
if err.errno != errno.EEXIST:
logging.exception("Mkdir failed on %s err: %s", path, err.strerror)
raise
return True
def do_makedirs(path):
try:
os.makedirs(path)
except OSError as err:
if err.errno != errno.EEXIST:
logging.exception("Makedirs failed on %s err: %s", path, err.strerror)
raise
return True
def do_listdir(path):
try:
buf = os.listdir(path)
except OSError as err:
logging.exception("Listdir failed on %s err: %s", path, err.strerror)
raise
return buf
def do_chown(path, uid, gid):
try:
os.chown(path, uid, gid)
except OSError as err:
logging.exception("Chown failed on %s err: %s", path, err.strerror)
raise
return True
def do_stat(path):
try:
#Check for fd.
if isinstance(path, int):
buf = os.fstat(path)
else:
buf = os.stat(path)
except OSError as err:
logging.exception("Stat failed on %s err: %s", path, err.strerror)
raise
return buf
def do_open(path, mode):
if isinstance(mode, int):
try:
fd = os.open(path, mode)
except OSError as err:
logging.exception("Open failed on %s err: %s", path, str(err))
raise
else:
try:
fd = open(path, mode)
except IOError as err:
logging.exception("Open failed on %s err: %s", path, str(err))
raise
return fd
def do_close(fd):
#fd could be file or int type.
try:
if isinstance(fd, int):
os.close(fd)
else:
fd.close()
except OSError as err:
logging.exception("Close failed on %s err: %s", fd, err.strerror)
raise
return True
def do_unlink(path, log = True):
try:
os.unlink(path)
except OSError as err:
if err.errno != errno.ENOENT:
if log:
logging.exception("Unlink failed on %s err: %s", path, err.strerror)
raise
return True
def do_rmdir(path):
try:
os.rmdir(path)
except OSError as err:
if err.errno != errno.ENOENT:
logging.exception("Rmdir failed on %s err: %s", path, err.strerror)
raise
res = False
else:
res = True
return res
def do_rename(old_path, new_path):
try:
os.rename(old_path, new_path)
except OSError as err:
logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \
err.strerror)
raise
return True
def mkdirs(path):
"""
Ensures the path is a directory or makes it if not. Errors if the path
exists but is a file or on permissions failure.
:param path: path to create
"""
if not os.path.isdir(path):
do_makedirs(path)
def dir_empty(path):
"""
Return true if directory/container is empty.
:param path: Directory path.
:returns: True/False.
"""
if os.path.isdir(path):
files = do_listdir(path)
return not files
elif not os.path.exists(path):
raise FileOrDirNotFoundError()
raise NotDirectoryError()
def rmdirs(path):
if not os.path.isdir(path):
return False
try:
os.rmdir(path)
except OSError as err:
if err.errno != errno.ENOENT:
logging.error("rmdirs failed on %s, err: %s", path, err.strerror)
return False
return True
def do_fsync(fd):
try:
tpool.execute(os.fsync, fd)
except OSError as err:
logging.exception("fsync failed with err: %s", err.strerror)
raise
return True

0
ufo/gluster/swift/common/middleware/__init__.py
View File

40
ufo/gluster/swift/common/middleware/gluster.py
View File

@ -1,40 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Noop Middleware that simply allows us to monkey patch the constraints
import gluster.swift.common.constraints
class Gluster(object):
"""
Noop middleware for use with the proxy server to get paste.deploy to load
this middleware such that the plugin constraints monkey patch the common
constraints ahead of their use.
"""
def __init__(self, app, conf):
self.app = app
self.conf = conf
def __call__(self, env, start_response):
return self.app(env, start_response)
def filter_factory(global_conf, **local_conf):
"""Returns a WSGI filter app for use with paste.deploy."""
conf = global_conf.copy()
conf.update(local_conf)
def gluster_filter(app):
return Gluster(app, conf)
return gluster_filter

111
ufo/gluster/swift/common/ring.py
View File

@ -1,111 +0,0 @@
# Copyright (c) 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ConfigParser import ConfigParser
from swift.common.ring import ring
from swift.common.utils import search_tree
from gluster.swift.common.Glusterfs import SWIFT_DIR
reseller_prefix = "AUTH_"
conf_files = search_tree(SWIFT_DIR, "proxy-server*", 'conf')
if conf_files:
conf_file = conf_files[0]
_conf = ConfigParser()
if conf_files and _conf.read(conf_file):
if _conf.defaults().get("reseller_prefix", None):
reseller_prefix = _conf.defaults().get("reseller_prefix")
else:
for key, value in _conf._sections.items():
if value.get("reseller_prefix", None):
reseller_prefix = value["reseller_prefix"]
break
if not reseller_prefix.endswith('_'):
reseller_prefix = reseller_prefix + '_'
class Ring(ring.Ring):
def _get_part_nodes(self, part):
seen_ids = set()
nodes = [dev for dev in self._devs \
if dev['device'] == self.acc_name \
and not (dev['id'] in seen_ids \
or seen_ids.add(dev['id']))]
if not nodes:
nodes = [self.false_node]
return nodes
def get_part_nodes(self, part):
"""
Get the nodes that are responsible for the partition. If one
node is responsible for more than one replica of the same
partition, it will only appear in the output once.
:param part: partition to get nodes for
:returns: list of node dicts
See :func:`get_nodes` for a description of the node dicts.
"""
return self._get_part_nodes(part)
def get_nodes(self, account, container=None, obj=None):
"""
Get the partition and nodes for an account/container/object.
If a node is responsible for more than one replica, it will
only appear in the output once.
:param account: account name
:param container: container name
:param obj: object name
:returns: a tuple of (partition, list of node dicts)
Each node dict will have at least the following keys:
====== ===============================================================
id unique integer identifier amongst devices
weight a float of the relative weight of this device as compared to
others; this indicates how many partitions the builder will try
to assign to this device
zone integer indicating which zone the device is in; a given
partition will not be assigned to multiple devices within the
same zone
ip the ip address of the device
port the tcp port of the device
device the device's name on disk (sdb1, for example)
meta general use 'extra' field; for example: the online date, the
hardware description
====== ===============================================================
"""
self.false_node = {'zone': 1, 'weight': 100.0, 'ip': '127.0.0.1', 'id': 0, \
'meta': '', 'device': 'volume_not_in_ring', \
'port': 6012}
if account.startswith(reseller_prefix):
self.acc_name = account.replace(reseller_prefix, '', 1)
else:
self.acc_name = account
part = 0
return part, self._get_part_nodes(part)
def get_more_nodes(self, part):
"""
Generator to get extra nodes for a partition for hinted handoff.
:param part: partition to get handoff nodes for
:returns: generator of node dicts
See :func:`get_nodes` for a description of the node dicts.
Should never be called in the swift UFO environment, so yield nothing
"""
yield self.false_node

496
ufo/gluster/swift/common/utils.py
View File

@ -1,496 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
import errno
import xattr
import random
from hashlib import md5
from eventlet import sleep
import cPickle as pickle
from ConfigParser import ConfigParser, NoSectionError, NoOptionError
from swift.common.utils import normalize_timestamp, TRUE_VALUES
from gluster.swift.common.fs_utils import *
from gluster.swift.common import Glusterfs
X_CONTENT_TYPE = 'Content-Type'
X_CONTENT_LENGTH = 'Content-Length'
X_TIMESTAMP = 'X-Timestamp'
X_PUT_TIMESTAMP = 'X-PUT-Timestamp'
X_TYPE = 'X-Type'
X_ETAG = 'ETag'
X_OBJECTS_COUNT = 'X-Object-Count'
X_BYTES_USED = 'X-Bytes-Used'
X_CONTAINER_COUNT = 'X-Container-Count'
X_OBJECT_TYPE = 'X-Object-Type'
DIR_TYPE = 'application/directory'
ACCOUNT = 'Account'
METADATA_KEY = 'user.swift.metadata'
MAX_XATTR_SIZE = 65536
CONTAINER = 'container'
DIR = 'dir'
MARKER_DIR = 'marker_dir'
TEMP_DIR = 'tmp'
ASYNCDIR = 'async_pending' # Keep in sync with swift.obj.server.ASYNCDIR
FILE = 'file'
FILE_TYPE = 'application/octet-stream'
OBJECT = 'Object'
OBJECT_TYPE = 'application/octet-stream'
DEFAULT_UID = -1
DEFAULT_GID = -1
PICKLE_PROTOCOL = 2
CHUNK_SIZE = 65536
MEMCACHE_KEY_PREFIX = 'gluster.swift.'
MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.'
MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.'
def read_metadata(path):
"""
Helper function to read the pickled metadata from a File/Directory.
:param path: File/Directory to read metadata from.
:returns: dictionary of metadata
"""
metadata = None
metadata_s = ''
key = 0
while metadata is None:
try:
metadata_s += xattr.getxattr(path, '%s%s' % (METADATA_KEY, (key or '')))
except IOError as err:
if err.errno == errno.ENODATA:
if key > 0:
# No errors reading the xattr keys, but since we have not
# been able to find enough chunks to get a successful
# unpickle operation, we consider the metadata lost, and
# drop the existing data so that the internal state can be
# recreated.
clean_metadata(path)
# We either could not find any metadata key, or we could find
# some keys, but were not successful in performing the
# unpickling (missing keys perhaps)? Either way, just report
# to the caller we have no metadata.
metadata = {}
else:
logging.exception("xattr.getxattr failed on %s key %s err: %s",
path, key, str(err))
# Note that we don't touch the keys on errors fetching the
# data since it could be a transient state.
raise
else:
try:
# If this key provides all or the remaining part of the pickle
# data, we don't need to keep searching for more keys. This
# means if we only need to store data in N xattr key/value
# pair, we only need to invoke xattr get N times. With large
# keys sizes we are shooting for N = 1.
metadata = pickle.loads(metadata_s)
assert isinstance(metadata, dict)
except EOFError, pickle.UnpicklingError:
# We still are not able recognize this existing data collected
# as a pickled object. Make sure we loop around to try to get
# more from another xattr key.
metadata = None
key += 1
return metadata
def write_metadata(path, metadata):
"""
Helper function to write pickled metadata for a File/Directory.
:param path: File/Directory path to write the metadata
:param metadata: dictionary to metadata write
"""
assert isinstance(metadata, dict)
metastr = pickle.dumps(metadata, PICKLE_PROTOCOL)
key = 0
while metastr:
try:
xattr.setxattr(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:MAX_XATTR_SIZE])
except IOError as err:
logging.exception("setxattr failed on %s key %s err: %s", path, key, str(err))
raise
metastr = metastr[MAX_XATTR_SIZE:]
key += 1
def clean_metadata(path):
key = 0
while True:
try:
xattr.removexattr(path, '%s%s' % (METADATA_KEY, (key or '')))
except IOError as err:
if err.errno == errno.ENODATA:
break
raise
key += 1
def check_user_xattr(path):
if not os_path.exists(path):
return False
try:
xattr.setxattr(path, 'user.test.key1', 'value1')
except IOError as err:
logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err))
raise
try:
xattr.removexattr(path, 'user.test.key1')
except IOError as err:
logging.exception("check_user_xattr: remove failed on %s err: %s", path, str(err))
#Remove xattr may fail in case of concurrent remove.
return True
def validate_container(metadata):
if not metadata:
logging.warn('validate_container: No metadata')
return False
if X_TYPE not in metadata.keys() or \
X_TIMESTAMP not in metadata.keys() or \
X_PUT_TIMESTAMP not in metadata.keys() or \
X_OBJECTS_COUNT not in metadata.keys() or \
X_BYTES_USED not in metadata.keys():
#logging.warn('validate_container: Metadata missing entries: %s' % metadata)
return False
(value, timestamp) = metadata[X_TYPE]
if value == CONTAINER:
return True
logging.warn('validate_container: metadata type is not CONTAINER (%r)' % (value,))
return False
def validate_account(metadata):
if not metadata:
logging.warn('validate_account: No metadata')
return False
if X_TYPE not in metadata.keys() or \
X_TIMESTAMP not in metadata.keys() or \
X_PUT_TIMESTAMP not in metadata.keys() or \
X_OBJECTS_COUNT not in metadata.keys() or \
X_BYTES_USED not in metadata.keys() or \
X_CONTAINER_COUNT not in metadata.keys():
#logging.warn('validate_account: Metadata missing entries: %s' % metadata)
return False
(value, timestamp) = metadata[X_TYPE]
if value == ACCOUNT:
return True
logging.warn('validate_account: metadata type is not ACCOUNT (%r)' % (value,))
return False
def validate_object(metadata):
if not metadata:
logging.warn('validate_object: No metadata')
return False
if X_TIMESTAMP not in metadata.keys() or \
X_CONTENT_TYPE not in metadata.keys() or \
X_ETAG not in metadata.keys() or \
X_CONTENT_LENGTH not in metadata.keys() or \
X_TYPE not in metadata.keys() or \
X_OBJECT_TYPE not in metadata.keys():
#logging.warn('validate_object: Metadata missing entries: %s' % metadata)
return False
if metadata[X_TYPE] == OBJECT:
return True
logging.warn('validate_object: metadata type is not OBJECT (%r)' % (metadata[X_TYPE],))
return False
def is_marker(metadata):
if not metadata:
logging.warn('is_marker: No metadata')
return False
if X_OBJECT_TYPE not in metadata.keys():
logging.warn('is_marker: X_OBJECT_TYPE missing from metadata: %s' % metadata)
return False
if metadata[X_OBJECT_TYPE] == MARKER_DIR:
return True
else:
return False
def _update_list(path, cont_path, src_list, reg_file=True, object_count=0,
bytes_used=0, obj_list=[]):
# strip the prefix off, also stripping the leading and trailing slashes
obj_path = path.replace(cont_path, '').strip(os.path.sep)
for obj_name in src_list:
if obj_path:
obj_list.append(os.path.join(obj_path, obj_name))
else:
obj_list.append(obj_name)
object_count += 1
if Glusterfs._do_getsize and reg_file:
bytes_used += os_path.getsize(os.path.join(path, obj_name))
sleep()
return object_count, bytes_used
def update_list(path, cont_path, dirs=[], files=[], object_count=0,
bytes_used=0, obj_list=[]):
if files:
object_count, bytes_used = _update_list(path, cont_path, files, True,
object_count, bytes_used,
obj_list)
if dirs:
object_count, bytes_used = _update_list(path, cont_path, dirs, False,
object_count, bytes_used,
obj_list)
return object_count, bytes_used
class ContainerDetails(object):
def __init__(self, bytes_used, object_count, obj_list, dir_list):
self.bytes_used = bytes_used
self.object_count = object_count
self.obj_list = obj_list
self.dir_list = dir_list
def _get_container_details_from_fs(cont_path):
"""
get container details by traversing the filesystem
"""
bytes_used = 0
object_count = 0
obj_list = []
dir_list = []
if os_path.isdir(cont_path):
for (path, dirs, files) in do_walk(cont_path):
object_count, bytes_used = update_list(path, cont_path, dirs, files,
object_count, bytes_used,
obj_list)
dir_list.append((path, do_stat(path).st_mtime))
sleep()
return ContainerDetails(bytes_used, object_count, obj_list, dir_list)
def get_container_details(cont_path, memcache=None):
"""
Return object_list, object_count and bytes_used.
"""
mkey = ''
if memcache:
mkey = MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX + cont_path
cd = memcache.get(mkey)
if cd:
if not cd.dir_list:
cd = None
else:
for (path, mtime) in cd.dir_list:
if mtime != do_stat(path).st_mtime:
cd = None
else:
cd = None
if not cd:
cd = _get_container_details_from_fs(cont_path)
if memcache:
memcache.set(mkey, cd)
return cd.obj_list, cd.object_count, cd.bytes_used
class AccountDetails(object):
""" A simple class to store the three pieces of information associated
with an account:
1. The last known modification time
2. The count of containers in the following list
3. The list of containers
"""
def __init__(self, mtime, container_count, container_list):
self.mtime = mtime
self.container_count = container_count
self.container_list = container_list
def _get_account_details_from_fs(acc_path, acc_stats):
container_list = []
container_count = 0
if not acc_stats:
acc_stats = do_stat(acc_path)
is_dir = (acc_stats.st_mode & 0040000) != 0
if is_dir:
for name in do_listdir(acc_path):
if name.lower() == TEMP_DIR \
or name.lower() == ASYNCDIR \
or not os_path.isdir(os.path.join(acc_path, name)):
continue
container_count += 1
container_list.append(name)
return AccountDetails(acc_stats.st_mtime, container_count, container_list)
def get_account_details(acc_path, memcache=None):
"""
Return container_list and container_count.
"""
acc_stats = None
mkey = ''
if memcache:
mkey = MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX + acc_path
ad = memcache.get(mkey)
if ad:
# FIXME: Do we really need to stat the file? If we are object
# only, then we can track the other Swift HTTP APIs that would
# modify the account and invalidate the cached entry there. If we
# are not object only, are we even called on this path?
acc_stats = do_stat(acc_path)
if ad.mtime != acc_stats.st_mtime:
ad = None
else:
ad = None
if not ad:
ad = _get_account_details_from_fs(acc_path, acc_stats)
if memcache:
memcache.set(mkey, ad)
return ad.container_list, ad.container_count
def _get_etag(path):
etag = md5()
with open(path, 'rb') as fp:
while True:
chunk = fp.read(CHUNK_SIZE)
if chunk:
etag.update(chunk)
else:
break
return etag.hexdigest()
def get_object_metadata(obj_path):
"""
Return metadata of object.
"""
try:
stats = do_stat(obj_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
metadata = {}
else:
is_dir = (stats.st_mode & 0040000) != 0
metadata = {
X_TYPE: OBJECT,
X_TIMESTAMP: normalize_timestamp(stats.st_ctime),
X_CONTENT_TYPE: DIR_TYPE if is_dir else FILE_TYPE,
X_OBJECT_TYPE: DIR if is_dir else FILE,
X_CONTENT_LENGTH: 0 if is_dir else stats.st_size,
X_ETAG: md5().hexdigest() if is_dir else _get_etag(obj_path),
}
return metadata
def _add_timestamp(metadata_i):
# At this point we have a simple key/value dictionary, turn it into
# key/(value,timestamp) pairs.
timestamp = 0
metadata = {}
for key, value_i in metadata_i.iteritems():
if not isinstance(value_i, tuple):
metadata[key] = (value_i, timestamp)
else:
metadata[key] = value_i
return metadata
def get_container_metadata(cont_path, memcache=None):
objects = []
object_count = 0
bytes_used = 0
objects, object_count, bytes_used = get_container_details(cont_path, memcache)
metadata = {X_TYPE: CONTAINER,
X_TIMESTAMP: normalize_timestamp(os_path.getctime(cont_path)),
X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(cont_path)),
X_OBJECTS_COUNT: object_count,
X_BYTES_USED: bytes_used}
return _add_timestamp(metadata)
def get_account_metadata(acc_path, memcache=None):
containers = []
container_count = 0
containers, container_count = get_account_details(acc_path, memcache)
metadata = {X_TYPE: ACCOUNT,
X_TIMESTAMP: normalize_timestamp(os_path.getctime(acc_path)),
X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(acc_path)),
X_OBJECTS_COUNT: 0,
X_BYTES_USED: 0,
X_CONTAINER_COUNT: container_count}
return _add_timestamp(metadata)
def restore_metadata(path, metadata):
meta_orig = read_metadata(path)
if meta_orig:
meta_new = meta_orig.copy()
meta_new.update(metadata)
else:
meta_new = metadata
if meta_orig != meta_new:
write_metadata(path, meta_new)
return meta_new
def create_object_metadata(obj_path):
metadata = get_object_metadata(obj_path)
return restore_metadata(obj_path, metadata)
def create_container_metadata(cont_path, memcache=None):
metadata = get_container_metadata(cont_path, memcache)
return restore_metadata(cont_path, metadata)
def create_account_metadata(acc_path, memcache=None):
metadata = get_account_metadata(acc_path, memcache)
return restore_metadata(acc_path, metadata)
def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
"""
Ensure that a pickle file gets written to disk. The file is first written
to a tmp file location in the destination directory path, ensured it is
synced to disk, then moved to its final destination name.
This version takes advantage of Gluster's dot-prefix-dot-suffix naming
where the a file named ".thefile.name.9a7aasv" is hashed to the same
Gluster node as "thefile.name". This ensures the renaming of a temp file
once written does not move it to another Gluster node.
:param obj: python object to be pickled
:param dest: path of final destination file
:param tmp: path to tmp to use, defaults to None (ignored)
:param pickle_protocol: protocol to pickle the obj with, defaults to 0
"""
dirname = os.path.dirname(dest)
basename = os.path.basename(dest)
tmpname = '.' + basename + '.' + md5(basename + str(random.random())).hexdigest()
tmppath = os.path.join(dirname, tmpname)
with open(tmppath, 'wb') as fo:
pickle.dump(obj, fo, pickle_protocol)
# TODO: This flush() method call turns into a flush() system call
# We'll need to wrap this as well, but we would do this by writing
#a context manager for our own open() method which returns an object
# in fo which makes the gluster API call.
fo.flush()
do_fsync(fo)
do_rename(tmppath, dest)
# Over-ride Swift's utils.write_pickle with ours
import swift.common.utils
swift.common.utils.write_pickle = write_pickle

0
ufo/gluster/swift/container/__init__.py
View File

46
ufo/gluster/swift/container/server.py
View File

@ -1,46 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Container Server for Gluster Swift UFO """
# Simply importing this monkey patches the constraint handling to fit our
# needs
import gluster.swift.common.constraints
from swift.container import server
from gluster.swift.common.DiskDir import DiskDir
class ContainerController(server.ContainerController):
def _get_container_broker(self, drive, part, account, container):
"""
Overriden to provide the GlusterFS specific broker that talks to
Gluster for the information related to servicing a given request
instead of talking to a database.
:param drive: drive that holds the container
:param part: partition the container is in
:param account: account name
:param container: container name
:returns: DiskDir object
"""
return DiskDir(self.root, drive, account, container, self.logger)
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI container server apps."""
conf = global_conf.copy()
conf.update(local_conf)
return ContainerController(conf)

0
ufo/gluster/swift/obj/__init__.py
View File

34
ufo/gluster/swift/obj/server.py
View File

@ -1,34 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Object Server for Gluster Swift UFO """
# Simply importing this monkey patches the constraint handling to fit our
# needs
import gluster.swift.common.constraints
import gluster.swift.common.utils
from swift.obj import server
from gluster.swift.common.DiskFile import Gluster_DiskFile
# Monkey patch the object server module to use Gluster's DiskFile definition
server.DiskFile = Gluster_DiskFile
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI object server apps"""
conf = global_conf.copy()
conf.update(local_conf)
return server.ObjectController(conf)

0
ufo/gluster/swift/proxy/__init__.py
View File

27
ufo/gluster/swift/proxy/server.py
View File

@ -1,27 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Simply importing this monkey patches the constraint handling to fit our
# needs
import gluster.swift.common.constraints
from swift.proxy import server
def app_factory(global_conf, **local_conf):
"""paste.deploy app factory for creating WSGI proxy apps."""
conf = global_conf.copy()
conf.update(local_conf)
return server.Application(conf)

57
ufo/setup.py
View File

@ -1,57 +0,0 @@
#!/usr/bin/python
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from setuptools import setup, find_packages
from gluster.swift import __canonical_version__ as version
name = 'gluster_swift_ufo'
setup(
name=name,
version=version,
description='Gluster Swift/UFO',
license='Apache License (2.0)',
author='Red Hat, Inc.',
author_email='gluster-users@gluster.org',
url='https://gluster.org/',
packages=find_packages(exclude=['test', 'bin']),
test_suite='nose.collector',
classifiers=[
'Development Status :: 4 - Beta',
'License :: OSI Approved :: Apache Software License',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python :: 2.6',
'Environment :: No Input/Output (Daemon)',
],
install_requires=[], # removed for better compat
scripts=[
'bin/gluster-swift-gen-builders',
],
entry_points={
'paste.app_factory': [
'proxy=gluster.swift.proxy.server:app_factory',
'object=gluster.swift.obj.server:app_factory',
'container=gluster.swift.container.server:app_factory',
'account=gluster.swift.account.server:app_factory',
],
'paste.filter_factory': [
'gluster=gluster.swift.common.middleware.gluster:filter_factory',
],
},
)

49
ufo/test/__init__.py
View File

@ -1,49 +0,0 @@
# See http://code.google.com/p/python-nose/issues/detail?id=373
# The code below enables nosetests to work with i18n _() blocks
import __builtin__
import sys
import os
from ConfigParser import MissingSectionHeaderError
from StringIO import StringIO
from swift.common.utils import readconf
setattr(__builtin__, '_', lambda x: x)
# Work around what seems to be a Python bug.
# c.f. https://bugs.launchpad.net/swift/+bug/820185.
import logging
logging.raiseExceptions = False
def get_config(section_name=None, defaults=None):
"""
Attempt to get a test config dictionary.
:param section_name: the section to read (all sections if not defined)
:param defaults: an optional dictionary namespace of defaults
"""
config_file = os.environ.get('SWIFT_TEST_CONFIG_FILE',
'/etc/swift/test.conf')
config = {}
if defaults is not None:
config.update(defaults)
try:
config = readconf(config_file, section_name)
except SystemExit:
if not os.path.exists(config_file):
print >>sys.stderr, \
'Unable to read test config %s - file not found' \
% config_file
elif not os.access(config_file, os.R_OK):
print >>sys.stderr, \
'Unable to read test config %s - permission denied' \
% config_file
else:
print >>sys.stderr, \
'Unable to read test config %s - section %s not found' \
% (config_file, section_name)
return config

95
ufo/test/unit/__init__.py
View File

@ -1,95 +0,0 @@
""" Gluster Swift Unit Tests """
import logging
from collections import defaultdict
from test import get_config
from swift.common.utils import TRUE_VALUES
class NullLoggingHandler(logging.Handler):
def emit(self, record):
pass
class FakeLogger(object):
# a thread safe logger
def __init__(self, *args, **kwargs):
self._clear()
self.level = logging.NOTSET
if 'facility' in kwargs:
self.facility = kwargs['facility']
def _clear(self):
self.log_dict = defaultdict(list)
def _store_in(store_name):
def stub_fn(self, *args, **kwargs):
self.log_dict[store_name].append((args, kwargs))
return stub_fn
error = _store_in('error')
info = _store_in('info')
warning = _store_in('warning')
debug = _store_in('debug')
def exception(self, *args, **kwargs):
self.log_dict['exception'].append((args, kwargs, str(exc_info()[1])))
# mock out the StatsD logging methods:
increment = _store_in('increment')
decrement = _store_in('decrement')
timing = _store_in('timing')
timing_since = _store_in('timing_since')
update_stats = _store_in('update_stats')
set_statsd_prefix = _store_in('set_statsd_prefix')
def setFormatter(self, obj):
self.formatter = obj
def close(self):
self._clear()
def set_name(self, name):
# don't touch _handlers
self._name = name
def acquire(self):
pass
def release(self):
pass
def createLock(self):
pass
def emit(self, record):
pass
def handle(self, record):
pass
def flush(self):
pass
def handleError(self, record):
pass
original_syslog_handler = logging.handlers.SysLogHandler
def fake_syslog_handler():
for attr in dir(original_syslog_handler):
if attr.startswith('LOG'):
setattr(FakeLogger, attr,
copy.copy(getattr(logging.handlers.SysLogHandler, attr)))
FakeLogger.priority_map = \
copy.deepcopy(logging.handlers.SysLogHandler.priority_map)
logging.handlers.SysLogHandler = FakeLogger
if get_config('unit_test').get('fake_syslog', 'False').lower() in TRUE_VALUES:
fake_syslog_handler()

0
ufo/test/unit/common/__init__.py
View File

3
ufo/test/unit/common/data/README.rings
View File

@ -1,3 +0,0 @@
The unit tests expect certain ring data built using the following command:
../../../../bin/gluster-swift-gen-builders test iops

BIN
ufo/test/unit/common/data/account.builder
View File

Binary file not shown.

BIN
ufo/test/unit/common/data/account.ring.gz
View File

Binary file not shown.

BIN
ufo/test/unit/common/data/account_tree.tar.bz2
View File

Binary file not shown.

BIN
ufo/test/unit/common/data/backups/1365124498.account.builder
View File

Binary file not shown.

BIN
ufo/test/unit/common/data/backups/1365124498.container.builder
View File

Binary file not shown.

BIN
ufo/test/unit/common/data/backups/1365124498.object.builder
View File

Binary file not shown.

BIN
ufo/test/unit/common/data/backups/1365124499.object.builder
View File

Binary file not shown.

BIN
ufo/test/unit/common/data/container.builder
View File

Binary file not shown.

BIN
ufo/test/unit/common/data/container.ring.gz
View File

Binary file not shown.

BIN
ufo/test/unit/common/data/container_tree.tar.bz2
View File

Binary file not shown.

BIN
ufo/test/unit/common/data/object.builder
View File

Binary file not shown.

BIN
ufo/test/unit/common/data/object.ring.gz
View File

Binary file not shown.

95
ufo/test/unit/common/test_Glusterfs.py
View File

@ -1,95 +0,0 @@
# Copyright (c) 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import os, fcntl, errno, shutil
from tempfile import mkdtemp
import gluster.swift.common.Glusterfs as gfs
def mock_os_path_ismount(path):
return True
def mock_get_export_list():
return ['test', 'test2']
def mock_os_system(cmd):
return False
def mock_fcntl_lockf(f, *a, **kw):
raise IOError(errno.EAGAIN)
def _init():
global _RUN_DIR, _OS_SYSTEM, _FCNTL_LOCKF
global _OS_PATH_ISMOUNT, __GET_EXPORT_LIST
_RUN_DIR = gfs.RUN_DIR
_OS_SYSTEM = os.system
_FCNTL_LOCKF = fcntl.lockf
_OS_PATH_ISMOUNT = os.path.ismount
__GET_EXPORT_LIST = gfs._get_export_list
def _init_mock_variables(tmpdir):
os.system = mock_os_system
os.path.ismount = mock_os_path_ismount
gfs.RUN_DIR = os.path.join(tmpdir, 'var/run/swift')
gfs._get_export_list = mock_get_export_list
def _reset_mock_variables():
gfs.RUN_DIR = _RUN_DIR
gfs._get_export_list = __GET_EXPORT_LIST
os.system = _OS_SYSTEM
fcntl.lockf = _FCNTL_LOCKF
os.path.ismount = _OS_PATH_ISMOUNT
class TestGlusterfs(unittest.TestCase):
""" Tests for common.GlusterFS """
def setUp(self):
_init()
def test_mount(self):
try:
tmpdir = mkdtemp()
root = os.path.join(tmpdir, 'mnt/gluster-object')
drive = 'test'
_init_mock_variables(tmpdir)
assert gfs.mount(root, drive)
finally:
_reset_mock_variables()
shutil.rmtree(tmpdir)
def test_mount_egain(self):
try:
tmpdir = mkdtemp()
root = os.path.join(tmpdir, 'mnt/gluster-object')
drive = 'test'
_init_mock_variables(tmpdir)
assert gfs.mount(root, drive)
fcntl.lockf = mock_fcntl_lockf
assert gfs.mount(root, drive)
finally:
_reset_mock_variables()
shutil.rmtree(tmpdir)
def test_mount_get_export_list_err(self):
gfs._get_export_list = mock_get_export_list
assert not gfs.mount(None, 'drive')
_reset_mock_variables()
def tearDown(self):
_reset_mock_variables()

932
ufo/test/unit/common/test_diskfile.py
View File

@ -1,932 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Tests for gluster.swift.common.DiskFile """
import os
import stat
import errno
import unittest
import tempfile
import shutil
from hashlib import md5
from swift.common.utils import normalize_timestamp
from swift.common.exceptions import DiskFileNotExist
import gluster.swift.common.DiskFile
import gluster.swift.common.utils
from gluster.swift.common.DiskFile import Gluster_DiskFile, \
AlreadyExistsAsDir
from gluster.swift.common.utils import DEFAULT_UID, DEFAULT_GID, X_TYPE, \
X_OBJECT_TYPE
from test_utils import _initxattr, _destroyxattr
from test.unit import FakeLogger
_metadata = {}
def _mock_read_metadata(filename):
if filename in _metadata:
md = _metadata[filename]
else:
md = {}
return md
def _mock_write_metadata(filename, metadata):
_metadata[filename] = metadata
def _mock_clear_metadata():
_metadata = {}
class MockException(Exception):
pass
def _mock_rmdirs(p):
raise MockException("gluster.swift.common.DiskFile.rmdirs() called")
def _mock_do_listdir(p):
raise MockException("gluster.swift.common.DiskFile.do_listdir() called")
def _mock_do_unlink(f):
ose = OSError()
ose.errno = errno.ENOENT
raise ose
def _mock_do_unlink_eacces_err(f):
ose = OSError()
ose.errno = errno.EACCES
raise ose
def _mock_getsize_eaccess_err(f):
ose = OSError()
ose.errno = errno.EACCES
raise ose
def _mock_do_rmdir_eacces_err(f):
ose = OSError()
ose.errno = errno.EACCES
raise ose
class MockRenamerCalled(Exception):
pass
def _mock_renamer(a, b):
raise MockRenamerCalled()
class TestDiskFile(unittest.TestCase):
""" Tests for gluster.swift.common.DiskFile """
def setUp(self):
self.lg = FakeLogger()
_initxattr()
_mock_clear_metadata()
self._saved_df_wm = gluster.swift.common.DiskFile.write_metadata
self._saved_df_rm = gluster.swift.common.DiskFile.read_metadata
gluster.swift.common.DiskFile.write_metadata = _mock_write_metadata
gluster.swift.common.DiskFile.read_metadata = _mock_read_metadata
self._saved_ut_wm = gluster.swift.common.utils.write_metadata
self._saved_ut_rm = gluster.swift.common.utils.read_metadata
gluster.swift.common.utils.write_metadata = _mock_write_metadata
gluster.swift.common.utils.read_metadata = _mock_read_metadata
def tearDown(self):
self.lg = None
_destroyxattr()
gluster.swift.common.DiskFile.write_metadata = self._saved_df_wm
gluster.swift.common.DiskFile.read_metadata = self._saved_df_rm
gluster.swift.common.utils.write_metadata = self._saved_ut_wm
gluster.swift.common.utils.read_metadata = self._saved_ut_rm
def test_constructor_no_slash(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf.name == "bar"
assert gdf.datadir == "/tmp/foo/vol0/bar"
assert gdf.device_path == "/tmp/foo/vol0"
assert gdf._container_path == "/tmp/foo/vol0/bar"
assert gdf.disk_chunk_size == 65536
assert gdf.iter_hook == None
assert gdf.logger == self.lg
assert gdf.uid == DEFAULT_UID
assert gdf.gid == DEFAULT_GID
assert gdf.metadata == {}
assert gdf.meta_file == None
assert gdf.data_file == None
assert gdf.fp == None
assert gdf.iter_etag == None
assert not gdf.started_at_0
assert not gdf.read_to_eof
assert gdf.quarantined_dir == None
assert not gdf.keep_cache
assert not gdf._is_dir
def test_constructor_leadtrail_slash(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"/b/a/z/", self.lg)
assert gdf._obj == "z"
assert gdf._obj_path == "b/a"
assert gdf.name == "bar/b/a"
assert gdf.datadir == "/tmp/foo/vol0/bar/b/a"
assert gdf.device_path == "/tmp/foo/vol0"
def test_constructor_no_metadata(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
stats = os.stat(the_file)
ts = normalize_timestamp(stats.st_ctime)
etag = md5()
etag.update("1234")
etag = etag.hexdigest()
exp_md = {
'Content-Length': 4,
'ETag': etag,
'X-Timestamp': ts,
'Content-Type': 'application/octet-stream'}
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
assert gdf.fp is None
assert gdf.metadata == exp_md
finally:
shutil.rmtree(td)
def test_constructor_existing_metadata(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'file',
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[the_file] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
assert gdf.fp is None
assert gdf.metadata == exp_md
finally:
shutil.rmtree(td)
def test_constructor_invalid_existing_metadata(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
inv_md = {
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[the_file] = inv_md
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
assert gdf.fp is None
assert gdf.metadata != inv_md
finally:
shutil.rmtree(td)
def test_constructor_isdir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
try:
os.makedirs(the_dir)
ini_md = {
'X-Type': 'Object',
'X-Object-Type': 'dir',
'Content-Length': 5,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/loctet-stream'}
_metadata[the_dir] = ini_md
exp_md = ini_md.copy()
del exp_md['X-Type']
del exp_md['X-Object-Type']
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"d", self.lg, keep_data_fp=True)
assert gdf._obj == "d"
assert gdf.data_file == the_dir
assert gdf._is_dir
assert gdf.fp is None
assert gdf.metadata == exp_md
finally:
shutil.rmtree(td)
def test_constructor_keep_data_fp(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg, keep_data_fp=True)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
assert gdf.fp is not None
finally:
shutil.rmtree(td)
def test_constructor_chunk_size(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg, disk_chunk_size=8192)
assert gdf.disk_chunk_size == 8192
def test_constructor_iter_hook(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg, iter_hook='hook')
assert gdf.iter_hook == 'hook'
def test_close(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
# Should be a no-op, as by default is_dir is False, but fp is None
gdf.close()
gdf._is_dir = True
gdf.fp = "123"
# Should still be a no-op as is_dir is True (marker directory)
gdf.close()
assert gdf.fp == "123"
gdf._is_dir = False
saved_dc = gluster.swift.common.DiskFile.do_close
self.called = False
def our_do_close(fp):
self.called = True
gluster.swift.common.DiskFile.do_close = our_do_close
try:
gdf.close()
assert self.called
assert gdf.fp is None
finally:
gluster.swift.common.DiskFile.do_close = saved_dc
def test_is_deleted(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.is_deleted()
gdf.data_file = "/tmp/foo/bar"
assert not gdf.is_deleted()
def test_create_dir_object(self):
td = tempfile.mkdtemp()
the_dir = os.path.join(td, "vol0", "bar", "dir")
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
gdf._create_dir_object(the_dir)
assert os.path.isdir(the_dir)
assert the_dir in _metadata
finally:
shutil.rmtree(td)
def test_create_dir_object_exists(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
os.makedirs(the_path)
with open(the_dir, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
# Not created, dir object path is different, just checking
assert gdf._obj == "z"
def _mock_do_chown(p, u, g):
assert u == DEFAULT_UID
assert g == DEFAULT_GID
dc = gluster.swift.common.DiskFile.do_chown
gluster.swift.common.DiskFile.do_chown = _mock_do_chown
try:
gdf._create_dir_object(the_dir)
finally:
gluster.swift.common.DiskFile.do_chown = dc
assert os.path.isdir(the_dir)
assert the_dir in _metadata
finally:
shutil.rmtree(td)
def test_put_metadata(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "z")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
md = { 'Content-Type': 'application/octet-stream', 'a': 'b' }
gdf.put_metadata(md.copy())
assert gdf.metadata == md, "gdf.metadata = %r, md = %r" % (gdf.metadata, md)
assert _metadata[the_dir] == md
finally:
shutil.rmtree(td)
def test_put_w_tombstone(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.metadata == {}
gdf.put_metadata({'x': '1'}, tombstone=True)
assert gdf.metadata == {}
def test_put_w_meta_file(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
newmd = gdf.metadata.copy()
newmd['X-Object-Meta-test'] = '1234'
gdf.put_metadata(newmd)
assert gdf.metadata == newmd
assert _metadata[the_file] == newmd
finally:
shutil.rmtree(td)
def test_put_w_meta_file_no_content_type(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
newmd = gdf.metadata.copy()
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
gdf.put_metadata(newmd)
assert gdf.metadata == newmd
assert _metadata[the_file] == newmd
finally:
shutil.rmtree(td)
def test_put_w_meta_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir", self.lg)
newmd = gdf.metadata.copy()
newmd['X-Object-Meta-test'] = '1234'
gdf.put_metadata(newmd)
assert gdf.metadata == newmd
assert _metadata[the_dir] == newmd
finally:
shutil.rmtree(td)
def test_put_w_marker_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir", self.lg)
newmd = gdf.metadata.copy()
newmd['X-Object-Meta-test'] = '1234'
gdf.put_metadata(newmd)
assert gdf.metadata == newmd
assert _metadata[the_dir] == newmd
finally:
shutil.rmtree(td)
def test_put_w_marker_dir_create(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir", self.lg)
assert gdf.metadata == {}
newmd = {
'Content-Length': 0,
'ETag': 'etag',
'X-Timestamp': 'ts',
'Content-Type': 'application/directory'}
gdf.put(None, newmd, extension='.dir')
assert gdf.data_file == the_dir
assert gdf.metadata == newmd
assert _metadata[the_dir] == newmd
finally:
shutil.rmtree(td)
def test_put_is_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir", self.lg)
origmd = gdf.metadata.copy()
origfmd = _metadata[the_dir]
newmd = gdf.metadata.copy()
# FIXME: This is a hack to get to the code-path; it is not clear
# how this can happen normally.
newmd['Content-Type'] = ''
newmd['X-Object-Meta-test'] = '1234'
try:
gdf.put(None, newmd, extension='.data')
except AlreadyExistsAsDir:
pass
else:
self.fail("Expected to encounter 'already-exists-as-dir' exception")
assert gdf.metadata == origmd
assert _metadata[the_dir] == origfmd
finally:
shutil.rmtree(td)
def test_put(self):
td = tempfile.mkdtemp()
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf._obj_path == ""
assert gdf.name == "bar"
assert gdf.datadir == os.path.join(td, "vol0", "bar")
assert gdf.data_file is None
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
with gdf.mkstemp() as fd:
assert gdf.tmppath is not None
tmppath = gdf.tmppath
os.write(fd, body)
gdf.put(fd, metadata)
assert gdf.data_file == os.path.join(td, "vol0", "bar", "z")
assert os.path.exists(gdf.data_file)
assert not os.path.exists(tmppath)
finally:
shutil.rmtree(td)
def test_put_obj_path(self):
the_obj_path = os.path.join("b", "a")
the_file = os.path.join(the_obj_path, "z")
td = tempfile.mkdtemp()
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
the_file, self.lg)
assert gdf._obj == "z"
assert gdf._obj_path == the_obj_path
assert gdf.name == os.path.join("bar", "b", "a")
assert gdf.datadir == os.path.join(td, "vol0", "bar", "b", "a")
assert gdf.data_file is None
body = '1234\n'
etag = md5()
etag.update(body)
etag = etag.hexdigest()
metadata = {
'X-Timestamp': '1234',
'Content-Type': 'file',
'ETag': etag,
'Content-Length': '5',
}
with gdf.mkstemp() as fd:
assert gdf.tmppath is not None
tmppath = gdf.tmppath
os.write(fd, body)
gdf.put(fd, metadata)
assert gdf.data_file == os.path.join(td, "vol0", "bar", "b", "a", "z")
assert os.path.exists(gdf.data_file)
assert not os.path.exists(tmppath)
finally:
shutil.rmtree(td)
def test_unlinkold_no_metadata(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.metadata == {}
_saved_rmdirs = gluster.swift.common.DiskFile.rmdirs
_saved_do_listdir = gluster.swift.common.DiskFile.do_listdir
gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs
gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir
try:
gdf.unlinkold(None)
except MockException as exp:
self.fail(str(exp))
finally:
gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs
gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir
def test_unlinkold_same_timestamp(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.metadata == {}
gdf.metadata['X-Timestamp'] = 1
_saved_rmdirs = gluster.swift.common.DiskFile.rmdirs
_saved_do_listdir = gluster.swift.common.DiskFile.do_listdir
gluster.swift.common.DiskFile.rmdirs = _mock_rmdirs
gluster.swift.common.DiskFile.do_listdir = _mock_do_listdir
try:
gdf.unlinkold(1)
except MockException as exp:
self.fail(str(exp))
finally:
gluster.swift.common.DiskFile.rmdirs = _saved_rmdirs
gluster.swift.common.DiskFile.do_listdir = _saved_do_listdir
def test_unlinkold_file(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
assert os.path.isdir(gdf.datadir)
assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj))
finally:
shutil.rmtree(td)
def test_unlinkold_file_not_found(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
# Handle the case the file is not in the directory listing.
os.unlink(the_file)
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
assert os.path.isdir(gdf.datadir)
assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj))
finally:
shutil.rmtree(td)
def test_unlinkold_file_unlink_error(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
stats = os.stat(the_path)
os.chmod(the_path, stats.st_mode & (~stat.S_IWUSR))
# Handle the case do_unlink() raises an OSError
__os_unlink = os.unlink
os.unlink = _mock_do_unlink_eacces_err
try:
gdf.unlinkold(normalize_timestamp(later))
except OSError as e:
assert e.errno == errno.EACCES
else:
self.fail("Excepted an OSError when unlinking file")
finally:
os.unlink = __os_unlink
os.chmod(the_path, stats.st_mode)
assert os.path.isdir(gdf.datadir)
assert os.path.exists(os.path.join(gdf.datadir, gdf._obj))
finally:
shutil.rmtree(td)
def test_unlinkold_is_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"d", self.lg, keep_data_fp=True)
assert gdf.data_file == the_dir
assert gdf._is_dir
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
assert os.path.isdir(gdf.datadir)
assert not os.path.exists(os.path.join(gdf.datadir, gdf._obj))
finally:
shutil.rmtree(td)
def test_unlinkold_is_dir_failure(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"d", self.lg, keep_data_fp=True)
assert gdf.data_file == the_dir
assert gdf._is_dir
stats = os.stat(gdf.datadir)
os.chmod(gdf.datadir, 0)
__os_rmdir = os.rmdir
os.rmdir = _mock_do_rmdir_eacces_err
try:
later = float(gdf.metadata['X-Timestamp']) + 1
gdf.unlinkold(normalize_timestamp(later))
finally:
os.chmod(gdf.datadir, stats.st_mode)
os.rmdir = __os_rmdir
assert os.path.isdir(gdf.datadir)
assert os.path.isdir(gdf.data_file)
finally:
shutil.rmtree(td)
def test_get_data_file_size(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
assert 4 == gdf.get_data_file_size()
finally:
shutil.rmtree(td)
def test_get_data_file_size(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
assert 4 == gdf.metadata['Content-Length']
gdf.metadata['Content-Length'] = 3
assert 4 == gdf.get_data_file_size()
assert 4 == gdf.metadata['Content-Length']
finally:
shutil.rmtree(td)
def test_get_data_file_size_dne(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"/b/a/z/", self.lg)
try:
s = gdf.get_data_file_size()
except DiskFileNotExist:
pass
else:
self.fail("Expected DiskFileNotExist exception")
def test_get_data_file_size_dne_os_err(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
gdf.data_file = gdf.data_file + ".dne"
try:
s = gdf.get_data_file_size()
except DiskFileNotExist:
pass
else:
self.fail("Expected DiskFileNotExist exception")
finally:
shutil.rmtree(td)
def test_get_data_file_size_os_err(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_file = os.path.join(the_path, "z")
try:
os.makedirs(the_path)
with open(the_file, "wb") as fd:
fd.write("1234")
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf._obj == "z"
assert gdf.data_file == the_file
assert not gdf._is_dir
stats = os.stat(the_path)
os.chmod(the_path, 0)
__os_path_getsize = os.path.getsize
os.path.getsize = _mock_getsize_eaccess_err
try:
s = gdf.get_data_file_size()
except OSError as err:
assert err.errno == errno.EACCES
else:
self.fail("Expected OSError exception")
finally:
os.path.getsize = __os_path_getsize
os.chmod(the_path, stats.st_mode)
finally:
shutil.rmtree(td)
def test_get_data_file_size_dir(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "d")
try:
os.makedirs(the_dir)
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"d", self.lg, keep_data_fp=True)
assert gdf._obj == "d"
assert gdf.data_file == the_dir
assert gdf._is_dir
assert 0 == gdf.get_data_file_size()
finally:
shutil.rmtree(td)
def test_filter_metadata(self):
assert not os.path.exists("/tmp/foo")
gdf = Gluster_DiskFile("/tmp/foo", "vol0", "p57", "ufo47", "bar",
"z", self.lg)
assert gdf.metadata == {}
gdf.filter_metadata()
assert gdf.metadata == {}
gdf.metadata[X_TYPE] = 'a'
gdf.metadata[X_OBJECT_TYPE] = 'b'
gdf.metadata['foobar'] = 'c'
gdf.filter_metadata()
assert X_TYPE not in gdf.metadata
assert X_OBJECT_TYPE not in gdf.metadata
assert 'foobar' in gdf.metadata
def test_mkstemp(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
saved_tmppath = ''
with gdf.mkstemp() as fd:
assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir")
assert os.path.isdir(gdf.datadir)
saved_tmppath = gdf.tmppath
assert os.path.dirname(saved_tmppath) == gdf.datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
os.write(fd, "123")
assert not os.path.exists(saved_tmppath)
finally:
shutil.rmtree(td)
def test_mkstemp_err_on_close(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
saved_tmppath = ''
with gdf.mkstemp() as fd:
assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir")
assert os.path.isdir(gdf.datadir)
saved_tmppath = gdf.tmppath
assert os.path.dirname(saved_tmppath) == gdf.datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
os.write(fd, "123")
# At the end of previous with block a close on fd is called.
# Calling os.close on the same fd will raise an OSError
# exception and we must catch it.
try:
os.close(fd)
except OSError as err:
pass
else:
self.fail("Exception expected")
assert not os.path.exists(saved_tmppath)
finally:
shutil.rmtree(td)
def test_mkstemp_err_on_unlink(self):
td = tempfile.mkdtemp()
the_path = os.path.join(td, "vol0", "bar")
the_dir = os.path.join(the_path, "dir")
try:
gdf = Gluster_DiskFile(td, "vol0", "p57", "ufo47", "bar",
"dir/z", self.lg)
saved_tmppath = ''
with gdf.mkstemp() as fd:
assert gdf.datadir == os.path.join(td, "vol0", "bar", "dir")
assert os.path.isdir(gdf.datadir)
saved_tmppath = gdf.tmppath
assert os.path.dirname(saved_tmppath) == gdf.datadir
assert os.path.basename(saved_tmppath)[:3] == '.z.'
assert os.path.exists(saved_tmppath)
os.write(fd, "123")
os.unlink(saved_tmppath)
assert not os.path.exists(saved_tmppath)
finally:
shutil.rmtree(td)

277
ufo/test/unit/common/test_fs_utils.py
View File

@ -1,277 +0,0 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import shutil
import random
import unittest
from tempfile import mkdtemp, mkstemp
from gluster.swift.common import fs_utils as fs
from gluster.swift.common.exceptions import NotDirectoryError, \
FileOrDirNotFoundError
class TestUtils(unittest.TestCase):
""" Tests for common.utils """
def test_do_walk(self):
try:
# create directory structure
tmpparent = mkdtemp()
tmpdirs = []
tmpfiles = []
for i in range(5):
tmpdirs.append(mkdtemp(dir=tmpparent).rsplit(os.path.sep, 1)[1])
tmpfiles.append(mkstemp(dir=tmpparent)[1].rsplit(os.path.sep, \
1)[1])
for path, dirnames, filenames in fs.do_walk(tmpparent):
assert path == tmpparent
assert dirnames.sort() == tmpdirs.sort()
assert filenames.sort() == tmpfiles.sort()
break
finally:
shutil.rmtree(tmpparent)
def test_do_open(self):
try:
fd, tmpfile = mkstemp()
f = fs.do_open(tmpfile, 'r')
try:
f.write('test')
except IOError as err:
pass
else:
self.fail("IOError expected")
finally:
f.close()
os.close(fd)
os.remove(tmpfile)
def test_do_open_err(self):
try:
fs.do_open(os.path.join('/tmp', str(random.random())), 'r')
except IOError:
pass
else:
self.fail("IOError expected")
def test_do_write(self):
try:
fd, tmpfile = mkstemp()
cnt = fs.do_write(fd, "test")
assert cnt == len("test")
finally:
os.close(fd)
os.remove(tmpfile)
def test_do_write_err(self):
try:
fd, tmpfile = mkstemp()
fd1 = os.open(tmpfile, os.O_RDONLY)
fs.do_write(fd1, "test")
except OSError:
pass
else:
self.fail("OSError expected")
finally:
os.close(fd)
os.close(fd1)
def test_do_mkdir(self):
try:
path = os.path.join('/tmp', str(random.random()))
fs.do_mkdir(path)
assert os.path.exists(path)
assert fs.do_mkdir(path)
finally:
os.rmdir(path)
def test_do_mkdir_err(self):
try:
path = os.path.join('/tmp', str(random.random()), str(random.random()))
fs.do_mkdir(path)
except OSError:
pass
else:
self.fail("OSError expected")
def test_do_makedirs(self):
try:
subdir = os.path.join('/tmp', str(random.random()))
path = os.path.join(subdir, str(random.random()))
fs.do_makedirs(path)
assert os.path.exists(path)
assert fs.do_makedirs(path)
finally:
shutil.rmtree(subdir)
def test_do_listdir(self):
try:
tmpdir = mkdtemp()
subdir = []
for i in range(5):
subdir.append(mkdtemp(dir=tmpdir).rsplit(os.path.sep, 1)[1])
assert subdir.sort() == fs.do_listdir(tmpdir).sort()
finally:
shutil.rmtree(tmpdir)
def test_do_listdir_err(self):
try:
path = os.path.join('/tmp', str(random.random()))
fs.do_listdir(path)
except OSError:
pass
else:
self.fail("OSError expected")
def test_do_stat(self):
try:
tmpdir = mkdtemp()
fd, tmpfile = mkstemp(dir=tmpdir)
buf1 = os.stat(tmpfile)
buf2 = fs.do_stat(fd)
buf3 = fs.do_stat(tmpfile)
assert buf1 == buf2
assert buf1 == buf3
finally:
os.close(fd)
os.remove(tmpfile)
os.rmdir(tmpdir)
def test_do_stat_err(self):
try:
fs.do_stat(os.path.join('/tmp', str(random.random())))
except OSError:
pass
else:
self.fail("OSError expected")
def test_do_close(self):
try:
fd, tmpfile = mkstemp()
fs.do_close(fd);
try:
os.write(fd, "test")
except OSError:
pass
else:
self.fail("OSError expected")
fp = open(tmpfile)
fs.do_close(fp)
finally:
os.remove(tmpfile)
def test_do_unlink(self):
try:
fd, tmpfile = mkstemp()
fs.do_unlink(tmpfile)
assert not os.path.exists(tmpfile)
assert fs.do_unlink(os.path.join('/tmp', str(random.random())))
finally:
os.close(fd)
def test_do_unlink_err(self):
try:
tmpdir = mkdtemp()
fs.do_unlink(tmpdir)
except OSError:
pass
else:
self.fail('OSError expected')
finally:
os.rmdir(tmpdir)
def test_do_rmdir(self):
tmpdir = mkdtemp()
fs.do_rmdir(tmpdir)
assert not os.path.exists(tmpdir)
assert not fs.do_rmdir(os.path.join('/tmp', str(random.random())))
def test_do_rmdir_err(self):
try:
fd, tmpfile = mkstemp()
fs.do_rmdir(tmpfile)
except OSError:
pass
else:
self.fail('OSError expected')
finally:
os.close(fd)
os.remove(tmpfile)
def test_do_rename(self):
try:
srcpath = mkdtemp()
destpath = os.path.join('/tmp', str(random.random()))
fs.do_rename(srcpath, destpath)
assert not os.path.exists(srcpath)
assert os.path.exists(destpath)
finally:
os.rmdir(destpath)
def test_do_rename_err(self):
try:
srcpath = os.path.join('/tmp', str(random.random()))
destpath = os.path.join('/tmp', str(random.random()))
fs.do_rename(srcpath, destpath)
except OSError:
pass
else:
self.fail("OSError expected")
def test_dir_empty(self):
try:
tmpdir = mkdtemp()
subdir = mkdtemp(dir=tmpdir)
assert not fs.dir_empty(tmpdir)
assert fs.dir_empty(subdir)
finally:
shutil.rmtree(tmpdir)
def test_dir_empty_err(self):
try:
try:
assert fs.dir_empty(os.path.join('/tmp', str(random.random())))
except FileOrDirNotFoundError:
pass
else:
self.fail("FileOrDirNotFoundError exception expected")
fd, tmpfile = mkstemp()
try:
fs.dir_empty(tmpfile)
except NotDirectoryError:
pass
else:
self.fail("NotDirectoryError exception expected")
finally:
os.close(fd)
os.unlink(tmpfile)
def test_rmdirs(self):
try:
tmpdir = mkdtemp()
subdir = mkdtemp(dir=tmpdir)
fd, tmpfile = mkstemp(dir=tmpdir)
assert not fs.rmdirs(tmpfile)
assert not fs.rmdirs(tmpdir)
assert fs.rmdirs(subdir)
assert not os.path.exists(subdir)
finally:
os.close(fd)
shutil.rmtree(tmpdir)

55
ufo/test/unit/common/test_ring.py
View File

@ -1,55 +0,0 @@
# Copyright (c) 2013 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import unittest
import gluster.swift.common.constraints
import swift.common.utils
from gluster.swift.common.ring import Ring
class TestRing(unittest.TestCase):
""" Tests for common.utils """
def setUp(self):
swift.common.utils.HASH_PATH_SUFFIX = 'endcap'
swiftdir = os.path.join(os.getcwd(), "common", "data")
self.ring = Ring(swiftdir, ring_name='object')
def test_first_device(self):
part, node = self.ring.get_nodes('test')
assert node[0]['device'] == 'test'
node = self.ring.get_part_nodes(0)
assert node[0]['device'] == 'test'
for node in self.ring.get_more_nodes(0):
assert node['device'] == 'volume_not_in_ring'
def test_invalid_device(self):
part, node = self.ring.get_nodes('test2')
assert node[0]['device'] == 'volume_not_in_ring'
node = self.ring.get_part_nodes(0)
assert node[0]['device'] == 'volume_not_in_ring'
def test_second_device(self):
part, node = self.ring.get_nodes('iops')
assert node[0]['device'] == 'iops'
node = self.ring.get_part_nodes(0)
assert node[0]['device'] == 'iops'
for node in self.ring.get_more_nodes(0):
assert node['device'] == 'volume_not_in_ring'
def test_second_device_with_reseller_prefix(self):
part, node = self.ring.get_nodes('AUTH_iops')
assert node[0]['device'] == 'iops'

1020
ufo/test/unit/common/test_utils.py
View File

File diff suppressed because it is too large Load Diff

6
ufo/tools/test-requires
View File

@ -1,6 +0,0 @@
coverage
nose
nosexcover
openstack.nose_plugin
nosehtmloutput
mock>=0.8.0

25
ufo/tox.ini
View File

@ -1,25 +0,0 @@
[tox]
envlist = py26,py27
[testenv]
setenv = VIRTUAL_ENV={envdir}
NOSE_WITH_OPENSTACK=1
NOSE_OPENSTACK_COLOR=1
NOSE_OPENSTACK_RED=0.05
NOSE_OPENSTACK_YELLOW=0.025
NOSE_OPENSTACK_SHOW_ELAPSED=1
NOSE_OPENSTACK_STDOUT=1
deps =
https://launchpad.net/swift/grizzly/1.8.0/+download/swift-1.8.0.tar.gz
-r{toxinidir}/tools/test-requires
changedir = {toxinidir}/test/unit
commands = nosetests -v --exe --with-coverage --cover-package gluster --cover-erase {posargs}
[tox:jenkins]
downloadcache = ~/cache/pip
[testenv:cover]
setenv = NOSE_WITH_COVERAGE=1
[testenv:venv]
commands = {posargs}

7
ufo/unittests.sh
View File

@ -1,7 +0,0 @@
#!/bin/bash
cd $(dirname $0)/test/unit
nosetests --exe --with-coverage --cover-package gluster --cover-erase $@
saved_status=$?
rm -f .coverage
exit $saved_status

54
xlators/cluster/dht/src/dht-common.c
View File

@ -1662,41 +1662,6 @@ err:
return 0;
}
static int
dht_ufo_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
int op_ret, int op_errno, dict_t *xdata)
{
dht_local_t *local = NULL;
int this_call_cnt = 0;
call_frame_t *prev = NULL;
local = frame->local;
prev = cookie;
LOCK (&frame->lock);
{
if (op_ret == -1) {
local->op_ret = -1;
local->op_errno = op_errno;
gf_log (this->name, GF_LOG_DEBUG,
"subvolume %s returned -1 (%s)",
prev->this->name, strerror (op_errno));
goto unlock;
}
}
unlock:
UNLOCK (&frame->lock);
this_call_cnt = dht_frame_return (frame);
if (is_last_call (this_call_cnt)) {
DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
local->op_errno, NULL);
}
return 0;
}
int
dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
int op_ret, int op_errno, dict_t *xdata)
@ -2461,25 +2426,6 @@ dht_setxattr (call_frame_t *frame, xlator_t *this,
local->call_cnt = call_cnt = layout->cnt;
/* This key is sent by Unified File and Object storage
* to test xattr support in backend.
*/
tmp = dict_get (xattr, "user.ufo-test");
if (tmp) {
if (IA_ISREG (loc->inode->ia_type)) {
op_errno = ENOTSUP;
goto err;
}
local->op_ret = 0;
for (i = 0; i < call_cnt; i++) {
STACK_WIND (frame, dht_ufo_xattr_cbk,
layout->list[i].xlator,
layout->list[i].xlator->fops->setxattr,
loc, xattr, flags, NULL);
}
return 0;
}
tmp = dict_get (xattr, "distribute.migrate-data");
if (tmp) {
if (IA_ISDIR (loc->inode->ia_type)) {