object-storage: Use the wrapper functions provided by fs_utils.py to make

system calls.

 The set of changes:
 * Unit test cases for fs_utils.py

 * Replaced os.path with os_path

 * Implemented wrapper functions do_write, do_chmod, etc in fs_utils.py

 * Replaced os.<sys-call> with the wrapper functions.

Change-Id: I770da878e83eda6b98e49d70193990406a2642a7
BUG: 887301
Signed-off-by: Mohammed Junaid <junaid@redhat.com>
Reviewed-on: http://review.gluster.org/4360
Reviewed-by: Peter Portante <pportant@redhat.com>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>
This commit is contained in:
Mohammed Junaid 2013-02-09 04:37:28 +05:30 committed by Anand Avati
parent 3b19a14b86
commit 311a5df884
6 changed files with 409 additions and 85 deletions

View File

@ -22,7 +22,7 @@ from gluster.swift.common.utils import clean_metadata, dir_empty, rmdirs, \
DEFAULT_UID, validate_object, create_object_metadata, read_metadata, \
write_metadata, X_CONTENT_TYPE, X_CONTENT_LENGTH, X_TIMESTAMP, \
X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, X_BYTES_USED, \
X_CONTAINER_COUNT, CONTAINER
X_CONTAINER_COUNT, CONTAINER, os_path
from gluster.swift.common import Glusterfs
from swift.common.constraints import CONTAINER_LISTING_LIMIT
@ -69,7 +69,7 @@ def _read_metadata(dd):
class DiskCommon(object):
def is_deleted(self):
return not os.path.exists(self.datadir)
return not os_path.exists(self.datadir)
def filter_prefix(self, objects, prefix):
"""
@ -170,7 +170,7 @@ class DiskDir(DiskCommon):
self.uid = int(uid)
self.gid = int(gid)
self.db_file = _db_file
self.dir_exists = os.path.exists(self.datadir)
self.dir_exists = os_path.exists(self.datadir)
if self.dir_exists:
try:
self.metadata = _read_metadata(self.datadir)
@ -201,7 +201,7 @@ class DiskDir(DiskCommon):
def delete(self):
if self.empty():
#For delete account.
if os.path.ismount(self.datadir):
if os_path.ismount(self.datadir):
clean_metadata(self.datadir)
else:
rmdirs(self.datadir)
@ -387,7 +387,7 @@ class DiskDir(DiskCommon):
"""
Create the container if it doesn't exist and update the timestamp
"""
if not os.path.exists(self.datadir):
if not os_path.exists(self.datadir):
self.put(self.metadata)
def delete_object(self, name, timestamp):

View File

@ -17,13 +17,13 @@ import os
import errno
import random
from hashlib import md5
from eventlet import tpool
from contextlib import contextmanager
from swift.common.utils import normalize_timestamp, renamer
from swift.common.exceptions import DiskFileNotExist
from gluster.swift.common.exceptions import AlreadyExistsAsDir
from gluster.swift.common.utils import mkdirs, rmdirs, validate_object, \
create_object_metadata, do_open, do_close, do_unlink, do_chown, \
do_stat, do_listdir, read_metadata, write_metadata
create_object_metadata, do_open, do_close, do_unlink, do_chown, \
do_listdir, read_metadata, write_metadata, os_path, do_fsync
from gluster.swift.common.utils import X_CONTENT_TYPE, X_CONTENT_LENGTH, \
X_TIMESTAMP, X_PUT_TIMESTAMP, X_TYPE, X_ETAG, X_OBJECTS_COUNT, \
X_BYTES_USED, X_OBJECT_TYPE, FILE, DIR, MARKER_DIR, OBJECT, DIR_TYPE, \
@ -38,10 +38,6 @@ DEFAULT_DISK_CHUNK_SIZE = 65536
DISALLOWED_HEADERS = set('content-length content-type deleted etag'.split())
class AlreadyExistsAsDir(Exception):
pass
def _adjust_metadata(metadata):
# Fix up the metadata to ensure it has a proper value for the
# Content-Type metadata, as well as an X_TYPE and X_OBJECT_TYPE
@ -119,7 +115,7 @@ class Gluster_DiskFile(DiskFile):
# Don't store a value for data_file until we know it exists.
self.data_file = None
data_file = os.path.join(self.datadir, self._obj)
if not os.path.exists(data_file):
if not os_path.exists(data_file):
return
self.data_file = os.path.join(data_file)
@ -134,7 +130,7 @@ class Gluster_DiskFile(DiskFile):
self.filter_metadata()
if os.path.isdir(data_file):
if os_path.isdir(data_file):
self._is_dir = True
else:
if keep_data_fp:
@ -170,7 +166,7 @@ class Gluster_DiskFile(DiskFile):
def _create_dir_object(self, dir_path):
#TODO: if object already exists???
if os.path.exists(dir_path) and not os.path.isdir(dir_path):
if os_path.exists(dir_path) and not os_path.isdir(dir_path):
self.logger.error("Deleting file %s", dir_path)
do_unlink(dir_path)
#If dir aleady exist just override metadata.
@ -228,7 +224,7 @@ class Gluster_DiskFile(DiskFile):
write_metadata(self.tmppath, metadata)
if X_CONTENT_LENGTH in metadata:
self.drop_cache(fd, 0, int(metadata[X_CONTENT_LENGTH]))
tpool.execute(os.fsync, fd)
do_fsync(fd)
if self._obj_path:
dir_objs = self._obj_path.split('/')
assert len(dir_objs) >= 1
@ -272,7 +268,7 @@ class Gluster_DiskFile(DiskFile):
def get_data_file_size(self):
"""
Returns the os.path.getsize for the file. Raises an exception if this
Returns the os_path.getsize for the file. Raises an exception if this
file does not match the Content-Length stored in the metadata. Or if
self.data_file does not exist.
@ -286,7 +282,7 @@ class Gluster_DiskFile(DiskFile):
try:
file_size = 0
if self.data_file:
file_size = os.path.getsize(self.data_file)
file_size = os_path.getsize(self.data_file)
if X_CONTENT_LENGTH in self.metadata:
metadata_size = int(self.metadata[X_CONTENT_LENGTH])
if file_size != metadata_size:
@ -314,28 +310,29 @@ class Gluster_DiskFile(DiskFile):
# if exists, then it means that it also has its metadata.
# Not checking for container, since the container should already
# exist for the call to come here.
if not os.path.exists(self.datadir):
if not os_path.exists(self.datadir):
path = self._container_path
subdir_list = self._obj_path.split(os.path.sep)
for i in range(len(subdir_list)):
path = os.path.join(path, subdir_list[i]);
if not os.path.exists(path):
if not os_path.exists(path):
self._create_dir_object(path)
tmpfile = '.' + self._obj + '.' + md5(self._obj + \
str(random.random())).hexdigest()
self.tmppath = os.path.join(self.datadir, tmpfile)
fd = os.open(self.tmppath, os.O_RDWR | os.O_CREAT | os.O_EXCL)
fd = do_open(self.tmppath, os.O_RDWR | os.O_CREAT | os.O_EXCL)
try:
yield fd
finally:
try:
os.close(fd)
do_close(fd)
except OSError:
pass
tmppath, self.tmppath = self.tmppath, None
try:
os.unlink(tmppath)
except OSError:
pass
do_unlink(tmppath)
except OSError as err:
if err.errno != errno.ENOENT:
raise

View File

@ -0,0 +1,27 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class GlusterfsException(Exception):
pass
class FileOrDirNotFoundError(GlusterfsException):
pass
class NotDirectoryError(GlusterfsException):
pass
class AlreadyExistsAsDir(GlusterfsException):
pass

View File

@ -16,38 +16,53 @@
import logging
import os
import errno
import os.path as os_path
from eventlet import tpool
from gluster.swift.common.exceptions import FileOrDirNotFoundError, \
NotDirectoryError
def do_walk(*args, **kwargs):
return os.walk(*args, **kwargs)
def do_write(fd, msg):
try:
cnt = os.write(fd, msg)
except OSError as err:
logging.exception("Write failed, err: %s", str(err))
raise
return cnt
def do_mkdir(path):
try:
os.mkdir(path)
except Exception, err:
logging.exception("Mkdir failed on %s err: %s", path, str(err))
except OSError as err:
if err.errno != errno.EEXIST:
logging.exception("Mkdir failed on %s err: %s", path, err.strerror)
raise
return True
def do_makedirs(path):
try:
os.makedirs(path)
except Exception, err:
logging.exception("Makedirs failed on %s err: %s", path, str(err))
except OSError as err:
if err.errno != errno.EEXIST:
logging.exception("Makedirs failed on %s err: %s", path, err.strerror)
raise
return True
def do_listdir(path):
try:
buf = os.listdir(path)
except Exception, err:
logging.exception("Listdir failed on %s err: %s", path, str(err))
except OSError as err:
logging.exception("Listdir failed on %s err: %s", path, err.strerror)
raise
return buf
def do_chown(path, uid, gid):
try:
os.chown(path, uid, gid)
except Exception, err:
logging.exception("Chown failed on %s err: %s", path, str(err))
except OSError as err:
logging.exception("Chown failed on %s err: %s", path, err.strerror)
raise
return True
@ -58,18 +73,24 @@ def do_stat(path):
buf = os.fstat(path)
else:
buf = os.stat(path)
except Exception, err:
logging.exception("Stat failed on %s err: %s", path, str(err))
except OSError as err:
logging.exception("Stat failed on %s err: %s", path, err.strerror)
raise
return buf
def do_open(path, mode):
try:
fd = open(path, mode)
except Exception, err:
logging.exception("Open failed on %s err: %s", path, str(err))
raise
if isinstance(mode, int):
try:
fd = os.open(path, mode)
except OSError as err:
logging.exception("Open failed on %s err: %s", path, str(err))
raise
else:
try:
fd = open(path, mode)
except IOError as err:
logging.exception("Open failed on %s err: %s", path, str(err))
raise
return fd
def do_close(fd):
@ -79,27 +100,27 @@ def do_close(fd):
os.close(fd)
else:
fd.close()
except Exception, err:
logging.exception("Close failed on %s err: %s", fd, str(err))
except OSError as err:
logging.exception("Close failed on %s err: %s", fd, err.strerror)
raise
return True
def do_unlink(path, log = True):
try:
os.unlink(path)
except Exception, err:
if log:
logging.exception("Unlink failed on %s err: %s", path, str(err))
except OSError as err:
if err.errno != errno.ENOENT:
if log:
logging.exception("Unlink failed on %s err: %s", path, err.strerror)
raise
return True
def do_rmdir(path):
try:
os.rmdir(path)
except Exception, err:
logging.exception("Rmdir failed on %s err: %s", path, str(err))
except OSError as err:
if err.errno != errno.ENOENT:
logging.exception("Rmdir failed on %s err: %s", path, err.strerror)
raise
res = False
else:
@ -109,9 +130,9 @@ def do_rmdir(path):
def do_rename(old_path, new_path):
try:
os.rename(old_path, new_path)
except Exception, err:
except OSError as err:
logging.exception("Rename failed on %s to %s err: %s", old_path, new_path, \
str(err))
err.strerror)
raise
return True
@ -123,13 +144,7 @@ def mkdirs(path):
:param path: path to create
"""
if not os.path.isdir(path):
try:
do_makedirs(path)
except OSError, err:
#TODO: check, isdir will fail if mounted and volume stopped.
#if err.errno != errno.EEXIST or not os.path.isdir(path)
if err.errno != errno.EEXIST:
raise
do_makedirs(path)
def dir_empty(path):
"""
@ -138,22 +153,27 @@ def dir_empty(path):
:returns: True/False.
"""
if os.path.isdir(path):
try:
files = do_listdir(path)
except Exception, err:
logging.exception("listdir failed on %s err: %s", path, str(err))
raise
if not files:
return True
else:
return False
else:
if not os.path.exists(path):
return True
files = do_listdir(path)
return not files
elif not os.path.exists(path):
raise FileOrDirNotFoundError()
raise NotDirectoryError()
def rmdirs(path):
if not os.path.isdir(path) or not dir_empty(path):
logging.error("rmdirs failed: %s may not be empty or not valid dir", path)
if not os.path.isdir(path):
return False
try:
os.rmdir(path)
except OSError as err:
if err.errno != errno.ENOENT:
logging.error("rmdirs failed on %s, err: %s", path, err.strerror)
return False
return True
return do_rmdir(path)
def do_fsync(fd):
try:
tpool.execute(os.fsync, fd)
except OSError as err:
logging.exception("fsync failed with err: %s", err.strerror)
raise
return True

View File

@ -57,7 +57,6 @@ MEMCACHE_KEY_PREFIX = 'gluster.swift.'
MEMCACHE_ACCOUNT_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'account.details.'
MEMCACHE_CONTAINER_DETAILS_KEY_PREFIX = MEMCACHE_KEY_PREFIX + 'container.details.'
def read_metadata(path):
"""
Helper function to read the pickled metadata from a File/Directory.
@ -140,7 +139,7 @@ def clean_metadata(path):
key += 1
def check_user_xattr(path):
if not os.path.exists(path):
if not os_path.exists(path):
return False
try:
xattr.set(path, 'user.test.key1', 'value1')
@ -243,7 +242,7 @@ def _update_list(path, cont_path, src_list, reg_file=True, object_count=0,
object_count += 1
if reg_file:
bytes_used += os.path.getsize(os.path.join(path, obj_name))
bytes_used += os_path.getsize(os.path.join(path, obj_name))
sleep()
return object_count, bytes_used
@ -278,8 +277,8 @@ def _get_container_details_from_fs(cont_path):
obj_list = []
dir_list = []
if os.path.isdir(cont_path):
for (path, dirs, files) in os.walk(cont_path):
if os_path.isdir(cont_path):
for (path, dirs, files) in do_walk(cont_path):
object_count, bytes_used = update_list(path, cont_path, dirs, files,
object_count, bytes_used,
obj_list)
@ -338,7 +337,7 @@ def _get_account_details_from_fs(acc_path, acc_stats):
for name in do_listdir(acc_path):
if name.lower() == TEMP_DIR \
or name.lower() == ASYNCDIR \
or not os.path.isdir(os.path.join(acc_path, name)):
or not os_path.isdir(os.path.join(acc_path, name)):
continue
container_count += 1
container_list.append(name)
@ -386,7 +385,7 @@ def get_object_metadata(obj_path):
Return metadata of object.
"""
try:
stats = os.stat(obj_path)
stats = do_stat(obj_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
@ -421,8 +420,8 @@ def get_container_metadata(cont_path, memcache=None):
bytes_used = 0
objects, object_count, bytes_used = get_container_details(cont_path, memcache)
metadata = {X_TYPE: CONTAINER,
X_TIMESTAMP: normalize_timestamp(os.path.getctime(cont_path)),
X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(cont_path)),
X_TIMESTAMP: normalize_timestamp(os_path.getctime(cont_path)),
X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(cont_path)),
X_OBJECTS_COUNT: object_count,
X_BYTES_USED: bytes_used}
return _add_timestamp(metadata)
@ -432,8 +431,8 @@ def get_account_metadata(acc_path, memcache=None):
container_count = 0
containers, container_count = get_account_details(acc_path, memcache)
metadata = {X_TYPE: ACCOUNT,
X_TIMESTAMP: normalize_timestamp(os.path.getctime(acc_path)),
X_PUT_TIMESTAMP: normalize_timestamp(os.path.getmtime(acc_path)),
X_TIMESTAMP: normalize_timestamp(os_path.getctime(acc_path)),
X_PUT_TIMESTAMP: normalize_timestamp(os_path.getmtime(acc_path)),
X_OBJECTS_COUNT: 0,
X_BYTES_USED: 0,
X_CONTAINER_COUNT: container_count}
@ -484,9 +483,13 @@ def write_pickle(obj, dest, tmp=None, pickle_protocol=0):
tmppath = os.path.join(dirname, tmpname)
with open(tmppath, 'wb') as fo:
pickle.dump(obj, fo, pickle_protocol)
# TODO: This flush() method call turns into a flush() system call
# We'll need to wrap this as well, but we would do this by writing
#a context manager for our own open() method which returns an object
# in fo which makes the gluster API call.
fo.flush()
os.fsync(fo)
os.rename(tmppath, dest)
do_fsync(fo)
do_rename(tmppath, dest)
# Over-ride Swift's utils.write_pickle with ours
import swift.common.utils

View File

@ -0,0 +1,277 @@
# Copyright (c) 2012 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import shutil
import random
import unittest
from tempfile import mkdtemp, mkstemp
from gluster.swift.common import fs_utils as fs
from gluster.swift.common.exceptions import NotDirectoryError, \
FileOrDirNotFoundError
class TestUtils(unittest.TestCase):
""" Tests for common.utils """
def test_do_walk(self):
try:
# create directory structure
tmpparent = mkdtemp()
tmpdirs = []
tmpfiles = []
for i in range(5):
tmpdirs.append(mkdtemp(dir=tmpparent).rsplit(os.path.sep, 1)[1])
tmpfiles.append(mkstemp(dir=tmpparent)[1].rsplit(os.path.sep, \
1)[1])
for path, dirnames, filenames in fs.do_walk(tmpparent):
assert path == tmpparent
assert dirnames.sort() == tmpdirs.sort()
assert filenames.sort() == tmpfiles.sort()
break
finally:
shutil.rmtree(tmpparent)
def test_do_open(self):
try:
fd, tmpfile = mkstemp()
f = fs.do_open(tmpfile, 'r')
try:
f.write('test')
except IOError as err:
pass
else:
self.fail("IOError expected")
finally:
f.close()
os.close(fd)
os.remove(tmpfile)
def test_do_open_err(self):
try:
fs.do_open(os.path.join('/tmp', str(random.random())), 'r')
except IOError:
pass
else:
self.fail("IOError expected")
def test_do_write(self):
try:
fd, tmpfile = mkstemp()
cnt = fs.do_write(fd, "test")
assert cnt == len("test")
finally:
os.close(fd)
os.remove(tmpfile)
def test_do_write_err(self):
try:
fd, tmpfile = mkstemp()
fd1 = os.open(tmpfile, os.O_RDONLY)
fs.do_write(fd1, "test")
except OSError:
pass
else:
self.fail("OSError expected")
finally:
os.close(fd)
os.close(fd1)
def test_do_mkdir(self):
try:
path = os.path.join('/tmp', str(random.random()))
fs.do_mkdir(path)
assert os.path.exists(path)
assert fs.do_mkdir(path)
finally:
os.rmdir(path)
def test_do_mkdir_err(self):
try:
path = os.path.join('/tmp', str(random.random()), str(random.random()))
fs.do_mkdir(path)
except OSError:
pass
else:
self.fail("OSError expected")
def test_do_makedirs(self):
try:
subdir = os.path.join('/tmp', str(random.random()))
path = os.path.join(subdir, str(random.random()))
fs.do_makedirs(path)
assert os.path.exists(path)
assert fs.do_makedirs(path)
finally:
shutil.rmtree(subdir)
def test_do_listdir(self):
try:
tmpdir = mkdtemp()
subdir = []
for i in range(5):
subdir.append(mkdtemp(dir=tmpdir).rsplit(os.path.sep, 1)[1])
assert subdir.sort() == fs.do_listdir(tmpdir).sort()
finally:
shutil.rmtree(tmpdir)
def test_do_listdir_err(self):
try:
path = os.path.join('/tmp', str(random.random()))
fs.do_listdir(path)
except OSError:
pass
else:
self.fail("OSError expected")
def test_do_stat(self):
try:
tmpdir = mkdtemp()
fd, tmpfile = mkstemp(dir=tmpdir)
buf1 = os.stat(tmpfile)
buf2 = fs.do_stat(fd)
buf3 = fs.do_stat(tmpfile)
assert buf1 == buf2
assert buf1 == buf3
finally:
os.close(fd)
os.remove(tmpfile)
os.rmdir(tmpdir)
def test_do_stat_err(self):
try:
fs.do_stat(os.path.join('/tmp', str(random.random())))
except OSError:
pass
else:
self.fail("OSError expected")
def test_do_close(self):
try:
fd, tmpfile = mkstemp()
fs.do_close(fd);
try:
os.write(fd, "test")
except OSError:
pass
else:
self.fail("OSError expected")
fp = open(tmpfile)
fs.do_close(fp)
finally:
os.remove(tmpfile)
def test_do_unlink(self):
try:
fd, tmpfile = mkstemp()
fs.do_unlink(tmpfile)
assert not os.path.exists(tmpfile)
assert fs.do_unlink(os.path.join('/tmp', str(random.random())))
finally:
os.close(fd)
def test_do_unlink_err(self):
try:
tmpdir = mkdtemp()
fs.do_unlink(tmpdir)
except OSError:
pass
else:
self.fail('OSError expected')
finally:
os.rmdir(tmpdir)
def test_do_rmdir(self):
tmpdir = mkdtemp()
fs.do_rmdir(tmpdir)
assert not os.path.exists(tmpdir)
assert not fs.do_rmdir(os.path.join('/tmp', str(random.random())))
def test_do_rmdir_err(self):
try:
fd, tmpfile = mkstemp()
fs.do_rmdir(tmpfile)
except OSError:
pass
else:
self.fail('OSError expected')
finally:
os.close(fd)
os.remove(tmpfile)
def test_do_rename(self):
try:
srcpath = mkdtemp()
destpath = os.path.join('/tmp', str(random.random()))
fs.do_rename(srcpath, destpath)
assert not os.path.exists(srcpath)
assert os.path.exists(destpath)
finally:
os.rmdir(destpath)
def test_do_rename_err(self):
try:
srcpath = os.path.join('/tmp', str(random.random()))
destpath = os.path.join('/tmp', str(random.random()))
fs.do_rename(srcpath, destpath)
except OSError:
pass
else:
self.fail("OSError expected")
def test_dir_empty(self):
try:
tmpdir = mkdtemp()
subdir = mkdtemp(dir=tmpdir)
assert not fs.dir_empty(tmpdir)
assert fs.dir_empty(subdir)
finally:
shutil.rmtree(tmpdir)
def test_dir_empty_err(self):
try:
try:
assert fs.dir_empty(os.path.join('/tmp', str(random.random())))
except FileOrDirNotFoundError:
pass
else:
self.fail("FileOrDirNotFoundError exception expected")
fd, tmpfile = mkstemp()
try:
fs.dir_empty(tmpfile)
except NotDirectoryError:
pass
else:
self.fail("NotDirectoryError exception expected")
finally:
os.close(fd)
os.unlink(tmpfile)
def test_rmdirs(self):
try:
tmpdir = mkdtemp()
subdir = mkdtemp(dir=tmpdir)
fd, tmpfile = mkstemp(dir=tmpdir)
assert not fs.rmdirs(tmpfile)
assert not fs.rmdirs(tmpdir)
assert fs.rmdirs(subdir)
assert not os.path.exists(subdir)
finally:
os.close(fd)
shutil.rmtree(tmpdir)