Further reduce extended attribute reads/writes
More refactoring towards reducing the number of extended attribute
reads/writes to the file system. See BZ 868120:
http://bugzilla.redhat.com/show_bug.cgi?id=868120
Basically the redundant routines restore_object, restore_account and
restore_container have been collapsed to one routine, restore_metadata, which
will only write out metadata if the new metadata is different from what was
originally read.
Along with these changes come a set of unit tests for all the functions
related to the routines that are in this module in the call tree where
restore_metadata is invoked.
Change-Id: I957ee2f8646cbe6df4d4420d3bdfb1f6ac62bdd2
BUG: 868120
Signed-off-by: Peter Portante <peter.portante@redhat.com>
Reviewed-on: http://review.gluster.org/4111
Reviewed-by: Kaleb KEITHLEY <kkeithle@redhat.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>
Tested-by: Anand Avati <avati@redhat.com>
This commit is contained in:
Peter Portante
Anand Avati
parent
3ab5404fab
commit
4f6aeb63d3
@ -1,6 +1,5 @@
|
||||
#!/bin/bash
|
||||
|
||||
cd swift/1.4.8/test/unit
|
||||
nosetests --exe --with-coverage --cover-package plugins --cover-erase $@
|
||||
rm -f .coverage
|
||||
cd swift/1.4.8/
|
||||
./.unittests
|
||||
cd -
|
||||
|
||||
6
swift/1.4.8/.unittests
Executable file
6
swift/1.4.8/.unittests
Executable file
@ -0,0 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
cd test/unit
|
||||
nosetests --exe --with-coverage --cover-package plugins --cover-erase $@
|
||||
rm -f .coverage
|
||||
cd -
|
||||
@ -98,7 +98,6 @@ def do_makedirs(path):
|
||||
raise
|
||||
return True
|
||||
|
||||
|
||||
def do_listdir(path):
|
||||
try:
|
||||
buf = os.listdir(path)
|
||||
@ -278,7 +277,6 @@ def dir_empty(path):
|
||||
if not os.path.exists(path):
|
||||
return True
|
||||
|
||||
|
||||
def get_device_from_account(account):
|
||||
if account.startswith(RESELLER_PREFIX):
|
||||
device = account.replace(RESELLER_PREFIX, '', 1)
|
||||
@ -517,7 +515,6 @@ def get_account_details_from_memcache(acc_path, memcache=None):
|
||||
container_count = memcache.get(strip_obj_storage_path(acc_path)+'_container_count')
|
||||
return container_list, container_count
|
||||
|
||||
|
||||
def get_account_details(acc_path, memcache=None):
|
||||
"""
|
||||
Return container_list and container_count.
|
||||
@ -527,8 +524,6 @@ def get_account_details(acc_path, memcache=None):
|
||||
else:
|
||||
return get_account_details_from_fs(acc_path, memcache)
|
||||
|
||||
|
||||
|
||||
def get_etag(path):
|
||||
etag = None
|
||||
if os.path.exists(path):
|
||||
@ -613,45 +608,28 @@ def get_account_metadata(acc_path, memcache=None):
|
||||
X_CONTAINER_COUNT: container_count}
|
||||
return _add_timestamp(metadata)
|
||||
|
||||
def restore_object(obj_path, metadata):
|
||||
meta = read_metadata(obj_path)
|
||||
if meta:
|
||||
meta.update(metadata)
|
||||
write_metadata(obj_path, meta)
|
||||
def restore_metadata(path, metadata):
|
||||
meta_orig = read_metadata(path)
|
||||
if meta_orig:
|
||||
meta_new = meta_orig.copy()
|
||||
meta_new.update(metadata)
|
||||
else:
|
||||
write_metadata(obj_path, metadata)
|
||||
|
||||
def restore_container(cont_path, metadata):
|
||||
meta = read_metadata(cont_path)
|
||||
if meta:
|
||||
meta.update(metadata)
|
||||
write_metadata(cont_path, meta)
|
||||
else:
|
||||
write_metadata(cont_path, metadata)
|
||||
|
||||
def restore_account(acc_path, metadata):
|
||||
meta = read_metadata(acc_path)
|
||||
if meta:
|
||||
meta.update(metadata)
|
||||
write_metadata(acc_path, meta)
|
||||
else:
|
||||
write_metadata(acc_path, metadata)
|
||||
meta_new = metadata
|
||||
if meta_orig != meta_new:
|
||||
write_metadata(path, meta_new)
|
||||
return meta_new
|
||||
|
||||
def create_object_metadata(obj_path):
|
||||
meta = get_object_metadata(obj_path)
|
||||
restore_object(obj_path, meta)
|
||||
return meta
|
||||
metadata = get_object_metadata(obj_path)
|
||||
return restore_metadata(obj_path, metadata)
|
||||
|
||||
def create_container_metadata(cont_path, memcache=None):
|
||||
meta = get_container_metadata(cont_path, memcache)
|
||||
restore_container(cont_path, meta)
|
||||
return meta
|
||||
metadata = get_container_metadata(cont_path, memcache)
|
||||
return restore_metadata(cont_path, metadata)
|
||||
|
||||
def create_account_metadata(acc_path, memcache=None):
|
||||
meta = get_account_metadata(acc_path, memcache)
|
||||
restore_account(acc_path, meta)
|
||||
return meta
|
||||
|
||||
metadata = get_account_metadata(acc_path, memcache)
|
||||
return restore_metadata(acc_path, metadata)
|
||||
|
||||
def check_account_exists(account, fs_object):
|
||||
if account not in get_account_list(fs_object):
|
||||
@ -666,7 +644,6 @@ def get_account_list(fs_object):
|
||||
account_list = fs_object.get_export_list()
|
||||
return account_list
|
||||
|
||||
|
||||
def get_account_id(account):
|
||||
return RESELLER_PREFIX + md5(account + HASH_PATH_SUFFIX).hexdigest()
|
||||
|
||||
|
||||
@ -15,10 +15,14 @@
|
||||
|
||||
""" Tests for plugins.utils """
|
||||
|
||||
import os
|
||||
import unittest
|
||||
import errno
|
||||
import xattr
|
||||
import cPickle as pickle
|
||||
import tempfile
|
||||
import hashlib
|
||||
from swift.common.utils import normalize_timestamp
|
||||
from collections import defaultdict
|
||||
from plugins import utils
|
||||
|
||||
@ -229,3 +233,257 @@ class TestUtils(unittest.TestCase):
|
||||
assert res_d == {}
|
||||
assert _xattr_op_cnt['get'] == 3, "%r" % _xattr_op_cnt
|
||||
assert len(_xattrs.keys()) == 0, "Expected 0 keys, found %d" % len(_xattrs.keys())
|
||||
|
||||
def test_restore_metadata_none(self):
|
||||
# No initial metadata
|
||||
path = "/tmp/foo/i"
|
||||
res_d = utils.restore_metadata(path, { 'b': 'y' })
|
||||
expected_d = { 'b': 'y' }
|
||||
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
|
||||
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
|
||||
assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt
|
||||
|
||||
def test_restore_metadata(self):
|
||||
# Initial metadata
|
||||
path = "/tmp/foo/i"
|
||||
initial_d = { 'a': 'z' }
|
||||
xkey = _xkey(path, utils.METADATA_KEY)
|
||||
_xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL)
|
||||
res_d = utils.restore_metadata(path, { 'b': 'y' })
|
||||
expected_d = { 'a': 'z', 'b': 'y' }
|
||||
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
|
||||
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
|
||||
assert _xattr_op_cnt['set'] == 1, "%r" % _xattr_op_cnt
|
||||
|
||||
def test_restore_metadata_nochange(self):
|
||||
# Initial metadata but no changes
|
||||
path = "/tmp/foo/i"
|
||||
initial_d = { 'a': 'z' }
|
||||
xkey = _xkey(path, utils.METADATA_KEY)
|
||||
_xattrs[xkey] = pickle.dumps(initial_d, utils.PICKLE_PROTOCOL)
|
||||
res_d = utils.restore_metadata(path, {})
|
||||
expected_d = { 'a': 'z' }
|
||||
assert res_d == expected_d, "Expected %r, result %r" % (expected_d, res_d)
|
||||
assert _xattr_op_cnt['get'] == 1, "%r" % _xattr_op_cnt
|
||||
assert _xattr_op_cnt['set'] == 0, "%r" % _xattr_op_cnt
|
||||
|
||||
def test_add_timestamp_empty(self):
|
||||
orig = {}
|
||||
res = utils._add_timestamp(orig)
|
||||
assert res == {}
|
||||
|
||||
def test_add_timestamp_none(self):
|
||||
orig = { 'a': 1, 'b': 2, 'c': 3 }
|
||||
exp = { 'a': (1, 0), 'b': (2, 0), 'c': (3, 0) }
|
||||
res = utils._add_timestamp(orig)
|
||||
assert res == exp
|
||||
|
||||
def test_add_timestamp_mixed(self):
|
||||
orig = { 'a': 1, 'b': (2, 1), 'c': 3 }
|
||||
exp = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) }
|
||||
res = utils._add_timestamp(orig)
|
||||
assert res == exp
|
||||
|
||||
def test_add_timestamp_all(self):
|
||||
orig = { 'a': (1, 0), 'b': (2, 1), 'c': (3, 0) }
|
||||
res = utils._add_timestamp(orig)
|
||||
assert res == orig
|
||||
|
||||
def test_get_etag_empty(self):
|
||||
tf = tempfile.NamedTemporaryFile()
|
||||
hd = utils.get_etag(tf.name)
|
||||
assert hd == hashlib.md5().hexdigest()
|
||||
|
||||
def test_get_etag(self):
|
||||
tf = tempfile.NamedTemporaryFile()
|
||||
tf.file.write('123' * utils.CHUNK_SIZE)
|
||||
tf.file.flush()
|
||||
hd = utils.get_etag(tf.name)
|
||||
tf.file.seek(0)
|
||||
md5 = hashlib.md5()
|
||||
while True:
|
||||
chunk = tf.file.read(utils.CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
md5.update(chunk)
|
||||
assert hd == md5.hexdigest()
|
||||
|
||||
def test_get_object_metadata_dne(self):
|
||||
md = utils.get_object_metadata("/tmp/doesNotEx1st")
|
||||
assert md == {}
|
||||
|
||||
obj_keys = (utils.X_TIMESTAMP, utils.X_CONTENT_TYPE, utils.X_ETAG,
|
||||
utils.X_CONTENT_LENGTH, utils.X_TYPE, utils.X_OBJECT_TYPE)
|
||||
|
||||
def test_get_object_metadata_file(self):
|
||||
tf = tempfile.NamedTemporaryFile()
|
||||
tf.file.write('123'); tf.file.flush()
|
||||
md = utils.get_object_metadata(tf.name)
|
||||
for key in self.obj_keys:
|
||||
assert key in md, "Expected key %s in %r" % (key, md)
|
||||
assert md[utils.X_TYPE] == utils.OBJECT
|
||||
assert md[utils.X_OBJECT_TYPE] == utils.FILE
|
||||
assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE
|
||||
assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name)
|
||||
assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name))
|
||||
assert md[utils.X_ETAG] == utils.get_etag(tf.name)
|
||||
|
||||
def test_get_object_metadata_dir(self):
|
||||
td = tempfile.mkdtemp()
|
||||
try:
|
||||
md = utils.get_object_metadata(td)
|
||||
for key in self.obj_keys:
|
||||
assert key in md, "Expected key %s in %r" % (key, md)
|
||||
assert md[utils.X_TYPE] == utils.OBJECT
|
||||
assert md[utils.X_OBJECT_TYPE] == utils.DIR
|
||||
assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
|
||||
assert md[utils.X_CONTENT_LENGTH] == 0
|
||||
assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td))
|
||||
assert md[utils.X_ETAG] == hashlib.md5().hexdigest()
|
||||
finally:
|
||||
os.rmdir(td)
|
||||
|
||||
def test_create_object_metadata_file(self):
|
||||
tf = tempfile.NamedTemporaryFile()
|
||||
tf.file.write('4567'); tf.file.flush()
|
||||
r_md = utils.create_object_metadata(tf.name)
|
||||
|
||||
xkey = _xkey(tf.name, utils.METADATA_KEY)
|
||||
assert len(_xattrs.keys()) == 1
|
||||
assert xkey in _xattrs
|
||||
assert _xattr_op_cnt['get'] == 1
|
||||
assert _xattr_op_cnt['set'] == 1
|
||||
md = pickle.loads(_xattrs[xkey])
|
||||
assert r_md == md
|
||||
|
||||
for key in self.obj_keys:
|
||||
assert key in md, "Expected key %s in %r" % (key, md)
|
||||
assert md[utils.X_TYPE] == utils.OBJECT
|
||||
assert md[utils.X_OBJECT_TYPE] == utils.FILE
|
||||
assert md[utils.X_CONTENT_TYPE] == utils.FILE_TYPE
|
||||
assert md[utils.X_CONTENT_LENGTH] == os.path.getsize(tf.name)
|
||||
assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(tf.name))
|
||||
assert md[utils.X_ETAG] == utils.get_etag(tf.name)
|
||||
|
||||
def test_create_object_metadata_dir(self):
|
||||
td = tempfile.mkdtemp()
|
||||
try:
|
||||
r_md = utils.create_object_metadata(td)
|
||||
|
||||
xkey = _xkey(td, utils.METADATA_KEY)
|
||||
assert len(_xattrs.keys()) == 1
|
||||
assert xkey in _xattrs
|
||||
assert _xattr_op_cnt['get'] == 1
|
||||
assert _xattr_op_cnt['set'] == 1
|
||||
md = pickle.loads(_xattrs[xkey])
|
||||
assert r_md == md
|
||||
|
||||
for key in self.obj_keys:
|
||||
assert key in md, "Expected key %s in %r" % (key, md)
|
||||
assert md[utils.X_TYPE] == utils.OBJECT
|
||||
assert md[utils.X_OBJECT_TYPE] == utils.DIR
|
||||
assert md[utils.X_CONTENT_TYPE] == utils.DIR_TYPE
|
||||
assert md[utils.X_CONTENT_LENGTH] == 0
|
||||
assert md[utils.X_TIMESTAMP] == normalize_timestamp(os.path.getctime(td))
|
||||
assert md[utils.X_ETAG] == hashlib.md5().hexdigest()
|
||||
finally:
|
||||
os.rmdir(td)
|
||||
|
||||
def test_get_container_metadata(self):
|
||||
def _mock_get_container_details(path, memcache=None):
|
||||
o_list = [ 'a', 'b', 'c' ]
|
||||
o_count = 3
|
||||
b_used = 47
|
||||
return o_list, o_count, b_used
|
||||
td = tempfile.mkdtemp()
|
||||
orig_gcd = utils.get_container_details
|
||||
utils.get_container_details = _mock_get_container_details
|
||||
try:
|
||||
exp_md = {
|
||||
utils.X_TYPE: (utils.CONTAINER, 0),
|
||||
utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0),
|
||||
utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0),
|
||||
utils.X_OBJECTS_COUNT: (3, 0),
|
||||
utils.X_BYTES_USED: (47, 0),
|
||||
}
|
||||
md = utils.get_container_metadata(td)
|
||||
assert md == exp_md
|
||||
finally:
|
||||
utils.get_container_details = orig_gcd
|
||||
os.rmdir(td)
|
||||
|
||||
def test_get_account_metadata(self):
|
||||
def _mock_get_account_details(path, memcache=None):
|
||||
c_list = [ '123', 'abc' ]
|
||||
c_count = 2
|
||||
return c_list, c_count
|
||||
td = tempfile.mkdtemp()
|
||||
orig_gad = utils.get_account_details
|
||||
utils.get_account_details = _mock_get_account_details
|
||||
try:
|
||||
exp_md = {
|
||||
utils.X_TYPE: (utils.ACCOUNT, 0),
|
||||
utils.X_TIMESTAMP: (normalize_timestamp(os.path.getctime(td)), 0),
|
||||
utils.X_PUT_TIMESTAMP: (normalize_timestamp(os.path.getmtime(td)), 0),
|
||||
utils.X_OBJECTS_COUNT: (0, 0),
|
||||
utils.X_BYTES_USED: (0, 0),
|
||||
utils.X_CONTAINER_COUNT: (2, 0),
|
||||
}
|
||||
md = utils.get_account_metadata(td)
|
||||
assert md == exp_md
|
||||
finally:
|
||||
utils.get_account_details = orig_gad
|
||||
os.rmdir(td)
|
||||
|
||||
cont_keys = [utils.X_TYPE, utils.X_TIMESTAMP, utils.X_PUT_TIMESTAMP,
|
||||
utils.X_OBJECTS_COUNT, utils.X_BYTES_USED]
|
||||
|
||||
def test_create_container_metadata(self):
|
||||
td = tempfile.mkdtemp()
|
||||
try:
|
||||
r_md = utils.create_container_metadata(td)
|
||||
|
||||
xkey = _xkey(td, utils.METADATA_KEY)
|
||||
assert len(_xattrs.keys()) == 1
|
||||
assert xkey in _xattrs
|
||||
assert _xattr_op_cnt['get'] == 1
|
||||
assert _xattr_op_cnt['set'] == 1
|
||||
md = pickle.loads(_xattrs[xkey])
|
||||
assert r_md == md
|
||||
|
||||
for key in self.cont_keys:
|
||||
assert key in md, "Expected key %s in %r" % (key, md)
|
||||
assert md[utils.X_TYPE] == (utils.CONTAINER, 0)
|
||||
assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0)
|
||||
assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0)
|
||||
assert md[utils.X_OBJECTS_COUNT] == (0, 0)
|
||||
assert md[utils.X_BYTES_USED] == (0, 0)
|
||||
finally:
|
||||
os.rmdir(td)
|
||||
|
||||
acct_keys = [val for val in cont_keys]
|
||||
acct_keys.append(utils.X_CONTAINER_COUNT)
|
||||
|
||||
def test_create_account_metadata(self):
|
||||
td = tempfile.mkdtemp()
|
||||
try:
|
||||
r_md = utils.create_account_metadata(td)
|
||||
|
||||
xkey = _xkey(td, utils.METADATA_KEY)
|
||||
assert len(_xattrs.keys()) == 1
|
||||
assert xkey in _xattrs
|
||||
assert _xattr_op_cnt['get'] == 1
|
||||
assert _xattr_op_cnt['set'] == 1
|
||||
md = pickle.loads(_xattrs[xkey])
|
||||
assert r_md == md
|
||||
|
||||
for key in self.acct_keys:
|
||||
assert key in md, "Expected key %s in %r" % (key, md)
|
||||
assert md[utils.X_TYPE] == (utils.ACCOUNT, 0)
|
||||
assert md[utils.X_TIMESTAMP] == (normalize_timestamp(os.path.getctime(td)), 0)
|
||||
assert md[utils.X_PUT_TIMESTAMP] == (normalize_timestamp(os.path.getmtime(td)), 0)
|
||||
assert md[utils.X_OBJECTS_COUNT] == (0, 0)
|
||||
assert md[utils.X_BYTES_USED] == (0, 0)
|
||||
assert md[utils.X_CONTAINER_COUNT] == (0, 0)
|
||||
finally:
|
||||
os.rmdir(td)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user