andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

storage/posix: Implement .unlink directory

Browse Source 
Problem: For EC volume, If a file descriptor is open and
file has been unlinked, any further write on that fd will
fail. When a write request comes, EC internally reads some
blocks using anonymous fd. This read will fail as the file
has already been unlinked.

Solution: To solve this issue, we are using .unlink directory
to keep track of unlinked file. If a file is to be unlinked
while its fd is open, move this to .unlink directory and unlink
it from .glusterfs and real path. Once all the fd will be closed,
remove this entry form .unlink directory.

Change-Id: I8344edb0d340bdb883dc46458c16edbc336916b9
BUG: 1286029
Signed-off-by: Ashish Pandey <aspandey@redhat.com>
Reviewed-on: http://review.gluster.org/12816
Reviewed-by: Krutika Dhananjay <kdhananj@redhat.com>
Reviewed-by: Xavier Hernandez <xhernandez@datalab.es>
Tested-by: NetBSD Build System <jenkins@build.gluster.org>
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Pranith Kumar Karampuri <pkarampu@redhat.com>
This commit is contained in:
Ashish Pandey 2015-11-26 14:35:49 +05:30 committed by Pranith Kumar Karampuri
parent be438f9aa7
commit c4f6521dfe
9 changed files with 324 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libglusterfs/src/common-utils.h
View File

@ -359,6 +359,8 @@ union gf_sock_union {
};
#define GF_HIDDEN_PATH ".glusterfs"
#define GF_UNLINK_PATH GF_HIDDEN_PATH"/unlink"
#define GF_LANDFILL_PATH GF_HIDDEN_PATH"/landfill"
#define IOV_MIN(n) min(IOV_MAX,n)

2
libglusterfs/src/inode.h
View File

@ -93,7 +93,7 @@ struct _inode {
struct list_head hash; /* hash table pointers */
struct list_head list; /* active/lru/purge */
struct _inode_ctx *_ctx; /* replacement for dict_t *(inode->ctx) */
struct _inode_ctx *_ctx; /* replacement for dict_t *(inode->ctx) */
};

42
tests/basic/ec/ec-anonymous-fd.t Normal file
View File

@ -0,0 +1,42 @@
#!/bin/bash
. $(dirname $0)/../../include.rc
. $(dirname $0)/../../volume.rc
. $(dirname $0)/../../fileio.rc
cleanup
function num_entries {
ls -l $1 | wc -l
}
TEST glusterd
TEST pidof glusterd
TEST $CLI volume create $V0 disperse 6 redundancy 2 $H0:$B0/${V0}{0..5}
TEST $CLI volume start $V0
TEST glusterfs --entry-timeout=0 --attribute-timeout=0 -s $H0 --volfile-id $V0 $M0
EXPECT_WITHIN $CHILD_UP_TIMEOUT "6" ec_child_up_count $V0 0
fd1=`fd_available`
TEST fd_open $fd1 'w' $M0/file1
TEST fd_write $fd1 testing
TEST cat $M0/file1
TEST rm -rf $M0/file1
TEST fd_write $fd1 testing
TEST fd_write $fd1 testing
TEST fd_write $fd1 testing
EXPECT "^2$" num_entries $B0/${V0}0/.glusterfs/unlink/
EXPECT "^2$" num_entries $B0/${V0}1/.glusterfs/unlink/
EXPECT "^2$" num_entries $B0/${V0}2/.glusterfs/unlink/
EXPECT "^2$" num_entries $B0/${V0}3/.glusterfs/unlink/
EXPECT "^2$" num_entries $B0/${V0}4/.glusterfs/unlink/
EXPECT "^2$" num_entries $B0/${V0}5/.glusterfs/unlink/
TEST fd_close $fd1;
EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}0/.glusterfs/unlink/
EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}1/.glusterfs/unlink/
EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}2/.glusterfs/unlink/
EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}3/.glusterfs/unlink/
EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}4/.glusterfs/unlink/
EXPECT_WITHIN $UNLINK_TIMEOUT "^1$" num_entries $B0/${V0}5/.glusterfs/unlink/
cleanup;

1
tests/include.rc
View File

@ -72,6 +72,7 @@ UMOUNT_TIMEOUT=5
CONFIG_UPDATE_TIMEOUT=5
AUTH_REFRESH_INTERVAL=10
GRAPH_SWITCH_TIMEOUT=10
UNLINK_TIMEOUT=5
LOGDIR=$(gluster --print-logdir)

3
xlators/storage/posix/src/posix-handle.c
View File

@ -16,6 +16,8 @@
#include <alloca.h>
#endif
#include "common-utils.h"
#include "posix-handle.h"
#include "posix.h"
#include "xlator.h"
@ -447,7 +449,6 @@ out:
return len;
}
int
posix_handle_init (xlator_t *this)
{

77
xlators/storage/posix/src/posix-helpers.c
View File

@ -1604,7 +1604,6 @@ out:
return ret;
}
static int
__posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p)
{
@ -1612,34 +1611,37 @@ __posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p)
struct posix_fd *pfd = NULL;
int ret = -1;
char *real_path = NULL;
char *unlink_path = NULL;
int _fd = -1;
DIR *dir = NULL;
struct posix_private *priv = NULL;
priv = this->private;
ret = __fd_ctx_get (fd, this, &tmp_pfd);
if (ret == 0) {
pfd = (void *)(long) tmp_pfd;
ret = 0;
goto out;
}
MAKE_HANDLE_PATH (real_path, this, fd->inode->gfid, NULL);
if (!real_path) {
gf_msg (this->name, GF_LOG_ERROR, 0,
P_MSG_HANDLE_PATH_CREATE_FAILED,
"Failed to create handle path (%s)",
uuid_utoa (fd->inode->gfid));
ret = -1;
goto out;
}
if (!fd_is_anonymous(fd)) {
gf_log (this->name, GF_LOG_ERROR,
gf_msg (this->name, GF_LOG_ERROR, 0,
P_MSG_READ_FAILED,
"Failed to get fd context for a non-anonymous fd, "
"file: %s, gfid: %s", real_path,
uuid_utoa (fd->inode->gfid));
goto out;
}
MAKE_HANDLE_PATH (real_path, this, fd->inode->gfid, NULL);
if (!real_path) {
gf_msg (this->name, GF_LOG_ERROR, 0,
P_MSG_READ_FAILED,
"Failed to create handle path (%s)",
uuid_utoa (fd->inode->gfid));
ret = -1;
goto out;
}
pfd = GF_CALLOC (1, sizeof (*pfd), gf_posix_mt_posix_fd);
if (!pfd) {
goto out;
@ -1663,6 +1665,16 @@ __posix_fd_ctx_get (fd_t *fd, xlator_t *this, struct posix_fd **pfd_p)
if (fd->inode->ia_type == IA_IFREG) {
_fd = open (real_path, fd->flags);
if (_fd == -1) {
POSIX_GET_FILE_UNLINK_PATH (priv->base_path,
fd->inode->gfid,
unlink_path);
_fd = open (unlink_path, fd->flags);
}
if (_fd == -1) {
gf_msg (this->name, GF_LOG_ERROR, errno,
P_MSG_READ_FAILED,
"Failed to get anonymous "
"real_path: %s _fd = %d", real_path, _fd);
GF_FREE (pfd);
pfd = NULL;
goto out;
@ -2164,3 +2176,40 @@ posix_fdget_objectsignature (int fd, dict_t *xattr)
error_return:
return -EINVAL;
}
int
posix_inode_ctx_get (inode_t *inode, xlator_t *this, uint64_t *ctx)
{
int ret = -1;
uint64_t ctx_int = 0;
GF_VALIDATE_OR_GOTO (this->name, this, out);
GF_VALIDATE_OR_GOTO (this->name, inode, out);
ret = inode_ctx_get (inode, this, &ctx_int);
if (ret)
return ret;
if (ctx)
*ctx = ctx_int;
out:
return ret;
}
int
posix_inode_ctx_set (inode_t *inode, xlator_t *this, uint64_t ctx)
{
int ret = -1;
GF_VALIDATE_OR_GOTO (this->name, this, out);
GF_VALIDATE_OR_GOTO (this->name, inode, out);
GF_VALIDATE_OR_GOTO (this->name, ctx, out);
ret = inode_ctx_set (inode, this, &ctx);
out:
return ret;
}

1
xlators/storage/posix/src/posix-mem-types.h
View File

@ -21,6 +21,7 @@ enum gf_posix_mem_types_ {
gf_posix_mt_posix_dev_t,
gf_posix_mt_trash_path,
gf_posix_mt_paiocb,
gf_posix_mt_inode_ctx_t,
gf_posix_mt_end
};
#endif

196
xlators/storage/posix/src/posix.c
View File

@ -101,10 +101,32 @@ int
posix_forget (xlator_t *this, inode_t *inode)
{
uint64_t tmp_cache = 0;
if (!inode_ctx_del (inode, this, &tmp_cache))
dict_destroy ((dict_t *)(long)tmp_cache);
int ret = 0;
char *unlink_path = NULL;
struct posix_private *priv_posix = NULL;
return 0;
priv_posix = (struct posix_private *) this->private;
ret = inode_ctx_del (inode, this, &tmp_cache);
if (ret < 0) {
ret = 0;
goto out;
}
if (tmp_cache == GF_UNLINK_TRUE) {
POSIX_GET_FILE_UNLINK_PATH(priv_posix->base_path,
inode->gfid, unlink_path);
if (!unlink_path) {
gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
P_MSG_UNLINK_FAILED,
"Failed to remove gfid :%s",
uuid_utoa (inode->gfid));
ret = -1;
goto out;
}
ret = sys_unlink(unlink_path);
}
out:
return ret;
}
/* Regular fops */
@ -1446,16 +1468,93 @@ out:
return 0;
}
int
posix_add_unlink_to_ctx (inode_t *inode, xlator_t *this, char *unlink_path)
{
uint64_t ctx = GF_UNLINK_FALSE;
int ret = 0;
if (!unlink_path) {
gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
P_MSG_UNLINK_FAILED,
"Creation of unlink entry failed for gfid: %s",
unlink_path);
ret = -1;
goto out;
}
ctx = GF_UNLINK_TRUE;
ret = posix_inode_ctx_set (inode, this, ctx);
if (ret < 0) {
goto out;
}
out:
return ret;
}
int32_t
posix_move_gfid_to_unlink (xlator_t *this, uuid_t gfid, loc_t *loc)
{
char *unlink_path = NULL;
char *gfid_path = NULL;
struct stat stbuf = {0, };
int ret = 0;
struct posix_private *priv_posix = NULL;
priv_posix = (struct posix_private *) this->private;
MAKE_HANDLE_GFID_PATH (gfid_path, this, gfid, NULL);
POSIX_GET_FILE_UNLINK_PATH (priv_posix->base_path,
loc->inode->gfid, unlink_path);
if (!unlink_path) {
ret = -1;
goto out;
}
gf_msg_debug (this->name, 0,
"Moving gfid: %s to unlink_path : %s",
gfid_path, unlink_path);
ret = sys_rename (gfid_path, unlink_path);
if (ret < 0) {
gf_msg (this->name, GF_LOG_ERROR, errno,
P_MSG_UNLINK_FAILED,
"Creation of unlink entry failed for gfid: %s",
unlink_path);
goto out;
}
ret = posix_add_unlink_to_ctx (loc->inode, this, unlink_path);
if (ret < 0)
goto out;
out:
return ret;
}
int32_t
posix_unlink_gfid_handle_and_entry (xlator_t *this, const char *real_path,
struct iatt *stbuf, int32_t *op_errno)
struct iatt *stbuf, int32_t *op_errno,
loc_t *loc)
{
int32_t ret = 0;
int32_t ret = 0;
struct posix_private *priv = NULL;
int fd_count = 0;
priv = this->private;
/* Unlink the gfid_handle_first */
if (stbuf && stbuf->ia_nlink == 1) {
ret = posix_handle_unset (this, stbuf->ia_gfid, NULL);
LOCK (&loc->inode->lock);
if (loc->inode->fd_count == 0) {
UNLOCK (&loc->inode->lock);
ret = posix_handle_unset (this, stbuf->ia_gfid, NULL);
} else {
UNLOCK (&loc->inode->lock);
ret = posix_move_gfid_to_unlink (this, stbuf->ia_gfid,
loc);
}
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, errno,
P_MSG_UNLINK_FAILED, "unlink of gfid handle "
@ -1469,7 +1568,6 @@ posix_unlink_gfid_handle_and_entry (xlator_t *this, const char *real_path,
if (ret == -1) {
if (op_errno)
*op_errno = errno;
gf_msg (this->name, GF_LOG_ERROR, errno, P_MSG_UNLINK_FAILED,
"unlink of %s failed", real_path);
goto err;
@ -1623,7 +1721,7 @@ posix_unlink (call_frame_t *frame, xlator_t *this,
}
op_ret = posix_unlink_gfid_handle_and_entry (this, real_path, &stbuf,
&op_errno);
&op_errno, loc);
if (op_ret == -1) {
goto out;
}
@ -6176,6 +6274,77 @@ out:
return ret;
}
int32_t
posix_create_unlink_dir (xlator_t *this) {
char *unlink_path = NULL;
char *landfill_path = NULL;
struct posix_private *priv = NULL;
struct stat stbuf;
int ret = -1;
uuid_t gfid = {0};
char gfid_str[64] = {0};
priv = this->private;
unlink_path = alloca (strlen (priv->base_path) + 1 +
strlen (GF_UNLINK_PATH) + 1);
sprintf (unlink_path, "%s/%s", priv->base_path,
GF_UNLINK_PATH);
gf_uuid_generate (gfid);
uuid_utoa_r (gfid, gfid_str);
landfill_path = alloca (strlen (priv->base_path) + 1 +
strlen (GF_LANDFILL_PATH) + 1 +
strlen (gfid_str) + 1);
sprintf (landfill_path, "%s/%s/%s", priv->base_path,
GF_LANDFILL_PATH, gfid_str);
ret = sys_stat (unlink_path, &stbuf);
switch (ret) {
case -1:
if (errno != ENOENT) {
gf_msg (this->name, GF_LOG_ERROR, errno,
P_MSG_HANDLE_CREATE,
"Checking for %s failed",
unlink_path);
return -1;
}
break;
case 0:
if (!S_ISDIR (stbuf.st_mode)) {
gf_msg (this->name, GF_LOG_ERROR, 0,
P_MSG_HANDLE_CREATE,
"Not a directory: %s",
unlink_path);
return -1;
}
ret = sys_rename (unlink_path, landfill_path);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, errno,
P_MSG_HANDLE_CREATE,
"Can not delete directory %s ",
unlink_path);
return -1;
}
break;
default:
break;
}
ret = sys_mkdir (unlink_path, 0600);
if (ret) {
gf_msg (this->name, GF_LOG_ERROR, errno,
P_MSG_HANDLE_CREATE,
"Creating directory %s failed",
unlink_path);
return -1;
}
return 0;
}
/**
* init -
@ -6587,6 +6756,15 @@ init (xlator_t *this)
goto out;
}
op_ret = posix_create_unlink_dir (this);
if (op_ret == -1) {
gf_msg (this->name, GF_LOG_ERROR, 0,
P_MSG_HANDLE_CREATE,
"Creation of unlink directory failed");
ret = -1;
goto out;
}
_private->aio_init_done = _gf_false;
_private->aio_capable = _gf_false;

26
xlators/storage/posix/src/posix.h
View File

@ -56,6 +56,8 @@
+ SLEN(GF_HIDDEN_PATH) + SLEN("/") \
+ SLEN("00/") \
+ SLEN("00/") + SLEN(UUID0_STR) + 1) /* '\0' */;
#define GF_UNLINK_TRUE 0x0000000000000001
#define GF_UNLINK_FALSE 0x0000000000000000
/**
* posix_fd - internal structure common to file and directory fd's
@ -184,14 +186,36 @@ typedef struct {
int32_t op_errno;
} posix_xattr_filler_t;
#define POSIX_BASE_PATH(this) (((struct posix_private *)this->private)->base_path)
#define POSIX_BASE_PATH_LEN(this) (((struct posix_private *)this->private)->base_path_length)
#define POSIX_PATH_MAX(this) (((struct posix_private *)this->private)->path_max)
#define POSIX_GET_FILE_UNLINK_PATH(base_path, gfid, unlink_path) \
do { \
int path_len = 0; \
char gfid_str[64] = {0}; \
uuid_utoa_r (gfid, gfid_str); \
path_len = strlen (base_path) + 1 + \
strlen (GF_UNLINK_PATH) + 1 + \
strlen (gfid_str) + 1; \
unlink_path = alloca (path_len); \
if (!unlink_path) { \
gf_msg ("posix", GF_LOG_ERROR, ENOMEM, \
P_MSG_UNLINK_FAILED, \
"Failed to get unlink_path"); \
break; \
} \
sprintf (unlink_path, "%s/%s/%s", \
base_path, GF_UNLINK_PATH, gfid_str); \
} while (0)
/* Helper functions */
int posix_inode_ctx_get (inode_t *inode, xlator_t *this, uint64_t *ctx);
int posix_inode_ctx_set (inode_t *inode, xlator_t *this, uint64_t ctx);
int posix_gfid_set (xlator_t *this, const char *path, loc_t *loc,
dict_t *xattr_req);
int posix_fdstat (xlator_t *this, int fd, struct iatt *stbuf_p);