andy/glusterfs
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

rpc: fix binding brick issue while bind-insecure is enabled

Browse Source 
problem:

When bind-insecure is turned on (which is the default now), it may happen
that brick is not able to bind to port assigned by Glusterd for example
49192-49195...

It seems to occur because the rpc_clnt connections are binding to ports in
the same range. so brick fails to bind to a port which is already used by
someone else

solution:

fix for now is to  make rpc_clnt to get port numbers from 65535 in a descending
order, as a result port clash is minimized

other fixes:

previously rdma binds to port >= 1024 if it cannot find a free port < 1024,
even when bind insecure was turned off(ref to commit '0e3fd04e'), this patch
add's a check for bind-insecure in gf_rdma_client_bind function

This patch also re-enable bind-insecure and allow insecure by default which was
reverted (ref: commit cef1720) previously

Change-Id: Ia1cfa93c5454e2ae0ff57813689b75de282ebd07
BUG: 1238661
Signed-off-by: Prasanna Kumar Kalever <prasanna.kalever@redhat.com>
Reviewed-on: http://review.gluster.org/11512
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Raghavendra G <rgowdapp@redhat.com>
This commit is contained in:
Prasanna Kumar Kalever
2015-06-24 12:21:02 +05:30
committed by Raghavendra G
parent e9b86d0b57
commit 9442e7bf80
7 changed files with 103 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
libglusterfs/src/common-utils.c
View File

@ -2789,7 +2789,7 @@ out:
} }
int int
gf_process_reserved_ports (gf_boolean_t *ports) gf_process_reserved_ports (gf_boolean_t *ports, uint32_t ceiling)
{ {
int ret = -1; int ret = -1;
#if defined GF_LINUX_HOST_OS #if defined GF_LINUX_HOST_OS
@ -2809,7 +2809,7 @@ gf_process_reserved_ports (gf_boolean_t *ports)
blocked_port = strtok_r (ports_info, ",\n",&tmp); blocked_port = strtok_r (ports_info, ",\n",&tmp);
while (blocked_port) { while (blocked_port) {
gf_ports_reserved (blocked_port, ports); gf_ports_reserved (blocked_port, ports, ceiling);
blocked_port = strtok_r (NULL, ",\n", &tmp); blocked_port = strtok_r (NULL, ",\n", &tmp);
} }
@ -2822,7 +2822,7 @@ out:
} }
gf_boolean_t gf_boolean_t
gf_ports_reserved (char *blocked_port, gf_boolean_t *ports) gf_ports_reserved (char *blocked_port, gf_boolean_t *ports, uint32_t ceiling)
{ {
gf_boolean_t result = _gf_false; gf_boolean_t result = _gf_false;
char *range_port = NULL; char *range_port = NULL;
@ -2834,7 +2834,7 @@ gf_ports_reserved (char *blocked_port, gf_boolean_t *ports)
if (blocked_port[strlen(blocked_port) -1] == '\n') if (blocked_port[strlen(blocked_port) -1] == '\n')
blocked_port[strlen(blocked_port) -1] = '\0'; blocked_port[strlen(blocked_port) -1] = '\0';
if (gf_string2int16 (blocked_port, &tmp_port1) == 0) { if (gf_string2int16 (blocked_port, &tmp_port1) == 0) {
if (tmp_port1 > (GF_CLIENT_PORT_CEILING - 1) if (tmp_port1 > ceiling
|| tmp_port1 < 0) { || tmp_port1 < 0) {
gf_msg ("glusterfs-socket", GF_LOG_WARNING, 0, gf_msg ("glusterfs-socket", GF_LOG_WARNING, 0,
LG_MSG_INVALID_PORT, "invalid port %d", LG_MSG_INVALID_PORT, "invalid port %d",
@ -2860,8 +2860,8 @@ gf_ports_reserved (char *blocked_port, gf_boolean_t *ports)
goto out; goto out;
} }
if (gf_string2int16 (range_port, &tmp_port1) == 0) { if (gf_string2int16 (range_port, &tmp_port1) == 0) {
if (tmp_port1 > (GF_CLIENT_PORT_CEILING - 1)) if (tmp_port1 > ceiling)
tmp_port1 = GF_CLIENT_PORT_CEILING - 1; tmp_port1 = ceiling;
if (tmp_port1 < 0) if (tmp_port1 < 0)
tmp_port1 = 0; tmp_port1 = 0;
} }
@ -2874,9 +2874,8 @@ gf_ports_reserved (char *blocked_port, gf_boolean_t *ports)
if (range_port[strlen(range_port) -1] == '\n') if (range_port[strlen(range_port) -1] == '\n')
range_port[strlen(range_port) - 1] = '\0'; range_port[strlen(range_port) - 1] = '\0';
if (gf_string2int16 (range_port, &tmp_port2) == 0) { if (gf_string2int16 (range_port, &tmp_port2) == 0) {
if (tmp_port2 > if (tmp_port2 > ceiling)
(GF_CLIENT_PORT_CEILING - 1)) tmp_port2 = ceiling;
tmp_port2 = GF_CLIENT_PORT_CEILING - 1;
if (tmp_port2 < 0) if (tmp_port2 < 0)
tmp_port2 = 0; tmp_port2 = 0;
} }

6
libglusterfs/src/common-utils.h
View File

@ -83,6 +83,7 @@ void trap (void);
*/ */
#define GF_NFS3_PORT 2049 #define GF_NFS3_PORT 2049
#define GF_CLIENT_PORT_CEILING 1024 #define GF_CLIENT_PORT_CEILING 1024
#define GF_PORT_MAX 65535
#define GF_MINUTE_IN_SECONDS 60 #define GF_MINUTE_IN_SECONDS 60
#define GF_HOUR_IN_SECONDS (60*60) #define GF_HOUR_IN_SECONDS (60*60)
@ -697,8 +698,9 @@ int gf_strip_whitespace (char *str, int len);
int gf_canonicalize_path (char *path); int gf_canonicalize_path (char *path);
char *generate_glusterfs_ctx_id (void); char *generate_glusterfs_ctx_id (void);
char *gf_get_reserved_ports(); char *gf_get_reserved_ports();
int gf_process_reserved_ports (gf_boolean_t ports[]); int gf_process_reserved_ports (gf_boolean_t ports[], uint32_t ceiling);
gf_boolean_t gf_ports_reserved (char *blocked_port, gf_boolean_t *ports); gf_boolean_t
gf_ports_reserved (char *blocked_port, gf_boolean_t *ports, uint32_t ceiling);
int gf_get_hostname_from_ip (char *client_ip, char **hostname); int gf_get_hostname_from_ip (char *client_ip, char **hostname);
gf_boolean_t gf_is_local_addr (char *hostname); gf_boolean_t gf_is_local_addr (char *hostname);
gf_boolean_t gf_is_same_address (char *host1, char *host2); gf_boolean_t gf_is_same_address (char *host1, char *host2);

3
rpc/rpc-lib/src/rpc-transport.c
View File

@ -262,7 +262,8 @@ rpc_transport_load (glusterfs_ctx_t *ctx, dict_t *options, char *trans_name)
else else
trans->bind_insecure = 0; trans->bind_insecure = 0;
} else { } else {
trans->bind_insecure = 0; /* By default allow bind insecure */
trans->bind_insecure = 1;
} }
ret = dict_get_str (options, "transport-type", &type); ret = dict_get_str (options, "transport-type", &type);

13
rpc/rpc-lib/src/rpcsvc-auth.c
View File

@ -221,9 +221,20 @@ rpcsvc_set_allow_insecure (rpcsvc_t *svc, dict_t *options)
else else
svc->allow_insecure = 0; svc->allow_insecure = 0;
} }
} else {
/* By default set allow-insecure to true */
svc->allow_insecure = 1;
/* setting in options for the sake of functions that look
* configuration params for allow insecure, eg: gf_auth
*/
ret = dict_set_str (options, "rpc-auth-allow-insecure", "on");
if (ret < 0)
gf_log ("rpc-auth", GF_LOG_DEBUG,
"dict_set failed for 'allow-insecure'");
} }
return 0; return ret;
} }
int int

6
rpc/rpc-lib/src/rpcsvc.c
View File

@ -632,8 +632,10 @@ rpcsvc_handle_rpc_call (rpcsvc_t *svc, rpc_transport_t *trans,
"Request received from non-" "Request received from non-"
"privileged port. Failing request for %s.", "privileged port. Failing request for %s.",
req->trans->peerinfo.identifier); req->trans->peerinfo.identifier);
rpcsvc_request_destroy (req); req->rpc_status = MSG_DENIED;
return -1; req->rpc_err = AUTH_ERROR;
req->auth_err = RPCSVC_AUTH_REJECT;
goto err_reply;
} }
/* DRC */ /* DRC */

67
rpc/rpc-transport/rdma/src/name.c
View File

@ -33,36 +33,41 @@ gf_resolve_ip6 (const char *hostname,
void **dnscache, void **dnscache,
struct addrinfo **addr_info); struct addrinfo **addr_info);
static void
_assign_port (struct sockaddr *sockaddr, uint16_t port)
{
switch (sockaddr->sa_family) {
case AF_INET6:
((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port);
break;
case AF_INET_SDP:
case AF_INET:
((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
break;
}
}
static int32_t static int32_t
af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id, af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id,
struct sockaddr *sockaddr, struct sockaddr *sockaddr,
socklen_t sockaddr_len, int ceiling) socklen_t sockaddr_len, uint32_t ceiling)
{ {
int32_t ret = -1; int32_t ret = -1;
uint16_t port = ceiling - 1; uint16_t port = ceiling - 1;
/* by default assume none of the ports are blocked and all are available */ /* by default assume none of the ports are blocked and all are available */
gf_boolean_t ports[1024] = {_gf_false,}; gf_boolean_t ports[GF_PORT_MAX] = {_gf_false,};
int i = 0; int i = 0;
ret = gf_process_reserved_ports (ports); ret = gf_process_reserved_ports (ports, ceiling);
if (ret != 0) { if (ret != 0) {
for (i = 0; i < 1024; i++) for (i = 0; i < GF_PORT_MAX; i++)
ports[i] = _gf_false; ports[i] = _gf_false;
} }
while (port) { while (port) {
switch (sockaddr->sa_family) { _assign_port (sockaddr, port);
case AF_INET6:
((struct sockaddr_in6 *)sockaddr)->sin6_port
= htons (port);
break;
case AF_INET_SDP:
case AF_INET:
((struct sockaddr_in *)sockaddr)->sin_port
= htons (port);
break;
}
/* ignore the reserved ports */ /* ignore the reserved ports */
if (ports[port] == _gf_true) { if (ports[port] == _gf_true) {
port--; port--;
@ -426,22 +431,26 @@ gf_rdma_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr,
*sockaddr_len = sizeof (struct sockaddr_in); *sockaddr_len = sizeof (struct sockaddr_in);
case AF_INET6: case AF_INET6:
ret = af_inet_bind_to_port_lt_ceiling (cm_id, sockaddr, if (!this->bind_insecure) {
ret = af_inet_bind_to_port_lt_ceiling (cm_id, sockaddr,
*sockaddr_len, *sockaddr_len,
GF_CLIENT_PORT_CEILING); GF_CLIENT_PORT_CEILING);
if (ret == -1) { if (ret == -1) {
gf_msg (this->name, GF_LOG_WARNING, errno, gf_msg (this->name, GF_LOG_WARNING, errno,
RDMA_MSG_PORT_BIND_FAILED, RDMA_MSG_PORT_BIND_FAILED,
"cannot bind rdma_cm_id to port " "cannot bind rdma_cm_id to port "
"less than %d", GF_CLIENT_PORT_CEILING); "less than %d", GF_CLIENT_PORT_CEILING);
if (sockaddr->sa_family == AF_INET6) { }
((struct sockaddr_in6 *)sockaddr)->sin6_port } else {
= htons (0); ret = af_inet_bind_to_port_lt_ceiling (cm_id, sockaddr,
} else { *sockaddr_len,
((struct sockaddr_in *)sockaddr)->sin_port GF_PORT_MAX);
= htons (0); if (ret == -1) {
gf_msg (this->name, GF_LOG_WARNING, errno,
RDMA_MSG_PORT_BIND_FAILED,
"cannot bind rdma_cm_id to port "
"less than %d", GF_PORT_MAX);
} }
ret = rdma_bind_addr (cm_id, sockaddr);
} }
break; break;

56
rpc/rpc-transport/socket/src/name.c
View File

@ -23,35 +23,40 @@
#include "socket.h" #include "socket.h"
#include "common-utils.h" #include "common-utils.h"
static void
_assign_port (struct sockaddr *sockaddr, uint16_t port)
{
switch (sockaddr->sa_family) {
case AF_INET6:
((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port);
break;
case AF_INET_SDP:
case AF_INET:
((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
break;
}
}
static int32_t static int32_t
af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr, af_inet_bind_to_port_lt_ceiling (int fd, struct sockaddr *sockaddr,
socklen_t sockaddr_len, int ceiling) socklen_t sockaddr_len, uint32_t ceiling)
{ {
int32_t ret = -1; int32_t ret = -1;
uint16_t port = ceiling - 1; uint16_t port = ceiling - 1;
// by default assume none of the ports are blocked and all are available // by default assume none of the ports are blocked and all are available
gf_boolean_t ports[1024] = {_gf_false,}; gf_boolean_t ports[GF_PORT_MAX] = {_gf_false,};
int i = 0; int i = 0;
ret = gf_process_reserved_ports (ports); ret = gf_process_reserved_ports (ports, ceiling);
if (ret != 0) { if (ret != 0) {
for (i = 0; i < 1024; i++) for (i = 0; i < GF_PORT_MAX; i++)
ports[i] = _gf_false; ports[i] = _gf_false;
} }
while (port) while (port)
{ {
switch (sockaddr->sa_family) _assign_port (sockaddr, port);
{
case AF_INET6:
((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port);
break;
case AF_INET_SDP:
case AF_INET:
((struct sockaddr_in *)sockaddr)->sin_port = htons (port);
break;
}
// ignore the reserved ports // ignore the reserved ports
if (ports[port] == _gf_true) { if (ports[port] == _gf_true) {
port--; port--;
@ -440,12 +445,21 @@ client_bind (rpc_transport_t *this,
if (!this->bind_insecure) { if (!this->bind_insecure) {
ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr, ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr,
*sockaddr_len, GF_CLIENT_PORT_CEILING); *sockaddr_len, GF_CLIENT_PORT_CEILING);
} if (ret == -1) {
if (ret == -1) { gf_log (this->name, GF_LOG_DEBUG,
gf_log (this->name, GF_LOG_DEBUG, "cannot bind inet socket (%d) to port less than %d (%s)",
"cannot bind inet socket (%d) to port less than %d (%s)", sock, GF_CLIENT_PORT_CEILING, strerror (errno));
sock, GF_CLIENT_PORT_CEILING, strerror (errno)); ret = 0;
ret = 0; }
} else {
ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr,
*sockaddr_len, GF_PORT_MAX);
if (ret == -1) {
gf_log (this->name, GF_LOG_DEBUG,
"failed while binding to less than %d (%s)",
GF_PORT_MAX, strerror (errno));
ret = 0;
}
} }
break; break;