2004-06-24 12:02:38 +04:00
/*
* Copyright ( C ) 2002 - 2004 Sistina Software , Inc . All rights reserved .
* Copyright ( C ) 2004 Red Hat , Inc . All rights reserved .
*
* This file is part of LVM2 .
*
* This copyrighted material is made available to anyone wishing to use ,
* modify , copy , or redistribute it subject to the terms and conditions
* of the GNU General Public License v .2 .
*
* You should have received a copy of the GNU General Public License
* along with this program ; if not , write to the Free Software Foundation ,
* Inc . , 59 Temple Place , Suite 330 , Boston , MA 02111 - 1307 USA
*/
/*
* CMAN communication layer for clvmd .
*/
2008-11-04 19:41:47 +03:00
# define _GNU_SOURCE
# define _FILE_OFFSET_BITS 64
# include <configure.h>
2004-06-24 12:02:38 +04:00
# include <pthread.h>
# include <sys/types.h>
# include <sys/stat.h>
# include <sys/socket.h>
# include <sys/uio.h>
# include <sys/un.h>
# include <sys/time.h>
# include <sys/ioctl.h>
# include <sys/utsname.h>
# include <syslog.h>
# include <netinet/in.h>
# include <stdio.h>
# include <stdlib.h>
# include <stddef.h>
# include <signal.h>
# include <unistd.h>
# include <fcntl.h>
# include <getopt.h>
# include <errno.h>
2008-05-09 11:20:04 +04:00
# include <libdevmapper.h>
2007-04-27 21:46:16 +04:00
# include <libdlm.h>
2004-06-24 12:02:38 +04:00
# include "clvmd-comms.h"
# include "clvm.h"
2008-11-04 19:41:47 +03:00
# include "lvm-logging.h"
2004-06-24 12:02:38 +04:00
# include "clvmd.h"
# include "lvm-functions.h"
# define LOCKSPACE_NAME "clvmd"
2008-05-09 11:20:04 +04:00
struct clvmd_node
{
struct cman_node * node ;
int clvmd_up ;
} ;
2004-06-24 12:02:38 +04:00
static int num_nodes ;
2006-03-14 17:18:34 +03:00
static struct cman_node * nodes = NULL ;
static struct cman_node this_node ;
2004-06-24 12:02:38 +04:00
static int count_nodes ; /* size of allocated nodes array */
2008-05-09 11:20:04 +04:00
static struct dm_hash_table * node_updown_hash ;
2004-06-24 12:02:38 +04:00
static dlm_lshandle_t * lockspace ;
2006-03-14 17:18:34 +03:00
static cman_handle_t c_handle ;
2004-06-24 12:02:38 +04:00
static void count_clvmds_running ( void ) ;
static void get_members ( void ) ;
2007-05-02 16:22:40 +04:00
static int nodeid_from_csid ( const char * csid ) ;
2004-06-24 12:02:38 +04:00
static int name_from_nodeid ( int nodeid , char * name ) ;
2006-03-14 17:18:34 +03:00
static void event_callback ( cman_handle_t handle , void * private , int reason , int arg ) ;
static void data_callback ( cman_handle_t handle , void * private ,
char * buf , int len , uint8_t port , int nodeid ) ;
2004-06-24 12:02:38 +04:00
struct lock_wait {
pthread_cond_t cond ;
pthread_mutex_t mutex ;
struct dlm_lksb lksb ;
} ;
2005-01-13 16:24:02 +03:00
static int _init_cluster ( void )
2004-06-24 12:02:38 +04:00
{
2008-05-09 11:20:04 +04:00
node_updown_hash = dm_hash_create ( 100 ) ;
2004-06-24 12:02:38 +04:00
/* Open the cluster communication socket */
2006-03-14 17:18:34 +03:00
c_handle = cman_init ( NULL ) ;
if ( ! c_handle ) {
syslog ( LOG_ERR , " Can't open cluster manager socket: %m " ) ;
2004-06-24 12:02:38 +04:00
return - 1 ;
}
2007-05-02 12:23:36 +04:00
DEBUGLOG ( " Connected to CMAN \n " ) ;
2004-06-24 12:02:38 +04:00
2006-03-14 17:18:34 +03:00
if ( cman_start_recv_data ( c_handle , data_callback , CLUSTER_PORT_CLVMD ) ) {
2004-11-11 17:51:23 +03:00
syslog ( LOG_ERR , " Can't bind cluster socket: %m " ) ;
2004-06-24 12:02:38 +04:00
return - 1 ;
}
2006-03-14 17:18:34 +03:00
if ( cman_start_notification ( c_handle , event_callback ) ) {
syslog ( LOG_ERR , " Can't start cluster event listening " ) ;
return - 1 ;
}
2004-06-24 12:02:38 +04:00
/* Get the cluster members list */
get_members ( ) ;
count_clvmds_running ( ) ;
2007-05-02 12:23:36 +04:00
DEBUGLOG ( " CMAN initialisation complete \n " ) ;
2004-06-24 12:02:38 +04:00
/* Create a lockspace for LV & VG locks to live in */
lockspace = dlm_create_lockspace ( LOCKSPACE_NAME , 0600 ) ;
if ( ! lockspace ) {
2004-11-11 17:51:23 +03:00
syslog ( LOG_ERR , " Unable to create lockspace for CLVM: %m " ) ;
2004-06-24 12:02:38 +04:00
return - 1 ;
}
dlm_ls_pthread_init ( lockspace ) ;
2007-05-02 12:23:36 +04:00
DEBUGLOG ( " DLM initialisation complete \n " ) ;
2004-06-24 12:02:38 +04:00
return 0 ;
}
2005-01-21 14:35:24 +03:00
static void _cluster_init_completed ( void )
{
clvmd_cluster_init_completed ( ) ;
}
2005-01-13 16:24:02 +03:00
static int _get_main_cluster_fd ( )
2004-06-24 12:02:38 +04:00
{
2006-03-14 17:18:34 +03:00
return cman_get_fd ( c_handle ) ;
2004-06-24 12:02:38 +04:00
}
2005-01-13 16:24:02 +03:00
static int _get_num_nodes ( )
2004-06-24 12:02:38 +04:00
{
2006-03-14 17:18:34 +03:00
int i ;
int nnodes = 0 ;
/* return number of ACTIVE nodes */
for ( i = 0 ; i < num_nodes ; i + + ) {
2007-04-23 18:55:28 +04:00
if ( nodes [ i ] . cn_member & & nodes [ i ] . cn_nodeid )
2006-03-14 17:18:34 +03:00
nnodes + + ;
}
return nnodes ;
2004-06-24 12:02:38 +04:00
}
/* send_message with the fd check removed */
2007-05-02 16:22:40 +04:00
static int _cluster_send_message ( const void * buf , int msglen , const char * csid ,
const char * errtext )
2004-06-24 12:02:38 +04:00
{
2006-03-14 17:18:34 +03:00
int nodeid = 0 ;
2004-06-24 12:02:38 +04:00
2006-03-14 17:18:34 +03:00
if ( csid )
memcpy ( & nodeid , csid , CMAN_MAX_CSID_LEN ) ;
2004-06-24 12:02:38 +04:00
2006-03-14 17:18:34 +03:00
if ( cman_send_data ( c_handle , buf , msglen , 0 , CLUSTER_PORT_CLVMD , nodeid ) < = 0 )
{
2007-05-02 16:22:40 +04:00
log_error ( " %s " , errtext ) ;
2006-03-14 17:18:34 +03:00
}
return msglen ;
2004-06-24 12:02:38 +04:00
}
2005-01-13 16:24:02 +03:00
static void _get_our_csid ( char * csid )
2004-06-24 12:02:38 +04:00
{
2006-03-14 17:18:34 +03:00
if ( this_node . cn_nodeid = = 0 ) {
cman_get_node ( c_handle , 0 , & this_node ) ;
2004-06-24 12:02:38 +04:00
}
2006-03-14 17:18:34 +03:00
memcpy ( csid , & this_node . cn_nodeid , CMAN_MAX_CSID_LEN ) ;
2004-06-24 12:02:38 +04:00
}
2006-03-14 17:18:34 +03:00
/* Call a callback routine for each node is that known (down means not running a clvmd) */
2005-01-13 16:24:02 +03:00
static int _cluster_do_node_callback ( struct local_client * client ,
2007-05-02 16:22:40 +04:00
void ( * callback ) ( struct local_client * ,
const char * ,
2006-08-24 16:45:05 +04:00
int ) )
2004-06-24 12:02:38 +04:00
{
int i ;
int somedown = 0 ;
2005-01-13 16:24:02 +03:00
for ( i = 0 ; i < _get_num_nodes ( ) ; i + + ) {
2007-04-23 18:55:28 +04:00
if ( nodes [ i ] . cn_member & & nodes [ i ] . cn_nodeid ) {
2008-05-09 11:20:04 +04:00
int up = ( int ) ( long ) dm_hash_lookup_binary ( node_updown_hash , ( char * ) & nodes [ i ] . cn_nodeid , sizeof ( int ) ) ;
callback ( client , ( char * ) & nodes [ i ] . cn_nodeid , up ) ;
if ( ! up )
2006-08-24 16:45:05 +04:00
somedown = - 1 ;
}
2004-06-24 12:02:38 +04:00
}
return somedown ;
}
2007-04-23 18:55:28 +04:00
/* Process OOB messages from the cluster socket */
2006-03-14 17:18:34 +03:00
static void event_callback ( cman_handle_t handle , void * private , int reason , int arg )
2004-06-24 12:02:38 +04:00
{
2006-10-06 14:06:10 +04:00
char namebuf [ MAX_CLUSTER_MEMBER_NAME_LEN ] ;
2006-03-14 17:18:34 +03:00
switch ( reason ) {
case CMAN_REASON_PORTCLOSED :
name_from_nodeid ( arg , namebuf ) ;
2004-06-24 12:02:38 +04:00
log_notice ( " clvmd on node %s has died \n " , namebuf ) ;
2006-03-21 13:31:08 +03:00
DEBUGLOG ( " Got port closed message, removing node %s \n " , namebuf ) ;
2004-06-24 12:02:38 +04:00
2008-05-09 11:20:04 +04:00
dm_hash_insert_binary ( node_updown_hash , ( char * ) & arg , sizeof ( int ) , ( void * ) 0 ) ;
2004-06-24 12:02:38 +04:00
break ;
2006-03-14 17:18:34 +03:00
case CMAN_REASON_STATECHANGE :
2006-03-21 13:31:08 +03:00
DEBUGLOG ( " Got state change message, re-reading members list \n " ) ;
2004-06-24 12:02:38 +04:00
get_members ( ) ;
break ;
2006-03-21 13:31:08 +03:00
# if defined(LIBCMAN_VERSION) && LIBCMAN_VERSION >= 2
case CMAN_REASON_PORTOPENED :
/* Ignore this, wait for startup message from clvmd itself */
break ;
case CMAN_REASON_TRY_SHUTDOWN :
DEBUGLOG ( " Got try shutdown, sending OK \n " ) ;
cman_replyto_shutdown ( c_handle , 1 ) ;
break ;
# endif
2004-06-24 12:02:38 +04:00
default :
/* ERROR */
2006-03-14 17:18:34 +03:00
DEBUGLOG ( " Got unknown event callback message: %d \n " , reason ) ;
break ;
2004-06-24 12:02:38 +04:00
}
}
2006-03-14 17:18:34 +03:00
static struct local_client * cman_client ;
2007-05-02 16:22:40 +04:00
static int _cluster_fd_callback ( struct local_client * fd , char * buf , int len ,
const char * csid ,
2006-08-24 16:45:05 +04:00
struct local_client * * new_client )
2004-06-24 12:02:38 +04:00
{
2006-03-14 17:18:34 +03:00
/* Save this for data_callback */
cman_client = fd ;
2004-06-24 12:02:38 +04:00
/* We never return a new client */
* new_client = NULL ;
2006-03-14 17:18:34 +03:00
return cman_dispatch ( c_handle , 0 ) ;
2006-03-21 13:31:08 +03:00
}
2004-06-24 12:02:38 +04:00
2006-03-14 17:18:34 +03:00
static void data_callback ( cman_handle_t handle , void * private ,
char * buf , int len , uint8_t port , int nodeid )
{
/* Ignore looped back messages */
if ( nodeid = = this_node . cn_nodeid )
return ;
process_message ( cman_client , buf , len , ( char * ) & nodeid ) ;
2004-06-24 12:02:38 +04:00
}
2007-05-02 16:22:40 +04:00
static void _add_up_node ( const char * csid )
2004-06-24 12:02:38 +04:00
{
/* It's up ! */
int nodeid = nodeid_from_csid ( csid ) ;
2008-05-09 11:20:04 +04:00
dm_hash_insert_binary ( node_updown_hash , ( char * ) & nodeid , sizeof ( int ) , ( void * ) 1 ) ;
2004-06-24 12:02:38 +04:00
DEBUGLOG ( " Added new node %d to updown list \n " , nodeid ) ;
}
2005-01-13 16:24:02 +03:00
static void _cluster_closedown ( )
2004-06-24 12:02:38 +04:00
{
unlock_all ( ) ;
dlm_release_lockspace ( LOCKSPACE_NAME , lockspace , 1 ) ;
2006-03-14 17:18:34 +03:00
cman_finish ( c_handle ) ;
2004-06-24 12:02:38 +04:00
}
static int is_listening ( int nodeid )
{
int status ;
do {
2006-03-14 17:18:34 +03:00
status = cman_is_listening ( c_handle , nodeid , CLUSTER_PORT_CLVMD ) ;
2004-06-24 12:02:38 +04:00
if ( status < 0 & & errno = = EBUSY ) { /* Don't busywait */
sleep ( 1 ) ;
errno = EBUSY ; /* In case sleep trashes it */
}
}
while ( status < 0 & & errno = = EBUSY ) ;
return status ;
}
/* Populate the list of CLVMDs running.
called only at startup time */
2005-01-13 16:24:02 +03:00
static void count_clvmds_running ( void )
2004-06-24 12:02:38 +04:00
{
int i ;
for ( i = 0 ; i < num_nodes ; i + + ) {
2008-05-09 11:20:04 +04:00
int nodeid = nodes [ i ] . cn_nodeid ;
if ( is_listening ( nodeid ) = = 1 )
dm_hash_insert_binary ( node_updown_hash , ( void * ) & nodeid , sizeof ( int ) , ( void * ) 1 ) ;
else
dm_hash_insert_binary ( node_updown_hash , ( void * ) & nodeid , sizeof ( int ) , ( void * ) 0 ) ;
2004-06-24 12:02:38 +04:00
}
}
/* Get a list of active cluster members */
static void get_members ( )
{
2006-03-14 17:18:34 +03:00
int retnodes ;
int status ;
2008-04-01 19:01:30 +04:00
int i ;
int high_nodeid = 0 ;
2004-06-24 12:02:38 +04:00
2006-03-14 17:18:34 +03:00
num_nodes = cman_get_node_count ( c_handle ) ;
2004-06-24 12:02:38 +04:00
if ( num_nodes = = - 1 ) {
2004-10-06 14:02:25 +04:00
log_error ( " Unable to get node count " ) ;
2006-03-14 17:18:34 +03:00
return ;
}
2006-08-24 16:45:05 +04:00
/* Not enough room for new nodes list ? */
if ( num_nodes > count_nodes & & nodes ) {
free ( nodes ) ;
nodes = NULL ;
}
2004-06-24 12:02:38 +04:00
2006-08-24 16:45:05 +04:00
if ( nodes = = NULL ) {
count_nodes = num_nodes + 10 ; /* Overallocate a little */
2006-03-14 17:18:34 +03:00
nodes = malloc ( count_nodes * sizeof ( struct cman_node ) ) ;
2006-08-24 16:45:05 +04:00
if ( ! nodes ) {
log_error ( " Unable to allocate nodes array \n " ) ;
exit ( 5 ) ;
2004-06-24 12:02:38 +04:00
}
2006-08-24 16:45:05 +04:00
}
2004-09-24 13:39:57 +04:00
2006-03-14 17:18:34 +03:00
status = cman_get_nodes ( c_handle , count_nodes , & retnodes , nodes ) ;
if ( status < 0 ) {
2006-08-24 16:45:05 +04:00
log_error ( " Unable to get node details " ) ;
exit ( 6 ) ;
}
2004-06-24 12:02:38 +04:00
2008-04-01 19:01:30 +04:00
/* Get the highest nodeid */
for ( i = 0 ; i < retnodes ; i + + ) {
if ( nodes [ i ] . cn_nodeid > high_nodeid )
high_nodeid = nodes [ i ] . cn_nodeid ;
}
2006-08-24 16:45:05 +04:00
}
2004-06-24 12:02:38 +04:00
/* Convert a node name to a CSID */
2007-05-02 16:22:40 +04:00
static int _csid_from_name ( char * csid , const char * name )
2004-06-24 12:02:38 +04:00
{
int i ;
for ( i = 0 ; i < num_nodes ; i + + ) {
2006-03-14 17:18:34 +03:00
if ( strcmp ( name , nodes [ i ] . cn_name ) = = 0 ) {
memcpy ( csid , & nodes [ i ] . cn_nodeid , CMAN_MAX_CSID_LEN ) ;
2004-06-24 12:02:38 +04:00
return 0 ;
}
}
return - 1 ;
}
/* Convert a CSID to a node name */
2007-05-02 16:22:40 +04:00
static int _name_from_csid ( const char * csid , char * name )
2004-06-24 12:02:38 +04:00
{
int i ;
for ( i = 0 ; i < num_nodes ; i + + ) {
2006-03-14 17:18:34 +03:00
if ( memcmp ( csid , & nodes [ i ] . cn_nodeid , CMAN_MAX_CSID_LEN ) = = 0 ) {
strcpy ( name , nodes [ i ] . cn_name ) ;
2004-06-24 12:02:38 +04:00
return 0 ;
}
}
/* Who?? */
strcpy ( name , " Unknown " ) ;
return - 1 ;
}
/* Convert a node ID to a node name */
2005-01-13 16:24:02 +03:00
static int name_from_nodeid ( int nodeid , char * name )
2004-06-24 12:02:38 +04:00
{
int i ;
for ( i = 0 ; i < num_nodes ; i + + ) {
2006-03-14 17:18:34 +03:00
if ( nodeid = = nodes [ i ] . cn_nodeid ) {
strcpy ( name , nodes [ i ] . cn_name ) ;
2004-06-24 12:02:38 +04:00
return 0 ;
}
}
/* Who?? */
strcpy ( name , " Unknown " ) ;
return - 1 ;
}
/* Convert a CSID to a node ID */
2007-05-02 16:22:40 +04:00
static int nodeid_from_csid ( const char * csid )
2004-06-24 12:02:38 +04:00
{
int nodeid ;
2005-01-13 16:24:02 +03:00
memcpy ( & nodeid , csid , CMAN_MAX_CSID_LEN ) ;
2004-06-24 12:02:38 +04:00
return nodeid ;
}
2005-01-13 16:24:02 +03:00
static int _is_quorate ( )
2004-06-24 12:02:38 +04:00
{
2006-03-14 17:18:34 +03:00
return cman_is_quorate ( c_handle ) ;
2004-06-24 12:02:38 +04:00
}
static void sync_ast_routine ( void * arg )
{
struct lock_wait * lwait = arg ;
pthread_mutex_lock ( & lwait - > mutex ) ;
pthread_cond_signal ( & lwait - > cond ) ;
pthread_mutex_unlock ( & lwait - > mutex ) ;
}
2005-01-13 16:24:02 +03:00
static int _sync_lock ( const char * resource , int mode , int flags , int * lockid )
2004-06-24 12:02:38 +04:00
{
int status ;
struct lock_wait lwait ;
if ( ! lockid ) {
errno = EINVAL ;
return - 1 ;
}
2004-09-24 13:39:57 +04:00
DEBUGLOG ( " sync_lock: '%s' mode:%d flags=%d \n " , resource , mode , flags ) ;
2004-06-24 12:02:38 +04:00
/* Conversions need the lockid in the LKSB */
if ( flags & LKF_CONVERT )
lwait . lksb . sb_lkid = * lockid ;
pthread_cond_init ( & lwait . cond , NULL ) ;
pthread_mutex_init ( & lwait . mutex , NULL ) ;
pthread_mutex_lock ( & lwait . mutex ) ;
status = dlm_ls_lock ( lockspace ,
mode ,
& lwait . lksb ,
flags ,
resource ,
strlen ( resource ) ,
0 , sync_ast_routine , & lwait , NULL , NULL ) ;
if ( status )
return status ;
/* Wait for it to complete */
pthread_cond_wait ( & lwait . cond , & lwait . mutex ) ;
pthread_mutex_unlock ( & lwait . mutex ) ;
* lockid = lwait . lksb . sb_lkid ;
errno = lwait . lksb . sb_status ;
2004-09-24 13:39:57 +04:00
DEBUGLOG ( " sync_lock: returning lkid %x \n " , * lockid ) ;
2004-06-24 12:02:38 +04:00
if ( lwait . lksb . sb_status )
return - 1 ;
else
return 0 ;
}
2005-01-13 16:24:02 +03:00
static int _sync_unlock ( const char * resource /* UNUSED */ , int lockid )
2004-06-24 12:02:38 +04:00
{
int status ;
struct lock_wait lwait ;
2004-09-24 13:39:57 +04:00
DEBUGLOG ( " sync_unlock: '%s' lkid:%x \n " , resource , lockid ) ;
2004-06-24 12:02:38 +04:00
pthread_cond_init ( & lwait . cond , NULL ) ;
pthread_mutex_init ( & lwait . mutex , NULL ) ;
pthread_mutex_lock ( & lwait . mutex ) ;
status = dlm_ls_unlock ( lockspace , lockid , 0 , & lwait . lksb , & lwait ) ;
if ( status )
return status ;
/* Wait for it to complete */
pthread_cond_wait ( & lwait . cond , & lwait . mutex ) ;
pthread_mutex_unlock ( & lwait . mutex ) ;
errno = lwait . lksb . sb_status ;
if ( lwait . lksb . sb_status ! = EUNLOCK )
return - 1 ;
else
return 0 ;
}
2005-01-13 16:24:02 +03:00
2006-10-09 18:11:57 +04:00
static int _get_cluster_name ( char * buf , int buflen )
{
cman_cluster_t cluster_info ;
int status ;
status = cman_get_cluster ( c_handle , & cluster_info ) ;
if ( ! status ) {
strncpy ( buf , cluster_info . ci_name , buflen ) ;
}
return status ;
}
2005-01-13 16:24:02 +03:00
static struct cluster_ops _cluster_cman_ops = {
2005-01-21 14:35:24 +03:00
. cluster_init_completed = _cluster_init_completed ,
2005-01-13 16:24:02 +03:00
. cluster_send_message = _cluster_send_message ,
. name_from_csid = _name_from_csid ,
. csid_from_name = _csid_from_name ,
. get_num_nodes = _get_num_nodes ,
. cluster_fd_callback = _cluster_fd_callback ,
. get_main_cluster_fd = _get_main_cluster_fd ,
. cluster_do_node_callback = _cluster_do_node_callback ,
. is_quorate = _is_quorate ,
. get_our_csid = _get_our_csid ,
. add_up_node = _add_up_node ,
. cluster_closedown = _cluster_closedown ,
2006-10-09 18:11:57 +04:00
. get_cluster_name = _get_cluster_name ,
2005-01-13 16:24:02 +03:00
. sync_lock = _sync_lock ,
. sync_unlock = _sync_unlock ,
} ;
struct cluster_ops * init_cman_cluster ( void )
{
if ( ! _init_cluster ( ) )
return & _cluster_cman_ops ;
else
return NULL ;
}