2007-05-18 14:06:29 +04:00
/*
monitoring links to all other nodes to detect dead nodes
Copyright ( C ) Ronnie Sahlberg 2007
2007-05-31 07:50:53 +04:00
This program is free software ; you can redistribute it and / or modify
it under the terms of the GNU General Public License as published by
2007-07-10 09:29:31 +04:00
the Free Software Foundation ; either version 3 of the License , or
2007-05-31 07:50:53 +04:00
( at your option ) any later version .
This program is distributed in the hope that it will be useful ,
2007-05-18 14:06:29 +04:00
but WITHOUT ANY WARRANTY ; without even the implied warranty of
2007-05-31 07:50:53 +04:00
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
GNU General Public License for more details .
You should have received a copy of the GNU General Public License
2007-07-10 09:29:31 +04:00
along with this program ; if not , see < http : //www.gnu.org/licenses/>.
2007-05-18 14:06:29 +04:00
*/
# include "includes.h"
2010-08-18 03:46:31 +04:00
# include "lib/tevent/tevent.h"
2007-05-18 14:06:29 +04:00
# include "system/filesys.h"
# include "system/wait.h"
# include "../include/ctdb_private.h"
2008-01-10 06:40:56 +03:00
struct ctdb_monitor_state {
uint32_t monitoring_mode ;
TALLOC_CTX * monitor_context ;
uint32_t next_interval ;
} ;
2007-06-06 04:25:46 +04:00
static void ctdb_check_health ( struct event_context * ev , struct timed_event * te ,
struct timeval t , void * private_data ) ;
2009-03-31 07:23:31 +04:00
/*
setup the notification script
*/
int ctdb_set_notification_script ( struct ctdb_context * ctdb , const char * script )
{
ctdb - > notification_script = talloc_strdup ( ctdb , script ) ;
CTDB_NO_MEMORY ( ctdb , ctdb - > notification_script ) ;
return 0 ;
}
static int ctdb_run_notification_script_child ( struct ctdb_context * ctdb , const char * event )
{
struct stat st ;
int ret ;
char * cmd ;
if ( stat ( ctdb - > notification_script , & st ) ! = 0 ) {
DEBUG ( DEBUG_ERR , ( " Could not stat notification script %s. Can not send notifications. \n " , ctdb - > notification_script ) ) ;
return - 1 ;
}
if ( ! ( st . st_mode & S_IXUSR ) ) {
DEBUG ( DEBUG_ERR , ( " Notification script %s is not executable. \n " , ctdb - > notification_script ) ) ;
return - 1 ;
}
cmd = talloc_asprintf ( ctdb , " %s %s \n " , ctdb - > notification_script , event ) ;
CTDB_NO_MEMORY ( ctdb , cmd ) ;
ret = system ( cmd ) ;
/* if the system() call was successful, translate ret into the
return code from the command
*/
if ( ret ! = - 1 ) {
ret = WEXITSTATUS ( ret ) ;
}
if ( ret ! = 0 ) {
DEBUG ( DEBUG_ERR , ( " Notification script \" %s \" failed with error %d \n " , cmd , ret ) ) ;
}
return ret ;
}
2010-01-19 12:07:14 +03:00
void ctdb_run_notification_script ( struct ctdb_context * ctdb , const char * event )
2009-03-31 07:23:31 +04:00
{
pid_t child ;
if ( ctdb - > notification_script = = NULL ) {
return ;
}
child = fork ( ) ;
if ( child = = ( pid_t ) - 1 ) {
DEBUG ( DEBUG_ERR , ( " Failed to fork() a notification child process \n " ) ) ;
return ;
}
if ( child = = 0 ) {
int ret ;
2010-07-19 13:59:09 +04:00
debug_extra = talloc_asprintf ( NULL , " notification-%s: " , event ) ;
2009-03-31 07:23:31 +04:00
ret = ctdb_run_notification_script_child ( ctdb , event ) ;
if ( ret ! = 0 ) {
DEBUG ( DEBUG_ERR , ( __location__ " Notification script failed \n " ) ) ;
}
_exit ( 0 ) ;
}
return ;
}
2007-06-06 04:25:46 +04:00
/*
called when a health monitoring event script finishes
*/
static void ctdb_health_callback ( struct ctdb_context * ctdb , int status , void * p )
{
2007-09-04 04:06:36 +04:00
struct ctdb_node * node = ctdb - > nodes [ ctdb - > pnn ] ;
2007-06-06 04:25:46 +04:00
TDB_DATA data ;
struct ctdb_node_flag_change c ;
2007-09-24 04:12:18 +04:00
uint32_t next_interval ;
2009-10-14 04:59:16 +04:00
int ret ;
TDB_DATA rddata ;
struct takeover_run_reply rd ;
2010-08-24 11:22:49 +04:00
const char * state_str = NULL ;
2007-06-06 04:25:46 +04:00
2007-09-04 04:33:10 +04:00
c . pnn = ctdb - > pnn ;
2007-08-21 11:25:15 +04:00
c . old_flags = node - > flags ;
2009-10-14 04:59:16 +04:00
rd . pnn = ctdb - > pnn ;
rd . srvid = CTDB_SRVID_TAKEOVER_RUN_RESPONSE ;
rddata . dptr = ( uint8_t * ) & rd ;
rddata . dsize = sizeof ( rd ) ;
2009-12-07 16:18:57 +03:00
if ( status = = - ETIME ) {
ctdb - > event_script_timeouts + + ;
2009-12-14 07:53:23 +03:00
if ( ctdb - > event_script_timeouts > = ctdb - > tunable . script_timeout_count ) {
DEBUG ( DEBUG_ERR , ( " Maximum timeout count %u reached for eventscript. Making node unhealthy \n " , ctdb - > tunable . script_timeout_count ) ) ;
2009-12-07 16:18:57 +03:00
} else {
/* We pretend this is OK. */
2009-12-07 16:22:01 +03:00
goto after_change_status ;
2009-12-07 16:18:57 +03:00
}
}
2007-06-07 05:15:22 +04:00
if ( status ! = 0 & & ! ( node - > flags & NODE_FLAGS_UNHEALTHY ) ) {
2008-02-04 12:07:15 +03:00
DEBUG ( DEBUG_NOTICE , ( " monitor event failed - disabling node \n " ) ) ;
2007-06-07 05:15:22 +04:00
node - > flags | = NODE_FLAGS_UNHEALTHY ;
2009-10-27 05:51:45 +03:00
ctdb - > monitor - > next_interval = 5 ;
2008-02-22 02:33:09 +03:00
2009-03-31 07:23:31 +04:00
ctdb_run_notification_script ( ctdb , " unhealthy " ) ;
2007-06-07 05:15:22 +04:00
} else if ( status = = 0 & & ( node - > flags & NODE_FLAGS_UNHEALTHY ) ) {
2008-02-04 09:44:24 +03:00
DEBUG ( DEBUG_NOTICE , ( " monitor event OK - node re-enabled \n " ) ) ;
2007-09-24 04:12:18 +04:00
node - > flags & = ~ NODE_FLAGS_UNHEALTHY ;
2009-10-27 05:51:45 +03:00
ctdb - > monitor - > next_interval = 5 ;
2009-03-31 07:23:31 +04:00
ctdb_run_notification_script ( ctdb , " healthy " ) ;
2007-09-24 04:12:18 +04:00
}
2009-12-07 16:22:01 +03:00
after_change_status :
2008-01-10 06:40:56 +03:00
next_interval = ctdb - > monitor - > next_interval ;
ctdb - > monitor - > next_interval * = 2 ;
if ( ctdb - > monitor - > next_interval > ctdb - > tunable . monitor_interval ) {
ctdb - > monitor - > next_interval = ctdb - > tunable . monitor_interval ;
2007-09-24 04:12:18 +04:00
}
2008-01-10 06:40:56 +03:00
event_add_timed ( ctdb - > ev , ctdb - > monitor - > monitor_context ,
2007-11-30 01:02:37 +03:00
timeval_current_ofs ( next_interval , 0 ) ,
ctdb_check_health , ctdb ) ;
2007-09-24 04:12:18 +04:00
if ( c . old_flags = = node - > flags ) {
2007-06-06 04:25:46 +04:00
return ;
}
2007-08-21 11:25:15 +04:00
c . new_flags = node - > flags ;
2007-06-06 04:25:46 +04:00
data . dptr = ( uint8_t * ) & c ;
data . dsize = sizeof ( c ) ;
2008-11-19 06:43:46 +03:00
/* ask the recovery daemon to push these changes out to all nodes */
ctdb_daemon_send_message ( ctdb , ctdb - > pnn ,
CTDB_SRVID_PUSH_NODE_FLAGS , data ) ;
2007-06-07 05:15:22 +04:00
2010-08-24 11:22:49 +04:00
if ( c . new_flags & NODE_FLAGS_UNHEALTHY ) {
state_str = " UNHEALTHY " ;
} else {
state_str = " HEALTHY " ;
}
/* ask the recmaster to reallocate all addresses */
DEBUG ( DEBUG_ERR , ( " Node became %s. Ask recovery master %u to perform ip reallocation \n " ,
state_str , ctdb - > recovery_master ) ) ;
ret = ctdb_daemon_send_message ( ctdb , ctdb - > recovery_master , CTDB_SRVID_TAKEOVER_RUN , rddata ) ;
if ( ret ! = 0 ) {
DEBUG ( DEBUG_ERR , ( __location__ " Failed to send ip takeover run request message to %u \n " , ctdb - > recovery_master ) ) ;
}
2007-06-06 04:25:46 +04:00
}
2007-11-12 02:53:11 +03:00
/*
called when the startup event script finishes
*/
static void ctdb_startup_callback ( struct ctdb_context * ctdb , int status , void * p )
{
if ( status ! = 0 ) {
2008-02-04 12:07:15 +03:00
DEBUG ( DEBUG_ERR , ( " startup event failed \n " ) ) ;
2007-11-12 02:53:11 +03:00
} else if ( status = = 0 ) {
2008-02-04 12:07:15 +03:00
DEBUG ( DEBUG_NOTICE , ( " startup event OK - enabling monitoring \n " ) ) ;
2007-11-12 02:53:11 +03:00
ctdb - > done_startup = true ;
2010-08-20 08:54:03 +04:00
ctdb - > monitor - > next_interval = 2 ;
2009-10-01 08:05:30 +04:00
ctdb_run_notification_script ( ctdb , " startup " ) ;
2007-11-12 02:53:11 +03:00
}
2008-01-10 06:40:56 +03:00
event_add_timed ( ctdb - > ev , ctdb - > monitor - > monitor_context ,
timeval_current_ofs ( ctdb - > monitor - > next_interval , 0 ) ,
ctdb_check_health , ctdb ) ;
2007-11-12 02:53:11 +03:00
}
2009-12-01 05:19:58 +03:00
/*
wait until we have finished initial recoveries before we start the
monitoring events
*/
static void ctdb_wait_until_recovered ( struct event_context * ev , struct timed_event * te ,
struct timeval t , void * private_data )
{
struct ctdb_context * ctdb = talloc_get_type ( private_data , struct ctdb_context ) ;
2009-12-07 15:28:11 +03:00
int ret ;
2009-12-01 05:19:58 +03:00
DEBUG ( DEBUG_NOTICE , ( " CTDB_WAIT_UNTIL_RECOVERED \n " ) ) ;
if ( ctdb - > vnn_map - > generation = = INVALID_GENERATION ) {
2009-12-07 15:28:11 +03:00
ctdb - > db_persistent_startup_generation = INVALID_GENERATION ;
2009-12-01 05:19:58 +03:00
DEBUG ( DEBUG_NOTICE , ( __location__ " generation is INVALID. Wait one more second \n " ) ) ;
event_add_timed ( ctdb - > ev , ctdb - > monitor - > monitor_context ,
timeval_current_ofs ( 1 , 0 ) ,
ctdb_wait_until_recovered , ctdb ) ;
return ;
}
if ( ctdb - > recovery_mode ! = CTDB_RECOVERY_NORMAL ) {
2009-12-07 15:28:11 +03:00
ctdb - > db_persistent_startup_generation = INVALID_GENERATION ;
2009-12-01 05:19:58 +03:00
DEBUG ( DEBUG_NOTICE , ( __location__ " in recovery. Wait one more second \n " ) ) ;
event_add_timed ( ctdb - > ev , ctdb - > monitor - > monitor_context ,
timeval_current_ofs ( 1 , 0 ) ,
ctdb_wait_until_recovered , ctdb ) ;
return ;
}
2010-06-22 17:22:34 +04:00
if ( ! fast_start & & timeval_elapsed ( & ctdb - > last_recovery_finished ) < ( ctdb - > tunable . rerecovery_timeout + 3 ) ) {
2009-12-07 15:28:11 +03:00
ctdb - > db_persistent_startup_generation = INVALID_GENERATION ;
2009-12-01 05:19:58 +03:00
DEBUG ( DEBUG_NOTICE , ( __location__ " wait for pending recoveries to end. Wait one more second. \n " ) ) ;
event_add_timed ( ctdb - > ev , ctdb - > monitor - > monitor_context ,
timeval_current_ofs ( 1 , 0 ) ,
ctdb_wait_until_recovered , ctdb ) ;
return ;
}
2009-12-07 15:28:11 +03:00
if ( ctdb - > vnn_map - > generation = = ctdb - > db_persistent_startup_generation ) {
DEBUG ( DEBUG_INFO , ( __location__ " skip ctdb_recheck_persistent_health() "
" until the next recovery \n " ) ) ;
event_add_timed ( ctdb - > ev , ctdb - > monitor - > monitor_context ,
timeval_current_ofs ( 1 , 0 ) ,
ctdb_wait_until_recovered , ctdb ) ;
return ;
}
ctdb - > db_persistent_startup_generation = ctdb - > vnn_map - > generation ;
ret = ctdb_recheck_persistent_health ( ctdb ) ;
if ( ret ! = 0 ) {
ctdb - > db_persistent_check_errors + + ;
if ( ctdb - > db_persistent_check_errors < ctdb - > max_persistent_check_errors ) {
DEBUG ( ctdb - > db_persistent_check_errors = = 1 ? DEBUG_ERR : DEBUG_WARNING ,
( __location__ " ctdb_recheck_persistent_health() "
" failed (%llu of %llu times) - retry later \n " ,
( unsigned long long ) ctdb - > db_persistent_check_errors ,
( unsigned long long ) ctdb - > max_persistent_check_errors ) ) ;
event_add_timed ( ctdb - > ev ,
ctdb - > monitor - > monitor_context ,
timeval_current_ofs ( 1 , 0 ) ,
ctdb_wait_until_recovered , ctdb ) ;
return ;
}
DEBUG ( DEBUG_ALERT , ( __location__
" ctdb_recheck_persistent_health() failed (%llu times) - prepare shutdown \n " ,
( unsigned long long ) ctdb - > db_persistent_check_errors ) ) ;
ctdb_stop_recoverd ( ctdb ) ;
ctdb_stop_keepalive ( ctdb ) ;
ctdb_stop_monitoring ( ctdb ) ;
ctdb_release_all_ips ( ctdb ) ;
if ( ctdb - > methods ! = NULL ) {
ctdb - > methods - > shutdown ( ctdb ) ;
}
ctdb_event_script ( ctdb , CTDB_EVENT_SHUTDOWN ) ;
DEBUG ( DEBUG_ALERT , ( " ctdb_recheck_persistent_health() failed - Stopping CTDB daemon \n " ) ) ;
exit ( 11 ) ;
}
ctdb - > db_persistent_check_errors = 0 ;
DEBUG ( DEBUG_NOTICE , ( __location__
" ctdb_start_monitoring: ctdb_recheck_persistent_health() OK \n " ) ) ;
2009-12-01 05:19:58 +03:00
DEBUG ( DEBUG_NOTICE , ( __location__ " Recoveries finished. Running the \" startup \" event. \n " ) ) ;
event_add_timed ( ctdb - > ev , ctdb - > monitor - > monitor_context ,
2010-06-22 17:20:45 +04:00
timeval_current ( ) ,
2009-12-01 05:19:58 +03:00
ctdb_check_health , ctdb ) ;
}
2007-06-06 04:25:46 +04:00
/*
see if the event scripts think we are healthy
*/
static void ctdb_check_health ( struct event_context * ev , struct timed_event * te ,
struct timeval t , void * private_data )
{
struct ctdb_context * ctdb = talloc_get_type ( private_data , struct ctdb_context ) ;
2009-10-28 08:11:54 +03:00
int ret = 0 ;
2007-06-06 04:25:46 +04:00
2007-11-12 05:10:15 +03:00
if ( ctdb - > recovery_mode ! = CTDB_RECOVERY_NORMAL | |
2008-01-10 06:40:56 +03:00
( ctdb - > monitor - > monitoring_mode = = CTDB_MONITORING_DISABLED & & ctdb - > done_startup ) ) {
event_add_timed ( ctdb - > ev , ctdb - > monitor - > monitor_context ,
timeval_current_ofs ( ctdb - > monitor - > next_interval , 0 ) ,
2007-06-06 04:25:46 +04:00
ctdb_check_health , ctdb ) ;
return ;
}
2007-11-12 02:53:11 +03:00
if ( ! ctdb - > done_startup ) {
ret = ctdb_event_script_callback ( ctdb ,
2008-01-10 06:40:56 +03:00
ctdb - > monitor - > monitor_context , ctdb_startup_callback ,
2009-11-26 07:49:49 +03:00
ctdb , false ,
CTDB_EVENT_STARTUP , " %s " , " " ) ;
2007-11-12 02:53:11 +03:00
} else {
2009-10-15 09:03:43 +04:00
int i ;
int skip_monitoring = 0 ;
if ( ctdb - > recovery_mode ! = CTDB_RECOVERY_NORMAL ) {
skip_monitoring = 1 ;
DEBUG ( DEBUG_ERR , ( " Skip monitoring during recovery \n " ) ) ;
}
for ( i = 1 ; i < = NUM_DB_PRIORITIES ; i + + ) {
2009-10-22 06:41:28 +04:00
if ( ctdb - > freeze_handles [ i ] ! = NULL ) {
2009-10-15 09:03:43 +04:00
DEBUG ( DEBUG_ERR , ( " Skip monitoring since databases are frozen \n " ) ) ;
skip_monitoring = 1 ;
break ;
}
}
2009-10-28 08:11:54 +03:00
if ( skip_monitoring ! = 0 ) {
2009-10-15 09:03:43 +04:00
event_add_timed ( ctdb - > ev , ctdb - > monitor - > monitor_context ,
timeval_current_ofs ( ctdb - > monitor - > next_interval , 0 ) ,
ctdb_check_health , ctdb ) ;
return ;
} else {
ret = ctdb_event_script_callback ( ctdb ,
2009-10-28 08:11:54 +03:00
ctdb - > monitor - > monitor_context , ctdb_health_callback ,
2009-11-26 07:49:49 +03:00
ctdb , false ,
CTDB_EVENT_MONITOR , " %s " , " " ) ;
2009-10-15 09:03:43 +04:00
}
2007-11-12 02:53:11 +03:00
}
2007-06-06 04:25:46 +04:00
if ( ret ! = 0 ) {
2008-02-04 12:07:15 +03:00
DEBUG ( DEBUG_ERR , ( " Unable to launch monitor event script \n " ) ) ;
2009-10-28 08:11:54 +03:00
ctdb - > monitor - > next_interval = 5 ;
2008-01-10 06:40:56 +03:00
event_add_timed ( ctdb - > ev , ctdb - > monitor - > monitor_context ,
2009-10-28 08:11:54 +03:00
timeval_current_ofs ( 5 , 0 ) ,
ctdb_check_health , ctdb ) ;
}
2007-06-06 04:25:46 +04:00
}
2007-11-30 02:09:54 +03:00
/*
( Temporaily ) Disabling monitoring will stop the monitor event scripts
from running but node health checks will still occur
*/
void ctdb_disable_monitoring ( struct ctdb_context * ctdb )
{
2008-01-10 06:40:56 +03:00
ctdb - > monitor - > monitoring_mode = CTDB_MONITORING_DISABLED ;
2008-02-04 09:44:24 +03:00
DEBUG ( DEBUG_INFO , ( " Monitoring has been disabled \n " ) ) ;
2007-11-30 02:09:54 +03:00
}
/*
Re - enable running monitor events after they have been disabled
*/
void ctdb_enable_monitoring ( struct ctdb_context * ctdb )
{
2008-01-10 06:40:56 +03:00
ctdb - > monitor - > monitoring_mode = CTDB_MONITORING_ACTIVE ;
2009-10-28 08:11:54 +03:00
ctdb - > monitor - > next_interval = 5 ;
2008-02-04 09:44:24 +03:00
DEBUG ( DEBUG_INFO , ( " Monitoring has been enabled \n " ) ) ;
2007-11-30 02:09:54 +03:00
}
/* stop any monitoring
this should only be done when shutting down the daemon
*/
2007-06-06 07:45:12 +04:00
void ctdb_stop_monitoring ( struct ctdb_context * ctdb )
{
2008-01-10 06:40:56 +03:00
talloc_free ( ctdb - > monitor - > monitor_context ) ;
ctdb - > monitor - > monitor_context = NULL ;
2007-11-30 00:44:34 +03:00
2008-01-10 06:40:56 +03:00
ctdb - > monitor - > monitoring_mode = CTDB_MONITORING_DISABLED ;
2009-10-28 08:11:54 +03:00
ctdb - > monitor - > next_interval = 5 ;
2008-02-04 12:07:15 +03:00
DEBUG ( DEBUG_NOTICE , ( " Monitoring has been stopped \n " ) ) ;
2007-06-06 07:45:12 +04:00
}
2007-06-06 04:25:46 +04:00
2007-05-18 17:23:36 +04:00
/*
start watching for nodes that might be dead
*/
2007-06-06 07:45:12 +04:00
void ctdb_start_monitoring ( struct ctdb_context * ctdb )
2007-05-18 14:06:29 +04:00
{
2008-01-10 06:40:56 +03:00
if ( ctdb - > monitor ! = NULL ) {
2007-11-30 00:44:34 +03:00
return ;
}
2008-01-10 06:40:56 +03:00
ctdb - > monitor = talloc ( ctdb , struct ctdb_monitor_state ) ;
CTDB_NO_MEMORY_FATAL ( ctdb , ctdb - > monitor ) ;
2009-10-28 08:11:54 +03:00
ctdb - > monitor - > next_interval = 5 ;
2007-06-06 07:45:12 +04:00
2008-01-10 06:40:56 +03:00
ctdb - > monitor - > monitor_context = talloc_new ( ctdb - > monitor ) ;
CTDB_NO_MEMORY_FATAL ( ctdb , ctdb - > monitor - > monitor_context ) ;
2007-11-30 00:44:34 +03:00
2009-12-01 05:19:58 +03:00
event_add_timed ( ctdb - > ev , ctdb - > monitor - > monitor_context ,
2008-01-10 06:49:43 +03:00
timeval_current_ofs ( 1 , 0 ) ,
2009-12-01 05:19:58 +03:00
ctdb_wait_until_recovered , ctdb ) ;
2007-11-30 00:44:34 +03:00
2008-01-10 06:40:56 +03:00
ctdb - > monitor - > monitoring_mode = CTDB_MONITORING_ACTIVE ;
2008-02-04 12:07:15 +03:00
DEBUG ( DEBUG_NOTICE , ( " Monitoring has been started \n " ) ) ;
2007-05-18 14:06:29 +04:00
}
2007-06-07 05:15:22 +04:00
/*
2007-06-07 09:18:55 +04:00
modify flags on a node
2007-06-07 05:15:22 +04:00
*/
2007-06-07 09:18:55 +04:00
int32_t ctdb_control_modflags ( struct ctdb_context * ctdb , TDB_DATA indata )
2007-06-07 05:15:22 +04:00
{
2008-11-19 06:43:46 +03:00
struct ctdb_node_flag_change * c = ( struct ctdb_node_flag_change * ) indata . dptr ;
struct ctdb_node * node ;
2009-07-09 07:20:14 +04:00
uint32_t old_flags ;
2009-10-12 05:08:39 +04:00
int i ;
2008-11-19 06:43:46 +03:00
if ( c - > pnn > = ctdb - > num_nodes ) {
DEBUG ( DEBUG_ERR , ( __location__ " Node %d is invalid, num_nodes :%d \n " , c - > pnn , ctdb - > num_nodes ) ) ;
return - 1 ;
}
2007-06-07 05:15:22 +04:00
2008-11-19 06:43:46 +03:00
node = ctdb - > nodes [ c - > pnn ] ;
2009-07-09 07:20:14 +04:00
old_flags = node - > flags ;
2009-10-09 17:47:49 +04:00
if ( c - > pnn ! = ctdb - > pnn ) {
c - > old_flags = node - > flags ;
}
2008-11-19 06:43:46 +03:00
node - > flags = c - > new_flags & ~ NODE_FLAGS_DISCONNECTED ;
node - > flags | = ( c - > old_flags & NODE_FLAGS_DISCONNECTED ) ;
2007-06-07 09:18:55 +04:00
2009-07-09 07:20:14 +04:00
/* we dont let other nodes modify our STOPPED status */
if ( c - > pnn = = ctdb - > pnn ) {
node - > flags & = ~ NODE_FLAGS_STOPPED ;
if ( old_flags & NODE_FLAGS_STOPPED ) {
node - > flags | = NODE_FLAGS_STOPPED ;
}
}
2009-09-03 20:20:39 +04:00
/* we dont let other nodes modify our BANNED status */
if ( c - > pnn = = ctdb - > pnn ) {
node - > flags & = ~ NODE_FLAGS_BANNED ;
if ( old_flags & NODE_FLAGS_BANNED ) {
node - > flags | = NODE_FLAGS_BANNED ;
}
}
2008-11-19 06:43:46 +03:00
if ( node - > flags = = c - > old_flags ) {
DEBUG ( DEBUG_INFO , ( " Control modflags on node %u - Unchanged - flags 0x%x \n " , c - > pnn , node - > flags ) ) ;
2007-06-07 09:18:55 +04:00
return 0 ;
2007-06-07 05:15:22 +04:00
}
2008-11-19 06:43:46 +03:00
DEBUG ( DEBUG_INFO , ( " Control modflags on node %u - flags now 0x%x \n " , c - > pnn , node - > flags ) ) ;
2007-06-07 10:34:33 +04:00
2009-12-07 15:28:11 +03:00
if ( node - > flags = = 0 & & ! ctdb - > done_startup ) {
DEBUG ( DEBUG_ERR , ( __location__ " Node %u became healthy - force recovery for startup \n " ,
c - > pnn ) ) ;
ctdb - > recovery_mode = CTDB_RECOVERY_ACTIVE ;
}
2007-06-07 05:15:22 +04:00
2008-11-19 06:43:46 +03:00
/* tell the recovery daemon something has changed */
ctdb_daemon_send_message ( ctdb , ctdb - > pnn ,
CTDB_SRVID_SET_NODE_FLAGS , indata ) ;
2007-06-07 10:34:33 +04:00
2008-11-19 06:43:46 +03:00
/* if we have become banned, we should go into recovery mode */
2009-06-10 04:28:47 +04:00
if ( ( node - > flags & NODE_FLAGS_BANNED ) & & ! ( c - > old_flags & NODE_FLAGS_BANNED ) & & ( node - > pnn = = ctdb - > pnn ) ) {
2007-06-07 10:34:33 +04:00
/* make sure we are frozen */
2008-02-04 12:07:15 +03:00
DEBUG ( DEBUG_NOTICE , ( " This node has been banned - forcing freeze and recovery \n " ) ) ;
2007-08-22 04:38:35 +04:00
/* Reset the generation id to 1 to make us ignore any
REQ / REPLY CALL / DMASTER someone sends to us .
We are now banned so we shouldnt service database calls
anymore .
*/
2007-08-22 06:38:31 +04:00
ctdb - > vnn_map - > generation = INVALID_GENERATION ;
2007-08-22 04:38:35 +04:00
2009-10-12 05:08:39 +04:00
for ( i = 1 ; i < = NUM_DB_PRIORITIES ; i + + ) {
if ( ctdb_start_freeze ( ctdb , i ) ! = 0 ) {
DEBUG ( DEBUG_ERR , ( __location__ " Failed to freeze db priority %u \n " , i ) ) ;
}
}
2007-06-10 02:46:33 +04:00
ctdb_release_all_ips ( ctdb ) ;
2007-06-07 10:34:33 +04:00
ctdb - > recovery_mode = CTDB_RECOVERY_ACTIVE ;
}
2007-06-07 05:15:22 +04:00
return 0 ;
}
2008-01-10 06:40:56 +03:00
/*
return the monitoring mode
*/
int32_t ctdb_monitoring_mode ( struct ctdb_context * ctdb )
{
if ( ctdb - > monitor = = NULL ) {
return CTDB_MONITORING_DISABLED ;
}
return ctdb - > monitor - > monitoring_mode ;
}
2008-04-02 04:13:30 +04:00