2019-11-21 21:32:02 +03:00
// +build !windows
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package graceful
import (
2019-11-30 17:40:22 +03:00
"context"
2019-11-21 21:32:02 +03:00
"errors"
"os"
"os/signal"
"sync"
"syscall"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
2019-12-15 12:51:28 +03:00
// Manager manages the graceful shutdown process
type Manager struct {
2019-11-21 21:32:02 +03:00
isChild bool
forked bool
lock * sync . RWMutex
state state
shutdown chan struct { }
hammer chan struct { }
terminate chan struct { }
2019-12-15 12:51:28 +03:00
done chan struct { }
2019-11-21 21:32:02 +03:00
runningServerWaitGroup sync . WaitGroup
createServerWaitGroup sync . WaitGroup
terminateWaitGroup sync . WaitGroup
}
2019-12-15 12:51:28 +03:00
func newGracefulManager ( ctx context . Context ) * Manager {
manager := & Manager {
2019-11-21 21:32:02 +03:00
isChild : len ( os . Getenv ( listenFDs ) ) > 0 && os . Getppid ( ) > 1 ,
lock : & sync . RWMutex { } ,
}
manager . createServerWaitGroup . Add ( numberOfServersToCreate )
2019-12-15 12:51:28 +03:00
manager . start ( ctx )
2019-11-21 21:32:02 +03:00
return manager
}
2019-12-15 12:51:28 +03:00
func ( g * Manager ) start ( ctx context . Context ) {
// Make channels
g . terminate = make ( chan struct { } )
g . shutdown = make ( chan struct { } )
g . hammer = make ( chan struct { } )
g . done = make ( chan struct { } )
// Set the running state & handle signals
2019-11-21 21:32:02 +03:00
g . setState ( stateRunning )
2019-11-30 17:40:22 +03:00
go g . handleSignals ( ctx )
2019-12-15 12:51:28 +03:00
// Handle clean up of unused provided listeners and delayed start-up
startupDone := make ( chan struct { } )
2019-11-21 21:32:02 +03:00
go func ( ) {
2019-12-15 12:51:28 +03:00
defer close ( startupDone )
2019-11-21 21:32:02 +03:00
// Wait till we're done getting all of the listeners and then close
// the unused ones
g . createServerWaitGroup . Wait ( )
// Ignore the error here there's not much we can do with it
// They're logged in the CloseProvidedListeners function
_ = CloseProvidedListeners ( )
} ( )
if setting . StartupTimeout > 0 {
go func ( ) {
select {
2019-12-15 12:51:28 +03:00
case <- startupDone :
2019-11-21 21:32:02 +03:00
return
case <- g . IsShutdown ( ) :
2019-12-15 12:51:28 +03:00
func ( ) {
// When waitgroup counter goes negative it will panic - we don't care about this so we can just ignore it.
defer func ( ) {
_ = recover ( )
} ( )
// Ensure that the createServerWaitGroup stops waiting
for {
g . createServerWaitGroup . Done ( )
}
} ( )
2019-11-21 21:32:02 +03:00
return
case <- time . After ( setting . StartupTimeout ) :
log . Error ( "Startup took too long! Shutting down" )
g . doShutdown ( )
}
} ( )
}
}
2019-12-15 12:51:28 +03:00
func ( g * Manager ) handleSignals ( ctx context . Context ) {
2019-11-21 21:32:02 +03:00
signalChannel := make ( chan os . Signal , 1 )
signal . Notify (
signalChannel ,
syscall . SIGHUP ,
syscall . SIGUSR1 ,
syscall . SIGUSR2 ,
syscall . SIGINT ,
syscall . SIGTERM ,
syscall . SIGTSTP ,
)
pid := syscall . Getpid ( )
for {
2019-11-30 17:40:22 +03:00
select {
case sig := <- signalChannel :
switch sig {
case syscall . SIGHUP :
2020-02-11 08:29:45 +03:00
log . Info ( "PID: %d. Received SIGHUP. Attempting GracefulRestart..." , pid )
g . DoGracefulRestart ( )
2019-11-30 17:40:22 +03:00
case syscall . SIGUSR1 :
2020-07-06 03:07:07 +03:00
log . Warn ( "PID %d. Received SIGUSR1. Releasing and reopening logs" , pid )
if err := log . ReleaseReopen ( ) ; err != nil {
log . Error ( "Error whilst releasing and reopening logs: %v" , err )
}
2019-11-30 17:40:22 +03:00
case syscall . SIGUSR2 :
log . Warn ( "PID %d. Received SIGUSR2. Hammering..." , pid )
2020-01-29 04:01:06 +03:00
g . DoImmediateHammer ( )
2019-11-30 17:40:22 +03:00
case syscall . SIGINT :
log . Warn ( "PID %d. Received SIGINT. Shutting down..." , pid )
2020-01-29 04:01:06 +03:00
g . DoGracefulShutdown ( )
2019-11-30 17:40:22 +03:00
case syscall . SIGTERM :
log . Warn ( "PID %d. Received SIGTERM. Shutting down..." , pid )
2020-01-29 04:01:06 +03:00
g . DoGracefulShutdown ( )
2019-11-30 17:40:22 +03:00
case syscall . SIGTSTP :
log . Info ( "PID %d. Received SIGTSTP." , pid )
default :
log . Info ( "PID %d. Received %v." , pid , sig )
2019-11-21 21:32:02 +03:00
}
2019-11-30 17:40:22 +03:00
case <- ctx . Done ( ) :
log . Warn ( "PID: %d. Background context for manager closed - %v - Shutting down..." , pid , ctx . Err ( ) )
2020-01-29 04:01:06 +03:00
g . DoGracefulShutdown ( )
2019-11-21 21:32:02 +03:00
}
}
}
2019-12-15 12:51:28 +03:00
func ( g * Manager ) doFork ( ) error {
2019-11-21 21:32:02 +03:00
g . lock . Lock ( )
if g . forked {
g . lock . Unlock ( )
return errors . New ( "another process already forked. Ignoring this one" )
}
g . forked = true
g . lock . Unlock ( )
// We need to move the file logs to append pids
setting . RestartLogsWithPIDSuffix ( )
_ , err := RestartProcess ( )
return err
}
2020-01-29 04:01:06 +03:00
// DoGracefulRestart causes a graceful restart
func ( g * Manager ) DoGracefulRestart ( ) {
if setting . GracefulRestartable {
log . Info ( "PID: %d. Forking..." , os . Getpid ( ) )
err := g . doFork ( )
if err != nil && err . Error ( ) != "another process already forked. Ignoring this one" {
log . Error ( "Error whilst forking from PID: %d : %v" , os . Getpid ( ) , err )
}
} else {
log . Info ( "PID: %d. Not set restartable. Shutting down..." , os . Getpid ( ) )
g . doShutdown ( )
}
}
// DoImmediateHammer causes an immediate hammer
func ( g * Manager ) DoImmediateHammer ( ) {
g . doHammerTime ( 0 * time . Second )
}
// DoGracefulShutdown causes a graceful shutdown
func ( g * Manager ) DoGracefulShutdown ( ) {
g . doShutdown ( )
}
2019-12-15 12:51:28 +03:00
// RegisterServer registers the running of a listening server, in the case of unix this means that the parent process can now die.
// Any call to RegisterServer must be matched by a call to ServerDone
func ( g * Manager ) RegisterServer ( ) {
2019-11-21 21:32:02 +03:00
KillParent ( )
g . runningServerWaitGroup . Add ( 1 )
}