1
0
mirror of https://github.com/go-gitea/gitea.git synced 2024-12-22 17:34:03 +03:00

Attempt to prevent the deadlock in the QueueDiskChannel Test again (#18415)

* Attempt to prevent the deadlock in the QueueDiskChannel Test again

This time we're going to adjust the pause tests to only test the right
flag.

* Only switch off pushback once we know that we are not pushing anything else
* Ensure full redirection occurs
* More nicely handle a closed datachan
* And handle similar problems in queue_channel_test

Signed-off-by: Andrew Thornton <art27@cantab.net>
This commit is contained in:
zeripath 2022-01-29 11:37:08 +00:00 committed by GitHub
parent 726715fcfb
commit 92b715e0f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 119 additions and 95 deletions

View File

@ -205,7 +205,10 @@ loop:
// tell the pool to shutdown.
q.baseCtxCancel()
return
case data := <-q.dataChan:
case data, ok := <-q.dataChan:
if !ok {
return
}
if err := q.PushBack(data); err != nil {
log.Error("Unable to push back data into queue %s", q.name)
}

View File

@ -117,7 +117,10 @@ func (q *ChannelQueue) FlushWithContext(ctx context.Context) error {
select {
case <-paused:
return nil
case data := <-q.dataChan:
case data, ok := <-q.dataChan:
if !ok {
return nil
}
if unhandled := q.handle(data); unhandled != nil {
log.Error("Unhandled Data whilst flushing queue %d", q.qid)
}

View File

@ -9,6 +9,7 @@ import (
"testing"
"time"
"code.gitea.io/gitea/modules/log"
"github.com/stretchr/testify/assert"
)
@ -111,7 +112,6 @@ func TestChannelQueue_Pause(t *testing.T) {
if pausable, ok := queue.(Pausable); ok {
pausable.Pause()
}
pushBack = false
lock.Unlock()
return data
}
@ -123,7 +123,9 @@ func TestChannelQueue_Pause(t *testing.T) {
}
return nil
}
nilFn := func(_ func()) {}
queueShutdown := []func(){}
queueTerminate := []func(){}
queue, err = NewChannelQueue(handle,
ChannelQueueConfiguration{
@ -139,7 +141,34 @@ func TestChannelQueue_Pause(t *testing.T) {
}, &testData{})
assert.NoError(t, err)
go queue.Run(nilFn, nilFn)
go queue.Run(func(shutdown func()) {
lock.Lock()
defer lock.Unlock()
queueShutdown = append(queueShutdown, shutdown)
}, func(terminate func()) {
lock.Lock()
defer lock.Unlock()
queueTerminate = append(queueTerminate, terminate)
})
// Shutdown and Terminate in defer
defer func() {
lock.Lock()
callbacks := make([]func(), len(queueShutdown))
copy(callbacks, queueShutdown)
lock.Unlock()
for _, callback := range callbacks {
callback()
}
lock.Lock()
log.Info("Finally terminating")
callbacks = make([]func(), len(queueTerminate))
copy(callbacks, queueTerminate)
lock.Unlock()
for _, callback := range callbacks {
callback()
}
}()
test1 := testData{"A", 1}
test2 := testData{"B", 2}
@ -155,14 +184,11 @@ func TestChannelQueue_Pause(t *testing.T) {
pausable.Pause()
paused, resumed := pausable.IsPausedIsResumed()
paused, _ := pausable.IsPausedIsResumed()
select {
case <-paused:
case <-resumed:
assert.Fail(t, "Queue should not be resumed")
return
default:
case <-time.After(100 * time.Millisecond):
assert.Fail(t, "Queue is not paused")
return
}
@ -179,10 +205,11 @@ func TestChannelQueue_Pause(t *testing.T) {
assert.Nil(t, result2)
pausable.Resume()
_, resumed := pausable.IsPausedIsResumed()
select {
case <-resumed:
default:
case <-time.After(100 * time.Millisecond):
assert.Fail(t, "Queue should be resumed")
}
@ -199,47 +226,47 @@ func TestChannelQueue_Pause(t *testing.T) {
pushBack = true
lock.Unlock()
paused, resumed = pausable.IsPausedIsResumed()
_, resumed = pausable.IsPausedIsResumed()
select {
case <-paused:
assert.Fail(t, "Queue should not be paused")
return
case <-resumed:
default:
case <-time.After(100 * time.Millisecond):
assert.Fail(t, "Queue is not resumed")
return
}
queue.Push(&test1)
paused, _ = pausable.IsPausedIsResumed()
select {
case <-paused:
case <-handleChan:
assert.Fail(t, "handler chan should not contain test1")
return
case <-time.After(500 * time.Millisecond):
case <-time.After(100 * time.Millisecond):
assert.Fail(t, "queue should be paused")
return
}
paused, resumed = pausable.IsPausedIsResumed()
lock.Lock()
pushBack = false
lock.Unlock()
paused, _ = pausable.IsPausedIsResumed()
select {
case <-paused:
case <-resumed:
assert.Fail(t, "Queue should not be resumed")
return
default:
case <-time.After(100 * time.Millisecond):
assert.Fail(t, "Queue is not paused")
return
}
pausable.Resume()
_, resumed = pausable.IsPausedIsResumed()
select {
case <-resumed:
default:
case <-time.After(100 * time.Millisecond):
assert.Fail(t, "Queue should be resumed")
}

View File

@ -313,14 +313,13 @@ func (q *PersistableChannelQueue) Shutdown() {
q.channelQueue.Wait()
q.internal.(*LevelQueue).Wait()
// Redirect all remaining data in the chan to the internal channel
go func() {
log.Trace("PersistableChannelQueue: %s Redirecting remaining data", q.delayedStarter.name)
for data := range q.channelQueue.dataChan {
_ = q.internal.Push(data)
atomic.AddInt64(&q.channelQueue.numInQueue, -1)
}
log.Trace("PersistableChannelQueue: %s Done Redirecting remaining data", q.delayedStarter.name)
}()
log.Trace("PersistableChannelQueue: %s Redirecting remaining data", q.delayedStarter.name)
close(q.channelQueue.dataChan)
for data := range q.channelQueue.dataChan {
_ = q.internal.Push(data)
atomic.AddInt64(&q.channelQueue.numInQueue, -1)
}
log.Trace("PersistableChannelQueue: %s Done Redirecting remaining data", q.delayedStarter.name)
log.Debug("PersistableChannelQueue: %s Shutdown", q.delayedStarter.name)
}

View File

@ -207,7 +207,6 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
log.Info("pausing")
pausable.Pause()
}
pushBack = false
lock.Unlock()
return data
}
@ -248,6 +247,25 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
queueTerminate = append(queueTerminate, terminate)
})
// Shutdown and Terminate in defer
defer func() {
lock.Lock()
callbacks := make([]func(), len(queueShutdown))
copy(callbacks, queueShutdown)
lock.Unlock()
for _, callback := range callbacks {
callback()
}
lock.Lock()
log.Info("Finally terminating")
callbacks = make([]func(), len(queueTerminate))
copy(callbacks, queueTerminate)
lock.Unlock()
for _, callback := range callbacks {
callback()
}
}()
test1 := testData{"A", 1}
test2 := testData{"B", 2}
@ -263,14 +281,11 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
assert.Equal(t, test1.TestInt, result1.TestInt)
pausable.Pause()
paused, resumed := pausable.IsPausedIsResumed()
paused, _ := pausable.IsPausedIsResumed()
select {
case <-paused:
case <-resumed:
assert.Fail(t, "Queue should not be resumed")
return
default:
case <-time.After(100 * time.Millisecond):
assert.Fail(t, "Queue is not paused")
return
}
@ -287,14 +302,11 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
assert.Nil(t, result2)
pausable.Resume()
paused, resumed = pausable.IsPausedIsResumed()
_, resumed := pausable.IsPausedIsResumed()
select {
case <-paused:
assert.Fail(t, "Queue should be resumed")
return
case <-resumed:
default:
case <-time.After(100 * time.Millisecond):
assert.Fail(t, "Queue should be resumed")
return
}
@ -308,24 +320,27 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
assert.Equal(t, test2.TestString, result2.TestString)
assert.Equal(t, test2.TestInt, result2.TestInt)
// Set pushBack to so that the next handle will result in a Pause
lock.Lock()
pushBack = true
lock.Unlock()
paused, resumed = pausable.IsPausedIsResumed()
// Ensure that we're still resumed
_, resumed = pausable.IsPausedIsResumed()
select {
case <-paused:
assert.Fail(t, "Queue should not be paused")
return
case <-resumed:
default:
case <-time.After(100 * time.Millisecond):
assert.Fail(t, "Queue is not resumed")
return
}
// push test1
queue.Push(&test1)
// Now as this is handled it should pause
paused, _ = pausable.IsPausedIsResumed()
select {
case <-paused:
case <-handleChan:
@ -336,27 +351,16 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
return
}
paused, resumed = pausable.IsPausedIsResumed()
select {
case <-paused:
case <-resumed:
assert.Fail(t, "Queue should not be resumed")
return
default:
assert.Fail(t, "Queue is not paused")
return
}
lock.Lock()
pushBack = false
lock.Unlock()
pausable.Resume()
paused, resumed = pausable.IsPausedIsResumed()
_, resumed = pausable.IsPausedIsResumed()
select {
case <-paused:
assert.Fail(t, "Queue should not be paused")
return
case <-resumed:
default:
case <-time.After(500 * time.Millisecond):
assert.Fail(t, "Queue should be resumed")
return
}
@ -373,6 +377,7 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
lock.Lock()
callbacks := make([]func(), len(queueShutdown))
copy(callbacks, queueShutdown)
queueShutdown = queueShutdown[:0]
lock.Unlock()
// Now shutdown the queue
for _, callback := range callbacks {
@ -402,6 +407,7 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
lock.Lock()
callbacks = make([]func(), len(queueTerminate))
copy(callbacks, queueTerminate)
queueShutdown = queueTerminate[:0]
lock.Unlock()
for _, callback := range callbacks {
callback()
@ -453,14 +459,11 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
case <-paused:
}
paused, resumed = pausable.IsPausedIsResumed()
paused, _ = pausable.IsPausedIsResumed()
select {
case <-paused:
case <-resumed:
assert.Fail(t, "Queue should not be resumed")
return
default:
case <-time.After(500 * time.Millisecond):
assert.Fail(t, "Queue is not paused")
return
}
@ -472,14 +475,15 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
default:
}
lock.Lock()
pushBack = false
lock.Unlock()
pausable.Resume()
paused, resumed = pausable.IsPausedIsResumed()
_, resumed = pausable.IsPausedIsResumed()
select {
case <-paused:
assert.Fail(t, "Queue should not be paused")
return
case <-resumed:
default:
case <-time.After(500 * time.Millisecond):
assert.Fail(t, "Queue should be resumed")
return
}
@ -506,18 +510,4 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
assert.Equal(t, test2.TestString, result4.TestString)
assert.Equal(t, test2.TestInt, result4.TestInt)
lock.Lock()
callbacks = make([]func(), len(queueShutdown))
copy(callbacks, queueShutdown)
lock.Unlock()
for _, callback := range callbacks {
callback()
}
lock.Lock()
callbacks = make([]func(), len(queueTerminate))
copy(callbacks, queueTerminate)
lock.Unlock()
for _, callback := range callbacks {
callback()
}
}

View File

@ -178,7 +178,10 @@ func (q *ChannelUniqueQueue) FlushWithContext(ctx context.Context) error {
default:
}
select {
case data := <-q.dataChan:
case data, ok := <-q.dataChan:
if !ok {
return nil
}
if unhandled := q.handle(data); unhandled != nil {
log.Error("Unhandled Data whilst flushing queue %d", q.qid)
}

View File

@ -282,13 +282,12 @@ func (q *PersistableChannelUniqueQueue) Shutdown() {
q.channelQueue.Wait()
q.internal.(*LevelUniqueQueue).Wait()
// Redirect all remaining data in the chan to the internal channel
go func() {
log.Trace("PersistableChannelUniqueQueue: %s Redirecting remaining data", q.delayedStarter.name)
for data := range q.channelQueue.dataChan {
_ = q.internal.Push(data)
}
log.Trace("PersistableChannelUniqueQueue: %s Done Redirecting remaining data", q.delayedStarter.name)
}()
close(q.channelQueue.dataChan)
log.Trace("PersistableChannelUniqueQueue: %s Redirecting remaining data", q.delayedStarter.name)
for data := range q.channelQueue.dataChan {
_ = q.internal.Push(data)
}
log.Trace("PersistableChannelUniqueQueue: %s Done Redirecting remaining data", q.delayedStarter.name)
log.Debug("PersistableChannelUniqueQueue: %s Shutdown", q.delayedStarter.name)
}