ignite-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Denis Chudov (Jira)" <j...@apache.org>
Subject [jira] [Created] (IGNITE-13418) Deadlock on multiple cache delete
Date Wed, 09 Sep 2020 07:51:00 GMT
Denis Chudov created IGNITE-13418:
-------------------------------------

             Summary: Deadlock on multiple cache delete
                 Key: IGNITE-13418
                 URL: https://issues.apache.org/jira/browse/IGNITE-13418
             Project: Ignite
          Issue Type: Bug
            Reporter: Denis Chudov
            Assignee: Denis Chudov


Thread 1:
 - acquires checkpoint read lock in GridCacheProcessor#processCacheStopRequestOnExchangeDone
 - acquires GridQueryProcessor#stateMux in GridQueryProcessor.onCacheStop
 - enters H2TreeIndex.destroy
 - releases checkpoint read lock in H2Tree.temporaryReleaseLock, can't take it again because
of db-checkpoint-thread

Thread 2:
 - acquires checkpoint read lock in GridCacheProcessor#processCacheStopRequestOnExchangeDone
 - trying to acquire GridQueryProcessor#stateMux in GridQueryProcessor.onCacheStop which is
held by thread 1

db-checkpoint-thread:
 - trying to acquire checkpoint write lock, can't do it because of Thread 2

Decision: H2Tree.temporaryReleaseLock should release lock only in case when tree deletion
is asynchronous (H2TreeIndex.destroy is called with async=true), i.e. it happens inside of
DurableBackgroundTask. Such tasks are executed in separate threads, which don't hold any other
locks.

Thread dump:
{code:java}
Thread [name="sys-#1220%DPL_GRID%DplGridNodeName%", id=3200, state=BLOCKED, blockCnt=1, waitCnt=0]
 Lock [object=java.lang.Object@6a9a92ba, ownerName=sys-#1215%DPL_GRID%DplGridNodeName%, ownerId=3195]
 at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop0(GridQueryProcessor.java:1695)
 at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop(GridQueryProcessor.java:902)
 at o.a.i.i.processors.cache.GridCacheProcessor.stopCache(GridCacheProcessor.java:1169)
 at o.a.i.i.processors.cache.GridCacheProcessor.prepareCacheStop(GridCacheProcessor.java:2644)
 at o.a.i.i.processors.cache.GridCacheProcessor.lambda$processCacheStopRequestOnExchangeDone$629e8679$1(GridCacheProcessor.java:2803)
 at o.a.i.i.processors.cache.GridCacheProcessor$$Lambda$620/1418386924.apply(Unknown Source)
 at o.a.i.i.util.IgniteUtils.lambda$null$1(IgniteUtils.java:10879)
 at o.a.i.i.util.IgniteUtils$$Lambda$436/321848940.call(Unknown Source)
 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
 at java.lang.Thread.run(Thread.java:748)
 Locked synchronizers:
 java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync@2582f93c
 java.util.concurrent.ThreadPoolExecutor$Worker@4da1cafe
Thread [name="sys-#1215%DPL_GRID%DplGridNodeName%", id=3195, state=BLOCKED, blockCnt=4, waitCnt=437520]
 Lock [object=o.a.i.i.processors.failure.FailureProcessor@78edb1e9, ownerName=async-durable-background-task-executor-1-#1222%DPL_GRID%DplGridNodeName%,
ownerId=3202]
 at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:162)
 at o.a.i.i.processors.failure.FailureProcessor.process(FailureProcessor.java:151)
 at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.failCheckpointReadLock(GridCacheDatabaseSharedManager.java:1787)
 at o.a.i.i.processors.cache.persistence.GridCacheDatabaseSharedManager.checkpointReadLock(GridCacheDatabaseSharedManager.java:1722)
 at o.a.i.i.processors.query.h2.database.H2Tree.temporaryReleaseLock(H2Tree.java:690)
 at o.a.i.i.processors.cache.persistence.tree.BPlusTree.temporaryReleaseLock(BPlusTree.java:2367)
 at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2548)
 at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522)
 at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522)
 at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroyDownPages(BPlusTree.java:2522)
 at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroy(BPlusTree.java:2441)
 at o.a.i.i.processors.cache.persistence.tree.BPlusTree.destroy(BPlusTree.java:2392)
 at o.a.i.i.processors.query.h2.database.H2TreeIndex.destroy0(H2TreeIndex.java:671)
 at o.a.i.i.processors.query.h2.database.H2TreeIndex.destroy(H2TreeIndex.java:639)
 at o.a.i.i.processors.query.h2.opt.GridH2Table.destroy(GridH2Table.java:567)
 at o.a.i.i.processors.query.h2.H2TableDescriptor.onDrop(H2TableDescriptor.java:347)
 at o.a.i.i.processors.query.h2.H2Schema.drop(H2Schema.java:127)
 at o.a.i.i.processors.query.h2.IgniteH2Indexing.unregisterCache(IgniteH2Indexing.java:2595)
 at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop0(GridQueryProcessor.java:1727)
 - locked java.lang.Object@6a9a92ba
 at o.a.i.i.processors.query.GridQueryProcessor.onCacheStop(GridQueryProcessor.java:902)
 at o.a.i.i.processors.cache.GridCacheProcessor.stopCache(GridCacheProcessor.java:1169)
 at o.a.i.i.processors.cache.GridCacheProcessor.prepareCacheStop(GridCacheProcessor.java:2644)
 at o.a.i.i.processors.cache.GridCacheProcessor.lambda$processCacheStopRequestOnExchangeDone$629e8679$1(GridCacheProcessor.java:2803)
 at o.a.i.i.processors.cache.GridCacheProcessor$$Lambda$620/1418386924.apply(Unknown Source)
 at o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:10898)
 at o.a.i.i.util.IgniteUtils.doInParallel(IgniteUtils.java:10800)
 at o.a.i.i.processors.cache.GridCacheProcessor.processCacheStopRequestOnExchangeDone(GridCacheProcessor.java:2775)
 at o.a.i.i.processors.cache.GridCacheProcessor.onExchangeDone(GridCacheProcessor.java:2899)
 at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onDone(GridDhtPartitionsExchangeFuture.java:2210)
 at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.processFullMessage(GridDhtPartitionsExchangeFuture.java:4197)
 at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.access$1600(GridDhtPartitionsExchangeFuture.java:143)
 at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$5.apply(GridDhtPartitionsExchangeFuture.java:3879)
 at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$5.apply(GridDhtPartitionsExchangeFuture.java:3867)
 at o.a.i.i.util.future.GridFutureAdapter.notifyListener(GridFutureAdapter.java:385)
 at o.a.i.i.util.future.GridFutureAdapter.listen(GridFutureAdapter.java:355)
 at o.a.i.i.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.onReceiveFullMessage(GridDhtPartitionsExchangeFuture.java:3867)
 at o.a.i.i.processors.cache.GridCachePartitionExchangeManager.processFullPartitionUpdate(GridCachePartitionExchangeManager.java:1798)
 at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$3.onMessage(GridCachePartitionExchangeManager.java:408)
 at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$3.onMessage(GridCachePartitionExchangeManager.java:395)
 at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3492)
 at o.a.i.i.processors.cache.GridCachePartitionExchangeManager$MessageHandler.apply(GridCachePartitionExchangeManager.java:3471)
 at o.a.i.i.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1106)
 at o.a.i.i.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:589)
 at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:386)
 at o.a.i.i.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:312)
 at o.a.i.i.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:102)
 at o.a.i.i.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:301)
 at o.a.i.i.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1574)
 at o.a.i.i.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1198)
 at o.a.i.i.managers.communication.GridIoManager.access$4300(GridIoManager.java:132)
 at o.a.i.i.managers.communication.GridIoManager$8.run(GridIoManager.java:1099)
 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
 at java.lang.Thread.run(Thread.java:748){code}



--
This message was sent by Atlassian Jira
(v8.3.4#803005)

Mime
View raw message