I currently see kubo eating a lot of CPU and see a lot of errors in the “docker logs” - also docker ps shows it as unhealthy:
devops@prod-ipfs-2 ➜ ~ docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
7ef35bf665c6 ipfs/kubo:latest "/sbin/tini -- /usr/…" 5 days ago Up 41 minutes (unhealthy) 0.0.0.0:4001->4001/tcp, 8081/tcp, 4001/udp, 0.0.0.0:5001->5001/tcp, 8180/tcp, 0.0.0.0:8180->8080/tcp ipfs
a125b07b00e3 quay.io/prometheus/haproxy-exporter:v0.12.0 "/bin/haproxy_export…" 2 months ago Up 6 days 8404/tcp, 0.0.0.0:9101->9101/tcp haproxy-exporter
the logs are a constant stream of this:
goroutine 13751324 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc07f996a40)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc011e23440, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc00abe6d80, 0x1, 0xc011e23230?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc00abe6d60, 0x1, 0xc01ec159d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc01366e960, 0x26}, {0xc00abe6d00, 0x1, 0x1}, {0x3af9aa8, 0x0, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc01366e960, 0x26}, {0x2a47f70, 0xc01bcca990})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc083152700})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc01e6a7f08, 0x13}, {0x7f9ec419ded0?, 0xc083152700})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 13751464 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc1b026b0e0)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc00af6c6f0, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc014684dc0, 0x1, 0xc013aaf350?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc014684db0, 0x1, 0xc2d42bf9d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc00dba78f0, 0x26}, {0xc014684d70, 0x1, 0x1}, {0xc083152980, 0x8, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc00dba78f0, 0x26}, {0x2a47f70, 0xc01d27d470})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc00e052e00})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc05db1c9f0, 0x13}, {0x7f9ec419ded0?, 0xc00e052e00})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 13751461 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc08aa9d960)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc0135a4780, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc015dc5910, 0x1, 0xc0135a4690?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc015dc5900, 0x1, 0xc0da1469d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc018693d40, 0x26}, {0xc015dc58c0, 0x1, 0x1}, {0x3af9aa8, 0x0, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc018693d40, 0x26}, {0x2a47f70, 0xc010468bd0})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc083152880})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc05db1c978, 0x13}, {0x7f9ec419ded0?, 0xc083152880})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 13754615 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc008f3fc80)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc01ad0af30, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc01da88d00, 0x1, 0xc01ad0abd0?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc01da88cf0, 0x1, 0xc03c95e9d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc01366e960, 0x26}, {0xc01da88cb0, 0x1, 0x1}, {0x3af9aa8, 0x0, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc01366e960, 0x26}, {0x2a47f70, 0xc00d539920})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc004833d00})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc07acef1a0, 0x13}, {0x7f9ec419ded0?, 0xc004833d00})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 13751520 [select, 68 minutes]:
github.com/libp2p/go-libp2p-kad-dht/providers.(*ProviderManager).GetProviders(0xc0003ea120, {0x2a2c2e0?, 0xc018881e60}, {0xc011069680, 0x22, 0x30})
pkg/mod/github.com/libp2p/go-libp2p-kad-dht@v0.16.0/providers/providers_manager.go:286 +0x179
github.com/libp2p/go-libp2p-kad-dht.(*IpfsDHT).handleGetProviders(0xc000e28a80, {0x2a2c2e0, 0xc018881e60}, {0xc01adbb110, 0x26}, 0xc08c2e0a80)
pkg/mod/github.com/libp2p/go-libp2p-kad-dht@v0.16.0/handlers.go:320 +0x1c3
github.com/libp2p/go-libp2p-kad-dht.(*IpfsDHT).handleNewMessage(0xc000e28a80, {0x2a415b0?, 0xc08c7b1a80})
pkg/mod/github.com/libp2p/go-libp2p-kad-dht@v0.16.0/dht_net.go:121 +0x9d8
github.com/libp2p/go-libp2p-kad-dht.(*IpfsDHT).handleNewStream(0xc08c7b1ac0?, {0x2a415b0, 0xc08c7b1a80})
pkg/mod/github.com/libp2p/go-libp2p-kad-dht@v0.16.0/dht_net.go:26 +0x25
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc056d5dc30, 0xf}, {0x7f9ec419ded0?, 0xc08c7b1a80})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 13751329 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc0038e1ae0)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc0183a6e10, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc01aec0060, 0x1, 0xc0183a6b70?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc01aec0050, 0x1, 0xc0283c39d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc00f070900, 0x26}, {0xc01aec0010, 0x1, 0x1}, {0x3af9aa8, 0x0, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc00f070900, 0x26}, {0x2a47f70, 0xc0059de750})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc014ce8900})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc079540240, 0x13}, {0x7f9ec419ded0?, 0xc014ce8900})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 14774157 [semacquire, 63 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc00f1df820)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc20f425470, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc010918ff0, 0x3, 0xc20ddb0ae0?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc010918fc0, 0x3, 0xc3fd6c39d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc003b265d0, 0x26}, {0xc013818ff0, 0x3, 0x3}, {0xc01d33fcc0, 0xa, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc003b265d0, 0x26}, {0x2a47f70, 0xc013818e70})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc2ca76ce80})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc012f84540, 0x13}, {0x7f9ec419ded0?, 0xc2ca76ce80})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 16463699 [chan receive, 5 minutes]:
github.com/ipfs/go-blockservice.getBlocks.func1()
pkg/mod/github.com/ipfs/go-blockservice@v0.3.0/blockservice.go:313 +0x2dd
created by github.com/ipfs/go-blockservice.getBlocks
pkg/mod/github.com/ipfs/go-blockservice@v0.3.0/blockservice.go:264 +0x16b
goroutine 13753627 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc083bd03e0)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc01528bb90, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc009dd9e50, 0x1, 0xc01528b9b0?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc009dd9e40, 0x1, 0xc01a6bc9d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc0136e0ba0, 0x26}, {0xc009dd9e00, 0x1, 0x1}, {0x3af9aa8, 0x0, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc0136e0ba0, 0x26}, {0x2a47f70, 0xc015b0da10})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc007c7d400})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc06b46a3f0, 0x13}, {0x7f9ec419ded0?, 0xc007c7d400})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 13889214 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc1fdd7eb40)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc1adb9b2c0, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc19d1775a0, 0x1, 0xc1adb9b1a0?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc19d177590, 0x1, 0xc0862379d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc003ad2930, 0x26}, {0xc19d177550, 0x1, 0x1}, {0x3af9aa8, 0x0, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc003ad2930, 0x26}, {0x2a47f70, 0xc1b477b440})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc3637fdc80})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc19d459b00, 0x13}, {0x7f9ec419ded0?, 0xc3637fdc80})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 16665689 [select]:
github.com/ipfs/go-merkledag.getNodesFromBG.func1()
pkg/mod/github.com/ipfs/go-merkledag@v0.6.0/merkledag.go:244 +0x145
created by github.com/ipfs/go-merkledag.getNodesFromBG
pkg/mod/github.com/ipfs/go-merkledag@v0.6.0/merkledag.go:241 +0x30a
goroutine 13888953 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc007832e60)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc20d04f5f0, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc1949086a0, 0x1, 0xc20d04f4a0?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc194908690, 0x1, 0xc07f6199d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc00ff137d0, 0x26}, {0xc194908650, 0x1, 0x1}, {0x3af9aa8, 0x0, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc00ff137d0, 0x26}, {0x2a47f70, 0xc1a43f6c60})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc369948500})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc1adfdad68, 0x13}, {0x7f9ec419ded0?, 0xc369948500})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 14196401 [select, 66 minutes]:
github.com/ipfs/go-bitswap/internal/session.(*sessionWantSender).Run(0xc1970f45d0)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/internal/session/sessionwantsender.go:186 +0x149
created by github.com/ipfs/go-bitswap/internal/session.(*Session).run
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/internal/session/session.go:297 +0x8d
goroutine 13889055 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc08c0b7de0)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc209fdb3e0, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc1943c7ba0, 0x1, 0xc209fdb320?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc1943c7b80, 0x1, 0xc067e149d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc00c3ed0e0, 0x26}, {0xc1943c7b20, 0x1, 0x1}, {0x3af9aa8, 0x0, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc00c3ed0e0, 0x26}, {0x2a47f70, 0xc1a90c4b10})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc3669c6480})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc1a1aeb1b8, 0x13}, {0x7f9ec419ded0?, 0xc3669c6480})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 13751563 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc1a35cc0e0)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc0111202d0, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc0134f49c0, 0x3, 0xc01934def0?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc0134f4990, 0x3, 0xc002b489d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc00f070900, 0x26}, {0xc018523d10, 0x3, 0x3}, {0x3af9aa8, 0x0, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc00f070900, 0x26}, {0x2a47f70, 0xc018523b30})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc084f5de80})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc07a510318, 0x13}, {0x7f9ec419ded0?, 0xc084f5de80})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:411 +0x715
goroutine 13753475 [semacquire, 68 minutes]:
sync.runtime_SemacquireMutex(0xc000494a20?, 0x20?, 0xd939bd?)
/usr/local/go/src/runtime/sema.go:71 +0x25
sync.(*Mutex).lockSlow(0xc07abf4660)
/usr/local/go/src/sync/mutex.go:162 +0x165
sync.(*Mutex).Lock(...)
/usr/local/go/src/sync/mutex.go:81
sync.(*RWMutex).Lock(0x222dd80?)
/usr/local/go/src/sync/rwmutex.go:139 +0x36
github.com/ipfs/go-ipfs-blockstore.(*arccache).lock(0xc0004a2e40, {0xc013e5d1a0, 0x22}, 0x1)
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:67 +0x12f
github.com/ipfs/go-ipfs-blockstore.(*arccache).PutMany(0xc0004a2e40, {0x2a2c270, 0xc000054060}, {0xc0147dd990, 0x1, 0xc013e5d050?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/arc_cache.go:331 +0x3b2
github.com/ipfs/go-ipfs-blockstore.(*idstore).PutMany(0xc000494ae0, {0x2a2c270, 0xc000054060}, {0xc0147dd980, 0x1, 0xc0341a89d8?})
pkg/mod/github.com/ipfs/go-ipfs-blockstore@v1.2.0/idstore.go:107 +0xa3
github.com/ipfs/go-bitswap.(*Bitswap).receiveBlocksFrom(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc012aae1e0, 0x22}, {0xc0147dd930, 0x1, 0x1}, {0x3af9aa8, 0x0, ...}, ...)
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:490 +0x190
github.com/ipfs/go-bitswap.(*Bitswap).ReceiveMessage(0xc0002dc2c0, {0x2a2c270, 0xc000054060}, {0xc012aae1e0, 0x22}, {0x2a47f70, 0xc0121b2930})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/bitswap.go:583 +0x29a
github.com/ipfs/go-bitswap/network.(*impl).handleNewStream(0xc0004f4000, {0x2a415b0?, 0xc010e96680})
pkg/mod/github.com/ipfs/go-bitswap@v0.7.0/network/ipfs_impl.go:428 +0x37c
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1({0xc079541a58, 0x13}, {0x7f9ec419ded0?, 0xc010e96680})
pkg/mod/github.com/libp2p/go-libp2p@v0.20.3/p2p/host/basic/basic_host.go:568 +0x76
any idea what I could do here?