Deadlocks in `get_picture()` on specific files with multi-threaded decoding while ffmpeg/dav1dplay don't

[sdroege]

File available from https://www.dropbox.com/s/s8o8paaeghcz3cy/dav1d-rs-get-picture-deadlock.ivf?dl=0. This is an AV1 version of https://www.youtube.com/watch?v=mkggXE5e2yk

I was quite certain that this is a bug in dav1d but... I can't reproduce it with dav1dplay (from the dav1d examples, using the placebo-gl renderer because the SDL one doesn't support this configuration) and also not with ffmpeg. But I can reproduce it with the dav1d GStreamer plugin, which uses dav1d-rs.

(gdb) thread apply all bt

Thread 33 (Thread 0x7f9a95a6a780 (LWP 1166877) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 32 (Thread 0x7f9a95b6e780 (LWP 1166876) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 31 (Thread 0x7f9a95c72780 (LWP 1166875) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 30 (Thread 0x7f9a95d76780 (LWP 1166874) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 29 (Thread 0x7f9a95e7a780 (LWP 1166873) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 28 (Thread 0x7f9a95f7e780 (LWP 1166872) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

--Type <RET> for more, q to quit, c to continue without paging--
Thread 27 (Thread 0x7f9a96082780 (LWP 1166871) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 26 (Thread 0x7f9a96186780 (LWP 1166870) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 25 (Thread 0x7f9a9628a780 (LWP 1166869) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 24 (Thread 0x7f9a9638e780 (LWP 1166868) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 23 (Thread 0x7f9a96492780 (LWP 1166867) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 22 (Thread 0x7f9a96596780 (LWP 1166866) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 21 (Thread 0x7f9a9669a780 (LWP 1166865) "dav1d-worker"):
--Type <RET> for more, q to quit, c to continue without paging--
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 20 (Thread 0x7f9a9679e780 (LWP 1166864) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 19 (Thread 0x7f9a968a2780 (LWP 1166863) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 18 (Thread 0x7f9a969a6780 (LWP 1166862) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 17 (Thread 0x7f9a96aaa780 (LWP 1166861) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 16 (Thread 0x7f9a96bae780 (LWP 1166860) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 15 (Thread 0x7f9a96cb2780 (LWP 1166859) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
--Type <RET> for more, q to quit, c to continue without paging--
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 14 (Thread 0x7f9a96db6780 (LWP 1166858) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 13 (Thread 0x7f9a96eba780 (LWP 1166857) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 12 (Thread 0x7f9a96fbe780 (LWP 1166856) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 11 (Thread 0x7f9a970c2780 (LWP 1166855) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 10 (Thread 0x7f9a971c6780 (LWP 1166854) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 9 (Thread 0x7f9a972ca780 (LWP 1166853) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
--Type <RET> for more, q to quit, c to continue without paging--
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 8 (Thread 0x7f9a973ce780 (LWP 1166852) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 7 (Thread 0x7f9a974d2780 (LWP 1166851) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 6 (Thread 0x7f9a975d6780 (LWP 1166850) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 5 (Thread 0x7f9a976da780 (LWP 1166849) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 4 (Thread 0x7f9a977de780 (LWP 1166848) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 3 (Thread 0x7f9a978e2780 (LWP 1166847) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
--Type <RET> for more, q to quit, c to continue without paging--
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 2 (Thread 0x7f9a979e6780 (LWP 1166846) "dav1d-worker"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98533d76 in dav1d_worker_task () from /lib64/libdav1d.so.6
#3  0x00007f9a9826312d in start_thread () from /lib64/libc.so.6
#4  0x00007f9a982e4bc0 in clone3 () from /lib64/libc.so.6

Thread 1 (Thread 0x7f9a981d6bc0 (LWP 1166844) "dav1d-tools"):
#0  0x00007f9a9825fd16 in __futex_abstimed_wait_common () from /lib64/libc.so.6
#1  0x00007f9a98262510 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libc.so.6
#2  0x00007f9a98531f5f in dav1d_get_picture () from /lib64/libdav1d.so.6
#3  0x00005612d32d2106 in dav1d::Decoder::get_picture (self=0x7ffeb6728538) at src/lib.rs:326
#4  0x00005612d32cad34 in dav1d_tools::handle_pending_pictures (dec=0x7ffeb6728538) at tools/src/main.rs:79
#5  0x00005612d32cb3fe in dav1d_tools::main () at tools/src/main.rs:128

[sdroege]

diff --git a/tools/src/main.rs b/tools/src/main.rs
index ca17a41..148ba90 100644
--- a/tools/src/main.rs
+++ b/tools/src/main.rs
@@ -95,7 +95,10 @@ fn main() -> std::io::Result<()> {
     let header = ivf::read_header(&mut r)?;
     println!("{:?}", header);
 
-    let mut dec = dav1d::Decoder::new().expect("failed to create decoder instance");
+    let mut settings = dav1d::Settings::default();
+    settings.set_n_threads(2);
+    let mut dec =
+        dav1d::Decoder::with_settings(&settings).expect("failed to create decoder instance");
 
     while let Ok(packet) = ivf::read_packet(&mut r) {
         println!("Packet {}", packet.pts);

With 1 thread it doesn't deadlock, with 2 or more threads it deadlocks.

[sdroege]

Actually VLC also deadlocks on this file in the same way. I'll report this to dav1d then.

[sdroege]

https://code.videolan.org/videolan/dav1d/-/issues/416

[lu-zero]

You may keep the issue open.