From c0750ab0e2fd7cc794f65ec00ffee85ea56847bf Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Wed, 24 Sep 2025 17:32:31 +0200 Subject: [PATCH 1/4] Force propertynames(::Py) to occur on the main thread --- src/C/context.jl | 46 ++++++++++++++++++++++++++++++++++++++++++++++ src/Core/Py.jl | 36 ++++++++++++++++++++---------------- test/Core.jl | 9 +++++++++ 3 files changed, 75 insertions(+), 16 deletions(-) diff --git a/src/C/context.jl b/src/C/context.jl index 8d2714c5..2d550f31 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -29,6 +29,20 @@ function _atpyexit() return end + +const MAIN_THREAD_TASK_LOCK = ReentrantLock() +const MAIN_THREAD_CHANNEL_INPUT = Channel(1) +const MAIN_THREAD_CHANNEL_OUTPUT = Channel(1) + +# Execute f() on the main thread. +function on_main_thread(f) + @lock MAIN_THREAD_TASK_LOCK begin + put!(MAIN_THREAD_CHANNEL_INPUT, f) + take!(MAIN_THREAD_CHANNEL_OUTPUT) + end +end + + function init_context() CTX.is_embedded = hasproperty(Base.Main, :__PythonCall_libptr) @@ -240,6 +254,15 @@ function init_context() "Only Python 3.9+ is supported, this is Python $(CTX.version) at $(CTX.exe_path===missing ? "unknown location" : CTX.exe_path).", ) + main_thread_task = Task() do + while true + f = take!(MAIN_THREAD_CHANNEL_INPUT) + put!(MAIN_THREAD_CHANNEL_OUTPUT, f()) + end + end + set_task_tid!(main_thread_task, Threads.threadid()) + schedule(main_thread_task) + @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version return @@ -260,3 +283,26 @@ const PYTHONCALL_PKGID = Base.PkgId(PYTHONCALL_UUID, "PythonCall") const PYCALL_UUID = Base.UUID("438e738f-606a-5dbb-bf0a-cddfbfd45ab0") const PYCALL_PKGID = Base.PkgId(PYCALL_UUID, "PyCall") + + +# taken from StableTasks.jl, itself taken from Dagger.jl +function set_task_tid!(task::Task, tid::Integer) + task.sticky = true + ctr = 0 + while true + ret = ccall(:jl_set_task_tid, Cint, (Any, Cint), task, tid-1) + if ret == 1 + break + elseif ret == 0 + yield() + else + error("Unexpected retcode from jl_set_task_tid: $ret") + end + ctr += 1 + if ctr > 10 + @warn "Setting task TID to $tid failed, giving up!" + return + end + end + @assert Threads.threadid(task) == tid "jl_set_task_tid failed!" +end diff --git a/src/Core/Py.jl b/src/Core/Py.jl index 9faea130..2074e303 100644 --- a/src/Core/Py.jl +++ b/src/Core/Py.jl @@ -257,25 +257,29 @@ Base.setproperty!(x::Py, k::Symbol, v) = pysetattr(x, string(k), v) Base.setproperty!(x::Py, k::String, v) = pysetattr(x, k, v) function Base.propertynames(x::Py, private::Bool = false) - # this follows the logic of rlcompleter.py - function classmembers(c) - r = pydir(c) - if pyhasattr(c, "__bases__") - for b in c.__bases__ - r = pyiadd(r, classmembers(b)) + properties = C.on_main_thread() do + # this follows the logic of rlcompleter.py + function classmembers(c) + r = pydir(c) + if pyhasattr(c, "__bases__") + for b in c.__bases__ + r = pyiadd(r, classmembers(b)) + end end + return r end - return r - end - words = pyset(pydir(x)) - words.discard("__builtins__") - if pyhasattr(x, "__class__") - words.add("__class__") - words.update(classmembers(x.__class__)) - end - words = map(pystr_asstring, words) + + words = pyset(pydir(x::Py)) + words.discard("__builtins__") + if pyhasattr(x, "__class__") + words.add("__class__") + words.update(classmembers(x.__class__)) + end + map(pystr_asstring, words) + end::Vector{String} # explicit type since on_main_thread() is type-unstable + # private || filter!(w->!startswith(w, "_"), words) - map(Symbol, words) + map(Symbol, properties) end Base.Bool(x::Py) = pytruth(x) diff --git a/test/Core.jl b/test/Core.jl index 32b4d59f..ef3034e4 100644 --- a/test/Core.jl +++ b/test/Core.jl @@ -827,3 +827,12 @@ end @test !isdir(tname) end end + +@testitem "propertynames" begin + x = pyint(7) + task = Threads.@spawn propertynames(x) + properties = propertynames(x) + @test :__init__ in properties + prop_task = fetch(task) + @test properties == prop_task +end From 992d48a1bf387084d5fd7dca2f886beed6b09c2f Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Thu, 25 Sep 2025 14:27:07 +0200 Subject: [PATCH 2/4] Refactor, document, make more robust and remove a lock --- src/C/context.jl | 91 +++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/src/C/context.jl b/src/C/context.jl index 2d550f31..e2b38b23 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -30,18 +30,59 @@ function _atpyexit() end -const MAIN_THREAD_TASK_LOCK = ReentrantLock() -const MAIN_THREAD_CHANNEL_INPUT = Channel(1) -const MAIN_THREAD_CHANNEL_OUTPUT = Channel(1) - -# Execute f() on the main thread. -function on_main_thread(f) - @lock MAIN_THREAD_TASK_LOCK begin - put!(MAIN_THREAD_CHANNEL_INPUT, f) - take!(MAIN_THREAD_CHANNEL_OUTPUT) +function setup_onfixedthread() + channel_input = Channel(1) + channel_output = Channel(1) + islaunched = Ref(false) # use Ref to avoid closure boxing of variable + function launch_worker(tid) + islaunched[] && error("Cannot launch more than once: call setup_onfixedthread again if need be.") + islaunched[] = true + worker_task = Task() do + while true + f = take!(channel_input) + put!(channel_output, f()) + end + end + # code adapted from set_task_tid! in StableTasks.jl, itself taken from Dagger.jl + worker_task.sticky = true + for _ in 1:100 + # try to fix the task id to tid, retrying up to 100 times + ret = ccall(:jl_set_task_tid, Cint, (Any, Cint), worker_task, tid-1) + if ret == 1 + break # success + elseif ret == 0 + yield() + else + error("Unexpected retcode from jl_set_task_tid: $ret") + end + end + if Threads.threadid(worker_task) != tid + error("Failed setting the thread ID to $tid.") + end + schedule(worker_task) + end + function onfixedthread(f) + put!(channel_input, f) + take!(channel_output) end + launch_worker, onfixedthread end +# launch_on_main_thread is used in init_context(), after which on_main_thread becomes usable +const launch_on_main_thread, on_main_thread = setup_onfixedthread() + +""" + on_main_thread(f) + +Execute `f()` on the main thread. + +!!! warning + The value returned by `on_main_thread(f)` cannot be type-inferred by the compiler: + if necessary, use explicit type annotations such as `on_main_thread(f)::T`, where `T` is + the expected return type. +""" +on_main_thread + function init_context() @@ -254,14 +295,7 @@ function init_context() "Only Python 3.9+ is supported, this is Python $(CTX.version) at $(CTX.exe_path===missing ? "unknown location" : CTX.exe_path).", ) - main_thread_task = Task() do - while true - f = take!(MAIN_THREAD_CHANNEL_INPUT) - put!(MAIN_THREAD_CHANNEL_OUTPUT, f()) - end - end - set_task_tid!(main_thread_task, Threads.threadid()) - schedule(main_thread_task) + launch_on_main_thread(Threads.threadid()) # makes on_main_thread usable @debug "Initialized PythonCall.jl" CTX.is_embedded CTX.is_initialized CTX.exe_path CTX.lib_path CTX.lib_ptr CTX.pyprogname CTX.pyhome CTX.version @@ -283,26 +317,3 @@ const PYTHONCALL_PKGID = Base.PkgId(PYTHONCALL_UUID, "PythonCall") const PYCALL_UUID = Base.UUID("438e738f-606a-5dbb-bf0a-cddfbfd45ab0") const PYCALL_PKGID = Base.PkgId(PYCALL_UUID, "PyCall") - - -# taken from StableTasks.jl, itself taken from Dagger.jl -function set_task_tid!(task::Task, tid::Integer) - task.sticky = true - ctr = 0 - while true - ret = ccall(:jl_set_task_tid, Cint, (Any, Cint), task, tid-1) - if ret == 1 - break - elseif ret == 0 - yield() - else - error("Unexpected retcode from jl_set_task_tid: $ret") - end - ctr += 1 - if ctr > 10 - @warn "Setting task TID to $tid failed, giving up!" - return - end - end - @assert Threads.threadid(task) == tid "jl_set_task_tid failed!" -end From 30f061262671beda3cb9d28e25ec4a7386b3c5ff Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Thu, 25 Sep 2025 15:40:16 +0200 Subject: [PATCH 3/4] Make setup_onfixedthread robust against errors --- src/C/context.jl | 22 ++++++++++++++++++++-- test/Core.jl | 10 ++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/C/context.jl b/src/C/context.jl index e2b38b23..3722895d 100644 --- a/src/C/context.jl +++ b/src/C/context.jl @@ -40,7 +40,15 @@ function setup_onfixedthread() worker_task = Task() do while true f = take!(channel_input) - put!(channel_output, f()) + ret = try + Some(invokelatest(f)) + # invokelatest is necessary for development and interactive use. + # Otherwise, only a method f defined in a world prior to the call of + # launch_worker would work. + catch e + e, catch_backtrace() + end + put!(channel_output, ret) end end # code adapted from set_task_tid! in StableTasks.jl, itself taken from Dagger.jl @@ -63,7 +71,17 @@ function setup_onfixedthread() end function onfixedthread(f) put!(channel_input, f) - take!(channel_output) + ret = take!(channel_output) + if ret isa Tuple + e, backtrace = ret + printstyled(stderr, "ERROR: "; color=:red, bold=true) + showerror(stderr, e) + Base.show_backtrace(stderr, backtrace) + println(stderr) + throw(e) # the stacktrace of the actual error is printed above + else + something(ret) + end end launch_worker, onfixedthread end diff --git a/test/Core.jl b/test/Core.jl index ef3034e4..ce53ae9a 100644 --- a/test/Core.jl +++ b/test/Core.jl @@ -836,3 +836,13 @@ end prop_task = fetch(task) @test properties == prop_task end + +@testitem "on_main_thread" begin + task = Threads.@spawn PythonCall.C.on_main_thread() do; Threads.threadid(); end + @test fetch(task) == 1 + @test_throws DivideError redirect_stderr(devnull) do + PythonCall.C.on_main_thread() do + throw(DivideError()) + end + end +end From 8a86e37cf5b041217b948d747dce0de64bc67820 Mon Sep 17 00:00:00 2001 From: Lionel Zoubritzky Date: Thu, 25 Sep 2025 16:11:28 +0200 Subject: [PATCH 4/4] Make on_main_thread test more reliable --- test/Core.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/Core.jl b/test/Core.jl index ce53ae9a..2379b0b0 100644 --- a/test/Core.jl +++ b/test/Core.jl @@ -838,8 +838,9 @@ end end @testitem "on_main_thread" begin - task = Threads.@spawn PythonCall.C.on_main_thread() do; Threads.threadid(); end - @test fetch(task) == 1 + refid = PythonCall.C.on_main_thread() do; Threads.threadid(); end + tasks = [Threads.@spawn(PythonCall.C.on_main_thread() do; Threads.threadid(); end) for _ in 1:20] + @test all(t -> fetch(t) == refid, tasks) @test_throws DivideError redirect_stderr(devnull) do PythonCall.C.on_main_thread() do throw(DivideError())