Split compiler integration to avoid duplicating Cthulhu when loading the compiler extension (#677)

* Initial implementation

* WIP: Overhaul internal architecture & user-facing API

* Bump major version

* Fix highlighter command naming

* Implement initial external AbstractInterpreter support

* Separate commands and their values

This way, commands merely register shortcuts
to execute callbacks, so they get simplified,
and at the same time we no longer have to worry
about keeping command "values" and configuration values
in sync.

* Properly implement way to conditionally disable commands

* Move quit/bookmark commands to beginning of action list

* Fix a few tests

* Adjust conditional Command printing

* `with_effects` -> `effects`, re-enable remark/effect/exception type annotations

The renaming was done to improve consistency with similar settings,
`remarks` and `exception_type`.

* Re-enable VSCode diagnostics & inlay types

* Add backspace shortcut to go back (ascend)

* Refactor bookmark functionality

* Remove disabled AbstractCursor interface

* Fix & clarify IRCode/CodeInfo interactions in LookupResult

`IRCode` is now used primarily for callsite detection and display.
`CodeInfo` may also be used for that purpose, if no `IRCode` is
available. The main requirement is that they are consistent with the
provided call infos.

`CodeInfo` is mainly used to extract slotnames, get more accurate
sources via TypedSyntax, and for module dumping for LLVM/native views.

* Minor interface refactors

* Move terminal utility functions to a separate file

* Start testing various AbstractProvider implementations

* Start adding docs

* Re-enable Compiler extension, test DefaultProvider with both
integrations

* Prevent terminal tests from hanging

* Fix test_terminal.jl

* FakeTerminal -> VirtualTerminal

* Revert minor printing changes

* Fix AbstractInterpreter tests

* Fix test_codeview

* Use unoptimized lookup if source is not available

* More fixes, add a few more tests

* Reimplement `descend_code_`typed/warntype

* Attempt to fix tests for 1.13

* Remove dead code

* Add ExternalProvider example

* Improve interface documentation

* exception_type -> exception_types

To match the plural used for `remarks`, `effects`, etc

* inline_cost -> inlining_costs

To improve consistency with `remarks`, `effects`, `exception_type`

* Update example `descend` usage in README

* Test fixes, avoid corrupting the global method table

* Remove `interruptexc`, add a few more docs

Now that we have backspace to ascend, we can have
'q' quit unconditionally and remove this option.
Furthermore, we now exit without having to handle an interrupt.

* Only reevaluate compiler-related functionality in CthulhuCompilerExt

* Fix tests

* Add more comments, make ir_to_src common

* Mention docstring of AbstractProvider in changelog

* Allow returning `nothing` after lookup to log an error

Author: serenity4

CHANGELOG.md

@@ -4,7 +4,7 @@
 
 ### Improvements
 
-- The way that packages integrate with Cthulhu to customize the behavior of code introspection has been redesigned (see https://github.com/JuliaDebug/Cthulhu.jl/pull/662 for more details).
+- The way that packages integrate with Cthulhu to customize the behavior of code introspection has been redesigned (https://github.com/JuliaDebug/Cthulhu.jl/pull/662 and https://github.com/JuliaDebug/Cthulhu.jl/pull/677). See the docstring of `Cthulhu.AbstractProvider` for more details.
 - A new UI command mapped to `DEL` (backspace, `'\x7f'`) now allows to go back (ascend) with a single key press.
 
 ### Breaking changes
@@ -17,4 +17,3 @@
 - The `with_effects` configuration option was renamed to `effects` for consistency with `remarks` and `exception_types`.
 - The `inline_cost` configuration option was renamed to `inlining_costs`, also for consistency reasons.
 - The `interruptexc` configuration option was removed. It used to control whether `q` exited (by throwing an `InterruptException`) or ascended, but now that backspace was added as a shortcut to ascend, we can now unconditionally exit with `q` (which actually matches its action description).
-

ext/CthulhuCompilerExt.jl

@@ -1,14 +1,32 @@
 module CthulhuCompilerExt
 
 using Compiler: Compiler as CC
-using Compiler.IRShow: IRShow
 using Cthulhu: Cthulhu
 
-function __init__()
-    Cthulhu.CTHULHU_MODULE[] = @__MODULE__
-    read_config!()
-end
+@static if CC.AbstractInterpreter !== Cthulhu.CC.AbstractInterpreter
+    using Compiler.IRShow: IRShow
+
+    using Accessors
+    using CodeTracking: CodeTracking, definition, whereis, maybe_fix_path
+    using InteractiveUtils
+    using UUIDs
+    using REPL: REPL, AbstractTerminal
+    using JuliaSyntax
+    using JuliaSyntax: SyntaxNode, AbstractSyntaxNode, child, children
+    using TypedSyntax
+    using WidthLimitedIO
+
+    using Core.IR
 
-include("../src/CthulhuBase.jl")
+    include("../src/CthulhuCompiler.jl")
+
+    function __init__()
+        Cthulhu.CompilerExt = @__MODULE__
+    end
+else
+    function __init__()
+        Cthulhu.CompilerExt = nothing
+    end
+end
 
 end

src/Cthulhu.jl

@@ -2,37 +2,61 @@ module Cthulhu
 
 export @descend, @descend_code_typed, @descend_code_warntype,
     descend, descend_code_typed, descend_code_warntype, ascend,
-    AbstractProvider
+    AbstractProvider,
+    is_compiler_loaded, is_compiler_extension_loaded, get_module_for_compiler_integration
 
 const CC = Base.Compiler
 const IRShow = Base.IRShow
 
-const CTHULHU_MODULE = Ref{Module}(@__MODULE__)
+using Accessors
+using CodeTracking: CodeTracking, definition, whereis, maybe_fix_path
+using InteractiveUtils
+using InteractiveUtils: is_expected_union
+using UUIDs
+using REPL: REPL, AbstractTerminal
+using JuliaSyntax: JuliaSyntax, children
+using TypedSyntax
+import WidthLimitedIO: TextWidthLimiter
+using Preferences
 
+using Core.IR
+using Base: default_tt, unwrap_unionall, mapany
+
+global CompilerExt::Union{Nothing, Module}
+
+is_compiler_extension_loaded() = isdefined(@__MODULE__, :CompilerExt)
 function is_compiler_loaded()
     pkgid = Base.PkgId(Base.UUID("807dbc54-b67e-4c79-8afb-eafe4df6f2e1"), "Compiler")
     return haskey(Base.loaded_modules, pkgid)
 end
-is_compiler_extension_loaded() = CTHULHU_MODULE[] !== @__MODULE__
-
-function resolve_module(Compiler::Module)
-    Compiler === Base.Compiler && return @__MODULE__
-    Compiler === CTHULHU_MODULE[].Compiler && return CTHULHU_MODULE[]
-    return resolve_module()
-end
-resolve_module() = CTHULHU_MODULE[]
-function resolve_module(@nospecialize(::T)) where {T}
-    mod = parentmodule(T)
-    nameof(mod) === :Compiler && return resolve_module(mod)
-    return resolve_module()
+function get_module_for_compiler_integration(; use_compiler_stdlib::Bool = true)
+    !use_compiler_stdlib && return @__MODULE__
+    is_compiler_extension_loaded() || error("The Cthulhu -> Compiler extension must be loaded first if `use_compiler_stdlib` is set to `true`")
+    return something(CompilerExt, @__MODULE__)
 end
 
+cached_exception_type(code::CodeInstance) = code.exctype
+get_mi(ci::CodeInstance) = CC.get_ci_mi(ci)
+get_mi(mi::MethodInstance) = mi
+
+include("config.jl")
+include("preferences.jl")
 __init__() = read_config!()
 
-include("CthulhuBase.jl")
+include("interface.jl")
+include("callsite.jl")
+include("state.jl")
+include("ui.jl")
+include("bookmark.jl")
+include("descend.jl")
+include("ascend.jl")
 include("backedges.jl")
 include("testing.jl")
 
+function ir_to_src end
+
+include("CthulhuCompiler.jl")
+
 """
     @descend
 
@@ -146,19 +170,8 @@ julia> descend() do
 [...]
 ```
 """
-function descend(@nospecialize(args...); interp=nothing,
-                                         provider=nothing,
-                                         @nospecialize(kwargs...))
-    if provider !== nothing
-        mod = resolve_module(provider)
-        mod.descend_with_error_handling(args...; provider, kwargs...)
-    elseif interp !== nothing
-        mod = resolve_module(interp)
-        mod.descend_with_error_handling(args...; interp, kwargs...)
-    else
-        mod = resolve_module()
-        mod.descend_with_error_handling(args...; kwargs...)
-    end
+function descend(@nospecialize(args...); @nospecialize(kwargs...))
+    descend_with_error_handling(args...; kwargs...)
     return nothing
 end
 
@@ -236,9 +249,7 @@ with the option to `descend` into intermediate calls.
 Keyword arguments `pagesize, dynamic, maxsize` are passed to `Cthulhu.FoldingTrees.TreeMenu`.
 Any remaining `kwargs` are passed to [`descend`](@ref).
 """
-function ascend(@nospecialize(args...); kwargs...)
-    CTHULHU_MODULE[].ascend_impl(args...; kwargs...)
-end
+ascend(@nospecialize(args...); kwargs...) = ascend_impl(args...; kwargs...)
 
 using PrecompileTools
 @setup_workload begin

src/CthulhuBase.jl renamed to src/CthulhuCompiler.jl

@@ -1,56 +1,29 @@
 Base.Experimental.@compiler_options compile=min optimize=1
 
-using Accessors
-using CodeTracking: CodeTracking, definition, whereis, maybe_fix_path
-using InteractiveUtils
-using UUIDs
-using REPL: REPL, AbstractTerminal
-using JuliaSyntax
-using JuliaSyntax: SyntaxNode, AbstractSyntaxNode, children, is_leaf
-using TypedSyntax
-using WidthLimitedIO
+import .Cthulhu: AbstractProvider, get_abstract_interpreter, get_inference_world, find_method_instance, generate_code_instance, get_override, lookup, find_caller_of, get_inlining_costs, show_parameters, get_ci, get_rt, get_pc_remarks, get_pc_effects, get_pc_excts, show_callsite, show_callinfo, print_callsite_info, cthulhu_source, cthulhu_typed, cthulhu_ast, cthulhu_llvm, cthulhu_native, find_callsites, ir_to_src
+using .Cthulhu: CthulhuState, CthulhuConfig, CallInfo, Callsite, cached_exception_type, get_mi
+
+using Base: isvarargtype, unwrapva, unwrap_unionall, mapany, get_world_counter
+using JuliaSyntax: JuliaSyntax, children, is_leaf
 
-using Core: MethodInstance, MethodMatch
-using Core.IR
 using .CC: AbstractInterpreter, CallMeta, ApplyCallInfo, CallInfo as CCCallInfo, ConstCallInfo,
     EFFECTS_TOTAL, Effects, IncrementalCompact, InferenceParams, InferenceResult,
     InferenceState, IRCode, LimitedAccuracy, MethodMatchInfo, MethodResultPure,
     NativeInterpreter, NoCallInfo, OptimizationParams, OptimizationState,
-    UnionSplitApplyCallInfo, UnionSplitInfo, WorldRange, WorldView, get_inference_world,
+    UnionSplitApplyCallInfo, UnionSplitInfo, WorldRange, WorldView,
     argextype, argtypes_to_type, compileable_specialization, ignorelimited, singleton_type,
-    specialize_method, sptypes_from_meth_instance, widenconst, method_table, findsup
-using Base: @constprop, default_tt, isvarargtype, unwrapva, unwrap_unionall, rewrap_unionall
-const mapany = Base.mapany
+    specialize_method, sptypes_from_meth_instance, widenconst, method_table, findsup,
+    cached_return_type
 
 const ArgTypes = Vector{Any}
 
-using Base: get_world_counter
-
-get_mi(ci::CodeInstance) = CC.get_ci_mi(ci)
-get_mi(mi::MethodInstance) = mi
-
-using Preferences
-include("config.jl")
-include("preferences.jl")
-
-include("interface.jl")
-include("callsite.jl")
-include("compiler.jl")
-include("state.jl")
-include("interpreter.jl")
-include("provider.jl")
-include("reflection.jl")
-include("ui.jl")
-include("codeview.jl")
-include("bookmark.jl")
-include("descend.jl")
-include("ascend.jl")
-
-resolve_module(::AbstractProvider) = @__MODULE__
-
-using .CC: cached_return_type
-
-cached_exception_type(code::CodeInstance) = code.exctype
+include("compiler/callsite.jl")
+include("compiler/interface.jl")
+include("compiler/lookup.jl")
+include("compiler/interpreter.jl")
+include("compiler/provider.jl")
+include("compiler/reflection.jl")
+include("compiler/codeview.jl")
 
 get_effects(codeinst::CodeInstance) = CC.decode_effects(codeinst.ipo_purity_bits)
 get_effects(codeinst::CodeInfo) = CC.decode_effects(codeinst.purity)

src/ascend.jl

@@ -1,6 +1,6 @@
 function ascend_impl(
         term, mi;
-        interp::AbstractInterpreter=NativeInterpreter(),
+        interp=Base.Compiler.NativeInterpreter(),
         provider::AbstractProvider=AbstractProvider(interp),
         pagesize::Int=10, dynamic::Bool=false, maxsize::Int=pagesize,
         menu_options=(; pagesize), kwargs...)

src/backedges.jl

@@ -18,7 +18,7 @@ function show_tuple_as_call(@nospecialize(highlighter), io::IO, name::Symbol, @n
     sig = (sig::DataType).parameters
 
     ft = sig[1]
-    uw = Base.unwrap_unionall(ft)
+    uw = unwrap_unionall(ft)
     if ft <: Function && isa(uw,DataType) && isempty(uw.parameters) &&
             isdefined(uw.name.module, get_fname(uw)) &&
             ft == typeof(getfield(uw.name.module, get_fname(uw)))
@@ -44,7 +44,7 @@ end
 
 function stripType(@nospecialize(typ))
     if isa(typ, UnionAll)
-        typ = Base.unwrap_unionall(typ)
+        typ = unwrap_unionall(typ)
     elseif isa(typ, TypeVar) || isa(typ, Union)
         return typ
     end

src/bookmark.jl

@@ -36,7 +36,7 @@ const BOOKMARKS = Bookmark[]
 function Base.show(io::IO, ::MIME"text/plain", bookmark::Bookmark; kwargs...)
     (; provider, ci) = bookmark
     state = CthulhuState(bookmark; kwargs...)
-    result = LookupResult(provider, ci, state.config.optimize)
+    result = lookup(provider, ci, state.config.optimize)
     world = get_inference_world(provider)
     if get(io, :typeinfo, Any) === Bookmark  # a hack to check if in Vector etc.
         info = EdgeCallInfo(ci, result.rt, Effects())
@@ -52,7 +52,7 @@ end
 function Base.code_typed(bookmark::Bookmark; kwargs...)
     (; provider, ci) = bookmark
     state = CthulhuState(bookmark; kwargs...)
-    result = LookupResult(provider, ci, state.config.optimize)
+    result = lookup(provider, ci, state.config.optimize)
     src = something(result.src, result.ir)::Union{CodeInfo, IRCode}
     return src => result.rt
 end
@@ -67,20 +67,20 @@ InteractiveUtils.code_native(bookmark::Bookmark; kwargs...) =
 function InteractiveUtils.code_warntype(io::IO, bookmark::Bookmark; kwargs...)
     (; provider, ci) = bookmark
     state = CthulhuState(bookmark; kwargs...)
-    result = LookupResult(provider, ci, state.config.optimize)
+    result = lookup(provider, ci, state.config.optimize)
     cthulhu_warntype(io, provider, state, result)
 end
 
 function InteractiveUtils.code_llvm(io::IO, bookmark::Bookmark; dump_module = false, raw = false, kwargs...)
     (; provider, ci) = bookmark
     state = CthulhuState(bookmark; kwargs...)
-    result = LookupResult(provider, ci, state.config.optimize)
+    result = lookup(provider, ci, state.config.optimize)
     cthulhu_llvm(io, provider, state, result; dump_module, raw)
 end
 
 function InteractiveUtils.code_native(io::IO, bookmark::Bookmark; dump_module = false, raw = false, kwargs...)
     (; provider, ci) = bookmark
     state = CthulhuState(bookmark; kwargs...)
-    result = LookupResult(provider, ci, state.config.optimize)
+    result = lookup(provider, ci, state.config.optimize)
     cthulhu_native(io, provider, state, result; dump_module, raw)
 end