update to JuliaSyntax 1.0 (#661)

* update to JuliaSyntax 1.0

* update to JuliaSyntax@1

* fix

* Adjust to `nothing` representing no children instead of an empty vector

* Import Core: Const to avoid having to qualify Core.Const

* Adjust handling of infix operators

JuliaSyntax now emits `op=` nodes,
with functions as `Identifier` nodes,
which are no longer detectable with
`is_operator` for e.g. `+` in `a += b`

* Fix remaining test failures

* Fix UInt/Int discrepancy

* More fixes for Cthulhu

* Don't attempt to retrieve definition for kwcall from base/boot.jl

* Fixes for 32-bit

* Minor comment fix

* Reapply fix for 32-bit

* Bump versions for Cthulhu and TypedSyntax

* Try testing on 1.12.0-rc2

* Try on 1.12.0-rc1

* Revert back to testing SnoopCompile on 1.12-nightly

---------

Co-authored-by: serenity4 <[email protected]>

Author: oscardssmith

Project.toml

@@ -1,14 +1,14 @@
 authors = ["Valentin Churavy <[email protected]> and contributors"]
 name = "Cthulhu"
 uuid = "f68482b8-f384-11e8-15f7-abe071a5a75f"
-version = "2.17.6"
+version = "2.17.7"
 
 [compat]
 CodeTracking = "0.5, 1, 2"
 Compiler = "0.1"
 FoldingTrees = "1"
 InteractiveUtils = "1.9"
-JuliaSyntax = "0.4"
+JuliaSyntax = "1"
 PrecompileTools = "1"
 Preferences = "1"
 REPL = "1.9"

TypedSyntax/Project.toml

@@ -1,15 +1,15 @@
 name = "TypedSyntax"
 uuid = "d265eb64-f81a-44ad-a842-4247ee1503de"
 authors = ["Tim Holy <[email protected]> and contributors"]
-version = "1.5.0"
+version = "1.5.1"
 
 [deps]
 CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"
 JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4"
 
 [compat]
 CodeTracking = "1.3, 2"
-JuliaSyntax = "0.4"
+JuliaSyntax = "1"
 julia = "1.12"
 
 [extras]

TypedSyntax/src/TypedSyntax.jl

@@ -2,7 +2,7 @@ module TypedSyntax
 
 using Core: CodeInfo, MethodInstance, SlotNumber, SSAValue
 using JuliaSyntax: JuliaSyntax, AbstractSyntaxData, SyntaxData, SyntaxNode, GreenNode, AbstractSyntaxNode, SyntaxHead, SourceFile,
-                   head, kind, child, children, haschildren, untokenize, first_byte, last_byte, source_line, source_location,
+                   head, kind, children, is_leaf, untokenize, first_byte, last_byte, source_line, source_location,
                    sourcetext, @K_str, @KSet_str, is_infix_op_call, is_prefix_op_call, is_prec_assignment, is_operator, is_literal
 using Base.Meta: isexpr
 using CodeTracking

TypedSyntax/src/node.jl

@@ -86,19 +86,22 @@ end
 
 # Recursive construction of the TypedSyntaxNode tree from the SyntaxNodeTree
 function addchildren!(tparent, parent, src::CodeInfo, node2ssa, symtyps, mappings)
-    if haschildren(parent) && tparent.children === nothing
-        tparent.children = TypedSyntaxNode[]
-    end
-    for child in children(parent)
-        tnode = TypedSyntaxNode(tparent, nothing, TypedSyntaxData(child.data::SyntaxData, src, gettyp(node2ssa, child, src)))
-        if tnode.typ === nothing && (#=is_literal(child) ||=# kind(child) == K"Identifier")
-            tnode.typ = get(symtyps, child, nothing)
+    if !is_leaf(parent)
+        if tparent.children === nothing
+            tparent.children = TypedSyntaxNode[]
+        end
+        for child in children(parent)
+            typ = gettyp(node2ssa, child, src)
+            tnode = TypedSyntaxNode(tparent, nothing, TypedSyntaxData(child.data::SyntaxData, src, typ))
+            if tnode.typ === nothing && (#=is_literal(child) ||=# kind(child) == K"Identifier")
+                tnode.typ = get(symtyps, child, nothing)
+            end
+            push!(tparent, tnode)
+            addchildren!(tnode, child, src, node2ssa, symtyps, mappings)
         end
-        push!(tparent, tnode)
-        addchildren!(tnode, child, src, node2ssa, symtyps, mappings)
     end
     # In `return f(args..)`, copy any types assigned to `f(args...)` up to the `[return]` node
-    if kind(tparent) == K"return" && haschildren(tparent)
+    if kind(tparent) == K"return" && !is_leaf(tparent)
         childs = children(tparent)
         tparent.typ = isempty(childs) ? Nothing : only(childs).typ
     end
@@ -121,33 +124,31 @@ function map_signature!(sig::TypedSyntaxNode, slotnames::Vector{Symbol}, slottyp
             nchildren = length(children(arg))
             nchildren == 1 && return nothing, defaultval
             @assert nchildren == 2
-            arg = child(arg, 1)
+            arg = arg[1]
         end
         if kind(arg) == K"$"
             return nothing, defaultval
         end
         if kind(arg) == K"."
             arg = last(children(arg))
-            if kind(arg) == K"inert"
-                @assert kind(only(children(arg))) == K"$"
-                return nothing, defaultval
+            if kind(arg) != K"Identifier"
+                @assert kind(arg) in KSet"quote $"
+                arg = only(children(arg))
             end
-            @assert kind(arg) == K"quote"
-            arg = only(children(arg))
         end
         if kind(arg) == K"curly"
             arg = first(children(arg))
         end
         if kind(arg) == K"var"
-            arg = child(arg, 1)
+            arg = arg[1]
         end
         kind(arg) == K"tuple" && return nothing, defaultval     # FIXME? see extrema2 test
         @assert kind(arg) == K"Identifier" || is_operator(arg)
         return arg, defaultval
     end
 
     while kind(sig) ∈ KSet"where ::"   # handle MyType{T}(args...) and return-type annotations
-        sig = child(sig, 1)
+        sig = sig[1]
     end
     @assert kind(sig) == K"call"
     # First, match named args, since those have to be unique
@@ -241,14 +242,14 @@ function map_signature!(sig::TypedSyntaxNode, slotnames::Vector{Symbol}, slottyp
         end
         idx == 0 && continue
         if kind(arg) == K"::" && length(children(arg)) == 2
-            arg = child(arg, 1)
+            arg = arg[1]
         end
         arg.typ = slottypes[idx]
     end
 
     # It's annoying to print the signature as `foo::typeof(foo)(a::Int)`
     # Strip the type annotation from the function name
-    arg, _ = argidentifier(child(sig, 1))
+    arg, _ = argidentifier(sig[1])
     if arg !== nothing
         arg.typ = nothing
     end
@@ -264,8 +265,8 @@ function striparg(arg)
             arg = only(children(arg))
         end
         if kind(arg) == K"="
-            defaultval = child(arg, 2)
-            arg = child(arg, 1)
+            defaultval = arg[2]
+            arg = arg[1]
         end
         if kind(arg) == K"macrocall"
             arg = last(children(arg))    # FIXME: is the variable always the final argument?
@@ -351,10 +352,10 @@ end
 
 function is_function_def(node)  # this is not `Base.is_function_def`
     kind(node) == K"function" && return true
-    if kind(node) == K"=" && length(children(node)) >= 1
-        sig = child(node, 1)
+    if kind(node) == K"=" && !is_leaf(node)
+        sig = node[1]
         while(kind(sig) ∈ KSet"where ::")   # allow MyType{T}(args...) and return-type annotations
-            sig = child(sig, 1)
+            sig = sig[1]
         end
         kind(sig) == K"call" && return true
     end
@@ -366,7 +367,7 @@ function get_function_def(rootnode)
     while kind(rootnode) ∈ KSet"macrocall global local const"
         idx = findlast(node -> is_function_def(node) || kind(node) == K"macrocall", children(rootnode))
         idx === nothing && break
-        rootnode = child(rootnode, idx)
+        rootnode = rootnode[idx]
     end
     return rootnode
 end
@@ -375,7 +376,7 @@ function num_positional_args(tsn::AbstractSyntaxNode)
     TypedSyntax.is_function_def(tsn) || return 0
     sig, _ = children(tsn)
     while kind(sig) ∈ KSet"where ::"
-        sig = child(sig, 1)
+        sig = sig[1]
     end
     @assert kind(sig) == K"call"
     for (i, node) in enumerate(children(sig))
@@ -392,7 +393,7 @@ function collect_symbol_nodes(rootnode)
     rootnode = get_function_def(rootnode)
     is_function_def(rootnode) || error("expected function definition, got ", sourcetext(rootnode))
     symlocs = Dict{Any,Vector{typeof(rootnode)}}()
-    return collect_symbol_nodes!(symlocs, child(rootnode, 2))
+    return collect_symbol_nodes!(symlocs, rootnode[2])
 end
 
 function collect_symbol_nodes!(symlocs::AbstractDict, node)
@@ -409,10 +410,10 @@ function collect_symbol_nodes!(symlocs::AbstractDict, node)
         locs = get!(Vector{typeof(node)}, symlocs, node.val)
         push!(locs, node)
     end
-    if haschildren(node)
+    if !is_leaf(node)
         if kind(node) == K"do"
             # process only `g(args...)` in `g(args...) do ... end`
-            collect_symbol_nodes!(symlocs, child(node, 1))
+            collect_symbol_nodes!(symlocs, node[1])
         elseif kind(node) == K"generator"
             for c in Iterators.drop(children(node), 1)
                 collect_symbol_nodes!(symlocs, c)
@@ -560,7 +561,7 @@ function map_ssas_to_source(src::CodeInfo, mi::MethodInstance, rootnode::SyntaxN
                 argnode = p
                 p = argnode.parent::MaybeTypedSyntaxNode
             end
-            is_prec_assignment(p) && argnode == child(p, 1 + is_rhs)  # is it the correct side of an assignment?
+            is_prec_assignment(p) && argnode == p[1 + is_rhs]  # is it the correct side of an assignment?
         end
         return targets
     end
@@ -719,7 +720,7 @@ function map_ssas_to_source(src::CodeInfo, mi::MethodInstance, rootnode::SyntaxN
                         while !is_prec_assignment(lhsnode) && (lhsparent = lhsnode.parent; lhsparent !== nothing)
                             lhsnode = lhsparent
                         end
-                        lhsnode, found = find_identifier_or_tuplechild(child(lhsnode, 1), sym)
+                        lhsnode, found = find_identifier_or_tuplechild(lhsnode[1], sym)
                         if found
                             symtyps[lhsnode] = ssavaluetypes[i]
                         end
@@ -750,7 +751,7 @@ function map_ssas_to_source(src::CodeInfo, mi::MethodInstance, rootnode::SyntaxN
                                 for t in itr
                                     haskey(symtyps, t) && continue
                                     if skipped_parent(t) == node
-                                        is_prec_assignment(node) && t == child(node, 1) && continue
+                                        is_prec_assignment(node) && t == node[1] && continue
                                         symtyps[t] = if j > 0
                                             ssavaluetypes[j]
                                         elseif have_slottypes
@@ -861,6 +862,7 @@ function skipped_parent(node::SyntaxNode)
     pnode = node.parent
     pnode === nothing && return node
     ppnode = pnode.parent
+    kind(pnode) == K"in" && kind(ppnode) == K"iteration" && return ppnode
     if ppnode !== nothing && kind(pnode) ∈ KSet"... quote"   # might need to add more things here
         if kind(node) == K"Identifier" && kind(pnode) == K"quote" && kind(ppnode) == K"." && sourcetext(node) == "materialize"
             return ppnode.parent

TypedSyntax/src/show.jl

@@ -12,14 +12,14 @@ function JuliaSyntax._show_syntax_node(io, current_filename, node::TypedSyntaxNo
         posstr *= "$(lpad(first_byte(node),6)):$(rpad(last_byte(node),6))│"
     end
     val = node.val
-    nodestr = haschildren(node) ? "[$(untokenize(head(node)))]" :
+    nodestr = !is_leaf(node) ? "[$(untokenize(head(node)))]" :
     isa(val, Symbol) ? string(val) : repr(val)
     treestr = string(indent, nodestr)
     if node.typ !== nothing
         treestr = string(rpad(treestr, 40), "│$(node.typ)")
     end
     println(io, posstr, treestr)
-    if haschildren(node)
+    if !is_leaf(node)
         new_indent = indent*"  "
         for n in children(node)
             JuliaSyntax._show_syntax_node(io, current_filename, n, new_indent, show_byte_offsets)
@@ -36,7 +36,7 @@ function Base.printstyled(io::IO, rootnode::MaybeTypedSyntaxNode;
     rt = gettyp(rootnode)
     nd = with_linenumber ? ndigits_linenumbers(rootnode, idxend) : 0
     rootnode = get_function_def(rootnode)
-    position = first_byte(rootnode) - 1
+    position = Int(first_byte(rootnode) - 1)
     with_linenumber && print_linenumber(io, rootnode, position + 1, nd)
     if is_function_def(rootnode)
         # We're printing a MethodInstance
@@ -66,21 +66,18 @@ end
 function show_src_expr(io::IO, node::MaybeTypedSyntaxNode, position::Int, pre::String, pre2::String; type_annotations::Bool=true, iswarn::Bool=false, hide_type_stable::Bool=false, nd::Int)
     _lastidx = last_byte(node)
     position = catchup(io, node, position, nd)
-    if haschildren(node)
-        cs = children(node)
-        if !isempty(cs)   # `haschildren(node)` returns `true` as long as the node has the *capacity* to store children
-            position = catchup(io, first(children(node)), position, nd)
-        end
+    if !is_leaf(node)
+        position = catchup(io, first(children(node)), position, nd)
     end
     _print(io, pre, node.source, position)
-    for (i, child) in enumerate(children(node))
+    !is_leaf(node) && for (i, child) in enumerate(children(node))
         i == 2 && _print(io, pre2, node.source, position)
         cT = gettyp(child)
         ctype_annotate, cpre, cpre2, cpost = type_annotation_mode(child, cT; type_annotations, hide_type_stable)
         position = show_src_expr(io, child, position, cpre, cpre2; type_annotations, iswarn, hide_type_stable, nd)
         ctype_annotate && show_annotation(io, cT, cpost, node.source, position; iswarn)
     end
-    return Int(catchup(io, node, position, nd, _lastidx+1))
+    return catchup(io, node, position, nd, _lastidx+1)
 end
 
 # should we print a type-annotation?
@@ -106,14 +103,17 @@ function is_callfunc(node::MaybeTypedSyntaxNode, @nospecialize(T))
         thisnode = pnode
         pnode = pnode.parent
     end
-    if pnode !== nothing && kind(pnode) ∈ (K"call", K"curly") && ((is_infix_op_call(pnode) && is_operator(thisnode)) || thisnode === pnode.children[1])
-        if isa(T, Core.Const)
-            T = T.val
-        end
-        if isa(T, Type) || isa(T, Function)
-            T === Colon() && sourcetext(node) == ":" && return true
-            return is_type_transparent(node, T)
-        end
+    pnode === nothing && return false
+    is_in_infix_context = kind(pnode) == K"op=" || kind(pnode) == K"call" && is_infix_op_call(pnode)
+    is_caller_function = kind(pnode) == K"call" && thisnode === pnode.children[1]
+    is_parametrized_type = kind(pnode) == K"curly" && thisnode === pnode.children[1]
+    is_in_infix_context || is_caller_function || is_parametrized_type || return false
+    if isa(T, Core.Const)
+        T = T.val
+    end
+    if isa(T, Type) || isa(T, Function)
+        T === Colon() && sourcetext(node) == ":" && return true
+        return is_type_transparent(node, T)
     end
     return false
 end
@@ -173,15 +173,15 @@ print_linenumber(io::IO, ln::Int, nd::Int) = printstyled(io, lpad(ln, nd), " ";
 
 # Do any "overdue" printing, generating a line number if needed. Mostly, this catches whitespace.
 # Printing occurs over indexes from `position:stop-1`.
-function catchup(io::IO, node::MaybeTypedSyntaxNode, position::Int, nd::Int, stop = first_byte(node))
+function catchup(io::IO, node::MaybeTypedSyntaxNode, position::Int, nd::Int, stop = Int(first_byte(node)))
     if position + 1 < stop
         for (i, c) in pairs(node.source[position+1:stop-1])
             print(io, c)
             if c == '\n' && nd > 0
                 print_linenumber(io, node, position + i + 1, nd)
             end
         end
-        position = stop - 1
+        position = Int(stop - 1)
     end
     return position
 end

TypedSyntax/test/exhaustive.jl

@@ -14,6 +14,7 @@ const goodmis = Core.MethodInstance[]
         m = mi.def::Method
         isdefined(m, :generator) && continue   # code_typed can't handle this
         m ∈ missingmethods && continue         # we tried before and couldn't find the source text
+        m.name === :kwcall && mi.specTypes.parameters[3] === Type{Union{}} && continue # skip kwcall definition in base/boot.jl
         cis = Base.code_typed_by_type(mi.specTypes; debuginfo=:source, optimize=false)
         if length(cis) == 1
             src, rt = cis[1]