Teach InteractiveUtils how to read `ncat` AST by BioTurboNick · Pull Request #45399 · JuliaLang/julia

BioTurboNick · 2022-05-20T17:33:01Z

Will resolve #43294

BioTurboNick · 2022-05-21T01:53:05Z

Well, that was a lot of work for an underwhelming result 😆

They all have the same entry point, so they all look like this:

julia> @code_lowered [3 4 ;;; 5 4]
CodeInfo(
1 ─ %1 = Core.tuple(dimsshape, row_first)
│   %2 = Core._apply_iterate(Base.iterate, Base._hvncat, %1, xs)
└──      return %2
)

The LLVM and assembly are more interesting

julia> @code_llvm [3 4 ;;; 5 4]
;  @ abstractarray.jl:2129 within `hvncat`
; Function Attrs: uwtable
define nonnull {}* @julia_hvncat_827([3 x i64]* nocapture nonnull readonly align 8 dereferenceable(24) %0, i8 zeroext %1, i64 signext %2, i64 signext %3, i64 signext %4, i64 signext %5) #0 {
top:
  %6 = alloca [7 x {}*], align 8
  %gcframe10 = alloca [7 x {}*], align 16
  %gcframe10.sub = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 0
  %.sub = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 0
  %7 = bitcast [7 x {}*]* %gcframe10 to i8*
  call void @llvm.memset.p0i8.i32(i8* nonnull align 16 dereferenceable(56) %7, i8 0, i32 56, i1 false)
  %8 = call {}*** inttoptr (i64 43440368 to {}*** ()*)() #4
; ┌ @ abstractarray.jl:2134 within `_hvncat`
   %9 = bitcast [7 x {}*]* %gcframe10 to i64*
   store i64 20, i64* %9, align 16
   %10 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 1
   %11 = bitcast {}** %10 to {}***
   %12 = load {}**, {}*** %8, align 8
   store {}** %12, {}*** %11, align 8
   %13 = bitcast {}*** %8 to {}***
   store {}** %gcframe10.sub, {}*** %13, align 8
   %ptls_field6 = getelementptr inbounds {}**, {}*** %8, i64 2305843009213693954
   %14 = bitcast {}*** %ptls_field6 to i8**
   %ptls_load78 = load i8*, i8** %14, align 8
   %15 = call noalias nonnull {}* @jl_gc_pool_alloc(i8* %ptls_load78, i32 1416, i32 32) #5
   %16 = bitcast {}* %15 to i64*
   %17 = getelementptr inbounds i64, i64* %16, i64 -1
   store atomic i64 289240400, i64* %17 unordered, align 8
   %18 = bitcast {}* %15 to i8*
   %19 = bitcast [3 x i64]* %0 to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 dereferenceable(24) %18, i8* nonnull align 8 dereferenceable(24) %19, i64 24, i1 false)
   %20 = and i8 %1, 1
   %.not = icmp eq i8 %20, 0
   %21 = select i1 %.not, {}* inttoptr (i64 286603920 to {}*), {}* inttoptr (i64 286531536 to {}*)
   %22 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 6
   store {}* %15, {}** %22, align 16
   %23 = call nonnull {}* @jl_box_int64(i64 signext %2)
   %24 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 5
   store {}* %23, {}** %24, align 8
   %25 = call nonnull {}* @jl_box_int64(i64 signext %3)
   %26 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 4
   store {}* %25, {}** %26, align 16
   %27 = call nonnull {}* @jl_box_int64(i64 signext %4)
   %28 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 3
   store {}* %27, {}** %28, align 8
   %29 = call nonnull {}* @jl_box_int64(i64 signext %5)
   %30 = getelementptr inbounds [7 x {}*], [7 x {}*]* %gcframe10, i64 0, i64 2
   store {}* %29, {}** %30, align 16
   store {}* inttoptr (i64 284363968 to {}*), {}** %.sub, align 8
   %31 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 1
   store {}* %15, {}** %31, align 8
   %32 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 2
   store {}* %21, {}** %32, align 8
   %33 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 3
   store {}* %23, {}** %33, align 8
   %34 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 4
   store {}* %25, {}** %34, align 8
   %35 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 5
   store {}* %27, {}** %35, align 8
   %36 = getelementptr inbounds [7 x {}*], [7 x {}*]* %6, i64 0, i64 6
   store {}* %29, {}** %36, align 8
   %37 = call nonnull {}* @jl_invoke({}* inttoptr (i64 352304192 to {}*), {}** nonnull %.sub, i32 7, {}* inttoptr (i64 206632560 to {}*))
   %38 = load {}*, {}** %10, align 8
   %39 = bitcast {}*** %8 to {}**
   store {}* %38, {}** %39, align 8
; └
  ret {}* %37
}

julia> @code_native [3 4 ;;; 5 4]
        .text
; ┌ @ abstractarray.jl:2129 within `hvncat`
        pushq   %rbp
        movq    %rsp, %rbp
        pushq   %r15
        pushq   %r14
        pushq   %r13
        pushq   %r12
        pushq   %rsi
        pushq   %rdi
        pushq   %rbx
        andq    $-32, %rsp
        subq    $192, %rsp
        movq    %r9, 56(%rsp)
        movq    %r8, %rsi
        movl    %edx, %ebx
        movq    %rcx, %r14
        vxorps  %xmm0, %xmm0, %xmm0
        vmovups %ymm0, 88(%rsp)
        vmovaps %ymm0, 64(%rsp)
        movl    $jl_get_pgcstack, %eax
        vzeroupper
        callq   *%rax
        movq    %rax, %r15
; │┌ @ abstractarray.jl:2134 within `_hvncat`
        movq    $20, 64(%rsp)
        movq    (%r15), %rax
        movq    %rax, 72(%rsp)
        leaq    64(%rsp), %rax
        movq    %rax, (%r15)
        movq    16(%r15), %rcx
        movabsq $jl_gc_pool_alloc, %rax
        movl    $1416, %edx                     # imm = 0x588
        movl    $32, %r8d
        callq   *%rax
        movq    %rax, %rdi
        movq    $289240400, -8(%rdi)            # imm = 0x113D7550
        movq    16(%r14), %rax
        movq    %rax, 16(%rdi)
        vmovups (%r14), %xmm0
        vmovups %xmm0, (%rdi)
        testb   $1, %bl
        movl    $286603920, %eax                # imm = 0x11153A90
        movl    $286531536, %r12d               # imm = 0x11141FD0
        cmoveq  %rax, %r12
        movq    %rdi, 112(%rsp)
        movabsq $jl_box_int64, %r13
        movq    %rsi, %rcx
        callq   *%r13
        movq    %rax, %r14
        movq    %r14, 104(%rsp)
        movq    56(%rsp), %rcx
        callq   *%r13
        movq    %rax, %rsi
        movq    %rsi, 96(%rsp)
        movq    48(%rbp), %rcx
        callq   *%r13
        movq    %rax, %rbx
        movq    %rbx, 88(%rsp)
        movq    56(%rbp), %rcx
        callq   *%r13
        movq    %rax, 80(%rsp)
        movq    $284363968, 128(%rsp)           # imm = 0x10F30CC0
        movq    %rdi, 136(%rsp)
        movq    %r12, 144(%rsp)
        movq    %r14, 152(%rsp)
        movq    %rsi, 160(%rsp)
        movq    %rbx, 168(%rsp)
        movq    %rax, 176(%rsp)
        movabsq $jl_invoke, %rax
        leaq    128(%rsp), %rdx
        movl    $352304192, %ecx                # imm = 0x14FFBC40
        movl    $206632560, %r9d                # imm = 0xC50F670
        movl    $7, %r8d
        callq   *%rax
        movq    72(%rsp), %rcx
        movq    %rcx, (%r15)
; │└
        leaq    -56(%rbp), %rsp
        popq    %rbx
        popq    %rdi
        popq    %rsi
        popq    %r12
        popq    %r13
        popq    %r14
        popq    %r15
        popq    %rbp
        retq
        nopl    (%rax,%rax)
; └

BioTurboNick · 2024-02-06T18:32:01Z

Might be good to get this out of the way and into 1.11, if there are no objections. @KristofferC

BioTurboNick · 2025-03-12T21:00:47Z

This should probably make it in at some point, right?

PallHaraldsson · 2025-10-15T13:55:40Z

Yes, merge it? Or are test failures real (and making this overlooked, rerun them?)? It seems like a false alarm on apple but not sure about the other, then REPL related (only on 32-bit, strange).

BioTurboNick · 2025-10-15T15:42:02Z

They both seem to be timeouts? The REPL error in the Linux 32-bit runner also occurs in 32-bit Windows CI but doesn't make it fail, so I think it's unrelated. I'll see if it's a known issue.

BioTurboNick · 2025-10-15T15:49:20Z

It's happening in other recent PRs: https://buildkite.com/julialang/julia-master/builds/51387/steps/canvas?sid=0199e85c-6990-43c3-bed9-f157220cbb02

adienes · 2025-10-15T16:09:26Z

I get this:

julia> @code_lowered [3 4 ;;; 5 4]
ERROR: LoadError: UndefVarError: `typesof` not defined in `InteractiveUtils`
Suggestion: check for spelling errors or missing imports.
Stacktrace:
 [1] gen_call_with_extracted_types(__module__::Module, fcn::Symbol, ex0::Expr, kws::Vector{…}; is_source_reflection::Bool, supports_binding_reflection::Bool, use_signature_tuple::Bool)
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:593
 [2] gen_call_with_extracted_types
   @ ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:386 [inlined]
 [3] #gen_call_with_extracted_types_and_kwargs#75
   @ ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:645 [inlined]
 [4] kwcall(::@NamedTuple{…}, ::typeof(InteractiveUtils.gen_call_with_extracted_types_and_kwargs), __module__::Module, fcn::Symbol, ex0::Tuple{…})
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:631
 [5] var"@code_lowered"(__source__::LineNumberNode, __module__::Module, ex0::Vararg{Any})
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:673
 [6] fl_lower
   @ ./flfrontend.jl:24 [inlined]
 [7] fl_lower(ex::Expr, mod::Module, filename::Ptr{UInt8}, lineno::UInt64)
   @ Base ./flfrontend.jl:23
in expression starting at REPL[2]:1
Some type information was truncated. Use `show(err)` to see complete types.

would it be possible to add a test case or two? maybe with ;; in different positions & combinations, mixing in spaces / commas, etc.

BioTurboNick · 2025-10-15T16:57:23Z

I get this:

julia> @code_lowered [3 4 ;;; 5 4]
ERROR: LoadError: UndefVarError: `typesof` not defined in `InteractiveUtils`
Suggestion: check for spelling errors or missing imports.
Stacktrace:
 [1] gen_call_with_extracted_types(__module__::Module, fcn::Symbol, ex0::Expr, kws::Vector{…}; is_source_reflection::Bool, supports_binding_reflection::Bool, use_signature_tuple::Bool)
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:593
 [2] gen_call_with_extracted_types
   @ ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:386 [inlined]
 [3] #gen_call_with_extracted_types_and_kwargs#75
   @ ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:645 [inlined]
 [4] kwcall(::@NamedTuple{…}, ::typeof(InteractiveUtils.gen_call_with_extracted_types_and_kwargs), __module__::Module, fcn::Symbol, ex0::Tuple{…})
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:631
 [5] var"@code_lowered"(__source__::LineNumberNode, __module__::Module, ex0::Vararg{Any})
   @ InteractiveUtils ~/.julia/juliaup/julia-pr45399/share/julia/stdlib/v1.13/InteractiveUtils/src/macros.jl:673
 [6] fl_lower
   @ ./flfrontend.jl:24 [inlined]
 [7] fl_lower(ex::Expr, mod::Module, filename::Ptr{UInt8}, lineno::UInt64)
   @ Base ./flfrontend.jl:23
in expression starting at REPL[2]:1
Some type information was truncated. Use `show(err)` to see complete types.

would it be possible to add a test case or two? maybe with ;; in different positions & combinations, mixing in spaces / commas, etc.

Seems that this commit in master, but not 1.12, broke it: c3e7b1b

I'll have to adjust.

BioTurboNick · 2025-11-21T04:33:11Z

I don't understand why test x86_64-linux-gnuassertrr keeps failing.

PallHaraldsson · 2025-11-21T13:04:31Z

I'm guessing most likely a false alarm (so just finally merge this?); maybe caused by this:

errorshow (10) | started at 2025-11-20T16:12:39.526 on pid 722
..
From worker 10: Warning: detected a stack overflow; program state may be corrupted, so further execution might be unreliable.
From worker 10: Warning: detected a stack overflow; program state may be corrupted, so further execution might be unreliable.

..
[more effects]
..
boundscheck (2) | failed at 2025-11-20T16:17:02.168
ProcessExitedException(2)

I'm pretty sure this is also a false alarm:

No test results found from test_x86_64-linux-gnuassertrr
This could mean:

The test step hasn't completed successfully yet
The test step failed and produced no results
This upload job was triggered manually before test completion
You can re-run this job manually once test results are available.
🚨 Error: The command exited with status 1

BioTurboNick · 2025-11-24T16:52:57Z

Thanks, ~~guess I'll keep updating the branch until CI passes and it gets merged.~~

BioTurboNick · 2025-11-24T16:54:05Z

Oh, I was going to add tests first.

BioTurboNick · 2025-12-12T21:54:56Z

It's ready; Windows CI failure in Profile is surely unrelated.

adienes · 2026-02-06T14:18:05Z

it doesn't seem that extract_elements here handles the leaves (aka actual items in the concatenation expression) and will always return [] or nothing, resulting in issues like these examples:

julia> @macroexpand @which [1;;;]
:(InteractiveUtils.which((InteractiveUtils.make_tuple_type)(InteractiveUtils.Any[Core.Typeof(hvncat), Core.Typeof(3), Core.Typeof(nothing)]); ))

julia> @macroexpand @which [rand(2);;;]
:(InteractiveUtils.which((InteractiveUtils.make_tuple_type)(InteractiveUtils.Any[Core.Typeof(hvncat), Core.Typeof(3), Core.Typeof(Any[])]); ))

so note how in the first, we passed a literal 1 but the macro sees Core.Typeof(nothing) and in the second we passed rand(2) (could be any function call) and the macro sees Core.Typeof(Any[])

two other general notes:

I think trying to recreate the lowering logic in a macro is going to be hard to get right, but more importantly it's going to be extremely hard to maintain, since if that lowering ever changes this macro would have to change in tandem. maybe instead, this PR could inspect the results of Meta.@lower [1;;;] directly? I know that's a bit atypical but it seems more maintainable
ideally, the tests would be more picky than just checking that the head is :hvncat and actually test that we hit the right signature

BioTurboNick · 2026-02-06T20:36:15Z

Thank you for finding those issues. I take your point about maintainability, but I got it working. Fixed extract_element code, then got frustrated with get_shape and tried out Claude Code on it; with proper description of the problem and examples, then having it make it more concise, it actually worked. Then I added test cases by hand. I don't love how I'm diving deep into an Expr object to do it though.

adienes · 2026-02-07T15:31:56Z

the tests are better, thanks. but I think the else return [] in extract_elements is still not correct. this will not observe the result of any expression in a concatenation, e.g. @which [string(1) string(2)] will just send you to the generic fallback even if some String specialization existed

BioTurboNick · 2026-02-07T22:47:03Z

Ah, got it. Thanks!

@BioTurboNick

Became an issue in #45399 (cc @BioTurboNick). - Port #58940 and steal most tests (we should figure out how to just run the existing tests soon) - Add a version of `@__FUNCTION__` that preserves provenance - Catch misuse in the validator - Fix a bug where generated functions weren't being counted as functions

@BioTurboNick

…#61150) Became an issue in JuliaLang#45399 (cc @BioTurboNick). - Port JuliaLang#58940 and steal most tests (we should figure out how to just run the existing tests soon) - Add a version of `@__FUNCTION__` that preserves provenance - Catch misuse in the validator - Fix a bug where generated functions weren't being counted as functions

BioTurboNick · 2026-09-24T18:27:41Z

Should we get this in for 1.14?

adienes · 2026-09-24T18:41:04Z

I'm out of tokens but if ^(astra|fable)$ don't find anything else IIRC it was looking good

…s, unbalanced shapes

BioTurboNick · 2026-09-25T20:30:24Z

Good call - Opus 5.5 (Medium) found a couple of issues and a bug in the Scheme lowering of the original implementation (that was already fixed in JuliaLowering). So for consistency, I'm also matching JuliaLowering and also fixing Scheme here.

Then after committing in a fresh session I repeated the request to Fable 5.1, which found a couple of minor nits.

BioTurboNick marked this pull request as ready for review May 22, 2022 01:29

BioTurboNick changed the title ~~Draft: Teach InteractiveUtils how to read ncat AST~~ Teach InteractiveUtils how to read ncat AST May 22, 2022

adienes reviewed Oct 15, 2025

View reviewed changes

Comment thread stdlib/InteractiveUtils/src/macros.jl Outdated

adienes reviewed Oct 15, 2025

View reviewed changes

Comment thread stdlib/InteractiveUtils/src/macros.jl Outdated

BioTurboNick requested a review from adienes February 3, 2026 03:41

BioTurboNick added 11 commits February 6, 2026 11:38

Initial work

b414e5f

Shape variant

841d12c

Shape variant

a88a28c

Rest of the way.

ca2fdfa

Remove what I think is a stray debug block?

e4ed2c5

Attempt to switch to typesof_expr

fa866a0

fix

328bf75

Make extraction more robust, adopt gen_call

9a0464b

Add tests

caf28fa

whitespace

fb74704

Fix element extraction

4e03261

dims was reversed for balanced case

680384e

Don't lose other args

1ea3e0d

adienes reviewed Feb 8, 2026

View reviewed changes

Comment thread stdlib/InteractiveUtils/src/macros.jl Outdated

adienes reviewed Feb 8, 2026

View reviewed changes

Comment thread stdlib/InteractiveUtils/src/macros.jl Outdated

BioTurboNick added 7 commits February 7, 2026 19:49

Fix tests

8b51a30

whitespace

f14dc33

Fix row_first

08c568e

Reduce boxing

17678e1

Get around local function boxing issue.

f71accd

More boxing defense

2c065de

map -> broadcasts so function reference works

1713348

This comment was marked as resolved.

Sign in to view

Merge branch 'master' into hvncat-interactiveutils

0a83d2d

mlechu mentioned this pull request Feb 24, 2026

[JuliaLowering] Support new @__FUNCTION__/thisfunction #61150

Merged

Skip InteractiveUtils

4f0a821

mlechu reviewed Feb 25, 2026

View reviewed changes

Comment thread test/JuliaLowering_stdlibs.jl Outdated

BioTurboNick added 4 commits February 25, 2026 14:38

Merge branch 'master' into hvncat-interactiveutils

69ee895

Merge branch 'master' into hvncat-interactiveutils

c16b93b

Merge branch 'master' into hvncat-interactiveutils

efa1a3a

Merge branch 'master' into hvncat-interactiveutils

0415be2

BioTurboNick added 2 commits September 25, 2026 16:09

Fix Opus 5.5-identified issues: string elements, symbols and variable…

d97fe00

…s, unbalanced shapes

Fix Fable 5.1-detected nitpicks

93d37fc

Uh oh!

Conversation

BioTurboNick commented May 20, 2022

Uh oh!

BioTurboNick commented May 21, 2022 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

BioTurboNick commented Feb 6, 2024

Uh oh!

BioTurboNick commented Mar 12, 2025

Uh oh!

PallHaraldsson commented Oct 15, 2025

Uh oh!

BioTurboNick commented Oct 15, 2025

Uh oh!

BioTurboNick commented Oct 15, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

adienes commented Oct 15, 2025

Uh oh!

Uh oh!

Uh oh!

BioTurboNick commented Oct 15, 2025

Uh oh!

BioTurboNick commented Nov 21, 2025

Uh oh!

PallHaraldsson commented Nov 21, 2025

Uh oh!

BioTurboNick commented Nov 24, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

BioTurboNick commented Nov 24, 2025

Uh oh!

BioTurboNick commented Dec 12, 2025

Uh oh!

adienes commented Feb 6, 2026

Uh oh!

BioTurboNick commented Feb 6, 2026

Uh oh!

adienes commented Feb 7, 2026

Uh oh!

BioTurboNick commented Feb 7, 2026

Uh oh!

Uh oh!

Uh oh!

This comment was marked as resolved.

Uh oh!

BioTurboNick commented Sep 24, 2026

Uh oh!

adienes commented Sep 24, 2026

Uh oh!

BioTurboNick commented Sep 25, 2026

Uh oh!

This branch has not been deployed

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

5 participants

BioTurboNick commented May 21, 2022 •

edited

Loading

BioTurboNick commented Oct 15, 2025 •

edited

Loading

BioTurboNick commented Nov 24, 2025 •

edited

Loading