From 6fe0e25fa07bccc8c37bc4e1805286b48cc143f3 Mon Sep 17 00:00:00 2001 From: TEC Date: Thu, 25 Dec 2025 09:24:12 +0800 Subject: [PATCH 01/23] Parameterise annotations by their value type Annotated strings held their annotation values as `Any`. Nothing about an `AnnotatedString{String}` said what its annotations held, so code that read or displayed one could not dispatch on it. Annotations now have a value type parameter: `AnnotatedString{S, V}`, and likewise `AnnotatedChar`, `Annotation`, `RegionAnnotation` and `AnnotatedIOBuffer`. A one-parameter form, such as `AnnotatedString{String}`, covers every `V`, and the one-parameter constructors infer it. - Construction infers `V` from the annotations given. Constructing from an existing annotated string or char copies its annotations, while `convert` returns it as it is. Adding annotations to one widens `V`, as `vcat` does. - `annotatedstring`, `*` and `replace` widen `V` to the union of the value types, up to three of them, and then to `Any`. `join` keeps a shared `V`, and otherwise gives `Any`. Values are not converted to a common type. - `annotate!` refuses a value outside `V` with an `ArgumentError`. Writing or replacing into a typed buffer converts the values to its `V`, and reading converts them to the requested type. - An annotated buffer merges adjacent runs only when their values are identical. Equal but distinct values keep their own runs. --- base/logging/ConsoleLogger.jl | 6 +- base/regex.jl | 10 +- base/strings/annotated.jl | 221 +++++++++++++++++++++------------ base/strings/annotated_io.jl | 86 +++++++------ base/strings/io.jl | 18 ++- contrib/generate_precompile.jl | 2 +- doc/src/manual/strings.md | 2 +- test/strings/annotated.jl | 132 ++++++++++++++++++-- 8 files changed, 329 insertions(+), 148 deletions(-) diff --git a/base/logging/ConsoleLogger.jl b/base/logging/ConsoleLogger.jl index cc4afa6b7b7bc..12f00f319758f 100644 --- a/base/logging/ConsoleLogger.jl +++ b/base/logging/ConsoleLogger.jl @@ -125,9 +125,9 @@ function handle_message(logger::ConsoleLogger, level::LogLevel, message, _module # split into lines. This is specialised to improve type inference, # and reduce the risk of resulting method invalidations. message = string(message) - msglines = if Base._isannotated(message) && !isempty(Base.annotations(message)::Vector{Base.RegionAnnotation}) - message = Base.AnnotatedString(String(message)::String, Base.annotations(message)::Vector{Base.RegionAnnotation}) - @NamedTuple{indent::Int, msg::Union{SubString{Base.AnnotatedString{String}}, SubString{String}}}[ + msglines = if Base._isannotated(message) + message = Base.AnnotatedString{String, Any}(String(message)::String, Base.annotations(message)) + @NamedTuple{indent::Int, msg::Union{SubString{Base.AnnotatedString{String, Any}}, SubString{String}}}[ (indent=0, msg=l) for l in split(chomp(message), '\n')] else [(indent=0, msg=l) for l in split( diff --git a/base/regex.jl b/base/regex.jl index a8bc21f9fc79f..37fa160999258 100644 --- a/base/regex.jl +++ b/base/regex.jl @@ -453,10 +453,10 @@ function match(re::Regex, str::DenseUTF8String, idx::Integer, return result end -function _annotatedmatch(m::RegexMatch{S}, str::AnnotatedString{S}) where {S<:AbstractString} - RegexMatch{AnnotatedString{S}}( +function _annotatedmatch(m::RegexMatch{S}, str::AnnotatedString{S, V}) where {S, V} + RegexMatch{AnnotatedString{S, V}}( (@inbounds raw_substring(str, m.match.offset + 1, m.match.ncodeunits)), - Union{Nothing,SubString{AnnotatedString{S}}}[ + Union{Nothing,SubString{AnnotatedString{S, V}}}[ if !isnothing(cap) (@inbounds raw_substring(str, cap.offset + 1, cap.ncodeunits)) end for cap in m.captures], @@ -739,8 +739,8 @@ struct RegexMatchIterator{S <: AbstractString} RegexMatchIterator(regex::Regex, string::AbstractString, ovr::Bool=false) = new{String}(regex, String(string), ovr) - RegexMatchIterator(regex::Regex, string::AnnotatedString, ovr::Bool=false) = - new{AnnotatedString{String}}(regex, AnnotatedString(String(string.string), string.annotations), ovr) + RegexMatchIterator(regex::Regex, string::AnnotatedString{S, V}, ovr::Bool=false) where {S, V} = + new{AnnotatedString{String, V}}(regex, AnnotatedString(String(string.string), string.annotations), ovr) end compile(itr::RegexMatchIterator) = (compile(itr.regex); itr) eltype(::Type{<:RegexMatchIterator}) = RegexMatch diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index a9653dc3b3356..962fd0a6b0163 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -1,10 +1,10 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -const Annotation = NamedTuple{(:label, :value), Tuple{Symbol, Any}} -const RegionAnnotation = NamedTuple{(:region, :label, :value), Tuple{UnitRange{Int}, Symbol, Any}} +const Annotation{V} = NamedTuple{(:label, :value), Tuple{Symbol, V}} +const RegionAnnotation{V} = NamedTuple{(:region, :label, :value), Tuple{UnitRange{Int}, Symbol, V}} """ - AnnotatedString{S <: AbstractString} <: AbstractString + AnnotatedString{S <: AbstractString, V} <: AbstractString A string with metadata, in the form of annotated regions. @@ -23,8 +23,8 @@ annotated with labeled values. The above diagram represents a `AnnotatedString` where three ranges have been annotated (labeled `A`, `B`, and `C`). Each annotation holds a label (`Symbol`) -and a value (`Any`). These three pieces of information are held as a -`$RegionAnnotation`. +and a value (`V`). These three pieces of information are held as a +`$(RegionAnnotation{TypeVar(:V)})`. Labels do not need to be unique, the same region can hold multiple annotations with the same label. @@ -46,10 +46,13 @@ See also [`AnnotatedChar`](@ref), [`annotatedstring`](@ref), # Constructors ```julia -AnnotatedString(s::S<:AbstractString) -> AnnotatedString{S} -AnnotatedString(s::S<:AbstractString, annotations::Vector{$RegionAnnotation}) +AnnotatedString(s::S<:AbstractString) -> AnnotatedString{S, Any} +AnnotatedString(s::S<:AbstractString, annotations::Vector{$(RegionAnnotation{TypeVar(:V)})}) -> AnnotatedString{S, V} +AnnotatedString{S, V}(s::S, annotations) -> AnnotatedString{S, V} ``` +Constructing from an existing `AnnotatedString` copies its annotations. + An AnnotatedString can also be created with [`annotatedstring`](@ref), which acts much like [`string`](@ref) but preserves any annotations present in the arguments. @@ -61,13 +64,13 @@ julia> AnnotatedString("this is an example annotated string", "this is an example annotated string" ``` """ -struct AnnotatedString{S <: AbstractString} <: AbstractString +struct AnnotatedString{S <: AbstractString, V} <: AbstractString string::S - annotations::Vector{RegionAnnotation} + annotations::Vector{RegionAnnotation{V}} end """ - AnnotatedChar{S <: AbstractChar} <: AbstractChar + AnnotatedChar{C <: AbstractChar, V} <: AbstractChar A Char with annotations. @@ -81,10 +84,13 @@ See also [`AnnotatedString`](@ref), [`annotatedstring`](@ref), `annotations`, # Constructors ```julia -AnnotatedChar(s::S) -> AnnotatedChar{S} -AnnotatedChar(s::S, annotations::Vector{$Annotation}) +AnnotatedChar(c::C<:AbstractChar) -> AnnotatedChar{C, Any} +AnnotatedChar(c::C<:AbstractChar, annotations::Vector{$(Annotation{TypeVar(:V)})}) -> AnnotatedChar{C, V} +AnnotatedChar{C, V}(c::C, annotations) -> AnnotatedChar{C, V} ``` +Constructing from an existing `AnnotatedChar` copies its annotations. + # Examples ```jldoctest; setup=:(using Base: AnnotatedChar) @@ -92,34 +98,54 @@ julia> AnnotatedChar('j', [(:label, 1)]) 'j': ASCII/Unicode U+006A (category Ll: Letter, lowercase) ``` """ -struct AnnotatedChar{C <: AbstractChar} <: AbstractChar +struct AnnotatedChar{C <: AbstractChar, V} <: AbstractChar char::C - annotations::Vector{Annotation} + annotations::Vector{Annotation{V}} end ## Constructors ## -# When called with overly-specialised arguments +function AnnotatedString{S}(s::S, annots) where {S <: AbstractString} + valtype(::Vector{RegionAnnotation{V}}) where {V} = V + valtype(::Vector) = Any + vannots = [(region = UnitRange{Int}(r), label = Symbol(l), value = v) for (r, l, v) in annots] + AnnotatedString{S, valtype(vannots)}(s, vannots) +end + +AnnotatedString(s::AbstractString, annots) = AnnotatedString{typeof(s)}(s, annots) + +AnnotatedString{S, V}(s::S) where {S, V} = AnnotatedString{S, V}(s, RegionAnnotation{V}[]) + +function AnnotatedChar{C}(c::C, annots) where {C <: AbstractChar} + valtype(::Vector{Annotation{V}}) where {V} = V + valtype(::Vector) = Any + vannots = [(label = Symbol(l), value = v) for (l, v) in annots] + AnnotatedChar{C, valtype(vannots)}(c, vannots) +end + +AnnotatedChar(c::AbstractChar, annots) = AnnotatedChar{typeof(c)}(c, annots) -AnnotatedString(s::AbstractString, annots::Vector) = - AnnotatedString(s, Vector{RegionAnnotation}(annots)) +# Copy, as the annotations are mutable -AnnotatedString(s::AbstractString, annots) = - AnnotatedString(s, collect(RegionAnnotation, annots)) +AnnotatedString{S, V}(s::AnnotatedString{S}) where {S, V} = + AnnotatedString{S, V}(s.string, Vector{RegionAnnotation{V}}(s.annotations)) +AnnotatedString(s::AnnotatedString{S, V}) where {S, V} = AnnotatedString{S, V}(s) -AnnotatedChar(c::AbstractChar, annots::Vector) = - AnnotatedChar(c, Vector{Annotation}(annots)) +AnnotatedChar{C, V}(c::AnnotatedChar{C}) where {C, V} = + AnnotatedChar{C, V}(c.char, Vector{Annotation{V}}(c.annotations)) +AnnotatedChar(c::AnnotatedChar{C, V}) where {C, V} = AnnotatedChar{C, V}(c) -AnnotatedChar(c::AbstractChar, annots) = - AnnotatedChar(c, collect(Annotation, annots)) +# Constructors to avoid recursive wrapping, which widen the value type as `vcat` does -# Constructors to avoid recursive wrapping +AnnotatedString(s::AnnotatedString{S, V}, annots::Vector{RegionAnnotation{T}}) where {S <: AbstractString, V, T} = + AnnotatedString(s.string, typed_vcat(RegionAnnotation{annot_promote_valtype(s, annots)}, s.annotations, annots)) -AnnotatedString(s::AnnotatedString, annots::Vector{RegionAnnotation}) = - AnnotatedString(s.string, vcat(s.annotations, annots)) +AnnotatedString(s::AnnotatedString, annots) = AnnotatedString(s, AnnotatedString(s.string, annots).annotations) -AnnotatedChar(c::AnnotatedChar, annots::Vector{Annotation}) = - AnnotatedChar(c.char, vcat(c.annotations, Vector{Annotation}(annots))) +AnnotatedChar(c::AnnotatedChar, annots::Vector{Annotation{T}}) where {T} = + AnnotatedChar(c.char, typed_vcat(Annotation{annot_promote_valtype(c, annots)}, c.annotations, annots)) + +AnnotatedChar(c::AnnotatedChar, annots) = AnnotatedChar(c, AnnotatedChar(c.char, annots).annotations) # To avoid pointless overhead String(s::AnnotatedString{String}) = s.string @@ -127,21 +153,25 @@ String(s::AnnotatedString{String}) = s.string ## Conversion/promotion ## convert(::Type{AnnotatedString}, s::AnnotatedString) = s -convert(::Type{AnnotatedString{S}}, s::S) where {S <: AbstractString} = - AnnotatedString(s, Vector{RegionAnnotation}()) convert(::Type{AnnotatedString}, s::S) where {S <: AbstractString} = convert(AnnotatedString{S}, s) +convert(::Type{AnnotatedString{S}}, s::S) where {S <: AbstractString} = + convert(AnnotatedString{S, Any}, s) +convert(::Type{AnnotatedString{S, V}}, s::S) where {S <: AbstractString, V} = + AnnotatedString{S, V}(s) + AnnotatedString(s::S) where {S <: AbstractString} = convert(AnnotatedString{S}, s) convert(::Type{AnnotatedChar}, c::AnnotatedChar) = c -convert(::Type{AnnotatedChar{C}}, c::C) where { C <: AbstractChar } = - AnnotatedChar{C}(c, Vector{Annotation}()) +convert(::Type{AnnotatedChar{C, V}}, c::C) where { C <: AbstractChar, V } = + AnnotatedChar{C, V}(c, Vector{Annotation{V}}()) convert(::Type{AnnotatedChar}, c::C) where { C <: AbstractChar } = - convert(AnnotatedChar{C}, c) + convert(AnnotatedChar{C, Any}, c) AnnotatedChar(c::AbstractChar) = convert(AnnotatedChar, c) AnnotatedChar(c::UInt32) = convert(AnnotatedChar, Char(c)) AnnotatedChar{C}(c::UInt32) where {C <: AbstractChar} = convert(AnnotatedChar, C(c)) +AnnotatedChar{C, V}(c::UInt32) where {C <: AbstractChar, V} = convert(AnnotatedChar{C, V}, C(c)) promote_rule(::Type{<:AnnotatedString}, ::Type{<:AbstractString}) = AnnotatedString @@ -154,6 +184,7 @@ codeunit(s::AnnotatedString, i::Integer) = codeunit(s.string, i) isvalid(s::AnnotatedString, i::Integer) = isvalid(s.string, i)::Bool @propagate_inbounds iterate(s::AnnotatedString, i::Integer=firstindex(s)) = if i <= lastindex(s.string); (s[i], nextind(s, i)) end +eltype(::Type{<:AnnotatedString{S, V}}) where {S, V} = AnnotatedChar{eltype(S), V} eltype(::Type{<:AnnotatedString{S}}) where {S} = AnnotatedChar{eltype(S)} firstindex(s::AnnotatedString) = firstindex(s.string) lastindex(s::AnnotatedString) = lastindex(s.string) @@ -189,10 +220,10 @@ function unannotate(s::SubString{<:AnnotatedString}) end -function getindex(s::AnnotatedString, i::Integer) +function getindex(s::AnnotatedString{S, V}, i::Integer) where {S, V} @boundscheck checkbounds(s, i) @inbounds if isvalid(s, i) - AnnotatedChar(s.string[i], Annotation[(; label, value) for (; label, value) in annotations(s, i)]) + AnnotatedChar(s.string[i], Annotation{V}[Annotation{V}((a.label, a.value)) for a in annotations(s, i)]) else string_index_err(s, i) end @@ -274,7 +305,8 @@ function annotatedstring(xs...) size = mapreduce(_str_sizehint, +, xs) buf = IOBuffer(sizehint=size) s = IOContext(buf, :color => true) - annotations = Vector{RegionAnnotation}() + V = annot_promote_valtype(xs...) + annotations = Vector{RegionAnnotation{V}}() for x in xs size = filesize(s.io) if x isa AnnotatedString @@ -305,20 +337,48 @@ function annotatedstring(xs...) AnnotatedString(str, annotations) end +annot_valtype(::Type{AnnotatedString{S, V}}) where {S, V} = V +annot_valtype(::Type{AnnotatedChar{C, V}}) where {C, V} = V +annot_valtype(::Type{SubString{S}}) where {S} = annot_valtype(S) +annot_valtype(::Type{Vector{RegionAnnotation{V}}}) where {V} = V +annot_valtype(::Type{Vector{Annotation{V}}}) where {V} = V +annot_valtype(::Type{Pair{P, V}}) where {P, V} = annot_valtype(V) +# An abstract type that may hold annotated values, such as the eltype of a mixed vector, gives +# `Any`. This uses subtyping because `typeintersect` does not fold at compile time. +annot_valtype(::Type{T}) where {T} = + if T === Union{} Union{} + elseif T isa Union Union{annot_valtype(T.a), annot_valtype(T.b)} + elseif T <: Union{AnnotatedString, AnnotatedChar, SubString{<:AnnotatedString}} || + AnnotatedString <: T || AnnotatedChar <: T || SubString{<:AnnotatedString} <: T Any + else Union{} end +annot_valtype(x) = annot_valtype(typeof(x)) + +# The union of the arguments' value types, widened to `Any` past 4 members, the most that +# inference splits. Without an annotated argument, `Any`, so the result can take annotations. +function annot_promote_valtype(x) + V = annot_valtype(x) + if V === Union{} Any else V end +end +function annot_promote_valtype(a, b, xs...) + @inline + union = Union{annot_valtype(a), annot_valtype(b), map(annot_valtype, xs)...} + if union === Union{} || unionlen(union) > 4 Any else union end +end + annotatedstring(s::AnnotatedString) = s annotatedstring(c::AnnotatedChar) = AnnotatedString(string(c.char), [(region=1:ncodeunits(c), annot...) for annot in c.annotations]) AnnotatedString(s::SubString{<:AnnotatedString}) = annotatedstring(s) +AnnotatedString{S, V}(s::SubString{<:AnnotatedString{S}}) where {S, V} = + AnnotatedString{S, V}(annotatedstring(s)) -function repeat(str::AnnotatedString, r::Integer) - r == 0 && return one(AnnotatedString) - r == 1 && return str +function repeat(str::AnnotatedString{S, V}, r::Integer) where {S, V} unannot = repeat(str.string, r) - annotations = Vector{RegionAnnotation}() + annotations = Vector{RegionAnnotation{V}}() len = ncodeunits(str) fullregion = firstindex(str):lastindex(str) - if isempty(str.annotations) + if r == 0 || isempty(str.annotations) elseif allequal(a -> a.region, str.annotations) && first(str.annotations).region == fullregion newfullregion = firstindex(unannot):lastindex(unannot) for annot in str.annotations @@ -361,48 +421,56 @@ reverse(s::SubString{<:AnnotatedString}) = reverse(AnnotatedString(s)) ## End AbstractString interface ## -function _annotate!(annlist::Vector{RegionAnnotation}, region::UnitRange{Int}, label::Symbol, @nospecialize(value::Any)) - if value === nothing +function _annotate!(annlist::Vector{RegionAnnotation{V}}, region::UnitRange{Int}, label::Symbol, value::Union{V, Nothing}) where {V} + if isnothing(value) deleteat!(annlist, findall(ann -> ann.region == region && ann.label === label, annlist)) else - push!(annlist, RegionAnnotation((; region, label, value))) + push!(annlist, RegionAnnotation{V}((; region, label, value))) end end +_annotate!(annlist::Vector{RegionAnnotation{V}}, region::UnitRange{Int}, label::Symbol, @nospecialize(value)) where {V} = + throw(ArgumentError(LazyString("cannot annotate with a ", typeof(value), " value, as the value type is ", V, + "; construct the string or buffer with a wider value type, such as Any"))) """ annotate!(str::AnnotatedString, [range::UnitRange{Int}], label::Symbol, value) annotate!(str::SubString{AnnotatedString}, [range::UnitRange{Int}], label::Symbol, value) Annotate a `range` of `str` (or the entire string) with a labeled value `(label, value)`. -To remove existing `label` annotations, use a value of `nothing`. +To remove existing `label` annotations, use a value of `nothing`. A `value` outside the +value type `V` of `str` raises an `ArgumentError`. The order in which annotations are applied to `str` is semantically meaningful, as described in [`AnnotatedString`](@ref). """ -annotate!(s::AnnotatedString, range::UnitRange{Int}, label::Symbol, @nospecialize(val::Any)) = +annotate!(s::AnnotatedString, range::UnitRange{Int}, label::Symbol, val) = (_annotate!(s.annotations, range, label, val); s) -annotate!(ss::AnnotatedString, label::Symbol, @nospecialize(val::Any)) = +annotate!(ss::AnnotatedString, label::Symbol, val) = annotate!(ss, firstindex(ss):lastindex(ss), label, val) -annotate!(s::SubString{<:AnnotatedString}, range::UnitRange{Int}, label::Symbol, @nospecialize(val::Any)) = +annotate!(s::SubString{<:AnnotatedString}, range::UnitRange{Int}, label::Symbol, val) = (annotate!(s.string, s.offset .+ (range), label, val); s) -annotate!(s::SubString{<:AnnotatedString}, label::Symbol, @nospecialize(val::Any)) = +annotate!(s::SubString{<:AnnotatedString}, label::Symbol, val) = (annotate!(s.string, s.offset .+ (1:s.ncodeunits), label, val); s) """ annotate!(char::AnnotatedChar, label::Symbol, value::Any) -Annotate `char` with the labeled value `(label, value)`. +Annotate `char` with the labeled value `(label, value)`. A `value` outside the value +type `V` of `char` raises an `ArgumentError`. """ -annotate!(c::AnnotatedChar, label::Symbol, @nospecialize(value::Any)) = - (push!(c.annotations, Annotation((; label, value))); c) +annotate!(c::AnnotatedChar{C, V}, label::Symbol, value::V) where {C, V} = + (push!(c.annotations, Annotation{V}((; label, value))); c) +annotate!(::AnnotatedChar{C, V}, ::Symbol, @nospecialize(value)) where {C, V} = + throw(ArgumentError(LazyString("cannot annotate with a ", typeof(value), " value, as the value type is ", V, + "; construct the char with a wider value type, such as Any"))) """ annotations(str::Union{AnnotatedString, SubString{AnnotatedString}}, [position::Union{Integer, UnitRange}]) -> - Vector{$RegionAnnotation} + Vector{$(RegionAnnotation{TypeVar(:V)})} Get all annotations that apply to `str`. Should `position` be provided, only annotations that overlap with `position` will be returned. @@ -417,9 +485,9 @@ See also [`annotate!`](@ref). """ annotations(s::AnnotatedString) = s.annotations -function annotations(s::SubString{<:AnnotatedString}) +function annotations(s::SubString{AnnotatedString{S, V}}) where {S, V} substr_range = s.offset+1:s.offset+s.ncodeunits - result = RegionAnnotation[] + result = RegionAnnotation{V}[] for ann in annotations(s.string, substr_range) # Shift the region to be relative to the substring start shifted_region = first(ann.region)-s.offset:last(ann.region)-s.offset @@ -429,9 +497,9 @@ function annotations(s::SubString{<:AnnotatedString}) return result end -function annotations(s::AnnotatedString, pos::UnitRange{<:Integer}) +function annotations(s::AnnotatedString{S, V}, pos::UnitRange{<:Integer}) where {S, V} # TODO optimise - result = RegionAnnotation[] + result = RegionAnnotation{V}[] for ann in s.annotations if !isempty(intersect(pos, ann.region)) clamped_region = max(first(pos), first(ann.region)):min(last(pos), last(ann.region)) @@ -451,7 +519,7 @@ annotations(s::SubString{<:AnnotatedString}, pos::UnitRange{<:Integer}) = annotations(s.string, first(pos)+s.offset:last(pos)+s.offset) """ - annotations(chr::AnnotatedChar)::Vector{$Annotation} + annotations(chr::AnnotatedChar)::Vector{$(Annotation{TypeVar(:V)})} Get all annotations of `chr`, in the form of a vector of annotation pairs. """ @@ -471,12 +539,12 @@ This works by comparing the number of code units of each character before and after transforming with `f`, recording and aggregating any differences, then applying them to the annotation regions. -Returns an `AnnotatedString{String}` (regardless of the original underlying -string type of `str`). +Returns an `AnnotatedString{String, V}` (regardless of the original underlying +string type of `str`), where `V` is the value type of `str`. """ -function annotated_chartransform(f::Function, str::AnnotatedString, state=nothing) +function annotated_chartransform(f::Function, str::AnnotatedString{S, V}, state=nothing) where {S, V} outstr = IOBuffer() - annots = RegionAnnotation[] + annots = RegionAnnotation{V}[] bytepos = firstindex(str) - 1 offsets = [bytepos => 0] for c in str.string @@ -498,13 +566,13 @@ function annotated_chartransform(f::Function, str::AnnotatedString, state=nothin stop_offset = last(offsets[findlast(<=(stop) ∘ first, offsets)::Int]) push!(annots, @inline(setindex(annot, (start + start_offset):(stop + stop_offset), :region))) end - AnnotatedString(takestring!(outstr), annots) + AnnotatedString{String, V}(takestring!(outstr), annots) end -struct RegionIterator{S <: AbstractString} +struct RegionIterator{S <: AbstractString, V} str::S regions::Vector{UnitRange{Int}} - annotations::Vector{Vector{Annotation}} + annotations::Vector{Vector{Annotation{V}}} end Base.length(si::RegionIterator) = length(si.regions) @@ -515,8 +583,8 @@ Base.@propagate_inbounds function Base.iterate(si::RegionIterator, i::Integer=1) end end -Base.eltype(::RegionIterator{S}) where { S <: AbstractString} = - Tuple{SubString{S}, Vector{Annotation}} +Base.eltype(::RegionIterator{S, V}) where { S <: AbstractString, V} = + Tuple{SubString{S}, Vector{Annotation{V}}} """ eachregion(s::AnnotatedString{S}) @@ -538,15 +606,15 @@ julia> collect(eachregion(AnnotatedString( ("there", [$Annotation((:face, :italic))]) ``` """ -function eachregion(s::AnnotatedString, subregion::UnitRange{Int}=firstindex(s):lastindex(s)) +function eachregion(s::AnnotatedString{S, V}, subregion::UnitRange{Int}=firstindex(s):lastindex(s)) where {S, V} isempty(s) || isempty(subregion) && - return RegionIterator(s.string, UnitRange{Int}[], Vector{Annotation}[]) + return RegionIterator(s.string, UnitRange{Int}[], Vector{Annotation{V}}[]) events = annotation_events(s, subregion) - isempty(events) && return RegionIterator(s.string, [subregion], [Annotation[]]) - annotvals = Annotation[ + isempty(events) && return RegionIterator(s.string, [subregion], [Annotation{V}[]]) + annotvals = Annotation{V}[ (; label, value) for (; label, value) in annotations(s)] regions = Vector{UnitRange{Int}}() - annots = Vector{Vector{Annotation}}() + annots = Vector{Vector{Annotation{V}}}() pos = first(events).pos if pos > first(subregion) push!(regions, thisind(s, first(subregion)):prevind(s, pos)) @@ -572,9 +640,9 @@ function eachregion(s::AnnotatedString, subregion::UnitRange{Int}=firstindex(s): RegionIterator(s.string, regions, annots) end -function eachregion(s::SubString{<:AnnotatedString}, pos::UnitRange{Int}=firstindex(s):lastindex(s)) +function eachregion(s::SubString{AnnotatedString{S, V}}, pos::UnitRange{Int}=firstindex(s):lastindex(s)) where {S, V} if isempty(s) - RegionIterator(s.string, Vector{UnitRange{Int}}(), Vector{Vector{Annotation}}()) + RegionIterator(s.string, Vector{UnitRange{Int}}(), Vector{Vector{Annotation{V}}}()) else eachregion(s.string, first(pos)+s.offset:last(pos)+s.offset) end @@ -592,8 +660,9 @@ index::Int}` where `pos` is the position of the event, `active` is a boolean indicating whether the annotation is being activated or deactivated, and `index` is the index of the annotation in question. """ -function annotation_events(s::AbstractString, annots::Vector{RegionAnnotation}, subregion::UnitRange{Int}) +function annotation_events(s::AbstractString, annots::Vector{<:RegionAnnotation}, subregion::UnitRange{Int}) events = Vector{NamedTuple{(:pos, :active, :index), Tuple{Int, Bool, Int}}}() # Position, Active?, Annotation index + sizehint!(events, 2 * length(annots)) for (i, (; region)) in enumerate(annots) if !isempty(intersect(subregion, region)) start, stop = max(first(subregion), first(region)), min(last(subregion), last(region)) @@ -602,7 +671,7 @@ function annotation_events(s::AbstractString, annots::Vector{RegionAnnotation}, push!(events, (pos=nextind(s, stop), active=false, index=i)) end end - sort(events, by=e -> e.pos) + sort!(events, by=e -> e.pos) end annotation_events(s::AnnotatedString, subregion::UnitRange{Int}) = diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl index ce30a2146335e..c0ca023c3ede5 100644 --- a/base/strings/annotated_io.jl +++ b/base/strings/annotated_io.jl @@ -2,13 +2,13 @@ ## AnnotatedIOBuffer -struct AnnotatedIOBuffer <: AbstractPipe +struct AnnotatedIOBuffer{V} <: AbstractPipe io::IOBuffer - annotations::Vector{RegionAnnotation} + annotations::Vector{RegionAnnotation{V}} end -AnnotatedIOBuffer(io::IOBuffer) = AnnotatedIOBuffer(io, Vector{RegionAnnotation}()) -AnnotatedIOBuffer() = AnnotatedIOBuffer(IOBuffer()) +AnnotatedIOBuffer{V}(io::IOBuffer = IOBuffer()) where {V} = AnnotatedIOBuffer(io, Vector{RegionAnnotation{V}}()) +AnnotatedIOBuffer(io::IOBuffer = IOBuffer()) = AnnotatedIOBuffer{Any}(io) function show(io::IO, aio::AnnotatedIOBuffer) show(io, AnnotatedIOBuffer) @@ -32,12 +32,12 @@ annotations(io::AnnotatedIOBuffer) = io.annotations annotate!(io::AnnotatedIOBuffer, range::UnitRange{Int}, label::Symbol, @nospecialize(val::Any)) = (_annotate!(io.annotations, range, label, val); io) -function write(io::AnnotatedIOBuffer, astr::Union{AnnotatedString, SubString{<:AnnotatedString}}) - astr = AnnotatedString(astr) +function write(io::AnnotatedIOBuffer{V}, astr::Union{AnnotatedString{S}, SubString{<:AnnotatedString{S}}}) where {V, S} + annots = convert(Vector{RegionAnnotation{V}}, annotations(astr)) offset = position(io.io) eof(io) || _clear_annotations_in_region!(io.annotations, offset+1:offset+ncodeunits(astr)) - _insert_annotations!(io, astr.annotations) - write(io.io, String(astr)) + _insert_annotations!(io, annots) + write(io.io, unannotate(astr))::Int end write(io::AnnotatedIOBuffer, c::AnnotatedChar) = @@ -47,21 +47,21 @@ write(io::AnnotatedIOBuffer, s::Union{SubString{String}, String}) = write(io.io, write(io::AnnotatedIOBuffer, s::StringViewAndSub) = write(io.io, s)::Int write(io::AnnotatedIOBuffer, b::UInt8) = write(io.io, b) -function write(dest::AnnotatedIOBuffer, src::AnnotatedIOBuffer) +function write(dest::AnnotatedIOBuffer{V}, src::AnnotatedIOBuffer) where {V} destpos = position(dest) isappending = eof(dest) srcpos = position(src) + srcannots = RegionAnnotation{V}[ # Before the text, so a value that doesn't fit leaves `dest` as it was + @inline(setindex(annot, max(1 + srcpos, first(annot.region)):last(annot.region), :region)) + for annot in src.annotations if first(annot.region) >= srcpos] nb = write(dest.io, src.io) isappending || _clear_annotations_in_region!(dest.annotations, destpos:destpos+nb) - srcannots = [@inline(setindex(annot, max(1 + srcpos, first(annot.region)):last(annot.region), :region)) - for annot in src.annotations if first(annot.region) >= srcpos] _insert_annotations!(dest, srcannots, destpos - srcpos) nb end -# So that read/writes with `IOContext` (and any similar `AbstractPipe` wrappers) -# work as expected. -function write(io::AbstractPipe, s::Union{AnnotatedString, SubString{<:AnnotatedString}}) +# So that read/writes with `IOContext` (and any similar `AbstractPipe` wrappers) work as expected. +function write(io::AbstractPipe, s::Union{AnnotatedString{S}, SubString{<:AnnotatedString{S}}}) where {S} if pipe_writer(io) isa AnnotatedIOBuffer write(pipe_writer(io), s) else @@ -78,25 +78,26 @@ function write(io::AbstractPipe, c::AnnotatedChar) end::Int end -function read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{T}}) where {T <: AbstractString} +function read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{S, V}}) where {S, V} start = position(io) - if start == 0 - AnnotatedString(read(io.io, T), copy(io.annotations)) - else - annots = [@inline(setindex(annot, UnitRange{Int}(max(1, first(annot.region) - start), last(annot.region)-start), :region)) - for annot in io.annotations if last(annot.region) > start] - AnnotatedString(read(io.io, T), annots) - end + annots = RegionAnnotation{V}[ + (region = max(1, first(annot.region) - start):last(annot.region)-start, + label = annot.label, + value = annot.value) + for annot in io.annotations if last(annot.region) > start] + AnnotatedString{S, V}(read(io.io, S), annots) end +read(io::AnnotatedIOBuffer{V}, ::Type{AnnotatedString{S}}) where {S, V} = read(io, AnnotatedString{S, V}) read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{AbstractString}}) = read(io, AnnotatedString{String}) read(io::AnnotatedIOBuffer, ::Type{AnnotatedString}) = read(io, AnnotatedString{String}) -function read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{T}}) where {T <: AbstractChar} +function read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{T, V}}) where {T <: AbstractChar, V} pos = position(io) char = read(io.io, T) - annots = [NamedTuple{(:label, :value)}(annot) for annot in io.annotations if pos+1 in annot.region] - AnnotatedChar(char, annots) + annots = Annotation{V}[Annotation{V}((annot.label, annot.value)) for annot in io.annotations if pos+1 in annot.region] + AnnotatedChar{T, V}(char, annots) end +read(io::AnnotatedIOBuffer{V}, ::Type{AnnotatedChar{T}}) where {T <: AbstractChar, V} = read(io, AnnotatedChar{T, V}) read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{AbstractChar}}) = read(io, AnnotatedChar{Char}) read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar}) = read(io, AnnotatedChar{Char}) @@ -117,10 +118,10 @@ This operates by removing all elements of `annotations` that are entirely contained in `span`, truncating ranges that partially overlap, and splitting annotations that subsume `span` to just exist either side of `span`. """ -function _clear_annotations_in_region!(annotations::Vector{RegionAnnotation}, span::UnitRange{Int}) +function _clear_annotations_in_region!(annotations::Vector{RegionAnnotation{V}}, span::UnitRange{Int}) where {V} # Clear out any overlapping pre-existing annotations. filter!(ann -> first(ann.region) < first(span) || last(ann.region) > last(span), annotations) - extras = Tuple{Int, RegionAnnotation}[] + extras = Tuple{Int, RegionAnnotation{V}}[] for i in eachindex(annotations) annot = annotations[i] region = annot.region @@ -165,7 +166,7 @@ This is implemented so that one can say write an `AnnotatedString` to an `AnnotatedIOBuffer` one character at a time without needlessly producing a new annotation for each character. """ -function _insert_annotations!(annots::Vector{RegionAnnotation}, newannots::Vector{RegionAnnotation}, offset::Int = 0) +function _insert_annotations!(annots::Vector{RegionAnnotation{V}}, newannots::Vector{RegionAnnotation{V′}}, offset::Int = 0) where {V, V′ <: V} run = @label search begin if !isempty(annots) && last(last(annots).region) == offset for i in reverse(axes(newannots, 1)) @@ -173,14 +174,14 @@ function _insert_annotations!(annots::Vector{RegionAnnotation}, newannots::Vecto first(annot.region) == 1 || continue i <= length(annots) || continue annot.label == last(annots).label || continue - annot.value == last(annots).value || continue + annot.value === last(annots).value || continue all(1:i) do runlen new = newannots[begin+runlen-1] old = annots[end-i+runlen] !(last(old.region) != offset || first(new.region) != 1 || old.label != new.label || - old.value != new.value) + old.value !== new.value) end || continue break search i end @@ -228,7 +229,7 @@ function _insert_annotations!(annots::Vector{RegionAnnotation}, newannots::Vecto end end -_insert_annotations!(io::AnnotatedIOBuffer, newannots::Vector{RegionAnnotation}, offset::Int = position(io)) = +_insert_annotations!(io::AnnotatedIOBuffer, newannots::Vector{<:RegionAnnotation}, offset::Int = position(io)) = _insert_annotations!(io.annotations, newannots, offset) # String replacement @@ -237,7 +238,7 @@ _insert_annotations!(io::AnnotatedIOBuffer, newannots::Vector{RegionAnnotation}, # substantial slowdown here. If we remove `; count` from the signature # and run the sample code above in `_insert_annotations!`, the runtime # drops from ~4400ns to ~580ns (~7x faster). I cannot guess why this is. -function replace(out::AnnotatedIOBuffer, str::AnnotatedString, pat_f::Pair...; count = typemax(Int)) +function replace(out::AnnotatedIOBuffer{V}, str::AnnotatedString, pat_f::Pair...; count = typemax(Int)) where {V} if count == 0 || isempty(pat_f) write(out, str) return out @@ -274,13 +275,13 @@ function replace(out::AnnotatedIOBuffer, str::AnnotatedString, pat_f::Pair...; c replacement = replacers[ridx] _isannotated(replacement) || continue annots = annotations(replacement) - annots′ = if eltype(annots) == Annotation # When it's a char not a string + annots′ = if eltype(annots) <: Annotation # When it's a char not a string region = 1:newbytes - [@NamedTuple{region::UnitRange{Int}, label::Symbol, value}((region, label, value)) + [@NamedTuple{region::UnitRange{Int}, label::Symbol, value::V}((region, label, value)) for (; label, value) in annots] else - annots - end::Vector{RegionAnnotation} + convert(Vector{RegionAnnotation{V}}, annots) + end _insert_annotations!(newannots, annots′, destoff) end push!(replacements, (region = e1:(e1-1), offset = last(replacements).offset)) @@ -331,8 +332,11 @@ replace(out::IO, str::AnnotatedString, pat_f::Pair...; count=typemax(Int)) = replace(out, str.string, pat_f...; count) function replace(str::AnnotatedString, pat_f::Pair...; count=typemax(Int)) - isempty(pat_f) || iszero(count) && return str - out = AnnotatedIOBuffer() + V = annot_promote_valtype(str, pat_f...) + # As read back below, so that the type doesn't depend on `count` + (isempty(pat_f) || iszero(count)) && + return AnnotatedString{String, V}(String(str.string), Vector{RegionAnnotation{V}}(str.annotations)) + out = AnnotatedIOBuffer{V}() replace(out, str, pat_f...; count) read(seekstart(out), AnnotatedString) end @@ -359,13 +363,13 @@ ansi_write(f::Function, io::IO, x::Any) = ansi_write_(f::Function, io::IO, @nospecialize(x::Any)) = invoke_in_world(tls_world_age(), ansi_write, f, io, x) -Base.write(io::IO, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = +Base.write(io::IO, s::Union{AnnotatedString{S}, SubString{<:AnnotatedString{S}}}) where {S} = ansi_write_(write, io, s)::Int Base.write(io::IO, c::AnnotatedChar) = ansi_write_(write, io, c)::Int -function Base.write(io::IO, aio::AnnotatedIOBuffer) +function Base.write(io::IO, aio::AnnotatedIOBuffer{V}) where {V} if get(io, :color, false) == true # This does introduce an overhead that technically # could be avoided, but I'm not sure that it's currently @@ -373,7 +377,7 @@ function Base.write(io::IO, aio::AnnotatedIOBuffer) # writing from an AnnotatedIOBuffer with style. # In the meantime, by converting to an `AnnotatedString` we can just # reuse all the work done to make that work. - ansi_write_(write, io, read(aio, AnnotatedString))::Int + ansi_write_(write, io, read(aio, AnnotatedString{String, V}))::Int else write(io, aio.io) end diff --git a/base/strings/io.jl b/base/strings/io.jl index 933290ebc84ed..e5543e18dc71d 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -358,17 +358,15 @@ function _join_preserve_annotations(iterator, args...) if isconcretetype(et) && !_isannotated(et) && !any(_isannotated, args) sprint(join, iterator, args...) else - io = AnnotatedIOBuffer() + V = annot_promote_valtype(et, args...) + io = AnnotatedIOBuffer{V}() join(io, iterator, args...) - # If we know (from compile time information, or dynamically in the case - # of iterators with a non-concrete eltype), that the result is annotated - # in nature, we extract an `AnnotatedString`, otherwise we just extract - # a plain `String` from `io`. - if isconcretetype(et) || !isempty(io.annotations) - seekstart(io) - read(io, AnnotatedString{String}) - else + # `@default_eltype` widens the `UnionAll` `AnnotatedString{String}` to `Any`, so `join` of a + # `Vector{AnnotatedString{String}}` infers as `Union{AnnotatedString{String, Any}, String}`. + if !isconcretetype(et) && !_isannotated(et) && isempty(io.annotations) String(take!(io.io)) + else + read(seekstart(io), AnnotatedString{String, V}) end end end @@ -820,7 +818,7 @@ function AnnotatedString(chars::AbstractVector{C}) where {C<:AbstractChar} end end end - annots = RegionAnnotation[] + annots = RegionAnnotation{annot_promote_valtype(C)}[] point = 1 for c in chars if c isa AnnotatedChar diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index 63ddb2dc3a1bc..6064176eb8054 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -35,7 +35,7 @@ precompile(Tuple{typeof(Base.reseteof), Base.Terminals.TTYTerminal}) precompile(Tuple{typeof(Base.Terminals.width), Base.Terminals.TTYTerminal}) precompile(Tuple{typeof(Base.Terminals.height), Base.Terminals.TTYTerminal}) precompile(Tuple{typeof(Base.write), Base.Terminals.TTYTerminal, Array{UInt8, 1}}) -precompile(Tuple{typeof(Base.isempty), Base.AnnotatedString{String}}) +precompile(Tuple{typeof(Base.isempty), Base.AnnotatedString{String, Any}}) # The owner-carrying write paths (String/Array) call _unsafe_write_owned # directly, so the workload session does not compile the raw-pointer # unsafe_write entry points on its own; interactive startup still reaches diff --git a/doc/src/manual/strings.md b/doc/src/manual/strings.md index 0ff93871049ca..2c9af890e703f 100644 --- a/doc/src/manual/strings.md +++ b/doc/src/manual/strings.md @@ -1269,7 +1269,7 @@ julia> lpad(str, 14) " hello there" julia> typeof(lpad(str, 7)) -Base.AnnotatedString{String} +Base.AnnotatedString{String, Any} julia> str2 = Base.AnnotatedString(" julia", [(2:6, :face, :magenta)]) " julia" diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index a839a07fce86b..fffe8396d75e7 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -2,15 +2,15 @@ @testset "AnnotatedString" begin str = Base.AnnotatedString("some string") - @test str == Base.AnnotatedString(str.string, Base.RegionAnnotation[]) @test length(str) == 11 @test ncodeunits(str) == 11 @test codeunits(str) == codeunits("some string") @test codeunit(str) == UInt8 @test codeunit(str, 1) == codeunit("some string", 1) @test firstindex(str) == firstindex("some string") - @test convert(Base.AnnotatedString, str) === str - @test eltype(str) == Base.AnnotatedChar{eltype(str.string)} + @test eltype(str) == Base.AnnotatedChar{eltype(str.string), Any} + @test eltype(Base.AnnotatedString{String}) == Base.AnnotatedChar{Char} + @test collect(Base.AnnotatedString("ab", [(1:1, :x, 1)])) isa Vector{Base.AnnotatedChar{Char, Int}} @test first(str) == Base.AnnotatedChar(first(str.string), Pair{Symbol, Any}[]) @test str[1:4] isa SubString{typeof(str)} @test str[1:4] == Base.AnnotatedString("some") @@ -46,8 +46,6 @@ @test str != Base.AnnotatedString("some string", [(1:1, :thing, 0x01), (1:11, :all, 0x03), (6:6, :other, 0x02)]) @test str != Base.AnnotatedString("some string", [(1:4, :thing, 0x11), (1:11, :all, 0x13), (6:11, :other, 0x12)]) @test str != Base.AnnotatedString("some thingg", [(1:4, :thing, 0x01), (1:11, :all, 0x03), (6:11, :other, 0x02)]) - @test Base.AnnotatedString([Base.AnnotatedChar('a', [(:a, 1)]), Base.AnnotatedChar('b', [(:b, 2)])]) == - Base.AnnotatedString("ab", [(1:1, :a, 1), (2:2, :b, 2)]) let allstrings = ['a', Base.AnnotatedChar('a'), Base.AnnotatedChar('a', [(:aaa, 0x04)]), "a string", Base.AnnotatedString("a string"), @@ -64,11 +62,46 @@ end end end + # Values of different types are kept as they are, under the union of their types + valued(v) = Base.AnnotatedString("x", [(1:1, :n, v)]) + mixed = Base.annotatedstring(valued(1), valued(2.5)) + @test mixed isa Base.AnnotatedString{String, Union{Int, Float64}} + @test Base.annotations(mixed) == [(region = 1:1, label = :n, value = 1), (region = 2:2, label = :n, value = 2.5)] + @test valued(true) * valued(1) isa Base.AnnotatedString{String, Union{Bool, Int}} + @test Base.annotatedstring(valued(1), "y", valued('c'), valued("s")) isa Base.AnnotatedString{String, Union{Int, Char, String}} + @test (@inferred Base.annotatedstring(valued(1), valued('c'), valued("s"), valued(1.0), valued(:s))) isa Base.AnnotatedString{String, Any} + @test join([valued(1), valued(2)], valued('c')) isa Base.AnnotatedString{String, Union{Int, Char}} + @test join([valued(1), valued(2.5)], valued('c')) isa Base.AnnotatedString{String, Any} # The eltype no longer says + @test replace(valued(1), "x" => valued(2.5)) isa Base.AnnotatedString{String, Union{Int, Float64}} + # The value type is settled at compile time, also from an eltype that only may be annotated + @test (@inferred Base.annotatedstring("y", valued(1))) isa Base.AnnotatedString{String, Int} + @test (@inferred String join(AbstractString["y", valued(1)])) isa Base.AnnotatedString{String, Any} + @test (@inferred String join(Union{Missing, Base.AnnotatedString{String, Int}}[valued(1), valued(2)])) isa Base.AnnotatedString{String, Int} + # An empty iterator, whose eltype is `Union{}`, joins to a plain `String` + @test join(()) === "" + @test join(x for x in ()) === "" + let sub = Base.AnnotatedString(SubString("abcd", 1, 3), [(1:1, :n, 1)]) # Typed as `repeat` of the wrapped string + @test repeat(sub, 0) == Base.AnnotatedString("") + @test (@inferred repeat(sub, 1)) == sub + @test repeat(sub, 1) isa Base.AnnotatedString{String, Int} + @test repeat(sub, 1) !== sub # A copy, as an annotated string is mutable + end + @test uppercase(Base.AnnotatedString(SubString("abcd", 1, 3), [(1:1, :n, 1)])) isa Base.AnnotatedString{String, Int} + # A value outside the value type is refused with a clear error + @test_throws ArgumentError Base.annotate!(valued(1), 1:1, :m, "s") + @test_throws ArgumentError Base.annotate!(Base.AnnotatedIOBuffer{Int}(), 1:1, :m, "s") + @test_throws ArgumentError Base.annotate!(Base.AnnotatedChar('c', [(:m, 1)]), :m, "s") + # Annotating with `nothing` removes the label, even where the value type admits `nothing` + let s = Base.AnnotatedString{String, Any}("abc") + Base.annotate!(s, 1:3, :face, :red) + Base.annotate!(s, 1:3, :face, nothing) + @test isempty(Base.annotations(s)) + end # @test collect(Base.eachstyle(str)) == # [("some", [:thing, 0x01, :all, 0x03]), # (" string", [:all, 0x03, :other, 0x02])] @test chopprefix(sprint(show, str), "Base.") == - "AnnotatedString{String}(\"some string\", [(1:4, :thing, 0x01), (6:11, :other, 0x02), (1:11, :all, 0x03)])" + "AnnotatedString{String, Any}(\"some string\", [(1:4, :thing, 0x01), (6:11, :other, 0x02), (1:11, :all, 0x03)])" @test eval(Meta.parse(repr(str))) == str @test sprint(show, MIME("text/plain"), str) == "\"some string\"" @@ -92,9 +125,6 @@ end @testset "AnnotatedChar" begin chr = Base.AnnotatedChar('c') - @test Base.AnnotatedChar(UInt32('c')) == chr - @test convert(Base.AnnotatedChar, chr) === chr - @test chr == Base.AnnotatedChar(chr.char, Pair{Symbol, Any}[]) @test uppercase(chr) == Base.AnnotatedChar('C') @test titlecase(chr) == Base.AnnotatedChar('C') @test lowercase(Base.AnnotatedChar('C')) == chr @@ -120,6 +150,46 @@ end @test readchomp(`$(Base.julia_cmd()) --startup-file=no -e $script`) == "false bγ" end +@testset "Construction" begin + # Strings and chars follow the same rules; char annotations just lack a region + for (Annotated, plain, Ann, region) in ((Base.AnnotatedString, "ab", Base.RegionAnnotation, (1:1,)), + (Base.AnnotatedChar, 'a', Base.Annotation, ())) + Plain, Abstract = typeof(plain), supertype(typeof(plain)) + tuples = [(region..., :x, 1)] + ints = Ann{Int}[Ann{Int}(only(tuples))] + for T in (Annotated, Annotated{Plain}, Annotated{Abstract}, Annotated{Plain, Int}, Annotated{Abstract, Int}), + annots in (ints, tuples) + @test T(plain, annots) isa T + @test T(plain, annots) isa Annotated{<:Any, Int} + end + @test Annotated{Abstract, Int}(plain) isa Annotated{Abstract, Int} + @test Annotated{Plain}(plain, [(region..., :x, 1), (region..., :y, 'c')]) isa Annotated{Plain, Any} + @test Annotated(plain, ints).annotations === ints + @test Annotated{Plain, Int}(plain, ints).annotations === ints + @test Annotated{Plain, Any}(plain, ints).annotations !== ints + annotated = Annotated(plain, ints) + for copied in (Annotated(annotated), typeof(annotated)(annotated), Annotated{Plain, Any}(annotated)) + @test copied == annotated + @test copied.annotations !== annotated.annotations + end + @test typeof(Annotated(annotated)) == typeof(annotated) + @test convert(Annotated, annotated) === annotated + @test convert(typeof(annotated), annotated) === annotated + # Adding annotations widens the value type, as `vcat` does + for added in ([(region..., :y, :s)], Ann{Symbol}[Ann{Symbol}((region..., :y, :s))]) + @test Annotated(annotated, added) isa Annotated{Plain, Union{Int, Symbol}} + end + end + @test Base.AnnotatedString("ab") == Base.AnnotatedString("ab", Base.RegionAnnotation[]) + @test Base.AnnotatedChar(UInt32('c')) == Base.AnnotatedChar('c') == Base.AnnotatedChar('c', Pair{Symbol, Any}[]) + @test Base.AnnotatedString([Base.AnnotatedChar('a', [(:a, 1)]), Base.AnnotatedChar('b', [(:b, 2)])]) == + Base.AnnotatedString("ab", [(1:1, :a, 1), (2:2, :b, 2)]) + @test (@inferred Base.AnnotatedString(Base.AnnotatedString("x", [(1:1, :n, 1)]), [(1:1, :m, 2)])) isa Base.AnnotatedString{String, Int} + @test (@inferred Base.AnnotatedChar(Base.AnnotatedChar('x', [(:n, 1)]), [(:m, 2)])) isa Base.AnnotatedChar{Char, Int} + @test Base.AnnotatedChar(Base.AnnotatedChar('m', [(:attr, "h0m1")]), [(label = :attr, value = "m2m3")]) == + Base.AnnotatedChar('m', [(:attr, "h0m1"), (:attr, "m2m3")]) +end + @testset "Styling preservation" begin str = Base.AnnotatedString("some string", [(1:4, :thing, 0x01), (1:11, :all, 0x03), (6:11, :other, 0x02)]) @test match(r".e", str).match == str[3:4] @@ -188,7 +258,7 @@ end @testset "AnnotatedIOBuffer" begin aio = Base.AnnotatedIOBuffer() - vec2ann(v::Vector{<:Tuple}) = collect(Base.RegionAnnotation, v) + vec2ann(v::Vector{Tuple{UnitRange{Int}, Symbol, V}}) where {V} = collect(Base.RegionAnnotation{V}, v) # Append-only writing @test write(aio, Base.AnnotatedString("hello", [(1:5, :tag, 1)])) == 5 @test write(aio, ' ') == 1 @@ -271,6 +341,30 @@ end @test read(seekstart(aio2), Base.AnnotatedString) == Base.AnnotatedString("ab", [(1:1, :b, 1), (2:2, :a, 1), (2:2, :b, 1)]) end + let aio2 = Base.AnnotatedIOBuffer() # Equal but distinct values are not merged + write(aio2, Base.AnnotatedString("a", [(1:1, :x, [1])])) + write(aio2, Base.AnnotatedString("b", [(1:1, :x, [1])])) + @test length(Base.annotations(aio2)) == 2 + end + let ints = Base.AnnotatedIOBuffer{Int}() # Between value types + write(ints, Base.AnnotatedString("ab", [(1:1, :x, 1)])) + anys = Base.AnnotatedIOBuffer() + write(anys, seekstart(ints)) + @test Base.annotations(anys) == vec2ann([(1:1, :x, 1)]) + @test read(seekstart(anys), Base.AnnotatedString{String, Int}) isa Base.AnnotatedString{String, Int} + write(anys, Base.AnnotatedString("c", [(1:1, :y, "c")])) + @test_throws MethodError read(seekstart(anys), Base.AnnotatedString{String, Int}) + end + let ints = Base.AnnotatedIOBuffer{Int}() # Chars are read with the buffer's value type, or a given one + write(ints, Base.AnnotatedString("ab", [(1:1, :x, 1)])) + @test read(seekstart(ints), Base.AnnotatedChar) isa Base.AnnotatedChar{Char, Int} + @test read(seekstart(ints), Base.AnnotatedChar{Char, Any}) == Base.AnnotatedChar{Char, Any}('a', [(:x, 1)]) + end + let nested = Base.AnnotatedString(SubString("hello", 1, 3), [(1:1, :x, 1)]) # Over a `SubString` + aio2 = Base.AnnotatedIOBuffer() + @test write(aio2, nested[1:2]) == 2 + @test read(seekstart(aio2), Base.AnnotatedString) == Base.AnnotatedString("he", [(1:1, :x, 1)]) + end # Working through an IOContext aio = Base.AnnotatedIOBuffer() wrapio = IOContext(aio) @@ -281,7 +375,13 @@ end # show-ing an AnnotatedIOBuffer aio = Base.AnnotatedIOBuffer() write(aio, Base.AnnotatedString("hello", [(1:5, :tag, 1)])) - @test sprint(show, aio) == "Base.AnnotatedIOBuffer(5 bytes, 1 annotation)" + @test chopprefix(sprint(show, aio), "Base.") == "AnnotatedIOBuffer(5 bytes, 1 annotation)" + # A value that doesn't fit the destination throws before any text is written + let src = Base.AnnotatedIOBuffer(), dest = Base.AnnotatedIOBuffer{Int}() + write(src, Base.AnnotatedString("ab", [(1:1, :x, "s")])) + @test_throws MethodError write(dest, seekstart(src)) + @test position(dest) == 0 + end end @testset "Eachregion" begin @@ -835,5 +935,15 @@ end green_annots = filter(a -> a.value == :green, Base.annotations(result3)) @test all(a -> first(a.region) >= pos + 1, green_annots) end + + @testset "Typed buffers" begin + wide = Base.AnnotatedString{String, Any}("Q", [(1:1, :n, 1)]) + anybuf = Base.AnnotatedIOBuffer() + replace(anybuf, astr("apple", (1:5, :red)), "p" => wide) + typedbuf = Base.AnnotatedIOBuffer{Union{Symbol, Int}}() + replace(typedbuf, astr("apple", (1:5, :red)), "p" => wide) + @test read(seekstart(typedbuf), Base.AnnotatedString) == read(seekstart(anybuf), Base.AnnotatedString) + @test_throws MethodError replace(Base.AnnotatedIOBuffer{Symbol}(), astr("apple", (1:5, :red)), "p" => wide) + end end end From 5f5a491e6ed4bb5c7891daebe385ad4c19622ddc Mon Sep 17 00:00:00 2001 From: TEC Date: Sun, 20 Sep 2026 18:02:31 +0800 Subject: [PATCH 02/23] Dispatch annotated display on the value type The `invoke_in_world` barrier around every annotated write is replaced by a trait on the value type, `Base.AnnotatedDisplay.AnnotationStyle`. A package declares a style for its value type and implements `awrite(textwriter, style, io, s)`; `write`, `print` and `escape_string` all go through it. A style may add a `text/html` method, which `showable` reports. `NoStyle` writes the plain text and has no MIME form. Only `Any`-valued strings still work out their style from the values they hold, behind one remaining barrier, and `showable` does the same for them. Escaping into an `AnnotatedIOBuffer`, bare or inside an `IOContext`, keeps the annotations, moved to cover the escaped text. `print` goes through `write`, so printing into an `IOContext` around such a buffer keeps them too. When two styles meet, a rule `AnnotationStyle(a, b)` settles which applies. It is tried in both orders, as `promote_rule` is, and a missing rule raises an `ArgumentError` naming the method to define. Both trait functions have `max_methods = 1`. A call whose value type is not known at compile time then sees two applicable methods, so inference leaves it dynamic and records no method-table edge. This means a package's style and writer definitions cannot invalidate Base's compiled callers, which a test checks in a fresh process. --- base/strings/annotated_io.jl | 204 +++++++++++++++++++++++++++++------ test/strings/annotated.jl | 125 ++++++++++++++++++++- 2 files changed, 293 insertions(+), 36 deletions(-) diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl index c0ca023c3ede5..5d0c9b47fd04a 100644 --- a/base/strings/annotated_io.jl +++ b/base/strings/annotated_io.jl @@ -345,29 +345,126 @@ end function printstyled end -# NOTE: This is an interim solution to the invalidations caused -# by the split styled display implementation. This should be -# replaced by a more robust solution (such as a consolidation of -# the type and method definitions) in the near future. +""" + AnnotatedDisplay + +How an annotated string, substring, or char is written and shown. The value type of its +annotations selects an [`AnnotationStyle`](@ref), and [`awrite`](@ref) renders it under +that style. Base provides `NoStyle`, which writes the plain text; a package whose values +carry display information (such as StyledStrings' `Face`) declares a style for its type +and implements `awrite` for it, and strings holding such values then render statically, +without a lookup at write time. + +!!! warning "Experimental" + This interface is experimental and may change or be removed in a future release + without deprecation. +""" module AnnotatedDisplay using ..Base: IO, SubString, AnnotatedString, AnnotatedChar, AnnotatedIOBuffer -using ..Base: eachregion, invoke_in_world, tls_world_age, unannotate +using ..Base: eachregion, unannotate, annotations, annotatedstring, annot_valtype, invoke_in_world, tls_world_age, Fix1 +using ..Base: escape_string, annotate!, _clear_annotations_in_region!, RegionAnnotation + +public AbstractAnnotationStyle, AnnotationStyle, NoStyle, awrite + +# Annotation styles + +""" + AbstractAnnotationStyle + +The supertype of the singletons that [`AnnotationStyle`](@ref) selects between. +""" +abstract type AbstractAnnotationStyle end + +struct NoStyle <: AbstractAnnotationStyle end + +# These are plain functions rather than constructors so that `max_methods` applies to them +# (a constructor shares `DataType`'s limit). When the value type is not a compile-time +# constant, a call sees at least two applicable methods (`NoStyle` and `DynamicStyle`), so +# inference leaves it dynamic and records no method-table edge. A package's style or writer +# definitions therefore cannot invalidate Base's compiled callers. +""" + AnnotationStyle(::Type{V}) -> AbstractAnnotationStyle + +Trait selecting how annotations whose values have type `V` are displayed. + +A type that carries display information (such as StyledStrings' `Face`) returns its own +`AbstractAnnotationStyle` singleton, for which [`awrite`](@ref) methods are defined. +A type that is mere metadata returns `NoStyle()`, the default. A `Union` value type reduces +over its members with `AnnotationStyle(a, b)`, so annotations of several types are displayed +by the member with a style. Two members whose styles differ have no display style in +common, and raise an `ArgumentError` until one of their packages defines an +`AnnotationStyle(a, b)` method that settles it, in either order, as a `promote_rule` +settles a promotion; a method may also return `nothing` to leave the pair unsettled. Where +both orders have a rule, the one in the order the styles are combined in is used. +`Any` is `DynamicStyle()`, which finds the style from the values held. +""" +function AnnotationStyle end +typeof(AnnotationStyle).name.max_methods = 0x1 # `Base.Experimental.@max_methods 1`, before it exists + +AnnotationStyle(::Type) = NoStyle() +# As `promote_type` reads `promote_rule`, `promotestyle` reads `AnnotationStyle` rules in either +# orientation; the fallback's `nothing` means no rule, so no reflection is needed and it stays foldable +AnnotationStyle(::AbstractAnnotationStyle, ::AbstractAnnotationStyle) = nothing + +# The identities belong to the combination, as `promote_type`'s do, so no rule can clash with them +promotestyle(a::S, ::S) where {S <: AbstractAnnotationStyle} = a +promotestyle(a::AbstractAnnotationStyle, ::NoStyle) = a +promotestyle(::NoStyle, b::AbstractAnnotationStyle) = b +promotestyle(::NoStyle, ::NoStyle) = NoStyle() +function promotestyle(a::AbstractAnnotationStyle, b::AbstractAnnotationStyle) + ab = AnnotationStyle(a, b) + isnothing(ab) || return ab + ba = AnnotationStyle(b, a) + isnothing(ba) || return ba + throw(ArgumentError(LazyString("annotation styles ", a, " and ", b, " have no style in common: define AnnotationStyle(::", + typeof(a), ", ::", typeof(b), ") to settle it"))) +end + +Base.@assume_effects :foldable AnnotationStyle(U::Union) = + promotestyle(AnnotationStyle(U.a), AnnotationStyle(U.b)) + +style(x) = AnnotationStyle(annot_valtype(x)) # Write -# Unstyled fallback for when StyledStrings is not loaded. -ansi_write(f::Function, io::IO, x::Any) = - f(io, x isa AnnotatedChar ? x.char : unannotate(x)) +""" + awrite(textwriter, style::AbstractAnnotationStyle, io::IO, x) + awrite(style::AbstractAnnotationStyle, io::IO, mime::MIME, x) + +Write `x`, an annotated string, substring, or char, to `io` under `style`, with each run of +text written by `textwriter(io, text)`, and return the number of bytes written; or, with a +`mime`, show `x` in that format. + +A package implements the first for its [`AnnotationStyle`](@ref), for annotated strings and +substrings (a char is written as a one-character string unless a method is added for it), +and may implement the second. Taking the text writer as an argument lets a transformation +of the text, such as escaping, keep its styling. `NoStyle` writes the plain text and has no +`mime` form. +""" +function awrite end +typeof(awrite).name.max_methods = 0x1 # As for `AnnotationStyle` + +awrite(style::AbstractAnnotationStyle, io::IO, x) = awrite(write, style, io, x) -ansi_write_(f::Function, io::IO, @nospecialize(x::Any)) = - invoke_in_world(tls_world_age(), ansi_write, f, io, x) +const AnnotatedStr = Union{AnnotatedString, SubString{<:AnnotatedString}} -Base.write(io::IO, s::Union{AnnotatedString{S}, SubString{<:AnnotatedString{S}}}) where {S} = - ansi_write_(write, io, s)::Int +awrite(textwriter::F, style::AbstractAnnotationStyle, io::IO, c::AnnotatedChar) where {F} = + awrite(textwriter, style, io, annotatedstring(c)) + +awrite(textwriter::F, ::NoStyle, io::IO, s::AnnotatedStr) where {F} = textwriter(io, unannotate(s)) +awrite(textwriter::F, ::NoStyle, io::IO, c::AnnotatedChar) where {F} = textwriter(io, c.char) + +# Thrown here rather than by dispatch, so that a non-constant style still sees two methods +awrite(::NoStyle, io::IO, m::MIME, x) = throw(MethodError(show, (io, m, x))) +# Via the writer form, so that a non-constant style sees two methods there as well +awrite(io::IO, x) = awrite(write, style(x), io, x) + +Base.write(io::IO, s::Union{AnnotatedString{S}, SubString{<:AnnotatedString{S}}}) where {S} = + awrite(io, s)::Int Base.write(io::IO, c::AnnotatedChar) = - ansi_write_(write, io, c)::Int + awrite(io, c)::Int function Base.write(io::IO, aio::AnnotatedIOBuffer{V}) where {V} if get(io, :color, false) == true @@ -377,7 +474,7 @@ function Base.write(io::IO, aio::AnnotatedIOBuffer{V}) where {V} # writing from an AnnotatedIOBuffer with style. # In the meantime, by converting to an `AnnotatedString` we can just # reuse all the work done to make that work. - ansi_write_(write, io, read(aio, AnnotatedString{String, V}))::Int + awrite(io, read(aio, AnnotatedString{String, V}))::Int else write(io, aio.io) end @@ -385,18 +482,12 @@ end # Print +# Via `write`, which keeps the annotations when the destination can hold them Base.print(io::IO, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = - (ansi_write_(write, io, s); nothing) - + (write(io, s); nothing) Base.print(io::IO, s::AnnotatedChar) = - (ansi_write_(write, io, s); nothing) - -Base.print(io::AnnotatedIOBuffer, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = (write(io, s); nothing) -Base.print(io::AnnotatedIOBuffer, c::AnnotatedChar) = - (write(io, c); nothing) - styled_print(io::AnnotatedIOBuffer, msg::Any, kwargs::Any) = print(io, msg...) styled_print_(io::AnnotatedIOBuffer, @nospecialize(msg), @nospecialize(kwargs)) = @@ -407,25 +498,68 @@ Base.printstyled(io::AnnotatedIOBuffer, msg...; kwargs...) = # Escape -Base.escape_string(io::IO, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}, - esc = ""; keep = (), ascii::Bool=false, fullhex::Bool=false) = - (ansi_write_((io, s) -> escape_string(io, s, esc; keep, ascii, fullhex), io, s); nothing) +function Base.escape_string(io::IO, s::AnnotatedStr, esc = ""; keep = (), ascii::Bool=false, fullhex::Bool=false) + aio = first(Base.unwrapcontext(io)) # Qualified, as `show.jl` defines it after this file + escape_annotated(aio, io, s, esc, keep, ascii, fullhex) + nothing +end + +# Into an annotated buffer, the escaped text keeps the annotations +function escape_annotated(aio::AnnotatedIOBuffer{V}, _, s, esc, keep, ascii, fullhex) where {V} + annots = convert(Vector{RegionAnnotation{V}}, annotations(s)) # As `write` does, before any text is written + ends, outs = Int[0], Int[position(aio)] # Where each region ends in `s`, and where its escaped text ends in `aio` + for (text, _) in eachregion(s) + escape_string(aio, text, esc; keep, ascii, fullhex) + push!(ends, last(ends) + ncodeunits(text)) + push!(outs, position(aio)) + end + eof(aio) || _clear_annotations_in_region!(aio.annotations, first(outs)+1:last(outs)) + outpos(i) = outs[searchsortedlast(ends, clamp(i, 0, last(ends)))] # Where byte `i` of `s` ends in `aio`, clipped to `s` + for (; region, label, value) in annots # A substring's annotations may reach beyond it + annotate!(aio, outpos(first(region) - 1) + 1:outpos(last(region)), label, value) + end +end +escape_annotated(_, io, s, esc, keep, ascii, fullhex) = + awrite(style(s), io, s) do io, str + escape_string(io, str, esc; keep, ascii, fullhex) + end # Show -show_annot(io::IO, ::Any) = nothing -show_annot(io::IO, ::MIME, ::Any) = nothing +Base.show(io::IO, m::MIME"text/html", s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = + (awrite(style(s), io, m, s); nothing) +Base.show(io::IO, m::MIME"text/html", c::AnnotatedChar) = show(io, m, annotatedstring(c)) + +function Base.showable(m::MIME"text/html", x::Union{AnnotatedStr, AnnotatedChar}) + s = style(x) + if s === DynamicStyle() + s = invoke_in_world(tls_world_age(), valuestyle, x) + end + written = if x isa AnnotatedChar AnnotatedString{String, annot_valtype(x)} else typeof(x) end + s !== NoStyle() && hasmethod(awrite, Tuple{typeof(s), IO, typeof(m), written}) +end + +# Dynamic styles -show_annot_(io::IO, @nospecialize(x::Any)) = - invoke_in_world(tls_world_age(), show_annot, io, x)::Nothing +# An `Any` value type says nothing about display, so `DynamicStyle` works the style out from +# the values held. It does this in the latest world, so that new style methods do not +# invalidate compiled callers of `Any`-valued strings. +struct DynamicStyle <: AbstractAnnotationStyle end +AnnotationStyle(::Type{Any}) = DynamicStyle() -show_annot_(io::IO, m::MIME, @nospecialize(x::Any)) = - invoke_in_world(tls_world_age(), show_annot, io, m, x)::Nothing +valuestyle(x) = mapfoldl(a -> AnnotationStyle(typeof(a.value)), promotestyle, annotations(x), init = NoStyle()) -Base.show(io::IO, m::MIME"text/html", s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = - show_annot_(io, m, s) +dynamic(g, @nospecialize(x), args...) = g(valuestyle(x), args...) -Base.show(io::IO, m::MIME"text/html", c::AnnotatedChar) = - show_annot_(io, m, c) +awrite(textwriter::F, ::DynamicStyle, io::IO, @nospecialize(s::AnnotatedStr)) where {F} = + if isempty(annotations(if s isa SubString s.string else s end)) # Plain text, with no style to find + textwriter(io, unannotate(s)) + else + invoke_in_world(tls_world_age(), dynamic, Fix1(awrite, textwriter), s, io, s) + end +awrite(textwriter::F, ::DynamicStyle, io::IO, @nospecialize(c::AnnotatedChar)) where {F} = + invoke_in_world(tls_world_age(), dynamic, Fix1(awrite, textwriter), c, io, c) +awrite(::DynamicStyle, io::IO, m::MIME, @nospecialize(x)) = + invoke_in_world(tls_world_age(), dynamic, awrite, x, io, m, x) end diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index fffe8396d75e7..13245cdf86b5f 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -140,7 +140,7 @@ end end @testset "Unstyled display fallback" begin - # In a fresh process, so that StyledStrings' `ansi_write` methods can't mask the fallback + # In a fresh process, so that StyledStrings' display methods can't mask the fallback script = """ s = SubString(Base.AnnotatedString("αb", [(1:3, :face, :red)]), 3, 3) c = Base.AnnotatedChar('γ', [(:face, :red)]) @@ -264,6 +264,11 @@ end @test write(aio, ' ') == 1 @test write(aio, Base.AnnotatedString("world", [(1:5, :tag, 2)])) == 5 @test Base.annotations(aio) == vec2ann([(1:5, :tag, 1), (7:11, :tag, 2)]) + # Printing through a context keeps the annotations + ctxaio = Base.AnnotatedIOBuffer() + print(IOContext(ctxaio, :color => true), Base.AnnotatedString("hi", [(1:2, :tag, 3)])) + print(IOContext(ctxaio), Base.AnnotatedChar('!', [(:tag, 4)])) + @test Base.annotations(ctxaio) == vec2ann([(1:2, :tag, 3), (3:3, :tag, 4)]) # Check `annotate!`, including region sorting @test truncate(aio, 0).io.size == 0 @test write(aio, "hello world") == ncodeunits("hello world") @@ -947,3 +952,121 @@ end end end end + +# A package with its own annotation value type and a style to display it. +module MarkedAnnotations + import Base.AnnotatedDisplay: AbstractAnnotationStyle, AnnotationStyle, awrite + struct Mark end + struct Mark2 end # A second value type displayed the same way + struct Tag end # A value type of another package, with its own style + struct Note end # As `Tag`, but with no agreed style against `Mark` + struct Hilite end # Whose style writes HTML for whole strings only + struct Gloss end # Whose rule against `Mark` is written in the orientation its unions don't present + struct MarkStyle <: AbstractAnnotationStyle end + struct TagStyle <: AbstractAnnotationStyle end + struct NoteStyle <: AbstractAnnotationStyle end + struct HiliteStyle <: AbstractAnnotationStyle end + struct GlossStyle <: AbstractAnnotationStyle end + AnnotationStyle(::Type{Mark}) = MarkStyle() + AnnotationStyle(::Type{Mark2}) = MarkStyle() + AnnotationStyle(::Type{Tag}) = TagStyle() + AnnotationStyle(::Type{Note}) = NoteStyle() + AnnotationStyle(::Type{Hilite}) = HiliteStyle() + AnnotationStyle(::Type{Gloss}) = GlossStyle() + AnnotationStyle(a::MarkStyle, ::TagStyle) = a + AnnotationStyle(a::MarkStyle, ::GlossStyle) = a + function awrite(textwriter::F, ::MarkStyle, io::IO, s::Union{Base.AnnotatedString, SubString{<:Base.AnnotatedString}}) where {F} + buf = IOBuffer() + for (str, annots) in Base.eachregion(s) + isempty(annots) || write(buf, "<") + textwriter(buf, str) + isempty(annots) || write(buf, ">") + end + write(io, take!(buf)) + end + awrite(::MarkStyle, io::IO, ::MIME"text/html", s) = sum(Base.eachregion(s)) do (str, annots) + if isempty(annots) write(io, str) else write(io, "") + write(io, str) + write(io, "") end + end + awrite(::HiliteStyle, io::IO, ::MIME"text/html", s::Base.AnnotatedString) = write(io, "", String(s), "") +end + +@testset "AnnotationStyle" begin + (; Mark, MarkStyle) = MarkedAnnotations + AnnotationStyle, NoStyle = Base.AnnotatedDisplay.AnnotationStyle, Base.AnnotatedDisplay.NoStyle + @test AnnotationStyle(String) === NoStyle() + @test AnnotationStyle(Mark) === MarkStyle() + @test AnnotationStyle(Union{Mark, String}) === MarkStyle() + @test AnnotationStyle(Union{String, Int, Mark}) === MarkStyle() + @test AnnotationStyle(Union{String, Int}) === NoStyle() + @test Base.AnnotatedDisplay.promotestyle(MarkStyle(), NoStyle()) === Base.AnnotatedDisplay.promotestyle(NoStyle(), MarkStyle()) === MarkStyle() + # Types sharing a style combine; styles that differ have no answer until one is given + @test AnnotationStyle(Union{Mark, MarkedAnnotations.Mark2}) === MarkStyle() + @test_throws ArgumentError AnnotationStyle(Union{Mark, MarkedAnnotations.Note}) + @test AnnotationStyle(Union{Mark, MarkedAnnotations.Tag}) === MarkStyle() # By `MarkedAnnotations` + @test Base.AnnotatedDisplay.promotestyle(MarkedAnnotations.TagStyle(), MarkStyle()) === MarkStyle() # The rule, in the other orientation + # A rule found only in its other orientation settles a union in compiled code, at compile time + glossmark() = AnnotationStyle(Union{MarkedAnnotations.Gloss, Mark}) + @test glossmark() === MarkStyle() + @test Base.infer_return_type(glossmark, ()) === MarkStyle + marked(V) = Base.AnnotatedString{String, V}("x", [(1:1, :m, Mark())]) + @test sprint(print, marked(Mark)) == "" + @test sprint(print, marked(Union{Mark, Int})) == "" + @test sprint(print, marked(Any)) == "" # The style is found from the values + @test sprint(print, Base.AnnotatedString{String, Any}("xy", [(1:1, :t, MarkedAnnotations.Tag()), (2:2, :m, Mark())])) == "" + @test sprint(print, Base.AnnotatedString{String, Int}("x", [(1:1, :n, 1)])) == "x" + @test sprint(print, Base.AnnotatedString{String, Any}("x", [(1:1, :n, 1)])) == "x" + # HTML is available exactly when the style provides it, as for a plain `String` + unstyled = Base.AnnotatedString{String, Int}("ax" + let hilited = Base.AnnotatedString("x", [(1:1, :h, MarkedAnnotations.Hilite())]) + @test showable(MIME("text/html"), hilited) && showable(MIME("text/html"), hilited[1]) + @test !showable(MIME("text/html"), SubString(hilited, 1:1)) + end + # Escaping keeps the annotations on their (longer) text + @test sprint(escape_string, Base.AnnotatedString{String, Mark}("a\nb", [(1:1, :m, Mark()), (3:3, :m, Mark())])) == "\\n" + escaped(s; wrap = identity) = (buf = Base.AnnotatedIOBuffer(); escape_string(wrap(buf), s); read(seekstart(buf), Base.AnnotatedString)) + @test escaped(Base.AnnotatedString("a\nb\tc", [(1:1, :x, 1), (3:3, :y, 2), (2:5, :z, 3)])) == Base.AnnotatedString("a\\nb\\tc", [(1:1, :x, 1), (4:4, :y, 2), (2:7, :z, 3)]) + @test escaped(SubString(Base.AnnotatedString("a\nbc", [(1:3, :x, 1)]), 2, 4)) == Base.AnnotatedString("\\nbc", [(1:3, :x, 1)]) + @test escaped(Base.AnnotatedString("x\ty", [(2:2, :t, 1)]); wrap = io -> IOContext(io, :color => true)) == Base.AnnotatedString("x\\ty", [(2:3, :t, 1)]) + let buf = Base.AnnotatedIOBuffer{Float64}() # Values are converted to the buffer's value type, as by `write` + escape_string(buf, Base.AnnotatedString("ab", [(1:1, :x, 1)])) + @test Base.annotations(buf) == [(region = 1:1, label = :x, value = 1.0)] + end + # The style of a known value type is resolved at compile time + folded(V) = only(code_typed(Base.AnnotatedDisplay.style, (Base.AnnotatedString{String, V},)))[1].code + @test folded(Mark) == Any[Core.ReturnNode(MarkStyle())] + @test folded(Union{Mark, Int}) == Any[Core.ReturnNode(MarkStyle())] + # Defining a style, or a writer for one, must not invalidate code compiled for strings of + # unknown value type. Checked in a fresh process, where only Base's methods exist as when a + # package is first loaded; here the loaded packages' methods would make inference give up + # for that reason alone. + guard = """ + import Base.AnnotatedDisplay: AbstractAnnotationStyle, AnnotationStyle, awrite + struct Late end + struct LateStyle <: AbstractAnnotationStyle end + struct Sink <: IO end + Base.write(::Sink, ::UInt8) = 1 + Base.unsafe_write(::Sink, ::Ptr{UInt8}, n::UInt) = Int(n) + # A new IO type and an unusual string type, so the print chain is compiled here rather than + # taken from the sysimage; `v[1]` is inferred with the value type unbound. + unknown_valtype(v::Vector{Base.AnnotatedString{SubString{String}}}) = print(Sink(), v[1]) + unknown_html(v::Vector{Base.AnnotatedString{SubString{String}}}) = show(Sink(), MIME("text/html"), v[1]) + strings = Base.AnnotatedString{SubString{String}}[Base.AnnotatedString{SubString{String}, Int}(SubString("x"))] + unknown_valtype(strings) + try unknown_html(strings) catch end # compiled, then correctly a `MethodError` without a style + log = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1) + AnnotationStyle(::Type{Late}) = LateStyle() + awrite(textwriter, ::LateStyle, io::IO, s::Base.AnnotatedString) = 0 + awrite(::LateStyle, io::IO, ::MIME"text/html", s::Base.AnnotatedString) = 0 + ccall(:jl_debug_method_invalidation, Any, (Cint,), 0) + print(join(unique(x.def.name for x in log if x isa Core.MethodInstance), ' ')) + """ + invalidated = split(read(`$(Base.julia_cmd()) --startup-file=no -e $guard`, String)) + @test invalidated ⊆ ["AnnotationStyle", "awrite"] +end From ea10a54f203a1bb4966c31c2aa3c0b20dc71da82 Mon Sep 17 00:00:00 2001 From: TEC Date: Tue, 22 Sep 2026 01:06:20 +0800 Subject: [PATCH 03/23] Make region iteration lazy `eachregion` built every region up front: a vector of ranges, and a fresh vector of annotations for each region, all before yielding anything. The iterator now walks the annotation events as it goes, keeping a sorted list of the annotations active over the current region. Each region is yielded as a substring, with its annotations as a view of that list, `ActiveAnnotations`. A view held after the iterator has moved on still works; it finds its annotations again by position, at a cost linear in the number of annotations. Events are packed into integers, so sorting them is a radix sort. The active list is updated by shifting entries within it, and `iterate` is inlined so the sweep's state stays in registers. A region now costs 8 ns in order and 15 ns under a nested annotation, down from 20 and 35, with nothing allocated per region. A subregion or an annotation that starts within a character is moved back to the start of that character, as the events are. The views read the string's annotations when they are used, so a string must not change while its regions are in use. If it does, reads stay in bounds: the iterator keeps the event layout it was built with, and an annotation index that is out of range is a `BoundsError`. Assisted-by: Claude Code (Fable 5.1) --- base/strings/annotated.jl | 185 ++++++++++++++++++++++---------------- test/strings/annotated.jl | 35 ++++++++ 2 files changed, 142 insertions(+), 78 deletions(-) diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index 962fd0a6b0163..c2ff58c13006f 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -571,108 +571,137 @@ end struct RegionIterator{S <: AbstractString, V} str::S - regions::Vector{UnitRange{Int}} - annotations::Vector{Vector{Annotation{V}}} + annotations::Vector{RegionAnnotation{V}} + events::Vector{UInt64} # Each an annotation's start or end, see `annotation_events` + shift::UInt # Bit offset of the positions in `events`, fixed when they were made + subregion::UnitRange{Int} end -Base.length(si::RegionIterator) = length(si.regions) +# One pass over the regions, so that passes made at once don't share their state +mutable struct RegionCursor{S, V} + const regions::RegionIterator{S, V} + const active::Vector{Int} # Sorted indices of the annotations over the current region + pos::Int # Where the current region starts +end -Base.@propagate_inbounds function Base.iterate(si::RegionIterator, i::Integer=1) - if i <= length(si.regions) - @inbounds ((SubString(si.str, si.regions[i]), si.annotations[i]), i+1) - end +# The annotations of one region. While the cursor is still on that region they are read +# from its state; afterwards they are found again by position, in time linear in the annotations. +struct ActiveAnnotations{S, V} <: AbstractVector{Annotation{V}} + cursor::RegionCursor{S, V} + pos::Int end -Base.eltype(::RegionIterator{S, V}) where { S <: AbstractString, V} = - Tuple{SubString{S}, Vector{Annotation{V}}} +IndexStyle(::Type{<:ActiveAnnotations}) = IndexLinear() +# An annotation that starts within a character covers the region of that character +function size(a::ActiveAnnotations) + a.cursor.pos == a.pos && return (length(a.cursor.active),) + stop = nextind(a.cursor.regions.str, a.pos) + (count(annot -> !isempty(annot.region) && first(annot.region) < stop && a.pos <= last(annot.region), + a.cursor.regions.annotations),) +end +function getindex(a::ActiveAnnotations{S, V}, i::Int) where {S, V} + @boundscheck checkbounds(a, i) + a.cursor.pos == a.pos || return first(Iterators.drop(a, i - 1)) + (; label, value) = a.cursor.regions.annotations[@inbounds a.cursor.active[i]] + Annotation{V}((label, value)) +end +# For a region the cursor has left, one pass over the annotations rather than a scan per element +function iterate(a::ActiveAnnotations{S, V}, i::Int = 1) where {S, V} + a.cursor.pos == a.pos && return if i <= length(a); (@inbounds(a[i]), i + 1) end + annots = a.cursor.regions.annotations + stop = nextind(a.cursor.regions.str, a.pos) + index = findnext(annot -> !isempty(annot.region) && first(annot.region) < stop && a.pos <= last(annot.region), + annots, i) + isnothing(index) && return nothing + (; label, value) = @inbounds annots[index] + (Annotation{V}((label, value)), index + 1) +end + +IteratorSize(::Type{<:RegionIterator}) = SizeUnknown() +eltype(::Type{RegionIterator{S, V}}) where {S, V} = Tuple{SubString{S}, ActiveAnnotations{S, V}} +eltype(::Type{RegionIterator{SubString{S}, V}}) where {S, V} = Tuple{SubString{S}, ActiveAnnotations{SubString{S}, V}} + +@inline function iterate(r::RegionIterator, (i, pos, cursor) = (1, first(r.subregion), RegionCursor(r, Int[], 0))) + pos > last(r.subregion) && return nothing + (; events, shift) = r + (; active) = cursor + while i <= length(events) && events[i] >> shift == pos + event = events[i] + index, isstart = Int(event & (UInt64(1) << shift - 1)) >> 1, isodd(event) + at = searchsortedfirst(active, index) + if isstart # Shift the entries by hand: `insert!` and `deleteat!` near the front reallocate on alternate calls + push!(active, index) + copyto!(active, at + 1, active, at, length(active) - at) + active[at] = index + else + copyto!(active, at, active, at + 1, length(active) - at) + pop!(active) + end + i += 1 + end + cursor.pos = pos + next = if i <= length(events) + Int(events[i] >> shift) + else # To the end of the character the subregion ends in + nextind(r.str, thisind(r.str, last(r.subregion))) + end + ((@inbounds(raw_substring(r.str, pos, next - pos)), ActiveAnnotations(cursor, pos)), (i, next, cursor)) +end """ eachregion(s::AnnotatedString{S}) eachregion(s::SubString{AnnotatedString{S}}) -Identify the contiguous substrings of `s` with a constant annotations, and return -an iterator which provides each substring and the applicable annotations as a -`Tuple{SubString{S}, Vector{$Annotation}}`. +Identify the contiguous substrings of `s` with constant annotations, and return an +iterator which provides each substring and the applicable annotations as a +`Tuple{SubString{S}, AbstractVector{$Annotation}}`. + +The annotations of each region are read from `s` when they are used, so `s` must +not be changed while its regions are in use. # Examples ```jldoctest; setup=:(using Base: AnnotatedString, eachregion) -julia> collect(eachregion(AnnotatedString( - "hey there", [(1:3, :face, :bold), - (5:9, :face, :italic)]))) -3-element Vector{Tuple{SubString{String}, Vector{$Annotation}}}: - ("hey", [$Annotation((:face, :bold))]) - (" ", []) - ("there", [$Annotation((:face, :italic))]) +julia> a = AnnotatedString("hey there", [(1:3, :face, :bold), (5:9, :face, :italic)]); + +julia> map(collect, eachregion(a)) +3-element Vector{Vector{Any}}: + ["hey", [(label = :face, value = :bold)]] + [" ", @NamedTuple{label::Symbol, value::Symbol}[]] + ["there", [(label = :face, value = :italic)]] ``` """ function eachregion(s::AnnotatedString{S, V}, subregion::UnitRange{Int}=firstindex(s):lastindex(s)) where {S, V} - isempty(s) || isempty(subregion) && - return RegionIterator(s.string, UnitRange{Int}[], Vector{Annotation{V}}[]) - events = annotation_events(s, subregion) - isempty(events) && return RegionIterator(s.string, [subregion], [Annotation{V}[]]) - annotvals = Annotation{V}[ - (; label, value) for (; label, value) in annotations(s)] - regions = Vector{UnitRange{Int}}() - annots = Vector{Vector{Annotation{V}}}() - pos = first(events).pos - if pos > first(subregion) - push!(regions, thisind(s, first(subregion)):prevind(s, pos)) - push!(annots, Annotation[]) - end - activelist = Int[] - for event in events - if event.pos != pos - push!(regions, pos:prevind(s, event.pos)) - push!(annots, annotvals[activelist]) - pos = event.pos - end - if event.active - insert!(activelist, searchsortedfirst(activelist, event.index), event.index) - else - deleteat!(activelist, searchsortedfirst(activelist, event.index)) - end - end - if last(events).pos < nextind(s, last(subregion)) - push!(regions, last(events).pos:thisind(s, last(subregion))) - push!(annots, Annotation[]) - end - RegionIterator(s.string, regions, annots) + # Whole characters, so a region's annotations read the same during iteration and after it + isempty(subregion) || (subregion = thisind(s, first(subregion)):nextind(s, thisind(s, last(subregion))) - 1) + RegionIterator{S, V}(s.string, s.annotations, annotation_events(s, subregion), + unsigned(top_set_bit(2 * length(s.annotations) + 1)), subregion) end -function eachregion(s::SubString{AnnotatedString{S, V}}, pos::UnitRange{Int}=firstindex(s):lastindex(s)) where {S, V} - if isempty(s) - RegionIterator(s.string, Vector{UnitRange{Int}}(), Vector{Vector{Annotation{V}}}()) - else - eachregion(s.string, first(pos)+s.offset:last(pos)+s.offset) - end -end +eachregion(s::SubString{AnnotatedString{S, V}}, pos::UnitRange{Int}=firstindex(s):lastindex(s)) where {S, V} = + eachregion(s.string, first(pos)+s.offset:last(pos)+s.offset) """ - annotation_events(string::AbstractString, annots::Vector{$RegionAnnotation}, subregion::UnitRange{Int}) annotation_events(string::AnnotatedString, subregion::UnitRange{Int}) -Find all annotation "change events" that occur within a `subregion` of `annots`, -with respect to `string`. When `string` is styled, `annots` is inferred. +Find all annotation "change events" that occur within a `subregion` of `string`. -Each change event is given in the form of a `@NamedTuple{pos::Int, active::Bool, -index::Int}` where `pos` is the position of the event, `active` is a boolean -indicating whether the annotation is being activated or deactivated, and `index` -is the index of the annotation in question. +Each change event is a `UInt64` packing the position of the event above the index of +the annotation in question, followed by a bit set when the annotation is being +activated, so that sorting the events orders them by position. The index and bit take +the low `top_set_bit(2length(annotations(string)) + 1)` bits, as the region iterator reads them. """ -function annotation_events(s::AbstractString, annots::Vector{<:RegionAnnotation}, subregion::UnitRange{Int}) - events = Vector{NamedTuple{(:pos, :active, :index), Tuple{Int, Bool, Int}}}() # Position, Active?, Annotation index - sizehint!(events, 2 * length(annots)) - for (i, (; region)) in enumerate(annots) - if !isempty(intersect(subregion, region)) - start, stop = max(first(subregion), first(region)), min(last(subregion), last(region)) - start <= stop || continue # Currently can't handle empty regions - push!(events, (pos=thisind(s, start), active=true, index=i)) - push!(events, (pos=nextind(s, stop), active=false, index=i)) - end +function annotation_events(s::AnnotatedString, subregion::UnitRange{Int}) + events = UInt64[] + sizehint!(events, 2 * length(s.annotations)) + shift = unsigned(top_set_bit(2 * length(s.annotations) + 1)) + leading_zeros(UInt64(ncodeunits(s) + 1)) >= shift || throw(ArgumentError(LazyString( + "too many annotations (", length(s.annotations), ") to order the regions of a ", ncodeunits(s), "-byte string"))) + for (i, (; region)) in enumerate(s.annotations) + start, stop = max(first(subregion), first(region)), min(last(subregion), last(region)) + start <= stop || continue # Outside the subregion, or empty + push!(events, UInt64(thisind(s, start)) << shift | UInt64(i) << 1 | true) + push!(events, UInt64(nextind(s, stop)) << shift | UInt64(i) << 1) end - sort!(events, by=e -> e.pos) + sort!(events) # Integer keys, so an out-of-order list gets a radix sort end - -annotation_events(s::AnnotatedString, subregion::UnitRange{Int}) = - annotation_events(s.string, annotations(s), subregion) diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index 13245cdf86b5f..66a5167bfa9a4 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -435,6 +435,34 @@ end @test annregions("𝟏x", [(1:4, :face, :red)]) == [("𝟏", [(:face, :red)]), ("x", [])] + # Annotations and subregions that start within a character + let str = Base.AnnotatedString("aébc", [(3:3, :face, :red)]) + regions = collect(Base.eachregion(str)) # Read after the iterator has moved on + @test [(s, Tuple.(a)) for (s, a) in regions] == [("a", []), ("é", [(:face, :red)]), ("bc", [])] + @test [s for (s, _) in Base.eachregion(str, 3:5)] == ["é", "bc"] + end + # A subregion starting within a character, whose regions read the same during iteration and after + let str = Base.AnnotatedString("aébcdef", [(1:8, :a, 1), (3:5, :b, 2), (5:5, :c, 3), (2:2, :d, 4)]) + @test [(String(s), Tuple.(a)) for (s, a) in Base.eachregion(str, 3:5)] == + [(String(s), Tuple.(a)) for (s, a) in collect(Base.eachregion(str, 3:5))] + end + # Read after the iterator has left them, past an annotation beyond the end of the string + @test [Tuple.(a) for (_, a) in collect(Base.eachregion(Base.AnnotatedString("ab", [(1:1, :a, 1), (5:6, :b, 2)])))] == [[(:a, 1)], []] + # Passes over one iterator at the same time don't share their state + let regions = Base.eachregion(Base.AnnotatedString("abcd", [(1:2, :a, 1), (2:3, :b, 2)])) + pass = [(String(s), Tuple.(a)) for (s, a) in regions] + @test [(String(s), Tuple.(a)) for ((s, a), _) in zip(regions, regions)] == pass + @test [(String(s), Tuple.(a)) for (s, a) in regions for _ in regions] == repeat(pass, inner = length(pass)) + end + # A string backed by a substring, whose regions are substrings of the same parent + @test length(collect(Base.eachregion(Base.AnnotatedString(SubString("xab", 2:3), [(1:1, :a, 1)])))) == 2 + # Changing a string while its regions are read is unsupported, but memory safe + let str = Base.AnnotatedString("abcdef", [(1:2, :a, 1), (3:4, :a, 2), (5:6, :a, 3)]) + @test count(_ -> (Base.annotate!(str, 1:6, :b, 9); true), Base.eachregion(str)) == 3 + regions = collect(Base.eachregion(str)) + empty!(str.annotations) + @test_throws BoundsError collect(last(regions)[2]) + end end @testset "Replacement" begin @@ -1028,6 +1056,13 @@ end @test showable(MIME("text/html"), hilited) && showable(MIME("text/html"), hilited[1]) @test !showable(MIME("text/html"), SubString(hilited, 1:1)) end + # Regions consumed as they are yielded carry the same annotations as regions held + nested = Base.AnnotatedString("abcdef", [(1:6, :a, Mark()), (2:5, :b, Mark()), (3:4, :c, Mark())]) + streamed = [(String(str), collect(annots)) for (str, annots) in Base.eachregion(nested)] + @test streamed == [(String(str), collect(annots)) for (str, annots) in collect(Base.eachregion(nested))] + @test map(last, streamed) == [[(label = :a, value = Mark())], [(label = :a, value = Mark()), (label = :b, value = Mark())], + [(label = :a, value = Mark()), (label = :b, value = Mark()), (label = :c, value = Mark())], + [(label = :a, value = Mark()), (label = :b, value = Mark())], [(label = :a, value = Mark())]] # Escaping keeps the annotations on their (longer) text @test sprint(escape_string, Base.AnnotatedString{String, Mark}("a\nb", [(1:1, :m, Mark()), (3:3, :m, Mark())])) == "\\n" escaped(s; wrap = identity) = (buf = Base.AnnotatedIOBuffer(); escape_string(wrap(buf), s); read(seekstart(buf), Base.AnnotatedString)) From 8d58551c404785140b1538e7685bb18ca9fd73db Mon Sep 17 00:00:00 2001 From: TEC Date: Tue, 29 Sep 2026 02:21:24 +0800 Subject: [PATCH 04/23] Bump StyledStrings and JuliaSyntaxHighlighting Stdlib: StyledStrings URL: https://github.com/JuliaLang/StyledStrings.jl.git Stdlib branch: face-primacy Old commit: 41c13c2 New commit: b8c9246 Diff: https://github.com/JuliaLang/StyledStrings.jl/compare/41c13c298edbc4e8ba4234bbae599e06c8283791...b8c92465c8912df92d17cf6f9a94be2870680f05 Stdlib: JuliaSyntaxHighlighting URL: https://github.com/JuliaLang/JuliaSyntaxHighlighting.jl.git Stdlib branch: faces-value-parameterisation Old commit: dae5817 New commit: 2061af0 Diff: https://github.com/JuliaLang/JuliaSyntaxHighlighting.jl/compare/dae5817ad3ab9a90d8912153eb665022cb777a92...2061af04bfe6a7c333d486a98423b884148a271e StyledStrings displays faces through the annotation style trait, and names faces with `Face` values held in module palettes (JuliaLang/StyledStrings.jl#131). Links are `Link` values, and `StyledValue` names the value type of styled strings. JuliaSyntaxHighlighting highlights with `Face` values from its own palette. --- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + stdlib/JuliaSyntaxHighlighting.version | 4 ++-- stdlib/StyledStrings.version | 4 ++-- 10 files changed, 8 insertions(+), 8 deletions(-) create mode 100644 deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/md5 create mode 100644 deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/sha512 delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/md5 delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/sha512 delete mode 100644 deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/md5 delete mode 100644 deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/sha512 create mode 100644 deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/md5 create mode 100644 deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/sha512 diff --git a/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/md5 new file mode 100644 index 0000000000000..753419d3ef21a --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/md5 @@ -0,0 +1 @@ +f14f37552be7d275ef7c30d26cf900e7 diff --git a/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/sha512 new file mode 100644 index 0000000000000..6c65735c0a077 --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/sha512 @@ -0,0 +1 @@ +215f952ceada63c6b375dcc6cd62fedce257d8b6835dc57638225941eea810d00003925a3893b8bc0f4c82cb41dbd00baee098080a0246a5fc9a5f335938ee8c diff --git a/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/md5 deleted file mode 100644 index ae83b1b01fc4c..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -e01101fae581c424f776e145a0998196 diff --git a/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/sha512 deleted file mode 100644 index f033fd54e4442..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-dae5817ad3ab9a90d8912153eb665022cb777a92.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -b94bdd91573a26046ec2120ca55de049189850627cae18cd7e66c9c66e42b7523d3a34dab124770b9b5d686304443457de304cdc4f4df1d631265c5601af1946 diff --git a/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/md5 b/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/md5 deleted file mode 100644 index 9144e5269d019..0000000000000 --- a/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -628eee7616058469a454fe3698557d09 diff --git a/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/sha512 b/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/sha512 deleted file mode 100644 index df42d7384c552..0000000000000 --- a/deps/checksums/StyledStrings-41c13c298edbc4e8ba4234bbae599e06c8283791.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -5346ec19f852e544a7c748b746bd2b8f7284a76d326eb6686c8f5a11dd595e0078fd4df5040f42eccee4f5df4e41935dd6ede25beab8ac9957ed892edb171f72 diff --git a/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/md5 b/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/md5 new file mode 100644 index 0000000000000..e94cde402b1ee --- /dev/null +++ b/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/md5 @@ -0,0 +1 @@ +1456e9796ebcf64f77e954555c743671 diff --git a/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/sha512 b/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/sha512 new file mode 100644 index 0000000000000..311c23d198449 --- /dev/null +++ b/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/sha512 @@ -0,0 +1 @@ +1f29ba805b6f947e9939d4b5f6b32836451a31ba29bbe2851f0e6a53f687b68d293b81a5030b1d5786c7fd066306b984fbc81ceaf3704d758eded1bcf8d8aa87 diff --git a/stdlib/JuliaSyntaxHighlighting.version b/stdlib/JuliaSyntaxHighlighting.version index f0a4325b2e51f..d1918de872b98 100644 --- a/stdlib/JuliaSyntaxHighlighting.version +++ b/stdlib/JuliaSyntaxHighlighting.version @@ -1,4 +1,4 @@ -JULIASYNTAXHIGHLIGHTING_BRANCH = main -JULIASYNTAXHIGHLIGHTING_SHA1 = dae5817ad3ab9a90d8912153eb665022cb777a92 +JULIASYNTAXHIGHLIGHTING_BRANCH = faces-value-parameterisation +JULIASYNTAXHIGHLIGHTING_SHA1 = 2061af04bfe6a7c333d486a98423b884148a271e JULIASYNTAXHIGHLIGHTING_GIT_URL := https://github.com/JuliaLang/JuliaSyntaxHighlighting.jl.git JULIASYNTAXHIGHLIGHTING_TAR_URL = https://api.github.com/repos/JuliaLang/JuliaSyntaxHighlighting.jl/tarball/$1 diff --git a/stdlib/StyledStrings.version b/stdlib/StyledStrings.version index f031b8de12286..1abddf56b76eb 100644 --- a/stdlib/StyledStrings.version +++ b/stdlib/StyledStrings.version @@ -1,4 +1,4 @@ -STYLEDSTRINGS_BRANCH = main -STYLEDSTRINGS_SHA1 = 41c13c298edbc4e8ba4234bbae599e06c8283791 +STYLEDSTRINGS_BRANCH = face-primacy +STYLEDSTRINGS_SHA1 = b8c92465c8912df92d17cf6f9a94be2870680f05 STYLEDSTRINGS_GIT_URL := https://github.com/JuliaLang/StyledStrings.jl.git STYLEDSTRINGS_TAR_URL = https://api.github.com/repos/JuliaLang/StyledStrings.jl/tarball/$1 From db98e85baff164d0d0840eca8177429b0d91652f Mon Sep 17 00:00:00 2001 From: TEC Date: Thu, 25 Dec 2025 09:24:12 +0800 Subject: [PATCH 05/23] Use faces from palettes in the stdlib Markdown, Profile and the REPL named their faces with prefixed `Symbol`s, registered with `addface!`. They now declare their faces with `@defpalette`, register them from `__init__`, and refer to them as `face"name"` or by bare name inside `styled""`. The REPL's styling passes, Markdown and Profile build their strings with `StyledValue` values, as styled strings have, so that they share one value type and the code specialised on it, and redisplaying the input takes the static display path. Each styling pass must return such a string, a `StylingPasses.StyledString`. Markdown keeps only the annotations of interpolated text that its buffer can hold, and marks its links with `StyledStrings.Link`. Markdown also never styled `ERROR:` in REPL examples, as the check compared an annotated substring with a plain string. It now compares the plain text. The REPL's history search styled the character before each condition by its byte offset, which threw when that character took more than one byte. It now styles it only where a single-byte prefix or separator can be. Assisted-by: Claude Code (Opus 5.5) --- stdlib/Markdown/src/Markdown.jl | 44 +++++----- .../src/render/terminal/formatting.jl | 4 +- stdlib/Markdown/src/render/terminal/render.jl | 86 ++++++++++--------- stdlib/Markdown/test/runtests.jl | 26 ++++-- stdlib/Profile/src/Profile.jl | 8 +- stdlib/REPL/src/History/History.jl | 24 +++--- stdlib/REPL/src/History/display.jl | 77 ++++++++--------- stdlib/REPL/src/History/resumablefiltering.jl | 28 +++--- stdlib/REPL/src/LineEdit.jl | 2 +- stdlib/REPL/src/StylingPasses.jl | 18 ++-- stdlib/REPL/src/precompile.jl | 1 + stdlib/REPL/test/lineedit.jl | 3 + stdlib/REPL/test/repl.jl | 12 ++- 13 files changed, 179 insertions(+), 154 deletions(-) diff --git a/stdlib/Markdown/src/Markdown.jl b/stdlib/Markdown/src/Markdown.jl index 7cf82bfd257ef..16f5b0ab7e030 100644 --- a/stdlib/Markdown/src/Markdown.jl +++ b/stdlib/Markdown/src/Markdown.jl @@ -12,12 +12,31 @@ module Markdown import Base: ==, AnnotatedIOBuffer, AnnotatedString, mapany, show using Base64: stringmime -using StyledStrings: @styled_str, Face, StyledStrings, addface!, styled -using JuliaSyntaxHighlighting: highlight! +using StyledStrings: StyledStrings, @defpalette, @registerpalette, @face_str, Face, StyledValue, @styled_str, styled +using JuliaSyntaxHighlighting: highlight, highlight! # Margin for printing in terminal. const margin = 2 +@defpalette begin + header = Face(weight = :bold) + h1 = Face(height=1.25, inherit = header) + h2 = Face(height=1.20, inherit = header) + h3 = Face(height=1.15, inherit = header) + h4 = Face(height=1.12, inherit = header) + h5 = Face(height=1.08, inherit = header) + h6 = Face(height=1.05, inherit = header) + admonition = Face(weight = :bold) + code = Face(inherit = StyledStrings.code) + julia_prompt = Face(slant = :italic, foreground = bright_green, inherit = REPL_prompt_julia) + footnote = Face(inherit = bright_yellow) + hrule = Face(inherit = shadow) + inlinecode = Face(inherit = code) + latex = Face(inherit = magenta) + link = Face(underline = bright_blue) + list = Face(foreground = blue) +end + include("parse/config.jl") include("parse/util.jl") include("parse/parse.jl") @@ -37,26 +56,7 @@ export @doc_str, @md_str public MD, parse -const MARKDOWN_FACES = [ - :markdown_header => Face(weight=:bold), - :markdown_h1 => Face(height=1.25, inherit=:markdown_header), - :markdown_h2 => Face(height=1.20, inherit=:markdown_header), - :markdown_h3 => Face(height=1.15, inherit=:markdown_header), - :markdown_h4 => Face(height=1.12, inherit=:markdown_header), - :markdown_h5 => Face(height=1.08, inherit=:markdown_header), - :markdown_h6 => Face(height=1.05, inherit=:markdown_header), - :markdown_admonition => Face(weight=:bold), - :markdown_code => Face(inherit=:code), - :markdown_julia_prompt => Face(slant=:italic, foreground=:bright_green, inherit=:repl_prompt_julia), - :markdown_footnote => Face(inherit=:bright_yellow), - :markdown_hrule => Face(inherit=:shadow), - :markdown_inlinecode => Face(inherit=:markdown_code), - :markdown_latex => Face(inherit=:magenta), - :markdown_link => Face(underline=:bright_blue), - :markdown_list => Face(foreground=:blue), -] - -__init__() = foreach(addface!, MARKDOWN_FACES) +__init__() = @registerpalette parse(markdown::String; flavor = julia) = parse(IOBuffer(markdown), flavor = flavor) diff --git a/stdlib/Markdown/src/render/terminal/formatting.jl b/stdlib/Markdown/src/render/terminal/formatting.jl index 09dff9e367631..812075c8e4820 100644 --- a/stdlib/Markdown/src/render/terminal/formatting.jl +++ b/stdlib/Markdown/src/render/terminal/formatting.jl @@ -1,9 +1,9 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -const AnnotIO = Union{AnnotatedIOBuffer, IOContext{AnnotatedIOBuffer}} +const AnnotIO = Union{AnnotatedIOBuffer{V}, IOContext{AnnotatedIOBuffer{V}}} where V function annotprint(f::Function, args...) - buf = AnnotatedIOBuffer() + buf = AnnotatedIOBuffer{StyledValue}() f(buf, args...) read(seekstart(buf), AnnotatedString) end diff --git a/stdlib/Markdown/src/render/terminal/render.jl b/stdlib/Markdown/src/render/terminal/render.jl index 26586c97dd043..5255af6a04058 100644 --- a/stdlib/Markdown/src/render/terminal/render.jl +++ b/stdlib/Markdown/src/render/terminal/render.jl @@ -42,34 +42,32 @@ function term(io::IO, md::BlockQuote, columns) end end +const ADMONITION_FACES = Dict( + "danger" => face"error", + "warning" => face"warning", + "info" => face"info", + "note" => face"info", + "tip" => face"tip", + "compat" => face"bright_cyan", + "todo" => face"magenta", +) + function term(io::IO, md::Admonition, columns) - accent = if md.category == "danger" - :error - elseif md.category in ("info", "note") - :info - elseif md.category in ("warning", "tip") - Symbol(md.category) - elseif md.category == "compat" - :bright_cyan - elseif md.category == "todo" - :magenta - else - :default - end + accent = get(ADMONITION_FACES, md.category, face"") title = if isempty(md.title) md.category else md.title end - print(io, ' '^margin, styled"{$accent,markdown_admonition:│ $title}", - '\n', ' '^margin, styled"{$accent,markdown_admonition:│}", '\n') + print(io, ' '^margin, styled"{$accent,admonition:│ $title}", + '\n', ' '^margin, styled"{$accent,admonition:│}", '\n') content = annotprint(term, md.content, columns - 10) lines = split(rstrip(content), '\n') for (i, line) in enumerate(lines) - print(io, ' '^margin, styled"{$accent,markdown_admonition:│}", line) + print(io, ' '^margin, styled"{$accent,admonition:│}", line) i < length(lines) && println(io) end end function term(io::IO, f::Footnote, columns) print(io, ' '^margin, "│ ") - print(io, styled"{markdown_footnote:[^$(f.id)]}") + print(io, styled"{footnote:[^$(f.id)]}") println(io, '\n', ' '^margin, '│') content = annotprint(term, f.text, columns - 10) lines = split(rstrip(content), '\n') @@ -102,8 +100,8 @@ function term(io::IO, md::List, columns, depth::Int = 1) for (i, point) in enumerate(md.items) list_marker = make_list_marker(i) - print(io, ' '^margin, styled"{markdown_list:$list_marker}") - buf = AnnotatedIOBuffer() + print(io, ' '^margin, styled"{list:$list_marker}") + buf = AnnotatedIOBuffer{StyledValue}() if point isa Vector && !isempty(point) for (i, elt) in enumerate(point[1:end-1]) dterm(buf, elt, columns, depth + 1) @@ -133,7 +131,7 @@ const _header_underlines = collect("≡=–-⋅ ") # TODO settle on another option with unicode e.g. "≡=≃–∼⋅" ? function term(io::AnnotIO, md::Header{l}, columns) where l - face = Symbol("markdown_h$l") + face = (face"h1", face"h2", face"h3", face"h4", face"h5", face"h6")[l] underline = _header_underlines[l] pre = ' '^margin line_width = with_output_annotations(io, :face => face) do io @@ -160,21 +158,18 @@ end function term(io::IO, md::Code, _) code = if md.language == "julia" - hl = AnnotatedString(md.code) - StyledStrings.face!(hl, :markdown_code) - highlight!(hl) + highlight(md.code) elseif md.language == "julia-repl" || Base.startswith(md.language, "jldoctest") - hl = AnnotatedString(md.code) - StyledStrings.face!(hl, :markdown_code) + hl = AnnotatedString{String, StyledValue}(md.code) for (; match) in eachmatch(r"(?:^|\n)julia>", hl) - StyledStrings.face!(match, :markdown_julia_prompt) + StyledStrings.face!(match, face"julia_prompt") afterprompt = match.offset + ncodeunits(match) + 1 _, exprend = Meta.parse(md.code, afterprompt, raise = false) highlight!(hl[afterprompt:prevind(md.code, exprend)]) if (nextspace = findnext(' ', md.code, exprend)) |> !isnothing nextword = hl[exprend:prevind(hl, nextspace)] - if nextword == "ERROR:" - StyledStrings.face!(nextword, :error) + if String(nextword) == "ERROR:" + StyledStrings.face!(nextword, face"error") end end end @@ -182,33 +177,38 @@ function term(io::IO, md::Code, _) elseif md.language == "styled" styled(md.code) else - styled"{markdown_code:$(md.code)}" + md.code end lines = split(code, '\n') for (i, line) in enumerate(lines) - print(io, ' '^margin, line) + print(io, ' '^margin, styled"{code:$line}") i < length(lines) && println(io) end end function term(io::IO, tex::LaTeX, _) - print(io, ' '^margin, styled"{markdown_latex:$(tex.formula)}") + print(io, ' '^margin, styled"{latex:$(tex.formula)}") end term(::IO, ::LineBreak, _) = nothing # line breaks already printed between subsequent elements function term(io::IO, ::HorizontalRule, columns) - print(io, ' '^margin, styled"{markdown_hrule:$('─'^(columns - 2margin))}") + print(io, ' '^margin, styled"{hrule:$('─'^(columns - 2margin))}") end function term(io::IO, md::MarkdownElement, columns) - a = IOContext(AnnotatedIOBuffer(), io) + a = IOContext(AnnotatedIOBuffer{StyledValue}(), io) term(a, md, columns) print(io, read(seekstart(a.io), AnnotatedString)) end +term(io::AnnotIO, md::AnnotatedString, _) = show(io, MIME"text/plain"(), holdable(io, md)) term(io::IO, x, _) = show(io, MIME"text/plain"(), x) +# Drop the annotations the buffer cannot hold, such as pre-1.14 face names +holdable(::AnnotIO{V}, s::AnnotatedString) where {V} = + AnnotatedString(s.string, filter(a -> a.value isa V, Base.annotations(s))) + # Inline Content terminline(io::IO, content...) = terminline(io, collect(content)) @@ -223,16 +223,18 @@ function terminline(io::IO, md::AbstractString) print(io, replace(md, r"[ \t\n]+" => ' ')) end +terminline(io::AnnotIO, md::AnnotatedString) = print(io, replace(holdable(io, md), r"[ \t\n]+" => ' ')) + function terminline(io::AnnotIO, md::Bold) - with_output_annotations(io -> terminline(io, md.text), io, :face => :bold) + with_output_annotations(io -> terminline(io, md.text), io, :face => face"bold") end function terminline(io::AnnotIO, md::Italic) - with_output_annotations(io -> terminline(io, md.text), io, :face => :italic) + with_output_annotations(io -> terminline(io, md.text), io, :face => face"italic") end function terminline(io::AnnotIO, md::Strikethrough) - with_output_annotations(io -> terminline(io, md.text), io, :face => :strikethrough) + with_output_annotations(io -> terminline(io, md.text), io, :face => face"strikethrough") end function terminline(io::IO, ::LineBreak) @@ -244,14 +246,14 @@ function terminline(io::IO, md::Image) end function terminline(io::IO, f::Footnote) - print(io, styled"{markdown_footnote:[^$(f.id)]}") + print(io, styled"{footnote:[^$(f.id)]}") end function terminline(io::AnnotIO, md::Link) annots = if occursin(r"^(https?|file)://", md.url) - (:face => :markdown_link, :link => md.url) + (:face => face"link", :link => StyledStrings.Link(md.url)) else - (:face => :markdown_link,) + (:face => face"link",) end with_output_annotations(io -> terminline(io, md.text), io, annots...) end @@ -262,17 +264,17 @@ function terminline(io::IO, code::Code) else code.code end - print(io, styled"{markdown_inlinecode:$body}") + print(io, styled"{inlinecode:$body}") end function terminline(io::IO, tex::LaTeX) - print(io, styled"{markdown_latex:$(tex.formula)}") + print(io, styled"{latex:$(tex.formula)}") end terminline(io::IO, md::HTMLInline) = print(io, md.content) function terminline(io::IO, md::MarkdownElement) - a = IOContext(AnnotatedIOBuffer(), io) + a = IOContext(AnnotatedIOBuffer{StyledValue}(), io) terminline(a, md) print(io, read(seekstart(a.io), AnnotatedString)) end diff --git a/stdlib/Markdown/test/runtests.jl b/stdlib/Markdown/test/runtests.jl index 619b458bd42e9..b3157a7251a62 100644 --- a/stdlib/Markdown/test/runtests.jl +++ b/stdlib/Markdown/test/runtests.jl @@ -1338,26 +1338,42 @@ end @test !occursin(Base.text_colors[:underline], lines[end]) end +using JuliaSyntaxHighlighting # Before the testset, whose `face""` paths are read when it is expanded + @testset "code blocks are faced as code #61456" begin + # Julia code blocks are padded to the full width so the background tint + # spans the block, hence the `rstrip`. function codefaces(md) buf = Base.AnnotatedIOBuffer() show(buf, MIME("text/plain"), md) str = read(seekstart(buf), Base.AnnotatedString) - [(String(str[a.region]), a.value) for a in Base.annotations(str) if a.label === :face] + [(rstrip(String(str[a.region])), a.value) for a in Base.annotations(str) if a.label === :face] end # Syntax highlighting is conservative, so a lone identifier picks up no # highlighting of its own; it must still be faced as code. for lang in ("", "julia", "julia-repl", "jldoctest", "text") @test codefaces(Markdown.MD(Markdown.Code(lang, "VERSION"))) == - [("VERSION", :markdown_code)] + [("VERSION", face"Markdown.code")] end # Highlighting is layered over the code face rather than replacing it. let faces = codefaces(Markdown.MD(Markdown.Code("julia", "f() = 1 # c"))) - @test ("f() = 1 # c", :markdown_code) ∈ faces - @test ("# c", :julia_comment) ∈ faces + @test ("f() = 1 # c", face"Markdown.code") ∈ faces + @test ("# c", face"JuliaSyntaxHighlighting.comment") ∈ faces end - @test ("julia>", :markdown_julia_prompt) ∈ + @test ("julia>", face"Markdown.julia_prompt") ∈ codefaces(Markdown.MD(Markdown.Code("julia-repl", "julia> x"))) + @test ("ERROR:", face"error") ∈ + codefaces(Markdown.MD(Markdown.Code("julia-repl", "julia> x\nERROR: bad"))) +end + +@testset "foreign annotations" begin + legacy = Base.AnnotatedString("hi", [(1:2, :face, :red), (1:2, :tag, 1)]) + @test sprint(show, MIME("text/plain"), Markdown.MD(Markdown.Paragraph(Any["a ", legacy]))) == " a hi" + # On its own in a block, as an interpolated value is + for block in (Markdown.Admonition("note", "", Any[legacy]), Markdown.BlockQuote(Any[legacy]), + Markdown.Admonition("note", "", Any[Markdown.List(Any[Any[legacy]])])) + @test contains(sprint(show, MIME("text/plain"), Markdown.MD(block)), "hi") + end end @testset "table rendering with term #25213" begin diff --git a/stdlib/Profile/src/Profile.jl b/stdlib/Profile/src/Profile.jl index 6a4ca01722a27..f37033d25e35a 100644 --- a/stdlib/Profile/src/Profile.jl +++ b/stdlib/Profile/src/Profile.jl @@ -39,7 +39,7 @@ public clear, import Base.StackTraces: lookup, UNKNOWN, show_spec_linfo, StackFrame import Base: AnnotatedString -using StyledStrings: @styled_str +using StyledStrings: @styled_str, StyledValue const nmeta = 4 # number of metadata fields per block (threadid, taskid, cpu_cycle_clock, thread_sleeping) @@ -89,7 +89,7 @@ end # An internal function called to show the report after an information request (SIGINFO or SIGUSR1). function _peek_report() - iob = Base.AnnotatedIOBuffer() + iob = Base.AnnotatedIOBuffer{StyledValue}() ioc = IOContext(IOContext(iob, stderr), :displaysize=>displaysize(stderr)) print(ioc, groupby = [:thread, :task]) Base.print(stderr, read(seekstart(iob), AnnotatedString)) @@ -1090,7 +1090,7 @@ function tree_format(frames::Vector{<:StackFrameTree}, level::Int, cols::Int, ma ndigline = ndigits(maximum(frame.frame.line for frame in frames)) + 6 ntext = max(30, cols - ndigoverhead - nindent - ndigcounts - ndigline - 6) widthfile = 2*ntext÷5 # min 12 - strs = Vector{AnnotatedString{String}}(undef, length(frames)) + strs = Vector{AnnotatedString{String, StyledValue}}(undef, length(frames)) showextra = false if level > nindent nextra = level - nindent @@ -1317,7 +1317,7 @@ end function print_tree(io::IO, bt::StackFrameTree{T}, cols::Int, fmt::ProfileFormat, is_subsection::Bool) where T maxes = maxstats(bt) filenamemap = FileNameMap() - worklist = [(bt, 0, 0, AnnotatedString(""))] + worklist = [(bt, 0, 0, AnnotatedString{String, StyledValue}(""))] if !is_subsection Base.print(io, "Overhead ╎ [+additional indent] Count File:Line Function\n") Base.print(io, "=========================================================\n") diff --git a/stdlib/REPL/src/History/History.jl b/stdlib/REPL/src/History/History.jl index 8ecf5b1e4730d..63b4832eac580 100644 --- a/stdlib/REPL/src/History/History.jl +++ b/stdlib/REPL/src/History/History.jl @@ -4,7 +4,7 @@ module History using ..REPL: REPL -using StyledStrings: @styled_str as @S_str, Face, addface!, face!, annotations, AnnotatedIOBuffer, AnnotatedString, AnnotatedChar +using StyledStrings: @styled_str as @S_str, @face_str, @defpalette, @registerpalette, face!, Face, StyledValue, annotations, AnnotatedString, AnnotatedChar using JuliaSyntaxHighlighting: highlight using Base.Threads using Dates @@ -13,16 +13,16 @@ import InteractiveUtils export HistoryFile, HistEntry, update!, runsearch -const FACES = ( - :REPL_History_search_separator => Face(foreground=:blue), - :REPL_History_search_prefix => Face(foreground=:magenta), - :REPL_History_search_selected => Face(foreground=:blue), - :REPL_History_search_unselected => Face(foreground=:grey), - # :REPL_History_search_preview_box => Face(foreground=:grey), - :REPL_History_search_hint => Face(foreground=:magenta, slant=:italic, weight=:light), - :REPL_History_search_results => Face(inherit=:shadow), - :REPL_History_search_match => Face(weight = :bold, underline = true), -) +@defpalette begin + search_separator = Face(foreground = blue) + search_prefix = Face(foreground = magenta) + search_selected = Face(foreground = blue) + search_unselected = Face(foreground = grey) + # search_preview_box = Face(foreground = grey) + search_hint = Face(foreground = magenta, slant = :italic, weight = :light) + search_results = Face(inherit = shadow) + search_match = Face(weight = :bold, underline = true) +end include("histfile.jl") include("resumablefiltering.jl") @@ -30,6 +30,6 @@ include("prompt.jl") include("display.jl") include("search.jl") -__init__() = foreach(addface!, FACES) +__init__() = @registerpalette end diff --git a/stdlib/REPL/src/History/display.jl b/stdlib/REPL/src/History/display.jl index 8dedd48f8b923..f4b8ef935ed26 100644 --- a/stdlib/REPL/src/History/display.jl +++ b/stdlib/REPL/src/History/display.jl @@ -18,7 +18,7 @@ const EMPTY_STATE = SelectorState((0, 0), "", FilterSpec(), [], 0, (active = Int const LABELS = ( gatherdivider = S"{italic:carried over}", preview_suggestion = S"Ctrl+S to save", - help_prompt = S"{REPL_History_search_hint,shadow:try {REPL_History_search_hint,(slant=normal):?} for help} ", + help_prompt = S"{search_hint,shadow:try {search_hint,(slant=normal):?} for help} ", ) const SYNC_UPDATE_BEGIN = "\eP=1s\e\\" @@ -154,10 +154,10 @@ end const BASE_MODE = :julia const MODE_FACES = Dict( - :julia => :green, - :shell => :red, - :pkg => :blue, - :help => :yellow, + :julia => face"green", + :shell => face"red", + :pkg => face"blue", + :help => face"yellow", ) """ @@ -185,20 +185,15 @@ function redisplay_prompt(io::IO, oldstate::SelectorState, newstate::SelectorSta for substr in substrs start, len = substr.offset, substr.ncodeunits patend = max(patend, start + len) - if start > 1 + # Only a one-byte prefix or separator can sit just before a condition + if start > 0 && isvalid(query, start) if query[start] == FILTER_SEPARATOR - face!(styquery[start:start], :REPL_History_search_separator) + face!(styquery[start:start], face"search_separator") else - face!(styquery[start:start], :REPL_History_search_prefix) - face!(styquery[start-1:start-1], :REPL_History_search_separator) + face!(styquery[start:start], face"search_prefix") + start > 1 && isvalid(query, start - 1) && + face!(styquery[start-1:start-1], face"search_separator") end - elseif start > 0 - face!(styquery[start:start], - if query[start] == FILTER_SEPARATOR - :REPL_History_search_separator - else - :REPL_History_search_prefix - end) end isempty(kindname) || continue if start <= qpos <= start + len @@ -209,30 +204,30 @@ function redisplay_prompt(io::IO, oldstate::SelectorState, newstate::SelectorSta end if patend < ncodeunits(query) if query[patend+1] == FILTER_SEPARATOR - face!(styquery[patend+1:patend+1], :REPL_History_search_separator) + face!(styquery[patend+1:patend+1], face"search_separator") if patend + 1 < ncodeunits(query) && query[patend+2] ∈ FILTER_PREFIXES - face!(styquery[patend+2:patend+2], :REPL_History_search_prefix) + face!(styquery[patend+2:patend+2], face"search_prefix") elseif isempty(kindname) kindname = "separator" end elseif ncodeunits(query) == 1 && query[1] ∈ FILTER_PREFIXES - face!(styquery[1:1], :REPL_History_search_prefix) + face!(styquery[1:1], face"search_prefix") end end prefix = S"{bold:▪:} " ncand = length(newstate.candidates) resultnum = if newstate.hover > 0 - S"{REPL_History_search_results:[$(ncand - newstate.hover + 1)/$ncand]}" + S"{search_results:[$(ncand - newstate.hover + 1)/$ncand]}" else # Carried-over entries are unnumbered - S"{REPL_History_search_results:[•/$ncand]}" + S"{search_results:[•/$ncand]}" end padspaces = newstate.area.width - sum(textwidth, (prefix, styquery, resultnum)) suffix = if isempty(styquery) LABELS.help_prompt elseif newstate.query ∈ (FILTER_SHORTHELP_QUERY, FILTER_LONGHELP_QUERY) - S"{REPL_History_search_hint:help} " + S"{search_hint:help} " elseif kindname != "" - S"{REPL_History_search_hint:$kindname} " + S"{search_hint:$kindname} " else S"" end @@ -254,17 +249,17 @@ end const LIST_MARKERS = if Sys.isapple() # '🞇' is not available by default, and '⬤' is oversized, so we must compromise. - (selected = AnnotatedChar('⏺', [(:face, :REPL_History_search_selected)]), - hover = AnnotatedChar('⦿', [(:face, :REPL_History_search_selected)]), - unselected = AnnotatedChar('◦', [(:face, :REPL_History_search_unselected)]), - pending = AnnotatedChar('·', [(:face, :shadow)])) + (selected = AnnotatedChar('⏺', [(:face, face"search_selected")]), + hover = AnnotatedChar('⦿', [(:face, face"search_selected")]), + unselected = AnnotatedChar('◦', [(:face, face"search_unselected")]), + pending = AnnotatedChar('·', [(:face, face"shadow")])) else # Linux tends to have pretty fantastic OOTB Unicode support, with fonts # like Symbola installed by default, so we can go for the best symbols. - (selected = AnnotatedChar('⬤', [(:face, :REPL_History_search_selected)]), - hover = AnnotatedChar('🞇', [(:face, :REPL_History_search_selected)]), - unselected = AnnotatedChar('◦', [(:face, :REPL_History_search_unselected)]), - pending = AnnotatedChar('🞄', [(:face, :shadow)])) + (selected = AnnotatedChar('⬤', [(:face, face"search_selected")]), + hover = AnnotatedChar('🞇', [(:face, face"search_selected")]), + unselected = AnnotatedChar('◦', [(:face, face"search_unselected")]), + pending = AnnotatedChar('🞄', [(:face, face"shadow")])) end const NEWLINE_MARKER = S"{shadow:↩ }" @@ -474,7 +469,7 @@ function print_candidate(io::IO, search::FilterSpec, cand::HistEntry, width::Int modehint = if cand.mode == BASE_MODE S"" else - modeface = get(MODE_FACES, cand.mode, :grey) + modeface = get(MODE_FACES, cand.mode, face"grey") if hover S"{region: {bold,inverse,$modeface: $(cand.mode) }}" elseif ncodeunits(age) == 2 @@ -487,8 +482,8 @@ function print_candidate(io::IO, search::FilterSpec, cand::HistEntry, width::Int flatcand = replace(highlightcand(cand), r"\r?\n\s*" => NEWLINE_MARKER) candstr = focus_matches(search, flatcand, width - decorationlen) if hover - face!(candstr, :region) - face!(agedec, :region) + face!(candstr, face"region") + face!(agedec, face"region") end println(io, candstr, modehint, agedec, ' ') end @@ -525,7 +520,7 @@ To best display matches, this function operates in multiple stages: 5. Expand the window rightwards similarly, inserting a trailing ellipsis if there is remaining text. 6. Slice out the computed substring from `content`, preserving existing annotations. -7. Re-apply the match highlight face (`:REPL_History_search_match`) to any +7. Re-apply the match highlight face (`search_match`) to any regions within the window. 8. Pad the result with spaces if its width is less than `targetwidth`. @@ -595,7 +590,7 @@ function focus_matches(search::FilterSpec, content::AnnotatedString{String}, tar for region in mregions (last(region) < left || first(region) > right) && continue adjregion = (max(left, first(region)) - left + 1):(min(right, last(region)) - left + 1) - face!(regstr, adjregion, :REPL_History_search_match) + face!(regstr, adjregion, face"search_match") end # Add ellipses ellipstr = if left > firstindex(cstr) && right < lastindex(cstr) @@ -661,10 +656,10 @@ function redisplay_preview(io::IO, oldstate::SelectorState, oldrows::Int, newsta if !isnothing(hovcand) hovcontent = highlightcand(hovcand) for region in matchregions(newstate.filter, String(hovcontent)) - face!(hovcontent[region], :REPL_History_search_match) + face!(hovcontent[region], face"search_match") end if hovcand.mode !== BASE_MODE - mcolor = get(MODE_FACES, hovcand.mode, :grey) + mcolor = get(MODE_FACES, hovcand.mode, face"grey") hovcontent = S"{bold,$mcolor:$(hovcand.mode)>} " * hovcontent end boxedcontent(io, hovcontent, newstate.area.width, newrows - 2) @@ -680,11 +675,11 @@ function redisplay_preview(io::IO, oldstate::SelectorState, oldrows::Int, newsta end else linesprinted = 0 - seltexts = AnnotatedString{String}[] + seltexts = AnnotatedString{String, StyledValue}[] for idx in getselidxs(newstate) entry = getcand(newstate, idx) content = highlightcand(entry) - ishover(newstate, idx) && face!(content, :region) + ishover(newstate, idx) && face!(content, face"region") push!(seltexts, content) end linecount = sum(t -> 1 + count('\n', String(t)), seltexts, init=0) @@ -731,7 +726,7 @@ Returns the number of printed lines. function boxedcontent(io::IO, content::AnnotatedString{String}, width::Int, maxlines::Int) function breaklines(content::AnnotatedString{String}, maxwidth::Int) textwidth(content) <= maxwidth && return [content] - spans = AnnotatedString{String}[] + spans = AnnotatedString{String, StyledValue}[] basestr = String(content) # Because of expensive char iteration start, pos, linewidth = 1, 0, 0 for char in basestr diff --git a/stdlib/REPL/src/History/resumablefiltering.jl b/stdlib/REPL/src/History/resumablefiltering.jl index 9e3fcf3151bba..5d7dbb9d355f2 100644 --- a/stdlib/REPL/src/History/resumablefiltering.jl +++ b/stdlib/REPL/src/History/resumablefiltering.jl @@ -49,25 +49,25 @@ const FILTER_SHORTHELP = S""" {bold,magenta:Interactive history search} Enter a search term at the prompt, and see matching candidates. - A search term that is {italic:just} '{REPL_History_search_prefix:?}' brings up this help page. + A search term that is {italic:just} '{search_prefix:?}' brings up this help page. - See more information on behaviour and keybindings with '{REPL_History_search_prefix:??}'. + See more information on behaviour and keybindings with '{search_prefix:??}'. By default, each word in the search string is looked for in any order. - Should the search string start with {REPL_History_search_prefix:xyz>}, then only xyz-mode entries are considered. + Should the search string start with {search_prefix:xyz>}, then only xyz-mode entries are considered. Different search modes are available via prefixes, as follows: - {emphasis:•} {REPL_History_search_prefix:=} looks for exact matches - {emphasis:•} {REPL_History_search_prefix:!} {italic:excludes} exact matches - {emphasis:•} {REPL_History_search_prefix:/} performs a regexp search - {emphasis:•} {REPL_History_search_prefix:~} looks for fuzzy matches - {emphasis:•} {REPL_History_search_prefix:`} looks for an initialism (text with matching initials) + {emphasis:•} {search_prefix:=} looks for exact matches + {emphasis:•} {search_prefix:!} {italic:excludes} exact matches + {emphasis:•} {search_prefix:/} performs a regexp search + {emphasis:•} {search_prefix:~} looks for fuzzy matches + {emphasis:•} {search_prefix:`} looks for an initialism (text with matching initials) - You can also apply multiple restrictions with the separator '{REPL_History_search_separator:$FILTER_SEPARATOR}'. + You can also apply multiple restrictions with the separator '{search_separator:$FILTER_SEPARATOR}'. - For example, {region:{REPL_History_search_prefix:/}^foo{REPL_History_search_separator:$FILTER_SEPARATOR}\ -{REPL_History_search_prefix:`}bar{REPL_History_search_separator:$FILTER_SEPARATOR}\ -{REPL_History_search_prefix:shell>}} will look for history entries that start with "{code:foo}", + For example, {region:{search_prefix:/}^foo{search_separator:$FILTER_SEPARATOR}\ +{search_prefix:`}bar{search_separator:$FILTER_SEPARATOR}\ +{search_prefix:shell>}} will look for history entries that start with "{code:foo}", contains "{code:b... a... r...}", {italic:and} are a shell history entry. """ @@ -76,12 +76,12 @@ const FILTER_LONGHELP = S""" Search your REPL history interactively by constructing filters. - With no mode specified (see the basic help with '{REPL_History_search_prefix:?}'), entries are matched + With no mode specified (see the basic help with '{search_prefix:?}'), entries are matched if they contain all of the words in the search string, in any order. If the entire search string is lowercase, the search is case-insensitive. - If you want to include the filter separator '{REPL_History_search_separator:$FILTER_SEPARATOR}' in a query, or start + If you want to include the filter separator '{search_separator:$FILTER_SEPARATOR}' in a query, or start a words filter with a prefix character, you may escape it with a backslash (e.g. {code:\\;}). Search results can be navigated with: diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl index 9d27476797b25..3a4be7e823030 100644 --- a/stdlib/REPL/src/LineEdit.jl +++ b/stdlib/REPL/src/LineEdit.jl @@ -862,7 +862,7 @@ function refresh_multi_line(termbuf::TerminalBuffer, terminal::UnixTerminal, buf end end - styled_buffer = AnnotatedString("") + styled_buffer = AnnotatedString{String, StyledStrings.StyledValue}("") if buf.size > 0 && buf.size <= max_highlight_size full_input = String(buf.data[1:buf.size]) if !isempty(full_input) diff --git a/stdlib/REPL/src/StylingPasses.jl b/stdlib/REPL/src/StylingPasses.jl index 919c2be82c0d5..ed79fa4af4468 100644 --- a/stdlib/REPL/src/StylingPasses.jl +++ b/stdlib/REPL/src/StylingPasses.jl @@ -1,17 +1,19 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -# Each pass takes the input string and returns an AnnotatedString with styling annotations +# Each pass takes the input string and returns a `StyledString` with styling annotations module StylingPasses using StyledStrings -using StyledStrings: Face +using StyledStrings: Face, StyledValue using JuliaSyntaxHighlighting import Base: AnnotatedString, annotate!, annotations, JuliaSyntax export StylingPass, StylingContext, SyntaxHighlightPass, RegionHighlightPass, EnclosingParenHighlightPass, apply_styling_passes, merge_annotations +const StyledString = AnnotatedString{String, StyledValue} + # Context information passed to all styling passes struct StylingContext cursor_pos::Int # -1 when cursor is not shown (e.g. after hitting enter) @@ -24,7 +26,7 @@ StylingContext(cursor_pos::Int) = StylingContext(cursor_pos, 0, 0) abstract type StylingPass end function merge_annotations(annotated_strings::Vector{<:AnnotatedString}) - isempty(annotated_strings) && return AnnotatedString("") + isempty(annotated_strings) && return StyledString("") result = AnnotatedString(annotated_strings[1]) @@ -38,12 +40,12 @@ function merge_annotations(annotated_strings::Vector{<:AnnotatedString}) end function apply_style(pass::StylingPass, input::String, ast, context::StylingContext) - return pass(input, ast, context)::AnnotatedString{String} + return pass(input, ast, context)::StyledString end function apply_styling_passes(input::String, passes::Vector{StylingPass}, context::StylingContext) if isempty(passes) - return AnnotatedString(input) + return StyledString(input) end # Parse once and share AST across all passes @@ -61,7 +63,7 @@ function (::SyntaxHighlightPass)(input::String, ast, ::StylingContext) return JuliaSyntaxHighlighting.highlight(input, ast) catch e @error "Error in SyntaxHighlightPass" exception=(e, catch_backtrace()) maxlog=1 - return AnnotatedString(input) + return StyledString(input) end end @@ -69,7 +71,7 @@ end struct RegionHighlightPass <: StylingPass end function (::RegionHighlightPass)(input::String, ::Any, context::StylingContext) - result = AnnotatedString(input) + result = StyledString(input) if context.region_start > 0 && context.region_stop >= context.region_start # Add inverse face to the region @@ -89,7 +91,7 @@ end EnclosingParenHighlightPass() = EnclosingParenHighlightPass(Face(weight=:bold, underline=true)) function (pass::EnclosingParenHighlightPass)(input::String, ast, context::StylingContext) - result = AnnotatedString(input) + result = StyledString(input) if isempty(input) return result diff --git a/stdlib/REPL/src/precompile.jl b/stdlib/REPL/src/precompile.jl index 43bb9016c8b51..a95ab88d9d6ec 100644 --- a/stdlib/REPL/src/precompile.jl +++ b/stdlib/REPL/src/precompile.jl @@ -205,6 +205,7 @@ let precompile(Tuple{typeof(Base.setindex!), Base.Dict{Any, Any}, Any, Char}) precompile(Tuple{typeof(Base.setindex!), Base.Dict{Any, Any}, Any, Int}) precompile(Tuple{typeof(Base.delete!), Base.Set{Any}, String}) + precompile(Tuple{typeof(Base.:(==)), Char, String}) finally ccall(:jl_tag_newly_inferred_disable, Cvoid, ()) end diff --git a/stdlib/REPL/test/lineedit.jl b/stdlib/REPL/test/lineedit.jl index 4d110784f7ebe..50a42a33e2ad5 100644 --- a/stdlib/REPL/test/lineedit.jl +++ b/stdlib/REPL/test/lineedit.jl @@ -1332,6 +1332,7 @@ end # tests on the same worker would otherwise see, so restore it afterwards. osc_saved_colors = copy(REPL.StyledStrings.FACES.basecolors) osc_saved_faces = copy(REPL.StyledStrings.FACES.current[]) +osc_saved_theme = REPL.StyledStrings.FACES.current_theme[] @testset "OSC colour responses" begin RGB(r, g, b) = (; r=UInt8(r), g=UInt8(g), b=UInt8(b)) # `awaiting` mirrors a pending `query_colors`, so the sentinel applies the palette. @@ -1405,3 +1406,5 @@ osc_saved_faces = copy(REPL.StyledStrings.FACES.current[]) end merge!(empty!(REPL.StyledStrings.FACES.basecolors), osc_saved_colors) merge!(empty!(REPL.StyledStrings.FACES.current[]), osc_saved_faces) +REPL.StyledStrings.FACES.current_theme[] = osc_saved_theme +REPL.StyledStrings.emptycache!(REPL.StyledStrings.FACES.cache[]) # Resolved during the tests diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 52e0030c48217..0107816e8b38f 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -2064,13 +2064,19 @@ end end # Test syntax highlighting in REPL input +# Imported outside the testset as `face"..."` resolves the module at lowering time. +using StyledStrings, JuliaSyntaxHighlighting @testset "Syntax highlighting" begin - using StyledStrings using REPL.StylingPasses + # The passes' strings have the value type of styled strings, for the static display path + @test StylingPasses.apply_styling_passes("f(x)", + StylingPasses.StylingPass[StylingPasses.SyntaxHighlightPass(), StylingPasses.RegionHighlightPass(), + StylingPasses.EnclosingParenHighlightPass()], + StylingPasses.StylingContext(3, 1, 2)) isa Base.AnnotatedString{String, StyledStrings.StyledValue} # Use withfaces to ensure consistent face definitions regardless of user config - StyledStrings.withfaces(:julia_keyword => StyledStrings.Face(foreground=:red), - :julia_number => StyledStrings.Face(foreground=:blue)) do + StyledStrings.withfaces(StyledStrings.face"JuliaSyntaxHighlighting.keyword" => StyledStrings.Face(foreground=StyledStrings.face"red"), + StyledStrings.face"JuliaSyntaxHighlighting.number" => StyledStrings.Face(foreground=StyledStrings.face"blue")) do # Test that julia_prompt has syntax highlighting passes fake_repl(options = REPL.Options(confirm_exit=false, style_input=true, auto_insert_closing_bracket=false)) do stdin_write, stdout_read, repl From 2ba1a88d0515b58c9ff5e987916a570420a51879 Mon Sep 17 00:00:00 2001 From: TEC Date: Sun, 20 Sep 2026 18:02:31 +0800 Subject: [PATCH 06/23] Make the terminfo loader safe to trim `--trim=safe` reported two dynamic calls when loading a terminfo entry. The parts of `TERMINFO_DIRS` are now converted to `String`s, rather than kept as `SubString`s, and each kind of capability is inserted with its own concrete value type. --- base/terminfo.jl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/base/terminfo.jl b/base/terminfo.jl index bece6b2322cda..a9c5b4261afb8 100644 --- a/base/terminfo.jl +++ b/base/terminfo.jl @@ -134,8 +134,17 @@ function extendedterminfo(data::IO, NumInt::Union{Type{Int16}, Type{Int32}}) table_indices[index] += table_halfoffset end labels = map(Symbol, _terminfo_read_strings(table_data, table_indices[string_count+1:end])) - Dict{Symbol, Union{Bool, Int, String, Nothing}}( - zip(labels, Iterators.flatten((flags, numbers, strings)))) + # Each kind of value is inserted separately, so that every insertion has a concrete value type. + capabilities = Dict{Symbol, Union{Bool, Int, String, Nothing}}() + function setcapabilities!(names, values) + for (name, value) in zip(names, values) + capabilities[name] = value + end + end + setcapabilities!(labels, flags) + setcapabilities!(Iterators.drop(labels, length(flags)), numbers) + setcapabilities!(Iterators.drop(labels, length(flags) + length(numbers)), strings) + capabilities end """ @@ -257,7 +266,7 @@ function find_terminfo_file(term::String) end haskey(ENV, "TERMINFO_DIRS") && append!(terminfo_dirs, - replace(split(ENV["TERMINFO_DIRS"], ':'), + replace(map(String, split(ENV["TERMINFO_DIRS"], ':')), "" => "/usr/share/terminfo")) push!(terminfo_dirs, normpath(Sys.BINDIR, DATAROOTDIR, "julia", "terminfo")) Sys.isunix() && From 6c61f0cf463920a738901fc31e0af4d5c0577511 Mon Sep 17 00:00:00 2001 From: TEC Date: Mon, 28 Sep 2026 03:42:20 +0800 Subject: [PATCH 07/23] Check styled output in the trim test The Trimmability program now prints a styled string with ANSI escapes and as HTML. The string has a height, an underline colour and a link, so the writers compare successive faces. The test checks both lines against the same output from an untrimmed process. Assisted-by: Claude Code (Opus 5.5) --- test/trim/Trimmability/Project.toml | 1 + test/trim/Trimmability/runtests.jl | 4 ++++ test/trim/Trimmability/src/Trimmability.jl | 11 +++++++++++ 3 files changed, 16 insertions(+) diff --git a/test/trim/Trimmability/Project.toml b/test/trim/Trimmability/Project.toml index a28e89627ac06..126e69ddaaba5 100644 --- a/test/trim/Trimmability/Project.toml +++ b/test/trim/Trimmability/Project.toml @@ -6,3 +6,4 @@ version = "0.1.0" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" +StyledStrings = "f489334b-da3d-4c2e-b8f0-e476e12c162b" diff --git a/test/trim/Trimmability/runtests.jl b/test/trim/Trimmability/runtests.jl index 5ff04f045410e..56beda0750036 100644 --- a/test/trim/Trimmability/runtests.jl +++ b/test/trim/Trimmability/runtests.jl @@ -1,5 +1,6 @@ # Verify the trimmed `Trimmability` executable exercises a range of constructs using Test +using StyledStrings outdir = ARGS[1] @@ -43,4 +44,7 @@ outdir = ARGS[1] @test lines[31] == "threads: 4160 204" @test lines[32] == "custom_library_ccall: $(VERSION.major)" @test lines[33] == "lazy_library_ccall: $(VERSION.major) 1" + sample = styled"{bold:bold} {(fg=red, underline=(blue, curly)):mixed} {(height=1.2):big} {link={https://julialang.org}:link}" + @test lines[end-1] == "styled ansi: " * sprint(print, sample; context = :color => true) + @test lines[end] == "styled html: " * sprint(show, MIME("text/html"), sample) end diff --git a/test/trim/Trimmability/src/Trimmability.jl b/test/trim/Trimmability/src/Trimmability.jl index c35c792446ed8..71c47448f861e 100644 --- a/test/trim/Trimmability/src/Trimmability.jl +++ b/test/trim/Trimmability/src/Trimmability.jl @@ -2,6 +2,7 @@ module Trimmability using Sockets +using StyledStrings include("linalg.jl") @@ -196,6 +197,15 @@ end # keeps in a merged union: each `print` must still resolve statically. @noinline interpolate_many(r, x, s, c) = "got $(r) vs $(first(r)) with $(x) and $(s) and $(c)" +# Styled output, both as ANSI escapes and as HTML. Faces with a height, underline colour and +# link exercise the comparisons between successive faces. +const styled_sample = styled"{bold:bold} {(fg=red, underline=(blue, curly)):mixed} {(height=1.2):big} {link={https://julialang.org}:link}" + +function _test_styled() + println(Core.stdout, "styled ansi: ", sprint(print, styled_sample; context = :color => true)) + println(Core.stdout, "styled html: ", sprint(show, MIME("text/html"), styled_sample)) +end + function _test_threads() v = zeros(Int, 64) Threads.@threads for i in eachindex(v) @@ -262,6 +272,7 @@ function @main(args::Vector{String})::Cint user_library_ccall() # prints as a side effect lazy_library_ccall() + _test_styled() # Last, as the test reads the final two lines Base.donotdelete(reshape([1,2,3],:,1,1)) From a429304db7845c70083fa520e9d607059f130569 Mon Sep 17 00:00:00 2001 From: TEC Date: Wed, 23 Sep 2026 21:40:33 +0800 Subject: [PATCH 08/23] Merge each styling pass's annotations once `merge_annotations` started from a copy of the first pass's string, then added the annotations of every pass. The first pass's annotations were therefore stored twice. The loop now skips the first pass. Assisted-by: Claude Code (Opus 5.5) --- stdlib/REPL/src/StylingPasses.jl | 2 +- stdlib/REPL/test/repl.jl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/stdlib/REPL/src/StylingPasses.jl b/stdlib/REPL/src/StylingPasses.jl index ed79fa4af4468..d8ed8b9325070 100644 --- a/stdlib/REPL/src/StylingPasses.jl +++ b/stdlib/REPL/src/StylingPasses.jl @@ -30,7 +30,7 @@ function merge_annotations(annotated_strings::Vector{<:AnnotatedString}) result = AnnotatedString(annotated_strings[1]) - for source in annotated_strings + for source in Iterators.drop(annotated_strings, 1) for ann in annotations(source) annotate!(result, ann.region, ann.label, ann.value) end diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 0107816e8b38f..8541346ef9d83 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -2068,6 +2068,10 @@ end using StyledStrings, JuliaSyntaxHighlighting @testset "Syntax highlighting" begin using REPL.StylingPasses + # Each pass's annotations are merged once + let passes = [Base.AnnotatedString("ab", [(1:1, :face, 1)]), Base.AnnotatedString("ab", [(2:2, :face, 2)])] + @test length(Base.annotations(StylingPasses.merge_annotations(passes))) == 2 + end # The passes' strings have the value type of styled strings, for the static display path @test StylingPasses.apply_styling_passes("f(x)", StylingPasses.StylingPass[StylingPasses.SyntaxHighlightPass(), StylingPasses.RegionHighlightPass(), From cf7e6a5913ec09b8a96a632ab16cfd1ddaa5827d Mon Sep 17 00:00:00 2001 From: TEC Date: Mon, 28 Sep 2026 03:24:30 +0800 Subject: [PATCH 09/23] Keep annotations when joining annotated substrings `join(split(s))` of an `AnnotatedString` returned a plain `String`. `join` asks `_isannotated` about the element type, and only the method for instances covered `SubString{<:AnnotatedString}`. A method for the type now sits beside it. Assisted-by: Claude Code (Opus 5.5) --- base/strings/substring.jl | 1 + test/strings/annotated.jl | 2 ++ 2 files changed, 3 insertions(+) diff --git a/base/strings/substring.jl b/base/strings/substring.jl index 911f81bc17f37..f2a06e3714533 100644 --- a/base/strings/substring.jl +++ b/base/strings/substring.jl @@ -204,6 +204,7 @@ hash(data::SubString{String}, h::UInt) = GC.@preserve data hash_bytes(pointer(data), sizeof(data), UInt64(h), HASH_SECRET) % UInt _isannotated(::SubString{T}) where {T} = _isannotated(T) +_isannotated(::Type{SubString{T}}) where {T} = _isannotated(T) string(a::String) = String(a) string(a::SubString{String}) = String(a) diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index 66a5167bfa9a4..08bb78cb54dff 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -77,6 +77,7 @@ @test (@inferred Base.annotatedstring("y", valued(1))) isa Base.AnnotatedString{String, Int} @test (@inferred String join(AbstractString["y", valued(1)])) isa Base.AnnotatedString{String, Any} @test (@inferred String join(Union{Missing, Base.AnnotatedString{String, Int}}[valued(1), valued(2)])) isa Base.AnnotatedString{String, Int} + @test join(split(Base.AnnotatedString("ab cd", [(1:2, :a, :x)]))) == Base.AnnotatedString("abcd", [(1:2, :a, :x)]) # An empty iterator, whose eltype is `Union{}`, joins to a plain `String` @test join(()) === "" @test join(x for x in ()) === "" @@ -87,6 +88,7 @@ @test repeat(sub, 1) !== sub # A copy, as an annotated string is mutable end @test uppercase(Base.AnnotatedString(SubString("abcd", 1, 3), [(1:1, :n, 1)])) isa Base.AnnotatedString{String, Int} + @test (@inferred join(split(valued(1) * " " * valued(2)))) isa Base.AnnotatedString{String, Int} # A value outside the value type is refused with a clear error @test_throws ArgumentError Base.annotate!(valued(1), 1:1, :m, "s") @test_throws ArgumentError Base.annotate!(Base.AnnotatedIOBuffer{Int}(), 1:1, :m, "s") From bd480e76e5cf5e373158f6e6d28e3c1ccbcac2d2 Mon Sep 17 00:00:00 2001 From: TEC Date: Fri, 9 Oct 2026 01:02:45 +0800 Subject: [PATCH 10/23] Give a substring the eltype of its string `eltype(SubString{T})` was `Char` whatever `T`, while iterating a substring of an annotated string yields `AnnotatedChar`s. So `collect` of such a substring gave a `Vector{Char}` and dropped the annotations. A substring's eltype is now that of its string. Assisted-by: Claude Code (Opus 5.5) --- base/strings/substring.jl | 1 + test/strings/annotated.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/base/strings/substring.jl b/base/strings/substring.jl index f2a06e3714533..a54959a9d9298 100644 --- a/base/strings/substring.jl +++ b/base/strings/substring.jl @@ -137,6 +137,7 @@ function String(s::SubString{String}) end ncodeunits(s::SubString) = s.ncodeunits +eltype(::Type{SubString{T}}) where {T} = eltype(T) # As iterating yields, such as `AnnotatedChar`s codeunit(s::SubString) = codeunit(s.string)::CodeunitType length(s::SubString) = length(s.string, s.offset+1, s.offset+s.ncodeunits) # nothrow: SubString invariants guarantee 0 ≤ offset and offset+ncodeunits ≤ ncodeunits(string), diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index 08bb78cb54dff..cdf55d62f1ca0 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -89,6 +89,7 @@ end @test uppercase(Base.AnnotatedString(SubString("abcd", 1, 3), [(1:1, :n, 1)])) isa Base.AnnotatedString{String, Int} @test (@inferred join(split(valued(1) * " " * valued(2)))) isa Base.AnnotatedString{String, Int} + @test collect(SubString(valued(1), 1, 1)) isa Vector{Base.AnnotatedChar{Char, Int}} # As its eltype says # A value outside the value type is refused with a clear error @test_throws ArgumentError Base.annotate!(valued(1), 1:1, :m, "s") @test_throws ArgumentError Base.annotate!(Base.AnnotatedIOBuffer{Int}(), 1:1, :m, "s") From 32a8e4fd0f0ddff92b85c172602bbdac97ac4fc8 Mon Sep 17 00:00:00 2001 From: TEC Date: Sun, 11 Oct 2026 09:33:40 +0800 Subject: [PATCH 11/23] Sum annotatedstring's size hints in a loop `annotatedstring` summed its arguments' size hints with `mapreduce`, which compiles a fold for each combination of argument types. As the value types of annotations are now mostly defined outside Base, these folds were compiled into each package image that builds annotated strings: about 440 methods across Markdown, Profile and the REPL. A loop, as `string` uses, compiles none. Assisted-by: Claude Code (Opus 5.5) --- base/strings/annotated.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index c2ff58c13006f..fb75546dcca98 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -302,7 +302,10 @@ julia> annotatedstring(AnnotatedString("annotated", [(1:9, :label, 1)]), ", and """ function annotatedstring(xs...) isempty(xs) && return AnnotatedString("") - size = mapreduce(_str_sizehint, +, xs) + size = 0 # Summed in a loop, as `mapreduce` would compile a fold for each combination of argument types + for x in xs + size += _str_sizehint(x) + end buf = IOBuffer(sizehint=size) s = IOContext(buf, :color => true) V = annot_promote_valtype(xs...) From 88c31510f63f2747d10dfd73ba63e4a880cddf7a Mon Sep 17 00:00:00 2001 From: TEC Date: Tue, 13 Oct 2026 09:28:18 +0800 Subject: [PATCH 12/23] Keep the types of completion and history strings `shell_completions` joined the arguments it parsed from a `Vector{Any}`, and `string` and `join` infer that an annotated string may result from that, so path completion was also compiled for `AnnotatedString{String, Any}`. The arguments are strings, so they are now joined as a `Vector{String}`. Likewise, the file name that the history search's save prompt reads lost its type through a `try`, and so did the styled message it is interpolated into. Together these take about 0.5 MB off the REPL image. Assisted-by: Claude Code (Opus 5.5) --- stdlib/REPL/src/History/search.jl | 2 +- stdlib/REPL/src/REPLCompletions.jl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stdlib/REPL/src/History/search.jl b/stdlib/REPL/src/History/search.jl index e57650e170e44..725c375f8dae5 100644 --- a/stdlib/REPL/src/History/search.jl +++ b/stdlib/REPL/src/History/search.jl @@ -376,7 +376,7 @@ function savefile(term::Base.Terminals.TTYTerminal, content::AbstractString) out = term.out_stream nlines = count('\n', content) + 1 print(out, S"\e[1G\e[2K{grey,bold:history>} {bold,emphasis:save file: }") - filename = try + filename::String = try readline(term.in_stream) catch err if err isa InterruptException diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index 97ec9c4161da1..60481d978e06a 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -1435,7 +1435,7 @@ function shell_completions(str, pos, hint::Bool=false; escape_backticks::Bool=fa # Now look at the last thing we parsed isempty(ex.args) && return Completion[], 1:0, false # Concatenate every string fragment so dir\file completes correctly. - lastarg = all(x -> x isa String, ex.args) ? string(ex.args...) : ex.args[end] + lastarg = all(x -> x isa String, ex.args) ? join(Vector{String}(ex.args)) : ex.args[end] # As Base.shell_parse throws away trailing spaces (unless they are escaped), # we need to special case here. @@ -1450,13 +1450,13 @@ function shell_completions(str, pos, hint::Bool=false; escape_backticks::Bool=fa paths, dir, success = complete_path(""; use_envpath=false, cmd_escape=true, escape_backticks, dirsep='/') return paths, r, success elseif all(@nospecialize(arg) -> arg isa AbstractString, ex.args) - path = join(ex.args) + path = join(Vector{String}(ex.args)) elseif Meta.isexpr(get(ex.args, 1, nothing), :call) && (call = ex.args[1]::Expr; call.args[1] === :expanduser) && (tilde_idx = findfirst(a -> a isa String, call.args); tilde_idx !== nothing) && all(@nospecialize(arg) -> arg isa AbstractString, ex.args[2:end]) tilde_str = call.args[tilde_idx]::String # "~" or "~alice" - rest = join(ex.args[2:end]) + rest = join(Vector{String}(ex.args[2:end])) r = last_arg_start:pos if isempty(rest) # No path after the tilde: complete usernames. From 2f5910432115bd8362fc5155c238d05a4108492d Mon Sep 17 00:00:00 2001 From: TEC Date: Fri, 16 Oct 2026 00:21:13 +0800 Subject: [PATCH 13/23] Display annotations through each value type's state Annotated strings were displayed through a trait: each value type named a style, a union's styles were promoted to one, and the style's writer showed the whole string. Two types without a promotion rule could not be shown together, and an `Any` string chose its style from its contents, so a string could print differently from the same text held more widely, or cut into parts. Now each value type says how its annotations look in a MIME type, with three functions. `context` reads what a display needs once, `reduction` gives a region's state from that type's annotations, and `transition` writes the change between two states. With no annotations, a reduction gives the baseline, where display starts and ends. A type that needs no context can leave it out of its reduction. `render` walks the regions and writes the transitions between their states. For plain text, each type's state changes on its own. For HTML, the types share one stack of nested elements, and an element is kept while its state still fits. A type without methods is not displayed, so a string displays the same whatever union of value types holds it. A known union's states are found by straight-line code. An `Any` string goes through one method, compiled once, which dispatches at runtime in the latest world, so new display methods do not invalidate its callers. `write`, `print`, `escape_string`, and `show` for HTML use `render`. HTML is offered for every annotated string, as `show` can always write it. Plain text without colour, or without annotations, is written as it is. StyledStrings now displays faces and links through these functions (JuliaLang/StyledStrings.jl#131), so it is bumped together: Stdlib: StyledStrings URL: https://github.com/JuliaLang/StyledStrings.jl.git Stdlib branch: face-primacy Old commit: b8c9246 New commit: 64e4cfd Diff: https://github.com/JuliaLang/StyledStrings.jl/compare/b8c92465c8912df92d17cf6f9a94be2870680f05...64e4cfd5cbe673f07947d3f983c579d0d11925b2 Assisted-by: Claude Code (Opus 5.5) --- base/strings/annotated_io.jl | 284 ++++++++++-------- .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/StyledStrings.version | 2 +- test/strings/annotated.jl | 162 +++++----- 7 files changed, 230 insertions(+), 222 deletions(-) create mode 100644 deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/md5 create mode 100644 deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/sha512 delete mode 100644 deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/md5 delete mode 100644 deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/sha512 diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl index 5d0c9b47fd04a..5b1bef09a4c8e 100644 --- a/base/strings/annotated_io.jl +++ b/base/strings/annotated_io.jl @@ -348,12 +348,12 @@ function printstyled end """ AnnotatedDisplay -How an annotated string, substring, or char is written and shown. The value type of its -annotations selects an [`AnnotationStyle`](@ref), and [`awrite`](@ref) renders it under -that style. Base provides `NoStyle`, which writes the plain text; a package whose values -carry display information (such as StyledStrings' `Face`) declares a style for its type -and implements `awrite` for it, and strings holding such values then render statically, -without a lookup at write time. +How annotated strings, substrings, and chars are displayed. A type of annotation value says how +its annotations look in a MIME type with three functions: `context`, `reduction`, and +`transition`. Each region of a string has a state for each value type, reduced from the region's +annotations of that type, and displaying the string writes the transitions between the states +of its regions. As each type's display depends only on its own annotations, a string displays +the same whatever union of value types holds it. !!! warning "Experimental" This interface is experimental and may change or be removed in a future release @@ -361,110 +361,163 @@ without a lookup at write time. """ module AnnotatedDisplay -using ..Base: IO, SubString, AnnotatedString, AnnotatedChar, AnnotatedIOBuffer -using ..Base: eachregion, unannotate, annotations, annotatedstring, annot_valtype, invoke_in_world, tls_world_age, Fix1 -using ..Base: escape_string, annotate!, _clear_annotations_in_region!, RegionAnnotation +using ..Base: IO, IOBuffer, SubString, AnnotatedString, AnnotatedChar, AnnotatedIOBuffer, Annotation +using ..Base: eachregion, ActiveAnnotations, unannotate, annotations, annotatedstring, uniontypes +using ..Base: invoke_in_world, tls_world_age, escape_string, annotate!, _clear_annotations_in_region!, RegionAnnotation -public AbstractAnnotationStyle, AnnotationStyle, NoStyle, awrite +public context, reduction, transition -# Annotation styles +const AnnotatedStr{S, V} = Union{AnnotatedString{S, V}, SubString{AnnotatedString{S, V}}} + +## The interface, for each MIME type and value type """ - AbstractAnnotationStyle + context(mime::MIME, T::Type) -> ctx -The supertype of the singletons that [`AnnotationStyle`](@ref) selects between. +What displaying the annotations of value type `T` as `mime` needs to read once per display, +passed on to each `reduction`. """ -abstract type AbstractAnnotationStyle end +context(::MIME, ::Type) = nothing -struct NoStyle <: AbstractAnnotationStyle end +""" + reduction(mime::MIME, T::Type[, ctx], annotations) -> state -# These are plain functions rather than constructors so that `max_methods` applies to them -# (a constructor shares `DataType`'s limit). When the value type is not a compile-time -# constant, a call sees at least two applicable methods (`NoStyle` and `DynamicStyle`), so -# inference leaves it dynamic and records no method-table edge. A package's style or writer -# definitions therefore cannot invalidate Base's compiled callers. +The state of a region with `annotations`, from those whose values have type `T`. With no +annotations, this is the baseline, where display starts and ends. A type without a method is +not displayed, and one without a `context` can leave out `ctx`. """ - AnnotationStyle(::Type{V}) -> AbstractAnnotationStyle - -Trait selecting how annotations whose values have type `V` are displayed. - -A type that carries display information (such as StyledStrings' `Face`) returns its own -`AbstractAnnotationStyle` singleton, for which [`awrite`](@ref) methods are defined. -A type that is mere metadata returns `NoStyle()`, the default. A `Union` value type reduces -over its members with `AnnotationStyle(a, b)`, so annotations of several types are displayed -by the member with a style. Two members whose styles differ have no display style in -common, and raise an `ArgumentError` until one of their packages defines an -`AnnotationStyle(a, b)` method that settles it, in either order, as a `promote_rule` -settles a promotion; a method may also return `nothing` to leave the pair unsettled. Where -both orders have a rule, the one in the order the styles are combined in is used. -`Any` is `DynamicStyle()`, which finds the style from the values held. +reduction(mime::MIME, ::Type{T}, _, annotations) where {T} = reduction(mime, T, annotations) +reduction(::MIME, ::Type, _) = nothing + """ -function AnnotationStyle end -typeof(AnnotationStyle).name.max_methods = 0x1 # `Base.Experimental.@max_methods 1`, before it exists - -AnnotationStyle(::Type) = NoStyle() -# As `promote_type` reads `promote_rule`, `promotestyle` reads `AnnotationStyle` rules in either -# orientation; the fallback's `nothing` means no rule, so no reflection is needed and it stays foldable -AnnotationStyle(::AbstractAnnotationStyle, ::AbstractAnnotationStyle) = nothing - -# The identities belong to the combination, as `promote_type`'s do, so no rule can clash with them -promotestyle(a::S, ::S) where {S <: AbstractAnnotationStyle} = a -promotestyle(a::AbstractAnnotationStyle, ::NoStyle) = a -promotestyle(::NoStyle, b::AbstractAnnotationStyle) = b -promotestyle(::NoStyle, ::NoStyle) = NoStyle() -function promotestyle(a::AbstractAnnotationStyle, b::AbstractAnnotationStyle) - ab = AnnotationStyle(a, b) - isnothing(ab) || return ab - ba = AnnotationStyle(b, a) - isnothing(ba) || return ba - throw(ArgumentError(LazyString("annotation styles ", a, " and ", b, " have no style in common: define AnnotationStyle(::", - typeof(a), ", ::", typeof(b), ") to settle it"))) -end + transition(io::IO, mime::MIME, from, to) -Base.@assume_effects :foldable AnnotationStyle(U::Union) = - promotestyle(AnnotationStyle(U.a), AnnotationStyle(U.b)) +Write to `io` the change from the state `from` to the state `to`. Each type of state belongs to +one value type, as `transition` dispatches on the states. -style(x) = AnnotationStyle(annot_valtype(x)) +For `text/html`, a state's fields are its visible properties, a transition opens an element +within the current one, and a transition to `nothing` closes the element its state opened. +""" +transition(io::IO, ::MIME, ::Nothing, ::Nothing) = nothing -# Write +## Rendering + +@generated function contexts(mime::MIME, ::Type{U}) where {U} + calls = (:(context(mime, $T)) for T in uniontypes(U)) + :(tuple($(calls...))) +end +@generated function reductions(mime::MIME, ::Type{U}, ctxs, annotations) where {U} + calls = (:(reduction(mime, $T, ctxs[$i], annotations)) for (i, T) in enumerate(uniontypes(U))) + :($(Expr(:meta, :inline)); tuple($(calls...))) # Keeping each region's states off the heap +end """ - awrite(textwriter, style::AbstractAnnotationStyle, io::IO, x) - awrite(style::AbstractAnnotationStyle, io::IO, mime::MIME, x) - -Write `x`, an annotated string, substring, or char, to `io` under `style`, with each run of -text written by `textwriter(io, text)`, and return the number of bytes written; or, with a -`mime`, show `x` in that format. - -A package implements the first for its [`AnnotationStyle`](@ref), for annotated strings and -substrings (a char is written as a one-character string unless a method is added for it), -and may implement the second. Taking the text writer as an argument lets a transformation -of the text, such as escaping, keep its styling. `NoStyle` writes the plain text and has no -`mime` form. + render(writetext, mime::MIME, io::IO, x::Union{AnnotatedStr, AnnotatedChar}) + +Write `x` to `io` as `mime`, with its text written by `writetext(io, str)`. """ -function awrite end -typeof(awrite).name.max_methods = 0x1 # As for `AnnotationStyle` +function render(writetext::F, mime::MIME, io::IO, s::AnnotatedStr{S, V}) where {F, S, V} + ctxs = contexts(mime, V) + render(writetext, mime, io, s, annots -> (@inline; reductions(mime, V, ctxs, annots))) +end -awrite(style::AbstractAnnotationStyle, io::IO, x) = awrite(write, style, io, x) +# Compiled once, dispatching at runtime on whatever types `s` holds, in the latest world so +# that new display methods do not invalidate compiled callers. Taken in a union's order, the +# types display as that union would. +render(writetext::F, mime::MIME, io::IO, s::AnnotatedStr{S, Any}) where {F, S} = + invoke_in_world(tls_world_age(), renderheld, writetext, mime, io, s) + +function renderheld(writetext, mime::MIME, io::IO, @nospecialize(s::AnnotatedStr)) + types = uniontypes(Union{unique(typeof(a.value) for a in annotations(s))...}) + ctxs = Any[context(mime, T) for T in types] + render(writetext, mime, io, s, + annots -> Any[reduction(mime, T, ctx, annots) for (T, ctx) in zip(types, ctxs)]) +end -const AnnotatedStr = Union{AnnotatedString, SubString{<:AnnotatedString}} +render(writetext::F, mime::MIME, io::IO, c::AnnotatedChar) where {F} = + render(writetext, mime, io, annotatedstring(c)) -awrite(textwriter::F, style::AbstractAnnotationStyle, io::IO, c::AnnotatedChar) where {F} = - awrite(textwriter, style, io, annotatedstring(c)) +function render(writetext, mime::MIME, io::IO, s::AnnotatedStr, statesof) + regions = eachregion(s) + bases = statesof(ActiveAnnotations(regions, regions.pos)) # Before its first region, the iterator's annotations are none + written = change(io, mime, nothing, bases) + for (str, annots) in regions + written = change(io, mime, written, statesof(annots)) + writetext(io, str) + end + change(io, mime, written, bases) +end -awrite(textwriter::F, ::NoStyle, io::IO, s::AnnotatedStr) where {F} = textwriter(io, unannotate(s)) -awrite(textwriter::F, ::NoStyle, io::IO, c::AnnotatedChar) where {F} = textwriter(io, c.char) +## Plain text: each type's state changes on its own -# Thrown here rather than by dispatch, so that a non-constant style still sees two methods -awrite(::NoStyle, io::IO, m::MIME, x) = throw(MethodError(show, (io, m, x))) +change(io::IO, ::MIME"text/plain", ::Nothing, bases) = bases +function change(io::IO, mime::MIME"text/plain", previous, states) + foreach((from, to) -> transition(io, mime, from, to), previous, states) + states +end -# Via the writer form, so that a non-constant style sees two methods there as well -awrite(io::IO, x) = awrite(write, style(x), io, x) +## HTML: the types share one stack of nested elements +# +# Each frame holds the states within an open element, from the baselines in, and differs from +# the frame outside it only in the state of the element's type. An element still fits when each +# property of each state is the outer frame's or the new state's. + +change(io::IO, ::MIME"text/html", ::Nothing, bases) = [bases] + +function change(io::IO, mime::MIME"text/html", frames, states) + # One method, as a local function with two is boxed where the closure below captures it + replaced(t, k, v) = + if t isa Tuple Base.setindex(t, v, k) else setindex!(copy(t), v, k) end + fits(el, parent, to) = + all(ntuple(Val(fieldcount(typeof(el)))) do i + property = getfield(el, i) + property === getfield(parent, i) || + property === getfield(to, i) + end) + keep = 1 + while keep < length(frames) && all(map(fits, frames[keep + 1], frames[keep], states)) + keep += 1 + end + while length(frames) > keep + inner = pop!(frames) + foreach(inner, last(frames)) do within, outside + within === outside || transition(io, mime, within, nothing) + end + end + # An unrolled `map`, as a loop or a closure out of line would box each state. In reverse, so + # a later type's element (a link) encloses an earlier type's (a face). + map(reverse(ntuple(identity, length(states))), reverse(states)) do i, to + @inline + within = last(frames)[i] + if within !== to + transition(io, mime, within, to) + push!(frames, replaced(last(frames), i, to)) + end + nothing + end + frames +end + +## Writing, printing, escaping, and showing + +# Through a buffer, so that the text and the codes between it reach `io` in one write +function writestyled(writetext::F, mime::MIME, io::IO, x) where {F} + raw = first(Base.unwrapcontext(io)) # Qualified, as `show.jl` defines it after this file + buf = if raw isa IOBuffer && !raw.append raw else IOBuffer() end + offset = position(buf) + render(writetext, mime, buf, x) + bytes = position(buf) - offset + buf === raw || write(io, seekstart(buf)) + bytes +end + +isstyled(io::IO, x::Union{AnnotatedStr, AnnotatedChar}) = + get(io, :color, false)::Bool && !isempty(if x isa SubString x.string.annotations else x.annotations end) Base.write(io::IO, s::Union{AnnotatedString{S}, SubString{<:AnnotatedString{S}}}) where {S} = - awrite(io, s)::Int + (if isstyled(io, s) writestyled(write, MIME"text/plain"(), io, s) else write(io, unannotate(s)) end)::Int Base.write(io::IO, c::AnnotatedChar) = - awrite(io, c)::Int + (if isstyled(io, c) writestyled(write, MIME"text/plain"(), io, c) else write(io, c.char) end)::Int function Base.write(io::IO, aio::AnnotatedIOBuffer{V}) where {V} if get(io, :color, false) == true @@ -474,19 +527,14 @@ function Base.write(io::IO, aio::AnnotatedIOBuffer{V}) where {V} # writing from an AnnotatedIOBuffer with style. # In the meantime, by converting to an `AnnotatedString` we can just # reuse all the work done to make that work. - awrite(io, read(aio, AnnotatedString{String, V}))::Int + write(io, read(aio, AnnotatedString{String, V}))::Int else write(io, aio.io) end end -# Print - # Via `write`, which keeps the annotations when the destination can hold them -Base.print(io::IO, s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = - (write(io, s); nothing) -Base.print(io::IO, s::AnnotatedChar) = - (write(io, s); nothing) +Base.print(io::IO, x::Union{AnnotatedString, SubString{<:AnnotatedString}, AnnotatedChar}) = (write(io, x); nothing) styled_print(io::AnnotatedIOBuffer, msg::Any, kwargs::Any) = print(io, msg...) @@ -496,15 +544,12 @@ styled_print_(io::AnnotatedIOBuffer, @nospecialize(msg), @nospecialize(kwargs)) Base.printstyled(io::AnnotatedIOBuffer, msg...; kwargs...) = styled_print_(io, msg, kwargs) -# Escape - function Base.escape_string(io::IO, s::AnnotatedStr, esc = ""; keep = (), ascii::Bool=false, fullhex::Bool=false) - aio = first(Base.unwrapcontext(io)) # Qualified, as `show.jl` defines it after this file + aio = first(Base.unwrapcontext(io)) escape_annotated(aio, io, s, esc, keep, ascii, fullhex) nothing end -# Into an annotated buffer, the escaped text keeps the annotations function escape_annotated(aio::AnnotatedIOBuffer{V}, _, s, esc, keep, ascii, fullhex) where {V} annots = convert(Vector{RegionAnnotation{V}}, annotations(s)) # As `write` does, before any text is written ends, outs = Int[0], Int[position(aio)] # Where each region ends in `s`, and where its escaped text ends in `aio` @@ -520,46 +565,21 @@ function escape_annotated(aio::AnnotatedIOBuffer{V}, _, s, esc, keep, ascii, ful end end escape_annotated(_, io, s, esc, keep, ascii, fullhex) = - awrite(style(s), io, s) do io, str - escape_string(io, str, esc; keep, ascii, fullhex) + if isstyled(io, s) + writestyled(MIME"text/plain"(), io, s) do io, str + escape_string(io, str, esc; keep, ascii, fullhex) + end + else + escape_string(io, unannotate(s), esc; keep, ascii, fullhex) end -# Show +Base.show(io::IO, m::MIME"text/html", x::Union{AnnotatedString, SubString{<:AnnotatedString}, AnnotatedChar}) = + (writestyled(escapehtml, m, io, x); nothing) -Base.show(io::IO, m::MIME"text/html", s::Union{<:AnnotatedString, SubString{<:AnnotatedString}}) = - (awrite(style(s), io, m, s); nothing) -Base.show(io::IO, m::MIME"text/html", c::AnnotatedChar) = show(io, m, annotatedstring(c)) - -function Base.showable(m::MIME"text/html", x::Union{AnnotatedStr, AnnotatedChar}) - s = style(x) - if s === DynamicStyle() - s = invoke_in_world(tls_world_age(), valuestyle, x) +function escapehtml(io::IO, str::AbstractString) + for c in str + print(io, if c == '&' "&" elseif c == '<' "<" elseif c == '>' ">" else c end) end - written = if x isa AnnotatedChar AnnotatedString{String, annot_valtype(x)} else typeof(x) end - s !== NoStyle() && hasmethod(awrite, Tuple{typeof(s), IO, typeof(m), written}) end -# Dynamic styles - -# An `Any` value type says nothing about display, so `DynamicStyle` works the style out from -# the values held. It does this in the latest world, so that new style methods do not -# invalidate compiled callers of `Any`-valued strings. -struct DynamicStyle <: AbstractAnnotationStyle end -AnnotationStyle(::Type{Any}) = DynamicStyle() - -valuestyle(x) = mapfoldl(a -> AnnotationStyle(typeof(a.value)), promotestyle, annotations(x), init = NoStyle()) - -dynamic(g, @nospecialize(x), args...) = g(valuestyle(x), args...) - -awrite(textwriter::F, ::DynamicStyle, io::IO, @nospecialize(s::AnnotatedStr)) where {F} = - if isempty(annotations(if s isa SubString s.string else s end)) # Plain text, with no style to find - textwriter(io, unannotate(s)) - else - invoke_in_world(tls_world_age(), dynamic, Fix1(awrite, textwriter), s, io, s) - end -awrite(textwriter::F, ::DynamicStyle, io::IO, @nospecialize(c::AnnotatedChar)) where {F} = - invoke_in_world(tls_world_age(), dynamic, Fix1(awrite, textwriter), c, io, c) -awrite(::DynamicStyle, io::IO, m::MIME, @nospecialize(x)) = - invoke_in_world(tls_world_age(), dynamic, awrite, x, io, m, x) - end diff --git a/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/md5 b/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/md5 new file mode 100644 index 0000000000000..6c8ee5b4c62b1 --- /dev/null +++ b/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/md5 @@ -0,0 +1 @@ +79bee96671cf360f02c4ab8f10709dbd diff --git a/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/sha512 b/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/sha512 new file mode 100644 index 0000000000000..065f2afd2ec68 --- /dev/null +++ b/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/sha512 @@ -0,0 +1 @@ +40925dd386b0aba3b2243fb9e9a31590461b0b6456b2e1e13255ae033afb273c2d0de9fbae062fcb1328291e39a642d36d31b331912dbd9264b153383e13a011 diff --git a/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/md5 b/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/md5 deleted file mode 100644 index e94cde402b1ee..0000000000000 --- a/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -1456e9796ebcf64f77e954555c743671 diff --git a/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/sha512 b/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/sha512 deleted file mode 100644 index 311c23d198449..0000000000000 --- a/deps/checksums/StyledStrings-b8c92465c8912df92d17cf6f9a94be2870680f05.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -1f29ba805b6f947e9939d4b5f6b32836451a31ba29bbe2851f0e6a53f687b68d293b81a5030b1d5786c7fd066306b984fbc81ceaf3704d758eded1bcf8d8aa87 diff --git a/stdlib/StyledStrings.version b/stdlib/StyledStrings.version index 1abddf56b76eb..8582a3643df25 100644 --- a/stdlib/StyledStrings.version +++ b/stdlib/StyledStrings.version @@ -1,4 +1,4 @@ STYLEDSTRINGS_BRANCH = face-primacy -STYLEDSTRINGS_SHA1 = b8c92465c8912df92d17cf6f9a94be2870680f05 +STYLEDSTRINGS_SHA1 = 64e4cfd5cbe673f07947d3f983c579d0d11925b2 STYLEDSTRINGS_GIT_URL := https://github.com/JuliaLang/StyledStrings.jl.git STYLEDSTRINGS_TAR_URL = https://api.github.com/repos/JuliaLang/StyledStrings.jl/tarball/$1 diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index cdf55d62f1ca0..505fa73b15f40 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -984,81 +984,59 @@ end end end -# A package with its own annotation value type and a style to display it. +# A package with its own annotation value types and how they display module MarkedAnnotations - import Base.AnnotatedDisplay: AbstractAnnotationStyle, AnnotationStyle, awrite + import Base.AnnotatedDisplay: reduction, transition struct Mark end - struct Mark2 end # A second value type displayed the same way - struct Tag end # A value type of another package, with its own style - struct Note end # As `Tag`, but with no agreed style against `Mark` - struct Hilite end # Whose style writes HTML for whole strings only - struct Gloss end # Whose rule against `Mark` is written in the orientation its unions don't present - struct MarkStyle <: AbstractAnnotationStyle end - struct TagStyle <: AbstractAnnotationStyle end - struct NoteStyle <: AbstractAnnotationStyle end - struct HiliteStyle <: AbstractAnnotationStyle end - struct GlossStyle <: AbstractAnnotationStyle end - AnnotationStyle(::Type{Mark}) = MarkStyle() - AnnotationStyle(::Type{Mark2}) = MarkStyle() - AnnotationStyle(::Type{Tag}) = TagStyle() - AnnotationStyle(::Type{Note}) = NoteStyle() - AnnotationStyle(::Type{Hilite}) = HiliteStyle() - AnnotationStyle(::Type{Gloss}) = GlossStyle() - AnnotationStyle(a::MarkStyle, ::TagStyle) = a - AnnotationStyle(a::MarkStyle, ::GlossStyle) = a - function awrite(textwriter::F, ::MarkStyle, io::IO, s::Union{Base.AnnotatedString, SubString{<:Base.AnnotatedString}}) where {F} - buf = IOBuffer() - for (str, annots) in Base.eachregion(s) - isempty(annots) || write(buf, "<") - textwriter(buf, str) - isempty(annots) || write(buf, ">") - end - write(io, take!(buf)) + struct Tag # Shown in plain text only + name::String + end + # The states: whether a region is marked, and the tag it is in, if any + struct Marked + on::Bool + end + struct Tagged + name::String end - awrite(::MarkStyle, io::IO, ::MIME"text/html", s) = sum(Base.eachregion(s)) do (str, annots) - if isempty(annots) write(io, str) else write(io, "") + write(io, str) + write(io, "") end + marked(annotations) = Marked(any(a -> a.value isa Mark, annotations)) + reduction(::MIME"text/plain", ::Type{Mark}, annotations) = marked(annotations) + reduction(::MIME"text/html", ::Type{Mark}, annotations) = marked(annotations) + transition(io::IO, ::MIME"text/plain", from::Marked, to::Marked) = from === to || print(io, if to.on "<" else ">" end) + transition(io::IO, ::MIME"text/html", ::Marked, ::Marked) = print(io, "") + transition(io::IO, ::MIME"text/html", ::Marked, ::Nothing) = print(io, "") + function reduction(::MIME"text/plain", ::Type{Tag}, annotations) + i = findlast(a -> a.value isa Tag, annotations) + Tagged(if isnothing(i) "" else annotations[i].value.name end) + end + function transition(io::IO, ::MIME"text/plain", from::Tagged, to::Tagged) + from === to && return + isempty(from.name) || print(io, "]") + isempty(to.name) || print(io, "[", to.name, " ") end - awrite(::HiliteStyle, io::IO, ::MIME"text/html", s::Base.AnnotatedString) = write(io, "", String(s), "") end -@testset "AnnotationStyle" begin - (; Mark, MarkStyle) = MarkedAnnotations - AnnotationStyle, NoStyle = Base.AnnotatedDisplay.AnnotationStyle, Base.AnnotatedDisplay.NoStyle - @test AnnotationStyle(String) === NoStyle() - @test AnnotationStyle(Mark) === MarkStyle() - @test AnnotationStyle(Union{Mark, String}) === MarkStyle() - @test AnnotationStyle(Union{String, Int, Mark}) === MarkStyle() - @test AnnotationStyle(Union{String, Int}) === NoStyle() - @test Base.AnnotatedDisplay.promotestyle(MarkStyle(), NoStyle()) === Base.AnnotatedDisplay.promotestyle(NoStyle(), MarkStyle()) === MarkStyle() - # Types sharing a style combine; styles that differ have no answer until one is given - @test AnnotationStyle(Union{Mark, MarkedAnnotations.Mark2}) === MarkStyle() - @test_throws ArgumentError AnnotationStyle(Union{Mark, MarkedAnnotations.Note}) - @test AnnotationStyle(Union{Mark, MarkedAnnotations.Tag}) === MarkStyle() # By `MarkedAnnotations` - @test Base.AnnotatedDisplay.promotestyle(MarkedAnnotations.TagStyle(), MarkStyle()) === MarkStyle() # The rule, in the other orientation - # A rule found only in its other orientation settles a union in compiled code, at compile time - glossmark() = AnnotationStyle(Union{MarkedAnnotations.Gloss, Mark}) - @test glossmark() === MarkStyle() - @test Base.infer_return_type(glossmark, ()) === MarkStyle - marked(V) = Base.AnnotatedString{String, V}("x", [(1:1, :m, Mark())]) - @test sprint(print, marked(Mark)) == "" - @test sprint(print, marked(Union{Mark, Int})) == "" - @test sprint(print, marked(Any)) == "" # The style is found from the values - @test sprint(print, Base.AnnotatedString{String, Any}("xy", [(1:1, :t, MarkedAnnotations.Tag()), (2:2, :m, Mark())])) == "" - @test sprint(print, Base.AnnotatedString{String, Int}("x", [(1:1, :n, 1)])) == "x" - @test sprint(print, Base.AnnotatedString{String, Any}("x", [(1:1, :n, 1)])) == "x" - # HTML is available exactly when the style provides it, as for a plain `String` - unstyled = Base.AnnotatedString{String, Int}("ax" - let hilited = Base.AnnotatedString("x", [(1:1, :h, MarkedAnnotations.Hilite())]) - @test showable(MIME("text/html"), hilited) && showable(MIME("text/html"), hilited[1]) - @test !showable(MIME("text/html"), SubString(hilited, 1:1)) - end +@testset "AnnotatedDisplay" begin + (; Mark, Tag, Marked) = MarkedAnnotations + colour(x) = sprint(print, x; context = :color => true) + html(x) = sprint(show, MIME("text/html"), x) + marked(V) = Base.AnnotatedString{String, V}("xy", [(1:1, :m, Mark())]) + # A type's display is written between the regions, and only to a terminal + @test colour(marked(Mark)) == "y" + @test sprint(print, marked(Mark)) == "xy" + @test colour(Base.AnnotatedString{String, Int}("x", [(1:1, :n, 1)])) == "x" # Not displayed + # A string displays the same whatever union of value types holds it + tagged(V) = Base.AnnotatedString{String, V}("abc", [(1:2, :m, Mark()), (2:3, :t, Tag("t")), (3:3, :n, 1)]) + @test colour(tagged(Union{Mark, Tag, Int})) == "c]" + @test colour(tagged(Union{Mark, Tag, Int, Symbol})) == colour(tagged(Any)) == colour(tagged(Union{Mark, Tag, Int})) + @test colour(Base.AnnotatedString{String, Any}("x")) == "x" + # Substrings and chars, as the strings made from them + @test colour(SubString(tagged(Any), 2, 3)) == colour(Base.AnnotatedString(SubString(tagged(Any), 2, 3))) + @test colour(marked(Mark)[1]) == "" + # The text of HTML is always escaped + @test html(marked(Mark)) == "xy" + @test html(SubString(marked(Any), 1, 1)) == html(marked(Mark)[1]) == "x" + plain = Base.AnnotatedString{String, Union{Tag, Int}}("a\\n" + # Escaping keeps the display around the (longer) text, and an annotated buffer the annotations + @test sprint(escape_string, marked(Mark); context = :color => true) == "y" + @test sprint(escape_string, Base.AnnotatedString{String, Mark}("a\nb", [(1:1, :m, Mark()), (3:3, :m, Mark())]); + context = :color => true) == "\\n" escaped(s; wrap = identity) = (buf = Base.AnnotatedIOBuffer(); escape_string(wrap(buf), s); read(seekstart(buf), Base.AnnotatedString)) @test escaped(Base.AnnotatedString("a\nb\tc", [(1:1, :x, 1), (3:3, :y, 2), (2:5, :z, 3)])) == Base.AnnotatedString("a\\nb\\tc", [(1:1, :x, 1), (4:4, :y, 2), (2:7, :z, 3)]) @test escaped(SubString(Base.AnnotatedString("a\nbc", [(1:3, :x, 1)]), 2, 4)) == Base.AnnotatedString("\\nbc", [(1:3, :x, 1)]) @@ -1076,35 +1056,43 @@ end escape_string(buf, Base.AnnotatedString("ab", [(1:1, :x, 1)])) @test Base.annotations(buf) == [(region = 1:1, label = :x, value = 1.0)] end - # The style of a known value type is resolved at compile time - folded(V) = only(code_typed(Base.AnnotatedDisplay.style, (Base.AnnotatedString{String, V},)))[1].code - @test folded(Mark) == Any[Core.ReturnNode(MarkStyle())] - @test folded(Union{Mark, Int}) == Any[Core.ReturnNode(MarkStyle())] - # Defining a style, or a writer for one, must not invalidate code compiled for strings of - # unknown value type. Checked in a fresh process, where only Base's methods exist as when a - # package is first loaded; here the loaded packages' methods would make inference give up - # for that reason alone. + # A known union's states are found without dispatch at runtime + none = Memory{Base.Annotation{Union{Mark, Int}}}(undef, 0) + states = @inferred Base.AnnotatedDisplay.reductions(MIME("text/plain"), Union{Mark, Int}, (nothing, nothing), none) + @test Set(states) == Set([Marked(false), nothing]) + # An `Any` string is displayed by code compiled once, whatever types it holds + specialisations() = sum(m -> length(Base.specializations(m)), methods(Base.AnnotatedDisplay.render)) + anyof(values...) = Base.AnnotatedString{String, Any}("abc", [(i:i, :v, v) for (i, v) in enumerate(values)]) + colour(anyof(Mark())) + before = specialisations() + for values in ((Mark(), Tag("t")), (Tag("t"), 1), (1.0, Mark(), :s)) + colour(anyof(values...)) + end + @test specialisations() == before + # Defining how a type displays must not invalidate code compiled for strings of unknown value + # type. Checked in a fresh process, where only Base's methods exist as when a package is first + # loaded; here the loaded packages' methods would make inference give up for that reason alone. guard = """ - import Base.AnnotatedDisplay: AbstractAnnotationStyle, AnnotationStyle, awrite + import Base.AnnotatedDisplay: context, reduction, transition struct Late end - struct LateStyle <: AbstractAnnotationStyle end + struct LateState end struct Sink <: IO end Base.write(::Sink, ::UInt8) = 1 Base.unsafe_write(::Sink, ::Ptr{UInt8}, n::UInt) = Int(n) # A new IO type and an unusual string type, so the print chain is compiled here rather than # taken from the sysimage; `v[1]` is inferred with the value type unbound. - unknown_valtype(v::Vector{Base.AnnotatedString{SubString{String}}}) = print(Sink(), v[1]) + unknown_valtype(v::Vector{Base.AnnotatedString{SubString{String}}}) = print(IOContext(Sink(), :color => true), v[1]) unknown_html(v::Vector{Base.AnnotatedString{SubString{String}}}) = show(Sink(), MIME("text/html"), v[1]) strings = Base.AnnotatedString{SubString{String}}[Base.AnnotatedString{SubString{String}, Int}(SubString("x"))] unknown_valtype(strings) - try unknown_html(strings) catch end # compiled, then correctly a `MethodError` without a style + unknown_html(strings) log = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1) - AnnotationStyle(::Type{Late}) = LateStyle() - awrite(textwriter, ::LateStyle, io::IO, s::Base.AnnotatedString) = 0 - awrite(::LateStyle, io::IO, ::MIME"text/html", s::Base.AnnotatedString) = 0 + context(::MIME, ::Type{Late}) = 0 + reduction(::MIME, ::Type{Late}, _, _) = LateState() + transition(::IO, ::MIME, ::LateState, ::LateState) = nothing ccall(:jl_debug_method_invalidation, Any, (Cint,), 0) print(join(unique(x.def.name for x in log if x isa Core.MethodInstance), ' ')) """ invalidated = split(read(`$(Base.julia_cmd()) --startup-file=no -e $guard`, String)) - @test invalidated ⊆ ["AnnotationStyle", "awrite"] + @test invalidated ⊆ ["context", "reduction", "transition"] end From e09cdb89b5483b26b18eafc605afcaa4d6a406cf Mon Sep 17 00:00:00 2001 From: TEC Date: Fri, 16 Oct 2026 02:08:18 +0800 Subject: [PATCH 14/23] Drop the labels of annotations The label of an annotation restated the type of its value: `:face` beside a `Face`, `:link` beside a `Link`. Storing both let them disagree, and labels shared one flat namespace across packages. An annotation is now a region and a value, `(; region, value)`. A char's annotations, and each region's from `eachregion`, are the values themselves. Annotations are given as `(region, value)`, `region => value`, or with `annotate!(str, [range], value)`. A labelled `(region, label, value)` is refused, rather than read as a region and a value. Markdown and the REPL annotate with values. Markdown's `with_output_annotations` is compiled once rather than for each closure it is given, so its call to the closure is dynamic. Markdown's terminal rendering already dispatches on untyped content, so this costs nothing under `--trim` now, but would need specialising if that rendering were made trimmable. StyledStrings and JuliaSyntaxHighlighting annotate with values alone, so they are bumped together: Stdlib: StyledStrings URL: https://github.com/JuliaLang/StyledStrings.jl.git Stdlib branch: face-primacy Old commit: 64e4cfd New commit: d6479d5 Diff: https://github.com/JuliaLang/StyledStrings.jl/compare/64e4cfd5cbe673f07947d3f983c579d0d11925b2...d6479d542ec51030bc0358076a8b532896b23b5c Stdlib: JuliaSyntaxHighlighting URL: https://github.com/JuliaLang/JuliaSyntaxHighlighting.jl.git Stdlib branch: faces-value-parameterisation Old commit: 2061af0 New commit: 9ef9da7 Diff: https://github.com/JuliaLang/JuliaSyntaxHighlighting.jl/compare/2061af04bfe6a7c333d486a98423b884148a271e...9ef9da7dbfd008b9d4097676b42e1e2492a2146b Assisted-by: Claude Code (Opus 5.5) --- base/strings/annotated.jl | 174 ++++--- base/strings/annotated_io.jl | 34 +- base/strings/io.jl | 4 +- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + doc/src/manual/strings.md | 8 +- stdlib/JuliaSyntaxHighlighting.version | 2 +- .../src/render/terminal/formatting.jl | 12 +- stdlib/Markdown/src/render/terminal/render.jl | 14 +- stdlib/Markdown/test/runtests.jl | 4 +- stdlib/REPL/src/History/display.jl | 16 +- stdlib/REPL/src/StylingPasses.jl | 8 +- stdlib/REPL/test/repl.jl | 2 +- stdlib/StyledStrings.version | 2 +- test/strings/annotated.jl | 433 +++++++++--------- 21 files changed, 354 insertions(+), 367 deletions(-) delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/md5 delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/sha512 create mode 100644 deps/checksums/JuliaSyntaxHighlighting-9ef9da7dbfd008b9d4097676b42e1e2492a2146b.tar.gz/md5 create mode 100644 deps/checksums/JuliaSyntaxHighlighting-9ef9da7dbfd008b9d4097676b42e1e2492a2146b.tar.gz/sha512 delete mode 100644 deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/md5 delete mode 100644 deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/sha512 create mode 100644 deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/md5 create mode 100644 deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/sha512 diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index fb75546dcca98..af4ce7f706208 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -1,7 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -const Annotation{V} = NamedTuple{(:label, :value), Tuple{Symbol, V}} -const RegionAnnotation{V} = NamedTuple{(:region, :label, :value), Tuple{UnitRange{Int}, Symbol, V}} +const RegionAnnotation{V} = NamedTuple{(:region, :value), Tuple{UnitRange{Int}, V}} """ AnnotatedString{S <: AbstractString, V} <: AbstractString @@ -10,7 +9,7 @@ A string with metadata, in the form of annotated regions. More specifically, this is a simple wrapper around any other [`AbstractString`](@ref) that allows for regions of the wrapped string to be -annotated with labeled values. +annotated with values. ```text C @@ -22,12 +21,11 @@ annotated with labeled values. ``` The above diagram represents a `AnnotatedString` where three ranges have been -annotated (labeled `A`, `B`, and `C`). Each annotation holds a label (`Symbol`) -and a value (`V`). These three pieces of information are held as a -`$(RegionAnnotation{TypeVar(:V)})`. +annotated (`A`, `B`, and `C`). Each annotation holds a region and a value (`V`), +as a `$(RegionAnnotation{TypeVar(:V)})`. The type of a value says what kind of +annotation it is. -Labels do not need to be unique, the same region can hold multiple annotations -with the same label. +The same region can hold multiple annotations of the same kind. Code written for `AnnotatedString`s in general should conserve the following properties: @@ -37,8 +35,8 @@ properties: Additional semantics may be introduced by specific uses of `AnnotatedString`s. A corollary of these rules is that adjacent, consecutively placed, annotations -with identical labels and values are equivalent to a single annotation spanning -the combined range. +with identical values are equivalent to a single annotation spanning the +combined range. See also [`AnnotatedChar`](@ref), [`annotatedstring`](@ref), [`annotations`](@ref), and [`annotate!`](@ref). @@ -51,7 +49,8 @@ AnnotatedString(s::S<:AbstractString, annotations::Vector{$(RegionAnnotation{Typ AnnotatedString{S, V}(s::S, annotations) -> AnnotatedString{S, V} ``` -Constructing from an existing `AnnotatedString` copies its annotations. +Each annotation may also be given as a `(region, value)` tuple or a `region => value` +pair. Constructing from an existing `AnnotatedString` copies its annotations. An AnnotatedString can also be created with [`annotatedstring`](@ref), which acts much like [`string`](@ref) but preserves any annotations present in the arguments. @@ -60,7 +59,7 @@ like [`string`](@ref) but preserves any annotations present in the arguments. ```jldoctest; setup=:(using Base: AnnotatedString) julia> AnnotatedString("this is an example annotated string", - [(1:18, :A, 1), (12:28, :B, 2), (18:35, :C, 3)]) + [(1:18, 1), (12:28, 2), (18:35, 3)]) "this is an example annotated string" ``` """ @@ -75,8 +74,8 @@ end A Char with annotations. More specifically, this is a simple wrapper around any other -[`AbstractChar`](@ref), which holds a list of arbitrary labelled annotations -(`$Annotation`) with the wrapped character. +[`AbstractChar`](@ref), which holds a list of annotation values (`V`) with the +wrapped character. See also [`AnnotatedString`](@ref), [`annotatedstring`](@ref), `annotations`, `annotate!`. @@ -85,7 +84,7 @@ See also [`AnnotatedString`](@ref), [`annotatedstring`](@ref), `annotations`, ```julia AnnotatedChar(c::C<:AbstractChar) -> AnnotatedChar{C, Any} -AnnotatedChar(c::C<:AbstractChar, annotations::Vector{$(Annotation{TypeVar(:V)})}) -> AnnotatedChar{C, V} +AnnotatedChar(c::C<:AbstractChar, annotations::Vector{V}) -> AnnotatedChar{C, V} AnnotatedChar{C, V}(c::C, annotations) -> AnnotatedChar{C, V} ``` @@ -94,13 +93,13 @@ Constructing from an existing `AnnotatedChar` copies its annotations. # Examples ```jldoctest; setup=:(using Base: AnnotatedChar) -julia> AnnotatedChar('j', [(:label, 1)]) +julia> AnnotatedChar('j', [1]) 'j': ASCII/Unicode U+006A (category Ll: Letter, lowercase) ``` """ struct AnnotatedChar{C <: AbstractChar, V} <: AbstractChar char::C - annotations::Vector{Annotation{V}} + annotations::Vector{V} end ## Constructors ## @@ -108,7 +107,12 @@ end function AnnotatedString{S}(s::S, annots) where {S <: AbstractString} valtype(::Vector{RegionAnnotation{V}}) where {V} = V valtype(::Vector) = Any - vannots = [(region = UnitRange{Int}(r), label = Symbol(l), value = v) for (r, l, v) in annots] + function annotation(annot) + length(annot) == 2 || throw(ArgumentError(LazyString("an annotation is a region and a value, as `(region, value)`, not ", annot))) + region, value = annot + (region = UnitRange{Int}(region), value) + end + vannots = [annotation(annot) for annot in annots] AnnotatedString{S, valtype(vannots)}(s, vannots) end @@ -117,10 +121,8 @@ AnnotatedString(s::AbstractString, annots) = AnnotatedString{typeof(s)}(s, annot AnnotatedString{S, V}(s::S) where {S, V} = AnnotatedString{S, V}(s, RegionAnnotation{V}[]) function AnnotatedChar{C}(c::C, annots) where {C <: AbstractChar} - valtype(::Vector{Annotation{V}}) where {V} = V - valtype(::Vector) = Any - vannots = [(label = Symbol(l), value = v) for (l, v) in annots] - AnnotatedChar{C, valtype(vannots)}(c, vannots) + vannots = collect(annots) + AnnotatedChar{C, eltype(vannots)}(c, vannots) end AnnotatedChar(c::AbstractChar, annots) = AnnotatedChar{typeof(c)}(c, annots) @@ -132,18 +134,19 @@ AnnotatedString{S, V}(s::AnnotatedString{S}) where {S, V} = AnnotatedString(s::AnnotatedString{S, V}) where {S, V} = AnnotatedString{S, V}(s) AnnotatedChar{C, V}(c::AnnotatedChar{C}) where {C, V} = - AnnotatedChar{C, V}(c.char, Vector{Annotation{V}}(c.annotations)) + AnnotatedChar{C, V}(c.char, Vector{V}(c.annotations)) AnnotatedChar(c::AnnotatedChar{C, V}) where {C, V} = AnnotatedChar{C, V}(c) -# Constructors to avoid recursive wrapping, which widen the value type as `vcat` does +# Constructors to avoid recursive wrapping, which widen the value type as `vcat` does. The +# `Vector{RegionAnnotation}` method also settles an ambiguity with the default constructor. AnnotatedString(s::AnnotatedString{S, V}, annots::Vector{RegionAnnotation{T}}) where {S <: AbstractString, V, T} = AnnotatedString(s.string, typed_vcat(RegionAnnotation{annot_promote_valtype(s, annots)}, s.annotations, annots)) AnnotatedString(s::AnnotatedString, annots) = AnnotatedString(s, AnnotatedString(s.string, annots).annotations) -AnnotatedChar(c::AnnotatedChar, annots::Vector{Annotation{T}}) where {T} = - AnnotatedChar(c.char, typed_vcat(Annotation{annot_promote_valtype(c, annots)}, c.annotations, annots)) +AnnotatedChar(c::AnnotatedChar, annots::Vector) = + AnnotatedChar(c.char, typed_vcat(annot_promote_valtype(c, AnnotatedChar(c.char, annots)), c.annotations, annots)) AnnotatedChar(c::AnnotatedChar, annots) = AnnotatedChar(c, AnnotatedChar(c.char, annots).annotations) @@ -164,7 +167,7 @@ AnnotatedString(s::S) where {S <: AbstractString} = convert(AnnotatedString{S}, convert(::Type{AnnotatedChar}, c::AnnotatedChar) = c convert(::Type{AnnotatedChar{C, V}}, c::C) where { C <: AbstractChar, V } = - AnnotatedChar{C, V}(c, Vector{Annotation{V}}()) + AnnotatedChar{C, V}(c, Vector{V}()) convert(::Type{AnnotatedChar}, c::C) where { C <: AbstractChar } = convert(AnnotatedChar{C, Any}, c) @@ -198,7 +201,7 @@ Get the underlying string of `s`, without copying. # Examples ```jldoctest; setup=:(using Base: AnnotatedString) -julia> s = AnnotatedString("abcde", [(1:3, :A, 4)]) +julia> s = AnnotatedString("abcde", [(1:3, 4)]) "abcde" julia> u = Base.unannotate(s) @@ -223,7 +226,7 @@ end function getindex(s::AnnotatedString{S, V}, i::Integer) where {S, V} @boundscheck checkbounds(s, i) @inbounds if isvalid(s, i) - AnnotatedChar(s.string[i], Annotation{V}[Annotation{V}((a.label, a.value)) for a in annotations(s, i)]) + AnnotatedChar(s.string[i], V[a.value for a in s.annotations if i in a.region]) else string_index_err(s, i) end @@ -237,7 +240,7 @@ function show(io::IO, s::A) where {A <: AnnotatedString} print(io, '(') show(io, s.string) print(io, ", ") - tupanns = Vector{Tuple{UnitRange{Int}, Symbol, Any}}(map(values, s.annotations)) + tupanns = Vector{Tuple{UnitRange{Int}, Any}}(map(values, s.annotations)) show(IOContext(io, :typeinfo => typeof(tupanns)), tupanns) print(io, ')') end @@ -296,7 +299,7 @@ See also [`AnnotatedString`](@ref), [`AnnotatedChar`](@ref). julia> annotatedstring("now an AnnotatedString") "now an AnnotatedString" -julia> annotatedstring(AnnotatedString("annotated", [(1:9, :label, 1)]), ", and unannotated") +julia> annotatedstring(AnnotatedString("annotated", [(1:9, 1)]), ", and unannotated") "annotated, and unannotated" ``` """ @@ -328,8 +331,8 @@ function annotatedstring(xs...) end print(s, unannotate(x)) elseif x isa AnnotatedChar - for annot in x.annotations - push!(annotations, (region=1+size:1+size, annot...)) + for value in x.annotations + push!(annotations, (region=1+size:1+size, value)) end print(s, x.char) else @@ -344,7 +347,6 @@ annot_valtype(::Type{AnnotatedString{S, V}}) where {S, V} = V annot_valtype(::Type{AnnotatedChar{C, V}}) where {C, V} = V annot_valtype(::Type{SubString{S}}) where {S} = annot_valtype(S) annot_valtype(::Type{Vector{RegionAnnotation{V}}}) where {V} = V -annot_valtype(::Type{Vector{Annotation{V}}}) where {V} = V annot_valtype(::Type{Pair{P, V}}) where {P, V} = annot_valtype(V) # An abstract type that may hold annotated values, such as the eltype of a mixed vector, gives # `Any`. This uses subtyping because `typeintersect` does not fold at compile time. @@ -369,8 +371,8 @@ function annot_promote_valtype(a, b, xs...) end annotatedstring(s::AnnotatedString) = s -annotatedstring(c::AnnotatedChar) = - AnnotatedString(string(c.char), [(region=1:ncodeunits(c), annot...) for annot in c.annotations]) +annotatedstring(c::AnnotatedChar{C, V}) where {C, V} = + AnnotatedString(string(c.char), RegionAnnotation{V}[(region=1:ncodeunits(c), value) for value in c.annotations]) AnnotatedString(s::SubString{<:AnnotatedString}) = annotatedstring(s) AnnotatedString{S, V}(s::SubString{<:AnnotatedString{S}}) where {S, V} = @@ -400,10 +402,10 @@ end repeat(str::SubString{<:AnnotatedString}, r::Integer) = repeat(AnnotatedString(str), r) -function repeat(c::AnnotatedChar, r::Integer) +function repeat(c::AnnotatedChar{C, V}, r::Integer) where {C, V} str = repeat(c.char, r) fullregion = firstindex(str):lastindex(str) - AnnotatedString(str, [(region=fullregion, annot...) for annot in c.annotations]) + AnnotatedString(str, RegionAnnotation{V}[(region=fullregion, value) for value in c.annotations]) end function reverse(s::AnnotatedString) @@ -424,49 +426,43 @@ reverse(s::SubString{<:AnnotatedString}) = reverse(AnnotatedString(s)) ## End AbstractString interface ## -function _annotate!(annlist::Vector{RegionAnnotation{V}}, region::UnitRange{Int}, label::Symbol, value::Union{V, Nothing}) where {V} - if isnothing(value) - deleteat!(annlist, findall(ann -> ann.region == region && ann.label === label, annlist)) - else - push!(annlist, RegionAnnotation{V}((; region, label, value))) - end -end -_annotate!(annlist::Vector{RegionAnnotation{V}}, region::UnitRange{Int}, label::Symbol, @nospecialize(value)) where {V} = +_annotate!(annlist::Vector{RegionAnnotation{V}}, region::UnitRange{Int}, value::V) where {V} = + push!(annlist, RegionAnnotation{V}((region, value))) +_annotate!(annlist::Vector{RegionAnnotation{V}}, region::UnitRange{Int}, @nospecialize(value)) where {V} = throw(ArgumentError(LazyString("cannot annotate with a ", typeof(value), " value, as the value type is ", V, "; construct the string or buffer with a wider value type, such as Any"))) """ - annotate!(str::AnnotatedString, [range::UnitRange{Int}], label::Symbol, value) - annotate!(str::SubString{AnnotatedString}, [range::UnitRange{Int}], label::Symbol, value) + annotate!(str::AnnotatedString, [range::UnitRange{Int}], value) + annotate!(str::SubString{AnnotatedString}, [range::UnitRange{Int}], value) -Annotate a `range` of `str` (or the entire string) with a labeled value `(label, value)`. -To remove existing `label` annotations, use a value of `nothing`. A `value` outside the +Annotate a `range` of `str` (or the entire string) with `value`. A `value` outside the value type `V` of `str` raises an `ArgumentError`. The order in which annotations are applied to `str` is semantically meaningful, as described in [`AnnotatedString`](@ref). """ -annotate!(s::AnnotatedString, range::UnitRange{Int}, label::Symbol, val) = - (_annotate!(s.annotations, range, label, val); s) +annotate!(s::AnnotatedString, range::UnitRange{Int}, value) = + (_annotate!(s.annotations, range, value); s) -annotate!(ss::AnnotatedString, label::Symbol, val) = - annotate!(ss, firstindex(ss):lastindex(ss), label, val) +annotate!(s::AnnotatedString, value) = + annotate!(s, firstindex(s):lastindex(s), value) -annotate!(s::SubString{<:AnnotatedString}, range::UnitRange{Int}, label::Symbol, val) = - (annotate!(s.string, s.offset .+ (range), label, val); s) +annotate!(s::SubString{<:AnnotatedString}, range::UnitRange{Int}, value) = + (annotate!(s.string, s.offset .+ (range), value); s) -annotate!(s::SubString{<:AnnotatedString}, label::Symbol, val) = - (annotate!(s.string, s.offset .+ (1:s.ncodeunits), label, val); s) +annotate!(s::SubString{<:AnnotatedString}, value) = + (annotate!(s.string, s.offset .+ (1:s.ncodeunits), value); s) """ - annotate!(char::AnnotatedChar, label::Symbol, value::Any) + annotate!(char::AnnotatedChar, value) -Annotate `char` with the labeled value `(label, value)`. A `value` outside the value -type `V` of `char` raises an `ArgumentError`. +Annotate `char` with `value`. A `value` outside the value type `V` of `char` raises an +`ArgumentError`. """ -annotate!(c::AnnotatedChar{C, V}, label::Symbol, value::V) where {C, V} = - (push!(c.annotations, Annotation{V}((; label, value))); c) -annotate!(::AnnotatedChar{C, V}, ::Symbol, @nospecialize(value)) where {C, V} = +annotate!(c::AnnotatedChar{C, V}, value::V) where {C, V} = + (push!(c.annotations, value); c) +annotate!(::AnnotatedChar{C, V}, @nospecialize(value)) where {C, V} = throw(ArgumentError(LazyString("cannot annotate with a ", typeof(value), " value, as the value type is ", V, "; construct the char with a wider value type, such as Any"))) @@ -522,9 +518,9 @@ annotations(s::SubString{<:AnnotatedString}, pos::UnitRange{<:Integer}) = annotations(s.string, first(pos)+s.offset:last(pos)+s.offset) """ - annotations(chr::AnnotatedChar)::Vector{$(Annotation{TypeVar(:V)})} + annotations(chr::AnnotatedChar)::Vector{V} -Get all annotations of `chr`, in the form of a vector of annotation pairs. +Get the values of all annotations of `chr`. """ annotations(c::AnnotatedChar) = c.annotations @@ -589,35 +585,33 @@ end # The annotations of one region. While the cursor is still on that region they are read # from its state; afterwards they are found again by position, in time linear in the annotations. -struct ActiveAnnotations{S, V} <: AbstractVector{Annotation{V}} +struct ActiveAnnotations{S, V} <: AbstractVector{V} cursor::RegionCursor{S, V} pos::Int end IndexStyle(::Type{<:ActiveAnnotations}) = IndexLinear() -# An annotation that starts within a character covers the region of that character -function size(a::ActiveAnnotations) - a.cursor.pos == a.pos && return (length(a.cursor.active),) - stop = nextind(a.cursor.regions.str, a.pos) - (count(annot -> !isempty(annot.region) && first(annot.region) < stop && a.pos <= last(annot.region), - a.cursor.regions.annotations),) -end -function getindex(a::ActiveAnnotations{S, V}, i::Int) where {S, V} +size(a::ActiveAnnotations) = + (if a.cursor.pos == a.pos length(a.cursor.active) else count(Returns(true), a) end,) +function getindex(a::ActiveAnnotations, i::Int) @boundscheck checkbounds(a, i) a.cursor.pos == a.pos || return first(Iterators.drop(a, i - 1)) - (; label, value) = a.cursor.regions.annotations[@inbounds a.cursor.active[i]] - Annotation{V}((label, value)) + a.cursor.regions.annotations[@inbounds a.cursor.active[i]].value end # For a region the cursor has left, one pass over the annotations rather than a scan per element -function iterate(a::ActiveAnnotations{S, V}, i::Int = 1) where {S, V} - a.cursor.pos == a.pos && return if i <= length(a); (@inbounds(a[i]), i + 1) end - annots = a.cursor.regions.annotations - stop = nextind(a.cursor.regions.str, a.pos) +@inline function iterate(a::ActiveAnnotations, i::Int = 1) + (; cursor) = a + (; annotations) = cursor.regions + if cursor.pos == a.pos # Not through `length(a)`, so this inlines and a tuple of a union of values isn't boxed + i <= length(cursor.active) || return nothing + return (annotations[@inbounds cursor.active[i]].value, i + 1) + end + # An annotation that starts within a character covers the region of that character + stop = nextind(cursor.regions.str, a.pos) index = findnext(annot -> !isempty(annot.region) && first(annot.region) < stop && a.pos <= last(annot.region), - annots, i) + annotations, i) isnothing(index) && return nothing - (; label, value) = @inbounds annots[index] - (Annotation{V}((label, value)), index + 1) + (@inbounds(annotations[index]).value, index + 1) end IteratorSize(::Type{<:RegionIterator}) = SizeUnknown() @@ -656,8 +650,8 @@ end eachregion(s::SubString{AnnotatedString{S}}) Identify the contiguous substrings of `s` with constant annotations, and return an -iterator which provides each substring and the applicable annotations as a -`Tuple{SubString{S}, AbstractVector{$Annotation}}`. +iterator which provides each substring and the values of the applicable annotations +as a `Tuple{SubString{S}, AbstractVector{V}}`. The annotations of each region are read from `s` when they are used, so `s` must not be changed while its regions are in use. @@ -665,13 +659,13 @@ not be changed while its regions are in use. # Examples ```jldoctest; setup=:(using Base: AnnotatedString, eachregion) -julia> a = AnnotatedString("hey there", [(1:3, :face, :bold), (5:9, :face, :italic)]); +julia> a = AnnotatedString("hey there", [(1:3, :bold), (5:9, :italic)]); julia> map(collect, eachregion(a)) 3-element Vector{Vector{Any}}: - ["hey", [(label = :face, value = :bold)]] - [" ", @NamedTuple{label::Symbol, value::Symbol}[]] - ["there", [(label = :face, value = :italic)]] + ["hey", [:bold]] + [" ", Symbol[]] + ["there", [:italic]] ``` """ function eachregion(s::AnnotatedString{S, V}, subregion::UnitRange{Int}=firstindex(s):lastindex(s)) where {S, V} diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl index 5b1bef09a4c8e..9f084cc7b6e13 100644 --- a/base/strings/annotated_io.jl +++ b/base/strings/annotated_io.jl @@ -29,8 +29,8 @@ copy(io::AnnotatedIOBuffer) = AnnotatedIOBuffer(copy(io.io), copy(io.annotations annotations(io::AnnotatedIOBuffer) = io.annotations -annotate!(io::AnnotatedIOBuffer, range::UnitRange{Int}, label::Symbol, @nospecialize(val::Any)) = - (_annotate!(io.annotations, range, label, val); io) +annotate!(io::AnnotatedIOBuffer, range::UnitRange{Int}, @nospecialize(val::Any)) = + (_annotate!(io.annotations, range, val); io) function write(io::AnnotatedIOBuffer{V}, astr::Union{AnnotatedString{S}, SubString{<:AnnotatedString{S}}}) where {V, S} annots = convert(Vector{RegionAnnotation{V}}, annotations(astr)) @@ -40,8 +40,7 @@ function write(io::AnnotatedIOBuffer{V}, astr::Union{AnnotatedString{S}, SubStri write(io.io, unannotate(astr))::Int end -write(io::AnnotatedIOBuffer, c::AnnotatedChar) = - write(io, AnnotatedString(string(c), [(region=1:ncodeunits(c), a...) for a in c.annotations])) +write(io::AnnotatedIOBuffer, c::AnnotatedChar) = write(io, annotatedstring(c)) write(io::AnnotatedIOBuffer, x::AbstractString) = write(io.io, x)::Int write(io::AnnotatedIOBuffer, s::Union{SubString{String}, String}) = write(io.io, s) write(io::AnnotatedIOBuffer, s::StringViewAndSub) = write(io.io, s)::Int @@ -82,7 +81,6 @@ function read(io::AnnotatedIOBuffer, ::Type{AnnotatedString{S, V}}) where {S, V} start = position(io) annots = RegionAnnotation{V}[ (region = max(1, first(annot.region) - start):last(annot.region)-start, - label = annot.label, value = annot.value) for annot in io.annotations if last(annot.region) > start] AnnotatedString{S, V}(read(io.io, S), annots) @@ -94,7 +92,7 @@ read(io::AnnotatedIOBuffer, ::Type{AnnotatedString}) = read(io, AnnotatedString{ function read(io::AnnotatedIOBuffer, ::Type{AnnotatedChar{T, V}}) where {T <: AbstractChar, V} pos = position(io) char = read(io.io, T) - annots = Annotation{V}[Annotation{V}((annot.label, annot.value)) for annot in io.annotations if pos+1 in annot.region] + annots = V[annot.value for annot in io.annotations if pos+1 in annot.region] AnnotatedChar{T, V}(char, annots) end read(io::AnnotatedIOBuffer{V}, ::Type{AnnotatedChar{T}}) where {T <: AbstractChar, V} = read(io, AnnotatedChar{T, V}) @@ -173,14 +171,12 @@ function _insert_annotations!(annots::Vector{RegionAnnotation{V}}, newannots::Ve annot = newannots[i] first(annot.region) == 1 || continue i <= length(annots) || continue - annot.label == last(annots).label || continue annot.value === last(annots).value || continue all(1:i) do runlen new = newannots[begin+runlen-1] old = annots[end-i+runlen] !(last(old.region) != offset || first(new.region) != 1 || - old.label != new.label || old.value !== new.value) end || continue break search i @@ -193,7 +189,6 @@ function _insert_annotations!(annots::Vector{RegionAnnotation{V}}, newannots::Ve old = annots[old_index] new = newannots[begin+runindex] extannot = (region = first(old.region):last(new.region)+offset, - label = old.label, value = old.value) annots[old_index] = extannot end @@ -223,7 +218,6 @@ function _insert_annotations!(annots::Vector{RegionAnnotation{V}}, newannots::Ve # What I have done is found that "direct tuple reconstruction" # (as below) is several times faster than using `setindex`. newannot = (region = start+offset:stop+offset, - label = annot.label, value = annot.value) push!(annots, newannot) end @@ -274,13 +268,11 @@ function replace(out::AnnotatedIOBuffer{V}, str::AnnotatedString, pat_f::Pair... push!(replacements, thisrep) replacement = replacers[ridx] _isannotated(replacement) || continue - annots = annotations(replacement) - annots′ = if eltype(annots) <: Annotation # When it's a char not a string + annots′ = if replacement isa AnnotatedChar region = 1:newbytes - [@NamedTuple{region::UnitRange{Int}, label::Symbol, value::V}((region, label, value)) - for (; label, value) in annots] + [RegionAnnotation{V}((region, value)) for value in annotations(replacement)] else - convert(Vector{RegionAnnotation{V}}, annots) + convert(Vector{RegionAnnotation{V}}, annotations(replacement)) end _insert_annotations!(newannots, annots′, destoff) end @@ -293,7 +285,7 @@ function replace(out::AnnotatedIOBuffer{V}, str::AnnotatedString, pat_f::Pair... # This means that the start of `.region`s in # `replacements` is monotonically increasing. isappending || _clear_annotations_in_region!(out.annotations, first(replacements).offset:position(out)) - for (; region, label, value) in str.annotations + for (; region, value) in str.annotations start, stop = first(region), last(region) prioridx = searchsortedlast( replacements, (region = start:start, offset = 0), @@ -308,7 +300,7 @@ function replace(out::AnnotatedIOBuffer{V}, str::AnnotatedString, pat_f::Pair... elseif postidx - prioridx <= 1 && start > last(priorrep.region) && stop < first(postrep.region) # Lies between replacements shiftregion = (start + priorrep.offset):(stop + priorrep.offset) - shiftann = (region = shiftregion, label, value) + shiftann = (region = shiftregion, value) push!(out.annotations, shiftann) else # Split between replacements @@ -317,7 +309,7 @@ function replace(out::AnnotatedIOBuffer{V}, str::AnnotatedString, pat_f::Pair... gap = max(start, last(prevrep.region)+1):min(stop, first(rep.region)-1) if !isempty(gap) shiftregion = (first(gap) + prevrep.offset):(last(gap) + prevrep.offset) - shiftann = (; region = shiftregion, label, value) + shiftann = (; region = shiftregion, value) push!(out.annotations, shiftann) end prevrep = rep @@ -361,7 +353,7 @@ the same whatever union of value types holds it. """ module AnnotatedDisplay -using ..Base: IO, IOBuffer, SubString, AnnotatedString, AnnotatedChar, AnnotatedIOBuffer, Annotation +using ..Base: IO, IOBuffer, SubString, AnnotatedString, AnnotatedChar, AnnotatedIOBuffer using ..Base: eachregion, ActiveAnnotations, unannotate, annotations, annotatedstring, uniontypes using ..Base: invoke_in_world, tls_world_age, escape_string, annotate!, _clear_annotations_in_region!, RegionAnnotation @@ -560,8 +552,8 @@ function escape_annotated(aio::AnnotatedIOBuffer{V}, _, s, esc, keep, ascii, ful end eof(aio) || _clear_annotations_in_region!(aio.annotations, first(outs)+1:last(outs)) outpos(i) = outs[searchsortedlast(ends, clamp(i, 0, last(ends)))] # Where byte `i` of `s` ends in `aio`, clipped to `s` - for (; region, label, value) in annots # A substring's annotations may reach beyond it - annotate!(aio, outpos(first(region) - 1) + 1:outpos(last(region)), label, value) + for (; region, value) in annots # A substring's annotations may reach beyond it + annotate!(aio, outpos(first(region) - 1) + 1:outpos(last(region)), value) end end escape_annotated(_, io, s, esc, keep, ascii, fullhex) = diff --git a/base/strings/io.jl b/base/strings/io.jl index e5543e18dc71d..f7839e4257c30 100644 --- a/base/strings/io.jl +++ b/base/strings/io.jl @@ -822,8 +822,8 @@ function AnnotatedString(chars::AbstractVector{C}) where {C<:AbstractChar} point = 1 for c in chars if c isa AnnotatedChar - for annot in c.annotations - push!(annots, (point:point, annot...)) + for value in c.annotations + push!(annots, (point:point, value)) end end point += ncodeunits(c) diff --git a/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/md5 deleted file mode 100644 index 753419d3ef21a..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -f14f37552be7d275ef7c30d26cf900e7 diff --git a/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/sha512 deleted file mode 100644 index 6c65735c0a077..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-2061af04bfe6a7c333d486a98423b884148a271e.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -215f952ceada63c6b375dcc6cd62fedce257d8b6835dc57638225941eea810d00003925a3893b8bc0f4c82cb41dbd00baee098080a0246a5fc9a5f335938ee8c diff --git a/deps/checksums/JuliaSyntaxHighlighting-9ef9da7dbfd008b9d4097676b42e1e2492a2146b.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-9ef9da7dbfd008b9d4097676b42e1e2492a2146b.tar.gz/md5 new file mode 100644 index 0000000000000..07728d7347c90 --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-9ef9da7dbfd008b9d4097676b42e1e2492a2146b.tar.gz/md5 @@ -0,0 +1 @@ +d83159574f26bcfca625615090327cda diff --git a/deps/checksums/JuliaSyntaxHighlighting-9ef9da7dbfd008b9d4097676b42e1e2492a2146b.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-9ef9da7dbfd008b9d4097676b42e1e2492a2146b.tar.gz/sha512 new file mode 100644 index 0000000000000..25d0fe22ccdde --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-9ef9da7dbfd008b9d4097676b42e1e2492a2146b.tar.gz/sha512 @@ -0,0 +1 @@ +c5fcbebfd48c37cd793d57afa6be64d0c87b7ec5f03615f46e48a78ffb192afffcc715115d590322f24732a20e660f6af55b79a5d27ef05b9905f32f9f7e6533 diff --git a/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/md5 b/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/md5 deleted file mode 100644 index 6c8ee5b4c62b1..0000000000000 --- a/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -79bee96671cf360f02c4ab8f10709dbd diff --git a/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/sha512 b/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/sha512 deleted file mode 100644 index 065f2afd2ec68..0000000000000 --- a/deps/checksums/StyledStrings-64e4cfd5cbe673f07947d3f983c579d0d11925b2.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -40925dd386b0aba3b2243fb9e9a31590461b0b6456b2e1e13255ae033afb273c2d0de9fbae062fcb1328291e39a642d36d31b331912dbd9264b153383e13a011 diff --git a/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/md5 b/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/md5 new file mode 100644 index 0000000000000..f1c9b08f8e149 --- /dev/null +++ b/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/md5 @@ -0,0 +1 @@ +ecc20f4d244c10d3a98424aee93bf6b4 diff --git a/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/sha512 b/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/sha512 new file mode 100644 index 0000000000000..b5ec01045eca3 --- /dev/null +++ b/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/sha512 @@ -0,0 +1 @@ +5384fdb0bf2d91927fcfb579d1e97eaecae2658507fb6021b23ecb0b31e052be9469f7f5165c37caab1e196cb9ad1cb1ccd24ab40ce5a2a142eb448cdbd6ae78 diff --git a/doc/src/manual/strings.md b/doc/src/manual/strings.md index 2c9af890e703f..8d854fa633c8f 100644 --- a/doc/src/manual/strings.md +++ b/doc/src/manual/strings.md @@ -1242,7 +1242,7 @@ last backslash escapes a quote, since these backslashes appear before a quote. It is sometimes useful to be able to hold metadata relating to regions of a string. An [`AnnotatedString`](@ref Base.AnnotatedString) wraps another string and -allows for regions of it to be annotated with labelled values (`:label => value`). +allows for regions of it to be annotated with values, whose types say what they are. All generic string operations are applied to the underlying string. However, when possible, styling information is preserved. This means you can manipulate a [`AnnotatedString`](@ref Base.AnnotatedString) —taking substrings, padding them, @@ -1250,7 +1250,7 @@ concatenating them with other strings— and the metadata annotations will "come along for the ride". This string type is fundamental to the [StyledStrings stdlib](@ref -stdlib-styledstrings), which uses `:face`-labelled annotations to hold styling +stdlib-styledstrings), which annotates with `Face` values to hold styling information. When concatenating a [`AnnotatedString`](@ref Base.AnnotatedString), take care to use @@ -1259,7 +1259,7 @@ to keep the string annotations. ```jldoctest julia> str = Base.AnnotatedString("hello there", - [(1:5, :word, :greeting), (7:11, :label, 1)]) + [(1:5, :greeting), (7:11, 1)]) "hello there" julia> length(str) @@ -1271,7 +1271,7 @@ julia> lpad(str, 14) julia> typeof(lpad(str, 7)) Base.AnnotatedString{String, Any} -julia> str2 = Base.AnnotatedString(" julia", [(2:6, :face, :magenta)]) +julia> str2 = Base.AnnotatedString(" julia", [(2:6, :magenta)]) " julia" julia> Base.annotatedstring(str, str2) diff --git a/stdlib/JuliaSyntaxHighlighting.version b/stdlib/JuliaSyntaxHighlighting.version index d1918de872b98..e8b5f8f7cd675 100644 --- a/stdlib/JuliaSyntaxHighlighting.version +++ b/stdlib/JuliaSyntaxHighlighting.version @@ -1,4 +1,4 @@ JULIASYNTAXHIGHLIGHTING_BRANCH = faces-value-parameterisation -JULIASYNTAXHIGHLIGHTING_SHA1 = 2061af04bfe6a7c333d486a98423b884148a271e +JULIASYNTAXHIGHLIGHTING_SHA1 = 9ef9da7dbfd008b9d4097676b42e1e2492a2146b JULIASYNTAXHIGHLIGHTING_GIT_URL := https://github.com/JuliaLang/JuliaSyntaxHighlighting.jl.git JULIASYNTAXHIGHLIGHTING_TAR_URL = https://api.github.com/repos/JuliaLang/JuliaSyntaxHighlighting.jl/tarball/$1 diff --git a/stdlib/Markdown/src/render/terminal/formatting.jl b/stdlib/Markdown/src/render/terminal/formatting.jl index 812075c8e4820..4d47296571536 100644 --- a/stdlib/Markdown/src/render/terminal/formatting.jl +++ b/stdlib/Markdown/src/render/terminal/formatting.jl @@ -9,19 +9,19 @@ function annotprint(f::Function, args...) end """ - with_output_annotations(f::Function, io::AnnotIO, annots::Pair{Symbol, <:Any}...) + with_output_annotations(f::Function, io::AnnotIO, values...) -Call `f(io)`, and apply `annots` to the output created by doing so. +Call `f(io)`, and annotate the output created by doing so with `values`. """ -function with_output_annotations(f::Function, io::AnnotIO, annots::Pair{Symbol, <:Any}...) - @nospecialize annots +function with_output_annotations(f::Function, io::AnnotIO, values...) + @nospecialize f values aio = if io isa AnnotatedIOBuffer io else io.io end start = position(aio) + 1 v = f(io) stop = position(aio) sortedindex = searchsortedlast(aio.annotations, (region=start:stop,), by=a -> a.region) - for (i, annot) in enumerate(annots) - insert!(aio.annotations, sortedindex + i, (start:stop, annot...)) + for (i, value) in enumerate(values) + insert!(aio.annotations, sortedindex + i, (start:stop, value)) end return v end diff --git a/stdlib/Markdown/src/render/terminal/render.jl b/stdlib/Markdown/src/render/terminal/render.jl index 5255af6a04058..dabee7f7823f4 100644 --- a/stdlib/Markdown/src/render/terminal/render.jl +++ b/stdlib/Markdown/src/render/terminal/render.jl @@ -134,7 +134,7 @@ function term(io::AnnotIO, md::Header{l}, columns) where l face = (face"h1", face"h2", face"h3", face"h4", face"h5", face"h6")[l] underline = _header_underlines[l] pre = ' '^margin - line_width = with_output_annotations(io, :face => face) do io + line_width = with_output_annotations(io, face) do io headline = annotprint(terminline, md.text) lines = wraplines(headline, columns - 4margin) for (i, line) in enumerate(lines) @@ -152,7 +152,7 @@ function term(io::AnnotIO, md::Header{l}, columns) where l header_width = max(0, line_width) if underline != ' ' && header_width > 0 print(io, '\n', ' '^(margin)) - with_output_annotations(io -> print(io, underline^header_width), io, :face => face) + with_output_annotations(io -> print(io, underline^header_width), io, face) end end @@ -226,15 +226,15 @@ end terminline(io::AnnotIO, md::AnnotatedString) = print(io, replace(holdable(io, md), r"[ \t\n]+" => ' ')) function terminline(io::AnnotIO, md::Bold) - with_output_annotations(io -> terminline(io, md.text), io, :face => face"bold") + with_output_annotations(io -> terminline(io, md.text), io, face"bold") end function terminline(io::AnnotIO, md::Italic) - with_output_annotations(io -> terminline(io, md.text), io, :face => face"italic") + with_output_annotations(io -> terminline(io, md.text), io, face"italic") end function terminline(io::AnnotIO, md::Strikethrough) - with_output_annotations(io -> terminline(io, md.text), io, :face => face"strikethrough") + with_output_annotations(io -> terminline(io, md.text), io, face"strikethrough") end function terminline(io::IO, ::LineBreak) @@ -251,9 +251,9 @@ end function terminline(io::AnnotIO, md::Link) annots = if occursin(r"^(https?|file)://", md.url) - (:face => face"link", :link => StyledStrings.Link(md.url)) + (face"link", StyledStrings.Link(md.url)) else - (:face => face"link",) + (face"link",) end with_output_annotations(io -> terminline(io, md.text), io, annots...) end diff --git a/stdlib/Markdown/test/runtests.jl b/stdlib/Markdown/test/runtests.jl index b3157a7251a62..4f3d4e21751ff 100644 --- a/stdlib/Markdown/test/runtests.jl +++ b/stdlib/Markdown/test/runtests.jl @@ -1347,7 +1347,7 @@ using JuliaSyntaxHighlighting # Before the testset, whose `face""` paths are rea buf = Base.AnnotatedIOBuffer() show(buf, MIME("text/plain"), md) str = read(seekstart(buf), Base.AnnotatedString) - [(rstrip(String(str[a.region])), a.value) for a in Base.annotations(str) if a.label === :face] + [(rstrip(String(str[a.region])), a.value) for a in Base.annotations(str) if a.value isa StyledStrings.Face] end # Syntax highlighting is conservative, so a lone identifier picks up no # highlighting of its own; it must still be faced as code. @@ -1367,7 +1367,7 @@ using JuliaSyntaxHighlighting # Before the testset, whose `face""` paths are rea end @testset "foreign annotations" begin - legacy = Base.AnnotatedString("hi", [(1:2, :face, :red), (1:2, :tag, 1)]) + legacy = Base.AnnotatedString("hi", [(1:2, :red), (1:2, :tag => 1)]) @test sprint(show, MIME("text/plain"), Markdown.MD(Markdown.Paragraph(Any["a ", legacy]))) == " a hi" # On its own in a block, as an interpolated value is for block in (Markdown.Admonition("note", "", Any[legacy]), Markdown.BlockQuote(Any[legacy]), diff --git a/stdlib/REPL/src/History/display.jl b/stdlib/REPL/src/History/display.jl index f4b8ef935ed26..f627c15ba5281 100644 --- a/stdlib/REPL/src/History/display.jl +++ b/stdlib/REPL/src/History/display.jl @@ -249,17 +249,17 @@ end const LIST_MARKERS = if Sys.isapple() # '🞇' is not available by default, and '⬤' is oversized, so we must compromise. - (selected = AnnotatedChar('⏺', [(:face, face"search_selected")]), - hover = AnnotatedChar('⦿', [(:face, face"search_selected")]), - unselected = AnnotatedChar('◦', [(:face, face"search_unselected")]), - pending = AnnotatedChar('·', [(:face, face"shadow")])) + (selected = AnnotatedChar('⏺', [face"search_selected"]), + hover = AnnotatedChar('⦿', [face"search_selected"]), + unselected = AnnotatedChar('◦', [face"search_unselected"]), + pending = AnnotatedChar('·', [face"shadow"])) else # Linux tends to have pretty fantastic OOTB Unicode support, with fonts # like Symbola installed by default, so we can go for the best symbols. - (selected = AnnotatedChar('⬤', [(:face, face"search_selected")]), - hover = AnnotatedChar('🞇', [(:face, face"search_selected")]), - unselected = AnnotatedChar('◦', [(:face, face"search_unselected")]), - pending = AnnotatedChar('🞄', [(:face, face"shadow")])) + (selected = AnnotatedChar('⬤', [face"search_selected"]), + hover = AnnotatedChar('🞇', [face"search_selected"]), + unselected = AnnotatedChar('◦', [face"search_unselected"]), + pending = AnnotatedChar('🞄', [face"shadow"])) end const NEWLINE_MARKER = S"{shadow:↩ }" diff --git a/stdlib/REPL/src/StylingPasses.jl b/stdlib/REPL/src/StylingPasses.jl index d8ed8b9325070..3684a585add03 100644 --- a/stdlib/REPL/src/StylingPasses.jl +++ b/stdlib/REPL/src/StylingPasses.jl @@ -32,7 +32,7 @@ function merge_annotations(annotated_strings::Vector{<:AnnotatedString}) for source in Iterators.drop(annotated_strings, 1) for ann in annotations(source) - annotate!(result, ann.region, ann.label, ann.value) + annotate!(result, ann.region, ann.value) end end @@ -77,7 +77,7 @@ function (::RegionHighlightPass)(input::String, ::Any, context::StylingContext) # Add inverse face to the region # Region positions are 1-based byte positions region_range = context.region_start:context.region_stop - annotate!(result, region_range, :face, Face(inverse=true)) + annotate!(result, region_range, Face(inverse=true)) end return result @@ -101,8 +101,8 @@ function (pass::EnclosingParenHighlightPass)(input::String, ast, context::Stylin paren_pairs = find_enclosing_parens(input, ast, context.cursor_pos) for (open_pos, close_pos) in paren_pairs - annotate!(result, open_pos:open_pos, :face, pass.face) - annotate!(result, close_pos:close_pos, :face, pass.face) + annotate!(result, open_pos:open_pos, pass.face) + annotate!(result, close_pos:close_pos, pass.face) end catch e @error "Error in EnclosingParenHighlightPass" exception=(e, catch_backtrace()) maxlog=1 diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 8541346ef9d83..a4e2af62e5688 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -2069,7 +2069,7 @@ using StyledStrings, JuliaSyntaxHighlighting @testset "Syntax highlighting" begin using REPL.StylingPasses # Each pass's annotations are merged once - let passes = [Base.AnnotatedString("ab", [(1:1, :face, 1)]), Base.AnnotatedString("ab", [(2:2, :face, 2)])] + let passes = [Base.AnnotatedString("ab", [(1:1, 1)]), Base.AnnotatedString("ab", [(2:2, 2)])] @test length(Base.annotations(StylingPasses.merge_annotations(passes))) == 2 end # The passes' strings have the value type of styled strings, for the static display path diff --git a/stdlib/StyledStrings.version b/stdlib/StyledStrings.version index 8582a3643df25..6195fe0a294ec 100644 --- a/stdlib/StyledStrings.version +++ b/stdlib/StyledStrings.version @@ -1,4 +1,4 @@ STYLEDSTRINGS_BRANCH = face-primacy -STYLEDSTRINGS_SHA1 = 64e4cfd5cbe673f07947d3f983c579d0d11925b2 +STYLEDSTRINGS_SHA1 = d6479d542ec51030bc0358076a8b532896b23b5c STYLEDSTRINGS_GIT_URL := https://github.com/JuliaLang/StyledStrings.jl.git STYLEDSTRINGS_TAR_URL = https://api.github.com/repos/JuliaLang/StyledStrings.jl/tarball/$1 diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index 505fa73b15f40..33e0b9477d47a 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -10,8 +10,8 @@ @test firstindex(str) == firstindex("some string") @test eltype(str) == Base.AnnotatedChar{eltype(str.string), Any} @test eltype(Base.AnnotatedString{String}) == Base.AnnotatedChar{Char} - @test collect(Base.AnnotatedString("ab", [(1:1, :x, 1)])) isa Vector{Base.AnnotatedChar{Char, Int}} - @test first(str) == Base.AnnotatedChar(first(str.string), Pair{Symbol, Any}[]) + @test collect(Base.AnnotatedString("ab", [(1:1, 1)])) isa Vector{Base.AnnotatedChar{Char, Int}} + @test first(str) == Base.AnnotatedChar(first(str.string), Any[]) @test str[1:4] isa SubString{typeof(str)} @test str[1:4] == Base.AnnotatedString("some") big_byte_str = Base.AnnotatedString("आख") @@ -23,14 +23,14 @@ @test cmp("some stringy thingy", str) == 1 @test str[3:4] == SubString("me") @test SubString("me") == str[3:4] - Base.annotate!(str, 1:4, :thing, 0x01) - Base.annotate!(str, 6:11, :other, 0x02) - Base.annotate!(str, 1:11, :all, 0x03) - # :thing :other + Base.annotate!(str, 1:4, 0x01) + Base.annotate!(str, 6:11, 0x02) + Base.annotate!(str, 1:11, 0x03) + # 0x01 0x02 # ┌┸─┐ ┌──┸─┐ # "some string" # └───┰─────┘ - # :all + # 0x03 @test str[3:4] == SubString(str, 3, 4) @test str[3:4] != SubString("me") @test SubString("me") != str[3:4] @@ -38,19 +38,19 @@ @test repeat(SubString(str, 3, 4), 2) == repeat(Base.AnnotatedString(str[3:4]), 2) @test reverse(SubString(str, 3, 4)) == reverse(Base.AnnotatedString(str[3:4])) @test Base.AnnotatedString(str[3:4]) == - Base.AnnotatedString("me", [(1:2, :thing, 0x01), (1:2, :all, 0x03)]) + Base.AnnotatedString("me", [(1:2, 0x01), (1:2, 0x03)]) @test Base.AnnotatedString(str[3:6]) == - Base.AnnotatedString("me s", [(1:2, :thing, 0x01), (4:4, :other, 0x02), (1:4, :all, 0x03)]) - @test str == Base.AnnotatedString("some string", [(1:4, :thing, 0x01), (6:11, :other, 0x02), (1:11, :all, 0x03)]) + Base.AnnotatedString("me s", [(1:2, 0x01), (4:4, 0x02), (1:4, 0x03)]) + @test str == Base.AnnotatedString("some string", [(1:4, 0x01), (6:11, 0x02), (1:11, 0x03)]) @test str != Base.AnnotatedString("some string") - @test str != Base.AnnotatedString("some string", [(1:1, :thing, 0x01), (1:11, :all, 0x03), (6:6, :other, 0x02)]) - @test str != Base.AnnotatedString("some string", [(1:4, :thing, 0x11), (1:11, :all, 0x13), (6:11, :other, 0x12)]) - @test str != Base.AnnotatedString("some thingg", [(1:4, :thing, 0x01), (1:11, :all, 0x03), (6:11, :other, 0x02)]) + @test str != Base.AnnotatedString("some string", [(1:1, 0x01), (1:11, 0x03), (6:6, 0x02)]) + @test str != Base.AnnotatedString("some string", [(1:4, 0x11), (1:11, 0x13), (6:11, 0x12)]) + @test str != Base.AnnotatedString("some thingg", [(1:4, 0x01), (1:11, 0x03), (6:11, 0x02)]) let allstrings = - ['a', Base.AnnotatedChar('a'), Base.AnnotatedChar('a', [(:aaa, 0x04)]), + ['a', Base.AnnotatedChar('a'), Base.AnnotatedChar('a', [0x04]), "a string", Base.AnnotatedString("a string"), - Base.AnnotatedString("a string", [(1:2, :hmm, '%')]), - SubString(Base.AnnotatedString("a string", [(1:2, :hmm, '%')]), 1:1)] + Base.AnnotatedString("a string", [(1:2, '%')]), + SubString(Base.AnnotatedString("a string", [(1:2, '%')]), 1:1)] for str1 in repeat(allstrings, 2) for str2 in repeat(allstrings, 2) @test String(str1 * str2) == @@ -63,10 +63,10 @@ end end # Values of different types are kept as they are, under the union of their types - valued(v) = Base.AnnotatedString("x", [(1:1, :n, v)]) + valued(v) = Base.AnnotatedString("x", [(1:1, v)]) mixed = Base.annotatedstring(valued(1), valued(2.5)) @test mixed isa Base.AnnotatedString{String, Union{Int, Float64}} - @test Base.annotations(mixed) == [(region = 1:1, label = :n, value = 1), (region = 2:2, label = :n, value = 2.5)] + @test Base.annotations(mixed) == [(region = 1:1, value = 1), (region = 2:2, value = 2.5)] @test valued(true) * valued(1) isa Base.AnnotatedString{String, Union{Bool, Int}} @test Base.annotatedstring(valued(1), "y", valued('c'), valued("s")) isa Base.AnnotatedString{String, Union{Int, Char, String}} @test (@inferred Base.annotatedstring(valued(1), valued('c'), valued("s"), valued(1.0), valued(:s))) isa Base.AnnotatedString{String, Any} @@ -77,46 +77,40 @@ @test (@inferred Base.annotatedstring("y", valued(1))) isa Base.AnnotatedString{String, Int} @test (@inferred String join(AbstractString["y", valued(1)])) isa Base.AnnotatedString{String, Any} @test (@inferred String join(Union{Missing, Base.AnnotatedString{String, Int}}[valued(1), valued(2)])) isa Base.AnnotatedString{String, Int} - @test join(split(Base.AnnotatedString("ab cd", [(1:2, :a, :x)]))) == Base.AnnotatedString("abcd", [(1:2, :a, :x)]) + @test join(split(Base.AnnotatedString("ab cd", [(1:2, :x)]))) == Base.AnnotatedString("abcd", [(1:2, :x)]) # An empty iterator, whose eltype is `Union{}`, joins to a plain `String` @test join(()) === "" @test join(x for x in ()) === "" - let sub = Base.AnnotatedString(SubString("abcd", 1, 3), [(1:1, :n, 1)]) # Typed as `repeat` of the wrapped string + let sub = Base.AnnotatedString(SubString("abcd", 1, 3), [(1:1, 1)]) # Typed as `repeat` of the wrapped string @test repeat(sub, 0) == Base.AnnotatedString("") @test (@inferred repeat(sub, 1)) == sub @test repeat(sub, 1) isa Base.AnnotatedString{String, Int} @test repeat(sub, 1) !== sub # A copy, as an annotated string is mutable end - @test uppercase(Base.AnnotatedString(SubString("abcd", 1, 3), [(1:1, :n, 1)])) isa Base.AnnotatedString{String, Int} + @test uppercase(Base.AnnotatedString(SubString("abcd", 1, 3), [(1:1, 1)])) isa Base.AnnotatedString{String, Int} @test (@inferred join(split(valued(1) * " " * valued(2)))) isa Base.AnnotatedString{String, Int} @test collect(SubString(valued(1), 1, 1)) isa Vector{Base.AnnotatedChar{Char, Int}} # As its eltype says # A value outside the value type is refused with a clear error - @test_throws ArgumentError Base.annotate!(valued(1), 1:1, :m, "s") - @test_throws ArgumentError Base.annotate!(Base.AnnotatedIOBuffer{Int}(), 1:1, :m, "s") - @test_throws ArgumentError Base.annotate!(Base.AnnotatedChar('c', [(:m, 1)]), :m, "s") - # Annotating with `nothing` removes the label, even where the value type admits `nothing` - let s = Base.AnnotatedString{String, Any}("abc") - Base.annotate!(s, 1:3, :face, :red) - Base.annotate!(s, 1:3, :face, nothing) - @test isempty(Base.annotations(s)) - end + @test_throws ArgumentError Base.annotate!(valued(1), 1:1, "s") + @test_throws ArgumentError Base.annotate!(Base.AnnotatedIOBuffer{Int}(), 1:1, "s") + @test_throws ArgumentError Base.annotate!(Base.AnnotatedChar('c', [1]), "s") # @test collect(Base.eachstyle(str)) == # [("some", [:thing, 0x01, :all, 0x03]), # (" string", [:all, 0x03, :other, 0x02])] @test chopprefix(sprint(show, str), "Base.") == - "AnnotatedString{String, Any}(\"some string\", [(1:4, :thing, 0x01), (6:11, :other, 0x02), (1:11, :all, 0x03)])" + "AnnotatedString{String, Any}(\"some string\", [(1:4, 0x01), (6:11, 0x02), (1:11, 0x03)])" @test eval(Meta.parse(repr(str))) == str @test sprint(show, MIME("text/plain"), str) == "\"some string\"" - a = Base.AnnotatedString("hello", [(1:5, :label, 1)]) - @test first(a) == Base.AnnotatedChar('h', [(:label, 1)]) + a = Base.AnnotatedString("hello", [(1:5, 1)]) + @test first(a) == Base.AnnotatedChar('h', [1]) @test Bool === Base.infer_return_type(isvalid, Tuple{Base.AnnotatedString, Vararg}) @test Int === Base.infer_return_type(ncodeunits, Tuple{Base.AnnotatedString}) @testset "unannotate" begin s = "some string" - str = Base.AnnotatedString(s, [(2:5, :A, 3)]) + str = Base.AnnotatedString(s, [(2:5, 3)]) @test Base.unannotate(str) === s str2 = SubString(str, 2:9) @@ -131,22 +125,22 @@ end @test uppercase(chr) == Base.AnnotatedChar('C') @test titlecase(chr) == Base.AnnotatedChar('C') @test lowercase(Base.AnnotatedChar('C')) == chr - str = Base.AnnotatedString("hmm", [(1:1, :attr, "h0h0"), - (1:2, :attr, "h0m1"), - (2:3, :attr, "m1m2")]) - @test str[1] == Base.AnnotatedChar('h', [(:attr, "h0h0")]) - @test str[2] == Base.AnnotatedChar('m', [(:attr, "h0m1"), (:attr, "m1m2")]) - @test str[3] == Base.AnnotatedChar('m', [(:attr, "m1m2")]) - chr = Base.annotate!(Base.AnnotatedChar('m'), :attr, "h0m1") - @test chr == Base.AnnotatedChar('m', [(:attr, "h0m1")]) - @test Base.annotate!(chr, :attr, "m1m2") == Base.AnnotatedChar('m', [(:attr, "h0m1"), (:attr, "m1m2")]) + str = Base.AnnotatedString("hmm", [(1:1, "h0h0"), + (1:2, "h0m1"), + (2:3, "m1m2")]) + @test str[1] == Base.AnnotatedChar('h', ["h0h0"]) + @test str[2] == Base.AnnotatedChar('m', ["h0m1", "m1m2"]) + @test str[3] == Base.AnnotatedChar('m', ["m1m2"]) + chr = Base.annotate!(Base.AnnotatedChar('m'), "h0m1") + @test chr == Base.AnnotatedChar('m', ["h0m1"]) + @test Base.annotate!(chr, "m1m2") == Base.AnnotatedChar('m', ["h0m1", "m1m2"]) end @testset "Unstyled display fallback" begin # In a fresh process, so that StyledStrings' display methods can't mask the fallback script = """ - s = SubString(Base.AnnotatedString("αb", [(1:3, :face, :red)]), 3, 3) - c = Base.AnnotatedChar('γ', [(:face, :red)]) + s = SubString(Base.AnnotatedString("αb", [(1:3, :red)]), 3, 3) + c = Base.AnnotatedChar('γ', [:red]) print(any(m -> nameof(m) === :StyledStrings, Base.loaded_modules_array()), ' ', String(s), sprint(print, c)) """ @@ -155,18 +149,18 @@ end @testset "Construction" begin # Strings and chars follow the same rules; char annotations just lack a region - for (Annotated, plain, Ann, region) in ((Base.AnnotatedString, "ab", Base.RegionAnnotation, (1:1,)), - (Base.AnnotatedChar, 'a', Base.Annotation, ())) + for (Annotated, plain, annotof) in ((Base.AnnotatedString, "ab", v -> (1:1, v)), + (Base.AnnotatedChar, 'a', identity)) Plain, Abstract = typeof(plain), supertype(typeof(plain)) - tuples = [(region..., :x, 1)] - ints = Ann{Int}[Ann{Int}(only(tuples))] + loose = [annotof(1)] + ints = Annotated(plain, loose).annotations for T in (Annotated, Annotated{Plain}, Annotated{Abstract}, Annotated{Plain, Int}, Annotated{Abstract, Int}), - annots in (ints, tuples) + annots in (ints, loose) @test T(plain, annots) isa T @test T(plain, annots) isa Annotated{<:Any, Int} end @test Annotated{Abstract, Int}(plain) isa Annotated{Abstract, Int} - @test Annotated{Plain}(plain, [(region..., :x, 1), (region..., :y, 'c')]) isa Annotated{Plain, Any} + @test Annotated{Plain}(plain, [annotof(1), annotof('c')]) isa Annotated{Plain, Any} @test Annotated(plain, ints).annotations === ints @test Annotated{Plain, Int}(plain, ints).annotations === ints @test Annotated{Plain, Any}(plain, ints).annotations !== ints @@ -179,22 +173,24 @@ end @test convert(Annotated, annotated) === annotated @test convert(typeof(annotated), annotated) === annotated # Adding annotations widens the value type, as `vcat` does - for added in ([(region..., :y, :s)], Ann{Symbol}[Ann{Symbol}((region..., :y, :s))]) + for added in ([annotof(:s)], Annotated(plain, [annotof(:s)]).annotations) @test Annotated(annotated, added) isa Annotated{Plain, Union{Int, Symbol}} end end @test Base.AnnotatedString("ab") == Base.AnnotatedString("ab", Base.RegionAnnotation[]) - @test Base.AnnotatedChar(UInt32('c')) == Base.AnnotatedChar('c') == Base.AnnotatedChar('c', Pair{Symbol, Any}[]) - @test Base.AnnotatedString([Base.AnnotatedChar('a', [(:a, 1)]), Base.AnnotatedChar('b', [(:b, 2)])]) == - Base.AnnotatedString("ab", [(1:1, :a, 1), (2:2, :b, 2)]) - @test (@inferred Base.AnnotatedString(Base.AnnotatedString("x", [(1:1, :n, 1)]), [(1:1, :m, 2)])) isa Base.AnnotatedString{String, Int} - @test (@inferred Base.AnnotatedChar(Base.AnnotatedChar('x', [(:n, 1)]), [(:m, 2)])) isa Base.AnnotatedChar{Char, Int} - @test Base.AnnotatedChar(Base.AnnotatedChar('m', [(:attr, "h0m1")]), [(label = :attr, value = "m2m3")]) == - Base.AnnotatedChar('m', [(:attr, "h0m1"), (:attr, "m2m3")]) + @test Base.AnnotatedChar(UInt32('c')) == Base.AnnotatedChar('c') == Base.AnnotatedChar('c', Any[]) + @test Base.AnnotatedString("ab", [1:1 => 1, (region = 2:2, value = 2)]) == Base.AnnotatedString("ab", [(1:1, 1), (2:2, 2)]) + @test_throws ArgumentError Base.AnnotatedString("ab", Any[(1:1, :label, 1)]) # Not read as a region and a value + @test Base.AnnotatedString([Base.AnnotatedChar('a', [1]), Base.AnnotatedChar('b', [2])]) == + Base.AnnotatedString("ab", [(1:1, 1), (2:2, 2)]) + @test (@inferred Base.AnnotatedString(Base.AnnotatedString("x", [(1:1, 1)]), [(1:1, 2)])) isa Base.AnnotatedString{String, Int} + @test (@inferred Base.AnnotatedChar(Base.AnnotatedChar('x', [1]), [2])) isa Base.AnnotatedChar{Char, Int} + @test Base.AnnotatedChar(Base.AnnotatedChar('m', ["h0m1"]), ["m2m3"]) == + Base.AnnotatedChar('m', ["h0m1", "m2m3"]) end @testset "Styling preservation" begin - str = Base.AnnotatedString("some string", [(1:4, :thing, 0x01), (1:11, :all, 0x03), (6:11, :other, 0x02)]) + str = Base.AnnotatedString("some string", [(1:4, 0x01), (1:11, 0x03), (6:11, 0x02)]) @test match(r".e", str).match == str[3:4] @test match(r"(.e)", str).captures == [str[3:4]] let m0 = match(r"(.)e", str) @@ -204,43 +200,43 @@ end end end @test lpad(str, 12) == - Base.AnnotatedString(" some string", [(2:5, :thing, 0x01), - (2:12, :all, 0x03), - (7:12, :other, 0x02)]) + Base.AnnotatedString(" some string", [(2:5, 0x01), + (2:12, 0x03), + (7:12, 0x02)]) @test rpad(str, 12) == - Base.AnnotatedString("some string ", [(1:4, :thing, 0x01), - (1:11, :all, 0x03), - (6:11, :other, 0x02)]) - str1 = Base.AnnotatedString("test", [(1:4, :label, 5)]) - str2 = Base.AnnotatedString("case", [(2:3, :label, "oomph")]) + Base.AnnotatedString("some string ", [(1:4, 0x01), + (1:11, 0x03), + (6:11, 0x02)]) + str1 = Base.AnnotatedString("test", [(1:4, 5)]) + str2 = Base.AnnotatedString("case", [(2:3, "oomph")]) @test join([str1, str1], ' ') == Base.AnnotatedString("test test", - [(1:4, :label, 5), - (6:9, :label, 5)]) - @test join([str1, str1], Base.AnnotatedString(" ", [(1:1, :label, 2)])) == + [(1:4, 5), + (6:9, 5)]) + @test join([str1, str1], Base.AnnotatedString(" ", [(1:1, 2)])) == Base.AnnotatedString("test test", - [(1:4, :label, 5), - (5:5, :label, 2), - (6:9, :label, 5)]) + [(1:4, 5), + (5:5, 2), + (6:9, 5)]) @test join((String(str1), str1), ' ') == - Base.AnnotatedString("test test", [(6:9, :label, 5)]) - @test repeat(str1, 2) == Base.AnnotatedString("testtest", [(1:8, :label, 5)]) - @test repeat(str2, 2) == Base.AnnotatedString("casecase", [(2:3, :label, "oomph"), - (6:7, :label, "oomph")]) - @test repeat(str1[1], 3) == Base.AnnotatedString("ttt", [(1:3, :label, 5)]) - @test reverse(str1) == Base.AnnotatedString("tset", [(1:4, :label, 5)]) - @test reverse(str2) == Base.AnnotatedString("esac", [(2:3, :label, "oomph")]) + Base.AnnotatedString("test test", [(6:9, 5)]) + @test repeat(str1, 2) == Base.AnnotatedString("testtest", [(1:8, 5)]) + @test repeat(str2, 2) == Base.AnnotatedString("casecase", [(2:3, "oomph"), + (6:7, "oomph")]) + @test repeat(str1[1], 3) == Base.AnnotatedString("ttt", [(1:3, 5)]) + @test reverse(str1) == Base.AnnotatedString("tset", [(1:4, 5)]) + @test reverse(str2) == Base.AnnotatedString("esac", [(2:3, "oomph")]) end @testset "Unicode" begin for words in (["ᲃase", "cɦɒnɡeȿ", "can", "CHⱯNGE", "Сodeunıts"], ["Сodeunıts", "ᲃase", "cɦɒnɡeȿ", "can", "CHⱯNGE"]) - ann_words = [Base.AnnotatedString(w, [(1:ncodeunits(w), :i, i)]) + ann_words = [Base.AnnotatedString(w, [(1:ncodeunits(w), i)]) for (i, w) in enumerate(words)] ann_str = join(ann_words, '-') for transform in (lowercase, uppercase, titlecase) t_words = map(transform, words) - ann_t_words = [Base.AnnotatedString(w, [(1:ncodeunits(w), :i, i)]) + ann_t_words = [Base.AnnotatedString(w, [(1:ncodeunits(w), i)]) for (i, w) in enumerate(t_words)] ann_t_str = join(ann_t_words, '-') t_ann_str = transform(ann_str) @@ -249,7 +245,7 @@ end end for transform in (uppercasefirst, lowercasefirst) t_words = vcat(transform(first(words)), words[2:end]) - ann_t_words = [Base.AnnotatedString(w, [(1:ncodeunits(w), :i, i)]) + ann_t_words = [Base.AnnotatedString(w, [(1:ncodeunits(w), i)]) for (i, w) in enumerate(t_words)] ann_t_str = join(ann_t_words, '-') t_ann_str = transform(ann_str) @@ -261,37 +257,37 @@ end @testset "AnnotatedIOBuffer" begin aio = Base.AnnotatedIOBuffer() - vec2ann(v::Vector{Tuple{UnitRange{Int}, Symbol, V}}) where {V} = collect(Base.RegionAnnotation{V}, v) + vec2ann(v::Vector{Tuple{UnitRange{Int}, V}}) where {V} = collect(Base.RegionAnnotation{V}, v) # Append-only writing - @test write(aio, Base.AnnotatedString("hello", [(1:5, :tag, 1)])) == 5 + @test write(aio, Base.AnnotatedString("hello", [(1:5, 1)])) == 5 @test write(aio, ' ') == 1 - @test write(aio, Base.AnnotatedString("world", [(1:5, :tag, 2)])) == 5 - @test Base.annotations(aio) == vec2ann([(1:5, :tag, 1), (7:11, :tag, 2)]) + @test write(aio, Base.AnnotatedString("world", [(1:5, 2)])) == 5 + @test Base.annotations(aio) == vec2ann([(1:5, 1), (7:11, 2)]) # Printing through a context keeps the annotations ctxaio = Base.AnnotatedIOBuffer() - print(IOContext(ctxaio, :color => true), Base.AnnotatedString("hi", [(1:2, :tag, 3)])) - print(IOContext(ctxaio), Base.AnnotatedChar('!', [(:tag, 4)])) - @test Base.annotations(ctxaio) == vec2ann([(1:2, :tag, 3), (3:3, :tag, 4)]) + print(IOContext(ctxaio, :color => true), Base.AnnotatedString("hi", [(1:2, 3)])) + print(IOContext(ctxaio), Base.AnnotatedChar('!', [4])) + @test Base.annotations(ctxaio) == vec2ann([(1:2, 3), (3:3, 4)]) # Check `annotate!`, including region sorting @test truncate(aio, 0).io.size == 0 @test write(aio, "hello world") == ncodeunits("hello world") - @test Base.annotate!(aio, 1:5, :tag, 1) === aio - @test Base.annotate!(aio, 7:11, :tag, 2) === aio - @test Base.annotations(aio) == vec2ann([(1:5, :tag, 1), (7:11, :tag, 2)]) + @test Base.annotate!(aio, 1:5, 1) === aio + @test Base.annotate!(aio, 7:11, 2) === aio + @test Base.annotations(aio) == vec2ann([(1:5, 1), (7:11, 2)]) # Reading @test read(seekstart(deepcopy(aio.io)), String) == "hello world" @test read(seekstart(deepcopy(aio)), String) == "hello world" - @test read(seek(aio, 0), Base.AnnotatedString) == Base.AnnotatedString("hello world", [(1:5, :tag, 1), (7:11, :tag, 2)]) - @test read(seek(aio, 1), Base.AnnotatedString) == Base.AnnotatedString("ello world", [(1:4, :tag, 1), (6:10, :tag, 2)]) - @test read(seek(aio, 4), Base.AnnotatedString) == Base.AnnotatedString("o world", [(1:1, :tag, 1), (3:7, :tag, 2)]) - @test read(seek(aio, 5), Base.AnnotatedString) == Base.AnnotatedString(" world", [(2:6, :tag, 2)]) + @test read(seek(aio, 0), Base.AnnotatedString) == Base.AnnotatedString("hello world", [(1:5, 1), (7:11, 2)]) + @test read(seek(aio, 1), Base.AnnotatedString) == Base.AnnotatedString("ello world", [(1:4, 1), (6:10, 2)]) + @test read(seek(aio, 4), Base.AnnotatedString) == Base.AnnotatedString("o world", [(1:1, 1), (3:7, 2)]) + @test read(seek(aio, 5), Base.AnnotatedString) == Base.AnnotatedString(" world", [(2:6, 2)]) @test read(seekend(aio), Base.AnnotatedString) == Base.AnnotatedString("") - @test read(seekstart(truncate(deepcopy(aio), 5)), Base.AnnotatedString) == Base.AnnotatedString("hello", [(1:5, :tag, 1)]) - @test read(seekstart(truncate(deepcopy(aio), 6)), Base.AnnotatedString) == Base.AnnotatedString("hello ", [(1:5, :tag, 1)]) - @test read(seekstart(truncate(deepcopy(aio), 7)), Base.AnnotatedString) == Base.AnnotatedString("hello w", [(1:5, :tag, 1), (7:7, :tag, 2)]) - @test read(seek(aio, 0), Base.AnnotatedChar) == Base.AnnotatedChar('h', [(:tag, 1)]) + @test read(seekstart(truncate(deepcopy(aio), 5)), Base.AnnotatedString) == Base.AnnotatedString("hello", [(1:5, 1)]) + @test read(seekstart(truncate(deepcopy(aio), 6)), Base.AnnotatedString) == Base.AnnotatedString("hello ", [(1:5, 1)]) + @test read(seekstart(truncate(deepcopy(aio), 7)), Base.AnnotatedString) == Base.AnnotatedString("hello w", [(1:5, 1), (7:7, 2)]) + @test read(seek(aio, 0), Base.AnnotatedChar) == Base.AnnotatedChar('h', [1]) @test read(seek(aio, 5), Base.AnnotatedChar) == Base.AnnotatedChar(' ', []) - @test read(seek(aio, 6), Base.AnnotatedChar) == Base.AnnotatedChar('w', [(:tag, 2)]) + @test read(seek(aio, 6), Base.AnnotatedChar) == Base.AnnotatedChar('w', [2]) # Check method compatibility with IOBuffer @test position(aio) == 7 @test seek(aio, 4) === aio @@ -301,19 +297,19 @@ end # Writing into the middle of the buffer @test write(seek(aio, 6), "alice") == 5 # Replace 'world' with 'alice' @test read(seekstart(aio), String) == "hello alice" - @test Base.annotations(aio) == vec2ann([(1:5, :tag, 1), (7:11, :tag, 2)]) # Should be unchanged - @test write(seek(aio, 0), Base.AnnotatedString("hey-o", [(1:5, :hey, 'o')])) == 5 + @test Base.annotations(aio) == vec2ann([(1:5, 1), (7:11, 2)]) # Should be unchanged + @test write(seek(aio, 0), Base.AnnotatedString("hey-o", [(1:5, 'o')])) == 5 @test read(seekstart(aio), String) == "hey-o alice" - @test Base.annotations(aio) == vec2ann([(7:11, :tag, 2), (1:5, :hey, 'o')]) # First annotation should have been entirely replaced - @test write(seek(aio, 7), Base.AnnotatedString("bbi", [(1:3, :hey, 'a')])) == 3 # a[lic, bbi]e ('alice', 'abbie') + @test Base.annotations(aio) == vec2ann([(7:11, 2), (1:5, 'o')]) # First annotation should have been entirely replaced + @test write(seek(aio, 7), Base.AnnotatedString("bbi", [(1:3, 'a')])) == 3 # a[lic, bbi]e ('alice', 'abbie') @test read(seekstart(aio), String) == "hey-o abbie" - @test Base.annotations(aio) == vec2ann([(7:7, :tag, 2), (11:11, :tag, 2), (1:5, :hey, 'o'), (8:10, :hey, 'a')]) + @test Base.annotations(aio) == vec2ann([(7:7, 2), (11:11, 2), (1:5, 'o'), (8:10, 'a')]) @test write(seek(aio, 0), Base.AnnotatedString("ab")) == 2 # Check first annotation's region is adjusted correctly @test read(seekstart(aio), String) == "aby-o abbie" - @test Base.annotations(aio) == vec2ann([(7:7, :tag, 2), (11:11, :tag, 2), (3:5, :hey, 'o'), (8:10, :hey, 'a')]) + @test Base.annotations(aio) == vec2ann([(7:7, 2), (11:11, 2), (3:5, 'o'), (8:10, 'a')]) @test write(seek(aio, 3), Base.AnnotatedString("ss")) == 2 @test read(seekstart(aio), String) == "abyss abbie" - @test Base.annotations(aio) == vec2ann([(7:7, :tag, 2), (11:11, :tag, 2), (3:3, :hey, 'o'), (8:10, :hey, 'a')]) + @test Base.annotations(aio) == vec2ann([(7:7, 2), (11:11, 2), (3:3, 'o'), (8:10, 'a')]) # Writing one buffer to another newaio = Base.AnnotatedIOBuffer() @test write(newaio, seekstart(aio)) == 11 @@ -324,144 +320,150 @@ end @test write(newaio, seek(aio, 5)) == 6 @test read(seekstart(newaio), String) == "abyss abbie abbie" @test sort(Base.annotations(newaio)) == - sort(vcat(Base.annotations(aio), vec2ann([(13:13, :tag, 2), (14:16, :hey, 'a'), (17:17, :tag, 2)]))) + sort(vcat(Base.annotations(aio), vec2ann([(13:13, 2), (14:16, 'a'), (17:17, 2)]))) # The `_insert_annotations!` cautious-merging optimisation aio = Base.AnnotatedIOBuffer() - @test write(aio, Base.AnnotatedChar('a', [(:a, 1), (:b, 2)])) == 1 - @test Base.annotations(aio) == vec2ann([(1:1, :a, 1), (1:1, :b, 2)]) - @test write(aio, Base.AnnotatedChar('b', [(:a, 1), (:b, 2)])) == 1 - @test Base.annotations(aio) == vec2ann([(1:2, :a, 1), (1:2, :b, 2)]) + @test write(aio, Base.AnnotatedChar('a', [1, 2])) == 1 + @test Base.annotations(aio) == vec2ann([(1:1, 1), (1:1, 2)]) + @test write(aio, Base.AnnotatedChar('b', [1, 2])) == 1 + @test Base.annotations(aio) == vec2ann([(1:2, 1), (1:2, 2)]) let aio2 = copy(aio) # A different start makes merging too risky to do. - @test write(aio2, Base.AnnotatedChar('c', [(:a, 0), (:b, 2)])) == 1 - @test Base.annotations(aio2) == vec2ann([(1:2, :a, 1), (1:2, :b, 2), (3:3, :a, 0), (3:3, :b, 2)]) + @test write(aio2, Base.AnnotatedChar('c', [0, 2])) == 1 + @test Base.annotations(aio2) == vec2ann([(1:2, 1), (1:2, 2), (3:3, 0), (3:3, 2)]) end let aio2 = copy(aio) # Merging some run of the most recent annotations is fine though. - @test write(aio2, Base.AnnotatedChar('c', [(:b, 2)])) == 1 - @test Base.annotations(aio2) == vec2ann([(1:2, :a, 1), (1:3, :b, 2)]) + @test write(aio2, Base.AnnotatedChar('c', [2])) == 1 + @test Base.annotations(aio2) == vec2ann([(1:2, 1), (1:3, 2)]) end let aio2 = copy(aio) # ...and any subsequent annotations after a matching run can just be copied over. - @test write(aio2, Base.AnnotatedChar('c', [(:b, 2), (:c, 3), (:d, 4)])) == 1 - @test Base.annotations(aio2) == vec2ann([(1:2, :a, 1), (1:3, :b, 2), (3:3, :c, 3), (3:3, :d, 4)]) + @test write(aio2, Base.AnnotatedChar('c', [2, 3, 4])) == 1 + @test Base.annotations(aio2) == vec2ann([(1:2, 1), (1:3, 2), (3:3, 3), (3:3, 4)]) end let aio2 = Base.AnnotatedIOBuffer() - @test write(aio2, Base.AnnotatedChar('a', [(:b, 1)])) == 1 - @test write(aio2, Base.AnnotatedChar('b', [(:a, 1), (:b, 1)])) == 1 + @test write(aio2, Base.AnnotatedChar('a', [2])) == 1 + @test write(aio2, Base.AnnotatedChar('b', [1, 2])) == 1 @test read(seekstart(aio2), Base.AnnotatedString) == - Base.AnnotatedString("ab", [(1:1, :b, 1), (2:2, :a, 1), (2:2, :b, 1)]) + Base.AnnotatedString("ab", [(1:1, 2), (2:2, 1), (2:2, 2)]) end let aio2 = Base.AnnotatedIOBuffer() # Equal but distinct values are not merged - write(aio2, Base.AnnotatedString("a", [(1:1, :x, [1])])) - write(aio2, Base.AnnotatedString("b", [(1:1, :x, [1])])) + write(aio2, Base.AnnotatedString("a", [(1:1, [1])])) + write(aio2, Base.AnnotatedString("b", [(1:1, [1])])) @test length(Base.annotations(aio2)) == 2 end let ints = Base.AnnotatedIOBuffer{Int}() # Between value types - write(ints, Base.AnnotatedString("ab", [(1:1, :x, 1)])) + write(ints, Base.AnnotatedString("ab", [(1:1, 1)])) anys = Base.AnnotatedIOBuffer() write(anys, seekstart(ints)) - @test Base.annotations(anys) == vec2ann([(1:1, :x, 1)]) + @test Base.annotations(anys) == vec2ann([(1:1, 1)]) @test read(seekstart(anys), Base.AnnotatedString{String, Int}) isa Base.AnnotatedString{String, Int} - write(anys, Base.AnnotatedString("c", [(1:1, :y, "c")])) + write(anys, Base.AnnotatedString("c", [(1:1, "c")])) @test_throws MethodError read(seekstart(anys), Base.AnnotatedString{String, Int}) end let ints = Base.AnnotatedIOBuffer{Int}() # Chars are read with the buffer's value type, or a given one - write(ints, Base.AnnotatedString("ab", [(1:1, :x, 1)])) + write(ints, Base.AnnotatedString("ab", [(1:1, 1)])) @test read(seekstart(ints), Base.AnnotatedChar) isa Base.AnnotatedChar{Char, Int} - @test read(seekstart(ints), Base.AnnotatedChar{Char, Any}) == Base.AnnotatedChar{Char, Any}('a', [(:x, 1)]) + @test read(seekstart(ints), Base.AnnotatedChar{Char, Any}) == Base.AnnotatedChar{Char, Any}('a', [1]) end - let nested = Base.AnnotatedString(SubString("hello", 1, 3), [(1:1, :x, 1)]) # Over a `SubString` + let nested = Base.AnnotatedString(SubString("hello", 1, 3), [(1:1, 1)]) # Over a `SubString` aio2 = Base.AnnotatedIOBuffer() @test write(aio2, nested[1:2]) == 2 - @test read(seekstart(aio2), Base.AnnotatedString) == Base.AnnotatedString("he", [(1:1, :x, 1)]) + @test read(seekstart(aio2), Base.AnnotatedString) == Base.AnnotatedString("he", [(1:1, 1)]) end # Working through an IOContext aio = Base.AnnotatedIOBuffer() wrapio = IOContext(aio) - @test write(wrapio, Base.AnnotatedString("hey", [(1:3, :x, 1)])) == 3 - @test write(wrapio, Base.AnnotatedChar('a', [(:y, 2)])) == 1 + @test write(wrapio, Base.AnnotatedString("hey", [(1:3, 1)])) == 3 + @test write(wrapio, Base.AnnotatedChar('a', [2])) == 1 @test read(seekstart(aio), Base.AnnotatedString) == - Base.AnnotatedString("heya", [(1:3, :x, 1), (4:4, :y, 2)]) + Base.AnnotatedString("heya", [(1:3, 1), (4:4, 2)]) # show-ing an AnnotatedIOBuffer aio = Base.AnnotatedIOBuffer() - write(aio, Base.AnnotatedString("hello", [(1:5, :tag, 1)])) + write(aio, Base.AnnotatedString("hello", [(1:5, 1)])) @test chopprefix(sprint(show, aio), "Base.") == "AnnotatedIOBuffer(5 bytes, 1 annotation)" # A value that doesn't fit the destination throws before any text is written let src = Base.AnnotatedIOBuffer(), dest = Base.AnnotatedIOBuffer{Int}() - write(src, Base.AnnotatedString("ab", [(1:1, :x, "s")])) + write(src, Base.AnnotatedString("ab", [(1:1, "s")])) @test_throws MethodError write(dest, seekstart(src)) @test position(dest) == 0 end end @testset "Eachregion" begin - annregions(str::String, annots::Vector{<:Tuple{UnitRange{Int}, Symbol, <:Any}}) = - [(s, Tuple.(a)) for (s, a) in Base.eachregion(Base.AnnotatedString(str, annots))] + annregions(str::String, annots::Vector{<:Tuple{UnitRange{Int}, <:Any}}) = + [(s, collect(a)) for (s, a) in Base.eachregion(Base.AnnotatedString(str, annots))] # Regions that do/don't extend to the left/right edges - @test annregions(" abc ", [(2:4, :face, :bold)]) == + @test annregions(" abc ", [(2:4, :bold)]) == [(" ", []), - ("abc", [(:face, :bold)]), + ("abc", [:bold]), (" ", [])] - @test annregions(" x ", [(2:2, :face, :bold)]) == + @test annregions(" x ", [(2:2, :bold)]) == [(" ", []), - ("x", [(:face, :bold)]), + ("x", [:bold]), (" ", [])] - @test annregions(" x", [(2:2, :face, :bold)]) == + @test annregions(" x", [(2:2, :bold)]) == [(" ", []), - ("x", [(:face, :bold)])] - @test annregions("x ", [(1:1, :face, :bold)]) == - [("x", [(:face, :bold)]), + ("x", [:bold])] + @test annregions("x ", [(1:1, :bold)]) == + [("x", [:bold]), (" ", [])] - @test annregions("x", [(1:1, :face, :bold)]) == - [("x", [(:face, :bold)])] + @test annregions("x", [(1:1, :bold)]) == + [("x", [:bold])] # Overlapping/nested regions - @test annregions(" abc ", [(2:4, :face, :bold), (3:3, :face, :italic)]) == + @test annregions(" abc ", [(2:4, :bold), (3:3, :italic)]) == [(" ", []), - ("a", [(:face, :bold)]), - ("b", [(:face, :bold), (:face, :italic)]), - ("c", [(:face, :bold)]), + ("a", [:bold]), + ("b", [:bold, :italic]), + ("c", [:bold]), (" ", [])] - @test annregions("abc-xyz", [(1:7, :face, :bold), (1:3, :face, :green), (4:4, :face, :yellow), (4:7, :face, :italic)]) == - [("abc", [(:face, :bold), (:face, :green)]), - ("-", [(:face, :bold), (:face, :yellow), (:face, :italic)]), - ("xyz", [(:face, :bold), (:face, :italic)])] + @test annregions("abc-xyz", [(1:7, :bold), (1:3, :green), (4:4, :yellow), (4:7, :italic)]) == + [("abc", [:bold, :green]), + ("-", [:bold, :yellow, :italic]), + ("xyz", [:bold, :italic])] # Preserving annotation order - @test annregions("abcd", [(1:3, :face, :red), (2:2, :face, :yellow), (2:3, :face, :green), (2:4, :face, :blue)]) == - [("a", [(:face, :red)]), - ("b", [(:face, :red), (:face, :yellow), (:face, :green), (:face, :blue)]), - ("c", [(:face, :red), (:face, :green), (:face, :blue)]), - ("d", [(:face, :blue)])] - @test annregions("abcd", [(2:4, :face, :blue), (1:3, :face, :red), (2:3, :face, :green), (2:2, :face, :yellow)]) == - [("a", [(:face, :red)]), - ("b", [(:face, :blue), (:face, :red), (:face, :green), (:face, :yellow)]), - ("c", [(:face, :blue), (:face, :red), (:face, :green)]), - ("d", [(:face, :blue)])] + @test annregions("abcd", [(1:3, :red), (2:2, :yellow), (2:3, :green), (2:4, :blue)]) == + [("a", [:red]), + ("b", [:red, :yellow, :green, :blue]), + ("c", [:red, :green, :blue]), + ("d", [:blue])] + @test annregions("abcd", [(2:4, :blue), (1:3, :red), (2:3, :green), (2:2, :yellow)]) == + [("a", [:red]), + ("b", [:blue, :red, :green, :yellow]), + ("c", [:blue, :red, :green]), + ("d", [:blue])] # Region starting after a character spanning multiple codepoints. - @test annregions("𝟏x", [(1:4, :face, :red)]) == - [("𝟏", [(:face, :red)]), + @test annregions("𝟏x", [(1:4, :red)]) == + [("𝟏", [:red]), ("x", [])] # Annotations and subregions that start within a character - let str = Base.AnnotatedString("aébc", [(3:3, :face, :red)]) + let str = Base.AnnotatedString("aébc", [(3:3, :red)]) regions = collect(Base.eachregion(str)) # Read after the iterator has moved on - @test [(s, Tuple.(a)) for (s, a) in regions] == [("a", []), ("é", [(:face, :red)]), ("bc", [])] + @test [(s, collect(a)) for (s, a) in regions] == [("a", []), ("é", [:red]), ("bc", [])] @test [s for (s, _) in Base.eachregion(str, 3:5)] == ["é", "bc"] end # A subregion starting within a character, whose regions read the same during iteration and after - let str = Base.AnnotatedString("aébcdef", [(1:8, :a, 1), (3:5, :b, 2), (5:5, :c, 3), (2:2, :d, 4)]) - @test [(String(s), Tuple.(a)) for (s, a) in Base.eachregion(str, 3:5)] == - [(String(s), Tuple.(a)) for (s, a) in collect(Base.eachregion(str, 3:5))] + let str = Base.AnnotatedString("aébcdef", [(1:8, 1), (3:5, 2), (5:5, 3), (2:2, 4)]) + @test [(String(s), collect(a)) for (s, a) in Base.eachregion(str, 3:5)] == + [(String(s), collect(a)) for (s, a) in collect(Base.eachregion(str, 3:5))] end # Read after the iterator has left them, past an annotation beyond the end of the string - @test [Tuple.(a) for (_, a) in collect(Base.eachregion(Base.AnnotatedString("ab", [(1:1, :a, 1), (5:6, :b, 2)])))] == [[(:a, 1)], []] + @test [collect(a) for (_, a) in collect(Base.eachregion(Base.AnnotatedString("ab", [(1:1, 1), (5:6, 2)])))] == [[1], []] # Passes over one iterator at the same time don't share their state - let regions = Base.eachregion(Base.AnnotatedString("abcd", [(1:2, :a, 1), (2:3, :b, 2)])) - pass = [(String(s), Tuple.(a)) for (s, a) in regions] - @test [(String(s), Tuple.(a)) for ((s, a), _) in zip(regions, regions)] == pass - @test [(String(s), Tuple.(a)) for (s, a) in regions for _ in regions] == repeat(pass, inner = length(pass)) + let regions = Base.eachregion(Base.AnnotatedString("abcd", [(1:2, 1), (2:3, 2)])) + pass = [(String(s), collect(a)) for (s, a) in regions] + @test [(String(s), collect(a)) for ((s, a), _) in zip(regions, regions)] == pass + @test [(String(s), collect(a)) for (s, a) in regions for _ in regions] == repeat(pass, inner = length(pass)) end # A string backed by a substring, whose regions are substrings of the same parent - @test length(collect(Base.eachregion(Base.AnnotatedString(SubString("xab", 2:3), [(1:1, :a, 1)])))) == 2 + @test length(collect(Base.eachregion(Base.AnnotatedString(SubString("xab", 2:3), [(1:1, 1)])))) == 2 + # A region's values of a union type are iterated without boxing, whether the iterator is on it or has left it + let regions = collect(Base.eachregion(Base.AnnotatedString{String, Union{Int, Symbol}}("ab", [(1:2, 1), (1:1, :x)]))) + ints(annots) = (n = 0; for value in annots; value isa Int && (n += 1) end; n) + @test ints(regions[1][2]) == 1 && ints(regions[2][2]) == 1 + @test (@allocated ints(regions[1][2])) == (@allocated ints(regions[2][2])) == 0 + end # Changing a string while its regions are read is unsupported, but memory safe - let str = Base.AnnotatedString("abcdef", [(1:2, :a, 1), (3:4, :a, 2), (5:6, :a, 3)]) - @test count(_ -> (Base.annotate!(str, 1:6, :b, 9); true), Base.eachregion(str)) == 3 + let str = Base.AnnotatedString("abcdef", [(1:2, 1), (3:4, 2), (5:6, 3)]) + @test count(_ -> (Base.annotate!(str, 1:6, 9); true), Base.eachregion(str)) == 3 regions = collect(Base.eachregion(str)) empty!(str.annotations) @test_throws BoundsError collect(last(regions)[2]) @@ -470,7 +472,7 @@ end @testset "Replacement" begin astr(s::String, faceregions::Tuple{UnitRange{Int}, Symbol}...) = - Base.AnnotatedString(s, [(r, :face, f) for (r, f) in faceregions]) + Base.AnnotatedString(s, [(r, f) for (r, f) in faceregions]) @testset "Basic Transformations" begin @testset "Deletion" begin @@ -556,9 +558,9 @@ end astr("cafe", (1:3, :green)) @test replace(astr("test"), 't' => astr("TTT", (1:3, :blue))) == astr("TTTesTTT", (1:3, :blue), (6:8, :blue)) - @test replace(astr("hello", (1:5, :red)), 'l' => Base.AnnotatedChar('L', [(label=:face, value=:blue)])) == + @test replace(astr("hello", (1:5, :red)), 'l' => Base.AnnotatedChar('L', [:blue])) == astr("heLLo", (1:2, :red), (5:5, :red), (3:4, :blue)) - @test replace(astr("abc", (1:3, :green)), 'b' => Base.AnnotatedChar('B', [(label=:face, value=:bold)])) == + @test replace(astr("abc", (1:3, :green)), 'b' => Base.AnnotatedChar('B', [:bold])) == astr("aBc", (1:1, :green), (3:3, :green), (2:2, :bold)) end @@ -595,11 +597,11 @@ end end @testset "AnnotatedChar" begin - @test replace(astr("test", (1:4, :red)), 't' => Base.AnnotatedChar('T', [(label=:face, value=:blue)])) == + @test replace(astr("test", (1:4, :red)), 't' => Base.AnnotatedChar('T', [:blue])) == astr("TesT", (2:3, :red), (1:1, :blue), (4:4, :blue)) - @test replace(astr("hello"), 'l' => Base.AnnotatedChar('L', [(label=:face, value=:bold), (label=:face, value=:red)])) == + @test replace(astr("hello"), 'l' => Base.AnnotatedChar('L', [:bold, :red])) == astr("heLLo", (3:4, :bold), (3:4, :red)) - @test replace(astr("a b c", (1:5, :green)), ' ' => Base.AnnotatedChar('_', [(label=:face, value=:underline)])) == + @test replace(astr("a b c", (1:5, :green)), ' ' => Base.AnnotatedChar('_', [:underline])) == astr("a_b_c", (1:1, :green), (3:3, :green), (5:5, :green), (2:2, :underline), (4:4, :underline)) end @@ -823,12 +825,12 @@ end end @testset "Multiple annotations same region" begin - # Same region with different labels + # Same region with different values str = astr("test", (1:4, :red), (1:4, :bold), (2:3, :italic)) @test replace(str, "es" => "ES") == astr("tESt", (1:1, :red), (4:4, :red), (1:1, :bold), (4:4, :bold)) - # Same label, different values on different regions + # Different values on different regions str2 = astr("test", (1:2, :red), (3:4, :blue)) @test replace(str2, "es" => "ES") == astr("tESt", (1:1, :red), (4:4, :blue)) @@ -839,7 +841,7 @@ end str = astr("abc", (1:3, :red)) result = replace(str, 'a' => astr("A", (1:1, :red)), 'b' => astr("B", (1:1, :red)), 'c' => astr("C", (1:1, :red))) @test String(result) == "ABC" - @test Base.annotations(result) == [(region=1:3, label=:face, value=:red)] + @test Base.annotations(result) == [(region=1:3, value=:red)] # Non-adjacent should not merge str2 = astr("axbxc", (1:5, :green)) @@ -863,7 +865,7 @@ end str = astr("a" * "b"^20, (1:21, :red)) result = replace(str, "b" => "B") @test String(result) == "a" * "B"^20 - @test Base.annotations(result) == [(region=1:1, label=:face, value=:red)] + @test Base.annotations(result) == [(region=1:1, value=:red)] # Multiple different patterns str2 = astr("ababababab", (1:10, :green)) @@ -918,7 +920,7 @@ end replace(buf, astr("test", (1:4, :green)), "t" => "T") result = read(seekstart(buf), Base.AnnotatedString) @test String(result) == "prefix TesT" - @test Base.annotations(result) == [(region=9:10, label=:face, value=:green)] + @test Base.annotations(result) == [(region=9:10, value=:green)] buf = Base.AnnotatedIOBuffer() replace(buf, astr("line1", (1:5, :red)), "1" => "A") @@ -943,8 +945,8 @@ end # Standard IOBuffer doesn't truncate, so we get "TesTinal" not "TesT" @test String(result) == "TesTinal" # Annotations in the replaced region should be cleared, old ones shifted - @test Base.annotations(result) == [(region=5:8, label=:face, value=:red), - (region=2:3, label=:face, value=:blue)] + @test Base.annotations(result) == [(region=5:8, value=:red), + (region=2:3, value=:blue)] # Multiple sequential replacements to same buffer (appending) buf2 = Base.AnnotatedIOBuffer() @@ -973,7 +975,7 @@ end end @testset "Typed buffers" begin - wide = Base.AnnotatedString{String, Any}("Q", [(1:1, :n, 1)]) + wide = Base.AnnotatedString{String, Any}("Q", [(1:1, 1)]) anybuf = Base.AnnotatedIOBuffer() replace(anybuf, astr("apple", (1:5, :red)), "p" => wide) typedbuf = Base.AnnotatedIOBuffer{Union{Symbol, Int}}() @@ -998,15 +1000,15 @@ module MarkedAnnotations struct Tagged name::String end - marked(annotations) = Marked(any(a -> a.value isa Mark, annotations)) + marked(annotations) = Marked(any(value -> value isa Mark, annotations)) reduction(::MIME"text/plain", ::Type{Mark}, annotations) = marked(annotations) reduction(::MIME"text/html", ::Type{Mark}, annotations) = marked(annotations) transition(io::IO, ::MIME"text/plain", from::Marked, to::Marked) = from === to || print(io, if to.on "<" else ">" end) transition(io::IO, ::MIME"text/html", ::Marked, ::Marked) = print(io, "") transition(io::IO, ::MIME"text/html", ::Marked, ::Nothing) = print(io, "") function reduction(::MIME"text/plain", ::Type{Tag}, annotations) - i = findlast(a -> a.value isa Tag, annotations) - Tagged(if isnothing(i) "" else annotations[i].value.name end) + i = findlast(value -> value isa Tag, annotations) + Tagged(if isnothing(i) "" else annotations[i].name end) end function transition(io::IO, ::MIME"text/plain", from::Tagged, to::Tagged) from === to && return @@ -1019,13 +1021,13 @@ end (; Mark, Tag, Marked) = MarkedAnnotations colour(x) = sprint(print, x; context = :color => true) html(x) = sprint(show, MIME("text/html"), x) - marked(V) = Base.AnnotatedString{String, V}("xy", [(1:1, :m, Mark())]) + marked(V) = Base.AnnotatedString{String, V}("xy", [(1:1, Mark())]) # A type's display is written between the regions, and only to a terminal @test colour(marked(Mark)) == "y" @test sprint(print, marked(Mark)) == "xy" - @test colour(Base.AnnotatedString{String, Int}("x", [(1:1, :n, 1)])) == "x" # Not displayed + @test colour(Base.AnnotatedString{String, Int}("x", [(1:1, 1)])) == "x" # Not displayed # A string displays the same whatever union of value types holds it - tagged(V) = Base.AnnotatedString{String, V}("abc", [(1:2, :m, Mark()), (2:3, :t, Tag("t")), (3:3, :n, 1)]) + tagged(V) = Base.AnnotatedString{String, V}("abc", [(1:2, Mark()), (2:3, Tag("t")), (3:3, 1)]) @test colour(tagged(Union{Mark, Tag, Int})) == "c]" @test colour(tagged(Union{Mark, Tag, Int, Symbol})) == colour(tagged(Any)) == colour(tagged(Union{Mark, Tag, Int})) @test colour(Base.AnnotatedString{String, Any}("x")) == "x" @@ -1035,34 +1037,33 @@ end # The text of HTML is always escaped @test html(marked(Mark)) == "xy" @test html(SubString(marked(Any), 1, 1)) == html(marked(Mark)[1]) == "x" - plain = Base.AnnotatedString{String, Union{Tag, Int}}("a true) == "y" - @test sprint(escape_string, Base.AnnotatedString{String, Mark}("a\nb", [(1:1, :m, Mark()), (3:3, :m, Mark())]); + @test sprint(escape_string, Base.AnnotatedString{String, Mark}("a\nb", [(1:1, Mark()), (3:3, Mark())]); context = :color => true) == "\\n" escaped(s; wrap = identity) = (buf = Base.AnnotatedIOBuffer(); escape_string(wrap(buf), s); read(seekstart(buf), Base.AnnotatedString)) - @test escaped(Base.AnnotatedString("a\nb\tc", [(1:1, :x, 1), (3:3, :y, 2), (2:5, :z, 3)])) == Base.AnnotatedString("a\\nb\\tc", [(1:1, :x, 1), (4:4, :y, 2), (2:7, :z, 3)]) - @test escaped(SubString(Base.AnnotatedString("a\nbc", [(1:3, :x, 1)]), 2, 4)) == Base.AnnotatedString("\\nbc", [(1:3, :x, 1)]) - @test escaped(Base.AnnotatedString("x\ty", [(2:2, :t, 1)]); wrap = io -> IOContext(io, :color => true)) == Base.AnnotatedString("x\\ty", [(2:3, :t, 1)]) + @test escaped(Base.AnnotatedString("a\nb\tc", [(1:1, 1), (3:3, 2), (2:5, 3)])) == Base.AnnotatedString("a\\nb\\tc", [(1:1, 1), (4:4, 2), (2:7, 3)]) + @test escaped(SubString(Base.AnnotatedString("a\nbc", [(1:3, 1)]), 2, 4)) == Base.AnnotatedString("\\nbc", [(1:3, 1)]) + @test escaped(Base.AnnotatedString("x\ty", [(2:2, 1)]); wrap = io -> IOContext(io, :color => true)) == Base.AnnotatedString("x\\ty", [(2:3, 1)]) let buf = Base.AnnotatedIOBuffer{Float64}() # Values are converted to the buffer's value type, as by `write` - escape_string(buf, Base.AnnotatedString("ab", [(1:1, :x, 1)])) - @test Base.annotations(buf) == [(region = 1:1, label = :x, value = 1.0)] + escape_string(buf, Base.AnnotatedString("ab", [(1:1, 1)])) + @test Base.annotations(buf) == [(region = 1:1, value = 1.0)] end # A known union's states are found without dispatch at runtime - none = Memory{Base.Annotation{Union{Mark, Int}}}(undef, 0) + none = Memory{Union{Mark, Int}}(undef, 0) states = @inferred Base.AnnotatedDisplay.reductions(MIME("text/plain"), Union{Mark, Int}, (nothing, nothing), none) @test Set(states) == Set([Marked(false), nothing]) # An `Any` string is displayed by code compiled once, whatever types it holds specialisations() = sum(m -> length(Base.specializations(m)), methods(Base.AnnotatedDisplay.render)) - anyof(values...) = Base.AnnotatedString{String, Any}("abc", [(i:i, :v, v) for (i, v) in enumerate(values)]) + anyof(values...) = Base.AnnotatedString{String, Any}("abc", [(i:i, v) for (i, v) in enumerate(values)]) colour(anyof(Mark())) before = specialisations() for values in ((Mark(), Tag("t")), (Tag("t"), 1), (1.0, Mark(), :s)) From ea5bf0b2fe6fcbc07f932a57bb957cf5cbd9c997 Mon Sep 17 00:00:00 2001 From: TEC Date: Fri, 16 Oct 2026 07:44:11 +0800 Subject: [PATCH 15/23] Deprecate the labelled forms of annotations Code written for labelled annotations keeps working. A labelled value is kept as it is when it has the kind of its label, and otherwise becomes `label => value`, a `Pair{Symbol, Any}`. The kind comes from the internal `labelled_kind`, which is the label itself unless a package gives it a type, as StyledStrings does for `:face` and `:link`. Deletion with `nothing` removes the annotations of that kind. Labelled writes with a value are deprecated. Deletion is not, as there is no other way to delete yet. A vector of labelled annotations is converted when all of it is labelled. Reading `.label` breaks. All of it is in `deprecated.jl`, so that it can be removed as one. StyledStrings gives `:face` and `:link` their kinds, so it is bumped together: Stdlib: StyledStrings URL: https://github.com/JuliaLang/StyledStrings.jl.git Stdlib branch: face-primacy Old commit: d6479d5 New commit: 10c37a7 Diff: https://github.com/JuliaLang/StyledStrings.jl/compare/d6479d542ec51030bc0358076a8b532896b23b5c...10c37a775558e11159430165edafccbb3fe9478e Assisted-by: Claude Code (Opus 5.5) --- base/deprecated.jl | 67 +++++++++++++++++++ .../md5 | 1 + .../sha512 | 1 + .../md5 | 1 - .../sha512 | 1 - stdlib/StyledStrings.version | 2 +- test/strings/annotated.jl | 35 ++++++++++ 7 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/md5 create mode 100644 deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/sha512 delete mode 100644 deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/md5 delete mode 100644 deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/sha512 diff --git a/base/deprecated.jl b/base/deprecated.jl index c263deff8d44f..cbb4635902dfb 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -747,4 +747,71 @@ end @noinline tan_domain_error(x) = throw_finite_domainerror(:tan, x) end +# Annotations with labels, from before the type of a value said what it is (#60527) + +# The kind of annotation a label stands for: a type of value, or by default the label itself, as +# the key of a `Pair{Symbol, Any}`. StyledStrings gives `:face` and `:link` their types. +labelled_kind(::Val{label}) where {label} = label + +# The value a labelled annotation stands for: its own when it is of the label's kind, and otherwise `label => value` +function labelled_value(label::Symbol, value) + kind = labelled_kind(Val(label)) + if kind isa Type && value isa kind value else Pair{Symbol, Any}(label, value) end +end + +function AnnotatedString{S}(s::S, annots::Vector{<:Union{Tuple{Any, Symbol, Any}, NamedTuple{(:region, :label, :value)}}}) where {S <: AbstractString} + isempty(annots) && return AnnotatedString{S, Any}(s) # Such as a `Vector{Union{}}`, which dispatch also puts here + depwarn("annotations with a label are deprecated; give `(region, value)`, where the type of the value says what it is", :AnnotatedString) + AnnotatedString{S}(s, [(region, labelled_value(Symbol(label), value)) for (region, label, value) in annots]) +end + +function AnnotatedChar(c::AbstractChar, annots::Vector{<:Union{Tuple{Symbol, Any}, NamedTuple{(:label, :value)}}}) + isempty(annots) && return AnnotatedChar(c, Any[]) + depwarn("annotations with a label are deprecated; give the value, whose type says what it is", :AnnotatedChar) + AnnotatedChar(c, [labelled_value(label, value) for (label, value) in annots]) +end +AnnotatedChar(c::AnnotatedChar, annots::Vector{<:Union{Tuple{Symbol, Any}, NamedTuple{(:label, :value)}}}) = + AnnotatedChar(c, AnnotatedChar(c.char, annots).annotations) + +function _annotate!(annlist::Vector{<:RegionAnnotation}, region::UnitRange{Int}, label::Symbol, value) + if isnothing(value) + kind = labelled_kind(Val(label)) + deleteat!(annlist, findall(annlist) do annot + annot.region == region && + if kind isa Type annot.value isa kind else annot.value isa Pair{Symbol} && first(annot.value) === kind end + end) + else + depwarn("`annotate!` with a label is deprecated; annotate with the value, whose type says what it is", :annotate!) + _annotate!(annlist, region, labelled_value(label, value)) + end +end + +""" + annotate!(str, [range::UnitRange{Int}], label::Symbol, nothing) + +Remove the annotations over exactly `range` of the kind that `label` stands for: `Face`s for +`:face` and `Link`s for `:link`, with StyledStrings, and otherwise `label => value` pairs. With +a value other than `nothing`, this labelled form is deprecated, and annotates with the value +when it is of that kind, or else with `label => value`. +""" +annotate!(s::AnnotatedString, range::UnitRange{Int}, label::Symbol, val) = + (_annotate!(s.annotations, range, label, val); s) + +annotate!(ss::AnnotatedString, label::Symbol, val) = + annotate!(ss, firstindex(ss):lastindex(ss), label, val) + +annotate!(s::SubString{<:AnnotatedString}, range::UnitRange{Int}, label::Symbol, val) = + (annotate!(s.string, s.offset .+ (range), label, val); s) + +annotate!(s::SubString{<:AnnotatedString}, label::Symbol, val) = + (annotate!(s.string, s.offset .+ (1:s.ncodeunits), label, val); s) + +annotate!(io::AnnotatedIOBuffer, range::UnitRange{Int}, label::Symbol, @nospecialize(val::Any)) = + (_annotate!(io.annotations, range, label, val); io) + +function annotate!(c::AnnotatedChar, label::Symbol, value) + depwarn("`annotate!` with a label is deprecated; annotate with the value, whose type says what it is", :annotate!) + annotate!(c, labelled_value(label, value)) +end + # END 1.14 deprecations diff --git a/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/md5 b/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/md5 new file mode 100644 index 0000000000000..d5fe2899d2b04 --- /dev/null +++ b/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/md5 @@ -0,0 +1 @@ +572b6d9e6c8d9cc9ae0ea7eaf819e3ea diff --git a/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/sha512 b/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/sha512 new file mode 100644 index 0000000000000..c5680ae81c7ee --- /dev/null +++ b/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/sha512 @@ -0,0 +1 @@ +d800368c3bd0f2e0a0a095e877960521f1669ad83e139d9d663c116a537f0a71c05b115926f2357de44296615f9111a612f8c030615154a238a8c27413c6986c diff --git a/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/md5 b/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/md5 deleted file mode 100644 index f1c9b08f8e149..0000000000000 --- a/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -ecc20f4d244c10d3a98424aee93bf6b4 diff --git a/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/sha512 b/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/sha512 deleted file mode 100644 index b5ec01045eca3..0000000000000 --- a/deps/checksums/StyledStrings-d6479d542ec51030bc0358076a8b532896b23b5c.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -5384fdb0bf2d91927fcfb579d1e97eaecae2658507fb6021b23ecb0b31e052be9469f7f5165c37caab1e196cb9ad1cb1ccd24ab40ce5a2a142eb448cdbd6ae78 diff --git a/stdlib/StyledStrings.version b/stdlib/StyledStrings.version index 6195fe0a294ec..4e9ec41f6738a 100644 --- a/stdlib/StyledStrings.version +++ b/stdlib/StyledStrings.version @@ -1,4 +1,4 @@ STYLEDSTRINGS_BRANCH = face-primacy -STYLEDSTRINGS_SHA1 = d6479d542ec51030bc0358076a8b532896b23b5c +STYLEDSTRINGS_SHA1 = 10c37a775558e11159430165edafccbb3fe9478e STYLEDSTRINGS_GIT_URL := https://github.com/JuliaLang/StyledStrings.jl.git STYLEDSTRINGS_TAR_URL = https://api.github.com/repos/JuliaLang/StyledStrings.jl/tarball/$1 diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index 33e0b9477d47a..d2fcef7b97761 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -147,6 +147,41 @@ end @test readchomp(`$(Base.julia_cmd()) --startup-file=no -e $script`) == "false bγ" end +# A package's value type, which it claims a label for +struct Weight + w::Int +end +Base.labelled_kind(::Val{:weight}) = Weight + +@testset "Labelled annotations" begin + # Deleting by label removes the annotations of its kind over exactly the region: a type of value, or a key + let s = Base.AnnotatedString{String, Any}("abc", [(1:3, :size => 2), (1:3, Weight(1)), (1:2, :size => 3), (1:3, :font => 4)]) + Base.annotate!(s, 1:3, :size, nothing) + @test Base.annotations(s) == [(region = 1:3, value = Weight(1)), (region = 1:2, value = :size => 3), (region = 1:3, value = :font => 4)] + Base.annotate!(s, 1:3, :weight, nothing) + @test Base.annotations(s) == [(region = 1:2, value = :size => 3), (region = 1:3, value = :font => 4)] + end + # Other labelled forms are deprecated: a value of the label's kind is kept, and any other becomes `label => value` + @test_deprecated Base.AnnotatedString("ab", [(1:1, :size, 2)]) + @test_deprecated Base.AnnotatedChar('a', [(:size, 2)]) + @test_deprecated Base.annotate!(Base.AnnotatedString{String, Any}("ab"), 1:1, :size, 2) + @test_deprecated Base.annotate!(Base.AnnotatedChar{Char, Any}('a'), :size, 2) + labelled = """ + struct Weight; w::Int end + Base.labelled_kind(::Val{:weight}) = Weight + size, w = Pair{Symbol, Any}(:size, 2), Weight(3) + s = Base.AnnotatedString("abc", [(1:1, :size, 2), (2:2, :weight, w), (3:3, :weight, 4)]) + c = Base.AnnotatedChar('a', [(:size, 2), (:weight, w)]) + print(all((Base.annotations(s) == [(region = 1:1, value = size), (region = 2:2, value = w), (region = 3:3, value = :weight => 4)], + Base.annotations(Base.AnnotatedString("a", [(region = 1:1, label = :weight, value = w)])) == [(region = 1:1, value = w)], + Base.annotations(c) == [size, w], + Base.annotations(Base.AnnotatedChar('a', [(label = :weight, value = w)])) == [w], + Base.annotate!(Base.AnnotatedString{String, Any}("ab"), 1:1, :size, 2) == Base.AnnotatedString("ab", [(1:1, size)]), + Base.annotate!(Base.AnnotatedChar{Char, Any}('a'), :weight, w) == Base.AnnotatedChar('a', [w])))) + """ + @test read(`$(Base.julia_cmd()) --startup-file=no --depwarn=no -e $labelled`, String) == "true" +end + @testset "Construction" begin # Strings and chars follow the same rules; char annotations just lack a region for (Annotated, plain, annotof) in ((Base.AnnotatedString, "ab", v -> (1:1, v)), From 24b3e7f33275df93981f5df8e066a5894da33463 Mon Sep 17 00:00:00 2001 From: TEC Date: Fri, 16 Oct 2026 20:47:53 +0800 Subject: [PATCH 16/23] Display any value type through its concrete types, compiled once A value type with a member that isn't concrete now displays from the types of its values, as `Any` does, so a string displays the same whatever value type holds it. A reduction is called for each concrete type, and the manual and its docstring say that it selects its values with `value isa T`, so a reduction written for several types sees each type's values apart. Strings of `Any` values display through code compiled once. Their types are found without compiling a set for each. They share a known union's methods for plain text and HTML, where each type of state is checked on its own. They call the interface through barriers, so that a new value type's methods invalidate none of it. In HTML, a state with no fields now closes its element when it changes, as a state with fields does. Assisted-by: Claude Code (Opus 5.5) --- base/strings/annotated_io.jl | 91 +++++++++++++++++++----------------- test/strings/annotated.jl | 56 +++++++++++++++------- 2 files changed, 86 insertions(+), 61 deletions(-) diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl index 9f084cc7b6e13..ae661c7fbece3 100644 --- a/base/strings/annotated_io.jl +++ b/base/strings/annotated_io.jl @@ -355,7 +355,7 @@ module AnnotatedDisplay using ..Base: IO, IOBuffer, SubString, AnnotatedString, AnnotatedChar, AnnotatedIOBuffer using ..Base: eachregion, ActiveAnnotations, unannotate, annotations, annotatedstring, uniontypes -using ..Base: invoke_in_world, tls_world_age, escape_string, annotate!, _clear_annotations_in_region!, RegionAnnotation +using ..Base: invoke_in_world, tls_world_age, inferencebarrier, escape_string, annotate!, _clear_annotations_in_region!, RegionAnnotation public context, reduction, transition @@ -372,13 +372,14 @@ passed on to each `reduction`. context(::MIME, ::Type) = nothing """ - reduction(mime::MIME, T::Type[, ctx], annotations) -> state + reduction(mime::MIME, T::Type[, ctx], values) -> state -The state of a region with `annotations`, from those whose values have type `T`. With no -annotations, this is the baseline, where display starts and ends. A type without a method is -not displayed, and one without a `context` can leave out `ctx`. +The state of a region, from those of its annotation `values` of type `T`, the concrete type it +is called for, which it selects with `value isa T`. With none, this is the baseline, where +display starts and ends. A type without a method is not displayed, and one without a `context` +can leave out `ctx`. """ -reduction(mime::MIME, ::Type{T}, _, annotations) where {T} = reduction(mime, T, annotations) +reduction(mime::MIME, ::Type{T}, _, values) where {T} = reduction(mime, T, values) reduction(::MIME, ::Type, _) = nothing """ @@ -394,7 +395,9 @@ transition(io::IO, ::MIME, ::Nothing, ::Nothing) = nothing ## Rendering +# A value type with a member that isn't concrete, as `Any`, has its types found from its values @generated function contexts(mime::MIME, ::Type{U}) where {U} + all(isconcretetype, uniontypes(U)) || return nothing calls = (:(context(mime, $T)) for T in uniontypes(U)) :(tuple($(calls...))) end @@ -410,20 +413,19 @@ Write `x` to `io` as `mime`, with its text written by `writetext(io, str)`. """ function render(writetext::F, mime::MIME, io::IO, s::AnnotatedStr{S, V}) where {F, S, V} ctxs = contexts(mime, V) + isnothing(ctxs) && return invoke_in_world(tls_world_age(), renderheld, writetext, mime, io, s) render(writetext, mime, io, s, annots -> (@inline; reductions(mime, V, ctxs, annots))) end -# Compiled once, dispatching at runtime on whatever types `s` holds, in the latest world so -# that new display methods do not invalidate compiled callers. Taken in a union's order, the -# types display as that union would. -render(writetext::F, mime::MIME, io::IO, s::AnnotatedStr{S, Any}) where {F, S} = - invoke_in_world(tls_world_age(), renderheld, writetext, mime, io, s) - +# Compiled once for whatever types `s` holds, in the latest world. Its interface calls go through +# barriers, here and in `render`, so a new value type's methods invalidate nothing compiled for it. +# Taken in a union's order, the types display as that union would. function renderheld(writetext, mime::MIME, io::IO, @nospecialize(s::AnnotatedStr)) - types = uniontypes(Union{unique(typeof(a.value) for a in annotations(s))...}) - ctxs = Any[context(mime, T) for T in types] - render(writetext, mime, io, s, - annots -> Any[reduction(mime, T, ctx, annots) for (T, ctx) in zip(types, ctxs)]) + # As a `Union`, which is built without compiling anything for the types it holds + types = uniontypes(Union{Type[typeof(value) for (; value) in annotations(s)]...}) + ctxs = Any[inferencebarrier(context)(mime, T) for T in types] + render(writetext, mime, io, s, annots -> + Any[inferencebarrier(reduction)(mime, T, ctx, annots) for (T, ctx) in zip(types, ctxs)]) end render(writetext::F, mime::MIME, io::IO, c::AnnotatedChar) where {F} = @@ -432,19 +434,22 @@ render(writetext::F, mime::MIME, io::IO, c::AnnotatedChar) where {F} = function render(writetext, mime::MIME, io::IO, s::AnnotatedStr, statesof) regions = eachregion(s) bases = statesof(ActiveAnnotations(regions, regions.pos)) # Before its first region, the iterator's annotations are none - written = change(io, mime, nothing, bases) + # Through a barrier for types found from the values, as in `renderheld`. Applied in the call, as + # a closure capturing `inferencebarrier(transition)` is specialised on what it captures. + transit = if bases isa Tuple transition else (args...) -> inferencebarrier(transition)(args...) end + written = change(transit, io, mime, nothing, bases) for (str, annots) in regions - written = change(io, mime, written, statesof(annots)) + written = change(transit, io, mime, written, statesof(annots)) writetext(io, str) end - change(io, mime, written, bases) + change(transit, io, mime, written, bases) end ## Plain text: each type's state changes on its own -change(io::IO, ::MIME"text/plain", ::Nothing, bases) = bases -function change(io::IO, mime::MIME"text/plain", previous, states) - foreach((from, to) -> transition(io, mime, from, to), previous, states) +change(transit, io::IO, ::MIME"text/plain", ::Nothing, bases) = bases +function change(transit, io::IO, mime::MIME"text/plain", previous, states) + foreach((from, to) -> transit(io, mime, from, to), previous, states) states end @@ -452,20 +457,18 @@ end # # Each frame holds the states within an open element, from the baselines in, and differs from # the frame outside it only in the state of the element's type. An element still fits when each -# property of each state is the outer frame's or the new state's. - -change(io::IO, ::MIME"text/html", ::Nothing, bases) = [bases] - -function change(io::IO, mime::MIME"text/html", frames, states) - # One method, as a local function with two is boxed where the closure below captures it - replaced(t, k, v) = - if t isa Tuple Base.setindex(t, v, k) else setindex!(copy(t), v, k) end - fits(el, parent, to) = - all(ntuple(Val(fieldcount(typeof(el)))) do i - property = getfield(el, i) - property === getfield(parent, i) || - property === getfield(to, i) - end) +# property, or field, of each state is the outer frame's or the new state's. A state without +# fields fits only as a whole. + +change(transit, io::IO, ::MIME"text/html", ::Nothing, bases) = [bases] + +# For a known union, `frames` holds tuples of states. For types found from the values, it holds +# vectors, and `fits` is dispatched for each type of state, so nothing compiles per combination. +function change(transit, io::IO, mime::MIME"text/html", frames, states) + function fits(el::T, parent::T, to::T) where {T} + fieldcount(T) == 0 && return el === parent || el === to + all(ntuple(i -> getfield(el, i) === getfield(parent, i) || getfield(el, i) === getfield(to, i), Val(fieldcount(T)))) + end keep = 1 while keep < length(frames) && all(map(fits, frames[keep + 1], frames[keep], states)) keep += 1 @@ -473,17 +476,19 @@ function change(io::IO, mime::MIME"text/html", frames, states) while length(frames) > keep inner = pop!(frames) foreach(inner, last(frames)) do within, outside - within === outside || transition(io, mime, within, nothing) + within === outside || transit(io, mime, within, nothing) end end - # An unrolled `map`, as a loop or a closure out of line would box each state. In reverse, so - # a later type's element (a link) encloses an earlier type's (a face). - map(reverse(ntuple(identity, length(states))), reverse(states)) do i, to + # An unrolled `map` for a tuple, as a loop would box its states. In reverse, so a later type's + # element (a link) encloses an earlier type's (a face). + indices = if states isa Tuple ntuple(identity, length(states)) else eachindex(states) end + map(reverse(indices), reverse(states)) do i, to @inline - within = last(frames)[i] + frame = last(frames) + within = frame[i] if within !== to - transition(io, mime, within, to) - push!(frames, replaced(last(frames), i, to)) + transit(io, mime, within, to) + push!(frames, if frame isa Tuple Base.setindex(frame, to, i) else setindex!(copy(frame), to, i) end) end nothing end diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index d2fcef7b97761..7c9431aa19c72 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -1035,21 +1035,34 @@ module MarkedAnnotations struct Tagged name::String end - marked(annotations) = Marked(any(value -> value isa Mark, annotations)) - reduction(::MIME"text/plain", ::Type{Mark}, annotations) = marked(annotations) - reduction(::MIME"text/html", ::Type{Mark}, annotations) = marked(annotations) + reduction(::MIME"text/plain", ::Type{Mark}, values) = Marked(any(value -> value isa Mark, values)) + reduction(::MIME"text/html", ::Type{Mark}, values) = Marked(any(value -> value isa Mark, values)) transition(io::IO, ::MIME"text/plain", from::Marked, to::Marked) = from === to || print(io, if to.on "<" else ">" end) transition(io::IO, ::MIME"text/html", ::Marked, ::Marked) = print(io, "") transition(io::IO, ::MIME"text/html", ::Marked, ::Nothing) = print(io, "") - function reduction(::MIME"text/plain", ::Type{Tag}, annotations) - i = findlast(value -> value isa Tag, annotations) - Tagged(if isnothing(i) "" else annotations[i].name end) - end + reduction(::MIME"text/plain", ::Type{Tag}, values) = + Tagged(foldl((name, value) -> if value isa Tag value.name else name end, values; init = "")) function transition(io::IO, ::MIME"text/plain", from::Tagged, to::Tagged) from === to && return isempty(from.name) || print(io, "]") isempty(to.name) || print(io, "[", to.name, " ") end + # A state with no fields: whether a region is flagged + struct Flag end + reduction(::MIME"text/html", ::Type{Flag}, values) = any(value -> value isa Flag, values) + transition(io::IO, ::MIME"text/html", ::Bool, flagged::Bool) = flagged && print(io, "") + transition(io::IO, ::MIME"text/html", ::Bool, ::Nothing) = print(io, "") + # Displayed by each concrete type, whatever the value type that holds them + abstract type Shade end + struct Dark <: Shade end + reduction(::MIME"text/plain", ::Type{Dark}, values) = Marked(any(value -> value isa Dark, values)) + struct Level{N} end # Each of its types is a level of brackets, counted by its own reduction + struct Depth + n::Int + end + reduction(::MIME"text/plain", ::Type{T}, values) where {T <: Level} = Depth(count(value -> value isa T, values)) + transition(io::IO, ::MIME"text/plain", from::Depth, to::Depth) = + print(io, "["^max(0, to.n - from.n), "]"^max(0, from.n - to.n)) end @testset "AnnotatedDisplay" begin @@ -1066,6 +1079,12 @@ end @test colour(tagged(Union{Mark, Tag, Int})) == "c]" @test colour(tagged(Union{Mark, Tag, Int, Symbol})) == colour(tagged(Any)) == colour(tagged(Union{Mark, Tag, Int})) @test colour(Base.AnnotatedString{String, Any}("x")) == "x" + # Each type of value displays its own values, so a value type with abstract members displays as `Any` does + (; Dark, Level, Flag) = MarkedAnnotations + shaded(V) = colour(Base.AnnotatedString{String, V}("xy", [(1:1, Dark())])) + @test shaded(Dark) == shaded(MarkedAnnotations.Shade) == shaded(Any) == "y" + levelled(V) = colour(Base.AnnotatedString{String, V}("ab", [(1:1, Level{1}()), (1:1, Level{2}())])) + @test levelled(Union{Level{1}, Level{2}}) == levelled(Level) == levelled(Any) == "[[a]]b" # Substrings and chars, as the strings made from them @test colour(SubString(tagged(Any), 2, 3)) == colour(Base.AnnotatedString(SubString(tagged(Any), 2, 3))) @test colour(marked(Mark)[1]) == "" @@ -1074,12 +1093,9 @@ end @test html(SubString(marked(Any), 1, 1)) == html(marked(Mark)[1]) == "x" plain = Base.AnnotatedString{String, Union{Tag, Int}}("abcde" # Escaping keeps the display around the (longer) text, and an annotated buffer the annotations @test sprint(escape_string, marked(Mark); context = :color => true) == "y" @test sprint(escape_string, Base.AnnotatedString{String, Mark}("a\nb", [(1:1, Mark()), (3:3, Mark())]); @@ -1096,13 +1112,14 @@ end none = Memory{Union{Mark, Int}}(undef, 0) states = @inferred Base.AnnotatedDisplay.reductions(MIME("text/plain"), Union{Mark, Int}, (nothing, nothing), none) @test Set(states) == Set([Marked(false), nothing]) - # An `Any` string is displayed by code compiled once, whatever types it holds - specialisations() = sum(m -> length(Base.specializations(m)), methods(Base.AnnotatedDisplay.render)) + # An `Any` string is displayed by code compiled once, whatever types it holds, and however many + D = Base.AnnotatedDisplay + specialisations() = sum(f -> sum(m -> length(Base.specializations(m)), methods(f)), (D.render, D.renderheld, D.change)) anyof(values...) = Base.AnnotatedString{String, Any}("abc", [(i:i, v) for (i, v) in enumerate(values)]) - colour(anyof(Mark())) + colour(anyof(Mark())); html(anyof(Mark())) before = specialisations() - for values in ((Mark(), Tag("t")), (Tag("t"), 1), (1.0, Mark(), :s)) - colour(anyof(values...)) + for values in ((Mark(), Tag("t")), (Tag("t"), 1), (1.0, Mark(), :s), (Flag(), Mark(), Flag())) + colour(anyof(values...)); html(anyof(values...)) end @test specialisations() == before # Defining how a type displays must not invalidate code compiled for strings of unknown value @@ -1122,6 +1139,9 @@ end strings = Base.AnnotatedString{SubString{String}}[Base.AnnotatedString{SubString{String}, Int}(SubString("x"))] unknown_valtype(strings) unknown_html(strings) + anything = Base.AnnotatedString{String, Any}("x", [(1:1, 1)]) # And through the code compiled once for any values + print(IOContext(Sink(), :color => true), anything) + show(Sink(), MIME("text/html"), anything) log = ccall(:jl_debug_method_invalidation, Any, (Cint,), 1) context(::MIME, ::Type{Late}) = 0 reduction(::MIME, ::Type{Late}, _, _) = LateState() From 997a8008407e31402bde37c634c58ff3fbc120ae Mon Sep 17 00:00:00 2001 From: TEC Date: Fri, 16 Oct 2026 20:48:03 +0800 Subject: [PATCH 17/23] Read labelled annotations wherever they are given The labelled forms of annotations were only read in vectors of them alone, and their values only by the kind of their label, so old code lost its links and some of its annotations without a word: - A package now says what its labels' values stand for, as well as their kinds. StyledStrings gives `:link` strings their `Link`s. - A string reads a labelled annotation where it reads each annotation, so it is read in a vector of any values too. - Every labelled form but deletion reads its values through `labelled_value`, which gives the one warning. Base builds its own chars with their value types, so a string whose values only look labelled keeps them as they are. Assisted-by: Claude Code (Opus 5.5) --- base/deprecated.jl | 21 +++++++++------------ base/strings/annotated.jl | 13 ++++++++++--- base/strings/unicode.jl | 6 +++--- test/strings/annotated.jl | 21 +++++++++++++++++++-- 4 files changed, 41 insertions(+), 20 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index cbb4635902dfb..2019121400506 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -753,21 +753,22 @@ end # the key of a `Pair{Symbol, Any}`. StyledStrings gives `:face` and `:link` their types. labelled_kind(::Val{label}) where {label} = label -# The value a labelled annotation stands for: its own when it is of the label's kind, and otherwise `label => value` +# The value a labelled annotation stands for, which a package can give for its own labels: by +# default its value when it is of the label's kind, and otherwise `label => value`. Every labelled +# form but deletion reads its values here. function labelled_value(label::Symbol, value) + depwarn("annotations with a label are deprecated; give the value alone, whose type says what it is", :labelled_value) + labelled_value(Val(label), value) +end +function labelled_value(::Val{label}, value) where {label} kind = labelled_kind(Val(label)) if kind isa Type && value isa kind value else Pair{Symbol, Any}(label, value) end end -function AnnotatedString{S}(s::S, annots::Vector{<:Union{Tuple{Any, Symbol, Any}, NamedTuple{(:region, :label, :value)}}}) where {S <: AbstractString} - isempty(annots) && return AnnotatedString{S, Any}(s) # Such as a `Vector{Union{}}`, which dispatch also puts here - depwarn("annotations with a label are deprecated; give `(region, value)`, where the type of the value says what it is", :AnnotatedString) - AnnotatedString{S}(s, [(region, labelled_value(Symbol(label), value)) for (region, label, value) in annots]) -end +# A string reads labelled annotations as it reads each annotation, in `AnnotatedString{S}` function AnnotatedChar(c::AbstractChar, annots::Vector{<:Union{Tuple{Symbol, Any}, NamedTuple{(:label, :value)}}}) isempty(annots) && return AnnotatedChar(c, Any[]) - depwarn("annotations with a label are deprecated; give the value, whose type says what it is", :AnnotatedChar) AnnotatedChar(c, [labelled_value(label, value) for (label, value) in annots]) end AnnotatedChar(c::AnnotatedChar, annots::Vector{<:Union{Tuple{Symbol, Any}, NamedTuple{(:label, :value)}}}) = @@ -781,7 +782,6 @@ function _annotate!(annlist::Vector{<:RegionAnnotation}, region::UnitRange{Int}, if kind isa Type annot.value isa kind else annot.value isa Pair{Symbol} && first(annot.value) === kind end end) else - depwarn("`annotate!` with a label is deprecated; annotate with the value, whose type says what it is", :annotate!) _annotate!(annlist, region, labelled_value(label, value)) end end @@ -809,9 +809,6 @@ annotate!(s::SubString{<:AnnotatedString}, label::Symbol, val) = annotate!(io::AnnotatedIOBuffer, range::UnitRange{Int}, label::Symbol, @nospecialize(val::Any)) = (_annotate!(io.annotations, range, label, val); io) -function annotate!(c::AnnotatedChar, label::Symbol, value) - depwarn("`annotate!` with a label is deprecated; annotate with the value, whose type says what it is", :annotate!) - annotate!(c, labelled_value(label, value)) -end +annotate!(c::AnnotatedChar, label::Symbol, value) = annotate!(c, labelled_value(label, value)) # END 1.14 deprecations diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index af4ce7f706208..ebcf3162d09cc 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -108,6 +108,10 @@ function AnnotatedString{S}(s::S, annots) where {S <: AbstractString} valtype(::Vector{RegionAnnotation{V}}) where {V} = V valtype(::Vector) = Any function annotation(annot) + if annot isa Union{Tuple{Any, Symbol, Any}, NamedTuple{(:region, :label, :value)}} # Deprecated + region, label, value = annot + return (region = UnitRange{Int}(region), value = labelled_value(Symbol(label), value)) + end length(annot) == 2 || throw(ArgumentError(LazyString("an annotation is a region and a value, as `(region, value)`, not ", annot))) region, value = annot (region = UnitRange{Int}(region), value) @@ -145,8 +149,10 @@ AnnotatedString(s::AnnotatedString{S, V}, annots::Vector{RegionAnnotation{T}}) w AnnotatedString(s::AnnotatedString, annots) = AnnotatedString(s, AnnotatedString(s.string, annots).annotations) -AnnotatedChar(c::AnnotatedChar, annots::Vector) = - AnnotatedChar(c.char, typed_vcat(annot_promote_valtype(c, AnnotatedChar(c.char, annots)), c.annotations, annots)) +function AnnotatedChar(c::AnnotatedChar{C}, annots::Vector) where {C <: AbstractChar} + V = annot_promote_valtype(c, AnnotatedChar(c.char, annots)) + AnnotatedChar{C, V}(c.char, typed_vcat(V, c.annotations, annots)) +end AnnotatedChar(c::AnnotatedChar, annots) = AnnotatedChar(c, AnnotatedChar(c.char, annots).annotations) @@ -226,7 +232,8 @@ end function getindex(s::AnnotatedString{S, V}, i::Integer) where {S, V} @boundscheck checkbounds(s, i) @inbounds if isvalid(s, i) - AnnotatedChar(s.string[i], V[a.value for a in s.annotations if i in a.region]) + c = s.string[i] + AnnotatedChar{typeof(c), V}(c, V[a.value for a in s.annotations if i in a.region]) else string_index_err(s, i) end diff --git a/base/strings/unicode.jl b/base/strings/unicode.jl index c04f3b1bf112b..43ae9395d10bd 100644 --- a/base/strings/unicode.jl +++ b/base/strings/unicode.jl @@ -316,7 +316,7 @@ julia> lowercase('Ö') lowercase(c::T) where {T<:AbstractChar} = isascii(c) ? ('A' <= c <= 'Z' ? c + 0x20 : c) : T(@assume_effects :foldable :nothrow @ccall utf8proc_tolower(c::UInt32)::UInt32) -lowercase(c::AnnotatedChar) = AnnotatedChar(lowercase(c.char), annotations(c)) +lowercase(c::AnnotatedChar{C, V}) where {C, V} = AnnotatedChar{C, V}(lowercase(c.char), copy(annotations(c))) """ uppercase(c::AbstractChar) @@ -337,7 +337,7 @@ julia> uppercase('ê') uppercase(c::T) where {T<:AbstractChar} = isascii(c) ? ('a' <= c <= 'z' ? c - 0x20 : c) : T(@assume_effects :foldable :nothrow @ccall utf8proc_toupper(c::UInt32)::UInt32) -uppercase(c::AnnotatedChar) = AnnotatedChar(uppercase(c.char), annotations(c)) +uppercase(c::AnnotatedChar{C, V}) where {C, V} = AnnotatedChar{C, V}(uppercase(c.char), copy(annotations(c))) """ titlecase(c::AbstractChar) @@ -362,7 +362,7 @@ julia> uppercase('dž') titlecase(c::T) where {T<:AbstractChar} = isascii(c) ? ('a' <= c <= 'z' ? c - 0x20 : c) : T(@assume_effects :foldable :nothrow @ccall utf8proc_totitle(c::UInt32)::UInt32) -titlecase(c::AnnotatedChar) = AnnotatedChar(titlecase(c.char), annotations(c)) +titlecase(c::AnnotatedChar{C, V}) where {C, V} = AnnotatedChar{C, V}(titlecase(c.char), copy(annotations(c))) ############################################################################ diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index 7c9431aa19c72..a3bbdf298920b 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -125,6 +125,12 @@ end @test uppercase(chr) == Base.AnnotatedChar('C') @test titlecase(chr) == Base.AnnotatedChar('C') @test lowercase(Base.AnnotatedChar('C')) == chr + # A char in another case has its own annotations + for transform in (lowercase, uppercase, titlecase) + original = Base.AnnotatedChar('a', [1]) + Base.annotate!(transform(original), 2) + @test Base.annotations(original) == [1] + end str = Base.AnnotatedString("hmm", [(1:1, "h0h0"), (1:2, "h0m1"), (2:3, "m1m2")]) @@ -166,9 +172,18 @@ Base.labelled_kind(::Val{:weight}) = Weight @test_deprecated Base.AnnotatedChar('a', [(:size, 2)]) @test_deprecated Base.annotate!(Base.AnnotatedString{String, Any}("ab"), 1:1, :size, 2) @test_deprecated Base.annotate!(Base.AnnotatedChar{Char, Any}('a'), :size, 2) + @test_deprecated Base.AnnotatedString("ab", Any[(1:1, :size, 2), (2:2, 3)]) # In a vector of any values + # A string's values that only look labelled are kept as they are, by its chars too + let s = Base.AnnotatedString("ab", [(1:1, (:a, 1))]) + @test s[1] == Base.AnnotatedChar{Char, Tuple{Symbol, Int}}('a', [(:a, 1)]) + @test collect(s) isa Vector{Base.AnnotatedChar{Char, Tuple{Symbol, Int}}} + @test Base.annotations(uppercase(s[1])) == [(:a, 1)] + end + @test Base.annotations(Base.AnnotatedChar('a', Any[(:weight, Weight(1))])) == [(:weight, Weight(1))] # Among any values, a value labelled = """ struct Weight; w::Int end Base.labelled_kind(::Val{:weight}) = Weight + Base.labelled_value(::Val{:heavy}, n::Int) = Weight(n) # A package's own upgrade of an old form size, w = Pair{Symbol, Any}(:size, 2), Weight(3) s = Base.AnnotatedString("abc", [(1:1, :size, 2), (2:2, :weight, w), (3:3, :weight, 4)]) c = Base.AnnotatedChar('a', [(:size, 2), (:weight, w)]) @@ -177,7 +192,9 @@ Base.labelled_kind(::Val{:weight}) = Weight Base.annotations(c) == [size, w], Base.annotations(Base.AnnotatedChar('a', [(label = :weight, value = w)])) == [w], Base.annotate!(Base.AnnotatedString{String, Any}("ab"), 1:1, :size, 2) == Base.AnnotatedString("ab", [(1:1, size)]), - Base.annotate!(Base.AnnotatedChar{Char, Any}('a'), :weight, w) == Base.AnnotatedChar('a', [w])))) + Base.annotate!(Base.AnnotatedChar{Char, Any}('a'), :weight, w) == Base.AnnotatedChar('a', [w]), + Base.annotations(Base.AnnotatedString("a", [(1:1, :heavy, 5)])) == [(region = 1:1, value = Weight(5))], + Base.annotations(Base.AnnotatedString("ab", Any[(1:1, :size, 2), (2:2, 3)])) == [(region = 1:1, value = size), (region = 2:2, value = 3)]))) """ @test read(`$(Base.julia_cmd()) --startup-file=no --depwarn=no -e $labelled`, String) == "true" end @@ -215,7 +232,7 @@ end @test Base.AnnotatedString("ab") == Base.AnnotatedString("ab", Base.RegionAnnotation[]) @test Base.AnnotatedChar(UInt32('c')) == Base.AnnotatedChar('c') == Base.AnnotatedChar('c', Any[]) @test Base.AnnotatedString("ab", [1:1 => 1, (region = 2:2, value = 2)]) == Base.AnnotatedString("ab", [(1:1, 1), (2:2, 2)]) - @test_throws ArgumentError Base.AnnotatedString("ab", Any[(1:1, :label, 1)]) # Not read as a region and a value + @test_throws ArgumentError Base.AnnotatedString("ab", Any[(1:1, 2, 3)]) # Not read as a region and a value @test Base.AnnotatedString([Base.AnnotatedChar('a', [1]), Base.AnnotatedChar('b', [2])]) == Base.AnnotatedString("ab", [(1:1, 1), (2:2, 2)]) @test (@inferred Base.AnnotatedString(Base.AnnotatedString("x", [(1:1, 1)]), [(1:1, 2)])) isa Base.AnnotatedString{String, Int} From c8138c8af7ae93db5fe92f27c57741946bd62af9 Mon Sep 17 00:00:00 2001 From: TEC Date: Fri, 16 Oct 2026 20:48:03 +0800 Subject: [PATCH 18/23] Document how annotations display The manual gains a section on `Base.AnnotatedDisplay`: what `context`, `reduction` and `transition` do, the rules an implementation follows, and a worked example, run as a doctest. Assisted-by: Claude Code (Opus 5.5) --- doc/src/base/strings.md | 66 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/doc/src/base/strings.md b/doc/src/base/strings.md index 760d05e44abc6..4ec9f0c3e97dc 100644 --- a/doc/src/base/strings.md +++ b/doc/src/base/strings.md @@ -240,3 +240,69 @@ Base.annotations Base.annotate! Base.unannotate ``` + +### [Displaying annotations](@id base-annotated-display) + +How an annotated string looks when it is printed to a terminal or shown as HTML is up to the +types of its annotation values. A package says how the values of its own types display with +three functions of [`Base.AnnotatedDisplay`](@ref): + +- [`context`](@ref Base.AnnotatedDisplay.context) reads what displaying them needs, once per display; +- [`reduction`](@ref Base.AnnotatedDisplay.reduction) gives the state of a region from its values of the type; +- [`transition`](@ref Base.AnnotatedDisplay.transition) writes the change from one state to another. + +These rules hold: + +- A value's concrete type says how it displays, so a string displays the same whatever value + type holds it, `Any` included. +- `reduction` is called for each concrete type `T` of the values, with all of a region's + values, and reads those with `value isa T`. With none, it gives the baseline state, where + display starts and ends. +- Each value type has one type of state, compared with `===`. For HTML, a state's fields are + its visible properties. +- In plain text, `transition` writes the change between two states. In HTML, each transition + opens one element within the current one, and a transition to `nothing` closes it. +- The types display in the order of the value type's union, and in HTML a later type's + elements enclose an earlier type's. + +For example, a `Tag` that displays in brackets in a terminal, and as a `` in HTML: + +```jldoctest annotated-display +julia> struct Tag + name::String + end + +julia> struct Tagged # The state: the innermost tag of a region, if it has one + name::String + end + +julia> Base.AnnotatedDisplay.reduction(::MIME, ::Type{Tag}, values) = + Tagged(foldl((name, value) -> if value isa Tag value.name else name end, values; init = "")) + +julia> function Base.AnnotatedDisplay.transition(io::IO, ::MIME"text/plain", from::Tagged, to::Tagged) + from === to && return + isempty(from.name) || print(io, ']') + isempty(to.name) || print(io, '[', to.name, ' ') + end + +julia> Base.AnnotatedDisplay.transition(io::IO, ::MIME"text/html", ::Tagged, to::Tagged) = + print(io, "") + +julia> Base.AnnotatedDisplay.transition(io::IO, ::MIME"text/html", ::Tagged, ::Nothing) = + print(io, "") + +julia> s = Base.AnnotatedString("an example", [(4:10, Tag("noun"))]); + +julia> print(IOContext(stdout, :color => true), s) +an [noun example] + +julia> show(stdout, MIME("text/html"), s) +an example +``` + +```@docs +Base.AnnotatedDisplay +Base.AnnotatedDisplay.context +Base.AnnotatedDisplay.reduction +Base.AnnotatedDisplay.transition +``` From 575ec1022705ff4cbaddfbe2f823198d2b0d9647 Mon Sep 17 00:00:00 2001 From: TEC Date: Fri, 16 Oct 2026 23:05:31 +0800 Subject: [PATCH 19/23] Specialise annotated replace on its pairs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `replace` into an `AnnotatedIOBuffer` takes its pairs as a vararg and only passes them on. So the body behind its `count` keyword was not specialised on them, and it boxed every value of the replacement loop. Taking them as `Vararg{Pair, N}` makes it specialise, as `_replace_` does for plain strings. Replacing two characters in a short styled string drops from 5.7 µs to 0.3 µs, and `replace` on an annotated string from 6.5 µs to 0.9 µs. Annotations were also built with the type of their own value, then converted to the buffer's value type as they were pushed. That allocated whenever the value type was a union. They are now built with the buffer's value type, which makes inserting them nine times faster. This removes two REVIEW comments that asked why these were slow. Assisted-by: Claude Code (Opus 5.5) --- base/strings/annotated_io.jl | 45 +++++------------------------------- 1 file changed, 6 insertions(+), 39 deletions(-) diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl index ae661c7fbece3..553c5451d0a21 100644 --- a/base/strings/annotated_io.jl +++ b/base/strings/annotated_io.jl @@ -188,38 +188,12 @@ function _insert_annotations!(annots::Vector{RegionAnnotation{V}}, newannots::Ve old_index = lastindex(annots) - run + 1 + runindex old = annots[old_index] new = newannots[begin+runindex] - extannot = (region = first(old.region):last(new.region)+offset, - value = old.value) - annots[old_index] = extannot + annots[old_index] = RegionAnnotation{V}((first(old.region):last(new.region)+offset, old.value)) end for index in run+1:lastindex(newannots) annot = newannots[index] start, stop = first(annot.region), last(annot.region) - # REVIEW: For some reason, construction of `newannot` - # can be a significant contributor to the overall runtime - # of this function. For instance, executing: - # - # replace(AnnotatedIOBuffer(), S"apple", - # 'e' => S"{red:x}", 'p' => S"{green:y}") - # - # results in 3 calls to `_insert_annotations!`. It takes - # ~570ns in total, compared to ~200ns if we push `annot` - # instead of `newannot`. Commenting out the `_insert_annotations!` - # line reduces the runtime to ~170ns, from which we can infer - # that constructing `newannot` is somehow responsible for - # a ~30ns -> ~400ns (~13x) increase in runtime!! - # This also comes with a marginal increase in allocations - # (compared to the commented out version) of 2 -> 14 (250b -> 720b). - # - # This seems quite strange, but I haven't dug into the generated - # LLVM or ASM code. If anybody reading this is interested in checking - # this out, that would be brilliant 🙏. - # - # What I have done is found that "direct tuple reconstruction" - # (as below) is several times faster than using `setindex`. - newannot = (region = start+offset:stop+offset, - value = annot.value) - push!(annots, newannot) + push!(annots, RegionAnnotation{V}((start+offset:stop+offset, annot.value))) end end @@ -228,11 +202,8 @@ _insert_annotations!(io::AnnotatedIOBuffer, newannots::Vector{<:RegionAnnotation # String replacement -# REVIEW: For some reason the `Core.kwcall` indirection seems to cause a -# substantial slowdown here. If we remove `; count` from the signature -# and run the sample code above in `_insert_annotations!`, the runtime -# drops from ~4400ns to ~580ns (~7x faster). I cannot guess why this is. -function replace(out::AnnotatedIOBuffer{V}, str::AnnotatedString, pat_f::Pair...; count = typemax(Int)) where {V} +# `Vararg{Pair, N}`, as the keyword body would otherwise not specialise on the pairs it passes on +function replace(out::AnnotatedIOBuffer{V}, str::AnnotatedString, pat_f::Vararg{Pair, N}; count = typemax(Int)) where {V, N} if count == 0 || isempty(pat_f) write(out, str) return out @@ -299,18 +270,14 @@ function replace(out::AnnotatedIOBuffer{V}, str::AnnotatedString, pat_f::Pair... continue elseif postidx - prioridx <= 1 && start > last(priorrep.region) && stop < first(postrep.region) # Lies between replacements - shiftregion = (start + priorrep.offset):(stop + priorrep.offset) - shiftann = (region = shiftregion, value) - push!(out.annotations, shiftann) + push!(out.annotations, RegionAnnotation{V}(((start + priorrep.offset):(stop + priorrep.offset), value))) else # Split between replacements prevrep = replacements[max(begin, prioridx - 1)] for rep in @view replacements[max(begin, prioridx - 1):min(end, postidx + 1)] gap = max(start, last(prevrep.region)+1):min(stop, first(rep.region)-1) if !isempty(gap) - shiftregion = (first(gap) + prevrep.offset):(last(gap) + prevrep.offset) - shiftann = (; region = shiftregion, value) - push!(out.annotations, shiftann) + push!(out.annotations, RegionAnnotation{V}(((first(gap) + prevrep.offset):(last(gap) + prevrep.offset), value))) end prevrep = rep end From f5996de32d48d5a81f844f7c014e62ad27bbe08c Mon Sep 17 00:00:00 2001 From: TEC Date: Fri, 16 Oct 2026 23:35:59 +0800 Subject: [PATCH 20/23] Remove duplicated replace tests Two replace tests appeared twice, once each in the splitting and size tests. They are kept under count and replacing a pattern with itself. Assisted-by: Claude Code (Opus 5.5) --- test/strings/annotated.jl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index a3bbdf298920b..511a3674354f2 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -563,8 +563,6 @@ end astr("a_b_c", (1:1, :red), (3:3, :red), (5:5, :red)) @test replace(astr("foobarbaz", (1:9, :green)), "o" => "0", "a" => "A") == astr("f00bArbAz", (1:1, :green), (4:4, :green), (6:7, :green), (9:9, :green)) - @test replace(astr("a b c", (1:5, :red)), " " => "_", count=1) == - astr("a_b c", (1:1, :red), (3:5, :red)) @test replace(astr("abcde", (2:4, :red)), "c" => "X") == astr("abXde", (2:2, :red), (4:4, :red)) @test replace(astr("a a a a", (1:7, :blue)), "a" => "b") == @@ -824,8 +822,6 @@ end astr("hi", (1:2, :red)) @test replace(astr("aabbcc", (1:6, :red)), "a" => "x", "b" => "y", "c" => "z") == astr("xxyyzz") - @test replace(astr("hello", (1:5, :red)), "hello" => astr("hello", (1:5, :blue))) == - astr("hello", (1:5, :blue)) end @testset "Complex" begin From 7a166d2a213795f5dcd033062045232e30353936 Mon Sep 17 00:00:00 2001 From: TEC Date: Sat, 17 Oct 2026 01:03:02 +0800 Subject: [PATCH 21/23] Render only the annotations that display A string was rendered by code compiled for its value type, so each new union compiled the whole region walk and every reduction again, even when the new member does not display. `render` now finds the types that display from their baseline states, those that are not `nothing`, and walks only their annotations. A type that does not display adds just a small filter, and the rest of the rendering is the code already compiled for the types that do. When nothing displays, the text is written as it is. The region iterator can be limited to one type of value, which it reads in place when the string holds nothing else, and otherwise gathers as it makes its events. Annotations that do not display no longer split regions, so a string that mixes them with faces renders faster. A string of `Any` values finds its types from the values, and drops those that do not display in the same way. Every state of a type is of its baseline's type. One of another type, which broke `transition` or the HTML frames, now displays as the baseline. Where the state types are known this check compiles away. A new value type that only adds a non-displayed member now compiles in about 55 ms rather than 190 ms. A long styled string with an `Int` among its values prints 60% faster, and allocates 8 KB rather than 51 KB. Strings of `StyledValue` print as fast as before. Assisted-by: Claude Code (Opus 5.5) --- base/strings/annotated.jl | 28 +++++++++++++++------------- base/strings/annotated_io.jl | 35 +++++++++++++++++++++++++++-------- test/strings/annotated.jl | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index ebcf3162d09cc..6282e881f9a82 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -675,33 +675,35 @@ julia> map(collect, eachregion(a)) ["there", [:italic]] ``` """ -function eachregion(s::AnnotatedString{S, V}, subregion::UnitRange{Int}=firstindex(s):lastindex(s)) where {S, V} +function eachregion(s::AnnotatedString{S, V}, subregion::UnitRange{Int}=firstindex(s):lastindex(s), ::Type{T}=V) where {S, V, T} # Whole characters, so a region's annotations read the same during iteration and after it isempty(subregion) || (subregion = thisind(s, first(subregion)):nextind(s, thisind(s, last(subregion))) - 1) - RegionIterator{S, V}(s.string, s.annotations, annotation_events(s, subregion), - unsigned(top_set_bit(2 * length(s.annotations) + 1)), subregion) + # Only this reads the values as `V`, so the iteration is compiled for `T` + annots = if V <: T s.annotations else RegionAnnotation{T}[annot for annot in s.annotations if annot.value isa T] end + RegionIterator(s.string, annots, annotation_events(s.string, annots, subregion), + unsigned(top_set_bit(2 * length(annots) + 1)), subregion) end -eachregion(s::SubString{AnnotatedString{S, V}}, pos::UnitRange{Int}=firstindex(s):lastindex(s)) where {S, V} = - eachregion(s.string, first(pos)+s.offset:last(pos)+s.offset) +eachregion(s::SubString{AnnotatedString{S, V}}, pos::UnitRange{Int}=firstindex(s):lastindex(s), ::Type{T}=V) where {S, V, T} = + eachregion(s.string, first(pos)+s.offset:last(pos)+s.offset, T) """ - annotation_events(string::AnnotatedString, subregion::UnitRange{Int}) + annotation_events(string::AbstractString, annots::Vector{$RegionAnnotation}, subregion::UnitRange{Int}) -Find all annotation "change events" that occur within a `subregion` of `string`. +Find all annotation "change events" of `annots` that occur within a `subregion` of `string`. Each change event is a `UInt64` packing the position of the event above the index of the annotation in question, followed by a bit set when the annotation is being activated, so that sorting the events orders them by position. The index and bit take -the low `top_set_bit(2length(annotations(string)) + 1)` bits, as the region iterator reads them. +the low `top_set_bit(2length(annots) + 1)` bits, as the region iterator reads them. """ -function annotation_events(s::AnnotatedString, subregion::UnitRange{Int}) +function annotation_events(s::AbstractString, annots::Vector{<:RegionAnnotation}, subregion::UnitRange{Int}) events = UInt64[] - sizehint!(events, 2 * length(s.annotations)) - shift = unsigned(top_set_bit(2 * length(s.annotations) + 1)) + sizehint!(events, 2 * length(annots)) + shift = unsigned(top_set_bit(2 * length(annots) + 1)) leading_zeros(UInt64(ncodeunits(s) + 1)) >= shift || throw(ArgumentError(LazyString( - "too many annotations (", length(s.annotations), ") to order the regions of a ", ncodeunits(s), "-byte string"))) - for (i, (; region)) in enumerate(s.annotations) + "too many annotations (", length(annots), ") to order the regions of a ", ncodeunits(s), "-byte string"))) + for (i, (; region)) in enumerate(annots) start, stop = max(first(subregion), first(region)), min(last(subregion), last(region)) start <= stop || continue # Outside the subregion, or empty push!(events, UInt64(thisind(s, start)) << shift | UInt64(i) << 1 | true) diff --git a/base/strings/annotated_io.jl b/base/strings/annotated_io.jl index 553c5451d0a21..aa133f08e405e 100644 --- a/base/strings/annotated_io.jl +++ b/base/strings/annotated_io.jl @@ -321,7 +321,7 @@ the same whatever union of value types holds it. module AnnotatedDisplay using ..Base: IO, IOBuffer, SubString, AnnotatedString, AnnotatedChar, AnnotatedIOBuffer -using ..Base: eachregion, ActiveAnnotations, unannotate, annotations, annotatedstring, uniontypes +using ..Base: eachregion, RegionIterator, unannotate, annotations, annotatedstring, uniontypes using ..Base: invoke_in_world, tls_world_age, inferencebarrier, escape_string, annotate!, _clear_annotations_in_region!, RegionAnnotation public context, reduction, transition @@ -345,6 +345,10 @@ The state of a region, from those of its annotation `values` of type `T`, the co is called for, which it selects with `value isa T`. With none, this is the baseline, where display starts and ends. A type without a method is not displayed, and one without a `context` can leave out `ctx`. + +Every state of a type is of its baseline's type, so that its reduction is type-stable and display +compiles once for it. A type whose baseline is `nothing` is not displayed, and a state of another +type displays as the baseline. """ reduction(mime::MIME, ::Type{T}, _, values) where {T} = reduction(mime, T, values) reduction(::MIME, ::Type, _) = nothing @@ -358,7 +362,7 @@ one value type, as `transition` dispatches on the states. For `text/html`, a state's fields are its visible properties, a transition opens an element within the current one, and a transition to `nothing` closes the element its state opened. """ -transition(io::IO, ::MIME, ::Nothing, ::Nothing) = nothing +function transition end ## Rendering @@ -381,9 +385,17 @@ Write `x` to `io` as `mime`, with its text written by `writetext(io, str)`. function render(writetext::F, mime::MIME, io::IO, s::AnnotatedStr{S, V}) where {F, S, V} ctxs = contexts(mime, V) isnothing(ctxs) && return invoke_in_world(tls_world_age(), renderheld, writetext, mime, io, s) - render(writetext, mime, io, s, annots -> (@inline; reductions(mime, V, ctxs, annots))) + # Only the types that display, so one that doesn't adds no rendering compiled for itself + D = displayed(V, reductions(mime, V, ctxs, Memory{Union{}}(undef, 0))) + D === Union{} && return writetext(io, unannotate(s)) + dctxs = contexts(mime, D) + render(writetext, mime, io, eachregion(s, firstindex(s):lastindex(s), D), annots -> (@inline; reductions(mime, D, dctxs, annots))) end +# The members of `V` whose baseline `states` are not `nothing` +@generated displayed(::Type{V}, ::B) where {V, B <: Tuple} = + Union{(T for (T, state) in zip(uniontypes(V), fieldtypes(B)) if state !== Nothing)...} + # Compiled once for whatever types `s` holds, in the latest world. Its interface calls go through # barriers, here and in `render`, so a new value type's methods invalidate nothing compiled for it. # Taken in a union's order, the types display as that union would. @@ -391,22 +403,29 @@ function renderheld(writetext, mime::MIME, io::IO, @nospecialize(s::AnnotatedStr # As a `Union`, which is built without compiling anything for the types it holds types = uniontypes(Union{Type[typeof(value) for (; value) in annotations(s)]...}) ctxs = Any[inferencebarrier(context)(mime, T) for T in types] - render(writetext, mime, io, s, annots -> + # Only the types that display, as for a known union in `render` + shown = [!isnothing(inferencebarrier(reduction)(mime, T, ctx, Memory{Union{}}(undef, 0))) for (T, ctx) in zip(types, ctxs)] + keepat!(types, shown) # In place, as reassigning variables the closure below captures would box them + keepat!(ctxs, shown) + render(writetext, mime, io, eachregion(s), annots -> Any[inferencebarrier(reduction)(mime, T, ctx, annots) for (T, ctx) in zip(types, ctxs)]) end render(writetext::F, mime::MIME, io::IO, c::AnnotatedChar) where {F} = render(writetext, mime, io, annotatedstring(c)) -function render(writetext, mime::MIME, io::IO, s::AnnotatedStr, statesof) - regions = eachregion(s) - bases = statesof(ActiveAnnotations(regions, regions.pos)) # Before its first region, the iterator's annotations are none +function render(writetext, mime::MIME, io::IO, regions::RegionIterator, statesof) + bases = statesof(Memory{Union{}}(undef, 0)) # Through a barrier for types found from the values, as in `renderheld`. Applied in the call, as # a closure capturing `inferencebarrier(transition)` is specialised on what it captures. transit = if bases isa Tuple transition else (args...) -> inferencebarrier(transition)(args...) end + # A state of another type than its baseline's breaks the interface, and displays as the baseline + conform(state, base) = if state isa typeof(base) state else base end written = change(transit, io, mime, nothing, bases) for (str, annots) in regions - written = change(transit, io, mime, written, statesof(annots)) + states = statesof(annots) + states = if states isa Tuple map(conform, states, bases) else map!(conform, states, states, bases) end + written = change(transit, io, mime, written, states) writetext(io, str) end change(transit, io, mime, written, bases) diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index 511a3674354f2..c3133fc9191b0 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -513,6 +513,14 @@ end @test ints(regions[1][2]) == 1 && ints(regions[2][2]) == 1 @test (@allocated ints(regions[1][2])) == (@allocated ints(regions[2][2])) == 0 end + # Only the annotations with values of a given type, which alone mark out the regions + let str = Base.AnnotatedString{String, Union{Int, Symbol}}("abcd", [(1:3, 1), (2:2, :x), (3:4, 2)]) + @test [(String(s), collect(a)) for (s, a) in Base.eachregion(str, 1:4, Int)] == [("ab", [1]), ("c", [1, 2]), ("d", [2])] + @test all(((_, a),) -> collect(a) isa Vector{Int}, Base.eachregion(str, 1:4, Int)) + @test [String(s) for (s, _) in Base.eachregion(SubString(str, 2, 4), 1:3, Int)] == ["b", "c", "d"] + @test collect(only(Base.eachregion(str, 1:4, Float64))[2]) == Float64[] + @test Base.eachregion(str, 1:4, Any).annotations === str.annotations # A wider type reads them in place + end # Changing a string while its regions are read is unsupported, but memory safe let str = Base.AnnotatedString("abcdef", [(1:2, 1), (3:4, 2), (5:6, 3)]) @test count(_ -> (Base.annotate!(str, 1:6, 9); true), Base.eachregion(str)) == 3 @@ -1076,6 +1084,11 @@ module MarkedAnnotations reduction(::MIME"text/plain", ::Type{T}, values) where {T <: Level} = Depth(count(value -> value isa T, values)) transition(io::IO, ::MIME"text/plain", from::Depth, to::Depth) = print(io, "["^max(0, to.n - from.n), "]"^max(0, from.n - to.n)) + # Types whose states break the interface: with no baseline state, or of another type + struct Patchy end + reduction(::MIME, ::Type{Patchy}, values) = if any(value -> value isa Patchy, values) Marked(true) end + struct Fickle end + reduction(::MIME, ::Type{Fickle}, values) = if any(value -> value isa Fickle, values) Tagged("f") else Marked(false) end end @testset "AnnotatedDisplay" begin @@ -1125,6 +1138,26 @@ end none = Memory{Union{Mark, Int}}(undef, 0) states = @inferred Base.AnnotatedDisplay.reductions(MIME("text/plain"), Union{Mark, Int}, (nothing, nothing), none) @test Set(states) == Set([Marked(false), nothing]) + # Values that don't display add no rendering of their own: a string displays through the + # code compiled for the types that do + walk = which(Base.AnnotatedDisplay.render, Tuple{Any, MIME, IO, Base.RegionIterator, Any}) + marked(Mark); html(marked(Mark)) + walks() = length(Base.specializations(walk)) + before = walks() + for x in (1, 1.0, :s) + mixed = Base.AnnotatedString{String, Union{Mark, typeof(x)}}("xyz", [(1:2, Mark()), (2:3, x)]) + @test colour(mixed) == colour(Base.AnnotatedString{String, Mark}("xyz", [(1:2, Mark())])) + html(mixed) + end + @test walks() == before + @test colour(Base.AnnotatedString{String, Int}("xy", [(1:1, 1)])) == "xy" # Nothing displays + # Neither changes the text, whatever the value type holding them + (; Patchy, Fickle) = MarkedAnnotations + for T in (Patchy, Fickle) + broken(V) = Base.AnnotatedString{String, V}("xy", [(1:1, T())]) + @test colour(broken(T)) == colour(broken(Union{T, Mark})) == colour(broken(Any)) == "xy" + @test html(broken(T)) == html(broken(Union{T, Mark})) == html(broken(Any)) == "xy" + end # An `Any` string is displayed by code compiled once, whatever types it holds, and however many D = Base.AnnotatedDisplay specialisations() = sum(f -> sum(m -> length(Base.specializations(m)), methods(f)), (D.render, D.renderheld, D.change)) From 5d29754233a2879456688bb3e25788bd4a7f7c65 Mon Sep 17 00:00:00 2001 From: TEC Date: Tue, 22 Sep 2026 19:53:08 +0800 Subject: [PATCH 22/23] Add NEWS for annotated strings and StyledStrings Under Base, note the value type parameter, how it is inferred and widened, that it selects the display, that a type written without it is no longer concrete, and that region iteration is lazy. Under JuliaSyntaxHighlighting, note its palette of faces. Under Markdown, note the renamed faces.toml section. Under StyledStrings, note face primacy, palettes and their use, when face names are resolved, and that `Symbol`-valued face annotations are no longer styled. --- NEWS.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/NEWS.md b/NEWS.md index 8ca44c35bcc53..84bc52c2c6c1f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -213,6 +213,16 @@ New library functions New library features -------------------- +* `AnnotatedString` and `AnnotatedChar` take a second type parameter, the type of their annotation + values, and `AnnotatedIOBuffer` takes it as its only parameter. It is inferred from the + annotations given, and widened as strings are combined. A type written without it, such as + `Vector{AnnotatedString{String}}`, no longer names a concrete type. Packages set how values of + their types display through the functions of `Base.AnnotatedDisplay` ([#60527]). +* Annotations no longer have a label: the type of an annotation's value says what it is. A string's + annotations are `(; region, value)` NamedTuples, given as `(region, value)`, `region => value`, or + with `annotate!(str, [range], value)`. The labelled forms are deprecated. Code that reads + annotations from both older and newer versions can check + `a.value isa Face || (hasproperty(a, :label) && a.label === :face)` ([#60527]). * `IOContext` supports a new boolean `hexunsigned` option that allows for printing unsigned integers in decimal instead of hexadecimal ([#60267]). * `lazy"..."` strings now support a flag `lazy"..."c` that adds `compact` and `limit` flags to the @@ -274,6 +284,12 @@ Standard library changes #### JuliaSyntaxHighlighting +* Highlighting annotates with `Face`s rather than `Symbol` face names, from a palette in the + `julia` namespace, so the faces are still customised as `julia_keyword` and so on. Other + packages can use them as `face"JuliaSyntaxHighlighting.keyword"`, or after + `@usepalette JuliaSyntaxHighlighting`. `highlight!` needs a string whose value type can hold a + `Face` ([#60527]). + #### Libdl * `Libdl.LazyLibrary` is not compatible with `--trim`. @@ -291,6 +307,8 @@ Standard library changes * Many, many bug fixes and minor tweaks; overall behavior is now much closer to CommonMark ([#59977], [#60502]). * Table columns whose delimiter cell has no `:` (such as `---`) are now left-aligned, as on GitHub, instead of right-aligned. +* Markdown's faces form a palette, so they are customised in `faces.toml` under `[Markdown]` + rather than `[markdown]` ([#60527]). ### Mmap @@ -326,6 +344,17 @@ Standard library changes array on all processes, e.g. so the file backing a file-backed `SharedArray` can be deleted immediately ([#62488]). +#### StyledStrings + +* Named faces are `Face` objects rather than `Symbol`s, looked up with `face"name"`. Packages define + theirs in palettes with `@defpalette` and `@registerpalette`, which also hold light and dark + variants, and `@usepalette` brings other modules' palettes into scope. In a module with a palette, + `styled"..."` and `face"..."` reject an unknown face name. Styled strings hold `Face` and `Link` + values, with the value type `StyledValue`, and other `{key=value:…}` annotations are + `Pair{Symbol, Any}` values. `remapfaces` substitutes the faces of a string, and a `faces.toml` + value of `"inherit"` resets an attribute to what the face inherits. Annotations whose values are + `Symbol`s naming faces are no longer styled ([#60527]). + #### Test * Pressing `^C` twice at an empty `julia>` prompt now cancels all still-running work started by earlier evaluations (e.g. a runaway `@async` task spewing From 6ea1e4cfc886449349ce7df549b31b0ba00c9fcf Mon Sep 17 00:00:00 2001 From: TEC Date: Sat, 17 Oct 2026 02:07:49 +0800 Subject: [PATCH 23/23] Bump StyledStrings The rest of JuliaLang/StyledStrings.jl#131. Links reduce without a context, old string links become `Link`s through `labelled_value`, and internal styled text has the value type of styled strings. The face, markup, and palette code is simpler, and the precompile workload covers what a fresh session first uses. Interpolated variables are read in order, markup keys must be identifiers, a customised default face's resets fall back on the standard ones, and face loops through palette variants are rejected. Interpolating a value of unknown type is faster, and StyledStrings' image holds about 330 KB less native code. Stdlib: StyledStrings URL: https://github.com/JuliaLang/StyledStrings.jl.git Stdlib branch: face-primacy Old commit: 10c37a7 New commit: bc6d6a5 Diff: https://github.com/JuliaLang/StyledStrings.jl/compare/10c37a775558e11159430165edafccbb3fe9478e...bc6d6a5dfb923fed48e710a6db895b8c967cd6a7 Assisted-by: Claude Code (Opus 5.5) --- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + stdlib/StyledStrings.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/md5 delete mode 100644 deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/sha512 create mode 100644 deps/checksums/StyledStrings-bc6d6a5dfb923fed48e710a6db895b8c967cd6a7.tar.gz/md5 create mode 100644 deps/checksums/StyledStrings-bc6d6a5dfb923fed48e710a6db895b8c967cd6a7.tar.gz/sha512 diff --git a/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/md5 b/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/md5 deleted file mode 100644 index d5fe2899d2b04..0000000000000 --- a/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -572b6d9e6c8d9cc9ae0ea7eaf819e3ea diff --git a/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/sha512 b/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/sha512 deleted file mode 100644 index c5680ae81c7ee..0000000000000 --- a/deps/checksums/StyledStrings-10c37a775558e11159430165edafccbb3fe9478e.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -d800368c3bd0f2e0a0a095e877960521f1669ad83e139d9d663c116a537f0a71c05b115926f2357de44296615f9111a612f8c030615154a238a8c27413c6986c diff --git a/deps/checksums/StyledStrings-bc6d6a5dfb923fed48e710a6db895b8c967cd6a7.tar.gz/md5 b/deps/checksums/StyledStrings-bc6d6a5dfb923fed48e710a6db895b8c967cd6a7.tar.gz/md5 new file mode 100644 index 0000000000000..da159a742a143 --- /dev/null +++ b/deps/checksums/StyledStrings-bc6d6a5dfb923fed48e710a6db895b8c967cd6a7.tar.gz/md5 @@ -0,0 +1 @@ +7d8f070d469647a99f924d3bf3ff5c65 diff --git a/deps/checksums/StyledStrings-bc6d6a5dfb923fed48e710a6db895b8c967cd6a7.tar.gz/sha512 b/deps/checksums/StyledStrings-bc6d6a5dfb923fed48e710a6db895b8c967cd6a7.tar.gz/sha512 new file mode 100644 index 0000000000000..0937241b78135 --- /dev/null +++ b/deps/checksums/StyledStrings-bc6d6a5dfb923fed48e710a6db895b8c967cd6a7.tar.gz/sha512 @@ -0,0 +1 @@ +f13dc637fce8997a3e8f11ce30d799a4964d90bca5ebd5166a1c0a835f192af0682d4a788a86596351a24807050ed420557d1dc096e90639c9a72c093b33ccc3 diff --git a/stdlib/StyledStrings.version b/stdlib/StyledStrings.version index 4e9ec41f6738a..d6186c5e36dea 100644 --- a/stdlib/StyledStrings.version +++ b/stdlib/StyledStrings.version @@ -1,4 +1,4 @@ STYLEDSTRINGS_BRANCH = face-primacy -STYLEDSTRINGS_SHA1 = 10c37a775558e11159430165edafccbb3fe9478e +STYLEDSTRINGS_SHA1 = bc6d6a5dfb923fed48e710a6db895b8c967cd6a7 STYLEDSTRINGS_GIT_URL := https://github.com/JuliaLang/StyledStrings.jl.git STYLEDSTRINGS_TAR_URL = https://api.github.com/repos/JuliaLang/StyledStrings.jl/tarball/$1