Skip to content

change(web): add tokenized prediction intermediate type for whitespace correction support 🚂 #15907

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
jahorton wants to merge 3 commits into
base: change/web/multi-token-suggestion-similarity
Choose a base branch
from
Draft

change(web): add tokenized prediction intermediate type for whitespace correction support 🚂 #15907

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ff53eed
change(web): add tokenized prediction intermediate type for whitespac…
jahorton May 4, 2026
d79d54a
fix(web): adjusts existing unit tests to match new intermediate-predi…
jahorton May 11, 2026
b763609
fix(web): apply original casing-application logic on a per-token basis
jahorton May 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 8 additions & 27 deletions web/src/engine/predictive-text/worker-thread/src/main/model-compositor.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
import * as models from '@keymanapp/models-templates';
import { LexicalModelTypes } from '@keymanapp/common-types';

import { applySuggestionCasing, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
import { detectCurrentCasing, determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';
import TransformUtils from './transformUtils.js';

import * as correction from './correction/index.js'
import { applySuggestionCasing, compositeIntermediatePredictions, correctAndEnumerate, createDefaultKeep, dedupeSuggestions, finalizeSuggestions, predictionAutoSelect, processSimilarity, toAnnotatedSuggestion, tupleDisplayOrderSort } from './predict-helpers.js';
import { determineModelTokenizer, determineModelWordbreaker, determinePunctuationFromModel } from './model-helpers.js';

import { ContextTracker } from './correction/context-tracker.js';
import { DEFAULT_ALLOTTED_CORRECTION_TIME_INTERVAL } from './correction/distance-modeler.js';

import CasingForm = LexicalModelTypes.CasingForm;
import Configuration = LexicalModelTypes.Configuration;
import Context = LexicalModelTypes.Context;
import Distribution = LexicalModelTypes.Distribution;
Expand Down Expand Up @@ -125,24 +123,6 @@ export class ModelCompositor {
const transformId = inputTransform.id;
this.initContextTracker(context, transformId);

const allowBksp = TransformUtils.isBackspace(inputTransform);
const allowWhitespace = TransformUtils.isWhitespace(inputTransform);

const postContext = models.applyTransform(inputTransform, context);

// TODO: It would be best for the correctAndEnumerate method to return the
// suggestion's prefix, as it already has lots of logic oriented to this.
// The context-tracker used there with v14+ models can determine this more
// robustly.
const truePrefix = this.wordbreak(postContext);
// Only use of `truePrefix`.
const basePrefix = (allowBksp || allowWhitespace) ? truePrefix : this.wordbreak(context);

// Used to restore whitespaces if operations would remove them.
const currentCasing: CasingForm = lexicalModel.languageUsesCasing
? detectCurrentCasing(lexicalModel, postContext)
: null;

// Section 1: determine 'prediction roots' - enumerate corrections from most to least likely,
// searching for results that yield viable predictions from the model.

Expand All @@ -160,9 +140,9 @@ export class ModelCompositor {
// Properly capitalizes the suggestions based on the existing context casing state.
// This may result in duplicates if multiple casing options exist within the
// lexicon for a word. (Example: "Apple" the company vs "apple" the fruit.)
for(let tuple of rawPredictions) {
if(currentCasing && currentCasing != 'lower') {
applySuggestionCasing(tuple.components.prediction, basePrefix, this.lexicalModel, currentCasing);
if(lexicalModel.languageUsesCasing) {
for(let tuple of rawPredictions) {
tuple.components.forEach((component) => applySuggestionCasing(component, this.lexicalModel));
}
}

Expand All @@ -171,9 +151,10 @@ export class ModelCompositor {

// We want to dedupe before trimming the list so that we can present a full set
// of viable distinct suggestions if available.
const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, rawPredictions, context);
const deduplicatedSuggestionTuples = dedupeSuggestions(this.lexicalModel, compositeIntermediatePredictions(rawPredictions), context);

// Needs "casing" to be applied first.
const postContext = postContextState?.context ?? models.applyTransform(inputTransform, context);
const hasExistingKeep = processSimilarity(this.lexicalModel, deduplicatedSuggestionTuples, context, postContext);

// If no existing suggestion directly matches the user-visible version of
Expand Down
Loading
Loading