86 lines
3.4 KiB
JavaScript
86 lines
3.4 KiB
JavaScript
// UI selection boundaries only: this does not replace the #3 NLP tokenizer.
|
|
const words = new Intl.Segmenter('en', { granularity: 'word' });
|
|
const graphemes = new Intl.Segmenter('en', { granularity: 'grapheme' });
|
|
const encoder = new TextEncoder();
|
|
|
|
export function tokenize(text) {
|
|
const tokens = [];
|
|
for (const part of words.segment(text)) {
|
|
if (part.isWordLike) {
|
|
tokens.push({ text: part.segment, start: part.index, end: part.index + part.segment.length, word: true });
|
|
} else {
|
|
for (const item of graphemes.segment(part.segment)) {
|
|
const start = part.index + item.index;
|
|
tokens.push({ text: item.segment, start, end: start + item.segment.length, word: /\p{Extended_Pictographic}/u.test(item.segment) });
|
|
}
|
|
}
|
|
}
|
|
return tokens;
|
|
}
|
|
|
|
function selection(text, tokens, first, last) {
|
|
const start = tokens[first].start;
|
|
const end = tokens[last].end;
|
|
return {
|
|
start, end, text: text.slice(start, end), first, last,
|
|
start_cp: [...text.slice(0, start)].length,
|
|
end_cp: [...text.slice(0, end)].length,
|
|
start_utf8: encoder.encode(text.slice(0, start)).length,
|
|
end_utf8: encoder.encode(text.slice(0, end)).length,
|
|
start_utf16: start, end_utf16: end,
|
|
};
|
|
}
|
|
|
|
// Inputs and outputs use half-open UTF-16 offsets into the unchanged original.
|
|
export function normalizeRange(text, tokens, start, end) {
|
|
if (![start, end].every(n => Number.isInteger(n) && n >= 0 && n <= text.length) || start === end) return null;
|
|
const low = Math.min(start, end);
|
|
const high = Math.max(start, end);
|
|
let first = -1;
|
|
let last = -1;
|
|
tokens.forEach((token, index) => {
|
|
if (token.word && token.start < high && token.end > low) {
|
|
if (first === -1) first = index;
|
|
last = index;
|
|
}
|
|
});
|
|
return first === -1 ? null : selection(text, tokens, first, last);
|
|
}
|
|
|
|
export function pointSelection(text, tokens, index) {
|
|
return Number.isInteger(index) && tokens[index]?.word ? selection(text, tokens, index, index) : null;
|
|
}
|
|
|
|
// direction -1/+1 moves a boundary left/right; it never collapses a selection.
|
|
export function adjustRange(text, tokens, range, edge, direction) {
|
|
if (!range || !['start', 'end'].includes(edge) || ![-1, 1].includes(direction)) return range;
|
|
let next = (edge === 'start' ? range.first : range.last) + direction;
|
|
while (next >= 0 && next < tokens.length && !tokens[next].word) next += direction;
|
|
if (next < 0 || next >= tokens.length) return range;
|
|
const first = edge === 'start' ? next : range.first;
|
|
const last = edge === 'end' ? next : range.last;
|
|
return first > last ? range : selection(text, tokens, first, last);
|
|
}
|
|
|
|
export function occurrences(text, needle) {
|
|
if (!needle) return [];
|
|
const tokens = tokenize(text);
|
|
const result = [];
|
|
for (let start = text.indexOf(needle); start !== -1; start = text.indexOf(needle, start + 1)) {
|
|
const end = start + needle.length;
|
|
const range = normalizeRange(text, tokens, start, end);
|
|
if (range?.start === start && range.end === end) result.push(range);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Display precedence only. Saved occurrence identities remain separate.
|
|
export function resolveOverlaps(ranges) {
|
|
const sorted = [...ranges].sort((a, b) => a.start - b.start || b.end - a.end);
|
|
const result = [];
|
|
for (const range of sorted) {
|
|
if (range.end > range.start && (!result.length || range.start >= result.at(-1).end)) result.push(range);
|
|
}
|
|
return result;
|
|
}
|