From ac9a6a99bac0a5506c197b43f6eea4bddce1f84f Mon Sep 17 00:00:00 2001 From: will wade Date: Thu, 13 Aug 2026 14:53:12 +0100 Subject: [PATCH 1/2] Add linguistic competence metrics for AAC spoken-output analysis Privacy-preserving, pure functions that analyse AAC *spoken output* (phrase history) across the four dimensions of linguistic competence (Light, 1989), grounded in AssistiveWare's 'Measuring AAC user linguistic competence' and Frisch/Wade et al. 'It's Complicated' (arXiv:2606.24854): - Semantic: MATTR-30 lexical diversity (headline, sample-length independent) - Syntactic: MA-UPC-TWR-30 preposition + conjunction diversity - Morphological: MA-UMORPH-TLWR-30 heuristic proxy - Phonological: spelling validity (optional, needs a dictionary) - Activity: utterances/words/unique words/active days/words-per-utterance All measures use 30-word moving-average windows (Covington & McFall, 2010), binned by calendar month with a weighted trend. No I/O, no platform deps - runs anywhere (browser included) and emits only aggregate statistics (no raw text, no word lists). Unit tested. --- src/analytics.ts | 1 + src/utilities/analytics/competence.ts | 997 ++++++++++++++++++++++++++ src/utilities/analytics/index.ts | 3 + test/competence.test.ts | 257 +++++++ 4 files changed, 1258 insertions(+) create mode 100644 src/utilities/analytics/competence.ts create mode 100644 test/competence.test.ts diff --git a/src/analytics.ts b/src/analytics.ts index 39fffb8..ac3435c 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -6,3 +6,4 @@ */ export * from './utilities/analytics/history'; +export * from './utilities/analytics/competence'; diff --git a/src/utilities/analytics/competence.ts b/src/utilities/analytics/competence.ts new file mode 100644 index 0000000..5fdd099 --- /dev/null +++ b/src/utilities/analytics/competence.ts @@ -0,0 +1,997 @@ +/** + * Linguistic Competence Metrics + * + * Privacy-preserving analysis of AAC *spoken output* (phrase history), based on: + * - Niemeijer, Sheldon & Hillary Zisk (2025), "Measuring AAC user linguistic + * competence: A novel approach", AssistiveWare (Communication Matters handout). + * - Frisch, Wade et al. (2026), "It's Complicated: On the Design and Evaluation + * of AI-Powered AAC Interfaces", arXiv:2606.24854. + * + * All functions here are pure and platform-agnostic (no I/O, no Bun/Node APIs). + * They compute only aggregate statistics and never return the raw text they are + * given. This makes them safe to run on-device and trivially unit-testable. + * + * The four dimensions of linguistic competence (Light, 1989) and the measures we + * use for each, following the AssistiveWare findings: + * + * Semantic -> MATTR-30 (lexical diversity, the strongest overall measure) + * Syntactic -> MA-UPC-TWR-30 (preposition + conjunction diversity) + * Morphological -> MA-UMORPH-TLWR-30 (proxy; documented below) + * Phonological -> proportion of unique words correctly spelled (optional, weak) + * + * Key design choices, straight from the AssistiveWare paper: + * - Diversity beats density (density plateaus); we use diversity measures. + * - Moving-average windows of 30 words (Covington & McFall, 2010) make the + * measures insensitive to sample length and usable for tiny language samples. + * - MLU is reported only as a *distribution* and never as a headline, because + * it conflates linguistic, operational, strategic and social competence. + */ + +/** A single spoken utterance with a production timestamp (epoch ms). */ +export interface CompetenceUtterance { + text: string; + timestampMs: number; +} + +/** A tokenised word stream produced in chronological order. */ +export type WordStream = string[]; + +/** Options shared by the dimension measures. */ +export interface DiversityOptions { + /** Moving-average window size in words. The papers use 30. */ + windowSize?: number; + /** BCP-47-ish language code, e.g. "en-GB", "nl-BE". Only the primary subtag matters. */ + lang?: string; +} + +export interface DiversityResult { + /** Median of the per-window values (the headline figure, per the paper). */ + median: number | null; + mean: number | null; + /** Number of windows that contributed a value (after any skipping). */ + nWindows: number; + /** The window size used. */ + windowSize: number; +} + +/* ------------------------------------------------------------------ * + * Small statistics helpers + * ------------------------------------------------------------------ */ + +function mean(values: number[]): number | null { + if (values.length === 0) return null; + let sum = 0; + for (const v of values) sum += v; + return sum / values.length; +} + +function median(values: number[]): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} + +function quantile(values: number[], q: number): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const pos = (sorted.length - 1) * q; + const base = Math.floor(pos); + const rest = pos - base; + if (sorted[base + 1] !== undefined) { + return sorted[base] + rest * (sorted[base + 1] - sorted[base]); + } + return sorted[base]; +} + +/* ------------------------------------------------------------------ * + * Tokenisation + * ------------------------------------------------------------------ */ + +const TOKEN_RE = /[\p{L}\p{N}]+(?:['’][\p{L}\p{N}]+)?/gu; + +/** + * Tokenise raw text into a lowercased word stream. + * Keeps intra-word apostrophes (don't, children's) but drops leading/trailing + * punctuation and pure whitespace. Accented characters are preserved (\p{L}). + */ +export function tokenize(text: string): WordStream { + if (!text) return []; + const out: string[] = []; + let m: RegExpExecArray | null; + TOKEN_RE.lastIndex = 0; + while ((m = TOKENReexec()) !== null) { + let tok = m[0].toLowerCase(); + // Strip stray leading/trailing apostrophes introduced by quotes. + tok = tok.replace(/^['’]+|['’]+$/g, ''); + if (tok.length > 0) out.push(tok); + } + return out; + + function TOKENReexec(): RegExpExecArray | null { + return TOKEN_RE.exec(text); + } +} + +/* ------------------------------------------------------------------ * + * Semantic competence: lexical diversity (MATTR-30) + * ------------------------------------------------------------------ */ + +/** + * Compute a type-token ratio for one segment of words. + */ +function segmentTTR(segment: WordStream): number { + if (segment.length === 0) return 0; + return new Set(segment).size / segment.length; +} + +/** + * Moving-Average Type-Token Ratio (Covington & McFall, 2010). + * + * Slides a fixed-size window across the word stream and computes the TTR for + * each window. The median across windows is sample-length independent, which is + * exactly why it is preferred over plain TTR for highly variable AAC samples. + * + * Returns the headline `median` plus mean and window count. + */ +export function movingAverageTTR(words: WordStream, windowSize = 30): DiversityResult { + const n = words.length; + const w = Math.max(1, Math.floor(windowSize)); + if (n === 0) return { median: null, mean: null, nWindows: 0, windowSize: w }; + + // Fewer words than one window: compute TTR over the whole sample. + if (n < w) { + const v = segmentTTR(words); + return { median: v, mean: v, nWindows: 1, windowSize: w }; + } + + const values: number[] = []; + for (let i = 0; i <= n - w; i++) { + values.push(segmentTTR(words.slice(i, i + w))); + } + return { + median: median(values), + mean: mean(values), + nWindows: values.length, + windowSize: w, + }; +} + +/** Convenience: MATTR-30 lexical diversity (the semantic headline). */ +export function lexicalDiversity(words: WordStream, windowSize = 30): DiversityResult { + return movingAverageTTR(words, windowSize); +} + +/* ------------------------------------------------------------------ * + * Syntactic competence: preposition + conjunction diversity (MA-UPC-TWR-30) + * ------------------------------------------------------------------ */ + +/** English prepositions (closed set, indicator of relational syntax). */ +const EN_PREPOSITIONS = new Set([ + 'about', + 'above', + 'across', + 'after', + 'against', + 'along', + 'alongside', + 'amid', + 'among', + 'amongst', + 'around', + 'as', + 'at', + 'atop', + 'before', + 'behind', + 'below', + 'beneath', + 'beside', + 'besides', + 'between', + 'beyond', + 'by', + 'despite', + 'down', + 'during', + 'except', + 'for', + 'from', + 'in', + 'inside', + 'into', + 'near', + 'of', + 'off', + 'on', + 'onto', + 'opposite', + 'out', + 'outside', + 'over', + 'past', + 'per', + 'plus', + 'round', + 'since', + 'than', + 'through', + 'throughout', + 'till', + 'to', + 'toward', + 'towards', + 'under', + 'underneath', + 'unlike', + 'until', + 'up', + 'upon', + 'versus', + 'via', + 'with', + 'within', + 'without', +]); + +/** English conjunctions (closed set, indicator of clausal complexity). */ +const EN_CONJUNCTIONS = new Set([ + 'and', + 'but', + 'or', + 'nor', + 'yet', + 'so', + 'although', + 'because', + 'considering', + 'if', + 'lest', + 'once', + 'provided', + 'since', + 'than', + 'that', + 'though', + 'unless', + 'until', + 'when', + 'whenever', + 'where', + 'whereas', + 'wherever', + 'whether', + 'while', + 'whilst', + 'why', + 'neither', + 'either', + 'both', +]); + +/** Dutch prepositions. */ +const NL_PREPOSITIONS = new Set([ + 'aan', + 'achter', + 'bij', + 'door', + 'tijdens', + 'in', + 'boven', + 'langs', + 'met', + 'na', + 'naar', + 'om', + 'onder', + 'op', + 'over', + 'rond', + 'per', + 'sinds', + 'tegen', + 'uit', + 'van', + 'voor', + 'tot', + 'tussen', + 'binnen', + 'buiten', + 'behalve', + 'wegens', + 'krachtens', + 'volgens', +]); + +/** Dutch conjunctions. */ +const NL_CONJUNCTIONS = new Set([ + 'en', + 'of', + 'maar', + 'want', + 'dus', + 'omdat', + 'toen', + 'voordat', + 'nadat', + 'terwijl', + 'indien', + 'mits', + 'tenzij', + 'hoewel', + 'ofschoon', + 'zodra', + 'zolang', + 'als', + 'dat', + 'noch', +]); + +/** + * Get the closed-set (prepositions ∪ conjunctions) for a language, or null if + * unsupported. Callers should skip the syntactic measure when this is null. + */ +export function getClosedClassSet(lang?: string): Set | null { + const primary = (lang || 'en').split(/[-_]/)[0].toLowerCase(); + switch (primary) { + case 'en': + return new Set([...EN_PREPOSITIONS, ...EN_CONJUNCTIONS]); + case 'nl': + return new Set([...NL_PREPOSITIONS, ...NL_CONJUNCTIONS]); + default: + return null; + } +} + +/** + * Syntactic diversity: MA-UPC-TWR-30. + * + * For each 30-word window, compute the type-token ratio restricted to the + * closed-class words that indicate syntactic complexity (prepositions and + * conjunctions). Windows containing none of these are skipped (no syntactic + * signal). The median across contributing windows is reported. + */ +export function syntacticDiversity( + words: WordStream, + options: DiversityOptions = {} +): DiversityResult { + const w = Math.max(1, Math.floor(options.windowSize ?? 30)); + const closed = getClosedClassSet(options.lang); + if (!closed) { + return { median: null, mean: null, nWindows: 0, windowSize: w }; + } + + const n = words.length; + if (n === 0) return { median: null, mean: null, nWindows: 0, windowSize: w }; + + const values: number[] = []; + const scan = (segment: WordStream): void => { + const upc = segment.filter((tok) => closed.has(tok)); + if (upc.length > 0) { + values.push(new Set(upc).size / upc.length); + } + }; + + if (n < w) { + scan(words); + } else { + for (let i = 0; i <= n - w; i++) { + scan(words.slice(i, i + w)); + } + } + + if (values.length === 0) { + return { median: null, mean: null, nWindows: 0, windowSize: w }; + } + return { median: median(values), mean: mean(values), nWindows: values.length, windowSize: w }; +} + +/* ------------------------------------------------------------------ * + * Morphological competence: MA-UMORPH-TLWR-30 (documented proxy) + * ------------------------------------------------------------------ */ + +export type InflectionCategory = + | 'base' + | 'plural' + | 'possessive' + | 'past' + | 'progressive' + | 'comparative' + | 'superlative' + | 'adverb'; + +const BASE_GUARD = new Set([ + // Words that look inflected but are base forms; avoids the worst false hits. + 'is', + 'as', + 'was', + 'has', + 'his', + 'its', + 'us', + 'bus', + 'gas', + 'yes', + 'this', + 'miss', + 'loss', + 'boss', + 'less', + 'dress', + 'guess', + 'press', + 'cross', + 'class', + 'glass', + 'pass', + 'mass', + 'ass', + 'bed', + 'red', + 'fed', + 'led', + 'wed', + 'shed', + 'ted', + 'bred', + 'her', + 'per', + 'der', + 'fer', + 'ring', + 'sing', + 'king', + 'thing', + 'bring', + 'long', + 'wing', + 'spring', + 'string', + 'sting', + 'cling', + 'swing', + 'fling', + 'slang', + 'best', + 'rest', + 'test', + 'west', + 'nest', + 'chest', + 'pest', + 'vest', + 'quest', + 'fest', + 'lest', + 'beast', + 'feast', + 'breast', + 'fly', + 'ply', + 'sly', + 'ally', + 'rally', + 'supply', + 'reply', + 'apply', + 'comply', + 'rely', + 'family', + 'only', + 'lonely', + 'likely', + 'lovely', + 'silly', + 'holy', + 'uly', + 'fully', +]); + +/** + * Crude, dependency-free inflection classifier for English. + * + * This is intentionally a *heuristic proxy*: it identifies the morphological + * operation encoded by a word's suffix so we can measure how diverse the user's + * morphological production is. It is NOT a lemmatiser. False positives/negatives + * are expected and documented; the measure is used comparatively across time + * bins, not as an absolute clinical score. + */ +export function classifyInflection(word: string): InflectionCategory { + const w = word.toLowerCase(); + if (w.length < 3) return 'base'; + + // Possessive first (apostrophe forms): "dad's", "dogs'". + if (w.endsWith("'s") || w.endsWith("s'")) return 'possessive'; + + if (BASE_GUARD.has(w)) return 'base'; + + if (w.endsWith('ing') && w.length > 4) return 'progressive'; + if (w.endsWith('est') && w.length > 4) return 'superlative'; + if (w.endsWith('ies') && w.length > 3) return 'plural'; // berries, stories + if (w.endsWith('ied') && w.length > 3) return 'past'; // carried + if (w.endsWith('ed') && w.length > 3 && !w.endsWith('eed')) return 'past'; + if (w.endsWith('er') && w.length > 3) return 'comparative'; + if (w.endsWith('ly') && w.length > 3) return 'adverb'; + if (w.endsWith('es') && w.length > 3) return 'plural'; + if ( + w.endsWith('s') && + !w.endsWith('ss') && + !w.endsWith('us') && + !w.endsWith('is') && + w.length > 2 + ) { + return 'plural'; + } + return 'base'; +} + +/** + * Morphological diversity (proxy for MA-UMORPH-TLWR-30). + * + * Within each 30-word window, look at the words carrying a morphological + * operation (any category other than "base") and compute the type-token ratio + * over their *surface forms*. A higher value means the user is producing a + * broader, less-repetitive range of inflected forms locally. Windows with no + * inflected words are skipped. + * + * Caveat (per AssistiveWare): for symbol-supported AAC, pre-stored morphology + * buttons (e.g. "finished", "all done", "is") heavily affect this measure, so + * interpret trends rather than absolutes. + */ +export function morphologicalDiversity( + words: WordStream, + options: DiversityOptions = {} +): DiversityResult { + const w = Math.max(1, Math.floor(options.windowSize ?? 30)); + const lang = (options.lang || 'en').split(/[-_]/)[0].toLowerCase(); + if (lang !== 'en') { + return { median: null, mean: null, nWindows: 0, windowSize: w }; + } + + const n = words.length; + if (n === 0) return { median: null, mean: null, nWindows: 0, windowSize: w }; + + const values: number[] = []; + const scan = (segment: WordStream): void => { + const inflected = segment.filter((tok) => classifyInflection(tok) !== 'base'); + if (inflected.length > 0) { + values.push(new Set(inflected).size / inflected.length); + } + }; + + if (n < w) { + scan(words); + } else { + for (let i = 0; i <= n - w; i++) { + scan(words.slice(i, i + w)); + } + } + + if (values.length === 0) { + return { median: null, mean: null, nWindows: 0, windowSize: w }; + } + return { median: median(values), mean: mean(values), nWindows: values.length, windowSize: w }; +} + +/* ------------------------------------------------------------------ * + * Phonological competence: spelling validity (optional, weak) + * ------------------------------------------------------------------ */ + +/** + * Proportion of unique alphabetic words present in the supplied dictionary. + * + * Focuses on *unique* words so it is not skewed by repetition or repeated + * misspellings. Pass a dictionary Set (e.g. loaded from a hunspell/aspell + * wordlist). Returns null if no dictionary is provided. + */ +export function spellingValidity(words: WordStream, dictionary?: Set): number | null { + if (!dictionary || dictionary.size === 0) return null; + const unique = new Set(words); + let checked = 0; + let correct = 0; + for (const tok of unique) { + // Only consider alphabetic tokens of reasonable length. + if (!/^\p{L}{2,}$/u.test(tok)) continue; + checked++; + if (dictionary.has(tok)) correct++; + } + return checked === 0 ? null : correct / checked; +} + +/* ------------------------------------------------------------------ * + * Activity / engagement statistics (multidimensional, distributional) + * ------------------------------------------------------------------ */ + +export interface DistributionStats { + median: number | null; + mean: number | null; + p25: number | null; + p75: number | null; + n: number; +} + +export interface ActivityStats { + utterances: number; + words: number; + /** Distinct tokens — vocabulary breadth (count only, never the words). */ + uniqueWords: number; + activeDays: number; + wordsPerUtterance: DistributionStats; +} + +function distribution(values: number[]): DistributionStats { + return { + median: median(values), + mean: mean(values), + p25: quantile(values, 0.25), + p75: quantile(values, 0.75), + n: values.length, + }; +} + +/** Compute engagement/activity stats for a set of utterances. */ +export function summarizeActivity(utterances: CompetenceUtterance[]): ActivityStats { + const days = new Set(); + let totalWords = 0; + const wpu: number[] = []; + for (const u of utterances) { + const toks = tokenize(u.text); + totalWords += toks.length; + wpu.push(toks.length); + const day = Math.floor(u.timestampMs / 86_400_000); + days.add(day); + } + return { + utterances: utterances.length, + words: totalWords, + uniqueWords: 0, // filled by caller from the ordered stream for accuracy + activeDays: days.size, + wordsPerUtterance: distribution(wpu), + }; +} + +/* ------------------------------------------------------------------ * + * Timeline analysis (the longitudinal engine) + * ------------------------------------------------------------------ */ + +export interface MonthBin { + /** Calendar month in local time, "YYYY-MM". */ + month: string; + utterances: number; + words: number; + uniqueWords: number; + activeDays: number; + wordsPerUtterance: DistributionStats; + /** Semantic — the headline measure. */ + lexicalDiversity: DiversityResult; + /** Syntactic. null if language unsupported. */ + syntacticDiversity: DiversityResult; + /** Morphological (proxy). null if language unsupported. */ + morphologicalDiversity: DiversityResult; + /** Phonological — only when a dictionary is supplied. */ + spellingValidity: number | null; + /** True when the month has too little data to trust the diversity figures. */ + suppressed: boolean; + suppressReason: string | null; +} + +export interface TrendResult { + metric: string; + /** Slope of the weighted linear regression, in metric units per month. */ + slopePerMonth: number | null; + firstHalf: number | null; + secondHalf: number | null; + /** secondHalf - firstHalf. */ + delta: number | null; + direction: 'up' | 'down' | 'flat' | 'unknown'; +} + +/** + * Structural / effort summary of the user's AAC pageset (gridset). + * + * Computed from the existing MetricsCalculator so competence numbers can be read + * alongside the vocabulary design that shapes them (grid size, vocabulary size, + * effort, prediction) — the AssistiveWare paper stresses that setup affects both + * competence and its measurement. Contains NO word labels (privacy). + */ +export interface PagesetSummary { + /** Hashed label unless the caller opts in to revealing the file name. */ + label: string; + gridsetIncluded: boolean; + analysisVersion?: string; + totalBoards: number; + totalButtons: number; + totalWords: number; + grid: { rows: number; columns: number }; + /** Effort distribution across all scored buttons (lower = easier). */ + effort: DistributionStats; + hasDynamicPrediction: boolean; + spellingEffort: { base: number | null; perLetter: number | null }; + /** Present if the gridset could not be loaded/analysed. */ + error?: string; +} + +/** + * Privacy-safe system-configuration context for the user (Grid 3 UserSettings). + * + * These are NOT chat content — they describe how the system is set up, which is + * essential for interpreting the competence numbers. Notably `onlineAiToolsOptIn` + * records whether the user has enabled the vendor's online AI tools, letting + * competence trends be read against whether AI assistance was even active. + */ +export interface UserSettingsSummary { + /** Vocabulary/gridset name configured as the startup set (a product name). */ + startupGridSet: string | null; + /** Whether the user opted in to the vendor's online AI tools. */ + onlineAiToolsOptIn: boolean | null; + /** Enabled access method names (e.g. Touch, Pointer, EyeGaze, Switch). */ + accessMethods: string[]; + /** Counts of user-created personalisation entries (aggregate across languages). */ + personalisation: { + pronunciations: number; + capitalisations: number; + abbreviationExpansions: number; + smallWords: number; + }; +} + +export interface CompetenceReport { + schema: string; + generatedAt: string; + privacy: { + rawUtterancesIncluded: boolean; + wordListsIncluded: boolean; + fringeWordFrequencyIncluded: boolean; + minAggregationWindowDays: number; + notes: string[]; + }; + source: { + platform: string; + langCode?: string; + userLabel?: string; + dbPathIncluded: boolean; + }; + config: { + months: number; + windowSize: number; + lang: string; + minWordsPerMonth: number; + dictionaryProvided: boolean; + }; + overall: { + windowStart: string; + windowEnd: string; + totalUtterances: number; + totalWords: number; + monthsCovered: number; + monthsSuppressed: number; + }; + timeline: MonthBin[]; + trend: TrendResult; + /** Structural metrics for the user's default gridset (null if unavailable). */ + pageset?: PagesetSummary | null; + /** System-configuration context (access method, AI opt-in, startup gridset). */ + userSettings?: UserSettingsSummary | null; +} + +export interface TimelineOptions { + /** How many trailing months to analyse. Default 12. */ + months?: number; + /** Moving-average window. Default 30. */ + windowSize?: number; + /** Language code. Default "en". */ + lang?: string; + /** Months with fewer than this many words are flagged suppressed. Default 150. */ + minWordsPerMonth?: number; + /** Optional dictionary Set for the spelling measure. */ + dictionary?: Set; + /** Epoch ms for "now". Defaults to Date.now(). Mainly for tests. */ + now?: number; + /** Platform label for the report. Default "Grid3". */ + platform?: string; + userLabel?: string; + langCode?: string; + dbPathIncluded?: boolean; +} + +/** + * Build a YYYY-MM key from epoch ms in local time. + */ +function monthKey(timestampMs: number): string { + const d = new Date(timestampMs); + const y = d.getFullYear(); + const m = String(d.getMonth() + 1).padStart(2, '0'); + return `${y}-${m}`; +} + +/** + * Weighted linear-regression slope. Weights account for uneven sample sizes so + * that a noisy low-volume month cannot dominate the trend. + */ +function weightedSlope(points: Array<{ x: number; y: number; w: number }>): number | null { + const usable = points.filter((p) => p.y !== null && isFinite(p.y)); + if (usable.length < 2) return null; + let sw = 0; + let swx = 0; + let swy = 0; + let swxx = 0; + let swxy = 0; + for (const p of usable) { + const wgt = Math.max(p.w, 1); + sw += wgt; + swx += wgt * p.x; + swy += wgt * p.y; + swxx += wgt * p.x * p.x; + swxy += wgt * p.x * p.y; + } + const denom = sw * swxx - swx * swx; + if (denom === 0) return null; + return (sw * swxy - swx * swy) / denom; +} + +/** + * Analyse a corpus of utterances as a longitudinal competence report. + * + * Utterances are filtered to the trailing `months` window, binned by calendar + * month, and each bin is scored on the four competence dimensions plus activity. + * The headline lexical-diversity trend across months is summarised. + * + * No raw text, word list, or fringe-vocabulary frequency is included in the + * returned report — only aggregate statistics. + */ +export function analyzeTimeline( + utterances: CompetenceUtterance[], + options: TimelineOptions = {} +): CompetenceReport { + const months = Math.max(1, Math.floor(options.months ?? 12)); + const windowSize = Math.max(1, Math.floor(options.windowSize ?? 30)); + const lang = options.lang ?? 'en'; + const minWordsPerMonth = Math.max(0, Math.floor(options.minWordsPerMonth ?? 150)); + const dictionary = options.dictionary; + const now = options.now ?? Date.now(); + + // ---- Filter to the trailing N months ---------------------------------- + const windowMs = months * 31 * 86_400_000; + const windowStartMs = now - windowMs; + const inWindow = utterances.filter( + (u) => + u.timestampMs <= now && u.timestampMs > windowStartMs && u.text && u.text.trim().length > 0 + ); + + // ---- Bin by month ----------------------------------------------------- + const bins = new Map(); + for (const u of inWindow) { + const key = monthKey(u.timestampMs); + const arr = bins.get(key) ?? []; + arr.push(u); + bins.set(key, arr); + } + + const sortedKeys = [...bins.keys()].sort(); + const timeline: MonthBin[] = []; + + for (const key of sortedKeys) { + const monthUtts = bins + .get(key)! + .slice() + .sort((a, b) => a.timestampMs - b.timestampMs); + + // Ordered word stream for the sliding-window measures. + const stream: WordStream = []; + for (const u of monthUtts) stream.push(...tokenize(u.text)); + + const activity = summarizeActivity(monthUtts); + activity.uniqueWords = new Set(stream).size; + + const suppressed = stream.length < minWordsPerMonth; + const suppressReason: string | null = suppressed + ? `fewer than ${minWordsPerMonth} words (${stream.length})` + : null; + + const lex = suppressed + ? { median: null, mean: null, nWindows: 0, windowSize } + : lexicalDiversity(stream, windowSize); + const syn = suppressed + ? { median: null, mean: null, nWindows: 0, windowSize } + : syntacticDiversity(stream, { windowSize, lang }); + const mor = suppressed + ? { median: null, mean: null, nWindows: 0, windowSize } + : morphologicalDiversity(stream, { windowSize, lang }); + const spell = suppressed ? null : spellingValidity(stream, dictionary); + + timeline.push({ + month: key, + utterances: activity.utterances, + words: activity.words, + uniqueWords: activity.uniqueWords, + activeDays: activity.activeDays, + wordsPerUtterance: activity.wordsPerUtterance, + lexicalDiversity: lex, + syntacticDiversity: syn, + morphologicalDiversity: mor, + spellingValidity: spell, + suppressed, + suppressReason, + }); + } + + // ---- Trend on the headline lexical-diversity median ------------------- + const points: Array<{ x: number; y: number; w: number }> = []; + for (let i = 0; i < timeline.length; i++) { + const b = timeline[i]; + if (b.suppressed) continue; + const y = b.lexicalDiversity.median; + if (y === null) continue; + points.push({ x: i, y, w: b.words }); + } + + const slope = weightedSlope(points); + const validYs = points.map((p) => p.y); + const half = Math.floor(validYs.length / 2); + let firstHalf: number | null = null; + let secondHalf: number | null = null; + if (validYs.length >= 2) { + const fh = validYs.slice(0, Math.max(1, half)); + const sh = validYs.slice(Math.max(1, half)); + firstHalf = mean(fh); + secondHalf = mean(sh); + } + const delta = firstHalf !== null && secondHalf !== null ? secondHalf - firstHalf : null; + let direction: TrendResult['direction'] = 'unknown'; + if (slope !== null) { + if (Math.abs(slope) < 0.0005) direction = 'flat'; + else direction = slope > 0 ? 'up' : 'down'; + } else if (delta !== null) { + if (Math.abs(delta) < 0.005) direction = 'flat'; + else direction = delta > 0 ? 'up' : 'down'; + } + + const totalUtts = timeline.reduce((s, b) => s + b.utterances, 0); + const totalWords = timeline.reduce((s, b) => s + b.words, 0); + + return { + schema: 'aac-competence-report/v1', + generatedAt: new Date(now).toISOString(), + privacy: { + rawUtterancesIncluded: false, + wordListsIncluded: false, + fringeWordFrequencyIncluded: false, + minAggregationWindowDays: 31, + notes: [ + 'All metrics computed locally; only aggregate statistics are emitted.', + 'Utterances are binned by calendar month so no pattern can be tied to a specific day or time.', + 'No word lists or fringe-vocabulary frequencies are included (per AssistiveWare privacy guidance).', + ], + }, + source: { + platform: options.platform ?? 'Grid3', + langCode: options.langCode, + userLabel: options.userLabel, + dbPathIncluded: options.dbPathIncluded === true, + }, + config: { + months, + windowSize, + lang, + minWordsPerMonth, + dictionaryProvided: !!dictionary, + }, + overall: { + windowStart: sortedKeys[0] ?? monthKey(windowStartMs), + windowEnd: sortedKeys[sortedKeys.length - 1] ?? monthKey(now), + totalUtterances: totalUtts, + totalWords: totalWords, + monthsCovered: timeline.length, + monthsSuppressed: timeline.filter((b) => b.suppressed).length, + }, + timeline, + trend: { + metric: 'lexicalDiversity.median', + slopePerMonth: slope, + firstHalf, + secondHalf, + delta, + direction, + }, + }; +} diff --git a/src/utilities/analytics/index.ts b/src/utilities/analytics/index.ts index 579711b..3f9a935 100644 --- a/src/utilities/analytics/index.ts +++ b/src/utilities/analytics/index.ts @@ -33,6 +33,9 @@ export { SentenceAnalyzer } from './metrics/sentence'; export { ComparisonAnalyzer } from './metrics/comparison'; export { ReferenceLoader } from './reference'; +// Export linguistic-competence measures (privacy-preserving spoken-output analysis) +export * from './competence'; + /** * Get the default reference data path */ diff --git a/test/competence.test.ts b/test/competence.test.ts new file mode 100644 index 0000000..4bb466d --- /dev/null +++ b/test/competence.test.ts @@ -0,0 +1,257 @@ +import { describe, expect, it } from '@jest/globals'; +import { + analyzeTimeline, + classifyInflection, + getClosedClassSet, + lexicalDiversity, + morphologicalDiversity, + movingAverageTTR, + spellingValidity, + summarizeActivity, + syntacticDiversity, + tokenize, + type CompetenceUtterance, +} from '../src/utilities/analytics/competence'; + +describe('competence / tokenize', () => { + it('lowercases and keeps apostrophes inside words', () => { + expect(tokenize("I DON'T want it")).toEqual(['i', "don't", 'want', 'it']); + }); + + it('drops punctuation but keeps accented characters', () => { + expect(tokenize('Hallo, wereld! Café — och "ja".')).toEqual([ + 'hallo', + 'wereld', + 'café', + 'och', + 'ja', + ]); + }); + + it('returns [] for empty input', () => { + expect(tokenize('')).toEqual([]); + expect(tokenize(' !!! ')).toEqual([]); + }); +}); + +describe('competence / movingAverageTTR (MATTR)', () => { + it('returns nulls for an empty stream', () => { + const r = movingAverageTTR([], 30); + expect(r.median).toBeNull(); + expect(r.nWindows).toBe(0); + }); + + it('treats a short sample as a single window TTR', () => { + const r = movingAverageTTR(['a', 'b', 'c'], 30); + // 3 unique / 3 = 1.0 + expect(r.median).toBeCloseTo(1.0); + expect(r.nWindows).toBe(1); + }); + + it('gives a higher diversity for a varied stream than a repetitive one', () => { + const varied = ( + 'the quick brown fox jumps over lazy dog cat hat bat ' + + 'sun moon star tree leaf rock river hill cloud rain wind' + ).split(/\s+/); + const repetitive = Array.from({ length: 40 }, () => 'yes'); + expect(lexicalDiversity(varied, 30).median!).toBeGreaterThan( + lexicalDiversity(repetitive, 30).median! + ); + // Repetitive single-word stream has TTR ~ 1/30 within windows. + expect(lexicalDiversity(repetitive, 30).median).toBeCloseTo(1 / 30, 5); + }); + + it('MATTR is sample-length insensitive (more repetition does not inflate it)', () => { + const base = 'a b c d e f g h i j k l m n o'.split(/\s+/); // 15 unique + const doubled = [...base, ...base]; // same words repeated + // With window 15, repeated unique set still yields TTR 1.0 per window. + expect(movingAverageTTR(base, 15).median).toBeCloseTo(1.0); + expect(movingAverageTTR(doubled, 15).median).toBeCloseTo(1.0); + }); +}); + +describe('competence / syntacticDiversity (MA-UPC-TWR-30)', () => { + it('returns null median for a language with no closed-class set', () => { + const r = syntacticDiversity(['and', 'but'], { lang: 'xx', windowSize: 30 }); + expect(r.median).toBeNull(); + }); + + it('detects prepositions and conjunctions in English', () => { + expect(getClosedClassSet('en-GB')!.has('because')).toBe(true); + expect(getClosedClassSet('en')!.has('under')).toBe(true); + expect(getClosedClassSet('en')!.has('banana')).toBe(false); + }); + + it('scores a window rich in varied connectors higher than one using only "and"', () => { + const rich = + 'I went to the shop and bought milk but the milk was off so I went back because it was bad although they refunded me'.split( + /\s+/ + ); + const onlyAnd = Array.from({ length: 40 }, (_, i) => (i % 2 === 0 ? 'and' : 'cat')); + const r1 = syntacticDiversity(rich, { lang: 'en', windowSize: 15 }); + const r2 = syntacticDiversity(onlyAnd, { lang: 'en', windowSize: 15 }); + expect(r1.median!).toBeGreaterThan(r2.median!); + // Only "and" repeated ~8x per window -> distinct/total = 1/8. + expect(r2.median!).toBeLessThan(0.2); + }); + + it('supports Dutch closed-class set', () => { + expect(getClosedClassSet('nl-BE')!.has('omdat')).toBe(true); + expect(getClosedClassSet('nl')!.has('onder')).toBe(true); + }); +}); + +describe('competence / morphologicalDiversity (proxy)', () => { + it('returns null for non-English (documented limitation)', () => { + const r = morphologicalDiversity(['lopen', 'gelopen'], { lang: 'nl', windowSize: 5 }); + expect(r.median).toBeNull(); + }); + + it('classifies obvious inflections', () => { + expect(classifyInflection('running')).toBe('progressive'); + expect(classifyInflection('jumped')).toBe('past'); + expect(classifyInflection('biggest')).toBe('superlative'); + expect(classifyInflection('happily')).toBe('adverb'); + expect(classifyInflection('cats')).toBe('plural'); + expect(classifyInflection("dad's")).toBe('possessive'); + }); + + it('keeps guarded base words as base', () => { + expect(classifyInflection('is')).toBe('base'); + expect(classifyInflection('ring')).toBe('base'); + expect(classifyInflection('best')).toBe('base'); + }); + + it('scores a stream with varied morphology above a base-only stream', () => { + const morphed = 'cats running jumped faster biggest quickly dogs walked eating smaller'.split( + /\s+/ + ); + const base = 'cat run jump fast big quick dog walk eat small'.split(/\s+/); + const r1 = morphologicalDiversity(morphed, { lang: 'en', windowSize: 10 }); + // base-only stream has no inflected words -> null + const r2 = morphologicalDiversity(base, { lang: 'en', windowSize: 10 }); + expect(r1.median).not.toBeNull(); + expect(r2.median).toBeNull(); + }); +}); + +describe('competence / spellingValidity', () => { + it('returns null without a dictionary', () => { + expect(spellingValidity(['hello', 'wrld'])).toBeNull(); + }); + + it('counts dictionary hits over unique alphabetic words', () => { + const dict = new Set(['hello', 'world']); + // unique alphabetic words: hello, world, xyz -> 2/3 + expect(spellingValidity(['hello', 'world', 'xyz', 'hello'], dict)).toBeCloseTo(2 / 3, 5); + }); + + it('ignores non-alphabetic tokens', () => { + const dict = new Set(['hello']); + expect(spellingValidity(['hello', '12', 'a'], dict)).toBeCloseTo(1.0, 5); // only "hello" checked + }); +}); + +describe('competence / summarizeActivity', () => { + it('counts utterances, words, active days and words per utterance', () => { + const DAY = 86_400_000; + const utts: CompetenceUtterance[] = [ + { text: 'hello world', timestampMs: 0 }, + { text: 'good morning', timestampMs: DAY }, // next day + { text: 'bye', timestampMs: DAY + 1000 }, // same day + ]; + const a = summarizeActivity(utts); + expect(a.utterances).toBe(3); + expect(a.words).toBe(5); // hello world good morning bye + expect(a.activeDays).toBe(2); + expect(a.wordsPerUtterance.median).toBe(2); + expect(a.wordsPerUtterance.n).toBe(3); + }); +}); + +describe('competence / analyzeTimeline', () => { + const DAY = 86_400_000; + + function buildCorpus(): CompetenceUtterance[] { + const now = Date.now(); + const utts: CompetenceUtterance[] = []; + // Two months ago: simple, repetitive vocabulary. + const simpleMonth = now - 60 * DAY; + for (let i = 0; i < 30; i++) { + utts.push({ text: 'I want it yes', timestampMs: simpleMonth + i * 1000 }); + } + // This month: richer, more syntactically complex vocabulary. + for (let i = 0; i < 30; i++) { + utts.push({ + text: 'I think that we should go to the park because the weather is lovely although it might rain', + timestampMs: now - i * 60_000, + }); + } + return utts; + } + + it('bins utterances by month and computes per-bin metrics', () => { + const report = analyzeTimeline(buildCorpus(), { + months: 3, + windowSize: 15, + minWordsPerMonth: 1, + lang: 'en', + }); + expect(report.schema).toBe('aac-competence-report/v1'); + expect(report.timeline.length).toBeGreaterThanOrEqual(1); + for (const bin of report.timeline) { + expect(bin.suppressed).toBe(false); + expect(bin.lexicalDiversity.median).not.toBeNull(); + } + }); + + it('flags sparse months as suppressed and omits their diversity figures', () => { + const now = Date.now(); + const report = analyzeTimeline([{ text: 'hi', timestampMs: now - 40 * DAY }], { + months: 3, + minWordsPerMonth: 150, + lang: 'en', + }); + const suppressed = report.timeline.filter((b) => b.suppressed); + expect(suppressed.length).toBeGreaterThan(0); + expect(suppressed[0].lexicalDiversity.median).toBeNull(); + expect(suppressed[0].suppressReason).toContain('fewer than'); + }); + + it('emits no raw text or word lists anywhere in the report', () => { + const report = analyzeTimeline(buildCorpus(), { + months: 3, + windowSize: 15, + minWordsPerMonth: 1, + lang: 'en', + }); + const json = JSON.stringify(report); + expect(json).not.toContain('hello'); + expect(json).not.toContain('because the weather'); + expect(report.privacy.rawUtterancesIncluded).toBe(false); + expect(report.privacy.wordListsIncluded).toBe(false); + expect(report.privacy.fringeWordFrequencyIncluded).toBe(false); + }); + + it('trends lexical diversity upward for an improving corpus', () => { + const report = analyzeTimeline(buildCorpus(), { + months: 3, + windowSize: 15, + minWordsPerMonth: 1, + lang: 'en', + }); + expect(report.trend.metric).toBe('lexicalDiversity.median'); + expect(report.trend.direction).toBe('up'); + expect(report.trend.delta!).toBeGreaterThan(0); + }); + + it('respects the trailing-months window (drops old utterances)', () => { + const now = Date.now(); + const report = analyzeTimeline( + [{ text: 'very old utterance here', timestampMs: now - 400 * DAY }], + { months: 3, minWordsPerMonth: 1, lang: 'en' } + ); + expect(report.timeline.length).toBe(0); + expect(report.overall.totalUtterances).toBe(0); + }); +}); From 0f0c9801fa9a195c4614a8cb6622512efa64a0f9 Mon Sep 17 00:00:00 2001 From: will wade Date: Thu, 13 Aug 2026 15:10:49 +0100 Subject: [PATCH 2/2] Refactor competence metrics: language-agnostic core, source-agnostic, no hardcoded lists Addresses review feedback: - No hardcoded word lists. Removed the inlined EN/NL preposition+conjunction sets and the morphology guard list. The core is now data-free; language resources (closed-class words, an inflection classifier) are injected via LanguageResources. Adding a language = provide resources, no core change. - Any language, no silent degradation. When a resource is missing, the affected measure is reported unavailable with a reason, and the report gains a support block + warnings list, so language gaps are explicit. - Source-agnostic. analyzeTimeline already took generic utterances; now historyEntriesToCompetenceUtterances adapts ANY HistoryEntry (Grid 3, Snap, OBF/OBFL, ...) into that stream. Added an OBF/OBFL test proving any speech-history source plugs in. - Removed non-null assertions; lint-clean. Semantic (MATTR-30) stays available for every language out of the box. --- src/utilities/analytics/competence.ts | 605 ++++++++------------------ src/utilities/analytics/history.ts | 24 + test/competence.test.ts | 205 ++++++--- 3 files changed, 356 insertions(+), 478 deletions(-) diff --git a/src/utilities/analytics/competence.ts b/src/utilities/analytics/competence.ts index 5fdd099..2148545 100644 --- a/src/utilities/analytics/competence.ts +++ b/src/utilities/analytics/competence.ts @@ -7,24 +7,32 @@ * - Frisch, Wade et al. (2026), "It's Complicated: On the Design and Evaluation * of AI-Powered AAC Interfaces", arXiv:2606.24854. * - * All functions here are pure and platform-agnostic (no I/O, no Bun/Node APIs). - * They compute only aggregate statistics and never return the raw text they are - * given. This makes them safe to run on-device and trivially unit-testable. + * DESIGN (read me): + * - **Source-agnostic.** The only input is `{ text, timestampMs }[]`. It does + * not know or care whether the speech history came from Grid 3, Snap, + * TouchChat, OBF/OBFL logs, or anything else. See `historyEntriesToCompetence* + * Utterances` (in history.ts) to adapt any `HistoryEntry[]` source. + * - **Language-agnostic core.** This module contains NO word lists. Language- + * specific resources (a closed-class word set, an inflection classifier) are + * INJECTED via `LanguageResources`. When a resource is missing for a + * language, the affected measure is reported as `unavailable` with a reason + * and a warning is raised — never silently wrong. + * - **Pure / no I/O.** No filesystem, no platform APIs. Runs anywhere (browser + * included) and emits only aggregate statistics (never the raw text). * * The four dimensions of linguistic competence (Light, 1989) and the measures we * use for each, following the AssistiveWare findings: * - * Semantic -> MATTR-30 (lexical diversity, the strongest overall measure) - * Syntactic -> MA-UPC-TWR-30 (preposition + conjunction diversity) - * Morphological -> MA-UMORPH-TLWR-30 (proxy; documented below) - * Phonological -> proportion of unique words correctly spelled (optional, weak) + * Semantic -> MATTR-30 lexical diversity (always available) + * Syntactic -> preposition/conjunction diversity (needs closedClassWords) + * Morphological -> inflected-form diversity (needs classifyInflection) + * Phonological -> proportion of unique words in a dictionary (needs a dictionary) * - * Key design choices, straight from the AssistiveWare paper: - * - Diversity beats density (density plateaus); we use diversity measures. - * - Moving-average windows of 30 words (Covington & McFall, 2010) make the - * measures insensitive to sample length and usable for tiny language samples. - * - MLU is reported only as a *distribution* and never as a headline, because - * it conflates linguistic, operational, strategic and social competence. + * All diversity measures use 30-word moving-average windows (Covington & McFall, + * 2010), making them sample-length independent and usable for the tiny, highly + * variable samples typical of AAC. MLU is intentionally NOT a headline (it + * conflates linguistic/operational/strategic/social competence in AAC); it is + * reported only as a distribution. */ /** A single spoken utterance with a production timestamp (epoch ms). */ @@ -36,12 +44,36 @@ export interface CompetenceUtterance { /** A tokenised word stream produced in chronological order. */ export type WordStream = string[]; -/** Options shared by the dimension measures. */ +/** Coarse inflection category used by the (injected) morphology classifier. */ +export type InflectionCategory = + | 'base' + | 'plural' + | 'possessive' + | 'past' + | 'progressive' + | 'comparative' + | 'superlative' + | 'adverb'; + +/** + * Language-specific resources, injected by the caller. Providing none leaves the + * language-specific measures unavailable (with explicit warnings) — the semantic + * measure still works for any language. + */ +export interface LanguageResources { + /** Closed-class words (prepositions, conjunctions, ...) for syntactic diversity. */ + closedClassWords?: Set; + /** Maps a lowercased word to an inflection category for morphological diversity. */ + classifyInflection?: (word: string) => InflectionCategory; +} + export interface DiversityOptions { /** Moving-average window size in words. The papers use 30. */ windowSize?: number; - /** BCP-47-ish language code, e.g. "en-GB", "nl-BE". Only the primary subtag matters. */ - lang?: string; + /** Closed-class word set (for the syntactic measure). */ + closedClassWords?: Set; + /** Inflection classifier (for the morphological measure). */ + classifyInflection?: (word: string) => InflectionCategory; } export interface DiversityResult { @@ -52,6 +84,8 @@ export interface DiversityResult { nWindows: number; /** The window size used. */ windowSize: number; + /** Present when the measure could not be computed (e.g. missing language data). */ + unavailable?: string; } /* ------------------------------------------------------------------ * @@ -100,26 +134,18 @@ export function tokenize(text: string): WordStream { const out: string[] = []; let m: RegExpExecArray | null; TOKEN_RE.lastIndex = 0; - while ((m = TOKENReexec()) !== null) { + while ((m = TOKEN_RE.exec(text)) !== null) { let tok = m[0].toLowerCase(); - // Strip stray leading/trailing apostrophes introduced by quotes. tok = tok.replace(/^['’]+|['’]+$/g, ''); if (tok.length > 0) out.push(tok); } return out; - - function TOKENReexec(): RegExpExecArray | null { - return TOKEN_RE.exec(text); - } } /* ------------------------------------------------------------------ * - * Semantic competence: lexical diversity (MATTR-30) + * Semantic competence: lexical diversity (MATTR-30) — always available * ------------------------------------------------------------------ */ -/** - * Compute a type-token ratio for one segment of words. - */ function segmentTTR(segment: WordStream): number { if (segment.length === 0) return 0; return new Set(segment).size / segment.length; @@ -131,15 +157,12 @@ function segmentTTR(segment: WordStream): number { * Slides a fixed-size window across the word stream and computes the TTR for * each window. The median across windows is sample-length independent, which is * exactly why it is preferred over plain TTR for highly variable AAC samples. - * - * Returns the headline `median` plus mean and window count. */ export function movingAverageTTR(words: WordStream, windowSize = 30): DiversityResult { const n = words.length; const w = Math.max(1, Math.floor(windowSize)); if (n === 0) return { median: null, mean: null, nWindows: 0, windowSize: w }; - // Fewer words than one window: compute TTR over the whole sample. if (n < w) { const v = segmentTTR(words); return { median: v, mean: v, nWindows: 1, windowSize: w }; @@ -163,202 +186,31 @@ export function lexicalDiversity(words: WordStream, windowSize = 30): DiversityR } /* ------------------------------------------------------------------ * - * Syntactic competence: preposition + conjunction diversity (MA-UPC-TWR-30) + * Syntactic competence: closed-class diversity (MA-UPC-TWR-30) * ------------------------------------------------------------------ */ -/** English prepositions (closed set, indicator of relational syntax). */ -const EN_PREPOSITIONS = new Set([ - 'about', - 'above', - 'across', - 'after', - 'against', - 'along', - 'alongside', - 'amid', - 'among', - 'amongst', - 'around', - 'as', - 'at', - 'atop', - 'before', - 'behind', - 'below', - 'beneath', - 'beside', - 'besides', - 'between', - 'beyond', - 'by', - 'despite', - 'down', - 'during', - 'except', - 'for', - 'from', - 'in', - 'inside', - 'into', - 'near', - 'of', - 'off', - 'on', - 'onto', - 'opposite', - 'out', - 'outside', - 'over', - 'past', - 'per', - 'plus', - 'round', - 'since', - 'than', - 'through', - 'throughout', - 'till', - 'to', - 'toward', - 'towards', - 'under', - 'underneath', - 'unlike', - 'until', - 'up', - 'upon', - 'versus', - 'via', - 'with', - 'within', - 'without', -]); - -/** English conjunctions (closed set, indicator of clausal complexity). */ -const EN_CONJUNCTIONS = new Set([ - 'and', - 'but', - 'or', - 'nor', - 'yet', - 'so', - 'although', - 'because', - 'considering', - 'if', - 'lest', - 'once', - 'provided', - 'since', - 'than', - 'that', - 'though', - 'unless', - 'until', - 'when', - 'whenever', - 'where', - 'whereas', - 'wherever', - 'whether', - 'while', - 'whilst', - 'why', - 'neither', - 'either', - 'both', -]); - -/** Dutch prepositions. */ -const NL_PREPOSITIONS = new Set([ - 'aan', - 'achter', - 'bij', - 'door', - 'tijdens', - 'in', - 'boven', - 'langs', - 'met', - 'na', - 'naar', - 'om', - 'onder', - 'op', - 'over', - 'rond', - 'per', - 'sinds', - 'tegen', - 'uit', - 'van', - 'voor', - 'tot', - 'tussen', - 'binnen', - 'buiten', - 'behalve', - 'wegens', - 'krachtens', - 'volgens', -]); - -/** Dutch conjunctions. */ -const NL_CONJUNCTIONS = new Set([ - 'en', - 'of', - 'maar', - 'want', - 'dus', - 'omdat', - 'toen', - 'voordat', - 'nadat', - 'terwijl', - 'indien', - 'mits', - 'tenzij', - 'hoewel', - 'ofschoon', - 'zodra', - 'zolang', - 'als', - 'dat', - 'noch', -]); - -/** - * Get the closed-set (prepositions ∪ conjunctions) for a language, or null if - * unsupported. Callers should skip the syntactic measure when this is null. - */ -export function getClosedClassSet(lang?: string): Set | null { - const primary = (lang || 'en').split(/[-_]/)[0].toLowerCase(); - switch (primary) { - case 'en': - return new Set([...EN_PREPOSITIONS, ...EN_CONJUNCTIONS]); - case 'nl': - return new Set([...NL_PREPOSITIONS, ...NL_CONJUNCTIONS]); - default: - return null; - } -} - /** - * Syntactic diversity: MA-UPC-TWR-30. + * Closed-class diversity (generalises AssistiveWare's MA-UPC-TWR-30). * - * For each 30-word window, compute the type-token ratio restricted to the - * closed-class words that indicate syntactic complexity (prepositions and - * conjunctions). Windows containing none of these are skipped (no syntactic - * signal). The median across contributing windows is reported. + * For each window, compute the type-token ratio restricted to the supplied + * closed-class words (prepositions + conjunctions in the original paper). Windows + * containing none are skipped. The caller supplies the set via + * `closedClassWords`, so this works for any language without hardcoding here. */ export function syntacticDiversity( words: WordStream, options: DiversityOptions = {} ): DiversityResult { const w = Math.max(1, Math.floor(options.windowSize ?? 30)); - const closed = getClosedClassSet(options.lang); - if (!closed) { - return { median: null, mean: null, nWindows: 0, windowSize: w }; + const closed = options.closedClassWords; + if (!closed || closed.size === 0) { + return { + median: null, + mean: null, + nWindows: 0, + windowSize: w, + unavailable: 'no closed-class word data provided for this language', + }; } const n = words.length; @@ -366,9 +218,9 @@ export function syntacticDiversity( const values: number[] = []; const scan = (segment: WordStream): void => { - const upc = segment.filter((tok) => closed.has(tok)); - if (upc.length > 0) { - values.push(new Set(upc).size / upc.length); + const cc = segment.filter((tok) => closed.has(tok)); + if (cc.length > 0) { + values.push(new Set(cc).size / cc.length); } }; @@ -381,171 +233,52 @@ export function syntacticDiversity( } if (values.length === 0) { - return { median: null, mean: null, nWindows: 0, windowSize: w }; + return { + median: null, + mean: null, + nWindows: 0, + windowSize: w, + unavailable: 'no closed-class words found in the sample', + }; } - return { median: median(values), mean: mean(values), nWindows: values.length, windowSize: w }; + return { + median: median(values), + mean: mean(values), + nWindows: values.length, + windowSize: w, + }; } /* ------------------------------------------------------------------ * - * Morphological competence: MA-UMORPH-TLWR-30 (documented proxy) + * Morphological competence: inflected-form diversity (MA-UMORPH-TLWR-30 proxy) * ------------------------------------------------------------------ */ -export type InflectionCategory = - | 'base' - | 'plural' - | 'possessive' - | 'past' - | 'progressive' - | 'comparative' - | 'superlative' - | 'adverb'; - -const BASE_GUARD = new Set([ - // Words that look inflected but are base forms; avoids the worst false hits. - 'is', - 'as', - 'was', - 'has', - 'his', - 'its', - 'us', - 'bus', - 'gas', - 'yes', - 'this', - 'miss', - 'loss', - 'boss', - 'less', - 'dress', - 'guess', - 'press', - 'cross', - 'class', - 'glass', - 'pass', - 'mass', - 'ass', - 'bed', - 'red', - 'fed', - 'led', - 'wed', - 'shed', - 'ted', - 'bred', - 'her', - 'per', - 'der', - 'fer', - 'ring', - 'sing', - 'king', - 'thing', - 'bring', - 'long', - 'wing', - 'spring', - 'string', - 'sting', - 'cling', - 'swing', - 'fling', - 'slang', - 'best', - 'rest', - 'test', - 'west', - 'nest', - 'chest', - 'pest', - 'vest', - 'quest', - 'fest', - 'lest', - 'beast', - 'feast', - 'breast', - 'fly', - 'ply', - 'sly', - 'ally', - 'rally', - 'supply', - 'reply', - 'apply', - 'comply', - 'rely', - 'family', - 'only', - 'lonely', - 'likely', - 'lovely', - 'silly', - 'holy', - 'uly', - 'fully', -]); - -/** - * Crude, dependency-free inflection classifier for English. - * - * This is intentionally a *heuristic proxy*: it identifies the morphological - * operation encoded by a word's suffix so we can measure how diverse the user's - * morphological production is. It is NOT a lemmatiser. False positives/negatives - * are expected and documented; the measure is used comparatively across time - * bins, not as an absolute clinical score. - */ -export function classifyInflection(word: string): InflectionCategory { - const w = word.toLowerCase(); - if (w.length < 3) return 'base'; - - // Possessive first (apostrophe forms): "dad's", "dogs'". - if (w.endsWith("'s") || w.endsWith("s'")) return 'possessive'; - - if (BASE_GUARD.has(w)) return 'base'; - - if (w.endsWith('ing') && w.length > 4) return 'progressive'; - if (w.endsWith('est') && w.length > 4) return 'superlative'; - if (w.endsWith('ies') && w.length > 3) return 'plural'; // berries, stories - if (w.endsWith('ied') && w.length > 3) return 'past'; // carried - if (w.endsWith('ed') && w.length > 3 && !w.endsWith('eed')) return 'past'; - if (w.endsWith('er') && w.length > 3) return 'comparative'; - if (w.endsWith('ly') && w.length > 3) return 'adverb'; - if (w.endsWith('es') && w.length > 3) return 'plural'; - if ( - w.endsWith('s') && - !w.endsWith('ss') && - !w.endsWith('us') && - !w.endsWith('is') && - w.length > 2 - ) { - return 'plural'; - } - return 'base'; -} - /** * Morphological diversity (proxy for MA-UMORPH-TLWR-30). * - * Within each 30-word window, look at the words carrying a morphological - * operation (any category other than "base") and compute the type-token ratio - * over their *surface forms*. A higher value means the user is producing a - * broader, less-repetitive range of inflected forms locally. Windows with no - * inflected words are skipped. + * Within each window, words the supplied classifier marks as inflected (any + * category other than "base") contribute their surface forms to a type-token + * ratio. Windows with no inflected words are skipped. The classifier is injected + * (`classifyInflection`) so the heuristic lives with the caller, per language. * * Caveat (per AssistiveWare): for symbol-supported AAC, pre-stored morphology - * buttons (e.g. "finished", "all done", "is") heavily affect this measure, so - * interpret trends rather than absolutes. + * buttons ("finished", "is", ...) heavily affect this measure — interpret trends + * rather than absolutes. */ export function morphologicalDiversity( words: WordStream, options: DiversityOptions = {} ): DiversityResult { const w = Math.max(1, Math.floor(options.windowSize ?? 30)); - const lang = (options.lang || 'en').split(/[-_]/)[0].toLowerCase(); - if (lang !== 'en') { - return { median: null, mean: null, nWindows: 0, windowSize: w }; + const classify = options.classifyInflection; + if (!classify) { + return { + median: null, + mean: null, + nWindows: 0, + windowSize: w, + unavailable: 'no inflection classifier provided for this language', + }; } const n = words.length; @@ -553,7 +286,7 @@ export function morphologicalDiversity( const values: number[] = []; const scan = (segment: WordStream): void => { - const inflected = segment.filter((tok) => classifyInflection(tok) !== 'base'); + const inflected = segment.filter((tok) => classify(tok) !== 'base'); if (inflected.length > 0) { values.push(new Set(inflected).size / inflected.length); } @@ -568,9 +301,20 @@ export function morphologicalDiversity( } if (values.length === 0) { - return { median: null, mean: null, nWindows: 0, windowSize: w }; + return { + median: null, + mean: null, + nWindows: 0, + windowSize: w, + unavailable: 'no inflected word forms found in the sample', + }; } - return { median: median(values), mean: mean(values), nWindows: values.length, windowSize: w }; + return { + median: median(values), + mean: mean(values), + nWindows: values.length, + windowSize: w, + }; } /* ------------------------------------------------------------------ * @@ -579,10 +323,8 @@ export function morphologicalDiversity( /** * Proportion of unique alphabetic words present in the supplied dictionary. - * - * Focuses on *unique* words so it is not skewed by repetition or repeated - * misspellings. Pass a dictionary Set (e.g. loaded from a hunspell/aspell - * wordlist). Returns null if no dictionary is provided. + * Unique words only, so it is not skewed by repetition or repeated misspellings. + * Returns null (unavailable) if no dictionary is provided. */ export function spellingValidity(words: WordStream, dictionary?: Set): number | null { if (!dictionary || dictionary.size === 0) return null; @@ -590,7 +332,6 @@ export function spellingValidity(words: WordStream, dictionary?: Set): n let checked = 0; let correct = 0; for (const tok of unique) { - // Only consider alphabetic tokens of reasonable length. if (!/^\p{L}{2,}$/u.test(tok)) continue; checked++; if (dictionary.has(tok)) correct++; @@ -599,7 +340,7 @@ export function spellingValidity(words: WordStream, dictionary?: Set): n } /* ------------------------------------------------------------------ * - * Activity / engagement statistics (multidimensional, distributional) + * Activity / engagement statistics * ------------------------------------------------------------------ */ export interface DistributionStats { @@ -644,7 +385,7 @@ export function summarizeActivity(utterances: CompetenceUtterance[]): ActivitySt return { utterances: utterances.length, words: totalWords, - uniqueWords: 0, // filled by caller from the ordered stream for accuracy + uniqueWords: 0, activeDays: days.size, wordsPerUtterance: distribution(wpu), }; @@ -664,9 +405,9 @@ export interface MonthBin { wordsPerUtterance: DistributionStats; /** Semantic — the headline measure. */ lexicalDiversity: DiversityResult; - /** Syntactic. null if language unsupported. */ + /** Syntactic. null/unavailable when no closed-class data for the language. */ syntacticDiversity: DiversityResult; - /** Morphological (proxy). null if language unsupported. */ + /** Morphological (proxy). null/unavailable when no classifier for the language. */ morphologicalDiversity: DiversityResult; /** Phonological — only when a dictionary is supplied. */ spellingValidity: number | null; @@ -686,16 +427,12 @@ export interface TrendResult { direction: 'up' | 'down' | 'flat' | 'unknown'; } -/** - * Structural / effort summary of the user's AAC pageset (gridset). - * - * Computed from the existing MetricsCalculator so competence numbers can be read - * alongside the vocabulary design that shapes them (grid size, vocabulary size, - * effort, prediction) — the AssistiveWare paper stresses that setup affects both - * competence and its measurement. Contains NO word labels (privacy). - */ +export interface DimensionSupport { + available: boolean; + reason?: string; +} + export interface PagesetSummary { - /** Hashed label unless the caller opts in to revealing the file name. */ label: string; gridsetIncluded: boolean; analysisVersion?: string; @@ -703,30 +440,16 @@ export interface PagesetSummary { totalButtons: number; totalWords: number; grid: { rows: number; columns: number }; - /** Effort distribution across all scored buttons (lower = easier). */ effort: DistributionStats; hasDynamicPrediction: boolean; spellingEffort: { base: number | null; perLetter: number | null }; - /** Present if the gridset could not be loaded/analysed. */ error?: string; } -/** - * Privacy-safe system-configuration context for the user (Grid 3 UserSettings). - * - * These are NOT chat content — they describe how the system is set up, which is - * essential for interpreting the competence numbers. Notably `onlineAiToolsOptIn` - * records whether the user has enabled the vendor's online AI tools, letting - * competence trends be read against whether AI assistance was even active. - */ export interface UserSettingsSummary { - /** Vocabulary/gridset name configured as the startup set (a product name). */ startupGridSet: string | null; - /** Whether the user opted in to the vendor's online AI tools. */ onlineAiToolsOptIn: boolean | null; - /** Enabled access method names (e.g. Touch, Pointer, EyeGaze, Switch). */ accessMethods: string[]; - /** Counts of user-created personalisation entries (aggregate across languages). */ personalisation: { pronunciations: number; capitalisations: number; @@ -768,6 +491,16 @@ export interface CompetenceReport { }; timeline: MonthBin[]; trend: TrendResult; + /** Per-dimension availability for the detected language (no silent degradation). */ + support: { + lang: string; + semantic: DimensionSupport; + syntactic: DimensionSupport; + morphological: DimensionSupport; + phonological: DimensionSupport; + }; + /** Human-readable notes about anything skipped or approximate. */ + warnings: string[]; /** Structural metrics for the user's default gridset (null if unavailable). */ pageset?: PagesetSummary | null; /** System-configuration context (access method, AI opt-in, startup gridset). */ @@ -779,12 +512,14 @@ export interface TimelineOptions { months?: number; /** Moving-average window. Default 30. */ windowSize?: number; - /** Language code. Default "en". */ + /** Language code. Default "en". Used for reporting only. */ lang?: string; /** Months with fewer than this many words are flagged suppressed. Default 150. */ minWordsPerMonth?: number; /** Optional dictionary Set for the spelling measure. */ dictionary?: Set; + /** Language-specific resources (closed-class words, inflection classifier). */ + resources?: LanguageResources; /** Epoch ms for "now". Defaults to Date.now(). Mainly for tests. */ now?: number; /** Platform label for the report. Default "Grid3". */ @@ -794,9 +529,6 @@ export interface TimelineOptions { dbPathIncluded?: boolean; } -/** - * Build a YYYY-MM key from epoch ms in local time. - */ function monthKey(timestampMs: number): string { const d = new Date(timestampMs); const y = d.getFullYear(); @@ -804,10 +536,6 @@ function monthKey(timestampMs: number): string { return `${y}-${m}`; } -/** - * Weighted linear-regression slope. Weights account for uneven sample sizes so - * that a noisy low-volume month cannot dominate the trend. - */ function weightedSlope(points: Array<{ x: number; y: number; w: number }>): number | null { const usable = points.filter((p) => p.y !== null && isFinite(p.y)); if (usable.length < 2) return null; @@ -834,7 +562,8 @@ function weightedSlope(points: Array<{ x: number; y: number; w: number }>): numb * * Utterances are filtered to the trailing `months` window, binned by calendar * month, and each bin is scored on the four competence dimensions plus activity. - * The headline lexical-diversity trend across months is summarised. + * Language-specific measures require matching `resources`; missing resources are + * reported under `support` and `warnings` rather than silently dropped. * * No raw text, word list, or fringe-vocabulary frequency is included in the * returned report — only aggregate statistics. @@ -848,6 +577,7 @@ export function analyzeTimeline( const lang = options.lang ?? 'en'; const minWordsPerMonth = Math.max(0, Math.floor(options.minWordsPerMonth ?? 150)); const dictionary = options.dictionary; + const resources = options.resources ?? {}; const now = options.now ?? Date.now(); // ---- Filter to the trailing N months ---------------------------------- @@ -871,12 +601,8 @@ export function analyzeTimeline( const timeline: MonthBin[] = []; for (const key of sortedKeys) { - const monthUtts = bins - .get(key)! - .slice() - .sort((a, b) => a.timestampMs - b.timestampMs); + const monthUtts = (bins.get(key) ?? []).slice().sort((a, b) => a.timestampMs - b.timestampMs); - // Ordered word stream for the sliding-window measures. const stream: WordStream = []; for (const u of monthUtts) stream.push(...tokenize(u.text)); @@ -893,10 +619,13 @@ export function analyzeTimeline( : lexicalDiversity(stream, windowSize); const syn = suppressed ? { median: null, mean: null, nWindows: 0, windowSize } - : syntacticDiversity(stream, { windowSize, lang }); + : syntacticDiversity(stream, { windowSize, closedClassWords: resources.closedClassWords }); const mor = suppressed ? { median: null, mean: null, nWindows: 0, windowSize } - : morphologicalDiversity(stream, { windowSize, lang }); + : morphologicalDiversity(stream, { + windowSize, + classifyInflection: resources.classifyInflection, + }); const spell = suppressed ? null : spellingValidity(stream, dictionary); timeline.push({ @@ -915,6 +644,32 @@ export function analyzeTimeline( }); } + // ---- Support + warnings (no silent language degradation) -------------- + const warnings: string[] = []; + const hasCC = !!resources.closedClassWords && resources.closedClassWords.size > 0; + const hasMorph = !!resources.classifyInflection; + if (!hasCC) { + warnings.push( + `Syntactic diversity unavailable: no closed-class word data for language '${lang}'. ` + + `Provide resources.closedClassWords to enable it.` + ); + } + if (!hasMorph) { + warnings.push( + `Morphological diversity unavailable: no inflection classifier for language '${lang}'. ` + + `Provide resources.classifyInflection to enable it.` + ); + } + if (!dictionary) { + warnings.push('Spelling validity unavailable: no dictionary provided.'); + } + const suppCount = timeline.filter((b) => b.suppressed).length; + if (suppCount > 0) { + warnings.push( + `${suppCount} month(s) suppressed for having fewer than ${minWordsPerMonth} words.` + ); + } + // ---- Trend on the headline lexical-diversity median ------------------- const points: Array<{ x: number; y: number; w: number }> = []; for (let i = 0; i < timeline.length; i++) { @@ -949,6 +704,9 @@ export function analyzeTimeline( const totalUtts = timeline.reduce((s, b) => s + b.utterances, 0); const totalWords = timeline.reduce((s, b) => s + b.words, 0); + const dim = (available: boolean, reason?: string): DimensionSupport => + available ? { available: true } : { available: false, reason }; + return { schema: 'aac-competence-report/v1', generatedAt: new Date(now).toISOString(), @@ -982,7 +740,7 @@ export function analyzeTimeline( totalUtterances: totalUtts, totalWords: totalWords, monthsCovered: timeline.length, - monthsSuppressed: timeline.filter((b) => b.suppressed).length, + monthsSuppressed: suppCount, }, timeline, trend: { @@ -993,5 +751,16 @@ export function analyzeTimeline( delta, direction, }, + support: { + lang, + semantic: dim(true), + syntactic: dim(hasCC, hasCC ? undefined : 'no closed-class data for this language'), + morphological: dim( + hasMorph, + hasMorph ? undefined : 'no inflection classifier for this language' + ), + phonological: dim(!!dictionary, dictionary ? undefined : 'no dictionary provided'), + }, + warnings, }; } diff --git a/src/utilities/analytics/history.ts b/src/utilities/analytics/history.ts index 682abcf..fd76165 100644 --- a/src/utilities/analytics/history.ts +++ b/src/utilities/analytics/history.ts @@ -13,6 +13,7 @@ import { SnapUserInfo, } from '../../processors/snap/helpers'; import { AACSemanticCategory, AACSemanticIntent } from '../../core/treeStructure'; +import type { CompetenceUtterance } from './competence'; export type HistorySource = 'Grid' | 'Snap' | 'OBL' | string; @@ -55,6 +56,29 @@ export interface HistoryEntry { export { dotNetTicksToDate }; +/** + * Adapt any `HistoryEntry[]` (Grid 3, Snap, OBF/OBFL logs, ...) into the generic + * utterance stream consumed by the linguistic-competence engine + * (`analyzeTimeline`). Each occurrence of each phrase becomes one utterance, + * timestamped by its occurrence time. This keeps the competence metrics fully + * source-agnostic: anything the library can read as history can be analysed. + */ +export function historyEntriesToCompetenceUtterances( + entries: HistoryEntry[] +): CompetenceUtterance[] { + const out: CompetenceUtterance[] = []; + for (const e of entries) { + const text = e.content; + if (!text || text.trim().length === 0) continue; + const occs = e.occurrences ?? []; + for (const occ of occs) { + if (!occ.timestamp) continue; + out.push({ text, timestampMs: occ.timestamp.getTime() }); + } + } + return out; +} + export interface BatonExportMetadata { timestamp: string; latitude?: number | null; diff --git a/test/competence.test.ts b/test/competence.test.ts index 4bb466d..a053f27 100644 --- a/test/competence.test.ts +++ b/test/competence.test.ts @@ -1,8 +1,6 @@ import { describe, expect, it } from '@jest/globals'; import { analyzeTimeline, - classifyInflection, - getClosedClassSet, lexicalDiversity, morphologicalDiversity, movingAverageTTR, @@ -11,7 +9,48 @@ import { syntacticDiversity, tokenize, type CompetenceUtterance, + type InflectionCategory, } from '../src/utilities/analytics/competence'; +import { + historyEntriesToCompetenceUtterances, + type HistoryEntry, +} from '../src/utilities/analytics/history'; + +/* ------------------------------------------------------------------ * + * Inline language fixtures. + * + * The library core ships NO word lists, so tests inject a tiny closed-class + * set + inflection classifier. Real callers pass curated language resources. + * ------------------------------------------------------------------ */ + +const FIXTURE_CC = new Set([ + 'and', + 'but', + 'or', + 'because', + 'if', + 'so', + 'although', + 'to', + 'for', + 'with', + 'in', + 'on', + 'at', + 'of', + 'from', + 'by', + 'under', + 'over', +]); + +function fixtureClassify(w: string): InflectionCategory { + if (w.endsWith('ing')) return 'progressive'; + if (w.endsWith('ed')) return 'past'; + if (w.endsWith("'s")) return 'possessive'; + if (w.endsWith('s')) return 'plural'; + return 'base'; +} describe('competence / tokenize', () => { it('lowercases and keeps apostrophes inside words', () => { @@ -43,7 +82,6 @@ describe('competence / movingAverageTTR (MATTR)', () => { it('treats a short sample as a single window TTR', () => { const r = movingAverageTTR(['a', 'b', 'c'], 30); - // 3 unique / 3 = 1.0 expect(r.median).toBeCloseTo(1.0); expect(r.nWindows).toBe(1); }); @@ -54,32 +92,28 @@ describe('competence / movingAverageTTR (MATTR)', () => { 'sun moon star tree leaf rock river hill cloud rain wind' ).split(/\s+/); const repetitive = Array.from({ length: 40 }, () => 'yes'); - expect(lexicalDiversity(varied, 30).median!).toBeGreaterThan( - lexicalDiversity(repetitive, 30).median! - ); + const v = lexicalDiversity(varied, 30).median; + const r = lexicalDiversity(repetitive, 30).median; + expect(v).not.toBeNull(); + expect(r).not.toBeNull(); + if (v !== null && r !== null) expect(v).toBeGreaterThan(r); // Repetitive single-word stream has TTR ~ 1/30 within windows. - expect(lexicalDiversity(repetitive, 30).median).toBeCloseTo(1 / 30, 5); + expect(r).toBeCloseTo(1 / 30, 5); }); - it('MATTR is sample-length insensitive (more repetition does not inflate it)', () => { + it('MATTR is sample-length insensitive', () => { const base = 'a b c d e f g h i j k l m n o'.split(/\s+/); // 15 unique const doubled = [...base, ...base]; // same words repeated - // With window 15, repeated unique set still yields TTR 1.0 per window. expect(movingAverageTTR(base, 15).median).toBeCloseTo(1.0); expect(movingAverageTTR(doubled, 15).median).toBeCloseTo(1.0); }); }); -describe('competence / syntacticDiversity (MA-UPC-TWR-30)', () => { - it('returns null median for a language with no closed-class set', () => { - const r = syntacticDiversity(['and', 'but'], { lang: 'xx', windowSize: 30 }); +describe('competence / syntacticDiversity (closed-class, injected)', () => { + it('is unavailable with no closed-class data (no silent degradation)', () => { + const r = syntacticDiversity(['and', 'but'], { windowSize: 30 }); expect(r.median).toBeNull(); - }); - - it('detects prepositions and conjunctions in English', () => { - expect(getClosedClassSet('en-GB')!.has('because')).toBe(true); - expect(getClosedClassSet('en')!.has('under')).toBe(true); - expect(getClosedClassSet('en')!.has('banana')).toBe(false); + expect(r.unavailable).toBeDefined(); }); it('scores a window rich in varied connectors higher than one using only "and"', () => { @@ -88,50 +122,44 @@ describe('competence / syntacticDiversity (MA-UPC-TWR-30)', () => { /\s+/ ); const onlyAnd = Array.from({ length: 40 }, (_, i) => (i % 2 === 0 ? 'and' : 'cat')); - const r1 = syntacticDiversity(rich, { lang: 'en', windowSize: 15 }); - const r2 = syntacticDiversity(onlyAnd, { lang: 'en', windowSize: 15 }); - expect(r1.median!).toBeGreaterThan(r2.median!); - // Only "and" repeated ~8x per window -> distinct/total = 1/8. - expect(r2.median!).toBeLessThan(0.2); + const r1 = syntacticDiversity(rich, { windowSize: 15, closedClassWords: FIXTURE_CC }); + const r2 = syntacticDiversity(onlyAnd, { windowSize: 15, closedClassWords: FIXTURE_CC }); + expect(r1.median).not.toBeNull(); + expect(r2.median).not.toBeNull(); + if (r1.median !== null && r2.median !== null) { + expect(r1.median).toBeGreaterThan(r2.median); + // Only "and" repeated ~8x per window -> distinct/total = 1/8. + expect(r2.median).toBeLessThan(0.2); + } }); - it('supports Dutch closed-class set', () => { - expect(getClosedClassSet('nl-BE')!.has('omdat')).toBe(true); - expect(getClosedClassSet('nl')!.has('onder')).toBe(true); + it('works for any language because the set is injected (Dutch example)', () => { + const nl = new Set(['en', 'of', 'maar', 'want', 'omdat', 'aan', 'in', 'op']); + const r = syntacticDiversity('ik wil graag naar buiten omdat het mooi weer is'.split(/\s+/), { + windowSize: 8, + closedClassWords: nl, + }); + expect(r.median).not.toBeNull(); }); }); -describe('competence / morphologicalDiversity (proxy)', () => { - it('returns null for non-English (documented limitation)', () => { - const r = morphologicalDiversity(['lopen', 'gelopen'], { lang: 'nl', windowSize: 5 }); +describe('competence / morphologicalDiversity (classifier injected)', () => { + it('is unavailable without a classifier', () => { + const r = morphologicalDiversity(['cats'], { windowSize: 5 }); expect(r.median).toBeNull(); + expect(r.unavailable).toBeDefined(); }); - it('classifies obvious inflections', () => { - expect(classifyInflection('running')).toBe('progressive'); - expect(classifyInflection('jumped')).toBe('past'); - expect(classifyInflection('biggest')).toBe('superlative'); - expect(classifyInflection('happily')).toBe('adverb'); - expect(classifyInflection('cats')).toBe('plural'); - expect(classifyInflection("dad's")).toBe('possessive'); - }); - - it('keeps guarded base words as base', () => { - expect(classifyInflection('is')).toBe('base'); - expect(classifyInflection('ring')).toBe('base'); - expect(classifyInflection('best')).toBe('base'); - }); - - it('scores a stream with varied morphology above a base-only stream', () => { - const morphed = 'cats running jumped faster biggest quickly dogs walked eating smaller'.split( - /\s+/ - ); - const base = 'cat run jump fast big quick dog walk eat small'.split(/\s+/); - const r1 = morphologicalDiversity(morphed, { lang: 'en', windowSize: 10 }); - // base-only stream has no inflected words -> null - const r2 = morphologicalDiversity(base, { lang: 'en', windowSize: 10 }); + it('scores inflected words above a base-only stream', () => { + const morphed = 'cats running jumped'.split(/\s+/); + const base = 'cat run jump'.split(/\s+/); + const r1 = morphologicalDiversity(morphed, { + windowSize: 3, + classifyInflection: fixtureClassify, + }); + const r2 = morphologicalDiversity(base, { windowSize: 3, classifyInflection: fixtureClassify }); expect(r1.median).not.toBeNull(); - expect(r2.median).toBeNull(); + expect(r2.median).toBeNull(); // no inflected words -> unavailable }); }); @@ -148,7 +176,7 @@ describe('competence / spellingValidity', () => { it('ignores non-alphabetic tokens', () => { const dict = new Set(['hello']); - expect(spellingValidity(['hello', '12', 'a'], dict)).toBeCloseTo(1.0, 5); // only "hello" checked + expect(spellingValidity(['hello', '12', 'a'], dict)).toBeCloseTo(1.0, 5); }); }); @@ -157,12 +185,12 @@ describe('competence / summarizeActivity', () => { const DAY = 86_400_000; const utts: CompetenceUtterance[] = [ { text: 'hello world', timestampMs: 0 }, - { text: 'good morning', timestampMs: DAY }, // next day - { text: 'bye', timestampMs: DAY + 1000 }, // same day + { text: 'good morning', timestampMs: DAY }, + { text: 'bye', timestampMs: DAY + 1000 }, ]; const a = summarizeActivity(utts); expect(a.utterances).toBe(3); - expect(a.words).toBe(5); // hello world good morning bye + expect(a.words).toBe(5); expect(a.activeDays).toBe(2); expect(a.wordsPerUtterance.median).toBe(2); expect(a.wordsPerUtterance.n).toBe(3); @@ -175,12 +203,10 @@ describe('competence / analyzeTimeline', () => { function buildCorpus(): CompetenceUtterance[] { const now = Date.now(); const utts: CompetenceUtterance[] = []; - // Two months ago: simple, repetitive vocabulary. const simpleMonth = now - 60 * DAY; for (let i = 0; i < 30; i++) { utts.push({ text: 'I want it yes', timestampMs: simpleMonth + i * 1000 }); } - // This month: richer, more syntactically complex vocabulary. for (let i = 0; i < 30; i++) { utts.push({ text: 'I think that we should go to the park because the weather is lovely although it might rain', @@ -196,6 +222,7 @@ describe('competence / analyzeTimeline', () => { windowSize: 15, minWordsPerMonth: 1, lang: 'en', + resources: { closedClassWords: FIXTURE_CC, classifyInflection: fixtureClassify }, }); expect(report.schema).toBe('aac-competence-report/v1'); expect(report.timeline.length).toBeGreaterThanOrEqual(1); @@ -205,6 +232,21 @@ describe('competence / analyzeTimeline', () => { } }); + it('reports support + warnings when language resources are missing', () => { + const report = analyzeTimeline(buildCorpus(), { + months: 3, + windowSize: 15, + minWordsPerMonth: 1, + lang: 'fr', // no resources provided -> syntactic/morphological unavailable + }); + expect(report.support.syntactic.available).toBe(false); + expect(report.support.morphological.available).toBe(false); + expect(report.support.semantic.available).toBe(true); + expect(report.warnings.some((w) => w.includes("'fr'"))).toBe(true); + // The measure itself reports unavailable in each bin. + expect(report.timeline[0].syntacticDiversity.unavailable).toBeDefined(); + }); + it('flags sparse months as suppressed and omits their diversity figures', () => { const now = Date.now(); const report = analyzeTimeline([{ text: 'hi', timestampMs: now - 40 * DAY }], { @@ -224,6 +266,7 @@ describe('competence / analyzeTimeline', () => { windowSize: 15, minWordsPerMonth: 1, lang: 'en', + resources: { closedClassWords: FIXTURE_CC }, }); const json = JSON.stringify(report); expect(json).not.toContain('hello'); @@ -239,10 +282,12 @@ describe('competence / analyzeTimeline', () => { windowSize: 15, minWordsPerMonth: 1, lang: 'en', + resources: { closedClassWords: FIXTURE_CC }, }); expect(report.trend.metric).toBe('lexicalDiversity.median'); expect(report.trend.direction).toBe('up'); - expect(report.trend.delta!).toBeGreaterThan(0); + expect(report.trend.delta).not.toBeNull(); + if (report.trend.delta !== null) expect(report.trend.delta).toBeGreaterThan(0); }); it('respects the trailing-months window (drops old utterances)', () => { @@ -255,3 +300,43 @@ describe('competence / analyzeTimeline', () => { expect(report.overall.totalUtterances).toBe(0); }); }); + +describe('competence / source-agnostic (OBF/OBFL via adapter)', () => { + it('analyses any HistoryEntry source, e.g. OBF/OBFL logs, identically to native utterances', () => { + const now = Date.now(); + // OBF/OBFL-style history: phrases with occurrences, source-tagged 'OBL'. + const obfEntries: HistoryEntry[] = [ + { + id: 'obf-1', + source: 'OBL', + content: 'I want a drink of water please', + occurrences: [ + { timestamp: new Date(now - 50 * 86_400_000) }, + { timestamp: new Date(now - 49 * 86_400_000) }, + ], + }, + { + id: 'obf-2', + source: 'OBL', + content: 'the cat sat on the mat because it was tired', + occurrences: [{ timestamp: new Date(now - 1000) }], + }, + ]; + + const utts = historyEntriesToCompetenceUtterances(obfEntries); + // Two occurrences of phrase 1 + one of phrase 2 = 3 utterances. + expect(utts.length).toBe(3); + + const report = analyzeTimeline(utts, { + months: 3, + windowSize: 10, + minWordsPerMonth: 1, + lang: 'en', + resources: { closedClassWords: FIXTURE_CC }, + }); + expect(report.source.platform).toBe('Grid3'); // default label; data came from OBF + expect(report.overall.totalUtterances).toBe(3); + // No raw phrase text leaks into the output. + expect(JSON.stringify(report)).not.toContain('drink of water'); + }); +});