diff --git a/src/analytics.ts b/src/analytics.ts index 39fffb8..ac3435c 100644 --- a/src/analytics.ts +++ b/src/analytics.ts @@ -6,3 +6,4 @@ */ export * from './utilities/analytics/history'; +export * from './utilities/analytics/competence'; diff --git a/src/utilities/analytics/competence.ts b/src/utilities/analytics/competence.ts new file mode 100644 index 0000000..2148545 --- /dev/null +++ b/src/utilities/analytics/competence.ts @@ -0,0 +1,766 @@ +/** + * Linguistic Competence Metrics + * + * Privacy-preserving analysis of AAC *spoken output* (phrase history), based on: + * - Niemeijer, Sheldon & Hillary Zisk (2025), "Measuring AAC user linguistic + * competence: A novel approach", AssistiveWare (Communication Matters handout). + * - Frisch, Wade et al. (2026), "It's Complicated: On the Design and Evaluation + * of AI-Powered AAC Interfaces", arXiv:2606.24854. + * + * DESIGN (read me): + * - **Source-agnostic.** The only input is `{ text, timestampMs }[]`. It does + * not know or care whether the speech history came from Grid 3, Snap, + * TouchChat, OBF/OBFL logs, or anything else. See `historyEntriesToCompetence* + * Utterances` (in history.ts) to adapt any `HistoryEntry[]` source. + * - **Language-agnostic core.** This module contains NO word lists. Language- + * specific resources (a closed-class word set, an inflection classifier) are + * INJECTED via `LanguageResources`. When a resource is missing for a + * language, the affected measure is reported as `unavailable` with a reason + * and a warning is raised — never silently wrong. + * - **Pure / no I/O.** No filesystem, no platform APIs. Runs anywhere (browser + * included) and emits only aggregate statistics (never the raw text). + * + * The four dimensions of linguistic competence (Light, 1989) and the measures we + * use for each, following the AssistiveWare findings: + * + * Semantic -> MATTR-30 lexical diversity (always available) + * Syntactic -> preposition/conjunction diversity (needs closedClassWords) + * Morphological -> inflected-form diversity (needs classifyInflection) + * Phonological -> proportion of unique words in a dictionary (needs a dictionary) + * + * All diversity measures use 30-word moving-average windows (Covington & McFall, + * 2010), making them sample-length independent and usable for the tiny, highly + * variable samples typical of AAC. MLU is intentionally NOT a headline (it + * conflates linguistic/operational/strategic/social competence in AAC); it is + * reported only as a distribution. + */ + +/** A single spoken utterance with a production timestamp (epoch ms). */ +export interface CompetenceUtterance { + text: string; + timestampMs: number; +} + +/** A tokenised word stream produced in chronological order. */ +export type WordStream = string[]; + +/** Coarse inflection category used by the (injected) morphology classifier. */ +export type InflectionCategory = + | 'base' + | 'plural' + | 'possessive' + | 'past' + | 'progressive' + | 'comparative' + | 'superlative' + | 'adverb'; + +/** + * Language-specific resources, injected by the caller. Providing none leaves the + * language-specific measures unavailable (with explicit warnings) — the semantic + * measure still works for any language. + */ +export interface LanguageResources { + /** Closed-class words (prepositions, conjunctions, ...) for syntactic diversity. */ + closedClassWords?: Set; + /** Maps a lowercased word to an inflection category for morphological diversity. */ + classifyInflection?: (word: string) => InflectionCategory; +} + +export interface DiversityOptions { + /** Moving-average window size in words. The papers use 30. */ + windowSize?: number; + /** Closed-class word set (for the syntactic measure). */ + closedClassWords?: Set; + /** Inflection classifier (for the morphological measure). */ + classifyInflection?: (word: string) => InflectionCategory; +} + +export interface DiversityResult { + /** Median of the per-window values (the headline figure, per the paper). */ + median: number | null; + mean: number | null; + /** Number of windows that contributed a value (after any skipping). */ + nWindows: number; + /** The window size used. */ + windowSize: number; + /** Present when the measure could not be computed (e.g. missing language data). */ + unavailable?: string; +} + +/* ------------------------------------------------------------------ * + * Small statistics helpers + * ------------------------------------------------------------------ */ + +function mean(values: number[]): number | null { + if (values.length === 0) return null; + let sum = 0; + for (const v of values) sum += v; + return sum / values.length; +} + +function median(values: number[]): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; +} + +function quantile(values: number[], q: number): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + const pos = (sorted.length - 1) * q; + const base = Math.floor(pos); + const rest = pos - base; + if (sorted[base + 1] !== undefined) { + return sorted[base] + rest * (sorted[base + 1] - sorted[base]); + } + return sorted[base]; +} + +/* ------------------------------------------------------------------ * + * Tokenisation + * ------------------------------------------------------------------ */ + +const TOKEN_RE = /[\p{L}\p{N}]+(?:['’][\p{L}\p{N}]+)?/gu; + +/** + * Tokenise raw text into a lowercased word stream. + * Keeps intra-word apostrophes (don't, children's) but drops leading/trailing + * punctuation and pure whitespace. Accented characters are preserved (\p{L}). + */ +export function tokenize(text: string): WordStream { + if (!text) return []; + const out: string[] = []; + let m: RegExpExecArray | null; + TOKEN_RE.lastIndex = 0; + while ((m = TOKEN_RE.exec(text)) !== null) { + let tok = m[0].toLowerCase(); + tok = tok.replace(/^['’]+|['’]+$/g, ''); + if (tok.length > 0) out.push(tok); + } + return out; +} + +/* ------------------------------------------------------------------ * + * Semantic competence: lexical diversity (MATTR-30) — always available + * ------------------------------------------------------------------ */ + +function segmentTTR(segment: WordStream): number { + if (segment.length === 0) return 0; + return new Set(segment).size / segment.length; +} + +/** + * Moving-Average Type-Token Ratio (Covington & McFall, 2010). + * + * Slides a fixed-size window across the word stream and computes the TTR for + * each window. The median across windows is sample-length independent, which is + * exactly why it is preferred over plain TTR for highly variable AAC samples. + */ +export function movingAverageTTR(words: WordStream, windowSize = 30): DiversityResult { + const n = words.length; + const w = Math.max(1, Math.floor(windowSize)); + if (n === 0) return { median: null, mean: null, nWindows: 0, windowSize: w }; + + if (n < w) { + const v = segmentTTR(words); + return { median: v, mean: v, nWindows: 1, windowSize: w }; + } + + const values: number[] = []; + for (let i = 0; i <= n - w; i++) { + values.push(segmentTTR(words.slice(i, i + w))); + } + return { + median: median(values), + mean: mean(values), + nWindows: values.length, + windowSize: w, + }; +} + +/** Convenience: MATTR-30 lexical diversity (the semantic headline). */ +export function lexicalDiversity(words: WordStream, windowSize = 30): DiversityResult { + return movingAverageTTR(words, windowSize); +} + +/* ------------------------------------------------------------------ * + * Syntactic competence: closed-class diversity (MA-UPC-TWR-30) + * ------------------------------------------------------------------ */ + +/** + * Closed-class diversity (generalises AssistiveWare's MA-UPC-TWR-30). + * + * For each window, compute the type-token ratio restricted to the supplied + * closed-class words (prepositions + conjunctions in the original paper). Windows + * containing none are skipped. The caller supplies the set via + * `closedClassWords`, so this works for any language without hardcoding here. + */ +export function syntacticDiversity( + words: WordStream, + options: DiversityOptions = {} +): DiversityResult { + const w = Math.max(1, Math.floor(options.windowSize ?? 30)); + const closed = options.closedClassWords; + if (!closed || closed.size === 0) { + return { + median: null, + mean: null, + nWindows: 0, + windowSize: w, + unavailable: 'no closed-class word data provided for this language', + }; + } + + const n = words.length; + if (n === 0) return { median: null, mean: null, nWindows: 0, windowSize: w }; + + const values: number[] = []; + const scan = (segment: WordStream): void => { + const cc = segment.filter((tok) => closed.has(tok)); + if (cc.length > 0) { + values.push(new Set(cc).size / cc.length); + } + }; + + if (n < w) { + scan(words); + } else { + for (let i = 0; i <= n - w; i++) { + scan(words.slice(i, i + w)); + } + } + + if (values.length === 0) { + return { + median: null, + mean: null, + nWindows: 0, + windowSize: w, + unavailable: 'no closed-class words found in the sample', + }; + } + return { + median: median(values), + mean: mean(values), + nWindows: values.length, + windowSize: w, + }; +} + +/* ------------------------------------------------------------------ * + * Morphological competence: inflected-form diversity (MA-UMORPH-TLWR-30 proxy) + * ------------------------------------------------------------------ */ + +/** + * Morphological diversity (proxy for MA-UMORPH-TLWR-30). + * + * Within each window, words the supplied classifier marks as inflected (any + * category other than "base") contribute their surface forms to a type-token + * ratio. Windows with no inflected words are skipped. The classifier is injected + * (`classifyInflection`) so the heuristic lives with the caller, per language. + * + * Caveat (per AssistiveWare): for symbol-supported AAC, pre-stored morphology + * buttons ("finished", "is", ...) heavily affect this measure — interpret trends + * rather than absolutes. + */ +export function morphologicalDiversity( + words: WordStream, + options: DiversityOptions = {} +): DiversityResult { + const w = Math.max(1, Math.floor(options.windowSize ?? 30)); + const classify = options.classifyInflection; + if (!classify) { + return { + median: null, + mean: null, + nWindows: 0, + windowSize: w, + unavailable: 'no inflection classifier provided for this language', + }; + } + + const n = words.length; + if (n === 0) return { median: null, mean: null, nWindows: 0, windowSize: w }; + + const values: number[] = []; + const scan = (segment: WordStream): void => { + const inflected = segment.filter((tok) => classify(tok) !== 'base'); + if (inflected.length > 0) { + values.push(new Set(inflected).size / inflected.length); + } + }; + + if (n < w) { + scan(words); + } else { + for (let i = 0; i <= n - w; i++) { + scan(words.slice(i, i + w)); + } + } + + if (values.length === 0) { + return { + median: null, + mean: null, + nWindows: 0, + windowSize: w, + unavailable: 'no inflected word forms found in the sample', + }; + } + return { + median: median(values), + mean: mean(values), + nWindows: values.length, + windowSize: w, + }; +} + +/* ------------------------------------------------------------------ * + * Phonological competence: spelling validity (optional, weak) + * ------------------------------------------------------------------ */ + +/** + * Proportion of unique alphabetic words present in the supplied dictionary. + * Unique words only, so it is not skewed by repetition or repeated misspellings. + * Returns null (unavailable) if no dictionary is provided. + */ +export function spellingValidity(words: WordStream, dictionary?: Set): number | null { + if (!dictionary || dictionary.size === 0) return null; + const unique = new Set(words); + let checked = 0; + let correct = 0; + for (const tok of unique) { + if (!/^\p{L}{2,}$/u.test(tok)) continue; + checked++; + if (dictionary.has(tok)) correct++; + } + return checked === 0 ? null : correct / checked; +} + +/* ------------------------------------------------------------------ * + * Activity / engagement statistics + * ------------------------------------------------------------------ */ + +export interface DistributionStats { + median: number | null; + mean: number | null; + p25: number | null; + p75: number | null; + n: number; +} + +export interface ActivityStats { + utterances: number; + words: number; + /** Distinct tokens — vocabulary breadth (count only, never the words). */ + uniqueWords: number; + activeDays: number; + wordsPerUtterance: DistributionStats; +} + +function distribution(values: number[]): DistributionStats { + return { + median: median(values), + mean: mean(values), + p25: quantile(values, 0.25), + p75: quantile(values, 0.75), + n: values.length, + }; +} + +/** Compute engagement/activity stats for a set of utterances. */ +export function summarizeActivity(utterances: CompetenceUtterance[]): ActivityStats { + const days = new Set(); + let totalWords = 0; + const wpu: number[] = []; + for (const u of utterances) { + const toks = tokenize(u.text); + totalWords += toks.length; + wpu.push(toks.length); + const day = Math.floor(u.timestampMs / 86_400_000); + days.add(day); + } + return { + utterances: utterances.length, + words: totalWords, + uniqueWords: 0, + activeDays: days.size, + wordsPerUtterance: distribution(wpu), + }; +} + +/* ------------------------------------------------------------------ * + * Timeline analysis (the longitudinal engine) + * ------------------------------------------------------------------ */ + +export interface MonthBin { + /** Calendar month in local time, "YYYY-MM". */ + month: string; + utterances: number; + words: number; + uniqueWords: number; + activeDays: number; + wordsPerUtterance: DistributionStats; + /** Semantic — the headline measure. */ + lexicalDiversity: DiversityResult; + /** Syntactic. null/unavailable when no closed-class data for the language. */ + syntacticDiversity: DiversityResult; + /** Morphological (proxy). null/unavailable when no classifier for the language. */ + morphologicalDiversity: DiversityResult; + /** Phonological — only when a dictionary is supplied. */ + spellingValidity: number | null; + /** True when the month has too little data to trust the diversity figures. */ + suppressed: boolean; + suppressReason: string | null; +} + +export interface TrendResult { + metric: string; + /** Slope of the weighted linear regression, in metric units per month. */ + slopePerMonth: number | null; + firstHalf: number | null; + secondHalf: number | null; + /** secondHalf - firstHalf. */ + delta: number | null; + direction: 'up' | 'down' | 'flat' | 'unknown'; +} + +export interface DimensionSupport { + available: boolean; + reason?: string; +} + +export interface PagesetSummary { + label: string; + gridsetIncluded: boolean; + analysisVersion?: string; + totalBoards: number; + totalButtons: number; + totalWords: number; + grid: { rows: number; columns: number }; + effort: DistributionStats; + hasDynamicPrediction: boolean; + spellingEffort: { base: number | null; perLetter: number | null }; + error?: string; +} + +export interface UserSettingsSummary { + startupGridSet: string | null; + onlineAiToolsOptIn: boolean | null; + accessMethods: string[]; + personalisation: { + pronunciations: number; + capitalisations: number; + abbreviationExpansions: number; + smallWords: number; + }; +} + +export interface CompetenceReport { + schema: string; + generatedAt: string; + privacy: { + rawUtterancesIncluded: boolean; + wordListsIncluded: boolean; + fringeWordFrequencyIncluded: boolean; + minAggregationWindowDays: number; + notes: string[]; + }; + source: { + platform: string; + langCode?: string; + userLabel?: string; + dbPathIncluded: boolean; + }; + config: { + months: number; + windowSize: number; + lang: string; + minWordsPerMonth: number; + dictionaryProvided: boolean; + }; + overall: { + windowStart: string; + windowEnd: string; + totalUtterances: number; + totalWords: number; + monthsCovered: number; + monthsSuppressed: number; + }; + timeline: MonthBin[]; + trend: TrendResult; + /** Per-dimension availability for the detected language (no silent degradation). */ + support: { + lang: string; + semantic: DimensionSupport; + syntactic: DimensionSupport; + morphological: DimensionSupport; + phonological: DimensionSupport; + }; + /** Human-readable notes about anything skipped or approximate. */ + warnings: string[]; + /** Structural metrics for the user's default gridset (null if unavailable). */ + pageset?: PagesetSummary | null; + /** System-configuration context (access method, AI opt-in, startup gridset). */ + userSettings?: UserSettingsSummary | null; +} + +export interface TimelineOptions { + /** How many trailing months to analyse. Default 12. */ + months?: number; + /** Moving-average window. Default 30. */ + windowSize?: number; + /** Language code. Default "en". Used for reporting only. */ + lang?: string; + /** Months with fewer than this many words are flagged suppressed. Default 150. */ + minWordsPerMonth?: number; + /** Optional dictionary Set for the spelling measure. */ + dictionary?: Set; + /** Language-specific resources (closed-class words, inflection classifier). */ + resources?: LanguageResources; + /** Epoch ms for "now". Defaults to Date.now(). Mainly for tests. */ + now?: number; + /** Platform label for the report. Default "Grid3". */ + platform?: string; + userLabel?: string; + langCode?: string; + dbPathIncluded?: boolean; +} + +function monthKey(timestampMs: number): string { + const d = new Date(timestampMs); + const y = d.getFullYear(); + const m = String(d.getMonth() + 1).padStart(2, '0'); + return `${y}-${m}`; +} + +function weightedSlope(points: Array<{ x: number; y: number; w: number }>): number | null { + const usable = points.filter((p) => p.y !== null && isFinite(p.y)); + if (usable.length < 2) return null; + let sw = 0; + let swx = 0; + let swy = 0; + let swxx = 0; + let swxy = 0; + for (const p of usable) { + const wgt = Math.max(p.w, 1); + sw += wgt; + swx += wgt * p.x; + swy += wgt * p.y; + swxx += wgt * p.x * p.x; + swxy += wgt * p.x * p.y; + } + const denom = sw * swxx - swx * swx; + if (denom === 0) return null; + return (sw * swxy - swx * swy) / denom; +} + +/** + * Analyse a corpus of utterances as a longitudinal competence report. + * + * Utterances are filtered to the trailing `months` window, binned by calendar + * month, and each bin is scored on the four competence dimensions plus activity. + * Language-specific measures require matching `resources`; missing resources are + * reported under `support` and `warnings` rather than silently dropped. + * + * No raw text, word list, or fringe-vocabulary frequency is included in the + * returned report — only aggregate statistics. + */ +export function analyzeTimeline( + utterances: CompetenceUtterance[], + options: TimelineOptions = {} +): CompetenceReport { + const months = Math.max(1, Math.floor(options.months ?? 12)); + const windowSize = Math.max(1, Math.floor(options.windowSize ?? 30)); + const lang = options.lang ?? 'en'; + const minWordsPerMonth = Math.max(0, Math.floor(options.minWordsPerMonth ?? 150)); + const dictionary = options.dictionary; + const resources = options.resources ?? {}; + const now = options.now ?? Date.now(); + + // ---- Filter to the trailing N months ---------------------------------- + const windowMs = months * 31 * 86_400_000; + const windowStartMs = now - windowMs; + const inWindow = utterances.filter( + (u) => + u.timestampMs <= now && u.timestampMs > windowStartMs && u.text && u.text.trim().length > 0 + ); + + // ---- Bin by month ----------------------------------------------------- + const bins = new Map(); + for (const u of inWindow) { + const key = monthKey(u.timestampMs); + const arr = bins.get(key) ?? []; + arr.push(u); + bins.set(key, arr); + } + + const sortedKeys = [...bins.keys()].sort(); + const timeline: MonthBin[] = []; + + for (const key of sortedKeys) { + const monthUtts = (bins.get(key) ?? []).slice().sort((a, b) => a.timestampMs - b.timestampMs); + + const stream: WordStream = []; + for (const u of monthUtts) stream.push(...tokenize(u.text)); + + const activity = summarizeActivity(monthUtts); + activity.uniqueWords = new Set(stream).size; + + const suppressed = stream.length < minWordsPerMonth; + const suppressReason: string | null = suppressed + ? `fewer than ${minWordsPerMonth} words (${stream.length})` + : null; + + const lex = suppressed + ? { median: null, mean: null, nWindows: 0, windowSize } + : lexicalDiversity(stream, windowSize); + const syn = suppressed + ? { median: null, mean: null, nWindows: 0, windowSize } + : syntacticDiversity(stream, { windowSize, closedClassWords: resources.closedClassWords }); + const mor = suppressed + ? { median: null, mean: null, nWindows: 0, windowSize } + : morphologicalDiversity(stream, { + windowSize, + classifyInflection: resources.classifyInflection, + }); + const spell = suppressed ? null : spellingValidity(stream, dictionary); + + timeline.push({ + month: key, + utterances: activity.utterances, + words: activity.words, + uniqueWords: activity.uniqueWords, + activeDays: activity.activeDays, + wordsPerUtterance: activity.wordsPerUtterance, + lexicalDiversity: lex, + syntacticDiversity: syn, + morphologicalDiversity: mor, + spellingValidity: spell, + suppressed, + suppressReason, + }); + } + + // ---- Support + warnings (no silent language degradation) -------------- + const warnings: string[] = []; + const hasCC = !!resources.closedClassWords && resources.closedClassWords.size > 0; + const hasMorph = !!resources.classifyInflection; + if (!hasCC) { + warnings.push( + `Syntactic diversity unavailable: no closed-class word data for language '${lang}'. ` + + `Provide resources.closedClassWords to enable it.` + ); + } + if (!hasMorph) { + warnings.push( + `Morphological diversity unavailable: no inflection classifier for language '${lang}'. ` + + `Provide resources.classifyInflection to enable it.` + ); + } + if (!dictionary) { + warnings.push('Spelling validity unavailable: no dictionary provided.'); + } + const suppCount = timeline.filter((b) => b.suppressed).length; + if (suppCount > 0) { + warnings.push( + `${suppCount} month(s) suppressed for having fewer than ${minWordsPerMonth} words.` + ); + } + + // ---- Trend on the headline lexical-diversity median ------------------- + const points: Array<{ x: number; y: number; w: number }> = []; + for (let i = 0; i < timeline.length; i++) { + const b = timeline[i]; + if (b.suppressed) continue; + const y = b.lexicalDiversity.median; + if (y === null) continue; + points.push({ x: i, y, w: b.words }); + } + + const slope = weightedSlope(points); + const validYs = points.map((p) => p.y); + const half = Math.floor(validYs.length / 2); + let firstHalf: number | null = null; + let secondHalf: number | null = null; + if (validYs.length >= 2) { + const fh = validYs.slice(0, Math.max(1, half)); + const sh = validYs.slice(Math.max(1, half)); + firstHalf = mean(fh); + secondHalf = mean(sh); + } + const delta = firstHalf !== null && secondHalf !== null ? secondHalf - firstHalf : null; + let direction: TrendResult['direction'] = 'unknown'; + if (slope !== null) { + if (Math.abs(slope) < 0.0005) direction = 'flat'; + else direction = slope > 0 ? 'up' : 'down'; + } else if (delta !== null) { + if (Math.abs(delta) < 0.005) direction = 'flat'; + else direction = delta > 0 ? 'up' : 'down'; + } + + const totalUtts = timeline.reduce((s, b) => s + b.utterances, 0); + const totalWords = timeline.reduce((s, b) => s + b.words, 0); + + const dim = (available: boolean, reason?: string): DimensionSupport => + available ? { available: true } : { available: false, reason }; + + return { + schema: 'aac-competence-report/v1', + generatedAt: new Date(now).toISOString(), + privacy: { + rawUtterancesIncluded: false, + wordListsIncluded: false, + fringeWordFrequencyIncluded: false, + minAggregationWindowDays: 31, + notes: [ + 'All metrics computed locally; only aggregate statistics are emitted.', + 'Utterances are binned by calendar month so no pattern can be tied to a specific day or time.', + 'No word lists or fringe-vocabulary frequencies are included (per AssistiveWare privacy guidance).', + ], + }, + source: { + platform: options.platform ?? 'Grid3', + langCode: options.langCode, + userLabel: options.userLabel, + dbPathIncluded: options.dbPathIncluded === true, + }, + config: { + months, + windowSize, + lang, + minWordsPerMonth, + dictionaryProvided: !!dictionary, + }, + overall: { + windowStart: sortedKeys[0] ?? monthKey(windowStartMs), + windowEnd: sortedKeys[sortedKeys.length - 1] ?? monthKey(now), + totalUtterances: totalUtts, + totalWords: totalWords, + monthsCovered: timeline.length, + monthsSuppressed: suppCount, + }, + timeline, + trend: { + metric: 'lexicalDiversity.median', + slopePerMonth: slope, + firstHalf, + secondHalf, + delta, + direction, + }, + support: { + lang, + semantic: dim(true), + syntactic: dim(hasCC, hasCC ? undefined : 'no closed-class data for this language'), + morphological: dim( + hasMorph, + hasMorph ? undefined : 'no inflection classifier for this language' + ), + phonological: dim(!!dictionary, dictionary ? undefined : 'no dictionary provided'), + }, + warnings, + }; +} diff --git a/src/utilities/analytics/history.ts b/src/utilities/analytics/history.ts index 682abcf..fd76165 100644 --- a/src/utilities/analytics/history.ts +++ b/src/utilities/analytics/history.ts @@ -13,6 +13,7 @@ import { SnapUserInfo, } from '../../processors/snap/helpers'; import { AACSemanticCategory, AACSemanticIntent } from '../../core/treeStructure'; +import type { CompetenceUtterance } from './competence'; export type HistorySource = 'Grid' | 'Snap' | 'OBL' | string; @@ -55,6 +56,29 @@ export interface HistoryEntry { export { dotNetTicksToDate }; +/** + * Adapt any `HistoryEntry[]` (Grid 3, Snap, OBF/OBFL logs, ...) into the generic + * utterance stream consumed by the linguistic-competence engine + * (`analyzeTimeline`). Each occurrence of each phrase becomes one utterance, + * timestamped by its occurrence time. This keeps the competence metrics fully + * source-agnostic: anything the library can read as history can be analysed. + */ +export function historyEntriesToCompetenceUtterances( + entries: HistoryEntry[] +): CompetenceUtterance[] { + const out: CompetenceUtterance[] = []; + for (const e of entries) { + const text = e.content; + if (!text || text.trim().length === 0) continue; + const occs = e.occurrences ?? []; + for (const occ of occs) { + if (!occ.timestamp) continue; + out.push({ text, timestampMs: occ.timestamp.getTime() }); + } + } + return out; +} + export interface BatonExportMetadata { timestamp: string; latitude?: number | null; diff --git a/src/utilities/analytics/index.ts b/src/utilities/analytics/index.ts index 579711b..3f9a935 100644 --- a/src/utilities/analytics/index.ts +++ b/src/utilities/analytics/index.ts @@ -33,6 +33,9 @@ export { SentenceAnalyzer } from './metrics/sentence'; export { ComparisonAnalyzer } from './metrics/comparison'; export { ReferenceLoader } from './reference'; +// Export linguistic-competence measures (privacy-preserving spoken-output analysis) +export * from './competence'; + /** * Get the default reference data path */ diff --git a/test/competence.test.ts b/test/competence.test.ts new file mode 100644 index 0000000..a053f27 --- /dev/null +++ b/test/competence.test.ts @@ -0,0 +1,342 @@ +import { describe, expect, it } from '@jest/globals'; +import { + analyzeTimeline, + lexicalDiversity, + morphologicalDiversity, + movingAverageTTR, + spellingValidity, + summarizeActivity, + syntacticDiversity, + tokenize, + type CompetenceUtterance, + type InflectionCategory, +} from '../src/utilities/analytics/competence'; +import { + historyEntriesToCompetenceUtterances, + type HistoryEntry, +} from '../src/utilities/analytics/history'; + +/* ------------------------------------------------------------------ * + * Inline language fixtures. + * + * The library core ships NO word lists, so tests inject a tiny closed-class + * set + inflection classifier. Real callers pass curated language resources. + * ------------------------------------------------------------------ */ + +const FIXTURE_CC = new Set([ + 'and', + 'but', + 'or', + 'because', + 'if', + 'so', + 'although', + 'to', + 'for', + 'with', + 'in', + 'on', + 'at', + 'of', + 'from', + 'by', + 'under', + 'over', +]); + +function fixtureClassify(w: string): InflectionCategory { + if (w.endsWith('ing')) return 'progressive'; + if (w.endsWith('ed')) return 'past'; + if (w.endsWith("'s")) return 'possessive'; + if (w.endsWith('s')) return 'plural'; + return 'base'; +} + +describe('competence / tokenize', () => { + it('lowercases and keeps apostrophes inside words', () => { + expect(tokenize("I DON'T want it")).toEqual(['i', "don't", 'want', 'it']); + }); + + it('drops punctuation but keeps accented characters', () => { + expect(tokenize('Hallo, wereld! Café — och "ja".')).toEqual([ + 'hallo', + 'wereld', + 'café', + 'och', + 'ja', + ]); + }); + + it('returns [] for empty input', () => { + expect(tokenize('')).toEqual([]); + expect(tokenize(' !!! ')).toEqual([]); + }); +}); + +describe('competence / movingAverageTTR (MATTR)', () => { + it('returns nulls for an empty stream', () => { + const r = movingAverageTTR([], 30); + expect(r.median).toBeNull(); + expect(r.nWindows).toBe(0); + }); + + it('treats a short sample as a single window TTR', () => { + const r = movingAverageTTR(['a', 'b', 'c'], 30); + expect(r.median).toBeCloseTo(1.0); + expect(r.nWindows).toBe(1); + }); + + it('gives a higher diversity for a varied stream than a repetitive one', () => { + const varied = ( + 'the quick brown fox jumps over lazy dog cat hat bat ' + + 'sun moon star tree leaf rock river hill cloud rain wind' + ).split(/\s+/); + const repetitive = Array.from({ length: 40 }, () => 'yes'); + const v = lexicalDiversity(varied, 30).median; + const r = lexicalDiversity(repetitive, 30).median; + expect(v).not.toBeNull(); + expect(r).not.toBeNull(); + if (v !== null && r !== null) expect(v).toBeGreaterThan(r); + // Repetitive single-word stream has TTR ~ 1/30 within windows. + expect(r).toBeCloseTo(1 / 30, 5); + }); + + it('MATTR is sample-length insensitive', () => { + const base = 'a b c d e f g h i j k l m n o'.split(/\s+/); // 15 unique + const doubled = [...base, ...base]; // same words repeated + expect(movingAverageTTR(base, 15).median).toBeCloseTo(1.0); + expect(movingAverageTTR(doubled, 15).median).toBeCloseTo(1.0); + }); +}); + +describe('competence / syntacticDiversity (closed-class, injected)', () => { + it('is unavailable with no closed-class data (no silent degradation)', () => { + const r = syntacticDiversity(['and', 'but'], { windowSize: 30 }); + expect(r.median).toBeNull(); + expect(r.unavailable).toBeDefined(); + }); + + it('scores a window rich in varied connectors higher than one using only "and"', () => { + const rich = + 'I went to the shop and bought milk but the milk was off so I went back because it was bad although they refunded me'.split( + /\s+/ + ); + const onlyAnd = Array.from({ length: 40 }, (_, i) => (i % 2 === 0 ? 'and' : 'cat')); + const r1 = syntacticDiversity(rich, { windowSize: 15, closedClassWords: FIXTURE_CC }); + const r2 = syntacticDiversity(onlyAnd, { windowSize: 15, closedClassWords: FIXTURE_CC }); + expect(r1.median).not.toBeNull(); + expect(r2.median).not.toBeNull(); + if (r1.median !== null && r2.median !== null) { + expect(r1.median).toBeGreaterThan(r2.median); + // Only "and" repeated ~8x per window -> distinct/total = 1/8. + expect(r2.median).toBeLessThan(0.2); + } + }); + + it('works for any language because the set is injected (Dutch example)', () => { + const nl = new Set(['en', 'of', 'maar', 'want', 'omdat', 'aan', 'in', 'op']); + const r = syntacticDiversity('ik wil graag naar buiten omdat het mooi weer is'.split(/\s+/), { + windowSize: 8, + closedClassWords: nl, + }); + expect(r.median).not.toBeNull(); + }); +}); + +describe('competence / morphologicalDiversity (classifier injected)', () => { + it('is unavailable without a classifier', () => { + const r = morphologicalDiversity(['cats'], { windowSize: 5 }); + expect(r.median).toBeNull(); + expect(r.unavailable).toBeDefined(); + }); + + it('scores inflected words above a base-only stream', () => { + const morphed = 'cats running jumped'.split(/\s+/); + const base = 'cat run jump'.split(/\s+/); + const r1 = morphologicalDiversity(morphed, { + windowSize: 3, + classifyInflection: fixtureClassify, + }); + const r2 = morphologicalDiversity(base, { windowSize: 3, classifyInflection: fixtureClassify }); + expect(r1.median).not.toBeNull(); + expect(r2.median).toBeNull(); // no inflected words -> unavailable + }); +}); + +describe('competence / spellingValidity', () => { + it('returns null without a dictionary', () => { + expect(spellingValidity(['hello', 'wrld'])).toBeNull(); + }); + + it('counts dictionary hits over unique alphabetic words', () => { + const dict = new Set(['hello', 'world']); + // unique alphabetic words: hello, world, xyz -> 2/3 + expect(spellingValidity(['hello', 'world', 'xyz', 'hello'], dict)).toBeCloseTo(2 / 3, 5); + }); + + it('ignores non-alphabetic tokens', () => { + const dict = new Set(['hello']); + expect(spellingValidity(['hello', '12', 'a'], dict)).toBeCloseTo(1.0, 5); + }); +}); + +describe('competence / summarizeActivity', () => { + it('counts utterances, words, active days and words per utterance', () => { + const DAY = 86_400_000; + const utts: CompetenceUtterance[] = [ + { text: 'hello world', timestampMs: 0 }, + { text: 'good morning', timestampMs: DAY }, + { text: 'bye', timestampMs: DAY + 1000 }, + ]; + const a = summarizeActivity(utts); + expect(a.utterances).toBe(3); + expect(a.words).toBe(5); + expect(a.activeDays).toBe(2); + expect(a.wordsPerUtterance.median).toBe(2); + expect(a.wordsPerUtterance.n).toBe(3); + }); +}); + +describe('competence / analyzeTimeline', () => { + const DAY = 86_400_000; + + function buildCorpus(): CompetenceUtterance[] { + const now = Date.now(); + const utts: CompetenceUtterance[] = []; + const simpleMonth = now - 60 * DAY; + for (let i = 0; i < 30; i++) { + utts.push({ text: 'I want it yes', timestampMs: simpleMonth + i * 1000 }); + } + for (let i = 0; i < 30; i++) { + utts.push({ + text: 'I think that we should go to the park because the weather is lovely although it might rain', + timestampMs: now - i * 60_000, + }); + } + return utts; + } + + it('bins utterances by month and computes per-bin metrics', () => { + const report = analyzeTimeline(buildCorpus(), { + months: 3, + windowSize: 15, + minWordsPerMonth: 1, + lang: 'en', + resources: { closedClassWords: FIXTURE_CC, classifyInflection: fixtureClassify }, + }); + expect(report.schema).toBe('aac-competence-report/v1'); + expect(report.timeline.length).toBeGreaterThanOrEqual(1); + for (const bin of report.timeline) { + expect(bin.suppressed).toBe(false); + expect(bin.lexicalDiversity.median).not.toBeNull(); + } + }); + + it('reports support + warnings when language resources are missing', () => { + const report = analyzeTimeline(buildCorpus(), { + months: 3, + windowSize: 15, + minWordsPerMonth: 1, + lang: 'fr', // no resources provided -> syntactic/morphological unavailable + }); + expect(report.support.syntactic.available).toBe(false); + expect(report.support.morphological.available).toBe(false); + expect(report.support.semantic.available).toBe(true); + expect(report.warnings.some((w) => w.includes("'fr'"))).toBe(true); + // The measure itself reports unavailable in each bin. + expect(report.timeline[0].syntacticDiversity.unavailable).toBeDefined(); + }); + + it('flags sparse months as suppressed and omits their diversity figures', () => { + const now = Date.now(); + const report = analyzeTimeline([{ text: 'hi', timestampMs: now - 40 * DAY }], { + months: 3, + minWordsPerMonth: 150, + lang: 'en', + }); + const suppressed = report.timeline.filter((b) => b.suppressed); + expect(suppressed.length).toBeGreaterThan(0); + expect(suppressed[0].lexicalDiversity.median).toBeNull(); + expect(suppressed[0].suppressReason).toContain('fewer than'); + }); + + it('emits no raw text or word lists anywhere in the report', () => { + const report = analyzeTimeline(buildCorpus(), { + months: 3, + windowSize: 15, + minWordsPerMonth: 1, + lang: 'en', + resources: { closedClassWords: FIXTURE_CC }, + }); + const json = JSON.stringify(report); + expect(json).not.toContain('hello'); + expect(json).not.toContain('because the weather'); + expect(report.privacy.rawUtterancesIncluded).toBe(false); + expect(report.privacy.wordListsIncluded).toBe(false); + expect(report.privacy.fringeWordFrequencyIncluded).toBe(false); + }); + + it('trends lexical diversity upward for an improving corpus', () => { + const report = analyzeTimeline(buildCorpus(), { + months: 3, + windowSize: 15, + minWordsPerMonth: 1, + lang: 'en', + resources: { closedClassWords: FIXTURE_CC }, + }); + expect(report.trend.metric).toBe('lexicalDiversity.median'); + expect(report.trend.direction).toBe('up'); + expect(report.trend.delta).not.toBeNull(); + if (report.trend.delta !== null) expect(report.trend.delta).toBeGreaterThan(0); + }); + + it('respects the trailing-months window (drops old utterances)', () => { + const now = Date.now(); + const report = analyzeTimeline( + [{ text: 'very old utterance here', timestampMs: now - 400 * DAY }], + { months: 3, minWordsPerMonth: 1, lang: 'en' } + ); + expect(report.timeline.length).toBe(0); + expect(report.overall.totalUtterances).toBe(0); + }); +}); + +describe('competence / source-agnostic (OBF/OBFL via adapter)', () => { + it('analyses any HistoryEntry source, e.g. OBF/OBFL logs, identically to native utterances', () => { + const now = Date.now(); + // OBF/OBFL-style history: phrases with occurrences, source-tagged 'OBL'. + const obfEntries: HistoryEntry[] = [ + { + id: 'obf-1', + source: 'OBL', + content: 'I want a drink of water please', + occurrences: [ + { timestamp: new Date(now - 50 * 86_400_000) }, + { timestamp: new Date(now - 49 * 86_400_000) }, + ], + }, + { + id: 'obf-2', + source: 'OBL', + content: 'the cat sat on the mat because it was tired', + occurrences: [{ timestamp: new Date(now - 1000) }], + }, + ]; + + const utts = historyEntriesToCompetenceUtterances(obfEntries); + // Two occurrences of phrase 1 + one of phrase 2 = 3 utterances. + expect(utts.length).toBe(3); + + const report = analyzeTimeline(utts, { + months: 3, + windowSize: 10, + minWordsPerMonth: 1, + lang: 'en', + resources: { closedClassWords: FIXTURE_CC }, + }); + expect(report.source.platform).toBe('Grid3'); // default label; data came from OBF + expect(report.overall.totalUtterances).toBe(3); + // No raw phrase text leaks into the output. + expect(JSON.stringify(report)).not.toContain('drink of water'); + }); +});