From f39d50079ac3904a0f869f3152bc7a9265f928fa Mon Sep 17 00:00:00 2001 From: Thomas Lamant Date: Sat, 25 Jul 2026 00:39:24 +0200 Subject: [PATCH 1/2] feat(parser): normalize full-width input syntax --- src/classes/recipe.ts | 20 +++++------ src/classes/shopping_list.ts | 8 ++--- src/utils/normalization.ts | 35 ++++++++++++++++++++ src/utils/parser_helpers.ts | 43 ++++++++++++++---------- src/utils/time.ts | 3 +- test/parser_helpers.test.ts | 57 ++++++++++++++++++++++++++++++++ test/recipe_parsing.test.ts | 29 ++++++++++++++++ test/shopping_list_files.test.ts | 20 +++++++++++ test/utils_time.test.ts | 6 ++++ 9 files changed, 188 insertions(+), 33 deletions(-) create mode 100644 src/utils/normalization.ts diff --git a/src/classes/recipe.ts b/src/classes/recipe.ts index 8f3d7e1..cf27cf1 100644 --- a/src/classes/recipe.ts +++ b/src/classes/recipe.ts @@ -59,6 +59,7 @@ import { getAlternativeSignature, parseMarkdownSegments, } from "../utils/parser_helpers"; +import { normalizeInputString } from "../utils/normalization"; import { addEquivalentsAndSimplify } from "../quantities/alternatives"; import { multiplyQuantityValue, @@ -775,8 +776,7 @@ export class Recipe { // Type for accumulated quantities type QuantityAccumulator = { quantities: ( - | QuantityWithExtendedUnit - | FlatOrGroup + QuantityWithExtendedUnit | FlatOrGroup )[]; alternativeQuantities: Map< number, @@ -1199,8 +1199,7 @@ export class Recipe { // Collect all raw quantities across all signature groups const quantities: ( - | QuantityWithExtendedUnit - | FlatOrGroup + QuantityWithExtendedUnit | FlatOrGroup )[] = []; if (usedAsPrimary) { @@ -1287,8 +1286,7 @@ export class Recipe { const groupsForIng = ingredientGroups.get(index); if (groupsForIng) { const quantityGroups: ( - | IngredientQuantityGroup - | IngredientQuantityAndGroup + IngredientQuantityGroup | IngredientQuantityAndGroup )[] = []; for (const [, group] of groupsForIng) { @@ -1421,10 +1419,12 @@ export class Recipe { */ parse(content: string) { // Remove noise - const cleanContent = content - .replace(metadataRegex, "") - .replace(commentRegex, "") - .replace(blockCommentRegex, "") + const cleanContent = normalizeInputString( + content + .replace(metadataRegex, "") + .replace(commentRegex, "") + .replace(blockCommentRegex, ""), + ) .trim() .split(/\r\n?|\n/); diff --git a/src/classes/shopping_list.ts b/src/classes/shopping_list.ts index edc9bb9..1689a0b 100644 --- a/src/classes/shopping_list.ts +++ b/src/classes/shopping_list.ts @@ -41,6 +41,7 @@ import { import { parseQuantityWithUnit } from "../utils/parser_helpers"; import { formatQuantity } from "../utils/render_helpers"; import { NoTabAsIndentError, UnknownRecipePathError } from "../errors"; +import { normalizeInputString } from "../utils/normalization"; /** * Shopping List generator. @@ -206,8 +207,7 @@ export class ShoppingList { // Separate text-value quantities (cannot be summed) from numeric ones const textEntries: QuantityWithExtendedUnit[] = []; const numericEntries: ( - | QuantityWithExtendedUnit - | FlatOrGroup + QuantityWithExtendedUnit | FlatOrGroup )[] = []; for (const q of rawQuantities) { if ( @@ -1058,7 +1058,7 @@ export class ShoppingList { */ private parseRecipeRefLine(line: string): ShoppingListRecipeRef { // this function is always called for lines starting with "./" so regex always matches - const match = line.match(recipeRefLineRegex)!; + const match = normalizeInputString(line).match(recipeRefLineRegex)!; return { path: match[1]!, servings: match[2] ? Number(match[2]) : undefined, @@ -1070,7 +1070,7 @@ export class ShoppingList { * @internal */ private parseManualItemLine(line: string): AddedIngredient { - const match = line.match(manualIngredientRegex); + const match = normalizeInputString(line).match(manualIngredientRegex); const groups = match!.groups as { name: string; quantity?: string }; const name = groups.name.trim(); if (groups.quantity) { diff --git a/src/utils/normalization.ts b/src/utils/normalization.ts new file mode 100644 index 0000000..15a56e5 --- /dev/null +++ b/src/utils/normalization.ts @@ -0,0 +1,35 @@ +const fullWidthSyntaxMap: Record = { + "＠": "@", + "＃": "#", + "～": "~", + "｛": "{", + "｝": "}", + "（": "(", + "）": ")", + "［": "[", + "］": "]", + "％": "%", + "｜": "|", + "：": ":", + "，": ",", + "．": ".", + "／": "/", + "－": "-", + "＝": "=", + "\u3000": " ", +}; + +/** + * Normalizes parser-significant full-width characters without changing prose. + */ +export function normalizeInputString(input: string): string { + return input.replace( + /[＠＃～〜｛｝（）［］％｜：，．／－＝\u3000０-９]/g, + (char) => { + if (char >= "０" && char <= "９") { + return String(char.charCodeAt(0) - "０".charCodeAt(0)); + } + return fullWidthSyntaxMap[char]!; + }, + ); +} diff --git a/src/utils/parser_helpers.ts b/src/utils/parser_helpers.ts index 7d41095..b717f58 100644 --- a/src/utils/parser_helpers.ts +++ b/src/utils/parser_helpers.ts @@ -41,6 +41,7 @@ import { ReferencedItemCannotBeRedefinedError, } from "../errors"; import { parseTimeToMinutes } from "./time"; +import { normalizeInputString } from "./normalization"; /** * Pushes a pending note to the section content if it has items. @@ -250,12 +251,13 @@ export function findAndUpsertCookware( export const parseFixedValue = ( input_str: string, ): TextValue | DecimalValue | FractionValue => { - if (!numberLikeRegex.test(input_str)) { - return { type: "text", text: input_str }; + const normalizedInput = normalizeInputString(input_str); + if (!numberLikeRegex.test(normalizedInput)) { + return { type: "text", text: normalizedInput }; } // After this we know that s is either a fraction or a decimal value - const s = input_str.trim().replace(",", "."); + const s = normalizedInput.trim().replace(",", "."); // fraction if (s.includes("/")) { @@ -323,7 +325,7 @@ function stringifyFixedValue(quantity: FixedValue): string { * ``` */ export function parseQuantityValue(input_str: string): FixedValue | Range { - const clean_str = String(input_str).trim(); + const clean_str = normalizeInputString(String(input_str)).trim(); if (rangeRegex.test(clean_str)) { const range_parts = clean_str.split("-"); @@ -355,7 +357,7 @@ export function parseQuantityWithUnit(input: string): { value: FixedValue | Range; unit?: string; } { - const trimmed = input.trim(); + const trimmed = normalizeInputString(input).trim(); const separatorIndex = trimmed.indexOf("%"); if (separatorIndex === -1) { return { value: parseQuantityValue(trimmed) }; @@ -677,11 +679,12 @@ export function parseBlockScalarMetaVar( * @throws {@link InvalidQuantityFormat} if the value is non-numeric. */ export function parseArbitraryQuantity(raw: string): ArbitraryScalable { - const quantityMatch = raw.trim().match(quantityAlternativeRegex); + const normalizedRaw = normalizeInputString(raw); + const quantityMatch = normalizedRaw.trim().match(quantityAlternativeRegex); /* v8 ignore next 4 -- @preserve: defensive guard; regex always matches */ if (!quantityMatch?.groups) { throw new InvalidQuantityFormat( - raw, + normalizedRaw, "Arbitrary quantities must have a numerical value", ); } @@ -689,7 +692,7 @@ export function parseArbitraryQuantity(raw: string): ArbitraryScalable { const unit = quantityMatch.groups.unit; if (!value || (value.type === "fixed" && value.value.type === "text")) { throw new InvalidQuantityFormat( - raw, + normalizedRaw, "Arbitrary quantities must have a numerical value", ); } @@ -713,11 +716,12 @@ export function parseServingsMetaVar( content: string, varName: "servings" | "serves", ): { numericValue: number; rawValue: number | string } | undefined { - const raw = parseSimpleMetaVar(content, varName); + const raw = parseSimpleMetaVar(normalizeInputString(content), varName); if (raw === undefined) return undefined; - const num = Number(raw); + const normalizedRaw = normalizeInputString(String(raw)); + const num = Number(normalizedRaw); if (isNaN(num)) { - return { numericValue: 1, rawValue: raw }; + return { numericValue: 1, rawValue: normalizedRaw }; } return { numericValue: num, rawValue: num }; } @@ -733,7 +737,7 @@ export function parseServingsMetaVar( * @throws {@link InvalidQuantityFormat} if the value is non-numeric. */ export function parseYieldMetaVar(content: string): Yield | undefined { - const match = content.match(yieldMetaValueRegex); + const match = normalizeInputString(content).match(yieldMetaValueRegex); if (!match) return undefined; // Complex format branch: matched the {{...}} pattern @@ -1006,21 +1010,22 @@ function parseListItems( * Parses a raw string value into appropriate type (number, string, or array). */ function parseSingleLineMetadataValue(rawValue: string): MetadataValue { + const normalizedRawValue = normalizeInputString(rawValue); // Check for inline array [a, b, c] - if (rawValue.startsWith("[") && rawValue.endsWith("]")) { - return rawValue + if (normalizedRawValue.startsWith("[") && normalizedRawValue.endsWith("]")) { + return normalizedRawValue .slice(1, -1) .split(",") .map((item) => item.trim()); } // Check for number (integer or decimal) - if (numericValueRegex.test(rawValue)) { - return Number(rawValue); + if (numericValueRegex.test(normalizedRawValue)) { + return Number(normalizedRawValue); } // Return as string - return rawValue; + return normalizedRawValue; } /** @@ -1063,7 +1068,9 @@ export function extractMetadata(content: string): MetadataExtract { let servings: number | undefined = undefined; // Is there front-matter at all? - const metadataContent = content.match(metadataRegex)?.[2]; + const metadataContent = normalizeInputString( + content.match(metadataRegex)?.[2] ?? "", + ); if (!metadataContent) { return { metadata }; } diff --git a/src/utils/time.ts b/src/utils/time.ts index 5077dfd..3f28bfa 100644 --- a/src/utils/time.ts +++ b/src/utils/time.ts @@ -9,6 +9,7 @@ */ import { compactTimeRegex, timeUnitTokenRegex } from "../regex"; +import { normalizeInputString } from "./normalization"; /** Maps unit strings to their value in minutes. */ const timeUnitToMinutes: Record = { @@ -60,7 +61,7 @@ export function parseTimeToMinutes(input: string | number): number | undefined { return input; } - const trimmed = input.trim(); + const trimmed = normalizeInputString(input).trim(); if (trimmed === "") return undefined; // Strategy 1: Plain number string → minutes diff --git a/test/parser_helpers.test.ts b/test/parser_helpers.test.ts index 028e5e2..95dcfd9 100644 --- a/test/parser_helpers.test.ts +++ b/test/parser_helpers.test.ts @@ -70,6 +70,12 @@ describe("parseServingsMetaVar", () => { rawValue: 6, }); }); + it("should parse full-width separators and digits", () => { + expect(parseServingsMetaVar("servings：６", "servings")).toEqual({ + numericValue: 6, + rawValue: 6, + }); + }); it("should parse serves", () => { expect(parseServingsMetaVar("serves: 4", "serves")).toEqual({ numericValue: 4, @@ -107,6 +113,12 @@ describe("parseYieldMetaVar", () => { unit: "g", }); }); + it("should parse full-width quantity punctuation and digits", () => { + expect(parseYieldMetaVar("yield：３００％g")).toEqual({ + quantity: { type: "fixed", value: { type: "decimal", decimal: 300 } }, + unit: "g", + }); + }); it("should parse plain fraction quantity with unit", () => { expect(parseYieldMetaVar("yield: 1/2%kg")).toEqual({ quantity: { type: "fixed", value: { type: "fraction", num: 1, den: 2 } }, @@ -160,6 +172,12 @@ describe("parseArbitraryQuantity", () => { unit: "g", }); }); + it("should parse full-width quantity punctuation and digits", () => { + expect(parseArbitraryQuantity("３００％g")).toEqual({ + quantity: { type: "fixed", value: { type: "decimal", decimal: 300 } }, + unit: "g", + }); + }); it("should parse a quantity without unit", () => { expect(parseArbitraryQuantity("5")).toEqual({ quantity: { type: "fixed", value: { type: "decimal", decimal: 5 } }, @@ -465,6 +483,27 @@ describe("extractMetadata", () => { expect(extractMetadata(content)).toEqual({ metadata: {} }); }); + it("should parse full-width separators and digits in metadata", () => { + const content = `--- +servings：２ +yield：３００％g +time：１時間 ３０分 +---`; + expect(extractMetadata(content)).toEqual({ + metadata: { + servings: 2, + yield: { + quantity: { type: "fixed", value: { type: "decimal", decimal: 300 } }, + unit: "g", + }, + time: { + total: 90, + }, + }, + servings: 2, + }); + }); + it("should return an empty object if metavars are declared outside of block", () => { const content = ` --- @@ -1077,6 +1116,11 @@ describe("parseQuantityValue", () => { min: { type: "fraction", num: 1, den: 2 }, max: { type: "decimal", decimal: 1 }, }); + expect(parseQuantityValue("１－２")).toEqual({ + type: "range", + min: { type: "decimal", decimal: 1 }, + max: { type: "decimal", decimal: 2 }, + }); }); it("correctly parses fixed values", () => { @@ -1088,6 +1132,10 @@ describe("parseQuantityValue", () => { type: "fixed", value: { type: "decimal", decimal: 1.2 }, }); + expect(parseQuantityValue("１．２")).toEqual({ + type: "fixed", + value: { type: "decimal", decimal: 1.2 }, + }); }); }); @@ -1612,6 +1660,15 @@ describe("parseQuantityWithUnit", () => { expect(result.unit).toBe("g"); }); + it("should parse full-width quantity punctuation and digits", () => { + const result = parseQuantityWithUnit("５００％g"); + expect(result.value).toMatchObject({ + type: "fixed", + value: { type: "decimal", decimal: 500 }, + }); + expect(result.unit).toBe("g"); + }); + it("should parse value without unit", () => { const result = parseQuantityWithUnit("6"); expect(result.value).toMatchObject({ diff --git a/test/recipe_parsing.test.ts b/test/recipe_parsing.test.ts index bd13da6..23c463d 100644 --- a/test/recipe_parsing.test.ts +++ b/test/recipe_parsing.test.ts @@ -12,6 +12,7 @@ import { ReferencedItemCannotBeRedefinedError, } from "../src/errors"; import type { + Cookware, Ingredient, IngredientItem, Note, @@ -803,6 +804,34 @@ describe("parse function", () => { expect(() => new Recipe(badInput)).toThrow(/Timer missing unit/); }); + it("normalizes full-width Cooklang tokens", () => { + const result = new Recipe( + "＠味噌｛２％大さじ｝を＃鍋｛１｝で〜煮る｛１０％分｝。｛｛３００％g｝｝", + ); + expect(result.ingredients[0]).toMatchObject({ + name: "味噌", + quantities: [ + { + quantity: { type: "fixed", value: { type: "decimal", decimal: 2 } }, + unit: "大さじ", + }, + ], + }); + expect(result.cookware[0]).toMatchObject({ + name: "鍋", + quantity: { type: "fixed", value: { type: "decimal", decimal: 1 } }, + }); + expect(result.timers[0]).toEqual({ + name: "煮る", + duration: { type: "fixed", value: { type: "decimal", decimal: 10 } }, + unit: "分", + }); + expect(result.arbitraries[0]).toEqual({ + quantity: { type: "fixed", value: { type: "decimal", decimal: 300 } }, + unit: "g", + }); + }); + it("ignores comments and comments blocks", () => { const recipeWithComments = ` This is a step. diff --git a/test/shopping_list_files.test.ts b/test/shopping_list_files.test.ts index c63fd9c..e1698c1 100644 --- a/test/shopping_list_files.test.ts +++ b/test/shopping_list_files.test.ts @@ -187,6 +187,26 @@ olive oil ]); }); + it("should parse manual item with full-width quantity syntax", () => { + const content = `味噌｛２％大さじ｝\n`; + const list = new ShoppingList(); + list.loadFile(content); + expect(list.manualItems).toMatchObject([ + { + name: "味噌", + quantities: [ + { + quantity: { + type: "fixed", + value: { type: "decimal", decimal: 2 }, + }, + unit: "大さじ", + }, + ], + }, + ]); + }); + it("should parse manual item with quantity but no unit", () => { const content = `eggs{6}\n`; const list = new ShoppingList(); diff --git a/test/utils_time.test.ts b/test/utils_time.test.ts index c807ae9..8e329c8 100644 --- a/test/utils_time.test.ts +++ b/test/utils_time.test.ts @@ -131,6 +131,12 @@ describe("parseTimeToMinutes", () => { expect(parseTimeToMinutes("1日 2時間 30分")).toBe(1590); }); + it("normalizes full-width digits and punctuation", () => { + expect(parseTimeToMinutes("１０分")).toBe(10); + expect(parseTimeToMinutes("１時間　３０分")).toBe(90); + expect(parseTimeToMinutes("１．５時間")).toBe(90); + }); + it("handles all second aliases", () => { expect(parseTimeToMinutes("60 s")).toBe(1); expect(parseTimeToMinutes("60 sec")).toBe(1); From 7101e92e872afb84f6b272aa86cb1c90e8c2a8da Mon Sep 17 00:00:00 2001 From: Thomas Lamant Date: Sat, 25 Jul 2026 01:07:06 +0200 Subject: [PATCH 2/2] fix: tests --- src/utils/normalization.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/normalization.ts b/src/utils/normalization.ts index 15a56e5..aa4f6b1 100644 --- a/src/utils/normalization.ts +++ b/src/utils/normalization.ts @@ -2,6 +2,7 @@ const fullWidthSyntaxMap: Record = { "＠": "@", "＃": "#", "～": "~", + "〜": "~", "｛": "{", "｝": "}", "（": "(",