From 87cf356737dc5c4e34bd7037ab875d29c30ed794 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Tue, 18 Aug 2026 22:25:56 +0900 Subject: [PATCH] Add `ordinary_normalize` --- include/iris/stdint.hpp | 6 +- include/iris/string_algo.hpp | 119 ++++++++++++++++++++++++++++++++++- test/string_algo.cpp | 62 ++++++++++++++++++ 3 files changed, 182 insertions(+), 5 deletions(-) diff --git a/include/iris/stdint.hpp b/include/iris/stdint.hpp index 1edf9e4..2d32189 100644 --- a/include/iris/stdint.hpp +++ b/include/iris/stdint.hpp @@ -98,7 +98,7 @@ using make_integer_of_size_impl_t = integer_of_size_impl struct make_signed_of_size { - static_assert(std::integral && !std::same_as, bool> || std::is_enum_v); + static_assert((std::integral && !std::same_as, bool>) || std::is_enum_v); using type = remove_cv::template apply; }; @@ -108,7 +108,7 @@ using make_signed_of_size_t = make_signed_of_size::type; template struct make_unsigned_of_size { - static_assert(std::integral && !std::same_as, bool> || std::is_enum_v); + static_assert((std::integral && !std::same_as, bool>) || std::is_enum_v); using type = remove_cv::template apply; }; @@ -118,7 +118,7 @@ using make_unsigned_of_size_t = make_unsigned_of_size::type; template struct make_integer_of_size { - static_assert(std::integral && !std::same_as, bool> || std::is_enum_v); + static_assert((std::integral && !std::same_as, bool>) || std::is_enum_v); using type = remove_cv::template apply; }; diff --git a/include/iris/string_algo.hpp b/include/iris/string_algo.hpp index 7b1aea6..f618d6e 100644 --- a/include/iris/string_algo.hpp +++ b/include/iris/string_algo.hpp @@ -3,13 +3,16 @@ // SPDX-License-Identifier: MIT -#include -#include +#include // IWYU pragma: keep +#include // IWYU pragma: keep +#include +#include #include #include #include #include +#include #include @@ -266,6 +269,118 @@ template return buf; } + +namespace detail { + +// Closed range [first, last] of character set +template +struct char_range +{ + using char_type = CharT; + using value_type = make_unsigned_of_size_t; + + CharT first, last; + + consteval char_range(CharT first, CharT last) + : first(first) + , last(last) + { + if (first > last) throw std::domain_error{"char_range must be proper"}; + } + + [[nodiscard]] constexpr std::size_t count() const noexcept + { + return static_cast( + static_cast(last) - static_cast(first) + ) + 1uz; + } +}; + +template +struct char_substitute +{ + using char_type = FromT::char_type; + using value_type = make_unsigned_of_size_t; + + FromT from; + ToT to; + + consteval char_substitute(FromT from, ToT to) + : from(from) + , to(to) + { + if (from.count() != to.count()) { + throw std::out_of_range{"from.count() does not match to.count()"}; + } + } + + [[nodiscard]] constexpr bool substitute(char_type& ch) const noexcept + { + if (from.first <= ch && ch <= from.last) { + value_type const ofs = static_cast(ch) - static_cast(from.first); + ch = static_cast(static_cast(to.first) + ofs); + return true; + } + return false; + } +}; + +template +inline constexpr auto jp_substitutes = std::to_array>>({ + {{U'０', U'９'}, {U'0', U'9'}}, + {{U'Ａ', U'Ｚ'}, {U'A', U'Z'}}, + {{U'ａ', U'ｚ'}, {U'a', U'z'}}, +}); + +} // detail + +// Normalizes occurrences of "ordinary" inconsistent spelling with +// the canonicalized letter. +// +// This algorithm guarantees the character count never changes regardless +// of the transformation, in contrast to fully-featured Unicode normalization +// such as NFKC. (Note: NFKC not only changes the character count; it may +// occasionally render the resulting character to empty set, or even *flips* +// the character positioning, which makes it require a rather complex algo.) +// +// The above design choice enables this function to be used like an *easy* +// canonicalization, where the original and the transformed string can both +// be mapped to the identical `interval`. +template +constexpr void ordinary_normalize(std::basic_string& input) +{ + static_assert(std::same_as, "sorry, not implemented"); + + for (CharT& ch : input) { + switch (ch) { + case U'　': ch = U' '; continue; + default: break; + } + + for (auto const& subs : detail::jp_substitutes) { + if (subs.substitute(ch)) { + break; + } + } + } +} + +//template +//[[nodiscard]] constexpr std::string ordinary_normalize_copy(std::string_view input) +//{ +// std::string buf(input); +// iris::ordinary_normalize(buf); +// return buf; +//} + +template +[[nodiscard]] constexpr std::u32string ordinary_normalize_copy(std::u32string_view input) +{ + std::u32string buf(input); + iris::ordinary_normalize(buf); + return buf; +} + } // iris #endif diff --git a/test/string_algo.cpp b/test/string_algo.cpp index 420a404..c62568d 100644 --- a/test/string_algo.cpp +++ b/test/string_algo.cpp @@ -205,4 +205,66 @@ TEST_CASE("string_algo: escape") #undef IRIS_TEST_ESCAPE } +TEST_CASE("string_algo: ordinary_normalize") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + std::u32string str; + iris::ordinary_normalize(str); + CHECK(str == U""sv); + } + + CHECK(iris::ordinary_normalize_copy(U"") == U""sv); + +#if 0 +let edge_chars = [['ａ', 'ｚ'], ['Ａ', 'Ｚ'], ['０', '９']]; +for (let char_range of edge_chars) { + console.log(`[${char_range[0].codePointAt(0).toString(16)}, ${char_range[1].codePointAt(0).toString(16)}]`); +} +// [ff41, ff5a] +// [ff21, ff3a] +// [ff10, ff19] +#endif + + CHECK(iris::ordinary_normalize_copy(U" ") == U" "sv); + CHECK(iris::ordinary_normalize_copy(U"　") == U" "sv); + + CHECK(iris::ordinary_normalize_copy(U"a") == U"a"sv); + CHECK(iris::ordinary_normalize_copy(U"z") == U"z"sv); + CHECK(iris::ordinary_normalize_copy(U"A") == U"A"sv); + CHECK(iris::ordinary_normalize_copy(U"Z") == U"Z"sv); + CHECK(iris::ordinary_normalize_copy(U"0") == U"0"sv); + CHECK(iris::ordinary_normalize_copy(U"9") == U"9"sv); + + CHECK(iris::ordinary_normalize_copy(U"ａ") == U"a"sv); + CHECK(iris::ordinary_normalize_copy(U"ｚ") == U"z"sv); + CHECK(iris::ordinary_normalize_copy(U"\uff5b") == U"\uff5b"sv); + + CHECK(iris::ordinary_normalize_copy(U"Ａ") == U"A"sv); + CHECK(iris::ordinary_normalize_copy(U"Ｚ") == U"Z"sv); + CHECK(iris::ordinary_normalize_copy(U"\uff3b") == U"\uff3b"sv); + + CHECK(iris::ordinary_normalize_copy(U"０") == U"0"sv); + CHECK(iris::ordinary_normalize_copy(U"９") == U"9"sv); + CHECK(iris::ordinary_normalize_copy(U"\uff20") == U"\uff20"sv); + + { + std::u32string_view const input = + U"ａｂｃｄｅｆｇｈｉｊｋｌｍｎｏｐｑｒｓｔｕｖｗｘｙｚ／" + U"ＡＢＣＤＥＦＧＨＩＪＫＬＭＮＯＰＱＲＳＴＵＶＷＸＹＺ／" + U"０１２３４５６７８９"; + + std::u32string_view const expected_normalized = + U"abcdefghijklmnopqrstuvwxyz／" + U"ABCDEFGHIJKLMNOPQRSTUVWXYZ／" + U"0123456789"; + + CHECK(iris::ordinary_normalize_copy(input) == expected_normalized); + CHECK(iris::ordinary_normalize_copy(input).size() == input.size()); + } +} + // NOLINTEND(readability-container-size-empty)