diff --git a/db/migrations/20260702_120000_drop_persistent_scratch_tables.sql b/db/migrations/20260702_120000_drop_persistent_scratch_tables.sql new file mode 100644 index 000000000..9ab819da2 --- /dev/null +++ b/db/migrations/20260702_120000_drop_persistent_scratch_tables.sql @@ -0,0 +1,21 @@ +-- Migration: Drop the persistent parse/import scratch tables. +-- +-- temp_word_occurrences, temp_words and tempexprs used to be persistent, +-- globally-shared InnoDB tables that every parse/import TRUNCATEd and refilled. +-- That caused two problems: +-- * "Tablespace is missing for a table" (InnoDB error 194) crashes on +-- TRUNCATE when the table's .ibd file went missing (notably on Windows) — +-- every text import then failed. See issue #247. +-- * Concurrent parses/imports corrupted each other's rows (no isolation). +-- +-- They are now created per database connection as CREATE TEMPORARY TABLE at +-- runtime (see src/Shared/Infrastructure/Database/ScratchTables.php), so the +-- persistent versions are no longer needed. Dropping them here also repairs +-- installs whose tablespace was already orphaned (the reported crash). +-- +-- DROP TABLE IF EXISTS is idempotent and succeeds even when the tablespace is +-- missing, so it clears the orphaned dictionary entry as well. + +DROP TABLE IF EXISTS temp_word_occurrences; +DROP TABLE IF EXISTS temp_words; +DROP TABLE IF EXISTS tempexprs; diff --git a/db/schema/baseline.sql b/db/schema/baseline.sql index e21f3c2f1..dc438f327 100644 --- a/db/schema/baseline.sql +++ b/db/schema/baseline.sql @@ -112,23 +112,12 @@ CREATE TABLE IF NOT EXISTS word_occurrences ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE IF NOT EXISTS temp_word_occurrences ( - TiCount smallint(5) unsigned NOT NULL, - TiSeID mediumint(8) unsigned NOT NULL, - TiOrder smallint(5) unsigned NOT NULL, - TiWordCount tinyint(3) unsigned NOT NULL, - TiText varchar(250) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; - -CREATE TABLE IF NOT EXISTS temp_words ( - WoText varchar(250) DEFAULT NULL, - WoTextLC varchar(250) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, - WoTranslation varchar(500) NOT NULL DEFAULT '*', - WoRomanization varchar(100) DEFAULT NULL, - WoSentence varchar(1000) DEFAULT NULL, - WoTaglist varchar(255) DEFAULT NULL, - PRIMARY KEY(WoTextLC) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; +-- The parse/import scratch tables (temp_word_occurrences, temp_words, tempexprs) +-- are no longer persistent shared tables. They are created per database +-- connection as CREATE TEMPORARY TABLE at runtime (see +-- src/Shared/Infrastructure/Database/ScratchTables.php). This avoids InnoDB +-- "Tablespace is missing" (error 194) crashes on TRUNCATE and isolates +-- concurrent parses/imports from each other. See issue #247. CREATE TABLE IF NOT EXISTS texts ( TxID smallint(5) unsigned NOT NULL AUTO_INCREMENT, diff --git a/src/Modules/Language/Application/Services/ParsingCoordinator.php b/src/Modules/Language/Application/Services/ParsingCoordinator.php deleted file mode 100644 index 3b4be44d1..000000000 --- a/src/Modules/Language/Application/Services/ParsingCoordinator.php +++ /dev/null @@ -1,348 +0,0 @@ - - * @license Unlicense - * @link https://hugofara.github.io/lwt/developer/api - * @since 3.0.0 - */ - -declare(strict_types=1); - -namespace Lwt\Modules\Language\Application\Services; - -use Lwt\Modules\Language\Domain\Language; -use Lwt\Modules\Language\Domain\Parser\ParserConfig; -use Lwt\Modules\Language\Domain\Parser\ParserResult; -use Lwt\Modules\Language\Infrastructure\Parser\ParserRegistry; -use Lwt\Shared\Infrastructure\Globals; -use Lwt\Shared\Infrastructure\Database\Connection; -use Lwt\Shared\Infrastructure\Database\Escaping; -use Lwt\Shared\Infrastructure\Database\QueryBuilder; -use Lwt\Shared\Infrastructure\Database\UserScopedQuery; - -/** - * Coordinates parsing operations, providing a facade for the parser system. - * - * This class handles parser selection, preprocessing, and database persistence. - * It serves as the main entry point for text parsing operations. - * - * @since 3.0.0 - */ -class ParsingCoordinator -{ - public function __construct( - private ParserRegistry $registry - ) { - } - - /** - * Split text into sentences without database operations. - * - * @param string $text Text to parse - * @param Language $language Language entity - * - * @return string[] Array of sentences - */ - public function splitIntoSentences(string $text, Language $language): array - { - $text = $this->preprocess($text, $language); - $config = ParserConfig::fromLanguage($language); - $parser = $this->registry->getParserForLanguage($language); - - $result = $parser->parse($text, $config); - return $result->getSentences(); - } - - /** - * Parse text and return the result without database operations. - * - * @param string $text Text to parse - * @param Language $language Language entity - * - * @return ParserResult Parsing result - */ - public function parseForPreview(string $text, Language $language): ParserResult - { - $text = $this->preprocess($text, $language); - $config = ParserConfig::fromLanguage($language); - $parser = $this->registry->getParserForLanguage($language); - - return $parser->parse($text, $config); - } - - /** - * Parse text and save to database. - * - * @param string $text Text to parse - * @param Language $language Language entity - * @param int $textId Text ID (must be positive) - * - * @return void - * - * @throws \InvalidArgumentException If textId is not positive - */ - public function parseAndSave(string $text, Language $language, int $textId): void - { - if ($textId <= 0) { - throw new \InvalidArgumentException( - "Text ID must be positive, got: $textId" - ); - } - - $text = $this->preprocess($text, $language); - $config = ParserConfig::fromLanguage($language); - $parser = $this->registry->getParserForLanguage($language); - - $result = $parser->parse($text, $config); - - $this->saveToDatabase($result, $language, $textId); - } - - /** - * Preprocess text before parsing. - * - * Applies character substitutions and other text cleanup. - * - * @param string $text Raw text - * @param Language $language Language entity - * - * @return string Preprocessed text - */ - protected function preprocess(string $text, Language $language): string - { - $text = Escaping::prepareTextdata($text); - - // Replace special characters that interfere with sentence parsing - $text = str_replace(['}', '{'], [']', '['], $text); - - // Apply character substitutions - $substitutions = $language->characterSubstitutions(); - if ($substitutions !== '') { - foreach (explode('|', $substitutions) as $value) { - $fromto = explode('=', trim($value)); - if (count($fromto) >= 2) { - $text = str_replace(trim($fromto[0]), trim($fromto[1]), $text); - } - } - } - - return $text; - } - - /** - * Save parsing result to database. - * - * @param ParserResult $result Parsing result - * @param Language $language Language entity - * @param int $textId Text ID - * - * @return void - */ - protected function saveToDatabase( - ParserResult $result, - Language $language, - int $textId - ): void { - $lid = $language->id()->toInt(); - - // Clear temporary table - QueryBuilder::table('temp_word_occurrences')->truncate(); - - // Get next sentence ID - $dbname = Globals::getDatabaseName(); - $sentencesTable = Globals::table('sentences'); - $nextSeID = (int) Connection::preparedFetchValue( - "SELECT AUTO_INCREMENT as value FROM information_schema.TABLES - WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?", - [$dbname, $sentencesTable] - ); - if ($nextSeID <= 0) { - $bindings = []; - $nextSeID = (int) Connection::preparedFetchValue( - "SELECT IFNULL(MAX(`SeID`)+1,1) as value FROM sentences WHERE 1=1" - . UserScopedQuery::forTablePrepared('sentences', $bindings), - $bindings - ); - } - - // Insert tokens into temp_word_occurrences - $this->insertTokensToTemp($result, $nextSeID); - - // Check for multi-word expressions - $hasMultiword = $this->checkExpressions($lid); - - // Register sentences and text items - $this->registerSentencesTextItems($textId, $lid, $hasMultiword); - } - - /** - * Insert tokens into temp_word_occurrences table. - * - * @param ParserResult $result Parsing result - * @param int $startSeID Starting sentence ID - * - * @return void - */ - protected function insertTokensToTemp(ParserResult $result, int $startSeID): void - { - $tokens = $result->getTokens(); - if (empty($tokens)) { - return; - } - - $rows = []; - $currentSeID = $startSeID; - $lastSentenceIndex = -1; - $count = 0; - - foreach ($tokens as $token) { - // Track sentence changes - if ($token->getSentenceIndex() !== $lastSentenceIndex) { - if ($lastSentenceIndex >= 0) { - $currentSeID++; - $count = 0; - } - $lastSentenceIndex = $token->getSentenceIndex(); - } - - $tiSeID = $currentSeID; - $tiCount = $count + 1; - $count += mb_strlen($token->getText()); - $tiOrder = $token->getOrder() + 1; // 1-based in DB - $tiWordCount = $token->isWord() ? 1 : 0; - - $rows[] = [$tiSeID, $tiCount, $tiOrder, $token->getText(), $tiWordCount]; - } - - $placeholders = []; - $flatParams = []; - foreach ($rows as $row) { - $placeholders[] = "(?, ?, ?, ?, ?)"; - $flatParams = array_merge($flatParams, $row); - } - - Connection::preparedExecute( - "INSERT INTO temp_word_occurrences ( - TiSeID, TiCount, TiOrder, TiText, TiWordCount - ) VALUES " . implode(',', $placeholders), - $flatParams - ); - } - - /** - * Check for multi-word expressions and populate tempexprs. - * - * @param int $lid Language ID - * - * @return bool True if multi-word expressions were found - */ - protected function checkExpressions(int $lid): bool - { - // This is a simplified version - the full implementation is in TextParsing - $bindings = [$lid, $lid]; - $result = Connection::preparedFetchAll( - "SELECT WoWordCount FROM words WHERE WoWordCount > 1 AND WoLgID = ?" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " GROUP BY WoWordCount", - $bindings - ); - - return !empty($result); - } - - /** - * Register sentences and text items in the database. - * - * @param int $tid Text ID - * @param int $lid Language ID - * @param bool $hasmultiword Whether to process multi-word expressions - * - * @return void - */ - protected function registerSentencesTextItems( - int $tid, - int $lid, - bool $hasmultiword - ): void { - // Insert sentences - Connection::query('SET @i=0;'); - Connection::preparedExecute( - "INSERT INTO sentences ( - SeLgID, SeTxID, SeOrder, SeFirstPos, SeText - ) SELECT - ?, - ?, - @i:=@i+1, - MIN(IF(TiWordCount=0, TiOrder+1, TiOrder)), - GROUP_CONCAT(TiText ORDER BY TiOrder SEPARATOR \"\") - FROM temp_word_occurrences - GROUP BY TiSeID - ORDER BY TiSeID", - [$lid, $tid] - ); - - // Insert text items - if ($hasmultiword) { - // Build SQL and bindings in lockstep so that the two user-scope - // injections inside the UNION land in left-to-right placeholder - // order. Pre-bundling all six values up-front and appending the - // two $userIds at the end mismatches positions 3-7 and swaps - // Ti2LgID/Ti2TxID, triggering an FK violation on word_occurrences. - $bindings = [$lid, $tid]; - $sql = "INSERT INTO word_occurrences ( - Ti2WoID, Ti2LgID, Ti2TxID, Ti2SeID, Ti2Order, Ti2WordCount, Ti2Text - ) SELECT WoID, ?, ?, sent, TiOrder - (2*(n-1)) TiOrder, - n TiWordCount, word - FROM tempexprs - JOIN words - ON WoTextLC = lword AND WoWordCount = n" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " WHERE lword IS NOT NULL AND WoLgID = ?"; - $bindings[] = $lid; - $bindings[] = $lid; - $bindings[] = $tid; - $sql .= " UNION ALL - SELECT WoID, ?, ?, TiSeID, TiOrder, TiWordCount, TiText - FROM temp_word_occurrences - LEFT JOIN words - ON LOWER(TiText) = WoTextLC AND TiWordCount=1 AND WoLgID = ?"; - $bindings[] = $lid; - $sql .= UserScopedQuery::forTablePrepared('words', $bindings, '') - . " ORDER BY TiOrder, TiWordCount"; - - $stmt = Connection::prepare($sql); - $stmt->bindValues($bindings); - $stmt->execute(); - } else { - $bindings = [$lid, $tid, $lid]; - Connection::preparedExecute( - "INSERT INTO word_occurrences ( - Ti2WoID, Ti2LgID, Ti2TxID, Ti2SeID, Ti2Order, Ti2WordCount, Ti2Text - ) SELECT WoID, ?, ?, TiSeID, TiOrder, TiWordCount, TiText - FROM temp_word_occurrences - LEFT JOIN words - ON LOWER(TiText) = WoTextLC AND TiWordCount=1 AND WoLgID = ?" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " ORDER BY TiSeID, TiOrder, TiWordCount", - $bindings - ); - } - } - - /** - * Get the parser registry. - * - * @return ParserRegistry Parser registry - */ - public function getRegistry(): ParserRegistry - { - return $this->registry; - } -} diff --git a/src/Modules/Language/LanguageServiceProvider.php b/src/Modules/Language/LanguageServiceProvider.php index ae26161e5..08759e013 100644 --- a/src/Modules/Language/LanguageServiceProvider.php +++ b/src/Modules/Language/LanguageServiceProvider.php @@ -47,7 +47,6 @@ // Parser Infrastructure use Lwt\Modules\Language\Infrastructure\Parser\ExternalParserLoader; use Lwt\Modules\Language\Infrastructure\Parser\ParserRegistry; -use Lwt\Modules\Language\Application\Services\ParsingCoordinator; /** * Service provider for the Language module. @@ -85,12 +84,6 @@ public function register(Container $container): void ); }); - $container->singleton(ParsingCoordinator::class, function (Container $c) { - return new ParsingCoordinator( - $c->getTyped(ParserRegistry::class) - ); - }); - // Register Repository Interface binding $container->singleton(LanguageRepositoryInterface::class, function (Container $_c) { return new MySqlLanguageRepository(); diff --git a/src/Modules/Vocabulary/Application/Services/CompleteImportService.php b/src/Modules/Vocabulary/Application/Services/CompleteImportService.php index 728be73ad..89a811800 100644 --- a/src/Modules/Vocabulary/Application/Services/CompleteImportService.php +++ b/src/Modules/Vocabulary/Application/Services/CompleteImportService.php @@ -20,6 +20,7 @@ use Lwt\Shared\Infrastructure\Database\Connection; use Lwt\Shared\Infrastructure\Database\DB; use Lwt\Shared\Infrastructure\Database\QueryBuilder; +use Lwt\Shared\Infrastructure\Database\ScratchTables; use Lwt\Shared\Infrastructure\Database\Settings; use Lwt\Shared\Infrastructure\Database\UserScopedQuery; use Lwt\Modules\Tags\Application\TagsFacade; @@ -131,6 +132,10 @@ public function importComplete( */ private function initTempTables(): void { + // Per-connection scratch table for the import rows (was a persistent + // shared table; see ScratchTables for why it is now TEMPORARY). + ScratchTables::ensureWords(); + Connection::execute("DELETE FROM temp_words"); Connection::execute( "CREATE TEMPORARY TABLE IF NOT EXISTS numbers( n tinyint(3) unsigned NOT NULL @@ -646,8 +651,11 @@ private function handleTagsImport(int $langId): void */ private function cleanupTempTables(): void { - Connection::execute("DROP TABLE IF EXISTS numbers"); - QueryBuilder::table('temp_words')->truncate(); + // DROP TEMPORARY (not DROP TABLE / TRUNCATE): the former does not + // implicitly commit, so it is safe to call inside the import transaction. + // Both tables are recreated by initTempTables() on the next import. + Connection::execute("DROP TEMPORARY TABLE IF EXISTS numbers"); + Connection::execute("DROP TEMPORARY TABLE IF EXISTS temp_words"); } /** @@ -662,6 +670,10 @@ private function cleanupTempTables(): void */ public function importTagsOnly(array $fields, string $tabType, string $fileName, bool $ignoreFirst): void { + // Create the per-connection scratch tables (temp_words + numbers) before + // loading the tag rows into temp_words below. + $this->initTempTables(); + $columns = ''; $tlField = $fields["tl"]; for ($j = 1; $j <= $tlField; $j++) { @@ -736,15 +748,6 @@ public function importTagsOnly(array $fields, string $tabType, string $fileName, } } - // Create numbers table and insert tags - Connection::execute( - "CREATE TEMPORARY TABLE IF NOT EXISTS numbers( - n tinyint(3) unsigned NOT NULL - )" - ); - Connection::execute("INSERT IGNORE INTO numbers(n) VALUES ('1'),('2'),('3'), - ('4'),('5'),('6'),('7'),('8'),('9')"); - // Stamp TgUsID so the imported tags are owned by the caller — // see handleTagsImport() for context. Without this the rows // land with TgUsID NULL and never appear on the user's tag diff --git a/src/Shared/Infrastructure/Database/JapaneseTextParser.php b/src/Shared/Infrastructure/Database/JapaneseTextParser.php index ae37a920e..d420205d2 100644 --- a/src/Shared/Infrastructure/Database/JapaneseTextParser.php +++ b/src/Shared/Infrastructure/Database/JapaneseTextParser.php @@ -23,7 +23,8 @@ /** * Japanese text parsing using MeCab. * - * Handles splitting, previewing, and database insertion for Japanese text. + * Handles splitting, previewing, and tokenization for Japanese text. Produces + * ParsedToken objects consumed by TokenPersistence — no scratch table involved. * * @since 3.0.0 */ @@ -63,14 +64,13 @@ public static function displayJapanesePreview(string $text): void } /** - * Parse Japanese text with MeCab and insert into temp_word_occurrences. + * Tokenize Japanese text with MeCab into ParsedToken objects. * - * @param string $text Preprocessed text - * @param bool $useMaxSeID Whether to query for max sentence ID (true for existing texts) + * @param string $text Preprocessed text (character substitutions applied) * - * @return void + * @return ParsedToken[] */ - public static function parseJapaneseToDatabase(string $text, bool $useMaxSeID): void + public static function tokenize(string $text): array { $text = preg_replace('/[ \t]+/u', ' ', $text) ?? $text; $text = trim($text); @@ -81,7 +81,7 @@ public static function parseJapaneseToDatabase(string $text, bool $useMaxSeID): } try { - // We use the format "word num num" for all nodes + // We use the format "word\ttype\tnode" for all nodes. $mecab_args = " -F %m\\t%t\\t%h\\n -U %m\\t%t\\t%h\\n -E EOP\\t3\\t7\\n"; $mecab_args .= " -o " . escapeshellarg($file_name) . " "; $mecab = (new TextParsingService())->getMecabPath($mecab_args); @@ -93,15 +93,6 @@ public static function parseJapaneseToDatabase(string $text, bool $useMaxSeID): pclose($handle); } - Connection::execute( - "CREATE TEMPORARY TABLE IF NOT EXISTS tempword_occurrences ( - TiCount smallint(5) unsigned NOT NULL, - TiSeID mediumint(8) unsigned NOT NULL, - TiOrder smallint(5) unsigned NOT NULL, - TiWordCount tinyint(3) unsigned NOT NULL, - TiText varchar(250) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL - ) DEFAULT CHARSET=utf8" - ); $handle = fopen($file_name, 'r'); $mecabed = ''; if ($handle !== false) { @@ -112,110 +103,7 @@ public static function parseJapaneseToDatabase(string $text, bool $useMaxSeID): } fclose($handle); } - $values = array(); - $order = 0; - $sid = 1; - if ($useMaxSeID) { - $sid = (int)Connection::fetchValue( - "SELECT IFNULL(MAX(`SeID`)+1,1) as value FROM sentences" - . UserScopedQuery::forTable('sentences') - ); - } - $term_type = 0; - $last_node_type = 0; - $count = 0; - $row = array(0, 0, 0, "", 0); - $separator = mb_chr(9); - if ($separator === false) { - $separator = "\t"; - } - foreach (explode(PHP_EOL, $mecabed) as $line) { - if (trim($line) == "") { - continue; - } - $parts = explode($separator, $line); - $term = $parts[0] ?? ''; - $node_type = $parts[1] ?? ''; - $third = $parts[2] ?? ''; - if ($term_type == 2 || $term == 'EOP' && $third == '7') { - $sid += 1; - } - $row[0] = $sid; // TiSeID - $row[1] = $count + 1; // TiCount - $count += mb_strlen($term); - $last_term_type = $term_type; - if ($third == '7') { - if ($term == 'EOP') { - $term = '¶'; - } - $term_type = 2; - } elseif (in_array($node_type, ['2', '6', '7', '8'])) { - $term_type = 0; - } else { - $term_type = 1; - } - - // Increase word order: - // Once if the current or the previous term were words - // Twice if current or the previous were not of unmanaged type - $order += (int)($term_type == 0 && $last_term_type == 0) + - (int)($term_type != 1 || $last_term_type != 1); - $row[2] = $order; // TiOrder - $row[3] = $term; // TiText (no escaping needed for prepared statement) - $row[4] = $term_type == 0 ? 1 : 0; // TiWordCount - $values[] = $row; - // Special case for kazu (numbers) - if ($last_node_type == 8 && $node_type == 8) { - $lastKey = array_key_last($values); - // $lastKey is int<0, max> since we just added an element - // We need at least 2 elements to access previous - if ($lastKey > 0 && isset($values[$lastKey - 1][3])) { - // Concatenate the previous value with the current term - $values[$lastKey - 1][3] = $values[$lastKey - 1][3] . $term; - // Remove last element to avoid repetition - array_pop($values); - } - } - $last_node_type = $node_type; - } - - // Build multi-row INSERT with prepared statement - // Generate placeholders for all rows: (?, ?, ?, ?, ?), (?, ?, ?, ?, ?), ... - $placeholders = array(); - $flatParams = array(); - foreach ($values as $row) { - $placeholders[] = "(?, ?, ?, ?, ?)"; - // Flatten the row values into a single array for binding - $flatParams[] = $row[0]; // TiSeID - $flatParams[] = $row[1]; // TiCount - $flatParams[] = $row[2]; // TiOrder - $flatParams[] = $row[3]; // TiText - $flatParams[] = $row[4]; // TiWordCount - } - - if (!empty($placeholders)) { - Connection::preparedExecute( - "INSERT INTO tempword_occurrences ( - TiSeID, TiCount, TiOrder, TiText, TiWordCount - ) VALUES " . implode(',', $placeholders), - $flatParams - ); - } - // Delete elements TiOrder=@order - Connection::preparedExecute( - "DELETE FROM tempword_occurrences WHERE TiOrder=?", - [$order] - ); - Connection::query( - "INSERT INTO temp_word_occurrences ( - TiCount, TiSeID, TiOrder, TiWordCount, TiText - ) - SELECT MIN(TiCount) s, TiSeID, TiOrder, TiWordCount, - group_concat(TiText ORDER BY TiCount SEPARATOR '') - FROM tempword_occurrences - GROUP BY TiOrder" - ); - Connection::execute("DROP TABLE tempword_occurrences"); + return self::buildTokensFromMecab($mecabed); } finally { if (file_exists($file_name)) { unlink($file_name); @@ -224,29 +112,97 @@ public static function parseJapaneseToDatabase(string $text, bool $useMaxSeID): } /** - * Parse a Japanese text using MeCab and add it to the database. - * - * @param string $text Text to parse. - * @param int $id Text ID. If $id = -1 print results, - * if $id = -2 return splitted texts + * Convert raw MeCab output into ParsedToken objects. * - * @return null|string[] Splitted sentence if $id = -2 + * Kept separate from the MeCab subprocess call so it can be unit-tested with + * captured/synthetic MeCab output. Mirrors the former SQL pipeline: build + * staging rows, drop the final EOP order group, then group rows sharing a + * TiOrder into one token (text concatenated in TiCount order). * - * @psalm-return non-empty-list|null + * @param string $mecabed Raw MeCab output ("word\ttype\tnode" per line) * - * @internal Use TextParsing::splitIntoSentences(), parseAndDisplayPreview(), or parseAndSave() instead. + * @return ParsedToken[] */ - public static function parseJapanese(string $text, int $id): ?array + public static function buildTokensFromMecab(string $mecabed): array { - if ($id == -2) { - return self::splitJapaneseSentences($text); + /** @var array $values */ + $values = array(); + $order = 0; + $sid = 1; + $term_type = 0; + $last_node_type = 0; + $count = 0; + $separator = mb_chr(9); + if ($separator === false) { + $separator = "\t"; } + foreach (explode(PHP_EOL, $mecabed) as $line) { + if (trim($line) == "") { + continue; + } + $parts = explode($separator, $line); + $term = $parts[0] ?? ''; + $node_type = $parts[1] ?? ''; + $third = $parts[2] ?? ''; + if ($term_type == 2 || $term == 'EOP' && $third == '7') { + $sid += 1; + } + $tiSeID = $sid; + $tiCount = $count + 1; + $count += mb_strlen($term); + $last_term_type = $term_type; + if ($third == '7') { + if ($term == 'EOP') { + $term = '¶'; + } + $term_type = 2; + } elseif (in_array($node_type, ['2', '6', '7', '8'])) { + $term_type = 0; + } else { + $term_type = 1; + } - if ($id == -1) { - self::displayJapanesePreview($text); + // Increase word order: + // Once if the current or the previous term were words + // Twice if current or the previous were not of unmanaged type + $order += (int)($term_type == 0 && $last_term_type == 0) + + (int)($term_type != 1 || $last_term_type != 1); + $tiWordCount = $term_type == 0 ? 1 : 0; + $values[] = array($tiSeID, $tiCount, $order, $term, $tiWordCount); + + // Special case for kazu (numbers): merge consecutive number nodes. + if ($last_node_type == 8 && $node_type == 8) { + $lastKey = array_key_last($values); + if ($lastKey > 0 && isset($values[$lastKey - 1][3])) { + $values[$lastKey - 1][3] = $values[$lastKey - 1][3] . $term; + array_pop($values); + } + } + $last_node_type = $node_type; } - self::parseJapaneseToDatabase($text, $id > 0); - return null; + // Group staging rows by TiOrder, dropping the final EOP order group. + // (This replaces "DELETE WHERE TiOrder=@order" + "GROUP BY TiOrder".) + /** @var array> $groups */ + $groups = array(); + foreach ($values as $r) { + if ($r[2] === $order) { + continue; + } + $groups[$r[2]][] = $r; + } + ksort($groups); + + $tokens = array(); + foreach ($groups as $ord => $rows) { + // Concatenate merged node text in TiCount order. + usort($rows, fn(array $a, array $b): int => $a[1] <=> $b[1]); + $text = ''; + foreach ($rows as $rr) { + $text .= $rr[3]; + } + $tokens[] = new ParsedToken($rows[0][0], $ord, $rows[0][4], $text); + } + return $tokens; } } diff --git a/src/Shared/Infrastructure/Database/ParsedToken.php b/src/Shared/Infrastructure/Database/ParsedToken.php new file mode 100644 index 000000000..a2d55b6a7 --- /dev/null +++ b/src/Shared/Infrastructure/Database/ParsedToken.php @@ -0,0 +1,64 @@ + + * @license Unlicense + * @link https://hugofara.github.io/lwt/developer/api + * @since 3.2.2 + */ + +declare(strict_types=1); + +namespace Lwt\Shared\Infrastructure\Database; + +/** + * One token produced while parsing a text: a word or a non-word run + * (whitespace / punctuation). + * + * This replaces the rows that used to be written to the `temp_word_occurrences` + * scratch table during parsing. Tokens are produced fully in PHP and consumed + * by TokenPersistence, so no scratch table is needed. See ScratchTables for the + * history of why the old temp tables were removed. + * + * Field meanings mirror the former temp_word_occurrences columns: + * - $sentence: 1-based sentence index within this text (was TiSeID, but local, + * not a pre-computed SeID). + * - $order: global monotonic token order across the whole text (was TiOrder). + * - $wordCount: 1 for a word token, 0 for a non-word run (was TiWordCount). + * - $text: the token text (was TiText). + * + * @since 3.2.2 + */ +final class ParsedToken +{ + /** + * @param int $sentence 1-based sentence index within the text + * @param int $order Global monotonic token order across the text + * @param int $wordCount 1 for a word, 0 for a non-word run + * @param string $text The token text + */ + public function __construct( + public readonly int $sentence, + public readonly int $order, + public readonly int $wordCount, + public readonly string $text + ) { + } + + /** + * Whether this token is a word (as opposed to whitespace/punctuation). + * + * @return bool + */ + public function isWord(): bool + { + return $this->wordCount !== 0; + } +} diff --git a/src/Shared/Infrastructure/Database/ScratchTables.php b/src/Shared/Infrastructure/Database/ScratchTables.php new file mode 100644 index 000000000..28ae11ba5 --- /dev/null +++ b/src/Shared/Infrastructure/Database/ScratchTables.php @@ -0,0 +1,75 @@ + + * @license Unlicense + * @link https://hugofara.github.io/lwt/developer/api + * @since 3.2.2 + */ + +declare(strict_types=1); + +namespace Lwt\Shared\Infrastructure\Database; + +/** + * Creates the per-connection scratch table used while importing vocabulary. + * + * `temp_words` used to be a persistent, globally-shared InnoDB table that every + * import `TRUNCATE`d and refilled. That design caused two problems: + * + * - **Tablespace crashes.** On file-per-table InnoDB, `TRUNCATE` physically + * drops and recreates the `.ibd`. When that file went missing (notably on + * Windows) the table was left with a missing tablespace and every subsequent + * import failed with InnoDB error 194 "Tablespace is missing for a table" + * (see issue #247). + * - **Concurrency corruption.** Because the table was shared, two imports + * running at once (or any two users in multi-user mode) read and truncated + * each other's rows. + * + * It is now created per database connection with `CREATE TEMPORARY TABLE`, so + * each request gets its own private copy. Temporary InnoDB tables live in the + * shared session temporary tablespace (there is no per-table `.ibd` to orphan), + * and they are dropped automatically when the connection closes. + * + * Callers must `ensureWords()` before use, then clear rows with `DELETE FROM` — + * never `TRUNCATE`, which is DDL that recreates the tablespace and implicitly + * commits any open transaction. + * + * (Text parsing no longer uses any scratch table — see TokenPersistence.) + * + * @since 3.2.2 + */ +final class ScratchTables +{ + /** + * Ensure the word-import staging table exists for this connection. + * + * Holds the rows of a vocabulary import file before they are merged into + * the `words` table. + * + * @return void + */ + public static function ensureWords(): void + { + Connection::query( + <<<'SQL' + CREATE TEMPORARY TABLE IF NOT EXISTS temp_words ( + WoText varchar(250) DEFAULT NULL, + WoTextLC varchar(250) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, + WoTranslation varchar(500) NOT NULL DEFAULT '*', + WoRomanization varchar(100) DEFAULT NULL, + WoSentence varchar(1000) DEFAULT NULL, + WoTaglist varchar(255) DEFAULT NULL, + PRIMARY KEY(WoTextLC) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci + SQL + ); + } +} diff --git a/src/Shared/Infrastructure/Database/StandardTextParser.php b/src/Shared/Infrastructure/Database/StandardTextParser.php index 4d2904697..e7486f94a 100644 --- a/src/Shared/Infrastructure/Database/StandardTextParser.php +++ b/src/Shared/Infrastructure/Database/StandardTextParser.php @@ -189,23 +189,21 @@ public static function displayStandardPreview(string $text, bool $rtlScript): vo } /** - * Parse standard text and insert into temp_word_occurrences. + * Build the tab-delimited token blob from preprocessed text. * - * @param string $text Preprocessed text + * Produces one line per token as "\t", where a term + * ending in "\r" marks the end of a sentence. This is the same + * serialization the old LOAD DATA path consumed; parseBlob() turns it + * into ParsedToken objects. + * + * @param string $text Text after word-splitting transformations * @param string $termchar Word character regex * @param string $removeSpaces Space removal setting - * @param bool $useMaxSeID Whether to query for max sentence ID - * - * @return void * - * @psalm-suppress MixedArgument + * @return string */ - public static function parseStandardToDatabase( - string $text, - string $termchar, - string $removeSpaces, - bool $useMaxSeID - ): void { + private static function buildTokenBlob(string $text, string $termchar, string $removeSpaces): string + { $qc = self::quoteChars(); $replaced = preg_replace( array( @@ -225,136 +223,120 @@ public static function parseStandardToDatabase( str_replace(array("\t", "\n\n"), array("\n", ""), $text) ); $text = trim($replaced ?? $text); - $text = StringUtils::removeSpaces( + return StringUtils::removeSpaces( preg_replace("/(\n|^)(?!1\t)/u", "\n0\t", $text) ?? $text, $removeSpaces ); + } - // It is faster to write to a file and let SQL do its magic, but may run into - // security restrictions - $use_local_infile = in_array( - Connection::fetchValue("SELECT @@GLOBAL.local_infile as value"), - array(1, '1', 'ON') - ); - // For database mode, we use a positive ID placeholder (1) since saveWithSql - // only checks if id > 0 for sentence ID calculation - $idForSql = $useMaxSeID ? 1 : 0; - if ($use_local_infile) { - TextParsingPersistence::saveWithSql($text, $idForSql); - } else { - $order = 0; - $sid = 1; - if ($useMaxSeID) { - // Get next auto-increment value from table status - // This is more reliable than MAX(SeID)+1 when there are gaps - $dbname = Globals::getDatabaseName(); - $sentencesTable = Globals::table('sentences'); - $sid = (int)Connection::preparedFetchValue( - "SELECT AUTO_INCREMENT as value FROM information_schema.TABLES - WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?", - [$dbname, $sentencesTable] - ); - // Fall back to MAX+1 if AUTO_INCREMENT is not available - if ($sid <= 0) { - $sid = (int)Connection::fetchValue( - "SELECT IFNULL(MAX(`SeID`)+1,1) as value FROM sentences" - . UserScopedQuery::forTable('sentences') - ); - } + /** + * Turn the token blob into ParsedToken objects. + * + * Replicates the semantics of the former LOAD DATA `SET` clause: each line + * is "\t"; a term ending with "\r" ends the current + * sentence (that token still belongs to the current sentence, the next one + * starts a new sentence). Order is a global 1-based counter. + * + * Unlike the old saveWithSqlFallback(), this does NOT trim the line, so the + * "\r" sentence markers and trailing-space tokens are preserved (that bug + * caused LOAD-DATA-less installs to parse every text as one sentence). + * + * @param string $blob Token blob from buildTokenBlob() + * + * @return ParsedToken[] + */ + private static function parseBlob(string $blob): array + { + $tokens = []; + $sentence = 1; + $order = 0; + foreach (explode("\n", $blob) as $line) { + $tab = strpos($line, "\t"); + if ($tab === false) { + // Blank or malformed line (e.g. a stray empty line): no token. + continue; } - $count = 0; - $rows = array(); - foreach (explode("\n", $text) as $line) { - if (trim($line) == "") { - continue; - } - list($word_count, $term) = explode("\t", $line); - $tiSeID = $sid; // TiSeID - $tiCount = $count + 1; // TiCount - $count += mb_strlen($term); - if (str_ends_with($term, "\r")) { - $term = str_replace("\r", '', $term); - $sid++; - $count = 0; - } - $tiOrder = ++$order; // TiOrder - $tiWordCount = (int)$word_count; // TiWordCount - $rows[] = array($tiSeID, $tiCount, $tiOrder, $term, $tiWordCount); + $wordCount = (int) substr($line, 0, $tab); + $term = substr($line, $tab + 1); + $sentenceEnd = str_ends_with($term, "\r"); + if ($sentenceEnd) { + $term = str_replace("\r", '', $term); } - - // Build multi-row INSERT with prepared statement - if (!empty($rows)) { - $placeholders = array(); - $flatParams = array(); - foreach ($rows as $row) { - $placeholders[] = "(?, ?, ?, ?, ?)"; - $flatParams = array_merge($flatParams, $row); - } - - Connection::preparedExecute( - "INSERT INTO temp_word_occurrences ( - TiSeID, TiCount, TiOrder, TiText, TiWordCount - ) VALUES " . implode(',', $placeholders), - $flatParams - ); + $order++; + $tokens[] = new ParsedToken($sentence, $order, $wordCount, $term); + if ($sentenceEnd) { + $sentence++; } } + return $tokens; } /** - * Parse a text using the default tools. It is a not-japanese text. - * - * @param string $text Text to parse - * @param int $id Text ID. If $id == -2, only split the text. - * @param int $lid Language ID. - * - * @return null|string[] If $id == -2 return a splitted version of the text. + * Tokenize a (character-substituted) standard text into ParsedToken objects. * - * @psalm-return non-empty-list|null + * @param string $text Preprocessed text (character substitutions applied) + * @param int $lid Language ID * - * @internal Use TextParsing::splitIntoSentences(), parseAndDisplayPreview(), or parseAndSave() instead. + * @return ParsedToken[] */ - public static function parseStandard(string $text, int $id, int $lid): ?array + public static function tokenize(string $text, int $lid): array { $settings = self::getLanguageSettings($lid); - - // Return null if language not found if ($settings === null) { - return null; + return []; } - - // Apply initial transformations (paragraph markers, trim, collapse spaces) - $text = self::applyInitialTransformations( + $text = self::applyInitialTransformations($text, $settings['splitEachChar']); + $text = self::applyWordSplitting( $text, - $settings['splitEachChar'] + $settings['splitSentence'], + $settings['noSentenceEnd'], + $settings['termchar'] ); + $blob = self::buildTokenBlob($text, $settings['termchar'], $settings['removeSpaces']); + return self::parseBlob($blob); + } - // Preview mode - display HTML BEFORE word splitting - if ($id == -1) { - self::displayStandardPreview($text, (bool)$settings['rtlScript']); + /** + * Split a (character-substituted) standard text into sentences only. + * + * @param string $text Preprocessed text (character substitutions applied) + * @param int $lid Language ID + * + * @return string[] + * + * @psalm-return non-empty-list + */ + public static function splitSentences(string $text, int $lid): array + { + $settings = self::getLanguageSettings($lid); + if ($settings === null) { + return ['']; } - - // Apply word-splitting transformations + $text = self::applyInitialTransformations($text, $settings['splitEachChar']); $text = self::applyWordSplitting( $text, $settings['splitSentence'], $settings['noSentenceEnd'], $settings['termchar'] ); + return self::splitStandardSentences($text, $settings['removeSpaces']); + } - // Split-only mode - if ($id == -2) { - return self::splitStandardSentences($text, $settings['removeSpaces']); + /** + * Echo the preview HTML for a (character-substituted) standard text. + * + * @param string $text Preprocessed text (character substitutions applied) + * @param int $lid Language ID + * + * @return void + */ + public static function echoPreview(string $text, int $lid): void + { + $settings = self::getLanguageSettings($lid); + if ($settings === null) { + return; } - - // Database insertion (for both preview mode -1 and actual save mode > 0) - self::parseStandardToDatabase( - $text, - $settings['termchar'], - $settings['removeSpaces'], - $id > 0 - ); - - return null; + $text = self::applyInitialTransformations($text, $settings['splitEachChar']); + self::displayStandardPreview($text, (bool)$settings['rtlScript']); } } diff --git a/src/Shared/Infrastructure/Database/TextParsing.php b/src/Shared/Infrastructure/Database/TextParsing.php index 1accbe363..5cb5f3819 100644 --- a/src/Shared/Infrastructure/Database/TextParsing.php +++ b/src/Shared/Infrastructure/Database/TextParsing.php @@ -23,7 +23,9 @@ /** * Text parsing and processing utilities (facade). * - * Delegates to JapaneseTextParser, StandardTextParser, and TextParsingPersistence. + * Delegates tokenization to JapaneseTextParser / StandardTextParser and + * persistence to TokenPersistence. Parsing happens entirely in PHP — there are + * no scratch tables involved. * * @since 3.0.0 */ @@ -32,9 +34,6 @@ class TextParsing /** * Split text into sentences without database operations. * - * Use this method when you only need to split text into sentences - * without saving to the database (e.g., for long text splitting). - * * @param string $text Text to parse * @param int $lid Language ID * @@ -44,15 +43,21 @@ class TextParsing */ public static function splitIntoSentences(string $text, int $lid): array { - $result = self::prepare($text, -2, $lid); - return $result ?? ['']; + $pre = self::preprocess($text, $lid); + if ($pre === null) { + return ['']; + } + [$ptext, $isMecab] = $pre; + if ($isMecab) { + return JapaneseTextParser::splitJapaneseSentences($ptext); + } + return StandardTextParser::splitSentences($ptext, $lid); } /** * Parse text and display preview HTML for validation. * - * Use this method for the text checking UI. Outputs HTML directly - * to show parsed sentences and word statistics. + * Outputs HTML directly to show parsed sentences and word statistics. * * @param string $text Text to parse * @param int $lid Language ID @@ -71,33 +76,28 @@ public static function parseAndDisplayPreview(string $text, int $lid): void } $rtlScript = (bool)$record['LgRightToLeft']; - // Parse text and display preview HTML (id=-1 triggers preview display in prepare) - self::prepare($text, -1, $lid); - - // Display sentences and word statistics - TextParsingPersistence::checkValid($lid); - - // Get multi-word expressions - $wl = TextParsingPersistence::getMultiWordLengths($lid); - - // Process multi-word expressions if any exist - if (!empty($wl)) { - TextParsingPersistence::checkExpressions($wl); + $pre = self::preprocess($text, $lid); + if ($pre === null) { + return; + } + [$ptext, $isMecab] = $pre; + + // Preview HTML is shown before word splitting. + if ($isMecab) { + JapaneseTextParser::displayJapanesePreview($ptext); + $tokens = JapaneseTextParser::tokenize($ptext); + } else { + StandardTextParser::echoPreview($ptext, $lid); + $tokens = StandardTextParser::tokenize($ptext, $lid); } - // Display statistics - TextParsingPersistence::displayStatistics($lid, $rtlScript, !empty($wl)); - - // Clean up - QueryBuilder::table('temp_word_occurrences')->truncate(); + TokenPersistence::echoCheckValid($tokens, $lid); + TokenPersistence::echoStatistics($tokens, $lid, $rtlScript); } /** * Parse text and save to database. * - * Use this method when creating or updating texts. Parses the text - * and inserts sentences and text items into the database. - * * @param string $text Text to parse * @param int $lid Language ID * @param int $textId Text ID (must be positive) @@ -123,28 +123,13 @@ public static function parseAndSave(string $text, int $lid, int $textId): void throw DatabaseException::recordNotFound('languages', 'LgID', $lid); } - // Parse text into temp_word_occurrences (id>0 uses MAX(SeID)+1 for sentence IDs) - self::prepare($text, $textId, $lid); - - // Get multi-word expressions - $wl = TextParsingPersistence::getMultiWordLengths($lid); - - // Process multi-word expressions if any exist - if (!empty($wl)) { - TextParsingPersistence::checkExpressions($wl); - } - - // Register sentences and text items in database - TextParsingPersistence::registerSentencesTextItems($textId, $lid, !empty($wl)); - - // Clean up - QueryBuilder::table('temp_word_occurrences')->truncate(); + $tokens = self::tokenize($text, $lid); + TokenPersistence::save($tokens, $lid, $textId); } /** * Check/preview text and return parsing statistics without saving. * - * Use this method to get text statistics for preview purposes. * Does not output any HTML or save to database. * * @param string $text Text to parse @@ -154,100 +139,45 @@ public static function parseAndSave(string $text, int $lid, int $textId): void */ public static function checkText(string $text, int $lid): array { - $settings = StandardTextParser::getLanguageSettings($lid); - - if ($settings === null) { - return [ - 'sentences' => 0, - 'words' => 0, - 'unknownPercent' => 100.0, - 'preview' => '' - ]; - } - - // Prepare text into temp_word_occurrences - self::prepare($text, -1, $lid); - - // Get sentence count - $sentences = Connection::fetchAll( - 'SELECT GROUP_CONCAT(TiText ORDER BY TiOrder SEPARATOR "") - AS Sent FROM temp_word_occurrences GROUP BY TiSeID' - ); - $sentenceCount = count($sentences); - - // Build preview from first few sentences - $preview = ''; - $previewSentences = array_slice($sentences, 0, 3); - foreach ($previewSentences as $record) { - if ($preview !== '') { - $preview .= ' '; - } - $preview .= (string) ($record['Sent'] ?? ''); - } - if (count($sentences) > 3) { - $preview .= '...'; - } - - // Get word statistics - $bindings = [$lid]; - $rows = Connection::preparedFetchAll( - "SELECT COUNT(`TiOrder`) AS cnt, IF(0=TiWordCount,0,1) AS len, - LOWER(TiText) AS word, WoTranslation - FROM temp_word_occurrences - LEFT JOIN words ON LOWER(TiText)=WoTextLC AND WoLgID=?" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " GROUP BY LOWER(TiText)", - $bindings - ); - - $totalWords = 0; - $unknownWords = 0; - - foreach ($rows as $record) { - if ($record['len'] == 1) { - $totalWords += (int) $record['cnt']; - // Word is unknown if it has no translation - if (empty($record['WoTranslation'])) { - $unknownWords += (int) $record['cnt']; - } - } - } - - $unknownPercent = $totalWords > 0 - ? round(($unknownWords / $totalWords) * 100, 1) - : 100.0; - - // Clean up temp_word_occurrences - QueryBuilder::table('temp_word_occurrences')->truncate(); - - return [ - 'sentences' => $sentenceCount, - 'words' => $totalWords, - 'unknownPercent' => $unknownPercent, - 'preview' => $preview - ]; + $tokens = self::tokenize($text, $lid); + return TokenPersistence::stats($tokens, $lid); } /** - * Pre-parse the input text before a definitive parsing by a specialized parser. + * Tokenize a text into ParsedToken objects (no database writes). * * @param string $text Text to parse - * @param int $id Text ID * @param int $lid Language ID * - * @return null|string[] If $id = -2 return a splitted version of the text + * @return ParsedToken[] + */ + private static function tokenize(string $text, int $lid): array + { + $pre = self::preprocess($text, $lid); + if ($pre === null) { + return []; + } + [$ptext, $isMecab] = $pre; + return $isMecab + ? JapaneseTextParser::tokenize($ptext) + : StandardTextParser::tokenize($ptext, $lid); + } + + /** + * Apply the language's text preprocessing (escaping + character + * substitutions) and report whether it uses the MeCab parser. * - * @psalm-return non-empty-list|null + * @param string $text Raw text + * @param int $lid Language ID * - * @internal Use splitIntoSentences(), parseAndDisplayPreview(), or parseAndSave() instead. + * @return array{0: string, 1: bool}|null [preprocessed text, isMecab] or null if language missing */ - private static function prepare(string $text, int $id, int $lid): ?array + private static function preprocess(string $text, int $lid): ?array { $record = QueryBuilder::table('languages') ->where('LgID', '=', $lid) ->firstPrepared(); - // Return null if language not found if ($record === null) { return null; } @@ -255,7 +185,6 @@ private static function prepare(string $text, int $id, int $lid): ?array $termchar = (string)$record['LgRegexpWordCharacters']; $replace = explode("|", (string) $record['LgCharacterSubstitutions']); $text = Escaping::prepareTextdata($text); - QueryBuilder::table('temp_word_occurrences')->truncate(); // because of sentence special characters $text = str_replace(array('}', '{'), array(']', '['), $text); @@ -266,9 +195,6 @@ private static function prepare(string $text, int $id, int $lid): ?array } } - if ('MECAB' == strtoupper(trim($termchar))) { - return JapaneseTextParser::parseJapanese($text, $id); - } - return StandardTextParser::parseStandard($text, $id, $lid); + return [$text, 'MECAB' === strtoupper(trim($termchar))]; } } diff --git a/src/Shared/Infrastructure/Database/TextParsingPersistence.php b/src/Shared/Infrastructure/Database/TextParsingPersistence.php deleted file mode 100644 index 9d6fe0156..000000000 --- a/src/Shared/Infrastructure/Database/TextParsingPersistence.php +++ /dev/null @@ -1,518 +0,0 @@ - - * @license Unlicense - * @link https://hugofara.github.io/lwt/developer/api - * @since 3.0.0 - */ - -declare(strict_types=1); - -namespace Lwt\Shared\Infrastructure\Database; - -use Lwt\Shared\Infrastructure\Globals; - -/** - * Shared database operations for text parsing. - * - * Handles SQL-based text saving, sentence/text-item registration, - * statistics display, and multi-word expression checking. - * - * @since 3.0.0 - */ -class TextParsingPersistence -{ - /** - * Insert a processed text in the data in pure SQL way. - * - * @param string $text Preprocessed text to insert - * @param int $id Text ID - * - * @return void - */ - public static function saveWithSql(string $text, int $id): void - { - $file_name = sys_get_temp_dir() . DIRECTORY_SEPARATOR . "tmpti.txt"; - $fp = fopen($file_name, 'w'); - if ($fp !== false) { - fwrite($fp, $text); - fclose($fp); - } - Connection::query("SET @order=0, @sid=1, @count = 0;"); - if ($id > 0) { - // Get next auto-increment value for accurate TiSeID calculation - $dbname = Globals::getDatabaseName(); - $sentencesTable = Globals::table('sentences'); - $autoInc = (int)Connection::preparedFetchValue( - "SELECT AUTO_INCREMENT as value FROM information_schema.TABLES - WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?", - [$dbname, $sentencesTable] - ); - // Fall back to MAX+1 if AUTO_INCREMENT is not available - if ($autoInc <= 0) { - $autoInc = (int)Connection::fetchValue( - "SELECT IFNULL(MAX(`SeID`)+1,1) as value FROM sentences" - . UserScopedQuery::forTable('sentences') - ); - } - Connection::query("SET @sid = $autoInc;"); - } - // LOAD DATA LOCAL INFILE does not support prepared statements for file path - // We need to use Connection::query() here, but we escape the file path manually - $connection = Globals::getDbConnection(); - if ($connection === null) { - throw new \RuntimeException('Database connection not available'); - } - $escaped_file_name = (string) mysqli_real_escape_string($connection, $file_name); - $sql = "LOAD DATA LOCAL INFILE '$escaped_file_name' - INTO TABLE temp_word_occurrences - FIELDS TERMINATED BY '\\t' LINES TERMINATED BY '\\n' (@word_count, @term) - SET - TiSeID = @sid, - TiCount = (@count:=@count+CHAR_LENGTH(@term))+1-CHAR_LENGTH(@term), - TiOrder = IF( - @term LIKE '%\\r', - CASE - WHEN (@term:=REPLACE(@term,'\\r','')) IS NULL THEN NULL - WHEN (@sid:=@sid+1) IS NULL THEN NULL - WHEN @count:= 0 IS NULL THEN NULL - ELSE @order := @order+1 - END, - @order := @order+1 - ), - TiText = @term, - TiWordCount = @word_count"; - - // Try LOAD DATA LOCAL INFILE, fall back to INSERT if it fails - try { - Connection::query($sql); - } catch (\RuntimeException $e) { - // If LOAD DATA LOCAL INFILE is disabled, use fallback method - if (strpos($e->getMessage(), 'LOAD DATA LOCAL INFILE is forbidden') !== false) { - self::saveWithSqlFallback($text, $id); - } else { - throw $e; - } - } - unlink($file_name); - } - - /** - * Fallback method to insert text data when LOAD DATA LOCAL INFILE is disabled. - * - * @param string $text Preprocessed text to insert - * @param int $id Text ID - * - * @return void - */ - public static function saveWithSqlFallback(string $text, int $id): void - { - // Get starting sentence ID - $sid = 1; - if ($id > 0) { - // Get next auto-increment value for accurate TiSeID calculation - $dbname = Globals::getDatabaseName(); - $sentencesTable = Globals::table('sentences'); - $sid = (int)Connection::preparedFetchValue( - "SELECT AUTO_INCREMENT as value FROM information_schema.TABLES - WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?", - [$dbname, $sentencesTable] - ); - // Fall back to MAX+1 if AUTO_INCREMENT is not available - if ($sid <= 0) { - $sid = (int)Connection::fetchValue( - "SELECT IFNULL(MAX(`SeID`)+1,1) as value FROM sentences" - . UserScopedQuery::forTable('sentences') - ); - } - } - - $lines = explode("\n", $text); - $order = 0; - $count = 0; - - foreach ($lines as $line) { - $line = trim($line); - if ($line === '') { - continue; - } - - $parts = explode("\t", $line); - if (count($parts) < 2) { - continue; - } - - $word_count = (int)$parts[0]; - $term = $parts[1]; - - // Handle line breaks (increase sentence ID) - if (substr($term, -1) === "\r") { - $term = rtrim($term, "\r"); - $order++; - $count = 0; - $sid++; - } else { - $order++; - } - - $current_count = $count; - $count += strlen($term) + 1; - - Connection::preparedExecute( - "INSERT INTO temp_word_occurrences - (TiSeID, TiCount, TiOrder, TiText, TiWordCount) - VALUES (?, ?, ?, ?, ?)", - [$sid, $current_count, $order, $term, $word_count] - ); - } - } - - /** - * Echo the sentences in a text. Prepare JS data for words and word count. - * - * @param int $lid Language ID - * - * @return void - */ - public static function checkValid(int $lid): void - { - $wo = $nw = array(); - $sentences = Connection::fetchAll( - 'SELECT GROUP_CONCAT(TiText order by TiOrder SEPARATOR "") - Sent FROM temp_word_occurrences group by TiSeID' - ); - echo '

Sentences

    '; - foreach ($sentences as $record) { - echo "
  1. " . \htmlspecialchars((string) ($record['Sent'] ?? ''), ENT_QUOTES, 'UTF-8') . "
    2. "; - } - echo '
'; - $bindings = [$lid]; - $rows = Connection::preparedFetchAll( - "SELECT count(`TiOrder`) cnt, if(0=TiWordCount,0,1) as len, - LOWER(TiText) as word, WoTranslation - FROM temp_word_occurrences - LEFT JOIN words ON lower(TiText)=WoTextLC AND WoLgID=?" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " GROUP BY lower(TiText)", - $bindings - ); - foreach ($rows as $record) { - if ($record['len'] == 1) { - $wo[] = array( - \htmlspecialchars((string) ($record['word'] ?? ''), ENT_QUOTES, 'UTF-8'), - $record['cnt'], - \htmlspecialchars((string) ($record['WoTranslation'] ?? ''), ENT_QUOTES, 'UTF-8') - ); - } else { - $nw[] = array( - htmlspecialchars((string)($record['word'] ?? ''), ENT_QUOTES, 'UTF-8'), - htmlspecialchars((string)($record['cnt'] ?? ''), ENT_QUOTES, 'UTF-8') - ); - } - } - // JavaScript moved to src/frontend/js/texts/text_check_display.ts - echo ''; - } - - /** - * Append sentences and text items in the database. - * - * TiSeID in temp_word_occurrences is pre-computed to match future SeID values. - * When parseStandardToDatabase runs with useMaxSeID=true, it sets TiSeID - * to MAX(SeID)+1, MAX(SeID)+2, etc. When we insert sentences here, they - * get those exact SeID values via auto-increment, so TiSeID = SeID. - * - * @param int $tid ID of text from which insert data - * @param int $lid ID of the language of the text - * @param bool $hasmultiword Set to true to insert multi-words as well. - * - * @return void - */ - public static function registerSentencesTextItems(int $tid, int $lid, bool $hasmultiword): void - { - // STEP 1: Insert sentences FIRST to satisfy FK constraint. - Connection::query('SET @i=0;'); - Connection::preparedExecute( - "INSERT INTO sentences ( - SeLgID, SeTxID, SeOrder, SeFirstPos, SeText - ) SELECT - ?, - ?, - @i:=@i+1, - MIN(IF(TiWordCount=0, TiOrder+1, TiOrder)), - GROUP_CONCAT(TiText ORDER BY TiOrder SEPARATOR \"\") - FROM temp_word_occurrences - GROUP BY TiSeID - ORDER BY TiSeID", - [$lid, $tid] - ); - - // STEP 1.5: Align TiSeID with actual SeID values. - // The pre-computed TiSeID may not match actual AUTO_INCREMENT values, - // so we update temp_word_occurrences to use the actual SeID from inserted sentences. - // ROW_NUMBER() maps TiSeID rank to SeOrder, which we JOIN to get SeID. - Connection::preparedExecute( - "UPDATE temp_word_occurrences t - JOIN ( - SELECT TiSeID AS old_seid, - ROW_NUMBER() OVER (ORDER BY TiSeID) AS rn - FROM temp_word_occurrences - GROUP BY TiSeID - ) mapping ON t.TiSeID = mapping.old_seid - JOIN sentences s ON s.SeOrder = mapping.rn AND s.SeTxID = ? - SET t.TiSeID = s.SeID", - [$tid] - ); - - // STEP 1.6: Also update tempexprs.sent if multiword expressions exist. - // tempexprs was populated before sentences were inserted, so its `sent` - // field also needs to be aligned with actual SeID values. - if ($hasmultiword) { - Connection::preparedExecute( - "UPDATE tempexprs e - JOIN ( - SELECT sent AS old_sent, - ROW_NUMBER() OVER (ORDER BY sent) AS rn - FROM tempexprs - WHERE sent IS NOT NULL - GROUP BY sent - ) mapping ON e.sent = mapping.old_sent - JOIN sentences s ON s.SeOrder = mapping.rn AND s.SeTxID = ? - SET e.sent = s.SeID", - [$tid] - ); - } - - // STEP 2: Insert text items. TiSeID and tempexprs.sent now equal actual SeID. - if ($hasmultiword) { - // Build SQL and bindings in lockstep. Each forTablePrepared() call - // both appends " AND WoUsID = ?" to the SQL and pushes $userId - // onto $bindings, so the bindings stay in left-to-right placeholder - // order even with two user-scope injections inside the UNION. - $bindings = [$lid, $tid]; - $sql = "INSERT INTO word_occurrences ( - Ti2WoID, Ti2LgID, Ti2TxID, Ti2SeID, Ti2Order, Ti2WordCount, Ti2Text - ) SELECT WoID, ?, ?, sent, TiOrder - (2*(n-1)) TiOrder, - n TiWordCount, word - FROM tempexprs - JOIN words - ON WoTextLC = lword AND WoWordCount = n" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " WHERE lword IS NOT NULL AND WoLgID = ?"; - $bindings[] = $lid; - $bindings[] = $lid; - $bindings[] = $tid; - $sql .= " UNION ALL - SELECT WoID, ?, ?, TiSeID, TiOrder, TiWordCount, TiText - FROM temp_word_occurrences - LEFT JOIN words - ON LOWER(TiText) = WoTextLC AND TiWordCount=1 AND WoLgID = ?"; - $bindings[] = $lid; - $sql .= UserScopedQuery::forTablePrepared('words', $bindings, '') - . " ORDER BY TiOrder, TiWordCount"; - - $stmt = Connection::prepare($sql); - $stmt->bindValues($bindings); - $stmt->execute(); - } else { - $bindings = [$lid, $tid, $lid]; - Connection::preparedExecute( - "INSERT INTO word_occurrences ( - Ti2WoID, Ti2LgID, Ti2TxID, Ti2SeID, Ti2Order, Ti2WordCount, Ti2Text - ) - SELECT WoID, ?, ?, TiSeID, TiOrder, TiWordCount, TiText - FROM temp_word_occurrences - LEFT JOIN words - ON LOWER(TiText) = WoTextLC AND TiWordCount=1 AND WoLgID = ?" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " ORDER BY TiOrder, TiWordCount", - $bindings - ); - } - } - - /** - * Display statistics about a text. - * - * @param int $lid Language ID - * @param bool $rtlScript true if language is right-to-left - * @param bool $multiwords Display if text has multi-words - * - * @return void - */ - public static function displayStatistics(int $lid, bool $rtlScript, bool $multiwords): void - { - $mw = array(); - if ($multiwords) { - $bindings = [$lid]; - $rows = Connection::preparedFetchAll( - "SELECT COUNT(WoID) cnt, n as len, - LOWER(WoText) AS word, WoTranslation - FROM tempexprs - JOIN words - ON WoTextLC = lword AND WoWordCount = n" - . UserScopedQuery::forTablePrepared('words', $bindings, '') - . " WHERE lword IS NOT NULL AND WoLgID = ? - GROUP BY WoID ORDER BY WoTextLC", - $bindings - ); - foreach ($rows as $record) { - $mw[] = array( - htmlspecialchars((string)($record['word'] ?? ''), ENT_QUOTES, 'UTF-8'), - $record['cnt'], - htmlspecialchars((string)($record['WoTranslation'] ?? ''), ENT_QUOTES, 'UTF-8') - ); - } - } - // JavaScript moved to src/frontend/js/texts/text_check_display.ts - echo ''; - } - - /** - * Get all multi-word expression lengths for a language. - * - * @param int $lid Language ID - * - * @return int[] Array of distinct word counts (e.g., [2, 3] for 2-word and 3-word expressions) - */ - public static function getMultiWordLengths(int $lid): array - { - $wl = []; - $rows = QueryBuilder::table('words') - ->select(['DISTINCT(WoWordCount) as WoWordCount']) - ->where('WoLgID', '=', $lid) - ->where('WoWordCount', '>', 1) - ->getPrepared(); - - foreach ($rows as $record) { - $wl[] = (int)$record['WoWordCount']; - } - - return $wl; - } - - /** - * Check a language that contains expressions. - * - * @param int[] $wl All the different expression length in the language. - * - * @return void - */ - public static function checkExpressions(array $wl): void - { - $wl_max = 0; - $mw_sql = ''; - foreach ($wl as $word_length) { - if ($wl_max < $word_length) { - $wl_max = $word_length; - } - $mw_sql .= ' WHEN ' . $word_length . - ' THEN @a' . ($word_length * 2 - 1); - } - $set_wo_sql = $set_wo_sql_2 = $del_wo_sql = $init_var = ''; - // For all possible multi-words length - for ($i = $wl_max * 2 - 1; $i > 1; $i--) { - $set_wo_sql .= "WHEN (@a$i := @a" . ($i - 1) . ") IS NULL THEN NULL "; - $set_wo_sql_2 .= "WHEN (@a$i := @a" . ($i - 2) . ") IS NULL THEN NULL "; - $del_wo_sql .= "WHEN (@a$i := @a0) IS NULL THEN NULL "; - $init_var .= "@a$i=0,"; - } - // 2.8.1-fork: @a0 is always 0? @f always '' but necessary to force code execution - Connection::query( - "SET $init_var@a1=0, @a0=0, @se_id=0, @c='', @d=0, @f='', @ti_or=0;" - ); - // Create a table to store length of each terms - Connection::query( - "CREATE TEMPORARY TABLE IF NOT EXISTS numbers( - n tinyint(3) unsigned NOT NULL - );" - ); - Connection::execute("TRUNCATE TABLE numbers"); - Connection::query( - "INSERT IGNORE INTO numbers(n) VALUES (" . - implode('),(', $wl) . - ');' - ); - // Store garbage - Connection::query( - "CREATE TABLE IF NOT EXISTS tempexprs ( - sent mediumint unsigned, - word varchar(250), - lword varchar(250), - TiOrder smallint unsigned, - n tinyint(3) unsigned NOT NULL - )" - ); - Connection::execute("TRUNCATE TABLE tempexprs"); - Connection::query( - "INSERT IGNORE INTO tempexprs - (sent, word, lword, TiOrder, n) - -- 2.10.0-fork: straight_join may be irrelevant as the query is less skewed - SELECT straight_join - IF( - @se_id=TiSeID and @ti_or=TiOrder, - IF((@ti_or:=TiOrder+@a0) is null,TiSeID,TiSeID), - IF( - @se_id=TiSeID, - IF( - (@d=1) and (0<>TiWordCount), - CASE $set_wo_sql_2 - WHEN (@a1:=TiCount+@a0) IS NULL THEN NULL - WHEN (@se_id:=TiSeID+@a0) IS NULL THEN NULL - WHEN (@ti_or:=TiOrder+@a0) IS NULL THEN NULL - WHEN (@c:=concat(@c,TiText)) IS NULL THEN NULL - WHEN (@d:=(0<>TiWordCount)+@a0) IS NULL THEN NULL - ELSE TiSeID - END, - CASE $set_wo_sql - WHEN (@a1:=TiCount+@a0) IS NULL THEN NULL - WHEN (@se_id:=TiSeID+@a0) IS NULL THEN NULL - WHEN (@ti_or:=TiOrder+@a0) IS NULL THEN NULL - WHEN (@c:=concat(@c,TiText)) IS NULL THEN NULL - WHEN (@d:=(0<>TiWordCount)+@a0) IS NULL THEN NULL - ELSE TiSeID - END - ), - CASE $del_wo_sql - WHEN (@a1:=TiCount+@a0) IS NULL THEN NULL - WHEN (@se_id:=TiSeID+@a0) IS NULL THEN NULL - WHEN (@ti_or:=TiOrder+@a0) IS NULL THEN NULL - WHEN (@c:=concat(TiText,@f)) IS NULL THEN NULL - WHEN (@d:=(0<>TiWordCount)+@a0) IS NULL THEN NULL - ELSE TiSeID - END - ) - ) sent, - if( - @d=0, - NULL, - if( - CRC32(@z:=substr(@c,CASE n$mw_sql END))<>CRC32(LOWER(@z)), - @z, - '' - ) - ) word, - if(@d=0 or ''=@z, NULL, lower(@z)) lword, - TiOrder, - n - FROM numbers , temp_word_occurrences" - ); - } -} diff --git a/src/Shared/Infrastructure/Database/TokenPersistence.php b/src/Shared/Infrastructure/Database/TokenPersistence.php new file mode 100644 index 000000000..d4f7daf0e --- /dev/null +++ b/src/Shared/Infrastructure/Database/TokenPersistence.php @@ -0,0 +1,495 @@ + + * @license Unlicense + * @link https://hugofara.github.io/lwt/developer/api + * @since 3.2.2 + */ + +declare(strict_types=1); + +namespace Lwt\Shared\Infrastructure\Database; + +/** + * Turns a parsed token stream into `sentences` and `word_occurrences` rows, + * detecting multi-word expressions along the way — all in PHP. + * + * This replaces the old scratch-table pipeline (temp_word_occurrences + + * tempexprs + numbers + the LOAD DATA path + the stateful @-variable multi-word + * detection SQL). Sentences are inserted first (to satisfy the FK on + * word_occurrences), their real SeIDs are read back, and word occurrences — + * single words and multi-word expressions — are inserted referencing them. + * + * @since 3.2.2 + */ +final class TokenPersistence +{ + /** @var int Rows per INSERT statement. */ + private const CHUNK = 500; + + /** + * Save parsed tokens as sentences + word occurrences for a text. + * + * @param ParsedToken[] $tokens Tokens for the whole text + * @param int $lid Language ID + * @param int $textId Text ID + * + * @return void + */ + public static function save(array $tokens, int $lid, int $textId): void + { + if (empty($tokens)) { + return; + } + $bySentence = self::groupBySentence($tokens); + $seIdMap = self::insertSentences($bySentence, $lid, $textId); + + $singleMap = self::singleWordTerms($lid, self::distinctWordLowercase($tokens)); + $mwTerms = self::multiWordTerms($lid); + + // Single tokens (words + non-words): every token becomes an occurrence. + $rows = []; + foreach ($tokens as $t) { + $woId = null; + if ($t->wordCount === 1) { + $woId = $singleMap[self::lc($t->text)]['id'] ?? null; + } + $rows[] = [$woId, $lid, $textId, $seIdMap[$t->sentence], $t->order, $t->wordCount, $t->text]; + } + // Multi-word expressions overlay the single words they span. + foreach (self::detectMultiWords($bySentence, $mwTerms) as $mw) { + // Store the span text only when it differs from its lowercase form + // (matches the historical Ti2Text storage optimisation). + $stored = $mw['text'] !== self::lc($mw['text']) ? $mw['text'] : ''; + $rows[] = [$mw['id'], $lid, $textId, $seIdMap[$mw['sentence']], $mw['order'], $mw['n'], $stored]; + } + + self::insertWordOccurrences($rows); + } + + /** + * Compute preview statistics for the check-text UI (no output). + * + * @param ParsedToken[] $tokens Tokens for the whole text + * @param int $lid Language ID + * + * @return array{sentences: int, words: int, unknownPercent: float, preview: string} + */ + public static function stats(array $tokens, int $lid): array + { + if (empty($tokens)) { + return ['sentences' => 0, 'words' => 0, 'unknownPercent' => 100.0, 'preview' => '']; + } + $bySentence = self::groupBySentence($tokens); + + $counts = []; + $total = 0; + foreach ($tokens as $t) { + if ($t->wordCount === 1) { + $lc = self::lc($t->text); + $counts[$lc] = ($counts[$lc] ?? 0) + 1; + $total++; + } + } + $single = self::singleWordTerms($lid, array_keys($counts)); + $unknown = 0; + foreach ($counts as $lc => $cnt) { + if (empty($single[$lc]['tr'] ?? '')) { + $unknown += $cnt; + } + } + $unknownPercent = $total > 0 ? round($unknown / $total * 100, 1) : 100.0; + + $texts = []; + foreach ($bySentence as $sTokens) { + $texts[] = self::sentenceText($sTokens); + } + $preview = implode(' ', array_slice($texts, 0, 3)); + if (count($texts) > 3) { + $preview .= '...'; + } + + return [ + 'sentences' => count($bySentence), + 'words' => $total, + 'unknownPercent' => $unknownPercent, + 'preview' => $preview, + ]; + } + + /** + * Echo the sentence list and per-word JSON for the check-text preview. + * + * @param ParsedToken[] $tokens Tokens for the whole text + * @param int $lid Language ID + * + * @return void + */ + public static function echoCheckValid(array $tokens, int $lid): void + { + $bySentence = self::groupBySentence($tokens); + echo '

Sentences

    '; + foreach ($bySentence as $sTokens) { + echo '
  1. ' . \htmlspecialchars(self::sentenceText($sTokens), ENT_QUOTES, 'UTF-8') . '
    2. '; + } + echo '
'; + + $wordCounts = []; + $nonWordCounts = []; + foreach ($tokens as $t) { + $lc = self::lc($t->text); + if ($t->wordCount === 1) { + $wordCounts[$lc] = ($wordCounts[$lc] ?? 0) + 1; + } else { + $nonWordCounts[$lc] = ($nonWordCounts[$lc] ?? 0) + 1; + } + } + $single = self::singleWordTerms($lid, array_keys($wordCounts)); + $wo = []; + foreach ($wordCounts as $lc => $cnt) { + $wo[] = [ + \htmlspecialchars($lc, ENT_QUOTES, 'UTF-8'), + $cnt, + \htmlspecialchars($single[$lc]['tr'] ?? '', ENT_QUOTES, 'UTF-8'), + ]; + } + $nw = []; + foreach ($nonWordCounts as $lc => $cnt) { + $nw[] = [ + \htmlspecialchars($lc, ENT_QUOTES, 'UTF-8'), + \htmlspecialchars((string)$cnt, ENT_QUOTES, 'UTF-8'), + ]; + } + echo ''; + } + + /** + * Echo the multi-word statistics JSON for the check-text preview. + * + * @param ParsedToken[] $tokens Tokens for the whole text + * @param int $lid Language ID + * @param bool $rtlScript Whether the language is right-to-left + * + * @return void + */ + public static function echoStatistics(array $tokens, int $lid, bool $rtlScript): void + { + $mwTerms = self::multiWordTerms($lid); + $occ = self::detectMultiWords(self::groupBySentence($tokens), $mwTerms); + + $idInfo = []; + foreach ($mwTerms as $terms) { + foreach ($terms as $info) { + $idInfo[$info['id']] = $info; + } + } + $byWo = []; + foreach ($occ as $o) { + $byWo[$o['id']] = ($byWo[$o['id']] ?? 0) + 1; + } + $mw = []; + foreach ($byWo as $id => $cnt) { + $info = $idInfo[$id] ?? ['text' => '', 'tr' => '']; + $mw[] = [ + \htmlspecialchars(self::lc($info['text']), ENT_QUOTES, 'UTF-8'), + $cnt, + \htmlspecialchars($info['tr'], ENT_QUOTES, 'UTF-8'), + ]; + } + \usort($mw, fn(array $a, array $b): int => \strcmp((string)$a[0], (string)$b[0])); + + echo ''; + } + + // ========================================================================= + // Internal helpers + // ========================================================================= + + /** + * Group tokens by their sentence index, preserving order. + * + * @param ParsedToken[] $tokens Tokens + * + * @return array> + */ + private static function groupBySentence(array $tokens): array + { + $by = []; + foreach ($tokens as $t) { + $by[$t->sentence][] = $t; + } + ksort($by); + return $by; + } + + /** + * Concatenate a sentence's token texts. + * + * @param list $sTokens Sentence tokens + * + * @return string + */ + private static function sentenceText(array $sTokens): string + { + $s = ''; + foreach ($sTokens as $t) { + $s .= $t->text; + } + return $s; + } + + /** + * Insert sentences and return a map of local sentence index -> real SeID. + * + * @param array> $bySentence Tokens grouped by sentence + * @param int $lid Language ID + * @param int $textId Text ID + * + * @return array + */ + private static function insertSentences(array $bySentence, int $lid, int $textId): array + { + $localSids = array_keys($bySentence); + $params = []; + $seOrder = 0; + foreach ($bySentence as $sTokens) { + $seOrder++; + $firstPos = null; + $text = ''; + foreach ($sTokens as $t) { + $pos = $t->wordCount === 0 ? $t->order + 1 : $t->order; + if ($firstPos === null || $pos < $firstPos) { + $firstPos = $pos; + } + $text .= $t->text; + } + array_push($params, $lid, $textId, $seOrder, (int)$firstPos, $text); + } + self::chunkedInsert( + 'INSERT INTO sentences (SeLgID, SeTxID, SeOrder, SeFirstPos, SeText) VALUES ', + '(?, ?, ?, ?, ?)', + 5, + $params + ); + + $bindings = [$textId]; + $rows = Connection::preparedFetchAll( + 'SELECT SeID FROM sentences WHERE SeTxID = ?' + . UserScopedQuery::forTablePrepared('sentences', $bindings) + . ' ORDER BY SeOrder', + $bindings + ); + $map = []; + foreach ($localSids as $i => $localSid) { + $map[$localSid] = (int)($rows[$i]['SeID'] ?? 0); + } + return $map; + } + + /** + * Insert word-occurrence rows in chunks. + * + * @param list $rows Rows + * + * @return void + */ + private static function insertWordOccurrences(array $rows): void + { + if (empty($rows)) { + return; + } + $params = []; + foreach ($rows as $r) { + array_push($params, $r[0], $r[1], $r[2], $r[3], $r[4], $r[5], $r[6]); + } + self::chunkedInsert( + 'INSERT INTO word_occurrences ' + . '(Ti2WoID, Ti2LgID, Ti2TxID, Ti2SeID, Ti2Order, Ti2WordCount, Ti2Text) VALUES ', + '(?, ?, ?, ?, ?, ?, ?)', + 7, + $params + ); + } + + /** + * Execute a multi-row INSERT in chunks of CHUNK rows. + * + * @param string $prefix SQL up to and including "VALUES " + * @param string $rowPlaceholder Placeholder for one row, e.g. "(?, ?)" + * @param int $colsPerRow Number of columns per row + * @param list $params Flat parameter list + * + * @return void + */ + private static function chunkedInsert( + string $prefix, + string $rowPlaceholder, + int $colsPerRow, + array $params + ): void { + $rowCount = intdiv(count($params), $colsPerRow); + for ($i = 0; $i < $rowCount; $i += self::CHUNK) { + $n = min(self::CHUNK, $rowCount - $i); + $placeholders = implode(',', array_fill(0, $n, $rowPlaceholder)); + $slice = array_slice($params, $i * $colsPerRow, $n * $colsPerRow); + Connection::preparedExecute($prefix . $placeholders, $slice); + } + } + + /** + * Distinct lowercased word-token texts. + * + * @param ParsedToken[] $tokens Tokens + * + * @return list + */ + private static function distinctWordLowercase(array $tokens): array + { + $set = []; + foreach ($tokens as $t) { + if ($t->wordCount === 1) { + $set[self::lc($t->text)] = true; + } + } + return array_keys($set); + } + + /** + * Load single-word terms for the given lowercased words. + * + * @param int $lid Language ID + * @param list $lowerWords Distinct lowercased words to look up + * + * @return array + */ + private static function singleWordTerms(int $lid, array $lowerWords): array + { + if (empty($lowerWords)) { + return []; + } + $bindings = [$lid]; + // Build the IN clause manually: these are string keys (WoTextLC), not + // the integer IDs that Connection::buildPreparedInClause() expects. + $inClause = '(' . implode(',', array_fill(0, count($lowerWords), '?')) . ')'; + foreach ($lowerWords as $word) { + $bindings[] = $word; + } + $sql = 'SELECT WoID, WoTextLC, WoTranslation FROM words ' + . 'WHERE WoLgID = ? AND WoWordCount = 1 AND WoTextLC IN ' . $inClause + . UserScopedQuery::forTablePrepared('words', $bindings); + $map = []; + foreach (Connection::preparedFetchAll($sql, $bindings) as $r) { + $map[(string)$r['WoTextLC']] = [ + 'id' => (int)$r['WoID'], + 'tr' => (string)($r['WoTranslation'] ?? ''), + ]; + } + return $map; + } + + /** + * Load multi-word terms grouped by word count. + * + * @param int $lid Language ID + * + * @return array> + */ + private static function multiWordTerms(int $lid): array + { + $bindings = [$lid]; + $sql = 'SELECT WoID, WoText, WoTextLC, WoTranslation, WoWordCount FROM words ' + . 'WHERE WoLgID = ? AND WoWordCount > 1' + . UserScopedQuery::forTablePrepared('words', $bindings); + $map = []; + foreach (Connection::preparedFetchAll($sql, $bindings) as $r) { + $n = (int)$r['WoWordCount']; + $map[$n][(string)$r['WoTextLC']] = [ + 'id' => (int)$r['WoID'], + 'text' => (string)$r['WoText'], + 'tr' => (string)($r['WoTranslation'] ?? ''), + ]; + } + return $map; + } + + /** + * Detect multi-word expression occurrences in the token stream. + * + * For each sentence and each known multi-word length n, slide a window of n + * words and match the concatenated span (words + intervening separators) + * against the language's n-word terms. + * + * @param array> $bySentence Tokens by sentence + * @param array> $mwTerms Multi-word terms + * + * @return list + */ + private static function detectMultiWords(array $bySentence, array $mwTerms): array + { + if (empty($mwTerms)) { + return []; + } + $lengths = array_keys($mwTerms); + $occ = []; + foreach ($bySentence as $localSid => $sTokens) { + $wordIdx = []; + foreach ($sTokens as $idx => $t) { + if ($t->wordCount === 1) { + $wordIdx[] = $idx; + } + } + $wordCount = count($wordIdx); + foreach ($lengths as $n) { + if ($n < 2) { + continue; + } + for ($i = 0; $i + $n - 1 < $wordCount; $i++) { + $firstIdx = $wordIdx[$i]; + $lastIdx = $wordIdx[$i + $n - 1]; + $span = ''; + for ($k = $firstIdx; $k <= $lastIdx; $k++) { + $span .= $sTokens[$k]->text; + } + $lc = self::lc($span); + if (isset($mwTerms[$n][$lc])) { + $occ[] = [ + 'id' => $mwTerms[$n][$lc]['id'], + 'sentence' => $localSid, + 'order' => $sTokens[$firstIdx]->order, + 'n' => $n, + 'text' => $span, + ]; + } + } + } + } + return $occ; + } + + /** + * Lowercase a string (UTF-8). + * + * @param string $s Input + * + * @return string + */ + private static function lc(string $s): string + { + return mb_strtolower($s, 'UTF-8'); + } +} diff --git a/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php b/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php index 046bd224b..495bd80ec 100644 --- a/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php +++ b/tests/backend/Shared/Infrastructure/Database/JapaneseTextParserTest.php @@ -128,34 +128,41 @@ public function splitJapaneseSentencesWithWhitespaceOnlyReturnsEmptyString(): vo } // ========================================================================= - // parseJapanese (split-only mode id=-2) + // buildTokensFromMecab (pure MeCab-output -> tokens, no binary needed) // ========================================================================= #[Test] - public function parseJapaneseWithIdMinus2ReturnsArrayOfSentences(): void + public function buildTokensFromMecabProducesExpectedTokens(): void { - $result = JapaneseTextParser::parseJapanese('Some text', -2); + // Captured MeCab output for "東京は大きいです。" in the parser's + // "-F %m\t%t\t%h" format, terminated with the EOP sentence marker. + $mecabed = "東京\t2\t46\nは\t6\t16\n大きい\t2\t10\n" + . "です\t6\t25\n。\t3\t7\nEOP\t3\t7\n"; - $this->assertIsArray($result); - $this->assertNotEmpty($result); - } + $tokens = JapaneseTextParser::buildTokensFromMecab($mecabed); - #[Test] - public function parseJapaneseWithIdMinus2MatchesSplitMethod(): void - { - $text = "Line A\nLine B"; - $splitResult = JapaneseTextParser::splitJapaneseSentences($text); - $parseResult = JapaneseTextParser::parseJapanese($text, -2); - - $this->assertSame($splitResult, $parseResult); + $this->assertCount(5, $tokens); + $this->assertSame( + ['東京', 'は', '大きい', 'です', '。'], + array_map(fn($t) => $t->text, $tokens) + ); + // Punctuation is a non-word; the rest are words. + $this->assertSame([1, 1, 1, 1, 0], array_map(fn($t) => $t->wordCount, $tokens)); + // All in the first sentence (the trailing EOP order group is dropped). + foreach ($tokens as $t) { + $this->assertSame(1, $t->sentence); + } + // Global order is strictly increasing. + $orders = array_map(fn($t) => $t->order, $tokens); + $sorted = $orders; + sort($sorted); + $this->assertSame($sorted, $orders); } #[Test] - public function parseJapaneseWithIdMinus2DoesNotReturnNull(): void + public function buildTokensFromMecabHandlesEmptyOutput(): void { - $result = JapaneseTextParser::parseJapanese('test', -2); - - $this->assertNotNull($result); + $this->assertSame([], JapaneseTextParser::buildTokensFromMecab('')); } // ========================================================================= @@ -204,32 +211,4 @@ public function displayJapanesePreviewCollapsesWhitespace(): void $this->assertStringContainsString('Word1 Word2', $output); } - - // ========================================================================= - // parseJapanese / parseJapaneseToDatabase (requires MeCab + DB) - // ========================================================================= - - #[Test] - public function parseJapaneseWithIdMinus1RequiresMecab(): void - { - $this->markTestSkipped( - 'MeCab installation required for display preview mode' - ); - } - - #[Test] - public function parseJapaneseWithPositiveIdRequiresMecabAndDatabase(): void - { - $this->markTestSkipped( - 'MeCab installation and database connection required for save mode' - ); - } - - #[Test] - public function parseJapaneseToDatabaseRequiresMecabAndDatabase(): void - { - $this->markTestSkipped( - 'MeCab installation and database connection required' - ); - } } diff --git a/tests/backend/Shared/Infrastructure/Database/StandardTextParserTest.php b/tests/backend/Shared/Infrastructure/Database/StandardTextParserTest.php index b8d0f4082..10cbf4069 100644 --- a/tests/backend/Shared/Infrastructure/Database/StandardTextParserTest.php +++ b/tests/backend/Shared/Infrastructure/Database/StandardTextParserTest.php @@ -288,20 +288,4 @@ public function getLanguageSettingsRequiresDatabase(): void { $this->markTestSkipped('Database connection required'); } - - // ========================================================================= - // parseStandard / parseStandardToDatabase (requires DB) - // ========================================================================= - - #[Test] - public function parseStandardRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - #[Test] - public function parseStandardToDatabaseRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } } diff --git a/tests/backend/Shared/Infrastructure/Database/TextParsingPersistenceTest.php b/tests/backend/Shared/Infrastructure/Database/TextParsingPersistenceTest.php deleted file mode 100644 index 7d50432fd..000000000 --- a/tests/backend/Shared/Infrastructure/Database/TextParsingPersistenceTest.php +++ /dev/null @@ -1,379 +0,0 @@ - - * @since 3.0.0 - */ - -declare(strict_types=1); - -namespace Tests\Backend\Shared\Infrastructure\Database; - -use Lwt\Shared\Infrastructure\Database\TextParsingPersistence; -use PHPUnit\Framework\Attributes\Test; -use PHPUnit\Framework\TestCase; -use PHPUnit\Framework\Attributes\CoversClass; - -/** - * Unit tests for TextParsingPersistence static methods. - * - * @since 3.0.0 - */ -#[CoversClass(TextParsingPersistence::class)] -class TextParsingPersistenceTest extends TestCase -{ - // ========================================================================= - // saveWithSql - signature and visibility - // ========================================================================= - - #[Test] - public function saveWithSqlIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSql' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function saveWithSqlAcceptsStringTextAndIntId(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSql' - ); - $params = $reflection->getParameters(); - - $this->assertCount(2, $params); - $this->assertSame('text', $params[0]->getName()); - $this->assertSame('string', $params[0]->getType()->getName()); - $this->assertSame('id', $params[1]->getName()); - $this->assertSame('int', $params[1]->getType()->getName()); - } - - #[Test] - public function saveWithSqlReturnsVoid(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSql' - ); - - $this->assertSame('void', $reflection->getReturnType()->getName()); - } - - #[Test] - public function saveWithSqlRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // saveWithSqlFallback - signature and visibility - // ========================================================================= - - #[Test] - public function saveWithSqlFallbackIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSqlFallback' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function saveWithSqlFallbackAcceptsStringTextAndIntId(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSqlFallback' - ); - $params = $reflection->getParameters(); - - $this->assertCount(2, $params); - $this->assertSame('text', $params[0]->getName()); - $this->assertSame('string', $params[0]->getType()->getName()); - $this->assertSame('id', $params[1]->getName()); - $this->assertSame('int', $params[1]->getType()->getName()); - } - - #[Test] - public function saveWithSqlFallbackReturnsVoid(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'saveWithSqlFallback' - ); - - $this->assertSame('void', $reflection->getReturnType()->getName()); - } - - #[Test] - public function saveWithSqlFallbackRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // checkValid - signature and visibility - // ========================================================================= - - #[Test] - public function checkValidIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkValid' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function checkValidAcceptsSingleIntParameter(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkValid' - ); - $params = $reflection->getParameters(); - - $this->assertCount(1, $params); - $this->assertSame('lid', $params[0]->getName()); - $this->assertSame('int', $params[0]->getType()->getName()); - } - - #[Test] - public function checkValidReturnsVoid(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkValid' - ); - - $this->assertSame('void', $reflection->getReturnType()->getName()); - } - - #[Test] - public function checkValidRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // registerSentencesTextItems - signature and visibility - // ========================================================================= - - #[Test] - public function registerSentencesTextItemsIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'registerSentencesTextItems' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function registerSentencesTextItemsAcceptsThreeParameters(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'registerSentencesTextItems' - ); - $params = $reflection->getParameters(); - - $this->assertCount(3, $params); - $this->assertSame('tid', $params[0]->getName()); - $this->assertSame('int', $params[0]->getType()->getName()); - $this->assertSame('lid', $params[1]->getName()); - $this->assertSame('int', $params[1]->getType()->getName()); - $this->assertSame('hasmultiword', $params[2]->getName()); - $this->assertSame('bool', $params[2]->getType()->getName()); - } - - #[Test] - public function registerSentencesTextItemsRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // getMultiWordLengths - signature and visibility - // ========================================================================= - - #[Test] - public function getMultiWordLengthsIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'getMultiWordLengths' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function getMultiWordLengthsAcceptsSingleIntParameter(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'getMultiWordLengths' - ); - $params = $reflection->getParameters(); - - $this->assertCount(1, $params); - $this->assertSame('lid', $params[0]->getName()); - $this->assertSame('int', $params[0]->getType()->getName()); - } - - #[Test] - public function getMultiWordLengthsReturnsArray(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'getMultiWordLengths' - ); - - $this->assertSame('array', $reflection->getReturnType()->getName()); - } - - #[Test] - public function getMultiWordLengthsRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // displayStatistics - signature and visibility - // ========================================================================= - - #[Test] - public function displayStatisticsIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'displayStatistics' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function displayStatisticsAcceptsThreeParameters(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'displayStatistics' - ); - $params = $reflection->getParameters(); - - $this->assertCount(3, $params); - $this->assertSame('lid', $params[0]->getName()); - $this->assertSame('int', $params[0]->getType()->getName()); - $this->assertSame('rtlScript', $params[1]->getName()); - $this->assertSame('bool', $params[1]->getType()->getName()); - $this->assertSame('multiwords', $params[2]->getName()); - $this->assertSame('bool', $params[2]->getType()->getName()); - } - - #[Test] - public function displayStatisticsRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // checkExpressions - signature and visibility - // ========================================================================= - - #[Test] - public function checkExpressionsIsPublicAndStatic(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkExpressions' - ); - - $this->assertTrue($reflection->isPublic()); - $this->assertTrue($reflection->isStatic()); - } - - #[Test] - public function checkExpressionsAcceptsSingleArrayParameter(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkExpressions' - ); - $params = $reflection->getParameters(); - - $this->assertCount(1, $params); - $this->assertSame('wl', $params[0]->getName()); - $this->assertSame('array', $params[0]->getType()->getName()); - } - - #[Test] - public function checkExpressionsReturnsVoid(): void - { - $reflection = new \ReflectionMethod( - TextParsingPersistence::class, - 'checkExpressions' - ); - - $this->assertSame('void', $reflection->getReturnType()->getName()); - } - - #[Test] - public function checkExpressionsRequiresDatabase(): void - { - $this->markTestSkipped('Database connection required'); - } - - // ========================================================================= - // Class-level checks - // ========================================================================= - - #[Test] - public function allPublicMethodsAreStatic(): void - { - $reflection = new \ReflectionClass(TextParsingPersistence::class); - $methods = $reflection->getMethods(\ReflectionMethod::IS_PUBLIC); - - foreach ($methods as $method) { - $this->assertTrue( - $method->isStatic(), - "Method {$method->getName()} should be static" - ); - } - } - - #[Test] - public function classHasExpectedNumberOfPublicMethods(): void - { - $reflection = new \ReflectionClass(TextParsingPersistence::class); - $methods = $reflection->getMethods(\ReflectionMethod::IS_PUBLIC); - - // saveWithSql, saveWithSqlFallback, checkValid, - // registerSentencesTextItems, displayStatistics, - // getMultiWordLengths, checkExpressions - $this->assertCount(7, $methods); - } -}