From d44633fda931813dbc2609288aa3817779be0ab2 Mon Sep 17 00:00:00 2001 From: kary zheng Date: Wed, 12 Aug 2026 14:31:28 -0700 Subject: [PATCH] feat(operator): reject Count Vectorizer on the Gaussian Naive Bayes operators Gaussian Naive Bayes and Training: Gaussian Naive Bayes offered a Count Vectorizer switch with no working setting. Turning it on ended the execution from inside scikit-learn: CountVectorizer emits a sparse matrix, and GaussianNB validates its input without accept_sparse because it estimates a mean and a variance per feature, which reads the zeros too. The refusal is deliberate on scikit-learn's side, since densifying a text matrix with a large vocabulary is what would exhaust memory. It is specific to this estimator, not to the switch. Fitting make_pipeline(CountVectorizer(), Estimator()) on the same documents, only GaussianNB raises; MultinomialNB, BernoulliNB and ComplementNB accept the sparse matrix, as do the rest of the estimators both groups use. The switch is declared once on SklearnModelOpDesc and both families splice it in the same way, so all of them inherit it whether or not their estimator can use it. An estimator that cannot take the sparse matrix now names what to reach for instead, and the shared getOutputSchemas turns that into a compile-time failure naming both. Hiding the switch would have been the closer match to how the family handles an impossible pairing already, but a hide keys on a sibling field's value and is evaluated in the frontend, so it cannot be made per estimator on a field the base declares once. It would also leave a workflow already saved with the switch on still failing at run time. The message names the estimator and the three Naive Bayes variants that do accept the matrix. Nothing fires while the switch is off, which is its default, so a freshly dropped operator is not reported invalid before it is configured. Co-Authored-By: Claude Opus 5 (1M context) --- .../SklearnGaussianNaiveBayesOpDesc.scala | 6 ++++ .../operator/sklearn/SklearnModelOpDesc.scala | 18 ++++++++++++ ...earnTrainingGaussianNaiveBayesOpDesc.scala | 5 ++++ .../SklearnGaussianNaiveBayesOpDescSpec.scala | 15 ++++++++++ .../sklearn/SklearnModelOpDescSpec.scala | 28 +++++++++++++++++++ ...TrainingGaussianNaiveBayesOpDescSpec.scala | 11 ++++++++ 6 files changed, 83 insertions(+) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDesc.scala index 0b9c9efecee..4f2a9b55257 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDesc.scala @@ -22,4 +22,10 @@ package org.apache.texera.amber.operator.sklearn class SklearnGaussianNaiveBayesOpDesc extends SklearnClassifierOpDesc { override def getImportStatements = "from sklearn.naive_bayes import GaussianNB" override def getUserFriendlyModelName = "Gaussian Naive Bayes" + + // GaussianNB estimates a mean and a variance for every feature, which reads the + // zeros as well, so scikit-learn validates its input without accept_sparse. The + // refusal is deliberate: densifying a text matrix is what would exhaust memory. + override protected def countVectorizerAlternatives: Option[String] = + Some("Multinomial, Bernoulli or Complement Naive Bayes") } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala index 6665477aeac..459a012d4f0 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala @@ -85,9 +85,27 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor { @JsonIgnore def getUserFriendlyModelName: String + /** An estimator that cannot be fitted on the sparse matrix `CountVectorizer` + * produces names here what to reach for instead, and turning the switch on + * stops the workflow at compile time rather than inside scikit-learn. + * + * The switch is declared on this base, so every estimator in both families + * inherits it whether or not its own can use it. + */ + @JsonIgnore + protected def countVectorizerAlternatives: Option[String] = None + override def getOutputSchemas( inputSchemas: Map[PortIdentity, Schema] ): Map[PortIdentity, Schema] = { + if (countVectorizer) { + countVectorizerAlternatives.foreach { alternatives => + throw new RuntimeException( + s"$getUserFriendlyModelName cannot be trained on the sparse matrix Count Vectorizer" + + s" produces. Turn Count Vectorizer off, or use $alternatives." + ) + } + } Map( operatorInfo.outputPorts.head.id -> Schema() .add("model_name", AttributeType.STRING) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDesc.scala index 8d4101b2689..f0326716e41 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDesc.scala @@ -22,4 +22,9 @@ package org.apache.texera.amber.operator.sklearn.training class SklearnTrainingGaussianNaiveBayesOpDesc extends SklearnTrainingOpDesc { override def getImportStatements = "from sklearn.naive_bayes import GaussianNB" override def getUserFriendlyModelName = "Training: Gaussian Naive Bayes" + + // The same estimator as the classifier family's, with the same limitation: a mean + // and a variance per feature cannot be read off a sparse matrix without densifying. + override protected def countVectorizerAlternatives: Option[String] = + Some("Multinomial, Bernoulli or Complement Naive Bayes") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala index 9c25894dc18..a8f6241bc2a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala @@ -78,4 +78,19 @@ class SklearnGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { r.target shouldBe "label" r.countVectorizer shouldBe true } + + "SklearnGaussianNaiveBayesOpDesc.getOutputSchemas" should + "reject Count Vectorizer, naming the estimator and what accepts it instead" in { + // GaussianNB is fitted on a mean and a variance per feature, which a sparse + // matrix cannot supply without being densified, so this used to end the run + // from inside scikit-learn. It stops at compile time now. + val d = new SklearnGaussianNaiveBayesOpDesc + d.target = "y" + d.text = "note" + d.countVectorizer = true + val thrown = intercept[RuntimeException](d.getOutputSchemas(Map.empty)) + thrown.getMessage should include("Gaussian Naive Bayes") + thrown.getMessage should include("Count Vectorizer") + thrown.getMessage should include("Multinomial, Bernoulli or Complement Naive Bayes") + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala index d94ae243e7c..1bf1ca4c4fa 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala @@ -75,4 +75,32 @@ class SklearnModelOpDescSpec extends AnyFlatSpec with Matchers { ) d.getOutputSchemas(arbitraryInput) shouldBe fromEmpty } + + it should "let Count Vectorizer through for an estimator that named no alternative" in { + // The default, and what every estimator but GaussianNB relies on: the sparse + // matrix is what the others accept. + val d = new TestSklearnModelOpDesc + d.countVectorizer = true + d.getOutputSchemas(Map.empty).keySet shouldBe Set(d.operatorInfo.outputPorts.head.id) + } + + it should "reject Count Vectorizer for an estimator that named one" in { + val d = new TestSklearnModelOpDesc { + override protected def countVectorizerAlternatives: Option[String] = Some("Some Other Model") + } + d.countVectorizer = true + val thrown = intercept[RuntimeException](d.getOutputSchemas(Map.empty)) + thrown.getMessage should include("Test Model") + thrown.getMessage should include("Count Vectorizer") + thrown.getMessage should include("Some Other Model") + } + + it should "stay silent while Count Vectorizer is off, whatever the estimator" in { + // The switch defaults to off, so a freshly dropped operator must not be + // reported invalid before anyone has configured it. + val d = new TestSklearnModelOpDesc { + override protected def countVectorizerAlternatives: Option[String] = Some("Some Other Model") + } + d.getOutputSchemas(Map.empty).keySet shouldBe Set(d.operatorInfo.outputPorts.head.id) + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala index 7d193883ccf..2e5d87481bb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala @@ -76,4 +76,15 @@ class SklearnTrainingGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Match r.target shouldBe "label" r.countVectorizer shouldBe true } + + "SklearnTrainingGaussianNaiveBayesOpDesc.getOutputSchemas" should + "reject Count Vectorizer the same way its classifier counterpart does" in { + val d = new SklearnTrainingGaussianNaiveBayesOpDesc + d.target = "y" + d.text = "note" + d.countVectorizer = true + val thrown = intercept[RuntimeException](d.getOutputSchemas(Map.empty)) + thrown.getMessage should include("Training: Gaussian Naive Bayes") + thrown.getMessage should include("Multinomial, Bernoulli or Complement Naive Bayes") + } }