diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDesc.scala index 0b9c9efecee..4f2a9b55257 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDesc.scala @@ -22,4 +22,10 @@ package org.apache.texera.amber.operator.sklearn class SklearnGaussianNaiveBayesOpDesc extends SklearnClassifierOpDesc { override def getImportStatements = "from sklearn.naive_bayes import GaussianNB" override def getUserFriendlyModelName = "Gaussian Naive Bayes" + + // GaussianNB estimates a mean and a variance for every feature, which reads the + // zeros as well, so scikit-learn validates its input without accept_sparse. The + // refusal is deliberate: densifying a text matrix is what would exhaust memory. + override protected def countVectorizerAlternatives: Option[String] = + Some("Multinomial, Bernoulli or Complement Naive Bayes") } diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala index 6665477aeac..459a012d4f0 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDesc.scala @@ -85,9 +85,27 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor { @JsonIgnore def getUserFriendlyModelName: String + /** An estimator that cannot be fitted on the sparse matrix `CountVectorizer` + * produces names here what to reach for instead, and turning the switch on + * stops the workflow at compile time rather than inside scikit-learn. + * + * The switch is declared on this base, so every estimator in both families + * inherits it whether or not its own can use it. + */ + @JsonIgnore + protected def countVectorizerAlternatives: Option[String] = None + override def getOutputSchemas( inputSchemas: Map[PortIdentity, Schema] ): Map[PortIdentity, Schema] = { + if (countVectorizer) { + countVectorizerAlternatives.foreach { alternatives => + throw new RuntimeException( + s"$getUserFriendlyModelName cannot be trained on the sparse matrix Count Vectorizer" + + s" produces. Turn Count Vectorizer off, or use $alternatives." + ) + } + } Map( operatorInfo.outputPorts.head.id -> Schema() .add("model_name", AttributeType.STRING) diff --git a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDesc.scala b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDesc.scala index 8d4101b2689..f0326716e41 100644 --- a/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDesc.scala +++ b/common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDesc.scala @@ -22,4 +22,9 @@ package org.apache.texera.amber.operator.sklearn.training class SklearnTrainingGaussianNaiveBayesOpDesc extends SklearnTrainingOpDesc { override def getImportStatements = "from sklearn.naive_bayes import GaussianNB" override def getUserFriendlyModelName = "Training: Gaussian Naive Bayes" + + // The same estimator as the classifier family's, with the same limitation: a mean + // and a variance per feature cannot be read off a sparse matrix without densifying. + override protected def countVectorizerAlternatives: Option[String] = + Some("Multinomial, Bernoulli or Complement Naive Bayes") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala index 9c25894dc18..a8f6241bc2a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala @@ -78,4 +78,19 @@ class SklearnGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { r.target shouldBe "label" r.countVectorizer shouldBe true } + + "SklearnGaussianNaiveBayesOpDesc.getOutputSchemas" should + "reject Count Vectorizer, naming the estimator and what accepts it instead" in { + // GaussianNB is fitted on a mean and a variance per feature, which a sparse + // matrix cannot supply without being densified, so this used to end the run + // from inside scikit-learn. It stops at compile time now. + val d = new SklearnGaussianNaiveBayesOpDesc + d.target = "y" + d.text = "note" + d.countVectorizer = true + val thrown = intercept[RuntimeException](d.getOutputSchemas(Map.empty)) + thrown.getMessage should include("Gaussian Naive Bayes") + thrown.getMessage should include("Count Vectorizer") + thrown.getMessage should include("Multinomial, Bernoulli or Complement Naive Bayes") + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala index d94ae243e7c..1bf1ca4c4fa 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescSpec.scala @@ -75,4 +75,32 @@ class SklearnModelOpDescSpec extends AnyFlatSpec with Matchers { ) d.getOutputSchemas(arbitraryInput) shouldBe fromEmpty } + + it should "let Count Vectorizer through for an estimator that named no alternative" in { + // The default, and what every estimator but GaussianNB relies on: the sparse + // matrix is what the others accept. + val d = new TestSklearnModelOpDesc + d.countVectorizer = true + d.getOutputSchemas(Map.empty).keySet shouldBe Set(d.operatorInfo.outputPorts.head.id) + } + + it should "reject Count Vectorizer for an estimator that named one" in { + val d = new TestSklearnModelOpDesc { + override protected def countVectorizerAlternatives: Option[String] = Some("Some Other Model") + } + d.countVectorizer = true + val thrown = intercept[RuntimeException](d.getOutputSchemas(Map.empty)) + thrown.getMessage should include("Test Model") + thrown.getMessage should include("Count Vectorizer") + thrown.getMessage should include("Some Other Model") + } + + it should "stay silent while Count Vectorizer is off, whatever the estimator" in { + // The switch defaults to off, so a freshly dropped operator must not be + // reported invalid before anyone has configured it. + val d = new TestSklearnModelOpDesc { + override protected def countVectorizerAlternatives: Option[String] = Some("Some Other Model") + } + d.getOutputSchemas(Map.empty).keySet shouldBe Set(d.operatorInfo.outputPorts.head.id) + } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala index 7d193883ccf..2e5d87481bb 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala @@ -76,4 +76,15 @@ class SklearnTrainingGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Match r.target shouldBe "label" r.countVectorizer shouldBe true } + + "SklearnTrainingGaussianNaiveBayesOpDesc.getOutputSchemas" should + "reject Count Vectorizer the same way its classifier counterpart does" in { + val d = new SklearnTrainingGaussianNaiveBayesOpDesc + d.target = "y" + d.text = "note" + d.countVectorizer = true + val thrown = intercept[RuntimeException](d.getOutputSchemas(Map.empty)) + thrown.getMessage should include("Training: Gaussian Naive Bayes") + thrown.getMessage should include("Multinomial, Bernoulli or Complement Naive Bayes") + } }