Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,10 @@ package org.apache.texera.amber.operator.sklearn
class SklearnGaussianNaiveBayesOpDesc extends SklearnClassifierOpDesc {
override def getImportStatements = "from sklearn.naive_bayes import GaussianNB"
override def getUserFriendlyModelName = "Gaussian Naive Bayes"

// GaussianNB estimates a mean and a variance for every feature, which reads the
// zeros as well, so scikit-learn validates its input without accept_sparse. The
// refusal is deliberate: densifying a text matrix is what would exhaust memory.
override protected def countVectorizerAlternatives: Option[String] =
Some("Multinomial, Bernoulli or Complement Naive Bayes")
}
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,27 @@ abstract class SklearnModelOpDesc extends PythonOperatorDescriptor {
@JsonIgnore
def getUserFriendlyModelName: String

/** An estimator that cannot be fitted on the sparse matrix `CountVectorizer`
* produces names here what to reach for instead, and turning the switch on
* stops the workflow at compile time rather than inside scikit-learn.
*
* The switch is declared on this base, so every estimator in both families
* inherits it whether or not its own can use it.
*/
@JsonIgnore
protected def countVectorizerAlternatives: Option[String] = None

override def getOutputSchemas(
inputSchemas: Map[PortIdentity, Schema]
): Map[PortIdentity, Schema] = {
if (countVectorizer) {
countVectorizerAlternatives.foreach { alternatives =>
throw new RuntimeException(
s"$getUserFriendlyModelName cannot be trained on the sparse matrix Count Vectorizer" +
s" produces. Turn Count Vectorizer off, or use $alternatives."
)
}
}
Map(
operatorInfo.outputPorts.head.id -> Schema()
.add("model_name", AttributeType.STRING)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,9 @@ package org.apache.texera.amber.operator.sklearn.training
class SklearnTrainingGaussianNaiveBayesOpDesc extends SklearnTrainingOpDesc {
override def getImportStatements = "from sklearn.naive_bayes import GaussianNB"
override def getUserFriendlyModelName = "Training: Gaussian Naive Bayes"

// The same estimator as the classifier family's, with the same limitation: a mean
// and a variance per feature cannot be read off a sparse matrix without densifying.
override protected def countVectorizerAlternatives: Option[String] =
Some("Multinomial, Bernoulli or Complement Naive Bayes")
}
Original file line number Diff line number Diff line change
Expand Up @@ -78,4 +78,19 @@ class SklearnGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers {
r.target shouldBe "label"
r.countVectorizer shouldBe true
}

"SklearnGaussianNaiveBayesOpDesc.getOutputSchemas" should
"reject Count Vectorizer, naming the estimator and what accepts it instead" in {
// GaussianNB is fitted on a mean and a variance per feature, which a sparse
// matrix cannot supply without being densified, so this used to end the run
// from inside scikit-learn. It stops at compile time now.
val d = new SklearnGaussianNaiveBayesOpDesc
d.target = "y"
d.text = "note"
d.countVectorizer = true
val thrown = intercept[RuntimeException](d.getOutputSchemas(Map.empty))
thrown.getMessage should include("Gaussian Naive Bayes")
thrown.getMessage should include("Count Vectorizer")
thrown.getMessage should include("Multinomial, Bernoulli or Complement Naive Bayes")
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -75,4 +75,32 @@ class SklearnModelOpDescSpec extends AnyFlatSpec with Matchers {
)
d.getOutputSchemas(arbitraryInput) shouldBe fromEmpty
}

it should "let Count Vectorizer through for an estimator that named no alternative" in {
// The default, and what every estimator but GaussianNB relies on: the sparse
// matrix is what the others accept.
val d = new TestSklearnModelOpDesc
d.countVectorizer = true
d.getOutputSchemas(Map.empty).keySet shouldBe Set(d.operatorInfo.outputPorts.head.id)
}

it should "reject Count Vectorizer for an estimator that named one" in {
val d = new TestSklearnModelOpDesc {
override protected def countVectorizerAlternatives: Option[String] = Some("Some Other Model")
}
d.countVectorizer = true
val thrown = intercept[RuntimeException](d.getOutputSchemas(Map.empty))
thrown.getMessage should include("Test Model")
thrown.getMessage should include("Count Vectorizer")
thrown.getMessage should include("Some Other Model")
}

it should "stay silent while Count Vectorizer is off, whatever the estimator" in {
// The switch defaults to off, so a freshly dropped operator must not be
// reported invalid before anyone has configured it.
val d = new TestSklearnModelOpDesc {
override protected def countVectorizerAlternatives: Option[String] = Some("Some Other Model")
}
d.getOutputSchemas(Map.empty).keySet shouldBe Set(d.operatorInfo.outputPorts.head.id)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -76,4 +76,15 @@ class SklearnTrainingGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Match
r.target shouldBe "label"
r.countVectorizer shouldBe true
}

"SklearnTrainingGaussianNaiveBayesOpDesc.getOutputSchemas" should
"reject Count Vectorizer the same way its classifier counterpart does" in {
val d = new SklearnTrainingGaussianNaiveBayesOpDesc
d.target = "y"
d.text = "note"
d.countVectorizer = true
val thrown = intercept[RuntimeException](d.getOutputSchemas(Map.empty))
thrown.getMessage should include("Training: Gaussian Naive Bayes")
thrown.getMessage should include("Multinomial, Bernoulli or Complement Naive Bayes")
}
}
Loading