diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3f61c37d..d97bc37a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -142,9 +142,6 @@ jobs:
 
       - run: nix develop --command sbt --client checkAll
 
-      - name: Run sample benchmarks
-        run: nix develop --command sbt --client 'bench/Jmh/run -i 1 -f1 -t1 -foe true'
-
 
   maven:
     runs-on: ubuntu-latest
diff --git a/build.sbt b/build.sbt
index c9c346c1..839157bd 100644
--- a/build.sbt
+++ b/build.sbt
@@ -654,17 +654,6 @@ lazy val snapshots = project
   .dependsOn(unit)
   .enablePlugins(BuildInfoPlugin)
 
-lazy val bench = project
-  .in(file("tests/benchmarks"))
-  .settings(
-    moduleName := "scip-java-bench",
-    Jmh / bspEnabled := false,
-    (run / fork) := true,
-    (publish / skip) := true
-  )
-  .dependsOn(unit)
-  .enablePlugins(JmhPlugin)
-
 lazy val javaOnlySettings = List[Def.Setting[_]](
   javafmtOnCompile := false,
   autoScalaLibrary := false,
diff --git a/project/plugins.sbt b/project/plugins.sbt
index abe711c7..7cf1e43b 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -6,7 +6,6 @@ addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.5")
 addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.14.3")
 addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.6")
 addSbtPlugin("com.lightbend.sbt" % "sbt-java-formatter" % "0.6.1")
-addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3")
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0")
 addSbtPlugin("org.jetbrains.scala" % "sbt-kotlin-plugin" % "3.1.6")
 addSbtPlugin("com.github.sbt.junit" % "sbt-jupiter-interface" % "0.15.1")
diff --git a/tests/benchmarks/src/main/scala/benchmarks/CompileBench.scala b/tests/benchmarks/src/main/scala/benchmarks/CompileBench.scala
deleted file mode 100644
index d6100be1..00000000
--- a/tests/benchmarks/src/main/scala/benchmarks/CompileBench.scala
+++ /dev/null
@@ -1,102 +0,0 @@
-package benchmarks
-
-import java.nio.charset.StandardCharsets
-import java.nio.file.FileSystems
-import java.nio.file.Files
-import java.nio.file.Path
-import java.util.concurrent.TimeUnit
-
-import scala.meta.inputs.Input
-import scala.meta.internal.io.FileIO
-import scala.meta.io.AbsolutePath
-
-import com.sourcegraph.io.DeleteVisitor
-import com.sourcegraph.scip_java.Dependencies
-import org.openjdk.jmh.annotations._
-import tests.TestCompiler
-
-@State(Scope.Benchmark)
-class CompileBench {
-
-  var deps: Dependencies = _
-  var tmp: Path = _
-  var compiler: TestCompiler = _
-
-  @Param(Array("bytebuddy", "guava"))
-  var lib: String = _
-
-  val libs = Map(
-    "bytebuddy" -> "net.bytebuddy:byte-buddy:1.10.20",
-    "guava" -> "com.google.guava:guava:30.1-jre"
-  )
-
-  @Setup()
-  def setup(): Unit = {
-    tmp = Files.createTempDirectory("benchmarks")
-    deps = Dependencies.resolveDependencies(List(libs(lib)))
-    compiler =
-      new TestCompiler(
-        deps.classpathSyntax,
-        javacOptions = Nil,
-        targetroot = tmp
-      )
-  }
-
-  @TearDown()
-  def teardown(): Unit = {
-    Files.walkFileTree(tmp, new DeleteVisitor)
-  }
-
-  @Benchmark
-  @BenchmarkMode(Array(Mode.SingleShotTime))
-  @OutputTimeUnit(TimeUnit.MILLISECONDS)
-  def compile(): Long = {
-    CompileBench.foreachSource(deps) { inputs =>
-      compiler.compile(inputs).byteCode.length
-    }
-  }
-
-  @Benchmark
-  @BenchmarkMode(Array(Mode.SingleShotTime))
-  @OutputTimeUnit(TimeUnit.MILLISECONDS)
-  def compileSemanticdb(): Long = {
-    CompileBench.foreachSource(deps) { inputs =>
-      compiler
-        .compileSemanticdb(inputs)
-        .textDocument
-        .map(_.getOccurrencesCount)
-        .getOrElse(0)
-    }
-  }
-
-}
-object CompileBench {
-  private val javaPattern = FileSystems
-    .getDefault
-    .getPathMatcher("glob:**.java")
-  def foreachSource(
-      deps: Dependencies
-  )(fn: Seq[Input.VirtualFile] => Int): Long = {
-    deps
-      .sources
-      .map { source =>
-        val path = AbsolutePath(source)
-        FileIO.withJarFileSystem(path, create = false, close = true) { root =>
-          val files =
-            FileIO
-              .listAllFilesRecursively(root)
-              .iterator
-              .filter(p => javaPattern.matches(p.toNIO))
-              .toArray
-          val inputs = files.map { source =>
-            val text = FileIO.slurp(source, StandardCharsets.UTF_8)
-            val relativePath = source.toString().stripPrefix("/")
-            Input.VirtualFile(relativePath, text)
-          }
-
-          fn(inputs)
-        }
-      }
-      .sum
-  }
-}
diff --git a/tests/benchmarks/src/main/scala/benchmarks/ScipSemanticdbBench.scala b/tests/benchmarks/src/main/scala/benchmarks/ScipSemanticdbBench.scala
deleted file mode 100644
index 9fa952bb..00000000
--- a/tests/benchmarks/src/main/scala/benchmarks/ScipSemanticdbBench.scala
+++ /dev/null
@@ -1,68 +0,0 @@
-package benchmarks
-
-import java.nio.file.Files
-import java.nio.file.Path
-import java.util.concurrent.TimeUnit
-
-import com.sourcegraph.io.DeleteVisitor
-import com.sourcegraph.scip_java.Dependencies
-import com.sourcegraph.scip_java.ScipJava
-import org.openjdk.jmh.annotations._
-import tests.TestCompiler
-
-@State(Scope.Benchmark)
-class ScipSemanticdbBench {
-
-  var targetroot: Path = _
-  var deps: Dependencies = _
-
-  @Setup
-  def setup(): Unit = {
-    targetroot = Files.createTempDirectory("scip-java")
-    deps = Dependencies.resolveDependencies(
-      List("com.google.guava:guava:30.1-jre")
-    )
-    val compiler =
-      new TestCompiler(deps.classpathSyntax, javacOptions = Nil, targetroot)
-    CompileBench.foreachSource(deps) { inputs =>
-      compiler.compileSemanticdb(inputs).byteCode.length
-    }
-  }
-
-  @TearDown
-  def teardown(): Unit = {
-    Files.walkFileTree(targetroot, new DeleteVisitor())
-  }
-
-  @Benchmark
-  @BenchmarkMode(Array(Mode.SingleShotTime))
-  @OutputTimeUnit(TimeUnit.MILLISECONDS)
-  def jsonParallel(): Unit = run("index.scip", parallel = true)
-
-  @Benchmark
-  @BenchmarkMode(Array(Mode.SingleShotTime))
-  @OutputTimeUnit(TimeUnit.MILLISECONDS)
-  def json(): Unit = run("index.scip", parallel = false)
-
-  private def run(filename: String, parallel: Boolean): Unit = {
-    val output = Files.createTempFile("scip-java", filename)
-    val parallelFlag =
-      if (parallel)
-        "--parallel"
-      else
-        "--no-parallel"
-    val exit = ScipJava
-      .app
-      .run(
-        List(
-          "index-semanticdb",
-          s"--output=$output",
-          parallelFlag,
-          targetroot.toString
-        )
-      )
-
-    require(exit == 0, exit)
-  }
-
-}