Index Kotlin with the Analysis API instead of a compiler plugin (take 2)

.github/renovate.json

@@ -4,5 +4,16 @@
     "config:recommended"
   ],
   "automerge": true,
-  "semanticCommits": false
+  "semanticCommits": false,
+  "packageRules": [
+    {
+      "description": "The Analysis API engine pin is managed manually: it lives in the JetBrains intellij-dependencies repository, uses IDE-aligned version schemes, and must move in lockstep with the `kotlin` toolchain version.",
+      "matchPackageNames": [
+        "org.jetbrains.kotlin:kotlin-compiler",
+        "*-for-ide",
+        "com.intellij.platform:*"
+      ],
+      "enabled": false
+    }
+  ]
 }

.github/workflows/ci.yaml

@@ -22,9 +22,9 @@ jobs:
 
       - run: gradle spotlessCheck --no-daemon
 
-  kotlin_plugin:
+  kotlin_indexer:
     runs-on: ubuntu-latest
-    name: scip-kotlinc
+    name: scip-kotlin-analysis
     steps:
       - uses: actions/checkout@v7
 
@@ -37,8 +37,8 @@ jobs:
         with:
           gradle-version: 9.8.2
 
-      - name: scip-kotlinc tests
-        run: gradle :scip-kotlinc:test --no-daemon
+      - name: scip-kotlin-analysis tests
+        run: gradle :scip-kotlin-analysis:test --no-daemon
 
   docker_test:
     runs-on: ubuntu-latest
@@ -69,7 +69,7 @@ jobs:
 
   ci-pass:
     if: always()
-    needs: [format, kotlin_plugin, docker_test]
+    needs: [format, kotlin_indexer, docker_test]
     runs-on: ubuntu-latest
     name: CI pass
     steps:

.github/workflows/repos.yaml

@@ -119,6 +119,39 @@ jobs:
             expected_language: kotlin
             covers: Gradle, Kotlin multiplatform JVM target, Kotlin 2.4.0, Java 25 runtime
 
+          - name: gradle-kotlin19-kotlinpoet
+            repository: square/kotlinpoet
+            ref: 1.16.0
+            directory: .
+            build_tool: gradle
+            build_command: ":kotlinpoet:compileKotlinJvm"
+            java: 17
+            bazel_version: ""
+            expected_language: kotlin
+            covers: Gradle, Kotlin multiplatform JVM target, legacy Kotlin 1.9.22, Java 17 runtime
+
+          - name: gradle-kotlin23-turbine
+            repository: cashapp/turbine
+            ref: ab1e9a0acbbbb175081a0c5d94cb4f4cf4da444a
+            directory: .
+            build_tool: gradle
+            build_command: "compileKotlinJvm"
+            java: 17
+            bazel_version: ""
+            expected_language: kotlin
+            covers: Gradle, Kotlin multiplatform JVM target, Kotlin 2.3.21, Java 17 runtime
+
+          - name: gradle-kotlin24-detekt
+            repository: detekt/detekt
+            ref: 04f97401d462e899383156b25d026b7b8c181401
+            directory: .
+            build_tool: gradle
+            build_command: ":detekt-api:compileKotlin"
+            java: 21
+            bazel_version: ""
+            expected_language: kotlin
+            covers: Gradle, Kotlin JVM, Kotlin 2.4.0, settings repositories (FAIL_ON_PROJECT_REPOS), Java 21 runtime
+
           - name: gradle-mixed-kotlinpoet
             repository: square/kotlinpoet
             ref: 0182a25089ee1f4f2378568b497f82a8ebc3017d

CONTRIBUTING.md

@@ -54,7 +54,7 @@ These are the main components of the project.
 | ------------------------------------------------------- | -------- | ----------------------------------------------------------------------- |
 | `gradle test --no-daemon`                               | terminal | Run all Gradle tests.                                                    |
 | `gradle :scip-java:test --no-daemon`                    | terminal | Run CLI build-tool integration tests (Gradle, Maven, SCIP config).       |
-| `gradle :scip-kotlinc:test --no-daemon`                 | terminal | Run Kotlin compiler-plugin tests.                                        |
+| `gradle :scip-kotlin-analysis:test --no-daemon`                 | terminal | Run Kotlin indexer tests.                                        |
 | `gradle :scip-snapshots:test --no-daemon`               | terminal | Compare Java and Kotlin snapshot goldens.                                |
 | `gradle :scip-snapshots:saveSnapshots --no-daemon`      | terminal | Regenerate Java and Kotlin snapshot goldens.                             |
 | `gradle :scip-java:installDist --no-daemon`             | terminal | Build a local `scip-java` distribution under `scip-java/build/install/`. |

build-logic/src/main/kotlin/org/scip_code/scip_java/buildlogic/ScipCompile.kt

@@ -4,7 +4,6 @@ import java.io.File
 import org.gradle.api.Task
 import org.gradle.api.artifacts.Configuration
 import org.gradle.api.file.Directory
-import org.gradle.api.file.FileSystemLocation
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.compile.JavaCompile
 
@@ -27,29 +26,6 @@ fun JavaCompile.useScipJavac(
         (options.forkOptions.jvmArgs ?: emptyList()) + JavacInternals.jvmOptions(rootDir)
 }
 
-/**
- * Builds the `kotlinc` arguments that load the scip-kotlinc compiler plugin from [pluginClasspath]
- * (the resolved shaded jar) and point it at [sourceroot]/[targetroot].
- *
- * The mapping lives here, in compiled build logic, rather than in a build script: a `.map {}`
- * lambda declared in a `.gradle.kts` file captures a hidden reference to the script object, which
- * the configuration cache cannot serialize.
- */
-fun scipKotlincPluginArgs(
-    pluginClasspath: Provider<Set<FileSystemLocation>>,
-    sourceroot: String,
-    targetroot: String,
-): Provider<List<String>> =
-    pluginClasspath.map { locations ->
-        listOf(
-            "-Xplugin=${locations.single().asFile.absolutePath}",
-            "-P",
-            "plugin:scip-kotlinc:sourceroot=$sourceroot",
-            "-P",
-            "plugin:scip-kotlinc:targetroot=$targetroot",
-        )
-    }
-
 /**
  * Registers a `doFirst` action that empties [dir] (deletes then recreates it) before the task runs.
  *

gradle/libs.versions.toml

@@ -2,9 +2,10 @@
 clikt = "5.1.0"
 gradle-api = "8.11.1"
 junit-jupiter = "5.11.4"
-kctfork = "0.14.0"
-kotest = "6.2.6"
 kotlin = "2.4.21"
+kotlin-analysis-api = "2.4.0"
+kotlinx-collections-immutable = "0.3.8"
+kotlinx-coroutines-intellij = "1.8.0-intellij-13"
 kotlinx-serialization = "1.12.0"
 lombok = "1.18.48"
 maven-plugin-annotations = "3.16.0"
@@ -18,20 +19,24 @@ spotless = "8.10.4"
 vanniktech-maven-publish = "0.37.0"
 
 [libraries]
+caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version = "2.9.3" }
 clikt-jvm = { module = "com.github.ajalt.clikt:clikt-jvm", version.ref = "clikt" }
 gradle-api = { module = "dev.gradleplugins:gradle-api", version.ref = "gradle-api" }
 gradle-test-kit = { module = "dev.gradleplugins:gradle-test-kit", version.ref = "gradle-api" }
-kctfork-core = { module = "dev.zacsweers.kctfork:core", version.ref = "kctfork" }
-kotest-assertions-core = { module = "io.kotest:kotest-assertions-core-jvm", version.ref = "kotest" }
-kotlin-compiler-embeddable = { module = "org.jetbrains.kotlin:kotlin-compiler-embeddable", version.ref = "kotlin" }
-kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect", version.ref = "kotlin" }
-kotlin-scripting-common = { module = "org.jetbrains.kotlin:kotlin-scripting-common", version.ref = "kotlin" }
-kotlin-scripting-dependencies = { module = "org.jetbrains.kotlin:kotlin-scripting-dependencies", version.ref = "kotlin" }
-kotlin-scripting-dependencies-maven = { module = "org.jetbrains.kotlin:kotlin-scripting-dependencies-maven", version.ref = "kotlin" }
-kotlin-scripting-jvm = { module = "org.jetbrains.kotlin:kotlin-scripting-jvm", version.ref = "kotlin" }
+kotlin-analysis-api-api = { module = "org.jetbrains.kotlin:analysis-api-for-ide", version.ref = "kotlin-analysis-api" }
+kotlin-analysis-api-impl-base = { module = "org.jetbrains.kotlin:analysis-api-impl-base-for-ide", version.ref = "kotlin-analysis-api" }
+kotlin-analysis-api-k2 = { module = "org.jetbrains.kotlin:analysis-api-k2-for-ide", version.ref = "kotlin-analysis-api" }
+kotlin-analysis-api-platform-interface = { module = "org.jetbrains.kotlin:analysis-api-platform-interface-for-ide", version.ref = "kotlin-analysis-api" }
+kotlin-analysis-api-standalone = { module = "org.jetbrains.kotlin:analysis-api-standalone-for-ide", version.ref = "kotlin-analysis-api" }
+kotlin-analysis-compiler = { module = "org.jetbrains.kotlin:kotlin-compiler", version.ref = "kotlin-analysis-api" }
+kotlin-analysis-compiler-common = { module = "org.jetbrains.kotlin:kotlin-compiler-common-for-ide", version.ref = "kotlin-analysis-api" }
+kotlin-analysis-low-level-api-fir = { module = "org.jetbrains.kotlin:low-level-api-fir-for-ide", version.ref = "kotlin-analysis-api" }
+kotlin-analysis-symbol-light-classes = { module = "org.jetbrains.kotlin:symbol-light-classes-for-ide", version.ref = "kotlin-analysis-api" }
 kotlin-stdlib = { module = "org.jetbrains.kotlin:kotlin-stdlib", version.ref = "kotlin" }
 kotlin-test = { module = "org.jetbrains.kotlin:kotlin-test", version.ref = "kotlin" }
 kotlin-test-junit5 = { module = "org.jetbrains.kotlin:kotlin-test-junit5", version.ref = "kotlin" }
+kotlinx-collections-immutable-jvm = { module = "org.jetbrains.kotlinx:kotlinx-collections-immutable-jvm", version.ref = "kotlinx-collections-immutable" }
+kotlinx-coroutines-core-jvm = { module = "com.intellij.platform:kotlinx-coroutines-core-jvm", version.ref = "kotlinx-coroutines-intellij" }
 kotlinx-serialization-json-jvm = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json-jvm", version.ref = "kotlinx-serialization" }
 lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" }
 maven-plugin-annotations = { module = "org.apache.maven.plugin-tools:maven-plugin-annotations", version.ref = "maven-plugin-annotations" }

scip-gradle-plugin/src/main/java/org/scip_code/scip_java/gradle/ScipGradlePlugin.java

@@ -1,12 +1,17 @@
 package org.scip_code.scip_java.gradle;
 
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.initialization.Settings;
 import org.gradle.api.internal.GradleInternal;
 import org.gradle.api.tasks.compile.JavaCompile;
@@ -110,15 +115,14 @@ private void configureProject(Project project) {
         triggers.add("compileKotlin");
         triggers.add("compileTestKotlin");
       }
-
-      // The CLI's init script provides the path of the embedded scip-kotlinc jar.
-      Object scipKotlinc = requiredExtra(extraProperties, "scipKotlincJar");
-      project
-          .getTasks()
-          .configureEach(
-              task -> configureKotlinCompileTask(task, scipKotlinc, sourceRoot, targetRoot));
     }
 
+    // Not limited to the plugin checks above: AGP 9's built-in Kotlin adds
+    // Kotlin compile tasks to Android modules without applying a Kotlin plugin.
+    project
+        .getTasks()
+        .configureEach(task -> configureKotlinCompileTask(task, sourceRoot, targetRoot));
+
     project.getTasks().create("scipCompileAll").dependsOn(triggers);
 
     project.getTasks().create("scipPrintDependencies", WriteDependencies.class);
@@ -172,39 +176,72 @@ private static boolean tryAddJavacPlugin(Project project, Object javacPluginDep)
     }
   }
 
-  private static void configureKotlinCompileTask(
-      Task task, Object scipKotlinc, String sourceRoot, String targetRoot) {
+  /**
+   * Kotlin sources are not indexed inside kotlinc: after each Kotlin compile task runs, its sources
+   * and classpath are dumped to {@code <targetroot>/kotlin-configs/<task>.txt} for the scip-java
+   * CLI to index after the build, decoupling indexing from the build's Kotlin compiler version.
+   */
+  private static void configureKotlinCompileTask(Task task, String sourceRoot, String targetRoot) {
     if (!task.getClass().getSimpleName().contains("KotlinCompile")) {
       return;
     }
 
-    // Referring to KotlinCompile directly here triggers NoClassDefFoundError -
-    // the plugin classpath is murky and we deliberately don't bundle the Kotlin
-    // Gradle plugin. So we commit the sins of reflection for our limited needs.
-    try {
-      Object kotlinOptions = task.getClass().getMethod("getKotlinOptions").invoke(task);
-
-      @SuppressWarnings("unchecked")
-      List<String> freeCompilerArgs =
-          (List<String>)
-              kotlinOptions.getClass().getMethod("getFreeCompilerArgs").invoke(kotlinOptions);
-
-      List<String> newArgs = new ArrayList<>(freeCompilerArgs.size() + 5);
-      newArgs.addAll(freeCompilerArgs);
-      newArgs.add("-Xplugin=" + scipKotlinc);
-      newArgs.add("-P");
-      newArgs.add("plugin:scip-kotlinc:sourceroot=" + sourceRoot);
-      newArgs.add("-P");
-      newArgs.add("plugin:scip-kotlinc:targetroot=" + targetRoot);
-
-      kotlinOptions
-          .getClass()
-          .getMethod("setFreeCompilerArgs", List.class)
-          .invoke(kotlinOptions, newArgs);
-    } catch (ReflectiveOperationException exc) {
-      throw new RuntimeException(
-          "scip-java: failed to configure Kotlin compile task '" + task.getName() + "'", exc);
-    }
+    task.doLast(
+        ignored -> {
+          try {
+            List<String> lines = new ArrayList<>();
+            lines.add("sourceroot " + sourceRoot);
+
+            // Referring to KotlinCompile directly triggers NoClassDefFoundError -
+            // the plugin classpath is murky and we deliberately don't bundle the
+            // Kotlin Gradle plugin. So we commit the sins of reflection.
+            FileCollection libraries;
+            try {
+              libraries = (FileCollection) task.getClass().getMethod("getLibraries").invoke(task);
+            } catch (ReflectiveOperationException exc) {
+              libraries = (FileCollection) task.getClass().getMethod("getClasspath").invoke(task);
+            }
+            for (File entry : libraries.getFiles()) {
+              lines.add("classpath " + entry.getAbsolutePath());
+            }
+
+            int sources = 0;
+            for (File file : task.getInputs().getSourceFiles().getFiles()) {
+              if (file.getName().endsWith(".kt") || file.getName().endsWith(".kts")) {
+                lines.add("source " + file.getAbsolutePath());
+                sources++;
+              }
+            }
+            // Java sources of the same module are listed too, so Kotlin
+            // references to them resolve. The indexer only emits Kotlin files.
+            try {
+              FileCollection javaSources =
+                  (FileCollection) task.getClass().getMethod("getJavaSources").invoke(task);
+              for (File file : javaSources.getFiles()) {
+                if (file.getName().endsWith(".java")) {
+                  lines.add("source " + file.getAbsolutePath());
+                }
+              }
+            } catch (ReflectiveOperationException exc) {
+              // Kotlin Gradle Plugin without javaSources (pre 1.7).
+            }
+            if (sources == 0) {
+              return;
+            }
+
+            Path configDir = Paths.get(targetRoot, "kotlin-configs");
+            Files.createDirectories(configDir);
+            String fileName = task.getPath().replaceAll("[^A-Za-z0-9]", "-") + ".txt";
+            Files.write(configDir.resolve(fileName), lines);
+          } catch (Exception exc) {
+            throw new RuntimeException(
+                "scip-java: failed to extract Kotlin compile configuration for task '"
+                    + task.getName()
+                    + "': "
+                    + exc,
+                exc);
+          }
+        });
   }
 
   private static Object requiredExtra(Map<String, Object> extraProperties, String name) {

scip-java/build.gradle.kts

@@ -13,17 +13,12 @@ description = "Java and Kotlin indexer for SCIP"
 
 val javacShadowJar = shadowJarArtifact(":scip-javac", "javacShadowJar")
 val gradlePluginShadowJar = shadowJarArtifact(":scip-gradle-plugin", "gradlePluginShadowJar")
-val kotlincShadowJar = shadowJarArtifact(":scip-kotlinc", "kotlincShadowJar")
 
 dependencies {
     implementation(project(":scip-aggregator"))
+    implementation(project(":scip-kotlin-analysis"))
     implementation(libs.clikt.jvm)
     implementation(libs.kotlin.stdlib)
-    implementation(libs.kotlin.compiler.embeddable)
-    implementation(libs.kotlin.scripting.common)
-    implementation(libs.kotlin.scripting.jvm)
-    implementation(libs.kotlin.scripting.dependencies)
-    implementation(libs.kotlin.scripting.dependencies.maven)
     implementation(libs.kotlinx.serialization.json.jvm)
 
     testImplementation(libs.kotlin.test)
@@ -46,9 +41,6 @@ val generateEmbeddedResources = tasks.register<Sync>("generateEmbeddedResources"
     from(gradlePluginShadowJar) {
         rename { "gradle-plugin.jar" }
     }
-    from(kotlincShadowJar) {
-        rename { "scip-kotlinc.jar" }
-    }
     into(layout.buildDirectory.dir("generated/resources/embedded"))
 }
 

scip-java/src/main/kotlin/org/scip_code/scip_java/Embedded.kt

@@ -25,8 +25,6 @@ object Embedded {
 
     fun gradlePluginJar(tmpDir: Path): Path = copyFile(tmpDir, "gradle-plugin.jar")
 
-    fun scipKotlincJar(tmpDir: Path): Path = copyFile(tmpDir, "scip-kotlinc.jar")
-
     private fun javacErrorpath(tmp: Path): Path = tmp.resolve("errorpath.txt")
 
     data class CustomJavac(val executable: Path, val environment: Map<String, String>)

scip-java/src/main/kotlin/org/scip_code/scip_java/ScipJava.kt

@@ -1,6 +1,7 @@
 package org.scip_code.scip_java
 
 import java.io.PrintStream
+import kotlin.system.exitProcess
 
 /**
  * Public entry point for the scip-java CLI. The single [app] instance is shared across test suites
@@ -17,6 +18,8 @@ object ScipJava {
     @JvmStatic
     fun main(args: Array<String>) {
         app.runAndExitIfNonZero(args.toList())
+        // The Analysis API indexer leaves non-daemon threads behind; exit explicitly.
+        exitProcess(0)
     }
 
     fun printHelp(out: PrintStream) {