Improvements to the pyinterfacegen tool

DEVELOPMENT.md

@@ -50,7 +50,7 @@ to the local repository also omits the license checks.
 ## Structure
 
 This repository is structured as a Maven multi-module project. There is also a Gradle project
-for the Gradle plugin: `org.graalvm.python.gradle.plugin`.
+for the Gradle plugin: `org.graalvm.python.gradle.plugin`, and for the `pyinterface` tool.
 
 A Maven project and pom.xml exist for the Gradle plugin, but solely to delegate most of the
 lifecycle tasks to Gradle. This allows you to run those tasks with a single Maven command.

pyinterfacegen/README.md

@@ -75,8 +75,6 @@ The Gradle plugin also provides a task that resolves a whole dependency graph to
 Example usage:
 
 ```kotlin
-import org.graalvm.python.pyinterfacegen.PyiFromDependencySources
-
 val commons by configurations.registering {
     isCanBeConsumed = false
     isCanBeResolved = true
@@ -87,7 +85,7 @@ dependencies {
 }
 
 // Register the task
-val pyi by tasks.registering(PyiFromDependencySources::class) {
+val pyi by tasks.registering(org.graalvm.python.pyinterfacegen.PyiFromDependencySources::class) {
     group = "verification"
     description = "Generate Python stubs from dependency sources in 'depStubs'"
     // Provide the configuration object directly so task inputs track changes correctly
@@ -120,3 +118,48 @@ This build includes a convenience task that downloads a matching GraalPy distrib
 Notes:
 - Override the GraalPy version with `-PgraalPyVersion=25.0.1` if needed.
 - The task uses the GraalPy community JVM distribution and sets `CLASSPATH` to your compiled classes so Java types are available at runtime.
+
+## Optional: type-check generated stubs
+
+You can run a Python type checker over the generated `.pyi` output to sanity-check internal consistency. A Gradle task type
+`TypeCheckPyiTask` is provided by the plugin. This project registers an example task:
+
+```bash
+./gradlew typecheckGraalPyStubs
+```
+
+By default it runs mypy via `python3 -m mypy`. If mypy isn't installed, the task logs and skips. To use pyright instead:
+
+```kotlin
+tasks.named<TypeCheckPyiTask>("typecheckGraalPyStubs") {
+    typeChecker.set("pyright")
+    // extraArgs.set(listOf("--verifytypes", "your_root_package"))
+}
+```
+
+Tip: install tools as needed:
+ - mypy: `python3 -m pip install mypy`
+ - pyright: `npm i -g pyright`
+
+### Namespace packages and mypy
+
+Generated modules may omit intermediate `__init__.py` files to allow multiple distributions to share a namespace
+(e.g., generating `foo.bar` and `foo.baz` separately without them conflicting on `foo/__init__.py`). This relies on
+[PEP 420] namespace packages.
+
+Mypy needs to be told to treat such directories as packages. The plugin does this automatically by passing:
+ - `--namespace-packages`: opt-in to PEP 420 package discovery.
+ - `--explicit-package-bases`: interpret the provided paths as package roots, improving resolution for PEP 420 trees.
+
+If you run mypy yourself, enable the same in your config:
+
+```ini
+# mypy.ini or pyproject.toml [tool.mypy]
+namespace_packages = true
+explicit_package_bases = true
+```
+
+For code that imports from namespace fragments installed in different locations, ensure mypy sees all fragments in its
+search path (e.g., by activating the venv where they’re installed, or by setting `MYPYPATH` to include those site dirs).
+
+[PEP 420]: https://peps.python.org/pep-0420/

pyinterfacegen/apache-commons-sample/build.gradle.kts

@@ -1,4 +1,5 @@
 import org.graalvm.python.pyinterfacegen.PyiFromDependencySources
+import org.graalvm.python.pyinterfacegen.TypeCheckPyiTask
 
 plugins {
     // Apply the plugin without a version; version resolution is handled in settings.gradle.kts
@@ -36,9 +37,18 @@ val pyi by tasks.registering(PyiFromDependencySources::class) {
 
     // Map Java package prefixes to nicer Python packages for both libraries
     packageMap.set(
-        "org.apache.commons.lang3=commons.lang," +
-        "org.apache.commons.collections4=commons.collections"
+        "org.apache.commons.lang3=commons_lang," +
+        // Avoid shadowing stdlib 'collections' by remapping to a safe name. TODO: Work out what to do about this properly.
+        "org.apache.commons.collections4=commons_collections"
     )
     moduleName.set("apache-commons")
     moduleVersion.set("0.1.0")
 }
+
+// Optional: type check the generated stubs (not part of default lifecycle)
+val typecheckApacheCommonsStubs by tasks.registering(TypeCheckPyiTask::class) {
+    description = "Run mypy or pyright over the generated Apache Commons stubs"
+    // The doclet assembles the module in build/pymodule (root), so point to that.
+    moduleDir.set(layout.buildDirectory.dir("pymodule"))
+    dependsOn(pyi)
+}

pyinterfacegen/build-logic/build.gradle.kts

@@ -0,0 +1,12 @@
+plugins {
+    `kotlin-dsl`
+}
+
+repositories {
+    gradlePluginPortal()
+    mavenCentral()
+}
+
+kotlin {
+    jvmToolchain(21)
+}

pyinterfacegen/build-logic/settings.gradle.kts

@@ -0,0 +1 @@
+rootProject.name = "j2pyi-build-logic"

pyinterfacegen/build-logic/src/main/kotlin/j2pyi.convention.gradle.kts

@@ -0,0 +1 @@
+// This is empty but required to make Gradle put the utility classes onto the classpath.

pyinterfacegen/build-logic/src/main/kotlin/org/graalvm/python/pyinterfacegen/build/RootPom.kt

@@ -0,0 +1,160 @@
+package org.graalvm.python.pyinterfacegen.build
+
+import org.gradle.api.Project
+import org.w3c.dom.Element
+import java.io.File
+import javax.xml.parsers.DocumentBuilderFactory
+import javax.xml.xpath.XPathConstants
+import javax.xml.xpath.XPathFactory
+
+data class LicenseInfo(val name: String, val url: String)
+data class DeveloperInfo(
+    val name: String,
+    val email: String,
+    val organization: String?,
+    val organizationUrl: String?
+)
+data class ScmInfo(
+    val connection: String,
+    val developerConnection: String,
+    val url: String,
+    val tag: String?
+)
+data class RootPomMetadata(
+    val version: String,
+    val url: String,
+    val licenses: List<LicenseInfo>,
+    val developers: List<DeveloperInfo>,
+    val scm: ScmInfo
+)
+
+private fun findPomUpwards(start: File): File? {
+    var cur: File? = start
+    while (cur != null) {
+        val pom = File(cur, "pom.xml")
+        if (pom.exists()) return pom
+        cur = cur.parentFile
+    }
+    return null
+}
+
+private val xp by lazy { XPathFactory.newInstance().newXPath() }
+
+private fun textOf(elem: Element?, expr: String): String? {
+    if (elem == null) return null
+    val node = xp.evaluate(expr, elem, XPathConstants.NODE) as? org.w3c.dom.Node ?: return null
+    return node.textContent?.trim()?.ifBlank { null }
+}
+
+private fun properties(elem: Element): Map<String, String> {
+    val propsElem = xp.evaluate("/*[local-name()='project']/*[local-name()='properties']",
+        elem, XPathConstants.NODE) as? Element ?: return emptyMap()
+    val map = linkedMapOf<String, String>()
+    val children = propsElem.childNodes
+    for (i in 0 until children.length) {
+        val n = children.item(i)
+        if (n is Element) {
+            val key = n.localName ?: n.tagName
+            val value = n.textContent?.trim().orEmpty()
+            if (key.isNotEmpty() && value.isNotEmpty()) {
+                map[key] = value
+            }
+        }
+    }
+    return map
+}
+
+private val PLACEHOLDER = Regex("\\$\\{([^}]+)}")
+private fun resolvePlaceholders(input: String, props: Map<String, String>): String {
+    // Resolve recursively but with a reasonable cap to avoid loops.
+    var prev = input
+    repeat(8) {
+        val next = PLACEHOLDER.replace(prev) { m ->
+            val key = m.groupValues[1]
+            props[key] ?: m.value
+        }
+        if (next == prev) return next
+        prev = next
+    }
+    return prev
+}
+
+fun readRootPomMetadata(project: Project): RootPomMetadata {
+    val pomFile = findPomUpwards(project.rootDir)
+        ?: error("Cannot locate root pom.xml starting at ${project.rootDir}")
+    val dbf = DocumentBuilderFactory.newInstance().apply { isNamespaceAware = true }
+    val doc = dbf.newDocumentBuilder().parse(pomFile)
+    val root = doc.documentElement
+
+    val props = properties(root)
+    val revision = props["revision"]
+        ?: error("Root pom.xml is missing <properties><revision>...</revision></properties>")
+
+    // Prefer explicit <url> first, then fall back to project.url.root property.
+    val urlRaw = xp.evaluate("/*[local-name()='project']/*[local-name()='url']/text()",
+        root, XPathConstants.STRING) as String?
+    val url = (urlRaw?.trim()?.takeIf { it.isNotEmpty() } ?: props["project.url.root"])
+        ?.let { resolvePlaceholders(it, props) }
+        ?: error("Root pom.xml is missing <url> or properties.project.url.root")
+
+    // Licenses
+    val licenseNodes = xp.evaluate(
+        "/*[local-name()='project']/*[local-name()='licenses']/*[local-name()='license']",
+        root, XPathConstants.NODESET
+    ) as org.w3c.dom.NodeList
+    val licenses = buildList {
+        for (i in 0 until licenseNodes.length) {
+            val e = licenseNodes.item(i) as Element
+            val name = textOf(e, "./*[local-name()='name']") ?: error("License name missing")
+            val lurl = textOf(e, "./*[local-name()='url']") ?: error("License url missing")
+            add(LicenseInfo(name, lurl))
+        }
+    }
+    require(licenses.isNotEmpty()) { "No <licenses> found in root pom.xml" }
+
+    // Developers
+    val devNodes = xp.evaluate(
+        "/*[local-name()='project']/*[local-name()='developers']/*[local-name()='developer']",
+        root, XPathConstants.NODESET
+    ) as org.w3c.dom.NodeList
+    val developers = buildList {
+        for (i in 0 until devNodes.length) {
+            val e = devNodes.item(i) as Element
+            val name = textOf(e, "./*[local-name()='name']") ?: error("Developer name missing")
+            val email = textOf(e, "./*[local-name()='email']") ?: error("Developer email missing")
+            val org = textOf(e, "./*[local-name()='organization']")
+            val orgUrl = textOf(e, "./*[local-name()='organizationUrl']")
+            add(DeveloperInfo(name, email, org, orgUrl))
+        }
+    }
+    require(developers.isNotEmpty()) { "No <developers> found in root pom.xml" }
+
+    // SCM
+    val scmElem = xp.evaluate(
+        "/*[local-name()='project']/*[local-name()='scm']",
+        root, XPathConstants.NODE
+    ) as? Element ?: error("No <scm> section in root pom.xml")
+    val scm = ScmInfo(
+        connection = resolvePlaceholders(
+            textOf(scmElem, "./*[local-name()='connection']")
+                ?: error("SCM connection missing"), props
+        ),
+        developerConnection = resolvePlaceholders(
+            textOf(scmElem, "./*[local-name()='developerConnection']")
+                ?: error("SCM developerConnection missing"), props
+        ),
+        url = resolvePlaceholders(
+            textOf(scmElem, "./*[local-name()='url']")
+                ?: error("SCM url missing"), props
+        ),
+        tag = textOf(scmElem, "./*[local-name()='tag']")?.let { resolvePlaceholders(it, props) }
+    )
+
+    return RootPomMetadata(
+        version = revision,
+        url = url,
+        licenses = licenses,
+        developers = developers,
+        scm = scm
+    )
+}

pyinterfacegen/build.gradle.kts

@@ -1,17 +1,74 @@
 import org.graalvm.python.pyinterfacegen.J2PyiTask
+import org.graalvm.python.pyinterfacegen.TypeCheckPyiTask
 import org.gradle.internal.os.OperatingSystem
+import org.gradle.api.publish.maven.MavenPublication
+import org.graalvm.python.pyinterfacegen.build.readRootPomMetadata
 import java.net.URI
 import java.util.*
 
 plugins {
     kotlin("jvm") version "2.2.10"
     java
-    id("org.graalvm.python.pyinterfacegen") version "1.3-SNAPSHOT"
+    // Use the locally included plugin (see settings.gradle.kts pluginManagement). Version is supplied there.
+    id("org.graalvm.python.pyinterfacegen")
+    id("j2pyi.convention")  // Local build logic.
 }
 
+// Read metadata and version from the repository root pom.xml
+val rootPomMeta = readRootPomMetadata(rootProject)
+
+allprojects {
+    group = "org.graalvm.python"
+    version = rootPomMeta.version
+}
+
+// When developing locally, always use the local doclet project for any requests to the published module,
+// regardless of version requested by the plugin.
 allprojects {
-    group = "com.oracle.graal.python"
-    version = "1.3-SNAPSHOT"
+    configurations.configureEach {
+        resolutionStrategy.dependencySubstitution {
+            substitute(module("org.graalvm.python.pyinterfacegen:j2pyi-doclet"))
+                .using(project(":doclet"))
+        }
+    }
+}
+
+// For projects that publish, project the POM data from the root pom.xml into their publications.
+subprojects {
+    plugins.withId("maven-publish") {
+        extensions.configure<PublishingExtension> {
+            publications.withType<MavenPublication>().configureEach {
+                pom {
+                    url.set(rootPomMeta.url)
+                    version = this@subprojects.version.toString()
+                    licenses {
+                        rootPomMeta.licenses.forEach { lic ->
+                            license {
+                                name.set(lic.name)
+                                url.set(lic.url)
+                            }
+                        }
+                    }
+                    developers {
+                        rootPomMeta.developers.forEach { d ->
+                            developer {
+                                name.set(d.name)
+                                email.set(d.email)
+                                d.organization?.let { organization.set(it) }
+                                d.organizationUrl?.let { organizationUrl.set(it) }
+                            }
+                        }
+                    }
+                    scm {
+                        url.set(rootPomMeta.scm.url)
+                        connection.set(rootPomMeta.scm.connection)
+                        developerConnection.set(rootPomMeta.scm.developerConnection)
+                        rootPomMeta.scm.tag?.let { tag.set(it) }
+                    }
+                }
+            }
+        }
+    }
 }
 
 repositories {
@@ -99,6 +156,17 @@ val graalPyBindingsMain by tasks.register<J2PyiTask>("graalPyBindingsMain") {
     setDestinationDir(layout.buildDirectory.dir("pymodule/${project.name}").get().asFile)
 }
 
+// Optional verification: run a Python type checker over the generated module.
+// Not wired into the standard 'check' lifecycle; invoke explicitly.
+tasks.register<TypeCheckPyiTask>("typecheckGraalPyStubs") {
+    description = "Run mypy (or pyright) over the generated .pyi module to detect internal inconsistencies"
+    moduleDir.set(layout.buildDirectory.dir("pymodule/${project.name}"))
+    // Ensure stubs are generated before checking them
+    dependsOn(graalPyBindingsMain)
+    // Default checker is mypy; customize via:
+    //   typeChecker.set("pyright")
+    //   extraArgs.set(listOf("--strict"))
+}
 // Execute a simple GraalPy run that imports the generated module and calls a method
 val graalPyIntegrationTest by tasks.registering {
     group = "verification"