Merge pull request #26 from dagger/sdk-module-interface-catchup

Update the Python SDK scope interface

Author: eunomie

.dagger/modules/e2e/fixtures/skip/.dagger-python-sdk-skip-generate

@@ -8,7 +8,6 @@ type E2e {
   let lookupModulePath: String! = fixtureRoot + "/lookup/app"
   let lookupNestedPath: String! = lookupModulePath + "/nested"
   let nonPythonModulePath: String! = fixtureRoot + "/lookup/not-python"
-  let skipModulePath: String! = fixtureRoot + "/skip/app"
   let generateModulePath: String! = fixtureRoot + "/generate/app"
   let configModulePath: String! = fixtureRoot + "/config/app"
   let configuredModulePath: String! = fixtureRoot + "/config/configured"
@@ -70,15 +69,6 @@ type E2e {
     null
   }
 
-  """
-  The helper should expose the generate skip marker used by callers.
-  """
-  pub skipGenerateFilenameCheck: Void @check {
-    assert(pythonSdk.skipGenerateFilename == ".dagger-python-sdk-skip-generate", "skip marker filename changed")
-
-    null
-  }
-
   """
   Mod lookup should find the owning Python SDK module when called from a nested path.
   """
@@ -118,26 +108,26 @@ type E2e {
   }
 
   """
-  detectScope answers with the directory of the nearest pyproject.toml as a
+  findClientRoot answers with the directory of the nearest pyproject.toml as a
   workspace-root-relative path, with the module rather than its vendored client
-  library, and with nothing where there is none.
+  library, and with null where there is none.
   """
-  pub detectScopeCheck(ws: Workspace!): Void @check {
-    assert(pythonSdk.detectScope(ws.withWorkdir(lookupNestedPath)) == lookupModulePath, "detectScope should find the module owning a nested path")
-    assert(pythonSdk.detectScope(ws.withWorkdir(lookupModulePath)) == lookupModulePath, "detectScope should answer with the module root itself")
-    assert(pythonSdk.detectScope(ws.withWorkdir(runtimeModulePath + "/sdk/src/dagger/client")) == runtimeModulePath, "detectScope should answer with the module owning a vendored sdk/, not the sdk/ itself")
-    assert(pythonSdk.detectScope(ws.withWorkdir("sdk/src/dagger/client")) == "sdk", "detectScope should not lift a standalone sdk/ project into a parent that is not a Python module")
+  pub findClientRootCheck(ws: Workspace!): Void @check {
+    assert(pythonSdk.findClientRoot(ws.withWorkdir(lookupNestedPath)) == lookupModulePath, "findClientRoot should find the module owning a nested path")
+    assert(pythonSdk.findClientRoot(ws.withWorkdir(lookupModulePath)) == lookupModulePath, "findClientRoot should answer with the module root itself")
+    assert(pythonSdk.findClientRoot(ws.withWorkdir(runtimeModulePath + "/sdk/src/dagger/client")) == runtimeModulePath, "findClientRoot should answer with the module owning a vendored sdk/, not the sdk/ itself")
+    assert(pythonSdk.findClientRoot(ws.withWorkdir("sdk/src/dagger/client")) == "sdk", "findClientRoot should not lift a standalone sdk/ project into a parent that is not a Python module")
 
     let nestedSdkWs = ws.withNewFile("/" + runtimeModulePath + "/sdk/dagger-module.toml", "name = \"nested\"\n\n[runtime]\nsource = \"python\"\n")
-    assert(pythonSdk.detectScope(nestedSdkWs.withWorkdir(runtimeModulePath + "/sdk/src/dagger/client")) == runtimeModulePath + "/sdk", "detectScope should answer with a nested module named sdk, not lift it into its parent")
-    assert(pythonSdk.detectScope(ws) == "", "detectScope should find no scope at the workspace root")
+    assert(pythonSdk.findClientRoot(nestedSdkWs.withWorkdir(runtimeModulePath + "/sdk/src/dagger/client")) == runtimeModulePath + "/sdk", "findClientRoot should answer with a nested module named sdk, not lift it into its parent")
+    assert(pythonSdk.findClientRoot(ws) == null, "findClientRoot should return null when it finds no client root")
 
     null
   }
 
   """
   generateScope initializes a module that has no config: the template, a
-  dagger-module.toml from the engine's builder and no dagger.json, and the
+  dagger-module.toml from the manifest builder and no dagger.json, and the
   generated client library, all under the scope, with the workspace cwd and
   existing files untouched. SDK settings select the template and its
   pyproject.toml values.
@@ -195,27 +185,28 @@ type E2e {
   }
 
   """
-  generateScope regenerates a module that already has a config, whether a
-  dagger-module.toml or a pre-1.0 dagger.json: only the generated client
-  library is added, and nothing else changes.
+  generateScope preserves a current manifest. It migrates a pre-1.0
+  dagger.json to dagger-module.toml. Both cases generate the client library.
   """
   pub generateScopeExistingCheck(ws: Workspace!): Void @check {
-    [
-      [tomlGenerateModulePath, "toml-generate-app"],
-      [generateModulePath, "generate-app"],
-    ].each { pair =>
-      let path = pair[0] ?? ""
-      let name = pair[1] ?? ""
-      let changes = pythonSdk
-        .generateScope(ws.withWorkdir(path), isModule: true, name: name, clients: [])
-        .withWorkdir(".")
-        .changes(ws)
-
-      assertAdded(changes, path + "/" + generatedMarkerPath)
-      assert(changes.modifiedPaths.length == 0, name + ": regenerating modified existing files: " + changes.modifiedPaths.join(", "))
-      assert(contains(changes.addedPaths, path + "/dagger-module.toml") == false, name + ": regenerating must not write a manifest")
-      null
-    }
+    let current = pythonSdk
+      .generateScope(ws.withWorkdir(tomlGenerateModulePath), isModule: true, name: "toml-generate-app", clients: [])
+      .withWorkdir(".")
+      .changes(ws)
+    assertAdded(current, tomlGenerateModulePath + "/" + generatedMarkerPath)
+    assertContains(current.after.file(tomlGenerateModulePath + "/dagger-module.toml").contents, "engineVersion = \"v1.0.0-0\"", "regenerating did not preserve engineVersion")
+
+    let legacy = pythonSdk
+      .generateScope(ws.withWorkdir(generateModulePath), isModule: true, name: "generate-app", clients: [])
+      .withWorkdir(".")
+      .changes(ws)
+    assertAdded(legacy, generateModulePath + "/" + generatedMarkerPath)
+    assertAdded(legacy, generateModulePath + "/dagger-module.toml")
+    assert(contains(legacy.removedPaths, generateModulePath + "/dagger.json"), "migration did not remove dagger.json")
+    assertContainsAll(legacy.after.file(generateModulePath + "/dagger-module.toml").contents, [
+      "name = \"generate-app\"",
+      "source = \"python\"",
+    ])
 
     null
   }
@@ -314,19 +305,6 @@ type E2e {
     null
   }
 
-  """
-  A skip marker above a module should make generate return an empty changeset.
-  """
-  pub skipGenerateCheck(ws: Workspace!): Void @check {
-    let mod = pythonSdk.mod(ws, path: skipModulePath)
-    let changes = mod.generate
-
-    assert(mod.skipGenerate, "skip marker was not detected")
-    assert(changes.isEmpty, "generate returned changes for a skipped module")
-
-    null
-  }
-
   """
   A module on this repository's runtime builds and runs, driven through a
   released CLI. The fixture points `[runtime] source` at `runtime/` by relative

.dagger/modules/e2e/fixtures/skip/app/dagger.json

@@ -6,4 +6,4 @@ engineVersion = "v1.0.0-0"
 
 [[dependencies]]
   name = "engine-dev"
-  source = "github.com/dagger/dagger/.dagger/modules/engine-dev@055def6f7794b9d0dfdcc4252f0695ea92257536"
+  source = "github.com/dagger/dagger/.dagger/modules/engine-dev@8fd9b22b5416f8dc7cb420ba37769adef6e874d2"

.dagger/modules/e2e/fixtures/skip/app/pyproject.toml

@@ -4,7 +4,7 @@ Checks the Python SDK against an engine built from dagger/dagger#13992
 dagger-module.toml.
 """
 type EngineE2e {
-  let engineCommit: String! = "055def6f7794b9d0dfdcc4252f0695ea92257536"
+  let engineCommit: String! = "8fd9b22b5416f8dc7cb420ba37769adef6e874d2"
   let modulePath: String! = ".dagger/modules/sdk-smoke"
 
   let assert(condition: Boolean!, message: String!): Void {
@@ -32,8 +32,10 @@ type EngineE2e {
     let listed = client.withExec(["dagger", "sdk", "list"])
     assert(listed.stdout.contains("python"), "the development Python SDK was not listed")
 
-    let initialized = listed
-      .withExec(["dagger", "--auto-apply", "module", "init", "python", "--name", "sdk-smoke", "--path", modulePath])
+    let checked = listed.withExec(["dagger", "-m", ".dagger/modules/e2e", "check"])
+
+    let initialized = checked
+      .withExec(["dagger", "module", "init", "python", "--auto-apply", "--name", "sdk-smoke", "--path", modulePath])
       .withExec(["test", "-f", modulePath + "/dagger-module.toml"])
       .withExec(["test", "!", "-e", modulePath + "/dagger.json"])
       .withExec(["test", "-f", modulePath + "/pyproject.toml"])
@@ -45,6 +47,21 @@ type EngineE2e {
       .stdout
     assert(release.trimSpace != "", "the generated module did not run")
 
+    initialized
+      .withExec([
+        "dagger", "module", "init", "python", "--auto-apply",
+        "--name", "sdk-settings",
+        "--path", ".dagger/modules/sdk-settings",
+        "--template", "empty",
+        "--python-version", "3.13",
+        "--use-uv=false",
+        "--base-image", "python:3.13-slim",
+      ])
+      .withExec(["grep", "-q", "class SdkSettings:", ".dagger/modules/sdk-settings/src/sdk_settings/__init__.py"])
+      .withExec(["grep", "-q", ">=3.13", ".dagger/modules/sdk-settings/pyproject.toml"])
+      .withExec(["grep", "-q", "use-uv = false", ".dagger/modules/sdk-settings/pyproject.toml"])
+      .withExec(["grep", "-q", "python:3.13-slim", ".dagger/modules/sdk-settings/pyproject.toml"])
+
     null
   }
 }

.dagger/modules/e2e/fixtures/skip/app/src/skip_app/__init__.py

@@ -6,16 +6,17 @@ SDK-specific module authoring (scaffolding new modules, language build config,
 codegen) lives in modules like this one. The engine drives the SDK
 (dagger/dagger#13992): it records a module scope in `dagger.toml`, sets the
 workspace cwd to it, and asks this module to generate the complete scope
-through `detectScope` and `generateScope`. The module writes the manifest and
+through `findClientRoot` and `generateScope`. The module writes the manifest and
 its own files; the engine owns the workspace bookkeeping.
 
-It uses the engine's native `Workspace` and `ModuleSource` APIs directly.
+It uses the engine's native `Workspace` and `ModuleSource` APIs. It uses
+`sdkHelpers.moduleManifest` from `dagger/sdk-helpers`.
 
 ## What lives here
 
 | Path | What it is |
 | --- | --- |
-| `python-sdk.dang`, `mod.dang`, `templates/` | authoring: `detectScope`, `generateScope`, `mod` (generate, config), templates |
+| `python-sdk.dang`, `mod.dang`, `templates/` | authoring: `findClientRoot`, `generateScope`, `mod` (generate, config), templates |
 | `sdk/` | the `dagger-io` client library and code generator |
 | `runtime/` | the module runtime the engine calls to run a module |
 
@@ -26,18 +27,18 @@ result into the module. The runtime never generates: it builds a module from its
 `dagger call`, and a module that has not been generated fails with an
 actionable error rather than being silently regenerated.
 
-Pre-1.0 `dagger.json` modules are the exception: they keep being generated and
-run by the Python SDK baked into the engine, exactly as before.
+When a managed pre-1.0 `dagger.json` scope is generated, the SDK writes
+`dagger-module.toml` and removes `dagger.json`. An unmanaged legacy module keeps
+using the Python SDK that is built into the engine.
 
 ## Two runtimes, one name
 
 Python modules reach one of two implementations, and which one is decided by
 the module's config format:
 
-- **Legacy** — a `dagger.json` with `"sdk": {"source": "python"}` resolves to
-  the runtime baked into the engine (`dagger/dagger`'s `sdk/python`), which
-  still generates bindings at module load. Nothing about those modules changes,
-  and they need no migration.
+- **Legacy** — an unmanaged `dagger.json` with `"sdk": {"source": "python"}`
+  resolves to the runtime built into the engine (`dagger/dagger`'s
+  `sdk/python`). It still generates bindings at module load.
 - **Modern** — a `dagger-module.toml` can point `[runtime] source` at this
   repository's `runtime/`, which is the no-codegen path above. Either a module
   ref or a path relative to the module works, for both `dagger generate` and
@@ -167,27 +168,12 @@ again.
 Standalone clients, in a scope without a module, are not generated yet; adding
 one to a Python scope is refused and the workspace is left unchanged.
 
-## Skipping generation
-
-A `.dagger-python-sdk-skip-generate` file at or above an existing module root
-makes `dagger generate` and `mod generate` leave that module as it is. Useful
-for fixtures, vendored modules, or anything you don't want regenerated. A new
-module is always generated.
-
-```sh
-touch some/fixture/.dagger-python-sdk-skip-generate
-```
-
 ## Test
 
 ```sh
 dagger check
 ```
 
-`engine-e-2-e:dev-sdk-check` builds an engine from dagger/dagger#13992 at the
-commit pinned in `.dagger/modules/engine-e2e` (the `engine-dev` dependency and
-`engineCommit`), installs this checkout as the `python` SDK, initializes a
-Python module, and calls it. Bump both to follow the branch. The `e-2-e:*`
-checks that call this module need an engine with that change as well; on the
-released engine they fail, because the module selects `moduleManifest`, which
-that engine does not have.
+`engine-e-2-e:dev-sdk-check` builds the pinned dagger/dagger#13992 engine. It
+runs the SDK interface checks, initializes Python modules with default and
+explicit settings, and calls a generated module.

.dagger/modules/e2e/main.dang

@@ -9,5 +9,11 @@
     "!.dagger",
     "!future",
     "!docs"
+  ],
+  "dependencies": [
+    {
+      "name": "sdk-helpers",
+      "source": "github.com/dagger/sdk-helpers"
+    }
   ]
 }

.dagger/modules/engine-e2e/dagger-module.toml

@@ -1,13 +1,15 @@
 [["version","2"]]
+["","git-latest",["https://github.com/dagger/sdk-helpers"],"refs/heads/main"]
 ["","git-latest",["https://github.com/dagger/sdk-sdk"],"refs/heads/main"]
 ["","git-sha",["https://github.com/containernetworking/plugins","refs/tags/v1.9.0"],"9b3772e1a7abf93cbb7c6526a28bc0d27b830e02"]
+["","git-sha",["https://github.com/dagger/sdk-helpers","refs/heads/main"],"64645f1967d3dba6fce951dd61ae4acd8d9b0861"]
 ["","git-sha",["https://github.com/dagger/sdk-sdk","refs/heads/main"],"334448911a8292fba0d677e5f31926c79ad80ad3"]
 ["","git-sha",["https://github.com/libfuse/sshfs.git","refs/tags/sshfs-3.7.6"],"7a2d988775446ebe7af9b01c99b3b8e86bddb05a"]
 ["","git-sha",["https://github.com/opencontainers/runc","refs/tags/v1.4.2"],"c241c0bb5e60a8e8c1b2e53d4eca8d0068d8d57e"]
 ["","oci-latest",["docker.io/library/busybox"],"1.38.0"]
 ["","oci-sha",["docker.io/library/alpine:3.22"],"sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce"]
 ["","oci-sha",["docker.io/library/busybox:1.38.0"],"sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616"]
 ["","oci-sha",["docker.io/library/golang:1.25-alpine"],"sha256:1ae0735f00daffa3aaf1363a5184c0d2dc55c78e3db4ec70241cdac97bf84b59"]
-["","oci-sha",["docker.io/library/golang:1.26-alpine"],"sha256:28d89ee9cc0ff9fec75c82ca201e6bf7fdf9a679d4b7b24dfa04f2bb766bb468"]
+["","oci-sha",["docker.io/library/golang:1.26-alpine"],"sha256:ce864e7223ac17b1775e6fd0b4c0db580c2eb50e7953a427916379e4b92a1628"]
 ["","oci-sha",["docker.io/tonistiigi/xx:1.2.1"],"sha256:8879a398dedf0aadaacfbd332b29ff2f84bc39ae6d4e9c0a1109db27ac5ba012"]
-["","oci-sha",["ghcr.io/astral-sh/uv:python3.14-alpine"],"sha256:3f43dc141a3a7e61948cc4c22e0a52666e80ef35ffe6c73e03d60e073b655327"]
+["","oci-sha",["ghcr.io/astral-sh/uv:python3.14-alpine"],"sha256:706cd9faf6d3fa7476fc470463bfca625d4b3ea5e46d6dc166ec1fb8ad3976e6"]