From d63af4f19e8a14827292aaafbf795aef9f2dbe49 Mon Sep 17 00:00:00 2001 From: politerealism Date: Fri, 13 Nov 2026 16:19:24 +0100 Subject: [PATCH] test(conformance): port workspace lifecycle tests to testkit Move e2e/rust workspace_lifecycle into a conformance workspaces suite on #[testkit::test], split into workspace_full_crud_lifecycle, template_is_scoped_to_its_workspace and workspace_terminating_rejects_creates. Use a sandbox template as the scoped resource so no provider profile is needed, and assert on exit status and listed state rather than message text. Drop the legacy test, its Cargo target, and its Podman archive exclusion. Supersedes #4607. Signed-off-by: Evan Lezar --- e2e/rust/Cargo.toml | 5 - e2e/rust/tests/workspace_lifecycle.rs | 278 --------------------- tests/suites/conformance/README.md | 1 + tests/suites/conformance/src/lib.rs | 1 + tests/suites/conformance/src/workspaces.rs | 260 +++++++++++++++++++ tests/tmachine/artifacts.nix | 1 - 6 files changed, 262 insertions(+), 284 deletions(-) delete mode 100644 e2e/rust/tests/workspace_lifecycle.rs create mode 100644 tests/suites/conformance/src/workspaces.rs diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index e73648fd36..141aa64e25 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -219,11 +219,6 @@ name = "forward_proxy_jsonrpc_l7" path = "tests/forward_proxy_jsonrpc_l7.rs" required-features = ["e2e-host-gateway"] -[[test]] -name = "workspace_lifecycle" -path = "tests/workspace_lifecycle.rs" -required-features = ["e2e"] - [[test]] name = "supervisor_middleware_http_hooks" path = "tests/supervisor_middleware_http_hooks.rs" diff --git a/e2e/rust/tests/workspace_lifecycle.rs b/e2e/rust/tests/workspace_lifecycle.rs deleted file mode 100644 index 2fe946389e..0000000000 --- a/e2e/rust/tests/workspace_lifecycle.rs +++ /dev/null @@ -1,278 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E tests for workspace resource lifecycle. -//! -//! Covers: -//! - Workspace CRUD -//! - Provider creation scoped to a workspace -//! - Workspace isolation (resources in one workspace are invisible in another) -//! - `--all-workspaces` listing -//! - Deletion guard (workspace cannot be deleted while resources exist) -//! - Successful deletion after resource cleanup - -use std::process::Stdio; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::output::strip_ansi; - -const WORKSPACE: &str = "lifecycle-test"; -const WORKSPACE_TERM: &str = "terminating-test"; -const PROVIDER: &str = "lifecycle-prov"; -const PROVIDER_TERM: &str = "term-prov"; - -struct CliResult { - output: String, - success: bool, -} - -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - CliResult { - output: strip_ansi(&combined), - success: output.status.success(), - } -} - -struct WorkspaceCleanup; - -impl Drop for WorkspaceCleanup { - fn drop(&mut self) { - let bin = openshell_bin(); - let _ = std::process::Command::new(&bin) - .args(["provider", "delete", PROVIDER, "--workspace", WORKSPACE]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - let _ = std::process::Command::new(&bin) - .args(["workspace", "delete", WORKSPACE]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -#[tokio::test] -async fn workspace_full_crud_lifecycle() { - let _cleanup = WorkspaceCleanup; - - // 1. Create workspace. - let res = run_cli(&["workspace", "create", "--name", WORKSPACE]).await; - assert!(res.success, "workspace create failed: {}", res.output); - - // 2. Verify workspace exists. - let res = run_cli(&["workspace", "get", WORKSPACE]).await; - assert!(res.success, "workspace get failed: {}", res.output); - assert!( - res.output.contains(WORKSPACE), - "workspace get output should contain workspace name: {}", - res.output - ); - - // 3. Create provider in the workspace. - let res = run_cli(&[ - "provider", - "create", - "--name", - PROVIDER, - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=test-value", - "--workspace", - WORKSPACE, - ]) - .await; - assert!( - res.success, - "provider create in workspace failed: {}", - res.output - ); - - // 4. List providers in workspace — should see our provider. - let res = run_cli(&["provider", "list", "--workspace", WORKSPACE]).await; - assert!(res.success, "provider list failed: {}", res.output); - assert!( - res.output.contains(PROVIDER), - "workspace-scoped list should show the provider: {}", - res.output - ); - - // 5. List providers in default workspace — should NOT see our provider. - let res = run_cli(&["provider", "list"]).await; - assert!( - res.success, - "provider list (default) failed: {}", - res.output - ); - assert!( - !res.output.contains(PROVIDER), - "default workspace should not contain the workspace-scoped provider: {}", - res.output - ); - - // 6. List providers with --all-workspaces — should see our provider. - let res = run_cli(&["provider", "list", "--all-workspaces"]).await; - assert!( - res.success, - "provider list --all-workspaces failed: {}", - res.output - ); - assert!( - res.output.contains(PROVIDER), - "--all-workspaces should include the workspace-scoped provider: {}", - res.output - ); - - // 7. Attempt workspace deletion — should fail because provider exists. - let res = run_cli(&["workspace", "delete", WORKSPACE]).await; - assert!( - !res.success, - "workspace delete should fail while resources exist: {}", - res.output - ); - assert!( - res.output.contains("still contains resources"), - "error should mention blocking resources: {}", - res.output - ); - - // 8. Delete the provider. - let res = run_cli(&["provider", "delete", PROVIDER, "--workspace", WORKSPACE]).await; - assert!(res.success, "provider delete failed: {}", res.output); - - // 9. Workspace deletion should now succeed. - let res = run_cli(&["workspace", "delete", WORKSPACE]).await; - assert!( - res.success, - "workspace delete should succeed after resources removed: {}", - res.output - ); - - // 10. Verify workspace is gone. - let res = run_cli(&["workspace", "get", WORKSPACE]).await; - assert!( - !res.success, - "workspace get should fail after deletion: {}", - res.output - ); -} - -struct TerminatingCleanup; - -impl Drop for TerminatingCleanup { - fn drop(&mut self) { - let bin = openshell_bin(); - let _ = std::process::Command::new(&bin) - .args([ - "provider", - "delete", - PROVIDER_TERM, - "--workspace", - WORKSPACE_TERM, - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - let _ = std::process::Command::new(&bin) - .args(["workspace", "delete", WORKSPACE_TERM]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -#[tokio::test] -async fn workspace_terminating_rejects_creates() { - let _cleanup = TerminatingCleanup; - - // 1. Create workspace. - let res = run_cli(&["workspace", "create", "--name", WORKSPACE_TERM]).await; - assert!(res.success, "workspace create failed: {}", res.output); - - // 2. Create a provider to block deletion. - let res = run_cli(&[ - "provider", - "create", - "--name", - PROVIDER_TERM, - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=test-value", - "--workspace", - WORKSPACE_TERM, - ]) - .await; - assert!(res.success, "provider create failed: {}", res.output); - - // 3. Attempt deletion — fails, but workspace is now Terminating. - let res = run_cli(&["workspace", "delete", WORKSPACE_TERM]).await; - assert!( - !res.success, - "delete should fail with blocker: {}", - res.output - ); - - // 4. Workspace list should show Terminating status. - let res = run_cli(&["workspace", "list"]).await; - assert!(res.success, "workspace list failed: {}", res.output); - assert!( - res.output.contains("Terminating"), - "workspace list should show Terminating status: {}", - res.output - ); - - // 5. Creating a new provider should be rejected. - let res = run_cli(&[ - "provider", - "create", - "--name", - "should-fail", - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=test-value", - "--workspace", - WORKSPACE_TERM, - ]) - .await; - assert!( - !res.success, - "provider create should fail in terminating workspace: {}", - res.output - ); - assert!( - res.output.contains("being deleted"), - "error should mention workspace is being deleted: {}", - res.output - ); - - // 6. Delete the blocking provider (deletes still work in Terminating). - let res = run_cli(&[ - "provider", - "delete", - PROVIDER_TERM, - "--workspace", - WORKSPACE_TERM, - ]) - .await; - assert!(res.success, "provider delete failed: {}", res.output); - - // 7. Retry deletion — should succeed (idempotent on already-Terminating). - let res = run_cli(&["workspace", "delete", WORKSPACE_TERM]).await; - assert!( - res.success, - "workspace delete should succeed after cleanup: {}", - res.output - ); -} diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index a6edb02a7c..1fc7cd14ac 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -19,6 +19,7 @@ nextest run the tests as the library's unit tests. The crate also builds the | `labels` | Labels are stored and select sandboxes in `sandbox list` | | `templates` | A sandbox created from a template uses its workload; duplicate, deleted and missing templates return the right errors | | `settings` | Sandbox-scoped settings | +| `workspaces` | Workspace create, get, list and delete; a template is scoped to its workspace; a workspace holding a resource goes Terminating, rejects creates and deletes once empty | | `process` | Core dumps are disabled for sandbox processes | | `filesystem-policy` | Landlock: a hard requirement accepts added device paths; a replaced policy path rejects the next launch | diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index cabd7f3a3b..04312ef8b1 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -21,3 +21,4 @@ mod process; mod settings; mod smoke; mod templates; +mod workspaces; diff --git a/tests/suites/conformance/src/workspaces.rs b/tests/suites/conformance/src/workspaces.rs new file mode 100644 index 0000000000..c0f744f52c --- /dev/null +++ b/tests/suites/conformance/src/workspaces.rs @@ -0,0 +1,260 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Workspaces suite: workspace create, inspect, scope and delete through the +//! `openshell` CLI. +//! +//! Ported from the legacy `workspace_lifecycle` e2e test. Workspace behavior is +//! not driver-specific, so it runs against any prepared gateway. +//! +//! The scoped resource is a sandbox template, not a provider: a template is +//! workspace-scoped and covered by the workspace delete guard, and needs no +//! provider profile, credential or image. +//! +//! Known limitation: the CLI has no structured error and no structured +//! `workspace get`, so assertions check exit status and listed state from +//! `--output json`, not message text. This suite is not meaningful where the +//! default workspace is not allowlisted (Kubernetes operator-workspace mode). + +use serde_json::Value; +use testkit::Context; + +const PAGE_SIZE: &str = "1000"; +/// More pages than any test gateway should have; stops a repeated token from +/// looping forever. +const MAX_PAGES: usize = 1000; + +/// `["sandbox", "template", "create", name, "--workspace", workspace]`. +fn template_create<'a>(workspace: &'a str, name: &'a str) -> [&'a str; 6] { + [ + "sandbox", + "template", + "create", + name, + "--workspace", + workspace, + ] +} + +fn template_delete<'a>(workspace: &'a str, name: &'a str) -> [&'a str; 6] { + [ + "sandbox", + "template", + "delete", + name, + "--workspace", + workspace, + ] +} + +/// Create workspace `name`. A cleanup deletes it after the test; it is +/// registered first, so it runs after the cleanups of its templates. A +/// workspace the test already deleted is not an error. +async fn create_workspace(cx: &Context, name: &str) -> testkit::Result { + let cli = cx.cli_handle(); + let owned = name.to_string(); + cx.on_cleanup(format!("delete workspace {name}"), async move || { + let out = cli.run(["workspace", "delete", &owned]).await?; + if out.exit_code != 0 && !is_not_found(&out.report_text()) { + out.success()?; + } + Ok(()) + }); + cx.cli(["workspace", "create", "--name", name]) + .await? + .success()?; + Ok(()) +} + +/// Create template `name` in `workspace`, deleted after the test unless the +/// test deleted it already. +async fn create_template(cx: &Context, workspace: &str, name: &str) -> testkit::Result { + let cli = cx.cli_handle(); + let (ws, owned) = (workspace.to_string(), name.to_string()); + cx.on_cleanup(format!("delete template {name}"), async move || { + let out = cli.run(template_delete(&ws, &owned)).await?; + if out.exit_code != 0 && !is_not_found(&out.report_text()) { + out.success()?; + } + Ok(()) + }); + cx.cli(template_create(workspace, name)).await?.success()?; + Ok(()) +} + +fn is_not_found(report: &str) -> bool { + report.to_ascii_lowercase().contains("not found") +} + +/// The entry called `name` in a paginated `--output json` listing, where +/// `collection` is the JSON key that holds the entries. A listing without that +/// key is an error, so a "not listed" check cannot pass on a malformed listing. +async fn find_entry( + cx: &Context, + list: &[&str], + collection: &str, + name: &str, +) -> testkit::Result> { + let mut token = String::new(); + for _ in 0..MAX_PAGES { + let mut args = list.to_vec(); + args.extend([ + "--page-size", + PAGE_SIZE, + "--page-token", + &token, + "--output", + "json", + ]); + let page: Value = cx.cli(args).await?.success()?.json()?; + let Some(entries) = page[collection].as_array() else { + return Err(testkit::Error::msg(format!( + "`{}` output has no '{collection}' array", + list.join(" ") + ))); + }; + if let Some(entry) = entries.iter().find(|entry| entry["name"] == name) { + return Ok(Some(entry.clone())); + } + token = page["next_page_token"] + .as_str() + .unwrap_or_default() + .to_string(); + if token.is_empty() { + return Ok(None); + } + } + Err(testkit::Error::msg(format!( + "`{}` did not finish within {MAX_PAGES} pages", + list.join(" ") + ))) +} + +/// Require workspace `name`'s listed status to be `expected`, where `None` +/// means it is not listed. +async fn expect_status(cx: &Context, name: &str, expected: Option<&str>) -> testkit::Result { + let entry = find_entry(cx, &["workspace", "list"], "workspaces", name).await?; + let actual = entry.as_ref().and_then(|entry| entry["status"].as_str()); + testkit::ensure!( + actual == expected, + "workspace {name} is {actual:?} in workspace list; expected {expected:?}" + ); + Ok(()) +} + +/// The workspace that owns `template` in the listing selected by `scope` +/// (`--workspace `, `--all-workspaces`, or nothing for the default +/// workspace), or `None` when it is not listed. +async fn template_owner( + cx: &Context, + scope: &[&str], + template: &str, +) -> testkit::Result> { + let mut list = vec!["sandbox", "template", "list"]; + list.extend_from_slice(scope); + let entry = find_entry(cx, &list, "templates", template).await?; + Ok(entry.map(|entry| entry["workspace"].as_str().unwrap_or_default().to_string())) +} + +/// A workspace can be created and is listed Active, cannot be deleted +/// while it holds a resource, can be deleted once empty, and is gone afterwards. +#[testkit::test] +async fn workspace_full_crud_lifecycle(cx: &mut Context) -> testkit::Result { + let workspace = cx.name("wc"); + let template = cx.name("tc"); + + create_workspace(cx, &workspace).await?; + cx.cli(["workspace", "get", &workspace]).await?.success()?; + expect_status(cx, &workspace, Some("Active")).await?; + + create_template(cx, &workspace, &template).await?; + cx.cli(["workspace", "delete", &workspace]) + .await? + .failure()?; + + cx.cli(template_delete(&workspace, &template)) + .await? + .success()?; + cx.cli(["workspace", "delete", &workspace]) + .await? + .success()?; + cx.cli(["workspace", "get", &workspace]).await?.failure()?; + expect_status(cx, &workspace, None).await +} + +/// A template is listed in its own workspace and in the all-workspaces view +/// with its owner, and is not visible in the default workspace. +#[testkit::test] +async fn template_is_scoped_to_its_workspace(cx: &mut Context) -> testkit::Result { + let workspace = cx.name("ws"); + let template = cx.name("ts"); + + create_workspace(cx, &workspace).await?; + create_template(cx, &workspace, &template).await?; + + let scoped = template_owner(cx, &["--workspace", &workspace], &template).await?; + let default = template_owner(cx, &[], &template).await?; + let all = template_owner(cx, &["--all-workspaces"], &template).await?; + testkit::ensure!( + scoped.as_deref() == Some(workspace.as_str()), + "{template} is owned by {scoped:?} in its workspace; expected {workspace}" + ); + testkit::ensure!( + all.as_deref() == Some(workspace.as_str()), + "{template} is owned by {all:?} across workspaces; expected {workspace}" + ); + testkit::ensure!( + default.is_none(), + "{template} is visible in the default workspace as {default:?}" + ); + + cx.cli(template_delete(&workspace, &template)) + .await? + .success()?; + cx.cli(["workspace", "delete", &workspace]) + .await? + .success() + .map(|_| ()) +} + +/// Deleting a workspace that still holds a resource is refused and leaves it +/// Terminating. While Terminating, new resources are rejected and existing +/// ones can still be deleted; retrying the delete then succeeds. +#[testkit::test] +async fn workspace_terminating_rejects_creates(cx: &mut Context) -> testkit::Result { + let workspace = cx.name("wt"); + let template = cx.name("tt"); + let blocked = cx.name("tb"); + + create_workspace(cx, &workspace).await?; + create_template(cx, &workspace, &template).await?; + + // Control: the same template can be created and deleted while the workspace + // is Active, so the later rejection can only be about the Terminating state. + create_template(cx, &workspace, &blocked).await?; + cx.cli(template_delete(&workspace, &blocked)) + .await? + .success()?; + + cx.cli(["workspace", "delete", &workspace]) + .await? + .failure()?; + expect_status(cx, &workspace, Some("Terminating")).await?; + + cx.cli(template_create(&workspace, &blocked)) + .await? + .failure()?; + let leaked = template_owner(cx, &["--workspace", &workspace], &blocked).await?; + testkit::ensure!( + leaked.is_none(), + "rejected template {blocked} exists in workspace {workspace}" + ); + + cx.cli(template_delete(&workspace, &template)) + .await? + .success()?; + cx.cli(["workspace", "delete", &workspace]) + .await? + .success()?; + expect_status(cx, &workspace, None).await +} diff --git a/tests/tmachine/artifacts.nix b/tests/tmachine/artifacts.nix index ef1be7ed98..a945cb61f0 100644 --- a/tests/tmachine/artifacts.nix +++ b/tests/tmachine/artifacts.nix @@ -113,7 +113,6 @@ let "sandbox_lifecycle" # Needs a prebuilt musl DNS probe in guest artifact mode; tracked in #3009. "transparent_tcp" - "workspace_lifecycle" ]; podmanE2eArchiveFilter = let