diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index e73648fd36..141aa64e25 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -219,11 +219,6 @@ name = "forward_proxy_jsonrpc_l7" path = "tests/forward_proxy_jsonrpc_l7.rs" required-features = ["e2e-host-gateway"] -[[test]] -name = "workspace_lifecycle" -path = "tests/workspace_lifecycle.rs" -required-features = ["e2e"] - [[test]] name = "supervisor_middleware_http_hooks" path = "tests/supervisor_middleware_http_hooks.rs" diff --git a/e2e/rust/tests/workspace_lifecycle.rs b/e2e/rust/tests/workspace_lifecycle.rs deleted file mode 100644 index 2fe946389e..0000000000 --- a/e2e/rust/tests/workspace_lifecycle.rs +++ /dev/null @@ -1,278 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E tests for workspace resource lifecycle. -//! -//! Covers: -//! - Workspace CRUD -//! - Provider creation scoped to a workspace -//! - Workspace isolation (resources in one workspace are invisible in another) -//! - `--all-workspaces` listing -//! - Deletion guard (workspace cannot be deleted while resources exist) -//! - Successful deletion after resource cleanup - -use std::process::Stdio; - -use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; -use openshell_e2e::harness::output::strip_ansi; - -const WORKSPACE: &str = "lifecycle-test"; -const WORKSPACE_TERM: &str = "terminating-test"; -const PROVIDER: &str = "lifecycle-prov"; -const PROVIDER_TERM: &str = "term-prov"; - -struct CliResult { - output: String, - success: bool, -} - -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - CliResult { - output: strip_ansi(&combined), - success: output.status.success(), - } -} - -struct WorkspaceCleanup; - -impl Drop for WorkspaceCleanup { - fn drop(&mut self) { - let bin = openshell_bin(); - let _ = std::process::Command::new(&bin) - .args(["provider", "delete", PROVIDER, "--workspace", WORKSPACE]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - let _ = std::process::Command::new(&bin) - .args(["workspace", "delete", WORKSPACE]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -#[tokio::test] -async fn workspace_full_crud_lifecycle() { - let _cleanup = WorkspaceCleanup; - - // 1. Create workspace. - let res = run_cli(&["workspace", "create", "--name", WORKSPACE]).await; - assert!(res.success, "workspace create failed: {}", res.output); - - // 2. Verify workspace exists. - let res = run_cli(&["workspace", "get", WORKSPACE]).await; - assert!(res.success, "workspace get failed: {}", res.output); - assert!( - res.output.contains(WORKSPACE), - "workspace get output should contain workspace name: {}", - res.output - ); - - // 3. Create provider in the workspace. - let res = run_cli(&[ - "provider", - "create", - "--name", - PROVIDER, - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=test-value", - "--workspace", - WORKSPACE, - ]) - .await; - assert!( - res.success, - "provider create in workspace failed: {}", - res.output - ); - - // 4. List providers in workspace — should see our provider. - let res = run_cli(&["provider", "list", "--workspace", WORKSPACE]).await; - assert!(res.success, "provider list failed: {}", res.output); - assert!( - res.output.contains(PROVIDER), - "workspace-scoped list should show the provider: {}", - res.output - ); - - // 5. List providers in default workspace — should NOT see our provider. - let res = run_cli(&["provider", "list"]).await; - assert!( - res.success, - "provider list (default) failed: {}", - res.output - ); - assert!( - !res.output.contains(PROVIDER), - "default workspace should not contain the workspace-scoped provider: {}", - res.output - ); - - // 6. List providers with --all-workspaces — should see our provider. - let res = run_cli(&["provider", "list", "--all-workspaces"]).await; - assert!( - res.success, - "provider list --all-workspaces failed: {}", - res.output - ); - assert!( - res.output.contains(PROVIDER), - "--all-workspaces should include the workspace-scoped provider: {}", - res.output - ); - - // 7. Attempt workspace deletion — should fail because provider exists. - let res = run_cli(&["workspace", "delete", WORKSPACE]).await; - assert!( - !res.success, - "workspace delete should fail while resources exist: {}", - res.output - ); - assert!( - res.output.contains("still contains resources"), - "error should mention blocking resources: {}", - res.output - ); - - // 8. Delete the provider. - let res = run_cli(&["provider", "delete", PROVIDER, "--workspace", WORKSPACE]).await; - assert!(res.success, "provider delete failed: {}", res.output); - - // 9. Workspace deletion should now succeed. - let res = run_cli(&["workspace", "delete", WORKSPACE]).await; - assert!( - res.success, - "workspace delete should succeed after resources removed: {}", - res.output - ); - - // 10. Verify workspace is gone. - let res = run_cli(&["workspace", "get", WORKSPACE]).await; - assert!( - !res.success, - "workspace get should fail after deletion: {}", - res.output - ); -} - -struct TerminatingCleanup; - -impl Drop for TerminatingCleanup { - fn drop(&mut self) { - let bin = openshell_bin(); - let _ = std::process::Command::new(&bin) - .args([ - "provider", - "delete", - PROVIDER_TERM, - "--workspace", - WORKSPACE_TERM, - ]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - let _ = std::process::Command::new(&bin) - .args(["workspace", "delete", WORKSPACE_TERM]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -#[tokio::test] -async fn workspace_terminating_rejects_creates() { - let _cleanup = TerminatingCleanup; - - // 1. Create workspace. - let res = run_cli(&["workspace", "create", "--name", WORKSPACE_TERM]).await; - assert!(res.success, "workspace create failed: {}", res.output); - - // 2. Create a provider to block deletion. - let res = run_cli(&[ - "provider", - "create", - "--name", - PROVIDER_TERM, - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=test-value", - "--workspace", - WORKSPACE_TERM, - ]) - .await; - assert!(res.success, "provider create failed: {}", res.output); - - // 3. Attempt deletion — fails, but workspace is now Terminating. - let res = run_cli(&["workspace", "delete", WORKSPACE_TERM]).await; - assert!( - !res.success, - "delete should fail with blocker: {}", - res.output - ); - - // 4. Workspace list should show Terminating status. - let res = run_cli(&["workspace", "list"]).await; - assert!(res.success, "workspace list failed: {}", res.output); - assert!( - res.output.contains("Terminating"), - "workspace list should show Terminating status: {}", - res.output - ); - - // 5. Creating a new provider should be rejected. - let res = run_cli(&[ - "provider", - "create", - "--name", - "should-fail", - "--type", - "openai", - "--credential", - "OPENAI_API_KEY=test-value", - "--workspace", - WORKSPACE_TERM, - ]) - .await; - assert!( - !res.success, - "provider create should fail in terminating workspace: {}", - res.output - ); - assert!( - res.output.contains("being deleted"), - "error should mention workspace is being deleted: {}", - res.output - ); - - // 6. Delete the blocking provider (deletes still work in Terminating). - let res = run_cli(&[ - "provider", - "delete", - PROVIDER_TERM, - "--workspace", - WORKSPACE_TERM, - ]) - .await; - assert!(res.success, "provider delete failed: {}", res.output); - - // 7. Retry deletion — should succeed (idempotent on already-Terminating). - let res = run_cli(&["workspace", "delete", WORKSPACE_TERM]).await; - assert!( - res.success, - "workspace delete should succeed after cleanup: {}", - res.output - ); -} diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index a6edb02a7c..1fc7cd14ac 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -19,6 +19,7 @@ nextest run the tests as the library's unit tests. The crate also builds the | `labels` | Labels are stored and select sandboxes in `sandbox list` | | `templates` | A sandbox created from a template uses its workload; duplicate, deleted and missing templates return the right errors | | `settings` | Sandbox-scoped settings | +| `workspaces` | Workspace create, get, list and delete; a template is scoped to its workspace; a workspace holding a resource goes Terminating, rejects creates and deletes once empty | | `process` | Core dumps are disabled for sandbox processes | | `filesystem-policy` | Landlock: a hard requirement accepts added device paths; a replaced policy path rejects the next launch | diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index cabd7f3a3b..04312ef8b1 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -21,3 +21,4 @@ mod process; mod settings; mod smoke; mod templates; +mod workspaces; diff --git a/tests/suites/conformance/src/workspaces.rs b/tests/suites/conformance/src/workspaces.rs new file mode 100644 index 0000000000..c0f744f52c --- /dev/null +++ b/tests/suites/conformance/src/workspaces.rs @@ -0,0 +1,260 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Workspaces suite: workspace create, inspect, scope and delete through the +//! `openshell` CLI. +//! +//! Ported from the legacy `workspace_lifecycle` e2e test. Workspace behavior is +//! not driver-specific, so it runs against any prepared gateway. +//! +//! The scoped resource is a sandbox template, not a provider: a template is +//! workspace-scoped and covered by the workspace delete guard, and needs no +//! provider profile, credential or image. +//! +//! Known limitation: the CLI has no structured error and no structured +//! `workspace get`, so assertions check exit status and listed state from +//! `--output json`, not message text. This suite is not meaningful where the +//! default workspace is not allowlisted (Kubernetes operator-workspace mode). + +use serde_json::Value; +use testkit::Context; + +const PAGE_SIZE: &str = "1000"; +/// More pages than any test gateway should have; stops a repeated token from +/// looping forever. +const MAX_PAGES: usize = 1000; + +/// `["sandbox", "template", "create", name, "--workspace", workspace]`. +fn template_create<'a>(workspace: &'a str, name: &'a str) -> [&'a str; 6] { + [ + "sandbox", + "template", + "create", + name, + "--workspace", + workspace, + ] +} + +fn template_delete<'a>(workspace: &'a str, name: &'a str) -> [&'a str; 6] { + [ + "sandbox", + "template", + "delete", + name, + "--workspace", + workspace, + ] +} + +/// Create workspace `name`. A cleanup deletes it after the test; it is +/// registered first, so it runs after the cleanups of its templates. A +/// workspace the test already deleted is not an error. +async fn create_workspace(cx: &Context, name: &str) -> testkit::Result { + let cli = cx.cli_handle(); + let owned = name.to_string(); + cx.on_cleanup(format!("delete workspace {name}"), async move || { + let out = cli.run(["workspace", "delete", &owned]).await?; + if out.exit_code != 0 && !is_not_found(&out.report_text()) { + out.success()?; + } + Ok(()) + }); + cx.cli(["workspace", "create", "--name", name]) + .await? + .success()?; + Ok(()) +} + +/// Create template `name` in `workspace`, deleted after the test unless the +/// test deleted it already. +async fn create_template(cx: &Context, workspace: &str, name: &str) -> testkit::Result { + let cli = cx.cli_handle(); + let (ws, owned) = (workspace.to_string(), name.to_string()); + cx.on_cleanup(format!("delete template {name}"), async move || { + let out = cli.run(template_delete(&ws, &owned)).await?; + if out.exit_code != 0 && !is_not_found(&out.report_text()) { + out.success()?; + } + Ok(()) + }); + cx.cli(template_create(workspace, name)).await?.success()?; + Ok(()) +} + +fn is_not_found(report: &str) -> bool { + report.to_ascii_lowercase().contains("not found") +} + +/// The entry called `name` in a paginated `--output json` listing, where +/// `collection` is the JSON key that holds the entries. A listing without that +/// key is an error, so a "not listed" check cannot pass on a malformed listing. +async fn find_entry( + cx: &Context, + list: &[&str], + collection: &str, + name: &str, +) -> testkit::Result> { + let mut token = String::new(); + for _ in 0..MAX_PAGES { + let mut args = list.to_vec(); + args.extend([ + "--page-size", + PAGE_SIZE, + "--page-token", + &token, + "--output", + "json", + ]); + let page: Value = cx.cli(args).await?.success()?.json()?; + let Some(entries) = page[collection].as_array() else { + return Err(testkit::Error::msg(format!( + "`{}` output has no '{collection}' array", + list.join(" ") + ))); + }; + if let Some(entry) = entries.iter().find(|entry| entry["name"] == name) { + return Ok(Some(entry.clone())); + } + token = page["next_page_token"] + .as_str() + .unwrap_or_default() + .to_string(); + if token.is_empty() { + return Ok(None); + } + } + Err(testkit::Error::msg(format!( + "`{}` did not finish within {MAX_PAGES} pages", + list.join(" ") + ))) +} + +/// Require workspace `name`'s listed status to be `expected`, where `None` +/// means it is not listed. +async fn expect_status(cx: &Context, name: &str, expected: Option<&str>) -> testkit::Result { + let entry = find_entry(cx, &["workspace", "list"], "workspaces", name).await?; + let actual = entry.as_ref().and_then(|entry| entry["status"].as_str()); + testkit::ensure!( + actual == expected, + "workspace {name} is {actual:?} in workspace list; expected {expected:?}" + ); + Ok(()) +} + +/// The workspace that owns `template` in the listing selected by `scope` +/// (`--workspace `, `--all-workspaces`, or nothing for the default +/// workspace), or `None` when it is not listed. +async fn template_owner( + cx: &Context, + scope: &[&str], + template: &str, +) -> testkit::Result> { + let mut list = vec!["sandbox", "template", "list"]; + list.extend_from_slice(scope); + let entry = find_entry(cx, &list, "templates", template).await?; + Ok(entry.map(|entry| entry["workspace"].as_str().unwrap_or_default().to_string())) +} + +/// A workspace can be created and is listed Active, cannot be deleted +/// while it holds a resource, can be deleted once empty, and is gone afterwards. +#[testkit::test] +async fn workspace_full_crud_lifecycle(cx: &mut Context) -> testkit::Result { + let workspace = cx.name("wc"); + let template = cx.name("tc"); + + create_workspace(cx, &workspace).await?; + cx.cli(["workspace", "get", &workspace]).await?.success()?; + expect_status(cx, &workspace, Some("Active")).await?; + + create_template(cx, &workspace, &template).await?; + cx.cli(["workspace", "delete", &workspace]) + .await? + .failure()?; + + cx.cli(template_delete(&workspace, &template)) + .await? + .success()?; + cx.cli(["workspace", "delete", &workspace]) + .await? + .success()?; + cx.cli(["workspace", "get", &workspace]).await?.failure()?; + expect_status(cx, &workspace, None).await +} + +/// A template is listed in its own workspace and in the all-workspaces view +/// with its owner, and is not visible in the default workspace. +#[testkit::test] +async fn template_is_scoped_to_its_workspace(cx: &mut Context) -> testkit::Result { + let workspace = cx.name("ws"); + let template = cx.name("ts"); + + create_workspace(cx, &workspace).await?; + create_template(cx, &workspace, &template).await?; + + let scoped = template_owner(cx, &["--workspace", &workspace], &template).await?; + let default = template_owner(cx, &[], &template).await?; + let all = template_owner(cx, &["--all-workspaces"], &template).await?; + testkit::ensure!( + scoped.as_deref() == Some(workspace.as_str()), + "{template} is owned by {scoped:?} in its workspace; expected {workspace}" + ); + testkit::ensure!( + all.as_deref() == Some(workspace.as_str()), + "{template} is owned by {all:?} across workspaces; expected {workspace}" + ); + testkit::ensure!( + default.is_none(), + "{template} is visible in the default workspace as {default:?}" + ); + + cx.cli(template_delete(&workspace, &template)) + .await? + .success()?; + cx.cli(["workspace", "delete", &workspace]) + .await? + .success() + .map(|_| ()) +} + +/// Deleting a workspace that still holds a resource is refused and leaves it +/// Terminating. While Terminating, new resources are rejected and existing +/// ones can still be deleted; retrying the delete then succeeds. +#[testkit::test] +async fn workspace_terminating_rejects_creates(cx: &mut Context) -> testkit::Result { + let workspace = cx.name("wt"); + let template = cx.name("tt"); + let blocked = cx.name("tb"); + + create_workspace(cx, &workspace).await?; + create_template(cx, &workspace, &template).await?; + + // Control: the same template can be created and deleted while the workspace + // is Active, so the later rejection can only be about the Terminating state. + create_template(cx, &workspace, &blocked).await?; + cx.cli(template_delete(&workspace, &blocked)) + .await? + .success()?; + + cx.cli(["workspace", "delete", &workspace]) + .await? + .failure()?; + expect_status(cx, &workspace, Some("Terminating")).await?; + + cx.cli(template_create(&workspace, &blocked)) + .await? + .failure()?; + let leaked = template_owner(cx, &["--workspace", &workspace], &blocked).await?; + testkit::ensure!( + leaked.is_none(), + "rejected template {blocked} exists in workspace {workspace}" + ); + + cx.cli(template_delete(&workspace, &template)) + .await? + .success()?; + cx.cli(["workspace", "delete", &workspace]) + .await? + .success()?; + expect_status(cx, &workspace, None).await +} diff --git a/tests/tmachine/artifacts.nix b/tests/tmachine/artifacts.nix index ef1be7ed98..a945cb61f0 100644 --- a/tests/tmachine/artifacts.nix +++ b/tests/tmachine/artifacts.nix @@ -113,7 +113,6 @@ let "sandbox_lifecycle" # Needs a prebuilt musl DNS probe in guest artifact mode; tracked in #3009. "transparent_tcp" - "workspace_lifecycle" ]; podmanE2eArchiveFilter = let