feat(vm): cocoon vm net --nics N — runtime NIC resize

.gitignore

@@ -51,3 +51,6 @@ tmp/*
 
 .claude/*
 cocoon-test
+
+# Local scratch / test scripts (not for upstream)
+/scratch/

KNOWN_ISSUES.md

@@ -242,6 +242,23 @@ Mitigations for production users:
 
 Control-plane traffic from cocoon-managed hosts (vk-cocoon, `cocoon vm exec`) goes through cocoon-agent over vsock and never depends on SSH credentials.
 
+## NIC hot-remove leaves the PCI slot pending on Cloud Hypervisor
+
+`cocoon vm net --nics N` tears down host TAP/veth/CNI plumbing immediately after `vm.remove-device` returns. CH only frees the PCI slot (and unregisters the device's ioeventfds) when the guest writes to the ACPI hot-plug controller (B0EJ) in response to the SCI raised by `remove-device`. If the guest is stopped, paused, or its NIC driver is wedged, the slot stays `Allocated` until the guest cooperates.
+
+Cocoon does not wait for B0EJ — there is no reliable signal from the CH HTTP API. The user accepts:
+
+- The guest may continue to reference a NIC whose host plumbing is gone, hanging the in-guest driver.
+- The pending eject may surface later as `Cannot register ioevent: File exists` when CH next tries to reuse that slot (e.g. a subsequent hot-add).
+
+Quiesce the guest NIC (ip link set down + NetworkManager remove on Linux; Disable-PnpDevice + driver unbind on Windows) before reducing the count.
+
+Firecracker is not supported: FC has no NIC hot-plug / hot-unplug API.
+
+## NIC hot-remove during clone hot-swap cannot wait for guest B0EJ
+
+Plain `cocoon vm net --nics N` polls CH's `device_tree` after `vm.remove-device` until the guest ACKs B0EJ via ACPI/SCI (typically < 1 s on Linux, a few seconds on Windows). The clone path's `hotSwapNets` runs while the VM is paused, so the guest cannot process SCI — eject stays pending until resume. The clone path therefore returns without waiting and adds the fresh NICs against half-removed device-tree state. This is the long-standing CH limitation cocoon was designed around; the new wait only applies to the running-VM resize.
+
 ## Android cocoon-agent service may be blocked by SELinux
 
 `os-image/android/{14.0,15.0}` install the cocoon-agent binary at `/system/bin/cocoon-agent` and register it via `/system/etc/init/cocoon-agent.rc`. Android's SELinux policies don't ship with a domain for cocoon-agent, so the service may run in `init`'s domain or be denied outright depending on the redroid build.

README.md

@@ -149,6 +149,7 @@ cocoon
 │   ├── device
 │   │   ├── attach [flags] VM     Attach a VFIO PCI device (CH only)
 │   │   └── detach [flags] VM     Detach a VFIO PCI device by --id
+│   ├── net [flags] VM             Resize NIC count on a running VM (CH only)
 │   └── debug [flags] IMAGE        Generate hypervisor launch command (dry run)
 ├── snapshot
 │   ├── save [flags] VM            Create a snapshot from a running VM
@@ -520,6 +521,22 @@ cocoon vm device detach my-vm --id mygpu
 
 `cocoon vm inspect VM` includes an `attached_devices` field for running VMs that surfaces every attached vhost-user-fs share and VFIO device, read live from CH `vm.info`. The field is omitted for stopped VMs.
 
+## NIC Hot-Resize (Cloud Hypervisor only)
+
+`cocoon vm net --nics N VM` brings the running VM's NIC count to `N`. To add NICs, cocoon allocates new host TAP/CNI/bridge plumbing and hot-plugs a fresh NIC into the guest. To remove NICs, it pops from the tail (LIFO) via `vm.remove-device` and tears down the host plumbing.
+
+```bash
+# Add a second NIC (or any number).
+cocoon vm net my-vm --nics 2
+
+# Remove a NIC (LIFO from the tail).
+cocoon vm net my-vm --nics 1
+```
+
+Cocoon manages **host-side** plumbing only. CH's `vm.remove-device` marks the slot for ejection but the actual eject only happens when the guest cooperates via ACPI (B0EJ write). The host TAP / veth / CNI lease are torn down immediately after the API call regardless. Quiesce in-guest NIC state (driver unbind, NetworkManager removal, Windows NDIS halt) **before** reducing the count, or the in-guest driver will reference plumbing that no longer exists.
+
+A VM started with zero NICs cannot be resized up (the VM record carries no provider hint). Start with at least one NIC if you plan to resize.
+
 ## Windows Support
 
 Cocoon supports Windows guests via the `--windows` flag:

cmd/vm/attach.go

@@ -9,13 +9,14 @@ import (
 	"github.com/spf13/cobra"
 
 	cmdcore "github.com/cocoonstack/cocoon/cmd/core"
+	"github.com/cocoonstack/cocoon/config"
 	"github.com/cocoonstack/cocoon/extend/fs"
 	"github.com/cocoonstack/cocoon/extend/vfio"
 	"github.com/cocoonstack/cocoon/hypervisor"
 )
 
 func (h Handler) FsAttach(cmd *cobra.Command, args []string) error {
-	ctx, a, err := resolveAttacher[fs.Attacher](h, cmd, args, "fs attach", fs.ErrUnsupportedBackend)
+	ctx, _, _, a, err := resolveAttacher[fs.Attacher](h, cmd, args, "fs attach", fs.ErrUnsupportedBackend)
 	if err != nil {
 		return err
 	}
@@ -35,7 +36,7 @@ func (h Handler) FsAttach(cmd *cobra.Command, args []string) error {
 }
 
 func (h Handler) FsDetach(cmd *cobra.Command, args []string) error {
-	ctx, a, err := resolveAttacher[fs.Attacher](h, cmd, args, "fs detach", fs.ErrUnsupportedBackend)
+	ctx, _, _, a, err := resolveAttacher[fs.Attacher](h, cmd, args, "fs detach", fs.ErrUnsupportedBackend)
 	if err != nil {
 		return err
 	}
@@ -51,7 +52,7 @@ func (h Handler) FsDetach(cmd *cobra.Command, args []string) error {
 }
 
 func (h Handler) DeviceAttach(cmd *cobra.Command, args []string) error {
-	ctx, a, err := resolveAttacher[vfio.Attacher](h, cmd, args, "device attach", vfio.ErrUnsupportedBackend)
+	ctx, _, _, a, err := resolveAttacher[vfio.Attacher](h, cmd, args, "device attach", vfio.ErrUnsupportedBackend)
 	if err != nil {
 		return err
 	}
@@ -69,7 +70,7 @@ func (h Handler) DeviceAttach(cmd *cobra.Command, args []string) error {
 }
 
 func (h Handler) DeviceDetach(cmd *cobra.Command, args []string) error {
-	ctx, a, err := resolveAttacher[vfio.Attacher](h, cmd, args, "device detach", vfio.ErrUnsupportedBackend)
+	ctx, _, _, a, err := resolveAttacher[vfio.Attacher](h, cmd, args, "device detach", vfio.ErrUnsupportedBackend)
 	if err != nil {
 		return err
 	}
@@ -84,24 +85,22 @@ func (h Handler) DeviceDetach(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-// resolveAttacher resolves args[0] to a hypervisor implementing A
-// (fs.Attacher / vfio.Attacher). op ("fs attach", "device detach") prefixes
-// both error wraps so callers see the operation, not just the type-assert.
-func resolveAttacher[A any](h Handler, cmd *cobra.Command, args []string, op string, errUnsupported error) (context.Context, A, error) {
+// resolveAttacher resolves args[0] to a hypervisor implementing A; also returns conf+hyper for further ops.
+func resolveAttacher[A any](h Handler, cmd *cobra.Command, args []string, op string, errUnsupported error) (context.Context, *config.Config, hypervisor.Hypervisor, A, error) {
 	var zero A
 	ctx, conf, err := h.Init(cmd)
 	if err != nil {
-		return ctx, zero, err
+		return ctx, nil, nil, zero, err
 	}
 	hyper, err := cmdcore.FindHypervisor(ctx, conf, args[0])
 	if err != nil {
-		return ctx, zero, fmt.Errorf("%s: %w", op, err)
+		return ctx, conf, nil, zero, fmt.Errorf("%s: %w", op, err)
 	}
 	a, ok := hyper.(A)
 	if !ok {
-		return ctx, zero, fmt.Errorf("%s: backend %s: %w", op, hyper.Type(), errUnsupported)
+		return ctx, conf, hyper, zero, fmt.Errorf("%s: backend %s: %w", op, hyper.Type(), errUnsupported)
 	}
-	return ctx, a, nil
+	return ctx, conf, hyper, a, nil
 }
 
 // classifyAttachErr surfaces ErrNotRunning more clearly than the generic wrap.

cmd/vm/commands.go

@@ -28,6 +28,7 @@ type Actions interface {
 	FsDetach(cmd *cobra.Command, args []string) error
 	DeviceAttach(cmd *cobra.Command, args []string) error
 	DeviceDetach(cmd *cobra.Command, args []string) error
+	NetResize(cmd *cobra.Command, args []string) error
 }
 
 // Command builds the "vm" parent command with all subcommands.
@@ -188,10 +189,24 @@ func Command(h Actions) *cobra.Command {
 		statusCmd,
 		buildFsCommand(h),
 		buildDeviceCommand(h),
+		buildNetCommand(h),
 	)
 	return vmCmd
 }
 
+func buildNetCommand(h Actions) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "net VM",
+		Short: "Resize a running VM's NIC count (CH only); quiesce in-guest NIC state before reducing",
+		Args:  cobra.ExactArgs(1),
+		RunE:  h.NetResize,
+	}
+	cmd.Flags().Int("nics", -1, "target NIC count (required, >= 0)")
+	_ = cmd.MarkFlagRequired("nics")
+	cmdcore.AddOutputFlag(cmd)
+	return cmd
+}
+
 func buildFsCommand(h Actions) *cobra.Command {
 	parent := &cobra.Command{
 		Use:   "fs",

cmd/vm/lifecycle.go

@@ -386,20 +386,20 @@ func (h Handler) recoverNetwork(ctx context.Context, conf *config.Config, hyper
 			continue
 		}
 		logger.Warnf(ctx, "network missing for VM %s, recovering", vm.ID)
-		if _, recoverErr := netProvider.Config(ctx, vm.ID, len(vm.NetworkConfigs), &vm.Config, vm.NetworkConfigs...); recoverErr != nil {
+		if _, recoverErr := netProvider.Add(ctx, vm.ID, &vm.Config, network.AddRecover(vm.NetworkConfigs)...); recoverErr != nil {
 			logger.Warnf(ctx, "recover network for VM %s: %v (start will fail)", vm.ID, recoverErr)
 		}
 	}
 }
 
-// providerForVM selects network provider from persisted NetworkConfig.
+// providerForVM picks the network provider from persisted NetworkConfig; cniProvider may be nil (lazy-init), bridgeCache must be non-nil.
 func providerForVM(conf *config.Config, cniProvider network.Network, bridgeCache map[string]network.Network, configs []*types.NetworkConfig) (network.Network, error) {
 	if len(configs) == 0 {
 		return nil, fmt.Errorf("no network configs")
 	}
 	// All NICs on a VM share the same backend.
 	cfg := configs[0]
-	if cfg.Backend == "bridge" {
+	if cfg.Backend == types.BackendBridge {
 		if cfg.BridgeDev == "" {
 			return nil, fmt.Errorf("bridge backend but no bridge device persisted")
 		}
@@ -414,10 +414,10 @@ func providerForVM(conf *config.Config, cniProvider network.Network, bridgeCache
 		return p, nil
 	}
 	// "cni" or empty (backward compat).
-	if cniProvider == nil {
-		return nil, fmt.Errorf("cni provider not available")
+	if cniProvider != nil {
+		return cniProvider, nil
 	}
-	return cniProvider, nil
+	return cmdcore.InitNetwork(conf)
 }
 
 func batchRoutedCmd(ctx context.Context, cmd *cobra.Command, name, pastTense string, routed map[hypervisor.Hypervisor][]string, fn func(hypervisor.Hypervisor, []string) ([]string, error)) error {

cmd/vm/netresize.go

@@ -0,0 +1,52 @@
+package vm
+
+import (
+	"fmt"
+
+	"github.com/projecteru2/core/log"
+	"github.com/spf13/cobra"
+
+	cmdcore "github.com/cocoonstack/cocoon/cmd/core"
+	"github.com/cocoonstack/cocoon/config"
+	"github.com/cocoonstack/cocoon/extend/netresize"
+	"github.com/cocoonstack/cocoon/network"
+	"github.com/cocoonstack/cocoon/types"
+)
+
+func (h Handler) NetResize(cmd *cobra.Command, args []string) error {
+	ctx, conf, hyper, resizer, err := resolveAttacher[netresize.Resizer](h, cmd, args, "vm net", netresize.ErrUnsupportedBackend)
+	if err != nil {
+		return err
+	}
+	vm, err := hyper.Inspect(ctx, args[0])
+	if err != nil {
+		return fmt.Errorf("vm net: %w", err)
+	}
+	plumbing, err := plumbingForVM(conf, vm.NetworkConfigs)
+	if err != nil {
+		return fmt.Errorf("vm net: %w", err)
+	}
+	target, _ := cmd.Flags().GetInt("nics")
+	res, err := resizer.NetResize(ctx, args[0], netresize.Spec{Target: target}, plumbing)
+	if err != nil {
+		return classifyAttachErr(err)
+	}
+	if done, jsonErr := cmdcore.MaybeOutputJSON(cmd, res); done {
+		return jsonErr
+	}
+	logger := log.WithFunc("cmd.vm.net")
+	logger.Infof(ctx, "resized %s: before=%d after=%d added=%d removed=%d",
+		args[0], res.Before, res.After, len(res.Added), len(res.Removed))
+	for _, w := range res.Warnings {
+		logger.Warnf(ctx, "%s: %s", args[0], w)
+	}
+	return nil
+}
+
+// plumbingForVM picks the provider for the VM's existing NICs; fails on zero NICs (VMConfig has no bridge hint).
+func plumbingForVM(conf *config.Config, configs []*types.NetworkConfig) (network.Network, error) {
+	if len(configs) == 0 {
+		return nil, fmt.Errorf("vm has zero NICs; resize up is not supported (start the VM with at least one NIC)")
+	}
+	return providerForVM(conf, nil, map[string]network.Network{}, configs)
+}

cmd/vm/run.go

@@ -480,7 +480,7 @@ func initNetwork(ctx context.Context, conf *config.Config, vmID string, nics int
 	// The network layer derives TAP queues from vmCfg.CPU.
 	origCPU := vmCfg.CPU
 	vmCfg.CPU = queues
-	configs, err := netProvider.Config(ctx, vmID, nics, vmCfg)
+	configs, err := netProvider.Add(ctx, vmID, vmCfg, network.AddRange(0, nics)...)
 	vmCfg.CPU = origCPU
 	if err != nil {
 		return nil, nil, fmt.Errorf("configure network: %w", err)

extend/netresize/netresize.go

@@ -0,0 +1,54 @@
+// Package netresize is the runtime interface for resizing a VM's NIC count.
+package netresize
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/cocoonstack/cocoon/network"
+	"github.com/cocoonstack/cocoon/types"
+)
+
+// ErrUnsupportedBackend signals the resolved hypervisor cannot resize NICs.
+var ErrUnsupportedBackend = errors.New("backend does not support net resize")
+
+// Spec is one resize request.
+type Spec struct {
+	Target int
+}
+
+// NIC is one NIC summary surfaced through Result.Added / Result.Removed.
+type NIC struct {
+	Index int    `json:"index"`
+	TAP   string `json:"tap"`
+	MAC   string `json:"mac"`
+}
+
+// Result reports the before/after count, NICs touched, and non-fatal Warnings.
+type Result struct {
+	Before   int      `json:"before"`
+	After    int      `json:"after"`
+	Added    []NIC    `json:"added,omitempty"`
+	Removed  []NIC    `json:"removed,omitempty"`
+	Warnings []string `json:"warnings,omitempty"`
+}
+
+// Plumbing is the host-side network ops NetResize delegates to; network.Network satisfies it.
+type Plumbing interface {
+	Add(ctx context.Context, vmID string, vmCfg *types.VMConfig, specs ...network.AddSpec) ([]*types.NetworkConfig, error)
+	Remove(ctx context.Context, vmID string, indices ...int) error
+}
+
+// Resizer resizes a running VM's NIC count.
+type Resizer interface {
+	NetResize(ctx context.Context, vmRef string, spec Spec, plumbing Plumbing) (Result, error)
+}
+
+// Normalize validates the spec.
+func (s *Spec) Normalize() error {
+	if s.Target < 0 {
+		return fmt.Errorf("--nics must be non-negative, got %d", s.Target)
+	}
+	return nil
+}

hypervisor/cloudhypervisor/api.go

@@ -1,5 +1,7 @@
 package cloudhypervisor
 
+import "encoding/json"
+
 type chVMConfig struct {
 	// Optional — pointer + omitempty (nil → omitted from JSON).
 	Payload *chPayload     `json:"payload,omitempty"`
@@ -110,7 +112,8 @@ type chPciDeviceInfo struct {
 }
 
 type chVMInfoResponse struct {
-	Config chVMInfoConfig `json:"config"`
+	Config     chVMInfoConfig             `json:"config"`
+	DeviceTree map[string]json.RawMessage `json:"device_tree,omitempty"`
 }
 
 type chVMInfoConfig struct {
@@ -119,4 +122,5 @@ type chVMInfoConfig struct {
 	Memory  chMemory      `json:"memory"`
 	Fs      []chFs        `json:"fs,omitempty"`
 	Devices []chDevice    `json:"devices,omitempty"`
+	Nets    []chNet       `json:"net,omitempty"`
 }

hypervisor/cloudhypervisor/args.go

@@ -17,6 +17,8 @@ const (
 	// defaultDiskQueueSize is the virtio-blk queue depth per device.
 	defaultDiskQueueSize = 512
 	cidataFile           = "cidata.img"
+
+	cocoonNetIDPrefix = "cocoon-net-"
 )
 
 // kvBuilder accumulates key=value CLI fragments.
@@ -179,6 +181,11 @@ func networkConfigToNet(nc *types.NetworkConfig) chNet {
 	}
 }
 
+// cocoonNetID is the deterministic CH device id for a cocoon-managed NIC.
+func cocoonNetID(mac string) string {
+	return cocoonNetIDPrefix + strings.ReplaceAll(mac, ":", "")
+}
+
 func storageConfigToDisk(storageConfig *types.StorageConfig, cpuCount, diskQueueSize int, noDirectIO bool) chDisk {
 	if diskQueueSize <= 0 {
 		diskQueueSize = defaultDiskQueueSize

hypervisor/cloudhypervisor/clone.go

@@ -320,8 +320,7 @@ func hotSwapNets(ctx context.Context, hc *http.Client, oldNets []chNet, networkC
 		logger.Infof(ctx, "removed snapshot NIC %s (old MAC %s)", oldNet.ID, oldNet.MAC)
 	}
 	for i, nc := range networkConfigs {
-		newNet := networkConfigToNet(nc)
-		if err := addNetVM(ctx, hc, newNet); err != nil {
+		if _, err := addCocoonNIC(ctx, hc, nc); err != nil {
 			return fmt.Errorf("add net device %d/%d (MAC %s, TAP %s): %w",
 				i+1, len(networkConfigs), nc.MAC, nc.TAP, err)
 		}