Fix logic errors that were preventing plugin dir configuration

scripts/build_antithesis_images.sh

@@ -85,5 +85,5 @@ else
                                 "${AVALANCHE_PATH}/build/antithesis/xsvm" \
                                 "AVALANCHEGO_PATH=${AVALANCHE_PATH}/build/avalanchego AVALANCHEGO_PLUGIN_DIR=${HOME}/.avalanchego/plugins"
 
-  build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/vms/example/xsvm/Dockerfile"
+  build_antithesis_images_for_avalanchego "${TEST_SETUP}" "${IMAGE_PREFIX}" "${AVALANCHE_PATH}/tests/antithesis/xsvm/Dockerfile.node"
 fi

tests/antithesis/xsvm/Dockerfile.node

@@ -24,8 +24,6 @@ FROM $AVALANCHEGO_NODE_IMAGE AS execution
 ARG BUILDER_WORKDIR
 
 # Copy the executable into the container
-RUN mkdir -p /root/.avalanchego/plugins
-COPY --from=builder $BUILDER_WORKDIR/build/xsvm \
-    /root/.avalanchego/plugins/v3m4wPxaHpvGr8qfMeyK6PRW3idZrPHmYcMTt7oXdK47yurVH
+COPY --from=builder $BUILDER_WORKDIR/build/xsvm /avalanchego/build/plugins/v3m4wPxaHpvGr8qfMeyK6PRW3idZrPHmYcMTt7oXdK47yurVH
 
 # The node image's entrypoint will be reused.

tests/e2e/faultinjection/duplicate_node_id.go

@@ -26,6 +26,11 @@ var _ = ginkgo.Describe("Duplicate node handling", func() {
 	ginkgo.It("should ensure that a given Node ID (i.e. staking keypair) can be used at most once on a network", func() {
 		network := e2e.GetEnv(tc).GetNetwork()
 
+		if network.DefaultRuntimeConfig.KubeRuntimeConfig != nil {
+			// Enabling this test for kube requires supporting a flexible name mapping
+			ginkgo.Skip("This test is not supported on kube because it relies on the mapping of network uuid + nodeid -> statefulset name")
+		}
+
 		tc.By("creating new node")
 		node1 := e2e.AddEphemeralNode(tc, network, tmpnet.FlagsMap{})
 		e2e.WaitForHealthy(tc, node1)

tests/e2e/p/l1.go

@@ -186,10 +186,11 @@ var _ = e2e.DescribePChain("[L1]", func() {
 		var (
 			networkID           = env.GetNetwork().GetNetworkID()
 			genesisPeerMessages = buffer.NewUnboundedBlockingDeque[p2pmessage.InboundMessage](1)
+			stakingAddress      = e2e.GetLocalStakingAddress(tc, subnetGenesisNode)
 		)
 		genesisPeer, err := peer.StartTestPeer(
 			tc.DefaultContext(),
-			subnetGenesisNode.StakingAddress,
+			stakingAddress,
 			networkID,
 			router.InboundHandlerFunc(func(_ context.Context, m p2pmessage.InboundMessage) {
 				tc.Log().Info("received a message",

tests/fixture/e2e/env.go

@@ -172,9 +172,13 @@ func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork
 			}
 
 			for _, subnet := range network.Subnets {
-				for _, uri := range uris {
+				for nodeID, uri := range uris {
 					infoClient := info.NewClient(uri)
 					for _, chain := range subnet.Chains {
+						tc.Log().Info("checking if chain is bootstrapped",
+							zap.Stringer("chainID", chain.ChainID),
+							zap.Stringer("nodeID", nodeID),
+						)
 						isBootstrapped, err := infoClient.IsBootstrapped(tc.DefaultContext(), chain.ChainID.String())
 						// Ignore errors since a chain id that is not yet known will result in a recoverable error.
 						if err != nil || !isBootstrapped {
@@ -204,7 +208,7 @@ func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork
 		uri, cancel, err := node.GetLocalURI(tc.DefaultContext())
 		require.NoError(err)
 		tc.DeferCleanup(cancel)
-		env.URIs[i] = tmpnet.NodeURI{
+		uris[i] = tmpnet.NodeURI{
 			NodeID: node.NodeID,
 			URI:    uri,
 		}

tests/fixture/e2e/helpers.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"net/netip"
 	"os"
 	"strings"
 	"time"
@@ -138,6 +139,7 @@ func AddEphemeralNode(tc tests.TestContext, network *tmpnet.Network, flags tmpne
 
 	node := tmpnet.NewEphemeralNode(flags)
 	require.NoError(network.StartNode(tc.DefaultContext(), node))
+	WaitForHealthy(tc, node)
 
 	tc.DeferCleanup(func() {
 		tc.Log().Info("shutting down ephemeral node",
@@ -367,3 +369,10 @@ func GetLocalURI(tc tests.TestContext, node *tmpnet.Node) string {
 	tc.DeferCleanup(cancel)
 	return uri
 }
+
+func GetLocalStakingAddress(tc tests.TestContext, node *tmpnet.Node) netip.AddrPort {
+	stakingAddress, cancel, err := node.GetLocalStakingAddress(tc.DefaultContext())
+	require.NoError(tc, err)
+	tc.DeferCleanup(cancel)
+	return stakingAddress
+}

tests/fixture/tmpnet/defaults.go

@@ -61,12 +61,17 @@ func DefaultProcessFlags() FlagsMap {
 }
 
 // Flags appropriate for a node running as a local process
-func DefaultKubeFlags() FlagsMap {
-	return FlagsMap{
+func DefaultKubeFlags(includePluginDir bool) FlagsMap {
+	flags := FlagsMap{
 		config.HTTPHostKey:        "0.0.0.0", // Need to bind to pod IP to ensure kubelet can access the http port for readiness check
 		config.LogDisplayLevelKey: logging.Info.String(),
-		config.LogLevelKey:        logging.Off.String(), // Assume collection of stdout logs
+		// TODO(marun) Revert
+		config.LogLevelKey: logging.Info.String(), // Assume collection of stdout logs
+	}
+	if includePluginDir {
+		flags[config.PluginDirKey] = "/avalanchego/build/plugins"
 	}
+	return flags
 }
 
 // Flags required by e2e testing
@@ -85,7 +90,8 @@ func DefaultChainConfigs() map[string]FlagsMap {
 		// defined in the `github.com/ava-labs/coreth/evm` package.
 		"C": {
 			"warp-api-enabled": true,
-			"log-level":        "trace",
+			// TODO(marun) Revert
+			"log-level": "info",
 		},
 	}
 }

tests/fixture/tmpnet/kube.go

@@ -99,6 +99,8 @@ func NewNodeStatefulSet(
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{
 						"app": name,
+						// TODO(marun) Need to add more labels when running tests via e.g. github actions (repo, workflow, job details, run)
+						"promtail-collect": "true",
 					},
 				},
 				Spec: corev1.PodSpec{

tests/fixture/tmpnet/node_pod.go

@@ -30,14 +30,21 @@ import (
 	restclient "k8s.io/client-go/rest"
 )
 
+// TODO(marun) need an easy way to cleanup stale nodes (maybe this suggests something cli-based)
+
 const (
 	containerName   = "avago"
 	volumeName      = "data"
 	volumeMountPath = "/data"
 
 	statusCheckInterval = 500 * time.Millisecond
 	// TODO(marun) Need to make this configurable
-	volumeSize = "128Mi"
+	// - EBS volume sizes are in GiB.
+	// - A node will report unhealthy if less than 1GiB is available.
+	// - On the local storage provider configured by kind, the volume
+	// size doesn't matter size the volumes are just paths on the host
+	// filesystem.
+	volumeSize = "2Gi"
 )
 
 type NodePod struct {
@@ -101,7 +108,11 @@ func (p *NodePod) readState(ctx context.Context) error {
 }
 
 func (p *NodePod) SetDefaultFlags() {
-	p.node.Flags.SetDefaults(DefaultKubeFlags())
+	// Only include the plugin dir if there are subnets to track to ensure an immutable path can be used
+	// TODO(marun) Revisit this comment
+	trackSubnets, err := p.node.Flags.GetStringVal(config.TrackSubnetsKey)
+	includePluginDir := err == nil && len(trackSubnets) > 0
+	p.node.Flags.SetDefaults(DefaultKubeFlags(includePluginDir))
 }
 
 // TODO(marun) Maybe better with a timestamp used as input to generateName and include uuid and node id as labels?
@@ -113,12 +124,21 @@ func (p *NodePod) getStatefulSetName() string {
 	return p.node.getNetwork().UUID + "-" + strings.ToLower(nodeIDString[startIndex:endIndex])
 }
 
+func (p *NodePod) getFlagsForPod() FlagsMap {
+	flags := p.node.Flags.Copy()
+	flags[config.DataDirKey] = volumeMountPath
+	// TODO(marun) Simplify this with SetDefaultFlags
+	trackSubnets, err := p.node.Flags.GetStringVal(config.TrackSubnetsKey)
+	if err == nil && len(trackSubnets) > 0 {
+		flags[config.PluginDirKey] = "/avalanchego/build/plugins"
+	}
+	return flags
+}
+
 // Start the node as a kubernetes statefulset.
 func (p *NodePod) Start(ctx context.Context) error {
 	// Create a statefulset for the pod and wait for it to become ready
 	runtimeConfig := p.runtimeConfig()
-	statefulSetFlags := p.node.Flags.Copy()
-	statefulSetFlags[config.DataDirKey] = volumeMountPath
 	statefulSet := NewNodeStatefulSet(
 		p.getStatefulSetName(),
 		false, // generateName
@@ -127,7 +147,7 @@ func (p *NodePod) Start(ctx context.Context) error {
 		volumeName,
 		volumeSize,
 		volumeMountPath,
-		statefulSetFlags,
+		p.getFlagsForPod(),
 	)
 
 	clientset, err := p.getClientset()
@@ -234,7 +254,7 @@ func (p *NodePod) Restart(ctx context.Context) error {
 	// TODO(marun) Reconsider usage of FlagsMap instead of just map[string]string
 	container := statefulset.Spec.Template.Spec.Containers[0]
 	sortEnvVars(container.Env) // Ensure both are sorted
-	nodeEnv := flagsToEnvVarSlice(p.node.Flags)
+	nodeEnv := flagsToEnvVarSlice(p.getFlagsForPod())
 	if !slices.Equal(container.Env, nodeEnv) {
 		patches = append(patches, map[string]any{
 			"op":    "replace",
@@ -343,6 +363,9 @@ func (p *NodePod) IsHealthy(ctx context.Context) (bool, error) {
 	healthReply, err := CheckNodeHealth(ctx, uri)
 	if errors.Is(ErrUnrecoverableNodeHealthCheck, err) {
 		return false, err
+	} else if err != nil {
+		// TODO(maybe debug log the error?)
+		return false, nil
 	}
 	return healthReply.Healthy, nil
 }
@@ -435,7 +458,7 @@ func (p *NodePod) GetLocalURI(ctx context.Context) (string, func(), error) {
 }
 
 func (p *NodePod) GetLocalStakingAddress(ctx context.Context) (netip.AddrPort, func(), error) {
-	if len(p.node.URI) == 0 {
+	if p.node.StakingAddress == (netip.AddrPort{}) {
 		return netip.AddrPort{}, func() {}, errNotRunning
 	}
 

vms/example/xsvm/Dockerfile

@@ -24,8 +24,7 @@ RUN ./scripts/build_xsvm.sh
 # ============= Cleanup Stage ================
 FROM $AVALANCHEGO_NODE_IMAGE AS execution
 
-# Copy the xsvm binary to the default plugin path
-RUN mkdir -p /root/.avalanchego/plugins
-COPY --from=builder /build/build/xsvm /root/.avalanchego/plugins/v3m4wPxaHpvGr8qfMeyK6PRW3idZrPHmYcMTt7oXdK47yurVH
+# Copy the xsvm binary to the container plugin path
+COPY --from=builder $BUILDER_WORKDIR/build/xsvm /avalanchego/build/plugins/v3m4wPxaHpvGr8qfMeyK6PRW3idZrPHmYcMTt7oXdK47yurVH
 
 # The node image's entrypoint will be reused.