[FEATURE] add ability to add ports to an existing loadbalancer (#615)

cmd/node/node.go

@@ -48,6 +48,7 @@ func NewCmdNode() *cobra.Command {
 	cmd.AddCommand(NewCmdNodeStop())
 	cmd.AddCommand(NewCmdNodeDelete())
 	cmd.AddCommand(NewCmdNodeList())
+	cmd.AddCommand(NewCmdNodeEdit())
 
 	// add flags
 

cmd/node/nodeEdit.go

@@ -0,0 +1,113 @@
+/*
+Copyright © 2020-2021 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package node
+
+import (
+	"github.com/docker/go-connections/nat"
+	"github.com/rancher/k3d/v4/cmd/util"
+	"github.com/rancher/k3d/v4/pkg/client"
+	"github.com/rancher/k3d/v4/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v4/pkg/types"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+// NewCmdNodeEdit returns a new cobra command
+func NewCmdNodeEdit() *cobra.Command {
+
+	// create new cobra command
+	cmd := &cobra.Command{
+		Use:               "edit NAME",
+		Short:             "[EXPERIMENTAL] Edit node(s).",
+		Long:              `[EXPERIMENTAL] Edit node(s).`,
+		Args:              cobra.ExactArgs(1),
+		Aliases:           []string{"update"},
+		ValidArgsFunction: util.ValidArgsAvailableNodes,
+		Run: func(cmd *cobra.Command, args []string) {
+
+			existingNode, changeset := parseEditNodeCmd(cmd, args)
+
+			log.Debugf("===== Current =====\n%+v\n===== Changeset =====\n%+v\n", existingNode, changeset)
+
+			if err := client.NodeEdit(cmd.Context(), runtimes.SelectedRuntime, existingNode, changeset); err != nil {
+				log.Fatalln(err)
+			}
+
+			log.Infof("Successfully updated %s", existingNode.Name)
+
+		},
+	}
+
+	// add subcommands
+
+	// add flags
+	cmd.Flags().StringArray("port-add", nil, "[EXPERIMENTAL] (serverlb only!) Map ports from the node container to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d node edit k3d-mycluster-serverlb --port-add 8080:80`")
+
+	// done
+	return cmd
+}
+
+// parseEditNodeCmd parses the command input into variables required to delete nodes
+func parseEditNodeCmd(cmd *cobra.Command, args []string) (*k3d.Node, *k3d.Node) {
+
+	existingNode, err := client.NodeGet(cmd.Context(), runtimes.SelectedRuntime, &k3d.Node{Name: args[0]})
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	if existingNode == nil {
+		log.Infof("Node %s not found", args[0])
+		return nil, nil
+	}
+
+	if existingNode.Role != k3d.LoadBalancerRole {
+		log.Fatalln("Currently only the loadbalancer can be updated!")
+	}
+
+	changeset := &k3d.Node{}
+
+	/*
+	 * --port-add
+	 */
+	portFlags, err := cmd.Flags().GetStringArray("port-add")
+	if err != nil {
+		log.Errorln(err)
+		return nil, nil
+	}
+
+	// init portmap
+	changeset.Ports = nat.PortMap{}
+
+	for _, flag := range portFlags {
+
+		portmappings, err := nat.ParsePortSpec(flag)
+		if err != nil {
+			log.Fatalf("Failed to parse port spec '%s': %+v", flag, err)
+		}
+
+		for _, pm := range portmappings {
+			changeset.Ports[pm.Port] = append(changeset.Ports[pm.Port], pm.Binding)
+		}
+	}
+
+	return existingNode, changeset
+}

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/heroku/docker-registry-client v0.0.0-20190909225348-afc9e1acc3d5
 	github.com/imdario/mergo v0.3.12
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de
+	github.com/mitchellh/copystructure v1.2.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf // indirect

go.sum

@@ -368,6 +368,8 @@ github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpe
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -380,6 +382,8 @@ github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:F
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM=
 github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM=

pkg/client/node.go

@@ -33,6 +33,9 @@ import (
 	"strings"
 	"time"
 
+	copystruct "github.com/mitchellh/copystructure"
+
+	"github.com/docker/go-connections/nat"
 	dockerunits "github.com/docker/go-units"
 	"github.com/imdario/mergo"
 	"github.com/rancher/k3d/v4/pkg/actions"
@@ -639,3 +642,150 @@ nodeLoop:
 
 	return resultList
 }
+
+// NodeEdit let's you update an existing node
+func NodeEdit(ctx context.Context, runtime runtimes.Runtime, existingNode, changeset *k3d.Node) error {
+
+	/*
+	 * Make a deep copy of the existing node
+	 */
+
+	result, err := CopyNode(ctx, existingNode, CopyNodeOpts{keepState: false})
+	if err != nil {
+		return err
+	}
+
+	/*
+	 * Apply changes
+	 */
+
+	// === Ports ===
+	if result.Ports == nil {
+		result.Ports = nat.PortMap{}
+	}
+	for port, portbindings := range changeset.Ports {
+	loopChangesetPortbindings:
+		for _, portbinding := range portbindings {
+
+			// loop over existing portbindings to avoid port collisions (docker doesn't check for it)
+			for _, existingPB := range result.Ports[port] {
+				if util.IsPortBindingEqual(portbinding, existingPB) { // also matches on "equal" HostIPs (127.0.0.1, "", 0.0.0.0)
+					log.Tracef("Skipping existing PortBinding: %+v", existingPB)
+					continue loopChangesetPortbindings
+				}
+			}
+			log.Tracef("Adding portbinding %+v for port %s", portbinding, port.Port())
+			result.Ports[port] = append(result.Ports[port], portbinding)
+		}
+	}
+
+	// --- Loadbalancer specifics ---
+	if result.Role == k3d.LoadBalancerRole {
+		nodeEditApplyLBSpecifics(ctx, result)
+	}
+
+	// replace existing node
+	return NodeReplace(ctx, runtime, existingNode, result)
+
+}
+
+func nodeEditApplyLBSpecifics(ctx context.Context, lbNode *k3d.Node) {
+	tcp_ports := []string{}
+	udp_ports := []string{}
+	for index, env := range lbNode.Env {
+		if strings.HasPrefix(env, "PORTS=") || strings.HasPrefix(env, "UDP_PORTS=") {
+			// Remove matching environment variable from slice (does not preserve order)
+			lbNode.Env[index] = lbNode.Env[len(lbNode.Env)-1] // copy last element to index of matching env
+			lbNode.Env[len(lbNode.Env)-1] = ""                // remove last element
+			lbNode.Env = lbNode.Env[:len(lbNode.Env)-1]       // truncate
+		}
+	}
+
+	for port := range lbNode.Ports {
+		switch port.Proto() {
+		case "tcp":
+			tcp_ports = append(tcp_ports, port.Port())
+			break
+		case "udp":
+			udp_ports = append(udp_ports, port.Port())
+			break
+		default:
+			log.Warnf("Unknown port protocol %s for port %s", port.Proto(), port.Port())
+		}
+	}
+	lbNode.Env = append(lbNode.Env, fmt.Sprintf("PORTS=%s", strings.Join(tcp_ports, ",")))
+	lbNode.Env = append(lbNode.Env, fmt.Sprintf("UDP_PORTS=%s", strings.Join(udp_ports, ",")))
+}
+
+func NodeReplace(ctx context.Context, runtime runtimes.Runtime, old, new *k3d.Node) error {
+
+	// rename existing node
+	oldNameTemp := fmt.Sprintf("%s-%s", old.Name, util.GenerateRandomString(5))
+	oldNameOriginal := old.Name
+	log.Infof("Renaming existing node %s to %s...", old.Name, oldNameTemp)
+	if err := runtime.RenameNode(ctx, old, oldNameTemp); err != nil {
+		return err
+	}
+	old.Name = oldNameTemp
+
+	// create (not start) new node
+	log.Infof("Creating new node %s...", new.Name)
+	if err := NodeCreate(ctx, runtime, new, k3d.NodeCreateOpts{Wait: true}); err != nil {
+		if err := runtime.RenameNode(ctx, old, oldNameOriginal); err != nil {
+			return fmt.Errorf("Failed to create new node. Also failed to rename %s back to %s: %+v", old.Name, oldNameOriginal, err)
+		}
+		return fmt.Errorf("Failed to create new node. Brought back old node: %+v", err)
+	}
+
+	// stop existing/old node
+	log.Infof("Stopping existing node %s...", old.Name)
+	if err := runtime.StopNode(ctx, old); err != nil {
+		return err
+	}
+
+	// start new node
+	log.Infof("Starting new node %s...", new.Name)
+	if err := NodeStart(ctx, runtime, new, k3d.NodeStartOpts{Wait: true}); err != nil {
+		if err := NodeDelete(ctx, runtime, new, k3d.NodeDeleteOpts{SkipLBUpdate: true}); err != nil {
+			return fmt.Errorf("Failed to start new node. Also failed to rollback: %+v", err)
+		}
+		if err := runtime.RenameNode(ctx, old, oldNameOriginal); err != nil {
+			return fmt.Errorf("Failed to start new node. Also failed to rename %s back to %s: %+v", old.Name, oldNameOriginal, err)
+		}
+		old.Name = oldNameOriginal
+		if err := NodeStart(ctx, runtime, old, k3d.NodeStartOpts{Wait: true}); err != nil {
+			return fmt.Errorf("Failed to start new node. Also failed to restart old node: %+v", err)
+		}
+		return fmt.Errorf("Failed to start new node. Rolled back: %+v", err)
+	}
+
+	// cleanup: delete old node
+	log.Infof("Deleting old node %s...", old.Name)
+	if err := NodeDelete(ctx, runtime, old, k3d.NodeDeleteOpts{SkipLBUpdate: true}); err != nil {
+		return err
+	}
+
+	// done
+	return nil
+}
+
+type CopyNodeOpts struct {
+	keepState bool
+}
+
+func CopyNode(ctx context.Context, src *k3d.Node, opts CopyNodeOpts) (*k3d.Node, error) {
+
+	targetCopy, err := copystruct.Copy(src)
+	if err != nil {
+		return nil, err
+	}
+
+	result := targetCopy.(*k3d.Node)
+
+	if !opts.keepState {
+		// ensure that node state is empty
+		result.State = k3d.NodeState{}
+	}
+
+	return result, err
+}

pkg/runtimes/docker/node.go

@@ -447,3 +447,21 @@ func (d Docker) GetNodesInNetwork(ctx context.Context, network string) ([]*k3d.N
 
 	return connectedNodes, nil
 }
+
+func (d Docker) RenameNode(ctx context.Context, node *k3d.Node, newName string) error {
+	// get the container for the given node
+	container, err := getNodeContainer(ctx, node)
+	if err != nil {
+		return err
+	}
+
+	// create docker client
+	docker, err := GetDockerClient()
+	if err != nil {
+		log.Errorln("Failed to create docker client")
+		return err
+	}
+	defer docker.Close()
+
+	return docker.ContainerRename(ctx, container.ID, newName)
+}

pkg/runtimes/docker/translate.go

@@ -243,14 +243,6 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
 		}
 	}
 
-	// env vars: only copy K3S_* and K3D_* // FIXME: should we really do this? Might be unexpected, if user has e.g. HTTP_PROXY vars
-	env := []string{}
-	for _, envVar := range containerDetails.Config.Env {
-		if strings.HasPrefix(envVar, "K3D_") || strings.HasPrefix(envVar, "K3S_") {
-			env = append(env, envVar)
-		}
-	}
-
 	// labels: only copy k3d.* labels
 	labels := map[string]string{}
 	for k, v := range containerDetails.Config.Labels {
@@ -277,7 +269,7 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
 		Role:          k3d.NodeRoles[containerDetails.Config.Labels[k3d.LabelRole]],
 		Image:         containerDetails.Image,
 		Volumes:       containerDetails.HostConfig.Binds,
-		Env:           env,
+		Env:           containerDetails.Config.Env,
 		Cmd:           containerDetails.Config.Cmd,
 		Args:          []string{}, // empty, since Cmd already contains flags
 		Ports:         containerDetails.HostConfig.PortBindings,

pkg/runtimes/runtime.go

@@ -52,6 +52,7 @@ type Runtime interface {
 	GetHost() string
 	CreateNode(context.Context, *k3d.Node) error
 	DeleteNode(context.Context, *k3d.Node) error
+	RenameNode(context.Context, *k3d.Node, string) error
 	GetNodesByLabel(context.Context, map[string]string) ([]*k3d.Node, error)
 	GetNode(context.Context, *k3d.Node) (*k3d.Node, error)
 	GetNodeStatus(context.Context, *k3d.Node) (bool, string, error)

pkg/util/ports.go

@@ -25,6 +25,7 @@ package util
 import (
 	"net"
 
+	"github.com/docker/go-connections/nat"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -45,3 +46,21 @@ func GetFreePort() (int, error) {
 
 	return tcpListener.Addr().(*net.TCPAddr).Port, nil
 }
+
+var equalHostIPs = map[string]interface{}{
+	"":          nil,
+	"127.0.0.1": nil,
+	"0.0.0.0":   nil,
+	"localhost": nil,
+}
+
+func IsPortBindingEqual(a, b nat.PortBinding) bool {
+	if a.HostPort == b.HostPort {
+		if _, ok := equalHostIPs[a.HostIP]; ok {
+			if _, ok := equalHostIPs[b.HostIP]; ok {
+				return true
+			}
+		}
+	}
+	return false
+}