[FAB-678|1820] Support peer-side Dockerfile generation

Our deployment payloads carry Dockerfile contents in most
contexts.  This is problematic for a variety of reasons.

For example, it assumes that the client is in a position to
understand what docker images are available to each and every
peer in the network.  Consider that some peers may have
different image availability or be running on different
architectures.

In addition, the client is then responsible for installing
a meaningful peer.crt without having any knowledge of the peers
the chaincode will need to execute against.

It is also a potential source of attack and/or misconfiguration
since it provides another method to execute code within the
endorser network outside of the normal constraints provided by
the chaincode execution environment.

To solve all this, this patch splits the
chaincode::Platform.WritePackage() into two distinct phases:

 1) GetDeploymentPayload()
 2) GenerateDockerBuild()

Phase (1) is executed in the same context as the previous
WritePackage() operation.  That is, solely during
deployment payload generation via the CLI methods.

Phase (2) is now executed only by Endorsers just before
they need to create a Docker image from a given codepackage.
This phase is responsible for synthesizing an appropriate
Dockerfile based on parameters such as the type of chaincode,
the architecture of the peer, the certificate of the peer, etc.

Fixes FAB-1820
Part of the fix for FAB-678.

Change-Id: Ibf04efcde24e3fec30c4b9857b924fbe7654d197
Signed-off-by: Greg Haskins <[email protected]>

Author: ghaskins

bddtests/context_endorser.go

@@ -26,7 +26,6 @@ import (
 	"github.com/DATA-DOG/godog/gherkin"
 	"github.com/hyperledger/fabric/common/util"
 	"github.com/hyperledger/fabric/core/chaincode/platforms"
-	"github.com/hyperledger/fabric/core/container"
 	pb "github.com/hyperledger/fabric/protos/peer"
 	"github.com/hyperledger/fabric/protos/utils"
 	"golang.org/x/net/context"
@@ -55,12 +54,9 @@ func (b *BDDContext) build(spec *pb.ChaincodeSpec) (*pb.ChaincodeDeploymentSpec,
 		return nil, err
 	}
 
-	vm, err := container.NewVM()
-	if err != nil {
-		return nil, fmt.Errorf("Error getting vm")
-	}
-
-	codePackageBytes, err = vm.BuildChaincodeContainer(spec)
+	// FIXME: This only returns a deployment spec...the chaincode is not compiled.
+	// Is compilation needed?
+	codePackageBytes, err := platforms.GetDeploymentPayload(spec)
 	if err != nil {
 		return nil, err
 	}

core/chaincode/chaincode_support.go

@@ -17,7 +17,6 @@ limitations under the License.
 package chaincode
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"strconv"
@@ -32,6 +31,7 @@ import (
 	"strings"
 
 	"github.com/hyperledger/fabric/common/flogging"
+	"github.com/hyperledger/fabric/core/chaincode/platforms"
 	"github.com/hyperledger/fabric/core/chaincode/shim"
 	"github.com/hyperledger/fabric/core/common/ccprovider"
 	"github.com/hyperledger/fabric/core/container"
@@ -314,10 +314,19 @@ func (chaincodeSupport *ChaincodeSupport) sendInitOrReady(context context.Contex
 func (chaincodeSupport *ChaincodeSupport) getArgsAndEnv(cccid *ccprovider.CCContext, cLang pb.ChaincodeSpec_Type) (args []string, envs []string, err error) {
 	canName := cccid.GetCanonicalName()
 	envs = []string{"CORE_CHAINCODE_ID_NAME=" + canName}
-	//if TLS is enabled, pass TLS material to chaincode
+
+	// ----------------------------------------------------------------------------
+	// Pass TLS options to chaincode
+	// ----------------------------------------------------------------------------
+	// Note: The peer certificate is only baked into the image during the build
+	// phase (see core/chaincode/platforms).  This logic below merely assumes the
+	// image is already configured appropriately and is simply toggling the feature
+	// on or off.  If the peer's x509 has changed since the chaincode was deployed,
+	// the image may be stale and the admin will need to remove the current containers
+	// before restarting the peer.
+	// ----------------------------------------------------------------------------
 	if chaincodeSupport.peerTLS {
 		envs = append(envs, "CORE_PEER_TLS_ENABLED=true")
-		envs = append(envs, "CORE_PEER_TLS_CERT_FILE="+chaincodeSupport.peerTLSCertFile)
 		if chaincodeSupport.peerTLSSvrHostOrd != "" {
 			envs = append(envs, "CORE_PEER_TLS_SERVERHOSTOVERRIDE="+chaincodeSupport.peerTLSSvrHostOrd)
 		}
@@ -540,7 +549,7 @@ func (chaincodeSupport *ChaincodeSupport) Launch(context context.Context, cccid
 	//launch container if it is a System container or not in dev mode
 	if (!chaincodeSupport.userRunsCC || cds.ExecEnv == pb.ChaincodeDeploymentSpec_SYSTEM) && (chrte == nil || chrte.handler == nil) {
 
-		builder := func() (io.Reader, error) { return bytes.NewBuffer(cds.CodePackage), nil }
+		builder := func() (io.Reader, error) { return platforms.GenerateDockerBuild(cds) }
 		err = chaincodeSupport.launchAndWaitForRegister(context, cccid, cds, cLang, builder)
 		if err != nil {
 			chaincodeLogger.Errorf("launchAndWaitForRegister failed %s", err)
@@ -600,7 +609,11 @@ func (chaincodeSupport *ChaincodeSupport) Deploy(context context.Context, cccid
 		return cds, fmt.Errorf("error getting args for chaincode %s", err)
 	}
 
-	var targz io.Reader = bytes.NewBuffer(cds.CodePackage)
+	targz, err := platforms.GenerateDockerBuild(cds)
+	if err != nil {
+		return cds, fmt.Errorf("error converting CodePackage to Docker: %s", err)
+	}
+
 	cir := &container.CreateImageReq{CCID: ccintf.CCID{ChaincodeSpec: cds.ChaincodeSpec, NetworkID: chaincodeSupport.peerNetworkID, PeerID: chaincodeSupport.peerID, ChainID: cccid.ChainID, Version: cccid.Version}, Args: args, Reader: targz, Env: envs}
 
 	vmtype, _ := chaincodeSupport.getVMType(cds)

core/chaincode/chaincodetest.yaml

@@ -355,7 +355,6 @@ chaincode:
         # be appended depedendent upon the chaincode specification.
         Dockerfile:  |
           FROM hyperledger/fabric-ccenv:$(ARCH)-$(PROJECT_VERSION)
-          COPY src $GOPATH/src
           WORKDIR $GOPATH
 
     car:

core/chaincode/platforms/car/package.go

@@ -18,77 +18,81 @@ package car
 
 import (
 	"archive/tar"
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
-	"os"
+	"net/url"
 	"strings"
-	"time"
 
 	cutil "github.com/hyperledger/fabric/core/container/util"
 	pb "github.com/hyperledger/fabric/protos/peer"
 )
 
-func download(path string) (string, error) {
-	if strings.HasPrefix(path, "http://") {
-		// The file is remote, so we need to download it to a temporary location first
-
-		var tmp *os.File
-		var err error
-		tmp, err = ioutil.TempFile("", "car")
-		if err != nil {
-			return "", fmt.Errorf("Error creating temporary file: %s", err)
-		}
-		defer os.Remove(tmp.Name())
-		defer tmp.Close()
-
-		resp, err := http.Get(path)
-		if err != nil {
-			return "", fmt.Errorf("Error with HTTP GET: %s", err)
-		}
-		defer resp.Body.Close()
-
-		_, err = io.Copy(tmp, resp.Body)
-		if err != nil {
-			return "", fmt.Errorf("Error downloading bytes: %s", err)
-		}
-
-		return tmp.Name(), nil
+func httpDownload(path string) ([]byte, error) {
+
+	// The file is remote, so we need to download it first
+	var err error
+
+	resp, err := http.Get(path)
+	if err != nil {
+		return nil, fmt.Errorf("Error with HTTP GET: %s", err)
+	}
+	defer resp.Body.Close()
+
+	buf := bytes.NewBuffer(nil)
+
+	// FIXME: Validate maximum size constraints are not violated
+	_, err = io.Copy(buf, resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("Error downloading bytes: %s", err)
 	}
 
-	return path, nil
+	return buf.Bytes(), nil
 }
 
-// WritePackage satisfies the platform interface for generating a docker package
-// that encapsulates the environment for a CAR based chaincode
-func (carPlatform *Platform) WritePackage(spec *pb.ChaincodeSpec, tw *tar.Writer) error {
+var httpSchemes = map[string]bool{
+	"http":  true,
+	"https": true,
+}
 
-	path, err := download(spec.ChaincodeID.Path)
+func (carPlatform *Platform) GetDeploymentPayload(spec *pb.ChaincodeSpec) ([]byte, error) {
+
+	path := spec.ChaincodeID.Path
+	url, err := url.Parse(path)
 	if err != nil {
-		return err
+		return nil, fmt.Errorf("Error decoding %s: %s", path, err)
+	}
+
+	if _, ok := httpSchemes[url.Scheme]; ok {
+		return httpDownload(path)
+	} else {
+		// All we know is its _not_ an HTTP scheme.  Assume the path is a local file
+		// and see if it works.
+		return ioutil.ReadFile(url.Path)
 	}
+}
+
+func (carPlatform *Platform) GenerateDockerBuild(cds *pb.ChaincodeDeploymentSpec, tw *tar.Writer) (string, error) {
 
 	var buf []string
+	var err error
+
+	spec := cds.ChaincodeSpec
 
 	//let the executable's name be chaincode ID's name
 	buf = append(buf, cutil.GetDockerfileFromConfig("chaincode.car.Dockerfile"))
-	buf = append(buf, "COPY package.car /tmp/package.car")
+	buf = append(buf, "COPY codepackage.car /tmp/codepackage.car")
 	// invoking directly for maximum JRE compatiblity
-	buf = append(buf, fmt.Sprintf("RUN java -jar /usr/local/bin/chaintool buildcar /tmp/package.car -o $GOPATH/bin/%s && rm /tmp/package.car", spec.ChaincodeID.Name))
+	buf = append(buf, fmt.Sprintf("RUN java -jar /usr/local/bin/chaintool buildcar /tmp/codepackage.car -o $GOPATH/bin/%s && rm /tmp/codepackage.car", spec.ChaincodeID.Name))
 
 	dockerFileContents := strings.Join(buf, "\n")
-	dockerFileSize := int64(len([]byte(dockerFileContents)))
-
-	//Make headers identical by using zero time
-	var zeroTime time.Time
-	tw.WriteHeader(&tar.Header{Name: "Dockerfile", Size: dockerFileSize, ModTime: zeroTime, AccessTime: zeroTime, ChangeTime: zeroTime})
-	tw.Write([]byte(dockerFileContents))
 
-	err = cutil.WriteFileToPackage(path, "package.car", tw)
+	err = cutil.WriteBytesToPackage("codepackage.car", cds.CodePackage, tw)
 	if err != nil {
-		return err
+		return "", err
 	}
 
-	return nil
+	return dockerFileContents, nil
 }

core/chaincode/platforms/car/test/car_test.go

@@ -47,7 +47,7 @@ func TestCar_BuildImage(t *testing.T) {
 
 	chaincodePath := cwd + "/org.hyperledger.chaincode.example02-0.1-SNAPSHOT.car"
 	spec := &pb.ChaincodeSpec{Type: pb.ChaincodeSpec_CAR, ChaincodeID: &pb.ChaincodeID{Name: "cartest", Path: chaincodePath}, Input: &pb.ChaincodeInput{Args: util.ToChaincodeArgs("f")}}
-	if _, err := vm.BuildChaincodeContainer(spec); err != nil {
+	if err := vm.BuildChaincodeContainer(spec); err != nil {
 		t.Error(err)
 	}
 }

core/chaincode/platforms/golang/package.go

@@ -19,12 +19,6 @@ package golang
 import (
 	"archive/tar"
 	"fmt"
-	"strings"
-	"time"
-
-	"github.com/spf13/viper"
-
-	"errors"
 
 	cutil "github.com/hyperledger/fabric/core/container/util"
 	pb "github.com/hyperledger/fabric/protos/peer"
@@ -34,53 +28,12 @@ import (
 //will just package rest of the bytes
 func writeChaincodePackage(spec *pb.ChaincodeSpec, tw *tar.Writer) error {
 
-	var urlLocation string
-	if strings.HasPrefix(spec.ChaincodeID.Path, "http://") {
-		urlLocation = spec.ChaincodeID.Path[7:]
-	} else if strings.HasPrefix(spec.ChaincodeID.Path, "https://") {
-		urlLocation = spec.ChaincodeID.Path[8:]
-	} else {
-		urlLocation = spec.ChaincodeID.Path
-	}
-
-	if urlLocation == "" {
-		return errors.New("ChaincodeSpec's path/URL cannot be empty")
-	}
-
-	if strings.LastIndex(urlLocation, "/") == len(urlLocation)-1 {
-		urlLocation = urlLocation[:len(urlLocation)-1]
-	}
-	toks := strings.Split(urlLocation, "/")
-	if toks == nil || len(toks) == 0 {
-		return fmt.Errorf("cannot get path components from %s", urlLocation)
-	}
-
-	chaincodeGoName := toks[len(toks)-1]
-	if chaincodeGoName == "" {
-		return fmt.Errorf("could not get chaincode name from path %s", urlLocation)
-	}
-
-	var buf []string
-
-	buf = append(buf, cutil.GetDockerfileFromConfig("chaincode.golang.Dockerfile"))
-	//let the executable's name be chaincode ID's name
-	buf = append(buf, fmt.Sprintf("RUN go install %s && mv $GOPATH/bin/%s $GOPATH/bin/%s", urlLocation, chaincodeGoName, spec.ChaincodeID.Name))
-	//NOTE-this could have been abstracted away so we could use it for all platforms in a common manner
-	//However, it would still be docker specific. Hence any such abstraction has to be done in a manner that
-	//is not just language dependent but also container depenedent. So lets make this change per platform for now
-	//in the interest of avoiding over-engineering without proper abstraction
-	if viper.GetBool("peer.tls.enabled") {
-		buf = append(buf, fmt.Sprintf("COPY src/certs/cert.pem %s", viper.GetString("peer.tls.cert.file")))
+	urlLocation, err := decodeUrl(spec)
+	if err != nil {
+		return fmt.Errorf("could not decode url: %s", err)
 	}
 
-	dockerFileContents := strings.Join(buf, "\n")
-	dockerFileSize := int64(len([]byte(dockerFileContents)))
-
-	//Make headers identical by using zero time
-	var zeroTime time.Time
-	tw.WriteHeader(&tar.Header{Name: "Dockerfile", Size: dockerFileSize, ModTime: zeroTime, AccessTime: zeroTime, ChangeTime: zeroTime})
-	tw.Write([]byte(dockerFileContents))
-	err := cutil.WriteGopathSrc(tw, urlLocation)
+	err = cutil.WriteGopathSrc(tw, urlLocation)
 	if err != nil {
 		return fmt.Errorf("Error writing Chaincode package contents: %s", err)
 	}