Add versioning support to our applications

This patch introduces a more robust versioning mechanism, summarized
as follows:

1) The top-level makefile defines $PROJECT_VERSION which is intended
   to be consumed throughout (though some places are left for a
   future patch)
2) The variable is expected to use Semantic Versioning with X.Y.Z for
   releases, and X.Y.Z-SNAPSHOT for intermediate development builds.
3) We bake the $PROJECT_VERSION into the binaries (peer and membersrvc)
   at compile time and remove the yaml configuration that attempted to
   provide competing versioning.
4) We tag our docker images (except for baseimage, that will be
   addressed later) with :$(ARCH)-$(PROJECT_VERSION), suitable for
   unambiguous long term hosting on something like dockerhub.
5) We add support for dynamically parsing yaml-based Dockerfile
   definitions to support variable subsitution.  This allows the yaml
   to set a Dockerfile definition with something like
   "FROM foo:$(ARCH)-$(PROJECT_VERSION)" and the variables will
   resolve at runtime based on the context it runs under.

This is all as a first step in supporting a more reasonable release
mechanism in general, and a docker-based release mechanism in particular.

Change-Id: Ic6df02be4046ca5dbb256f26658a4c393da55c0f
Signed-off-by: Greg Haskins <[email protected]>

Author: ghaskins

Makefile

@@ -36,8 +36,21 @@
 #   - clean - cleans the build area
 #   - dist-clean - superset of 'clean' that also removes persistent state
 
-PROJECT_NAME=hyperledger/fabric
+PROJECT_NAME   = hyperledger/fabric
+BASE_VERSION   = 0.6.0
+IS_RELEASE     = false # commit as 'true' only once for a given $(BASE_VERSION)
+
+ifneq ($(IS_RELEASE),true)
+EXTRA_VERSION ?= SNAPSHOT-$(shell git rev-parse --short HEAD)
+PROJECT_VERSION=$(BASE_VERSION)-$(EXTRA_VERSION)
+else
+PROJECT_VERSION=$(BASE_VERSION)
+endif
+
+DOCKER_TAG=$(shell uname -m)-$(PROJECT_VERSION)
+
 PKGNAME = github.com/$(PROJECT_NAME)
+GO_LDFLAGS = -X github.com/hyperledger/fabric/metadata.Version=$(PROJECT_VERSION)
 CGO_FLAGS = CGO_CFLAGS=" " CGO_LDFLAGS="-lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy"
 UID = $(shell id -u)
 CHAINTOOL_RELEASE=v0.8.1
@@ -76,7 +89,7 @@ membersrvc: build/bin/membersrvc
 membersrvc-image: build/image/membersrvc/.dummy
 
 unit-test: peer-image gotools
-	@./scripts/goUnitTests.sh
+	@./scripts/goUnitTests.sh $(DOCKER_TAG) "$(GO_LDFLAGS)"
 
 .PHONY: images
 images: $(patsubst %,build/image/%/.dummy, $(IMAGES))
@@ -137,7 +150,7 @@ build/docker/bin/%: build/image/src/.dummy $(PROJECT_FILES)
 		--user=$(UID) \
 		-v $(abspath build/docker/bin):/opt/gopath/bin \
 		-v $(abspath build/docker/pkg):/opt/gopath/pkg \
-		hyperledger/fabric-src go install github.com/hyperledger/fabric/$(TARGET)
+		hyperledger/fabric-src:$(DOCKER_TAG) go install -ldflags "$(GO_LDFLAGS)" github.com/hyperledger/fabric/$(TARGET)
 	@touch $@
 
 build/bin:
@@ -156,7 +169,7 @@ build/bin/block-listener:
 build/bin/%: build/image/base/.dummy $(PROJECT_FILES)
 	@mkdir -p $(@D)
 	@echo "$@"
-	$(CGO_FLAGS) GOBIN=$(abspath $(@D)) go install $(PKGNAME)/$(@F)
+	$(CGO_FLAGS) GOBIN=$(abspath $(@D)) go install -ldflags "$(GO_LDFLAGS)" $(PKGNAME)/$(@F)
 	@echo "Binary available as $@"
 	@touch $@
 
@@ -171,16 +184,22 @@ build/image/base/.dummy: $(BASEIMAGE_DEPS)
 build/image/src/.dummy: build/image/base/.dummy $(PROJECT_FILES)
 	@echo "Building docker src-image"
 	@mkdir -p $(@D)
-	@cat images/src/Dockerfile.in > $(@D)/Dockerfile
+	@cat images/src/Dockerfile.in \
+		| sed -e 's/_TAG_/$(DOCKER_TAG)/g' \
+		> $(@D)/Dockerfile
 	@git ls-files | tar -jcT - > $(@D)/gopath.tar.bz2
-	docker build -t $(PROJECT_NAME)-src:latest $(@D)
+	docker build -t $(PROJECT_NAME)-src $(@D)
+	docker tag $(PROJECT_NAME)-src $(PROJECT_NAME)-src:$(DOCKER_TAG)
 	@touch $@
 
 # Special override for ccenv-image (chaincode-environment)
 build/image/ccenv/.dummy: build/image/src/.dummy build/image/ccenv/bin/protoc-gen-go build/image/ccenv/bin/chaintool Makefile
 	@echo "Building docker ccenv-image"
-	@cat images/ccenv/Dockerfile.in > $(@D)/Dockerfile
-	docker build -t $(PROJECT_NAME)-ccenv:latest $(@D)
+	@cat images/ccenv/Dockerfile.in \
+		| sed -e 's/_TAG_/$(DOCKER_TAG)/g' \
+		> $(@D)/Dockerfile
+	docker build -t $(PROJECT_NAME)-ccenv $(@D)
+	docker tag $(PROJECT_NAME)-ccenv $(PROJECT_NAME)-ccenv:$(DOCKER_TAG)
 	@touch $@
 
 # Special override for java-image
@@ -194,17 +213,21 @@ build/image/javaenv/.dummy: Makefile $(JAVASHIM_DEPS)
 	# 3. Gradle settings file
 	@git ls-files core/chaincode/shim/java | tar -jcT - > $(@D)/javashimsrc.tar.bz2
 	@git ls-files protos settings.gradle  | tar -jcT - > $(@D)/protos.tar.bz2
-	docker build -t $(PROJECT_NAME)-javaenv:latest $(@D)
+	docker build -t $(PROJECT_NAME)-javaenv $(@D)
+	docker tag $(PROJECT_NAME)-javaenv $(PROJECT_NAME)-javaenv:$(DOCKER_TAG)
 	@touch $@
 
 # Default rule for image creation
 build/image/%/.dummy: build/image/src/.dummy build/docker/bin/%
 	$(eval TARGET = ${patsubst build/image/%/.dummy,%,${@}})
 	@echo "Building docker $(TARGET)-image"
 	@mkdir -p $(@D)/bin
-	@cat images/app/Dockerfile.in | sed -e 's/_TARGET_/$(TARGET)/g' > $(@D)/Dockerfile
+	@cat images/app/Dockerfile.in \
+		| sed -e 's/_TAG_/$(DOCKER_TAG)/g' \
+		> $(@D)/Dockerfile
 	cp build/docker/bin/$(TARGET) $(@D)/bin
-	docker build -t $(PROJECT_NAME)-$(TARGET):latest $(@D)
+	docker build -t $(PROJECT_NAME)-$(TARGET) $(@D)
+	docker tag $(PROJECT_NAME)-$(TARGET) $(PROJECT_NAME)-$(TARGET):$(DOCKER_TAG)
 	@touch $@
 
 .PHONY: protos
@@ -215,9 +238,11 @@ base-image-clean:
 	-docker rmi -f $(PROJECT_NAME)-baseimage
 	-@rm -rf build/image/base ||:
 
+src-image-clean: ccenv-image-clean peer-image-clean membersrvc-image-clean
+
 %-image-clean:
 	$(eval TARGET = ${patsubst %-image-clean,%,${@}})
-	-docker rmi -f $(PROJECT_NAME)-$(TARGET)
+	-docker images -q $(PROJECT_NAME)-$(TARGET) | xargs -r docker rmi -f
 	-@rm -rf build/image/$(TARGET) ||:
 
 images-clean: $(patsubst %,%-image-clean, $(IMAGES))

core/chaincode/platforms/car/package.go

@@ -28,7 +28,6 @@ import (
 
 	cutil "github.com/hyperledger/fabric/core/container/util"
 	pb "github.com/hyperledger/fabric/protos"
-	"github.com/spf13/viper"
 )
 
 func download(path string) (string, error) {
@@ -78,7 +77,7 @@ func (carPlatform *Platform) WritePackage(spec *pb.ChaincodeSpec, tw *tar.Writer
 	var buf []string
 
 	//let the executable's name be chaincode ID's name
-	buf = append(buf, viper.GetString("chaincode.car.Dockerfile"))
+	buf = append(buf, cutil.GetDockerfileFromConfig("chaincode.car.Dockerfile"))
 	buf = append(buf, "COPY package.car /tmp/package.car")
 	buf = append(buf, fmt.Sprintf("RUN chaintool buildcar /tmp/package.car -o $GOPATH/bin/%s && rm /tmp/package.car", spec.ChaincodeID.Name))
 

core/chaincode/platforms/golang/package.go

@@ -69,7 +69,7 @@ func writeChaincodePackage(spec *pb.ChaincodeSpec, tw *tar.Writer) error {
 		newRunLine = fmt.Sprintf("%s\nCOPY src/certs/cert.pem %s", newRunLine, viper.GetString("peer.tls.cert.file"))
 	}
 
-	dockerFileContents := fmt.Sprintf("%s\n%s", viper.GetString("chaincode.golang.Dockerfile"), newRunLine)
+	dockerFileContents := fmt.Sprintf("%s\n%s", cutil.GetDockerfileFromConfig("chaincode.golang.Dockerfile"), newRunLine)
 	dockerFileSize := int64(len([]byte(dockerFileContents)))
 
 	//Make headers identical by using zero time

core/chaincode/platforms/java/package.go

@@ -44,7 +44,7 @@ func writeChaincodePackage(spec *pb.ChaincodeSpec, tw *tar.Writer) error {
 	if viper.GetBool("security.enabled") {
 		//todo
 	} else {
-		buf = append(buf, viper.GetString("chaincode.java.Dockerfile"))
+		buf = append(buf, cutil.GetDockerfileFromConfig("chaincode.java.Dockerfile"))
 		buf = append(buf, "COPY src /root")
 		buf = append(buf, "RUN gradle -b build.gradle build")
 		buf = append(buf, "RUN unzip -od /root build/distributions/Chaincode.zip")

core/container/util/dockerutil.go

@@ -17,7 +17,11 @@ limitations under the License.
 package util
 
 import (
+	"runtime"
+	"strings"
+
 	"github.com/fsouza/go-dockerclient"
+	"github.com/hyperledger/fabric/metadata"
 	"github.com/spf13/viper"
 )
 
@@ -35,3 +39,30 @@ func NewDockerClient() (client *docker.Client, err error) {
 	}
 	return
 }
+
+// Our docker images retrieve $ARCH via "uname -m", which is typically "x86_64" for, well, x86_64.
+// However, GOARCH uses "amd64".  We therefore need to normalize any discrepancies between "uname -m"
+// and GOARCH here.
+var archRemap = map[string]string{
+	"amd64": "x86_64",
+}
+
+func getArch() string {
+	if remap, ok := archRemap[runtime.GOARCH]; ok {
+		return remap
+	} else {
+		return runtime.GOARCH
+	}
+}
+
+func parseDockerfileTemplate(template string) string {
+	r := strings.NewReplacer(
+		"$(ARCH)", getArch(),
+		"$(PROJECT_VERSION)", metadata.Version)
+
+	return r.Replace(template)
+}
+
+func GetDockerfileFromConfig(path string) string {
+	return parseDockerfileTemplate(viper.GetString(path))
+}

core/container/util/dockerutil_test.go

@@ -0,0 +1,31 @@
+/*
+Copyright London Stock Exchange 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+
+	"github.com/hyperledger/fabric/metadata"
+)
+
+func TestUtil_DockerfileTemplateParser(t *testing.T) {
+	expected := "FROM foo:" + getArch() + "-" + metadata.Version
+	actual := parseDockerfileTemplate("FROM foo:$(ARCH)-$(PROJECT_VERSION)")
+	if actual != expected {
+		t.Errorf("Error parsing Dockerfile Template.  Expected \"%s\", got \"%s\"", expected, actual)
+	}
+}

images/app/Dockerfile.in

@@ -1,4 +1,4 @@
-FROM hyperledger/fabric-src:latest
+FROM hyperledger/fabric-src:_TAG_
 RUN mkdir -p /var/hyperledger/db
 COPY bin/* $GOPATH/bin/
 WORKDIR $GOPATH/src/github.com/hyperledger/fabric

images/ccenv/Dockerfile.in

@@ -1,2 +1,2 @@
-FROM hyperledger/fabric-src:latest
+FROM hyperledger/fabric-src:_TAG_
 COPY bin/* /usr/local/bin/

membersrvc/membersrvc.yaml

@@ -1,8 +1,6 @@
 # CA server parameters
 #
 server:
-        # current version of the CA
-        version: "0.1"
 
         # limits the number of operating system threads used by the CA
         # set to negative to use the system default setting

membersrvc/server.go

@@ -27,6 +27,7 @@ import (
 	"github.com/hyperledger/fabric/core/crypto"
 	"github.com/hyperledger/fabric/flogging"
 	"github.com/hyperledger/fabric/membersrvc/ca"
+	"github.com/hyperledger/fabric/metadata"
 	"github.com/op/go-logging"
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
@@ -67,7 +68,7 @@ func main() {
 	// cache configure
 	ca.CacheConfiguration()
 
-	logger.Infof("CA Server (" + viper.GetString("server.version") + ")")
+	logger.Infof("CA Server (" + metadata.Version + ")")
 
 	aca := ca.NewACA()
 	defer aca.Stop()

metadata/metadata.go

@@ -0,0 +1,19 @@
+/*
+Copyright London Stock Exchange 2016 All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+		 http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metadata
+
+var Version string // defined by the Makefile and passed in with ldflags

peer/core.yaml

@@ -76,10 +76,6 @@ logging:
 ###############################################################################
 peer:
 
-    # Peer Version following version semantics as described here http://semver.org/
-    # The Peer supplies this version in communications with other Peers
-    version:  0.1.0
-
     # The Peer id is used for identifying this Peer instance.
     id: jdoe
 
@@ -292,14 +288,15 @@ chaincode:
         # This is the basis for the CAR Dockerfile.  Additional commands will
         # be appended depedendent upon the chaincode specification.
         Dockerfile:  |
-            FROM hyperledger/fabric-ccenv
+            FROM hyperledger/fabric-ccenv:$(ARCH)-$(PROJECT_VERSION)
+
     java:
         # This is an image based on java:openjdk-8 with addition compiler
         # tools added for java shim layer packaging.
         # This image is packed with shim layer libraries that are necessary
         # for Java chaincode runtime.
         Dockerfile:  |
-            from hyperledger/fabric-javaenv
+            from hyperledger/fabric-javaenv:$(ARCH)-$(PROJECT_VERSION)
 
     # timeout in millisecs for starting up a container and waiting for Register
     # to come through. 1sec should be plenty for chaincode unit tests

peer/version/version.go

@@ -19,8 +19,8 @@ package version
 import (
 	"fmt"
 
+	"github.com/hyperledger/fabric/metadata"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 )
 
 // Cmd returns the Cobra Command for Version
@@ -39,6 +39,5 @@ var cobraCommand = &cobra.Command{
 
 // Print outputs the current executable version to stdout
 func Print() {
-	version := viper.GetString("peer.version")
-	fmt.Printf("Fabric peer server version %s\n", version)
+	fmt.Printf("Fabric peer server version %s\n", metadata.Version)
 }

scripts/goUnitTests.sh

@@ -2,6 +2,19 @@
 
 set -e
 
+TAG=$1
+GO_LDFLAGS=$2
+
+BASEIMAGE="hyperledger/fabric-peer"
+IMAGE=$BASEIMAGE
+
+if [ "$TAG" != "" ]
+then
+    IMAGE="$BASEIMAGE:$TAG"
+fi
+
+echo "Running unit tests using $IMAGE"
+
 echo "Cleaning membership services folder"
 rm -rf membersrvc/ca/.ca/
 
@@ -10,7 +23,7 @@ PKGS=`go list github.com/hyperledger/fabric/... | grep -v /vendor/ | grep -v /ex
 echo "DONE!"
 
 echo -n "Starting peer.."
-CID=`docker run -dit -p 7051:7051 hyperledger/fabric-peer peer node start`
+CID=`docker run -dit -p 7051:7051 $IMAGE peer node start`
 cleanup() {
     echo "Stopping peer.."
     docker kill $CID 2>&1 > /dev/null
@@ -19,4 +32,4 @@ trap cleanup 0
 echo "DONE!"
 
 echo "Running tests..."
-gocov test $PKGS -p 1 -timeout=20m | gocov-xml > report.xml
+gocov test -ldflags "$GO_LDFLAGS" $PKGS -p 1 -timeout=20m | gocov-xml > report.xml