Skip to content

Commit

Permalink
[FAB-5060] Update DB schema
Browse files Browse the repository at this point in the history 
The database schema was updated to increase
character limit on certain columns. One reason
for this change is to allow a larger number
of attributes to be registered for an identity.

Change-Id: Ia220adab7039b59ac793703005dc82d5a67450f1
Signed-off-by: Saad Karim <[email protected]>
  • Loading branch information
Saad Karim committed Sep 26, 2017
1 parent baea861 commit 57c0cf3
Show file tree
Hide file tree
Showing 5 changed files with 332 additions and 28 deletions.
6 changes: 6 additions & 0 deletions lib/ca.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,12 @@ func (ca *CA) initDB() error {
return errors.Errorf("Invalid db.type in config file: '%s'; must be 'sqlite3', 'postgres', or 'mysql'", db.Type)
}

// Update the database to use the latest schema
err = dbutil.UpdateSchema(ca.db)
if err != nil {
return errors.Wrap(err, "Failed to update schema")
}

// Set the certificate DB accessor
ca.certDBAccessor = NewCertDBAccessor(ca.db)

Expand Down
17 changes: 14 additions & 3 deletions lib/dbaccessor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,11 +276,22 @@ func (d *Accessor) InsertAffiliation(name string, prekey string) error {
if err != nil {
return err
}
dbType := d.db.DriverName()
// InnoDB store engine for MySQL does not allow more than 767 bytes
// in a 'UNIQUE' column. To work around this, the UNIQUE constraint was removed
// from the 'name' column in the affiliations table for MySQL to allow for up to 1024
// characters to be stored. In doing this, a check is needed on MySQL to check
// if the affiliation exists before adding it to prevent duplicate entries.
if dbType == "mysql" {
aff, _ := d.GetAffiliation(name)
if aff != nil {
log.Debugf("Affiliation '%s' already exists", name)
return nil
}
}
_, err = d.db.Exec(d.db.Rebind(insertAffiliation), name, prekey)
if err != nil {
errStr := err.Error()
dbType := d.db.DriverName()
if (!strings.Contains(errStr, "Duplicate entry") && dbType == "mysql") || (!strings.Contains(err.Error(), "duplicate key value") && dbType == "postgres") {
if (!strings.Contains(err.Error(), "UNIQUE constraint failed") && dbType == "sqlite3") || (!strings.Contains(err.Error(), "duplicate key value") && dbType == "postgres") {
return err
}
log.Debugf("Affiliation '%s' already exists", name)
Expand Down
101 changes: 92 additions & 9 deletions lib/dbutil/dbutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,15 +69,15 @@ func createSQLiteDBTables(datasource string) error {
defer db.Close()

log.Debug("Creating users table if it does not exist")
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS users (id VARCHAR(64), token bytea, type VARCHAR(64), affiliation VARCHAR(64), attributes VARCHAR(256), state INTEGER, max_enrollments INTEGER)"); err != nil {
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS users (id VARCHAR(255), token bytea, type VARCHAR(256), affiliation VARCHAR(1024), attributes TEXT, state INTEGER, max_enrollments INTEGER)"); err != nil {
return errors.Wrap(err, "Error creating users table")
}
log.Debug("Creating affiliations table if it does not exist")
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS affiliations (name VARCHAR(64) NOT NULL UNIQUE, prekey VARCHAR(64))"); err != nil {
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS affiliations (name VARCHAR(1024) NOT NULL UNIQUE, prekey VARCHAR(1024))"); err != nil {
return errors.Wrap(err, "Error creating affiliations table")
}
log.Debug("Creating certificates table if it does not exist")
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS certificates (id VARCHAR(64), serial_number blob NOT NULL, authority_key_identifier blob NOT NULL, ca_label blob, status blob NOT NULL, reason int, expiry timestamp, revoked_at timestamp, pem blob NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier))"); err != nil {
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS certificates (id VARCHAR(255), serial_number blob NOT NULL, authority_key_identifier blob NOT NULL, ca_label blob, status blob NOT NULL, reason int, expiry timestamp, revoked_at timestamp, pem blob NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier))"); err != nil {
return errors.Wrap(err, "Error creating certificates table")
}

Expand Down Expand Up @@ -155,15 +155,15 @@ func createPostgresDatabase(dbName string, db *sqlx.DB) error {
// createPostgresDB creates postgres database
func createPostgresTables(dbName string, db *sqlx.DB) error {
log.Debug("Creating users table if it does not exist")
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS users (id VARCHAR(64), token bytea, type VARCHAR(64), affiliation VARCHAR(64), attributes VARCHAR(256), state INTEGER, max_enrollments INTEGER)"); err != nil {
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS users (id VARCHAR(255), token bytea, type VARCHAR(256), affiliation VARCHAR(1024), attributes TEXT, state INTEGER, max_enrollments INTEGER)"); err != nil {
return errors.Wrap(err, "Error creating users table")
}
log.Debug("Creating affiliations table if it does not exist")
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS affiliations (name VARCHAR(64) NOT NULL UNIQUE, prekey VARCHAR(64))"); err != nil {
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS affiliations (name VARCHAR(1024) NOT NULL UNIQUE, prekey VARCHAR(1024))"); err != nil {
return errors.Wrap(err, "Error creating affiliations table")
}
log.Debug("Creating certificates table if it does not exist")
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS certificates (id VARCHAR(64), serial_number bytea NOT NULL, authority_key_identifier bytea NOT NULL, ca_label bytea, status bytea NOT NULL, reason int, expiry timestamp, revoked_at timestamp, pem bytea NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier))"); err != nil {
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS certificates (id VARCHAR(255), serial_number bytea NOT NULL, authority_key_identifier bytea NOT NULL, ca_label bytea, status bytea NOT NULL, reason int, expiry timestamp, revoked_at timestamp, pem bytea NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier))"); err != nil {
return errors.Wrap(err, "Error creating certificates table")
}
return nil
Expand Down Expand Up @@ -231,17 +231,24 @@ func createMySQLDatabase(dbName string, db *sqlx.DB) error {

func createMySQLTables(dbName string, db *sqlx.DB) error {
log.Debug("Creating users table if it doesn't exist")
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS users (id VARCHAR(64) NOT NULL, token blob, type VARCHAR(64), affiliation VARCHAR(64), attributes VARCHAR(256), state INTEGER, max_enrollments INTEGER, PRIMARY KEY (id)) DEFAULT CHARSET=utf8 COLLATE utf8_bin"); err != nil {
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS users (id VARCHAR(255) NOT NULL, token blob, type VARCHAR(256), affiliation VARCHAR(1024), attributes TEXT, state INTEGER, max_enrollments INTEGER, PRIMARY KEY (id)) DEFAULT CHARSET=utf8 COLLATE utf8_bin"); err != nil {
return errors.Wrap(err, "Error creating users table")
}

log.Debug("Creating affiliations table if it doesn't exist")
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS affiliations (name VARCHAR(64) NOT NULL UNIQUE, prekey VARCHAR(64))"); err != nil {
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS affiliations (name VARCHAR(1024) NOT NULL, prekey VARCHAR(1024))"); err != nil {
return errors.Wrap(err, "Error creating affiliations table")
}

log.Debug("Creating index on 'name' in the affiliations table")
if _, err := db.Exec("CREATE INDEX name_index on affiliations (name)"); err != nil {
if !strings.Contains(err.Error(), "Error 1061") { // Error 1061: Duplicate key name, index already exists
return errors.Wrap(err, "Error creating index on affiliations table")
}
}

log.Debug("Creating certificates table if it doesn't exist")
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS certificates (id VARCHAR(64), serial_number varbinary(128) NOT NULL, authority_key_identifier varbinary(128) NOT NULL, ca_label varbinary(128), status varbinary(128) NOT NULL, reason int, expiry timestamp DEFAULT 0, revoked_at timestamp DEFAULT 0, pem varbinary(4096) NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier)) DEFAULT CHARSET=utf8 COLLATE utf8_bin"); err != nil {
if _, err := db.Exec("CREATE TABLE IF NOT EXISTS certificates (id VARCHAR(255), serial_number varbinary(128) NOT NULL, authority_key_identifier varbinary(128) NOT NULL, ca_label varbinary(128), status varbinary(128) NOT NULL, reason int, expiry timestamp DEFAULT 0, revoked_at timestamp DEFAULT 0, pem varbinary(4096) NOT NULL, PRIMARY KEY(serial_number, authority_key_identifier)) DEFAULT CHARSET=utf8 COLLATE utf8_bin"); err != nil {
return errors.Wrap(err, "Error creating certificates table")
}

Expand Down Expand Up @@ -288,3 +295,79 @@ func MaskDBCred(connStr, dbType string) string {
}
return connStr
}

// UpdateSchema updates the database tables to use the latest schema
func UpdateSchema(db *sqlx.DB) error {
log.Debug("Checking database schema...")

switch db.DriverName() {
case "sqlite3": // SQLite does not support altering columns. However, data types in SQLite are not rigid and thus no action is really required
return nil
case "mysql":
return updateMySQLSchema(db)
case "postgres":
return updatePostgresSchema(db)
default:
return errors.Errorf("Unsupported database type: %s", db.DriverName())
}
}

func updateMySQLSchema(db *sqlx.DB) error {
log.Debug("Update MySQL schema if using outdated schema")
var err error

_, err = db.Exec("ALTER TABLE users MODIFY id VARCHAR(255), MODIFY type VARCHAR(256), MODIFY affiliation VARCHAR(256)")
if err != nil {
return err
}
_, err = db.Exec("ALTER TABLE users MODIFY attributes TEXT")
if err != nil {
return err
}
_, err = db.Exec("ALTER TABLE affiliations DROP INDEX name;")
if err != nil {
if !strings.Contains(err.Error(), "Error 1091") { // Indicates that index not found
return err
}
}
_, err = db.Exec("ALTER TABLE affiliations MODIFY name VARCHAR(1024), MODIFY prekey VARCHAR(1024)")
if err != nil {
return err
}
_, err = db.Exec("ALTER TABLE affiliations ADD INDEX name_index (name)")
if err != nil {
if !strings.Contains(err.Error(), "Error 1061") { // Error 1061: Duplicate key name, index already exists
return err
}
}
_, err = db.Exec("ALTER TABLE certificates MODIFY id VARCHAR(255)")
if err != nil {
return err
}

return nil
}

func updatePostgresSchema(db *sqlx.DB) error {
log.Debug("Update Postgres schema if using outdated schema")
var err error

_, err = db.Exec("ALTER TABLE users ALTER COLUMN id TYPE VARCHAR(255), ALTER COLUMN type TYPE VARCHAR(256), ALTER COLUMN affiliation TYPE VARCHAR(256)")
if err != nil {
return err
}
_, err = db.Exec("ALTER TABLE users ALTER COLUMN attributes TYPE TEXT")
if err != nil {
return err
}
_, err = db.Exec("ALTER TABLE affiliations ALTER COLUMN name TYPE VARCHAR(1024), ALTER COLUMN prekey TYPE VARCHAR(1024)")
if err != nil {
return err
}
_, err = db.Exec("ALTER TABLE certificates ALTER COLUMN id TYPE VARCHAR(255)")
if err != nil {
return err
}

return nil
}
55 changes: 39 additions & 16 deletions scripts/fvt/db_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ function cleanup {
rm $SERVERLOG
}

function killserver {
echo "killing server $1"
kill -9 $1
}

function existingIdentity {
grep "Identity '$1' already registered, loaded identity" $2 &> /dev/null
if [ $? != 0 ]; then
Expand Down Expand Up @@ -161,12 +166,16 @@ mysql --host=localhost --user=root --password=mysql -e "drop database $DBNAME;"
mysql --host=localhost --user=root --password=mysql --database=$DBNAME -e "CREATE TABLE users (id VARCHAR(64) NOT NULL, token blob, type VARCHAR(64), affiliation VARCHAR(64), attributes VARCHAR(256), state INTEGER, max_enrollments INTEGER, PRIMARY KEY (id)) DEFAULT CHARSET=utf8 COLLATE utf8_bin;" &> /dev/null

# Starting server first time with one bootstrap user
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $MYSQLSERVERCONFIG 2>&1 > $SERVERLOG
$SCRIPTDIR/fabric-ca_setup.sh -K
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $MYSQLSERVERCONFIG 2>&1 | tee $SERVERLOG &
pollServer fabric-ca-server 127.0.0.1 17054 10 start
pid=$(pidof fabric-ca-server)
killserver $pid

# Starting server second time with a second bootstrap user
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $MYSQLSERVERCONFIG2 2>&1 >> $SERVERLOG
$SCRIPTDIR/fabric-ca_setup.sh -K
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $MYSQLSERVERCONFIG2 2>&1 | tee $SERVERLOG &
pollServer fabric-ca-server 127.0.0.1 17054 10 start
pid=$(pidof fabric-ca-server)
killserver $pid

existingIdentity "a" $SERVERLOG # Check to see that appropriate error message was seen for an already registered user
checkIdentity "c" $SERVERLOG # Check to see that a new identity properly got registered
Expand All @@ -182,8 +191,10 @@ echo "Test2: Fabric-ca should create the tables and bootstrap"
echo "Dropping and creating an empty '$DBNAME' database"
mysql --host=localhost --user=root --password=mysql -e "drop database fabric_ca;" -e "create database fabric_ca;" &> /dev/null

$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $MYSQLSERVERCONFIG2 2>&1 > $SERVERLOG
$SCRIPTDIR/fabric-ca_setup.sh -K
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $MYSQLSERVERCONFIG2 2>&1 | tee $SERVERLOG &
pollServer fabric-ca-server 127.0.0.1 17054 10 start
pid=$(pidof fabric-ca-server)
killserver $pid

checkIdentity "a" $SERVERLOG # Check to see that a new identity properly got registered
checkIdentity "c" $SERVERLOG # Check to see that a new identity properly got registered
Expand All @@ -196,8 +207,10 @@ echo "Test3: Fabric-ca should create the database and tables, and bootstrap"
echo "Dropping '$DBNAME' database"
mysql --host=localhost --user=root --password=mysql -e "drop database fabric_ca;" &> /dev/null

$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $MYSQLSERVERCONFIG2 2>&1 > $SERVERLOG
$SCRIPTDIR/fabric-ca_setup.sh -K
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $MYSQLSERVERCONFIG2 2>&1 | tee $SERVERLOG &
pollServer fabric-ca-server 127.0.0.1 17054 10 start
pid=$(pidof fabric-ca-server)
killserver $pid

checkIdentity "a" $SERVERLOG # Check to see that a new identity properly got registered
checkIdentity "c" $SERVERLOG # Check to see that a new identity properly got registered
Expand All @@ -216,12 +229,18 @@ psql -c "create database $DBNAME"
psql -d fabric_ca -c "CREATE TABLE users (id VARCHAR(64), token bytea, type VARCHAR(64), affiliation VARCHAR(64), attributes VARCHAR(256), state INTEGER, max_enrollments INTEGER)"

# Starting server first time with one bootstrap user
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $PGSQLSERVERCONFIG 2>&1 > $SERVERLOG
$SCRIPTDIR/fabric-ca_setup.sh -K
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $PGSQLSERVERCONFIG 2>&1 | tee $SERVERLOG &

pollServer fabric-ca-server 127.0.0.1 17054 10 start
pid=$(pidof fabric-ca-server)
killserver $pid

sleep 1
# Starting server second time with a second bootstrap user
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $PGSQLSERVERCONFIG2 2>&1 >> $SERVERLOG
$SCRIPTDIR/fabric-ca_setup.sh -K
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $PGSQLSERVERCONFIG2 2>&1 | tee $SERVERLOG &
pollServer fabric-ca-server 127.0.0.1 17054 10 start
pid=$(pidof fabric-ca-server)
killserver $pid

existingIdentity "a" $SERVERLOG # Check to see that appropriate error message was seen for an already registered user
checkIdentity "c" $SERVERLOG # Check to see that a new identity properly got registered
Expand All @@ -237,8 +256,10 @@ echo "Test2: Fabric-ca should create the tables and bootstrap"
psql -c "drop database $DBNAME"
psql -c "create database $DBNAME"

$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $PGSQLSERVERCONFIG2 2>&1 > $SERVERLOG
$SCRIPTDIR/fabric-ca_setup.sh -K
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $PGSQLSERVERCONFIG2 2>&1 | tee $SERVERLOG &
pollServer fabric-ca-server 127.0.0.1 17054 10 start
pid=$(pidof fabric-ca-server)
killserver $pid

checkIdentity "a" $SERVERLOG # Check to see that a new identity properly got registered
checkIdentity "c" $SERVERLOG # Check to see that a new identity properly got registered
Expand All @@ -250,9 +271,11 @@ echo "Test3: Database does not exist"
echo "Test3: Fabric-ca should create the database and tables, and bootstrap"
psql -c "drop database $DBNAME"

$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $PGSQLSERVERCONFIG2 2>&1 > $SERVERLOG
$SCRIPTDIR/fabric-ca_setup.sh -S -X -g $PGSQLSERVERCONFIG2 2>&1 | tee $SERVERLOG &
sleep 6 # Need to allow for Postgres to complete database and table creation
$SCRIPTDIR/fabric-ca_setup.sh -K
pollServer fabric-ca-server 127.0.0.1 17054 10 start
pid=$(pidof fabric-ca-server)
killserver $pid

checkIdentity "a" $SERVERLOG # Check to see that a new identity properly got registered
checkIdentity "c" $SERVERLOG # Check to see that a new identity properly got registered
Expand Down
Loading

0 comments on commit 57c0cf3

Please sign in to comment.