Skip to content

Commit d69cd02

Browse files
author
Luis Sanchez
committed
[FAB-1165] Use configured PartitionID
Subtask of FAB-890 When publishing to Kafka we have been relying on the topic only containing one partition (num.partitions = 1), and the default PartitionID (0) in the config lining up for a valid combination. This commit configures our producer PartitionID configured. Change-Id: Ibba476425a15aff9ff447afdfacbf4b77a41ce0c Signed-off-by: Luis Sanchez <[email protected]>
1 parent 7420a61 commit d69cd02

File tree

2 files changed

+105
-0
lines changed

2 files changed

+105
-0
lines changed

orderer/kafka/util.go

+21
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ const (
3030
func newBrokerConfig(conf *config.TopLevel) *sarama.Config {
3131
brokerConfig := sarama.NewConfig()
3232
brokerConfig.Version = conf.Kafka.Version
33+
brokerConfig.Producer.Partitioner = newStaticPartitioner(conf.Kafka.PartitionID)
3334
return brokerConfig
3435
}
3536

@@ -50,3 +51,23 @@ func newOffsetReq(conf *config.TopLevel, seek int64) *sarama.OffsetRequest {
5051
req.AddBlock(conf.Kafka.Topic, conf.Kafka.PartitionID, seek, 1)
5152
return req
5253
}
54+
55+
// newStaticPartitioner returns a PartitionerConstructor that returns a Partitioner
56+
// that always chooses the specified partition.
57+
func newStaticPartitioner(partition int32) sarama.PartitionerConstructor {
58+
return func(topic string) sarama.Partitioner {
59+
return &staticPartitioner{partition}
60+
}
61+
}
62+
63+
type staticPartitioner struct {
64+
partitionID int32
65+
}
66+
67+
func (p *staticPartitioner) Partition(message *sarama.ProducerMessage, numPartitions int32) (int32, error) {
68+
return p.partitionID, nil
69+
}
70+
71+
func (p *staticPartitioner) RequiresConsistency() bool {
72+
return true
73+
}

orderer/kafka/util_test.go

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
Copyright IBM Corp. 2016 All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package kafka
18+
19+
import (
20+
"testing"
21+
22+
"github.com/Shopify/sarama"
23+
)
24+
25+
func TestStaticPartitioner(t *testing.T) {
26+
27+
var partition int32 = 3
28+
var numberOfPartitions int32 = 6
29+
30+
partitionerConstructor := newStaticPartitioner(partition)
31+
partitioner := partitionerConstructor(testConf.Kafka.Topic)
32+
33+
for i := 0; i < 10; i++ {
34+
assignedPartition, err := partitioner.Partition(new(sarama.ProducerMessage), numberOfPartitions)
35+
if err != nil {
36+
t.Fatal(err)
37+
}
38+
if assignedPartition != partition {
39+
t.Fatalf("Expected: %v. Actual: %v", partition, assignedPartition)
40+
}
41+
}
42+
}
43+
44+
func TestNewBrokerConfig(t *testing.T) {
45+
46+
topic := testConf.Kafka.Topic
47+
48+
// use a partition id that is not the 'default' 0
49+
var partition int32 = 2
50+
originalPartitionID := testConf.Kafka.PartitionID
51+
defer func() {
52+
testConf.Kafka.PartitionID = originalPartitionID
53+
}()
54+
testConf.Kafka.PartitionID = partition
55+
56+
// setup a mock broker that reports that it has 3 partitions for the topic
57+
broker := sarama.NewMockBroker(t, 1000)
58+
broker.SetHandlerByMap(map[string]sarama.MockResponse{
59+
"MetadataRequest": sarama.NewMockMetadataResponse(t).
60+
SetBroker(broker.Addr(), broker.BrokerID()).
61+
SetLeader(topic, 0, broker.BrokerID()).
62+
SetLeader(topic, 1, broker.BrokerID()).
63+
SetLeader(topic, 2, broker.BrokerID()),
64+
"ProduceRequest": sarama.NewMockProduceResponse(t),
65+
})
66+
67+
config := newBrokerConfig(testConf)
68+
producer, err := sarama.NewSyncProducer([]string{broker.Addr()}, config)
69+
if err != nil {
70+
t.Fatal(err)
71+
}
72+
73+
for i := 0; i < 10; i++ {
74+
assignedPartition, _, err := producer.SendMessage(&sarama.ProducerMessage{Topic: topic})
75+
if err != nil {
76+
t.Fatal(err)
77+
}
78+
if assignedPartition != partition {
79+
t.Fatalf("Expected: %v. Actual: %v", partition, assignedPartition)
80+
}
81+
}
82+
producer.Close()
83+
broker.Close()
84+
}

0 commit comments

Comments
 (0)