Go Kafka Consumer Group Usage

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

// ConsumerGroup is responsible for dividing up processing of topics and partitions
// over a collection of processes (the members of the consumer group).
type ConsumerGroup interface {
	// Consume joins a cluster of consumers for a given list of topics and
	// starts a blocking ConsumerGroupSession through the ConsumerGroupHandler.
	//
	// The life-cycle of a session is represented by the following steps:
	//
	// 1. The consumers join the group (as explained in https://kafka.apache.org/documentation/#intro_consumers)
	//    and is assigned their "fair share" of partitions, aka 'claims'.
	// 2. Before processing starts, the handler's Setup() hook is called to notify the user
	//    of the claims and allow any necessary preparation or alteration of state.
	// 3. For each of the assigned claims the handler's ConsumeClaim() function is then called
	//    in a separate goroutine which requires it to be thread-safe. Any state must be carefully protected
	//    from concurrent reads/writes.
	// 4. The session will persist until one of the ConsumeClaim() functions exits. This can be either when the
	//    parent context is canceled or when a server-side rebalance cycle is initiated.
	// 5. Once all the ConsumeClaim() loops have exited, the handler's Cleanup() hook is called
	//    to allow the user to perform any final tasks before a rebalance.
	// 6. Finally, marked offsets are committed one last time before claims are released.
	//
	// Please note, that once a rebalance is triggered, sessions must be completed within
	// Config.Consumer.Group.Rebalance.Timeout. This means that ConsumeClaim() functions must exit
	// as quickly as possible to allow time for Cleanup() and the final offset commit. If the timeout
	// is exceeded, the consumer will be removed from the group by Kafka, which will cause offset
	// commit failures.
	// This method should be called inside an infinite loop, when a
	// server-side rebalance happens, the consumer session will need to be
	// recreated to get the new claims.
	Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error

	// Errors returns a read channel of errors that occurred during the consumer life-cycle.
	// By default, errors are logged and not returned over this channel.
	// If you want to implement any custom error handling, set your config's
	// Consumer.Return.Errors setting to true, and read from this channel.
	Errors() <-chan error

	// Close stops the ConsumerGroup and detaches any running sessions. It is required to call
	// this function before the object passes out of scope, as it will otherwise leak memory.
	Close() error
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100

package kafka

import (
	"context"
	"log"
	"time"

	"github.com/Shopify/sarama"
	"github.com/pkg/errors"
)

type ConsumerGroup interface {
	Consume(topics []string, handler sarama.ConsumerGroupHandler)
}

type consumerGroup struct {
	ctx           context.Context
	consumerGroup sarama.ConsumerGroup
	groupId       string
	brokers       []string
	done          chan struct{}
	cancel        context.CancelFunc
}

func NewConsumerGroup(ctx context.Context, groupId string, brokers []string) ConsumerGroup {
	config := sarama.NewConfig()
	config.Version = sarama.V2_4_1_0
	config.Consumer.Return.Errors = true
	config.Net.ReadTimeout = time.Second * 75                  // default 30s
	config.Consumer.Group.Rebalance.Timeout = time.Second * 60 // default 60s
	config.Consumer.Group.Session.Timeout = time.Second * 45   // default 10s
	config.Consumer.Group.Heartbeat.Interval = time.Second * 5 // default 3s, must <= 1/3 * Group.Session.Timeout
	config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategySticky
	config.Consumer.MaxWaitTime = time.Millisecond * 250 // default 250ms
	config.Consumer.Retry.Backoff = time.Second * 2      // default 2s

	cg, err := sarama.NewConsumerGroup(brokers, groupId, config)
	if err != nil {
		panic(errors.Wrap(err, "consumerGroup create err"))
	}

	go func() {
		for err := range cg.Errors() {
			log.Printf("consumerGroup, group_id:%v, err:%v", groupId, err)
		}
	}()

	ctx, cancel := context.WithCancel(ctx)
	return &consumerGroup{
		ctx:           ctx,
		consumerGroup: cg,
		groupId:       groupId,
		brokers:       brokers,
		done:          make(chan struct{}),
		cancel:        cancel,
	}
}
func (c *consumerGroup) Consume(topics []string, handler sarama.ConsumerGroupHandler) {
	go func() {
		defer func(consumerGroup sarama.ConsumerGroup) {
			err := consumerGroup.Close()
			if err != nil {

			}
		}(c.consumerGroup)

	Loop:
		for {
			if err := c.consumerGroup.Consume(c.ctx, topics, handler); err != nil {
				switch err {
				case sarama.ErrClosedClient, sarama.ErrClosedConsumerGroup:
					log.Printf("consumerGroup.Consume() topic:%v, shutdown with err:%v", topics, err)
					break Loop
				case sarama.ErrOutOfBrokers:
					log.Printf("consumerGroup.Consume() topic:%v, err:%v", topics, sarama.ErrOutOfBrokers)
				default:
					log.Printf("consumerGroup.Consume() topic:%v, err:%v", topics, err)
				}
				time.Sleep(time.Second)
			}
			select {
			case <-c.done:
				log.Printf("session exit after close signal")
				break Loop
			default:
				log.Printf("server-side reBalance happens, consumer session exit, will recreated")
			}
		}
	}()
}

func (c *consumerGroup) Close() {
	go func() { c.done <- struct{}{} }()
	c.cancel()
	if err := c.consumerGroup.Close(); err != nil {
		log.Printf("consumerGroup close error, %v", err)
	} else {
		log.Printf("consumerGroup exit gracefully")
	}
}