|
| 1 | +package com.morningstar.kafka; |
| 2 | + |
| 3 | +import com.google.common.base.Preconditions; |
| 4 | +import com.google.common.base.Strings; |
| 5 | +import com.google.common.collect.Sets; |
| 6 | +import com.google.gson.annotations.Expose; |
| 7 | + |
| 8 | +import java.util.Optional; |
| 9 | +import java.util.Set; |
| 10 | +import java.util.stream.Collectors; |
| 11 | + |
| 12 | + |
| 13 | +public class KafkaConsumerGroup { |
| 14 | + |
| 15 | + private final long COMPLETE_THRESHOLD = 10; |
| 16 | + |
| 17 | + @Expose private String consumerGroupName; |
| 18 | + @Expose private boolean isActive; |
| 19 | + @Expose private boolean complete; |
| 20 | + @Expose private long mostRecentCommittedMillis; |
| 21 | + @Expose private Status status; |
| 22 | + @Expose private Set<KafkaTopicPartition> topicPartitions; |
| 23 | + |
| 24 | + |
| 25 | + public KafkaConsumerGroup(String consumerGroupName) { |
| 26 | + |
| 27 | + Preconditions.checkArgument(!Strings.isNullOrEmpty(consumerGroupName), "consumerGroupName cannot be NULL or empty."); |
| 28 | + |
| 29 | + this.consumerGroupName = consumerGroupName; |
| 30 | + this.isActive = false; |
| 31 | + this.complete = false; |
| 32 | + this.mostRecentCommittedMillis = -1; |
| 33 | + this.status = Status.OK; |
| 34 | + this.topicPartitions = Sets.newConcurrentHashSet(); |
| 35 | + } |
| 36 | + |
| 37 | + |
| 38 | + public String getConsumerGroupName() { |
| 39 | + return consumerGroupName; |
| 40 | + } |
| 41 | + |
| 42 | + public boolean getComplete() { return complete; } |
| 43 | + |
| 44 | + public long getMaxCommitedMillis() { |
| 45 | + return mostRecentCommittedMillis; |
| 46 | + } |
| 47 | + |
| 48 | + public boolean isActive() { |
| 49 | + return isActive; |
| 50 | + } |
| 51 | + |
| 52 | + public Set<KafkaTopicPartition> getTopicPartitions() { |
| 53 | + |
| 54 | + return topicPartitions; |
| 55 | + } |
| 56 | + |
| 57 | + public Set<String> getTopics() { |
| 58 | + |
| 59 | + return topicPartitions.stream() |
| 60 | + .map(KafkaTopicPartition::getTopicName) |
| 61 | + .collect(Collectors.toSet()); |
| 62 | + } |
| 63 | + |
| 64 | + public synchronized void updateStatus() { |
| 65 | + |
| 66 | + if (!isActive) { |
| 67 | + this.status = Status.ERR; |
| 68 | + return; |
| 69 | + } |
| 70 | + |
| 71 | + Status newStatus = Status.OK; |
| 72 | + |
| 73 | + for (KafkaTopicPartition topicPartition : topicPartitions) { |
| 74 | + |
| 75 | + // Set group status to ERROR if any topicPartition's status is STOP |
| 76 | + if (Status.STOP == topicPartition.getStatus()) { |
| 77 | + newStatus = Status.ERR; |
| 78 | + break; |
| 79 | + } |
| 80 | + |
| 81 | + // Set group status to ERROR if any topicPartition's status is REWIND |
| 82 | + if (Status.REWIND == topicPartition.getStatus()) { |
| 83 | + newStatus = Status.ERR; |
| 84 | + break; |
| 85 | + } |
| 86 | + |
| 87 | + // Set group status to ERROR if any topicPartition's status is STALL |
| 88 | + if (Status.STALL == topicPartition.getStatus()) { |
| 89 | + newStatus = Status.ERR; |
| 90 | + break; |
| 91 | + } |
| 92 | + |
| 93 | + // Set group status to WARN if any topicPartition's status is WARN |
| 94 | + if (Status.WARN == topicPartition.getStatus()) { |
| 95 | + newStatus = Status.WARN; |
| 96 | + break; |
| 97 | + } |
| 98 | + } |
| 99 | + |
| 100 | + this.status = newStatus; |
| 101 | + } |
| 102 | + |
| 103 | + |
| 104 | + private Optional<KafkaTopicPartition> getTopicPartition(String topic, int partitionId) { |
| 105 | + |
| 106 | + //return committedOffsets.keySet().stream().filter(tp -> (tp.getTopicName().equals(topic) && tp.getPartitionId() == partitionId)).findFirst(); |
| 107 | + return topicPartitions.stream() |
| 108 | + .filter(tp -> (tp.getTopicName().equals(topic) && tp.getPartitionId() == partitionId)) |
| 109 | + .findFirst(); |
| 110 | + } |
| 111 | + |
| 112 | + private void upsertTopicPartition(KafkaCommittedOffset kafkaCommittedOffset) { |
| 113 | + |
| 114 | + Preconditions.checkArgument(!Strings.isNullOrEmpty(kafkaCommittedOffset.getTopicName()), "topic cannot be NULL or empty."); |
| 115 | + Preconditions.checkArgument(kafkaCommittedOffset.getPartitionId() >= 0, "partitionId must be greater-than or equal-to zero."); |
| 116 | + |
| 117 | + String incomingTopicName = kafkaCommittedOffset.getTopicName(); |
| 118 | + int incomingPartitionId = kafkaCommittedOffset.getPartitionId(); |
| 119 | + |
| 120 | + Optional<KafkaTopicPartition> existingTopicPartition = getTopicPartition(incomingTopicName, incomingPartitionId); |
| 121 | + |
| 122 | + if (existingTopicPartition.isPresent()) { |
| 123 | + // Append committed offset info to existing set item |
| 124 | + existingTopicPartition.get().addCommittedOffset(kafkaCommittedOffset.getCommittedOffset()); |
| 125 | + } else { |
| 126 | + // Add a new entry to the map |
| 127 | + KafkaTopicPartition newTopicPartition = new KafkaTopicPartition(incomingTopicName, incomingPartitionId); |
| 128 | + newTopicPartition.addCommittedOffset(kafkaCommittedOffset.getCommittedOffset()); |
| 129 | + topicPartitions.add(newTopicPartition); |
| 130 | + } |
| 131 | + } |
| 132 | + |
| 133 | + private void setMostRecentCommittedMillis(long mostRecentCommittedMillis) { |
| 134 | + if (this.mostRecentCommittedMillis < mostRecentCommittedMillis) { |
| 135 | + this.mostRecentCommittedMillis = mostRecentCommittedMillis; |
| 136 | + } |
| 137 | + } |
| 138 | + |
| 139 | + private void updateCompleteFlag() { |
| 140 | + |
| 141 | + this.complete = topicPartitions.stream() |
| 142 | + .noneMatch(f -> f.getCommittedOffsets().size() < COMPLETE_THRESHOLD); |
| 143 | + } |
| 144 | + |
| 145 | + public void addCommittedOffsetInfo(KafkaCommittedOffset kafkaCommittedOffset) { |
| 146 | + |
| 147 | + setMostRecentCommittedMillis(kafkaCommittedOffset.getCommittedOffset().getTimestamp()); |
| 148 | + this.isActive = kafkaCommittedOffset.getGroupIsActive(); |
| 149 | + upsertTopicPartition(kafkaCommittedOffset); |
| 150 | + updateCompleteFlag(); |
| 151 | + } |
| 152 | + |
| 153 | + |
| 154 | + @Override |
| 155 | + public boolean equals(Object o) { |
| 156 | + if (this == o) return true; |
| 157 | + if (o == null || getClass() != o.getClass()) return false; |
| 158 | + |
| 159 | + KafkaConsumerGroup that = (KafkaConsumerGroup) o; |
| 160 | + |
| 161 | + return getConsumerGroupName().equals(that.getConsumerGroupName()); |
| 162 | + } |
| 163 | + |
| 164 | + @Override |
| 165 | + public int hashCode() { |
| 166 | + return getConsumerGroupName().hashCode(); |
| 167 | + } |
| 168 | +} |
0 commit comments