Skip to content

Commit d1ebe22

Browse files
garyrussellartembilan
garyrussell
authored and
artembilan
committed
GH-1032: Consumer Batching Phase 2 @RabbitListener
Resolves #1032 Add basic consumer batching support to the `@RabbitListener` infrastructure. * * Polishing - simplify batch configuration in container factories * Add a test to ensure producer debatching works for DMLC too * * Add more tests * * Fix javadoc; add one more test * * Fix type in test name
1 parent 845a7e0 commit d1ebe22

14 files changed

+766
-40
lines changed

‎spring-rabbit/src/main/java/org/springframework/amqp/rabbit/annotation/RabbitListenerAnnotationBeanPostProcessor.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,10 @@ public void setCharset(Charset charset) {
236236
this.charset = charset;
237237
}
238238

239+
MessageHandlerMethodFactory getMessageHandlerMethodFactory() {
240+
return this.messageHandlerMethodFactory;
241+
}
242+
239243
@Override
240244
public void afterSingletonsInstantiated() {
241245
this.registrar.setBeanFactory(this.beanFactory);

‎spring-rabbit/src/main/java/org/springframework/amqp/rabbit/config/AbstractRabbitListenerContainerFactory.java

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,8 @@ public abstract class AbstractRabbitListenerContainerFactory<C extends AbstractM
125125

126126
private BatchingStrategy batchingStrategy;
127127

128+
private Boolean deBatchingEnabled;
129+
128130
/**
129131
* @param connectionFactory The connection factory.
130132
* @see AbstractMessageListenerContainer#setConnectionFactory(ConnectionFactory)
@@ -375,6 +377,17 @@ public void setBatchingStrategy(BatchingStrategy batchingStrategy) {
375377
this.batchingStrategy = batchingStrategy;
376378
}
377379

380+
/**
381+
* Determine whether or not the container should de-batch batched
382+
* messages (true) or call the listener with the batch (false). Default: true.
383+
* @param deBatchingEnabled whether or not to disable de-batching of messages.
384+
* @since 2.2
385+
* @see AbstractMessageListenerContainer#setDeBatchingEnabled(boolean)
386+
*/
387+
public void setDeBatchingEnabled(final Boolean deBatchingEnabled) {
388+
this.deBatchingEnabled = deBatchingEnabled;
389+
}
390+
378391
@Override
379392
public C createListenerContainer(RabbitListenerEndpoint endpoint) {
380393
C instance = createContainerInstance();
@@ -404,8 +417,12 @@ public C createListenerContainer(RabbitListenerEndpoint endpoint) {
404417
.acceptIfNotNull(this.applicationEventPublisher, instance::setApplicationEventPublisher)
405418
.acceptIfNotNull(this.autoStartup, instance::setAutoStartup)
406419
.acceptIfNotNull(this.phase, instance::setPhase)
407-
.acceptIfNotNull(this.afterReceivePostProcessors, instance::setAfterReceivePostProcessors);
408-
instance.setDeBatchingEnabled(!this.batchListener);
420+
.acceptIfNotNull(this.afterReceivePostProcessors, instance::setAfterReceivePostProcessors)
421+
.acceptIfNotNull(this.deBatchingEnabled, instance::setDeBatchingEnabled);
422+
if (this.batchListener && this.deBatchingEnabled == null) {
423+
// turn off container debatching by default for batch listeners
424+
instance.setDeBatchingEnabled(false);
425+
}
409426
if (endpoint != null) { // endpoint settings overriding default factory settings
410427
javaUtils
411428
.acceptIfNotNull(endpoint.getAutoStartup(), instance::setAutoStartup)

‎spring-rabbit/src/main/java/org/springframework/amqp/rabbit/config/SimpleRabbitListenerContainerFactory.java

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
public class SimpleRabbitListenerContainerFactory
3939
extends AbstractRabbitListenerContainerFactory<SimpleMessageListenerContainer> {
4040

41-
private Integer txSize;
41+
private Integer batchSize;
4242

4343
private Integer concurrentConsumers;
4444

@@ -54,14 +54,25 @@ public class SimpleRabbitListenerContainerFactory
5454

5555
private Long receiveTimeout;
5656

57-
private Boolean deBatchingEnabled;
57+
private Boolean consumerBatchEnabled;
5858

5959
/**
6060
* @param txSize the transaction size.
61-
* @see SimpleMessageListenerContainer#setTxSize
61+
* @see SimpleMessageListenerContainer#setBatchSize
62+
* @deprecated in favor of {@link #setBatchSize(Integer)}
6263
*/
64+
@Deprecated
6365
public void setTxSize(Integer txSize) {
64-
this.txSize = txSize;
66+
setBatchSize(txSize);
67+
}
68+
69+
/**
70+
* @param batchSize the batch size.
71+
* @since 2.2
72+
* @see SimpleMessageListenerContainer#setBatchSize
73+
*/
74+
public void setBatchSize(Integer batchSize) {
75+
this.batchSize = batchSize;
6576
}
6677

6778
/**
@@ -121,13 +132,14 @@ public void setReceiveTimeout(Long receiveTimeout) {
121132
}
122133

123134
/**
124-
* Determine whether or not the container should de-batch batched
125-
* messages (true) or call the listener with the batch (false). Default: true.
126-
* @param deBatchingEnabled whether or not to disable de-batching of messages.
127-
* @see SimpleMessageListenerContainer#setDeBatchingEnabled(boolean)
135+
* Set to true to present a list of messages based on the {@link #setBatchSize(Integer)},
136+
* if the listener supports it.
137+
* @param consumerBatchEnabled true to create message batches in the container.
138+
* @since 2.2
139+
* @see #setBatchSize(Integer)
128140
*/
129-
public void setDeBatchingEnabled(final Boolean deBatchingEnabled) {
130-
this.deBatchingEnabled = deBatchingEnabled;
141+
public void setConsumerBatchEnabled(boolean consumerBatchEnabled) {
142+
this.consumerBatchEnabled = consumerBatchEnabled;
131143
}
132144

133145
@Override
@@ -140,7 +152,7 @@ protected void initializeContainer(SimpleMessageListenerContainer instance, Rabb
140152
super.initializeContainer(instance, endpoint);
141153

142154
JavaUtils javaUtils = JavaUtils.INSTANCE
143-
.acceptIfNotNull(this.txSize, instance::setBatchSize);
155+
.acceptIfNotNull(this.batchSize, instance::setBatchSize);
144156
String concurrency = null;
145157
if (endpoint != null) {
146158
concurrency = endpoint.getConcurrency();
@@ -149,14 +161,22 @@ protected void initializeContainer(SimpleMessageListenerContainer instance, Rabb
149161
javaUtils
150162
.acceptIfCondition(concurrency == null && this.concurrentConsumers != null, this.concurrentConsumers,
151163
instance::setConcurrentConsumers)
152-
.acceptIfCondition((concurrency == null || !(concurrency.contains("-"))) && this.maxConcurrentConsumers != null,
164+
.acceptIfCondition((concurrency == null || !(concurrency.contains("-")))
165+
&& this.maxConcurrentConsumers != null,
153166
this.maxConcurrentConsumers, instance::setMaxConcurrentConsumers)
154167
.acceptIfNotNull(this.startConsumerMinInterval, instance::setStartConsumerMinInterval)
155168
.acceptIfNotNull(this.stopConsumerMinInterval, instance::setStopConsumerMinInterval)
156169
.acceptIfNotNull(this.consecutiveActiveTrigger, instance::setConsecutiveActiveTrigger)
157170
.acceptIfNotNull(this.consecutiveIdleTrigger, instance::setConsecutiveIdleTrigger)
158-
.acceptIfNotNull(this.receiveTimeout, instance::setReceiveTimeout)
159-
.acceptIfNotNull(this.deBatchingEnabled, instance::setDeBatchingEnabled);
171+
.acceptIfNotNull(this.receiveTimeout, instance::setReceiveTimeout);
172+
if (Boolean.TRUE.equals(this.consumerBatchEnabled)) {
173+
instance.setConsumerBatchEnabled(true);
174+
/*
175+
* 'batchListener=true' turns off container debatching by default, it must be
176+
* true when consumer batching is enabled.
177+
*/
178+
instance.setDeBatchingEnabled(true);
179+
}
160180
}
161181

162182
}

‎spring-rabbit/src/main/java/org/springframework/amqp/rabbit/listener/AbstractMessageListenerContainer.java

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1495,7 +1495,7 @@ protected void doInvokeListener(ChannelAwareMessageListener listener, Channel ch
14951495
}
14961496
}
14971497
catch (Exception e) {
1498-
throw wrapToListenerExecutionFailedExceptionIfNeeded(e, message);
1498+
throw wrapToListenerExecutionFailedExceptionIfNeeded(e, data);
14991499
}
15001500
}
15011501
finally {
@@ -1551,7 +1551,7 @@ protected void doInvokeListener(MessageListener listener, Object data) {
15511551
}
15521552
}
15531553
catch (Exception e) {
1554-
throw wrapToListenerExecutionFailedExceptionIfNeeded(e, message);
1554+
throw wrapToListenerExecutionFailedExceptionIfNeeded(e, data);
15551555
}
15561556
}
15571557

@@ -1590,16 +1590,23 @@ protected void handleListenerException(Throwable ex) {
15901590

15911591
/**
15921592
* @param e The Exception.
1593-
* @param message The failed message.
1593+
* @param data The failed message.
15941594
* @return If 'e' is of type {@link ListenerExecutionFailedException} - return 'e' as it is, otherwise wrap it to
15951595
* {@link ListenerExecutionFailedException} and return.
15961596
*/
1597+
@SuppressWarnings("unchecked")
15971598
protected ListenerExecutionFailedException wrapToListenerExecutionFailedExceptionIfNeeded(Exception e,
1598-
Message message) {
1599+
Object data) {
15991600

16001601
if (!(e instanceof ListenerExecutionFailedException)) {
16011602
// Wrap exception to ListenerExecutionFailedException.
1602-
return new ListenerExecutionFailedException("Listener threw exception", e, message);
1603+
if (data instanceof List) {
1604+
return new ListenerExecutionFailedException("Listener threw exception", e,
1605+
((List<Message>) data).toArray(new Message[0]));
1606+
}
1607+
else {
1608+
return new ListenerExecutionFailedException("Listener threw exception", e, (Message) data);
1609+
}
16031610
}
16041611
return (ListenerExecutionFailedException) e;
16051612
}

‎spring-rabbit/src/main/java/org/springframework/amqp/rabbit/listener/SimpleMessageListenerContainer.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,6 +518,8 @@ protected void doInitialize() {
518518
Assert.state(!this.consumerBatchEnabled || getMessageListener() instanceof BatchMessageListener
519519
|| getMessageListener() instanceof ChannelAwareBatchMessagelistener,
520520
"When setting 'consumerBatchEnabled' to true, the listener must support batching");
521+
Assert.state(!this.consumerBatchEnabled || isDeBatchingEnabled(),
522+
"When setting 'consumerBatchEnabled' to true, 'deBatchingEnabled' must also be true");
521523
}
522524

523525
@ManagedMetric(metricType = MetricType.GAUGE)

‎spring-rabbit/src/main/java/org/springframework/amqp/rabbit/listener/adapter/BatchMessagingMessageListenerAdapter.java

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,20 +22,25 @@
2222

2323
import org.springframework.amqp.rabbit.batch.BatchingStrategy;
2424
import org.springframework.amqp.rabbit.batch.SimpleBatchingStrategy;
25+
import org.springframework.amqp.rabbit.listener.api.ChannelAwareBatchMessagelistener;
2526
import org.springframework.amqp.rabbit.listener.api.RabbitListenerErrorHandler;
27+
import org.springframework.amqp.rabbit.support.RabbitExceptionTranslator;
2628
import org.springframework.lang.Nullable;
2729
import org.springframework.messaging.Message;
2830
import org.springframework.messaging.support.GenericMessage;
2931
import org.springframework.messaging.support.MessageBuilder;
3032

33+
import com.rabbitmq.client.Channel;
34+
3135
/**
3236
* A listener adapter for batch listeners.
3337
*
3438
* @author Gary Russell
3539
* @since 2.2
3640
*
3741
*/
38-
public class BatchMessagingMessageListenerAdapter extends MessagingMessageListenerAdapter {
42+
public class BatchMessagingMessageListenerAdapter extends MessagingMessageListenerAdapter
43+
implements ChannelAwareBatchMessagelistener {
3944

4045
private final MessagingMessageConverterAdapter converterAdapter;
4146

@@ -49,6 +54,36 @@ public BatchMessagingMessageListenerAdapter(Object bean, Method method, boolean
4954
this.batchingStrategy = batchingStrategy == null ? new SimpleBatchingStrategy(0, 0, 0L) : batchingStrategy;
5055
}
5156

57+
@Override
58+
public void onMessageBatch(List<org.springframework.amqp.core.Message> messages, Channel channel) {
59+
Message<?> converted;
60+
if (this.converterAdapter.isAmqpMessageList()) {
61+
converted = new GenericMessage<>(messages);
62+
}
63+
else {
64+
List<Message<?>> messagingMessages = new ArrayList<>();
65+
for (org.springframework.amqp.core.Message message : messages) {
66+
messagingMessages.add(toMessagingMessage(message));
67+
}
68+
if (this.converterAdapter.isMessageList()) {
69+
converted = new GenericMessage<>(messagingMessages);
70+
}
71+
else {
72+
List<Object> payloads = new ArrayList<>();
73+
for (Message<?> message : messagingMessages) {
74+
payloads.add(message.getPayload());
75+
}
76+
converted = new GenericMessage<>(payloads);
77+
}
78+
}
79+
try {
80+
invokeHandlerAndProcessResult(null, channel, converted);
81+
}
82+
catch (Exception e) {
83+
throw RabbitExceptionTranslator.convertRabbitAccessException(e);
84+
}
85+
}
86+
5287
@Override
5388
protected Message<?> toMessagingMessage(org.springframework.amqp.core.Message amqpMessage) {
5489
if (this.batchingStrategy.canDebatch(amqpMessage.getMessageProperties())) {

‎spring-rabbit/src/main/java/org/springframework/amqp/rabbit/listener/adapter/MessagingMessageListenerAdapter.java

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.springframework.amqp.support.converter.MessageConverter;
3232
import org.springframework.amqp.support.converter.MessagingMessageConverter;
3333
import org.springframework.core.MethodParameter;
34+
import org.springframework.lang.Nullable;
3435
import org.springframework.messaging.Message;
3536
import org.springframework.messaging.MessagingException;
3637
import org.springframework.messaging.handler.annotation.Payload;
@@ -128,6 +129,12 @@ public void setMessageConverter(MessageConverter messageConverter) {
128129
@Override
129130
public void onMessage(org.springframework.amqp.core.Message amqpMessage, Channel channel) throws Exception { // NOSONAR
130131
Message<?> message = toMessagingMessage(amqpMessage);
132+
invokeHandlerAndProcessResult(amqpMessage, channel, message);
133+
}
134+
135+
protected void invokeHandlerAndProcessResult(@Nullable org.springframework.amqp.core.Message amqpMessage,
136+
Channel channel, Message<?> message) throws Exception { // NOSONAR
137+
131138
if (logger.isDebugEnabled()) {
132139
logger.debug("Processing [" + message + "]");
133140
}
@@ -197,8 +204,9 @@ protected Message<?> toMessagingMessage(org.springframework.amqp.core.Message am
197204
* @param message the messaging message.
198205
* @return the result of invoking the handler.
199206
*/
200-
private InvocationResult invokeHandler(org.springframework.amqp.core.Message amqpMessage, Channel channel,
207+
private InvocationResult invokeHandler(@Nullable org.springframework.amqp.core.Message amqpMessage, Channel channel,
201208
Message<?> message) {
209+
202210
try {
203211
return this.handlerAdapter.invoke(message, amqpMessage, channel);
204212
}
@@ -267,6 +275,8 @@ protected final class MessagingMessageConverterAdapter extends MessagingMessageC
267275

268276
private boolean isMessageList;
269277

278+
private boolean isAmqpMessageList;
279+
270280
MessagingMessageConverterAdapter(Object bean, Method method, boolean batch) {
271281
this.bean = bean;
272282
this.method = method;
@@ -281,6 +291,14 @@ protected boolean isMessageList() {
281291
return this.isMessageList;
282292
}
283293

294+
protected boolean isAmqpMessageList() {
295+
return this.isAmqpMessageList;
296+
}
297+
298+
protected Method getMethod() {
299+
return this.method;
300+
}
301+
284302
@Override
285303
protected Object extractPayload(org.springframework.amqp.core.Message message) {
286304
MessageProperties messageProperties = message.getMessageProperties();
@@ -362,6 +380,7 @@ else if (parameterizedType.getRawType().equals(List.class)
362380
boolean messageHasGeneric = paramType instanceof ParameterizedType
363381
&& ((ParameterizedType) paramType).getRawType().equals(Message.class);
364382
this.isMessageList = paramType.equals(Message.class) || messageHasGeneric;
383+
this.isAmqpMessageList = paramType.equals(org.springframework.amqp.core.Message.class);
365384
if (messageHasGeneric) {
366385
genericParameterType = ((ParameterizedType) paramType).getActualTypeArguments()[0];
367386
}

‎spring-rabbit/src/main/java/org/springframework/amqp/rabbit/support/ListenerExecutionFailedException.java

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@
1616

1717
package org.springframework.amqp.rabbit.support;
1818

19+
import java.util.ArrayList;
20+
import java.util.Arrays;
21+
import java.util.Collection;
22+
import java.util.Collections;
23+
import java.util.List;
24+
1925
import org.springframework.amqp.AmqpException;
2026
import org.springframework.amqp.core.Message;
2127

@@ -31,21 +37,25 @@
3137
@SuppressWarnings("serial")
3238
public class ListenerExecutionFailedException extends AmqpException {
3339

34-
private final Message failedMessage;
40+
private final List<Message> failedMessages = new ArrayList<>();
3541

3642
/**
3743
* Constructor for ListenerExecutionFailedException.
3844
* @param msg the detail message
3945
* @param cause the exception thrown by the listener method
40-
* @param failedMessage the message that failed
46+
* @param failedMessage the message(s) that failed
4147
*/
42-
public ListenerExecutionFailedException(String msg, Throwable cause, Message failedMessage) {
48+
public ListenerExecutionFailedException(String msg, Throwable cause, Message... failedMessage) {
4349
super(msg, cause);
44-
this.failedMessage = failedMessage;
50+
this.failedMessages.addAll(Arrays.asList(failedMessage));
4551
}
4652

4753
public Message getFailedMessage() {
48-
return this.failedMessage;
54+
return this.failedMessages.get(0);
55+
}
56+
57+
public Collection<Message> getFailedMessages() {
58+
return Collections.unmodifiableList(this.failedMessages);
4959
}
5060

5161
}

0 commit comments

Comments
 (0)