001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.broker.region.policy;
018
019import java.util.ArrayList;
020import java.util.HashMap;
021import java.util.List;
022import java.util.Map;
023import java.util.Map.Entry;
024import java.util.concurrent.ConcurrentHashMap;
025import java.util.concurrent.atomic.AtomicBoolean;
026
027import org.apache.activemq.broker.Broker;
028import org.apache.activemq.broker.Connection;
029import org.apache.activemq.broker.ConnectionContext;
030import org.apache.activemq.broker.region.Destination;
031import org.apache.activemq.broker.region.Subscription;
032import org.apache.activemq.command.ConsumerControl;
033import org.apache.activemq.command.RemoveInfo;
034import org.apache.activemq.state.CommandVisitor;
035import org.apache.activemq.thread.Scheduler;
036import org.apache.activemq.transport.InactivityIOException;
037import org.slf4j.Logger;
038import org.slf4j.LoggerFactory;
039
040/**
041 * Abort slow consumers when they reach the configured threshold of slowness, default is slow for 30 seconds
042 *
043 * @org.apache.xbean.XBean
044 */
045public class AbortSlowConsumerStrategy implements SlowConsumerStrategy, Runnable {
046
047    private static final Logger LOG = LoggerFactory.getLogger(AbortSlowConsumerStrategy.class);
048
049    protected String name = "AbortSlowConsumerStrategy@" + hashCode();
050    protected Scheduler scheduler;
051    protected Broker broker;
052    protected final AtomicBoolean taskStarted = new AtomicBoolean(false);
053    protected final Map<Subscription, SlowConsumerEntry> slowConsumers =
054        new ConcurrentHashMap<Subscription, SlowConsumerEntry>();
055
056    private long maxSlowCount = -1;
057    private long maxSlowDuration = 30*1000;
058    private long checkPeriod = 30*1000;
059    private boolean abortConnection = false;
060
061    @Override
062    public void setBrokerService(Broker broker) {
063       this.scheduler = broker.getScheduler();
064       this.broker = broker;
065    }
066
067    @Override
068    public void slowConsumer(ConnectionContext context, Subscription subs) {
069        if (maxSlowCount < 0 && maxSlowDuration < 0) {
070            // nothing to do
071            LOG.info("no limits set, slowConsumer strategy has nothing to do");
072            return;
073        }
074
075        if (taskStarted.compareAndSet(false, true)) {
076            scheduler.executePeriodically(this, checkPeriod);
077        }
078
079        if (!slowConsumers.containsKey(subs)) {
080            slowConsumers.put(subs, new SlowConsumerEntry(context));
081        } else if (maxSlowCount > 0) {
082            slowConsumers.get(subs).slow();
083        }
084    }
085
086    @Override
087    public void run() {
088        if (maxSlowDuration > 0) {
089            // mark
090            for (SlowConsumerEntry entry : slowConsumers.values()) {
091                entry.mark();
092            }
093        }
094
095        HashMap<Subscription, SlowConsumerEntry> toAbort = new HashMap<Subscription, SlowConsumerEntry>();
096        for (Entry<Subscription, SlowConsumerEntry> entry : slowConsumers.entrySet()) {
097            if (entry.getKey().isSlowConsumer()) {
098                if (maxSlowDuration > 0 && (entry.getValue().markCount * checkPeriod >= maxSlowDuration)
099                        || maxSlowCount > 0 && entry.getValue().slowCount >= maxSlowCount) {
100                    toAbort.put(entry.getKey(), entry.getValue());
101                    slowConsumers.remove(entry.getKey());
102                }
103            } else {
104                LOG.info("sub: " + entry.getKey().getConsumerInfo().getConsumerId() + " is no longer slow");
105                slowConsumers.remove(entry.getKey());
106            }
107        }
108
109        abortSubscription(toAbort, abortConnection);
110    }
111
112    protected void abortSubscription(Map<Subscription, SlowConsumerEntry> toAbort, boolean abortSubscriberConnection) {
113
114        Map<Connection, List<Subscription>> abortMap = new HashMap<Connection, List<Subscription>>();
115
116        for (final Entry<Subscription, SlowConsumerEntry> entry : toAbort.entrySet()) {
117            ConnectionContext connectionContext = entry.getValue().context;
118            if (connectionContext == null) {
119                continue;
120            }
121
122            Connection connection = connectionContext.getConnection();
123            if (connection == null) {
124                LOG.debug("slowConsumer abort ignored, no connection in context:"  + connectionContext);
125            }
126
127            if (!abortMap.containsKey(connection)) {
128                abortMap.put(connection, new ArrayList<Subscription>());
129            }
130
131            abortMap.get(connection).add(entry.getKey());
132        }
133
134        for (Entry<Connection, List<Subscription>> entry : abortMap.entrySet()) {
135            final Connection connection = entry.getKey();
136            final List<Subscription> subscriptions = entry.getValue();
137
138            if (abortSubscriberConnection) {
139
140                LOG.info("aborting connection:{} with {} slow consumers",
141                         connection.getConnectionId(), subscriptions.size());
142
143                if (LOG.isTraceEnabled()) {
144                    for (Subscription subscription : subscriptions) {
145                        LOG.trace("Connection {} being aborted because of slow consumer: {} on destination: {}",
146                                  new Object[] { connection.getConnectionId(),
147                                                 subscription.getConsumerInfo().getConsumerId(),
148                                                 subscription.getActiveMQDestination() });
149                    }
150                }
151
152                try {
153                    scheduler.executeAfterDelay(new Runnable() {
154                        @Override
155                        public void run() {
156                            connection.serviceException(new InactivityIOException(
157                                    subscriptions.size() + " Consumers was slow too often (>"
158                                    + maxSlowCount +  ") or too long (>"
159                                    + maxSlowDuration + "): "));
160                        }}, 0l);
161                } catch (Exception e) {
162                    LOG.info("exception on aborting connection {} with {} slow consumers",
163                             connection.getConnectionId(), subscriptions.size());
164                }
165            } else {
166                // just abort each consumer
167                for (Subscription subscription : subscriptions) {
168                    final Subscription subToClose = subscription;
169                    LOG.info("aborting slow consumer: {} for destination:{}",
170                             subscription.getConsumerInfo().getConsumerId(),
171                             subscription.getActiveMQDestination());
172
173                    // tell the remote consumer to close
174                    try {
175                        ConsumerControl stopConsumer = new ConsumerControl();
176                        stopConsumer.setConsumerId(subscription.getConsumerInfo().getConsumerId());
177                        stopConsumer.setClose(true);
178                        connection.dispatchAsync(stopConsumer);
179                    } catch (Exception e) {
180                        LOG.info("exception on aborting slow consumer: {}", subscription.getConsumerInfo().getConsumerId(), e);
181                    }
182
183                    // force a local remove in case remote is unresponsive
184                    try {
185                        scheduler.executeAfterDelay(new Runnable() {
186                            @Override
187                            public void run() {
188                                try {
189                                    RemoveInfo removeCommand = subToClose.getConsumerInfo().createRemoveCommand();
190                                    if (connection instanceof CommandVisitor) {
191                                        // avoid service exception handling and logging
192                                        removeCommand.visit((CommandVisitor) connection);
193                                    } else {
194                                        connection.service(removeCommand);
195                                    }
196                                } catch (IllegalStateException ignoredAsRemoteHasDoneTheJob) {
197                                } catch (Exception e) {
198                                    LOG.info("exception on local remove of slow consumer: {}", subToClose.getConsumerInfo().getConsumerId(), e);
199                                }
200                            }}, 1000l);
201
202                    } catch (Exception e) {
203                        LOG.info("exception on local remove of slow consumer: {}", subscription.getConsumerInfo().getConsumerId(), e);
204                    }
205                }
206            }
207        }
208    }
209
210    public void abortConsumer(Subscription sub, boolean abortSubscriberConnection) {
211        if (sub != null) {
212            SlowConsumerEntry entry = slowConsumers.remove(sub);
213            if (entry != null) {
214                Map<Subscription, SlowConsumerEntry> toAbort = new HashMap<Subscription, SlowConsumerEntry>();
215                toAbort.put(sub, entry);
216                abortSubscription(toAbort, abortSubscriberConnection);
217            } else {
218                LOG.warn("cannot abort subscription as it no longer exists in the map of slow consumers: " + sub);
219            }
220        }
221    }
222
223    public long getMaxSlowCount() {
224        return maxSlowCount;
225    }
226
227    /**
228     * number of times a subscription can be deemed slow before triggering abort
229     * effect depends on dispatch rate as slow determination is done on dispatch
230     */
231    public void setMaxSlowCount(long maxSlowCount) {
232        this.maxSlowCount = maxSlowCount;
233    }
234
235    public long getMaxSlowDuration() {
236        return maxSlowDuration;
237    }
238
239    /**
240     * time in milliseconds that a sub can remain slow before triggering
241     * an abort.
242     * @param maxSlowDuration
243     */
244    public void setMaxSlowDuration(long maxSlowDuration) {
245        this.maxSlowDuration = maxSlowDuration;
246    }
247
248    public long getCheckPeriod() {
249        return checkPeriod;
250    }
251
252    /**
253     * time in milliseconds between checks for slow subscriptions
254     * @param checkPeriod
255     */
256    public void setCheckPeriod(long checkPeriod) {
257        this.checkPeriod = checkPeriod;
258    }
259
260    public boolean isAbortConnection() {
261        return abortConnection;
262    }
263
264    /**
265     * abort the consumers connection rather than sending a stop command to the remote consumer
266     * @param abortConnection
267     */
268    public void setAbortConnection(boolean abortConnection) {
269        this.abortConnection = abortConnection;
270    }
271
272    public void setName(String name) {
273        this.name = name;
274    }
275
276    public String getName() {
277        return name;
278    }
279
280    public Map<Subscription, SlowConsumerEntry> getSlowConsumers() {
281        return slowConsumers;
282    }
283
284    @Override
285    public void addDestination(Destination destination) {
286        // Not needed for this strategy.
287    }
288}