Process Thread Dump: Event Debug Server 5.14.3 (#4 built by jenkins on 20180414-0409 git: 7f2725eea4edeb1d184a2888db790d332167b6f8) 46 active threads Thread 6116300 (Keep-Alive-Timer): State: TIMED_WAITING Blocked count: 0 Waited count: 1014 Stack: java.lang.Thread.sleep(Native Method) sun.net.www.http.KeepAliveCache.run(KeepAliveCache.java:172) java.lang.Thread.run(Thread.java:745) Thread 6115955 (1684908225@event-store-http-2303): State: TIMED_WAITING Blocked count: 2528 Waited count: 2529 Stack: java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Thread 6115424 (505338999@DebugServer(port8084)-6909): State: RUNNABLE Blocked count: 1063 Waited count: 1212 Stack: sun.management.ThreadImpl.getThreadInfo1(Native Method) sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:174) sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:139) com.cloudera.enterprise.debug.ThreadUtil.printThreadInfo(ThreadUtil.java:34) com.cloudera.enterprise.debug.ThreadUtil.printThreadInfo(ThreadUtil.java:74) com.cloudera.enterprise.debug.BacktraceServlet.getStackTraces(BacktraceServlet.java:56) com.cloudera.enterprise.debug.BacktraceServlet.doGet(BacktraceServlet.java:49) javax.servlet.http.HttpServlet.service(HttpServlet.java:707) javax.servlet.http.HttpServlet.service(HttpServlet.java:820) org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:401) org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:767) org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) org.mortbay.jetty.Server.handle(Server.java:326) org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928) org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Thread 6114673 (1667470509@event-store-http-2302): State: TIMED_WAITING Blocked count: 13111 Waited count: 13111 Stack: java.lang.Object.wait(Native Method) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) Thread 161 (New I/O worker #21): State: RUNNABLE Blocked count: 959 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 160 (AvroEventStorePublisher-NettyThread-boss_1): State: WAITING Blocked count: 0 Waited count: 1 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@38357540 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 159 (EventStorePublisherWithRetry-0): State: WAITING Blocked count: 1 Waited count: 5400 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@11c8a32a Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 158 (New I/O worker #6): State: RUNNABLE Blocked count: 976 Waited count: 140 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 157 (New I/O worker #19): State: RUNNABLE Blocked count: 954 Waited count: 143 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 155 (pool-5-thread-18): State: WAITING Blocked count: 985 Waited count: 297 Waiting on java.util.concurrent.SynchronousQueue$TransferStack@17b56471 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:925) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 154 (New I/O worker #12): State: RUNNABLE Blocked count: 727 Waited count: 186 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 153 (New I/O worker #16): State: RUNNABLE Blocked count: 1198 Waited count: 89 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 106 (New I/O worker #20): State: RUNNABLE Blocked count: 796 Waited count: 68 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 64 (New I/O worker #14): State: RUNNABLE Blocked count: 1238 Waited count: 278 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 54 (New I/O worker #18): State: RUNNABLE Blocked count: 950 Waited count: 235 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 52 (New I/O worker #13): State: RUNNABLE Blocked count: 735 Waited count: 118 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 50 (New I/O worker #11): State: RUNNABLE Blocked count: 762 Waited count: 406 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 47 (New I/O worker #7): State: RUNNABLE Blocked count: 651 Waited count: 166 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 45 (New I/O worker #17): State: RUNNABLE Blocked count: 1027 Waited count: 197 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 43 (New I/O worker #15): State: RUNNABLE Blocked count: 788 Waited count: 154 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 42 (New I/O worker #10): State: RUNNABLE Blocked count: 1015 Waited count: 184 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 40 (New I/O worker #9): State: RUNNABLE Blocked count: 826 Waited count: 132 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 37 (New I/O worker #5): State: RUNNABLE Blocked count: 847 Waited count: 111 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 34 (New I/O worker #4): State: RUNNABLE Blocked count: 751 Waited count: 118 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 33 (New I/O worker #3): State: RUNNABLE Blocked count: 947 Waited count: 180 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 32 (tagger-writer-1): State: WAITING Blocked count: 2685924 Waited count: 20891492 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@5ed8d2bf Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 31 (New I/O worker #2): State: RUNNABLE Blocked count: 1553 Waited count: 485 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 30 (tagger-writer-0): State: WAITING Blocked count: 2677716 Waited count: 20885700 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@5ed8d2bf Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 29 (New I/O worker #1): State: RUNNABLE Blocked count: 1285 Waited count: 386 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52) org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208) org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 28 (DestroyJavaVM): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: Thread 27 (426122736@event-store-http-1 - Acceptor0 SelectChannelConnector@0.0.0.0:7185): State: RUNNABLE Blocked count: 122781 Waited count: 1 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:511) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:193) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Thread 25 (1215661217@DebugServer(port8084)-1 - Acceptor0 SocketConnector@0.0.0.0:8084): State: RUNNABLE Blocked count: 1809 Waited count: 1 Stack: java.net.PlainSocketImpl.socketAccept(Native Method) java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398) java.net.ServerSocket.implAccept(ServerSocket.java:530) java.net.ServerSocket.accept(ServerSocket.java:498) org.mortbay.jetty.bio.SocketConnector.accept(SocketConnector.java:99) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Thread 23 (pool-4-thread-1): State: TIMED_WAITING Blocked count: 0 Waited count: 55211 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 22 (EventMetricsPublisher): State: TIMED_WAITING Blocked count: 330 Waited count: 3312550 Stack: java.lang.Object.wait(Native Method) com.cloudera.enterprise.PeriodicEnterpriseService$UnexceptionablePeriodicRunnable.run(PeriodicEnterpriseService.java:93) java.lang.Thread.run(Thread.java:745) Thread 21 (IndexManagerTask-2): State: TIMED_WAITING Blocked count: 9645 Waited count: 19425879 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 20 (IndexManagerTask-1): State: TIMED_WAITING Blocked count: 9813 Waited count: 19423038 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 19 (IndexManagerTask-0): State: WAITING Blocked count: 9722 Waited count: 19425827 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@532a98d6 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1085) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 18 (New I/O server boss #1 ([id: 0x64bfc93b, /0.0.0.0:7184])): State: RUNNABLE Blocked count: 10 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) org.jboss.netty.channel.socket.nio.NioServerSocketPipelineSink$Boss.run(NioServerSocketPipelineSink.java:227) org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102) org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) java.lang.Thread.run(Thread.java:745) Thread 16 (RateMonitor-tagger-writer): State: TIMED_WAITING Blocked count: 1 Waited count: 99369669 Stack: java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Thread 15 (RateMonitor-EventIngester): State: TIMED_WAITING Blocked count: 1 Waited count: 99369667 Stack: java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Thread 14 (JvmPauseMonitor): State: TIMED_WAITING Blocked count: 7 Waited count: 397390586 Stack: java.lang.Thread.sleep(Native Method) com.cloudera.enterprise.debug.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:201) java.lang.Thread.run(Thread.java:745) Thread 13 (com.cloudera.cmf.PollingScmProxy): State: TIMED_WAITING Blocked count: 14712 Waited count: 198709637 Stack: java.lang.Object.wait(Native Method) com.cloudera.enterprise.PeriodicEnterpriseService$UnexceptionablePeriodicRunnable.run(PeriodicEnterpriseService.java:93) java.lang.Thread.run(Thread.java:745) Thread 9 (event filterer): State: TIMED_WAITING Blocked count: 2 Waited count: 99760389 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor.runFiltering(LogEventProcessor.java:132) com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor.access$000(LogEventProcessor.java:28) com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor$1.run(LogEventProcessor.java:81) Thread 5 (Signal Dispatcher): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: Thread 3 (Finalizer): State: WAITING Blocked count: 743483 Waited count: 742707 Waiting on java.lang.ref.ReferenceQueue$Lock@5186508b Stack: java.lang.Object.wait(Native Method) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209) Thread 2 (Reference Handler): State: WAITING Blocked count: 786420 Waited count: 781315 Waiting on java.lang.ref.Reference$Lock@3c9c8d05 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:503) java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)