AnsweredAssumed Answered

Crawl Database Unavailable error and workflow is in halted state

Question asked by Harpreet Kalsi on Oct 20, 2017
Latest reply on Oct 23, 2017 by Alan Bryant

HCI fails to run a workflow with Crawl Database Unavailable error. What can we do to figure out what happens?

Job-driver log contains the following stack trace:

 

2017-10-20 18:57:49,430 WARN [] [main] com.hds.ensemble.cassandra.CassandraConnector [CassandraConnector.java:184] Retries exceeded (1)

com.datastax.driver.core.exceptions.InvalidQueryException: Batch too large

        at com.datastax.driver.core.exceptions.InvalidQueryException.copy(InvalidQueryException.java:50)

        at com.datastax.driver.core.DriverThrowables.propagateCause(DriverThrowables.java:37)

        at com.datastax.driver.core.DefaultResultSetFuture.getUninterruptibly(DefaultResultSetFuture.java:245)

        at com.datastax.driver.core.AbstractSession.execute(AbstractSession.java:68)

        at com.hds.ensemble.cassandra.CassandraConnector.lambda$executeBatch$8(CassandraConnector.java:260)

        at com.hds.ensemble.cassandra.CassandraConnector.lambda$wrapRunnable$1(CassandraConnector.java:110)

        at com.hds.ensemble.cassandra.CassandraConnector.wrapSupplierWithRetry(CassandraConnector.java:171)

        at com.hds.ensemble.cassandra.CassandraConnector.wrapSupplier(CassandraConnector.java:122)

        at com.hds.ensemble.cassandra.CassandraConnector.wrapRunnable(CassandraConnector.java:109)

        at com.hds.ensemble.cassandra.CassandraConnector.executeBatch(CassandraConnector.java:260)

        at com.hds.ensemble.crawl.CrawlerDbapi.addNewDocument(CrawlerDbapi.java:134)

        at com.hds.ensemble.crawl.Crawler.crawlOneDocument(Crawler.java:674)

        at com.hds.ensemble.crawl.Crawler.crawlOneContainer(Crawler.java:597)

        at com.hds.ensemble.crawl.Crawler.crawlOneListBatch(Crawler.java:531)

        at com.hds.ensemble.crawl.Crawler.crawl(Crawler.java:309)

        at com.hds.ensemble.job.DocumentProducer.run(DocumentProducer.java:53)

        at com.hds.ensemble.job.SparkHelper.runWorkflow(SparkHelper.java:349)

        at com.hds.ensemble.job.JobDriver.runWorkflow(JobDriver.java:244)

        at com.hds.ensemble.job.JobDriver.run(JobDriver.java:144)

        at com.hds.ensemble.job.JobDriver.main(JobDriver.java:102)

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)

        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

        at java.lang.reflect.Method.invoke(Method.java:498)

        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:736)

        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:185)

        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:210)

        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:124)

        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

Caused by: com.datastax.driver.core.exceptions.InvalidQueryException: Batch too large

        at com.datastax.driver.core.Responses$Error.asException(Responses.java:136)

        at com.datastax.driver.core.DefaultResultSetFuture.onSet(DefaultResultSetFuture.java:179)

        at com.datastax.driver.core.RequestHandler.setFinalResult(RequestHandler.java:177)

        at com.datastax.driver.core.RequestHandler.access$2500(RequestHandler.java:46)

        at com.datastax.driver.core.RequestHandler$SpeculativeExecution.setFinalResult(RequestHandler.java:797)

        at com.datastax.driver.core.RequestHandler$SpeculativeExecution.onSet(RequestHandler.java:631)

        at com.datastax.driver.core.Connection$Dispatcher.channelRead0(Connection.java:1070)

        at com.datastax.driver.core.Connection$Dispatcher.channelRead0(Connection.java:993)

        at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:308)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294)

        at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:266)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:308)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294)

        at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:308)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294)

        at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:244)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:308)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294)

        at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:846)

        at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:831)

        at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:346)

        at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:254)

        at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)

        at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)

        at java.lang.Thread.run(Thread.java:745)

2017-10-20 18:57:49,440 ERROR [] [main] com.hds.ensemble.job.JobDriver [JobDriver.java:250] run workflow exception: Crawl Database unavailable

com.hds.ensemble.sdk.exception.ConfigurationException: Crawl Database unavailable

        at com.hds.ensemble.crawl.Crawler.throwAppropriateDbException(Crawler.java:911)

        at com.hds.ensemble.crawl.Crawler.crawlOneListBatch(Crawler.java:553)

        at com.hds.ensemble.crawl.Crawler.crawl(Crawler.java:309)

        at com.hds.ensemble.job.DocumentProducer.run(DocumentProducer.java:53)

        at com.hds.ensemble.job.SparkHelper.runWorkflow(SparkHelper.java:349)

        at com.hds.ensemble.job.JobDriver.runWorkflow(JobDriver.java:244)

        at com.hds.ensemble.job.JobDriver.run(JobDriver.java:144)

        at com.hds.ensemble.job.JobDriver.main(JobDriver.java:102)

        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)

        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)

        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

        at java.lang.reflect.Method.invoke(Method.java:498)

        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:736)

        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:185)

        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:210)

        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:124)

        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

Caused by: com.hds.ensemble.exception.DatabaseException: Batch too large

        at com.hds.ensemble.cassandra.CassandraConnector.mapToDatabaseException(CassandraConnector.java:141)

        at com.hds.ensemble.cassandra.CassandraConnector.wrapSupplierWithRetry(CassandraConnector.java:186)

        at com.hds.ensemble.cassandra.CassandraConnector.wrapSupplier(CassandraConnector.java:122)

        at com.hds.ensemble.cassandra.CassandraConnector.wrapRunnable(CassandraConnector.java:109)

        at com.hds.ensemble.cassandra.CassandraConnector.executeBatch(CassandraConnector.java:260)

        at com.hds.ensemble.crawl.CrawlerDbapi.addNewDocument(CrawlerDbapi.java:134)

        at com.hds.ensemble.crawl.Crawler.crawlOneDocument(Crawler.java:674)

        at com.hds.ensemble.crawl.Crawler.crawlOneContainer(Crawler.java:597)

        at com.hds.ensemble.crawl.Crawler.crawlOneListBatch(Crawler.java:531)

        ... 15 common frames omitted

Caused by: com.datastax.driver.core.exceptions.InvalidQueryException: Batch too large

        at com.datastax.driver.core.exceptions.InvalidQueryException.copy(InvalidQueryException.java:50)

        at com.datastax.driver.core.DriverThrowables.propagateCause(DriverThrowables.java:37)

        at com.datastax.driver.core.DefaultResultSetFuture.getUninterruptibly(DefaultResultSetFuture.java:245)

        at com.datastax.driver.core.AbstractSession.execute(AbstractSession.java:68)

        at com.hds.ensemble.cassandra.CassandraConnector.lambda$executeBatch$8(CassandraConnector.java:260)

        at com.hds.ensemble.cassandra.CassandraConnector.lambda$wrapRunnable$1(CassandraConnector.java:110)

        at com.hds.ensemble.cassandra.CassandraConnector.wrapSupplierWithRetry(CassandraConnector.java:171)

        ... 22 common frames omitted

Caused by: com.datastax.driver.core.exceptions.InvalidQueryException: Batch too large

        at com.datastax.driver.core.Responses$Error.asException(Responses.java:136)

        at com.datastax.driver.core.DefaultResultSetFuture.onSet(DefaultResultSetFuture.java:179)

        at com.datastax.driver.core.RequestHandler.setFinalResult(RequestHandler.java:177)

        at com.datastax.driver.core.RequestHandler.access$2500(RequestHandler.java:46)

        at com.datastax.driver.core.RequestHandler$SpeculativeExecution.setFinalResult(RequestHandler.java:797)

        at com.datastax.driver.core.RequestHandler$SpeculativeExecution.onSet(RequestHandler.java:631)

        at com.datastax.driver.core.Connection$Dispatcher.channelRead0(Connection.java:1070)

        at com.datastax.driver.core.Connection$Dispatcher.channelRead0(Connection.java:993)

        at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:308)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294)

        at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:266)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:308)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294)

        at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:308)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294)

        at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:244)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:308)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:294)

        at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:846)

        at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:831)

        at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:346)

        at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:254)

        at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:111)

        at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:137)

        at java.lang.Thread.run(Thread.java:745)

Outcomes