i think part of the problem may be the fact that we have a split commit failing
2025/06/04 10
5644.349 INFO [LLCSegmentCompletionHandlers] [grizzly-http-server-7] Response to segmentConsumed for segment: purchases__13__13__20250603T1152Z is: {"status":"FAILED","streamPartitionMsgOffset":null,"buildTimeSec":-1,"isSplitCommitType":true}
2025/06/04 10
5644.349 INFO [ControllerResponseFilter] [grizzly-http-server-7] Handled request from 10.52.0.3 GET
http://pinot-controller-1.pinot-controller.pinot.svc.cluster.local:9000/segmentConsumed?reason=rowLimit&streamPartitionMsgOffset=9057670&instance=Server_pinot-server-5.pinot-server.pinot.svc.cluster.local_8098&name=purchases__13__13__20250603T1152Z&rowCount=39923&memoryUsedBytes=83171438, content-type null status code 200 OK
2025/06/04 10
5644.350 ERROR [SegmentCompletionManager] [grizzly-http-server-15] Caught exception in segmentConsumed for segment purchases__13__13__20250603T1152Z
java.lang.IllegalStateException: Failed to find segment ZK metadata for segment: purchases__13__13__20250603T1152Z of table: purchases_REALTIME
at
org.apache.pinot.shaded.com.google.common.base.Preconditions.checkState(Preconditions.java:836) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.apache.pinot.controller.helix.core.realtime.PinotLLCRealtimeSegmentManager.getSegmentZKMetadata(PinotLLCRealtimeSegmentManager.java:411) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.apache.pinot.controller.helix.core.realtime.SegmentCompletionManager.createFsm(SegmentCompletionManager.java:128) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.apache.pinot.controller.helix.core.realtime.SegmentCompletionManager.lambda$lookupOrCreateFsm$0(SegmentCompletionManager.java:122) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at java.base/java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1708) ~[?:?]
at org.apache.pinot.controller.helix.core.realtime.SegmentCompletionManager.lookupOrCreateFsm(SegmentCompletionManager.java:122) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.apache.pinot.controller.helix.core.realtime.SegmentCompletionManager.segmentConsumed(SegmentCompletionManager.java:175) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.apache.pinot.controller.api.resources.LLCSegmentCompletionHandlers.segmentConsumed(LLCSegmentCompletionHandlers.java:136) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) ~[?:?]
at java.base/java.lang.reflect.Method.invoke(Method.java:580) ~[?:?]
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory.lambda$static$0(ResourceMethodInvocationHandlerFactory.java:52) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:146) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:189) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$TypeOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:219) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:93) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:478) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:400) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:81) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:256) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.internal.Errors.process(Errors.java:292) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.internal.Errors.process(Errors.java:274) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.internal.Errors.process(Errors.java:244) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:235) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:684) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpContainer.service(GrizzlyHttpContainer.java:356) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.grizzly.http.server.HttpHandler$1.run(HttpHandler.java:190) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.grizzly.threadpool.AbstractThreadPool$Worker.doWork(AbstractThreadPool.java:535) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at org.glassfish.grizzly.threadpool.AbstractThreadPool$Worker.run(AbstractThreadPool.java:515) ~[pinot-all-1.3.0-jar-with-dependencies.jar:1.3.0-c0023da298126af6a01b802a04b66da34ba16134]
at java.base/java.lang.Thread.run(Thread.java:1583) [?:?]