Hey team 👋 getting an error running bentoml, it seems to work when I run bentoml service in debug mode aka svc.asgi_app, but when I run it using bentoml serve cli command it fails to return prediction from a runner, following is the log trace

2023-07-05T10:50:52-0400 [INFO] [api_server:prediction-service:2] 127.0.0.1:60494 (scheme=http,method=POST,path=/predict,type=application/json,length=2666) (status=500,type=text/plain; charset=utf-8,length=21) 7803.511ms (trace=d0270f57de48b12efa904216372f1d05,span=22c95048afda9d30,sampled=0,service.name=prediction-service)
2023-07-05T10:50:52-0400 [ERROR] [runner:prediction_runner:1] Exception in ASGI application
Traceback (most recent call last):
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/uvicorn/protocols/http/h11_impl.py", line 407, in run_asgi
    result = await app(  # type: ignore[func-returns-value]
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
    return await <http://self.app|self.app>(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/uvicorn/middleware/message_logger.py", line 86, in __call__
    raise exc from None
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/uvicorn/middleware/message_logger.py", line 82, in __call__
    await <http://self.app|self.app>(scope, inner_receive, inner_send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/applications.py", line 122, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/errors.py", line 184, in __call__
    raise exc
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__
    await <http://self.app|self.app>(scope, receive, _send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/bentoml/_internal/server/http/traffic.py", line 26, in __call__
    await <http://self.app|self.app>(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/opentelemetry/instrumentation/asgi/__init__.py", line 579, in __call__
    await <http://self.app|self.app>(scope, otel_receive, otel_send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/bentoml/_internal/server/http/instruments.py", line 293, in __call__
    await <http://self.app|self.app>(scope, receive, wrapped_send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/bentoml/_internal/server/http/access.py", line 126, in __call__
    await <http://self.app|self.app>(scope, receive, wrapped_send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
    raise exc
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
    await <http://self.app|self.app>(scope, receive, sender)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__
    await route.handle(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle
    await <http://self.app|self.app>(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/routing.py", line 66, in app
    response = await func(request)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/bentoml/_internal/server/runner_app.py", line 368, in _run
    payload = AutoContainer.to_payload(ret, 0)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/bentoml/_internal/runner/container.py", line 637, in to_payload
    return container_cls.to_payload(batch, batch_dim)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/bentoml/_internal/runner/container.py", line 304, in to_payload
    batch.shape[batch_dim],
IndexError: tuple index out of range
2023-07-05T10:50:52-0400 [ERROR] [api_server:prediction-service:2] Exception in ASGI application
Traceback (most recent call last):
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/uvicorn/protocols/http/h11_impl.py", line 407, in run_asgi
    result = await app(  # type: ignore[func-returns-value]
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
    return await <http://self.app|self.app>(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/uvicorn/middleware/message_logger.py", line 86, in __call__
    raise exc from None
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/uvicorn/middleware/message_logger.py", line 82, in __call__
    await <http://self.app|self.app>(scope, inner_receive, inner_send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/applications.py", line 122, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/errors.py", line 184, in __call__
    raise exc
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__
    await <http://self.app|self.app>(scope, receive, _send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/bentoml/_internal/server/http/traffic.py", line 26, in __call__
    await <http://self.app|self.app>(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/bentoml/_internal/server/http/instruments.py", line 176, in __call__
    await <http://self.app|self.app>(scope, receive, wrapped_send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/opentelemetry/instrumentation/asgi/__init__.py", line 579, in __call__
    await <http://self.app|self.app>(scope, otel_receive, otel_send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/bentoml/_internal/server/http/access.py", line 126, in __call__
    await <http://self.app|self.app>(scope, receive, wrapped_send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
    raise exc
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
    await <http://self.app|self.app>(scope, receive, sender)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__
    await route.handle(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/routing.py", line 443, in handle
    await <http://self.app|self.app>(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/fastapi/applications.py", line 282, in __call__
    await super().__call__(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/applications.py", line 122, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/errors.py", line 184, in __call__
    raise exc
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__
    await <http://self.app|self.app>(scope, receive, _send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/base.py", line 108, in __call__
    response = await self.dispatch_func(request, call_next)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/canvass_ml_scikit_learn/serve/metrics/middleware.py", line 35, in dispatch
    response = await call_next(request)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/base.py", line 84, in call_next
    raise app_exc
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/base.py", line 70, in coro
    await <http://self.app|self.app>(scope, receive_or_disconnect, send_no_error)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
    raise exc
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
    await <http://self.app|self.app>(scope, receive, sender)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 20, in __call__
    raise e
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 17, in __call__
    await <http://self.app|self.app>(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__
    await route.handle(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle
    await <http://self.app|self.app>(scope, receive, send)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/starlette/routing.py", line 66, in app
    response = await func(request)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/fastapi/routing.py", line 241, in app
    raw_response = await run_endpoint_function(
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/fastapi/routing.py", line 167, in run_endpoint_function
    return await dependant.call(**values)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/canvass_ml_scikit_learn/serve/predict.py", line 57, in generate_predictions
    prediction: np.array = await serve_artifacts.runners.prediction_runner.predict.async_run(
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/bentoml/_internal/runner/runner.py", line 55, in async_run
    return await self.runner._runner_handle.async_run_method(self, *args, **kwargs)
  File "/Users/ashish/Desktop/Projects/canvass-ml-scikit-learn/.venv/lib/python3.10/site-packages/bentoml/_internal/runner/runner_handle/remote.py", line 246, in async_run_method
    raise RemoteException(
bentoml.exceptions.RemoteException: An unexpected exception occurred in remote runner prediction_runner: [500] Internal Server Error