2024-01-18T03:36:18+0000 [INFO] [api_server:prediction-service:1] 10.1.2.24:35014 (scheme=http,method=POST,path=/optimizer-predict,type=application/json) (status=500,type=text/plain; charset=utf-8) 55858.373ms (trace=d4f71197e4328e29bc776edd93a1fbf3,span=8ad05ccf0b184674,sampled=0,service.name=prediction-service)
2024-01-18T03:36:18+0000 [ERROR] [api_server:prediction-service:1] Exception in ASGI application
  + Exception Group Traceback (most recent call last):
  |   File "/opt/venv/lib/python3.10/site-packages/uvicorn/protocols/http/h11_impl.py", line 407, in run_asgi
  |     result = await app(  # type: ignore[func-returns-value]
  |   File "/opt/venv/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 78, in __call__
  |     return await <http://self.app|self.app>(scope, receive, send)
  |   File "/opt/venv/lib/python3.10/site-packages/uvicorn/middleware/message_logger.py", line 86, in __call__
  |     raise exc from None
  |   File "/opt/venv/lib/python3.10/site-packages/uvicorn/middleware/message_logger.py", line 82, in __call__
  |     await <http://self.app|self.app>(scope, inner_receive, inner_send)
  |   File "/opt/venv/lib/python3.10/site-packages/starlette/applications.py", line 122, in __call__
  |     await self.middleware_stack(scope, receive, send)
  |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/errors.py", line 184, in __call__
  |     raise exc
  |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__
  |     await <http://self.app|self.app>(scope, receive, _send)
  |   File "/opt/venv/lib/python3.10/site-packages/bentoml/_internal/server/http/traffic.py", line 23, in __call__
  |     async with anyio.create_task_group():
  |   File "/opt/venv/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 664, in __aexit__
  |     raise BaseExceptionGroup(
  | exceptiongroup.ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
  +-+---------------- 1 ----------------
    | Traceback (most recent call last):
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/base.py", line 108, in __call__
    |     response = await self.dispatch_func(request, call_next)
    |   File "/app/canvass_ml_scikit_learn/serve/metrics/middleware.py", line 35, in dispatch
    |     response = await call_next(request)
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/base.py", line 84, in call_next
    |     raise app_exc
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/base.py", line 70, in coro
    |     await <http://self.app|self.app>(scope, receive_or_disconnect, send_no_error)
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
    |     raise exc
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
    |     await <http://self.app|self.app>(scope, receive, sender)
    |   File "/opt/venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 20, in __call__
    |     raise e
    |   File "/opt/venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 17, in __call__
    |     await <http://self.app|self.app>(scope, receive, send)
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__
    |     await route.handle(scope, receive, send)
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle
    |     await <http://self.app|self.app>(scope, receive, send)
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/routing.py", line 66, in app
    |     response = await func(request)
    |   File "/opt/venv/lib/python3.10/site-packages/fastapi/routing.py", line 241, in app
    |     raw_response = await run_endpoint_function(
    |   File "/opt/venv/lib/python3.10/site-packages/fastapi/routing.py", line 167, in run_endpoint_function
    |     return await dependant.call(**values)
    |   File "/app/canvass_ml_scikit_learn/serve/optimizer/optimizer.py", line 51, in generate_optimizer_predictions
    |     predictions, input_guesses = await generate_predictions_from_synthetic_data(
    |   File "/app/canvass_ml_scikit_learn/serve/optimizer/optimizer_helpers.py", line 304, in generate_predictions_from_synthetic_data
    |     prediction = await trained_models[i].prediction_runner.predict.async_run(input_data)
    |   File "/opt/venv/lib/python3.10/site-packages/bentoml/_internal/runner/runner.py", line 55, in async_run
    |     return await self.runner._runner_handle.async_run_method(self, *args, **kwargs)
    |   File "/opt/venv/lib/python3.10/site-packages/bentoml/_internal/runner/runner_handle/remote.py", line 242, in async_run_method
    |     raise RemoteException(
    | bentoml.exceptions.RemoteException: An unexpected exception occurred in remote runner prediction_runner_2: [400] Invalid HTTP request received.
    |
    | During handling of the above exception, another exception occurred:
    |
    | Exception Group Traceback (most recent call last):
    |   File "/opt/venv/lib/python3.10/site-packages/bentoml/_internal/server/http/traffic.py", line 26, in __call__
    |     await <http://self.app|self.app>(scope, receive, send)
    |   File "/opt/venv/lib/python3.10/site-packages/bentoml/_internal/server/http/instruments.py", line 135, in __call__
    |     await <http://self.app|self.app>(scope, receive, wrapped_send)
    |   File "/opt/venv/lib/python3.10/site-packages/opentelemetry/instrumentation/asgi/__init__.py", line 580, in __call__
    |     await <http://self.app|self.app>(scope, otel_receive, otel_send)
    |   File "/opt/venv/lib/python3.10/site-packages/bentoml/_internal/server/http/access.py", line 126, in __call__
    |     await <http://self.app|self.app>(scope, receive, wrapped_send)
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
    |     raise exc
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
    |     await <http://self.app|self.app>(scope, receive, sender)
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__
    |     await route.handle(scope, receive, send)
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/routing.py", line 443, in handle
    |     await <http://self.app|self.app>(scope, receive, send)
    |   File "/opt/venv/lib/python3.10/site-packages/fastapi/applications.py", line 282, in __call__
    |     await super().__call__(scope, receive, send)
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/applications.py", line 122, in __call__
    |     await self.middleware_stack(scope, receive, send)
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/errors.py", line 184, in __call__
    |     raise exc
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__
    |     await <http://self.app|self.app>(scope, receive, _send)
    |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/base.py", line 106, in __call__
    |     async with anyio.create_task_group() as task_group:
    |   File "/opt/venv/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 664, in __aexit__
    |     raise BaseExceptionGroup(
    | exceptiongroup.ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
    +-+---------------- 1 ----------------
      | Traceback (most recent call last):
      |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/base.py", line 108, in __call__
      |     response = await self.dispatch_func(request, call_next)
      |   File "/app/canvass_ml_scikit_learn/serve/metrics/middleware.py", line 35, in dispatch
      |     response = await call_next(request)
      |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/base.py", line 84, in call_next
      |     raise app_exc
      |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/base.py", line 70, in coro
      |     await <http://self.app|self.app>(scope, receive_or_disconnect, send_no_error)
      |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
      |     raise exc
      |   File "/opt/venv/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
      |     await <http://self.app|self.app>(scope, receive, sender)
      |   File "/opt/venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 20, in __call__
      |     raise e
      |   File "/opt/venv/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 17, in __call__
      |     await <http://self.app|self.app>(scope, receive, send)
      |   File "/opt/venv/lib/python3.10/site-packages/starlette/routing.py", line 718, in __call__
      |     await route.handle(scope, receive, send)
      |   File "/opt/venv/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle
      |     await <http://self.app|self.app>(scope, receive, send)
      |   File "/opt/venv/lib/python3.10/site-packages/starlette/routing.py", line 66, in app
      |     response = await func(request)
      |   File "/opt/venv/lib/python3.10/site-packages/fastapi/routing.py", line 241, in app
      |     raw_response = await run_endpoint_function(
      |   File "/opt/venv/lib/python3.10/site-packages/fastapi/routing.py", line 167, in run_endpoint_function
      |     return await dependant.call(**values)
      |   File "/app/canvass_ml_scikit_learn/serve/optimizer/optimizer.py", line 51, in generate_optimizer_predictions
      |     predictions, input_guesses = await generate_predictions_from_synthetic_data(
      |   File "/app/canvass_ml_scikit_learn/serve/optimizer/optimizer_helpers.py", line 304, in generate_predictions_from_synthetic_data
      |     prediction = await trained_models[i].prediction_runner.predict.async_run(input_data)
      |   File "/opt/venv/lib/python3.10/site-packages/bentoml/_internal/runner/runner.py", line 55, in async_run
      |     return await self.runner._runner_handle.async_run_method(self, *args, **kwargs)
      |   File "/opt/venv/lib/python3.10/site-packages/bentoml/_internal/runner/runner_handle/remote.py", line 242, in async_run_method
      |     raise RemoteException(
      | bentoml.exceptions.RemoteException: An unexpected exception occurred in remote runner prediction_runner_2: [400] Invalid HTTP request received.
      +------------------------------------