This message was deleted.

I used the file below for testing. With the endpoints, I am able to load large arrays on the gpu with both cupy and torch. However, when using a text input of longer than a few sentences, the nlp endpoint yields the incorrect device ordinal error.

import spacy
import bentoml
from <http://bentoml.io|bentoml.io> import JSON
import torch
import cupy
from pydantic import BaseModel


class SpacyRunner(bentoml.Runnable):
    SUPPORTED_RESOURCES = ('<http://nvidia.com/gpu|nvidia.com/gpu>')
    SUPPORTS_CPU_MULTI_THREADING = True

    def __init__(self):
        spacy.require_gpu()

        self.nlp = spacy.load('models/en_core_web_trf-3.5.0')
        self.nlp.add_pipe("doc_cleaner", config={"attrs": {"tensor": None}})

    @bentoml.Runnable.method()
    def test_nlp(self, text):
        try:
            doc = self.nlp(text)
            return doc.text
        except Exception as e:
            print(e)

    @bentoml.Runnable.method()
    def test_torch(self, array):
        print('cuda available:', torch.cuda.is_available())
        tensor = torch.tensor(array, dtype=torch.int64, device="cuda:0")
        print('tensor device type:', tensor.device.type)

        try:
            tensor = torch.tensor(array, dtype=torch.int64, device="cuda:1")
        except Exception as e:
            print(e)

        return tensor.device.__str__()

    @bentoml.Runnable.method()
    def test_cupy(self, array):
        print('current cuda device:', cupy.cuda.runtime.getDevice())
        with cupy.cuda.Device(0):
            tensor = cupy.array(array)
        print('tensor device:', tensor.device.__str__())

        try:
            with cupy.cuda.Device(1):
                tensor = cupy.array(array)
        except Exception as e:
            print(e)

        return tensor.device.__str__()


bento_runner = bentoml.Runner(SpacyRunner, name='spacy_runner')
svc = bentoml.Service('debug_service', runners=[bento_runner])


class InputData(BaseModel):
    text: str = ""
    array: list = [1, 2, 3]


@svc.api(input=JSON(pydantic_model=InputData), output=JSON())
async def test_nlp(input_data):
    return (await bento_runner.test_nlp.async_run(text=input_data.text))


@svc.api(input=JSON(pydantic_model=InputData), output=JSON())
async def test_torch(input_data):
    return (await bento_runner.test_torch.async_run(array=input_data.array))


@svc.api(input=JSON(pydantic_model=InputData), output=JSON())
async def test_cupy(input_data):
    return (await bento_runner.test_cupy.async_run(array=input_data.array))