https://promptfoo.dev/ logo
Failure using my custom provider
# ❓|questions
x
I am using the customprivider.js, sometime it is not working. The failure output is : Reading prompts from ["file://D:\\github\\promptfoo\\mystarprompt\\test\\prompts.txt"] Loading var body from file: D:\…\context1.txt Loading var body from file: D:\…\context2.txt Eval #1 complete (1 of 2) Eval #2 complete (2 of 2) At least one evaluation failure occurred. This might be caused by the underlying call to the provider, or a test failure. Context: [{"provider":{"id":"customProvider.js","label":"My custom provider"},"prompt":{"raw":"需要分析的图书书名是 你好","label":"prompts.txt: 需要分析的图书书名是 {{body}}"},"vars":{"body":"你好"},"response":{"output":"书名“你好”是一个…。","tokenUsage":{"total":210,"prompt":28,"completion":182}},"success":true,"score":1,"namedScores":{},"latencyMs":43951,"gradingResult":{"pass":true,"score":1,"reason":"No assertions","tokensUsed":{"total":0,"prompt":0,"completion":0,"cached":0},"assertion":null}},{"provider":{"id":"customProvider.js","label":"My custom provider"},"prompt":{"raw":"需要分析的图书书名是 我饿了","label":"prompts.txt: 需要分析的图书书名是 {{body}}"},"vars":{"body":"我饿了"},"error":"Error: Request failed after 4 retries: Request timed out after 10000 ms\n\nError: Request failed after 4 retries: Request timed out after 10000 ms\n at fetchWithRetries (D:\\github\\promptfoo\\dist\\src\\fetch.js:82:11)\n at async D:\\github\\promptfoo\\dist\\src\\cache.js:73:26","success":false,"score":0,"namedScores":{},"latencyMs":0}] Running db migrations... Running migrations from C:\Users\xutwe\AppData\Roaming\npm\node_modules\promptfoo\dist\drizzle... Inserting eval eval-2024-08-26T03:11:29 Inserting prompt ea0f47c6633d1e5e8ae9abb4da2368cd0309147e7e2420b6603ca079389e854f Inserting dataset 5c914c0f5bddc2e550a175c7def595b345444cf3e334a908b517db387894d1b4 Awaiting 5 promises to database... ✔ Evaluation complete. Successes: 1 Failures: 1 Pass Rate: 50.00% Token usage: Total 210, Prompt 28, Completion 182, Cached 0 Done.
i
have you tried outputting debug information in your custom provider? what is the URL it's trying to fetch?
2 Views