ad0ll
01/12/2025, 7:10 PMxml
<reasoning>
(The reasoning for the classification)
</reasoning>
<tags>
<tag>TAG1</tag>
<tag>TAG2</tag>
</tags>
My tests are checking for tags and are in a CSV:
csv
email_input,__expected
"{""subject"":""email subject"",""from"":""\""email_sender@test.com\"", ""body"": ""email body"", ""attachments"":[]}","contains-all:tag1, tag2"
I'm extracting tags from the full output using a custom transform function:
yaml
defaultTest:
options:
transform: file://transform.js:get_transform
javascript
//TL;DR: Extract an array of tags from the output, with example from CSV you'd end up with ["tag1","tag2"]
export function get_transform(output, context) {
// Extract tags from XML-like output
const tagMatch = output.match(/<tags>.*?<\/tags>/s);
if (!tagMatch) return [];
const tagSection = tagMatch[0];
const tags =
tagSection.match(/<tag>(.+?)<\/tag>/g)?.map((tag) => {
return tag.replace(/<\/?tag>/g, "");
}) || [];
return tags;
}
This custom function strips the tag for code based answer checking, but this information might be useful to help me tune the prompt.
Question: Is there a way to render the content tag in the UI, even though it's not a part of the test, so I can see how it reached its conclusion?ad0ll
01/12/2025, 10:20 PMyaml
defaultTest:
assert:
- type: javascript
value: file://assert.js:getAssert
javascript
export function getAssert(output, context) {
const expectedTags = context.vars.tags.split(",");
// Extract actual tags from output
const actualTags =
output.match(/<tag>(.+?)<\/tag>/g)?.map((tag) => {
return tag.replace(/<\/?tag>/g, "");
}) || [];
// Check if all expected tags are present
const missingTags = expectedTags.filter((tag) => !actualTags.includes(tag));
// Check if there are any extra tags
const extraTags = actualTags.filter((tag) => !expectedTags.includes(tag));
const passResult = missingTags.length === 0 && extraTags.length === 0;
let reason = "";
if (missingTags.length > 0) {
reason += `Missing tags: ${missingTags.join(", ")}. `;
}
if (extraTags.length > 0) {
reason += `Extra tags: ${extraTags.join(", ")}. `;
}
if (reason === "") {
reason = "All expected tags present with no extras";
}
return {
pass: passResult,
score: passResult ? 1 : 0,
reason,
};
}