chrematt
11/28/2023, 10:49 AMyml
prompts:
- '{{query}}'
providers:
- exec:python retrieve.py --collection squad_train-v2.0_100
tests:
- assert:
- type: contains-all
value:
- context_1.txt
- type: python
value: "import sys\nexpected_output = 'context_1.txt'\noutput_lines = [line.strip()\
\ for line in output.split('\\n') if line.strip()]\nresult_recall_at_one = output_lines[0].startswith(expected_output)\
\ if output_lines else False\nprint(json.dumps({\n 'pass': result_recall_at_one,\n\
\ 'score': 1.0 if result_recall_at_one else 0.0,\n 'metric': 'Recall at\
\ One'\n}))"ian
11/28/2023, 2:04 PMian
11/28/2023, 3:34 PMyaml
- # ...
assert:
- type: icontains
value: arrr
metric: Pirateness
- # ...
assert:
- type: is-json
metric: Consistency
And the tags appear in the webview.
Do you think this would help your use case?chrematt
11/28/2023, 5:22 PMian
11/29/2023, 3:50 PMchrematt
11/30/2023, 10:00 AMjs
function calculateAndDisplayScores() {
const metricContainers = document.querySelectorAll('.custom-metric-container');
if (metricContainers.length === 0) {
console.error('No metric containers found');
return;
}
const metrics = {};
metricContainers.forEach(container => {
const metricElements = container.querySelectorAll('div');
metricElements.forEach(element => {
const parts = element.textContent.split(':');
const metricName = parts[0].trim();
const score = parseFloat(parts[1]);
if (!metrics[metricName]) {
metrics[metricName] = { trueCount: 0, totalCount: 0 };
}
metrics[metricName].totalCount++;
if (!isNaN(score) && score > 0) {
metrics[metricName].trueCount++;
}
});
});
// Calculate and display the results in the console
console.log('Scores:');
for (const [metric, data] of Object.entries(metrics)) {
const score = (data.trueCount / data.totalCount * 100).toFixed(2);
console.log(`${metric}: ${score}% (${data.trueCount}/${data.totalCount})`);
}
}
calculateAndDisplayScores();ian
12/05/2023, 2:07 AMyschermer
06/06/2024, 9:30 AMian
06/07/2024, 9:13 AM