Add Krea2 eval entry templates
This commit is contained in:
@@ -42,6 +42,47 @@ def _require_text(errors: list[str], entry: dict[str, Any], key: str, min_len: i
|
||||
errors.append(f"{key} must be at least {min_len} characters")
|
||||
|
||||
|
||||
def _entry_id_slug(variant_key: str) -> str:
|
||||
value = variant_key.removeprefix("pov_")
|
||||
chars = [char.lower() if char.isalnum() else "-" for char in value]
|
||||
slug = "".join(chars).strip("-")
|
||||
while "--" in slug:
|
||||
slug = slug.replace("--", "-")
|
||||
return slug or "krea2-eval"
|
||||
|
||||
|
||||
def entry_template(
|
||||
variant_key: str,
|
||||
*,
|
||||
seed: int,
|
||||
source: str = "sxcp_eval_mcp",
|
||||
date: str = "",
|
||||
result: str = "inconclusive",
|
||||
decision: str = "needs_more_tests",
|
||||
commit: str = "pending",
|
||||
) -> dict[str, Any]:
|
||||
if not isinstance(seed, int) or isinstance(seed, bool):
|
||||
raise ValueError("seed must be an integer")
|
||||
variant = _text(variant_key).strip()
|
||||
if not variant:
|
||||
raise ValueError("variant_key is required")
|
||||
return {
|
||||
"id": f"{_entry_id_slug(variant)}-{seed}-eval",
|
||||
"date": date,
|
||||
"variant_key": variant,
|
||||
"seed": seed,
|
||||
"source": source,
|
||||
"result": result,
|
||||
"decision": decision,
|
||||
"baseline_prompt_summary": f"Replace this with what the generated {variant} prompt did before the edit.",
|
||||
"candidate_prompt_summary": f"Replace this with what the same-seed candidate prompt changed for {variant}.",
|
||||
"observation": f"Replace this with the fixed-seed Krea2 image comparison observation for {variant}.",
|
||||
"baseline_image": "",
|
||||
"candidate_image": "",
|
||||
"commit": commit,
|
||||
}
|
||||
|
||||
|
||||
def validate_entry(
|
||||
entry: dict[str, Any],
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user