{"serverId":"mcp-registry/ai.openmark/openmark","tools":[{"name":"estimate_cost","fixed":{"name":"estimate_cost","description":"Credits a run_benchmark call would use, before spending anything: the expected total, the split per model, the balance needed to start, the worst case, and the user's balance. Same inputs and run settings as run_benchmark. Use it to drop an expensive model, or to compare settings (e.g. max_tokens), before running.","inputSchema":{"type":"object","required":["task_yaml","models"],"properties":{"models":{"type":"array","items":{"type":"string"},"maxItems":20,"minItems":1},"preset":{"enum":["standard","thorough"],"type":"string","description":"Like the app's presets: standard = 2 passes + snappy timeouts (the default), thorough = 3 passes + patient timeouts. Explicit stability_runs / timeout_profile override it"},"fail_fast":{"type":"boolean","default":true,"description":"Stop a model after repeated errors to save credits. Set false to let every model finish, e.g. to diagnose failures"},"task_yaml":{"type":"string","description":"The full task YAML (a JSON object with the same fields also works)"},"max_tokens":{"type":"integer","default":500,"maximum":16000,"minimum":0,"description":"Output token limit per answer, 16 to 16000; 0 = no limit. Keep the default unless results show it was too low: a higher limit raises the credits needed to start. get_results flags models whose answers came back empty or near the limit; re-run those with a higher value"},"upload_uid":{"type":"string","description":"From prepare_attachments, so image and document tokens are counted"},"custom_timeout":{"type":"integer","maximum":600,"minimum":120,"description":"Seconds per answer, with timeout_profile custom"},"stability_runs":{"type":"integer","default":2,"maximum":5,"minimum":1,"description":"Passes per model; 2+ shows which models are consistent"},"temp_preference":{"enum":["low","high"],"type":"string","default":"low","description":"With temperature_discovery: which temperature wins a tie"},"timeout_profile":{"enum":["snappy","patient","custom"],"type":"string","default":"snappy","description":"How long to wait per answer: snappy 45s, patient 90s, custom = custom_timeout"},"temperature_discovery":{"type":"boolean","default":false,"description":"Find each model's best temperature for this task. Costs about 6x as much"}}},"annotations":{"readOnlyHint":true},"title":"Estimate a run's cost"},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":633,"tokensAfter":633},{"name":"get_account","fixed":{"name":"get_account","description":"The user's plan, credit balance and limits, and what a credit pack, Pro or Expert would unlock. Use it before large runs, or when a run is refused for credits or limits.","inputSchema":{"type":"object","properties":{}},"annotations":{"readOnlyHint":true},"title":"Account, credits and plans"},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":72,"tokensAfter":72},{"name":"get_results","fixed":{"name":"get_results","description":"Status of a benchmark job, and once completed the ranked results (score, stability, cost per run, latency), the tests each model missed with its answer, a recommendation for the chosen objective and quick picks for the others.","inputSchema":{"type":"object","required":["job_id"],"properties":{"job_id":{"type":"string"},"tolerance":{"type":"number","default":2,"maximum":50,"minimum":0,"description":"Score points below the top score that still count as good enough"},"show_misses":{"type":"boolean","default":true,"description":"List the tests each model got wrong, with the expected answer and the model's reply"},"optimize_for":{"enum":["accuracy","cost","speed","stability"],"type":"string","default":"accuracy","description":"accuracy (same order as the app's results table): best score, then cheapest, then fastest. cost / speed / stability: cheapest / fastest / most consistent model within `tolerance` points of the top score"}}},"annotations":{"readOnlyHint":true},"title":"Get benchmark results"},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":272,"tokensAfter":272},{"name":"get_task_format","fixed":{"name":"get_task_format","description":"How to write an OpenMark task: the YAML structure, all 18 scoring modes, and tips. Call this before writing a task. Pass a topic for the full detail on one subject (the app's Task Guide): fields, scoring, composite, chaining, documents, examples, best_practices, or a scoring mode name.","inputSchema":{"type":"object","properties":{"topic":{"type":"string","description":"Optional: fields, scoring, composite, chaining, documents, examples, best_practices, or a scoring mode name such as regex_match"}}},"annotations":{"readOnlyHint":true},"title":"Task YAML format"},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":152,"tokensAfter":152},{"name":"list_models","fixed":{"name":"list_models","description":"Current OpenMark model ids with list prices ($ per 1M tokens), as text and as a structured list. Use these ids in run_benchmark. Append @low, @medium or @high to a reasoning model id to set its reasoning effort.","inputSchema":{"type":"object","properties":{"search":{"type":"string","description":"Filter by id, name or provider, e.g. 'claude', 'gemini flash'"},"images_only":{"type":"boolean","description":"Only models that accept image attachments"},"max_output_price":{"type":"number","description":"Only models at or below this $ per 1M output tokens"}}},"annotations":{"readOnlyHint":true},"title":"List models"},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":167,"tokensAfter":167},{"name":"run_benchmark","fixed":{"name":"run_benchmark","description":"Save a task (YAML, see get_task_format) to the user's OpenMark account and run it on the given models with real API calls. Returns a job_id and the expected cost; poll get_results. Spends the user's credits; pass max_credits to only start within a budget. Optional run settings match the app's Configure & Run panel (preset, stability_runs, max_tokens, temperature_discovery, temp_preference, timeout_profile, custom_timeout, fail_fast).","inputSchema":{"type":"object","required":["task_yaml","models"],"properties":{"models":{"type":"array","items":{"type":"string"},"maxItems":20,"minItems":1,"description":"Model ids from list_models"},"preset":{"enum":["standard","thorough"],"type":"string","description":"Like the app's presets: standard = 2 passes + snappy timeouts (the default), thorough = 3 passes + patient timeouts. Explicit stability_runs / timeout_profile override it"},"fail_fast":{"type":"boolean","default":true,"description":"Stop a model after repeated errors to save credits. Set false to let every model finish, e.g. to diagnose failures"},"task_yaml":{"type":"string","description":"The full task YAML (a JSON object with the same fields also works)"},"max_tokens":{"type":"integer","default":500,"maximum":16000,"minimum":0,"description":"Output token limit per answer, 16 to 16000; 0 = no limit. Keep the default unless results show it was too low: a higher limit raises the credits needed to start. get_results flags models whose answers came back empty or near the limit; re-run those with a higher value"},"upload_uid":{"type":"string","description":"From prepare_attachments, when tests use attachments"},"max_credits":{"type":"integer","minimum":1,"description":"Budget: if the expected cost is higher, nothing runs and you get the quote instead"},"custom_timeout":{"type":"integer","maximum":600,"minimum":120,"description":"Seconds per answer, with timeout_profile custom"},"stability_runs":{"type":"integer","default":2,"maximum":5,"minimum":1,"description":"Passes per model; 2+ shows which models are consistent"},"temp_preference":{"enum":["low","high"],"type":"string","default":"low","description":"With temperature_discovery: which temperature wins a tie"},"timeout_profile":{"enum":["snappy","patient","custom"],"type":"string","default":"snappy","description":"How long to wait per answer: snappy 45s, patient 90s, custom = custom_timeout"},"temperature_discovery":{"type":"boolean","default":false,"description":"Find each model's best temperature for this task. Costs about 6x as much"}}},"annotations":{"readOnlyHint":false,"destructiveHint":false,"idempotentHint":false},"title":"Run a benchmark"},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: false, destructiveHint: false, idempotentHint: false (inferred from the name).","verify":true}],"tokensBefore":715,"tokensAfter":715},{"name":"validate_task","fixed":{"name":"validate_task","description":"Check a task YAML the way OpenMark will before running it: structure, scoring settings, and the test limit of the user's plan. Free, saves nothing. Returns the problems to fix and warnings worth a look.","inputSchema":{"type":"object","required":["task_yaml"],"properties":{"task_yaml":{"type":"string","description":"The full task YAML (a JSON object with the same fields also works)"}}},"annotations":{"readOnlyHint":true},"title":"Check a task"},"changes":[{"kind":"annotations","detail":"Added readOnlyHint: true (inferred from the name).","verify":true}],"tokensBefore":119,"tokensAfter":119}],"toolsChanged":7,"tokensSaved":0,"todo":[{"tool":"estimate_cost","params":["models"]},{"tool":"get_results","params":["job_id"]},{"tool":"watch_task","params":["job_id"]}],"note":"Deterministic fixes. Changes marked verify need a human look before shipping (renames, inferred annotations, inferred types, removed text). Parameter descriptions in `todo` need to be written by someone who knows the API."}