", "one_liner_zh": "
", "primary_category": { "category_id": "software-development", "confidence": "medium", "name_en": "Software Development", "name_zh": "软件开发与交付", "reason": "matched_keywords:git" }, "target_user": "使用 chatgpt 等宿主 AI 的用户", "title_en": "arksim", "title_zh": "arksim 能力包", "visible_tags": [ { "label_en": "Browser Agents", "label_zh": "浏览器 Agent", "source": "repo_evidence_project_characteristics", "tag_id": "product_domain-browser-agents", "type": "product_domain" }, { "label_en": "Web Task Automation", "label_zh": "网页任务自动化", "source": "repo_evidence_project_characteristics", "tag_id": "user_job-web-task-automation", "type": "user_job" }, { "label_en": "Natural-language Web Actions", "label_zh": "自然语言网页操作", "source": "repo_evidence_project_characteristics", "tag_id": "core_capability-natural-language-web-actions", "type": "core_capability" }, { "label_en": "Page Observation and Action Planning", "label_zh": "页面观察与动作规划", "source": "repo_evidence_project_characteristics", "tag_id": "workflow_pattern-page-observation-and-action-planning", "type": "workflow_pattern" }, { "label_en": "Evaluation Suite", "label_zh": "评测体系", "source": "repo_evidence_project_characteristics", "tag_id": "selection_signal-evaluation-suite", "type": "selection_signal" } ] }, "packet_id": "phit_26cb20ea266a424793bec80ed8842fc0", "page_model": { "artifacts": { "artifact_slug": "arksim", "files": [ "PROJECT_PACK.json", "QUICK_START.md", "PROMPT_PREVIEW.md", "HUMAN_MANUAL.md", "AI_CONTEXT_PACK.md", "BOUNDARY_RISK_CARD.md", "PITFALL_LOG.md", "REPO_INSPECTION.json", "REPO_INSPECTION.md", "CAPABILITY_CONTRACT.json", "EVIDENCE_INDEX.json", "CLAIM_GRAPH.json" ], "required_files": [ "PROJECT_PACK.json", "QUICK_START.md", "PROMPT_PREVIEW.md", "HUMAN_MANUAL.md", "AI_CONTEXT_PACK.md", "BOUNDARY_RISK_CARD.md", "PITFALL_LOG.md", "REPO_INSPECTION.json" ] }, "detail": { "capability_source": "Project Hit Packet + DownstreamValidationResult", "commands": [ { "command": "pip install arksim", "label": "Python / pip · 官方安装入口", "source": "https://github.com/arklexai/arksim#readme", "verified": true } ], "display_tags": [ "浏览器 Agent", "网页任务自动化", "自然语言网页操作", "页面观察与动作规划", "评测体系" ], "eyebrow": "软件开发与交付", "glance": [ { "body": "判断自己是不是目标用户。", "label": "最适合谁", "value": "需要软件开发与交付能力，并使用 chatgpt的用户" }, { "body": "先理解能力边界，再决定是否继续。", "label": "核心价值", "value": "
" }, { "body": "未完成验证前保持审慎。", "label": "继续前", "value": "publish to Doramagic.ai project surfaces" } ], "guardrail_source": "Boundary & Risk Card", "guardrails": [ { "body": "Prompt Preview 只展示流程，不证明项目已安装或运行。", "label": "Check 1", "value": "不要把试用当真实运行" }, { "body": "chatgpt", "label": "Check 2", "value": "确认宿主兼容" }, { "body": "publish to Doramagic.ai project surfaces", "label": "Check 3", "value": "先隔离验证" } ], "mode": "skill, recipe, host_instruction, eval, preflight", "pitfall_log": { "items": [ { "body": "GitHub 社区证据显示该项目存在一个安装相关的待验证问题：v0.1.0", "category": "安装坑", "evidence": [ "community_evidence:github | cevd_d55b6435222f4142bf979809f0cf0794 | https://github.com/arklexai/arksim/releases/tag/v0.1.0 | 来源类型 github_release 暴露的待验证使用条件。" ], "severity": "medium", "suggested_check": "来源显示可能已有修复、规避或版本变化，说明书中必须标注适用版本。", "title": "来源证据：v0.1.0", "user_impact": "可能影响升级、迁移或版本选择。" }, { "body": "GitHub 社区证据显示该项目存在一个配置相关的待验证问题：v0.0.6", "category": "配置坑", "evidence": [ "community_evidence:github | cevd_57b2e0dc0cf94da8b4fc97ea744ae7e9 | https://github.com/arklexai/arksim/releases/tag/v0.0.6 | 来源类型 github_release 暴露的待验证使用条件。" ], "severity": "medium", "suggested_check": "来源显示可能已有修复、规避或版本变化，说明书中必须标注适用版本。", "title": "来源证据：v0.0.6", "user_impact": "可能增加新用户试用和生产接入成本。" }, { "body": "GitHub 社区证据显示该项目存在一个配置相关的待验证问题：v0.3.2", "category": "配置坑", "evidence": [ "community_evidence:github | cevd_3a2981d341cb4512a410e7d75e03baf0 | https://github.com/arklexai/arksim/releases/tag/v0.3.2 | 来源类型 github_release 暴露的待验证使用条件。" ], "severity": "medium", "suggested_check": "来源显示可能已有修复、规避或版本变化，说明书中必须标注适用版本。", "title": "来源证据：v0.3.2", "user_impact": "可能增加新用户试用和生产接入成本。" }, { "body": "GitHub 社区证据显示该项目存在一个配置相关的待验证问题：v0.3.4", "category": "配置坑", "evidence": [ "community_evidence:github | cevd_f949170a7f5d440cb0559c40fb441178 | https://github.com/arklexai/arksim/releases/tag/v0.3.4 | 来源类型 github_release 暴露的待验证使用条件。" ], "severity": "medium", "suggested_check": "来源显示可能已有修复、规避或版本变化，说明书中必须标注适用版本。", "title": "来源证据：v0.3.4", "user_impact": "可能阻塞安装或首次运行。" }, { "body": "GitHub 社区证据显示该项目存在一个配置相关的待验证问题：v0.3.5", "category": "配置坑", "evidence": [ "community_evidence:github | cevd_314def1c94614699b6c9ad728c36e9ab | https://github.com/arklexai/arksim/releases/tag/v0.3.5 | 来源类型 github_release 暴露的待验证使用条件。" ], "severity": "medium", "suggested_check": "来源显示可能已有修复、规避或版本变化，说明书中必须标注适用版本。", "title": "来源证据：v0.3.5", "user_impact": "可能增加新用户试用和生产接入成本。" }, { "body": "GitHub 社区证据显示该项目存在一个能力理解相关的待验证问题：v0.3.1", "category": "能力坑", "evidence": [ "community_evidence:github | cevd_c3e7552e9af84b95b2aec63c73dc7c26 | https://github.com/arklexai/arksim/releases/tag/v0.3.1 | 来源类型 github_release 暴露的待验证使用条件。" ], "severity": "medium", "suggested_check": "来源显示可能已有修复、规避或版本变化，说明书中必须标注适用版本。", "title": "来源证据：v0.3.1", "user_impact": "可能增加新用户试用和生产接入成本。" }, { "body": "README/documentation is current enough for a first validation pass.", "category": "能力坑", "evidence": [ "capability.assumptions | art_e3064c2689144cfb89a534e0544c6bfc | https://github.com/arklexai/arksim#readme | README/documentation is current enough for a first validation pass." ], "severity": "medium", "suggested_check": "将假设转成下游验证清单。", "title": "能力判断依赖假设", "user_impact": "假设不成立时，用户拿不到承诺的能力。" }, { "body": "GitHub 社区证据显示该项目存在一个维护/版本相关的待验证问题：v0.3.3", "category": "维护坑", "evidence": [ "community_evidence:github | cevd_03b0d8ce6f8a4bc2bc8e33a7636aa07c | https://github.com/arklexai/arksim/releases/tag/v0.3.3 | 来源类型 github_release 暴露的待验证使用条件。" ], "severity": "medium", "suggested_check": "来源显示可能已有修复、规避或版本变化，说明书中必须标注适用版本。", "title": "来源证据：v0.3.3", "user_impact": "可能增加新用户试用和生产接入成本。" }, { "body": "未记录 last_activity_observed。", "category": "维护坑", "evidence": [ "evidence.maintainer_signals | art_e3064c2689144cfb89a534e0544c6bfc | https://github.com/arklexai/arksim#readme | last_activity_observed missing" ], "severity": "medium", "suggested_check": "补 GitHub 最近 commit、release、issue/PR 响应信号。", "title": "维护活跃度未知", "user_impact": "新项目、停更项目和活跃项目会被混在一起，推荐信任度下降。" }, { "body": "no_demo", "category": "安全/权限坑", "evidence": [ "downstream_validation.risk_items | art_e3064c2689144cfb89a534e0544c6bfc | https://github.com/arklexai/arksim#readme | no_demo; severity=medium" ], "severity": "medium", "suggested_check": "进入安全/权限治理复核队列。", "title": "下游验证发现风险项", "user_impact": "下游已经要求复核，不能在页面中弱化。" }, { "body": "no_demo", "category": "安全/权限坑", "evidence": [ "risks.scoring_risks | art_e3064c2689144cfb89a534e0544c6bfc | https://github.com/arklexai/arksim#readme | no_demo; severity=medium" ], "severity": "medium", "suggested_check": "把风险写入边界卡，并确认是否需要人工复核。", "title": "存在评分风险", "user_impact": "风险会影响是否适合普通用户安装。" }, { "body": "GitHub 社区证据显示该项目存在一个安全/权限相关的待验证问题：v0.2.0", "category": "安全/权限坑", "evidence": [ "community_evidence:github | cevd_3c70cc242a1242b98d50cfe5078d6752 | https://github.com/arklexai/arksim/releases/tag/v0.2.0 | 来源讨论提到 python 相关条件，需在安装/试用前复核。" ], "severity": "medium", "suggested_check": "来源显示可能已有修复、规避或版本变化，说明书中必须标注适用版本。", "title": "来源证据：v0.2.0", "user_impact": "可能影响升级、迁移或版本选择。" }, { "body": "GitHub 社区证据显示该项目存在一个安全/权限相关的待验证问题：v0.3.0", "category": "安全/权限坑", "evidence": [ "community_evidence:github | cevd_69bd9f91a5064aeeb87f325f2f56f115 | https://github.com/arklexai/arksim/releases/tag/v0.3.0 | 来源类型 github_release 暴露的待验证使用条件。" ], "severity": "medium", "suggested_check": "来源显示可能已有修复、规避或版本变化，说明书中必须标注适用版本。", "title": "来源证据：v0.3.0", "user_impact": "可能影响升级、迁移或版本选择。" }, { "body": "issue_or_pr_quality=unknown。", "category": "维护坑", "evidence": [ "evidence.maintainer_signals | art_e3064c2689144cfb89a534e0544c6bfc | https://github.com/arklexai/arksim#readme | issue_or_pr_quality=unknown" ], "severity": "low", "suggested_check": "抽样最近 issue/PR，判断是否长期无人处理。", "title": "issue/PR 响应质量未知", "user_impact": "用户无法判断遇到问题后是否有人维护。" }, { "body": "release_recency=unknown。", "category": "维护坑", "evidence": [ "evidence.maintainer_signals | art_e3064c2689144cfb89a534e0544c6bfc | https://github.com/arklexai/arksim#readme | release_recency=unknown" ], "severity": "low", "suggested_check": "确认最近 release/tag 和 README 安装命令是否一致。", "title": "发布节奏不明确", "user_impact": "安装命令和文档可能落后于代码，用户踩坑概率升高。" } ], "source": "ProjectPitfallLog + ProjectHitPacket + validation + community signals", "summary": "发现 15 个潜在踩坑项，其中 0 个为 high/blocking；最高优先级：安装坑 - 来源证据：v0.1.0。", "title": "踩坑日志" }, "snapshot": { "contributors": null, "forks": null, "license": "unknown", "note": "站点快照，非实时质量证明；用于开工前背景判断。", "stars": null }, "source_url": "https://github.com/arklexai/arksim", "steps": [ { "body": "不安装项目，先体验能力节奏。", "code": "preview", "title": "先试 Prompt" }, { "body": "理解输入、输出、失败模式和边界。", "code": "manual", "title": "读说明书" }, { "body": "把上下文交给宿主 AI 继续工作。", "code": "context", "title": "带给 AI" }, { "body": "进入主力环境前先完成安装入口与风险边界验证。", "code": "verify", "title": "沙箱验证" } ], "subtitle": "
", "title": "arksim 能力包", "trial_prompt": "# arksim - Prompt Preview\n\n> Copy the prompt below into your AI host before installing anything.\n> Its purpose is to let you safely feel the project's workflow, not to claim the project has already run.\n\n## Copy this prompt\n\n```text\nYou are using an independent Doramagic capability pack for arklexai/arksim.\n\nProject:\n- Name: arksim\n- Repository: https://github.com/arklexai/arksim\n- Summary:
\n- Host target: chatgpt\n\nGoal:\nHelp me evaluate this project for the following task without installing it yet:
\n\nBefore taking action:\n1. Restate my task, success standard, and boundary.\n2. Identify whether the next step requires tools, browser access, network access, filesystem access, credentials, package installation, or host configuration.\n3. Use only the Doramagic Project Pack, the upstream repository, and the source-linked evidence listed below.\n4. If a real command, install step, API call, file write, or host integration is required, mark it as \"requires post-install verification\" and ask for approval first.\n5. If evidence is missing, say \"evidence is missing\" instead of filling the gap.\n\nPreviewable capabilities:\n- Capability 1: Use the source-backed project context to guide one small, checkable workflow step.\n\nCapabilities that require post-install verification:\n- Capability 1: Use the source-backed project context to guide one small, checkable workflow step.\n\nCore service flow:\n1. intro: Introduction to ArkSim. Produce one small intermediate artifact and wait for confirmation.\n2. quickstart: Quickstart Guide. Produce one small intermediate artifact and wait for confirmation.\n3. arch-overview: System Architecture Overview. Produce one small intermediate artifact and wait for confirmation.\n4. simulation-engine: Simulation Engine. Produce one small intermediate artifact and wait for confirmation.\n5. evaluation-system: Evaluation System. Produce one small intermediate artifact and wait for confirmation.\n\nSource-backed evidence to keep in mind:\n- https://github.com/arklexai/arksim#readme\n- .claude/skills/draft-pr/SKILL.md\n- .claude/skills/pre-review/SKILL.md\n- .claude/skills/review-pr/SKILL.md\n- README.md\n- arksim/__init__.py\n- arksim/constants.py\n- arksim/cli.py\n- arksim/templates/my_agent.py\n- arksim/templates/config.yaml\n\nFirst response rules:\n1. Start Step 1 only.\n2. Explain the one service action you will perform first.\n3. Ask exactly three questions about my target workflow, success standard, and sandbox boundary.\n4. Stop and wait for my answers.\n\nStep 1 follow-up protocol:\n- After I answer the first three questions, stay in Step 1.\n- Produce six parts only: clarified task, success standard, boundary conditions, two or three options, tradeoffs for each option, and one recommendation.\n- End by asking whether I confirm the recommendation.\n- Do not move to Step 2 until I explicitly confirm.\n\nConversation rules:\n- Advance one step at a time and wait for confirmation after each small artifact.\n- Write outputs as recommendations or planned checks, not as completed execution.\n- Do not claim tests passed, files changed, commands ran, APIs were called, or the project was installed.\n- If the user asks for execution, first provide the sandbox setup, expected output, rollback, and approval checkpoint.\n```\n", "voices": [ { "body": "来源平台：github, x。github/github_release: v0.3.7（https://github.com/arklexai/arksim/releases/tag/v0.3.7）；github/github_release: v0.3.6（https://github.com/arklexai/arksim/releases/tag/v0.3.6）；github/github_release: v0.3.5（https://github.com/arklexai/arksim/releases/tag/v0.3.5）；github/github_release: v0.3.4（https://github.com/arklexai/arksim/releases/tag/v0.3.4）；github/github_release: v0.3.3（https://github.com/arklexai/arksim/releases/tag/v0.3.3）；github/github_release: v0.3.2（https://github.com/arklexai/arksim/releases/tag/v0.3.2）；github/github_release: v0.3.1（https://github.com/arklexai/arksim/releases/tag/v0.3.1）；github/github_release: v0.3.0（https://github.com/arklexai/arksim/releases/tag/v0.3.0）；github/github_release: v0.2.0（https://github.com/arklexai/arksim/releases/tag/v0.2.0）；github/github_release: v0.1.0（https://github.com/arklexai/arksim/releases/tag/v0.1.0）；github/github_release: v0.0.6（https://github.com/arklexai/arksim/releases/tag/v0.0.6）；x: Zhou Yu (@Zhou_Yu_AI) / Highlights / X（https://x.com/Zhou_Yu_AI/highlights）。这些是项目级外部声音，不作为单独质量证明。", "items": [ { "kind": "github_release", "source": "github", "title": "v0.3.7", "url": "https://github.com/arklexai/arksim/releases/tag/v0.3.7" }, { "kind": "github_release", "source": "github", "title": "v0.3.6", "url": "https://github.com/arklexai/arksim/releases/tag/v0.3.6" }, { "kind": "github_release", "source": "github", "title": "v0.3.5", "url": "https://github.com/arklexai/arksim/releases/tag/v0.3.5" }, { "kind": "github_release", "source": "github", "title": "v0.3.4", "url": "https://github.com/arklexai/arksim/releases/tag/v0.3.4" }, { "kind": "github_release", "source": "github", "title": "v0.3.3", "url": "https://github.com/arklexai/arksim/releases/tag/v0.3.3" }, { "kind": "github_release", "source": "github", "title": "v0.3.2", "url": "https://github.com/arklexai/arksim/releases/tag/v0.3.2" }, { "kind": "github_release", "source": "github", "title": "v0.3.1", "url": "https://github.com/arklexai/arksim/releases/tag/v0.3.1" }, { "kind": "github_release", "source": "github", "title": "v0.3.0", "url": "https://github.com/arklexai/arksim/releases/tag/v0.3.0" }, { "kind": "github_release", "source": "github", "title": "v0.2.0", "url": "https://github.com/arklexai/arksim/releases/tag/v0.2.0" }, { "kind": "github_release", "source": "github", "title": "v0.1.0", "url": "https://github.com/arklexai/arksim/releases/tag/v0.1.0" }, { "kind": "github_release", "source": "github", "title": "v0.0.6", "url": "https://github.com/arklexai/arksim/releases/tag/v0.0.6" }, { "kind": "searxng_indexed", "source": "x", "title": "Zhou Yu (@Zhou_Yu_AI) / Highlights / X", "url": "https://x.com/Zhou_Yu_AI/highlights" } ], "status": "已收录 12 条来源", "title": "社区讨论" } ] }, "homepage_card": { "category": "软件开发与交付", "desc": "
",
"effort": "安装已验证",
"forks": null,
"icon": "code",
"name": "arksim 能力包",
"risk": "可发布",
"slug": "arksim",
"stars": null,
"tags": [
"浏览器 Agent",
"网页任务自动化",
"自然语言网页操作",
"页面观察与动作规划",
"评测体系"
],
"thumb": "gray",
"type": "Skill Pack"
},
"manual": {
"markdown": "# https://github.com/arklexai/arksim 项目说明书\n\n生成时间：2026-05-15 12:11:39 UTC\n\n## 目录\n\n- [Introduction to ArkSim](#intro)\n- [Quickstart Guide](#quickstart)\n- [System Architecture Overview](#arch-overview)\n- [Simulation Engine](#simulation-engine)\n- [Evaluation System](#evaluation-system)\n- [Agent Types and Integration](#agent-types)\n- [LLM Provider Integration](#llm-providers)\n- [Tool Call Capture](#tool-call-capture)\n- [Scenario Management](#scenario-management)\n- [Configuration System](#configuration)\n\n\n\n## Introduction to ArkSim\n\n### 相关页面\n\n相关主题：[Quickstart Guide](#quickstart), [System Architecture Overview](#arch-overview)\n\n How arksim connects to your agent. How arksim connects to your agent. \n- Host target: chatgpt\n\nGoal:\nHelp me evaluate this project for the following task without installing it yet: \n\nBefore taking action:\n1. Restate my task, success standard, and boundary.\n2. Identify whether the next step requires tools, browser access, network access, filesystem access, credentials, package installation, or host configuration.\n3. Use only the Doramagic Project Pack, the upstream repository, and the source-linked evidence listed below.\n4. If a real command, install step, API call, file write, or host integration is required, mark it as \"requires post-install verification\" and ask for approval first.\n5. If evidence is missing, say \"evidence is missing\" instead of filling the gap.\n\nPreviewable capabilities:\n- Capability 1: Use the source-backed project context to guide one small, checkable workflow step.\n\nCapabilities that require post-install verification:\n- Capability 1: Use the source-backed project context to guide one small, checkable workflow step.\n\nCore service flow:\n1. intro: Introduction to ArkSim. Produce one small intermediate artifact and wait for confirmation.\n2. quickstart: Quickstart Guide. Produce one small intermediate artifact and wait for confirmation.\n3. arch-overview: System Architecture Overview. Produce one small intermediate artifact and wait for confirmation.\n4. simulation-engine: Simulation Engine. Produce one small intermediate artifact and wait for confirmation.\n5. evaluation-system: Evaluation System. Produce one small intermediate artifact and wait for confirmation.\n\nSource-backed evidence to keep in mind:\n- https://github.com/arklexai/arksim#readme\n- .claude/skills/draft-pr/SKILL.md\n- .claude/skills/pre-review/SKILL.md\n- .claude/skills/review-pr/SKILL.md\n- README.md\n- arksim/__init__.py\n- arksim/constants.py\n- arksim/cli.py\n- arksim/templates/my_agent.py\n- arksim/templates/config.yaml\n\nFirst response rules:\n1. Start Step 1 only.\n2. Explain the one service action you will perform first.\n3. Ask exactly three questions about my target workflow, success standard, and sandbox boundary.\n4. Stop and wait for my answers.\n\nStep 1 follow-up protocol:\n- After I answer the first three questions, stay in Step 1.\n- Produce six parts only: clarified task, success standard, boundary conditions, two or three options, tradeoffs for each option, and one recommendation.\n- End by asking whether I confirm the recommendation.\n- Do not move to Step 2 until I explicitly confirm.\n\nConversation rules:\n- Advance one step at a time and wait for confirmation after each small artifact.\n- Write outputs as recommendations or planned checks, not as completed execution.\n- Do not claim tests passed, files changed, commands ran, APIs were called, or the project was installed.\n- If the user asks for execution, first provide the sandbox setup, expected output, rollback, and approval checkpoint.\n```\n",
"summary": "不安装项目也能感受能力节奏的安全试用 Prompt。",
"title": "Prompt Preview / 安装前试用 Prompt"
},
"quick_start": {
"asset_id": "quick_start",
"filename": "QUICK_START.md",
"markdown": "# Quick Start / 官方入口\n\n项目：arklexai/arksim\n\n## 官方安装入口\n\n### Python / pip · 官方安装入口\n\n```bash\npip install arksim\n```\n\n来源：https://github.com/arklexai/arksim#readme\n\n## 来源\n\n- docs: https://github.com/arklexai/arksim#readme\n",
"summary": "从项目官方 README 或安装文档提取的开工入口。",
"title": "Quick Start / 官方入口"
}
},
"validation_id": "dval_3536c77cf68640e68b2a7b029788aeed"
}
相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [README.md](https://github.com/arklexai/arksim/blob/main/README.md)\n- [CONTRIBUTING.md](https://github.com/arklexai/arksim/blob/main/CONTRIBUTING.md)\n- [CLAUDE.md](https://github.com/arklexai/arksim/blob/main/CLAUDE.md)\n- [arksim/simulation_engine/tool_types.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n- [examples/customer-service/custom_metrics.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n- [examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n- [examples/integrations/langgraph/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/langgraph/README.md)\n- [examples/integrations/autogen/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/autogen/README.md)\n- [examples/ci/README.md](https://github.com/arklexai/arksim/blob/main/examples/ci/README.md)\n
config.yaml] --> B[Simulation Engine]\n C[Scenarios
scenarios.json] --> B\n D[Agent
custom_agent.py] --> B\n B --> E[Conversation Data]\n E --> F[Evaluator]\n F --> G[HTML Report
final_report.html]\n H[Custom Metrics
custom_metrics.py] --> F\n```\n\n### Core Components\n\n| Component | Description |\n|-----------|-------------|\n| **Simulation Engine** | Orchestrates multi-turn conversations between simulated users and the agent |\n| **Evaluator** | Scores agent performance using built-in and custom metrics |\n| **CLI Interface** | Command-line interface for running simulations (`arksim simulate-evaluate`) |\n| **Report Generator** | Produces HTML reports with visualizations and conversation transcripts |\n\n资料来源：[examples/customer-service/README.md:1-25](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n## Agent Integration Methods\n\nArkSim supports multiple agent integration patterns to accommodate different agent architectures.\n\n### Python Class Integration (BaseAgent)\n\nThe default integration method uses a Python class inheriting from `BaseAgent`. This pattern provides maximum flexibility for connecting any agent implementation.\n\n```python\nfrom arksim.simulation_engine.agent.base import BaseAgent\nfrom arksim.simulation_engine.tool_types import AgentResponse\n\nclass MyAgent(BaseAgent):\n async def get_chat_id(self) -> str:\n return \"unique-id\"\n\n async def execute(self, user_query: str, **kwargs: object) -> str | AgentResponse:\n # Replace with your agent logic\n return \"agent response\"\n```\n\nThe agent can return either:\n- A plain string response\n- An `AgentResponse` object containing both content and tool calls\n\n资料来源：[README.md:40-55](https://github.com/arklexai/arksim/blob/main/README.md)\n\n### AgentResponse Data Model\n\nThe `AgentResponse` model encapsulates structured agent outputs:\n\n```python\nclass AgentResponse(BaseModel):\n \"\"\"Structured return from agent execution, carrying both text and tool calls.\"\"\"\n model_config = ConfigDict(extra=\"ignore\")\n \n content: str\n tool_calls: list[ToolCall] = Field(default_factory=list)\n```\n\n资料来源：[arksim/simulation_engine/tool_types.py:32-42](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n\n### ToolCall Model\n\nTool calls are captured using the `ToolCall` model:\n\n```python\nclass ToolCall(BaseModel):\n \"\"\"A single tool/function call observed during a turn.\"\"\"\n model_config = ConfigDict(extra=\"ignore\")\n \n id: str\n name: str\n arguments: dict[str, Any] = Field(default_factory=dict)\n result: str | None = None\n error: str | None = None\n source: ToolCallSource | None = None\n```\n\nThe `extra=\"ignore\"` configuration ensures forward compatibility with future versions.\n\n资料来源：[arksim/simulation_engine/tool_types.py:18-30](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n\n### Chat Completions Endpoint Integration\n\nFor agents exposing a standard OpenAI-compatible API:\n\n```yaml\nagent_config:\n agent_type: chat_completions\n agent_name: my-agent\n api_config:\n endpoint: http://localhost:8000/v1/chat/completions\n```\n\n资料来源：[README.md:57-62](https://github.com/arklexai/arksim/blob/main/README.md)\n\n### A2A Protocol Integration\n\nFor agents implementing the Agent-to-Agent (A2A) protocol:\n\n```yaml\nagent_config:\n agent_type: a2a\n agent_name: my-agent\n api_config:\n endpoint: http://localhost:9999/agent\n```\n\n资料来源：[README.md:64-70](https://github.com/arklexai/arksim/blob/main/README.md)\n\n### Automatic Tool Call Capture (Tracing)\n\nArkSim supports automatic tool call capture through a tracing processor, eliminating the need for agents to explicitly return tool calls:\n\n```mermaid\ngraph LR\n A[Simulator sets
routing context] --> B[agent.execute()]\n B --> C[SDK fires
TracingProcessor.on_span_end]\n C --> D[arksim captures
tool calls]\n D --> E[Evaluator scores]\n```\n\n```bash\npip install -r requirements-traced.txt\narksim simulate-evaluate config_traced.yaml\n```\n\n资料来源：[examples/customer-service/README.md:35-55](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n## Configuration System\n\nArkSim uses YAML configuration files to define simulation and evaluation parameters.\n\n### Configuration File Structure\n\n```yaml\n# Simulation settings\nsimulation:\n max_turns: 10\n timeout: 300\n\n# Agent configuration\nagent_config:\n agent_type: chat_completions # or 'a2a', 'custom'\n agent_name: my-agent\n api_config:\n endpoint: http://localhost:8000/v1/chat/completions\n\n# Trace receiver (optional)\ntrace_receiver:\n enabled: true\n wait_timeout: 5\n\n# Custom metrics\ncustom_metrics_file_paths:\n - ./custom_metrics.py\n\nmetrics_to_run:\n - goal_completion\n - verification_compliance\n```\n\n资料来源：[examples/ci/README.md:1-25](https://github.com/arklexai/arksim/blob/main/examples/ci/README.md)\n\n### Scenario Definition\n\nScenarios are defined in `scenarios.json` with the following structure:\n\n| Field | Description |\n|-------|-------------|\n| `scenario_id` | Unique identifier for the scenario |\n| `user_profile` | Description of the simulated user |\n| `knowledge` | Context and information available to the user |\n| `goals` | Objectives the user aims to achieve |\n| `expected_behavior` | Expected agent behavior patterns |\n\n资料来源：[examples/integrations/dify/README.md:1-20](https://github.com/arklexai/arksim/blob/main/examples/integrations/dify/README.md)\n\n## Evaluation Metrics\n\nArkSim provides built-in metrics and supports custom metric definitions.\n\n### Built-in Metrics\n\n| Metric | Description | Weight |\n|--------|-------------|--------|\n| Goal Completion | Whether the agent achieved the user's objectives | 60% |\n| Turn Success Ratio | Ratio of successful turns to total turns | 40% |\n| Final Score | Weighted average of other metrics | - |\n\n资料来源：[arksim/utils/html_report/report_template.html:80-95](https://github.com/arklexai/arksim/blob/main/arksim/utils/html_report/report_template.html)\n\n### Custom Metrics\n\nCustom metrics can be implemented as either quantitative (scored numerically) or qualitative (evaluated via LLM).\n\n#### Quantitative Metric Pattern\n\n```python\nfrom arksim.evaluator import (\n QuantitativeMetric,\n QuantResult,\n ScoreInput,\n format_chat_history,\n)\n\nclass VerificationComplianceMetric(Queresult):\n identity_verification: float # 0.0-1.0\n action_gating: float # 0.0-1.0\n reason: str\n\nVERIFICATION_COMPLIANCE_SYSTEM_PROMPT = \"\"\"\\\nYou are an impartial evaluator for a customer service agent.\nScore how well the agent followed identity verification protocols...\"\"\"\n\ndef verification_compliance_metric(\n input_data: ScoreInput,\n) -> QuantResult:\n # Implementation\n return QuantResult(score=0.85, reason=\"Verification completed\")\n```\n\n资料来源：[examples/customer-service/custom_metrics.py:15-40](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n\n#### Qualitative Metric Pattern\n\n```python\nfrom arksim.evaluator import (\n QualitativeMetric,\n QualResult,\n ScoreInput,\n format_chat_history,\n)\n\ndef helpfulness_metric(input_data: ScoreInput) -> QualResult:\n system_prompt = \"\"\"Evaluate the helpfulness of the agent response...\"\"\"\n return QualResult(\n assessment=\"The agent provided comprehensive assistance\",\n score=0.9,\n reason=\"Clear explanation with actionable steps\"\n )\n```\n\n### Metric Configuration\n\nTo use custom metrics:\n\n1. Implement the metric function in a Python file\n2. Reference the file path in `custom_metrics_file_paths` in config.yaml\n3. Add the metric name to `metrics_to_run`\n\n资料来源：[examples/customer-service/custom_metrics.py:1-15](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n\n## CLI Usage\n\nArkSim provides a command-line interface for running simulations and evaluations.\n\n### Primary Commands\n\n| Command | Description |\n|---------|-------------|\n| `arksim simulate-evaluate config.yaml` | Run simulation and evaluation in one step |\n| `arksim simulate config_simulate.yaml` | Run simulation only |\n| `arksim evaluate config_evaluate.yaml` | Evaluate existing simulation results |\n\n资料来源：[examples/customer-service/README.md:15-22](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n### Workflow Examples\n\n#### Combined Simulation and Evaluation\n\n```bash\narksim simulate-evaluate config.yaml\n```\n\n#### Separate Simulation and Evaluation\n\n```bash\n# Step 1: Simulate\narksim simulate config_simulate.yaml\n\n# Step 2: Evaluate\narksim evaluate config_evaluate.yaml\n```\n\nResults are written to `./results/simulation/simulation.json`. The evaluation report is printed to stdout with per-scenario metric scores and failure analysis.\n\n资料来源：[examples/integrations/dify/README.md:15-25](https://github.com/arklexai/arksim/blob/main/examples/integrations/dify/README.md)\n\n## Framework Integrations\n\nArkSim provides integration examples for popular agent frameworks.\n\n### Integration Matrix\n\n| Framework | Package Required | Documentation |\n|-----------|-----------------|---------------|\n| LangGraph | `langgraph langchain-openai` | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/langgraph/README.md) |\n| LangChain | `langgraph langchain-openai` | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/langchain/README.md) |\n| AutoGen | `autogen-agentchat autogen-ext[openai]` | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/autogen/README.md) |\n| Claude Agent SDK | `claude-agent-sdk` | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/claude-agent-sdk/README.md) |\n| Rasa | `rasa-pro` | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/rasa/README.md) |\n| Dify | (custom agent) | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/dify/README.md) |\n| OpenClaw | (custom agent) | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/openclaw/README.md) |\n\n资料来源：[examples/integrations/langgraph/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/langgraph/README.md), [examples/integrations/autogen/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/autogen/README.md)\n\n### General Integration Pattern\n\nRegardless of the framework, the integration follows this pattern:\n\n1. Create a `custom_agent.py` file with a `BaseAgent` subclass\n2. Implement the `execute()` method to call the framework's agent\n3. Return either a string or `AgentResponse` with tool calls\n4. Configure `config.yaml` to use the custom agent\n5. Create `scenarios.json` with test cases\n\n```python\nfrom arksim.simulation_engine.agent.base import BaseAgent\nfrom arksim.simulation_engine.tool_types import AgentResponse\n\nclass FrameworkAgent(BaseAgent):\n async def get_chat_id(self) -> str:\n return \"unique-session-id\"\n\n async def execute(self, user_query: str, **kwargs: object) -> str | AgentResponse:\n # Call your framework's agent here\n result = await your_agent.run(user_query)\n return str(result)\n```\n\n## Report Generation\n\nArkSim generates comprehensive HTML evaluation reports containing:\n\n### Report Sections\n\n| Section | Content |\n|---------|---------|\n| **Summary** | Overall scores, pass/fail status |\n| **Per-Scenario Scores** | Individual metric scores for each scenario |\n| **Failure Categories** | Grouped failure patterns with counts |\n| **Conversation Viewer** | Full transcript of each simulated conversation |\n| **Score Explanations** | Rationale for each score with references to conversation turns |\n\n资料来源：[README.md:25-30](https://github.com/arklexai/arksim/blob/main/README.md)\n\n### Report Output\n\nThe report tells you where your agent is strong and where it breaks. You get per-metric scores, categorized failures, and full conversation transcripts so you can read the exact turns where things went wrong.\n\n资料来源：[README.md:30-35](https://github.com/arklexai/arksim/blob/main/README.md)\n\n## Development Guidelines\n\n### Project Setup\n\n```bash\n# Fork and clone\ngit clone https://github.com/相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/cli.py](https://github.com/arklexai/arksim/blob/main/arksim/cli.py)\n- [arksim/simulation_engine/agent/base.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/agent/base.py)\n- [arksim/simulation_engine/tool_types.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n- [examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n- [examples/integrations/langchain/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/langchain/README.md)\n- [README.md](https://github.com/arklexai/arksim/blob/main/README.md)\n相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/simulation_engine/tool_types.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n- [arksim/cli.py](https://github.com/arklexai/arksim/blob/main/arksim/cli.py)\n- [README.md](https://github.com/arklexai/arksim/blob/main/README.md)\n- [examples/customer-service/custom_metrics.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n- [examples/e-commerce/custom_metrics.py](https://github.com/arklexai/arksim/blob/main/examples/e-commerce/custom_metrics.py)\n- [arksim/utils/html_report/report_template.html](https://github.com/arklexai/arksim/blob/main/arksim/utils/html_report/report_template.html)\n
Python class extending BaseAgent]\n HTTP[Chat Completions
HTTP endpoint /v1/chat/completions]\n A2A[A2A Protocol
Agent-to-Agent standard]\n end\n```\n\n1. **Custom Agent (Python class)**: Extend `BaseAgent` and implement the `execute()` method\n2. **Chat Completions**: Configure an HTTP endpoint for OpenAI-compatible chat completions API\n3. **A2A Protocol**: Connect via the Agent-to-Agent protocol standard\n\n资料来源：[README.md](https://github.com/arklexai/arksim/blob/main/README.md)\n\n## Simulation Engine\n\nThe simulation engine orchestrates multi-turn conversations between a user simulator and the agent under test. Each turn consists of:\n\n1. User simulator generates a response based on conversation history and scenario knowledge\n2. Agent executes the user query and returns a response (optionally with tool calls)\n3. Simulator captures the interaction for evaluation\n\n### Tool Call Capture\n\nArkSim captures tool/function calls in two ways:\n\n| Capture Method | Mechanism | Configuration |\n|----------------|-----------|---------------|\n| AgentResponse | Agent returns structured `AgentResponse` with `tool_calls` list | Default behavior |\n| TracingProcessor | SDK's `TracingProcessor.on_span_end` captures calls automatically | `trace_receiver.enabled: true` |\n\n资料来源：[examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n#### Tool Call Data Model\n\nTool calls are represented by the `ToolCall` class:\n\n```python\nclass ToolCall(BaseModel):\n id: str\n name: str\n arguments: dict[str, Any] = Field(default_factory=dict)\n result: str | None = None\n error: str | None = None\n source: ToolCallSource | None = None\n```\n\nThe `AgentResponse` wraps both content and tool calls:\n\n```python\nclass AgentResponse(BaseModel):\n content: str\n tool_calls: list[ToolCall] = Field(default_factory=list)\n```\n\n资料来源：[arksim/simulation_engine/tool_types.py:1-50](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n\n### Traced Agent Flow\n\n```mermaid\nsequenceDiagram\n participant Simulator\n participant Agent\n participant Tracing as ArksimTracingProcessor\n participant Evaluator\n\n Simulator->>Agent: execute(user_query)\n Agent->>Tracing: on_span_end(span)\n Tracing->>Simulator: captured_tool_calls\n Simulator->>Agent: RunResult\n Agent->>Simulator: AgentResponse\n Simulator->>Evaluator: Tool calls + Conversation\n Evaluator->>Simulator: Metric Scores\n```\n\n资料来源：[examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n## Evaluator\n\nThe evaluator scores agent performance using both quantitative and qualitative metrics. The evaluation framework supports custom metrics defined via Python files.\n\n### Metric Types\n\n| Type | Description | Scoring Method |\n|------|-------------|----------------|\n| QuantitativeMetric | Numerical scores (0.0-1.0 scale) | Structured JSON schema validation |\n| QualitativeMetric | Free-form evaluation with reasoning | LLM-generated analysis |\n\n资料来源：[examples/customer-service/custom_metrics.py:1-60](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n\n### Custom Metrics Structure\n\nCustom metrics require:\n\n1. A Pydantic `BaseModel` defining the output schema\n2. A system prompt describing evaluation criteria\n3. Implementation of `QuantitativeMetric` or `QualitativeMetric`\n\n```python\nclass ConversionSchema(BaseModel):\n intent_strength: float\n conversion_outcome: float\n evidence: list[str]\n reason: str\n\nCONVERSION_SYSTEM_PROMPT = \"\"\"\\\nYou are an impartial evaluator for an e-commerce shopping agent.\nYour job is to score (1) the shopper's purchase intent and (2) whether the agent achieved a conversion outcome...\n\"\"\"\n```\n\n资料来源：[examples/e-commerce/custom_metrics.py:1-40](https://github.com/arklexai/arksim/blob/main/examples/e-commerce/custom_metrics.py)\n\n### Score Calculation\n\nThe evaluator computes a **Final Score** as a weighted average:\n\n| Component | Weight | Description |\n|-----------|--------|-------------|\n| Turn Success Ratio | 40% | Ratio of successful turns to total turns |\n| Goal Completion Score | 60% | LLM-assessed goal achievement score |\n\n| Status | Condition |\n|--------|-----------|\n| Done | Final score = 1.0 |\n| Partial Failure | 0.0 < Final score < 1.0 |\n| Complete Failure | Final score = 0.0 |\n\n资料来源：[arksim/utils/html_report/report_template.html:1-50](https://github.com/arklexai/arksim/blob/main/arksim/utils/html_report/report_template.html)\n\n## Data Flow\n\n```mermaid\ngraph LR\n subgraph Input[\"Input\"]\n Config[config.yaml]\n Scenarios[scenarios.json]\n Metrics[custom_metrics.py]\n end\n\n subgraph Process[\"Processing\"]\n Sim[Simulation]\n Eval[Evaluation]\n Score[Scoring]\n end\n\n subgraph Output[\"Output\"]\n Results[simulation.json]\n Report[evaluation.html]\n end\n\n Config --> Sim\n Scenarios --> Sim\n Sim --> Eval\n Metrics --> Eval\n Eval --> Score\n Score --> Results\n Score --> Report\n```\n\n## Agent Connector Types\n\nArkSim supports multiple agent integration patterns via the configuration system:\n\n```yaml\n# Option 1: Custom Python Agent\nagent_config:\n agent_type: custom\n agent_name: my-agent\n\n# Option 2: Chat Completions HTTP Endpoint\nagent_config:\n agent_type: chat_completions\n agent_name: my-agent\n api_config:\n endpoint: http://localhost:8000/v1/chat/completions\n\n# Option 3: A2A Protocol\nagent_config:\n agent_type: a2a\n agent_name: my-agent\n api_config:\n endpoint: http://localhost:9999/agent\n```\n\n资料来源：[README.md](https://github.com/arklexai/arksim/blob/main/README.md)\n\n## User Simulator\n\nThe user simulator generates realistic multi-turn conversations based on:\n\n- **Scenario definitions**: Predefined conversation flows and expected behaviors\n- **User profiles**: Demographic and behavioral attributes for persona simulation\n- **Knowledge bases**: Domain-specific information the simulated user possesses\n- **Conversation history**: Full context of the multi-turn interaction\n\nThe simulator uses LLM-based generation to produce contextually appropriate user responses that evolve through the conversation based on agent actions and conversation state.\n\n## HTML Report Generation\n\nAfter evaluation completes, ArkSim generates an interactive HTML report containing:\n\n| Section | Content |\n|---------|---------|\n| Summary Statistics | Overall scores, pass/fail rates |\n| Per-Scenario Metrics | Individual metric scores with reasoning |\n| Failure Categories | Grouped analysis of failure types |\n| Conversation Transcripts | Full turn-by-turn dialogue viewer |\n\n资料来源：[arksim/utils/html_report/report_template.html:1-100](https://github.com/arklexai/arksim/blob/main/arksim/utils/html_report/report_template.html)\n\n## Integration Examples\n\nArkSim provides example integrations for popular agent frameworks:\n\n| Framework | Example Location | Connection Method |\n|-----------|-----------------|-------------------|\n| LangGraph | `examples/integrations/langgraph/` | Custom agent connector |\n| LangChain | `examples/integrations/langchain/` | Custom agent connector |\n| Claude Agent SDK | `examples/integrations/claude-agent-sdk/` | Custom agent connector |\n| AutoGen | `examples/integrations/autogen/` | Custom agent connector |\n| Pydantic AI | `examples/integrations/pydantic-ai/` | Custom agent connector |\n| Dify | `examples/integrations/dify/` | HTTP client |\n\n资料来源：[examples/integrations/langgraph/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/langgraph/README.md), [examples/integrations/claude-agent-sdk/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/claude-agent-sdk/README.md), [examples/integrations/autogen/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/autogen/README.md)\n\n## Configuration Schema\n\nThe main configuration file (`config.yaml`) controls all aspects of simulation and evaluation:\n\n| Section | Key Options |\n|---------|-------------|\n| `agent_config` | `agent_type`, `agent_name`, `api_config.endpoint` |\n| `scenario_file` | Path to scenarios.json |\n| `metrics_to_run` | List of metric names to execute |\n| `custom_metrics_file_paths` | Paths to custom metric Python files |\n| `trace_receiver.enabled` | Enable TracingProcessor capture (default: false) |\n| `trace_receiver.wait_timeout` | Timeout for trace capture (seconds) |\n\n资料来源：[examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n## Web UI\n\nArkSim includes a web-based control plane for managing simulations:\n\n```mermaid\ngraph TD\n subgraph UI[\"Web UI\"]\n Build[Build Scenarios]\n Load[Load Existing]\n Run[Run Simulation]\n View[View Results]\n end\n\n subgraph Features[\"UI Features\"]\n AutoGen[Auto-generate Scenarios PRO]\n Browse[File Browser]\n Refresh[Refresh Results]\n end\n```\n\nFeatures include:\n- Scenario building and loading\n- Auto-generate scenarios (PRO feature)\n- File browser integration\n- Results viewing and refresh\n\n资料来源：[arksim/ui/frontend/index.html:1-80](https://github.com/arklexai/arksim/blob/main/arksim/ui/frontend/index.html)\n\n## Summary\n\nArkSim provides a comprehensive multi-turn agent evaluation framework with:\n\n- **Flexible agent integration**: Support for custom Python agents, HTTP endpoints, and A2A protocol\n- **Tool call capture**: Two mechanisms for capturing agent tool executions\n- **Customizable metrics**: Both quantitative and qualitative evaluation approaches\n- **Interactive reporting**: HTML-based results with conversation viewer\n- **CLI and UI**: Command-line and web-based interfaces for running evaluations\n- **Framework integrations**: Pre-built examples for LangGraph, LangChain, Claude SDK, AutoGen, Pydantic AI, and Dify\n\n---\n\n\n\n## Simulation Engine\n\n### 相关页面\n\n相关主题：[System Architecture Overview](#arch-overview), [Evaluation System](#evaluation-system), [Scenario Management](#scenario-management)\n\n相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/simulation_engine/simulator.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/simulator.py)\n- [arksim/simulation_engine/entities.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/entities.py)\n- [arksim/simulation_engine/core/multi_knowledge_handling.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/core/multi_knowledge_handling.py)\n- [arksim/simulation_engine/agent/base.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/agent/base.py)\n- [arksim/simulation_engine/tool_types.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n- [arksim/simulation_engine/utils/prompts.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/utils/prompts.py)\nRelevant Source Files
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/evaluator/evaluator.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/evaluator.py)\n- [arksim/evaluator/builtin_metrics.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/builtin_metrics.py)\n- [arksim/evaluator/base_metric.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/base_metric.py)\n- [arksim/evaluator/tool_call_metrics.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/tool_call_metrics.py)\n- [arksim/evaluator/trajectory_matching.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/trajectory_matching.py)\n- [arksim/evaluator/thresholds.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/thresholds.py)\n- [arksim/utils/html_report/generate_html_report.py](https://github.com/arklexai/arksim/blob/main/arksim/utils/html_report/generate_html_report.py)\nRelevant Source Files
\n\n以下源码文件用于生成本页说明：\n\n- [README.md](https://github.com/arklexai/arksim/blob/main/README.md) - Main documentation with agent type examples\n- [arksim/cli.py](https://github.com/arklexai/arksim/blob/main/arksim/cli.py) - CLI commands for agent initialization\n- [arksim/ui/frontend/index.html](https://github.com/arklexai/arksim/blob/main/arksim/ui/frontend/index.html) - UI Agent Config section\n- [examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md) - Custom agent implementation example\n- [examples/integrations/dify/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/dify/README.md) - HTTP agent integration example\n- [examples/integrations/rasa/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/rasa/README.md) - External agent server integration\nAgent Config
\n 相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/llms/chat/providers/openai.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/providers/openai.py)\n- [arksim/llms/chat/providers/anthropic.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/providers/anthropic.py)\n- [arksim/llms/chat/providers/google.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/providers/google.py)\n- [arksim/llms/chat/providers/azure_openai.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/providers/azure_openai.py)\n- [arksim/llms/chat/llm.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/llm.py)\n- [arksim/llms/chat/base/base_llm.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/base/base_llm.py)\n
arksim/llms/chat/llm.py]\n B --> C[Base LLM
arksim/llms/chat/base/base_llm.py]\n C --> D[OpenAI Provider
providers/openai.py]\n C --> E[Anthropic Provider
providers/anthropic.py]\n C --> F[Google Provider
providers/google.py]\n C --> G[Azure OpenAI Provider
providers/azure_openai.py]\n \n H[Configuration YAML] --> B\n I[Environment Variables] --> D\n I --> E\n I --> F\n I --> G\n```\n\n### Provider Class Hierarchy\n\n```mermaid\ngraph TD\n A[BaseLLM
base/base_llm.py] --> B[OpenAIProvider
providers/openai.py]\n A --> C[AnthropicProvider
providers/anthropic.py]\n A --> D[GoogleProvider
providers/google.py]\n A --> E[AzureOpenAIProvider
providers/azure_openai.py]\n \n F[Provider Enum] --> A\n G[ChatMessage Model] --> A\n H[ChatCompletionResponse] --> A\n```\n\n## Supported Providers\n\nArkSim supports the following LLM providers through dedicated provider implementations:\n\n| Provider | Provider Class | API Style | Streaming Support |\n|----------|----------------|-----------|-------------------|\n| OpenAI | `OpenAIProvider` | OpenAI API | Yes |\n| Anthropic | `AnthropicProvider` | Anthropic API | Yes |\n| Google | `GoogleProvider` | Google AI API | Yes |\n| Azure OpenAI | `AzureOpenAIProvider` | Azure API | Yes |\n\nEach provider class inherits from `BaseLLM` and implements provider-specific API call logic while adhering to the common interface contract.\n\n## Base LLM Interface\n\nThe `BaseLLM` class defines the contract that all provider implementations must follow. This ensures consistent behavior across different LLM backends.\n\n### Core Methods\n\n| Method | Purpose | Parameters |\n|--------|---------|------------|\n| `chat()` | Send a chat completion request | `messages`, `model`, `temperature`, `max_tokens`, `**kwargs` |\n| `chat_stream()` | Stream chat completion responses | `messages`, `model`, `temperature`, `max_tokens`, `**kwargs` |\n| `count_tokens()` | Calculate token usage for messages | `messages`, `model` |\n\n### Data Models\n\nThe base module defines essential data structures used across all providers:\n\n| Model | Purpose |\n|-------|---------|\n| `ChatMessage` | Represents a single message with role and content |\n| `ChatCompletionResponse` | Wraps the API response from providers |\n| `Provider` | Enum identifying supported LLM providers |\n| `ModelInfo` | Metadata about available models per provider |\n\n## Provider Implementations\n\n### OpenAI Provider\n\nThe OpenAI provider connects to OpenAI's API endpoints for chat completions. It supports both standard and streaming responses.\n\n**Configuration Requirements:**\n\n| Parameter | Source | Description |\n|-----------|--------|-------------|\n| `OPENAI_API_KEY` | Environment variable | API key for authentication |\n| `model` | Config/Parameter | Model identifier (e.g., `gpt-4`, `gpt-4-turbo`) |\n\n**API Endpoint:**\n```\nPOST https://api.openai.com/v1/chat/completions\n```\n\n资料来源：[arksim/llms/chat/providers/openai.py]()\n\n### Anthropic Provider\n\nThe Anthropic provider integrates with Anthropic's Claude models through their API. It handles the distinct message format and API conventions used by Anthropic.\n\n**Configuration Requirements:**\n\n| Parameter | Source | Description |\n|-----------|--------|-------------|\n| `ANTHROPIC_API_KEY` | Environment variable | API key for authentication |\n| `model` | Config/Parameter | Model identifier (e.g., `claude-3-opus-20240229`) |\n\n**API Endpoint:**\n```\nPOST https://api.anthropic.com/v1/messages\n```\n\n资料来源：[arksim/llms/chat/providers/anthropic.py]()\n\n### Google Provider\n\nThe Google provider connects to Google's Gemini models via the Google AI API.\n\n**Configuration Requirements:**\n\n| Parameter | Source | Description |\n|-----------|--------|-------------|\n| `GOOGLE_API_KEY` | Environment variable | API key for authentication |\n| `model` | Config/Parameter | Model identifier (e.g., `gemini-pro`) |\n\n**API Endpoint:**\n```\nPOST https://generativelanguage.googleapis.com/v1/models/{model}:generateContent\n```\n\n资料来源：[arksim/llms/chat/providers/google.py]()\n\n### Azure OpenAI Provider\n\nThe Azure OpenAI provider enables integration with Azure-hosted OpenAI models, supporting enterprise deployments with Azure-specific authentication and endpoint configuration.\n\n**Configuration Requirements:**\n\n| Parameter | Source | Description |\n|-----------|--------|-------------|\n| `AZURE_OPENAI_API_KEY` | Environment variable | API key for Azure authentication |\n| `AZURE_OPENAI_ENDPOINT` | Environment variable | Azure endpoint URL |\n| `AZURE_OPENAI_DEPLOYMENT` | Config | Deployment name in Azure |\n| `AZURE_OPENAI_API_VERSION` | Config | Azure API version |\n\n资料来源：[arksim/llms/chat/providers/azure_openai.py]()\n\n## Configuration\n\n### YAML Configuration Structure\n\nLLM providers are configured through the `config.yaml` file used by the simulator:\n\n```yaml\nllm:\n provider: openai # or anthropic, google, azure_openai\n model: gpt-4\n temperature: 0.7\n max_tokens: 2048\n \nagent_config:\n # Agent-specific LLM settings\n provider: anthropic\n model: claude-3-opus-20240229\n```\n\n### Environment Variables\n\n| Variable | Providers | Purpose |\n|----------|-----------|---------|\n| `OPENAI_API_KEY` | OpenAI, AutoGen, LangChain | OpenAI API authentication |\n| `ANTHROPIC_API_KEY` | Claude Agent SDK | Anthropic API authentication |\n| `GOOGLE_API_KEY` | Google | Google AI API authentication |\n| `AZURE_OPENAI_API_KEY` | Azure OpenAI | Azure API authentication |\n| `AZURE_OPENAI_ENDPOINT` | Azure OpenAI | Azure resource endpoint |\n\n## Integration with Custom Agents\n\nThe LLM provider system integrates with the custom agent connector pattern used in ArkSim simulations. Custom agents can be configured to use any supported LLM provider.\n\n```mermaid\ngraph LR\n A[Scenario JSON] --> B[Simulation Engine]\n B --> C[Custom Agent
custom_agent.py]\n C --> D[LLM Provider]\n D --> E[External LLM API]\n```\n\n### Integration Examples\n\nArkSim provides integration examples for various agent frameworks:\n\n| Framework | Example Path | LLM Provider Used |\n|-----------|--------------|-------------------|\n| LangChain/LangGraph | `examples/integrations/langchain/` | OpenAI |\n| Claude Agent SDK | `examples/integrations/claude-agent-sdk/` | Anthropic |\n| LlamaIndex | `examples/integrations/llamaindex/` | OpenAI |\n| CrewAI | `examples/integrations/crewai/` | OpenAI |\n| AutoGen | `examples/integrations/autogen/` | OpenAI |\n| Pydantic AI | `examples/integrations/pydantic-ai/` | OpenAI |\n| Smolagents | `examples/integrations/smolagents/` | OpenAI |\n| Dify | `examples/integrations/dify/` | Custom HTTP |\n\nEach integration demonstrates how to connect the framework's agent to ArkSim's simulation engine while delegating LLM calls to the appropriate provider.\n\n## Usage Flow\n\n```mermaid\nsequenceDiagram\n participant User\n participant Config as config.yaml\n participant LLM as LLM Manager\n participant Provider as Provider Class\n participant API as External LLM API\n \n User->>Config: Load configuration\n User->>LLM: Initialize with provider type\n LLM->>Provider: Create provider instance\n User->>LLM: chat(messages, model)\n LLM->>Provider: _chat_completion()\n Provider->>API: HTTP POST request\n API-->>Provider: Completion response\n Provider-->>LLM: Normalized response\n LLM-->>User: ChatCompletionResponse\n```\n\n## Adding New Providers\n\nTo add support for a new LLM provider:\n\n1. Create a new provider class inheriting from `BaseLLM`\n2. Implement the required methods: `chat()`, `chat_stream()`, `count_tokens()`\n3. Add the provider to the `Provider` enum in `base_llm.py`\n4. Update the provider factory logic in `llm.py` to instantiate the new provider\n5. Add integration tests and documentation\n\n资料来源：[arksim/llms/chat/base/base_llm.py]()\n资料来源：[arksim/llms/chat/llm.py]()\n\n## Error Handling\n\nProvider implementations handle common error scenarios:\n\n| Error Type | Handling Strategy |\n|------------|-------------------|\n| Authentication failures | Raise `AuthenticationError` with helpful message |\n| Rate limiting | Implement automatic retry with backoff |\n| Invalid request parameters | Raise `ValidationError` with parameter details |\n| Network timeouts | Retry with exponential backoff |\n| Model not found | Raise `ModelNotFoundError` listing available models |\n\n## Best Practices\n\n1. **API Key Security**: Store API keys in environment variables, never in configuration files committed to version control. ArkSim automatically loads keys from environment variables.\n\n2. **Token Tracking**: Use the `count_tokens()` method to monitor token usage and estimate costs before running large-scale simulations.\n\n3. **Streaming for Large Responses**: Enable streaming (`chat_stream()`) for scenarios expecting long agent responses to improve perceived responsiveness.\n\n4. **Provider Selection**: Choose Azure OpenAI for enterprise deployments requiring compliance certifications and dedicated infrastructure.\n\n5. **Model Selection**: Refer to the integration examples for recommended model configurations per provider and use case.\n\n---\n\n\n\n## Tool Call Capture\n\n### 相关页面\n\n相关主题：[Evaluation System](#evaluation-system), [Agent Types and Integration](#agent-types)\n\n相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/simulation_engine/tool_types.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n- [arksim/simulation_engine/agent/clients/a2a.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/agent/clients/a2a.py)\n- [arksim/tracing/openai.py](https://github.com/arklexai/arksim/blob/main/arksim/tracing/openai.py)\n- [examples/customer-service/custom_agent.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_agent.py)\n- [examples/customer-service/traced_agent.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/traced_agent.py)\n- [examples/customer-service/a2a_server/agent.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/a2a_server/agent.py)\n相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/scenario/entities.py](https://github.com/arklexai/arksim/blob/main/arksim/scenario/entities.py)\n- [arksim/templates/scenarios.json](https://github.com/arklexai/arksim/blob/main/arksim/templates/scenarios.json)\n- [examples/e-commerce/scenarios.json](https://github.com/arklexai/arksim/blob/main/examples/e-commerce/scenarios.json)\n- [examples/bank-insurance/scenarios.json](https://github.com/arklexai/arksim/blob/main/examples/bank-insurance/scenarios.json)\n- [examples/customer-service/scenarios.json](https://github.com/arklexai/arksim/blob/main/examples/customer-service/scenarios.json)\n相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/config/types.py](https://github.com/arklexai/arksim/blob/main/arksim/config/types.py)\n- [arksim/config/core/agent.py](https://github.com/arklexai/arksim/blob/main/arksim/config/core/agent.py)\n- [arksim/config/utils.py](https://github.com/arklexai/arksim/blob/main/arksim/config/utils.py)\n- [arksim/config.yaml](https://github.com/arklexai/arksim/blob/main/arksim/config.yaml)\n- [arksim/config_simulate.yaml](https://github.com/arklexai/arksim/blob/main/arksim/config_simulate.yaml)\n- [arksim/config_evaluate.yaml](https://github.com/arklexai/arksim/blob/main/arksim/config_evaluate.yaml)\n- [arksim/cli.py](https://github.com/arklexai/arksim/blob/main/arksim/cli.py)\n- [examples/customer-service/custom_metrics.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [README.md](https://github.com/arklexai/arksim/blob/main/README.md)\n- [CONTRIBUTING.md](https://github.com/arklexai/arksim/blob/main/CONTRIBUTING.md)\n- [CLAUDE.md](https://github.com/arklexai/arksim/blob/main/CLAUDE.md)\n- [arksim/simulation_engine/tool_types.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n- [examples/customer-service/custom_metrics.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n- [examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n- [examples/integrations/langgraph/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/langgraph/README.md)\n- [examples/integrations/autogen/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/autogen/README.md)\n- [examples/ci/README.md](https://github.com/arklexai/arksim/blob/main/examples/ci/README.md)\n
config.yaml] --> B[Simulation Engine]\n C[Scenarios
scenarios.json] --> B\n D[Agent
custom_agent.py] --> B\n B --> E[Conversation Data]\n E --> F[Evaluator]\n F --> G[HTML Report
final_report.html]\n H[Custom Metrics
custom_metrics.py] --> F\n```\n\n### Core Components\n\n| Component | Description |\n|-----------|-------------|\n| **Simulation Engine** | Orchestrates multi-turn conversations between simulated users and the agent |\n| **Evaluator** | Scores agent performance using built-in and custom metrics |\n| **CLI Interface** | Command-line interface for running simulations (`arksim simulate-evaluate`) |\n| **Report Generator** | Produces HTML reports with visualizations and conversation transcripts |\n\n资料来源：[examples/customer-service/README.md:1-25](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n## Agent Integration Methods\n\nArkSim supports multiple agent integration patterns to accommodate different agent architectures.\n\n### Python Class Integration (BaseAgent)\n\nThe default integration method uses a Python class inheriting from `BaseAgent`. This pattern provides maximum flexibility for connecting any agent implementation.\n\n```python\nfrom arksim.simulation_engine.agent.base import BaseAgent\nfrom arksim.simulation_engine.tool_types import AgentResponse\n\nclass MyAgent(BaseAgent):\n async def get_chat_id(self) -> str:\n return \"unique-id\"\n\n async def execute(self, user_query: str, **kwargs: object) -> str | AgentResponse:\n # Replace with your agent logic\n return \"agent response\"\n```\n\nThe agent can return either:\n- A plain string response\n- An `AgentResponse` object containing both content and tool calls\n\n资料来源：[README.md:40-55](https://github.com/arklexai/arksim/blob/main/README.md)\n\n### AgentResponse Data Model\n\nThe `AgentResponse` model encapsulates structured agent outputs:\n\n```python\nclass AgentResponse(BaseModel):\n \"\"\"Structured return from agent execution, carrying both text and tool calls.\"\"\"\n model_config = ConfigDict(extra=\"ignore\")\n \n content: str\n tool_calls: list[ToolCall] = Field(default_factory=list)\n```\n\n资料来源：[arksim/simulation_engine/tool_types.py:32-42](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n\n### ToolCall Model\n\nTool calls are captured using the `ToolCall` model:\n\n```python\nclass ToolCall(BaseModel):\n \"\"\"A single tool/function call observed during a turn.\"\"\"\n model_config = ConfigDict(extra=\"ignore\")\n \n id: str\n name: str\n arguments: dict[str, Any] = Field(default_factory=dict)\n result: str | None = None\n error: str | None = None\n source: ToolCallSource | None = None\n```\n\nThe `extra=\"ignore\"` configuration ensures forward compatibility with future versions.\n\n资料来源：[arksim/simulation_engine/tool_types.py:18-30](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n\n### Chat Completions Endpoint Integration\n\nFor agents exposing a standard OpenAI-compatible API:\n\n```yaml\nagent_config:\n agent_type: chat_completions\n agent_name: my-agent\n api_config:\n endpoint: http://localhost:8000/v1/chat/completions\n```\n\n资料来源：[README.md:57-62](https://github.com/arklexai/arksim/blob/main/README.md)\n\n### A2A Protocol Integration\n\nFor agents implementing the Agent-to-Agent (A2A) protocol:\n\n```yaml\nagent_config:\n agent_type: a2a\n agent_name: my-agent\n api_config:\n endpoint: http://localhost:9999/agent\n```\n\n资料来源：[README.md:64-70](https://github.com/arklexai/arksim/blob/main/README.md)\n\n### Automatic Tool Call Capture (Tracing)\n\nArkSim supports automatic tool call capture through a tracing processor, eliminating the need for agents to explicitly return tool calls:\n\n```mermaid\ngraph LR\n A[Simulator sets
routing context] --> B[agent.execute()]\n B --> C[SDK fires
TracingProcessor.on_span_end]\n C --> D[arksim captures
tool calls]\n D --> E[Evaluator scores]\n```\n\n```bash\npip install -r requirements-traced.txt\narksim simulate-evaluate config_traced.yaml\n```\n\n资料来源：[examples/customer-service/README.md:35-55](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n## Configuration System\n\nArkSim uses YAML configuration files to define simulation and evaluation parameters.\n\n### Configuration File Structure\n\n```yaml\n# Simulation settings\nsimulation:\n max_turns: 10\n timeout: 300\n\n# Agent configuration\nagent_config:\n agent_type: chat_completions # or 'a2a', 'custom'\n agent_name: my-agent\n api_config:\n endpoint: http://localhost:8000/v1/chat/completions\n\n# Trace receiver (optional)\ntrace_receiver:\n enabled: true\n wait_timeout: 5\n\n# Custom metrics\ncustom_metrics_file_paths:\n - ./custom_metrics.py\n\nmetrics_to_run:\n - goal_completion\n - verification_compliance\n```\n\n资料来源：[examples/ci/README.md:1-25](https://github.com/arklexai/arksim/blob/main/examples/ci/README.md)\n\n### Scenario Definition\n\nScenarios are defined in `scenarios.json` with the following structure:\n\n| Field | Description |\n|-------|-------------|\n| `scenario_id` | Unique identifier for the scenario |\n| `user_profile` | Description of the simulated user |\n| `knowledge` | Context and information available to the user |\n| `goals` | Objectives the user aims to achieve |\n| `expected_behavior` | Expected agent behavior patterns |\n\n资料来源：[examples/integrations/dify/README.md:1-20](https://github.com/arklexai/arksim/blob/main/examples/integrations/dify/README.md)\n\n## Evaluation Metrics\n\nArkSim provides built-in metrics and supports custom metric definitions.\n\n### Built-in Metrics\n\n| Metric | Description | Weight |\n|--------|-------------|--------|\n| Goal Completion | Whether the agent achieved the user's objectives | 60% |\n| Turn Success Ratio | Ratio of successful turns to total turns | 40% |\n| Final Score | Weighted average of other metrics | - |\n\n资料来源：[arksim/utils/html_report/report_template.html:80-95](https://github.com/arklexai/arksim/blob/main/arksim/utils/html_report/report_template.html)\n\n### Custom Metrics\n\nCustom metrics can be implemented as either quantitative (scored numerically) or qualitative (evaluated via LLM).\n\n#### Quantitative Metric Pattern\n\n```python\nfrom arksim.evaluator import (\n QuantitativeMetric,\n QuantResult,\n ScoreInput,\n format_chat_history,\n)\n\nclass VerificationComplianceMetric(Queresult):\n identity_verification: float # 0.0-1.0\n action_gating: float # 0.0-1.0\n reason: str\n\nVERIFICATION_COMPLIANCE_SYSTEM_PROMPT = \"\"\"\\\nYou are an impartial evaluator for a customer service agent.\nScore how well the agent followed identity verification protocols...\"\"\"\n\ndef verification_compliance_metric(\n input_data: ScoreInput,\n) -> QuantResult:\n # Implementation\n return QuantResult(score=0.85, reason=\"Verification completed\")\n```\n\n资料来源：[examples/customer-service/custom_metrics.py:15-40](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n\n#### Qualitative Metric Pattern\n\n```python\nfrom arksim.evaluator import (\n QualitativeMetric,\n QualResult,\n ScoreInput,\n format_chat_history,\n)\n\ndef helpfulness_metric(input_data: ScoreInput) -> QualResult:\n system_prompt = \"\"\"Evaluate the helpfulness of the agent response...\"\"\"\n return QualResult(\n assessment=\"The agent provided comprehensive assistance\",\n score=0.9,\n reason=\"Clear explanation with actionable steps\"\n )\n```\n\n### Metric Configuration\n\nTo use custom metrics:\n\n1. Implement the metric function in a Python file\n2. Reference the file path in `custom_metrics_file_paths` in config.yaml\n3. Add the metric name to `metrics_to_run`\n\n资料来源：[examples/customer-service/custom_metrics.py:1-15](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n\n## CLI Usage\n\nArkSim provides a command-line interface for running simulations and evaluations.\n\n### Primary Commands\n\n| Command | Description |\n|---------|-------------|\n| `arksim simulate-evaluate config.yaml` | Run simulation and evaluation in one step |\n| `arksim simulate config_simulate.yaml` | Run simulation only |\n| `arksim evaluate config_evaluate.yaml` | Evaluate existing simulation results |\n\n资料来源：[examples/customer-service/README.md:15-22](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n### Workflow Examples\n\n#### Combined Simulation and Evaluation\n\n```bash\narksim simulate-evaluate config.yaml\n```\n\n#### Separate Simulation and Evaluation\n\n```bash\n# Step 1: Simulate\narksim simulate config_simulate.yaml\n\n# Step 2: Evaluate\narksim evaluate config_evaluate.yaml\n```\n\nResults are written to `./results/simulation/simulation.json`. The evaluation report is printed to stdout with per-scenario metric scores and failure analysis.\n\n资料来源：[examples/integrations/dify/README.md:15-25](https://github.com/arklexai/arksim/blob/main/examples/integrations/dify/README.md)\n\n## Framework Integrations\n\nArkSim provides integration examples for popular agent frameworks.\n\n### Integration Matrix\n\n| Framework | Package Required | Documentation |\n|-----------|-----------------|---------------|\n| LangGraph | `langgraph langchain-openai` | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/langgraph/README.md) |\n| LangChain | `langgraph langchain-openai` | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/langchain/README.md) |\n| AutoGen | `autogen-agentchat autogen-ext[openai]` | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/autogen/README.md) |\n| Claude Agent SDK | `claude-agent-sdk` | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/claude-agent-sdk/README.md) |\n| Rasa | `rasa-pro` | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/rasa/README.md) |\n| Dify | (custom agent) | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/dify/README.md) |\n| OpenClaw | (custom agent) | [Link](https://github.com/arklexai/arksim/blob/main/examples/integrations/openclaw/README.md) |\n\n资料来源：[examples/integrations/langgraph/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/langgraph/README.md), [examples/integrations/autogen/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/autogen/README.md)\n\n### General Integration Pattern\n\nRegardless of the framework, the integration follows this pattern:\n\n1. Create a `custom_agent.py` file with a `BaseAgent` subclass\n2. Implement the `execute()` method to call the framework's agent\n3. Return either a string or `AgentResponse` with tool calls\n4. Configure `config.yaml` to use the custom agent\n5. Create `scenarios.json` with test cases\n\n```python\nfrom arksim.simulation_engine.agent.base import BaseAgent\nfrom arksim.simulation_engine.tool_types import AgentResponse\n\nclass FrameworkAgent(BaseAgent):\n async def get_chat_id(self) -> str:\n return \"unique-session-id\"\n\n async def execute(self, user_query: str, **kwargs: object) -> str | AgentResponse:\n # Call your framework's agent here\n result = await your_agent.run(user_query)\n return str(result)\n```\n\n## Report Generation\n\nArkSim generates comprehensive HTML evaluation reports containing:\n\n### Report Sections\n\n| Section | Content |\n|---------|---------|\n| **Summary** | Overall scores, pass/fail status |\n| **Per-Scenario Scores** | Individual metric scores for each scenario |\n| **Failure Categories** | Grouped failure patterns with counts |\n| **Conversation Viewer** | Full transcript of each simulated conversation |\n| **Score Explanations** | Rationale for each score with references to conversation turns |\n\n资料来源：[README.md:25-30](https://github.com/arklexai/arksim/blob/main/README.md)\n\n### Report Output\n\nThe report tells you where your agent is strong and where it breaks. You get per-metric scores, categorized failures, and full conversation transcripts so you can read the exact turns where things went wrong.\n\n资料来源：[README.md:30-35](https://github.com/arklexai/arksim/blob/main/README.md)\n\n## Development Guidelines\n\n### Project Setup\n\n```bash\n# Fork and clone\ngit clone https://github.com/相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/cli.py](https://github.com/arklexai/arksim/blob/main/arksim/cli.py)\n- [arksim/simulation_engine/agent/base.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/agent/base.py)\n- [arksim/simulation_engine/tool_types.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n- [examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n- [examples/integrations/langchain/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/langchain/README.md)\n- [README.md](https://github.com/arklexai/arksim/blob/main/README.md)\n相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/simulation_engine/tool_types.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n- [arksim/cli.py](https://github.com/arklexai/arksim/blob/main/arksim/cli.py)\n- [README.md](https://github.com/arklexai/arksim/blob/main/README.md)\n- [examples/customer-service/custom_metrics.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n- [examples/e-commerce/custom_metrics.py](https://github.com/arklexai/arksim/blob/main/examples/e-commerce/custom_metrics.py)\n- [arksim/utils/html_report/report_template.html](https://github.com/arklexai/arksim/blob/main/arksim/utils/html_report/report_template.html)\n
Python class extending BaseAgent]\n HTTP[Chat Completions
HTTP endpoint /v1/chat/completions]\n A2A[A2A Protocol
Agent-to-Agent standard]\n end\n```\n\n1. **Custom Agent (Python class)**: Extend `BaseAgent` and implement the `execute()` method\n2. **Chat Completions**: Configure an HTTP endpoint for OpenAI-compatible chat completions API\n3. **A2A Protocol**: Connect via the Agent-to-Agent protocol standard\n\n资料来源：[README.md](https://github.com/arklexai/arksim/blob/main/README.md)\n\n## Simulation Engine\n\nThe simulation engine orchestrates multi-turn conversations between a user simulator and the agent under test. Each turn consists of:\n\n1. User simulator generates a response based on conversation history and scenario knowledge\n2. Agent executes the user query and returns a response (optionally with tool calls)\n3. Simulator captures the interaction for evaluation\n\n### Tool Call Capture\n\nArkSim captures tool/function calls in two ways:\n\n| Capture Method | Mechanism | Configuration |\n|----------------|-----------|---------------|\n| AgentResponse | Agent returns structured `AgentResponse` with `tool_calls` list | Default behavior |\n| TracingProcessor | SDK's `TracingProcessor.on_span_end` captures calls automatically | `trace_receiver.enabled: true` |\n\n资料来源：[examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n#### Tool Call Data Model\n\nTool calls are represented by the `ToolCall` class:\n\n```python\nclass ToolCall(BaseModel):\n id: str\n name: str\n arguments: dict[str, Any] = Field(default_factory=dict)\n result: str | None = None\n error: str | None = None\n source: ToolCallSource | None = None\n```\n\nThe `AgentResponse` wraps both content and tool calls:\n\n```python\nclass AgentResponse(BaseModel):\n content: str\n tool_calls: list[ToolCall] = Field(default_factory=list)\n```\n\n资料来源：[arksim/simulation_engine/tool_types.py:1-50](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n\n### Traced Agent Flow\n\n```mermaid\nsequenceDiagram\n participant Simulator\n participant Agent\n participant Tracing as ArksimTracingProcessor\n participant Evaluator\n\n Simulator->>Agent: execute(user_query)\n Agent->>Tracing: on_span_end(span)\n Tracing->>Simulator: captured_tool_calls\n Simulator->>Agent: RunResult\n Agent->>Simulator: AgentResponse\n Simulator->>Evaluator: Tool calls + Conversation\n Evaluator->>Simulator: Metric Scores\n```\n\n资料来源：[examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n## Evaluator\n\nThe evaluator scores agent performance using both quantitative and qualitative metrics. The evaluation framework supports custom metrics defined via Python files.\n\n### Metric Types\n\n| Type | Description | Scoring Method |\n|------|-------------|----------------|\n| QuantitativeMetric | Numerical scores (0.0-1.0 scale) | Structured JSON schema validation |\n| QualitativeMetric | Free-form evaluation with reasoning | LLM-generated analysis |\n\n资料来源：[examples/customer-service/custom_metrics.py:1-60](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n\n### Custom Metrics Structure\n\nCustom metrics require:\n\n1. A Pydantic `BaseModel` defining the output schema\n2. A system prompt describing evaluation criteria\n3. Implementation of `QuantitativeMetric` or `QualitativeMetric`\n\n```python\nclass ConversionSchema(BaseModel):\n intent_strength: float\n conversion_outcome: float\n evidence: list[str]\n reason: str\n\nCONVERSION_SYSTEM_PROMPT = \"\"\"\\\nYou are an impartial evaluator for an e-commerce shopping agent.\nYour job is to score (1) the shopper's purchase intent and (2) whether the agent achieved a conversion outcome...\n\"\"\"\n```\n\n资料来源：[examples/e-commerce/custom_metrics.py:1-40](https://github.com/arklexai/arksim/blob/main/examples/e-commerce/custom_metrics.py)\n\n### Score Calculation\n\nThe evaluator computes a **Final Score** as a weighted average:\n\n| Component | Weight | Description |\n|-----------|--------|-------------|\n| Turn Success Ratio | 40% | Ratio of successful turns to total turns |\n| Goal Completion Score | 60% | LLM-assessed goal achievement score |\n\n| Status | Condition |\n|--------|-----------|\n| Done | Final score = 1.0 |\n| Partial Failure | 0.0 < Final score < 1.0 |\n| Complete Failure | Final score = 0.0 |\n\n资料来源：[arksim/utils/html_report/report_template.html:1-50](https://github.com/arklexai/arksim/blob/main/arksim/utils/html_report/report_template.html)\n\n## Data Flow\n\n```mermaid\ngraph LR\n subgraph Input[\"Input\"]\n Config[config.yaml]\n Scenarios[scenarios.json]\n Metrics[custom_metrics.py]\n end\n\n subgraph Process[\"Processing\"]\n Sim[Simulation]\n Eval[Evaluation]\n Score[Scoring]\n end\n\n subgraph Output[\"Output\"]\n Results[simulation.json]\n Report[evaluation.html]\n end\n\n Config --> Sim\n Scenarios --> Sim\n Sim --> Eval\n Metrics --> Eval\n Eval --> Score\n Score --> Results\n Score --> Report\n```\n\n## Agent Connector Types\n\nArkSim supports multiple agent integration patterns via the configuration system:\n\n```yaml\n# Option 1: Custom Python Agent\nagent_config:\n agent_type: custom\n agent_name: my-agent\n\n# Option 2: Chat Completions HTTP Endpoint\nagent_config:\n agent_type: chat_completions\n agent_name: my-agent\n api_config:\n endpoint: http://localhost:8000/v1/chat/completions\n\n# Option 3: A2A Protocol\nagent_config:\n agent_type: a2a\n agent_name: my-agent\n api_config:\n endpoint: http://localhost:9999/agent\n```\n\n资料来源：[README.md](https://github.com/arklexai/arksim/blob/main/README.md)\n\n## User Simulator\n\nThe user simulator generates realistic multi-turn conversations based on:\n\n- **Scenario definitions**: Predefined conversation flows and expected behaviors\n- **User profiles**: Demographic and behavioral attributes for persona simulation\n- **Knowledge bases**: Domain-specific information the simulated user possesses\n- **Conversation history**: Full context of the multi-turn interaction\n\nThe simulator uses LLM-based generation to produce contextually appropriate user responses that evolve through the conversation based on agent actions and conversation state.\n\n## HTML Report Generation\n\nAfter evaluation completes, ArkSim generates an interactive HTML report containing:\n\n| Section | Content |\n|---------|---------|\n| Summary Statistics | Overall scores, pass/fail rates |\n| Per-Scenario Metrics | Individual metric scores with reasoning |\n| Failure Categories | Grouped analysis of failure types |\n| Conversation Transcripts | Full turn-by-turn dialogue viewer |\n\n资料来源：[arksim/utils/html_report/report_template.html:1-100](https://github.com/arklexai/arksim/blob/main/arksim/utils/html_report/report_template.html)\n\n## Integration Examples\n\nArkSim provides example integrations for popular agent frameworks:\n\n| Framework | Example Location | Connection Method |\n|-----------|-----------------|-------------------|\n| LangGraph | `examples/integrations/langgraph/` | Custom agent connector |\n| LangChain | `examples/integrations/langchain/` | Custom agent connector |\n| Claude Agent SDK | `examples/integrations/claude-agent-sdk/` | Custom agent connector |\n| AutoGen | `examples/integrations/autogen/` | Custom agent connector |\n| Pydantic AI | `examples/integrations/pydantic-ai/` | Custom agent connector |\n| Dify | `examples/integrations/dify/` | HTTP client |\n\n资料来源：[examples/integrations/langgraph/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/langgraph/README.md), [examples/integrations/claude-agent-sdk/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/claude-agent-sdk/README.md), [examples/integrations/autogen/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/autogen/README.md)\n\n## Configuration Schema\n\nThe main configuration file (`config.yaml`) controls all aspects of simulation and evaluation:\n\n| Section | Key Options |\n|---------|-------------|\n| `agent_config` | `agent_type`, `agent_name`, `api_config.endpoint` |\n| `scenario_file` | Path to scenarios.json |\n| `metrics_to_run` | List of metric names to execute |\n| `custom_metrics_file_paths` | Paths to custom metric Python files |\n| `trace_receiver.enabled` | Enable TracingProcessor capture (default: false) |\n| `trace_receiver.wait_timeout` | Timeout for trace capture (seconds) |\n\n资料来源：[examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md)\n\n## Web UI\n\nArkSim includes a web-based control plane for managing simulations:\n\n```mermaid\ngraph TD\n subgraph UI[\"Web UI\"]\n Build[Build Scenarios]\n Load[Load Existing]\n Run[Run Simulation]\n View[View Results]\n end\n\n subgraph Features[\"UI Features\"]\n AutoGen[Auto-generate Scenarios PRO]\n Browse[File Browser]\n Refresh[Refresh Results]\n end\n```\n\nFeatures include:\n- Scenario building and loading\n- Auto-generate scenarios (PRO feature)\n- File browser integration\n- Results viewing and refresh\n\n资料来源：[arksim/ui/frontend/index.html:1-80](https://github.com/arklexai/arksim/blob/main/arksim/ui/frontend/index.html)\n\n## Summary\n\nArkSim provides a comprehensive multi-turn agent evaluation framework with:\n\n- **Flexible agent integration**: Support for custom Python agents, HTTP endpoints, and A2A protocol\n- **Tool call capture**: Two mechanisms for capturing agent tool executions\n- **Customizable metrics**: Both quantitative and qualitative evaluation approaches\n- **Interactive reporting**: HTML-based results with conversation viewer\n- **CLI and UI**: Command-line and web-based interfaces for running evaluations\n- **Framework integrations**: Pre-built examples for LangGraph, LangChain, Claude SDK, AutoGen, Pydantic AI, and Dify\n\n---\n\n\n\n## Simulation Engine\n\n### 相关页面\n\n相关主题：[System Architecture Overview](#arch-overview), [Evaluation System](#evaluation-system), [Scenario Management](#scenario-management)\n\n相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/simulation_engine/simulator.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/simulator.py)\n- [arksim/simulation_engine/entities.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/entities.py)\n- [arksim/simulation_engine/core/multi_knowledge_handling.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/core/multi_knowledge_handling.py)\n- [arksim/simulation_engine/agent/base.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/agent/base.py)\n- [arksim/simulation_engine/tool_types.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n- [arksim/simulation_engine/utils/prompts.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/utils/prompts.py)\nRelevant Source Files
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/evaluator/evaluator.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/evaluator.py)\n- [arksim/evaluator/builtin_metrics.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/builtin_metrics.py)\n- [arksim/evaluator/base_metric.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/base_metric.py)\n- [arksim/evaluator/tool_call_metrics.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/tool_call_metrics.py)\n- [arksim/evaluator/trajectory_matching.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/trajectory_matching.py)\n- [arksim/evaluator/thresholds.py](https://github.com/arklexai/arksim/blob/main/arksim/evaluator/thresholds.py)\n- [arksim/utils/html_report/generate_html_report.py](https://github.com/arklexai/arksim/blob/main/arksim/utils/html_report/generate_html_report.py)\nRelevant Source Files
\n\n以下源码文件用于生成本页说明：\n\n- [README.md](https://github.com/arklexai/arksim/blob/main/README.md) - Main documentation with agent type examples\n- [arksim/cli.py](https://github.com/arklexai/arksim/blob/main/arksim/cli.py) - CLI commands for agent initialization\n- [arksim/ui/frontend/index.html](https://github.com/arklexai/arksim/blob/main/arksim/ui/frontend/index.html) - UI Agent Config section\n- [examples/customer-service/README.md](https://github.com/arklexai/arksim/blob/main/examples/customer-service/README.md) - Custom agent implementation example\n- [examples/integrations/dify/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/dify/README.md) - HTTP agent integration example\n- [examples/integrations/rasa/README.md](https://github.com/arklexai/arksim/blob/main/examples/integrations/rasa/README.md) - External agent server integration\nAgent Config
\n 相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/llms/chat/providers/openai.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/providers/openai.py)\n- [arksim/llms/chat/providers/anthropic.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/providers/anthropic.py)\n- [arksim/llms/chat/providers/google.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/providers/google.py)\n- [arksim/llms/chat/providers/azure_openai.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/providers/azure_openai.py)\n- [arksim/llms/chat/llm.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/llm.py)\n- [arksim/llms/chat/base/base_llm.py](https://github.com/arklexai/arksim/blob/main/arksim/llms/chat/base/base_llm.py)\n
arksim/llms/chat/llm.py]\n B --> C[Base LLM
arksim/llms/chat/base/base_llm.py]\n C --> D[OpenAI Provider
providers/openai.py]\n C --> E[Anthropic Provider
providers/anthropic.py]\n C --> F[Google Provider
providers/google.py]\n C --> G[Azure OpenAI Provider
providers/azure_openai.py]\n \n H[Configuration YAML] --> B\n I[Environment Variables] --> D\n I --> E\n I --> F\n I --> G\n```\n\n### Provider Class Hierarchy\n\n```mermaid\ngraph TD\n A[BaseLLM
base/base_llm.py] --> B[OpenAIProvider
providers/openai.py]\n A --> C[AnthropicProvider
providers/anthropic.py]\n A --> D[GoogleProvider
providers/google.py]\n A --> E[AzureOpenAIProvider
providers/azure_openai.py]\n \n F[Provider Enum] --> A\n G[ChatMessage Model] --> A\n H[ChatCompletionResponse] --> A\n```\n\n## Supported Providers\n\nArkSim supports the following LLM providers through dedicated provider implementations:\n\n| Provider | Provider Class | API Style | Streaming Support |\n|----------|----------------|-----------|-------------------|\n| OpenAI | `OpenAIProvider` | OpenAI API | Yes |\n| Anthropic | `AnthropicProvider` | Anthropic API | Yes |\n| Google | `GoogleProvider` | Google AI API | Yes |\n| Azure OpenAI | `AzureOpenAIProvider` | Azure API | Yes |\n\nEach provider class inherits from `BaseLLM` and implements provider-specific API call logic while adhering to the common interface contract.\n\n## Base LLM Interface\n\nThe `BaseLLM` class defines the contract that all provider implementations must follow. This ensures consistent behavior across different LLM backends.\n\n### Core Methods\n\n| Method | Purpose | Parameters |\n|--------|---------|------------|\n| `chat()` | Send a chat completion request | `messages`, `model`, `temperature`, `max_tokens`, `**kwargs` |\n| `chat_stream()` | Stream chat completion responses | `messages`, `model`, `temperature`, `max_tokens`, `**kwargs` |\n| `count_tokens()` | Calculate token usage for messages | `messages`, `model` |\n\n### Data Models\n\nThe base module defines essential data structures used across all providers:\n\n| Model | Purpose |\n|-------|---------|\n| `ChatMessage` | Represents a single message with role and content |\n| `ChatCompletionResponse` | Wraps the API response from providers |\n| `Provider` | Enum identifying supported LLM providers |\n| `ModelInfo` | Metadata about available models per provider |\n\n## Provider Implementations\n\n### OpenAI Provider\n\nThe OpenAI provider connects to OpenAI's API endpoints for chat completions. It supports both standard and streaming responses.\n\n**Configuration Requirements:**\n\n| Parameter | Source | Description |\n|-----------|--------|-------------|\n| `OPENAI_API_KEY` | Environment variable | API key for authentication |\n| `model` | Config/Parameter | Model identifier (e.g., `gpt-4`, `gpt-4-turbo`) |\n\n**API Endpoint:**\n```\nPOST https://api.openai.com/v1/chat/completions\n```\n\n资料来源：[arksim/llms/chat/providers/openai.py]()\n\n### Anthropic Provider\n\nThe Anthropic provider integrates with Anthropic's Claude models through their API. It handles the distinct message format and API conventions used by Anthropic.\n\n**Configuration Requirements:**\n\n| Parameter | Source | Description |\n|-----------|--------|-------------|\n| `ANTHROPIC_API_KEY` | Environment variable | API key for authentication |\n| `model` | Config/Parameter | Model identifier (e.g., `claude-3-opus-20240229`) |\n\n**API Endpoint:**\n```\nPOST https://api.anthropic.com/v1/messages\n```\n\n资料来源：[arksim/llms/chat/providers/anthropic.py]()\n\n### Google Provider\n\nThe Google provider connects to Google's Gemini models via the Google AI API.\n\n**Configuration Requirements:**\n\n| Parameter | Source | Description |\n|-----------|--------|-------------|\n| `GOOGLE_API_KEY` | Environment variable | API key for authentication |\n| `model` | Config/Parameter | Model identifier (e.g., `gemini-pro`) |\n\n**API Endpoint:**\n```\nPOST https://generativelanguage.googleapis.com/v1/models/{model}:generateContent\n```\n\n资料来源：[arksim/llms/chat/providers/google.py]()\n\n### Azure OpenAI Provider\n\nThe Azure OpenAI provider enables integration with Azure-hosted OpenAI models, supporting enterprise deployments with Azure-specific authentication and endpoint configuration.\n\n**Configuration Requirements:**\n\n| Parameter | Source | Description |\n|-----------|--------|-------------|\n| `AZURE_OPENAI_API_KEY` | Environment variable | API key for Azure authentication |\n| `AZURE_OPENAI_ENDPOINT` | Environment variable | Azure endpoint URL |\n| `AZURE_OPENAI_DEPLOYMENT` | Config | Deployment name in Azure |\n| `AZURE_OPENAI_API_VERSION` | Config | Azure API version |\n\n资料来源：[arksim/llms/chat/providers/azure_openai.py]()\n\n## Configuration\n\n### YAML Configuration Structure\n\nLLM providers are configured through the `config.yaml` file used by the simulator:\n\n```yaml\nllm:\n provider: openai # or anthropic, google, azure_openai\n model: gpt-4\n temperature: 0.7\n max_tokens: 2048\n \nagent_config:\n # Agent-specific LLM settings\n provider: anthropic\n model: claude-3-opus-20240229\n```\n\n### Environment Variables\n\n| Variable | Providers | Purpose |\n|----------|-----------|---------|\n| `OPENAI_API_KEY` | OpenAI, AutoGen, LangChain | OpenAI API authentication |\n| `ANTHROPIC_API_KEY` | Claude Agent SDK | Anthropic API authentication |\n| `GOOGLE_API_KEY` | Google | Google AI API authentication |\n| `AZURE_OPENAI_API_KEY` | Azure OpenAI | Azure API authentication |\n| `AZURE_OPENAI_ENDPOINT` | Azure OpenAI | Azure resource endpoint |\n\n## Integration with Custom Agents\n\nThe LLM provider system integrates with the custom agent connector pattern used in ArkSim simulations. Custom agents can be configured to use any supported LLM provider.\n\n```mermaid\ngraph LR\n A[Scenario JSON] --> B[Simulation Engine]\n B --> C[Custom Agent
custom_agent.py]\n C --> D[LLM Provider]\n D --> E[External LLM API]\n```\n\n### Integration Examples\n\nArkSim provides integration examples for various agent frameworks:\n\n| Framework | Example Path | LLM Provider Used |\n|-----------|--------------|-------------------|\n| LangChain/LangGraph | `examples/integrations/langchain/` | OpenAI |\n| Claude Agent SDK | `examples/integrations/claude-agent-sdk/` | Anthropic |\n| LlamaIndex | `examples/integrations/llamaindex/` | OpenAI |\n| CrewAI | `examples/integrations/crewai/` | OpenAI |\n| AutoGen | `examples/integrations/autogen/` | OpenAI |\n| Pydantic AI | `examples/integrations/pydantic-ai/` | OpenAI |\n| Smolagents | `examples/integrations/smolagents/` | OpenAI |\n| Dify | `examples/integrations/dify/` | Custom HTTP |\n\nEach integration demonstrates how to connect the framework's agent to ArkSim's simulation engine while delegating LLM calls to the appropriate provider.\n\n## Usage Flow\n\n```mermaid\nsequenceDiagram\n participant User\n participant Config as config.yaml\n participant LLM as LLM Manager\n participant Provider as Provider Class\n participant API as External LLM API\n \n User->>Config: Load configuration\n User->>LLM: Initialize with provider type\n LLM->>Provider: Create provider instance\n User->>LLM: chat(messages, model)\n LLM->>Provider: _chat_completion()\n Provider->>API: HTTP POST request\n API-->>Provider: Completion response\n Provider-->>LLM: Normalized response\n LLM-->>User: ChatCompletionResponse\n```\n\n## Adding New Providers\n\nTo add support for a new LLM provider:\n\n1. Create a new provider class inheriting from `BaseLLM`\n2. Implement the required methods: `chat()`, `chat_stream()`, `count_tokens()`\n3. Add the provider to the `Provider` enum in `base_llm.py`\n4. Update the provider factory logic in `llm.py` to instantiate the new provider\n5. Add integration tests and documentation\n\n资料来源：[arksim/llms/chat/base/base_llm.py]()\n资料来源：[arksim/llms/chat/llm.py]()\n\n## Error Handling\n\nProvider implementations handle common error scenarios:\n\n| Error Type | Handling Strategy |\n|------------|-------------------|\n| Authentication failures | Raise `AuthenticationError` with helpful message |\n| Rate limiting | Implement automatic retry with backoff |\n| Invalid request parameters | Raise `ValidationError` with parameter details |\n| Network timeouts | Retry with exponential backoff |\n| Model not found | Raise `ModelNotFoundError` listing available models |\n\n## Best Practices\n\n1. **API Key Security**: Store API keys in environment variables, never in configuration files committed to version control. ArkSim automatically loads keys from environment variables.\n\n2. **Token Tracking**: Use the `count_tokens()` method to monitor token usage and estimate costs before running large-scale simulations.\n\n3. **Streaming for Large Responses**: Enable streaming (`chat_stream()`) for scenarios expecting long agent responses to improve perceived responsiveness.\n\n4. **Provider Selection**: Choose Azure OpenAI for enterprise deployments requiring compliance certifications and dedicated infrastructure.\n\n5. **Model Selection**: Refer to the integration examples for recommended model configurations per provider and use case.\n\n---\n\n\n\n## Tool Call Capture\n\n### 相关页面\n\n相关主题：[Evaluation System](#evaluation-system), [Agent Types and Integration](#agent-types)\n\n相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/simulation_engine/tool_types.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/tool_types.py)\n- [arksim/simulation_engine/agent/clients/a2a.py](https://github.com/arklexai/arksim/blob/main/arksim/simulation_engine/agent/clients/a2a.py)\n- [arksim/tracing/openai.py](https://github.com/arklexai/arksim/blob/main/arksim/tracing/openai.py)\n- [examples/customer-service/custom_agent.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_agent.py)\n- [examples/customer-service/traced_agent.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/traced_agent.py)\n- [examples/customer-service/a2a_server/agent.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/a2a_server/agent.py)\n相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/scenario/entities.py](https://github.com/arklexai/arksim/blob/main/arksim/scenario/entities.py)\n- [arksim/templates/scenarios.json](https://github.com/arklexai/arksim/blob/main/arksim/templates/scenarios.json)\n- [examples/e-commerce/scenarios.json](https://github.com/arklexai/arksim/blob/main/examples/e-commerce/scenarios.json)\n- [examples/bank-insurance/scenarios.json](https://github.com/arklexai/arksim/blob/main/examples/bank-insurance/scenarios.json)\n- [examples/customer-service/scenarios.json](https://github.com/arklexai/arksim/blob/main/examples/customer-service/scenarios.json)\n相关源码文件
\n\n以下源码文件用于生成本页说明：\n\n- [arksim/config/types.py](https://github.com/arklexai/arksim/blob/main/arksim/config/types.py)\n- [arksim/config/core/agent.py](https://github.com/arklexai/arksim/blob/main/arksim/config/core/agent.py)\n- [arksim/config/utils.py](https://github.com/arklexai/arksim/blob/main/arksim/config/utils.py)\n- [arksim/config.yaml](https://github.com/arklexai/arksim/blob/main/arksim/config.yaml)\n- [arksim/config_simulate.yaml](https://github.com/arklexai/arksim/blob/main/arksim/config_simulate.yaml)\n- [arksim/config_evaluate.yaml](https://github.com/arklexai/arksim/blob/main/arksim/config_evaluate.yaml)\n- [arksim/cli.py](https://github.com/arklexai/arksim/blob/main/arksim/cli.py)\n- [examples/customer-service/custom_metrics.py](https://github.com/arklexai/arksim/blob/main/examples/customer-service/custom_metrics.py)\n