Skip to content

F
Ftt_Common
Commit 23830cb4 authored by 段廷瑞's avatar 段廷瑞
Browse files
Options

feat: 智能体交互优化：图片，命令行等

parent c64815c5
Expand all Show whitespace changes
Inline Side-by-side
Showing with 793 additions and 86 deletions
Tools/MyAgent/Agent/AgentEngine.cs
View file @ 23830cb4
......@@ -85,6 +85,9 @@ namespace Tools.MyAgent.Agent
/// <summary>是否向模型下发可选/高级工具（默认只下发核心工具）</summary>
private bool _includeOptionalTools;
/// <summary>v2.2 多模态：是否启用图片提问（附图以 image_url 内容块发送）</summary>
private bool _imageInputEnabled;
/// <summary>单次回答总时长兜底</summary>
private static readonly TimeSpan MaxRunDuration = TimeSpan.FromMinutes(10);
......@@ -120,7 +123,7 @@ namespace Tools.MyAgent.Agent
_contextTokens = cfg.MaxContextTokens > 0 ? cfg.MaxContextTokens : 32000;
_prompts.CustomLayer = cfg.CustomSystemPrompt ?? "";
_prompts.SystemPromptEnabled = cfg.SystemPromptEnabled;
_prompts.AutoMemory = cfg.EnableAutoMemory; // E3 问答沉淀提示（只提示确认，绝不静默写入）
_imageInputEnabled = cfg.EnableImageInput; // v2.2 多模态：图片提问开关
// #31 故障转移配置
_failover.Configure(cfg.EnableFailover, cfg.BackupBaseUrl, cfg.BackupModel, cfg.BackupApiKey);
......@@ -445,7 +448,7 @@ namespace Tools.MyAgent.Agent
{
if (m.Role == "user")
{
list.Add(new JObject { ["role"] = "user", ["content"] = m.Content ?? "" });
list.Add(new JObject { ["role"] = "user", ["content"] = BuildUserContent(m) });
}
else if (m.Role == "assistant" && !string.IsNullOrEmpty(m.Content))
{
......@@ -470,6 +473,80 @@ namespace Tools.MyAgent.Agent
return list;
}
/// <summary>
/// 组装用户消息 content：启用图片提问（v2.2）且消息带有效附图时，
/// 返回多模态内容块数组（text + 多个 image_url，base64 data URL）；否则返回纯文本。
/// <para>v2.3：支持多图（Images 列表优先，兼容旧会话单图 ImagePath）；全部图片无效时降级纯文本。</para>
/// </summary>
private JToken BuildUserContent(ChatMessage m)
{
string text = m.Content ?? "";
if (!_imageInputEnabled) return text;
var images = new List<string>();
if (m.Images != null)
foreach (var p in m.Images)
if (!string.IsNullOrEmpty(p) && !images.Contains(p))
images.Add(p);
if (!string.IsNullOrEmpty(m.ImagePath) && !images.Contains(m.ImagePath))
images.Add(m.ImagePath);
if (images.Count == 0) return text;
var arr = new JArray();
arr.Add(new JObject { ["type"] = "text", ["text"] = text });
int added = 0;
foreach (var p in images)
{
try
{
string dataUrl = ImageToDataUrl(p);
if (dataUrl != null)
{
arr.Add(new JObject { ["type"] = "image_url", ["image_url"] = new JObject { ["url"] = dataUrl } });
added++;
}
}
catch { }
}
return added > 0 ? (JToken)arr : (JToken)text; // 全部图片不可读/超限 → 降级纯文本
}
/// <summary>本地图片 → base64 data URL（原图上限 MaxImageBytes=4MB，避免超出 API 限制）；失败返回 null</summary>
private static string ImageToDataUrl(string path)
{
const long MaxImageBytes = 4 * 1024 * 1024;
if (string.IsNullOrEmpty(path) || !System.IO.File.Exists(path)) return null;
var fi = new System.IO.FileInfo(path);
if (fi.Length <= 0 || fi.Length > MaxImageBytes) return null;
string ext = System.IO.Path.GetExtension(path).ToLowerInvariant();
string mime = ext == ".jpg" || ext == ".jpeg" ? "image/jpeg"
: ext == ".gif" ? "image/gif"
: ext == ".bmp" ? "image/bmp"
: "image/png";
byte[] bytes = System.IO.File.ReadAllBytes(path);
return "data:" + mime + ";base64," + Convert.ToBase64String(bytes);
}
/// <summary>
/// 安全读取消息 content 文本（v2.2 多模态兼容）：
/// content 为数组（text + image_url 内容块）时取 text 块文本（忽略图片）；
/// 为字符串直接返回；其他返回 null。避免 JArray.Value&lt;string&gt;() 抛 InvalidCastException。
/// </summary>
private static string ContentText(JToken content)
{
if (content == null) return null;
if (content.Type == JTokenType.Array)
{
foreach (var item in content)
{
if (string.Equals(item["type"]?.Value<string>(), "text", StringComparison.OrdinalIgnoreCase))
return item["text"]?.Value<string>() ?? "";
}
return "";
}
return content.Value<string>();
}
/// <summary>从消息列表头部（跳过 system）裁剪最早 user/assistant 对，直到预算内或只剩 3 条</summary>
private static int DropOldestUntilUnder(JArray list, int budget)
{
......@@ -510,9 +587,19 @@ namespace Tools.MyAgent.Agent
int take = Math.Min(6, list.Count - 1);
if (take < 2) return 0;
var earliest = new JArray();
// 拼接最早消息文本（多模态消息仅取文本块，忽略图片，避免 data URL 大图进入压缩 prompt）
var sb = new StringBuilder();
for (int i = 1; i <= take; i++)
earliest.Add(list[i]);
{
var m = list[i];
string role = m["role"]?.Value<string>() ?? "user";
string text = ContentText(m["content"]);
if (string.IsNullOrEmpty(text)) continue;
if (text.Length > 400) text = text.Substring(0, 400) + "…";
sb.Append(role).Append(": ").Append(text).Append('\n');
}
string earliestText = sb.ToString();
if (earliestText.Length == 0) return 0;
string summary;
try
......@@ -520,7 +607,7 @@ namespace Tools.MyAgent.Agent
var prompt = new JArray
{
new JObject { ["role"] = "system", ["content"] = HistorySummaryPrompt },
new JObject { ["role"] = "user", ["content"] = earliest.ToString(Newtonsoft.Json.Formatting.None) }
new JObject { ["role"] = "user", ["content"] = earliestText }
};
summary = await _client.CompleteAsync(prompt, 512, ct).ConfigureAwait(false);
}
......@@ -554,7 +641,7 @@ namespace Tools.MyAgent.Agent
foreach (var m in messages)
{
total += 4; // role / 分隔符开销
var content = m["content"]?.Value<string>();
var content = ContentText(m["content"]);
if (!string.IsNullOrEmpty(content)) total += EstimateTokens(content);
var tc = m["tool_calls"] as JArray;
if (tc != null) total += 8 * tc.Count;
......
Tools/MyAgent/Agent/PromptBuilder.cs
View file @ 23830cb4
......@@ -18,9 +18,6 @@ namespace Tools.MyAgent.Agent
/// <summary>系统提示词启用开关（由设置注入；仅启用后才注入 CustomLayer）</summary>
public bool SystemPromptEnabled { get; set; }
/// <summary>E3 问答沉淀提示：开启后在工具规则尾部追加「可询问用户是否沉淀结论」（绝不静默写入）</summary>
public bool AutoMemory { get; set; }
public string BuildSystemMessage(KbService kb)
{
// B2 提示词外置：静态层优先读 输出目录/prompts/*.md 模板（用户可定制），
......@@ -42,13 +39,6 @@ namespace Tools.MyAgent.Agent
sb.AppendLine(LoadLayer(promptsDir, "security.md", SecurityLayer));
sb.AppendLine();
sb.AppendLine(LoadLayer(promptsDir, "tool_rules.md", ToolRules));
if (AutoMemory)
{
sb.AppendLine();
sb.AppendLine("【知识沉淀提示（E3）】");
sb.AppendLine("- 当你产出了有价值的结论（问题解答、流程梳理、新知识）时，可在回答末尾询问用户是否沉淀到知识库（kb_write_kb）。");
sb.AppendLine("- 未获得用户明确同意前，绝不允许写入知识库；用户拒绝则不再追问。");
}
return sb.ToString();
}
......
Tools/MyAgent/Config/AgentConfig.cs
View file @ 23830cb4
......@@ -90,7 +90,7 @@
/// <summary>是否启用可选/高级工具（db_query、module_patch、img_search、/workflow）</summary>
public bool EnableOptionalTools { get; set; } = false;
/// <summary>回答结束后是否提示把有价值结论沉淀到知识库（只提示确认，绝不静默写入；默认关闭）</summary>
public bool EnableAutoMemory { get; set; } = false;
/// <summary>是否启用图片提问（多模态，v2.2）：启用后粘贴截图随提问以 image_url 发送，需视觉模型支持（如 DeepSeek-VL / qwen-vl）；默认启用</summary>
public bool EnableImageInput { get; set; } = true;
}
}
Tools/MyAgent/Forms/AgentSettingsForm.Designer.cs
View file @ 23830cb4
......@@ -82,7 +82,7 @@
this.tabOther = new System.Windows.Forms.TabPage();
this.grpOpt = new System.Windows.Forms.GroupBox();
this.chkOpt = new System.Windows.Forms.CheckBox();
this.chkAutoMemory = new System.Windows.Forms.CheckBox();
this.chkImageInput = new System.Windows.Forms.CheckBox();
this.grpFail = new System.Windows.Forms.GroupBox();
this.chkFail = new System.Windows.Forms.CheckBox();
this.lblBUrl = new System.Windows.Forms.Label();
......@@ -717,7 +717,7 @@
// grpOpt
//
this.grpOpt.Controls.Add(this.chkOpt);
this.grpOpt.Controls.Add(this.chkAutoMemory);
this.grpOpt.Controls.Add(this.chkImageInput);
this.grpOpt.Location = new System.Drawing.Point(8, 12);
this.grpOpt.Name = "grpOpt";
this.grpOpt.Size = new System.Drawing.Size(596, 92);
......@@ -735,15 +735,17 @@
this.chkOpt.Text = "启用可选/高级工具（db_query、module_patch、img_search、/workflow）";
this.chkOpt.UseVisualStyleBackColor = true;
//
// chkAutoMemory
// chkImageInput
//
this.chkAutoMemory.AutoSize = true;
this.chkAutoMemory.Location = new System.Drawing.Point(16, 54);
this.chkAutoMemory.Name = "chkAutoMemory";
this.chkAutoMemory.Size = new System.Drawing.Size(351, 21);
this.chkAutoMemory.TabIndex = 1;
this.chkAutoMemory.Text = "回答后提示沉淀有价值结论到知识库（只提示，不静默写入）";
this.chkAutoMemory.UseVisualStyleBackColor = true;
this.chkImageInput.AutoSize = true;
this.chkImageInput.Checked = true;
this.chkImageInput.CheckState = System.Windows.Forms.CheckState.Checked;
this.chkImageInput.Location = new System.Drawing.Point(16, 54);
this.chkImageInput.Name = "chkImageInput";
this.chkImageInput.Size = new System.Drawing.Size(540, 21);
this.chkImageInput.TabIndex = 1;
this.chkImageInput.Text = "启用图片提问（多模态）：粘贴/拖入截图随提问以 image_url 发送，需视觉模型支持（如 qwen-vl）";
this.chkImageInput.UseVisualStyleBackColor = true;
//
// grpFail
//
......@@ -957,7 +959,7 @@
private System.Windows.Forms.TabPage tabOther;
private System.Windows.Forms.GroupBox grpOpt;
private System.Windows.Forms.CheckBox chkOpt;
private System.Windows.Forms.CheckBox chkAutoMemory;
private System.Windows.Forms.CheckBox chkImageInput;
private System.Windows.Forms.GroupBox grpFail;
private System.Windows.Forms.CheckBox chkFail;
private System.Windows.Forms.Label lblBUrl;
......
Tools/MyAgent/Forms/AgentSettingsForm.cs
View file @ 23830cb4
......@@ -137,7 +137,7 @@ namespace Tools.MyAgent.Forms
// 其他：可选能力
chkOpt.Checked = _config.EnableOptionalTools;
chkAutoMemory.Checked = _config.EnableAutoMemory;
chkImageInput.Checked = _config.EnableImageInput; // v2.2 多模态：图片提问开关
// 其他：故障转移
chkFail.Checked = _config.EnableFailover;
......@@ -429,7 +429,7 @@ namespace Tools.MyAgent.Forms
_config.CustomSystemPrompt = txtCustomPrompt.Text.Trim();
// 其他：可选能力
_config.EnableOptionalTools = chkOpt.Checked;
_config.EnableAutoMemory = chkAutoMemory.Checked;
_config.EnableImageInput = chkImageInput.Checked; // v2.2 多模态：图片提问开关
// 其他：故障转移
_config.EnableFailover = chkFail.Checked;
_config.BackupBaseUrl = txtBUrl.Text.Trim();
......
Tools/MyAgent/Forms/wBitmapView.cs 0 → 100644
View file @ 23830cb4
using System;
using System.Drawing;
using System.Windows.Forms;
namespace Tools.MyAgent
{
/// <summary>
/// 图片查看器：优先按图片原始尺寸展示（1:1 像素），但不超过屏幕工作区大小；
/// 图片大于工作区时窗体限制为工作区大小，可滚动查看。带边框与标题栏（可点 ✕ 关闭），置顶悬浮。
/// <para>点击任意处或按 Esc 也可关闭。纯代码构造，无 Designer 依赖。</para>
/// </summary>
public sealed class wBitmapView : Form
{
private Image _img;
private Label _hint;
private PictureBox _picture;
public wBitmapView(Image img)
{
_img = img;
//
this.TopMost = true;
this.ShowInTaskbar = false;
this.StartPosition = FormStartPosition.Manual;
// 带边框 + 标题栏（含 ✕ 关闭按钮），避免无边框窗体被误解为无法关闭
this.FormBorderStyle = FormBorderStyle.FixedToolWindow;
this.Text = $"图片查看 {img.Width} × {img.Height}";
this.AutoScroll = true;
this.KeyPreview = true;
// 原始尺寸：PictureBox 随图片自动扩展，窗体收缩时出现滚动条
_picture = new PictureBox
{
Location = new Point(0, 0),
SizeMode = PictureBoxSizeMode.AutoSize,
Image = img,
};
this.Controls.Add(_picture);
// 底部提示条：图片尺寸 + 关闭方式
_hint = new Label
{
Text = $" {img.Width} × {img.Height} · 点击任意处关闭 ",
AutoSize = true,
BackColor = Color.FromArgb(170, 0, 0, 0),
ForeColor = Color.White,
Font = new Font("微软雅黑", 9F),
Padding = new Padding(8, 3, 8, 3),
Anchor = AnchorStyles.Bottom,
};
this.Controls.Add(_hint);
_hint.BringToFront();
this.Resize += (s, e) => PositionHint();
this.Scroll += (s, e) => PositionHint();
// 点击任意处 / Esc 关闭
this.Click += (s, e) => this.Close();
_picture.Click += (s, e) => this.Close();
this.KeyDown += (s, e) => { if (e.KeyCode == Keys.Escape) this.Close(); };
// 关闭时释放位图
this.FormClosed += (s, e) =>
{
if (_img != null) { _img.Dispose(); _img = null; }
};
}
/// <summary>提示条水平居中于可视区底部</summary>
private void PositionHint()
{
if (_hint == null || !_hint.IsHandleCreated) return;
_hint.Location = new Point(
Math.Max(0, (this.ClientSize.Width - _hint.Width) / 2),
Math.Max(0, this.ClientSize.Height - _hint.Height - 8));
}
/// <summary>
/// 居中显示：优先按图片原始尺寸，但不超过屏幕工作区大小（超屏则限制在工作区内，滚动查看）。
/// </summary>
/// <param name="owner">调用方窗体：用于选取其所在屏幕的工作区（多屏场景），可传 null</param>
public void ShowView(Form owner = null)
{
// 选取屏幕：优先调用方窗体所在屏，其次鼠标所在屏
Rectangle wa;
if (owner != null && owner.IsHandleCreated)
wa = Screen.FromControl(owner).WorkingArea;
else
wa = Screen.FromPoint(Cursor.Position).WorkingArea;
// 核心限制：优先原尺寸，超屏取工作区大小（至少保留最小可视尺寸）
// 先强制创建句柄，使 Width/Height 反映窗体边框+标题栏装饰，保证窗体整体不超屏
IntPtr dummy = this.Handle;
int decoW = this.Width - this.ClientSize.Width;
int decoH = this.Height - this.ClientSize.Height;
int w = Math.Max(Math.Min(_img.Width, wa.Width - decoW), 120);
int h = Math.Max(Math.Min(_img.Height, wa.Height - decoH), 80);
this.ClientSize = new Size(w, h);
this.Location = new Point(wa.Left + (wa.Width - this.Width) / 2,
wa.Top + (wa.Height - this.Height) / 2);
this.Show();
this.Activate();
PositionHint();
}
/// <summary>从文件加载图片并按原始尺寸弹出查看（超屏自动限制在工作区内；内部负责释放位图）</summary>
public static wBitmapView ShowImage(string file, Form owner = null)
{
var img = Image.FromFile(file);
var v = new wBitmapView(img);
if (owner != null && owner.IsHandleCreated && owner.Visible)
v.Owner = owner; // 跟随主窗体：最小化/置顶关系一致
v.ShowView(owner);
return v;
}
}
}
Tools/MyAgent/Support/ChatMessage.cs
View file @ 23830cb4
using System;
using System.Collections.Generic;
namespace Tools.MyAgent
{
......@@ -12,5 +13,11 @@ namespace Tools.MyAgent
public string ThinkingContent { get; set; }
public bool IsStreaming { get; set; }
public DateTime Timestamp { get; set; } = DateTime.Now;
/// <summary>附图（多模态，v2.2）：本地图片绝对路径；启用图片提问时随消息以 image_url 内容块发送</summary>
public string ImagePath { get; set; }
/// <summary>附图列表（多图，v2.3）：一次提问可携带多张图片；优先于 ImagePath（旧会话兼容保留单图字段）</summary>
public List<string> Images { get; set; }
}
}
Tools/MyAgent/Support/NoCaretRichTextBox.cs 0 → 100644
View file @ 23830cb4
using System;
using System.Runtime.InteropServices;
using System.Windows.Forms;
namespace Tools.MyAgent
{
/// <summary>
/// 只读展示用 RichTextBox：隐藏文本插入光标（I 型插入符，不闪烁）。
/// <para>问答区不支持输入，无需显示插入光标；仍保留鼠标拖动选中与复制能力。
/// RichEdit 在获得焦点 / 鼠标按下 / 按键移动光标时会重新显示插入符，故在这些消息处理后再次隐藏。</para>
/// </summary>
public sealed class NoCaretRichTextBox : RichTextBox
{
private const int WM_SETFOCUS = 0x0007;
private const int WM_KEYDOWN = 0x0100;
private const int WM_LBUTTONDOWN = 0x0201;
private const int WM_LBUTTONUP = 0x0202;
[DllImport("user32.dll")]
private static extern bool HideCaret(IntPtr hWnd);
protected override void WndProc(ref Message m)
{
base.WndProc(ref m);
if (m.Msg == WM_SETFOCUS || m.Msg == WM_LBUTTONDOWN || m.Msg == WM_LBUTTONUP || m.Msg == WM_KEYDOWN)
{
try { HideCaret(m.HWnd); } catch { }
}
}
}
}
Tools/MyAgent/wMyAgent.cs
View file @ 23830cb4
This diff is collapsed.
Tools/Tools.csproj
View file @ 23830cb4
......@@ -190,6 +190,9 @@
<Compile Include="MyAgent\Forms\AgentSettingsForm.Designer.cs">
<DependentUpon>AgentSettingsForm.cs</DependentUpon>
</Compile>
<Compile Include="MyAgent\Forms\wBitmapView.cs">
<SubType>Form</SubType>
</Compile>
<Compile Include="MyAgent\Kb\KbConfig.cs" />
<Compile Include="MyAgent\Kb\KbService.cs" />
<Compile Include="MyAgent\Kb\KbIndex.cs" />
......@@ -204,6 +207,9 @@
<Compile Include="MyAgent\Support\ChatSession.cs" />
<Compile Include="MyAgent\Support\ChatRtfCache.cs" />
<Compile Include="MyAgent\Support\MarkdownRenderer.cs" />
<Compile Include="MyAgent\Support\NoCaretRichTextBox.cs">
<SubType>Component</SubType>
</Compile>
<Compile Include="MyAgent\wMyAgent.cs">
<SubType>Form</SubType>
</Compile>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment