feat(ai): 支持多语言AI分析响应并优化药品识别流程
- 饮食分析与药品分析服务新增多语言支持(zh-CN/en-US),根据用户偏好动态调整 Prompt 和返回信息 - 重构药品识别流程,利用 GLM-4.5v 模型将多阶段分析合并为单次全量分析,提升响应速度 - 增加用户语言获取逻辑,并在异步任务状态更新中支持本地化文案 - 移除废弃的药品分析 V1 接口,升级底层模型配置
This commit is contained in:
@@ -1,14 +1 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"context7": {
|
||||
"command": "npx",
|
||||
"args": [
|
||||
"-y",
|
||||
"@upstash/context7-mcp"
|
||||
],
|
||||
"env": {
|
||||
"DEFAULT_MINIMUM_TOKENS": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
{"mcpServers":{"context7":{"command":"npx","args":["-y","@upstash/context7-mcp"],"env":{"DEFAULT_MINIMUM_TOKENS":""},"alwaysAllow":["get-library-docs","resolve-library-id"]}}}
|
||||
@@ -175,7 +175,8 @@ export class AiCoachController {
|
||||
): Promise<FoodRecognitionResponseDto> {
|
||||
this.logger.log(`Food recognition request from user: ${user.sub}, images: ${body.imageUrls?.length || 0}`);
|
||||
|
||||
const result = await this.dietAnalysisService.recognizeFoodForConfirmation(body.imageUrls);
|
||||
const language = await this.usersService.getUserLanguage(user.sub);
|
||||
const result = await this.dietAnalysisService.recognizeFoodForConfirmation(body.imageUrls, language);
|
||||
|
||||
// 转换为DTO格式
|
||||
const response: FoodRecognitionResponseDto = {
|
||||
@@ -220,7 +221,8 @@ export class AiCoachController {
|
||||
): Promise<FoodRecognitionResponseDto> {
|
||||
this.logger.log(`Text food analysis request from user: ${user.sub}, text: "${body.text}"`);
|
||||
|
||||
const result = await this.dietAnalysisService.analyzeTextFoodForConfirmation(body.text);
|
||||
const language = await this.usersService.getUserLanguage(user.sub);
|
||||
const result = await this.dietAnalysisService.analyzeTextFoodForConfirmation(body.text, language);
|
||||
|
||||
// 转换为DTO格式
|
||||
const response: FoodRecognitionResponseDto = {
|
||||
|
||||
@@ -413,7 +413,8 @@ export class AiCoachService {
|
||||
): Promise<Readable | { type: 'structured'; data: any }> {
|
||||
if (params.imageUrls) {
|
||||
// 处理图片饮食记录
|
||||
const recognitionResult = await this.dietAnalysisService.recognizeFoodForConfirmation(params.imageUrls);
|
||||
const language = await this.usersService.getUserLanguage(params.userId);
|
||||
const recognitionResult = await this.dietAnalysisService.recognizeFoodForConfirmation(params.imageUrls, language);
|
||||
|
||||
if (recognitionResult.items.length > 0) {
|
||||
const choices = recognitionResult.items.map(item => ({
|
||||
@@ -467,6 +468,8 @@ export class AiCoachService {
|
||||
}
|
||||
} else {
|
||||
// 处理文本饮食记录
|
||||
// const language = await this.usersService.getUserLanguage(params.userId);
|
||||
// TODO: analyzeDietFromText 也需要支持多语言
|
||||
const textAnalysisResult = await this.dietAnalysisService.analyzeDietFromText(commandResult.cleanText);
|
||||
|
||||
if (textAnalysisResult.shouldRecord && textAnalysisResult.extractedData) {
|
||||
|
||||
@@ -5,6 +5,53 @@ import { DietRecordsService } from '../../diet-records/diet-records.service';
|
||||
import { CreateDietRecordDto } from '../../users/dto/diet-record.dto';
|
||||
import { MealType, DietRecordSource } from '../../users/models/user-diet-history.model';
|
||||
|
||||
const MESSAGES = {
|
||||
'zh-CN': {
|
||||
recognitionFailed: '食物识别失败,请稍后重试',
|
||||
serviceUnavailable: '服务暂时不可用,请稍后重试',
|
||||
analysisFailed: '图片分析失败,请稍后重试',
|
||||
textAnalysisFailed: '文本饮食分析失败,请稍后重试',
|
||||
textFoodAnalysisFailed: '文本食物分析失败,请稍后重试',
|
||||
noFoodInText: '未能从文本中识别到具体食物信息',
|
||||
provideMoreDetails: '请描述更具体的食物信息,如"吃了一碗米饭"、"喝了一杯牛奶"等',
|
||||
basedOnDescription: (text: string) => `基于您的描述"${text}",识别出以下食物`,
|
||||
textAnalysisParseFailed: '文本分析失败:无法解析识别结果',
|
||||
textAnalysisFailedRetry: '文本分析失败,请重新描述您吃的食物',
|
||||
recognizedCount: (text: string, count: number) => `基于您的描述"${text}",识别出 ${count} 种食物`,
|
||||
imageAnalysisFailed: '图片分析失败:无法解析识别结果',
|
||||
uploadImageRetry: '图片分析失败,请重新上传图片',
|
||||
noFoodDetected: '图片中未检测到食物',
|
||||
imageBlurred: '图片模糊,无法准确识别食物',
|
||||
foodRecognized: '已识别图片中的食物',
|
||||
uploadFoodImage: '图片中未检测到食物,请上传包含食物的图片',
|
||||
parseError: '图片分析失败:无法解析分析结果',
|
||||
noAnalysisDesc: '未提供分析说明',
|
||||
unknownFood: '未知食物'
|
||||
},
|
||||
'en-US': {
|
||||
recognitionFailed: 'Food recognition failed, please try again later',
|
||||
serviceUnavailable: 'Service temporarily unavailable, please try again later',
|
||||
analysisFailed: 'Image analysis failed, please try again later',
|
||||
textAnalysisFailed: 'Text diet analysis failed, please try again later',
|
||||
textFoodAnalysisFailed: 'Text food analysis failed, please try again later',
|
||||
noFoodInText: 'No specific food information identified from the text',
|
||||
provideMoreDetails: 'Please describe more specific food information, e.g., "ate a bowl of rice", "drank a glass of milk"',
|
||||
basedOnDescription: (text: string) => `Based on your description "${text}", the following foods were identified`,
|
||||
textAnalysisParseFailed: 'Text analysis failed: Unable to parse recognition result',
|
||||
textAnalysisFailedRetry: 'Text analysis failed, please describe your food again',
|
||||
recognizedCount: (text: string, count: number) => `Based on your description "${text}", ${count} foods were identified`,
|
||||
imageAnalysisFailed: 'Image analysis failed: Unable to parse recognition result',
|
||||
uploadImageRetry: 'Image analysis failed, please upload image again',
|
||||
noFoodDetected: 'No food detected in the image',
|
||||
imageBlurred: 'Image is blurred, unable to accurately recognize food',
|
||||
foodRecognized: 'Food recognized in the image',
|
||||
uploadFoodImage: 'No food detected in the image, please upload an image containing food',
|
||||
parseError: 'Image analysis failed: Unable to parse analysis result',
|
||||
noAnalysisDesc: 'No analysis description provided',
|
||||
unknownFood: 'Unknown Food'
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* 饮食分析结果接口
|
||||
*/
|
||||
@@ -88,7 +135,7 @@ export class DietAnalysisService {
|
||||
});
|
||||
|
||||
this.model = this.configService.get<string>('GLM_MODEL') || 'glm-4-flash';
|
||||
this.visionModel = this.configService.get<string>('GLM_VISION_MODEL') || 'glm-4v-plus';
|
||||
this.visionModel = 'glm-4v-flash'
|
||||
} else {
|
||||
// DashScope Configuration (default)
|
||||
const dashScopeApiKey = this.configService.get<string>('DASHSCOPE_API_KEY') || 'sk-e3ff4494c2f1463a8910d5b3d05d3143';
|
||||
@@ -178,29 +225,31 @@ export class DietAnalysisService {
|
||||
/**
|
||||
* 食物识别用于用户确认 - 新的确认流程
|
||||
* @param imageUrls 图片URL数组
|
||||
* @param language 语言代码,默认 zh-CN
|
||||
* @returns 食物识别确认结果
|
||||
*/
|
||||
async recognizeFoodForConfirmation(imageUrls: string[]): Promise<FoodRecognitionResult> {
|
||||
async recognizeFoodForConfirmation(imageUrls: string[], language: string = 'zh-CN'): Promise<FoodRecognitionResult> {
|
||||
try {
|
||||
const currentHour = new Date().getHours();
|
||||
const suggestedMealType = this.getSuggestedMealType(currentHour);
|
||||
|
||||
const prompt = this.buildFoodRecognitionPrompt(suggestedMealType);
|
||||
const prompt = this.buildFoodRecognitionPrompt(suggestedMealType, language);
|
||||
|
||||
const completion = await this.makeVisionApiCall(prompt, imageUrls);
|
||||
|
||||
const rawResult = completion.choices?.[0]?.message?.content || '{}';
|
||||
this.logger.log(`Food recognition result: ${rawResult}`);
|
||||
|
||||
return this.parseRecognitionResult(rawResult, suggestedMealType);
|
||||
return this.parseRecognitionResult(rawResult, suggestedMealType, language);
|
||||
} catch (error) {
|
||||
this.logger.error(`食物识别失败: ${error instanceof Error ? error.message : String(error)}`);
|
||||
const msgs = this.getMessages(language);
|
||||
return {
|
||||
items: [],
|
||||
analysisText: '食物识别失败,请稍后重试',
|
||||
analysisText: msgs.recognitionFailed,
|
||||
confidence: 0,
|
||||
isFoodDetected: false,
|
||||
nonFoodMessage: '服务暂时不可用,请稍后重试'
|
||||
nonFoodMessage: msgs.serviceUnavailable
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -264,9 +313,10 @@ export class DietAnalysisService {
|
||||
/**
|
||||
* 分析文本中的食物用于用户确认 - 与图片识别接口保持数据结构一致
|
||||
* @param userText 用户输入的文本描述
|
||||
* @param language 语言代码,默认 zh-CN
|
||||
* @returns 食物识别确认结果
|
||||
*/
|
||||
async analyzeTextFoodForConfirmation(userText: string): Promise<FoodRecognitionResult> {
|
||||
async analyzeTextFoodForConfirmation(userText: string, language: string = 'zh-CN'): Promise<FoodRecognitionResult> {
|
||||
try {
|
||||
this.logger.log(`Text food analysis request: ${userText}`);
|
||||
|
||||
@@ -274,22 +324,23 @@ export class DietAnalysisService {
|
||||
const suggestedMealType = this.getSuggestedMealType(currentHour);
|
||||
|
||||
// 使用专门的多食物文本分析 prompt
|
||||
const prompt = this.buildMultiFoodTextAnalysisPrompt(suggestedMealType);
|
||||
const prompt = this.buildMultiFoodTextAnalysisPrompt(suggestedMealType, language);
|
||||
const completion = await this.makeTextApiCall(prompt, userText);
|
||||
|
||||
const rawResult = completion.choices?.[0]?.message?.content || '{}';
|
||||
this.logger.log(`Multi-food text analysis result: ${rawResult}`);
|
||||
|
||||
// 直接解析为多食物结构
|
||||
return this.parseMultiFoodTextResult(rawResult, suggestedMealType, userText);
|
||||
return this.parseMultiFoodTextResult(rawResult, suggestedMealType, userText, language);
|
||||
} catch (error) {
|
||||
this.logger.error(`文本食物分析失败: ${error instanceof Error ? error.message : String(error)}`);
|
||||
const msgs = this.getMessages(language);
|
||||
return {
|
||||
items: [],
|
||||
analysisText: '文本食物分析失败,请稍后重试',
|
||||
analysisText: msgs.textFoodAnalysisFailed,
|
||||
confidence: 0,
|
||||
isFoodDetected: false,
|
||||
nonFoodMessage: '服务暂时不可用,请稍后重试'
|
||||
nonFoodMessage: msgs.serviceUnavailable
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -461,9 +512,10 @@ export class DietAnalysisService {
|
||||
/**
|
||||
* 构建食物识别提示(用于确认流程)
|
||||
* @param suggestedMealType 建议的餐次类型
|
||||
* @param language 语言代码
|
||||
* @returns 提示文本
|
||||
*/
|
||||
private buildFoodRecognitionPrompt(suggestedMealType: MealType): string {
|
||||
private buildFoodRecognitionPrompt(suggestedMealType: MealType, language: string): string {
|
||||
return `作为专业营养分析师,请分析这张图片并判断是否包含食物。
|
||||
|
||||
当前时间建议餐次:${suggestedMealType}
|
||||
@@ -475,17 +527,17 @@ export class DietAnalysisService {
|
||||
返回以下格式的JSON:
|
||||
{
|
||||
"confidence": number, // 整体识别置信度 0-100
|
||||
"analysisText": string, // 简短的识别说明文字
|
||||
"analysisText": string, // 简短的识别说明文字,请使用${language}语言
|
||||
"isFoodDetected": boolean, // 是否检测到食物
|
||||
"nonFoodMessage": string, // 当isFoodDetected为false时的提示信息
|
||||
"nonFoodMessage": string, // 当isFoodDetected为false时的提示信息,请使用${language}语言
|
||||
"recognizedItems": [ // 识别的食物列表(如果是食物才有内容)
|
||||
{
|
||||
"id": string, // 唯一标识符
|
||||
"foodName": string, // 食物名称
|
||||
"portion": string, // 份量描述(如"1碗"、"150g"等)
|
||||
"foodName": string, // 食物名称,请使用${language}语言
|
||||
"portion": string, // 份量描述,请使用${language}语言(如"1碗"、"150g"等)
|
||||
"calories": number, // 估算热量
|
||||
"mealType": "${suggestedMealType}", // 餐次类型
|
||||
"label": string, // 显示给用户的完整选项文本(如"一条鱼 200卡")
|
||||
"label": string, // 显示给用户的完整选项文本,请使用${language}语言(如"一条鱼 200卡")
|
||||
"nutritionData": {
|
||||
"proteinGrams": number, // 蛋白质
|
||||
"carbohydrateGrams": number, // 碳水化合物
|
||||
@@ -515,7 +567,11 @@ export class DietAnalysisService {
|
||||
|
||||
3. **模糊情况:**
|
||||
- 如果图片模糊但能看出是食物相关,设置 isFoodDetected: true,但返回空的recognizedItems数组
|
||||
- analysisText 说明"图片模糊,无法准确识别食物"`;
|
||||
- analysisText 说明"图片模糊,无法准确识别食物"
|
||||
|
||||
**重要提示:**
|
||||
请使用 ${language} 语言返回所有文本内容(包括label, analysisText, nonFoodMessage, foodName, portion等)。
|
||||
Please respond in ${language}.`;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -605,9 +661,10 @@ export class DietAnalysisService {
|
||||
/**
|
||||
* 构建多食物文本分析提示 - 支持识别多种食物
|
||||
* @param suggestedMealType 建议的餐次类型
|
||||
* @param language 语言代码
|
||||
* @returns 提示文本
|
||||
*/
|
||||
private buildMultiFoodTextAnalysisPrompt(suggestedMealType: MealType): string {
|
||||
private buildMultiFoodTextAnalysisPrompt(suggestedMealType: MealType, language: string): string {
|
||||
return `作为专业营养分析师,请分析用户描述的饮食内容,支持识别多种食物。
|
||||
|
||||
当前时间建议餐次:${suggestedMealType}
|
||||
@@ -615,17 +672,17 @@ export class DietAnalysisService {
|
||||
请返回以下格式的JSON(不要包含其他文本):
|
||||
{
|
||||
"confidence": number, // 整体识别置信度 0-100
|
||||
"analysisText": string, // 简短的识别说明文字
|
||||
"analysisText": string, // 简短的识别说明文字,请使用${language}语言
|
||||
"isFoodDetected": boolean, // 是否检测到食物
|
||||
"nonFoodMessage": string, // 当isFoodDetected为false时的提示信息
|
||||
"nonFoodMessage": string, // 当isFoodDetected为false时的提示信息,请使用${language}语言
|
||||
"recognizedItems": [ // 识别的食物列表
|
||||
{
|
||||
"id": string, // 唯一标识符(使用 food_1, food_2 等)
|
||||
"foodName": string, // 食物名称(简洁)
|
||||
"portion": string, // 份量描述(如"1碗"、"1份"等)
|
||||
"foodName": string, // 食物名称(简洁),请使用${language}语言
|
||||
"portion": string, // 份量描述(如"1碗"、"1份"等),请使用${language}语言
|
||||
"calories": number, // 估算热量
|
||||
"mealType": "${suggestedMealType}", // 餐次类型
|
||||
"label": string, // 显示给用户的完整选项文本(如"一碗米饭 280卡")
|
||||
"label": string, // 显示给用户的完整选项文本(如"一碗米饭 280卡"),请使用${language}语言
|
||||
"nutritionData": {
|
||||
"proteinGrams": number, // 蛋白质
|
||||
"carbohydrateGrams": number, // 碳水化合物
|
||||
@@ -660,7 +717,11 @@ export class DietAnalysisService {
|
||||
- "今天中午吃了一碗米饭,一份麻辣香锅" → 识别为2个选项
|
||||
- "早餐吃了燕麦粥加香蕉和牛奶" → 可识别为1个复合选项或3个独立选项
|
||||
- "晚上吃了牛肉面" → 识别为1个选项(面条+牛肉的复合菜品)
|
||||
- "喝了水" → isFoodDetected: false(水不是营养食物)`;
|
||||
- "喝了水" → isFoodDetected: false(水不是营养食物)
|
||||
|
||||
**重要提示:**
|
||||
请使用 ${language} 语言返回所有文本内容(包括label, analysisText, nonFoodMessage, foodName, portion等)。
|
||||
Please respond in ${language}.`;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -716,9 +777,11 @@ export class DietAnalysisService {
|
||||
* @param rawResult 原始结果字符串
|
||||
* @param suggestedMealType 建议的餐次类型
|
||||
* @param originalText 原始用户文本
|
||||
* @param language 语言代码
|
||||
* @returns 解析后的识别结果
|
||||
*/
|
||||
private parseMultiFoodTextResult(rawResult: string, suggestedMealType: MealType, originalText: string): FoodRecognitionResult {
|
||||
private parseMultiFoodTextResult(rawResult: string, suggestedMealType: MealType, originalText: string, language: string): FoodRecognitionResult {
|
||||
const msgs = this.getMessages(language);
|
||||
let parsedResult: any;
|
||||
try {
|
||||
parsedResult = JSON.parse(rawResult);
|
||||
@@ -726,10 +789,10 @@ export class DietAnalysisService {
|
||||
this.logger.error(`多食物文本分析JSON解析失败: ${parseError}`);
|
||||
return {
|
||||
items: [],
|
||||
analysisText: '文本分析失败:无法解析识别结果',
|
||||
analysisText: msgs.textAnalysisParseFailed,
|
||||
confidence: 0,
|
||||
isFoodDetected: false,
|
||||
nonFoodMessage: '文本分析失败,请重新描述您吃的食物'
|
||||
nonFoodMessage: msgs.textAnalysisFailedRetry
|
||||
};
|
||||
}
|
||||
|
||||
@@ -764,11 +827,11 @@ export class DietAnalysisService {
|
||||
// 根据是否识别到食物设置不同的分析文本
|
||||
let analysisText = parsedResult.analysisText || '';
|
||||
if (!isFoodDetected) {
|
||||
analysisText = analysisText || '文本中未检测到具体食物信息';
|
||||
analysisText = analysisText || msgs.noFoodInText;
|
||||
} else if (recognizedItems.length === 0) {
|
||||
analysisText = analysisText || '无法准确解析食物信息';
|
||||
analysisText = analysisText || msgs.noFoodInText;
|
||||
} else {
|
||||
analysisText = analysisText || `基于您的描述"${originalText}",识别出 ${recognizedItems.length} 种食物`;
|
||||
analysisText = analysisText || msgs.recognizedCount(originalText, recognizedItems.length);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -776,7 +839,7 @@ export class DietAnalysisService {
|
||||
analysisText,
|
||||
confidence: Math.min(100, Math.max(0, parsedResult.confidence || 0)),
|
||||
isFoodDetected,
|
||||
nonFoodMessage: !isFoodDetected ? (nonFoodMessage || '请描述更具体的食物信息,如"吃了一碗米饭"、"喝了一杯牛奶"等') : undefined
|
||||
nonFoodMessage: !isFoodDetected ? (nonFoodMessage || msgs.provideMoreDetails) : undefined
|
||||
};
|
||||
}
|
||||
|
||||
@@ -784,9 +847,11 @@ export class DietAnalysisService {
|
||||
* 解析食物识别结果
|
||||
* @param rawResult 原始结果字符串
|
||||
* @param suggestedMealType 建议的餐次类型
|
||||
* @param language 语言代码
|
||||
* @returns 解析后的识别结果
|
||||
*/
|
||||
private parseRecognitionResult(rawResult: string, suggestedMealType: MealType): FoodRecognitionResult {
|
||||
private parseRecognitionResult(rawResult: string, suggestedMealType: MealType, language: string): FoodRecognitionResult {
|
||||
const msgs = this.getMessages(language);
|
||||
let parsedResult: any;
|
||||
try {
|
||||
parsedResult = JSON.parse(rawResult);
|
||||
@@ -794,10 +859,10 @@ export class DietAnalysisService {
|
||||
this.logger.error(`食物识别JSON解析失败: ${parseError}`);
|
||||
return {
|
||||
items: [],
|
||||
analysisText: '图片分析失败:无法解析识别结果',
|
||||
analysisText: msgs.imageAnalysisFailed,
|
||||
confidence: 0,
|
||||
isFoodDetected: false,
|
||||
nonFoodMessage: '图片分析失败,请重新上传图片'
|
||||
nonFoodMessage: msgs.uploadImageRetry
|
||||
};
|
||||
}
|
||||
|
||||
@@ -832,11 +897,11 @@ export class DietAnalysisService {
|
||||
// 根据是否识别到食物设置不同的分析文本
|
||||
let analysisText = parsedResult.analysisText || '';
|
||||
if (!isFoodDetected) {
|
||||
analysisText = analysisText || '图片中未检测到食物';
|
||||
analysisText = analysisText || msgs.noFoodDetected;
|
||||
} else if (recognizedItems.length === 0) {
|
||||
analysisText = analysisText || '图片模糊,无法准确识别食物';
|
||||
analysisText = analysisText || msgs.imageBlurred;
|
||||
} else {
|
||||
analysisText = analysisText || '已识别图片中的食物';
|
||||
analysisText = analysisText || msgs.foodRecognized;
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -844,7 +909,7 @@ export class DietAnalysisService {
|
||||
analysisText,
|
||||
confidence: Math.min(100, Math.max(0, parsedResult.confidence || 0)),
|
||||
isFoodDetected,
|
||||
nonFoodMessage: !isFoodDetected ? (nonFoodMessage || '图片中未检测到食物,请上传包含食物的图片') : undefined
|
||||
nonFoodMessage: !isFoodDetected ? (nonFoodMessage || msgs.uploadFoodImage) : undefined
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1025,4 +1090,15 @@ export class DietAnalysisService {
|
||||
if (isNaN(num)) return undefined;
|
||||
return Math.max(min, Math.min(max, num));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取多语言消息
|
||||
*/
|
||||
private getMessages(language: string) {
|
||||
let langCode = 'zh-CN';
|
||||
if (language.toLowerCase().startsWith('en')) {
|
||||
langCode = 'en-US';
|
||||
}
|
||||
return MESSAGES[langCode] || MESSAGES['zh-CN'];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,6 +144,7 @@ export class DietRecordsController {
|
||||
): Promise<FoodRecognitionToDietRecordsResponseDto> {
|
||||
this.logger.log(`识别食物转饮食记录 - 用户ID: ${user.sub}, 图片URL: ${requestDto.imageUrl}`);
|
||||
return this.dietRecordsService.recognizeFoodToDietRecords(
|
||||
user.sub,
|
||||
requestDto.imageUrl,
|
||||
requestDto.mealType
|
||||
);
|
||||
@@ -164,6 +165,7 @@ export class DietRecordsController {
|
||||
): Promise<FoodRecognitionResponseDto> {
|
||||
this.logger.log(`识别食物 - 用户ID: ${user.sub}, 图片URL: ${requestDto.imageUrl}`);
|
||||
return this.dietRecordsService.recognizeFood(
|
||||
user.sub,
|
||||
requestDto.imageUrl,
|
||||
requestDto.mealType
|
||||
);
|
||||
|
||||
@@ -8,6 +8,7 @@ import { CreateDietRecordDto, UpdateDietRecordDto, GetDietHistoryQueryDto, DietR
|
||||
import { DietRecordSource, MealType } from '../users/models/user-diet-history.model';
|
||||
import { ResponseCode } from '../base.dto';
|
||||
import { DietAnalysisService } from '../ai-coach/services/diet-analysis.service';
|
||||
import { UsersService } from '../users/users.service';
|
||||
|
||||
@Injectable()
|
||||
export class DietRecordsService {
|
||||
@@ -21,6 +22,7 @@ export class DietRecordsService {
|
||||
private readonly sequelize: Sequelize,
|
||||
@Inject(forwardRef(() => DietAnalysisService))
|
||||
private readonly dietAnalysisService: DietAnalysisService,
|
||||
private readonly usersService: UsersService,
|
||||
) { }
|
||||
|
||||
/**
|
||||
@@ -296,14 +298,17 @@ export class DietRecordsService {
|
||||
* @returns 食物识别结果转换为饮食记录格式
|
||||
*/
|
||||
async recognizeFoodToDietRecords(
|
||||
userId: string,
|
||||
imageUrl: string,
|
||||
suggestedMealType?: MealType
|
||||
): Promise<FoodRecognitionToDietRecordsResponseDto> {
|
||||
try {
|
||||
this.logger.log(`recognizeFoodToDietRecords - imageUrl: ${imageUrl}, suggestedMealType: ${suggestedMealType}`);
|
||||
this.logger.log(`recognizeFoodToDietRecords - userId: ${userId}, imageUrl: ${imageUrl}, suggestedMealType: ${suggestedMealType}`);
|
||||
|
||||
const language = await this.usersService.getUserLanguage(userId);
|
||||
|
||||
// 调用 DietAnalysisService 进行食物识别
|
||||
const recognitionResult = await this.dietAnalysisService.recognizeFoodForConfirmation([imageUrl]);
|
||||
const recognitionResult = await this.dietAnalysisService.recognizeFoodForConfirmation([imageUrl], language);
|
||||
|
||||
// 将识别结果转换为 CreateDietRecordDto 格式
|
||||
const dietRecords: CreateDietRecordDto[] = recognitionResult.items.map(item => ({
|
||||
@@ -344,14 +349,17 @@ export class DietRecordsService {
|
||||
* @returns 食物识别结果
|
||||
*/
|
||||
async recognizeFood(
|
||||
userId: string,
|
||||
imageUrl: string,
|
||||
suggestedMealType?: MealType
|
||||
): Promise<FoodRecognitionResponseDto> {
|
||||
try {
|
||||
this.logger.log(`recognizeFood - imageUrl: ${imageUrl}, suggestedMealType: ${suggestedMealType}`);
|
||||
this.logger.log(`recognizeFood - userId: ${userId}, imageUrl: ${imageUrl}, suggestedMealType: ${suggestedMealType}`);
|
||||
|
||||
const language = await this.usersService.getUserLanguage(userId);
|
||||
|
||||
// 调用 DietAnalysisService 进行食物识别
|
||||
const recognitionResult = await this.dietAnalysisService.recognizeFoodForConfirmation([imageUrl]);
|
||||
const recognitionResult = await this.dietAnalysisService.recognizeFoodForConfirmation([imageUrl], language);
|
||||
|
||||
// 如果指定了建议的餐次类型,更新所有识别项的餐次类型
|
||||
if (suggestedMealType) {
|
||||
|
||||
@@ -8,11 +8,9 @@ import {
|
||||
Param,
|
||||
Query,
|
||||
UseGuards,
|
||||
Res,
|
||||
Logger,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger';
|
||||
import { Response } from 'express';
|
||||
import { MedicationsService } from './medications.service';
|
||||
import { CreateMedicationDto } from './dto/create-medication.dto';
|
||||
import { UpdateMedicationDto } from './dto/update-medication.dto';
|
||||
@@ -156,88 +154,6 @@ export class MedicationsController {
|
||||
return ApiResponseDto.success(medication, '激活成功');
|
||||
}
|
||||
|
||||
@Post(':id/ai-analysis')
|
||||
@ApiOperation({
|
||||
summary: '获取药品AI分析',
|
||||
description: '使用大模型分析药品信息,提供专业的用药指导、注意事项和健康建议。支持视觉识别药品图片。返回Server-Sent Events流式响应。'
|
||||
})
|
||||
@ApiResponse({
|
||||
status: 200,
|
||||
description: '返回流式文本分析结果',
|
||||
content: {
|
||||
'text/event-stream': {
|
||||
schema: {
|
||||
type: 'string',
|
||||
example: '药品分析内容...'
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
@ApiResponse({
|
||||
status: 403,
|
||||
description: '免费使用次数已用完'
|
||||
})
|
||||
async getAiAnalysis(
|
||||
@CurrentUser() user: any,
|
||||
@Param('id') id: string,
|
||||
@Res() res: Response,
|
||||
) {
|
||||
try {
|
||||
// 检查用户免费使用次数
|
||||
const userUsageCount = await this.usersService.getUserUsageCount(user.sub);
|
||||
|
||||
// 如果用户不是VIP且免费次数不足,返回错误
|
||||
if (userUsageCount <= 0) {
|
||||
this.logger.warn(`药品AI分析失败 - 用户ID: ${user.sub}, 免费次数不足`);
|
||||
res.status(403).json(
|
||||
ApiResponseDto.error('免费使用次数已用完,请开通会员获取更多使用次数'),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// 设置SSE响应头
|
||||
res.setHeader('Content-Type', 'text/event-stream');
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
res.setHeader('Connection', 'keep-alive');
|
||||
res.setHeader('X-Accel-Buffering', 'no'); // 禁用nginx缓冲
|
||||
|
||||
// 获取分析流
|
||||
const stream = await this.analysisService.analyzeMedication(id, user.sub);
|
||||
|
||||
// 分析成功后扣减用户免费使用次数
|
||||
try {
|
||||
await this.usersService.deductUserUsageCount(user.sub, 1);
|
||||
this.logger.log(`药品AI分析成功,已扣减用户免费次数 - 用户ID: ${user.sub}, 剩余次数: ${userUsageCount - 1}`);
|
||||
} catch (deductError) {
|
||||
this.logger.error(`扣减用户免费次数失败 - 用户ID: ${user.sub}, 错误: ${deductError instanceof Error ? deductError.message : String(deductError)}`);
|
||||
// 不影响主流程,继续返回分析结果
|
||||
}
|
||||
|
||||
// 将流式数据写入响应
|
||||
stream.on('data', (chunk: Buffer) => {
|
||||
res.write(chunk.toString());
|
||||
});
|
||||
|
||||
stream.on('end', () => {
|
||||
res.end();
|
||||
});
|
||||
|
||||
stream.on('error', (error) => {
|
||||
res.status(500).json(
|
||||
ApiResponseDto.error(
|
||||
error instanceof Error ? error.message : '分析过程中发生错误',
|
||||
),
|
||||
);
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json(
|
||||
ApiResponseDto.error(
|
||||
error instanceof Error ? error.message : '药品分析失败',
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@Post(':id/ai-analysis/v2')
|
||||
@ApiOperation({
|
||||
summary: '获取药品AI分析 (V2)',
|
||||
|
||||
@@ -3,10 +3,18 @@ import { ConfigService } from '@nestjs/config';
|
||||
import { InjectModel } from '@nestjs/sequelize';
|
||||
import { OpenAI } from 'openai';
|
||||
import { Readable } from 'stream';
|
||||
import { UsersService } from '../../users/users.service';
|
||||
import { MedicationsService } from '../medications.service';
|
||||
import { Medication } from '../models/medication.model';
|
||||
import { AiAnalysisResultDto } from '../dto/ai-analysis-result.dto';
|
||||
|
||||
interface LanguageConfig {
|
||||
label: string;
|
||||
analysisInstruction: string;
|
||||
jsonInstruction: string;
|
||||
unableToIdentifyMessage: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* 药品AI分析服务
|
||||
* 使用 GLM-4.5V 大模型分析药品信息,提供专业的用药指导和健康建议
|
||||
@@ -21,6 +29,7 @@ export class MedicationAnalysisService {
|
||||
constructor(
|
||||
private readonly configService: ConfigService,
|
||||
private readonly medicationsService: MedicationsService,
|
||||
private readonly usersService: UsersService,
|
||||
@InjectModel(Medication)
|
||||
private readonly medicationModel: typeof Medication,
|
||||
) {
|
||||
@@ -33,8 +42,8 @@ export class MedicationAnalysisService {
|
||||
baseURL: glmBaseURL,
|
||||
});
|
||||
|
||||
this.model = this.configService.get<string>('GLM_MODEL') || 'glm-4-flash';
|
||||
this.visionModel = this.configService.get<string>('GLM_VISION_MODEL') || 'glm-4v-plus';
|
||||
this.model = this.configService.get<string>('GLM_MODEL') || 'glm-4.6';
|
||||
this.visionModel = this.configService.get<string>('GLM_VISION_MODEL') || 'glm-4.5v';
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -48,10 +57,11 @@ export class MedicationAnalysisService {
|
||||
try {
|
||||
// 1. 获取药品信息
|
||||
const medication = await this.medicationsService.findOne(medicationId, userId);
|
||||
const languageConfig = await this.getUserLanguageConfig(userId);
|
||||
|
||||
this.logger.log(`获取到药品信息: ${JSON.stringify(medication, null, 2)}`)
|
||||
// 2. 构建专业医药分析提示
|
||||
const prompt = this.buildMedicationAnalysisPrompt(medication);
|
||||
const prompt = this.buildMedicationAnalysisPrompt(medication, languageConfig);
|
||||
|
||||
// 3. 调用AI模型进行分析
|
||||
if (medication.photoUrl) {
|
||||
@@ -78,10 +88,11 @@ export class MedicationAnalysisService {
|
||||
try {
|
||||
// 1. 获取药品信息
|
||||
const medication = await this.medicationsService.findOne(medicationId, userId);
|
||||
const languageConfig = await this.getUserLanguageConfig(userId);
|
||||
|
||||
this.logger.log(`获取到药品信息: ${JSON.stringify(medication, null, 2)}`);
|
||||
// 2. 构建专业医药分析提示
|
||||
const prompt = this.buildMedicationAnalysisPromptV2(medication);
|
||||
const prompt = this.buildMedicationAnalysisPromptV2(medication, languageConfig);
|
||||
|
||||
let result: AiAnalysisResultDto;
|
||||
|
||||
@@ -346,7 +357,7 @@ export class MedicationAnalysisService {
|
||||
* @param medication 药品信息
|
||||
* @returns 分析提示文本
|
||||
*/
|
||||
private buildMedicationAnalysisPrompt(medication: Medication): string {
|
||||
private buildMedicationAnalysisPrompt(medication: Medication, languageConfig: LanguageConfig): string {
|
||||
const formName = this.getMedicationFormName(medication.form);
|
||||
const dosageInfo = `${medication.dosageValue}${medication.dosageUnit}`;
|
||||
|
||||
@@ -387,6 +398,10 @@ ${medication.note ? `- 用户备注:${medication.note}` : ''}
|
||||
6. 给予健康关怀和鼓励
|
||||
7. 如果有图片,请结合图片信息提供更准确的分析
|
||||
|
||||
**语言要求**:
|
||||
${languageConfig.analysisInstruction}
|
||||
- 将所有标题、要点、提醒翻译为${languageConfig.label}(保留药品名称、成分等专有名词的常用写法),不要混用其他语言
|
||||
|
||||
**输出格式要求**:
|
||||
|
||||
**情况A:无法识别药品时**(药品名称不明确、过于笼统、随意输入、或缺少必要信息),请使用以下格式:
|
||||
@@ -491,7 +506,7 @@ ${medication.note ? `- 用户备注:${medication.note}` : ''}
|
||||
* @param medication 药品信息
|
||||
* @returns 分析提示文本
|
||||
*/
|
||||
private buildMedicationAnalysisPromptV2(medication: Medication): string {
|
||||
private buildMedicationAnalysisPromptV2(medication: Medication, languageConfig: LanguageConfig): string {
|
||||
const formName = this.getMedicationFormName(medication.form);
|
||||
const dosageInfo = `${medication.dosageValue}${medication.dosageUnit}`;
|
||||
|
||||
@@ -507,6 +522,11 @@ ${medication.note ? `- 用户备注:${medication.note}` : ''}
|
||||
${medication.photoUrl ? '- 药品图片:已提供(请结合图片中的药品外观、包装、说明书等信息进行分析)' : ''}
|
||||
${medication.note ? `- 用户备注:${medication.note}` : ''}
|
||||
|
||||
**语言要求**:
|
||||
- ${languageConfig.jsonInstruction}
|
||||
- 如果需要描述或解释,请使用${languageConfig.label}
|
||||
- 无法识别药品时,mainUsage 字段返回 "${languageConfig.unableToIdentifyMessage}"
|
||||
|
||||
**重要指示**:
|
||||
请以严格的 JSON 格式返回分析结果,不要包含任何 Markdown 标记或其他文本。JSON 结构如下:
|
||||
|
||||
@@ -529,7 +549,7 @@ ${medication.note ? `- 用户备注:${medication.note}` : ''}
|
||||
6. storageAdvice: 储存和保管建议,字符串数组
|
||||
7. healthAdvice: 健康关怀建议(生活方式、饮食等),字符串数组
|
||||
|
||||
如果无法识别药品,请在所有数组字段返回空数组,mainUsage 返回 "无法识别药品,请提供更准确的名称或图片"。
|
||||
如果无法识别药品,请在所有数组字段返回空数组,mainUsage 返回 "${languageConfig.unableToIdentifyMessage}"。
|
||||
`;
|
||||
}
|
||||
|
||||
@@ -552,4 +572,40 @@ ${medication.note ? `- 用户备注:${medication.note}` : ''}
|
||||
};
|
||||
return formNames[form] || form;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据用户ID获取语言配置
|
||||
*/
|
||||
private async getUserLanguageConfig(userId: string): Promise<LanguageConfig> {
|
||||
try {
|
||||
const language = await this.usersService.getUserLanguage(userId);
|
||||
return this.buildLanguageConfig(language);
|
||||
} catch (error) {
|
||||
this.logger.error(`获取用户语言失败,使用默认中文: ${error instanceof Error ? error.message : String(error)}`);
|
||||
return this.buildLanguageConfig();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 将语言代码映射为提示配置
|
||||
*/
|
||||
private buildLanguageConfig(language?: string): LanguageConfig {
|
||||
const normalized = (language || '').toLowerCase();
|
||||
|
||||
if (normalized.startsWith('en')) {
|
||||
return {
|
||||
label: 'English',
|
||||
analysisInstruction: 'Respond entirely in English. Translate every section title, bullet point and paragraph; keep emojis unchanged.',
|
||||
jsonInstruction: 'Return all JSON values in English. Keep JSON keys exactly as defined in the schema.',
|
||||
unableToIdentifyMessage: 'Unable to identify the medication, please provide a more accurate name or image.',
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
label: '简体中文',
|
||||
analysisInstruction: '请使用简体中文输出全部内容(包括标题、要点和提醒),不要混用其他语言。',
|
||||
jsonInstruction: '请确保 JSON 中的值使用简体中文,字段名保持英文。',
|
||||
unableToIdentifyMessage: '无法识别药品,请提供更准确的名称或图片。',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,34 @@ import {
|
||||
RecognitionStatusEnum,
|
||||
RECOGNITION_STATUS_DESCRIPTIONS,
|
||||
} from '../enums/recognition-status.enum';
|
||||
import { UsersService } from '../../users/users.service';
|
||||
|
||||
const STATUS_MESSAGES = {
|
||||
'zh-CN': {
|
||||
[RecognitionStatusEnum.PENDING]: '任务已创建,等待处理',
|
||||
[RecognitionStatusEnum.ANALYZING_PRODUCT]: '正在全方位分析药品信息...',
|
||||
ANALYZING_PRODUCT_DONE: '药品分析完成',
|
||||
[RecognitionStatusEnum.ANALYZING_SUITABILITY]: '正在分析适宜人群...',
|
||||
ANALYZING_SUITABILITY_DONE: '适宜人群分析完成',
|
||||
[RecognitionStatusEnum.ANALYZING_INGREDIENTS]: '正在分析主要成分...',
|
||||
ANALYZING_INGREDIENTS_DONE: '成分分析完成',
|
||||
[RecognitionStatusEnum.ANALYZING_EFFECTS]: '正在分析副作用和健康建议...',
|
||||
[RecognitionStatusEnum.COMPLETED]: '识别完成',
|
||||
[RecognitionStatusEnum.FAILED]: '识别失败',
|
||||
},
|
||||
'en-US': {
|
||||
[RecognitionStatusEnum.PENDING]: 'Task created, waiting for processing',
|
||||
[RecognitionStatusEnum.ANALYZING_PRODUCT]: 'Analyzing medication information comprehensively...',
|
||||
ANALYZING_PRODUCT_DONE: 'Medication analysis completed',
|
||||
[RecognitionStatusEnum.ANALYZING_SUITABILITY]: 'Analyzing suitable users...',
|
||||
ANALYZING_SUITABILITY_DONE: 'Suitability analysis completed',
|
||||
[RecognitionStatusEnum.ANALYZING_INGREDIENTS]: 'Analyzing main ingredients...',
|
||||
ANALYZING_INGREDIENTS_DONE: 'Ingredients analysis completed',
|
||||
[RecognitionStatusEnum.ANALYZING_EFFECTS]: 'Analyzing side effects and health advice...',
|
||||
[RecognitionStatusEnum.COMPLETED]: 'Recognition completed',
|
||||
[RecognitionStatusEnum.FAILED]: 'Recognition failed',
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* 药物AI识别服务
|
||||
@@ -26,6 +54,7 @@ export class MedicationRecognitionService {
|
||||
private readonly configService: ConfigService,
|
||||
@InjectModel(MedicationRecognitionTask)
|
||||
private readonly taskModel: typeof MedicationRecognitionTask,
|
||||
private readonly usersService: UsersService,
|
||||
) {
|
||||
const glmApiKey = this.configService.get<string>('GLM_API_KEY');
|
||||
const glmBaseURL =
|
||||
@@ -40,7 +69,7 @@ export class MedicationRecognitionService {
|
||||
this.visionModel =
|
||||
this.configService.get<string>('GLM_VISION_MODEL') || 'glm-4.5v';
|
||||
this.textModel =
|
||||
this.configService.get<string>('GLM_MODEL') || 'glm-4.5-air';
|
||||
this.configService.get<string>('GLM_MODEL') || 'glm-4.5-flash';
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -54,6 +83,13 @@ export class MedicationRecognitionService {
|
||||
|
||||
this.logger.log(`创建药物识别任务: ${taskId}, 用户: ${userId}`);
|
||||
|
||||
// 获取用户语言
|
||||
const language = await this.usersService.getUserLanguage(userId);
|
||||
const currentStep = this.getStatusMessage(
|
||||
RecognitionStatusEnum.PENDING,
|
||||
language,
|
||||
);
|
||||
|
||||
await this.taskModel.create({
|
||||
id: taskId,
|
||||
userId,
|
||||
@@ -61,7 +97,7 @@ export class MedicationRecognitionService {
|
||||
sideImageUrl: dto.sideImageUrl,
|
||||
auxiliaryImageUrl: dto.auxiliaryImageUrl,
|
||||
status: RecognitionStatusEnum.PENDING,
|
||||
currentStep: RECOGNITION_STATUS_DESCRIPTIONS[RecognitionStatusEnum.PENDING],
|
||||
currentStep,
|
||||
progress: 0,
|
||||
});
|
||||
|
||||
@@ -112,66 +148,25 @@ export class MedicationRecognitionService {
|
||||
const task = await this.taskModel.findByPk(taskId);
|
||||
if (!task) return;
|
||||
|
||||
// 阶段1: 产品识别分析 (0-40%)
|
||||
// 获取用户语言
|
||||
const language = await this.usersService.getUserLanguage(task.userId);
|
||||
|
||||
// 阶段1: 全量识别分析 (0-90%)
|
||||
// 使用 GLM-4.5v 强大的多模态能力,一次性提取所有信息,避免多次调用
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_PRODUCT,
|
||||
'正在识别药品基本信息...',
|
||||
this.getStatusMessage(RecognitionStatusEnum.ANALYZING_PRODUCT, language),
|
||||
10,
|
||||
);
|
||||
const productInfo = await this.recognizeProduct(task);
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_PRODUCT,
|
||||
'药品基本信息识别完成',
|
||||
40,
|
||||
);
|
||||
|
||||
this.logger.log(`任务 ${taskId} 开始执行全量识别分析(视觉+知识库)`);
|
||||
|
||||
const recognitionResult = await this.recognizeProduct(task, language);
|
||||
|
||||
// 阶段2: 适宜人群分析 (40-60%)
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_SUITABILITY,
|
||||
'正在分析适宜人群...',
|
||||
50,
|
||||
);
|
||||
const suitabilityInfo = await this.analyzeSuitability(productInfo);
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_SUITABILITY,
|
||||
'适宜人群分析完成',
|
||||
60,
|
||||
);
|
||||
|
||||
// 阶段3: 成分分析 (60-80%)
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_INGREDIENTS,
|
||||
'正在分析主要成分...',
|
||||
70,
|
||||
);
|
||||
const ingredientsInfo = await this.analyzeIngredients(productInfo);
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_INGREDIENTS,
|
||||
'成分分析完成',
|
||||
80,
|
||||
);
|
||||
|
||||
// 阶段4: 副作用分析 (80-100%)
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_EFFECTS,
|
||||
'正在分析副作用和健康建议...',
|
||||
90,
|
||||
);
|
||||
const effectsInfo = await this.analyzeEffects(productInfo);
|
||||
|
||||
// 合并所有结果,透传所有原始图片URL(避免被AI模型修改)
|
||||
// 合并结果,透传所有原始图片URL
|
||||
const finalResult = {
|
||||
...productInfo,
|
||||
...suitabilityInfo,
|
||||
...ingredientsInfo,
|
||||
...effectsInfo,
|
||||
...recognitionResult,
|
||||
// 强制使用任务记录中存储的原始图片URL,覆盖AI可能返回的不正确链接
|
||||
photoUrl: task.frontImageUrl,
|
||||
sideImageUrl: task.sideImageUrl,
|
||||
@@ -179,23 +174,33 @@ export class MedicationRecognitionService {
|
||||
} as RecognitionResultDto;
|
||||
|
||||
// 完成识别
|
||||
await this.completeTask(taskId, finalResult);
|
||||
await this.completeTask(taskId, finalResult, language);
|
||||
this.logger.log(`识别任务 ${taskId} 完成`);
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
this.logger.error(`识别任务 ${taskId} 失败: ${errorMessage}`);
|
||||
await this.failTask(taskId, errorMessage);
|
||||
// 尝试获取任务和语言信息以更新失败状态
|
||||
try {
|
||||
const taskInfo = await this.taskModel.findByPk(taskId);
|
||||
const lang = taskInfo
|
||||
? await this.usersService.getUserLanguage(taskInfo.userId)
|
||||
: 'zh-CN';
|
||||
await this.failTask(taskId, errorMessage, lang);
|
||||
} catch (e) {
|
||||
this.logger.error(`更新失败状态出错: ${e}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 阶段1: 识别药品基本信息
|
||||
* 执行全量药品识别(包含基本信息和详细分析)
|
||||
*/
|
||||
private async recognizeProduct(
|
||||
task: MedicationRecognitionTask,
|
||||
language: string,
|
||||
): Promise<Partial<RecognitionResultDto>> {
|
||||
const prompt = this.buildProductRecognitionPrompt();
|
||||
const prompt = this.buildProductRecognitionPrompt(language);
|
||||
const images = [task.frontImageUrl, task.sideImageUrl];
|
||||
if (task.auxiliaryImageUrl) images.push(task.auxiliaryImageUrl);
|
||||
|
||||
@@ -256,100 +261,10 @@ export class MedicationRecognitionService {
|
||||
}
|
||||
|
||||
/**
|
||||
* 阶段2: 分析适宜人群
|
||||
* 构建全量产品识别提示词
|
||||
*/
|
||||
private async analyzeSuitability(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): Promise<Partial<RecognitionResultDto>> {
|
||||
const prompt = this.buildSuitabilityAnalysisPrompt(productInfo);
|
||||
|
||||
this.logger.log(`分析适宜人群: ${productInfo.name}`);
|
||||
|
||||
const response = await this.client.chat.completions.create({
|
||||
model: this.textModel,
|
||||
temperature: 0.7,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
response_format: { type: 'json_object' },
|
||||
});
|
||||
|
||||
const content = response.choices[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error('AI模型返回内容为空');
|
||||
}
|
||||
|
||||
return this.parseJsonResponse(content);
|
||||
}
|
||||
|
||||
/**
|
||||
* 阶段3: 分析主要成分
|
||||
*/
|
||||
private async analyzeIngredients(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): Promise<Partial<RecognitionResultDto>> {
|
||||
const prompt = this.buildIngredientsAnalysisPrompt(productInfo);
|
||||
|
||||
this.logger.log(`分析主要成分: ${productInfo.name}`);
|
||||
|
||||
const response = await this.client.chat.completions.create({
|
||||
model: this.textModel,
|
||||
temperature: 0.7,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
response_format: { type: 'json_object' },
|
||||
});
|
||||
|
||||
const content = response.choices[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error('AI模型返回内容为空');
|
||||
}
|
||||
|
||||
return this.parseJsonResponse(content);
|
||||
}
|
||||
|
||||
/**
|
||||
* 阶段4: 分析副作用和健康建议
|
||||
*/
|
||||
private async analyzeEffects(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): Promise<Partial<RecognitionResultDto>> {
|
||||
const prompt = this.buildEffectsAnalysisPrompt(productInfo);
|
||||
|
||||
this.logger.log(`分析副作用和健康建议: ${productInfo.name}`);
|
||||
|
||||
const response = await this.client.chat.completions.create({
|
||||
model: this.textModel,
|
||||
temperature: 0.7,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
response_format: { type: 'json_object' },
|
||||
});
|
||||
|
||||
const content = response.choices[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error('AI模型返回内容为空');
|
||||
}
|
||||
|
||||
return this.parseJsonResponse(content);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建产品识别提示词
|
||||
*/
|
||||
private buildProductRecognitionPrompt(): string {
|
||||
return `你是一位拥有20年从业经验的资深药剂师,请根据提供的药品图片(包括正面、侧面和可能的辅助面)进行详细分析。
|
||||
private buildProductRecognitionPrompt(language: string): string {
|
||||
return `你是一位拥有20年从业经验的资深药剂师,请根据提供的药品图片(包括正面、侧面和可能的辅助面)进行全方位的详细分析。
|
||||
|
||||
**重要前提条件 - 图片可读性判断**:
|
||||
⚠️ 在进行任何识别之前,你必须首先判断图片是否足够清晰可读:
|
||||
@@ -360,10 +275,10 @@ export class MedicationRecognitionService {
|
||||
|
||||
**只有在图片清晰可读的情况下才能继续分析**:
|
||||
1. 仔细观察药品包装、说明书上的所有信息
|
||||
2. 识别药品的完整名称(通用名和商品名)
|
||||
3. 确定药物剂型(片剂/胶囊/注射剂等)
|
||||
4. 提取规格剂量信息
|
||||
5. 推荐合理的服用次数和时间
|
||||
2. 识别药品的完整名称、剂型、规格剂量
|
||||
3. 分析适宜人群、禁忌人群
|
||||
4. 提取主要成分、副作用、储存建议
|
||||
5. 给出健康建议和服用时间
|
||||
|
||||
**置信度评估标准(仅在图片可读时评估)**:
|
||||
- 如果图片清晰且信息完整,置信度应 >= 0.8
|
||||
@@ -371,9 +286,13 @@ export class MedicationRecognitionService {
|
||||
- 如果关键信息缺失或模糊不清,置信度 < 0.6,name返回"无法识别"
|
||||
- 置信度评估必须严格基于实际可见信息,不能猜测或臆断
|
||||
|
||||
**重要提示**:
|
||||
请使用 ${language} 语言返回所有文本内容。
|
||||
Please respond in ${language}.
|
||||
|
||||
**返回严格的JSON格式**(不要包含任何markdown标记):
|
||||
{
|
||||
"isReadable": true或false(图片是否足够清晰可读),
|
||||
"isReadable": true或false,
|
||||
"name": "药品完整名称",
|
||||
"photoUrl": "使用正面图片URL",
|
||||
"form": "剂型(tablet/capsule/injection/drops/syrup/ointment/powder/granules)",
|
||||
@@ -381,97 +300,26 @@ export class MedicationRecognitionService {
|
||||
"dosageUnit": "剂量单位",
|
||||
"timesPerDay": 建议每日服用次数(数字),
|
||||
"medicationTimes": ["建议的服药时间,格式HH:mm"],
|
||||
"confidence": 识别置信度(0-1的小数)
|
||||
}
|
||||
|
||||
**关键规则(必须遵守)**:
|
||||
1. isReadable 是最重要的字段,如果为 false,其他识别结果将被忽略
|
||||
2. 当图片模糊、反光、文字不清晰时,必须设置 isReadable 为 false
|
||||
3. 只有在确实能看清并理解图片内容时,才能设置 isReadable 为 true
|
||||
4. confidence 必须反映真实的识别把握程度,不能虚高
|
||||
5. 如果 isReadable 为 false,name 必须返回"无法识别",confidence 设为 0
|
||||
6. dosageValue 和 timesPerDay 必须是数字类型,不要加引号
|
||||
7. medicationTimes 必须是 HH:mm 格式的时间数组
|
||||
8. form 必须是枚举值之一
|
||||
|
||||
**宁可识别失败,也不要提供不准确的药品信息。用药安全高于一切!**`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建适宜人群分析提示词
|
||||
*/
|
||||
private buildSuitabilityAnalysisPrompt(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): string {
|
||||
return `作为资深药剂师,请分析以下药品的适宜人群和禁忌人群:
|
||||
|
||||
**药品信息**:
|
||||
- 名称:${productInfo.name}
|
||||
- 剂型:${productInfo.form}
|
||||
- 剂量:${productInfo.dosageValue}${productInfo.dosageUnit}
|
||||
|
||||
请以严格的JSON格式返回(不要包含任何markdown标记):
|
||||
{
|
||||
"confidence": 识别置信度(0-1),
|
||||
"suitableFor": ["适合人群1", "适合人群2", "适合人群3"],
|
||||
"unsuitableFor": ["不适合人群1", "不适合人群2", "不适合人群3"],
|
||||
"mainUsage": "药品的主要用途和适应症描述"
|
||||
}
|
||||
|
||||
**要求**:
|
||||
- suitableFor 和 unsuitableFor 必须是字符串数组,至少包含3项
|
||||
- mainUsage 是字符串,描述药品的主要治疗用途
|
||||
- 如果无法识别药品,所有数组返回空数组,mainUsage返回"无法识别药品"`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建成分分析提示词
|
||||
*/
|
||||
private buildIngredientsAnalysisPrompt(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): string {
|
||||
return `作为资深药剂师,请分析以下药品的主要成分:
|
||||
|
||||
**药品信息**:
|
||||
- 名称:${productInfo.name}
|
||||
- 用途:${productInfo.mainUsage}
|
||||
|
||||
请以严格的JSON格式返回(不要包含任何markdown标记):
|
||||
{
|
||||
"mainIngredients": ["主要成分1", "主要成分2", "主要成分3"]
|
||||
}
|
||||
|
||||
**要求**:
|
||||
- mainIngredients 必须是字符串数组,列出药品的主要活性成分
|
||||
- 至少包含1-3个主要成分
|
||||
- 如果无法确定,返回空数组`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建副作用分析提示词
|
||||
*/
|
||||
private buildEffectsAnalysisPrompt(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): string {
|
||||
return `作为资深药剂师,请分析以下药品的副作用、储存建议和健康建议:
|
||||
|
||||
**药品信息**:
|
||||
- 名称:${productInfo.name}
|
||||
- 用途:${productInfo.mainUsage}
|
||||
- 成分:${productInfo.mainIngredients?.join('、')}
|
||||
|
||||
请以严格的JSON格式返回(不要包含任何markdown标记):
|
||||
{
|
||||
"mainUsage": "药品的主要用途和适应症描述",
|
||||
"mainIngredients": ["主要成分1", "主要成分2", "主要成分3"],
|
||||
"sideEffects": ["副作用1", "副作用2", "副作用3"],
|
||||
"storageAdvice": ["储存建议1", "储存建议2", "储存建议3"],
|
||||
"healthAdvice": ["健康建议1", "健康建议2", "健康建议3"]
|
||||
}
|
||||
|
||||
**要求**:
|
||||
- 所有字段都是字符串数组
|
||||
- sideEffects: 列出常见和严重的副作用,至少3项
|
||||
- storageAdvice: 提供正确的储存方法,至少2项
|
||||
- healthAdvice: 给出配合用药的生活建议,至少3项
|
||||
- 如果无法确定,返回空数组`;
|
||||
**关键规则(必须遵守)**:
|
||||
1. isReadable 是最重要的字段,如果为 false,其他识别结果将被忽略,name返回"无法识别",confidence 为 0
|
||||
2. dosageValue 和 timesPerDay 必须是数字类型,不要加引号
|
||||
3. medicationTimes 必须是 HH:mm 格式的时间数组
|
||||
4. form 必须是枚举值之一
|
||||
5. suitableFor/unsuitableFor/mainIngredients/sideEffects/storageAdvice/healthAdvice 必须是字符串数组
|
||||
6. 数组字段至少包含 1-3 项,如无信息返回空数组
|
||||
7. 必须使用 ${language} 语言回答所有文本描述性内容
|
||||
|
||||
**宁可识别失败,也不要提供不准确的药品信息。用药安全高于一切!**`;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -529,11 +377,15 @@ export class MedicationRecognitionService {
|
||||
private async completeTask(
|
||||
taskId: string,
|
||||
result: RecognitionResultDto,
|
||||
language: string = 'zh-CN',
|
||||
): Promise<void> {
|
||||
await this.taskModel.update(
|
||||
{
|
||||
status: RecognitionStatusEnum.COMPLETED,
|
||||
currentStep: RECOGNITION_STATUS_DESCRIPTIONS[RecognitionStatusEnum.COMPLETED],
|
||||
currentStep: this.getStatusMessage(
|
||||
RecognitionStatusEnum.COMPLETED,
|
||||
language,
|
||||
),
|
||||
progress: 100,
|
||||
recognitionResult: JSON.stringify(result),
|
||||
completedAt: new Date(),
|
||||
@@ -547,11 +399,18 @@ export class MedicationRecognitionService {
|
||||
/**
|
||||
* 任务失败
|
||||
*/
|
||||
private async failTask(taskId: string, errorMessage: string): Promise<void> {
|
||||
private async failTask(
|
||||
taskId: string,
|
||||
errorMessage: string,
|
||||
language: string = 'zh-CN',
|
||||
): Promise<void> {
|
||||
await this.taskModel.update(
|
||||
{
|
||||
status: RecognitionStatusEnum.FAILED,
|
||||
currentStep: RECOGNITION_STATUS_DESCRIPTIONS[RecognitionStatusEnum.FAILED],
|
||||
currentStep: this.getStatusMessage(
|
||||
RecognitionStatusEnum.FAILED,
|
||||
language,
|
||||
),
|
||||
progress: 0,
|
||||
errorMessage,
|
||||
completedAt: new Date(),
|
||||
@@ -561,4 +420,18 @@ export class MedicationRecognitionService {
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取多语言状态描述
|
||||
*/
|
||||
private getStatusMessage(key: string, language: string): string {
|
||||
// 简化语言代码,如 'zh-TW' -> 'zh-CN', 'en-GB' -> 'en-US'
|
||||
let langCode = 'zh-CN'; // 默认中文
|
||||
if (language.toLowerCase().startsWith('en')) {
|
||||
langCode = 'en-US';
|
||||
}
|
||||
|
||||
const messages = STATUS_MESSAGES[langCode] || STATUS_MESSAGES['zh-CN'];
|
||||
return messages[key] || key;
|
||||
}
|
||||
}
|
||||
@@ -197,6 +197,22 @@ export class UsersService {
|
||||
}
|
||||
}
|
||||
|
||||
async getUserLanguage(userId: string): Promise<string> {
|
||||
try {
|
||||
const user = await this.userModel.findOne({ where: { id: userId } });
|
||||
|
||||
if (!user) {
|
||||
this.logger.warn(`getUserLanguage: ${userId} not found, default zh-CN`);
|
||||
return 'zh-CN';
|
||||
}
|
||||
|
||||
return user.language || 'zh-CN';
|
||||
} catch (error) {
|
||||
this.logger.error(`getUserLanguage error: ${error instanceof Error ? error.message : String(error)}`);
|
||||
return 'zh-CN';
|
||||
}
|
||||
}
|
||||
|
||||
// 扣减用户免费次数
|
||||
async deductUserUsageCount(userId: string, count: number = 1): Promise<void> {
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user