feat(medications): 增加基于视觉AI的药品智能录入系统
构建了从照片到药品档案的自动化处理流程，通过GLM多模态大模型实现药品信息的智能采集： 核心能力： - 创建任务追踪表 t_medication_recognition_tasks 存储识别任务状态 - 四阶段渐进式分析：基础识别→人群适配→成分解析→风险评估 - 提供三个REST端点支持任务创建、进度查询和结果确认 - 前端可通过轮询方式获取0-100%的实时进度反馈 - VIP用户免费使用，普通用户按次扣费 技术实现： - 利用GLM-4V-Plus模型处理多角度药品图像（正面+侧面+说明书） - 采用GLM-4-Flash模型进行文本深度分析 - 异步任务执行机制避免接口阻塞 - 完整的异常处理和任务失败恢复策略 - 新增AI_RECOGNITION.md文档详细说明集成方式 同步修复： - 修正会员用户AI配额扣减逻辑，避免不必要的次数消耗 - 优化APNs推送中无效设备令牌的检测和清理流程 - 将服药提醒的提前通知时间从15分钟缩短为5分钟
This commit is contained in:
richarjiang
523
src/medications/services/medication-recognition.service.ts
Normal file
523
src/medications/services/medication-recognition.service.ts
Normal file
@@ -0,0 +1,523 @@
|
||||
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { InjectModel } from '@nestjs/sequelize';
|
||||
import { OpenAI } from 'openai';
|
||||
import { MedicationRecognitionTask } from '../models/medication-recognition-task.model';
|
||||
import { CreateRecognitionTaskDto } from '../dto/create-recognition-task.dto';
|
||||
import { RecognitionStatusDto } from '../dto/recognition-status.dto';
|
||||
import { RecognitionResultDto } from '../dto/recognition-result.dto';
|
||||
import {
|
||||
RecognitionStatusEnum,
|
||||
RECOGNITION_STATUS_DESCRIPTIONS,
|
||||
} from '../enums/recognition-status.enum';
|
||||
import { MedicationFormEnum } from '../enums/medication-form.enum';
|
||||
|
||||
/**
|
||||
* 药物AI识别服务
|
||||
* 负责多图片药物识别、分析和结构化数据提取
|
||||
*/
|
||||
@Injectable()
|
||||
export class MedicationRecognitionService {
|
||||
private readonly logger = new Logger(MedicationRecognitionService.name);
|
||||
private readonly client: OpenAI;
|
||||
private readonly visionModel: string;
|
||||
private readonly textModel: string;
|
||||
|
||||
constructor(
|
||||
private readonly configService: ConfigService,
|
||||
@InjectModel(MedicationRecognitionTask)
|
||||
private readonly taskModel: typeof MedicationRecognitionTask,
|
||||
) {
|
||||
const glmApiKey = this.configService.get<string>('GLM_API_KEY');
|
||||
const glmBaseURL =
|
||||
this.configService.get<string>('GLM_BASE_URL') ||
|
||||
'https://open.bigmodel.cn/api/paas/v4';
|
||||
|
||||
this.client = new OpenAI({
|
||||
apiKey: glmApiKey,
|
||||
baseURL: glmBaseURL,
|
||||
});
|
||||
|
||||
this.visionModel =
|
||||
this.configService.get<string>('GLM_VISION_MODEL') || 'glm-4v-plus';
|
||||
this.textModel =
|
||||
this.configService.get<string>('GLM_MODEL') || 'glm-4-flash';
|
||||
}
|
||||
|
||||
/**
|
||||
* 创建识别任务
|
||||
*/
|
||||
async createRecognitionTask(
|
||||
userId: string,
|
||||
dto: CreateRecognitionTaskDto,
|
||||
): Promise<{ taskId: string; status: RecognitionStatusEnum }> {
|
||||
const taskId = `task_${userId}_${Date.now()}`;
|
||||
|
||||
this.logger.log(`创建药物识别任务: ${taskId}, 用户: ${userId}`);
|
||||
|
||||
await this.taskModel.create({
|
||||
id: taskId,
|
||||
userId,
|
||||
frontImageUrl: dto.frontImageUrl,
|
||||
sideImageUrl: dto.sideImageUrl,
|
||||
auxiliaryImageUrl: dto.auxiliaryImageUrl,
|
||||
status: RecognitionStatusEnum.PENDING,
|
||||
currentStep: RECOGNITION_STATUS_DESCRIPTIONS[RecognitionStatusEnum.PENDING],
|
||||
progress: 0,
|
||||
});
|
||||
|
||||
// 异步开始识别过程（不阻塞当前请求）
|
||||
this.startRecognitionProcess(taskId).catch((error) => {
|
||||
this.logger.error(
|
||||
`识别任务 ${taskId} 处理失败: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
});
|
||||
|
||||
return { taskId, status: RecognitionStatusEnum.PENDING };
|
||||
}
|
||||
|
||||
/**
|
||||
* 查询识别状态
|
||||
*/
|
||||
async getRecognitionStatus(
|
||||
taskId: string,
|
||||
userId: string,
|
||||
): Promise<RecognitionStatusDto> {
|
||||
const task = await this.taskModel.findOne({
|
||||
where: { id: taskId, userId },
|
||||
});
|
||||
|
||||
if (!task) {
|
||||
throw new NotFoundException('识别任务不存在');
|
||||
}
|
||||
|
||||
return {
|
||||
taskId: task.id,
|
||||
status: task.status as RecognitionStatusEnum,
|
||||
currentStep: task.currentStep,
|
||||
progress: task.progress,
|
||||
result: task.recognitionResult
|
||||
? JSON.parse(task.recognitionResult)
|
||||
: undefined,
|
||||
errorMessage: task.errorMessage,
|
||||
createdAt: task.createdAt,
|
||||
completedAt: task.completedAt,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 开始识别处理流程
|
||||
*/
|
||||
private async startRecognitionProcess(taskId: string): Promise<void> {
|
||||
try {
|
||||
const task = await this.taskModel.findByPk(taskId);
|
||||
if (!task) return;
|
||||
|
||||
// 阶段1: 产品识别分析 (0-40%)
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_PRODUCT,
|
||||
'正在识别药品基本信息...',
|
||||
10,
|
||||
);
|
||||
const productInfo = await this.recognizeProduct(task);
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_PRODUCT,
|
||||
'药品基本信息识别完成',
|
||||
40,
|
||||
);
|
||||
|
||||
// 阶段2: 适宜人群分析 (40-60%)
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_SUITABILITY,
|
||||
'正在分析适宜人群...',
|
||||
50,
|
||||
);
|
||||
const suitabilityInfo = await this.analyzeSuitability(productInfo);
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_SUITABILITY,
|
||||
'适宜人群分析完成',
|
||||
60,
|
||||
);
|
||||
|
||||
// 阶段3: 成分分析 (60-80%)
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_INGREDIENTS,
|
||||
'正在分析主要成分...',
|
||||
70,
|
||||
);
|
||||
const ingredientsInfo = await this.analyzeIngredients(productInfo);
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_INGREDIENTS,
|
||||
'成分分析完成',
|
||||
80,
|
||||
);
|
||||
|
||||
// 阶段4: 副作用分析 (80-100%)
|
||||
await this.updateTaskStatus(
|
||||
taskId,
|
||||
RecognitionStatusEnum.ANALYZING_EFFECTS,
|
||||
'正在分析副作用和健康建议...',
|
||||
90,
|
||||
);
|
||||
const effectsInfo = await this.analyzeEffects(productInfo);
|
||||
|
||||
// 合并所有结果
|
||||
const finalResult = {
|
||||
...productInfo,
|
||||
...suitabilityInfo,
|
||||
...ingredientsInfo,
|
||||
...effectsInfo,
|
||||
} as RecognitionResultDto;
|
||||
|
||||
// 完成识别
|
||||
await this.completeTask(taskId, finalResult);
|
||||
this.logger.log(`识别任务 ${taskId} 完成`);
|
||||
} catch (error) {
|
||||
const errorMessage =
|
||||
error instanceof Error ? error.message : String(error);
|
||||
this.logger.error(`识别任务 ${taskId} 失败: ${errorMessage}`);
|
||||
await this.failTask(taskId, errorMessage);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 阶段1: 识别药品基本信息
|
||||
*/
|
||||
private async recognizeProduct(
|
||||
task: MedicationRecognitionTask,
|
||||
): Promise<Partial<RecognitionResultDto>> {
|
||||
const prompt = this.buildProductRecognitionPrompt();
|
||||
const images = [task.frontImageUrl, task.sideImageUrl];
|
||||
if (task.auxiliaryImageUrl) images.push(task.auxiliaryImageUrl);
|
||||
|
||||
this.logger.log(
|
||||
`调用视觉模型识别药品，图片数量: ${images.length}, 任务ID: ${task.id}`,
|
||||
);
|
||||
|
||||
const response = await this.client.chat.completions.create({
|
||||
model: this.visionModel,
|
||||
temperature: 0.3,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: prompt },
|
||||
...images.map((url) => ({
|
||||
type: 'image_url',
|
||||
image_url: { url },
|
||||
})),
|
||||
] as any,
|
||||
},
|
||||
],
|
||||
response_format: { type: 'json_object' },
|
||||
} as any);
|
||||
|
||||
const content = response.choices[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error('AI模型返回内容为空');
|
||||
}
|
||||
|
||||
const parsed = this.parseJsonResponse(content);
|
||||
this.logger.log(`药品基本信息识别完成: ${parsed.name}, 置信度: ${parsed.confidence}`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* 阶段2: 分析适宜人群
|
||||
*/
|
||||
private async analyzeSuitability(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): Promise<Partial<RecognitionResultDto>> {
|
||||
const prompt = this.buildSuitabilityAnalysisPrompt(productInfo);
|
||||
|
||||
this.logger.log(`分析适宜人群: ${productInfo.name}`);
|
||||
|
||||
const response = await this.client.chat.completions.create({
|
||||
model: this.textModel,
|
||||
temperature: 0.7,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
response_format: { type: 'json_object' },
|
||||
});
|
||||
|
||||
const content = response.choices[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error('AI模型返回内容为空');
|
||||
}
|
||||
|
||||
return this.parseJsonResponse(content);
|
||||
}
|
||||
|
||||
/**
|
||||
* 阶段3: 分析主要成分
|
||||
*/
|
||||
private async analyzeIngredients(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): Promise<Partial<RecognitionResultDto>> {
|
||||
const prompt = this.buildIngredientsAnalysisPrompt(productInfo);
|
||||
|
||||
this.logger.log(`分析主要成分: ${productInfo.name}`);
|
||||
|
||||
const response = await this.client.chat.completions.create({
|
||||
model: this.textModel,
|
||||
temperature: 0.7,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
response_format: { type: 'json_object' },
|
||||
});
|
||||
|
||||
const content = response.choices[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error('AI模型返回内容为空');
|
||||
}
|
||||
|
||||
return this.parseJsonResponse(content);
|
||||
}
|
||||
|
||||
/**
|
||||
* 阶段4: 分析副作用和健康建议
|
||||
*/
|
||||
private async analyzeEffects(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): Promise<Partial<RecognitionResultDto>> {
|
||||
const prompt = this.buildEffectsAnalysisPrompt(productInfo);
|
||||
|
||||
this.logger.log(`分析副作用和健康建议: ${productInfo.name}`);
|
||||
|
||||
const response = await this.client.chat.completions.create({
|
||||
model: this.textModel,
|
||||
temperature: 0.7,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: prompt,
|
||||
},
|
||||
],
|
||||
response_format: { type: 'json_object' },
|
||||
});
|
||||
|
||||
const content = response.choices[0]?.message?.content;
|
||||
if (!content) {
|
||||
throw new Error('AI模型返回内容为空');
|
||||
}
|
||||
|
||||
return this.parseJsonResponse(content);
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建产品识别提示词
|
||||
*/
|
||||
private buildProductRecognitionPrompt(): string {
|
||||
return `你是一位拥有20年从业经验的资深药剂师，请根据提供的药品图片（包括正面、侧面和可能的辅助面）进行详细分析。
|
||||
|
||||
**分析要求**：
|
||||
1. 仔细观察药品包装、说明书上的所有信息
|
||||
2. 识别药品的完整名称（通用名和商品名）
|
||||
3. 确定药物剂型（片剂/胶囊/注射剂等）
|
||||
4. 提取规格剂量信息
|
||||
5. 推荐合理的服用次数和时间
|
||||
|
||||
**置信度评估标准**：
|
||||
- 如果图片清晰且信息完整，置信度应 >= 0.8
|
||||
- 如果部分信息不清晰但可推断，置信度 0.5-0.8
|
||||
- 如果无法准确识别，置信度 < 0.5，name返回"无法识别"
|
||||
|
||||
**返回严格的JSON格式**（不要包含任何markdown标记）：
|
||||
{
|
||||
"name": "药品完整名称",
|
||||
"photoUrl": "使用正面图片URL",
|
||||
"form": "剂型(tablet/capsule/injection/drops/syrup/ointment/powder/granules)",
|
||||
"dosageValue": 剂量数值(数字),
|
||||
"dosageUnit": "剂量单位",
|
||||
"timesPerDay": 建议每日服用次数(数字),
|
||||
"medicationTimes": ["建议的服药时间，格式HH:mm"],
|
||||
"confidence": 识别置信度(0-1的小数)
|
||||
}
|
||||
|
||||
**重要**：
|
||||
- dosageValue 和 timesPerDay 必须是数字类型，不要加引号
|
||||
- confidence 必须是 0-1 之间的小数
|
||||
- medicationTimes 必须是 HH:mm 格式的时间数组
|
||||
- form 必须是枚举值之一
|
||||
- 如果无法识别，name返回"无法识别"，其他字段返回合理的默认值`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建适宜人群分析提示词
|
||||
*/
|
||||
private buildSuitabilityAnalysisPrompt(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): string {
|
||||
return `作为资深药剂师，请分析以下药品的适宜人群和禁忌人群：
|
||||
|
||||
**药品信息**：
|
||||
- 名称：${productInfo.name}
|
||||
- 剂型：${productInfo.form}
|
||||
- 剂量：${productInfo.dosageValue}${productInfo.dosageUnit}
|
||||
|
||||
请以严格的JSON格式返回（不要包含任何markdown标记）：
|
||||
{
|
||||
"suitableFor": ["适合人群1", "适合人群2", "适合人群3"],
|
||||
"unsuitableFor": ["不适合人群1", "不适合人群2", "不适合人群3"],
|
||||
"mainUsage": "药品的主要用途和适应症描述"
|
||||
}
|
||||
|
||||
**要求**：
|
||||
- suitableFor 和 unsuitableFor 必须是字符串数组，至少包含3项
|
||||
- mainUsage 是字符串，描述药品的主要治疗用途
|
||||
- 如果无法识别药品，所有数组返回空数组，mainUsage返回"无法识别药品"`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建成分分析提示词
|
||||
*/
|
||||
private buildIngredientsAnalysisPrompt(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): string {
|
||||
return `作为资深药剂师，请分析以下药品的主要成分：
|
||||
|
||||
**药品信息**：
|
||||
- 名称：${productInfo.name}
|
||||
- 用途：${productInfo.mainUsage}
|
||||
|
||||
请以严格的JSON格式返回（不要包含任何markdown标记）：
|
||||
{
|
||||
"mainIngredients": ["主要成分1", "主要成分2", "主要成分3"]
|
||||
}
|
||||
|
||||
**要求**：
|
||||
- mainIngredients 必须是字符串数组，列出药品的主要活性成分
|
||||
- 至少包含1-3个主要成分
|
||||
- 如果无法确定，返回空数组`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 构建副作用分析提示词
|
||||
*/
|
||||
private buildEffectsAnalysisPrompt(
|
||||
productInfo: Partial<RecognitionResultDto>,
|
||||
): string {
|
||||
return `作为资深药剂师，请分析以下药品的副作用、储存建议和健康建议：
|
||||
|
||||
**药品信息**：
|
||||
- 名称：${productInfo.name}
|
||||
- 用途：${productInfo.mainUsage}
|
||||
- 成分：${productInfo.mainIngredients?.join('、')}
|
||||
|
||||
请以严格的JSON格式返回（不要包含任何markdown标记）：
|
||||
{
|
||||
"sideEffects": ["副作用1", "副作用2", "副作用3"],
|
||||
"storageAdvice": ["储存建议1", "储存建议2", "储存建议3"],
|
||||
"healthAdvice": ["健康建议1", "健康建议2", "健康建议3"]
|
||||
}
|
||||
|
||||
**要求**：
|
||||
- 所有字段都是字符串数组
|
||||
- sideEffects: 列出常见和严重的副作用，至少3项
|
||||
- storageAdvice: 提供正确的储存方法，至少2项
|
||||
- healthAdvice: 给出配合用药的生活建议，至少3项
|
||||
- 如果无法确定，返回空数组`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析JSON响应
|
||||
*/
|
||||
private parseJsonResponse(content: string): any {
|
||||
try {
|
||||
// 移除可能的 markdown 代码块标记
|
||||
let jsonString = content.trim();
|
||||
const jsonMatch = content.match(/```json\s*([\s\S]*?)\s*```/);
|
||||
if (jsonMatch) {
|
||||
jsonString = jsonMatch[1];
|
||||
} else {
|
||||
// 尝试提取第一个 { 到最后一个 }
|
||||
const firstBrace = content.indexOf('{');
|
||||
const lastBrace = content.lastIndexOf('}');
|
||||
if (firstBrace !== -1 && lastBrace !== -1) {
|
||||
jsonString = content.substring(firstBrace, lastBrace + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return JSON.parse(jsonString);
|
||||
} catch (error) {
|
||||
this.logger.error(
|
||||
`解析JSON响应失败: ${error instanceof Error ? error.message : String(error)}, Content: ${content}`,
|
||||
);
|
||||
throw new Error('AI响应格式错误，无法解析');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 更新任务状态
|
||||
*/
|
||||
private async updateTaskStatus(
|
||||
taskId: string,
|
||||
status: RecognitionStatusEnum,
|
||||
currentStep: string,
|
||||
progress: number,
|
||||
): Promise<void> {
|
||||
await this.taskModel.update(
|
||||
{
|
||||
status,
|
||||
currentStep,
|
||||
progress,
|
||||
},
|
||||
{
|
||||
where: { id: taskId },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 完成任务
|
||||
*/
|
||||
private async completeTask(
|
||||
taskId: string,
|
||||
result: RecognitionResultDto,
|
||||
): Promise<void> {
|
||||
await this.taskModel.update(
|
||||
{
|
||||
status: RecognitionStatusEnum.COMPLETED,
|
||||
currentStep: RECOGNITION_STATUS_DESCRIPTIONS[RecognitionStatusEnum.COMPLETED],
|
||||
progress: 100,
|
||||
recognitionResult: JSON.stringify(result),
|
||||
completedAt: new Date(),
|
||||
},
|
||||
{
|
||||
where: { id: taskId },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* 任务失败
|
||||
*/
|
||||
private async failTask(taskId: string, errorMessage: string): Promise<void> {
|
||||
await this.taskModel.update(
|
||||
{
|
||||
status: RecognitionStatusEnum.FAILED,
|
||||
currentStep: RECOGNITION_STATUS_DESCRIPTIONS[RecognitionStatusEnum.FAILED],
|
||||
progress: 0,
|
||||
errorMessage,
|
||||
completedAt: new Date(),
|
||||
},
|
||||
{
|
||||
where: { id: taskId },
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user