feat: 新增食物识别API，调整字段名，扩展识别功能与提示逻辑

2025-09-04 09:36:07 +08:00
parent 02f21f0858
commit d34f752776
5 changed files with 253 additions and 27 deletions
--- a/src/ai-coach/ai-coach.controller.ts
+++ b/src/ai-coach/ai-coach.controller.ts
@@ -7,6 +7,8 @@ import { AccessTokenPayload } from '../users/services/apple-auth.service';
 import { AiCoachService } from './ai-coach.service';
 import { AiChatRequestDto, AiChatResponseDto, AiResponseDataDto } from './dto/ai-chat.dto';
 import { PostureAssessmentRequestDto, PostureAssessmentResponseDto } from './dto/posture-assessment.dto';
+import { FoodRecognitionRequestDto, FoodRecognitionResponseDto } from './dto/food-recognition.dto';
+import { DietAnalysisService } from './services/diet-analysis.service';
 import { UsersService } from '../users/users.service';

@ApiTags('ai-coach')
@@ -16,6 +18,7 @@ export class AiCoachController {
  private readonly logger = new Logger(AiCoachController.name);
  constructor(
    private readonly aiCoachService: AiCoachService,
+    private readonly dietAnalysisService: DietAnalysisService,
    private readonly usersService: UsersService,
  ) { }

@@ -159,6 +162,51 @@ export class AiCoachController {
    });
    return res as any;
  }
+
+  @Post('food-recognition')
+  @ApiOperation({
+    summary: '食物识别服务',
+    description: '识别图片中的食物并返回数组格式的选项列表，支持多食物识别。如果图片中不包含食物，会返回相应提示信息。'
+  })
+  @ApiBody({ type: FoodRecognitionRequestDto })
+  async recognizeFood(
+    @Body() body: FoodRecognitionRequestDto,
+    @CurrentUser() user: AccessTokenPayload,
+  ): Promise<FoodRecognitionResponseDto> {
+    this.logger.log(`Food recognition request from user: ${user.sub}, images: ${body.imageUrls?.length || 0}`);
+
+    const result = await this.dietAnalysisService.recognizeFoodForConfirmation(body.imageUrls);
+
+    // 转换为DTO格式
+    const response: FoodRecognitionResponseDto = {
+      items: result.items.map(item => ({
+        id: item.id,
+        label: item.label,
+        foodName: item.foodName,
+        portion: item.portion,
+        calories: item.calories,
+        mealType: item.mealType,
+        nutritionData: {
+          proteinGrams: item.nutritionData.proteinGrams,
+          carbohydrateGrams: item.nutritionData.carbohydrateGrams,
+          fatGrams: item.nutritionData.fatGrams,
+          fiberGrams: item.nutritionData.fiberGrams,
+        }
+      })),
+      analysisText: result.analysisText,
+      confidence: result.confidence,
+      isFoodDetected: result.isFoodDetected,
+      nonFoodMessage: result.nonFoodMessage
+    };
+
+    if (!result.isFoodDetected) {
+      this.logger.log(`Non-food detected for user: ${user.sub}, message: ${result.nonFoodMessage}`);
+    } else {
+      this.logger.log(`Food recognition completed: ${result.items.length} items recognized`);
+    }
+
+    return response;
+  }
 }


--- a/src/ai-coach/ai-coach.service.ts
+++ b/src/ai-coach/ai-coach.service.ts
@@ -415,12 +415,12 @@ export class AiCoachService {
      // 处理图片饮食记录
      const recognitionResult = await this.dietAnalysisService.recognizeFoodForConfirmation(params.imageUrls);

-      if (recognitionResult.recognizedItems.length > 0) {
-        const choices = recognitionResult.recognizedItems.map(item => ({
+      if (recognitionResult.items.length > 0) {
+        const choices = recognitionResult.items.map(item => ({
          id: item.id,
          label: item.label,
          value: item,
-          recommended: recognitionResult.recognizedItems.indexOf(item) === 0
+          recommended: recognitionResult.items.indexOf(item) === 0
        }));

        const responseContent = `我识别到了以下食物，请选择要记录的内容：\n\n${recognitionResult.analysisText}`;
--- a/src/ai-coach/dto/food-recognition.dto.ts
+++ b/src/ai-coach/dto/food-recognition.dto.ts
@@ -0,0 +1,135 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { IsArray, IsBoolean, IsNotEmpty, IsOptional, IsString, IsInt, Min, Max } from 'class-validator';
+
+/**
+ * 食物营养数据DTO
+ */
+export class FoodNutritionDataDto {
+  @ApiProperty({ description: '蛋白质含量（克）', required: false })
+  @IsOptional()
+  @IsInt()
+  @Min(0)
+  proteinGrams?: number;
+
+  @ApiProperty({ description: '碳水化合物含量（克）', required: false })
+  @IsOptional()
+  @IsInt()
+  @Min(0)
+  carbohydrateGrams?: number;
+
+  @ApiProperty({ description: '脂肪含量（克）', required: false })
+  @IsOptional()
+  @IsInt()
+  @Min(0)
+  fatGrams?: number;
+
+  @ApiProperty({ description: '膳食纤维含量（克）', required: false })
+  @IsOptional()
+  @IsInt()
+  @Min(0)
+  fiberGrams?: number;
+}
+
+/**
+ * 食物确认选项DTO
+ */
+export class FoodConfirmationOptionDto {
+  @ApiProperty({ description: '食物选项唯一标识符' })
+  @IsString()
+  @IsNotEmpty()
+  id: string;
+
+  @ApiProperty({ description: '显示给用户的完整选项文本' })
+  @IsString()
+  @IsNotEmpty()
+  label: string;
+
+  @ApiProperty({ description: '食物名称' })
+  @IsString()
+  @IsNotEmpty()
+  foodName: string;
+
+  @ApiProperty({ description: '份量描述' })
+  @IsString()
+  @IsNotEmpty()
+  portion: string;
+
+  @ApiProperty({ description: '估算热量' })
+  @IsInt()
+  @Min(0)
+  @Max(5000)
+  calories: number;
+
+  @ApiProperty({ description: '餐次类型' })
+  @IsString()
+  @IsNotEmpty()
+  mealType: string;
+
+  @ApiProperty({ description: '营养数据', type: FoodNutritionDataDto })
+  nutritionData: FoodNutritionDataDto;
+}
+
+/**
+ * 食物识别请求DTO
+ */
+export class FoodRecognitionRequestDto {
+  @ApiProperty({ description: '图片URL数组' })
+  @IsArray()
+  @IsString({ each: true })
+  @IsNotEmpty({ each: true })
+  imageUrls: string[];
+}
+
+/**
+ * 食物识别响应DTO - 总是返回数组结构
+ */
+export class FoodRecognitionResponseDto {
+  @ApiProperty({
+    description: '识别到的食物列表，即使只有一种食物也返回数组格式',
+    type: [FoodConfirmationOptionDto]
+  })
+  @IsArray()
+  items: FoodConfirmationOptionDto[];
+
+  @ApiProperty({ description: '识别说明文字' })
+  @IsString()
+  analysisText: string;
+
+  @ApiProperty({
+    description: '识别置信度',
+    minimum: 0,
+    maximum: 100
+  })
+  @IsInt()
+  @Min(0)
+  @Max(100)
+  confidence: number;
+
+  @ApiProperty({
+    description: '是否识别到食物',
+    default: true
+  })
+  @IsBoolean()
+  isFoodDetected: boolean;
+
+  @ApiProperty({
+    description: '非食物提示信息，当isFoodDetected为false时显示',
+    required: false
+  })
+  @IsOptional()
+  @IsString()
+  nonFoodMessage?: string;
+}
+
+/**
+ * 食物确认请求DTO
+ */
+export class FoodConfirmationRequestDto {
+  @ApiProperty({ description: '用户选择的食物选项' })
+  selectedOption: FoodConfirmationOptionDto;
+
+  @ApiProperty({ description: '图片URL', required: false })
+  @IsOptional()
+  @IsString()
+  imageUrl?: string;
+}
--- a/src/ai-coach/services/diet-analysis.service.ts
+++ b/src/ai-coach/services/diet-analysis.service.ts
@@ -44,12 +44,14 @@ export interface FoodConfirmationOption {
 }

 /**
- * 食物识别确认结果接口
+ * 食物识别确认结果接口 - 现在总是返回数组结构
 */
 export interface FoodRecognitionResult {
-  recognizedItems: FoodConfirmationOption[];
+  items: FoodConfirmationOption[];  // 修改：统一使用items作为数组字段名
  analysisText: string;
  confidence: number;
+  isFoodDetected: boolean;  // 是否识别到食物
+  nonFoodMessage?: string;  // 非食物提示信息
 }

 /**
@@ -113,9 +115,11 @@ export class DietAnalysisService {
    } catch (error) {
      this.logger.error(`食物识别失败: ${error instanceof Error ? error.message : String(error)}`);
      return {
-        recognizedItems: [],
+        items: [],
        analysisText: '食物识别失败，请稍后重试',
-        confidence: 0
+        confidence: 0,
+        isFoodDetected: false,
+        nonFoodMessage: '服务暂时不可用，请稍后重试'
      };
    }
  }
@@ -369,15 +373,21 @@ export class DietAnalysisService {
   * @returns 提示文本
   */
  private buildFoodRecognitionPrompt(suggestedMealType: MealType): string {
-    return `作为专业营养分析师，请分析这张食物图片并生成用户确认选项。
+    return `作为专业营养分析师，请分析这张图片并判断是否包含食物。

 当前时间建议餐次：${suggestedMealType}

-请识别图片中的食物，并为每种食物生成确认选项。返回以下格式的JSON：
+**首先判断图片内容：**
+- 如果图片中包含食物，请识别并生成确认选项
+- 如果图片中不包含食物（如风景、人物、物品、文字等），请明确标识
+
+返回以下格式的JSON：
 {
-  "confidence": number,     // 整体识别置信度 0-100
-  "analysisText": string,   // 简短的识别说明文字
-  "recognizedItems": [      // 识别的食物列表
+  "confidence": number,        // 整体识别置信度 0-100
+  "analysisText": string,      // 简短的识别说明文字
+  "isFoodDetected": boolean,   // 是否检测到食物
+  "nonFoodMessage": string,    // 当isFoodDetected为false时的提示信息
+  "recognizedItems": [         // 识别的食物列表（如果是食物才有内容）
    {
      "id": string,               // 唯一标识符
      "foodName": string,         // 食物名称
@@ -395,12 +405,26 @@ export class DietAnalysisService {
  ]
 }

-要求：
-1. 如果图片中有多种食物，为每种主要食物生成一个选项
-2. label字段要简洁易懂，格式如"一条鱼 200卡"、"一碗米饭 150卡"
-3. 营养数据要合理估算
-4. 如果图片模糊或无法识别，返回空的recognizedItems数组
-5. 最多生成5个选项，优先选择主要食物`;
+**识别规则：**
+1. **非食物情况：**
+   - 如果图片中没有食物，设置 isFoodDetected: false
+   - recognizedItems 返回空数组
+   - nonFoodMessage 提供友好的提示，如"图片中未检测到食物，请上传包含食物的图片"
+   - confidence 设置为对判断的置信度
+
+2. **食物识别情况：**
+   - 设置 isFoodDetected: true
+   - nonFoodMessage 可以为空或不设置
+   - 如果图片中有多种食物，为每种主要食物生成一个选项
+   - 如果图片中只有一种食物，也要生成一个包含该食物的数组选项
+   - label字段要简洁易懂，格式如"一条鱼 200卡"、"一碗米饭 150卡"
+   - 营养数据要合理估算
+   - 最多生成8个选项，优先选择主要食物
+   - 对于复合菜品（如盖浇饭、汤面等），既可以作为整体记录，也可以分解为主要组成部分
+
+3. **模糊情况：**
+   - 如果图片模糊但能看出是食物相关，设置 isFoodDetected: true，但返回空的recognizedItems数组
+   - analysisText 说明"图片模糊，无法准确识别食物"`;
  }

  /**
@@ -503,15 +527,22 @@ export class DietAnalysisService {
    } catch (parseError) {
      this.logger.error(`食物识别JSON解析失败: ${parseError}`);
      return {
-        recognizedItems: [],
+        items: [],
        analysisText: '图片分析失败：无法解析识别结果',
-        confidence: 0
+        confidence: 0,
+        isFoodDetected: false,
+        nonFoodMessage: '图片分析失败，请重新上传图片'
      };
    }

+    // 检查是否识别到食物
+    const isFoodDetected = parsedResult.isFoodDetected !== false; // 默认为true，除非明确设置为false
+    const nonFoodMessage = parsedResult.nonFoodMessage || '';
+
    const recognizedItems: FoodConfirmationOption[] = [];

-    if (parsedResult.recognizedItems && Array.isArray(parsedResult.recognizedItems)) {
+    // 只有在识别到食物时才处理食物项目
+    if (isFoodDetected && parsedResult.recognizedItems && Array.isArray(parsedResult.recognizedItems)) {
      parsedResult.recognizedItems.forEach((item: any, index: number) => {
        if (item.foodName && item.calories) {
          recognizedItems.push({
@@ -532,10 +563,22 @@ export class DietAnalysisService {
      });
    }

+    // 根据是否识别到食物设置不同的分析文本
+    let analysisText = parsedResult.analysisText || '';
+    if (!isFoodDetected) {
+      analysisText = analysisText || '图片中未检测到食物';
+    } else if (recognizedItems.length === 0) {
+      analysisText = analysisText || '图片模糊，无法准确识别食物';
+    } else {
+      analysisText = analysisText || '已识别图片中的食物';
+    }
+
    return {
-      recognizedItems,
-      analysisText: parsedResult.analysisText || '已识别图片中的食物',
-      confidence: Math.min(100, Math.max(0, parsedResult.confidence || 0))
+      items: recognizedItems,
+      analysisText,
+      confidence: Math.min(100, Math.max(0, parsedResult.confidence || 0)),
+      isFoodDetected,
+      nonFoodMessage: !isFoodDetected ? (nonFoodMessage || '图片中未检测到食物，请上传包含食物的图片') : undefined
    };
  }

--- a/src/diet-records/diet-records.service.ts
+++ b/src/diet-records/diet-records.service.ts
@@ -306,7 +306,7 @@ export class DietRecordsService {
      const recognitionResult = await this.dietAnalysisService.recognizeFoodForConfirmation([imageUrl]);

      // 将识别结果转换为 CreateDietRecordDto 格式
-      const dietRecords: CreateDietRecordDto[] = recognitionResult.recognizedItems.map(item => ({
+      const dietRecords: CreateDietRecordDto[] = recognitionResult.items.map(item => ({
        mealType: suggestedMealType || item.mealType,
        foodName: item.foodName,
        portionDescription: item.portion,
@@ -355,13 +355,13 @@ export class DietRecordsService {

      // 如果指定了建议的餐次类型，更新所有识别项的餐次类型
      if (suggestedMealType) {
-        recognitionResult.recognizedItems.forEach(item => {
+        recognitionResult.items.forEach(item => {
          item.mealType = suggestedMealType;
        });
      }

      return {
-        recognizedItems: recognitionResult.recognizedItems,
+        recognizedItems: recognitionResult.items,
        analysisText: recognitionResult.analysisText,
        confidence: recognitionResult.confidence,
        imageUrl: imageUrl