feat: 添加食物识别功能,支持根据图片URL识别食物并转换为饮食记录格式

This commit is contained in:
richarjiang
2025-08-31 14:14:33 +08:00
parent d0b02b6228
commit 0488fe62a1
5 changed files with 216 additions and 7 deletions

View File

@@ -1,4 +1,4 @@
import { Module } from '@nestjs/common'; import { Module, forwardRef } from '@nestjs/common';
import { SequelizeModule } from '@nestjs/sequelize'; import { SequelizeModule } from '@nestjs/sequelize';
import { ConfigModule } from '@nestjs/config'; import { ConfigModule } from '@nestjs/config';
import { AiCoachController } from './ai-coach.controller'; import { AiCoachController } from './ai-coach.controller';
@@ -14,11 +14,12 @@ import { DietRecordsModule } from '../diet-records/diet-records.module';
imports: [ imports: [
ConfigModule, ConfigModule,
UsersModule, UsersModule,
DietRecordsModule, forwardRef(() => DietRecordsModule),
SequelizeModule.forFeature([AiConversation, AiMessage, PostureAssessment]), SequelizeModule.forFeature([AiConversation, AiMessage, PostureAssessment]),
], ],
controllers: [AiCoachController], controllers: [AiCoachController],
providers: [AiCoachService, DietAnalysisService], providers: [AiCoachService, DietAnalysisService],
exports: [DietAnalysisService],
}) })
export class AiCoachModule { } export class AiCoachModule { }

View File

@@ -15,7 +15,7 @@ import {
} from '@nestjs/common'; } from '@nestjs/common';
import { ApiOperation, ApiBody, ApiResponse, ApiTags, ApiQuery } from '@nestjs/swagger'; import { ApiOperation, ApiBody, ApiResponse, ApiTags, ApiQuery } from '@nestjs/swagger';
import { DietRecordsService } from './diet-records.service'; import { DietRecordsService } from './diet-records.service';
import { CreateDietRecordDto, UpdateDietRecordDto, GetDietHistoryQueryDto, DietRecordResponseDto, DietHistoryResponseDto, NutritionSummaryDto } from '../users/dto/diet-record.dto'; import { CreateDietRecordDto, UpdateDietRecordDto, GetDietHistoryQueryDto, DietRecordResponseDto, DietHistoryResponseDto, NutritionSummaryDto, FoodRecognitionRequestDto, FoodRecognitionResponseDto, FoodRecognitionToDietRecordsResponseDto } from '../users/dto/diet-record.dto';
import { JwtAuthGuard } from '../common/guards/jwt-auth.guard'; import { JwtAuthGuard } from '../common/guards/jwt-auth.guard';
import { CurrentUser } from '../common/decorators/current-user.decorator'; import { CurrentUser } from '../common/decorators/current-user.decorator';
import { AccessTokenPayload } from '../users/services/apple-auth.service'; import { AccessTokenPayload } from '../users/services/apple-auth.service';
@@ -121,4 +121,44 @@ export class DietRecordsController {
const count = mealCount ? parseInt(mealCount) : 10; const count = mealCount ? parseInt(mealCount) : 10;
return this.dietRecordsService.getRecentNutritionSummary(user.sub, count); return this.dietRecordsService.getRecentNutritionSummary(user.sub, count);
} }
/**
* 根据图片URL识别食物并转换为饮食记录格式
*/
@UseGuards(JwtAuthGuard)
@Post('recognize-food-to-records')
@HttpCode(HttpStatus.OK)
@ApiOperation({ summary: '根据图片URL识别食物并转换为饮食记录格式' })
@ApiBody({ type: FoodRecognitionRequestDto })
@ApiResponse({ status: 200, description: '成功识别食物并转换为饮食记录格式', type: FoodRecognitionToDietRecordsResponseDto })
async recognizeFoodToDietRecords(
@Body() requestDto: FoodRecognitionRequestDto,
@CurrentUser() user: AccessTokenPayload,
): Promise<FoodRecognitionToDietRecordsResponseDto> {
this.logger.log(`识别食物转饮食记录 - 用户ID: ${user.sub}, 图片URL: ${requestDto.imageUrl}`);
return this.dietRecordsService.recognizeFoodToDietRecords(
requestDto.imageUrl,
requestDto.mealType
);
}
/**
* 根据图片URL识别食物原始格式
*/
@UseGuards(JwtAuthGuard)
@Post('recognize-food')
@HttpCode(HttpStatus.OK)
@ApiOperation({ summary: '根据图片URL识别食物' })
@ApiBody({ type: FoodRecognitionRequestDto })
@ApiResponse({ status: 200, description: '成功识别食物', type: FoodRecognitionResponseDto })
async recognizeFood(
@Body() requestDto: FoodRecognitionRequestDto,
@CurrentUser() user: AccessTokenPayload,
): Promise<FoodRecognitionResponseDto> {
this.logger.log(`识别食物 - 用户ID: ${user.sub}, 图片URL: ${requestDto.imageUrl}`);
return this.dietRecordsService.recognizeFood(
requestDto.imageUrl,
requestDto.mealType
);
}
} }

View File

@@ -1,15 +1,17 @@
import { Module } from '@nestjs/common'; import { Module, forwardRef } from '@nestjs/common';
import { SequelizeModule } from '@nestjs/sequelize'; import { SequelizeModule } from '@nestjs/sequelize';
import { DietRecordsController } from './diet-records.controller'; import { DietRecordsController } from './diet-records.controller';
import { DietRecordsService } from './diet-records.service'; import { DietRecordsService } from './diet-records.service';
import { UserDietHistory } from '../users/models/user-diet-history.model'; import { UserDietHistory } from '../users/models/user-diet-history.model';
import { ActivityLog } from '../activity-logs/models/activity-log.model'; import { ActivityLog } from '../activity-logs/models/activity-log.model';
import { UsersModule } from '../users/users.module'; import { UsersModule } from '../users/users.module';
import { AiCoachModule } from '../ai-coach/ai-coach.module';
@Module({ @Module({
imports: [ imports: [
SequelizeModule.forFeature([UserDietHistory, ActivityLog]), SequelizeModule.forFeature([UserDietHistory, ActivityLog]),
UsersModule, UsersModule,
forwardRef(() => AiCoachModule),
], ],
controllers: [DietRecordsController], controllers: [DietRecordsController],
providers: [DietRecordsService], providers: [DietRecordsService],

View File

@@ -1,12 +1,13 @@
import { Injectable, Logger, NotFoundException } from '@nestjs/common'; import { Injectable, Logger, NotFoundException, Inject, forwardRef } from '@nestjs/common';
import { InjectModel } from '@nestjs/sequelize'; import { InjectModel } from '@nestjs/sequelize';
import { Op, Transaction } from 'sequelize'; import { Op, Transaction } from 'sequelize';
import { Sequelize } from 'sequelize-typescript'; import { Sequelize } from 'sequelize-typescript';
import { UserDietHistory } from '../users/models/user-diet-history.model'; import { UserDietHistory } from '../users/models/user-diet-history.model';
import { ActivityActionType, ActivityEntityType, ActivityLog } from '../activity-logs/models/activity-log.model'; import { ActivityActionType, ActivityEntityType, ActivityLog } from '../activity-logs/models/activity-log.model';
import { CreateDietRecordDto, UpdateDietRecordDto, GetDietHistoryQueryDto, DietRecordResponseDto, DietHistoryResponseDto, NutritionSummaryDto } from '../users/dto/diet-record.dto'; import { CreateDietRecordDto, UpdateDietRecordDto, GetDietHistoryQueryDto, DietRecordResponseDto, DietHistoryResponseDto, NutritionSummaryDto, FoodRecognitionRequestDto, FoodRecognitionResponseDto, FoodRecognitionToDietRecordsResponseDto } from '../users/dto/diet-record.dto';
import { DietRecordSource } from '../users/models/user-diet-history.model'; import { DietRecordSource, MealType } from '../users/models/user-diet-history.model';
import { ResponseCode } from '../base.dto'; import { ResponseCode } from '../base.dto';
import { DietAnalysisService } from '../ai-coach/services/diet-analysis.service';
@Injectable() @Injectable()
export class DietRecordsService { export class DietRecordsService {
@@ -18,6 +19,8 @@ export class DietRecordsService {
@InjectModel(ActivityLog) @InjectModel(ActivityLog)
private readonly activityLogModel: typeof ActivityLog, private readonly activityLogModel: typeof ActivityLog,
private readonly sequelize: Sequelize, private readonly sequelize: Sequelize,
@Inject(forwardRef(() => DietAnalysisService))
private readonly dietAnalysisService: DietAnalysisService,
) { } ) { }
/** /**
@@ -286,6 +289,89 @@ export class DietRecordsService {
}; };
} }
/**
* 根据图片URL识别食物并转换为饮食记录格式
* @param imageUrl 图片URL
* @param suggestedMealType 建议的餐次类型(可选)
* @returns 食物识别结果转换为饮食记录格式
*/
async recognizeFoodToDietRecords(
imageUrl: string,
suggestedMealType?: MealType
): Promise<FoodRecognitionToDietRecordsResponseDto> {
try {
this.logger.log(`recognizeFoodToDietRecords - imageUrl: ${imageUrl}, suggestedMealType: ${suggestedMealType}`);
// 调用 DietAnalysisService 进行食物识别
const recognitionResult = await this.dietAnalysisService.recognizeFoodForConfirmation([imageUrl]);
// 将识别结果转换为 CreateDietRecordDto 格式
const dietRecords: CreateDietRecordDto[] = recognitionResult.recognizedItems.map(item => ({
mealType: suggestedMealType || item.mealType,
foodName: item.foodName,
portionDescription: item.portion,
estimatedCalories: item.calories,
proteinGrams: item.nutritionData.proteinGrams,
carbohydrateGrams: item.nutritionData.carbohydrateGrams,
fatGrams: item.nutritionData.fatGrams,
fiberGrams: item.nutritionData.fiberGrams,
source: DietRecordSource.Vision,
imageUrl: imageUrl,
aiAnalysisResult: {
recognitionId: item.id,
confidence: recognitionResult.confidence,
analysisText: recognitionResult.analysisText,
originalLabel: item.label
}
}));
return {
dietRecords,
analysisText: recognitionResult.analysisText,
confidence: recognitionResult.confidence,
imageUrl: imageUrl
};
} catch (error) {
this.logger.error(`recognizeFoodToDietRecords error: ${error instanceof Error ? error.message : String(error)}`);
throw error;
}
}
/**
* 根据图片URL识别食物原始格式
* @param imageUrl 图片URL
* @param suggestedMealType 建议的餐次类型(可选)
* @returns 食物识别结果
*/
async recognizeFood(
imageUrl: string,
suggestedMealType?: MealType
): Promise<FoodRecognitionResponseDto> {
try {
this.logger.log(`recognizeFood - imageUrl: ${imageUrl}, suggestedMealType: ${suggestedMealType}`);
// 调用 DietAnalysisService 进行食物识别
const recognitionResult = await this.dietAnalysisService.recognizeFoodForConfirmation([imageUrl]);
// 如果指定了建议的餐次类型,更新所有识别项的餐次类型
if (suggestedMealType) {
recognitionResult.recognizedItems.forEach(item => {
item.mealType = suggestedMealType;
});
}
return {
recognizedItems: recognitionResult.recognizedItems,
analysisText: recognitionResult.analysisText,
confidence: recognitionResult.confidence,
imageUrl: imageUrl
};
} catch (error) {
this.logger.error(`recognizeFood error: ${error instanceof Error ? error.message : String(error)}`);
throw error;
}
}
/** /**
* 将数据库记录映射为DTO * 将数据库记录映射为DTO
*/ */

View File

@@ -375,3 +375,83 @@ export class DietAnalysisResponseDto {
@ApiProperty({ description: '改善建议' }) @ApiProperty({ description: '改善建议' })
recommendations: string[]; recommendations: string[];
} }
/**
* 食物识别请求 DTO
*/
export class FoodRecognitionRequestDto {
@ApiProperty({ description: '食物图片URL', example: 'https://example.com/food-image.jpg' })
@IsString()
@IsNotEmpty()
imageUrl: string;
@ApiProperty({ enum: MealType, description: '餐次类型(可选,系统会根据时间自动推断)', required: false })
@IsOptional()
@IsEnum(MealType)
mealType?: MealType;
}
/**
* 食物识别选项
*/
export class FoodRecognitionOptionDto {
@ApiProperty({ description: '选项ID' })
id: string;
@ApiProperty({ description: '显示标签,如"一碗米饭 150卡"' })
label: string;
@ApiProperty({ description: '食物名称' })
foodName: string;
@ApiProperty({ description: '份量描述' })
portion: string;
@ApiProperty({ description: '估算热量' })
calories: number;
@ApiProperty({ enum: MealType, description: '餐次类型' })
mealType: MealType;
@ApiProperty({ description: '营养数据' })
nutritionData: {
proteinGrams?: number;
carbohydrateGrams?: number;
fatGrams?: number;
fiberGrams?: number;
};
}
/**
* 食物识别响应 DTO
*/
export class FoodRecognitionResponseDto {
@ApiProperty({ type: [FoodRecognitionOptionDto], description: '识别到的食物选项列表' })
recognizedItems: FoodRecognitionOptionDto[];
@ApiProperty({ description: '分析说明文字' })
analysisText: string;
@ApiProperty({ description: '识别置信度 0-100' })
confidence: number;
@ApiProperty({ description: '原始图片URL' })
imageUrl: string;
}
/**
* 食物识别转换为饮食记录的响应 DTO
*/
export class FoodRecognitionToDietRecordsResponseDto {
@ApiProperty({ type: [CreateDietRecordDto], description: '转换后的饮食记录数据列表' })
dietRecords: CreateDietRecordDto[];
@ApiProperty({ description: '分析说明文字' })
analysisText: string;
@ApiProperty({ description: '识别置信度 0-100' })
confidence: number;
@ApiProperty({ description: '原始图片URL' })
imageUrl: string;
}