feat: add token usage tracking and leaderboard
- Add token_usage table with composite unique index for claw_id + date - Add API endpoints: POST /token, GET /token/leaderboard, GET /token/stats - Add TokenLeaderboard component with daily/total period toggle - Add CLI commands: `token` and `stats` for reporting and viewing usage - Add Redis caching for leaderboard with 1-minute TTL - Add shared utilities: authenticateRequest, getTodayDateString - Use UPSERT pattern for atomic token updates - Add i18n translations (en/zh) for new UI elements
This commit is contained in:
richarjiang
113
app/api/v1/token/leaderboard/route.ts
Normal file
113
app/api/v1/token/leaderboard/route.ts
Normal file
@@ -0,0 +1,113 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { eq, desc, sql } from "drizzle-orm";
|
||||
import { db } from "@/lib/db";
|
||||
import { claws, tokenUsage } from "@/lib/db/schema";
|
||||
import {
|
||||
getTokenLeaderboardCache,
|
||||
setTokenLeaderboardCache,
|
||||
} from "@/lib/redis";
|
||||
import { getTodayDateString } from "@/lib/utils";
|
||||
|
||||
interface LeaderboardEntry {
|
||||
clawId: string;
|
||||
clawName: string;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
totalTokens: number;
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const period = searchParams.get("period") || "daily";
|
||||
const limit = Math.min(parseInt(searchParams.get("limit") || "100"), 100);
|
||||
|
||||
if (period !== "daily" && period !== "total") {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid period. Must be 'daily' or 'total'" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const today = getTodayDateString();
|
||||
|
||||
// Check cache
|
||||
const cached = await getTokenLeaderboardCache(
|
||||
period as "daily" | "total",
|
||||
today
|
||||
);
|
||||
if (cached) {
|
||||
return NextResponse.json(JSON.parse(cached));
|
||||
}
|
||||
|
||||
let leaderboard: LeaderboardEntry[];
|
||||
|
||||
if (period === "daily") {
|
||||
// Daily leaderboard - tokens for today only
|
||||
const dailyData = await db
|
||||
.select({
|
||||
clawId: tokenUsage.clawId,
|
||||
clawName: claws.name,
|
||||
inputTokens: tokenUsage.inputTokens,
|
||||
outputTokens: tokenUsage.outputTokens,
|
||||
totalTokens: sql<number>`(${tokenUsage.inputTokens} + ${tokenUsage.outputTokens})`.as("totalTokens"),
|
||||
})
|
||||
.from(tokenUsage)
|
||||
.innerJoin(claws, eq(tokenUsage.clawId, claws.id))
|
||||
.where(eq(tokenUsage.date, today))
|
||||
.orderBy(desc(sql`(${tokenUsage.inputTokens} + ${tokenUsage.outputTokens})`))
|
||||
.limit(limit);
|
||||
|
||||
leaderboard = dailyData.map((d) => ({
|
||||
clawId: d.clawId,
|
||||
clawName: d.clawName,
|
||||
inputTokens: d.inputTokens,
|
||||
outputTokens: d.outputTokens,
|
||||
totalTokens: d.inputTokens + d.outputTokens,
|
||||
}));
|
||||
} else {
|
||||
// Total leaderboard - sum of all time
|
||||
// Use a raw query for aggregation since Drizzle's groupBy is complex
|
||||
const totalData = await db.execute(sql`
|
||||
SELECT
|
||||
t.claw_id AS clawId,
|
||||
c.name AS clawName,
|
||||
CAST(SUM(t.input_tokens) AS SIGNED) AS inputTokens,
|
||||
CAST(SUM(t.output_tokens) AS SIGNED) AS outputTokens,
|
||||
CAST(SUM(t.input_tokens + t.output_tokens) AS SIGNED) AS totalTokens
|
||||
FROM token_usage t
|
||||
JOIN claws c ON t.claw_id = c.id
|
||||
GROUP BY t.claw_id, c.name
|
||||
ORDER BY totalTokens DESC
|
||||
LIMIT ${limit}
|
||||
`);
|
||||
|
||||
leaderboard = totalData[0] as unknown as LeaderboardEntry[];
|
||||
}
|
||||
|
||||
// Add rank
|
||||
const responseData = {
|
||||
period,
|
||||
date: today,
|
||||
leaderboard: leaderboard.map((item, index) => ({
|
||||
rank: index + 1,
|
||||
...item,
|
||||
})),
|
||||
};
|
||||
|
||||
// Cache for 1 minute
|
||||
await setTokenLeaderboardCache(
|
||||
period as "daily" | "total",
|
||||
today,
|
||||
JSON.stringify(responseData)
|
||||
);
|
||||
|
||||
return NextResponse.json(responseData);
|
||||
} catch (error) {
|
||||
console.error("Token leaderboard error:", error);
|
||||
return NextResponse.json(
|
||||
{ error: "Internal server error" },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
55
app/api/v1/token/route.ts
Normal file
55
app/api/v1/token/route.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { db } from "@/lib/db";
|
||||
import { invalidateTokenLeaderboardCache } from "@/lib/redis";
|
||||
import { authenticateRequest, getTodayDateString } from "@/lib/utils";
|
||||
import { tokenSchema } from "@/lib/validators/schemas";
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
const auth = await authenticateRequest(req);
|
||||
if (auth instanceof NextResponse) {
|
||||
return auth;
|
||||
}
|
||||
const { claw } = auth;
|
||||
|
||||
const body = await req.json();
|
||||
const parsed = tokenSchema.safeParse(body);
|
||||
if (!parsed.success) {
|
||||
return NextResponse.json(
|
||||
{ error: "Validation failed", details: parsed.error.flatten() },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { inputTokens, outputTokens, date } = parsed.data;
|
||||
const targetDate = date || getTodayDateString();
|
||||
|
||||
// Use UPSERT (INSERT ... ON DUPLICATE KEY UPDATE) for atomic operation
|
||||
await db.execute(sql`
|
||||
INSERT INTO token_usage (claw_id, date, input_tokens, output_tokens)
|
||||
VALUES (${claw.id}, ${targetDate}, ${inputTokens}, ${outputTokens})
|
||||
ON DUPLICATE KEY UPDATE
|
||||
input_tokens = ${inputTokens},
|
||||
output_tokens = ${outputTokens},
|
||||
updated_at = NOW()
|
||||
`);
|
||||
|
||||
// Invalidate leaderboard cache
|
||||
await invalidateTokenLeaderboardCache();
|
||||
|
||||
return NextResponse.json({
|
||||
ok: true,
|
||||
date: targetDate,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens: inputTokens + outputTokens,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Token report error:", error);
|
||||
return NextResponse.json(
|
||||
{ error: "Internal server error" },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
91
app/api/v1/token/stats/route.ts
Normal file
91
app/api/v1/token/stats/route.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { eq, and, gte, sql } from "drizzle-orm";
|
||||
import { db } from "@/lib/db";
|
||||
import { tokenUsage } from "@/lib/db/schema";
|
||||
import { authenticateRequest, getTodayDateString } from "@/lib/utils";
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
const auth = await authenticateRequest(req);
|
||||
if (auth instanceof NextResponse) {
|
||||
return auth;
|
||||
}
|
||||
const { claw } = auth;
|
||||
|
||||
const today = getTodayDateString();
|
||||
|
||||
// Get total usage using raw query for aggregation
|
||||
const totalResult = await db.execute(sql`
|
||||
SELECT
|
||||
COALESCE(SUM(input_tokens), 0) AS inputTokens,
|
||||
COALESCE(SUM(output_tokens), 0) AS outputTokens
|
||||
FROM token_usage
|
||||
WHERE claw_id = ${claw.id}
|
||||
`);
|
||||
|
||||
const totalRow = totalResult[0] as unknown as { inputTokens: number; outputTokens: number } | undefined;
|
||||
const totalStats = totalRow
|
||||
? {
|
||||
inputTokens: Number(totalRow.inputTokens || 0),
|
||||
outputTokens: Number(totalRow.outputTokens || 0),
|
||||
totalTokens: Number(totalRow.inputTokens || 0) + Number(totalRow.outputTokens || 0),
|
||||
}
|
||||
: {
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
totalTokens: 0,
|
||||
};
|
||||
|
||||
// Get last 30 days history
|
||||
const thirtyDaysAgo = new Date();
|
||||
thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
|
||||
const thirtyDaysAgoStr = thirtyDaysAgo.toISOString().split("T")[0];
|
||||
|
||||
const history = await db
|
||||
.select({
|
||||
date: tokenUsage.date,
|
||||
inputTokens: tokenUsage.inputTokens,
|
||||
outputTokens: tokenUsage.outputTokens,
|
||||
})
|
||||
.from(tokenUsage)
|
||||
.where(
|
||||
and(
|
||||
eq(tokenUsage.clawId, claw.id),
|
||||
gte(tokenUsage.date, thirtyDaysAgoStr)
|
||||
)
|
||||
);
|
||||
|
||||
// Derive today's stats from history (avoids redundant query)
|
||||
const todayRecord = history.find((h) => h.date === today);
|
||||
const todayStats = todayRecord
|
||||
? {
|
||||
inputTokens: todayRecord.inputTokens,
|
||||
outputTokens: todayRecord.outputTokens,
|
||||
totalTokens: todayRecord.inputTokens + todayRecord.outputTokens,
|
||||
}
|
||||
: {
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
totalTokens: 0,
|
||||
};
|
||||
|
||||
return NextResponse.json({
|
||||
clawId: claw.id,
|
||||
clawName: claw.name,
|
||||
today: todayStats,
|
||||
total: totalStats,
|
||||
history: history.map((h) => ({
|
||||
date: h.date,
|
||||
inputTokens: h.inputTokens,
|
||||
outputTokens: h.outputTokens,
|
||||
totalTokens: h.inputTokens + h.outputTokens,
|
||||
})),
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Token stats error:", error);
|
||||
return NextResponse.json(
|
||||
{ error: "Internal server error" },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user