feat: stats api

This commit is contained in:
pinks 2023-10-09 21:03:31 +02:00
parent a8d36db20b
commit bf75cce20c
9 changed files with 343 additions and 14 deletions

View File

@ -14,7 +14,7 @@ export async function serveUi() {
aliasMap: { aliasMap: {
"/utils/*": "../utils/", "/utils/*": "../utils/",
}, },
quiet: true, log: (_request, response) => response.status >= 400,
}), }),
})); }));

View File

@ -1,14 +1,19 @@
import { createPathFilter } from "t_rest/server"; import { createLoggerMiddleware, createPathFilter } from "t_rest/server";
import { jobsRoute } from "./jobsRoute.ts"; import { jobsRoute } from "./jobsRoute.ts";
import { sessionsRoute } from "./sessionsRoute.ts";
import { usersRoute } from "./usersRoute.ts";
import { paramsRoute } from "./paramsRoute.ts"; import { paramsRoute } from "./paramsRoute.ts";
import { sessionsRoute } from "./sessionsRoute.ts";
import { statsRoute } from "./statsRoute.ts";
import { usersRoute } from "./usersRoute.ts";
export const serveApi = createPathFilter({ export const serveApi = createLoggerMiddleware(
"jobs": jobsRoute, createPathFilter({
"sessions": sessionsRoute, "jobs": jobsRoute,
"users": usersRoute, "sessions": sessionsRoute,
"settings/params": paramsRoute, "users": usersRoute,
}); "settings/params": paramsRoute,
"stats": statsRoute,
}),
{ filterStatus: (status) => status >= 400 },
);
export type ApiHandler = typeof serveApi; export type ApiHandler = typeof serveApi;

46
api/statsRoute.ts Normal file
View File

@ -0,0 +1,46 @@
import { createEndpoint, createMethodFilter, createPathFilter } from "t_rest/server";
import { liveGlobalStats } from "../app/globalStatsStore.ts";
import { getDailyStats } from "../app/dailyStatsStore.ts";
export const statsRoute = createPathFilter({
"global": createMethodFilter({
GET: createEndpoint(
{ query: null, body: null },
async () => {
return {
status: 200,
body: {
type: "application/json",
data: liveGlobalStats,
},
};
},
),
}),
"daily/{year}/{month}/{day}": createMethodFilter({
GET: createEndpoint(
{ query: null, body: null },
async ({ params }) => {
const year = Number(params.year);
const month = Number(params.month);
const day = Number(params.day);
const minDate = new Date("2023-01-01");
const maxDate = new Date();
const date = new Date(Date.UTC(year, month - 1, day));
if (date < minDate || date > maxDate) {
return {
status: 404,
body: { type: "text/plain", data: "Not found" },
};
}
return {
status: 200,
body: {
type: "application/json",
data: await getDailyStats(year, month, day),
},
};
},
),
}),
});

65
app/dailyStatsStore.ts Normal file
View File

@ -0,0 +1,65 @@
import { UTCDateMini } from "@date-fns/utc";
import { hoursToMilliseconds, isSameDay, minutesToMilliseconds } from "date-fns";
import { getLogger } from "std/log/mod.ts";
import { JsonSchema, jsonType } from "t_rest/server";
import { db } from "./db.ts";
import { generationStore } from "./generationStore.ts";
import { kvMemoize } from "./kvMemoize.ts";
const logger = () => getLogger();
export const dailyStatsSchema = {
type: "object",
properties: {
userIds: { type: "array", items: { type: "number" } },
imageCount: { type: "number" },
pixelCount: { type: "number" },
timestamp: { type: "number" },
},
required: ["userIds", "imageCount", "pixelCount", "timestamp"],
} as const satisfies JsonSchema;
export type DailyStats = jsonType<typeof dailyStatsSchema>;
export const getDailyStats = kvMemoize(
db,
["dailyStats"],
async (year: number, month: number, day: number): Promise<DailyStats> => {
const userIdSet = new Set<number>();
let imageCount = 0;
let pixelCount = 0;
const after = new Date(Date.UTC(year, month - 1, day));
const before = new Date(Date.UTC(year, month - 1, day + 1));
logger().info(`Calculating daily stats for ${year}-${month}-${day}`);
for await (
const generation of generationStore.listAll({ after, before })
) {
userIdSet.add(generation.value.from.id);
imageCount++;
pixelCount += (generation.value.info?.width ?? 0) * (generation.value.info?.height ?? 0);
}
return {
userIds: [...userIdSet],
imageCount,
pixelCount,
timestamp: Date.now(),
};
},
{
// expire in 1 minute if was calculated on the same day, otherwise 7-14 days.
expireIn: (result, year, month, day) => {
const requestDate = new UTCDateMini(year, month - 1, day);
const calculatedDate = new UTCDateMini(result.timestamp);
return isSameDay(requestDate, calculatedDate)
? minutesToMilliseconds(1)
: hoursToMilliseconds(24 * 7 + Math.random() * 24 * 7);
},
// should cache if the stats are non-zero
shouldCache: (result) =>
result.userIds.length > 0 || result.imageCount > 0 || result.pixelCount > 0,
},
);

56
app/globalStatsStore.ts Normal file
View File

@ -0,0 +1,56 @@
import { addDays } from "date-fns";
import { JsonSchema, jsonType } from "t_rest/server";
import { decodeTime } from "ulid";
import { getDailyStats } from "./dailyStatsStore.ts";
import { generationStore } from "./generationStore.ts";
export const globalStatsSchema = {
type: "object",
properties: {
userIds: { type: "array", items: { type: "number" } },
imageCount: { type: "number" },
pixelCount: { type: "number" },
timestamp: { type: "number" },
},
required: ["userIds", "imageCount", "pixelCount", "timestamp"],
} as const satisfies JsonSchema;
export type GlobalStats = jsonType<typeof globalStatsSchema>;
export const liveGlobalStats: GlobalStats = await getGlobalStats();
export async function getGlobalStats(): Promise<GlobalStats> {
// find the year/month/day of the first generation
const startDate = await generationStore.getAll({}, { limit: 1 })
.then((generations) => generations[0]?.id)
.then((generationId) => generationId ? new Date(decodeTime(generationId)) : new Date());
// iterate to today and sum up stats
const userIdSet = new Set<number>();
let imageCount = 0;
let pixelCount = 0;
const tomorrow = addDays(new Date(), 1);
for (
let date = startDate;
date < tomorrow;
date = addDays(date, 1)
) {
const dailyStats = await getDailyStats(
date.getUTCFullYear(),
date.getUTCMonth() + 1,
date.getUTCDate(),
);
for (const userId of dailyStats.userIds) userIdSet.add(userId);
imageCount += dailyStats.imageCount;
pixelCount += dailyStats.pixelCount;
}
return {
userIds: [...userIdSet],
imageCount,
pixelCount,
timestamp: Date.now(),
};
}

45
app/kvMemoize.ts Normal file
View File

@ -0,0 +1,45 @@
/**
* Memoizes the function result in KV storage.
*/
export function kvMemoize<A extends Deno.KvKey, R>(
db: Deno.Kv,
key: Deno.KvKey,
fn: (...args: A) => Promise<R>,
options?: {
expireIn?: number | ((result: R, ...args: A) => number);
shouldRecalculate?: (result: R, ...args: A) => boolean;
shouldCache?: (result: R, ...args: A) => boolean;
override?: {
set: (key: Deno.KvKey, args: A, value: R, options: { expireIn?: number }) => Promise<void>;
get: (key: Deno.KvKey, args: A) => Promise<R | undefined>;
};
},
): (...args: A) => Promise<R> {
return async (...args) => {
const cachedResult = options?.override?.get
? await options.override.get(key, args)
: (await db.get<R>([...key, ...args])).value;
if (cachedResult != null) {
if (!options?.shouldRecalculate?.(cachedResult, ...args)) {
return cachedResult;
}
}
const result = await fn(...args);
const expireIn = typeof options?.expireIn === "function"
? options.expireIn(result, ...args)
: options?.expireIn;
if (options?.shouldCache?.(result, ...args) ?? (result != null)) {
if (options?.override?.set) {
await options.override.set(key, args, result, { expireIn });
} else {
await db.set([...key, ...args], result, { expireIn });
}
}
return result;
};
}

View File

@ -9,6 +9,7 @@ import { bot } from "../bot/mod.ts";
import { formatUserChat } from "../utils/formatUserChat.ts"; import { formatUserChat } from "../utils/formatUserChat.ts";
import { db, fs } from "./db.ts"; import { db, fs } from "./db.ts";
import { generationStore, SdGenerationInfo } from "./generationStore.ts"; import { generationStore, SdGenerationInfo } from "./generationStore.ts";
import { liveGlobalStats } from "./globalStatsStore.ts";
const logger = () => getLogger(); const logger = () => getLogger();
@ -109,6 +110,15 @@ export async function processUploadQueue() {
info: state.info, info: state.info,
}); });
// update live stats
{
liveGlobalStats.imageCount++;
liveGlobalStats.pixelCount += state.info.width * state.info.height;
const userIdSet = new Set(liveGlobalStats.userIds);
userIdSet.add(state.from.id);
liveGlobalStats.userIds = [...userIdSet];
}
// delete the status message // delete the status message
await bot.api.deleteMessage(state.replyMessage.chat.id, state.replyMessage.message_id) await bot.api.deleteMessage(state.replyMessage.chat.id, state.replyMessage.message_id)
.catch(() => undefined); .catch(() => undefined);

100
app/userStatsStore.ts Normal file
View File

@ -0,0 +1,100 @@
import { minutesToMilliseconds } from "date-fns";
import { Store } from "indexed_kv";
import { getLogger } from "std/log/mod.ts";
import { JsonSchema, jsonType } from "t_rest/server";
import { db } from "./db.ts";
import { generationStore } from "./generationStore.ts";
import { kvMemoize } from "./kvMemoize.ts";
const logger = () => getLogger();
export const userStatsSchema = {
type: "object",
properties: {
userId: { type: "number" },
imageCount: { type: "number" },
pixelCount: { type: "number" },
tagsCount: {
type: "object",
additionalProperties: { type: "number" },
},
timestamp: { type: "number" },
},
required: ["userId", "imageCount", "pixelCount", "tagsCount", "timestamp"],
} as const satisfies JsonSchema;
export type UserStats = jsonType<typeof userStatsSchema>;
type UserStatsIndices = {
userId: number;
imageCount: number;
pixelCount: number;
};
const userStatsStore = new Store<UserStats, UserStatsIndices>(
db,
"userStats",
{
indices: {
userId: { getValue: (item) => item.userId },
imageCount: { getValue: (item) => item.imageCount },
pixelCount: { getValue: (item) => item.pixelCount },
},
},
);
export const getUserStats = kvMemoize(
db,
["userStats"],
async (userId: number): Promise<UserStats> => {
let imageCount = 0;
let pixelCount = 0;
const tagsCount: Record<string, number> = {};
logger().info(`Calculating user stats for ${userId}`);
for await (
const generation of generationStore.listBy("fromId", { value: userId })
) {
imageCount++;
pixelCount += (generation.value.info?.width ?? 0) * (generation.value.info?.height ?? 0);
const tags = generation.value.info?.prompt.split(/[,;.\n]/)
.map((tag) => tag.trim())
.filter((tag) => tag.length > 0)
.map((tag) => tag.toLowerCase())
.map((tag) => tag.replace(/[()[\]]/, ""))
.map((tag) => tag.replace(/:[\d.]/g, ""))
.map((tag) => tag.replace(/ +/g, " ")) ?? [];
for (const tag of tags) {
tagsCount[tag] = (tagsCount[tag] ?? 0) + 1;
}
}
return {
userId,
imageCount,
pixelCount,
tagsCount,
timestamp: Date.now(),
};
},
{
// expire in random time between 5-10 minutes
expireIn: () => minutesToMilliseconds(5 + Math.random() * 5),
// override default set/get behavior to use userStatsStore
override: {
get: async (_key, [userId]) => {
const items = await userStatsStore.getBy("userId", { value: userId }, { reverse: true });
return items[0]?.value;
},
set: async (_key, [userId], value, options) => {
// delete old stats
for await (const item of userStatsStore.listBy("userId", { value: userId })) {
await item.delete();
}
// set new stats
await userStatsStore.create(value, options);
},
},
},
);

View File

@ -19,10 +19,10 @@
"async": "https://deno.land/x/async@v2.0.2/mod.ts", "async": "https://deno.land/x/async@v2.0.2/mod.ts",
"ulid": "https://deno.land/x/ulid@v0.3.0/mod.ts", "ulid": "https://deno.land/x/ulid@v0.3.0/mod.ts",
"indexed_kv": "https://deno.land/x/indexed_kv@v0.4.0/mod.ts", "indexed_kv": "https://deno.land/x/indexed_kv@v0.5.0/mod.ts",
"kvmq": "https://deno.land/x/kvmq@v0.3.0/mod.ts", "kvmq": "https://deno.land/x/kvmq@v0.3.0/mod.ts",
"kvfs": "https://deno.land/x/kvfs@v0.1.0/mod.ts", "kvfs": "https://deno.land/x/kvfs@v0.1.0/mod.ts",
"serve_spa": "https://deno.land/x/serve_spa@v0.1.0/mod.ts", "serve_spa": "https://deno.land/x/serve_spa@v0.2.0/mod.ts",
"reroute": "https://deno.land/x/reroute@v0.1.0/mod.ts", "reroute": "https://deno.land/x/reroute@v0.1.0/mod.ts",
"grammy": "https://lib.deno.dev/x/grammy@1/mod.ts", "grammy": "https://lib.deno.dev/x/grammy@1/mod.ts",
"grammy_types": "https://lib.deno.dev/x/grammy_types@3/mod.ts", "grammy_types": "https://lib.deno.dev/x/grammy_types@3/mod.ts",
@ -30,13 +30,15 @@
"grammy_parse_mode": "https://lib.deno.dev/x/grammy_parse_mode@1/mod.ts", "grammy_parse_mode": "https://lib.deno.dev/x/grammy_parse_mode@1/mod.ts",
"grammy_stateless_question": "https://lib.deno.dev/x/grammy_stateless_question_alpha@3/mod.ts", "grammy_stateless_question": "https://lib.deno.dev/x/grammy_stateless_question_alpha@3/mod.ts",
"grammy_files": "https://lib.deno.dev/x/grammy_files@1/mod.ts", "grammy_files": "https://lib.deno.dev/x/grammy_files@1/mod.ts",
"date-fns": "https://cdn.skypack.dev/date-fns@2.30.0?dts",
"@date-fns/utc": "https://cdn.skypack.dev/@date-fns/utc@1.1.0?dts",
"file_type": "https://esm.sh/file-type@18.5.0", "file_type": "https://esm.sh/file-type@18.5.0",
"png_chunks_extract": "https://esm.sh/png-chunks-extract@1.0.0", "png_chunks_extract": "https://esm.sh/png-chunks-extract@1.0.0",
"png_chunk_text": "https://esm.sh/png-chunk-text@1.0.0", "png_chunk_text": "https://esm.sh/png-chunk-text@1.0.0",
"openapi_fetch": "https://esm.sh/openapi-fetch@0.7.6", "openapi_fetch": "https://esm.sh/openapi-fetch@0.7.6",
"t_rest/server": "https://esm.sh/ty-rest@0.3.1/server?dev", "t_rest/server": "https://esm.sh/ty-rest@0.3.2/server?dev",
"t_rest/client": "https://esm.sh/ty-rest@0.3.1/client?dev", "t_rest/client": "https://esm.sh/ty-rest@0.3.2/client?dev",
"react": "https://esm.sh/react@18.2.0?dev", "react": "https://esm.sh/react@18.2.0?dev",
"react-dom/client": "https://esm.sh/react-dom@18.2.0/client?dev", "react-dom/client": "https://esm.sh/react-dom@18.2.0/client?dev",
"react-router-dom": "https://esm.sh/react-router-dom@6.16.0?dev", "react-router-dom": "https://esm.sh/react-router-dom@6.16.0?dev",