review fixes

feat: user daily stats
refactor: move sendChatAction
2023-11-12 02:33:35 +00:00 · 2023-11-11 04:13:26 +03:00 · 2023-11-11 04:13:26 +03:00 · 2023-11-11 04:13:26 +03:00 · 2023-11-11 04:13:26 +03:00 · 2023-11-11 04:13:26 +03:00
14 changed files with 208 additions and 81 deletions
--- a/api/adminsRoute.ts
+++ b/api/adminsRoute.ts
@ -109,12 +109,12 @@ export const adminsRoute = createPathFilter({
          },
        },
        async ({ params, query }) => {
+          return withAdmin(query, async (chat) => {
            const deletedAdminEntry = await adminStore.getById(params.adminId!);
            if (!deletedAdminEntry) {
              return { status: 404, body: { type: "text/plain", data: `Admin not found` } };
            }
            const deletedAdminUser = await getUser(deletedAdminEntry.value.tgUserId);
-          return withUser(query, async (chat) => {
            await deletedAdminEntry.delete();
            info(`User ${chat.first_name} demoted user ${deletedAdminUser.first_name} from admin`);
            return {
--- a/api/jobsRoute.ts
+++ b/api/jobsRoute.ts
@ -4,12 +4,29 @@ import { generationQueue } from "../app/generationQueue.ts";
 export const jobsRoute = createMethodFilter({
  GET: createEndpoint(
    { query: null, body: null },
-    async () => ({
+    async () => {
+      const allJobs = await generationQueue.getAllJobs();
+      const filteredJobsData = allJobs.map((job) => ({
+        id: job.id,
+        place: job.place,
+        state: {
+          from: {
+            language_code: job.state.from.language_code,
+            first_name: job.state.from.first_name,
+            last_name: job.state.from.last_name,
+            username: job.state.from.username,
+          },
+          progress: job.state.progress,
+          workerInstanceKey: job.state.workerInstanceKey,
+        },
+      }));
+      return {
        status: 200,
        body: {
          type: "application/json",
-        data: await generationQueue.getAllJobs(),
+          data: filteredJobsData,
+        },
+      };
    },
-    }),
  ),
 });
--- a/api/statsRoute.ts
+++ b/api/statsRoute.ts
@ -5,6 +5,7 @@ import { generationStore } from "../app/generationStore.ts";
 import { globalStats } from "../app/globalStats.ts";
 import { getUserDailyStats } from "../app/userDailyStatsStore.ts";
 import { getUserStats } from "../app/userStatsStore.ts";
+import { withAdmin } from "./withUser.ts";

 const STATS_INTERVAL_MIN = 3;

@ -91,11 +92,31 @@ export const statsRoute = createPathFilter({
            data: {
              imageCount: stats.imageCount,
              pixelCount: stats.pixelCount,
+              timestamp: stats.timestamp,
+            },
+          },
+        };
+      },
+    ),
+  }),
+  "users/{userId}/tagcount": createMethodFilter({
+    GET: createEndpoint(
+      { query: { sessionId: { type: "string" } }, body: null },
+      async ({ params, query }) => {
+        return withAdmin(query, async () => {
+          const userId = Number(params.userId);
+          const stats = await getUserStats(userId);
+          return {
+            status: 200,
+            body: {
+              type: "application/json",
+              data: {
                tagCountMap: stats.tagCountMap,
                timestamp: stats.timestamp,
              },
            },
          };
+        });
      },
    ),
  }),
--- a/app/generationQueue.ts
+++ b/app/generationQueue.ts
@ -70,7 +70,8 @@ export async function processGenerationQueue() {
          return response;
        })
        .catch((error) => {
-          workerInstance.update({ lastError: { message: error.message, time: Date.now() } })
+          const cleanedErrorMessage = error.message.replace(/url \([^)]+\)/, "");
+          workerInstance.update({ lastError: { message: cleanedErrorMessage, time: Date.now() } })
            .catch(() => undefined);
          debug(`Worker ${workerInstance.value.key} is down: ${error}`);
        });
@ -240,8 +241,6 @@ async function processGenerationJob(
    if (progressResponse.data.progress > state.progress) {
      state.progress = progressResponse.data.progress;
      await updateJob({ state: state });
-      await bot.api.sendChatAction(state.chat.id, "upload_photo", { maxAttempts: 1 })
-        .catch(() => undefined);
      await bot.api.editMessageText(
        state.replyMessage.chat.id,
        state.replyMessage.message_id,
@ -297,6 +296,10 @@ async function processGenerationJob(
    { maxAttempts: 1 },
  ).catch(() => undefined);

+  // send upload photo action
+  await bot.api.sendChatAction(state.chat.id, "upload_photo", { maxAttempts: 1 })
+    .catch(() => undefined);
+
  debug(`Generation finished for ${formatUserChat(state)}`);
 }

--- a/app/userDailyStatsStore.ts
+++ b/app/userDailyStatsStore.ts
@ -2,6 +2,8 @@ import { JsonSchema, jsonType } from "t_rest/server";
 import { kvMemoize } from "./kvMemoize.ts";
 import { db } from "./db.ts";
 import { generationStore } from "./generationStore.ts";
+import { hoursToMilliseconds, isSameDay, minutesToMilliseconds } from "date-fns";
+import { UTCDateMini } from "date-fns/utc";

 export const userDailyStatsSchema = {
  type: "object",
@ -19,6 +21,36 @@ export const getUserDailyStats = kvMemoize(
  db,
  ["userDailyStats"],
  async (userId: number, year: number, month: number, day: number): Promise<UserDailyStats> => {
-    throw new Error("Not implemented");
+    let imageCount = 0;
+    let pixelCount = 0;
+
+    for await (
+      const generation of generationStore.listBy("fromId", {
+        after: new Date(Date.UTC(year, month - 1, day)),
+        before: new Date(Date.UTC(year, month - 1, day + 1)),
+        value: userId,
+      })
+    ) {
+      imageCount++;
+      pixelCount += (generation.value.info?.width ?? 0) * (generation.value.info?.height ?? 0);
+    }
+
+    return {
+      imageCount,
+      pixelCount,
+      timestamp: Date.now(),
+    };
+  },
+  {
+    // expire in 1 minute if was calculated on the same day, otherwise 7-14 days.
+    expireIn: (result, year, month, day) => {
+      const requestDate = new UTCDateMini(year, month - 1, day);
+      const calculatedDate = new UTCDateMini(result.timestamp);
+      return isSameDay(requestDate, calculatedDate)
+        ? minutesToMilliseconds(1)
+        : hoursToMilliseconds(24 * 7 + Math.random() * 24 * 7);
+    },
+    // should cache if the stats are non-zero
+    shouldCache: (result) => result.imageCount > 0 || result.pixelCount > 0,
  },
 );
--- a/bot/mod.ts
+++ b/bot/mod.ts
@ -115,7 +115,7 @@ bot.api.setMyDescription(
 bot.api.setMyCommands([
  { command: "txt2img", description: "Generate image from text" },
  { command: "img2img", description: "Generate image from image" },
-  { command: "pnginfo", description: "Show generation parameters of an image" },
+  { command: "pnginfo", description: "Try to extract prompt from raw file" },
  { command: "queue", description: "Show the current queue" },
  { command: "cancel", description: "Cancel all your requests" },
 ]);
--- a/bot/parsePngInfo.ts
+++ b/bot/parsePngInfo.ts
@ -1,12 +1,24 @@
-import { decode } from "png_chunk_text";
-import extractChunks from "png_chunks_extract";
+import * as ExifReader from "exifreader";

-export function getPngInfo(pngData: Uint8Array): string | undefined {
-  return extractChunks(pngData)
-    .filter((chunk) => chunk.name === "tEXt")
-    .map((chunk) => decode(chunk.data))
-    .find((textChunk) => textChunk.keyword === "parameters")
-    ?.text;
+export function getPngInfo(pngData: ArrayBuffer): string | undefined {
+  const info = ExifReader.load(pngData);
+
+  if (info.UserComment?.value && Array.isArray(info.UserComment.value)) {
+    // JPEG image
+    return String.fromCharCode(
+      ...info.UserComment.value
+        .filter((char): char is number => typeof char == "number")
+        .filter((char) => char !== 0),
+    ).replace("UNICODE", "");
+  }
+
+  if (info.parameters?.description) {
+    // PNG image
+    return info.parameters.description;
+  }
+
+  // Unknown image type
+  return undefined;
 }

 export interface PngInfo {
--- a/bot/pnginfoCommand.ts
+++ b/bot/pnginfoCommand.ts
@ -20,9 +20,9 @@ async function pnginfo(ctx: ErisContext, includeRepliedTo: boolean): Promise<voi
  const document = ctx.message?.document ||
    (includeRepliedTo ? ctx.message?.reply_to_message?.document : undefined);

-  if (document?.mime_type !== "image/png") {
+  if (document?.mime_type !== "image/png" && document?.mime_type !== "image/jpeg") {
    await ctx.reply(
-      "Please send me a PNG file." +
+      "Please send me a PNG or JPEG file." +
        pnginfoQuestion.messageSuffixMarkdown(),
      omitUndef(
        {
@ -37,7 +37,14 @@ async function pnginfo(ctx: ErisContext, includeRepliedTo: boolean): Promise<voi

  const file = await ctx.api.getFile(document.file_id);
  const buffer = await fetch(file.getUrl()).then((resp) => resp.arrayBuffer());
-  const params = parsePngInfo(getPngInfo(new Uint8Array(buffer)) ?? "");
+  const info = getPngInfo(buffer);
+  if (!info) {
+    return await ctx.reply(
+      "No info found in file.",
+      omitUndef({ reply_to_message_id: ctx.message?.message_id }),
+    );
+  }
+  const params = parsePngInfo(info, undefined, true);

  const paramsText = fmt([
    `${params.prompt}\n`,
--- a/bot/txt2imgCommand.ts
+++ b/bot/txt2imgCommand.ts
@ -6,6 +6,7 @@ import { generationQueue } from "../app/generationQueue.ts";
 import { formatUserChat } from "../utils/formatUserChat.ts";
 import { ErisContext } from "./mod.ts";
 import { getPngInfo, parsePngInfo, PngInfo } from "./parsePngInfo.ts";
+import { adminStore } from "../app/adminStore.ts";

 export const txt2imgQuestion = new StatelessQuestion<ErisContext>(
  "txt2img",
@ -29,6 +30,7 @@ async function txt2img(ctx: ErisContext, match: string, includeRepliedTo: boolea
  }

  const config = await getConfig();
+  let priority = 0;

  if (config.pausedReason != null) {
    await ctx.reply(`I'm paused: ${config.pausedReason || "No reason given"}`, {
@ -37,6 +39,11 @@ async function txt2img(ctx: ErisContext, match: string, includeRepliedTo: boolea
    return;
  }

+  const [admin] = await adminStore.getBy("tgUserId", { value: ctx.from.id });
+
+  if (admin) {
+    priority = 1;
+  } else {
    const jobs = await generationQueue.getAllJobs();
    if (jobs.length >= config.maxJobs) {
      await ctx.reply(`The queue is full. Try again later. (Max queue size: ${config.maxJobs})`, {
@ -47,11 +54,12 @@ async function txt2img(ctx: ErisContext, match: string, includeRepliedTo: boolea

    const userJobs = jobs.filter((job) => job.state.from.id === ctx.message?.from?.id);
    if (userJobs.length >= config.maxUserJobs) {
-    await ctx.reply(`You already have ${userJobs.length} jobs in queue. Try again later.`, {
+      await ctx.reply(`You already have ${userJobs.length} jobs in the queue. Try again later.`, {
        reply_to_message_id: ctx.message?.message_id,
      });
      return;
    }
+  }

  let params: Partial<PngInfo> = {};

@ -60,17 +68,26 @@ async function txt2img(ctx: ErisContext, match: string, includeRepliedTo: boolea
  if (includeRepliedTo && repliedToMsg?.document?.mime_type === "image/png") {
    const file = await ctx.api.getFile(repliedToMsg.document.file_id);
    const buffer = await fetch(file.getUrl()).then((resp) => resp.arrayBuffer());
-    params = parsePngInfo(getPngInfo(new Uint8Array(buffer)) ?? "", params);
+    params = parsePngInfo(getPngInfo(buffer) ?? "", params);
  }

  const repliedToText = repliedToMsg?.text || repliedToMsg?.caption;
-  if (includeRepliedTo && repliedToText) {
+  const isReply = includeRepliedTo && repliedToText;
+
+  if (isReply) {
    // TODO: remove bot command from replied to text
    params = parsePngInfo(repliedToText, params);
  }

  params = parsePngInfo(match, params, true);

+  if (isReply) {
+    const parsedInfo = parsePngInfo(repliedToText, undefined, true);
+    if (parsedInfo.prompt !== params.prompt) {
+      params.seed = parsedInfo.seed ?? -1;
+    }
+  }
+
  if (!params.prompt) {
    await ctx.reply(
      "Please tell me what you want to see." +
@ -94,7 +111,7 @@ async function txt2img(ctx: ErisContext, match: string, includeRepliedTo: boolea
    chat: ctx.message.chat,
    requestMessage: ctx.message,
    replyMessage: replyMessage,
-  }, { retryCount: 3, retryDelayMs: 10_000 });
+  }, { retryCount: 3, retryDelayMs: 10_000, priority: priority });

  debug(`Generation (txt2img) enqueued for ${formatUserChat(ctx.message)}`);
 }
--- a/deno.json
+++ b/deno.json
@ -11,6 +11,7 @@
    "async": "https://deno.land/x/async@v2.0.2/mod.ts",
    "date-fns": "https://cdn.skypack.dev/date-fns@2.30.0?dts",
    "date-fns/utc": "https://cdn.skypack.dev/@date-fns/utc@1.1.0?dts",
+    "exifreader": "https://esm.sh/exifreader@4.14.1",
    "file_type": "https://esm.sh/file-type@18.5.0",
    "grammy": "https://lib.deno.dev/x/grammy@1/mod.ts",
    "grammy_autoquote": "https://lib.deno.dev/x/grammy_autoquote@1/mod.ts",
@ -23,8 +24,6 @@
    "kvfs": "https://deno.land/x/kvfs@v0.1.0/mod.ts",
    "kvmq": "https://deno.land/x/kvmq@v0.3.0/mod.ts",
    "openapi_fetch": "https://esm.sh/openapi-fetch@0.7.6",
-    "png_chunk_text": "https://esm.sh/png-chunk-text@1.0.0",
-    "png_chunks_extract": "https://esm.sh/png-chunks-extract@1.0.0",
    "react": "https://esm.sh/react@18.2.0?dev",
    "react-dom/client": "https://esm.sh/react-dom@18.2.0/client?external=react&dev",
    "react-flip-move": "https://esm.sh/react-flip-move@3.0.5?external=react&dev",
--- a/deno.lock
+++ b/deno.lock
@ -303,10 +303,9 @@
    "https://esm.sh/@grammyjs/types@2.12.1": "eebe448d3bf3d4fdaeacee50e31b9e3947ce965d2591f1036e5c0273cb7aec36",
    "https://esm.sh/@twind/core@1.1.3": "bf3e64c13de95b061a721831bf2aed322f6e7335848b06d805168c3212686c1d",
    "https://esm.sh/@twind/preset-tailwind@1.1.4": "29f5e4774c344ff08d2636e3d86382060a8b40d6bce1c158b803df1823c2804c",
+    "https://esm.sh/exifreader@4.14.1": "d0f21973393b0d1a6ed329dac8fcfb2f87ce47fe40b8172e205e7d6d85790bb6",
    "https://esm.sh/file-type@18.5.0": "f01f6eddb05d365925545e26bd449f934dc042bead882b2795c35c4f48d690a3",
    "https://esm.sh/openapi-fetch@0.7.6": "43eff6df93e773801cb6ade02b0f47c05d0bfe2fb044adbf2b94fa74ff30d35b",
-    "https://esm.sh/png-chunk-text@1.0.0": "08beb86f31b5ff70240650fe095b6c4e4037e5c1d5917f9338e7633cae680356",
-    "https://esm.sh/png-chunks-extract@1.0.0": "da06bbd3c08199d72ab16354abe5ffd2361cb891ccbb44d757a0a0a4fbfa12b5",
    "https://esm.sh/react-dom@18.2.0/client?external=react&dev": "2eb39c339720d727591fd55fb44ffcb6f14b06812af0a71e7a2268185b5b6e73",
    "https://esm.sh/react-flip-move@3.0.5?external=react&dev": "4390c0777a0bec583d3e6cb5e4b33831ac937d670d894a20e4f192ce8cd21bae",
    "https://esm.sh/react-intl@6.4.7?external=react&dev": "60e68890e2c5ef3c02d37a89c53e056b1bbd1c8467a7aae62f0b634abc7a8a5f",
@ -333,6 +332,7 @@
    "https://esm.sh/v133/@remix-run/router@1.9.0/X-ZS9yZWFjdA/denonext/router.development.mjs": "97f96f031a7298b0afbbadb23177559ee9b915314d7adf0b9c6a8d7f452c70e7",
    "https://esm.sh/v133/client-only@0.0.1/X-ZS9yZWFjdA/denonext/client-only.development.mjs": "b7efaef2653f7f628d084e24a4d5a857c9cd1a5e5060d1d9957e185ee98c8a28",
    "https://esm.sh/v133/crc-32@0.3.0/denonext/crc-32.mjs": "92bbd96cd5a92e45267cf4b3d3928b9355d16963da1ba740637fb10f1daca590",
+    "https://esm.sh/v133/exifreader@4.14.1/denonext/exifreader.mjs": "691e1c1d1337ccaf092bf39115fdac56bf69e93259d04bb03985e918202317ab",
    "https://esm.sh/v133/file-type@18.5.0/denonext/file-type.mjs": "785cac1bc363448647871e1b310422e01c9cfb7b817a68690710b786d3598311",
    "https://esm.sh/v133/hoist-non-react-statics@3.3.2/X-ZS9yZWFjdA/denonext/hoist-non-react-statics.development.mjs": "fcafac9e3c33810f18ecb43dfc32ce80efc88adc63d3f49fd7ada0665d2146c6",
    "https://esm.sh/v133/ieee754@1.2.1/denonext/ieee754.mjs": "9ec2806065f50afcd4cf3f3f2f38d93e777a92a5954dda00d219f57d4b24832f",
--- a/ui/AdminsPage.tsx
+++ b/ui/AdminsPage.tsx
@ -48,14 +48,20 @@ export function AdminsPage(props: { sessionId: string | null }) {

      {getAdmins.data?.length
        ? (
-          <ul className="my-4 flex flex-col gap-2">
+          <ul className="flex flex-col gap-2">
            {getAdmins.data.map((admin) => (
              <AdminListItem key={admin.id} admin={admin} sessionId={sessionId} />
            ))}
          </ul>
        )
        : getAdmins.data?.length === 0
-        ? <p>No admins</p>
+        ? (
+          <ul className="flex flex-col gap-2">
+            <li className="flex flex-col gap-2 rounded-md bg-zinc-100 dark:bg-zinc-800 p-2">
+              <p key="no-admins" className="text-center text-gray-500">No admins.</p>
+            </li>
+          </ul>
+        )
        : getAdmins.error
        ? <p className="alert">Loading admins failed</p>
        : <div className="spinner self-center" />}
--- a/ui/QueuePage.tsx
+++ b/ui/QueuePage.tsx
@ -15,36 +15,43 @@ export function QueuePage() {
  return (
    <FlipMove
      typeName={"ul"}
-      className="flex flex-col items-stretch gap-2 p-2 bg-zinc-200 dark:bg-zinc-800 rounded-xl"
+      className="flex flex-col items-stretch gap-2 p-2 bg-zinc-200 dark:bg-zinc-800 rounded-md"
      enterAnimation="fade"
      leaveAnimation="fade"
    >
-      {getJobs.data?.map((job) => (
+      {getJobs.data && getJobs.data.length === 0
+        ? <li key="no-jobs" className="text-center text-gray-500">Queue is empty.</li>
+        : (
+          getJobs.data?.map((job) => (
            <li
-          className="flex items-baseline gap-2 bg-zinc-100 dark:bg-zinc-700 px-2 py-1 rounded-xl"
+              className="flex items-baseline gap-2 bg-zinc-100 dark:bg-zinc-700 px-2 py-1 rounded-md"
              key={job.id.join("/")}
            >
-          <span className="">
-            {job.place}.
-          </span>
+              <span className="">{job.place}.</span>
              <span>{getFlagEmoji(job.state.from.language_code)}</span>
              <strong>{job.state.from.first_name} {job.state.from.last_name}</strong>
              {job.state.from.username
                ? (
-              <a className="link" href={`https://t.me/${job.state.from.username}`} target="_blank">
+                  <a
+                    className="link"
+                    href={`https://t.me/${job.state.from.username}`}
+                    target="_blank"
+                  >
                    @{job.state.from.username}
                  </a>
                )
                : null}
              <span className="flex-grow self-center h-full">
-            {job.state.progress != null &&
-              <Progress className="w-full h-full" value={job.state.progress} />}
+                {job.state.progress != null && (
+                  <Progress className="w-full h-full" value={job.state.progress} />
+                )}
              </span>
              <span className="text-sm text-zinc-500 dark:text-zinc-400">
                {job.state.workerInstanceKey}
              </span>
            </li>
-      ))}
+          ))
+        )}
    </FlipMove>
  );
 }
--- a/ui/WorkersPage.tsx
+++ b/ui/WorkersPage.tsx
@ -31,14 +31,20 @@ export function WorkersPage(props: { sessionId: string | null }) {
    <>
      {getWorkers.data?.length
        ? (
-          <ul className="my-4 flex flex-col gap-2">
+          <ul className="flex flex-col gap-2">
            {getWorkers.data?.map((worker) => (
              <WorkerListItem key={worker.id} worker={worker} sessionId={sessionId} />
            ))}
          </ul>
        )
        : getWorkers.data?.length === 0
-        ? <p>No workers</p>
+        ? (
+          <ul className="flex flex-col gap-2">
+            <li className="flex flex-col gap-2 rounded-md bg-zinc-100 dark:bg-zinc-800 p-2">
+              <p key="no-workers" className="text-center text-gray-500">No workers.</p>
+            </li>
+          </ul>
+        )
        : getWorkers.error
        ? <p className="alert">Loading workers failed</p>
        : <div className="spinner self-center" />}
Author	SHA1	Message	Date
nameless	2f62c17e32	review fixes	2023-11-12 02:33:35 +00:00
nameless	602a63f2c9	feat: user daily stats	2023-11-11 04:13:26 +03:00
nameless	8ced57c175	refactor: move sendChatAction resolves pinks/eris#18	2023-11-11 04:13:26 +03:00
nameless	d22848691c	feat: remove url from error	2023-11-11 04:13:26 +03:00
nameless	af7b370321	feat: tagcount endpoint removes tagCountMap from user endpoint, returns it only via admin-only endpoint	2023-11-11 04:13:26 +03:00
nameless	5b61576315	fix: withUser -> withAdmin	2023-11-11 04:13:26 +03:00
nameless	e270a3ab1f	refactor: optimize jobs endpoint output	2023-11-11 04:13:26 +03:00
nameless	84ad11b709	refactor: webui adjustments - remove unnecessary margin in admins and workers page - use rounded-md instead of rounded-xl for queue page - output "No worker/admin/jobs" as item in a list	2023-11-11 04:13:26 +03:00
nameless	ac63e39373	feat: admin priority ignore queue limits and assign higher priority on admin jobs	2023-11-11 04:13:26 +03:00
nameless	73810d3204	fix: use exifreader for png info - replace pnginfo module - allow jpeg for tag extraction - rename pnginfo to getprompt	2023-11-11 04:13:26 +03:00
nameless	f4e7b5e1a7	change txt2img seed behaviour will ignore seed only on /txt2img or new requests	2023-11-11 04:13:15 +03:00