wait before retrying failed job

db/jobStore.ts

@@ -21,6 +21,7 @@ export interface JobSchema {
     | {
       type: "waiting";
       message?: GrammyTypes.Message.TextMessage;
+      lastErrorDate?: Date;
     }
     | {
       type: "processing";

tasks/processJobs.ts

@@ -33,8 +33,12 @@ export async function processJobs(): Promise<never> {
     await new Promise((resolve) => setTimeout(resolve, 1000));
 
     try {
-      // get first waiting job
+      const jobs = await jobStore.getBy("status.type", "waiting");
-      const job = await jobStore.getBy("status.type", "waiting").then((jobs) => jobs[0]);
+      // get first waiting job which hasn't errored in last minute
+      const job = jobs.find((job) =>
+        job.value.status.type === "waiting" &&
+        (job.value.status.lastErrorDate?.getTime() ?? 0) < Date.now() - 60_000
+      );
       if (!job) continue;
 
       // find a worker to handle the job
@@ -62,13 +66,20 @@ export async function processJobs(): Promise<never> {
           logger().error(
             `Job failed for ${formatUserChat(job.value)} via ${worker.id}: ${err}`,
           );
+          if (job.value.status.type === "processing" && job.value.status.message) {
+            await bot.api.deleteMessage(
+              job.value.status.message.chat.id,
+              job.value.status.message.message_id,
+            ).catch(() => undefined);
+          }
           if (err instanceof Grammy.GrammyError || err instanceof SdApiError) {
             await bot.api.sendMessage(
               job.value.chat.id,
               `Failed to generate your prompt using ${worker.name}: ${err.message}`,
               { reply_to_message_id: job.value.requestMessageId },
             ).catch(() => undefined);
-            await job.update({ status: { type: "waiting" } }).catch(() => undefined);
+            await job.update({ status: { type: "waiting", lastErrorDate: new Date() } })
+              .catch(() => undefined);
           }
           if (
             err instanceof SdApiError &&
@@ -169,10 +180,9 @@ async function processJob(job: IKV.Model<JobSchema>, worker: WorkerData, config:
   const handleProgress = async (progress: SdProgressResponse) => {
     // Important: don't let any errors escape this function
     if (job.value.status.type === "processing" && job.value.status.message) {
-      if (job.value.status.progress === progress.progress) return;
       await Promise.all([
         bot.api.sendChatAction(job.value.chat.id, "upload_photo", { maxAttempts: 1 }),
-        bot.api.editMessageText(
+        progress.progress > job.value.status.progress && bot.api.editMessageText(
           job.value.status.message.chat.id,
           job.value.status.message.message_id,
           `Generating your prompt now... ${

tasks/returnHangedJobs.ts

@@ -17,7 +17,13 @@ export async function returnHangedJobs(): Promise<never> {
         // if job wasn't updated for 2 minutes, return it to the queue
         const timeSinceLastUpdateMs = Date.now() - job.value.status.updatedDate.getTime();
         if (timeSinceLastUpdateMs > 2 * 60 * 1000) {
-          await job.update({ status: { type: "waiting" } });
+          await job.update((value) => ({
+            ...value,
+            status: {
+              type: "waiting",
+              message: value.status.type !== "done" ? value.status.message : undefined,
+            },
+          }));
           logger().warning(
             `Job for ${formatUserChat(job.value)} was returned to the queue because it hanged for ${
               FmtDuration.format(Math.trunc(timeSinceLastUpdateMs / 1000) * 1000, {