From 5a7ba81f462401a1282778e5800f111b1cd65be2 Mon Sep 17 00:00:00 2001 From: pinks Date: Mon, 11 Sep 2023 22:48:38 +0200 Subject: [PATCH] retry optimizations --- bot/mod.ts | 6 ++-- sd.ts | 2 +- tasks/processJobs.ts | 75 +++++++++++++++++++++++++++++--------------- 3 files changed, 55 insertions(+), 28 deletions(-) diff --git a/bot/mod.ts b/bot/mod.ts index befe43a..1426a19 100644 --- a/bot/mod.ts +++ b/bot/mod.ts @@ -40,7 +40,7 @@ bot.api.config.use(async (prev, method, payload, signal) => { return result; } logger().warning( - `Retrying ${method} after attempt ${attempt} failed with ${result.error_code} error`, + `${method} (attempt ${attempt}) failed: ${result.error_code} ${result.description}`, ); const retryAfterMs = (result.parameters?.retry_after ?? (attempt * 5)) * 1000; await new Promise((resolve) => setTimeout(resolve, retryAfterMs)); @@ -48,7 +48,9 @@ bot.api.config.use(async (prev, method, payload, signal) => { }); bot.catch((err) => { - logger().error(`Handling update from ${formatUserChat(err.ctx)} failed: ${err}`); + logger().error( + `Handling update from ${formatUserChat(err.ctx)} failed: ${err.name} ${err.message}`, + ); }); // if error happened, try to reply to the user with the error diff --git a/sd.ts b/sd.ts index ff290da..9e30a27 100644 --- a/sd.ts +++ b/sd.ts @@ -52,7 +52,7 @@ export async function sdTxt2Img( try { while (true) { - await Async.abortable(Promise.race([request, Async.delay(3000)]), signal); + await Async.abortable(Promise.race([request, Async.delay(4000)]), signal); if (await AsyncX.promiseState(request) !== "pending") return await request; onProgress?.(await fetchSdApi(api, "sdapi/v1/progress")); } diff --git a/tasks/processJobs.ts b/tasks/processJobs.ts index 1cfff8f..45b8f87 100644 --- a/tasks/processJobs.ts +++ b/tasks/processJobs.ts @@ -1,4 +1,14 @@ -import { Base64, FileType, FmtDuration, Grammy, GrammyParseMode, IKV, Log } from "../deps.ts"; +import { + Async, + Base64, + FileType, + FmtDuration, + Grammy, + GrammyParseMode, + GrammyTypes, + IKV, + Log, +} from "../deps.ts"; import { bot } from "../bot/mod.ts"; import { getGlobalSession, GlobalData, WorkerData } from "../bot/session.ts"; import { fmt, formatUserChat } from "../utils.ts"; @@ -43,7 +53,7 @@ export async function processJobs(): Promise { if (err instanceof Grammy.GrammyError || err instanceof SdApiError) { await bot.api.sendMessage( job.value.request.chat.id, - `Failed to generate your prompt: ${err.message}`, + `Failed to generate your prompt using ${worker.name}: ${err.message}`, { reply_to_message_id: job.value.request.message_id }, ).catch(() => undefined); await job.update({ status: { type: "waiting" } }).catch(() => undefined); @@ -141,7 +151,6 @@ async function processJob(job: IKV.Model, worker: WorkerData, config: job.value.reply.chat.id, job.value.reply.message_id, `Uploading your images...`, - { maxAttempts: 1 }, ).catch(() => undefined); } @@ -168,31 +177,45 @@ async function processJob(job: IKV.Model, worker: WorkerData, config: : [], ]); - // parse files from reply JSON - const inputFiles = await Promise.all( - response.images.map(async (imageBase64, idx) => { - const imageBuffer = Base64.decode(imageBase64); - const imageType = await FileType.fileTypeFromBuffer(imageBuffer); - if (!imageType) throw new Error("Unknown file type returned from worker"); - return Grammy.InputMediaBuilder.photo( - new Grammy.InputFile(imageBuffer, `image${idx}.${imageType.ext}`), - // if it can fit, add caption for first photo - idx === 0 && caption.text.length <= 1024 - ? { caption: caption.text, caption_entities: caption.entities } - : undefined, - ); - }), - ); + let sendMediaAttempt = 0; + let resultMessages: GrammyTypes.Message.MediaMessage[] | undefined; + while (true) { + sendMediaAttempt++; + + // parse files from reply JSON + const inputFiles = await Promise.all( + response.images.map(async (imageBase64, idx) => { + const imageBuffer = Base64.decode(imageBase64); + const imageType = await FileType.fileTypeFromBuffer(imageBuffer); + if (!imageType) throw new Error("Unknown file type returned from worker"); + return Grammy.InputMediaBuilder.photo( + new Grammy.InputFile(imageBuffer, `image${idx}.${imageType.ext}`), + // if it can fit, add caption for first photo + idx === 0 && caption.text.length <= 1024 + ? { caption: caption.text, caption_entities: caption.entities } + : undefined, + ); + }), + ); + + // send the result to telegram + try { + resultMessages = await bot.api.sendMediaGroup(job.value.request.chat.id, inputFiles, { + reply_to_message_id: job.value.request.message_id, + maxAttempts: 5, + }); + break; + } catch (err) { + logger().warning(`Sending images (attempt ${sendMediaAttempt}) failed: ${err}`); + if (sendMediaAttempt >= 5) throw err; + await Async.delay(15000); + } + } - // send the result to telegram - const resultMessage = await bot.api.sendMediaGroup(job.value.request.chat.id, inputFiles, { - reply_to_message_id: job.value.request.message_id, - maxAttempts: 5, - }); // send caption in separate message if it couldn't fit if (caption.text.length > 1024 && caption.text.length <= 4096) { await bot.api.sendMessage(job.value.request.chat.id, caption.text, { - reply_to_message_id: resultMessage[0].message_id, + reply_to_message_id: resultMessages[0].message_id, entities: caption.entities, }); } @@ -210,7 +233,9 @@ async function processJob(job: IKV.Model, worker: WorkerData, config: status: { type: "done", info: response.info, startDate, endDate: new Date() }, }); logger().debug( - `Job finished for ${formatUserChat(job.value.request)} using ${worker.name}`, + `Job finished for ${formatUserChat(job.value.request)} using ${worker.name}${ + sendMediaAttempt > 1 ? ` after ${sendMediaAttempt} attempts` : "" + }`, ); }