pinks
/
eris
1
0
Fork 4
Code Issues 15 Pull Requests 1 Activity

wait before retrying failed job

This commit is contained in:
pinks 2023-09-14 03:23:55 +02:00
parent ee4c2091f0
commit 9155d513b5
3 changed files with 23 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
db/jobStore.ts
View File

@ -21,6 +21,7 @@ export interface JobSchema {
| { | {
type: "waiting"; type: "waiting";
message?: GrammyTypes.Message.TextMessage; message?: GrammyTypes.Message.TextMessage;
lastErrorDate?: Date;
} }
| { | {
type: "processing"; type: "processing";

20
tasks/processJobs.ts
View File

@ -33,8 +33,12 @@ export async function processJobs(): Promise<never> {
await new Promise((resolve) => setTimeout(resolve, 1000)); await new Promise((resolve) => setTimeout(resolve, 1000));
try { try {
// get first waiting job const jobs = await jobStore.getBy("status.type", "waiting");
const job = await jobStore.getBy("status.type", "waiting").then((jobs) => jobs[0]); // get first waiting job which hasn't errored in last minute
const job = jobs.find((job) =>
job.value.status.type === "waiting" &&
(job.value.status.lastErrorDate?.getTime() ?? 0) < Date.now() - 60_000
);
if (!job) continue; if (!job) continue;
// find a worker to handle the job // find a worker to handle the job
@ -62,13 +66,20 @@ export async function processJobs(): Promise<never> {
logger().error( logger().error(
`Job failed for ${formatUserChat(job.value)} via ${worker.id}: ${err}`, `Job failed for ${formatUserChat(job.value)} via ${worker.id}: ${err}`,
); );
if (job.value.status.type === "processing" && job.value.status.message) {
await bot.api.deleteMessage(
job.value.status.message.chat.id,
job.value.status.message.message_id,
).catch(() => undefined);
}
if (err instanceof Grammy.GrammyError || err instanceof SdApiError) { if (err instanceof Grammy.GrammyError || err instanceof SdApiError) {
await bot.api.sendMessage( await bot.api.sendMessage(
job.value.chat.id, job.value.chat.id,
`Failed to generate your prompt using ${worker.name}: ${err.message}`, `Failed to generate your prompt using ${worker.name}: ${err.message}`,
{ reply_to_message_id: job.value.requestMessageId }, { reply_to_message_id: job.value.requestMessageId },
).catch(() => undefined); ).catch(() => undefined);
await job.update({ status: { type: "waiting" } }).catch(() => undefined); await job.update({ status: { type: "waiting", lastErrorDate: new Date() } })
.catch(() => undefined);
} }
if ( if (
err instanceof SdApiError && err instanceof SdApiError &&
@ -169,10 +180,9 @@ async function processJob(job: IKV.Model<JobSchema>, worker: WorkerData, config:
const handleProgress = async (progress: SdProgressResponse) => { const handleProgress = async (progress: SdProgressResponse) => {
// Important: don't let any errors escape this function // Important: don't let any errors escape this function
if (job.value.status.type === "processing" && job.value.status.message) { if (job.value.status.type === "processing" && job.value.status.message) {
if (job.value.status.progress === progress.progress) return;
await Promise.all([ await Promise.all([
bot.api.sendChatAction(job.value.chat.id, "upload_photo", { maxAttempts: 1 }), bot.api.sendChatAction(job.value.chat.id, "upload_photo", { maxAttempts: 1 }),
bot.api.editMessageText( progress.progress > job.value.status.progress && bot.api.editMessageText(
job.value.status.message.chat.id, job.value.status.message.chat.id,
job.value.status.message.message_id, job.value.status.message.message_id,
`Generating your prompt now... ${ `Generating your prompt now... ${

8
tasks/returnHangedJobs.ts
View File

@ -17,7 +17,13 @@ export async function returnHangedJobs(): Promise<never> {
// if job wasn't updated for 2 minutes, return it to the queue // if job wasn't updated for 2 minutes, return it to the queue
const timeSinceLastUpdateMs = Date.now() - job.value.status.updatedDate.getTime(); const timeSinceLastUpdateMs = Date.now() - job.value.status.updatedDate.getTime();
if (timeSinceLastUpdateMs > 2 * 60 * 1000) { if (timeSinceLastUpdateMs > 2 * 60 * 1000) {
await job.update({ status: { type: "waiting" } }); await job.update((value) => ({
...value,
status: {
type: "waiting",
message: value.status.type !== "done" ? value.status.message : undefined,
},
}));
logger().warning( logger().warning(
`Job for ${formatUserChat(job.value)} was returned to the queue because it hanged for ${ `Job for ${formatUserChat(job.value)} was returned to the queue because it hanged for ${
FmtDuration.format(Math.trunc(timeSinceLastUpdateMs / 1000) * 1000, { FmtDuration.format(Math.trunc(timeSinceLastUpdateMs / 1000) * 1000, {