forked from pinks/eris
wait before retrying failed job
This commit is contained in:
parent
ee4c2091f0
commit
9155d513b5
|
@ -21,6 +21,7 @@ export interface JobSchema {
|
|||
| {
|
||||
type: "waiting";
|
||||
message?: GrammyTypes.Message.TextMessage;
|
||||
lastErrorDate?: Date;
|
||||
}
|
||||
| {
|
||||
type: "processing";
|
||||
|
|
|
@ -33,8 +33,12 @@ export async function processJobs(): Promise<never> {
|
|||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
|
||||
try {
|
||||
// get first waiting job
|
||||
const job = await jobStore.getBy("status.type", "waiting").then((jobs) => jobs[0]);
|
||||
const jobs = await jobStore.getBy("status.type", "waiting");
|
||||
// get first waiting job which hasn't errored in last minute
|
||||
const job = jobs.find((job) =>
|
||||
job.value.status.type === "waiting" &&
|
||||
(job.value.status.lastErrorDate?.getTime() ?? 0) < Date.now() - 60_000
|
||||
);
|
||||
if (!job) continue;
|
||||
|
||||
// find a worker to handle the job
|
||||
|
@ -62,13 +66,20 @@ export async function processJobs(): Promise<never> {
|
|||
logger().error(
|
||||
`Job failed for ${formatUserChat(job.value)} via ${worker.id}: ${err}`,
|
||||
);
|
||||
if (job.value.status.type === "processing" && job.value.status.message) {
|
||||
await bot.api.deleteMessage(
|
||||
job.value.status.message.chat.id,
|
||||
job.value.status.message.message_id,
|
||||
).catch(() => undefined);
|
||||
}
|
||||
if (err instanceof Grammy.GrammyError || err instanceof SdApiError) {
|
||||
await bot.api.sendMessage(
|
||||
job.value.chat.id,
|
||||
`Failed to generate your prompt using ${worker.name}: ${err.message}`,
|
||||
{ reply_to_message_id: job.value.requestMessageId },
|
||||
).catch(() => undefined);
|
||||
await job.update({ status: { type: "waiting" } }).catch(() => undefined);
|
||||
await job.update({ status: { type: "waiting", lastErrorDate: new Date() } })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
if (
|
||||
err instanceof SdApiError &&
|
||||
|
@ -169,10 +180,9 @@ async function processJob(job: IKV.Model<JobSchema>, worker: WorkerData, config:
|
|||
const handleProgress = async (progress: SdProgressResponse) => {
|
||||
// Important: don't let any errors escape this function
|
||||
if (job.value.status.type === "processing" && job.value.status.message) {
|
||||
if (job.value.status.progress === progress.progress) return;
|
||||
await Promise.all([
|
||||
bot.api.sendChatAction(job.value.chat.id, "upload_photo", { maxAttempts: 1 }),
|
||||
bot.api.editMessageText(
|
||||
progress.progress > job.value.status.progress && bot.api.editMessageText(
|
||||
job.value.status.message.chat.id,
|
||||
job.value.status.message.message_id,
|
||||
`Generating your prompt now... ${
|
||||
|
|
|
@ -17,7 +17,13 @@ export async function returnHangedJobs(): Promise<never> {
|
|||
// if job wasn't updated for 2 minutes, return it to the queue
|
||||
const timeSinceLastUpdateMs = Date.now() - job.value.status.updatedDate.getTime();
|
||||
if (timeSinceLastUpdateMs > 2 * 60 * 1000) {
|
||||
await job.update({ status: { type: "waiting" } });
|
||||
await job.update((value) => ({
|
||||
...value,
|
||||
status: {
|
||||
type: "waiting",
|
||||
message: value.status.type !== "done" ? value.status.message : undefined,
|
||||
},
|
||||
}));
|
||||
logger().warning(
|
||||
`Job for ${formatUserChat(job.value)} was returned to the queue because it hanged for ${
|
||||
FmtDuration.format(Math.trunc(timeSinceLastUpdateMs / 1000) * 1000, {
|
||||
|
|
Loading…
Reference in New Issue