use import map instead of deps.ts

This commit is contained in:
pinks 2023-09-24 15:08:35 +02:00
parent aa380c63fb
commit ff5320bcf4
15 changed files with 141 additions and 130 deletions

View File

@ -1,4 +1,4 @@
import { KVFS } from "../deps.ts";
import { KvFs } from "kvfs";
export const db = await Deno.openKv("./app.db");
export const fs = new KVFS.KvFs(db);
export const fs = new KvFs(db);

View File

@ -1,25 +1,23 @@
import { promiseState } from "async";
import { Chat, Message, User } from "grammy_types";
import { JobData, Queue, Worker } from "kvmq";
import createOpenApiClient from "openapi_fetch";
import { delay } from "std/async";
import { decode, encode } from "std/encoding/base64";
import { getLogger } from "std/log";
import { ulid } from "ulid";
import { bot } from "../bot/mod.ts";
import { SdError } from "../sd/SdError.ts";
import { PngInfo } from "../sd/parsePngInfo.ts";
import * as SdApi from "../sd/sdApi.ts";
import { formatOrdinal } from "../utils/formatOrdinal.ts";
import { formatUserChat } from "../utils/formatUserChat.ts";
import { getConfig, SdInstanceData } from "./config.ts";
import { db, fs } from "./db.ts";
import { SdGenerationInfo } from "./generationStore.ts";
import {
Async,
AsyncX,
Base64,
createOpenApiClient,
GrammyTypes,
KVMQ,
Log,
ULID,
} from "../deps.ts";
import { formatOrdinal } from "../utils/formatOrdinal.ts";
import { SdError } from "../sd/SdError.ts";
import { uploadQueue } from "./uploadQueue.ts";
const logger = () => Log.getLogger();
const logger = () => getLogger();
interface GenerationJob {
task:
@ -32,17 +30,17 @@ interface GenerationJob {
params: Partial<PngInfo>;
fileId: string;
};
from: GrammyTypes.User;
chat: GrammyTypes.Chat;
requestMessage: GrammyTypes.Message;
replyMessage: GrammyTypes.Message;
from: User;
chat: Chat;
requestMessage: Message;
replyMessage: Message;
sdInstanceId?: string;
progress?: number;
}
export const generationQueue = new KVMQ.Queue<GenerationJob>(db, "jobQueue");
export const generationQueue = new Queue<GenerationJob>(db, "jobQueue");
export const activeGenerationWorkers = new Map<string, KVMQ.Worker<GenerationJob>>();
export const activeGenerationWorkers = new Map<string, Worker<GenerationJob>>();
/**
* Initializes queue workers for each SD instance when they become online.
@ -104,7 +102,7 @@ export async function processGenerationQueue() {
activeGenerationWorkers.set(sdInstance.id, newWorker);
logger().info(`Started worker ${sdInstance.id}`);
}
await Async.delay(60_000);
await delay(60_000);
}
}
@ -113,7 +111,7 @@ export async function processGenerationQueue() {
*/
async function processGenerationJob(
state: GenerationJob,
updateJob: (job: Partial<KVMQ.JobData<GenerationJob>>) => Promise<void>,
updateJob: (job: Partial<JobData<GenerationJob>>) => Promise<void>,
sdInstance: SdInstanceData,
) {
const startDate = new Date();
@ -185,7 +183,7 @@ async function processGenerationJob(
? state.task.params.negative_prompt
: config.defaultParams?.negative_prompt,
init_images: [
Base64.encode(
encode(
await fetch(
`https://api.telegram.org/file/bot${bot.token}/${await bot.api.getFile(
state.task.fileId,
@ -229,8 +227,8 @@ async function processGenerationJob(
{ maxAttempts: 1 },
).catch(() => undefined);
await Promise.race([Async.delay(3000), responsePromise]).catch(() => undefined);
} while (await AsyncX.promiseState(responsePromise) === "pending");
await Promise.race([delay(3000), responsePromise]).catch(() => undefined);
} while (await promiseState(responsePromise) === "pending");
// check response
const response = await responsePromise;
@ -247,8 +245,8 @@ async function processGenerationJob(
// save images to db
const imageKeys: Deno.KvKey[] = [];
for (const imageBase64 of response.data.images) {
const imageBuffer = Base64.decode(imageBase64);
const imageKey = ["images", "upload", ULID.ulid()];
const imageBuffer = decode(imageBase64);
const imageKey = ["images", "upload", ulid()];
await fs.set(imageKey, imageBuffer, { expireIn: 30 * 60 * 1000 });
imageKeys.push(imageKey);
}
@ -301,6 +299,6 @@ export async function updateGenerationQueue() {
{ maxAttempts: 1 },
).catch(() => undefined);
}
await Async.delay(3000);
await delay(3000);
}
}

View File

@ -1,9 +1,10 @@
import { GrammyTypes, IKV } from "../deps.ts";
import { Chat, User } from "grammy_types";
import { Store } from "indexed_kv";
import { db } from "./db.ts";
export interface GenerationSchema {
from: GrammyTypes.User;
chat: GrammyTypes.Chat;
from: User;
chat: Chat;
sdInstanceId?: string;
info?: SdGenerationInfo;
startDate?: Date;
@ -49,7 +50,7 @@ type GenerationIndices = {
chatId: number;
};
export const generationStore = new IKV.Store<GenerationSchema, GenerationIndices>(
export const generationStore = new Store<GenerationSchema, GenerationIndices>(
db,
"generations",
{

View File

@ -1,16 +1,22 @@
import { fileTypeFromBuffer } from "file_type";
import { InputFile, InputMediaBuilder } from "grammy";
import { bold, fmt } from "grammy_parse_mode";
import { Chat, Message, User } from "grammy_types";
import { Queue } from "kvmq";
import { format } from "std/fmt/duration";
import { getLogger } from "std/log";
import { bot } from "../bot/mod.ts";
import { formatUserChat } from "../utils/formatUserChat.ts";
import { db, fs } from "./db.ts";
import { generationStore, SdGenerationInfo } from "./generationStore.ts";
import { FileType, FmtDuration, Grammy, GrammyParseMode, GrammyTypes, KVMQ, Log } from "../deps.ts";
const logger = () => Log.getLogger();
const logger = () => getLogger();
interface UploadJob {
from: GrammyTypes.User;
chat: GrammyTypes.Chat;
requestMessage: GrammyTypes.Message;
replyMessage: GrammyTypes.Message;
from: User;
chat: Chat;
requestMessage: Message;
replyMessage: Message;
sdInstanceId: string;
startDate: Date;
endDate: Date;
@ -18,7 +24,7 @@ interface UploadJob {
info: SdGenerationInfo;
}
export const uploadQueue = new KVMQ.Queue<UploadJob>(db, "uploadQueue");
export const uploadQueue = new Queue<UploadJob>(db, "uploadQueue");
/**
* Initializes queue worker for uploading images to Telegram.
@ -37,7 +43,6 @@ export async function processUploadQueue() {
// const detailedReply = Object.keys(job.value.params).filter((key) => key !== "prompt").length > 0;
const detailedReply = true;
const jobDurationMs = Math.trunc((Date.now() - state.startDate.getTime()) / 1000) * 1000;
const { bold, fmt } = GrammyParseMode;
const caption = fmt([
`${state.info.prompt}\n`,
...detailedReply
@ -51,7 +56,7 @@ export async function processUploadQueue() {
fmt`${bold("Seed:")} ${state.info.seed}, `,
fmt`${bold("Size")}: ${state.info.width}x${state.info.height}, `,
fmt`${bold("Worker")}: ${state.sdInstanceId}, `,
fmt`${bold("Time taken")}: ${FmtDuration.format(jobDurationMs, { ignoreZero: true })}`,
fmt`${bold("Time taken")}: ${format(jobDurationMs, { ignoreZero: true })}`,
]
: [],
]);
@ -61,10 +66,10 @@ export async function processUploadQueue() {
state.imageKeys.map(async (fileKey, idx) => {
const imageBuffer = await fs.get(fileKey).then((entry) => entry.value);
if (!imageBuffer) throw new Error("File not found");
const imageType = await FileType.fileTypeFromBuffer(imageBuffer);
const imageType = await fileTypeFromBuffer(imageBuffer);
if (!imageType) throw new Error("Image has unknown type");
return Grammy.InputMediaBuilder.photo(
new Grammy.InputFile(imageBuffer, `image${idx}.${imageType.ext}`),
return InputMediaBuilder.photo(
new InputFile(imageBuffer, `image${idx}.${imageType.ext}`),
// if it can fit, add caption for first photo
idx === 0 && caption.text.length <= 1024
? { caption: caption.text, caption_entities: caption.entities }

View File

@ -1,7 +1,7 @@
import { generationQueue } from "../app/generationQueue.ts";
import { Context } from "./mod.ts";
import { ErisContext } from "./mod.ts";
export async function cancelCommand(ctx: Context) {
export async function cancelCommand(ctx: ErisContext) {
const jobs = await generationQueue.getAllJobs();
const userJobs = jobs
.filter((job) => job.lockUntil < new Date())

View File

@ -1,11 +1,13 @@
import { Collections, Grammy, GrammyStatelessQ } from "../deps.ts";
import { formatUserChat } from "../utils/formatUserChat.ts";
import { parsePngInfo, PngInfo } from "../sd/parsePngInfo.ts";
import { Context, logger } from "./mod.ts";
import { generationQueue } from "../app/generationQueue.ts";
import { CommandContext } from "grammy";
import { StatelessQuestion } from "grammy_stateless_question";
import { maxBy } from "std/collections";
import { getConfig } from "../app/config.ts";
import { generationQueue } from "../app/generationQueue.ts";
import { parsePngInfo, PngInfo } from "../sd/parsePngInfo.ts";
import { formatUserChat } from "../utils/formatUserChat.ts";
import { ErisContext, logger } from "./mod.ts";
export const img2imgQuestion = new GrammyStatelessQ.StatelessQuestion<Context>(
export const img2imgQuestion = new StatelessQuestion<ErisContext>(
"img2img",
async (ctx, state) => {
// todo: also save original image size in state
@ -13,12 +15,12 @@ export const img2imgQuestion = new GrammyStatelessQ.StatelessQuestion<Context>(
},
);
export async function img2imgCommand(ctx: Grammy.CommandContext<Context>) {
export async function img2imgCommand(ctx: CommandContext<ErisContext>) {
await img2img(ctx, ctx.match, true);
}
async function img2img(
ctx: Context,
ctx: ErisContext,
match: string | undefined,
includeRepliedTo: boolean,
fileId?: string,
@ -57,7 +59,7 @@ async function img2img(
if (includeRepliedTo && repliedToMsg?.photo) {
const photos = repliedToMsg.photo;
const biggestPhoto = Collections.maxBy(photos, (p) => p.width * p.height);
const biggestPhoto = maxBy(photos, (p) => p.width * p.height);
if (!biggestPhoto) throw new Error("Message was a photo but had no photos?");
fileId = biggestPhoto.file_id;
params.width = biggestPhoto.width;
@ -66,7 +68,7 @@ async function img2img(
if (ctx.message.photo) {
const photos = ctx.message.photo;
const biggestPhoto = Collections.maxBy(photos, (p) => p.width * p.height);
const biggestPhoto = maxBy(photos, (p) => p.width * p.height);
if (!biggestPhoto) throw new Error("Message was a photo but had no photos?");
fileId = biggestPhoto.file_id;
params.width = biggestPhoto.width;

View File

@ -1,52 +1,53 @@
import { Grammy, GrammyAutoQuote, GrammyFiles, GrammyParseMode, Log } from "../deps.ts";
import { Api, Bot, Context, RawApi, session, SessionFlavor } from "grammy";
import { autoQuote } from "grammy_autoquote";
import { FileFlavor, hydrateFiles } from "grammy_files";
import { hydrateReply, ParseModeFlavor } from "grammy_parse_mode";
import { getLogger } from "std/log";
import { getConfig, setConfig } from "../app/config.ts";
import { formatUserChat } from "../utils/formatUserChat.ts";
import { cancelCommand } from "./cancelCommand.ts";
import { img2imgCommand, img2imgQuestion } from "./img2imgCommand.ts";
import { pnginfoCommand, pnginfoQuestion } from "./pnginfoCommand.ts";
import { queueCommand } from "./queueCommand.ts";
import { txt2imgCommand, txt2imgQuestion } from "./txt2imgCommand.ts";
import { pnginfoCommand, pnginfoQuestion } from "./pnginfoCommand.ts";
import { img2imgCommand, img2imgQuestion } from "./img2imgCommand.ts";
import { cancelCommand } from "./cancelCommand.ts";
import { getConfig, setConfig } from "../app/config.ts";
export const logger = () => Log.getLogger();
export const logger = () => getLogger();
interface SessionData {
chat: ChatData;
user: UserData;
chat: ErisChatData;
user: ErisUserData;
}
interface ChatData {
interface ErisChatData {
language?: string;
}
interface UserData {
interface ErisUserData {
params?: Record<string, string>;
}
export type Context =
& GrammyFiles.FileFlavor<GrammyParseMode.ParseModeFlavor<Grammy.Context>>
& Grammy.SessionFlavor<SessionData>;
export type ErisContext =
& FileFlavor<ParseModeFlavor<Context>>
& SessionFlavor<SessionData>;
type WithRetryApi<T extends Grammy.RawApi> = {
type WithRetryApi<T extends RawApi> = {
[M in keyof T]: T[M] extends (args: infer P, ...rest: infer A) => infer R
? (args: P extends object ? P & { maxAttempts?: number; maxWait?: number } : P, ...rest: A) => R
: T[M];
};
type Api = Grammy.Api<WithRetryApi<Grammy.RawApi>>;
type ErisApi = Api<WithRetryApi<RawApi>>;
export const bot = new Grammy.Bot<Context, Api>(
export const bot = new Bot<ErisContext, ErisApi>(
Deno.env.get("TG_BOT_TOKEN")!,
{
client: { timeoutSeconds: 20 },
},
);
bot.use(GrammyAutoQuote.autoQuote);
bot.use(GrammyParseMode.hydrateReply);
bot.use(Grammy.session<
SessionData,
Grammy.Context & Grammy.SessionFlavor<SessionData>
>({
bot.use(autoQuote);
bot.use(hydrateReply);
bot.use(session<SessionData, ErisContext>({
type: "multi",
chat: {
initial: () => ({}),
@ -57,7 +58,7 @@ bot.use(Grammy.session<
},
}));
bot.api.config.use(GrammyFiles.hydrateFiles(bot.token));
bot.api.config.use(hydrateFiles(bot.token));
// Automatically retry bot requests if we get a "too many requests" or telegram internal error
bot.api.config.use(async (prev, method, payload, signal) => {

View File

@ -1,19 +1,21 @@
import { Grammy, GrammyParseMode, GrammyStatelessQ } from "../deps.ts";
import { CommandContext } from "grammy";
import { bold, fmt } from "grammy_parse_mode";
import { StatelessQuestion } from "grammy_stateless_question";
import { getPngInfo, parsePngInfo } from "../sd/parsePngInfo.ts";
import { Context } from "./mod.ts";
import { ErisContext } from "./mod.ts";
export const pnginfoQuestion = new GrammyStatelessQ.StatelessQuestion<Context>(
export const pnginfoQuestion = new StatelessQuestion<ErisContext>(
"pnginfo",
async (ctx) => {
await pnginfo(ctx, false);
},
);
export async function pnginfoCommand(ctx: Grammy.CommandContext<Context>) {
export async function pnginfoCommand(ctx: CommandContext<ErisContext>) {
await pnginfo(ctx, true);
}
async function pnginfo(ctx: Context, includeRepliedTo: boolean): Promise<void> {
async function pnginfo(ctx: ErisContext, includeRepliedTo: boolean): Promise<void> {
const document = ctx.message?.document ||
(includeRepliedTo ? ctx.message?.reply_to_message?.document : undefined);
@ -30,8 +32,6 @@ async function pnginfo(ctx: Context, includeRepliedTo: boolean): Promise<void> {
const buffer = await fetch(file.getUrl()).then((resp) => resp.arrayBuffer());
const params = parsePngInfo(getPngInfo(new Uint8Array(buffer)) ?? "");
const { bold, fmt } = GrammyParseMode;
const paramsText = fmt([
`${params.prompt}\n`,
params.negative_prompt ? fmt`${bold("Negative prompt:")} ${params.negative_prompt}\n` : "",

View File

@ -1,10 +1,11 @@
import { Grammy, GrammyParseMode } from "../deps.ts";
import { Context } from "./mod.ts";
import { getFlagEmoji } from "../utils/getFlagEmoji.ts";
import { activeGenerationWorkers, generationQueue } from "../app/generationQueue.ts";
import { CommandContext } from "grammy";
import { bold, fmt } from "grammy_parse_mode";
import { getConfig } from "../app/config.ts";
import { activeGenerationWorkers, generationQueue } from "../app/generationQueue.ts";
import { getFlagEmoji } from "../utils/getFlagEmoji.ts";
import { ErisContext } from "./mod.ts";
export async function queueCommand(ctx: Grammy.CommandContext<Context>) {
export async function queueCommand(ctx: CommandContext<ErisContext>) {
let formattedMessage = await getMessageText();
const queueMessage = await ctx.replyFmt(formattedMessage, { disable_notification: true });
handleFutureUpdates().catch(() => undefined);
@ -18,7 +19,6 @@ export async function queueCommand(ctx: Grammy.CommandContext<Context>) {
.filter((job) => job.lockUntil <= new Date())
.map((job, index) => ({ ...job, index: index + 1 }));
const jobs = [...processingJobs, ...waitingJobs];
const { bold, fmt } = GrammyParseMode;
return fmt([
"Current queue:\n",

View File

@ -1,11 +1,12 @@
import { Grammy, GrammyStatelessQ } from "../deps.ts";
import { formatUserChat } from "../utils/formatUserChat.ts";
import { getPngInfo, parsePngInfo, PngInfo } from "../sd/parsePngInfo.ts";
import { Context, logger } from "./mod.ts";
import { generationQueue } from "../app/generationQueue.ts";
import { CommandContext } from "grammy";
import { StatelessQuestion } from "grammy_stateless_question";
import { getConfig } from "../app/config.ts";
import { generationQueue } from "../app/generationQueue.ts";
import { getPngInfo, parsePngInfo, PngInfo } from "../sd/parsePngInfo.ts";
import { formatUserChat } from "../utils/formatUserChat.ts";
import { ErisContext, logger } from "./mod.ts";
export const txt2imgQuestion = new GrammyStatelessQ.StatelessQuestion<Context>(
export const txt2imgQuestion = new StatelessQuestion<ErisContext>(
"txt2img",
async (ctx) => {
if (!ctx.message.text) return;
@ -13,11 +14,11 @@ export const txt2imgQuestion = new GrammyStatelessQ.StatelessQuestion<Context>(
},
);
export async function txt2imgCommand(ctx: Grammy.CommandContext<Context>) {
export async function txt2imgCommand(ctx: CommandContext<ErisContext>) {
await txt2img(ctx, ctx.match, true);
}
async function txt2img(ctx: Context, match: string, includeRepliedTo: boolean): Promise<void> {
async function txt2img(ctx: ErisContext, match: string, includeRepliedTo: boolean): Promise<void> {
if (!ctx.message?.from?.id) {
await ctx.reply("I don't know who you are");
return;

View File

@ -4,5 +4,27 @@
},
"fmt": {
"lineWidth": 100
},
"imports": {
"std/log": "https://deno.land/std@0.201.0/log/mod.ts",
"std/async": "https://deno.land/std@0.201.0/async/mod.ts",
"std/fmt/duration": "https://deno.land/std@0.202.0/fmt/duration.ts",
"std/collections": "https://deno.land/std@0.202.0/collections/mod.ts",
"std/encoding/base64": "https://deno.land/std@0.202.0/encoding/base64.ts",
"async": "https://deno.land/x/async@v2.0.2/mod.ts",
"ulid": "https://deno.land/x/ulid@v0.3.0/mod.ts",
"indexed_kv": "https://deno.land/x/indexed_kv@v0.4.0/mod.ts",
"kvmq": "https://deno.land/x/kvmq@v0.2.0/mod.ts",
"kvfs": "https://deno.land/x/kvfs@v0.1.0/mod.ts",
"grammy": "https://lib.deno.dev/x/grammy@1/mod.ts",
"grammy_types": "https://lib.deno.dev/x/grammy_types@3/mod.ts",
"grammy_autoquote": "https://lib.deno.dev/x/grammy_autoquote@1/mod.ts",
"grammy_parse_mode": "https://lib.deno.dev/x/grammy_parse_mode@1/mod.ts",
"grammy_stateless_question": "https://lib.deno.dev/x/grammy_stateless_question_alpha@3/mod.ts",
"grammy_files": "https://lib.deno.dev/x/grammy_files@1/mod.ts",
"file_type": "https://esm.sh/file-type@18.5.0",
"png_chunks_extract": "https://esm.sh/png-chunks-extract@1.0.0",
"png_chunk_text": "https://esm.sh/png-chunk-text@1.0.0",
"openapi_fetch": "https://esm.sh/openapi-fetch@0.7.6"
}
}

20
deps.ts
View File

@ -1,20 +0,0 @@
export * as Log from "https://deno.land/std@0.201.0/log/mod.ts";
export * as Async from "https://deno.land/std@0.201.0/async/mod.ts";
export * as FmtDuration from "https://deno.land/std@0.202.0/fmt/duration.ts";
export * as Collections from "https://deno.land/std@0.202.0/collections/mod.ts";
export * as Base64 from "https://deno.land/std@0.202.0/encoding/base64.ts";
export * as AsyncX from "https://deno.land/x/async@v2.0.2/mod.ts";
export * as ULID from "https://deno.land/x/ulid@v0.3.0/mod.ts";
export * as IKV from "https://deno.land/x/indexed_kv@v0.4.0/mod.ts";
export * as KVMQ from "https://deno.land/x/kvmq@v0.2.0/mod.ts";
export * as KVFS from "https://deno.land/x/kvfs@v0.1.0/mod.ts";
export * as Grammy from "https://deno.land/x/grammy@v1.18.3/mod.ts";
export * as GrammyTypes from "https://deno.land/x/grammy_types@v3.2.2/mod.ts";
export * as GrammyAutoQuote from "https://deno.land/x/grammy_autoquote@v1.1.2/mod.ts";
export * as GrammyParseMode from "https://deno.land/x/grammy_parse_mode@1.8.1/mod.ts";
export * as GrammyStatelessQ from "https://deno.land/x/grammy_stateless_question_alpha@v3.0.4/mod.ts";
export * as GrammyFiles from "https://deno.land/x/grammy_files@v1.0.4/mod.ts";
export * as FileType from "https://esm.sh/file-type@18.5.0";
export { default as pngChunksExtract } from "https://esm.sh/png-chunks-extract@1.0.0";
export { decode as pngChunkTextDecode } from "https://esm.sh/png-chunk-text@1.0.0";
export { default as createOpenApiClient } from "https://esm.sh/openapi-fetch@0.7.6";

View File

@ -1,11 +1,11 @@
import "https://deno.land/std@0.201.0/dotenv/load.ts";
import { Log } from "./deps.ts";
import { bot } from "./bot/mod.ts";
import { handlers, setup } from "std/log";
import { runAllTasks } from "./app/mod.ts";
import { bot } from "./bot/mod.ts";
Log.setup({
setup({
handlers: {
console: new Log.handlers.ConsoleHandler("DEBUG"),
console: new handlers.ConsoleHandler("DEBUG"),
},
loggers: {
default: { level: "DEBUG", handlers: ["console"] },

View File

@ -1,9 +1,10 @@
import { pngChunksExtract, pngChunkTextDecode } from "../deps.ts";
import { decode } from "png_chunk_text";
import extractChunks from "png_chunks_extract";
export function getPngInfo(pngData: Uint8Array): string | undefined {
return pngChunksExtract(pngData)
return extractChunks(pngData)
.filter((chunk) => chunk.name === "tEXt")
.map((chunk) => pngChunkTextDecode(chunk.data))
.map((chunk) => decode(chunk.data))
.find((textChunk) => textChunk.keyword === "parameters")
?.text;
}

View File

@ -1,7 +1,7 @@
import { GrammyTypes } from "../deps.ts";
import { Chat, User } from "grammy_types";
export function formatUserChat(
ctx: { from?: GrammyTypes.User; chat?: GrammyTypes.Chat; sdInstanceId?: string },
ctx: { from?: User; chat?: Chat; sdInstanceId?: string },
) {
const msg: string[] = [];
if (ctx.from) {