diff --git a/eris.json b/eris.json new file mode 100644 index 0000000..fc39103 --- /dev/null +++ b/eris.json @@ -0,0 +1,282 @@ +{ + "lora": { + "civitai": [ + 8058, + 58390, + 9155, + 21640, + 107279, + 139340, + 4982, + 62342, + 9620, + 100070, + 108007, + 72456, + 151678, + 71596, + 18472, + 61361, + 113807, + 19777, + 19777, + 61442, + 58964, + 5975, + 26449, + 131093, + 15898, + 151686, + 52352, + 91403, + 134737, + 146173, + 28431, + 155045, + 9025, + 78887, + 8821, + 13495, + 151734, + 118352, + 109401, + 131567, + 156270, + 60132, + 128811, + 70618, + 60003, + 113845, + 115008, + 115848, + 27206, + 25360, + 77121, + 102838, + 80837, + 159037, + 10564, + 119172, + 156550, + 24584, + 89285, + 76732, + 150656, + 120887, + 13941, + 13941, + 13941, + 52023, + 150686, + 6976, + 113904, + 62978, + 68641, + 72446, + 150697, + 6433, + 72930, + 127533, + 59880, + 19620, + 116855, + 9272, + 13090, + 151772, + 70572, + 6526, + 114307, + 150706, + 159524, + 53983, + 141555, + 100707, + 123690, + 154996, + 53504, + 8678, + 151785, + 141128, + 12423, + 49021, + 159679, + 71813, + 88678, + 136710, + 150721, + 93201, + 53198, + 115720, + 52886, + 15907, + 115773, + 17824, + 151824, + 150729, + 114794, + 135870, + 87363, + 6610, + 28614, + 151894, + 119389, + 85366, + 48139, + 151921, + 151932, + 136067, + 25175, + 71616, + 116311, + 63999, + 155302, + 159528, + 152232, + 152247, + 8399, + 152293, + 152331, + 159000, + 155294, + 24937, + 101489, + 16775, + 85387, + 128095, + 61442, + 91226, + 152344, + 18299, + 89051, + 152369, + 10668, + 152397, + 114525, + 22615, + 58295, + 51293, + 90903, + 114881, + 88312, + 146171, + 159662, + 15798, + 15798, + 15798, + 132408, + 98229, + 43943, + 7958, + 80808, + 58012, + 82355, + 61442, + 90504, + 159360, + 152865, + 115062, + 141385, + 9630, + 46630, + 158942, + 131399, + 50808, + 114293, + 70591, + 70591, + 94795, + 25236, + 155300, + 115076, + 80910, + 11858, + 152885, + 115080, + 24751, + 150740, + 89196, + 11886, + 107014, + 132696, + 32207, + 116407, + 34372, + 33251, + 114718, + 82132, + 155296, + 150759, + 69323, + 23337, + 22764, + 150784, + 53076, + 30195, + 113762, + 60701, + 63492, + 120152, + 61442, + 61442, + 152893, + 41491, + 26818, + 67983, + 160623, + 142949, + 157025, + 119077, + 160652, + 154971, + 70556, + 75416, + 151659 + ], + "sideload": [ + "fr_female-x-feral_canid_rear-mounting_v1.2_e200.safetensors", + "fr_female-x-feral_canid_rear-mounting_v1.2_e200.preview.png", + "Y1-BadDragon.safetensors", + "Y1-BadDragon.preview.png" + ] + }, + "sd": { + "civitai": [ + 65203, + 5012, + 4108, + 3666, + 144962 + ], + "sideload": [ + "EasyFluffV10-PreRelease2.safetensors", + "EasyFluffV11.2.safetensors", + "fluffyrock-576-704-832-960-1088-lion-low-lr-e219-terminal-snr-e192.safetensors" + ] + }, + "emb": { + "civitai": [ + 16807, + 5224, + 116230, + 142280, + 87781, + 87781, + 87781, + 187118, + 16807, + 187118, + 7808, + 150087, + 56519, + 4629, + 187118, + 4499, + 187118, + 18410 + ], + "sideload": [ + "boring_e621_fluffyrock.pt", + "boring_e621_fluffyrock_v3.pt" + ] + } +} \ No newline at end of file diff --git a/erisscan.py b/erisscan.py new file mode 100644 index 0000000..bdb4861 --- /dev/null +++ b/erisscan.py @@ -0,0 +1,104 @@ +import os +import json +import hashlib +import urllib3 +from pathlib import Path + + +def scanfolder(path: str) -> list: + civitailist = [] + sideloaddict = [] + modelfolder = path + modelfiles = os.listdir(modelfolder) + for modelfile in modelfiles: + modelname = Path(modelfile).stem + sideload = False + if modelfile.endswith('.safetensors') or modelfile.endswith('.pt'): + #print(f"Found '{lorafile}' - Searching civitai.info") + if os.path.exists(f"{modelfolder}{modelname}.civitai.info"): + civitaiinfo = open(f"{modelfolder}{modelname}.civitai.info", "r", encoding='utf-8') + civitaiinfocontent = civitaiinfo.read() + civitaiinfo.close() + if civitaiinfocontent[:2] == "{}": + os.remove(f"{modelfolder}{modelname}.civitai.info") + sideload = True + else: + with open(f"{modelfolder}{modelname}.civitai.info", "r") as f: + #fstring = "dd" + print("Reading", modelfolder, modelfile) + modelinfoarray = json.load(f) + #print(loraarray) + try: + modelsha256 = modelinfoarray.get('files')[0].get('hashes').get('SHA256') + modelid = modelinfoarray.get('modelId') + civitailist.append(modelid) + #print(lorasha256, " " , loramodelid) + except Exception as e: + print("[Model Step local civitai.info parse] ", path, modelfile , " | ",e) + #print("Content:", civitaiinfocontent[:2]) + else: + sideload = True + if sideload: + print(f"{modelfolder}{modelname}.civitai.info not found. Getting SHA256 Hash and fetch data from civitai API") + with open(f"{modelfolder}{modelfile}", "rb") as f: + bytes = f.read() # read entire file as bytes + modelsha256 = hashlib.sha256(bytes).hexdigest(); + #print(readable_hash) + http = urllib3.PoolManager() + url = f"https://civitai.com/api/v1/model-versions/by-hash/{modelsha256}" + resp = http.request('GET', url) + modelinfoarray = json.loads(resp.data.decode('utf-8')) + try: + if modelinfoarray.get('error') == "Model not found": + print(f"Model {modelfile} is not a CivitAi Model") + sideloaddict.append(modelfile) + if os.path.exists(f"{modelfolder}{modelname}.preview.jpg"): + sideloaddict.append(f"{modelname}.preview.jpg") + if os.path.exists(f"{modelfolder}{modelname}.preview.jpeg"): + sideloaddict.append(f"{modelname}.preview.jpeg") + if os.path.exists(f"{modelfolder}{modelname}.preview.png"): + sideloaddict.append(f"{modelname}.preview.png") + else: + print(f"Model {modelfile} found on CivitAi") + modelid = modelinfoarray.get('modelId') + civitailist.append(modelid) + print(f"Creating {modelfolder}{modelname}.civitai.info for future use") + modelinfofile = open(f"{modelfolder}{modelname}.civitai.info", "w") + modelinfofile.write(resp.data.decode('utf-8')) + modelinfofile.close() + + except Exception as e: + print("[Model Step SHA256 civitai.info parse] ", path, modelfile , " | ",e) + #if os.path.exists(f"{modelfolder}{modelname}.civitai.info"): + #print("Civitai Info file found.") + # do json stuff + + #else: + + return list([civitailist, sideloaddict]) + + +def scanall(): + erislist = {} + erislist['lora'] = {} + result = scanfolder("./models/lora/") + erislist['lora']['civitai'] = result[0] + erislist['lora']['sideload'] = result[1] + + erislist['sd'] = {} + result = scanfolder("./models/stable-diffusion/") + erislist['sd']['civitai'] = result[0] + erislist['sd']['sideload'] = result[1] + + erislist['emb'] = {} + result = scanfolder("./embeddings/") + erislist['emb']['civitai'] = result[0] + erislist['emb']['sideload'] = result[1] + + erisjsonfile = open("./eris.json", "w") + erisjsonfile.write(json.dumps(erislist, indent=2)) + erisjsonfile.close() + + +if __name__ == "__main__": + scanall() \ No newline at end of file diff --git a/erisupdate.py b/erisupdate.py new file mode 100644 index 0000000..eb32101 --- /dev/null +++ b/erisupdate.py @@ -0,0 +1,421 @@ +import json +import urllib3 +import urllib.request +import os +import cgi +from pathlib import Path +import argparse +import string +#import getpass +#print("Env thinks the user is [%s]" % (os.getlogin())) +#print("Effective user is [%s]" % (getpass.getuser())) + +import erisscan + + +def filter_str(s: str) -> str: + return "".join(filter(lambda x: x in string.printable, s)) + + +def downloadcivitai(modelid: str, type: str, image: bool) -> bool: + + match type: + case "lora": + dlpath = f"./models/Lora/" + case "sd": + dlpath = f"./models/Stable-diffusion/" + case "emb": + dlpath = f"./embeddings/" + + http = urllib3.PoolManager() + urlid = f"https://civitai.com/api/v1/models/{modelid}" + resp = http.request('GET', urlid) + modelarray = json.loads(resp.data.decode('utf-8')) + modelsha256 = modelarray.get('modelVersions')[0].get('files')[0].get('hashes').get("SHA256") + urlhash = f"https://civitai.com/api/v1/model-versions/by-hash/{modelsha256}" + modelimageurl = modelarray.get('modelVersions')[0].get('images')[0].get('url') + modelfilename = modelarray.get('modelVersions')[0].get('files')[0].get('name') + #modelfilesize = int(modelarray.get('modelVersions')[0].get('files')[0].get('sizeKB')) + modelname = Path(modelfilename).stem + modelfileurl = modelarray.get('modelVersions')[0].get('files')[0].get('downloadUrl') + #modeldisplayname = modelarray.get('name') + #trainedwords = modelarray.get('modelVersions')[0].get('trainedWords') + + modelimagename = [f"{modelname}.preview.jpg", f"{modelname}.preview.jpeg", f"{modelname}.preview.png"] + #modelimagename = "" + modelinfofile = f"{modelname}.civitai.info" + + + def downloadmodel(url: str, dlpath: str) -> bool: + print(f"Download {modelfilename} to {dlpath}") + dlrequest = http.request('GET', url, preload_content=False) + filesize = dlrequest.getheader('content-length') + contentdis = dlrequest.getheader('Content-Disposition') + try: + type, filename = cgi.parse_header(contentdis) + #print("Found file", dict(filename).get('filename')) + filename = dict(filename).get('filename') + #filename = filename.encode('utf-8') + #filename = filter_str(filename) + except Exception as e: + print("Cannot get filename", e) + if filesize: + filesize = int(filesize) + blocksize = max(4096, filesize//100) + print(f"Filesize: {filesize//1024} kB | {filesize//1024//1024} MB") + else: + blocksize = 10000 + #with open(f"{dlpath}{filename}", 'wb') as out: + with open(Path(dlpath) / filename, 'wb') as out: + size = 0 + while True: + data = dlrequest.read(blocksize) + if not data: + break + size += len(data) + out.write(data) + if filesize: + Percent = int((size / filesize)*100) + if Percent == 100: + print(f"Download: {Percent}%") + else: + print(f"Download: {Percent}%", end='\r') + dlrequest.release_conn() + + + + def downloadimage(url: str, dlpath: str) -> bool: + print(f"Download Image for {modelfilename}") + imagerequest = http.request('GET', url, preload_content=False) + #path = f"{os.getcwd()}/models/Loras/" + filesize = imagerequest.getheader('content-length') + #contentdis = imagerequest.getheader('Content-Disposition') + #type, filename = cgi.parse_header(contentdis) + filename = f"{Path(modelfilename).stem}.preview{Path(filename).suffix}" + global modelimagename + modelimagename = filename + #print(filename['filename']) + if filesize: + filesize = int(filesize) + blocksize = max(4096, filesize//100) + print(f"Filesize: {filesize//1024} kB | {filesize//1024//1024} MB") + else: + blocksize = 1000 + with open(f"{dlpath}{filename}", 'wb') as out: + size = 0 + while True: + data = imagerequest.read(blocksize) + if not data: + break + size += len(data) + out.write(data) + if filesize: + Percent = int((size / filesize)*100) + if Percent == 100: + print(f"Download: {Percent}%") + else: + print(f"Download: {Percent}%", end='\r') + imagerequest.release_conn() + + def downloadinfo(url: str, dlfullpath: str) -> bool: + print(f"Download Infofile for {modelfilename}") + resp = http.request('GET', url) + infofile = open(f"{dlfullpath}", "w") + infofile.write(resp.data.decode('utf-8')) + infofile.close() + + # download Model file + try: + if not os.path.exists(f"{dlpath}{modelfilename}"): + downloadmodel(modelfileurl, dlpath) + else: + print(f"{modelfilename} already exist, skip download") + except Exception as e: + print(e) + finally: + if not os.path.exists(f"{dlpath}{modelfilename}"): + print(f"{modelfilename} could not be saved, abort.") + return False + # download Preview image, if not disabled + if image: + try: + if os.path.exists(f"{dlpath}{modelimagename[0]}") == False and os.path.exists(f"{dlpath}{modelimagename[1]}") == False and os.path.exists(f"{dlpath}{modelimagename[2]}") == False: + downloadimage(modelimageurl, dlpath) + else: + print(f"Preview Image for {modelfilename} already exist, skip download") + + except Exception as e: + print(e) + finally: + if isinstance(modelfilename, str): + if not os.path.exists(f"{dlpath}{modelimagename}"): + print(f"{modelfilename} could not be saved, continue.") + # save infofile + try: + if not os.path.exists(f"{dlpath}{modelinfofile}"): + downloadinfo(urlhash, f"{dlpath}{modelinfofile}") + else: + print(f"{modelinfofile} already exist, skip creation") + except Exception as e: + print(e) + finally: + if not os.path.exists(f"{dlpath}{modelinfofile}"): + print(f"{modelinfofile} could not be created, abort.") + return False + return True + +# ______________________________________________________________________ + +def downloadsideload(name: str, modeltype: str) -> bool: + sideloadserver = f"https://eris.hitmare.me/eris/{modeltype}/" + + match modeltype: + case "lora": + dlpath = "./models/lora/" + case "sd": + dlpath = "./models/stable-diffusion/" + case "emb": + dlpath = "./embeddings/" + + http = urllib3.PoolManager() + url = f"{sideloadserver}{name}" + try: + print(f"Download {name}") + dlrequest = http.request('GET', url, preload_content=False) + + if not int(dlrequest.status) == 200: + print(f"File {name} not found") + return False + + filesize = dlrequest.getheader('content-length') + #print(filesize) + #contentdis = dlrequest.getheader('Content-Disposition') + #type, filename = cgi.parse_header(contentdis) + filename = name + if filesize: + filesize = int(filesize) + blocksize = max(4096, filesize//100) + print(f"Filesize: {filesize//1024} kB | {filesize//1024//1024} MB") + else: + blocksize = 10000 + with open(f"{dlpath}{filename}", 'wb') as out: + size = 0 + while True: + data = dlrequest.read(blocksize) + if not data: + break + size += len(data) + out.write(data) + if filesize: + Percent = int((size / filesize)*100) + if Percent == 100: + print(f"Download: {Percent}%") + else: + print(f"Download: {Percent}%", end='\r') + print(f"Downloaded {dlpath}{filename}") + dlrequest.release_conn() + return True + except Exception as e: + print(name, "not found on Sideloadserver", " | Python Error:" ,e) + return False + + + +# ______________________________________________________________________ +# check current status of online and local lora +def checkcivitai(locallist: list, erislist: list) -> tuple: + #print("Check Civitai") + #url = "https://eris.hitmare.me/lora.txt" + erismodels = erislist + localmodels = locallist + missingmodels = set() + additionalmodels = set() + savedmodels = set() + # check online list against local list + for loraid in erismodels: + if not loraid in localmodels: + missingmodels.add(loraid) + else: + savedmodels.add(loraid) + + # check local list against online list + for loraid in localmodels: + if not loraid in erismodels: + additionalmodels.add(loraid) + + return tuple((missingmodels, additionalmodels, savedmodels)) + + +def checksideload(locallist: list, erislist: list) -> tuple: + #print("Check Sideload") + #url = "https://eris.hitmare.me/lora.txt" + erismodels = erislist + localmodels = locallist + missingmodels = set() + additionalmodels = set() + savedmodels = set() + # check online list against local list + for loraid in erismodels: + if not loraid in localmodels: + missingmodels.add(loraid) + else: + savedmodels.add(loraid) + + # check local list against online list + for loraid in localmodels: + if not loraid in erismodels: + additionalmodels.add(loraid) + + return tuple((missingmodels, additionalmodels, savedmodels)) + +def deleteFiles(nameid: str, modeltype: str, localfile: bool = False) -> bool: + match modeltype: + case "lora": + dlpath = "./models/lora/" + case "sd": + dlpath = "./models/stable-diffusion/" + case "emb": + dlpath = "./embeddings/" + + try: + if localfile: + modelfilename = nameid + else: + http = urllib3.PoolManager() + urlid = f"https://civitai.com/api/v1/models/{nameid}" + resp = http.request('GET', urlid) + modelarray = json.loads(resp.data.decode('utf-8')) + modelfilename = modelarray.get('modelVersions')[0].get('files')[0].get('name') + + modelfiles = os.listdir(dlpath) + for modelfile in modelfiles: + if Path(modelfile).stem == Path(modelfilename).stem: + os.remove(f"{dlpath}{modelfile}") + return True + except Exception as e: + print(f"Could not delete {nameid}. Error: {e}") + return False + +parser = argparse.ArgumentParser( + prog='Eris File Updater', + description='Updates Lora, Embeddings and Stable-diffusion Models', + epilog='Thanks for being a part of the Eris Bot ^-^') + +parser.add_argument('-si', '--skipimage', help=r"Skips the download of the preview images, Applies to all Model Types", action="store_true") +parser.add_argument('-sl', '--skiplora', help=r"Skips check of lora models", action="store_true") +parser.add_argument('-ss', '--skipsdmodel', help=r"Skips check of stable-diffusion models", action="store_true") +parser.add_argument('-se', '--skipemb', help=r"Skips check of embedding models", action="store_true") +parser.add_argument('-c', '--cleanup', help=r"Deletes any models that are locally but not on the eris model list. Affected by Skip Triggers", action="store_true") + +args = parser.parse_args() + +if not os.path.exists("./eris.json"): + # start scan python or exit + #print("Please run the erisscan bat/sh file first") + #exit() + print("No eris.json file found. Starting scan ...") + erisscan.scanall() + + +# get the eris list from the web and local +http = urllib3.PoolManager() +erisurl = "https://eris.hitmare.me/eris.json" +resp = http.request('GET', erisurl) +erisremote = json.loads(resp.data.decode('utf-8')) +erislocal = json.load(open("./eris.json", "r")) + +if args.skipimage: + dlimage = False +else: + dlimage = True + +if args.skiplora: + print("Skip Lora Model check") +else: + lorastate = checkcivitai(erislocal.get('lora')['civitai'], erisremote.get('lora')['civitai']) + if len(lorastate[0]) > 0 : + print(f"Missing lora ids: {lorastate[0]}") + for lora in lorastate[0]: + if downloadcivitai(lora, 'lora', dlimage): + erislocal.get('lora')['civitai'].append(lora) + lorasideloadstate = checksideload(erislocal.get('lora')['sideload'], erisremote.get('lora')['sideload']) + if len(lorasideloadstate[0]) > 0 : + print(f"Missing lora files: {lorasideloadstate[0]}") + for lora in lorasideloadstate[0]: + if downloadsideload(lora, 'lora'): + erislocal.get('lora')['sideload'].append(lora) + if args.cleanup: + if len(lorastate[1]) > 0: + print("Additional CivitAI lora ids:", lorastate[1]) + for dellora in lorastate[1]: + if deleteFiles(dellora, 'lora'): + erislocal.get('lora')['civitai'].remove(dellora) + if len(lorasideloadstate[1]) > 0: + print("Additional CivitAI lora files:", lorastate[1]) + for delslora in lorasideloadstate[1]: + if deleteFiles(delslora, 'lora', True): + erislocal.get('lora')['sideload'].remove(delslora) + #print(len(lorastate[1])) +if args.skipsdmodel: + print("Skip Stable-Diffusion Model check") +else: + sdstate = checkcivitai(erislocal.get('sd')['civitai'], erisremote.get('sd')['civitai']) + if len(sdstate[0]) > 0 : + print(f"Missing CivitAI stable-diffusion ids: {sdstate[0]}") + for sd in sdstate[0]: + if downloadcivitai(sd, 'sd', dlimage): + erislocal.get('sd')['civitai'].append(sd) + sdsideloadstate = checksideload(erislocal.get('sd')['sideload'], erisremote.get('sd')['sideload']) + if len(sdsideloadstate[0]) > 0 : + print(f"Missing Sideload stable-diffusion files: {sdsideloadstate[0]}") + for sd in sdsideloadstate[0]: + if downloadsideload(sd, 'sd'): + erislocal.get('sd')['sideload'].append(sd) + if args.cleanup: + if len(sdstate[1]) > 0: + print("Additional CivitAI stable-diffusion ids:", sdstate[1]) + for delsd in sdstate[1]: + if deleteFiles(delsd, 'lora'): + erislocal.get('sd')['civitai'].remove(delsd) + if len(sdsideloadstate[1]) > 0: + print("Additional Sideload stable-diffusion files:", sdstate[1]) + for delssd in sdsideloadstate[1]: + if deleteFiles(delssd, 'lora', True): + erislocal.get('sd')['sideload'].remove(delssd) + +if args.skipemb: + print("Skip Embedding Model check") +else: + embstate = checkcivitai(erislocal.get('emb')['civitai'], erisremote.get('emb')['civitai']) + if len(embstate[0]) > 0 : + print(f"Missing embedding ids: {embstate[0]}") + for emb in embstate[0]: + if downloadcivitai(emb, 'emb', dlimage): + erislocal.get('emb')['civitai'].append(emb) + embsideloadstate = checksideload(erislocal.get('emb')['sideload'], erisremote.get('emb')['sideload']) + if len(embsideloadstate[0]) > 0 : + print(f"Missing embedding files: {embsideloadstate[0]}") + for emb in embsideloadstate[0]: + if downloadsideload(emb, 'emb'): + erislocal.get('emb')['sideload'].append(emb) + if args.cleanup: + if len(embstate[1]) > 0: + print("Additional CivitAI embedding ids:", embstate[1]) + for delemb in embstate[1]: + if deleteFiles(delemb, 'lora'): + erislocal.get('emb')['civitai'].remove(delemb) + if len(embsideloadstate[1]) > 0: + print("Additional Sideload embedding files:", embsideloadstate[1]) + for delsemb in embsideloadstate[1]: + if deleteFiles(delsemb, 'lora', True): + erislocal.get('emb')['sideload'].remove(delsemb) + +erisjsonfile = open("./eris.json", "w") +erisjsonfile.write(json.dumps(erislocal, indent=2)) +erisjsonfile.close() + + + + + +