Hitmare
/
Eris-Update-script
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Dateien nach "/" hochladen

This commit is contained in:
Hitmare 2023-11-11 14:19:44 +00:00
parent e46c0da96c
commit cc5ddf288f
3 changed files with 807 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

282
eris.json Normal file
View File

@ -0,0 +1,282 @@
{
"lora": {
"civitai": [
8058,
58390,
9155,
21640,
107279,
139340,
4982,
62342,
9620,
100070,
108007,
72456,
151678,
71596,
18472,
61361,
113807,
19777,
19777,
61442,
58964,
5975,
26449,
131093,
15898,
151686,
52352,
91403,
134737,
146173,
28431,
155045,
9025,
78887,
8821,
13495,
151734,
118352,
109401,
131567,
156270,
60132,
128811,
70618,
60003,
113845,
115008,
115848,
27206,
25360,
77121,
102838,
80837,
159037,
10564,
119172,
156550,
24584,
89285,
76732,
150656,
120887,
13941,
13941,
13941,
52023,
150686,
6976,
113904,
62978,
68641,
72446,
150697,
6433,
72930,
127533,
59880,
19620,
116855,
9272,
13090,
151772,
70572,
6526,
114307,
150706,
159524,
53983,
141555,
100707,
123690,
154996,
53504,
8678,
151785,
141128,
12423,
49021,
159679,
71813,
88678,
136710,
150721,
93201,
53198,
115720,
52886,
15907,
115773,
17824,
151824,
150729,
114794,
135870,
87363,
6610,
28614,
151894,
119389,
85366,
48139,
151921,
151932,
136067,
25175,
71616,
116311,
63999,
155302,
159528,
152232,
152247,
8399,
152293,
152331,
159000,
155294,
24937,
101489,
16775,
85387,
128095,
61442,
91226,
152344,
18299,
89051,
152369,
10668,
152397,
114525,
22615,
58295,
51293,
90903,
114881,
88312,
146171,
159662,
15798,
15798,
15798,
132408,
98229,
43943,
7958,
80808,
58012,
82355,
61442,
90504,
159360,
152865,
115062,
141385,
9630,
46630,
158942,
131399,
50808,
114293,
70591,
70591,
94795,
25236,
155300,
115076,
80910,
11858,
152885,
115080,
24751,
150740,
89196,
11886,
107014,
132696,
32207,
116407,
34372,
33251,
114718,
82132,
155296,
150759,
69323,
23337,
22764,
150784,
53076,
30195,
113762,
60701,
63492,
120152,
61442,
61442,
152893,
41491,
26818,
67983,
160623,
142949,
157025,
119077,
160652,
154971,
70556,
75416,
151659
],
"sideload": [
"fr_female-x-feral_canid_rear-mounting_v1.2_e200.safetensors",
"fr_female-x-feral_canid_rear-mounting_v1.2_e200.preview.png",
"Y1-BadDragon.safetensors",
"Y1-BadDragon.preview.png"
]
},
"sd": {
"civitai": [
65203,
5012,
4108,
3666,
144962
],
"sideload": [
"EasyFluffV10-PreRelease2.safetensors",
"EasyFluffV11.2.safetensors",
"fluffyrock-576-704-832-960-1088-lion-low-lr-e219-terminal-snr-e192.safetensors"
]
},
"emb": {
"civitai": [
16807,
5224,
116230,
142280,
87781,
87781,
87781,
187118,
16807,
187118,
7808,
150087,
56519,
4629,
187118,
4499,
187118,
18410
],
"sideload": [
"boring_e621_fluffyrock.pt",
"boring_e621_fluffyrock_v3.pt"
]
}
}

104
erisscan.py Normal file
View File

@ -0,0 +1,104 @@
import os
import json
import hashlib
import urllib3
from pathlib import Path
def scanfolder(path: str) -> list:
civitailist = []
sideloaddict = []
modelfolder = path
modelfiles = os.listdir(modelfolder)
for modelfile in modelfiles:
modelname = Path(modelfile).stem
sideload = False
if modelfile.endswith('.safetensors') or modelfile.endswith('.pt'):
#print(f"Found '{lorafile}' - Searching civitai.info")
if os.path.exists(f"{modelfolder}{modelname}.civitai.info"):
civitaiinfo = open(f"{modelfolder}{modelname}.civitai.info", "r", encoding='utf-8')
civitaiinfocontent = civitaiinfo.read()
civitaiinfo.close()
if civitaiinfocontent[:2] == "{}":
os.remove(f"{modelfolder}{modelname}.civitai.info")
sideload = True
else:
with open(f"{modelfolder}{modelname}.civitai.info", "r") as f:
#fstring = "dd"
print("Reading", modelfolder, modelfile)
modelinfoarray = json.load(f)
#print(loraarray)
try:
modelsha256 = modelinfoarray.get('files')[0].get('hashes').get('SHA256')
modelid = modelinfoarray.get('modelId')
civitailist.append(modelid)
#print(lorasha256, " " , loramodelid)
except Exception as e:
print("[Model Step local civitai.info parse] ", path, modelfile , " | ",e)
#print("Content:", civitaiinfocontent[:2])
else:
sideload = True
if sideload:
print(f"{modelfolder}{modelname}.civitai.info not found. Getting SHA256 Hash and fetch data from civitai API")
with open(f"{modelfolder}{modelfile}", "rb") as f:
bytes = f.read() # read entire file as bytes
modelsha256 = hashlib.sha256(bytes).hexdigest();
#print(readable_hash)
http = urllib3.PoolManager()
url = f"https://civitai.com/api/v1/model-versions/by-hash/{modelsha256}"
resp = http.request('GET', url)
modelinfoarray = json.loads(resp.data.decode('utf-8'))
try:
if modelinfoarray.get('error') == "Model not found":
print(f"Model {modelfile} is not a CivitAi Model")
sideloaddict.append(modelfile)
if os.path.exists(f"{modelfolder}{modelname}.preview.jpg"):
sideloaddict.append(f"{modelname}.preview.jpg")
if os.path.exists(f"{modelfolder}{modelname}.preview.jpeg"):
sideloaddict.append(f"{modelname}.preview.jpeg")
if os.path.exists(f"{modelfolder}{modelname}.preview.png"):
sideloaddict.append(f"{modelname}.preview.png")
else:
print(f"Model {modelfile} found on CivitAi")
modelid = modelinfoarray.get('modelId')
civitailist.append(modelid)
print(f"Creating {modelfolder}{modelname}.civitai.info for future use")
modelinfofile = open(f"{modelfolder}{modelname}.civitai.info", "w")
modelinfofile.write(resp.data.decode('utf-8'))
modelinfofile.close()
except Exception as e:
print("[Model Step SHA256 civitai.info parse] ", path, modelfile , " | ",e)
#if os.path.exists(f"{modelfolder}{modelname}.civitai.info"):
#print("Civitai Info file found.")
# do json stuff
#else:
return list([civitailist, sideloaddict])
def scanall():
erislist = {}
erislist['lora'] = {}
result = scanfolder("./models/lora/")
erislist['lora']['civitai'] = result[0]
erislist['lora']['sideload'] = result[1]
erislist['sd'] = {}
result = scanfolder("./models/stable-diffusion/")
erislist['sd']['civitai'] = result[0]
erislist['sd']['sideload'] = result[1]
erislist['emb'] = {}
result = scanfolder("./embeddings/")
erislist['emb']['civitai'] = result[0]
erislist['emb']['sideload'] = result[1]
erisjsonfile = open("./eris.json", "w")
erisjsonfile.write(json.dumps(erislist, indent=2))
erisjsonfile.close()
if __name__ == "__main__":
scanall()

421
erisupdate.py Normal file
View File

@ -0,0 +1,421 @@
import json
import urllib3
import urllib.request
import os
import cgi
from pathlib import Path
import argparse
import string
#import getpass
#print("Env thinks the user is [%s]" % (os.getlogin()))
#print("Effective user is [%s]" % (getpass.getuser()))
import erisscan
def filter_str(s: str) -> str:
return "".join(filter(lambda x: x in string.printable, s))
def downloadcivitai(modelid: str, type: str, image: bool) -> bool:
match type:
case "lora":
dlpath = f"./models/Lora/"
case "sd":
dlpath = f"./models/Stable-diffusion/"
case "emb":
dlpath = f"./embeddings/"
http = urllib3.PoolManager()
urlid = f"https://civitai.com/api/v1/models/{modelid}"
resp = http.request('GET', urlid)
modelarray = json.loads(resp.data.decode('utf-8'))
modelsha256 = modelarray.get('modelVersions')[0].get('files')[0].get('hashes').get("SHA256")
urlhash = f"https://civitai.com/api/v1/model-versions/by-hash/{modelsha256}"
modelimageurl = modelarray.get('modelVersions')[0].get('images')[0].get('url')
modelfilename = modelarray.get('modelVersions')[0].get('files')[0].get('name')
#modelfilesize = int(modelarray.get('modelVersions')[0].get('files')[0].get('sizeKB'))
modelname = Path(modelfilename).stem
modelfileurl = modelarray.get('modelVersions')[0].get('files')[0].get('downloadUrl')
#modeldisplayname = modelarray.get('name')
#trainedwords = modelarray.get('modelVersions')[0].get('trainedWords')
modelimagename = [f"{modelname}.preview.jpg", f"{modelname}.preview.jpeg", f"{modelname}.preview.png"]
#modelimagename = ""
modelinfofile = f"{modelname}.civitai.info"
def downloadmodel(url: str, dlpath: str) -> bool:
print(f"Download {modelfilename} to {dlpath}")
dlrequest = http.request('GET', url, preload_content=False)
filesize = dlrequest.getheader('content-length')
contentdis = dlrequest.getheader('Content-Disposition')
try:
type, filename = cgi.parse_header(contentdis)
#print("Found file", dict(filename).get('filename'))
filename = dict(filename).get('filename')
#filename = filename.encode('utf-8')
#filename = filter_str(filename)
except Exception as e:
print("Cannot get filename", e)
if filesize:
filesize = int(filesize)
blocksize = max(4096, filesize//100)
print(f"Filesize: {filesize//1024} kB | {filesize//1024//1024} MB")
else:
blocksize = 10000
#with open(f"{dlpath}{filename}", 'wb') as out:
with open(Path(dlpath) / filename, 'wb') as out:
size = 0
while True:
data = dlrequest.read(blocksize)
if not data:
break
size += len(data)
out.write(data)
if filesize:
Percent = int((size / filesize)*100)
if Percent == 100:
print(f"Download: {Percent}%")
else:
print(f"Download: {Percent}%", end='\r')
dlrequest.release_conn()
def downloadimage(url: str, dlpath: str) -> bool:
print(f"Download Image for {modelfilename}")
imagerequest = http.request('GET', url, preload_content=False)
#path = f"{os.getcwd()}/models/Loras/"
filesize = imagerequest.getheader('content-length')
#contentdis = imagerequest.getheader('Content-Disposition')
#type, filename = cgi.parse_header(contentdis)
filename = f"{Path(modelfilename).stem}.preview{Path(filename).suffix}"
global modelimagename
modelimagename = filename
#print(filename['filename'])
if filesize:
filesize = int(filesize)
blocksize = max(4096, filesize//100)
print(f"Filesize: {filesize//1024} kB | {filesize//1024//1024} MB")
else:
blocksize = 1000
with open(f"{dlpath}{filename}", 'wb') as out:
size = 0
while True:
data = imagerequest.read(blocksize)
if not data:
break
size += len(data)
out.write(data)
if filesize:
Percent = int((size / filesize)*100)
if Percent == 100:
print(f"Download: {Percent}%")
else:
print(f"Download: {Percent}%", end='\r')
imagerequest.release_conn()
def downloadinfo(url: str, dlfullpath: str) -> bool:
print(f"Download Infofile for {modelfilename}")
resp = http.request('GET', url)
infofile = open(f"{dlfullpath}", "w")
infofile.write(resp.data.decode('utf-8'))
infofile.close()
# download Model file
try:
if not os.path.exists(f"{dlpath}{modelfilename}"):
downloadmodel(modelfileurl, dlpath)
else:
print(f"{modelfilename} already exist, skip download")
except Exception as e:
print(e)
finally:
if not os.path.exists(f"{dlpath}{modelfilename}"):
print(f"{modelfilename} could not be saved, abort.")
return False
# download Preview image, if not disabled
if image:
try:
if os.path.exists(f"{dlpath}{modelimagename[0]}") == False and os.path.exists(f"{dlpath}{modelimagename[1]}") == False and os.path.exists(f"{dlpath}{modelimagename[2]}") == False:
downloadimage(modelimageurl, dlpath)
else:
print(f"Preview Image for {modelfilename} already exist, skip download")
except Exception as e:
print(e)
finally:
if isinstance(modelfilename, str):
if not os.path.exists(f"{dlpath}{modelimagename}"):
print(f"{modelfilename} could not be saved, continue.")
# save infofile
try:
if not os.path.exists(f"{dlpath}{modelinfofile}"):
downloadinfo(urlhash, f"{dlpath}{modelinfofile}")
else:
print(f"{modelinfofile} already exist, skip creation")
except Exception as e:
print(e)
finally:
if not os.path.exists(f"{dlpath}{modelinfofile}"):
print(f"{modelinfofile} could not be created, abort.")
return False
return True
# ______________________________________________________________________
def downloadsideload(name: str, modeltype: str) -> bool:
sideloadserver = f"https://eris.hitmare.me/eris/{modeltype}/"
match modeltype:
case "lora":
dlpath = "./models/lora/"
case "sd":
dlpath = "./models/stable-diffusion/"
case "emb":
dlpath = "./embeddings/"
http = urllib3.PoolManager()
url = f"{sideloadserver}{name}"
try:
print(f"Download {name}")
dlrequest = http.request('GET', url, preload_content=False)
if not int(dlrequest.status) == 200:
print(f"File {name} not found")
return False
filesize = dlrequest.getheader('content-length')
#print(filesize)
#contentdis = dlrequest.getheader('Content-Disposition')
#type, filename = cgi.parse_header(contentdis)
filename = name
if filesize:
filesize = int(filesize)
blocksize = max(4096, filesize//100)
print(f"Filesize: {filesize//1024} kB | {filesize//1024//1024} MB")
else:
blocksize = 10000
with open(f"{dlpath}{filename}", 'wb') as out:
size = 0
while True:
data = dlrequest.read(blocksize)
if not data:
break
size += len(data)
out.write(data)
if filesize:
Percent = int((size / filesize)*100)
if Percent == 100:
print(f"Download: {Percent}%")
else:
print(f"Download: {Percent}%", end='\r')
print(f"Downloaded {dlpath}{filename}")
dlrequest.release_conn()
return True
except Exception as e:
print(name, "not found on Sideloadserver", " | Python Error:" ,e)
return False
# ______________________________________________________________________
# check current status of online and local lora
def checkcivitai(locallist: list, erislist: list) -> tuple:
#print("Check Civitai")
#url = "https://eris.hitmare.me/lora.txt"
erismodels = erislist
localmodels = locallist
missingmodels = set()
additionalmodels = set()
savedmodels = set()
# check online list against local list
for loraid in erismodels:
if not loraid in localmodels:
missingmodels.add(loraid)
else:
savedmodels.add(loraid)
# check local list against online list
for loraid in localmodels:
if not loraid in erismodels:
additionalmodels.add(loraid)
return tuple((missingmodels, additionalmodels, savedmodels))
def checksideload(locallist: list, erislist: list) -> tuple:
#print("Check Sideload")
#url = "https://eris.hitmare.me/lora.txt"
erismodels = erislist
localmodels = locallist
missingmodels = set()
additionalmodels = set()
savedmodels = set()
# check online list against local list
for loraid in erismodels:
if not loraid in localmodels:
missingmodels.add(loraid)
else:
savedmodels.add(loraid)
# check local list against online list
for loraid in localmodels:
if not loraid in erismodels:
additionalmodels.add(loraid)
return tuple((missingmodels, additionalmodels, savedmodels))
def deleteFiles(nameid: str, modeltype: str, localfile: bool = False) -> bool:
match modeltype:
case "lora":
dlpath = "./models/lora/"
case "sd":
dlpath = "./models/stable-diffusion/"
case "emb":
dlpath = "./embeddings/"
try:
if localfile:
modelfilename = nameid
else:
http = urllib3.PoolManager()
urlid = f"https://civitai.com/api/v1/models/{nameid}"
resp = http.request('GET', urlid)
modelarray = json.loads(resp.data.decode('utf-8'))
modelfilename = modelarray.get('modelVersions')[0].get('files')[0].get('name')
modelfiles = os.listdir(dlpath)
for modelfile in modelfiles:
if Path(modelfile).stem == Path(modelfilename).stem:
os.remove(f"{dlpath}{modelfile}")
return True
except Exception as e:
print(f"Could not delete {nameid}. Error: {e}")
return False
parser = argparse.ArgumentParser(
prog='Eris File Updater',
description='Updates Lora, Embeddings and Stable-diffusion Models',
epilog='Thanks for being a part of the Eris Bot ^-^')
parser.add_argument('-si', '--skipimage', help=r"Skips the download of the preview images, Applies to all Model Types", action="store_true")
parser.add_argument('-sl', '--skiplora', help=r"Skips check of lora models", action="store_true")
parser.add_argument('-ss', '--skipsdmodel', help=r"Skips check of stable-diffusion models", action="store_true")
parser.add_argument('-se', '--skipemb', help=r"Skips check of embedding models", action="store_true")
parser.add_argument('-c', '--cleanup', help=r"Deletes any models that are locally but not on the eris model list. Affected by Skip Triggers", action="store_true")
args = parser.parse_args()
if not os.path.exists("./eris.json"):
# start scan python or exit
#print("Please run the erisscan bat/sh file first")
#exit()
print("No eris.json file found. Starting scan ...")
erisscan.scanall()
# get the eris list from the web and local
http = urllib3.PoolManager()
erisurl = "https://eris.hitmare.me/eris.json"
resp = http.request('GET', erisurl)
erisremote = json.loads(resp.data.decode('utf-8'))
erislocal = json.load(open("./eris.json", "r"))
if args.skipimage:
dlimage = False
else:
dlimage = True
if args.skiplora:
print("Skip Lora Model check")
else:
lorastate = checkcivitai(erislocal.get('lora')['civitai'], erisremote.get('lora')['civitai'])
if len(lorastate[0]) > 0 :
print(f"Missing lora ids: {lorastate[0]}")
for lora in lorastate[0]:
if downloadcivitai(lora, 'lora', dlimage):
erislocal.get('lora')['civitai'].append(lora)
lorasideloadstate = checksideload(erislocal.get('lora')['sideload'], erisremote.get('lora')['sideload'])
if len(lorasideloadstate[0]) > 0 :
print(f"Missing lora files: {lorasideloadstate[0]}")
for lora in lorasideloadstate[0]:
if downloadsideload(lora, 'lora'):
erislocal.get('lora')['sideload'].append(lora)
if args.cleanup:
if len(lorastate[1]) > 0:
print("Additional CivitAI lora ids:", lorastate[1])
for dellora in lorastate[1]:
if deleteFiles(dellora, 'lora'):
erislocal.get('lora')['civitai'].remove(dellora)
if len(lorasideloadstate[1]) > 0:
print("Additional CivitAI lora files:", lorastate[1])
for delslora in lorasideloadstate[1]:
if deleteFiles(delslora, 'lora', True):
erislocal.get('lora')['sideload'].remove(delslora)
#print(len(lorastate[1]))
if args.skipsdmodel:
print("Skip Stable-Diffusion Model check")
else:
sdstate = checkcivitai(erislocal.get('sd')['civitai'], erisremote.get('sd')['civitai'])
if len(sdstate[0]) > 0 :
print(f"Missing CivitAI stable-diffusion ids: {sdstate[0]}")
for sd in sdstate[0]:
if downloadcivitai(sd, 'sd', dlimage):
erislocal.get('sd')['civitai'].append(sd)
sdsideloadstate = checksideload(erislocal.get('sd')['sideload'], erisremote.get('sd')['sideload'])
if len(sdsideloadstate[0]) > 0 :
print(f"Missing Sideload stable-diffusion files: {sdsideloadstate[0]}")
for sd in sdsideloadstate[0]:
if downloadsideload(sd, 'sd'):
erislocal.get('sd')['sideload'].append(sd)
if args.cleanup:
if len(sdstate[1]) > 0:
print("Additional CivitAI stable-diffusion ids:", sdstate[1])
for delsd in sdstate[1]:
if deleteFiles(delsd, 'lora'):
erislocal.get('sd')['civitai'].remove(delsd)
if len(sdsideloadstate[1]) > 0:
print("Additional Sideload stable-diffusion files:", sdstate[1])
for delssd in sdsideloadstate[1]:
if deleteFiles(delssd, 'lora', True):
erislocal.get('sd')['sideload'].remove(delssd)
if args.skipemb:
print("Skip Embedding Model check")
else:
embstate = checkcivitai(erislocal.get('emb')['civitai'], erisremote.get('emb')['civitai'])
if len(embstate[0]) > 0 :
print(f"Missing embedding ids: {embstate[0]}")
for emb in embstate[0]:
if downloadcivitai(emb, 'emb', dlimage):
erislocal.get('emb')['civitai'].append(emb)
embsideloadstate = checksideload(erislocal.get('emb')['sideload'], erisremote.get('emb')['sideload'])
if len(embsideloadstate[0]) > 0 :
print(f"Missing embedding files: {embsideloadstate[0]}")
for emb in embsideloadstate[0]:
if downloadsideload(emb, 'emb'):
erislocal.get('emb')['sideload'].append(emb)
if args.cleanup:
if len(embstate[1]) > 0:
print("Additional CivitAI embedding ids:", embstate[1])
for delemb in embstate[1]:
if deleteFiles(delemb, 'lora'):
erislocal.get('emb')['civitai'].remove(delemb)
if len(embsideloadstate[1]) > 0:
print("Additional Sideload embedding files:", embsideloadstate[1])
for delsemb in embsideloadstate[1]:
if deleteFiles(delsemb, 'lora', True):
erislocal.get('emb')['sideload'].remove(delsemb)
erisjsonfile = open("./eris.json", "w")
erisjsonfile.write(json.dumps(erislocal, indent=2))
erisjsonfile.close()