2024-03-02 15:34:56 +00:00
18 changed files with 394 additions and 217 deletions
--- a/admin.py
+++ b/admin.py
@ -1,4 +1,5 @@
 from sanic import response, redirect, Blueprint, exceptions
+from email_util import send_missing_propic_message
 from room import unconfirm_room_by_order
 from config import *
 from utils import *
@ -20,7 +21,8 @@ async def credentials_check(request: Request):

@bp.get('/cache/clear')
 async def clear_cache(request, order:Order):
-	await request.app.ctx.om.fill_cache()
+	success = await request.app.ctx.om.fill_cache()
+	if not success: raise exceptions.ServerError("An error occurred while loading the cache")
 	return redirect(f'/manage/admin')

@bp.get('/loginas/<code>')
@ -40,8 +42,8 @@ async def login_as(request, code, order:Order):

@bp.get('/room/verify')
 async def verify_rooms(request, order:Order):
-	already_checked = await request.app.ctx.om.update_cache()
-	if not already_checked:
+	already_checked, success = await request.app.ctx.om.update_cache()
+	if not already_checked and success:
 		orders = filter(lambda x: x.status not in ['c', 'e'] and x.room_id == x.code, request.app.ctx.om.cache.values())
 		await validate_rooms(request, orders, None)
 	return redirect(f'/manage/admin')
@ -82,3 +84,18 @@ async def rename_room(request, code, order:Order):
 	await dOrder.edit_answer("room_name", name)
 	await dOrder.send_answers()
 	return redirect(f'/manage/nosecount')
+
+@bp.get('/propic/remind')
+async def propic_remind_missing(request, order:Order):
+	await clear_cache(request, order)
+
+	orders = request.app.ctx.om.cache.values()
+	order: Order
+	for order in orders:
+		missingPropic = order.propic is None
+		missingFursuitPropic = order.is_fursuiter and order.propic_fursuiter is None
+		if(missingPropic or missingFursuitPropic):
+			# print(f"{order.code}: prp={missingPropic} fpr={missingFursuitPropic} - {order.name}")
+			await send_missing_propic_message(order, missingPropic, missingFursuitPropic)
+
+	return redirect(f'/manage/admin')
--- a/app.py
+++ b/app.py
@ -16,6 +16,9 @@ import requests
 import sys
 from sanic.log import logger, logging, access_logger
 from metrics import *
+from email_util import killSmptClient
+import pretixClient
+import traceback

 app = Sanic(__name__)
 app.static("/res", "res/")
@ -35,17 +38,27 @@ from admin import bp as admin_bp

 app.blueprint([room_bp, karaoke_bp, propic_bp, export_bp, stats_bp, api_bp, carpooling_bp, checkin_bp, admin_bp])

-@app.exception(exceptions.SanicException)
-async def clear_session(request, exception):
-	logger.warning(f"{request} -> {exception}")
-	tpl = app.ctx.tpl.get_template('error.html')
-	r = html(tpl.render(exception=exception))

-	if exception.status_code == 403:
-		r.delete_cookie("foxo_code")
-		r.delete_cookie("foxo_secret")
+async def clear_session(response):
+	response.delete_cookie("foxo_code")
+	response.delete_cookie("foxo_secret")
+
+@app.exception(exceptions.SanicException if DEV_MODE else Exception)
+async def handleException(request, exception):
+	incErrorNo()
+	logger.warning(f"{request} -> {exception}")
+	statusCode = exception.status_code if hasattr(exception, 'status_code') else 500
+	try:
+		tpl = app.ctx.tpl.get_template('error.html')
+		r = html(tpl.render(exception=exception, status_code=statusCode))
+	except:
+		traceback.print_exc()
+
+	if statusCode == 403:
+		clear_session(r)
 	return r

+
@app.before_server_start
 async def main_start(*_):
 	logger.info(f"[{app.name}] >>>>>> main_start <<<<<<")
@ -56,7 +69,10 @@ async def main_start(*_):
 	
 	app.ctx.om = OrderManager()
 	if FILL_CACHE:
-		await app.ctx.om.update_cache()
+		checked, success = await app.ctx.om.update_cache(check_itemsQuestions=True)
+		if checked and not success:
+			logger.error(f"[{app.name}] Failure in app startup: An error occurred while loading items or questions or cache.")
+			app.stop()
 	
 	app.ctx.nfc_counts = sqlite3.connect('data/nfc_counts.db')
 	
@ -90,9 +106,7 @@ async def redirect_explore(request, code, secret, order: Order, secret2=None):
 	if order and order.code != code: order = None

 	if not order:
-		async with httpx.AsyncClient() as client:
-			incPretixRead()
-			res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers)
+		res = await pretixClient.get(f"orders/{code}/", expectedStatusCodes=None)
 		
 		if res.status_code != 200:
 			raise exceptions.NotFound("This order code does not exist. Check that your order wasn't deleted, or the link is correct.")
@ -155,11 +169,9 @@ async def download_ticket(request, order: Order):
 	if not order.status != 'confirmed':
 		raise exceptions.Forbidden("You are not allowed to download this ticket.")
 		
-	async with httpx.AsyncClient() as client:
-		incPretixRead()
-		res = await client.get(join(base_url_event, f"orders/{order.code}/download/pdf/"), headers=headers)
+	res = await pretixClient.get(f"orders/{order.code}/download/pdf/", expectedStatusCodes=[200, 404, 409, 403])
 	
-	if res.status_code == 409:
+	if res.status_code == 409 or res.status_code == 404:
 		raise exceptions.SanicException("Your ticket is still being generated. Please try again later!", status_code=res.status_code)
 	elif res.status_code == 403:
 		raise exceptions.SanicException("You can download your ticket only after the order has been confirmed and paid. Try later!", status_code=400)
@ -191,6 +203,10 @@ async def logout(request):

 	raise exceptions.Forbidden("You have been logged out.")

+@app.signal("server.shutdown.before")
+async def sigintHandler(app, loop):
+	killSmptClient()
+
@app.get(METRICS_PATH)
 async def metrics(request):
 	return text(getMetricsText() + "\n" + getRoomCountersText(request))
@ -208,7 +224,7 @@ if __name__ == "__main__":
 	# to let it start in the correct order. The following piece of code makes sure that pretix is running and can talk to 
 	# postgres before actually starting the reserved area, since this operation requires a cache-fill in startup
 	print("Waiting for pretix to be up and running", file=sys.stderr)
-	while True:
+	while not SKIP_HEALTHCHECK:
 		print("Trying connecting to pretix...", file=sys.stderr)
 		try:
 			incPretixRead()
@ -218,7 +234,7 @@ if __name__ == "__main__":
 				print("Healtchecking...", file=sys.stderr)
 				incPretixRead()
 				res = requests.get(join(domain, "healthcheck"), headers=headers)
-				if(res.status_code == 200 or SKIP_HEALTHCHECK):
+				if(res.status_code == 200):
 					break
 		except:
 			pass
--- a/checkin.py
+++ b/checkin.py
@ -12,6 +12,7 @@ from time import time
 from urllib.parse import unquote
 import json
 from metrics import *
+import pretixClient

 bp = Blueprint("checkin", url_prefix="/checkin")

@ -64,9 +65,7 @@ async def do_checkin(request):
 	await order.send_answers()
 	
 	if not order.checked_in:
-		async with httpx.AsyncClient() as client:
-			incPretixWrite()
-			res = await client.post(base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]}, headers=headers)
+		await pretixClient.post("", baseUrl=base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]})
 	
 	tpl = request.app.ctx.tpl.get_template('checkin_3.html')
 	return html(tpl.render(order=order, room_owner=room_owner, roommates=roommates))
--- a/config.example.py
+++ b/config.example.py
@ -21,6 +21,11 @@ PROPIC_MIN_SIZE = (125, 125) # (Width, Height)
 TG_BOT_API = '123456789:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
 TG_CHAT_ID = -1234567

+import httpx
+# Number of tries for a request to the pretix's backend
+PRETIX_REQUESTS_MAX = 3
+PRETIX_REQUESTS_TIMEOUT = httpx.Timeout(15.0, read=30.0, connect=45.0, pool=None) # Timeout for httpx requests in seconds
+
 # These order codes have additional functions.
 ADMINS = ['XXXXX', 'YYYYY']
 # A list of staff_roles 
--- a/email_util.py
+++ b/email_util.py
@ -54,6 +54,13 @@ async def sendEmail(message : MIMEMultipart):
 	smptSender.sendmail(message['From'], message['to'], message.as_string())
 	sslLock.release()

+def render_email_template(title = "", body = ""):
+	tpl = Environment(loader=FileSystemLoader("tpl"), autoescape=False).get_template('email/comunication.html')
+	return str(tpl.render(title=title, body=body))
+
+
+
+
 async def send_unconfirm_message(room_order, orders):
 	memberMessages = []

@ -72,8 +79,8 @@ async def send_unconfirm_message(room_order, orders):
 	issues_html += "</ul>"

 	for member in orders:
-		plain_body = ROOM_UNCONFIRM_TEXT['plain'].format(member.name, room_order.room_name, issues_plain)
-		html_body = render_email_template(ROOM_UNCONFIRM_TITLE, ROOM_UNCONFIRM_TEXT['html'].format(member.name, room_order.room_name, issues_html))
+		plain_body = EMAILS_TEXT["ROOM_UNCONFIRM_TEXT"]['plain'].format(member.name, room_order.room_name, issues_plain)
+		html_body = render_email_template(EMAILS_TEXT["ROOM_UNCONFIRM_TITLE"], EMAILS_TEXT["ROOM_UNCONFIRM_TEXT"]['html'].format(member.name, room_order.room_name, issues_html))
 		plain_text = MIMEText(plain_body, "plain")
 		html_text = MIMEText(html_body, "html")
 		message = MIMEMultipart("alternative")
@ -89,6 +96,22 @@ async def send_unconfirm_message(room_order, orders):
 	for message in memberMessages:
 		await sendEmail(message)

-def render_email_template(title = "", body = ""):
-	tpl = Environment(loader=FileSystemLoader("tpl"), autoescape=False).get_template('email/comunication.html')
-	return str(tpl.render(title=title, body=body))
+async def send_missing_propic_message(order, missingPropic, missingFursuitPropic):
+	t = []
+	if(missingPropic): t.append("your propic")
+	if(missingFursuitPropic): t.append("your fursuit's badge")
+	missingText = " and ".join(t)
+
+	plain_body = EMAILS_TEXT["MISSING_PROPIC_TEXT"]['plain'].format(order.name, missingText)
+	html_body = render_email_template(EMAILS_TEXT["MISSING_PROPIC_TITLE"], EMAILS_TEXT["MISSING_PROPIC_TEXT"]['html'].format(order.name, missingText))
+	plain_text = MIMEText(plain_body, "plain")
+	html_text = MIMEText(html_body, "html")
+	message = MIMEMultipart("alternative")
+	message.attach(plain_text)
+	message.attach(html_text)
+	message['Subject'] = f"[{EMAIL_SENDER_NAME}] You haven't uploaded your badges yet!"
+	message['From'] = f'{EMAIL_SENDER_NAME} <{EMAIL_SENDER_MAIL}>'
+	message['To'] = f"{order.name} <{order.email}>"
+
+	await sendEmail(message)
+	
--- a/ext.py
+++ b/ext.py
@ -10,6 +10,9 @@ from sanic.log import logger
 from time import time
 from metrics import *
 import asyncio
+from threading import Lock
+import pretixClient
+import traceback

@dataclass
 class Order:
@ -156,12 +159,10 @@ class Order:
 		
 	async def edit_answer_fileUpload(self, name, fileName, mimeType, data : bytes):
 		if(mimeType != None and data != None):
-			async with httpx.AsyncClient() as client:
 			localHeaders = dict(headers)
 			localHeaders['Content-Type'] = mimeType
 			localHeaders['Content-Disposition'] = f'attachment; filename="{fileName}"'
-				incPretixWrite()
-				res = await client.post(join(base_url, 'upload'), headers=localHeaders, content=data)
+			res = await pretixClient.post("upload", baseUrl=base_url, headers=localHeaders, content=data)
 			res = res.json()
 			await self.edit_answer(name, res['id'])
 		else:
@ -184,10 +185,7 @@ class Order:
 				break
 					
 		if (not found) and (new_answer is not None):
-			
-			async with httpx.AsyncClient() as client:
-				incPretixRead()
-				res = await client.get(join(base_url_event, 'questions/'), headers=headers)
+			res = await pretixClient.get("questions/")
 			res = res.json()
 			for r in res['results']:
 				if r['identifier'] != name: continue
@ -201,7 +199,6 @@ class Order:
 		self.loadAns()
 			
 	async def send_answers(self):
-		async with httpx.AsyncClient() as client:
 		if DEV_MODE and EXTRA_PRINTS: logger.debug("[ANSWER POST] POSITION ID IS %s", self.position_id)
 		
 		for i, ans in enumerate(self.answers):
@ -214,8 +211,7 @@ class Order:
 			#	del self.answers[i]['options']
 			#	del self.answers[i]['option_identifiers']
 		
-			incPretixWrite()
-			res = await client.patch(join(base_url_event, f'orderpositions/{self.position_id}/'), headers=headers, json={'answers': self.answers})
+		res = await pretixClient.patch(f'orderpositions/{self.position_id}/', json={'answers': self.answers}, expectedStatusCodes=None)
 		
 		if res.status_code != 200:
 			for ans, err in zip(self.answers, res.json()['answers']):
@ -247,9 +243,7 @@ class Quotas:
 		return 0	

 async def get_quotas(request: Request=None):
-	async with httpx.AsyncClient() as client:
-		incPretixRead()
-		res = await client.get(join(base_url_event, 'quotas/?order=id&with_availability=true'), headers=headers)
+	res = await pretixClient.get('quotas/?order=id&with_availability=true')
 	res = res.json()
 	
 	return Quotas(res)
@ -261,7 +255,7 @@ async def get_order(request: Request=None):
 class OrderManager:
 	def __init__(self):
 		self.lastCacheUpdate = 0
-		self.updating = False
+		self.updating : Lock = Lock()
 		self.empty()

 	def empty(self):
@ -269,65 +263,90 @@ class OrderManager:
 		self.order_list = []

 	# Will fill cache once the last cache update is greater than cache expire time
-	async def update_cache(self):
+	async def update_cache(self, check_itemsQuestions=False):
 		t = time()
 		to_return = False
-		if(t - self.lastCacheUpdate > CACHE_EXPIRE_TIME and not self.updating):
+		success = True
+		if(t - self.lastCacheUpdate > CACHE_EXPIRE_TIME and not self.updating.locked()):
 			to_return = True
-			await self.fill_cache()
-		return to_return
+			success = await self.fill_cache(check_itemsQuestions=check_itemsQuestions)
+		return (to_return, success)

-	def add_cache(self, order):
-		self.cache[order.code] = order
-		if not order.code in self.order_list:
-			self.order_list.append(order.code)
+	def add_cache(self, order, cache=None, orderList=None):
+		# Extra params for dry runs
+		if(cache is None):
+			cache = self.cache
+		if(orderList is None):
+			orderList = self.order_list

-	def remove_cache(self, code):
-		if code in self.cache:
-			del self.cache[code]
-			self.order_list.remove(code)
+		cache[order.code] = order
+		if not order.code in orderList:
+			orderList.append(order.code)

-	async def fill_cache(self):
+	def remove_cache(self, code, cache=None, orderList=None):
+		# Extra params for dry runs
+		if(cache is None):
+			cache = self.cache
+		if(orderList is None):
+			orderList = self.order_list
+
+		if code in cache:
+			del cache[code]
+			orderList.remove(code)
+	
+	async def fill_cache(self, check_itemsQuestions=False) -> bool:
 		# Check cache lock
-		if self.updating == True: return
-		# Set cache lock
-		self.updating = True
+		self.updating.acquire()
 		start_time = time()
 		logger.info("[CACHE] Filling cache...")
 		# Index item's ids
-		await load_items()
+		r = await load_items()
+		if(not r and check_itemsQuestions):
+			logger.error("[CACHE] Items were not loading correctly. Aborting filling cache...")
+			return False

 		# Index questions' types
-		await load_questions()
+		r = await load_questions()
+		if(not r and check_itemsQuestions):
+			logger.error("[CACHE] Questions were not loading correctly. Aborting filling cache...")
+			return False

-		# Clear cache data completely
-		self.empty()
+		cache = {}
+		orderList = []
+		success = True
 		p = 0
 		try:
-			async with httpx.AsyncClient() as client:
 			while 1:
 				p += 1
-					incPretixRead()
-					res = await client.get(join(base_url_event, f"orders/?page={p}"), headers=headers)
+				res = await pretixClient.get(f"orders/?page={p}", expectedStatusCodes=[200, 404])
 				if res.status_code == 404: break
 				# Parse order data
 				data = res.json()
 				for o in data['results']:
 					o = Order(o)
 					if o.status in ['canceled', 'expired']:
-							self.remove_cache(o.code)
+						self.remove_cache(o.code, cache=cache, orderList=orderList)
 					else:
-							self.add_cache(Order(o))
+						self.add_cache(Order(o), cache=cache, orderList=orderList)
 			self.lastCacheUpdate = time()
 			logger.info(f"[CACHE] Cache filled in {self.lastCacheUpdate - start_time}s.")
-		except Exception as ex:
-			logger.error("[CACHE] Error while refreshing cache.", ex)
+		except Exception:
+			logger.error(f"[CACHE] Error while refreshing cache.\n{traceback.format_exc()}")
+			success = False
 		finally:
-			self.updating = False
+			self.updating.release()
+
+		# Apply new cache if there were no errors
+		if(success):
+			self.cache = cache
+			self.order_list = orderList
+
 		# Validating rooms
 		rooms = list(filter(lambda o: (o.code == o.room_id), self.cache.values()))
 		asyncio.create_task(validate_rooms(None, rooms, self))

+		return success
+	
 	async def get_order(self, request=None, code=None, secret=None, nfc_id=None, cached=False):

 		# if it's a nfc id, just retorn it
@ -349,9 +368,7 @@ class OrderManager:
 		if re.match('^[A-Z0-9]{5}$', code or '') and (secret is None or re.match('^[a-z0-9]{16,}$', secret)):
 			if DEV_MODE and EXTRA_PRINTS: logger.debug(f'Fetching {code} with secret {secret}')

-			async with httpx.AsyncClient() as client:
-				incPretixRead()
-				res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers)
+			res = await pretixClient.get(f"orders/{code}/", expectedStatusCodes=None)
 			if res.status_code != 200:
 				if request:
 					raise exceptions.Forbidden("Your session has expired due to order deletion or change! Please check your E-Mail for more info.")
--- a/messages.py
+++ b/messages.py
@ -6,12 +6,24 @@ ROOM_ERROR_TYPES = {
 	'capacity_mismatch': "The number of people in your room mismatches your type of ticket."
 }

-ROOM_UNCONFIRM_TITLE = "Your room got unconfirmed"
-ROOM_UNCONFIRM_TEXT = {
+EMAILS_TEXT = {
+	"ROOM_UNCONFIRM_TITLE": "Your room got unconfirmed",
+	"ROOM_UNCONFIRM_TEXT": {
 		'html': "Hello <b>{0}</b><br>We had to <b>unconfirm</b> your room <i>'{1}'</i> due to the following issues:<br></p>{2}<br><p>Please contact your room's owner or contact our support for further informations at <a href=\"https://furizon.net/contact/\"> https://furizon.net/contact/</a>.<br>Thank you.<br><br><a class=\"link\" style=\"background-color: #1095c1; color: #fff;\" href=\"https://reg.furizon.net/manage/welcome\">Manage booking</a>",
+
 		'plain': "Hello {0}\nWe had to unconfirm your room '{1}' due to the following issues:\n{2}\nPlease contact your room's owner or contact our support for further informations at https://furizon.net/contact/.\nThank you\n\nTo manage your booking: https://reg.furizon.net/manage/welcome"
+	},
+	
+
+	"MISSING_PROPIC_TITLE": "You haven't uploaded your badges yet!",
+	"MISSING_PROPIC_TEXT": {
+		'html': "Hello <b>{0}</b><br>We noticed you still have to <b>upload {1}</b>!<br>Please enter your booking page at <a href=\"https://reg.furizon.net/manage/welcome\"> https://reg.furizon.net/manage/welcome</a> and <b>upload them</b> under the <i>\"Badge Customization\"</i> section.<br>Thank you.<br><br><a class=\"link\" style=\"background-color: #1095c1; color: #fff;\" href=\"https://reg.furizon.net/manage/welcome\">Manage booking</a>",
+
+        'plain': "Hello {0}\nWe noticed you still have to upload {1}!\nPlease enter your booking page at https://reg.furizon.net/manage/welcome and upload them under the \"Badge Customization\" section.\nThank you."
+    },
 }

+
 NOSECOUNT = {
 	'filters': {
 		'capacity': "Here are some furs that share your room type and don't have a confirmed room."
--- a/metrics.py
+++ b/metrics.py
@ -3,8 +3,10 @@ from logging import LogRecord
 from config import *

 METRICS_REQ_NO = 0
+METRICS_ERR_NO = 0 # Errors served to the clients
 METRICS_PRETIX_READ = 0
 METRICS_PRETIX_WRITE = 0
+METRICS_PRETIX_ERRORS = 0 # Errors requesting pretix's backend

 def incPretixRead():
 	global METRICS_PRETIX_READ
@ -14,19 +16,32 @@ def incPretixWrite():
 	global METRICS_PRETIX_WRITE
 	METRICS_PRETIX_WRITE += 1

+def incPretixErrors():
+	global METRICS_PRETIX_ERRORS
+	METRICS_PRETIX_ERRORS += 1
+
 def incReqNo():
 	global METRICS_REQ_NO
 	METRICS_REQ_NO += 1

+def incErrorNo(): # Errors served to the clients
+	global METRICS_ERR_NO
+	METRICS_ERR_NO += 1
+
+
 def getMetricsText():
 	global METRICS_REQ_NO
+	global METRICS_ERR_NO
 	global METRICS_PRETIX_READ
 	global METRICS_PRETIX_WRITE
+	global METRICS_PRETIX_ERRORS
 	out = []

 	out.append(f'sanic_request_count{{}} {METRICS_REQ_NO}')
+	out.append(f'sanic_error_count{{}} {METRICS_ERR_NO}')
 	out.append(f'webint_pretix_read_count{{}} {METRICS_PRETIX_READ}')
 	out.append(f'webint_pretix_write_count{{}} {METRICS_PRETIX_WRITE}')
+	out.append(f'webint_pretix_error_count{{}} {METRICS_PRETIX_ERRORS}')

 	return "\n".join(out)

@ -35,17 +50,29 @@ def getRoomCountersText(request):
 	try :
 		daily = 0
 		counters = {}
+		counters_early = {}
+		counters_late = {}
 		for id in ROOM_TYPE_NAMES.keys():
 			counters[id] = 0
+			counters_early[id] = 0
+			counters_late[id] = 0

 		for order in request.app.ctx.om.cache.values():
 			if(order.daily):
 				daily += 1
 			else:
 				counters[order.bed_in_room] += 1
+				if(order.has_early):
+					counters_early[order.bed_in_room] += 1
+				if(order.has_late):
+					counters_late[order.bed_in_room] += 1

 		for id, count in counters.items():
-			out.append(f'webint_order_room_counter{{label="{ROOM_TYPE_NAMES[id]}"}} {count}')
+			out.append(f'webint_order_room_counter{{days="normal", label="{ROOM_TYPE_NAMES[id]}"}} {count}')
+		for id, count in counters_early.items():
+			out.append(f'webint_order_room_counter{{days="early", label="{ROOM_TYPE_NAMES[id]}"}} {count}')
+		for id, count in counters_late.items():
+			out.append(f'webint_order_room_counter{{days="late", label="{ROOM_TYPE_NAMES[id]}"}} {count}')
 		out.append(f'webint_order_room_counter{{label="Daily"}} {daily}')

 	except Exception as e:
--- a/pretixClient.py
+++ b/pretixClient.py
@ -0,0 +1,49 @@
+import httpx
+from utils import *
+from config import *
+from sanic.log import logger
+from metrics import *
+import traceback
+import asyncio
+
+async def get(url, baseUrl=base_url_event, headers=headers, expectedStatusCodes=[200]) -> httpx.Response:
+	async def func(client : httpx.AsyncClient) -> httpx.Request:
+		return await client.get(join(baseUrl, url), headers=headers)
+	return await doReq(url, func, incPretixRead, expectedStatusCodes, "GETing")
+
+async def post(url, content=None, json=None, baseUrl=base_url_event, headers=headers, expectedStatusCodes=[200]) -> httpx.Response:
+	async def func(client : httpx.AsyncClient) -> httpx.Request:
+		return await client.post(join(baseUrl, url), headers=headers, content=content, json=json)
+	return await doReq(url, func, incPretixWrite, expectedStatusCodes, "POSTing")
+	
+async def patch(url, json, baseUrl=base_url_event, headers=headers, expectedStatusCodes=[200]) -> httpx.Response:
+	async def func(client : httpx.AsyncClient) -> httpx.Request:
+		return await client.patch(join(baseUrl, url), headers=headers, json=json)
+	return await doReq(url, func, incPretixWrite, expectedStatusCodes, "PATCHing")
+
+
+
+async def doReq(url, httpxFunc, metricsFunc, expectedStatusCodes, opLogString) -> httpx.Response:
+	res = None
+	async with httpx.AsyncClient(timeout=PRETIX_REQUESTS_TIMEOUT) as client:
+		requests = 0
+		for requests in range(PRETIX_REQUESTS_MAX):
+			try:
+				metricsFunc()
+				res = await httpxFunc(client)
+
+				if expectedStatusCodes is not None and res.status_code not in expectedStatusCodes:
+					incPretixErrors()
+					logger.warning(f"[PRETIX] Got an unexpected status code ({res.status_code}) while {opLogString} '{url}'. Allowed status codes: {', '.join(expectedStatusCodes)}")
+					continue
+				break
+			except Exception as e:
+				incPretixErrors()
+				logger.warning(f"[PRETIX] An error ({requests}) occurred while {opLogString} '{url}':\n{traceback.format_exc()}")
+
+			requests += 1
+		else:
+			logger.error(f"[PRETIX] Reached PRETIX_REQUESTS_MAX ({PRETIX_REQUESTS_MAX}) while {opLogString} '{url}'. Aborting")
+			raise httpx.TimeoutException(f"PRETIX_REQUESTS_MAX reached while {opLogString} to pretix.")
+
+	return res
--- a/res/styles/base.css
+++ b/res/styles/base.css
@ -12,6 +12,21 @@ summary:has(span.status) {
    background-color: #ffaf0377;
 }

+.rainbow-text {
+    background-image: repeating-linear-gradient(90deg, #ff0000, #ffff00, #00ff00, #00ffff, #0000ff, #ff00ff, #ff0000, #ffff00, #00ff00, #00ffff, #0000ff, #ff00ff);
+    background-size: 2000% 2000%;
+    color: transparent;
+    background-clip: text;
+    -webkit-background-clip: text;
+    -webkit-text-fill-color: transparent;
+    animation: rainbow 4s linear infinite;
+}
+
+@keyframes rainbow { 
+    0% { background-position:0% 0%; }
+    100% { background-position:57.75% 0%; }
+}
+
 /* Dark theme */
@media only screen and (prefers-color-scheme: dark) {
    .icon {filter: invert(1);}
--- a/res/styles/navbar.css
+++ b/res/styles/navbar.css
@ -14,6 +14,7 @@ nav#topbar {
    top: 0rem;
    transition: top 300ms;
    line-height: 2em;
+    max-width:98vw;
 }

 nav#topbar a {
@ -42,21 +43,6 @@ nav img {
    box-sizing: border-box;
 }

-a.rainbow-text {
-    background-image: repeating-linear-gradient(90deg, #ff0000, #ffff00, #00ff00, #00ffff, #0000ff, #ff00ff, #ff0000, #ffff00, #00ff00, #00ffff, #0000ff, #ff00ff);
-    background-size: 2000% 2000%;
-    color: transparent;
-    background-clip: text;
-    -webkit-background-clip: text;
-    -webkit-text-fill-color: transparent;
-    animation: rainbow 4s linear infinite;
-}
-
-@keyframes rainbow { 
-    0% { background-position:0% 0%; }
-    100% { background-position:57.75% 0%; }
-}
-
 nav a#mobileMenu {
    display: none;
 }
--- a/room.py
+++ b/room.py
@ -335,21 +335,6 @@ async def confirm_room(request, order: Order, quotas: Quotas):
 		await rm.edit_answer('pending_roommates', None)
 		await rm.edit_answer('pending_room', None)
 	
-	# This should now be useless because in the ticket there already is the ticket/room type
-	# thing = {
-	# 	'order': order.code,
-	# 	'addon_to': order.position_positionid,
-	# 	'item': ITEM_IDS['room'],
-	# 	'variation': ROOM_MAP[len(room_members)]
-	# }
-	# 
-	# async with httpx.AsyncClient() as client:
-	#	incPretixRead()
-	# 	res = await client.post(join(base_url_event, "orderpositions/"), headers=headers, json=thing)
-	# 	
-	# 	if res.status_code != 201:
-	# 		raise exceptions.BadRequest("Something has gone wrong! Please contact support immediately")
-	
 	for rm in room_members:
 		await rm.send_answers()

--- a/stuff/rclone/clone.sh
+++ b/stuff/rclone/clone.sh
@ -0,0 +1,7 @@
+#!/bin/bash
+
+/usr/bin/tar -czvf /tmp/backupRclone.tar.gz /home/ /root/ /etc/ /var/backups/ /var/log/ /var/mail/ /var/pretix-data/ /var/prometheus-data/ /var/spool/ /var/www/ /var/lib/grafana/ /var/lib/redis/
+
+/usr/bin/rclone sync /tmp/backupRclone.tar.gz webservice:/backups/backupRclone.tar.gz -P --stats=1s --bwlimit 15M
+
+/usr/bin/rm /tmp/backupRclone.tar.gz
--- a/stuff/rclone/cloneHeadless.sh
+++ b/stuff/rclone/cloneHeadless.sh
@ -0,0 +1,8 @@
+#!/bin/bash
+source /root/.profile
+
+/usr/bin/tar -czf /tmp/backupRclone.tar.gz /home/ /root/ /etc/ /var/backups/ /var/log/ /var/mail/ /var/pretix-data/ /var/prometheus-data/ /var/spool/ /var/www/ /var/lib/grafana/ /var/lib/redis/
+
+/usr/bin/rclone sync /tmp/backupRclone.tar.gz webservice:/backups/backupRclone.tar.gz --bwlimit 15M
+
+/usr/bin/rm /tmp/backupRclone.tar.gz
--- a/tpl/admin.html
+++ b/tpl/admin.html
@ -13,6 +13,7 @@
 		<a href="/manage/admin/cache/clear" role="button" title="Reload the orders' data and the re-sync items' indexes from pretix">Clear cache</a>
 		<a href="/manage/nosecount" role="button" title="Shortcut to the nosecount's admin data">Manage rooms</a>
 		<a href="/manage/admin/room/verify" role="button" title="Will unconfirm rooms that fail the default check. Useful when editing answers from Pretix">Verify Rooms</a>
+		<a href="/manage/admin/propic/remind" role="button" title="Will remind via mail all people who event uploaded a badge to do it">Remind badge upload</a>
 		<hr>
 	</main>
 	
--- a/tpl/blocks/room.html
+++ b/tpl/blocks/room.html
@ -5,7 +5,7 @@
 	<p class="notice" style="background:#0881c0"><b>Note! </b> Only people with the same room type can be roommates. If you need help, contact the <a href="https://furizon.net/contact/">Furizon's Staff</a>.</p>
 	
 	{% if not order.room_confirmed %}
-	<p class="notice" style="background:#0881c0"><b><a href="https://furizon.net/manage/nosecount?filter=capacity">Check here</a> for any fur who share your room type.</p>
+	<p class="notice" style="background:#0881c0"><b><a href="/manage/nosecount?filter=capacity">Check here</a> for any fur who share your room type.</p>
 	{% endif %}

 	{# Show alert if room owner has wrong people inside #}
--- a/tpl/error.html
+++ b/tpl/error.html
@ -1,10 +1,10 @@
 {% extends "base.html" %}
-{% block title %}Error {{exception.status_code}}{% endblock %}
+{% block title %}Error {{status_code}}{% endblock %}
 {% block main %}
 	<main class="container">
-		<h1>{{exception.status_code}}</h1>
+		<h1>{{status_code}}</h1>
 		<p>{{exception}}</p>
-		{% if exception.status_code == 409 %}
+		{% if status_code == 409 %}
 			<p>Retrying in 1 second...</p>
 			<meta http-equiv="refresh" content="1">
 		{% endif %}
--- a/utils.py
+++ b/utils.py
@ -7,6 +7,8 @@ from email_util import send_unconfirm_message
 from sanic.response import text, html, redirect, raw
 from sanic.log import logger
 from metrics import *
+import pretixClient
+import traceback

 METADATA_TAG = "meta_data"
 VARIATIONS_TAG = "variations"
@ -28,34 +30,38 @@ QUESTION_TYPES = { #https://docs.pretix.eu/en/latest/api/resources/questions.htm
 TYPE_OF_QUESTIONS = {} # maps questionId -> type


-async def load_questions():
+async def load_questions() -> bool:
 	global TYPE_OF_QUESTIONS
-	TYPE_OF_QUESTIONS.clear()
-	async with httpx.AsyncClient() as client:
+	# TYPE_OF_QUESTIONS.clear() It should not be needed
+	logger.info("[QUESTIONS] Loading questions...")
+	success = True
+	try:
 		p = 0
 		while 1:
 			p += 1
-			incPretixRead()
-			res = await client.get(join(base_url_event, f"questions/?page={p}"), headers=headers)
-
+			res = await pretixClient.get(f"questions/?page={p}", expectedStatusCodes=[200, 404])
 			if res.status_code == 404: break

 			data = res.json()
 			for q in data['results']:
 				TYPE_OF_QUESTIONS[q['id']] = q['type']
+	except Exception:
+		logger.warning(f"[QUESTIONS] Error while loading questions.\n{traceback.format_exc()}")
+		success = False
+	return success

-async def load_items():
+async def load_items() -> bool:
 	global ITEMS_ID_MAP
 	global ITEM_VARIATIONS_MAP
 	global CATEGORIES_LIST_MAP
 	global ROOM_TYPE_NAMES
-	async with httpx.AsyncClient() as client:
+	logger.info("[ITEMS] Loading items...")
+	success = True
+	try:
 		p = 0
 		while 1:
 			p += 1
-			incPretixRead()
-			res = await client.get(join(base_url_event, f"items/?page={p}"), headers=headers)
-
+			res = await pretixClient.get(f"items/?page={p}", expectedStatusCodes=[200, 404])
 			if res.status_code == 404: break

 			data = res.json()
@ -85,6 +91,10 @@ async def load_items():
 			logger.debug(f'Mapped Variations: %s', ITEM_VARIATIONS_MAP)
 			logger.debug(f'Mapped categories: %s', CATEGORIES_LIST_MAP)
 			logger.debug(f'Mapped Rooms: %s', ROOM_TYPE_NAMES)
+	except Exception:
+		logger.warning(f"[ITEMS] Error while loading items.\n{traceback.format_exc()}")
+		success = False
+	return success

 # Tries to get an item name from metadata. Prints a warning if an item has no metadata
 def check_and_get_name(type, q):
@ -141,7 +151,7 @@ async def get_people_in_room_by_code(request, code, om=None):
 	await om.update_cache()
 	return filter(lambda rm: rm.room_id == code, om.cache.values())

-async def unconfirm_room_by_order(order, room_members:[]=None, throw=True, request=None, om=None):
+async def unconfirm_room_by_order(order, room_members=None, throw=True, request=None, om=None):
 	if not om: om = request.app.ctx.om
 	if not order.room_confirmed:
 		if throw: