Better handling of pretix's unresponsiviness

admin.py

@@ -20,7 +20,8 @@ async def credentials_check(request: Request):
 
 @bp.get('/cache/clear')
 async def clear_cache(request, order:Order):
-	await request.app.ctx.om.fill_cache()
+	success = await request.app.ctx.om.fill_cache()
+	if not success: raise exceptions.ServerError("An error occurred while loading the cache")
 	return redirect(f'/manage/admin')
 
 @bp.get('/loginas/<code>')
@@ -40,8 +41,8 @@ async def login_as(request, code, order:Order):
 
 @bp.get('/room/verify')
 async def verify_rooms(request, order:Order):
-	already_checked = await request.app.ctx.om.update_cache()
+	already_checked, success = await request.app.ctx.om.update_cache()
-	if not already_checked:
+	if not already_checked and success:
 		orders = filter(lambda x: x.status not in ['c', 'e'] and x.room_id == x.code, request.app.ctx.om.cache.values())
 		await validate_rooms(request, orders, None)
 	return redirect(f'/manage/admin')

app.py

@@ -16,6 +16,8 @@ import requests
 import sys
 from sanic.log import logger, logging, access_logger
 from metrics import *
+import pretixClient
+import traceback
 
 app = Sanic(__name__)
 app.static("/res", "res/")
@@ -35,17 +37,27 @@ from admin import bp as admin_bp
 
 app.blueprint([room_bp, karaoke_bp, propic_bp, export_bp, stats_bp, api_bp, carpooling_bp, checkin_bp, admin_bp])
 
-@app.exception(exceptions.SanicException)
-async def clear_session(request, exception):
-	logger.warning(f"{request} -> {exception}")
-	tpl = app.ctx.tpl.get_template('error.html')
-	r = html(tpl.render(exception=exception))
 
-	if exception.status_code == 403:
+async def clear_session(response):
-		r.delete_cookie("foxo_code")
+	response.delete_cookie("foxo_code")
-		r.delete_cookie("foxo_secret")
+	response.delete_cookie("foxo_secret")
+
+@app.exception(exceptions.SanicException if DEV_MODE else Exception)
+async def handleException(request, exception):
+	incErrorNo()
+	logger.warning(f"{request} -> {exception}")
+	statusCode = exception.status_code if hasattr(exception, 'status_code') else 500
+	try:
+		tpl = app.ctx.tpl.get_template('error.html')
+		r = html(tpl.render(exception=exception, status_code=statusCode))
+	except:
+		traceback.print_exc()
+
+	if statusCode == 403:
+		clear_session(r)
 	return r
 
+
 @app.before_server_start
 async def main_start(*_):
 	logger.info(f"[{app.name}] >>>>>> main_start <<<<<<")
@@ -56,7 +68,10 @@ async def main_start(*_):
 	
 	app.ctx.om = OrderManager()
 	if FILL_CACHE:
-		await app.ctx.om.update_cache()
+		checked, success = await app.ctx.om.update_cache(check_itemsQuestions=True)
+		if checked and not success:
+			logger.error(f"[{app.name}] Failure in app startup: An error occurred while loading items or questions or cache.")
+			app.stop()
 	
 	app.ctx.nfc_counts = sqlite3.connect('data/nfc_counts.db')
 	
@@ -90,9 +105,7 @@ async def redirect_explore(request, code, secret, order: Order, secret2=None):
 	if order and order.code != code: order = None
 
 	if not order:
-		async with httpx.AsyncClient() as client:
+		res = await pretixClient.get(f"orders/{code}/", expectedStatusCodes=None)
-			incPretixRead()
-			res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers)
 		
 		if res.status_code != 200:
 			raise exceptions.NotFound("This order code does not exist. Check that your order wasn't deleted, or the link is correct.")
@@ -155,11 +168,9 @@ async def download_ticket(request, order: Order):
 	if not order.status != 'confirmed':
 		raise exceptions.Forbidden("You are not allowed to download this ticket.")
 		
-	async with httpx.AsyncClient() as client:
+	res = await pretixClient.get(f"orders/{order.code}/download/pdf/", expectedStatusCodes=[200, 404, 409, 403])
-		incPretixRead()
-		res = await client.get(join(base_url_event, f"orders/{order.code}/download/pdf/"), headers=headers)
 	
-	if res.status_code == 409:
+	if res.status_code == 409 or res.status_code == 404:
 		raise exceptions.SanicException("Your ticket is still being generated. Please try again later!", status_code=res.status_code)
 	elif res.status_code == 403:
 		raise exceptions.SanicException("You can download your ticket only after the order has been confirmed and paid. Try later!", status_code=400)
@@ -208,7 +219,7 @@ if __name__ == "__main__":
 	# to let it start in the correct order. The following piece of code makes sure that pretix is running and can talk to 
 	# postgres before actually starting the reserved area, since this operation requires a cache-fill in startup
 	print("Waiting for pretix to be up and running", file=sys.stderr)
-	while True:
+	while not SKIP_HEALTHCHECK:
 		print("Trying connecting to pretix...", file=sys.stderr)
 		try:
 			incPretixRead()
@@ -218,7 +229,7 @@ if __name__ == "__main__":
 				print("Healtchecking...", file=sys.stderr)
 				incPretixRead()
 				res = requests.get(join(domain, "healthcheck"), headers=headers)
-				if(res.status_code == 200 or SKIP_HEALTHCHECK):
+				if(res.status_code == 200):
 					break
 		except:
 			pass

checkin.py

@@ -12,6 +12,7 @@ from time import time
 from urllib.parse import unquote
 import json
 from metrics import *
+import pretixClient
 
 bp = Blueprint("checkin", url_prefix="/checkin")
 
@@ -64,9 +65,7 @@ async def do_checkin(request):
 	await order.send_answers()
 	
 	if not order.checked_in:
-		async with httpx.AsyncClient() as client:
+		await pretixClient.post("", baseUrl=base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]})
-			incPretixWrite()
-			res = await client.post(base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]}, headers=headers)
 	
 	tpl = request.app.ctx.tpl.get_template('checkin_3.html')
 	return html(tpl.render(order=order, room_owner=room_owner, roommates=roommates))

config.example.py

@@ -21,6 +21,11 @@ PROPIC_MIN_SIZE = (125, 125) # (Width, Height)
 TG_BOT_API = '123456789:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
 TG_CHAT_ID = -1234567
 
+import httpx
+# Number of tries for a request to the pretix's backend
+PRETIX_REQUESTS_MAX = 3
+PRETIX_REQUESTS_TIMEOUT = httpx.Timeout(15.0, read=30.0, connect=45.0, pool=None) # Timeout for httpx requests in seconds
+
 # These order codes have additional functions.
 ADMINS = ['XXXXX', 'YYYYY']
 # A list of staff_roles 

ext.py

@@ -10,6 +10,9 @@ from sanic.log import logger
 from time import time
 from metrics import *
 import asyncio
+from threading import Lock
+import pretixClient
+import traceback
 
 @dataclass
 class Order:
@@ -156,12 +159,10 @@ class Order:
 		
 	async def edit_answer_fileUpload(self, name, fileName, mimeType, data : bytes):
 		if(mimeType != None and data != None):
-			async with httpx.AsyncClient() as client:
 			localHeaders = dict(headers)
 			localHeaders['Content-Type'] = mimeType
 			localHeaders['Content-Disposition'] = f'attachment; filename="{fileName}"'
-				incPretixWrite()
+			res = await pretixClient.post("upload", baseUrl=base_url, headers=localHeaders, content=data)
-				res = await client.post(join(base_url, 'upload'), headers=localHeaders, content=data)
 			res = res.json()
 			await self.edit_answer(name, res['id'])
 		else:
@@ -184,10 +185,7 @@ class Order:
 				break
 					
 		if (not found) and (new_answer is not None):
-			
+			res = await pretixClient.get("questions/")
-			async with httpx.AsyncClient() as client:
-				incPretixRead()
-				res = await client.get(join(base_url_event, 'questions/'), headers=headers)
 			res = res.json()
 			for r in res['results']:
 				if r['identifier'] != name: continue
@@ -201,7 +199,6 @@ class Order:
 		self.loadAns()
 			
 	async def send_answers(self):
-		async with httpx.AsyncClient() as client:
 		if DEV_MODE and EXTRA_PRINTS: logger.debug("[ANSWER POST] POSITION ID IS %s", self.position_id)
 		
 		for i, ans in enumerate(self.answers):
@@ -214,8 +211,7 @@ class Order:
 			#	del self.answers[i]['options']
 			#	del self.answers[i]['option_identifiers']
 		
-			incPretixWrite()
+		res = await pretixClient.patch(f'orderpositions/{self.position_id}/', json={'answers': self.answers}, expectedStatusCodes=None)
-			res = await client.patch(join(base_url_event, f'orderpositions/{self.position_id}/'), headers=headers, json={'answers': self.answers})
 		
 		if res.status_code != 200:
 			for ans, err in zip(self.answers, res.json()['answers']):
@@ -247,9 +243,7 @@ class Quotas:
 		return 0	
 
 async def get_quotas(request: Request=None):
-	async with httpx.AsyncClient() as client:
+	res = await pretixClient.get('quotas/?order=id&with_availability=true')
-		incPretixRead()
-		res = await client.get(join(base_url_event, 'quotas/?order=id&with_availability=true'), headers=headers)
 	res = res.json()
 	
 	return Quotas(res)
@@ -261,7 +255,7 @@ async def get_order(request: Request=None):
 class OrderManager:
 	def __init__(self):
 		self.lastCacheUpdate = 0
-		self.updating = False
+		self.updating : Lock = Lock()
 		self.empty()
 
 	def empty(self):
@@ -269,65 +263,90 @@ class OrderManager:
 		self.order_list = []
 
 	# Will fill cache once the last cache update is greater than cache expire time
-	async def update_cache(self):
+	async def update_cache(self, check_itemsQuestions=False):
 		t = time()
 		to_return = False
-		if(t - self.lastCacheUpdate > CACHE_EXPIRE_TIME and not self.updating):
+		success = True
+		if(t - self.lastCacheUpdate > CACHE_EXPIRE_TIME and not self.updating.locked()):
 			to_return = True
-			await self.fill_cache()
+			success = await self.fill_cache(check_itemsQuestions=check_itemsQuestions)
-		return to_return
+		return (to_return, success)
 
-	def add_cache(self, order):
+	def add_cache(self, order, cache=None, orderList=None):
-		self.cache[order.code] = order
+		# Extra params for dry runs
-		if not order.code in self.order_list:
+		if(cache is None):
-			self.order_list.append(order.code)
+			cache = self.cache
+		if(orderList is None):
+			orderList = self.order_list
 
-	def remove_cache(self, code):
+		cache[order.code] = order
-		if code in self.cache:
+		if not order.code in orderList:
-			del self.cache[code]
+			orderList.append(order.code)
-			self.order_list.remove(code)
 
-	async def fill_cache(self):
+	def remove_cache(self, code, cache=None, orderList=None):
+		# Extra params for dry runs
+		if(cache is None):
+			cache = self.cache
+		if(orderList is None):
+			orderList = self.order_list
+
+		if code in cache:
+			del cache[code]
+			orderList.remove(code)
+	
+	async def fill_cache(self, check_itemsQuestions=False) -> bool:
 		# Check cache lock
-		if self.updating == True: return
+		self.updating.acquire()
-		# Set cache lock
-		self.updating = True
 		start_time = time()
 		logger.info("[CACHE] Filling cache...")
 		# Index item's ids
-		await load_items()
+		r = await load_items()
+		if(not r and check_itemsQuestions):
+			logger.error("[CACHE] Items were not loading correctly. Aborting filling cache...")
+			return False
 
 		# Index questions' types
-		await load_questions()
+		r = await load_questions()
+		if(not r and check_itemsQuestions):
+			logger.error("[CACHE] Questions were not loading correctly. Aborting filling cache...")
+			return False
 
-		# Clear cache data completely
+		cache = {}
-		self.empty()
+		orderList = []
+		success = True
 		p = 0
 		try:
-			async with httpx.AsyncClient() as client:
 			while 1:
 				p += 1
-					incPretixRead()
+				res = await pretixClient.get(f"orders/?page={p}", expectedStatusCodes=[200, 404])
-					res = await client.get(join(base_url_event, f"orders/?page={p}"), headers=headers)
 				if res.status_code == 404: break
 				# Parse order data
 				data = res.json()
 				for o in data['results']:
 					o = Order(o)
 					if o.status in ['canceled', 'expired']:
-							self.remove_cache(o.code)
+						self.remove_cache(o.code, cache=cache, orderList=orderList)
 					else:
-							self.add_cache(Order(o))
+						self.add_cache(Order(o), cache=cache, orderList=orderList)
 			self.lastCacheUpdate = time()
 			logger.info(f"[CACHE] Cache filled in {self.lastCacheUpdate - start_time}s.")
-		except Exception as ex:
+		except Exception:
-			logger.error("[CACHE] Error while refreshing cache.", ex)
+			logger.error(f"[CACHE] Error while refreshing cache.\n{traceback.format_exc()}")
+			success = False
 		finally:
-			self.updating = False
+			self.updating.release()
+
+		# Apply new cache if there were no errors
+		if(success):
+			self.cache = cache
+			self.order_list = orderList
+
 		# Validating rooms
 		rooms = list(filter(lambda o: (o.code == o.room_id), self.cache.values()))
 		asyncio.create_task(validate_rooms(None, rooms, self))
 
+		return success
+	
 	async def get_order(self, request=None, code=None, secret=None, nfc_id=None, cached=False):
 
 		# if it's a nfc id, just retorn it
@@ -349,9 +368,7 @@ class OrderManager:
 		if re.match('^[A-Z0-9]{5}$', code or '') and (secret is None or re.match('^[a-z0-9]{16,}$', secret)):
 			if DEV_MODE and EXTRA_PRINTS: logger.debug(f'Fetching {code} with secret {secret}')
 
-			async with httpx.AsyncClient() as client:
+			res = await pretixClient.get(f"orders/{code}/", expectedStatusCodes=None)
-				incPretixRead()
-				res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers)
 			if res.status_code != 200:
 				if request:
 					raise exceptions.Forbidden("Your session has expired due to order deletion or change! Please check your E-Mail for more info.")

metrics.py

@@ -3,8 +3,10 @@ from logging import LogRecord
 from config import *
 
 METRICS_REQ_NO = 0
+METRICS_ERR_NO = 0 # Errors served to the clients
 METRICS_PRETIX_READ = 0
 METRICS_PRETIX_WRITE = 0
+METRICS_PRETIX_ERRORS = 0 # Errors requesting pretix's backend
 
 def incPretixRead():
 	global METRICS_PRETIX_READ
@@ -14,19 +16,32 @@ def incPretixWrite():
 	global METRICS_PRETIX_WRITE
 	METRICS_PRETIX_WRITE += 1
 
+def incPretixErrors():
+	global METRICS_PRETIX_ERRORS
+	METRICS_PRETIX_ERRORS += 1
+
 def incReqNo():
 	global METRICS_REQ_NO
 	METRICS_REQ_NO += 1
 
+def incErrorNo(): # Errors served to the clients
+	global METRICS_ERR_NO
+	METRICS_ERR_NO += 1
+
+
 def getMetricsText():
 	global METRICS_REQ_NO
+	global METRICS_ERR_NO
 	global METRICS_PRETIX_READ
 	global METRICS_PRETIX_WRITE
+	global METRICS_PRETIX_ERRORS
 	out = []
 
 	out.append(f'sanic_request_count{{}} {METRICS_REQ_NO}')
+	out.append(f'sanic_error_count{{}} {METRICS_ERR_NO}')
 	out.append(f'webint_pretix_read_count{{}} {METRICS_PRETIX_READ}')
 	out.append(f'webint_pretix_write_count{{}} {METRICS_PRETIX_WRITE}')
+	out.append(f'webint_pretix_error_count{{}} {METRICS_PRETIX_ERRORS}')
 
 	return "\n".join(out)
 

pretixClient.py

@@ -0,0 +1,49 @@
+import httpx
+from utils import *
+from config import *
+from sanic.log import logger
+from metrics import *
+import traceback
+import asyncio
+
+async def get(url, baseUrl=base_url_event, headers=headers, expectedStatusCodes=[200]) -> httpx.Response:
+	async def func(client : httpx.AsyncClient) -> httpx.Request:
+		return await client.get(join(baseUrl, url), headers=headers)
+	return await doReq(url, func, incPretixRead, expectedStatusCodes, "GETing")
+
+async def post(url, content=None, json=None, baseUrl=base_url_event, headers=headers, expectedStatusCodes=[200]) -> httpx.Response:
+	async def func(client : httpx.AsyncClient) -> httpx.Request:
+		return await client.post(join(baseUrl, url), headers=headers, content=content, json=json)
+	return await doReq(url, func, incPretixWrite, expectedStatusCodes, "POSTing")
+	
+async def patch(url, json, baseUrl=base_url_event, headers=headers, expectedStatusCodes=[200]) -> httpx.Response:
+	async def func(client : httpx.AsyncClient) -> httpx.Request:
+		return await client.patch(join(baseUrl, url), headers=headers, json=json)
+	return await doReq(url, func, incPretixWrite, expectedStatusCodes, "PATCHing")
+
+
+
+async def doReq(url, httpxFunc, metricsFunc, expectedStatusCodes, opLogString) -> httpx.Response:
+	res = None
+	async with httpx.AsyncClient(timeout=PRETIX_REQUESTS_TIMEOUT) as client:
+		requests = 0
+		for requests in range(PRETIX_REQUESTS_MAX):
+			try:
+				metricsFunc()
+				res = await httpxFunc(client)
+
+				if expectedStatusCodes is not None and res.status_code not in expectedStatusCodes:
+					incPretixErrors()
+					logger.warning(f"[PRETIX] Got an unexpected status code ({res.status_code}) while {opLogString} '{url}'. Allowed status codes: {', '.join(expectedStatusCodes)}")
+					continue
+				break
+			except Exception as e:
+				incPretixErrors()
+				logger.warning(f"[PRETIX] An error ({requests}) occurred while {opLogString} '{url}':\n{traceback.format_exc()}")
+
+			requests += 1
+		else:
+			logger.error(f"[PRETIX] Reached PRETIX_REQUESTS_MAX ({PRETIX_REQUESTS_MAX}) while {opLogString} '{url}'. Aborting")
+			raise httpx.TimeoutException(f"PRETIX_REQUESTS_MAX reached while {opLogString} to pretix.")
+
+	return res

room.py

@@ -335,21 +335,6 @@ async def confirm_room(request, order: Order, quotas: Quotas):
 		await rm.edit_answer('pending_roommates', None)
 		await rm.edit_answer('pending_room', None)
 	
-	# This should now be useless because in the ticket there already is the ticket/room type
-	# thing = {
-	# 	'order': order.code,
-	# 	'addon_to': order.position_positionid,
-	# 	'item': ITEM_IDS['room'],
-	# 	'variation': ROOM_MAP[len(room_members)]
-	# }
-	# 
-	# async with httpx.AsyncClient() as client:
-	#	incPretixRead()
-	# 	res = await client.post(join(base_url_event, "orderpositions/"), headers=headers, json=thing)
-	# 	
-	# 	if res.status_code != 201:
-	# 		raise exceptions.BadRequest("Something has gone wrong! Please contact support immediately")
-	
 	for rm in room_members:
 		await rm.send_answers()
 

tpl/error.html

@@ -1,10 +1,10 @@
 {% extends "base.html" %}
-{% block title %}Error {{exception.status_code}}{% endblock %}
+{% block title %}Error {{status_code}}{% endblock %}
 {% block main %}
 	<main class="container">
-		<h1>{{exception.status_code}}</h1>
+		<h1>{{status_code}}</h1>
 		<p>{{exception}}</p>
-		{% if exception.status_code == 409 %}
+		{% if status_code == 409 %}
 			<p>Retrying in 1 second...</p>
 			<meta http-equiv="refresh" content="1">
 		{% endif %}

utils.py

@@ -7,6 +7,8 @@ from email_util import send_unconfirm_message
 from sanic.response import text, html, redirect, raw
 from sanic.log import logger
 from metrics import *
+import pretixClient
+import traceback
 
 METADATA_TAG = "meta_data"
 VARIATIONS_TAG = "variations"
@@ -28,34 +30,38 @@ QUESTION_TYPES = { #https://docs.pretix.eu/en/latest/api/resources/questions.htm
 TYPE_OF_QUESTIONS = {} # maps questionId -> type
 
 
-async def load_questions():
+async def load_questions() -> bool:
 	global TYPE_OF_QUESTIONS
-	TYPE_OF_QUESTIONS.clear()
+	# TYPE_OF_QUESTIONS.clear() It should not be needed
-	async with httpx.AsyncClient() as client:
+	logger.info("[QUESTIONS] Loading questions...")
+	success = True
+	try:
 		p = 0
 		while 1:
 			p += 1
-			incPretixRead()
+			res = await pretixClient.get(f"questions/?page={p}", expectedStatusCodes=[200, 404])
-			res = await client.get(join(base_url_event, f"questions/?page={p}"), headers=headers)
-
 			if res.status_code == 404: break
 
 			data = res.json()
 			for q in data['results']:
 				TYPE_OF_QUESTIONS[q['id']] = q['type']
+	except Exception:
+		logger.warning(f"[QUESTIONS] Error while loading questions.\n{traceback.format_exc()}")
+		success = False
+	return success
 
-async def load_items():
+async def load_items() -> bool:
 	global ITEMS_ID_MAP
 	global ITEM_VARIATIONS_MAP
 	global CATEGORIES_LIST_MAP
 	global ROOM_TYPE_NAMES
-	async with httpx.AsyncClient() as client:
+	logger.info("[ITEMS] Loading items...")
+	success = True
+	try:
 		p = 0
 		while 1:
 			p += 1
-			incPretixRead()
+			res = await pretixClient.get(f"items/?page={p}", expectedStatusCodes=[200, 404])
-			res = await client.get(join(base_url_event, f"items/?page={p}"), headers=headers)
-
 			if res.status_code == 404: break
 
 			data = res.json()
@@ -85,6 +91,10 @@ async def load_items():
 			logger.debug(f'Mapped Variations: %s', ITEM_VARIATIONS_MAP)
 			logger.debug(f'Mapped categories: %s', CATEGORIES_LIST_MAP)
 			logger.debug(f'Mapped Rooms: %s', ROOM_TYPE_NAMES)
+	except Exception:
+		logger.warning(f"[ITEMS] Error while loading items.\n{traceback.format_exc()}")
+		success = False
+	return success
 
 # Tries to get an item name from metadata. Prints a warning if an item has no metadata
 def check_and_get_name(type, q):