From 4e1120a93f2afd6f7b0a4fa590eb02b2ca071ed8 Mon Sep 17 00:00:00 2001 From: Stranck Date: Thu, 22 Feb 2024 11:25:55 +0100 Subject: [PATCH 1/9] Dumb fix for horizontal scrolling --- res/styles/navbar.css | 1 + 1 file changed, 1 insertion(+) diff --git a/res/styles/navbar.css b/res/styles/navbar.css index ec1ade5..91f083d 100644 --- a/res/styles/navbar.css +++ b/res/styles/navbar.css @@ -14,6 +14,7 @@ nav#topbar { top: 0rem; transition: top 300ms; line-height: 2em; + max-width:98vw; } nav#topbar a { -- 2.43.5 From 968f5f09ed8a276fe59f4e4f7a4aa33c066f702f Mon Sep 17 00:00:00 2001 From: Stranck Date: Thu, 22 Feb 2024 15:53:06 +0100 Subject: [PATCH 2/9] Added back rainbow animation to sponsorcount text --- res/styles/base.css | 15 +++++++++++++++ res/styles/navbar.css | 15 --------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/res/styles/base.css b/res/styles/base.css index 52cf4fb..37d9a1a 100644 --- a/res/styles/base.css +++ b/res/styles/base.css @@ -12,6 +12,21 @@ summary:has(span.status) { background-color: #ffaf0377; } +.rainbow-text { + background-image: repeating-linear-gradient(90deg, #ff0000, #ffff00, #00ff00, #00ffff, #0000ff, #ff00ff, #ff0000, #ffff00, #00ff00, #00ffff, #0000ff, #ff00ff); + background-size: 2000% 2000%; + color: transparent; + background-clip: text; + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; + animation: rainbow 4s linear infinite; +} + +@keyframes rainbow { + 0% { background-position:0% 0%; } + 100% { background-position:57.75% 0%; } +} + /* Dark theme */ @media only screen and (prefers-color-scheme: dark) { .icon {filter: invert(1);} diff --git a/res/styles/navbar.css b/res/styles/navbar.css index 91f083d..4d34f22 100644 --- a/res/styles/navbar.css +++ b/res/styles/navbar.css @@ -43,21 +43,6 @@ nav img { box-sizing: border-box; } -a.rainbow-text { - background-image: repeating-linear-gradient(90deg, #ff0000, #ffff00, #00ff00, #00ffff, #0000ff, #ff00ff, #ff0000, #ffff00, #00ff00, #00ffff, #0000ff, #ff00ff); - background-size: 2000% 2000%; - color: transparent; - background-clip: text; - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; - animation: rainbow 4s linear infinite; -} - -@keyframes rainbow { - 0% { background-position:0% 0%; } - 100% { background-position:57.75% 0%; } -} - nav a#mobileMenu { display: none; } -- 2.43.5 From 1d7ea375c17e17845c0d9b87cd3fa32ca40c2d48 Mon Sep 17 00:00:00 2001 From: Stranck Date: Thu, 22 Feb 2024 19:15:33 +0100 Subject: [PATCH 3/9] Better handling of pretix's unresponsiviness --- admin.py | 7 +- app.py | 67 ++++++++------ checkin.py | 5 +- config.example.py | 5 + ext.py | 229 +++++++++++++++++++++++++--------------------- metrics.py | 15 +++ pretixClient.py | 49 ++++++++++ room.py | 15 --- tpl/error.html | 6 +- utils.py | 32 ++++--- 10 files changed, 261 insertions(+), 169 deletions(-) create mode 100644 pretixClient.py diff --git a/admin.py b/admin.py index 63f3b4f..e3651a7 100644 --- a/admin.py +++ b/admin.py @@ -20,7 +20,8 @@ async def credentials_check(request: Request): @bp.get('/cache/clear') async def clear_cache(request, order:Order): - await request.app.ctx.om.fill_cache() + success = await request.app.ctx.om.fill_cache() + if not success: raise exceptions.ServerError("An error occurred while loading the cache") return redirect(f'/manage/admin') @bp.get('/loginas/') @@ -40,8 +41,8 @@ async def login_as(request, code, order:Order): @bp.get('/room/verify') async def verify_rooms(request, order:Order): - already_checked = await request.app.ctx.om.update_cache() - if not already_checked: + already_checked, success = await request.app.ctx.om.update_cache() + if not already_checked and success: orders = filter(lambda x: x.status not in ['c', 'e'] and x.room_id == x.code, request.app.ctx.om.cache.values()) await validate_rooms(request, orders, None) return redirect(f'/manage/admin') diff --git a/app.py b/app.py index 018cbf5..b60b60a 100644 --- a/app.py +++ b/app.py @@ -16,6 +16,8 @@ import requests import sys from sanic.log import logger, logging, access_logger from metrics import * +import pretixClient +import traceback app = Sanic(__name__) app.static("/res", "res/") @@ -34,18 +36,28 @@ from checkin import bp as checkin_bp from admin import bp as admin_bp app.blueprint([room_bp, karaoke_bp, propic_bp, export_bp, stats_bp, api_bp, carpooling_bp, checkin_bp, admin_bp]) - -@app.exception(exceptions.SanicException) -async def clear_session(request, exception): + + +async def clear_session(response): + response.delete_cookie("foxo_code") + response.delete_cookie("foxo_secret") + +@app.exception(exceptions.SanicException if DEV_MODE else Exception) +async def handleException(request, exception): + incErrorNo() logger.warning(f"{request} -> {exception}") - tpl = app.ctx.tpl.get_template('error.html') - r = html(tpl.render(exception=exception)) - - if exception.status_code == 403: - r.delete_cookie("foxo_code") - r.delete_cookie("foxo_secret") + statusCode = exception.status_code if hasattr(exception, 'status_code') else 500 + try: + tpl = app.ctx.tpl.get_template('error.html') + r = html(tpl.render(exception=exception, status_code=statusCode)) + except: + traceback.print_exc() + + if statusCode == 403: + clear_session(r) return r + @app.before_server_start async def main_start(*_): logger.info(f"[{app.name}] >>>>>> main_start <<<<<<") @@ -56,7 +68,10 @@ async def main_start(*_): app.ctx.om = OrderManager() if FILL_CACHE: - await app.ctx.om.update_cache() + checked, success = await app.ctx.om.update_cache(check_itemsQuestions=True) + if checked and not success: + logger.error(f"[{app.name}] Failure in app startup: An error occurred while loading items or questions or cache.") + app.stop() app.ctx.nfc_counts = sqlite3.connect('data/nfc_counts.db') @@ -90,18 +105,16 @@ async def redirect_explore(request, code, secret, order: Order, secret2=None): if order and order.code != code: order = None if not order: - async with httpx.AsyncClient() as client: - incPretixRead() - res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers) - - if res.status_code != 200: - raise exceptions.NotFound("This order code does not exist. Check that your order wasn't deleted, or the link is correct.") - - res = res.json() - if secret != res['secret']: - raise exceptions.Forbidden("The secret part of the url is not correct. Check your E-Mail for the correct link, or contact support!") - r.cookies['foxo_code'] = code - r.cookies['foxo_secret'] = secret + res = await pretixClient.get(f"orders/{code}/", expectedStatusCodes=None) + + if res.status_code != 200: + raise exceptions.NotFound("This order code does not exist. Check that your order wasn't deleted, or the link is correct.") + + res = res.json() + if secret != res['secret']: + raise exceptions.Forbidden("The secret part of the url is not correct. Check your E-Mail for the correct link, or contact support!") + r.cookies['foxo_code'] = code + r.cookies['foxo_secret'] = secret return r @app.route("/manage/privacy") @@ -155,11 +168,9 @@ async def download_ticket(request, order: Order): if not order.status != 'confirmed': raise exceptions.Forbidden("You are not allowed to download this ticket.") - async with httpx.AsyncClient() as client: - incPretixRead() - res = await client.get(join(base_url_event, f"orders/{order.code}/download/pdf/"), headers=headers) + res = await pretixClient.get(f"orders/{order.code}/download/pdf/", expectedStatusCodes=[200, 404, 409, 403]) - if res.status_code == 409: + if res.status_code == 409 or res.status_code == 404: raise exceptions.SanicException("Your ticket is still being generated. Please try again later!", status_code=res.status_code) elif res.status_code == 403: raise exceptions.SanicException("You can download your ticket only after the order has been confirmed and paid. Try later!", status_code=400) @@ -208,7 +219,7 @@ if __name__ == "__main__": # to let it start in the correct order. The following piece of code makes sure that pretix is running and can talk to # postgres before actually starting the reserved area, since this operation requires a cache-fill in startup print("Waiting for pretix to be up and running", file=sys.stderr) - while True: + while not SKIP_HEALTHCHECK: print("Trying connecting to pretix...", file=sys.stderr) try: incPretixRead() @@ -218,7 +229,7 @@ if __name__ == "__main__": print("Healtchecking...", file=sys.stderr) incPretixRead() res = requests.get(join(domain, "healthcheck"), headers=headers) - if(res.status_code == 200 or SKIP_HEALTHCHECK): + if(res.status_code == 200): break except: pass diff --git a/checkin.py b/checkin.py index f61f590..fc662a6 100644 --- a/checkin.py +++ b/checkin.py @@ -12,6 +12,7 @@ from time import time from urllib.parse import unquote import json from metrics import * +import pretixClient bp = Blueprint("checkin", url_prefix="/checkin") @@ -64,9 +65,7 @@ async def do_checkin(request): await order.send_answers() if not order.checked_in: - async with httpx.AsyncClient() as client: - incPretixWrite() - res = await client.post(base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]}, headers=headers) + await pretixClient.post("", baseUrl=base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]}) tpl = request.app.ctx.tpl.get_template('checkin_3.html') return html(tpl.render(order=order, room_owner=room_owner, roommates=roommates)) diff --git a/config.example.py b/config.example.py index 861ec21..1d5a880 100644 --- a/config.example.py +++ b/config.example.py @@ -21,6 +21,11 @@ PROPIC_MIN_SIZE = (125, 125) # (Width, Height) TG_BOT_API = '123456789:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' TG_CHAT_ID = -1234567 +import httpx +# Number of tries for a request to the pretix's backend +PRETIX_REQUESTS_MAX = 3 +PRETIX_REQUESTS_TIMEOUT = httpx.Timeout(15.0, read=30.0, connect=45.0, pool=None) # Timeout for httpx requests in seconds + # These order codes have additional functions. ADMINS = ['XXXXX', 'YYYYY'] # A list of staff_roles diff --git a/ext.py b/ext.py index 5b1f913..33f125b 100644 --- a/ext.py +++ b/ext.py @@ -10,6 +10,9 @@ from sanic.log import logger from time import time from metrics import * import asyncio +from threading import Lock +import pretixClient +import traceback @dataclass class Order: @@ -156,14 +159,12 @@ class Order: async def edit_answer_fileUpload(self, name, fileName, mimeType, data : bytes): if(mimeType != None and data != None): - async with httpx.AsyncClient() as client: - localHeaders = dict(headers) - localHeaders['Content-Type'] = mimeType - localHeaders['Content-Disposition'] = f'attachment; filename="{fileName}"' - incPretixWrite() - res = await client.post(join(base_url, 'upload'), headers=localHeaders, content=data) - res = res.json() - await self.edit_answer(name, res['id']) + localHeaders = dict(headers) + localHeaders['Content-Type'] = mimeType + localHeaders['Content-Disposition'] = f'attachment; filename="{fileName}"' + res = await pretixClient.post("upload", baseUrl=base_url, headers=localHeaders, content=data) + res = res.json() + await self.edit_answer(name, res['id']) else: await self.edit_answer(name, None) self.loadAns() @@ -184,11 +185,8 @@ class Order: break if (not found) and (new_answer is not None): - - async with httpx.AsyncClient() as client: - incPretixRead() - res = await client.get(join(base_url_event, 'questions/'), headers=headers) - res = res.json() + res = await pretixClient.get("questions/") + res = res.json() for r in res['results']: if r['identifier'] != name: continue @@ -201,32 +199,30 @@ class Order: self.loadAns() async def send_answers(self): - async with httpx.AsyncClient() as client: - if DEV_MODE and EXTRA_PRINTS: logger.debug("[ANSWER POST] POSITION ID IS %s", self.position_id) - - for i, ans in enumerate(self.answers): - if TYPE_OF_QUESTIONS[ans['question']] == QUESTION_TYPES["multiple_choice_from_list"]: # if multiple choice - identifier = ans['question_identifier'] - if self.ans(identifier) == "": #if empty answer - await self.edit_answer(identifier, None) - # Fix for karaoke fields - #if ans['question'] == 40: - # del self.answers[i]['options'] - # del self.answers[i]['option_identifiers'] - - incPretixWrite() - res = await client.patch(join(base_url_event, f'orderpositions/{self.position_id}/'), headers=headers, json={'answers': self.answers}) - - if res.status_code != 200: - for ans, err in zip(self.answers, res.json()['answers']): - if err: - logger.error ('[ANSWERS SENDING] ERROR ON %s %s', ans, err) + if DEV_MODE and EXTRA_PRINTS: logger.debug("[ANSWER POST] POSITION ID IS %s", self.position_id) + + for i, ans in enumerate(self.answers): + if TYPE_OF_QUESTIONS[ans['question']] == QUESTION_TYPES["multiple_choice_from_list"]: # if multiple choice + identifier = ans['question_identifier'] + if self.ans(identifier) == "": #if empty answer + await self.edit_answer(identifier, None) + # Fix for karaoke fields + #if ans['question'] == 40: + # del self.answers[i]['options'] + # del self.answers[i]['option_identifiers'] + + res = await pretixClient.patch(f'orderpositions/{self.position_id}/', json={'answers': self.answers}, expectedStatusCodes=None) + + if res.status_code != 200: + for ans, err in zip(self.answers, res.json()['answers']): + if err: + logger.error ('[ANSWERS SENDING] ERROR ON %s %s', ans, err) - raise exceptions.ServerError('There has been an error while updating this answers.') - - for i, ans in enumerate(self.answers): - if(TYPE_OF_QUESTIONS[self.answers[i]['question']] == QUESTION_TYPES['file_upload']): - self.answers[i]['answer'] = "file:keep" + raise exceptions.ServerError('There has been an error while updating this answers.') + + for i, ans in enumerate(self.answers): + if(TYPE_OF_QUESTIONS[self.answers[i]['question']] == QUESTION_TYPES['file_upload']): + self.answers[i]['answer'] = "file:keep" self.pending_update = False self.time = -1 @@ -247,12 +243,10 @@ class Quotas: return 0 async def get_quotas(request: Request=None): - async with httpx.AsyncClient() as client: - incPretixRead() - res = await client.get(join(base_url_event, 'quotas/?order=id&with_availability=true'), headers=headers) - res = res.json() - - return Quotas(res) + res = await pretixClient.get('quotas/?order=id&with_availability=true') + res = res.json() + + return Quotas(res) async def get_order(request: Request=None): await request.receive_body() @@ -261,7 +255,7 @@ async def get_order(request: Request=None): class OrderManager: def __init__(self): self.lastCacheUpdate = 0 - self.updating = False + self.updating : Lock = Lock() self.empty() def empty(self): @@ -269,64 +263,89 @@ class OrderManager: self.order_list = [] # Will fill cache once the last cache update is greater than cache expire time - async def update_cache(self): + async def update_cache(self, check_itemsQuestions=False): t = time() to_return = False - if(t - self.lastCacheUpdate > CACHE_EXPIRE_TIME and not self.updating): + success = True + if(t - self.lastCacheUpdate > CACHE_EXPIRE_TIME and not self.updating.locked()): to_return = True - await self.fill_cache() - return to_return + success = await self.fill_cache(check_itemsQuestions=check_itemsQuestions) + return (to_return, success) - def add_cache(self, order): - self.cache[order.code] = order - if not order.code in self.order_list: - self.order_list.append(order.code) + def add_cache(self, order, cache=None, orderList=None): + # Extra params for dry runs + if(cache is None): + cache = self.cache + if(orderList is None): + orderList = self.order_list - def remove_cache(self, code): - if code in self.cache: - del self.cache[code] - self.order_list.remove(code) + cache[order.code] = order + if not order.code in orderList: + orderList.append(order.code) + + def remove_cache(self, code, cache=None, orderList=None): + # Extra params for dry runs + if(cache is None): + cache = self.cache + if(orderList is None): + orderList = self.order_list + + if code in cache: + del cache[code] + orderList.remove(code) - async def fill_cache(self): + async def fill_cache(self, check_itemsQuestions=False) -> bool: # Check cache lock - if self.updating == True: return - # Set cache lock - self.updating = True + self.updating.acquire() start_time = time() logger.info("[CACHE] Filling cache...") # Index item's ids - await load_items() + r = await load_items() + if(not r and check_itemsQuestions): + logger.error("[CACHE] Items were not loading correctly. Aborting filling cache...") + return False # Index questions' types - await load_questions() + r = await load_questions() + if(not r and check_itemsQuestions): + logger.error("[CACHE] Questions were not loading correctly. Aborting filling cache...") + return False - # Clear cache data completely - self.empty() + cache = {} + orderList = [] + success = True p = 0 try: - async with httpx.AsyncClient() as client: - while 1: - p += 1 - incPretixRead() - res = await client.get(join(base_url_event, f"orders/?page={p}"), headers=headers) - if res.status_code == 404: break - # Parse order data - data = res.json() - for o in data['results']: - o = Order(o) - if o.status in ['canceled', 'expired']: - self.remove_cache(o.code) - else: - self.add_cache(Order(o)) - self.lastCacheUpdate = time() - logger.info(f"[CACHE] Cache filled in {self.lastCacheUpdate - start_time}s.") - except Exception as ex: - logger.error("[CACHE] Error while refreshing cache.", ex) + while 1: + p += 1 + res = await pretixClient.get(f"orders/?page={p}", expectedStatusCodes=[200, 404]) + if res.status_code == 404: break + # Parse order data + data = res.json() + for o in data['results']: + o = Order(o) + if o.status in ['canceled', 'expired']: + self.remove_cache(o.code, cache=cache, orderList=orderList) + else: + self.add_cache(Order(o), cache=cache, orderList=orderList) + self.lastCacheUpdate = time() + logger.info(f"[CACHE] Cache filled in {self.lastCacheUpdate - start_time}s.") + except Exception: + logger.error(f"[CACHE] Error while refreshing cache.\n{traceback.format_exc()}") + success = False finally: - self.updating = False + self.updating.release() + + # Apply new cache if there were no errors + if(success): + self.cache = cache + self.order_list = orderList + # Validating rooms rooms = list(filter(lambda o: (o.code == o.room_id), self.cache.values())) asyncio.create_task(validate_rooms(None, rooms, self)) + + return success async def get_order(self, request=None, code=None, secret=None, nfc_id=None, cached=False): @@ -349,26 +368,24 @@ class OrderManager: if re.match('^[A-Z0-9]{5}$', code or '') and (secret is None or re.match('^[a-z0-9]{16,}$', secret)): if DEV_MODE and EXTRA_PRINTS: logger.debug(f'Fetching {code} with secret {secret}') - async with httpx.AsyncClient() as client: - incPretixRead() - res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers) - if res.status_code != 200: - if request: - raise exceptions.Forbidden("Your session has expired due to order deletion or change! Please check your E-Mail for more info.") - else: - self.remove_cache(code) - return None - - res = res.json() - - order = Order(res) - if order.status in ['canceled', 'expired']: - self.remove_cache(order.code) - if request: - raise exceptions.Forbidden(f"Your order has been deleted. Contact support with your order identifier ({res['code']}) for further info.") + res = await pretixClient.get(f"orders/{code}/", expectedStatusCodes=None) + if res.status_code != 200: + if request: + raise exceptions.Forbidden("Your session has expired due to order deletion or change! Please check your E-Mail for more info.") else: - self.add_cache(order) - - if request and secret != res['secret']: - raise exceptions.Forbidden("Your session has expired due to a token change. Please check your E-Mail for an updated link!") - return order + self.remove_cache(code) + return None + + res = res.json() + + order = Order(res) + if order.status in ['canceled', 'expired']: + self.remove_cache(order.code) + if request: + raise exceptions.Forbidden(f"Your order has been deleted. Contact support with your order identifier ({res['code']}) for further info.") + else: + self.add_cache(order) + + if request and secret != res['secret']: + raise exceptions.Forbidden("Your session has expired due to a token change. Please check your E-Mail for an updated link!") + return order diff --git a/metrics.py b/metrics.py index 805f45c..64bdff3 100644 --- a/metrics.py +++ b/metrics.py @@ -3,8 +3,10 @@ from logging import LogRecord from config import * METRICS_REQ_NO = 0 +METRICS_ERR_NO = 0 # Errors served to the clients METRICS_PRETIX_READ = 0 METRICS_PRETIX_WRITE = 0 +METRICS_PRETIX_ERRORS = 0 # Errors requesting pretix's backend def incPretixRead(): global METRICS_PRETIX_READ @@ -14,19 +16,32 @@ def incPretixWrite(): global METRICS_PRETIX_WRITE METRICS_PRETIX_WRITE += 1 +def incPretixErrors(): + global METRICS_PRETIX_ERRORS + METRICS_PRETIX_ERRORS += 1 + def incReqNo(): global METRICS_REQ_NO METRICS_REQ_NO += 1 +def incErrorNo(): # Errors served to the clients + global METRICS_ERR_NO + METRICS_ERR_NO += 1 + + def getMetricsText(): global METRICS_REQ_NO + global METRICS_ERR_NO global METRICS_PRETIX_READ global METRICS_PRETIX_WRITE + global METRICS_PRETIX_ERRORS out = [] out.append(f'sanic_request_count{{}} {METRICS_REQ_NO}') + out.append(f'sanic_error_count{{}} {METRICS_ERR_NO}') out.append(f'webint_pretix_read_count{{}} {METRICS_PRETIX_READ}') out.append(f'webint_pretix_write_count{{}} {METRICS_PRETIX_WRITE}') + out.append(f'webint_pretix_error_count{{}} {METRICS_PRETIX_ERRORS}') return "\n".join(out) diff --git a/pretixClient.py b/pretixClient.py new file mode 100644 index 0000000..ea596a1 --- /dev/null +++ b/pretixClient.py @@ -0,0 +1,49 @@ +import httpx +from utils import * +from config import * +from sanic.log import logger +from metrics import * +import traceback +import asyncio + +async def get(url, baseUrl=base_url_event, headers=headers, expectedStatusCodes=[200]) -> httpx.Response: + async def func(client : httpx.AsyncClient) -> httpx.Request: + return await client.get(join(baseUrl, url), headers=headers) + return await doReq(url, func, incPretixRead, expectedStatusCodes, "GETing") + +async def post(url, content=None, json=None, baseUrl=base_url_event, headers=headers, expectedStatusCodes=[200]) -> httpx.Response: + async def func(client : httpx.AsyncClient) -> httpx.Request: + return await client.post(join(baseUrl, url), headers=headers, content=content, json=json) + return await doReq(url, func, incPretixWrite, expectedStatusCodes, "POSTing") + +async def patch(url, json, baseUrl=base_url_event, headers=headers, expectedStatusCodes=[200]) -> httpx.Response: + async def func(client : httpx.AsyncClient) -> httpx.Request: + return await client.patch(join(baseUrl, url), headers=headers, json=json) + return await doReq(url, func, incPretixWrite, expectedStatusCodes, "PATCHing") + + + +async def doReq(url, httpxFunc, metricsFunc, expectedStatusCodes, opLogString) -> httpx.Response: + res = None + async with httpx.AsyncClient(timeout=PRETIX_REQUESTS_TIMEOUT) as client: + requests = 0 + for requests in range(PRETIX_REQUESTS_MAX): + try: + metricsFunc() + res = await httpxFunc(client) + + if expectedStatusCodes is not None and res.status_code not in expectedStatusCodes: + incPretixErrors() + logger.warning(f"[PRETIX] Got an unexpected status code ({res.status_code}) while {opLogString} '{url}'. Allowed status codes: {', '.join(expectedStatusCodes)}") + continue + break + except Exception as e: + incPretixErrors() + logger.warning(f"[PRETIX] An error ({requests}) occurred while {opLogString} '{url}':\n{traceback.format_exc()}") + + requests += 1 + else: + logger.error(f"[PRETIX] Reached PRETIX_REQUESTS_MAX ({PRETIX_REQUESTS_MAX}) while {opLogString} '{url}'. Aborting") + raise httpx.TimeoutException(f"PRETIX_REQUESTS_MAX reached while {opLogString} to pretix.") + + return res \ No newline at end of file diff --git a/room.py b/room.py index e521e82..3e1382f 100644 --- a/room.py +++ b/room.py @@ -335,21 +335,6 @@ async def confirm_room(request, order: Order, quotas: Quotas): await rm.edit_answer('pending_roommates', None) await rm.edit_answer('pending_room', None) - # This should now be useless because in the ticket there already is the ticket/room type - # thing = { - # 'order': order.code, - # 'addon_to': order.position_positionid, - # 'item': ITEM_IDS['room'], - # 'variation': ROOM_MAP[len(room_members)] - # } - # - # async with httpx.AsyncClient() as client: - # incPretixRead() - # res = await client.post(join(base_url_event, "orderpositions/"), headers=headers, json=thing) - # - # if res.status_code != 201: - # raise exceptions.BadRequest("Something has gone wrong! Please contact support immediately") - for rm in room_members: await rm.send_answers() diff --git a/tpl/error.html b/tpl/error.html index d5c6267..2533115 100644 --- a/tpl/error.html +++ b/tpl/error.html @@ -1,10 +1,10 @@ {% extends "base.html" %} -{% block title %}Error {{exception.status_code}}{% endblock %} +{% block title %}Error {{status_code}}{% endblock %} {% block main %}
-

{{exception.status_code}}

+

{{status_code}}

{{exception}}

- {% if exception.status_code == 409 %} + {% if status_code == 409 %}

Retrying in 1 second...

{% endif %} diff --git a/utils.py b/utils.py index a582ff1..b257544 100644 --- a/utils.py +++ b/utils.py @@ -7,6 +7,8 @@ from email_util import send_unconfirm_message from sanic.response import text, html, redirect, raw from sanic.log import logger from metrics import * +import pretixClient +import traceback METADATA_TAG = "meta_data" VARIATIONS_TAG = "variations" @@ -28,34 +30,38 @@ QUESTION_TYPES = { #https://docs.pretix.eu/en/latest/api/resources/questions.htm TYPE_OF_QUESTIONS = {} # maps questionId -> type -async def load_questions(): +async def load_questions() -> bool: global TYPE_OF_QUESTIONS - TYPE_OF_QUESTIONS.clear() - async with httpx.AsyncClient() as client: + # TYPE_OF_QUESTIONS.clear() It should not be needed + logger.info("[QUESTIONS] Loading questions...") + success = True + try: p = 0 while 1: p += 1 - incPretixRead() - res = await client.get(join(base_url_event, f"questions/?page={p}"), headers=headers) - + res = await pretixClient.get(f"questions/?page={p}", expectedStatusCodes=[200, 404]) if res.status_code == 404: break data = res.json() for q in data['results']: TYPE_OF_QUESTIONS[q['id']] = q['type'] + except Exception: + logger.warning(f"[QUESTIONS] Error while loading questions.\n{traceback.format_exc()}") + success = False + return success -async def load_items(): +async def load_items() -> bool: global ITEMS_ID_MAP global ITEM_VARIATIONS_MAP global CATEGORIES_LIST_MAP global ROOM_TYPE_NAMES - async with httpx.AsyncClient() as client: + logger.info("[ITEMS] Loading items...") + success = True + try: p = 0 while 1: p += 1 - incPretixRead() - res = await client.get(join(base_url_event, f"items/?page={p}"), headers=headers) - + res = await pretixClient.get(f"items/?page={p}", expectedStatusCodes=[200, 404]) if res.status_code == 404: break data = res.json() @@ -85,6 +91,10 @@ async def load_items(): logger.debug(f'Mapped Variations: %s', ITEM_VARIATIONS_MAP) logger.debug(f'Mapped categories: %s', CATEGORIES_LIST_MAP) logger.debug(f'Mapped Rooms: %s', ROOM_TYPE_NAMES) + except Exception: + logger.warning(f"[ITEMS] Error while loading items.\n{traceback.format_exc()}") + success = False + return success # Tries to get an item name from metadata. Prints a warning if an item has no metadata def check_and_get_name(type, q): -- 2.43.5 From 5d79052e59192bcc1ad8f5579f5bc150a315a324 Mon Sep 17 00:00:00 2001 From: Stranck Date: Mon, 26 Feb 2024 16:06:44 +0100 Subject: [PATCH 4/9] Added rclone scripts --- stuff/rclone/clone.sh | 7 +++++++ stuff/rclone/cloneHeadless.sh | 8 ++++++++ 2 files changed, 15 insertions(+) create mode 100644 stuff/rclone/clone.sh create mode 100644 stuff/rclone/cloneHeadless.sh diff --git a/stuff/rclone/clone.sh b/stuff/rclone/clone.sh new file mode 100644 index 0000000..d2aca4e --- /dev/null +++ b/stuff/rclone/clone.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +/usr/bin/tar -czvf /tmp/backupRclone.tar.gz /home/ /root/ /etc/ /var/backups/ /var/log/ /var/mail/ /var/pretix-data/ /var/prometheus-data/ /var/spool/ /var/www/ + +/usr/bin/rclone sync /tmp/backupRclone.tar.gz webservice:/backups/backupRclone.tar.gz -P --stats=1s --bwlimit 15M + +/usr/bin/rm /tmp/backupRclone.tar.gz diff --git a/stuff/rclone/cloneHeadless.sh b/stuff/rclone/cloneHeadless.sh new file mode 100644 index 0000000..7038839 --- /dev/null +++ b/stuff/rclone/cloneHeadless.sh @@ -0,0 +1,8 @@ +#!/bin/bash +source /root/.profile + +/usr/bin/tar -czf /tmp/backupRclone.tar.gz /home/ /root/ /etc/ /var/backups/ /var/log/ /var/mail/ /var/pretix-data/ /var/prometheus-data/ /var/spool/ /var/www/ + +/usr/bin/rclone sync /tmp/backupRclone.tar.gz webservice:/backups/backupRclone.tar.gz --bwlimit 15M + +/usr/bin/rm /tmp/backupRclone.tar.gz -- 2.43.5 From 3919d512a14f88b28f7d112e8b619f31870ace1a Mon Sep 17 00:00:00 2001 From: "Luca Sorace \"Stranck" Date: Tue, 27 Feb 2024 19:53:07 +0100 Subject: [PATCH 5/9] Added metrics for early/late orders --- metrics.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/metrics.py b/metrics.py index 64bdff3..43f037d 100644 --- a/metrics.py +++ b/metrics.py @@ -50,17 +50,29 @@ def getRoomCountersText(request): try : daily = 0 counters = {} + counters_early = {} + counters_late = {} for id in ROOM_TYPE_NAMES.keys(): counters[id] = 0 + counters_early[id] = 0 + counters_late[id] = 0 for order in request.app.ctx.om.cache.values(): if(order.daily): daily += 1 else: counters[order.bed_in_room] += 1 + if(order.has_early): + counters_early[order.bed_in_room] += 1 + if(order.has_late): + counters_late[order.bed_in_room] += 1 for id, count in counters.items(): - out.append(f'webint_order_room_counter{{label="{ROOM_TYPE_NAMES[id]}"}} {count}') + out.append(f'webint_order_room_counter{{days="normal", label="{ROOM_TYPE_NAMES[id]}"}} {count}') + for id, count in counters_early.items(): + out.append(f'webint_order_room_counter{{days="early", label="{ROOM_TYPE_NAMES[id]}"}} {count}') + for id, count in counters_late.items(): + out.append(f'webint_order_room_counter{{days="late", label="{ROOM_TYPE_NAMES[id]}"}} {count}') out.append(f'webint_order_room_counter{{label="Daily"}} {daily}') except Exception as e: -- 2.43.5 From 5b66a5839985120a97fed07429f624d76d15a228 Mon Sep 17 00:00:00 2001 From: Stranck Date: Thu, 29 Feb 2024 10:10:05 +0100 Subject: [PATCH 6/9] Included grafana and redis folders in backups --- stuff/rclone/clone.sh | 2 +- stuff/rclone/cloneHeadless.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stuff/rclone/clone.sh b/stuff/rclone/clone.sh index d2aca4e..7aae1e6 100644 --- a/stuff/rclone/clone.sh +++ b/stuff/rclone/clone.sh @@ -1,6 +1,6 @@ #!/bin/bash -/usr/bin/tar -czvf /tmp/backupRclone.tar.gz /home/ /root/ /etc/ /var/backups/ /var/log/ /var/mail/ /var/pretix-data/ /var/prometheus-data/ /var/spool/ /var/www/ +/usr/bin/tar -czvf /tmp/backupRclone.tar.gz /home/ /root/ /etc/ /var/backups/ /var/log/ /var/mail/ /var/pretix-data/ /var/prometheus-data/ /var/spool/ /var/www/ /var/lib/grafana/ /var/lib/redis/ /usr/bin/rclone sync /tmp/backupRclone.tar.gz webservice:/backups/backupRclone.tar.gz -P --stats=1s --bwlimit 15M diff --git a/stuff/rclone/cloneHeadless.sh b/stuff/rclone/cloneHeadless.sh index 7038839..ca5007a 100644 --- a/stuff/rclone/cloneHeadless.sh +++ b/stuff/rclone/cloneHeadless.sh @@ -1,7 +1,7 @@ #!/bin/bash source /root/.profile -/usr/bin/tar -czf /tmp/backupRclone.tar.gz /home/ /root/ /etc/ /var/backups/ /var/log/ /var/mail/ /var/pretix-data/ /var/prometheus-data/ /var/spool/ /var/www/ +/usr/bin/tar -czf /tmp/backupRclone.tar.gz /home/ /root/ /etc/ /var/backups/ /var/log/ /var/mail/ /var/pretix-data/ /var/prometheus-data/ /var/spool/ /var/www/ /var/lib/grafana/ /var/lib/redis/ /usr/bin/rclone sync /tmp/backupRclone.tar.gz webservice:/backups/backupRclone.tar.gz --bwlimit 15M -- 2.43.5 From c6bc9c65acb42bfc196a009261d92641514f5d57 Mon Sep 17 00:00:00 2001 From: Stranck Date: Thu, 29 Feb 2024 11:41:49 +0100 Subject: [PATCH 7/9] Added remind propic feature for admins --- admin.py | 18 ++++++++++++++++- email_util.py | 33 +++++++++++++++++++++++++++----- messages.py | 52 +++++++++++++++++++++++++++++++------------------- tpl/admin.html | 7 ++++--- utils.py | 4 ++-- 5 files changed, 83 insertions(+), 31 deletions(-) diff --git a/admin.py b/admin.py index e3651a7..9c37864 100644 --- a/admin.py +++ b/admin.py @@ -1,4 +1,5 @@ from sanic import response, redirect, Blueprint, exceptions +from email_util import send_missing_propic_message from room import unconfirm_room_by_order from config import * from utils import * @@ -82,4 +83,19 @@ async def rename_room(request, code, order:Order): await dOrder.edit_answer("room_name", name) await dOrder.send_answers() - return redirect(f'/manage/nosecount') \ No newline at end of file + return redirect(f'/manage/nosecount') + +@bp.get('/propic/remind') +async def propic_remind_missing(request, order:Order): + await clear_cache(request, order) + + orders = request.app.ctx.om.cache.values() + order: Order + for order in orders: + missingPropic = order.propic is None + missingFursuitPropic = order.is_fursuiter and order.propic_fursuiter is None + if(missingPropic or missingFursuitPropic): + # print(f"{order.code}: prp={missingPropic} fpr={missingFursuitPropic} - {order.name}") + await send_missing_propic_message(order, missingPropic, missingFursuitPropic) + + return redirect(f'/manage/admin') \ No newline at end of file diff --git a/email_util.py b/email_util.py index 6621e66..4687c5b 100644 --- a/email_util.py +++ b/email_util.py @@ -54,6 +54,13 @@ async def sendEmail(message : MIMEMultipart): smptSender.sendmail(message['From'], message['to'], message.as_string()) sslLock.release() +def render_email_template(title = "", body = ""): + tpl = Environment(loader=FileSystemLoader("tpl"), autoescape=False).get_template('email/comunication.html') + return str(tpl.render(title=title, body=body)) + + + + async def send_unconfirm_message(room_order, orders): memberMessages = [] @@ -72,8 +79,8 @@ async def send_unconfirm_message(room_order, orders): issues_html += "" for member in orders: - plain_body = ROOM_UNCONFIRM_TEXT['plain'].format(member.name, room_order.room_name, issues_plain) - html_body = render_email_template(ROOM_UNCONFIRM_TITLE, ROOM_UNCONFIRM_TEXT['html'].format(member.name, room_order.room_name, issues_html)) + plain_body = EMAILS_TEXT["ROOM_UNCONFIRM_TEXT"]['plain'].format(member.name, room_order.room_name, issues_plain) + html_body = render_email_template(EMAILS_TEXT["ROOM_UNCONFIRM_TITLE"], EMAILS_TEXT["ROOM_UNCONFIRM_TEXT"]['html'].format(member.name, room_order.room_name, issues_html)) plain_text = MIMEText(plain_body, "plain") html_text = MIMEText(html_body, "html") message = MIMEMultipart("alternative") @@ -89,6 +96,22 @@ async def send_unconfirm_message(room_order, orders): for message in memberMessages: await sendEmail(message) -def render_email_template(title = "", body = ""): - tpl = Environment(loader=FileSystemLoader("tpl"), autoescape=False).get_template('email/comunication.html') - return str(tpl.render(title=title, body=body)) \ No newline at end of file +async def send_missing_propic_message(order, missingPropic, missingFursuitPropic): + t = [] + if(missingPropic): t.append("your propic") + if(missingFursuitPropic): t.append("your fursuit's badge") + missingText = " and ".join(t) + + plain_body = EMAILS_TEXT["MISSING_PROPIC_TEXT"]['plain'].format(order.name, missingText) + html_body = render_email_template(EMAILS_TEXT["MISSING_PROPIC_TITLE"], EMAILS_TEXT["MISSING_PROPIC_TEXT"]['html'].format(order.name, missingText)) + plain_text = MIMEText(plain_body, "plain") + html_text = MIMEText(html_body, "html") + message = MIMEMultipart("alternative") + message.attach(plain_text) + message.attach(html_text) + message['Subject'] = f"[{EMAIL_SENDER_NAME}] You haven't uploaded your badges yet!" + message['From'] = f'{EMAIL_SENDER_NAME} <{EMAIL_SENDER_MAIL}>' + message['To'] = f"{order.name} <{order.email}>" + + await sendEmail(message) + diff --git a/messages.py b/messages.py index 98fc102..5664829 100644 --- a/messages.py +++ b/messages.py @@ -1,30 +1,42 @@ ROOM_ERROR_TYPES = { - 'room_id_mismatch': "There's a member in your room that is actually in another room, too. Please contact us as soon as possible in order to fix this issue.", - 'unpaid': "Somebody in your room has not paid for their reservation, yet.", - 'type_mismatch': "A member in your room has a ticket for a different type of room capacity. This happens when users swap their room types with others, without abandoning the room.", - 'daily': "Some member in your room has a Daily ticket. These tickets do not include a hotel reservation.", - 'capacity_mismatch': "The number of people in your room mismatches your type of ticket." + 'room_id_mismatch': "There's a member in your room that is actually in another room, too. Please contact us as soon as possible in order to fix this issue.", + 'unpaid': "Somebody in your room has not paid for their reservation, yet.", + 'type_mismatch': "A member in your room has a ticket for a different type of room capacity. This happens when users swap their room types with others, without abandoning the room.", + 'daily': "Some member in your room has a Daily ticket. These tickets do not include a hotel reservation.", + 'capacity_mismatch': "The number of people in your room mismatches your type of ticket." } -ROOM_UNCONFIRM_TITLE = "Your room got unconfirmed" -ROOM_UNCONFIRM_TEXT = { - 'html': "Hello {0}
We had to unconfirm your room '{1}' due to the following issues:

{2}

Please contact your room's owner or contact our support for further informations at https://furizon.net/contact/.
Thank you.

Manage booking", - 'plain': "Hello {0}\nWe had to unconfirm your room '{1}' due to the following issues:\n{2}\nPlease contact your room's owner or contact our support for further informations at https://furizon.net/contact/.\nThank you\n\nTo manage your booking: https://reg.furizon.net/manage/welcome" +EMAILS_TEXT = { + "ROOM_UNCONFIRM_TITLE": "Your room got unconfirmed", + "ROOM_UNCONFIRM_TEXT": { + 'html': "Hello {0}
We had to unconfirm your room '{1}' due to the following issues:

{2}

Please contact your room's owner or contact our support for further informations at https://furizon.net/contact/.
Thank you.

Manage booking", + + 'plain': "Hello {0}\nWe had to unconfirm your room '{1}' due to the following issues:\n{2}\nPlease contact your room's owner or contact our support for further informations at https://furizon.net/contact/.\nThank you\n\nTo manage your booking: https://reg.furizon.net/manage/welcome" + }, + + + "MISSING_PROPIC_TITLE": "You haven't uploaded your badges yet!", + "MISSING_PROPIC_TEXT": { + 'html': "Hello {0}
We noticed you still have to upload {1}!
Please enter your booking page at https://reg.furizon.net/manage/welcome and upload them under the \"Badge Customization\" section.
Thank you.

Manage booking", + + 'plain': "Hello {0}\nWe noticed you still have to upload {1}!\nPlease enter your booking page at https://reg.furizon.net/manage/welcome and upload them under the \"Badge Customization\" section.\nThank you." + }, } + NOSECOUNT = { - 'filters': { - 'capacity': "Here are some furs that share your room type and don't have a confirmed room." - } + 'filters': { + 'capacity': "Here are some furs that share your room type and don't have a confirmed room." + } } LOCALES = { - 'shuttle_link': { - 'en': 'Book now', - 'it': 'Prenota ora' - }, - 'shuttle_link_url': { - 'en': 'https://visitfiemme.regiondo.com/furizon?_ga=2.129644046.307369854.1705325023-235291123.1705325023', - 'it': 'https://experience.visitfiemme.it/furizon' - } + 'shuttle_link': { + 'en': 'Book now', + 'it': 'Prenota ora' + }, + 'shuttle_link_url': { + 'en': 'https://visitfiemme.regiondo.com/furizon?_ga=2.129644046.307369854.1705325023-235291123.1705325023', + 'it': 'https://experience.visitfiemme.it/furizon' + } } \ No newline at end of file diff --git a/tpl/admin.html b/tpl/admin.html index 0b2829f..2c1b0fb 100644 --- a/tpl/admin.html +++ b/tpl/admin.html @@ -9,10 +9,11 @@ -

Admin panel

- Clear cache - Manage rooms +

Admin panel

+ Clear cache + Manage rooms Verify Rooms + Remind badge upload
diff --git a/utils.py b/utils.py index b257544..d347419 100644 --- a/utils.py +++ b/utils.py @@ -151,7 +151,7 @@ async def get_people_in_room_by_code(request, code, om=None): await om.update_cache() return filter(lambda rm: rm.room_id == code, om.cache.values()) -async def unconfirm_room_by_order(order, room_members:[]=None, throw=True, request=None, om=None): +async def unconfirm_room_by_order(order, room_members=None, throw=True, request=None, om=None): if not om: om = request.app.ctx.om if not order.room_confirmed: if throw: @@ -216,7 +216,7 @@ async def validate_rooms(request, rooms, om): order = rtu[0] member_orders = rtu[2] try: - await send_unconfirm_message (order, member_orders) + await send_unconfirm_message(order, member_orders) sent_count += len(member_orders) except Exception as ex: if EXTRA_PRINTS: logger.exception(str(ex)) -- 2.43.5 From 9f191c3dec145a29ba0d56fed480ab3247356909 Mon Sep 17 00:00:00 2001 From: Stranck Date: Thu, 29 Feb 2024 11:42:05 +0100 Subject: [PATCH 8/9] Now smpt client will shutdown with SIGINTS --- app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app.py b/app.py index b60b60a..2e51d66 100644 --- a/app.py +++ b/app.py @@ -16,6 +16,7 @@ import requests import sys from sanic.log import logger, logging, access_logger from metrics import * +from email_util import killSmptClient import pretixClient import traceback @@ -202,6 +203,10 @@ async def logout(request): raise exceptions.Forbidden("You have been logged out.") +@app.signal("server.shutdown.before") +async def sigintHandler(app, loop): + killSmptClient() + @app.get(METRICS_PATH) async def metrics(request): return text(getMetricsText() + "\n" + getRoomCountersText(request)) -- 2.43.5 From f233fb0b6825d937d675ec446e9c008de3e8224b Mon Sep 17 00:00:00 2001 From: Stranck Date: Thu, 29 Feb 2024 14:57:46 +0100 Subject: [PATCH 9/9] Update room.html --- tpl/blocks/room.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tpl/blocks/room.html b/tpl/blocks/room.html index a10fcfd..4fca1dd 100644 --- a/tpl/blocks/room.html +++ b/tpl/blocks/room.html @@ -5,7 +5,7 @@

Note! Only people with the same room type can be roommates. If you need help, contact the Furizon's Staff.

{% if not order.room_confirmed %} -

Check here for any fur who share your room type.

+

Check here for any fur who share your room type.

{% endif %} {# Show alert if room owner has wrong people inside #} -- 2.43.5