Compare commits

...

3 Commits

Author SHA1 Message Date
Luca Sorace "Stranck 61447ee768 Added custom metrics
And removed old library
2024-02-14 20:22:56 +01:00
Luca Sorace "Stranck cfefd44435 moved logOrders in stuff/ 2024-02-14 16:42:21 +01:00
Luca Sorace "Stranck f59c288908 Added metrics 2024-02-14 16:10:27 +01:00
9 changed files with 186 additions and 3 deletions

20
app.py
View File

@ -14,7 +14,8 @@ from messages import LOCALES
import sqlite3 import sqlite3
import requests import requests
import sys import sys
from sanic.log import logger, logging from sanic.log import logger, logging, access_logger
from metrics import *
app = Sanic(__name__) app = Sanic(__name__)
app.static("/res", "res/") app.static("/res", "res/")
@ -36,6 +37,7 @@ app.blueprint([room_bp, karaoke_bp, propic_bp, export_bp, stats_bp, api_bp, carp
@app.exception(exceptions.SanicException) @app.exception(exceptions.SanicException)
async def clear_session(request, exception): async def clear_session(request, exception):
logger.warning(f"{request} -> {exception}")
tpl = app.ctx.tpl.get_template('error.html') tpl = app.ctx.tpl.get_template('error.html')
r = html(tpl.render(exception=exception)) r = html(tpl.render(exception=exception))
@ -48,6 +50,7 @@ async def clear_session(request, exception):
async def main_start(*_): async def main_start(*_):
logger.info(f"[{app.name}] >>>>>> main_start <<<<<<") logger.info(f"[{app.name}] >>>>>> main_start <<<<<<")
logger.setLevel(LOG_LEVEL) logger.setLevel(LOG_LEVEL)
access_logger.addFilter(MetricsFilter())
app.config.REQUEST_MAX_SIZE = PROPIC_MAX_FILE_SIZE * 3 app.config.REQUEST_MAX_SIZE = PROPIC_MAX_FILE_SIZE * 3
@ -88,6 +91,7 @@ async def redirect_explore(request, code, secret, order: Order, secret2=None):
if not order: if not order:
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixRead()
res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers) res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers)
if res.status_code != 200: if res.status_code != 200:
@ -152,6 +156,7 @@ async def download_ticket(request, order: Order):
raise exceptions.Forbidden("You are not allowed to download this ticket.") raise exceptions.Forbidden("You are not allowed to download this ticket.")
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixRead()
res = await client.get(join(base_url_event, f"orders/{order.code}/download/pdf/"), headers=headers) res = await client.get(join(base_url_event, f"orders/{order.code}/download/pdf/"), headers=headers)
if res.status_code == 409: if res.status_code == 409:
@ -186,6 +191,16 @@ async def logout(request):
raise exceptions.Forbidden("You have been logged out.") raise exceptions.Forbidden("You have been logged out.")
@app.get(METRICS_PATH)
async def metrics(request):
return text(getMetricsText() + "\n" + getRoomCountersText(request))
@app.on_request
async def countReqs(request : Request):
global METRICS_REQ_NO
if(request.path != METRICS_PATH):
incReqNo()
if __name__ == "__main__": if __name__ == "__main__":
# Wait for pretix in server reboot # Wait for pretix in server reboot
# Using a docker configuration, pretix may be unable to talk with postgres if postgres' service started before it. # Using a docker configuration, pretix may be unable to talk with postgres if postgres' service started before it.
@ -196,10 +211,12 @@ if __name__ == "__main__":
while True: while True:
print("Trying connecting to pretix...", file=sys.stderr) print("Trying connecting to pretix...", file=sys.stderr)
try: try:
incPretixRead()
res = requests.get(base_url_event, headers=headers) res = requests.get(base_url_event, headers=headers)
res = res.json() res = res.json()
if(res['slug'] == EVENT_NAME): if(res['slug'] == EVENT_NAME):
print("Healtchecking...", file=sys.stderr) print("Healtchecking...", file=sys.stderr)
incPretixRead()
res = requests.get(join(domain, "healthcheck"), headers=headers) res = requests.get(join(domain, "healthcheck"), headers=headers)
if(res.status_code == 200): if(res.status_code == 200):
break break
@ -207,4 +224,5 @@ if __name__ == "__main__":
pass pass
sleep(5) sleep(5)
print("Connected to pretix!", file=sys.stderr) print("Connected to pretix!", file=sys.stderr)
app.run(host="127.0.0.1", port=8188, dev=DEV_MODE, access_log=ACCESS_LOG) app.run(host="127.0.0.1", port=8188, dev=DEV_MODE, access_log=ACCESS_LOG)

View File

@ -11,6 +11,7 @@ from hashlib import sha224
from time import time from time import time
from urllib.parse import unquote from urllib.parse import unquote
import json import json
from metrics import *
bp = Blueprint("checkin", url_prefix="/checkin") bp = Blueprint("checkin", url_prefix="/checkin")
@ -64,6 +65,7 @@ async def do_checkin(request):
if not order.checked_in: if not order.checked_in:
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixWrite()
res = await client.post(base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]}, headers=headers) res = await client.post(base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]}, headers=headers)
tpl = request.app.ctx.tpl.get_template('checkin_3.html') tpl = request.app.ctx.tpl.get_template('checkin_3.html')

View File

@ -40,6 +40,8 @@ DEV_MODE = True
ACCESS_LOG = True ACCESS_LOG = True
EXTRA_PRINTS = True EXTRA_PRINTS = True
METRICS_PATH = "/welcome/metrics"
# Additional configured locales. # Additional configured locales.
# If an order has a country that's not listed here, # If an order has a country that's not listed here,
# Will default to an english preference. # Will default to an english preference.
@ -132,5 +134,5 @@ ROOM_CAPACITY_MAP = {
'bed_in_room_overflow1_2': 2, 'bed_in_room_overflow1_2': 2,
} }
# Autofilled # Autofilled. Maps roomTypeId -> roomName
ROOM_TYPE_NAMES = { } ROOM_TYPE_NAMES = { }

7
ext.py
View File

@ -8,6 +8,7 @@ from os.path import join
import json import json
from sanic.log import logger from sanic.log import logger
from time import time from time import time
from metrics import *
import asyncio import asyncio
@dataclass @dataclass
@ -159,6 +160,7 @@ class Order:
localHeaders = dict(headers) localHeaders = dict(headers)
localHeaders['Content-Type'] = mimeType localHeaders['Content-Type'] = mimeType
localHeaders['Content-Disposition'] = f'attachment; filename="{fileName}"' localHeaders['Content-Disposition'] = f'attachment; filename="{fileName}"'
incPretixWrite()
res = await client.post(join(base_url, 'upload'), headers=localHeaders, content=data) res = await client.post(join(base_url, 'upload'), headers=localHeaders, content=data)
res = res.json() res = res.json()
await self.edit_answer(name, res['id']) await self.edit_answer(name, res['id'])
@ -184,6 +186,7 @@ class Order:
if (not found) and (new_answer is not None): if (not found) and (new_answer is not None):
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixRead()
res = await client.get(join(base_url_event, 'questions/'), headers=headers) res = await client.get(join(base_url_event, 'questions/'), headers=headers)
res = res.json() res = res.json()
for r in res['results']: for r in res['results']:
@ -211,6 +214,7 @@ class Order:
# del self.answers[i]['options'] # del self.answers[i]['options']
# del self.answers[i]['option_identifiers'] # del self.answers[i]['option_identifiers']
incPretixWrite()
res = await client.patch(join(base_url_event, f'orderpositions/{self.position_id}/'), headers=headers, json={'answers': self.answers}) res = await client.patch(join(base_url_event, f'orderpositions/{self.position_id}/'), headers=headers, json={'answers': self.answers})
if res.status_code != 200: if res.status_code != 200:
@ -244,6 +248,7 @@ class Quotas:
async def get_quotas(request: Request=None): async def get_quotas(request: Request=None):
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixRead()
res = await client.get(join(base_url_event, 'quotas/?order=id&with_availability=true'), headers=headers) res = await client.get(join(base_url_event, 'quotas/?order=id&with_availability=true'), headers=headers)
res = res.json() res = res.json()
@ -302,6 +307,7 @@ class OrderManager:
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
while 1: while 1:
p += 1 p += 1
incPretixRead()
res = await client.get(join(base_url_event, f"orders/?page={p}"), headers=headers) res = await client.get(join(base_url_event, f"orders/?page={p}"), headers=headers)
if res.status_code == 404: break if res.status_code == 404: break
# Parse order data # Parse order data
@ -344,6 +350,7 @@ class OrderManager:
if DEV_MODE and EXTRA_PRINTS: logger.debug(f'Fetching {code} with secret {secret}') if DEV_MODE and EXTRA_PRINTS: logger.debug(f'Fetching {code} with secret {secret}')
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixRead()
res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers) res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers)
if res.status_code != 200: if res.status_code != 200:
if request: if request:

58
metrics.py Normal file
View File

@ -0,0 +1,58 @@
from sanic.log import logger, logging
from logging import LogRecord
from config import *
METRICS_REQ_NO = 0
METRICS_PRETIX_READ = 0
METRICS_PRETIX_WRITE = 0
def incPretixRead():
global METRICS_PRETIX_READ
METRICS_PRETIX_READ += 1
def incPretixWrite():
global METRICS_PRETIX_WRITE
METRICS_PRETIX_WRITE += 1
def incReqNo():
global METRICS_REQ_NO
METRICS_REQ_NO += 1
def getMetricsText():
global METRICS_REQ_NO
global METRICS_PRETIX_READ
global METRICS_PRETIX_WRITE
out = []
out.append(f'sanic_request_count{{}} {METRICS_REQ_NO}')
out.append(f'webint_pretix_read_count{{}} {METRICS_PRETIX_READ}')
out.append(f'webint_pretix_write_count{{}} {METRICS_PRETIX_WRITE}')
return "\n".join(out)
def getRoomCountersText(request):
out = []
try :
daily = 0
counters = {}
for id in ROOM_TYPE_NAMES.keys():
counters[id] = 0
for order in request.app.ctx.om.cache.values():
if(order.daily):
daily += 1
else:
counters[order.bed_in_room] += 1
for id, count in counters.items():
out.append(f'webint_order_room_counter{{label="{ROOM_TYPE_NAMES[id]}"}} {count}')
out.append(f'webint_order_room_counter{{label="Daily"}} {daily}')
except Exception as e:
print(e)
logger.warning("Error in loading metrics rooms")
return "\n".join(out)
class MetricsFilter(logging.Filter):
def filter(self, record : LogRecord):
return not (record.request.endswith("/manage/metrics") and record.status == 200)

View File

@ -344,6 +344,7 @@ async def confirm_room(request, order: Order, quotas: Quotas):
# } # }
# #
# async with httpx.AsyncClient() as client: # async with httpx.AsyncClient() as client:
# incPretixRead()
# res = await client.post(join(base_url_event, "orderpositions/"), headers=headers, json=thing) # res = await client.post(join(base_url_event, "orderpositions/"), headers=headers, json=thing)
# #
# if res.status_code != 201: # if res.status_code != 201:

91
stuff/systemMetrics.py Normal file
View File

@ -0,0 +1,91 @@
import psutil
import time
import sys
from sanic import Sanic
from sanic import response as res
DEV_MODE = False
ACCESS_LOG = False
app = Sanic(__name__)
@app.route("/")
async def main(req):
return res.text("I\'m a teapot", status=418)
@app.route("/metrics")
async def metrics(req):
out = []
cpus = psutil.cpu_percent(percpu=True)
totalCpu = 0
for i in range(len(cpus)):
out.append(f'monitor_cpu_core_usage_percent{{core="{i}"}} {cpus[i]}')
totalCpu += cpus[i]
out.append(f'monitor_cpu_core_usage_percent{{core="avg"}} {"%.2f" % (totalCpu / len(cpus))}')
diskIo = psutil.disk_io_counters(nowrap=True)
out.append(f'monitor_diskio{{value="read_count"}} {diskIo.read_count}')
out.append(f'monitor_diskio{{value="write_count"}} {diskIo.write_count}')
out.append(f'monitor_diskio{{value="read_bytes"}} {diskIo.read_bytes}')
out.append(f'monitor_diskio{{value="write_bytes"}} {diskIo.write_bytes}')
out.append(f'monitor_diskio{{value="read_time"}} {diskIo.read_time}')
out.append(f'monitor_diskio{{value="write_time"}} {diskIo.write_time}')
disks = psutil.disk_partitions()
for disk in disks:
try:
dUsage = psutil.disk_usage(disk.mountpoint)
out.append(f'monitor_disk_usage_percent{{partition="{disk.mountpoint}"}} {dUsage.percent}')
except:
pass
mem = psutil.virtual_memory()
out.append(f'monitor_memory{{value="total"}} {mem.total}')
out.append(f'monitor_memory{{value="available"}} {mem.available}')
out.append(f'monitor_memory{{value="percent"}} {mem.percent}')
out.append(f'monitor_memory{{value="used"}} {mem.used}')
out.append(f'monitor_memory{{value="free"}} {mem.free}')
swap = psutil.swap_memory()
out.append(f'monitor_swap{{value="total"}} {swap.total}')
out.append(f'monitor_swap{{value="used"}} {swap.used}')
out.append(f'monitor_swap{{value="free"}} {swap.free}')
out.append(f'monitor_swap{{value="percent"}} {swap.percent}')
out.append(f'monitor_swap{{value="sin"}} {swap.sin}')
out.append(f'monitor_swap{{value="sout"}} {swap.sout}')
bootTime = psutil.boot_time()
out.append(f'monitor_boot_time{{}} {int(time.time() - bootTime)}')
netioConnections = psutil.net_connections()
out.append(f'monitor_netio_connections{{}} {len(netioConnections)}')
netioCounters = psutil.net_io_counters(nowrap=True)
out.append(f'monitor_netio_counters{{value="bytes_sent"}} {netioCounters.bytes_sent}')
out.append(f'monitor_netio_counters{{value="bytes_recv"}} {netioCounters.bytes_recv}')
out.append(f'monitor_netio_counters{{value="packets_sent"}} {netioCounters.packets_sent}')
out.append(f'monitor_netio_counters{{value="packets_recv"}} {netioCounters.packets_recv}')
out.append(f'monitor_netio_counters{{value="errin"}} {netioCounters.errin}')
out.append(f'monitor_netio_counters{{value="errout"}} {netioCounters.errout}')
out.append(f'monitor_netio_counters{{value="dropin"}} {netioCounters.dropin}')
out.append(f'monitor_netio_counters{{value="dropout"}} {netioCounters.dropout}')
return res.text("\n".join(out), status=200)
if __name__ == '__main__':
ip = "127.0.0.1"
port = 2611
if(len(sys.argv) > 1):
ip = sys.argv[1]
if(len(sys.argv) > 2):
try:
port = int(sys.argv[2])
except:
print("Port must be a numeric value!")
exit(1)
if(port < 1 or port > 0xffff):
print("Port must be in [1, 65535] range!")
exit(1)
app.run(host=ip, port=port, dev=DEV_MODE, access_log=ACCESS_LOG)

View File

@ -4,7 +4,9 @@ from config import *
import httpx import httpx
from messages import ROOM_ERROR_TYPES from messages import ROOM_ERROR_TYPES
from email_util import send_unconfirm_message from email_util import send_unconfirm_message
from sanic.response import text, html, redirect, raw
from sanic.log import logger from sanic.log import logger
from metrics import *
METADATA_TAG = "meta_data" METADATA_TAG = "meta_data"
VARIATIONS_TAG = "variations" VARIATIONS_TAG = "variations"
@ -33,6 +35,7 @@ async def load_questions():
p = 0 p = 0
while 1: while 1:
p += 1 p += 1
incPretixRead()
res = await client.get(join(base_url_event, f"questions/?page={p}"), headers=headers) res = await client.get(join(base_url_event, f"questions/?page={p}"), headers=headers)
if res.status_code == 404: break if res.status_code == 404: break
@ -50,6 +53,7 @@ async def load_items():
p = 0 p = 0
while 1: while 1:
p += 1 p += 1
incPretixRead()
res = await client.get(join(base_url_event, f"items/?page={p}"), headers=headers) res = await client.get(join(base_url_event, f"items/?page={p}"), headers=headers)
if res.status_code == 404: break if res.status_code == 404: break
@ -253,4 +257,4 @@ async def check_room(request, order, om=None):
if order.room_confirmed: if order.room_confirmed:
allOk = False allOk = False
order.set_room_errors(room_errors) order.set_room_errors(room_errors)
return (order, allOk, room_members) return (order, allOk, room_members)