diff --git a/app.py b/app.py index 1c81301..6bbdd11 100644 --- a/app.py +++ b/app.py @@ -14,7 +14,8 @@ from messages import LOCALES import sqlite3 import requests import sys -from sanic.log import logger, logging +from sanic.log import logger, logging, access_logger +from metrics import * app = Sanic(__name__) app.static("/res", "res/") @@ -36,6 +37,7 @@ app.blueprint([room_bp, karaoke_bp, propic_bp, export_bp, stats_bp, api_bp, carp @app.exception(exceptions.SanicException) async def clear_session(request, exception): + logger.warning(f"{request} -> {exception}") tpl = app.ctx.tpl.get_template('error.html') r = html(tpl.render(exception=exception)) @@ -48,6 +50,7 @@ async def clear_session(request, exception): async def main_start(*_): logger.info(f"[{app.name}] >>>>>> main_start <<<<<<") logger.setLevel(LOG_LEVEL) + access_logger.addFilter(MetricsFilter()) app.config.REQUEST_MAX_SIZE = PROPIC_MAX_FILE_SIZE * 3 @@ -88,6 +91,7 @@ async def redirect_explore(request, code, secret, order: Order, secret2=None): if not order: async with httpx.AsyncClient() as client: + incPretixRead() res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers) if res.status_code != 200: @@ -152,6 +156,7 @@ async def download_ticket(request, order: Order): raise exceptions.Forbidden("You are not allowed to download this ticket.") async with httpx.AsyncClient() as client: + incPretixRead() res = await client.get(join(base_url_event, f"orders/{order.code}/download/pdf/"), headers=headers) if res.status_code == 409: @@ -186,6 +191,16 @@ async def logout(request): raise exceptions.Forbidden("You have been logged out.") +@app.get(METRICS_PATH) +async def metrics(request): + return text(getMetricsText() + "\n" + getRoomCountersText(request)) + +@app.on_request +async def countReqs(request : Request): + global METRICS_REQ_NO + if(request.path != METRICS_PATH): + incReqNo() + if __name__ == "__main__": # Wait for pretix in server reboot # Using a docker configuration, pretix may be unable to talk with postgres if postgres' service started before it. @@ -196,10 +211,12 @@ if __name__ == "__main__": while True: print("Trying connecting to pretix...", file=sys.stderr) try: + incPretixRead() res = requests.get(base_url_event, headers=headers) res = res.json() if(res['slug'] == EVENT_NAME): print("Healtchecking...", file=sys.stderr) + incPretixRead() res = requests.get(join(domain, "healthcheck"), headers=headers) if(res.status_code == 200): break @@ -207,4 +224,5 @@ if __name__ == "__main__": pass sleep(5) print("Connected to pretix!", file=sys.stderr) + app.run(host="127.0.0.1", port=8188, dev=DEV_MODE, access_log=ACCESS_LOG) diff --git a/checkin.py b/checkin.py index a51e14f..f61f590 100644 --- a/checkin.py +++ b/checkin.py @@ -11,6 +11,7 @@ from hashlib import sha224 from time import time from urllib.parse import unquote import json +from metrics import * bp = Blueprint("checkin", url_prefix="/checkin") @@ -64,6 +65,7 @@ async def do_checkin(request): if not order.checked_in: async with httpx.AsyncClient() as client: + incPretixWrite() res = await client.post(base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]}, headers=headers) tpl = request.app.ctx.tpl.get_template('checkin_3.html') diff --git a/config.example.py b/config.example.py index 28db10e..f72d50a 100644 --- a/config.example.py +++ b/config.example.py @@ -40,6 +40,8 @@ DEV_MODE = True ACCESS_LOG = True EXTRA_PRINTS = True +METRICS_PATH = "/welcome/metrics" + # Additional configured locales. # If an order has a country that's not listed here, # Will default to an english preference. @@ -132,5 +134,5 @@ ROOM_CAPACITY_MAP = { 'bed_in_room_overflow1_2': 2, } -# Autofilled +# Autofilled. Maps roomTypeId -> roomName ROOM_TYPE_NAMES = { } \ No newline at end of file diff --git a/ext.py b/ext.py index 5ececfc..4360783 100644 --- a/ext.py +++ b/ext.py @@ -8,6 +8,7 @@ from os.path import join import json from sanic.log import logger from time import time +from metrics import * import asyncio @dataclass @@ -159,6 +160,7 @@ class Order: localHeaders = dict(headers) localHeaders['Content-Type'] = mimeType localHeaders['Content-Disposition'] = f'attachment; filename="{fileName}"' + incPretixWrite() res = await client.post(join(base_url, 'upload'), headers=localHeaders, content=data) res = res.json() await self.edit_answer(name, res['id']) @@ -184,6 +186,7 @@ class Order: if (not found) and (new_answer is not None): async with httpx.AsyncClient() as client: + incPretixRead() res = await client.get(join(base_url_event, 'questions/'), headers=headers) res = res.json() for r in res['results']: @@ -211,6 +214,7 @@ class Order: # del self.answers[i]['options'] # del self.answers[i]['option_identifiers'] + incPretixWrite() res = await client.patch(join(base_url_event, f'orderpositions/{self.position_id}/'), headers=headers, json={'answers': self.answers}) if res.status_code != 200: @@ -244,6 +248,7 @@ class Quotas: async def get_quotas(request: Request=None): async with httpx.AsyncClient() as client: + incPretixRead() res = await client.get(join(base_url_event, 'quotas/?order=id&with_availability=true'), headers=headers) res = res.json() @@ -302,6 +307,7 @@ class OrderManager: async with httpx.AsyncClient() as client: while 1: p += 1 + incPretixRead() res = await client.get(join(base_url_event, f"orders/?page={p}"), headers=headers) if res.status_code == 404: break # Parse order data @@ -344,6 +350,7 @@ class OrderManager: if DEV_MODE and EXTRA_PRINTS: logger.debug(f'Fetching {code} with secret {secret}') async with httpx.AsyncClient() as client: + incPretixRead() res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers) if res.status_code != 200: if request: diff --git a/metrics.py b/metrics.py new file mode 100644 index 0000000..805f45c --- /dev/null +++ b/metrics.py @@ -0,0 +1,58 @@ +from sanic.log import logger, logging +from logging import LogRecord +from config import * + +METRICS_REQ_NO = 0 +METRICS_PRETIX_READ = 0 +METRICS_PRETIX_WRITE = 0 + +def incPretixRead(): + global METRICS_PRETIX_READ + METRICS_PRETIX_READ += 1 + +def incPretixWrite(): + global METRICS_PRETIX_WRITE + METRICS_PRETIX_WRITE += 1 + +def incReqNo(): + global METRICS_REQ_NO + METRICS_REQ_NO += 1 + +def getMetricsText(): + global METRICS_REQ_NO + global METRICS_PRETIX_READ + global METRICS_PRETIX_WRITE + out = [] + + out.append(f'sanic_request_count{{}} {METRICS_REQ_NO}') + out.append(f'webint_pretix_read_count{{}} {METRICS_PRETIX_READ}') + out.append(f'webint_pretix_write_count{{}} {METRICS_PRETIX_WRITE}') + + return "\n".join(out) + +def getRoomCountersText(request): + out = [] + try : + daily = 0 + counters = {} + for id in ROOM_TYPE_NAMES.keys(): + counters[id] = 0 + + for order in request.app.ctx.om.cache.values(): + if(order.daily): + daily += 1 + else: + counters[order.bed_in_room] += 1 + + for id, count in counters.items(): + out.append(f'webint_order_room_counter{{label="{ROOM_TYPE_NAMES[id]}"}} {count}') + out.append(f'webint_order_room_counter{{label="Daily"}} {daily}') + + except Exception as e: + print(e) + logger.warning("Error in loading metrics rooms") + return "\n".join(out) + +class MetricsFilter(logging.Filter): + def filter(self, record : LogRecord): + return not (record.request.endswith("/manage/metrics") and record.status == 200) \ No newline at end of file diff --git a/reg.furizon.net/index_beyond.html b/reg.furizon.net/index_beyond.html index e244324..c75d549 100644 --- a/reg.furizon.net/index_beyond.html +++ b/reg.furizon.net/index_beyond.html @@ -66,21 +66,6 @@ #clock {display:block;} - - -
diff --git a/room.py b/room.py index 4474c24..e521e82 100644 --- a/room.py +++ b/room.py @@ -344,6 +344,7 @@ async def confirm_room(request, order: Order, quotas: Quotas): # } # # async with httpx.AsyncClient() as client: + # incPretixRead() # res = await client.post(join(base_url_event, "orderpositions/"), headers=headers, json=thing) # # if res.status_code != 201: diff --git a/logOrders.py b/stuff/logOrders.py similarity index 100% rename from logOrders.py rename to stuff/logOrders.py diff --git a/stuff/systemMetrics.py b/stuff/systemMetrics.py new file mode 100644 index 0000000..696b499 --- /dev/null +++ b/stuff/systemMetrics.py @@ -0,0 +1,91 @@ +import psutil +import time +import sys + +from sanic import Sanic +from sanic import response as res + +DEV_MODE = False +ACCESS_LOG = False + +app = Sanic(__name__) + +@app.route("/") +async def main(req): + return res.text("I\'m a teapot", status=418) + +@app.route("/metrics") +async def metrics(req): + out = [] + + cpus = psutil.cpu_percent(percpu=True) + totalCpu = 0 + for i in range(len(cpus)): + out.append(f'monitor_cpu_core_usage_percent{{core="{i}"}} {cpus[i]}') + totalCpu += cpus[i] + out.append(f'monitor_cpu_core_usage_percent{{core="avg"}} {"%.2f" % (totalCpu / len(cpus))}') + + diskIo = psutil.disk_io_counters(nowrap=True) + out.append(f'monitor_diskio{{value="read_count"}} {diskIo.read_count}') + out.append(f'monitor_diskio{{value="write_count"}} {diskIo.write_count}') + out.append(f'monitor_diskio{{value="read_bytes"}} {diskIo.read_bytes}') + out.append(f'monitor_diskio{{value="write_bytes"}} {diskIo.write_bytes}') + out.append(f'monitor_diskio{{value="read_time"}} {diskIo.read_time}') + out.append(f'monitor_diskio{{value="write_time"}} {diskIo.write_time}') + + disks = psutil.disk_partitions() + for disk in disks: + try: + dUsage = psutil.disk_usage(disk.mountpoint) + out.append(f'monitor_disk_usage_percent{{partition="{disk.mountpoint}"}} {dUsage.percent}') + except: + pass + + mem = psutil.virtual_memory() + out.append(f'monitor_memory{{value="total"}} {mem.total}') + out.append(f'monitor_memory{{value="available"}} {mem.available}') + out.append(f'monitor_memory{{value="percent"}} {mem.percent}') + out.append(f'monitor_memory{{value="used"}} {mem.used}') + out.append(f'monitor_memory{{value="free"}} {mem.free}') + + swap = psutil.swap_memory() + out.append(f'monitor_swap{{value="total"}} {swap.total}') + out.append(f'monitor_swap{{value="used"}} {swap.used}') + out.append(f'monitor_swap{{value="free"}} {swap.free}') + out.append(f'monitor_swap{{value="percent"}} {swap.percent}') + out.append(f'monitor_swap{{value="sin"}} {swap.sin}') + out.append(f'monitor_swap{{value="sout"}} {swap.sout}') + + bootTime = psutil.boot_time() + out.append(f'monitor_boot_time{{}} {int(time.time() - bootTime)}') + + netioConnections = psutil.net_connections() + out.append(f'monitor_netio_connections{{}} {len(netioConnections)}') + netioCounters = psutil.net_io_counters(nowrap=True) + out.append(f'monitor_netio_counters{{value="bytes_sent"}} {netioCounters.bytes_sent}') + out.append(f'monitor_netio_counters{{value="bytes_recv"}} {netioCounters.bytes_recv}') + out.append(f'monitor_netio_counters{{value="packets_sent"}} {netioCounters.packets_sent}') + out.append(f'monitor_netio_counters{{value="packets_recv"}} {netioCounters.packets_recv}') + out.append(f'monitor_netio_counters{{value="errin"}} {netioCounters.errin}') + out.append(f'monitor_netio_counters{{value="errout"}} {netioCounters.errout}') + out.append(f'monitor_netio_counters{{value="dropin"}} {netioCounters.dropin}') + out.append(f'monitor_netio_counters{{value="dropout"}} {netioCounters.dropout}') + + return res.text("\n".join(out), status=200) + + +if __name__ == '__main__': + ip = "127.0.0.1" + port = 2611 + if(len(sys.argv) > 1): + ip = sys.argv[1] + if(len(sys.argv) > 2): + try: + port = int(sys.argv[2]) + except: + print("Port must be a numeric value!") + exit(1) + if(port < 1 or port > 0xffff): + print("Port must be in [1, 65535] range!") + exit(1) + app.run(host=ip, port=port, dev=DEV_MODE, access_log=ACCESS_LOG) diff --git a/tpl/base.html b/tpl/base.html index a832e47..e63aa76 100644 --- a/tpl/base.html +++ b/tpl/base.html @@ -48,28 +48,6 @@ grid-template-rows: repeat(2, 1fr); } - - - - - - {% include 'blocks/navbar.html' %} diff --git a/tpl/privacy.html b/tpl/privacy.html index 42be31b..becb163 100644 --- a/tpl/privacy.html +++ b/tpl/privacy.html @@ -28,7 +28,6 @@
  • First three octets of your ip address
  • List of status codes (404, 500, 403...) you encountered, and when you encountered them
    • -

    This info is collected through our first-party domain y.foxo.me

    Backups and export of data

    All data is stored in servers managed by Furizon APS, inside of facilities in the Italian territory. No data is exported outside of the italian territory. All communication uses end to end encryption.

    diff --git a/utils.py b/utils.py index 86f5b2f..a582ff1 100644 --- a/utils.py +++ b/utils.py @@ -4,7 +4,9 @@ from config import * import httpx from messages import ROOM_ERROR_TYPES from email_util import send_unconfirm_message +from sanic.response import text, html, redirect, raw from sanic.log import logger +from metrics import * METADATA_TAG = "meta_data" VARIATIONS_TAG = "variations" @@ -33,6 +35,7 @@ async def load_questions(): p = 0 while 1: p += 1 + incPretixRead() res = await client.get(join(base_url_event, f"questions/?page={p}"), headers=headers) if res.status_code == 404: break @@ -50,6 +53,7 @@ async def load_items(): p = 0 while 1: p += 1 + incPretixRead() res = await client.get(join(base_url_event, f"items/?page={p}"), headers=headers) if res.status_code == 404: break @@ -253,4 +257,4 @@ async def check_room(request, order, om=None): if order.room_confirmed: allOk = False order.set_room_errors(room_errors) - return (order, allOk, room_members) + return (order, allOk, room_members) \ No newline at end of file