stranck-dev #18

Merged
drew merged 4 commits from stranck-dev into drew-dev 2024-02-17 22:50:03 +00:00
12 changed files with 186 additions and 41 deletions

20
app.py
View File

@ -14,7 +14,8 @@ from messages import LOCALES
import sqlite3 import sqlite3
import requests import requests
import sys import sys
from sanic.log import logger, logging from sanic.log import logger, logging, access_logger
from metrics import *
app = Sanic(__name__) app = Sanic(__name__)
app.static("/res", "res/") app.static("/res", "res/")
@ -36,6 +37,7 @@ app.blueprint([room_bp, karaoke_bp, propic_bp, export_bp, stats_bp, api_bp, carp
@app.exception(exceptions.SanicException) @app.exception(exceptions.SanicException)
async def clear_session(request, exception): async def clear_session(request, exception):
logger.warning(f"{request} -> {exception}")
tpl = app.ctx.tpl.get_template('error.html') tpl = app.ctx.tpl.get_template('error.html')
r = html(tpl.render(exception=exception)) r = html(tpl.render(exception=exception))
@ -48,6 +50,7 @@ async def clear_session(request, exception):
async def main_start(*_): async def main_start(*_):
logger.info(f"[{app.name}] >>>>>> main_start <<<<<<") logger.info(f"[{app.name}] >>>>>> main_start <<<<<<")
logger.setLevel(LOG_LEVEL) logger.setLevel(LOG_LEVEL)
access_logger.addFilter(MetricsFilter())
app.config.REQUEST_MAX_SIZE = PROPIC_MAX_FILE_SIZE * 3 app.config.REQUEST_MAX_SIZE = PROPIC_MAX_FILE_SIZE * 3
@ -88,6 +91,7 @@ async def redirect_explore(request, code, secret, order: Order, secret2=None):
if not order: if not order:
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixRead()
res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers) res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers)
if res.status_code != 200: if res.status_code != 200:
@ -152,6 +156,7 @@ async def download_ticket(request, order: Order):
raise exceptions.Forbidden("You are not allowed to download this ticket.") raise exceptions.Forbidden("You are not allowed to download this ticket.")
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixRead()
res = await client.get(join(base_url_event, f"orders/{order.code}/download/pdf/"), headers=headers) res = await client.get(join(base_url_event, f"orders/{order.code}/download/pdf/"), headers=headers)
if res.status_code == 409: if res.status_code == 409:
@ -186,6 +191,16 @@ async def logout(request):
raise exceptions.Forbidden("You have been logged out.") raise exceptions.Forbidden("You have been logged out.")
@app.get(METRICS_PATH)
async def metrics(request):
return text(getMetricsText() + "\n" + getRoomCountersText(request))
@app.on_request
async def countReqs(request : Request):
global METRICS_REQ_NO
if(request.path != METRICS_PATH):
incReqNo()
if __name__ == "__main__": if __name__ == "__main__":
# Wait for pretix in server reboot # Wait for pretix in server reboot
# Using a docker configuration, pretix may be unable to talk with postgres if postgres' service started before it. # Using a docker configuration, pretix may be unable to talk with postgres if postgres' service started before it.
@ -196,10 +211,12 @@ if __name__ == "__main__":
while True: while True:
print("Trying connecting to pretix...", file=sys.stderr) print("Trying connecting to pretix...", file=sys.stderr)
try: try:
incPretixRead()
res = requests.get(base_url_event, headers=headers) res = requests.get(base_url_event, headers=headers)
res = res.json() res = res.json()
if(res['slug'] == EVENT_NAME): if(res['slug'] == EVENT_NAME):
print("Healtchecking...", file=sys.stderr) print("Healtchecking...", file=sys.stderr)
incPretixRead()
res = requests.get(join(domain, "healthcheck"), headers=headers) res = requests.get(join(domain, "healthcheck"), headers=headers)
if(res.status_code == 200): if(res.status_code == 200):
break break
@ -207,4 +224,5 @@ if __name__ == "__main__":
pass pass
sleep(5) sleep(5)
print("Connected to pretix!", file=sys.stderr) print("Connected to pretix!", file=sys.stderr)
app.run(host="127.0.0.1", port=8188, dev=DEV_MODE, access_log=ACCESS_LOG) app.run(host="127.0.0.1", port=8188, dev=DEV_MODE, access_log=ACCESS_LOG)

View File

@ -11,6 +11,7 @@ from hashlib import sha224
from time import time from time import time
from urllib.parse import unquote from urllib.parse import unquote
import json import json
from metrics import *
bp = Blueprint("checkin", url_prefix="/checkin") bp = Blueprint("checkin", url_prefix="/checkin")
@ -64,6 +65,7 @@ async def do_checkin(request):
if not order.checked_in: if not order.checked_in:
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixWrite()
res = await client.post(base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]}, headers=headers) res = await client.post(base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]}, headers=headers)
tpl = request.app.ctx.tpl.get_template('checkin_3.html') tpl = request.app.ctx.tpl.get_template('checkin_3.html')

View File

@ -40,6 +40,8 @@ DEV_MODE = True
ACCESS_LOG = True ACCESS_LOG = True
EXTRA_PRINTS = True EXTRA_PRINTS = True
METRICS_PATH = "/welcome/metrics"
# Additional configured locales. # Additional configured locales.
# If an order has a country that's not listed here, # If an order has a country that's not listed here,
# Will default to an english preference. # Will default to an english preference.
@ -132,5 +134,5 @@ ROOM_CAPACITY_MAP = {
'bed_in_room_overflow1_2': 2, 'bed_in_room_overflow1_2': 2,
} }
# Autofilled # Autofilled. Maps roomTypeId -> roomName
ROOM_TYPE_NAMES = { } ROOM_TYPE_NAMES = { }

7
ext.py
View File

@ -8,6 +8,7 @@ from os.path import join
import json import json
from sanic.log import logger from sanic.log import logger
from time import time from time import time
from metrics import *
import asyncio import asyncio
@dataclass @dataclass
@ -159,6 +160,7 @@ class Order:
localHeaders = dict(headers) localHeaders = dict(headers)
localHeaders['Content-Type'] = mimeType localHeaders['Content-Type'] = mimeType
localHeaders['Content-Disposition'] = f'attachment; filename="{fileName}"' localHeaders['Content-Disposition'] = f'attachment; filename="{fileName}"'
incPretixWrite()
res = await client.post(join(base_url, 'upload'), headers=localHeaders, content=data) res = await client.post(join(base_url, 'upload'), headers=localHeaders, content=data)
res = res.json() res = res.json()
await self.edit_answer(name, res['id']) await self.edit_answer(name, res['id'])
@ -184,6 +186,7 @@ class Order:
if (not found) and (new_answer is not None): if (not found) and (new_answer is not None):
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixRead()
res = await client.get(join(base_url_event, 'questions/'), headers=headers) res = await client.get(join(base_url_event, 'questions/'), headers=headers)
res = res.json() res = res.json()
for r in res['results']: for r in res['results']:
@ -211,6 +214,7 @@ class Order:
# del self.answers[i]['options'] # del self.answers[i]['options']
# del self.answers[i]['option_identifiers'] # del self.answers[i]['option_identifiers']
incPretixWrite()
res = await client.patch(join(base_url_event, f'orderpositions/{self.position_id}/'), headers=headers, json={'answers': self.answers}) res = await client.patch(join(base_url_event, f'orderpositions/{self.position_id}/'), headers=headers, json={'answers': self.answers})
if res.status_code != 200: if res.status_code != 200:
@ -244,6 +248,7 @@ class Quotas:
async def get_quotas(request: Request=None): async def get_quotas(request: Request=None):
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixRead()
res = await client.get(join(base_url_event, 'quotas/?order=id&with_availability=true'), headers=headers) res = await client.get(join(base_url_event, 'quotas/?order=id&with_availability=true'), headers=headers)
res = res.json() res = res.json()
@ -302,6 +307,7 @@ class OrderManager:
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
while 1: while 1:
p += 1 p += 1
incPretixRead()
res = await client.get(join(base_url_event, f"orders/?page={p}"), headers=headers) res = await client.get(join(base_url_event, f"orders/?page={p}"), headers=headers)
if res.status_code == 404: break if res.status_code == 404: break
# Parse order data # Parse order data
@ -344,6 +350,7 @@ class OrderManager:
if DEV_MODE and EXTRA_PRINTS: logger.debug(f'Fetching {code} with secret {secret}') if DEV_MODE and EXTRA_PRINTS: logger.debug(f'Fetching {code} with secret {secret}')
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
incPretixRead()
res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers) res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers)
if res.status_code != 200: if res.status_code != 200:
if request: if request:

58
metrics.py Normal file
View File

@ -0,0 +1,58 @@
from sanic.log import logger, logging
from logging import LogRecord
from config import *
METRICS_REQ_NO = 0
METRICS_PRETIX_READ = 0
METRICS_PRETIX_WRITE = 0
def incPretixRead():
global METRICS_PRETIX_READ
METRICS_PRETIX_READ += 1
def incPretixWrite():
global METRICS_PRETIX_WRITE
METRICS_PRETIX_WRITE += 1
def incReqNo():
global METRICS_REQ_NO
METRICS_REQ_NO += 1
def getMetricsText():
global METRICS_REQ_NO
global METRICS_PRETIX_READ
global METRICS_PRETIX_WRITE
out = []
out.append(f'sanic_request_count{{}} {METRICS_REQ_NO}')
out.append(f'webint_pretix_read_count{{}} {METRICS_PRETIX_READ}')
out.append(f'webint_pretix_write_count{{}} {METRICS_PRETIX_WRITE}')
return "\n".join(out)
def getRoomCountersText(request):
out = []
try :
daily = 0
counters = {}
for id in ROOM_TYPE_NAMES.keys():
counters[id] = 0
for order in request.app.ctx.om.cache.values():
if(order.daily):
daily += 1
else:
counters[order.bed_in_room] += 1
for id, count in counters.items():
out.append(f'webint_order_room_counter{{label="{ROOM_TYPE_NAMES[id]}"}} {count}')
out.append(f'webint_order_room_counter{{label="Daily"}} {daily}')
except Exception as e:
print(e)
logger.warning("Error in loading metrics rooms")
return "\n".join(out)
class MetricsFilter(logging.Filter):
def filter(self, record : LogRecord):
return not (record.request.endswith("/manage/metrics") and record.status == 200)

View File

@ -66,21 +66,6 @@
#clock {display:block;} #clock {display:block;}
</style> </style>
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://y.foxo.me/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '2']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
</head> </head>
<body> <body>
<main> <main>

View File

@ -344,6 +344,7 @@ async def confirm_room(request, order: Order, quotas: Quotas):
# } # }
# #
# async with httpx.AsyncClient() as client: # async with httpx.AsyncClient() as client:
# incPretixRead()
# res = await client.post(join(base_url_event, "orderpositions/"), headers=headers, json=thing) # res = await client.post(join(base_url_event, "orderpositions/"), headers=headers, json=thing)
# #
# if res.status_code != 201: # if res.status_code != 201:

91
stuff/systemMetrics.py Normal file
View File

@ -0,0 +1,91 @@
import psutil
import time
import sys
from sanic import Sanic
from sanic import response as res
DEV_MODE = False
ACCESS_LOG = False
app = Sanic(__name__)
@app.route("/")
async def main(req):
return res.text("I\'m a teapot", status=418)
@app.route("/metrics")
async def metrics(req):
out = []
cpus = psutil.cpu_percent(percpu=True)
totalCpu = 0
for i in range(len(cpus)):
out.append(f'monitor_cpu_core_usage_percent{{core="{i}"}} {cpus[i]}')
totalCpu += cpus[i]
out.append(f'monitor_cpu_core_usage_percent{{core="avg"}} {"%.2f" % (totalCpu / len(cpus))}')
diskIo = psutil.disk_io_counters(nowrap=True)
out.append(f'monitor_diskio{{value="read_count"}} {diskIo.read_count}')
out.append(f'monitor_diskio{{value="write_count"}} {diskIo.write_count}')
out.append(f'monitor_diskio{{value="read_bytes"}} {diskIo.read_bytes}')
out.append(f'monitor_diskio{{value="write_bytes"}} {diskIo.write_bytes}')
out.append(f'monitor_diskio{{value="read_time"}} {diskIo.read_time}')
out.append(f'monitor_diskio{{value="write_time"}} {diskIo.write_time}')
disks = psutil.disk_partitions()
for disk in disks:
try:
dUsage = psutil.disk_usage(disk.mountpoint)
out.append(f'monitor_disk_usage_percent{{partition="{disk.mountpoint}"}} {dUsage.percent}')
except:
pass
mem = psutil.virtual_memory()
out.append(f'monitor_memory{{value="total"}} {mem.total}')
out.append(f'monitor_memory{{value="available"}} {mem.available}')
out.append(f'monitor_memory{{value="percent"}} {mem.percent}')
out.append(f'monitor_memory{{value="used"}} {mem.used}')
out.append(f'monitor_memory{{value="free"}} {mem.free}')
swap = psutil.swap_memory()
out.append(f'monitor_swap{{value="total"}} {swap.total}')
out.append(f'monitor_swap{{value="used"}} {swap.used}')
out.append(f'monitor_swap{{value="free"}} {swap.free}')
out.append(f'monitor_swap{{value="percent"}} {swap.percent}')
out.append(f'monitor_swap{{value="sin"}} {swap.sin}')
out.append(f'monitor_swap{{value="sout"}} {swap.sout}')
bootTime = psutil.boot_time()
out.append(f'monitor_boot_time{{}} {int(time.time() - bootTime)}')
netioConnections = psutil.net_connections()
out.append(f'monitor_netio_connections{{}} {len(netioConnections)}')
netioCounters = psutil.net_io_counters(nowrap=True)
out.append(f'monitor_netio_counters{{value="bytes_sent"}} {netioCounters.bytes_sent}')
out.append(f'monitor_netio_counters{{value="bytes_recv"}} {netioCounters.bytes_recv}')
out.append(f'monitor_netio_counters{{value="packets_sent"}} {netioCounters.packets_sent}')
out.append(f'monitor_netio_counters{{value="packets_recv"}} {netioCounters.packets_recv}')
out.append(f'monitor_netio_counters{{value="errin"}} {netioCounters.errin}')
out.append(f'monitor_netio_counters{{value="errout"}} {netioCounters.errout}')
out.append(f'monitor_netio_counters{{value="dropin"}} {netioCounters.dropin}')
out.append(f'monitor_netio_counters{{value="dropout"}} {netioCounters.dropout}')
return res.text("\n".join(out), status=200)
if __name__ == '__main__':
ip = "127.0.0.1"
port = 2611
if(len(sys.argv) > 1):
ip = sys.argv[1]
if(len(sys.argv) > 2):
try:
port = int(sys.argv[2])
except:
print("Port must be a numeric value!")
exit(1)
if(port < 1 or port > 0xffff):
print("Port must be in [1, 65535] range!")
exit(1)
app.run(host=ip, port=port, dev=DEV_MODE, access_log=ACCESS_LOG)

View File

@ -48,28 +48,6 @@
grid-template-rows: repeat(2, 1fr); grid-template-rows: repeat(2, 1fr);
} }
</style> </style>
<!-- Matomo -->
<script>
var _paq = window._paq = window._paq || [];
/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
_paq.push(["setCampaignNameKey", "private_area"]);
{% if order %}
_paq.push(['setUserId', '{{order.code}}']);
{% endif %}
_paq.push(['trackPageView']);
_paq.push(['enableLinkTracking']);
(function() {
var u="https://y.foxo.me/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '2']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<script src="/res/scripts/base.js" defer></script>
<noscript><p><img src="https://y.foxo.me/matomo.php?idsite=2&amp;rec=1" style="border:0;" alt="" /></p></noscript>
<!-- End Matomo Code -->
</head> </head>
<body onscroll="onScrollNav()"> <body onscroll="onScrollNav()">
{% include 'blocks/navbar.html' %} {% include 'blocks/navbar.html' %}

View File

@ -28,7 +28,6 @@
<li>First three octets of your ip address</li> <li>First three octets of your ip address</li>
<li>List of status codes (404, 500, 403...) you encountered, and when you encountered them</li> <li>List of status codes (404, 500, 403...) you encountered, and when you encountered them</li>
</ul> </ul>
<p>This info is collected through our first-party domain <em>y.foxo.me</em></p>
<h2>Backups and export of data</h2> <h2>Backups and export of data</h2>
<p>All data is stored in servers managed by Furizon APS, inside of facilities in the Italian territory. No data is exported outside of the italian territory. All communication uses end to end encryption.</p> <p>All data is stored in servers managed by Furizon APS, inside of facilities in the Italian territory. No data is exported outside of the italian territory. All communication uses end to end encryption.</p>

View File

@ -4,7 +4,9 @@ from config import *
import httpx import httpx
from messages import ROOM_ERROR_TYPES from messages import ROOM_ERROR_TYPES
from email_util import send_unconfirm_message from email_util import send_unconfirm_message
from sanic.response import text, html, redirect, raw
from sanic.log import logger from sanic.log import logger
from metrics import *
METADATA_TAG = "meta_data" METADATA_TAG = "meta_data"
VARIATIONS_TAG = "variations" VARIATIONS_TAG = "variations"
@ -33,6 +35,7 @@ async def load_questions():
p = 0 p = 0
while 1: while 1:
p += 1 p += 1
incPretixRead()
res = await client.get(join(base_url_event, f"questions/?page={p}"), headers=headers) res = await client.get(join(base_url_event, f"questions/?page={p}"), headers=headers)
if res.status_code == 404: break if res.status_code == 404: break
@ -50,6 +53,7 @@ async def load_items():
p = 0 p = 0
while 1: while 1:
p += 1 p += 1
incPretixRead()
res = await client.get(join(base_url_event, f"items/?page={p}"), headers=headers) res = await client.get(join(base_url_event, f"items/?page={p}"), headers=headers)
if res.status_code == 404: break if res.status_code == 404: break
@ -253,4 +257,4 @@ async def check_room(request, order, om=None):
if order.room_confirmed: if order.room_confirmed:
allOk = False allOk = False
order.set_room_errors(room_errors) order.set_room_errors(room_errors)
return (order, allOk, room_members) return (order, allOk, room_members)