Merge pull request 'stranck-dev' (#18) from stranck-dev into drew-dev

Reviewed-on: #18

app.py

@@ -14,7 +14,8 @@ from messages import LOCALES
 import sqlite3
 import requests
 import sys
-from sanic.log import logger, logging
+from sanic.log import logger, logging, access_logger
+from metrics import *
 
 app = Sanic(__name__)
 app.static("/res", "res/")
@@ -36,6 +37,7 @@ app.blueprint([room_bp, karaoke_bp, propic_bp, export_bp, stats_bp, api_bp, carp
 				
 @app.exception(exceptions.SanicException)
 async def clear_session(request, exception):
+	logger.warning(f"{request} -> {exception}")
 	tpl = app.ctx.tpl.get_template('error.html')
 	r = html(tpl.render(exception=exception))
 	
@@ -48,6 +50,7 @@ async def clear_session(request, exception):
 async def main_start(*_):
 	logger.info(f"[{app.name}] >>>>>> main_start <<<<<<")
 	logger.setLevel(LOG_LEVEL)
+	access_logger.addFilter(MetricsFilter())
 
 	app.config.REQUEST_MAX_SIZE = PROPIC_MAX_FILE_SIZE * 3
 	
@@ -88,6 +91,7 @@ async def redirect_explore(request, code, secret, order: Order, secret2=None):
 
 	if not order:
 		async with httpx.AsyncClient() as client:
+			incPretixRead()
 			res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers)
 			
 			if res.status_code != 200:
@@ -152,6 +156,7 @@ async def download_ticket(request, order: Order):
 		raise exceptions.Forbidden("You are not allowed to download this ticket.")
 		
 	async with httpx.AsyncClient() as client:
+		incPretixRead()
 		res = await client.get(join(base_url_event, f"orders/{order.code}/download/pdf/"), headers=headers)
 	
 	if res.status_code == 409:
@@ -186,6 +191,16 @@ async def logout(request):
 
 	raise exceptions.Forbidden("You have been logged out.")
 
+@app.get(METRICS_PATH)
+async def metrics(request):
+	return text(getMetricsText() + "\n" + getRoomCountersText(request))
+
+@app.on_request
+async def countReqs(request : Request):
+	global METRICS_REQ_NO
+	if(request.path != METRICS_PATH):
+		incReqNo()
+
 if __name__ == "__main__":
 	# Wait for pretix in server reboot
 	# Using a docker configuration, pretix may be unable to talk with postgres if postgres' service started before it. 
@@ -196,10 +211,12 @@ if __name__ == "__main__":
 	while True:
 		print("Trying connecting to pretix...", file=sys.stderr)
 		try:
+			incPretixRead()
 			res = requests.get(base_url_event, headers=headers)
 			res = res.json()
 			if(res['slug'] == EVENT_NAME):
 				print("Healtchecking...", file=sys.stderr)
+				incPretixRead()
 				res = requests.get(join(domain, "healthcheck"), headers=headers)
 				if(res.status_code == 200):
 					break
@@ -207,4 +224,5 @@ if __name__ == "__main__":
 			pass
 		sleep(5)
 	print("Connected to pretix!", file=sys.stderr)
+
 	app.run(host="127.0.0.1", port=8188, dev=DEV_MODE, access_log=ACCESS_LOG)

checkin.py

@@ -11,6 +11,7 @@ from hashlib import sha224
 from time import time
 from urllib.parse import unquote
 import json
+from metrics import *
 
 bp = Blueprint("checkin", url_prefix="/checkin")
 
@@ -64,6 +65,7 @@ async def do_checkin(request):
 	
 	if not order.checked_in:
 		async with httpx.AsyncClient() as client:
+			incPretixWrite()
 			res = await client.post(base_url_event.replace(f'events/{EVENT_NAME}/', 'checkinrpc/redeem/'), json={'secret': order.barcode, 'source_type': 'barcode', 'type': 'entry', 'lists': [3,]}, headers=headers)
 	
 	tpl = request.app.ctx.tpl.get_template('checkin_3.html')

config.example.py

@@ -40,6 +40,8 @@ DEV_MODE = True
 ACCESS_LOG = True
 EXTRA_PRINTS = True
 
+METRICS_PATH = "/welcome/metrics"
+
 # Additional configured locales.
 # If an order has a country that's not listed here,
 # Will default to an english preference.
@@ -132,5 +134,5 @@ ROOM_CAPACITY_MAP = {
     'bed_in_room_overflow1_2': 2,
 }
 
-# Autofilled
+# Autofilled. Maps roomTypeId -> roomName
 ROOM_TYPE_NAMES = { }

ext.py

@@ -8,6 +8,7 @@ from os.path import join
 import json
 from sanic.log import logger
 from time import time
+from metrics import *
 import asyncio
 
 @dataclass
@@ -159,6 +160,7 @@ class Order:
 				localHeaders = dict(headers)
 				localHeaders['Content-Type'] = mimeType
 				localHeaders['Content-Disposition'] = f'attachment; filename="{fileName}"'
+				incPretixWrite()
 				res = await client.post(join(base_url, 'upload'), headers=localHeaders, content=data)
 				res = res.json()
 				await self.edit_answer(name, res['id'])
@@ -184,6 +186,7 @@ class Order:
 		if (not found) and (new_answer is not None):
 			
 			async with httpx.AsyncClient() as client:
+				incPretixRead()
 				res = await client.get(join(base_url_event, 'questions/'), headers=headers)
 				res = res.json()
 			for r in res['results']:
@@ -211,6 +214,7 @@ class Order:
 				#	del self.answers[i]['options']
 				#	del self.answers[i]['option_identifiers']
 			
+			incPretixWrite()
 			res = await client.patch(join(base_url_event, f'orderpositions/{self.position_id}/'), headers=headers, json={'answers': self.answers})
 			
 			if res.status_code != 200:
@@ -244,6 +248,7 @@ class Quotas:
 
 async def get_quotas(request: Request=None):
 	async with httpx.AsyncClient() as client:
+		incPretixRead()
 		res = await client.get(join(base_url_event, 'quotas/?order=id&with_availability=true'), headers=headers)
 		res = res.json()
 		
@@ -302,6 +307,7 @@ class OrderManager:
 			async with httpx.AsyncClient() as client:
 				while 1:
 					p += 1
+					incPretixRead()
 					res = await client.get(join(base_url_event, f"orders/?page={p}"), headers=headers)
 					if res.status_code == 404: break
 					# Parse order data
@@ -344,6 +350,7 @@ class OrderManager:
 			if DEV_MODE and EXTRA_PRINTS: logger.debug(f'Fetching {code} with secret {secret}')
 
 			async with httpx.AsyncClient() as client:
+				incPretixRead()
 				res = await client.get(join(base_url_event, f"orders/{code}/"), headers=headers)
 				if res.status_code != 200:
 					if request:

metrics.py

@@ -0,0 +1,58 @@
+from sanic.log import logger, logging
+from logging import LogRecord
+from config import *
+
+METRICS_REQ_NO = 0
+METRICS_PRETIX_READ = 0
+METRICS_PRETIX_WRITE = 0
+
+def incPretixRead():
+	global METRICS_PRETIX_READ
+	METRICS_PRETIX_READ += 1
+
+def incPretixWrite():
+	global METRICS_PRETIX_WRITE
+	METRICS_PRETIX_WRITE += 1
+
+def incReqNo():
+	global METRICS_REQ_NO
+	METRICS_REQ_NO += 1
+
+def getMetricsText():
+	global METRICS_REQ_NO
+	global METRICS_PRETIX_READ
+	global METRICS_PRETIX_WRITE
+	out = []
+
+	out.append(f'sanic_request_count{{}} {METRICS_REQ_NO}')
+	out.append(f'webint_pretix_read_count{{}} {METRICS_PRETIX_READ}')
+	out.append(f'webint_pretix_write_count{{}} {METRICS_PRETIX_WRITE}')
+
+	return "\n".join(out)
+
+def getRoomCountersText(request):
+	out = []
+	try :
+		daily = 0
+		counters = {}
+		for id in ROOM_TYPE_NAMES.keys():
+			counters[id] = 0
+
+		for order in request.app.ctx.om.cache.values():
+			if(order.daily):
+				daily += 1
+			else:
+				counters[order.bed_in_room] += 1
+
+		for id, count in counters.items():
+			out.append(f'webint_order_room_counter{{label="{ROOM_TYPE_NAMES[id]}"}} {count}')
+		out.append(f'webint_order_room_counter{{label="Daily"}} {daily}')
+
+	except Exception as e:
+		print(e)
+		logger.warning("Error in loading metrics rooms")
+	return "\n".join(out)
+
+class MetricsFilter(logging.Filter):
+	def filter(self, record : LogRecord):
+		return not (record.request.endswith("/manage/metrics") and record.status == 200)

reg.furizon.net/index_beyond.html

@@ -66,21 +66,6 @@
 		#clock {display:block;}
 		
 	</style>
-	<!-- Matomo -->
-	<script>
-		var _paq = window._paq = window._paq || [];
-		/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
-		_paq.push(['trackPageView']);
-		_paq.push(['enableLinkTracking']);
-		(function() {
-			var u="https://y.foxo.me/";
-			_paq.push(['setTrackerUrl', u+'matomo.php']);
-			_paq.push(['setSiteId', '2']);
-			var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
-			g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
-		})();
-	</script>
-	<!-- End Matomo Code -->
 </head>
 <body>
 	<main>

room.py

@@ -344,6 +344,7 @@ async def confirm_room(request, order: Order, quotas: Quotas):
 	# }
 	# 
 	# async with httpx.AsyncClient() as client:
+	#	incPretixRead()
 	# 	res = await client.post(join(base_url_event, "orderpositions/"), headers=headers, json=thing)
 	# 	
 	# 	if res.status_code != 201:

stuff/systemMetrics.py

@@ -0,0 +1,91 @@
+import psutil
+import time
+import sys
+
+from sanic import Sanic
+from sanic import response as res
+
+DEV_MODE = False
+ACCESS_LOG = False
+
+app = Sanic(__name__)
+
+@app.route("/")
+async def main(req):
+	return res.text("I\'m a teapot", status=418)
+
+@app.route("/metrics")
+async def metrics(req):
+	out = []
+
+	cpus = psutil.cpu_percent(percpu=True)
+	totalCpu = 0
+	for i in range(len(cpus)):
+		out.append(f'monitor_cpu_core_usage_percent{{core="{i}"}} {cpus[i]}')
+		totalCpu += cpus[i]
+	out.append(f'monitor_cpu_core_usage_percent{{core="avg"}} {"%.2f" % (totalCpu / len(cpus))}')
+
+	diskIo = psutil.disk_io_counters(nowrap=True)
+	out.append(f'monitor_diskio{{value="read_count"}} {diskIo.read_count}')
+	out.append(f'monitor_diskio{{value="write_count"}} {diskIo.write_count}')
+	out.append(f'monitor_diskio{{value="read_bytes"}} {diskIo.read_bytes}')
+	out.append(f'monitor_diskio{{value="write_bytes"}} {diskIo.write_bytes}')
+	out.append(f'monitor_diskio{{value="read_time"}} {diskIo.read_time}')
+	out.append(f'monitor_diskio{{value="write_time"}} {diskIo.write_time}')
+
+	disks = psutil.disk_partitions()
+	for disk in disks:
+		try:
+			dUsage = psutil.disk_usage(disk.mountpoint)
+			out.append(f'monitor_disk_usage_percent{{partition="{disk.mountpoint}"}} {dUsage.percent}')
+		except: 
+			pass
+
+	mem = psutil.virtual_memory()
+	out.append(f'monitor_memory{{value="total"}} {mem.total}')
+	out.append(f'monitor_memory{{value="available"}} {mem.available}')
+	out.append(f'monitor_memory{{value="percent"}} {mem.percent}')
+	out.append(f'monitor_memory{{value="used"}} {mem.used}')
+	out.append(f'monitor_memory{{value="free"}} {mem.free}')
+
+	swap = psutil.swap_memory()
+	out.append(f'monitor_swap{{value="total"}} {swap.total}')
+	out.append(f'monitor_swap{{value="used"}} {swap.used}')
+	out.append(f'monitor_swap{{value="free"}} {swap.free}')
+	out.append(f'monitor_swap{{value="percent"}} {swap.percent}')
+	out.append(f'monitor_swap{{value="sin"}} {swap.sin}')
+	out.append(f'monitor_swap{{value="sout"}} {swap.sout}')
+
+	bootTime = psutil.boot_time()
+	out.append(f'monitor_boot_time{{}} {int(time.time() - bootTime)}')
+
+	netioConnections = psutil.net_connections()
+	out.append(f'monitor_netio_connections{{}} {len(netioConnections)}')
+	netioCounters = psutil.net_io_counters(nowrap=True)
+	out.append(f'monitor_netio_counters{{value="bytes_sent"}} {netioCounters.bytes_sent}')
+	out.append(f'monitor_netio_counters{{value="bytes_recv"}} {netioCounters.bytes_recv}')
+	out.append(f'monitor_netio_counters{{value="packets_sent"}} {netioCounters.packets_sent}')
+	out.append(f'monitor_netio_counters{{value="packets_recv"}} {netioCounters.packets_recv}')
+	out.append(f'monitor_netio_counters{{value="errin"}} {netioCounters.errin}')
+	out.append(f'monitor_netio_counters{{value="errout"}} {netioCounters.errout}')
+	out.append(f'monitor_netio_counters{{value="dropin"}} {netioCounters.dropin}')
+	out.append(f'monitor_netio_counters{{value="dropout"}} {netioCounters.dropout}')
+
+	return res.text("\n".join(out), status=200)
+
+
+if __name__ == '__main__':
+	ip = "127.0.0.1"
+	port = 2611
+	if(len(sys.argv) > 1):
+		ip = sys.argv[1]
+	if(len(sys.argv) > 2):
+		try:
+			port = int(sys.argv[2])
+		except:
+			print("Port must be a numeric value!")
+			exit(1)
+		if(port < 1 or port > 0xffff):
+			print("Port must be in [1, 65535] range!")
+			exit(1)
+	app.run(host=ip, port=port, dev=DEV_MODE, access_log=ACCESS_LOG)

tpl/base.html

@@ -48,28 +48,6 @@
 				grid-template-rows: repeat(2, 1fr);
 			}
 		</style>
-		
-		<!-- Matomo -->
-		<script>
-			var _paq = window._paq = window._paq || [];
-			/* tracker methods like "setCustomDimension" should be called before "trackPageView" */
-			_paq.push(["setCampaignNameKey", "private_area"]);
-			{% if order %}
-				_paq.push(['setUserId', '{{order.code}}']);
-			{% endif %}
-			_paq.push(['trackPageView']);
-			_paq.push(['enableLinkTracking']);
-			(function() {
-				var u="https://y.foxo.me/";
-				_paq.push(['setTrackerUrl', u+'matomo.php']);
-				_paq.push(['setSiteId', '2']);
-				var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
-				g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
-			})();
-		</script>
-		<script src="/res/scripts/base.js" defer></script>
-		<noscript><p><img src="https://y.foxo.me/matomo.php?idsite=2&amp;rec=1" style="border:0;" alt="" /></p></noscript>
-		<!-- End Matomo Code -->
 	</head>
 	<body onscroll="onScrollNav()">
 		{% include 'blocks/navbar.html' %}

tpl/privacy.html

@@ -28,7 +28,6 @@
 			<li>First three octets of your ip address</li>
 			<li>List of status codes (404, 500, 403...) you encountered, and when you encountered them</li>
 		</ul>
-		<p>This info is collected through our first-party domain <em>y.foxo.me</em></p>
 				
 		<h2>Backups and export of data</h2>
 		<p>All data is stored in servers managed by Furizon APS, inside of facilities in the Italian territory. No data is exported outside of the italian territory. All communication uses end to end encryption.</p>

utils.py

@@ -4,7 +4,9 @@ from config import *
 import httpx
 from messages import ROOM_ERROR_TYPES
 from email_util import send_unconfirm_message
+from sanic.response import text, html, redirect, raw
 from sanic.log import logger
+from metrics import *
 
 METADATA_TAG = "meta_data"
 VARIATIONS_TAG = "variations"
@@ -33,6 +35,7 @@ async def load_questions():
 		p = 0
 		while 1:
 			p += 1
+			incPretixRead()
 			res = await client.get(join(base_url_event, f"questions/?page={p}"), headers=headers)
 
 			if res.status_code == 404: break
@@ -50,6 +53,7 @@ async def load_items():
 		p = 0
 		while 1:
 			p += 1
+			incPretixRead()
 			res = await client.get(join(base_url_event, f"items/?page={p}"), headers=headers)
 
 			if res.status_code == 404: break
@@ -253,4 +257,4 @@ async def check_room(request, order, om=None):
 		if order.room_confirmed:
 			allOk = False
 	order.set_room_errors(room_errors)
-	return (order, allOk, room_members)
+	return (order, allOk, room_members)