From 5ea7d06dae3dae5dc3ff7eb9db7cf63d5909795a Mon Sep 17 00:00:00 2001 From: Conatum Date: Tue, 26 Oct 2021 17:33:44 +0300 Subject: [PATCH 01/17] refactor: Update `study` module structure. Prepare `study` module for session and timer systems. Move regular sync to the studybadge loop. --- bot/core/module.py | 7 ++++--- bot/modules/study/__init__.py | 9 ++++----- bot/modules/study/badges/__init__.py | 2 ++ bot/modules/study/{ => badges}/badge_tracker.py | 3 ++- bot/modules/study/{ => badges}/data.py | 2 -- bot/modules/study/{ => badges}/studybadge_cmd.py | 2 +- bot/modules/study/timers/__init__.py | 0 bot/modules/study/tracking/__init__.py | 4 ++++ bot/modules/study/tracking/data.py | 3 +++ bot/modules/study/tracking/session_tracker.py | 0 .../study/{admin.py => tracking/settings.py} | 0 bot/modules/study/{ => tracking}/time_tracker.py | 16 ++++++++-------- 12 files changed, 28 insertions(+), 20 deletions(-) create mode 100644 bot/modules/study/badges/__init__.py rename bot/modules/study/{ => badges}/badge_tracker.py (99%) rename bot/modules/study/{ => badges}/data.py (92%) rename bot/modules/study/{ => badges}/studybadge_cmd.py (99%) create mode 100644 bot/modules/study/timers/__init__.py create mode 100644 bot/modules/study/tracking/__init__.py create mode 100644 bot/modules/study/tracking/data.py create mode 100644 bot/modules/study/tracking/session_tracker.py rename bot/modules/study/{admin.py => tracking/settings.py} (100%) rename bot/modules/study/{ => tracking}/time_tracker.py (85%) diff --git a/bot/core/module.py b/bot/core/module.py index daaa4bc7..0bf870c0 100644 --- a/bot/core/module.py +++ b/bot/core/module.py @@ -68,9 +68,10 @@ async def preload_studying_members(client): ) -@module.launch_task -async def launch_lion_sync_loop(client): - asyncio.create_task(_lion_sync_loop()) +# Removing the sync loop in favour of the studybadge sync. +# @module.launch_task +# async def launch_lion_sync_loop(client): +# asyncio.create_task(_lion_sync_loop()) @module.unload_task diff --git a/bot/modules/study/__init__.py b/bot/modules/study/__init__.py index eec16e1d..30f59149 100644 --- a/bot/modules/study/__init__.py +++ b/bot/modules/study/__init__.py @@ -1,9 +1,8 @@ from .module import module -from . import data -from . import admin -from . import badge_tracker -from . import time_tracker +from . import badges +from . import timers +from . import tracking + from . import top_cmd -from . import studybadge_cmd from . import stats_cmd diff --git a/bot/modules/study/badges/__init__.py b/bot/modules/study/badges/__init__.py new file mode 100644 index 00000000..8db92c34 --- /dev/null +++ b/bot/modules/study/badges/__init__.py @@ -0,0 +1,2 @@ +from . import badge_tracker +from . import studybadge_cmd diff --git a/bot/modules/study/badge_tracker.py b/bot/modules/study/badges/badge_tracker.py similarity index 99% rename from bot/modules/study/badge_tracker.py rename to bot/modules/study/badges/badge_tracker.py index cf69057a..34e97c3a 100644 --- a/bot/modules/study/badge_tracker.py +++ b/bot/modules/study/badges/badge_tracker.py @@ -13,7 +13,7 @@ from core.data import lions from utils.lib import strfdur from settings import GuildSettings -from .module import module +from ..module import module from .data import new_study_badges, study_badges @@ -287,6 +287,7 @@ async def study_badge_tracker(): """ while True: try: + Lion.sync() await update_study_badges() except Exception: # Unknown exception. Catch it so the loop doesn't die. diff --git a/bot/modules/study/data.py b/bot/modules/study/badges/data.py similarity index 92% rename from bot/modules/study/data.py rename to bot/modules/study/badges/data.py index 005765bb..eca5f220 100644 --- a/bot/modules/study/data.py +++ b/bot/modules/study/badges/data.py @@ -2,8 +2,6 @@ from cachetools import cached from data import Table, RowTable -untracked_channels = Table('untracked_channels') - study_badges = RowTable( 'study_badges', ('badgeid', 'guildid', 'roleid', 'required_time'), diff --git a/bot/modules/study/studybadge_cmd.py b/bot/modules/study/badges/studybadge_cmd.py similarity index 99% rename from bot/modules/study/studybadge_cmd.py rename to bot/modules/study/badges/studybadge_cmd.py index 99e28fec..1d9aea9c 100644 --- a/bot/modules/study/studybadge_cmd.py +++ b/bot/modules/study/badges/studybadge_cmd.py @@ -12,7 +12,7 @@ from wards import is_guild_admin from core.data import lions from settings import GuildSettings -from .module import module +from ..module import module from .data import study_badges, guild_role_cache, new_study_badges from .badge_tracker import _update_guild_badges diff --git a/bot/modules/study/timers/__init__.py b/bot/modules/study/timers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bot/modules/study/tracking/__init__.py b/bot/modules/study/tracking/__init__.py new file mode 100644 index 00000000..a80dd7a7 --- /dev/null +++ b/bot/modules/study/tracking/__init__.py @@ -0,0 +1,4 @@ +from . import data +from . import settings +from . import time_tracker +from . import session_tracker diff --git a/bot/modules/study/tracking/data.py b/bot/modules/study/tracking/data.py new file mode 100644 index 00000000..12bd8d77 --- /dev/null +++ b/bot/modules/study/tracking/data.py @@ -0,0 +1,3 @@ +from data import Table + +untracked_channels = Table('untracked_channels') diff --git a/bot/modules/study/tracking/session_tracker.py b/bot/modules/study/tracking/session_tracker.py new file mode 100644 index 00000000..e69de29b diff --git a/bot/modules/study/admin.py b/bot/modules/study/tracking/settings.py similarity index 100% rename from bot/modules/study/admin.py rename to bot/modules/study/tracking/settings.py diff --git a/bot/modules/study/time_tracker.py b/bot/modules/study/tracking/time_tracker.py similarity index 85% rename from bot/modules/study/time_tracker.py rename to bot/modules/study/tracking/time_tracker.py index 26afddd6..1cb35fa0 100644 --- a/bot/modules/study/time_tracker.py +++ b/bot/modules/study/tracking/time_tracker.py @@ -7,8 +7,8 @@ from time import time from meta import client from core import Lion -from .module import module -from . import admin +from ..module import module +from .settings import untracked_channels, hourly_reward, hourly_live_bonus last_scan = {} # guildid -> timestamp @@ -36,9 +36,9 @@ def _scan(guild): if interval > 60 * 20: return - untracked = admin.untracked_channels.get(guild.id).data - hourly_reward = admin.hourly_reward.get(guild.id).data - hourly_live_bonus = admin.hourly_live_bonus.get(guild.id).data + untracked = untracked_channels.get(guild.id).data + guild_hourly_reward = hourly_reward.get(guild.id).data + guild_hourly_live_bonus = hourly_live_bonus.get(guild.id).data channel_members = ( channel.members for channel in guild.voice_channels if channel.id not in untracked @@ -61,9 +61,9 @@ def _scan(guild): lion.addTime(interval, flush=False) # Add coins - hour_reward = hourly_reward + hour_reward = guild_hourly_reward if member.voice.self_stream or member.voice.self_video: - hour_reward += hourly_live_bonus + hour_reward += guild_hourly_live_bonus lion.addCoins(hour_reward * interval / (3600), flush=False) @@ -102,7 +102,7 @@ async def _study_tracker(): @module.launch_task async def launch_study_tracker(client): # First pre-load the untracked channels - await admin.untracked_channels.launch_task(client) + await untracked_channels.launch_task(client) asyncio.create_task(_study_tracker()) From 65fcfe0289f5cbb88c508613eefa6531dc62d531 Mon Sep 17 00:00:00 2001 From: Conatum Date: Sun, 31 Oct 2021 20:15:32 +0200 Subject: [PATCH 02/17] feature (sessions): Core session tracker. Base `Session` system with event trackers. --- bot/modules/study/tracking/__init__.py | 1 - bot/modules/study/tracking/data.py | 31 +++- bot/modules/study/tracking/session_tracker.py | 150 ++++++++++++++++++ bot/modules/study/tracking/settings.py | 11 ++ data/migration/v5-v6/migration.sql | 136 ++++++++++++++++ data/schema.sql | 120 +++++++++++++- 6 files changed, 440 insertions(+), 9 deletions(-) create mode 100644 data/migration/v5-v6/migration.sql diff --git a/bot/modules/study/tracking/__init__.py b/bot/modules/study/tracking/__init__.py index a80dd7a7..ba8de231 100644 --- a/bot/modules/study/tracking/__init__.py +++ b/bot/modules/study/tracking/__init__.py @@ -1,4 +1,3 @@ from . import data from . import settings -from . import time_tracker from . import session_tracker diff --git a/bot/modules/study/tracking/data.py b/bot/modules/study/tracking/data.py index 12bd8d77..deab3816 100644 --- a/bot/modules/study/tracking/data.py +++ b/bot/modules/study/tracking/data.py @@ -1,3 +1,32 @@ -from data import Table +from data import Table, RowTable, tables untracked_channels = Table('untracked_channels') + +session_history = Table('session_history') +current_sessions = RowTable( + 'current_sessions', + ('guildid', 'userid', 'channelid', 'channel_type', + 'start_time', + 'live_duration', 'live_start', + 'stream_duration', 'stream_start', + 'video_duration', 'video_start', + 'hourly_coins', 'hourly_live_coins'), + ('guildid', 'userid'), + cache={} # Keep all current sessions in cache +) + + +@current_sessions.save_query +def close_study_session(guildid, userid): + """ + Close a member's current session if it exists and update the member cache. + """ + # Execute the `close_study_session` database function + with current_sessions.conn as conn: + cursor = conn.cursor() + cursor.callproc('close_study_session', (guildid, userid)) + rows = cursor.fetchall() + # The row has been deleted, remove the from current sessions cache + current_sessions.row_cache.pop((guildid, userid), None) + # Use the function output to update the member cache + tables.lions._make_rows(*rows) diff --git a/bot/modules/study/tracking/session_tracker.py b/bot/modules/study/tracking/session_tracker.py index e69de29b..e44a2bc5 100644 --- a/bot/modules/study/tracking/session_tracker.py +++ b/bot/modules/study/tracking/session_tracker.py @@ -0,0 +1,150 @@ +import asyncio +import discord +from collections import defaultdict + +from utils.lib import utc_now +from ..module import module +from .data import current_sessions +from .settings import untracked_channels, hourly_reward, hourly_live_bonus, max_daily_study + + +class Session: + # TODO: Slots + sessions = defaultdict(dict) + + def __init__(self, guildid, userid): + self.guildid = guildid + self.userid = userid + self.key = (guildid, userid) + + @classmethod + def get(cls, guildid, userid): + """ + Fetch the current session for the provided member. + If there is no current session, returns `None`. + """ + return cls.sessions[guildid].get(userid, None) + + @classmethod + def start(cls, member: discord.Member, state: discord.VoiceState): + """ + Start a new study session for the provided member. + """ + guildid = member.guild.id + userid = member.id + now = utc_now() + + if userid in cls.sessions[guildid]: + raise ValueError("A session for this member already exists!") + # TODO: Handle daily study cap + # TODO: Calculate channel type + # TODO: Ensure lion + current_sessions.create_row( + guildid=guildid, + userid=userid, + channelid=state.channel.id, + channel_type=None, + start_time=now, + live_start=now if (state.self_video or state.self_stream) else None, + stream_start=now if state.self_stream else None, + video_start=now if state.self_video else None, + hourly_coins=hourly_reward.get(guildid).value, + hourly_live_coins=hourly_live_bonus.get(guildid).value + ) + session = cls(guildid, userid) + cls.sessions[guildid][userid] = session + return session + + @property + def data(self): + return current_sessions.fetch(self.key) + + def finish(self): + """ + Close the study session. + """ + self.sessions[self.guildid].pop(self.userid, None) + # Note that save_live_status doesn't need to be called here + # The database saving procedure will account for the values. + current_sessions.queries.close_study_session(*self.key) + + def save_live_status(self, state: discord.VoiceState): + """ + Update the saved live status of the member. + """ + has_video = state.self_video + has_stream = state.self_stream + is_live = has_video or has_stream + + now = utc_now() + data = self.data + + with data.batch_update(): + # Update video session stats + if data.video_start: + data.video_duration += (now - data.video_start).total_seconds() + data.video_start = now if has_video else None + + # Update stream session stats + if data.stream_start: + data.stream_duration += (now - data.stream_start).total_seconds() + data.stream_start = now if has_stream else None + + # Update overall live session stats + if data.live_start: + data.live_duration += (now - data.live_start).total_seconds() + data.live_start = now if is_live else None + + +async def session_voice_tracker(client, member, before, after): + """ + Voice update event dispatcher for study session tracking. + """ + guild = member.guild + session = Session.get(guild.id, member.id) + + if before.channel == after.channel: + # Voice state change without moving channel + if session and ((before.self_video != after.self_video) or (before.self_stream != after.self_stream)): + # Live status has changed! + session.save_live_status(after) + else: + # Member changed channel + # End the current session and start a new one, if applicable + # TODO: Max daily study session tasks + if session: + # End the current session + session.finish() + if after.channel: + blacklist = client.objects['blacklisted_users'] + guild_blacklist = client.objects['ignored_members'][guild.id] + untracked = untracked_channels.get(guild.id).data + start_session = ( + (after.channel.id not in untracked) + and (member.id not in blacklist) + and (member.id not in guild_blacklist) + ) + if start_session: + # Start a new session for the member + Session.start(member, after) + + +async def _init_session_tracker(client): + """ + Load ongoing saved study sessions into the session cache, + update them depending on the current voice states, + and attach the voice event handler. + """ + await client.wait_until_ready() + await untracked_channels.launch_task(client) + client.add_after_event("voice_state_update", session_voice_tracker) + + +@module.launch_task +async def launch_session_tracker(client): + """ + Launch the study session initialiser. + Doesn't block on the client being ready. + """ + client.objects['sessions'] = Session.sessions + asyncio.create_task(_init_session_tracker(client)) diff --git a/bot/modules/study/tracking/settings.py b/bot/modules/study/tracking/settings.py index 5861ab95..dc93cc00 100644 --- a/bot/modules/study/tracking/settings.py +++ b/bot/modules/study/tracking/settings.py @@ -111,3 +111,14 @@ class hourly_live_bonus(settings.Integer, settings.GuildSetting): @property def success_response(self): return "Members will be rewarded an extra `{}` LionCoins per hour if they stream.".format(self.formatted) + + +@GuildSettings.attach_setting +class max_daily_study(settings.Duration, settings.GuildSetting): + category = "Study Tracking" + + attr_name = "max_daily_study" + _data_column = "max_daily_study" + + display_name = "max_daily_study" + desc = "Maximum amount of study time ..." diff --git a/data/migration/v5-v6/migration.sql b/data/migration/v5-v6/migration.sql new file mode 100644 index 00000000..70ec3843 --- /dev/null +++ b/data/migration/v5-v6/migration.sql @@ -0,0 +1,136 @@ +/* DROP TYPE IF EXISTS SessionChannelType CASCADE; */ +/* DROP TABLE IF EXISTS session_history CASCADE; */ +/* DROP TABLE IF EXISTS current_sessions CASCADE; */ +/* DROP FUNCTION IF EXISTS close_study_session; */ + +/* DROP VIEW IF EXISTS current_sessions_totals CASCADE; */ +/* DROP VIEW IF EXISTS member_totals CASCADE; */ +/* DROP VIEW IF EXISTS member_ranks CASCADE; */ +/* DROP VIEW IF EXISTS current_study_badges CASCADE; */ +/* DROP VIEW IF EXISTS new_study_badges CASCADE; */ + +CREATE TYPE SessionChannelType AS ENUM ( + 'ACCOUNTABILITY', + 'RENTED', + 'EXTERNAL', + 'MIGRATED' +); + +CREATE TABLE session_history( + sessionid SERIAL PRIMARY KEY, + guildid BIGINT NOT NULL, + userid BIGINT NOT NULL, + channelid BIGINT, + channel_type SessionChannelType, + start_time TIMESTAMPTZ NOT NULL, + duration INTEGER NOT NULL, + coins_earned INTEGER NOT NULL, + live_duration INTEGER DEFAULT 0, + stream_duration INTEGER DEFAULT 0, + video_duration INTEGER DEFAULT 0, + FOREIGN KEY (guildid, userid) REFERENCES members (guildid, userid) ON DELETE CASCADE +); +CREATE INDEX session_history_members ON session_history (guildid, userid, start_time); + +CREATE TABLE current_sessions( + guildid BIGINT NOT NULL, + userid BIGINT NOT NULL, + channelid BIGINT, + channel_type SessionChannelType, + start_time TIMESTAMPTZ DEFAULT now(), + live_duration INTEGER DEFAULT 0, + live_start TIMESTAMPTZ, + stream_duration INTEGER DEFAULT 0, + stream_start TIMESTAMPTZ, + video_duration INTEGER DEFAULT 0, + video_start TIMESTAMPTZ, + hourly_coins INTEGER NOT NULL, + hourly_live_coins INTEGER NOT NULL, + FOREIGN KEY (guildid, userid) REFERENCES members (guildid, userid) ON DELETE CASCADE +); +CREATE UNIQUE INDEX current_session_members ON current_sessions (guildid, userid); + + +CREATE FUNCTION close_study_session(_guildid BIGINT, _userid BIGINT) + RETURNS SETOF members +AS $$ + BEGIN + RETURN QUERY + WITH + current_sesh AS ( + DELETE FROM current_sessions + WHERE guildid=_guildid AND userid=_userid + RETURNING + *, + EXTRACT(EPOCH FROM (NOW() - start_time)) AS total_duration, + stream_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - stream_start)), 0) AS total_stream_duration, + video_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - video_start)), 0) AS total_video_duration, + live_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - live_start)), 0) AS total_live_duration + ), saved_sesh AS ( + INSERT INTO session_history ( + guildid, userid, channelid, channel_type, start_time, + duration, stream_duration, video_duration, live_duration, + coins_earned + ) SELECT + guildid, userid, channelid, channel_type, start_time, + total_duration, total_stream_duration, total_video_duration, total_live_duration, + (total_duration * hourly_coins + live_duration * hourly_live_coins) / 60 + FROM current_sesh + RETURNING * + ) + UPDATE members + SET + tracked_time=(tracked_time + saved_sesh.duration), + coins=(coins + saved_sesh.coins_earned) + FROM saved_sesh + WHERE members.guildid=saved_sesh.guildid AND members.userid=saved_sesh.userid + RETURNING members.*; + END; +$$ LANGUAGE PLPGSQL; + + + +CREATE VIEW current_sessions_totals AS + SELECT + *, + EXTRACT(EPOCH FROM (NOW() - start_time)) AS total_duration, + stream_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - stream_start)), 0) AS total_stream_duration, + video_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - video_start)), 0) AS total_video_duration, + live_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - live_start)), 0) AS total_live_duration + FROM current_sessions; + + +CREATE VIEW members_totals AS + SELECT + *, + sesh.start_time AS session_start, + tracked_time + COALESCE(sesh.total_duration, 0) AS total_tracked_time, + coins + COALESCE((sesh.total_duration * sesh.hourly_coins + sesh.live_duration * sesh.hourly_live_coins) / 60, 0) AS total_coins + FROM members + LEFT JOIN current_sessions_totals sesh USING (guildid, userid); + + +CREATE VIEW member_ranks AS + SELECT + *, + row_number() OVER (PARTITION BY guildid ORDER BY total_tracked_time DESC, userid ASC) AS time_rank, + row_number() OVER (PARTITION BY guildid ORDER BY total_coins DESC, userid ASC) AS coin_rank + FROM members_totals; + +CREATE VIEW current_study_badges AS + SELECT + *, + (SELECT r.badgeid + FROM study_badges r + WHERE r.guildid = members_totals.guildid AND members_totals.tracked_time > r.required_time + ORDER BY r.required_time DESC + LIMIT 1) AS current_study_badgeid + FROM members_totals; + +CREATE VIEW new_study_badges AS + SELECT + current_study_badges.* + FROM current_study_badges + WHERE + last_study_badgeid IS DISTINCT FROM current_study_badgeid + ORDER BY guildid; diff --git a/data/schema.sql b/data/schema.sql index fe0158be..aba91dee 100644 --- a/data/schema.sql +++ b/data/schema.sql @@ -77,7 +77,8 @@ CREATE TABLE guild_config( greeting_message TEXT, returning_message TEXT, starting_funds INTEGER, - persist_roles BOOLEAN + persist_roles BOOLEAN, + max_daily_study INTEGER ); CREATE TABLE ignored_members( @@ -407,24 +408,127 @@ CREATE INDEX member_timestamps ON members (_timestamp); CREATE TRIGGER update_members_timstamp BEFORE UPDATE ON members FOR EACH ROW EXECUTE PROCEDURE update_timestamp_column(); +-- }}} + +-- Study Session Data {{{ +CREATE TYPE SessionChannelType AS ENUM ( + 'ACCOUNTABILITY', + 'RENTED', + 'EXTERNAL', + 'MIGRATED' +); + +CREATE TABLE session_history( + sessionid SERIAL PRIMARY KEY, + guildid BIGINT NOT NULL, + userid BIGINT NOT NULL, + channelid BIGINT, + channel_type SessionChannelType, + start_time TIMESTAMPTZ NOT NULL, + duration INTEGER NOT NULL, + coins_earned INTEGER NOT NULL, + live_duration INTEGER DEFAULT 0, + stream_duration INTEGER DEFAULT 0, + video_duration INTEGER DEFAULT 0, + FOREIGN KEY (guildid, userid) REFERENCES members (guildid, userid) ON DELETE CASCADE +); +CREATE INDEX session_history_members ON session_history (guildid, userid, start_time); + +CREATE TABLE current_sessions( + guildid BIGINT NOT NULL, + userid BIGINT NOT NULL, + channelid BIGINT, + channel_type SessionChannelType, + start_time TIMESTAMPTZ DEFAULT now(), + live_duration INTEGER DEFAULT 0, + live_start TIMESTAMPTZ, + stream_duration INTEGER DEFAULT 0, + stream_start TIMESTAMPTZ, + video_duration INTEGER DEFAULT 0, + video_start TIMESTAMPTZ, + hourly_coins INTEGER NOT NULL, + hourly_live_coins INTEGER NOT NULL, + FOREIGN KEY (guildid, userid) REFERENCES members (guildid, userid) ON DELETE CASCADE +); +CREATE UNIQUE INDEX current_session_members ON current_sessions (guildid, userid); + + +CREATE FUNCTION close_study_session(_guildid BIGINT, _userid BIGINT) + RETURNS SETOF members +AS $$ + BEGIN + RETURN QUERY + WITH + current_sesh AS ( + DELETE FROM current_sessions + WHERE guildid=_guildid AND userid=_userid + RETURNING + *, + EXTRACT(EPOCH FROM (NOW() - start_time)) AS total_duration, + stream_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - stream_start)), 0) AS total_stream_duration, + video_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - video_start)), 0) AS total_video_duration, + live_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - live_start)), 0) AS total_live_duration + ), saved_sesh AS ( + INSERT INTO session_history ( + guildid, userid, channelid, channel_type, start_time, + duration, stream_duration, video_duration, live_duration, + coins_earned + ) SELECT + guildid, userid, channelid, channel_type, start_time, + total_duration, total_stream_duration, total_video_duration, total_live_duration, + (total_duration * hourly_coins + live_duration * hourly_live_coins) / 60 + FROM current_sesh + RETURNING * + ) + UPDATE members + SET + tracked_time=(tracked_time + saved_sesh.duration), + coins=(coins + saved_sesh.coins_earned) + FROM saved_sesh + WHERE members.guildid=saved_sesh.guildid AND members.userid=saved_sesh.userid + RETURNING members.*; + END; +$$ LANGUAGE PLPGSQL; + + +CREATE VIEW current_sessions_totals AS + SELECT + *, + EXTRACT(EPOCH FROM (NOW() - start_time)) AS total_duration, + stream_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - stream_start)), 0) AS total_stream_duration, + video_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - video_start)), 0) AS total_video_duration, + live_duration + COALESCE(EXTRACT(EPOCH FROM (NOW() - live_start)), 0) AS total_live_duration + FROM current_sessions; + + +CREATE VIEW members_totals AS + SELECT + *, + sesh.start_time AS session_start, + tracked_time + COALESCE(sesh.total_duration, 0) AS total_tracked_time, + coins + COALESCE((sesh.total_duration * sesh.hourly_coins + sesh.live_duration * sesh.hourly_live_coins) / 60, 0) AS total_coins + FROM members + LEFT JOIN current_sessions_totals sesh USING (guildid, userid); + CREATE VIEW member_ranks AS SELECT *, - row_number() OVER (PARTITION BY guildid ORDER BY tracked_time DESC, userid ASC) AS time_rank, - row_number() OVER (PARTITION BY guildid ORDER BY coins DESC, userid ASC) AS coin_rank - FROM members; - + row_number() OVER (PARTITION BY guildid ORDER BY total_tracked_time DESC, userid ASC) AS time_rank, + row_number() OVER (PARTITION BY guildid ORDER BY total_coins DESC, userid ASC) AS coin_rank + FROM members_totals; +-- }}} +-- Study Badge Data {{{ CREATE VIEW current_study_badges AS SELECT *, (SELECT r.badgeid FROM study_badges r - WHERE r.guildid = members.guildid AND members.tracked_time > r.required_time + WHERE r.guildid = members_totals.guildid AND members_totals.tracked_time > r.required_time ORDER BY r.required_time DESC LIMIT 1) AS current_study_badgeid - FROM members; + FROM members_totals; CREATE VIEW new_study_badges AS SELECT @@ -527,6 +631,7 @@ CREATE TABLE reaction_role_expiring( reactionid INTEGER REFERENCES reaction_role_reactions (reactionid) ON DELETE SET NULL ); CREATE UNIQUE INDEX reaction_role_expiry_members ON reaction_role_expiring (guildid, userid, roleid); +-- }}} -- Member Role Data {{{ CREATE TABLE past_member_roles( @@ -538,4 +643,5 @@ CREATE TABLE past_member_roles( ); CREATE INDEX member_role_persistence_members ON past_member_roles (guildid, userid); -- }}} + -- vim: set fdm=marker: From 9c8dfd6a3af036372ab071adae5a30a5fa5507ac Mon Sep 17 00:00:00 2001 From: Conatum Date: Mon, 29 Nov 2021 13:31:37 +0200 Subject: [PATCH 03/17] (sessions): Complete launch and init pathway. --- bot/modules/study/tracking/session_tracker.py | 126 +++++++++++++++++- 1 file changed, 122 insertions(+), 4 deletions(-) diff --git a/bot/modules/study/tracking/session_tracker.py b/bot/modules/study/tracking/session_tracker.py index e44a2bc5..fb9b9341 100644 --- a/bot/modules/study/tracking/session_tracker.py +++ b/bot/modules/study/tracking/session_tracker.py @@ -1,14 +1,23 @@ import asyncio import discord +import logging +import traceback from collections import defaultdict from utils.lib import utc_now +from data import tables +from core import Lion + from ..module import module -from .data import current_sessions +from .data import current_sessions, SessionChannelType from .settings import untracked_channels, hourly_reward, hourly_live_bonus, max_daily_study class Session: + """ + A `Session` is a guild member that is currently studying (i.e. that is in a tracked voice channel). + This class acts as an opaque interface to the corresponding `sessions` data row. + """ # TODO: Slots sessions = defaultdict(dict) @@ -37,13 +46,20 @@ class Session: if userid in cls.sessions[guildid]: raise ValueError("A session for this member already exists!") # TODO: Handle daily study cap - # TODO: Calculate channel type - # TODO: Ensure lion + + # TODO: More reliable channel type determination + if state.channel.id in tables.rented.row_cache: + channel_type = SessionChannelType.RENTED + elif state.channel.id in tables.accountability_rooms.row_cache: + channel_type = SessionChannelType.ACCOUNTABILITY + else: + channel_type = SessionChannelType.STANDARD + current_sessions.create_row( guildid=guildid, userid=userid, channelid=state.channel.id, - channel_type=None, + channel_type=channel_type, start_time=now, live_start=now if (state.self_video or state.self_stream) else None, stream_start=now if state.self_stream else None, @@ -101,6 +117,7 @@ async def session_voice_tracker(client, member, before, after): Voice update event dispatcher for study session tracking. """ guild = member.guild + Lion.fetch(guild.id, member.id) session = Session.get(guild.id, member.id) if before.channel == after.channel: @@ -112,6 +129,7 @@ async def session_voice_tracker(client, member, before, after): # Member changed channel # End the current session and start a new one, if applicable # TODO: Max daily study session tasks + # TODO: Error if before is None but we have a current session if session: # End the current session session.finish() @@ -135,8 +153,108 @@ async def _init_session_tracker(client): update them depending on the current voice states, and attach the voice event handler. """ + # Ensure the client caches are ready and guilds are chunked await client.wait_until_ready() + + # Pre-cache the untracked channels await untracked_channels.launch_task(client) + + # Log init start and define logging counters + client.log( + "Loading ongoing study sessions.", + context="SESSION_INIT", + level=logging.DEBUG + ) + resumed = 0 + ended = 0 + + # Grab all ongoing sessions from data + rows = current_sessions.fetch_rows_where() + + # Iterate through, resume or end as needed + for row in rows: + if (guild := client.get_guild(row.guildid)) is not None and row.channelid is not None: + try: + # Load the Session + session = Session(row.guildid, row.userid) + + # Find the channel and member voice state + voice = None + if channel := guild.get_channel(row.channelid): + voice = next((member.voice for member in channel.members if member.id == row.userid), None) + + # Resume or end as required + if voice and voice.channel: + client.log( + "Resuming ongoing session: {}".format(row), + context="SESSION_INIT", + level=logging.DEBUG + ) + Session.sessions[row.guildid][row.userid] = session + session.save_live_status(voice) + resumed += 1 + else: + client.log( + "Ending already completed session: {}".format(row), + context="SESSION_INIT", + level=logging.DEBUG + ) + session.finish() + ended += 1 + except Exception: + # Fatal error + client.log( + "Fatal error occurred initialising session: {}\n{}".format(row, traceback.format_exc()), + context="SESSION_INIT", + level=logging.CRITICAL + ) + module.ready = False + return + + # Log resumed sessions + client.log( + "Resumed {} ongoing study sessions, and ended {}.".format(resumed, ended), + context="SESSION_INIT", + level=logging.INFO + ) + + # Now iterate through members of all tracked voice channels + # Start sessions if they don't already exist + tracked_channels = [ + channel + for guild in client.guilds + for channel in guild.voice_channels + if channel.members and channel.id not in untracked_channels.get(guild.id).data + ] + new_members = [ + member + for channel in tracked_channels + for member in channel.members + if not Session.get(member.guild.id, member.id) + ] + for member in new_members: + client.log( + "Starting new session for '{}' (uid: {}) in '{}' (cid: {}) of '{}' (gid: {})".format( + member.name, + member.id, + member.voice.channel.name, + member.voice.channel.id, + member.guild.name, + member.guild.id + ), + context="SESSION_INIT", + level=logging.DEBUG + ) + Session.start(member, member.voice) + + # Log newly started sessions + client.log( + "Started {} new study sessions from current voice channel members.".format(len(new_members)), + context="SESSION_INIT", + level=logging.INFO + ) + + # Now that we are in a valid initial state, attach the session event handler client.add_after_event("voice_state_update", session_voice_tracker) From ac71c4da9bd50ea6fe8eced4a28e62c16fe9a8ec Mon Sep 17 00:00:00 2001 From: Conatum Date: Mon, 29 Nov 2021 13:33:21 +0200 Subject: [PATCH 04/17] (sessions): Add `study_time_since` function. New `study_time_since` database function from session history. Add `Lion.timezone`. Add `Lion.day_start`. Add `Lion.studied_today`. Made `Table.queries` an instance variable. Renamed the session channel types. --- bot/core/data.py | 62 +++++++++++++++--------------- bot/core/lion.py | 35 +++++++++++++++++ bot/data/interfaces.py | 2 +- bot/modules/study/tracking/data.py | 27 +++++++++++++ data/migration/v5-v6/migration.sql | 62 ++++++++++++++++++++++++------ data/schema.sql | 40 ++++++++++++++++++- 6 files changed, 184 insertions(+), 44 deletions(-) diff --git a/bot/core/data.py b/bot/core/data.py index c33914d7..8ae74874 100644 --- a/bot/core/data.py +++ b/bot/core/data.py @@ -20,37 +20,6 @@ user_config = RowTable( ) -@user_config.save_query -def add_pending(pending): - """ - pending: - List of tuples of the form `(userid, pending_coins, pending_time)`. - """ - with lions.conn: - cursor = lions.conn.cursor() - data = execute_values( - cursor, - """ - UPDATE members - SET - coins = coins + t.coin_diff, - tracked_time = tracked_time + t.time_diff - FROM - (VALUES %s) - AS - t (guildid, userid, coin_diff, time_diff) - WHERE - members.guildid = t.guildid - AND - members.userid = t.userid - RETURNING * - """, - pending, - fetch=True - ) - return lions._make_rows(*data) - - guild_config = RowTable( 'guild_config', ('guildid', 'admin_role', 'mod_role', 'event_log_channel', 'alert_channel', @@ -84,6 +53,37 @@ lions = RowTable( lion_ranks = Table('member_ranks', attach_as='lion_ranks') +@lions.save_query +def add_pending(pending): + """ + pending: + List of tuples of the form `(userid, pending_coins, pending_time)`. + """ + with lions.conn: + cursor = lions.conn.cursor() + data = execute_values( + cursor, + """ + UPDATE members + SET + coins = coins + t.coin_diff, + tracked_time = tracked_time + t.time_diff + FROM + (VALUES %s) + AS + t (guildid, userid, coin_diff, time_diff) + WHERE + members.guildid = t.guildid + AND + members.userid = t.userid + RETURNING * + """, + pending, + fetch=True + ) + return lions._make_rows(*data) + + global_guild_blacklist = Table('global_guild_blacklist') global_user_blacklist = Table('global_user_blacklist') ignored_members = Table('ignored_members') diff --git a/bot/core/lion.py b/bot/core/lion.py index b9b10092..d9eb7ea1 100644 --- a/bot/core/lion.py +++ b/bot/core/lion.py @@ -1,4 +1,5 @@ import pytz +import datetime from meta import client from data import tables as tb @@ -41,6 +42,7 @@ class Lion: if key in cls._lions: return cls._lions[key] else: + # TODO: Debug log lion = tb.lions.fetch(key) if not lion: tb.lions.create_row( @@ -95,6 +97,39 @@ class Lion: """ return int(self.data.coins + self._pending_coins) + @property + def session(self): + """ + The current study session the user is in, if any. + """ + if 'sessions' not in client.objects: + raise ValueError("Cannot retrieve session before Study module is initialised!") + return client.objects['sessions'][self.guildid].get(self.userid, None) + + @property + def timezone(self): + """ + The user's configured timezone. + Shortcut to `Lion.settings.timezone.value`. + """ + return self.settings.timezone.value + + @property + def day_start(self): + """ + A timezone aware datetime representing the start of the user's day (in their configured timezone). + """ + now = datetime.datetime.now(tz=self.timezone) + return now.replace(hour=0, minute=0, second=0, microsecond=0) + + @property + def studied_today(self): + """ + The amount of time, in seconds, that the member has studied today. + Extracted from the session history. + """ + return tb.session_history.queries.study_time_since(self.guildid, self.userid, self.day_start) + def localize(self, naive_utc_dt): """ Localise the provided naive UTC datetime into the user's timezone. diff --git a/bot/data/interfaces.py b/bot/data/interfaces.py index 42810e72..7673b0e0 100644 --- a/bot/data/interfaces.py +++ b/bot/data/interfaces.py @@ -45,10 +45,10 @@ class Table: Intended to be subclassed to provide more derivative access for specific tables. """ conn = conn - queries = DotDict() def __init__(self, name, attach_as=None): self.name = name + self.queries = DotDict() tables[attach_as or name] = self @_connection_guard diff --git a/bot/modules/study/tracking/data.py b/bot/modules/study/tracking/data.py index deab3816..5940bf63 100644 --- a/bot/modules/study/tracking/data.py +++ b/bot/modules/study/tracking/data.py @@ -1,7 +1,21 @@ from data import Table, RowTable, tables +from utils.lib import FieldEnum + untracked_channels = Table('untracked_channels') + +class SessionChannelType(FieldEnum): + """ + The possible session channel types. + """ + # NOTE: "None" stands for Unknown, and the STANDARD description should be replaced with the channel name + STANDARD = 'STANDARD', "Standard" + ACCOUNTABILITY = 'ACCOUNTABILITY', "Accountability Room" + RENTED = 'RENTED', "Private Room" + EXTERNAL = 'EXTERNAL', "Unknown" + + session_history = Table('session_history') current_sessions = RowTable( 'current_sessions', @@ -30,3 +44,16 @@ def close_study_session(guildid, userid): current_sessions.row_cache.pop((guildid, userid), None) # Use the function output to update the member cache tables.lions._make_rows(*rows) + + +@session_history.save_query +def study_time_since(guildid, userid, timestamp): + """ + Retrieve the total member study time (in seconds) since the given timestamp. + Includes the current session, if it exists. + """ + with session_history.conn as conn: + cursor = conn.cursor() + cursor.callproc('study_time_since', (guildid, userid, timestamp)) + rows = cursor.fetchall() + return (rows[0][0] if rows else None) or 0 diff --git a/data/migration/v5-v6/migration.sql b/data/migration/v5-v6/migration.sql index 70ec3843..9e404e13 100644 --- a/data/migration/v5-v6/migration.sql +++ b/data/migration/v5-v6/migration.sql @@ -1,19 +1,22 @@ -/* DROP TYPE IF EXISTS SessionChannelType CASCADE; */ -/* DROP TABLE IF EXISTS session_history CASCADE; */ -/* DROP TABLE IF EXISTS current_sessions CASCADE; */ -/* DROP FUNCTION IF EXISTS close_study_session; */ +DROP TYPE IF EXISTS SessionChannelType CASCADE; +DROP TABLE IF EXISTS session_history CASCADE; +DROP TABLE IF EXISTS current_sessions CASCADE; +DROP FUNCTION IF EXISTS close_study_session; + +DROP VIEW IF EXISTS current_sessions_totals CASCADE; +DROP VIEW IF EXISTS member_totals CASCADE; +DROP VIEW IF EXISTS member_ranks CASCADE; +DROP VIEW IF EXISTS current_study_badges CASCADE; +DROP VIEW IF EXISTS new_study_badges CASCADE; + +DROP FUNCTION IF EXISTS study_time_since; -/* DROP VIEW IF EXISTS current_sessions_totals CASCADE; */ -/* DROP VIEW IF EXISTS member_totals CASCADE; */ -/* DROP VIEW IF EXISTS member_ranks CASCADE; */ -/* DROP VIEW IF EXISTS current_study_badges CASCADE; */ -/* DROP VIEW IF EXISTS new_study_badges CASCADE; */ CREATE TYPE SessionChannelType AS ENUM ( + 'STANDARD', 'ACCOUNTABILITY', 'RENTED', - 'EXTERNAL', - 'MIGRATED' + 'EXTERNAL' ); CREATE TABLE session_history( @@ -134,3 +137,40 @@ CREATE VIEW new_study_badges AS WHERE last_study_badgeid IS DISTINCT FROM current_study_badgeid ORDER BY guildid; + + +CREATE FUNCTION study_time_since(_guildid BIGINT, _userid BIGINT, _timestamp TIMESTAMPTZ) + RETURNS INTEGER +AS $$ + BEGIN + RETURN ( + SELECT + SUM( + CASE + WHEN start_time >= _timestamp THEN duration + ELSE EXTRACT(EPOCH FROM (end_time - _timestamp)) + END + ) + FROM ( + SELECT + start_time, + duration, + (start_time + duration * interval '1 second') AS end_time + FROM session_history + WHERE + guildid=_guildid + AND userid=_userid + AND (start_time + duration * interval '1 second') >= _timestamp + UNION + SELECT + start_time, + EXTRACT(EPOCH FROM (NOW() - start_time)) AS duration, + NOW() AS end_time + FROM current_sessions + WHERE + guildid=_guildid + AND userid=_userid + ) AS sessions + ); + END; +$$ LANGUAGE PLPGSQL; diff --git a/data/schema.sql b/data/schema.sql index aba91dee..51ba3ecd 100644 --- a/data/schema.sql +++ b/data/schema.sql @@ -412,12 +412,13 @@ update_timestamp_column(); -- Study Session Data {{{ CREATE TYPE SessionChannelType AS ENUM ( + 'STANDARD', 'ACCOUNTABILITY', 'RENTED', 'EXTERNAL', - 'MIGRATED' ); + CREATE TABLE session_history( sessionid SERIAL PRIMARY KEY, guildid BIGINT NOT NULL, @@ -453,6 +454,43 @@ CREATE TABLE current_sessions( CREATE UNIQUE INDEX current_session_members ON current_sessions (guildid, userid); +CREATE FUNCTION study_time_since(_guildid BIGINT, _userid BIGINT, _timestamp TIMESTAMPTZ) + RETURNS INTEGER +AS $$ + BEGIN + RETURN ( + SELECT + SUM( + CASE + WHEN start_time >= _timestamp THEN duration + ELSE EXTRACT(EPOCH FROM (end_time - _timestamp)) + END + ) + FROM ( + SELECT + start_time, + duration, + (start_time + duration * interval '1 second') AS end_time + FROM session_history + WHERE + guildid=_guildid + AND userid=_userid + AND (start_time + duration * interval '1 second') >= _timestamp + UNION + SELECT + start_time, + EXTRACT(EPOCH FROM (NOW() - start_time)) AS duration, + NOW() AS end_time + FROM current_sessions + WHERE + guildid=_guildid + AND userid=_userid + ) AS sessions + ); + END; +$$ LANGUAGE PLPGSQL; + + CREATE FUNCTION close_study_session(_guildid BIGINT, _userid BIGINT) RETURNS SETOF members AS $$ From 734436e2a61e55e2fb00bf8452d67f9e35d9b17f Mon Sep 17 00:00:00 2001 From: Conatum Date: Thu, 2 Dec 2021 11:48:04 +0200 Subject: [PATCH 05/17] (sessions): Add `daily_study_cap` system. --- bot/core/data.py | 2 +- bot/core/lion.py | 36 +++- bot/modules/study/tracking/session_tracker.py | 177 ++++++++++++++++-- bot/modules/study/tracking/settings.py | 14 +- data/migration/v5-v6/migration.sql | 2 + data/schema.sql | 2 +- 6 files changed, 207 insertions(+), 26 deletions(-) diff --git a/bot/core/data.py b/bot/core/data.py index 8ae74874..cd3f850b 100644 --- a/bot/core/data.py +++ b/bot/core/data.py @@ -25,7 +25,7 @@ guild_config = RowTable( ('guildid', 'admin_role', 'mod_role', 'event_log_channel', 'alert_channel', 'min_workout_length', 'workout_reward', 'max_tasks', 'task_reward', 'task_reward_limit', - 'study_hourly_reward', 'study_hourly_live_bonus', + 'study_hourly_reward', 'study_hourly_live_bonus', 'daily_study_cap', 'study_ban_role', 'max_study_bans'), 'guildid', cache=TTLCache(1000, ttl=60*5) diff --git a/bot/core/lion.py b/bot/core/lion.py index d9eb7ea1..15bed9e0 100644 --- a/bot/core/lion.py +++ b/bot/core/lion.py @@ -1,5 +1,5 @@ import pytz -import datetime +from datetime import datetime, timedelta from meta import client from data import tables as tb @@ -79,10 +79,17 @@ class Lion: @property def settings(self): """ - The UserSettings object for this user. + The UserSettings interface for this member. """ return UserSettings(self.userid) + @property + def guild_settings(self): + """ + The GuildSettings interface for this member. + """ + return GuildSettings(self.guildid) + @property def time(self): """ @@ -118,10 +125,15 @@ class Lion: def day_start(self): """ A timezone aware datetime representing the start of the user's day (in their configured timezone). + NOTE: This might not be accurate over DST boundaries. """ - now = datetime.datetime.now(tz=self.timezone) + now = datetime.now(tz=self.timezone) return now.replace(hour=0, minute=0, second=0, microsecond=0) + @property + def remaining_in_day(self): + return ((self.day_start + timedelta(days=1)) - datetime.now(self.timezone)).total_seconds() + @property def studied_today(self): """ @@ -130,6 +142,24 @@ class Lion: """ return tb.session_history.queries.study_time_since(self.guildid, self.userid, self.day_start) + @property + def remaining_study_today(self): + """ + Maximum remaining time (in seconds) this member can study today. + + May not account for DST boundaries and leap seconds. + """ + studied_today = self.studied_today + study_cap = self.guild_settings.daily_study_cap.value + + remaining_in_day = self.remaining_in_day + if remaining_in_day >= (study_cap - studied_today): + remaining = study_cap - studied_today + else: + remaining = remaining_in_day + study_cap + + return remaining + def localize(self, naive_utc_dt): """ Localise the provided naive UTC datetime into the user's timezone. diff --git a/bot/modules/study/tracking/session_tracker.py b/bot/modules/study/tracking/session_tracker.py index fb9b9341..146f1195 100644 --- a/bot/modules/study/tracking/session_tracker.py +++ b/bot/modules/study/tracking/session_tracker.py @@ -2,29 +2,42 @@ import asyncio import discord import logging import traceback +from typing import Dict from collections import defaultdict from utils.lib import utc_now from data import tables from core import Lion +from meta import client from ..module import module from .data import current_sessions, SessionChannelType -from .settings import untracked_channels, hourly_reward, hourly_live_bonus, max_daily_study +from .settings import untracked_channels, hourly_reward, hourly_live_bonus class Session: """ - A `Session` is a guild member that is currently studying (i.e. that is in a tracked voice channel). + A `Session` describes an ongoing study session by a single guild member. + A member is counted as studying when they are in a tracked voice channel. + This class acts as an opaque interface to the corresponding `sessions` data row. """ - # TODO: Slots - sessions = defaultdict(dict) + __slots__ = ( + 'guildid', + 'userid', + '_expiry_task' + ) + # Global cache of ongoing sessions + sessions: Dict[int, Dict[int, 'Session']] = defaultdict(dict) + + # Global cache of members pending session start (waiting for daily cap reset) + members_pending: Dict[int, Dict[int, asyncio.Task]] = defaultdict(dict) def __init__(self, guildid, userid): self.guildid = guildid self.userid = userid - self.key = (guildid, userid) + + self._expiry_task: asyncio.Task = None @classmethod def get(cls, guildid, userid): @@ -45,7 +58,22 @@ class Session: if userid in cls.sessions[guildid]: raise ValueError("A session for this member already exists!") - # TODO: Handle daily study cap + + # If the user is study capped, schedule the session start for the next day + if (lion := Lion.fetch(guildid, userid)).remaining_study_today <= 0: + if pending := cls.members_pending[guildid].pop(userid, None): + pending.cancel() + task = asyncio.create_task(cls._delayed_start(guildid, userid, member, state)) + cls.members_pending[guildid][userid] = task + client.log( + "Member (uid:{}) in (gid:{}) is study capped, " + "delaying session start for {} seconds until start of next day.".format( + userid, guildid, lion.remaining_in_day + ), + context="SESSION_TRACKER", + level=logging.DEBUG + ) + return # TODO: More reliable channel type determination if state.channel.id in tables.rented.row_cache: @@ -67,23 +95,104 @@ class Session: hourly_coins=hourly_reward.get(guildid).value, hourly_live_coins=hourly_live_bonus.get(guildid).value ) - session = cls(guildid, userid) - cls.sessions[guildid][userid] = session - return session + session = cls(guildid, userid).activate() + client.log( + "Started session: {}".format(session.data), + context="SESSION_TRACKER", + level=logging.DEBUG, + ) + + @classmethod + async def _delayed_start(cls, guildid, userid, *args): + delay = Lion.fetch(guildid, userid).remaining_in_day + try: + await asyncio.sleep(delay) + except asyncio.CancelledError: + pass + else: + cls.start(*args) + + @property + def key(self): + """ + RowTable Session identification key. + """ + return (self.guildid, self.userid) + + @property + def lion(self): + """ + The Lion member object associated with this member. + """ + return Lion.fetch(self.guildid, self.userid) @property def data(self): return current_sessions.fetch(self.key) + def activate(self): + """ + Activate the study session. + This adds the session to the studying members cache, + and schedules the session expiry, based on the daily study cap. + """ + # Add to the active cache + self.sessions[self.guildid][self.userid] = self + + # Schedule the session expiry + self.schedule_expiry() + + # Return self for easy chaining + return self + + def schedule_expiry(self): + """ + Schedule session termination when the user reaches the maximum daily study time. + """ + asyncio.create_task(self._schedule_expiry()) + + async def _schedule_expiry(self): + # Cancel any existing expiry + if self._expiry_task and not self._expiry_task.done(): + self._expiry_task.cancel() + + # Wait for the maximum session length + try: + self._expiry_task = await asyncio.sleep(self.lion.remaining_study_today) + except asyncio.CancelledError: + pass + else: + if self.lion.remaining_study_today <= 0: + # End the session + # Note that the user will not automatically start a new session when the day starts + # TODO: Notify user? Disconnect them? + client.log( + "Session for (uid:{}) in (gid:{}) reached daily guild study cap.\n{}".format( + self.userid, self.guildid, self.data + ), + context="SESSION_TRACKER" + ) + self.finish() + else: + # It's possible the expiry time was pushed forwards while waiting + # If so, reschedule + self.schedule_expiry() + def finish(self): """ Close the study session. """ - self.sessions[self.guildid].pop(self.userid, None) # Note that save_live_status doesn't need to be called here # The database saving procedure will account for the values. current_sessions.queries.close_study_session(*self.key) + # Remove session from active cache + self.sessions[self.guildid].pop(self.userid, None) + + # Cancel any existing expiry task + if self._expiry_task and not self._expiry_task.done(): + self._expiry_task.cancel() + def save_live_status(self, state: discord.VoiceState): """ Update the saved live status of the member. @@ -128,11 +237,43 @@ async def session_voice_tracker(client, member, before, after): else: # Member changed channel # End the current session and start a new one, if applicable - # TODO: Max daily study session tasks - # TODO: Error if before is None but we have a current session if session: + if (scid := session.data.channelid) and (not before.channel or scid != before.channel.id): + client.log( + "The previous voice state for " + "member {member.name} (uid:{member.id}) in {guild.name} (gid:{guild.id}) " + "does not match their current study session!\n" + "Session channel is (cid:{scid}), but the previous channel is {previous}.".format( + member=member, + guild=member.guild, + scid=scid, + previous="{0.name} (cid:{0.id})".format(before.channel) if before.channel else "None" + ), + context="SESSION_TRACKER", + level=logging.ERROR + ) + client.log( + "Ending study session for {member.name} (uid:{member.id}) " + "in {member.guild.id} (gid:{member.guild.id}) since they left the voice channel.\n{session}".format( + member=member, + session=session.data + ), + context="SESSION_TRACKER", + post=False + ) # End the current session session.finish() + elif pending := Session.members_pending[guild.id].pop(member.id, None): + client.log( + "Cancelling pending study session for {member.name} (uid:{member.id}) " + "in {member.guild.name} (gid:{member.guild.id}) since they left the voice channel.".format( + member=member + ), + context="SESSION_TRACKER", + post=False + ) + pending.cancel() + if after.channel: blacklist = client.objects['blacklisted_users'] guild_blacklist = client.objects['ignored_members'][guild.id] @@ -144,7 +285,15 @@ async def session_voice_tracker(client, member, before, after): ) if start_session: # Start a new session for the member - Session.start(member, after) + client.log( + "Starting a new voice channel study session for {member.name} (uid:{member.id}) " + "in {member.guild.name} (gid:{member.guild.id}).".format( + member=member, + ), + context="SESSION_TRACKER", + post=False + ) + session = Session.start(member, after) async def _init_session_tracker(client): @@ -190,7 +339,7 @@ async def _init_session_tracker(client): context="SESSION_INIT", level=logging.DEBUG ) - Session.sessions[row.guildid][row.userid] = session + session.activate() session.save_live_status(voice) resumed += 1 else: diff --git a/bot/modules/study/tracking/settings.py b/bot/modules/study/tracking/settings.py index dc93cc00..d7d583ab 100644 --- a/bot/modules/study/tracking/settings.py +++ b/bot/modules/study/tracking/settings.py @@ -1,5 +1,3 @@ -from collections import defaultdict - import settings from settings import GuildSettings from wards import guild_admin @@ -52,10 +50,10 @@ class untracked_channels(settings.ChannelList, settings.ListData, settings.Setti if any(channel.members for channel in guild.voice_channels) ] if active_guildids: + cache = {guildid: [] for guildid in active_guildids} rows = cls._table_interface.select_where( guildid=active_guildids ) - cache = defaultdict(list) for row in rows: cache[row['guildid']].append(row['channelid']) cls._cache.update(cache) @@ -114,11 +112,13 @@ class hourly_live_bonus(settings.Integer, settings.GuildSetting): @GuildSettings.attach_setting -class max_daily_study(settings.Duration, settings.GuildSetting): +class daily_study_cap(settings.Duration, settings.GuildSetting): category = "Study Tracking" - attr_name = "max_daily_study" - _data_column = "max_daily_study" + attr_name = "daily_study_cap" + _data_column = "daily_study_cap" - display_name = "max_daily_study" + display_name = "daily_study_cap" desc = "Maximum amount of study time ..." + + _default = 16 * 60 * 60 diff --git a/data/migration/v5-v6/migration.sql b/data/migration/v5-v6/migration.sql index 9e404e13..3e64007a 100644 --- a/data/migration/v5-v6/migration.sql +++ b/data/migration/v5-v6/migration.sql @@ -174,3 +174,5 @@ AS $$ ); END; $$ LANGUAGE PLPGSQL; + +ALTER TABLE guild_config ADD COLUMN daily_study_cap INTEGER; diff --git a/data/schema.sql b/data/schema.sql index 51ba3ecd..9f821f01 100644 --- a/data/schema.sql +++ b/data/schema.sql @@ -78,7 +78,7 @@ CREATE TABLE guild_config( returning_message TEXT, starting_funds INTEGER, persist_roles BOOLEAN, - max_daily_study INTEGER + daily_study_cap INTEGER ); CREATE TABLE ignored_members( From 3cb9d97e1ecde33d29409053d447d3360672bf27 Mon Sep 17 00:00:00 2001 From: Conatum Date: Thu, 2 Dec 2021 12:01:12 +0200 Subject: [PATCH 06/17] fix: Improve launch caching systems. --- bot/core/module.py | 5 +++-- bot/modules/moderation/video/admin.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bot/core/module.py b/bot/core/module.py index 0bf870c0..2f51408a 100644 --- a/bot/core/module.py +++ b/bot/core/module.py @@ -61,9 +61,10 @@ async def preload_studying_members(client): """ userids = list(set(member.id for guild in client.guilds for ch in guild.voice_channels for member in ch.members)) if userids: - rows = client.data.lions.fetch_rows_where(userid=userids) + users = client.data.user_config.fetch_rows_where(userid=userids) + members = client.data.lions.fetch_rows_where(userid=userids) client.log( - "Preloaded member data for {} members.".format(len(rows)), + "Preloaded data for {} user with {} members.".format(len(users), len(members)), context="CORE_LOADING" ) diff --git a/bot/modules/moderation/video/admin.py b/bot/modules/moderation/video/admin.py index 1f0ddaab..8d658c60 100644 --- a/bot/modules/moderation/video/admin.py +++ b/bot/modules/moderation/video/admin.py @@ -56,10 +56,10 @@ class video_channels(settings.ChannelList, settings.ListData, settings.Setting): if any(channel.members for channel in guild.voice_channels) ] if active_guildids: + cache = {guildid: [] for guildid in active_guildids} rows = cls._table_interface.select_where( guildid=active_guildids ) - cache = defaultdict(list) for row in rows: cache[row['guildid']].append(row['channelid']) cls._cache.update(cache) From 3abac6337956c66b6caa222ae7e4809dd6f3ecf5 Mon Sep 17 00:00:00 2001 From: Conatum Date: Thu, 2 Dec 2021 12:50:23 +0200 Subject: [PATCH 07/17] style (config): Add closest-match suggestions. --- bot/modules/guild_admin/guild_config.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bot/modules/guild_admin/guild_config.py b/bot/modules/guild_admin/guild_config.py index 608786fb..c79d62bb 100644 --- a/bot/modules/guild_admin/guild_config.py +++ b/bot/modules/guild_admin/guild_config.py @@ -1,3 +1,4 @@ +import difflib import discord from cmdClient.lib import SafeCancellation @@ -121,9 +122,15 @@ async def cmd_config(ctx, flags): name = parts[0] setting = setting_displaynames.get(name.lower(), None) if setting is None: + matches = difflib.get_close_matches(name, setting_displaynames.keys(), n=2) + match = "`{}`".format('` or `'.join(matches)) if matches else None return await ctx.error_reply( - "Server setting `{}` doesn't exist! Use `{}config` to see all server settings".format( - name, ctx.best_prefix + "Couldn't find a setting called `{}`!\n" + "{}" + "Use `{}config info` to see all the server settings.".format( + name, + "Maybe you meant {}?\n".format(match) if match else "", + ctx.best_prefix ) ) From fe1cd85511d1ac81c33f32c99ba6ea08d9461cfe Mon Sep 17 00:00:00 2001 From: Conatum Date: Thu, 2 Dec 2021 12:51:14 +0200 Subject: [PATCH 08/17] (sessions): Add UI for `daily_study_cap`. --- bot/modules/study/tracking/settings.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/bot/modules/study/tracking/settings.py b/bot/modules/study/tracking/settings.py index d7d583ab..53f7ecf8 100644 --- a/bot/modules/study/tracking/settings.py +++ b/bot/modules/study/tracking/settings.py @@ -119,6 +119,23 @@ class daily_study_cap(settings.Duration, settings.GuildSetting): _data_column = "daily_study_cap" display_name = "daily_study_cap" - desc = "Maximum amount of study time ..." + desc = "Maximum amount of recorded study time per member per day." _default = 16 * 60 * 60 + _default_multiplier = 60 * 60 + + _max = 25 * 60 * 60 + + long_desc = ( + "The maximum amount of study time that can be recorded for a member per day, " + "intended to remove system encouragement for unhealthy or obsessive behaviour.\n" + "The member may study for longer, but their sessions will not be tracked. " + "The start and end of the day are determined by the member's configured timezone." + ) + + @property + def success_response(self): + # Refresh expiry for all sessions in the guild + [session.schedule_expiry() for session in self.client.objects['sessions'][self.id].values()] + + return "The maximum tracked daily study time is now {}.".format(self.formatted) From cffdfb693b730780d4b6fe08a18ab0f8ecce4676 Mon Sep 17 00:00:00 2001 From: Conatum Date: Thu, 2 Dec 2021 13:16:36 +0200 Subject: [PATCH 09/17] (sessions): Add guild join and leave hooks. --- bot/modules/study/tracking/session_tracker.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/bot/modules/study/tracking/session_tracker.py b/bot/modules/study/tracking/session_tracker.py index 146f1195..24ffde49 100644 --- a/bot/modules/study/tracking/session_tracker.py +++ b/bot/modules/study/tracking/session_tracker.py @@ -296,6 +296,63 @@ async def session_voice_tracker(client, member, before, after): session = Session.start(member, after) +async def leave_guild_sessions(client, guild): + """ + `guild_leave` hook. + Close all sessions in the guild when we leave. + """ + sessions = list(Session.sessions[guild.id].values()) + for session in sessions: + session.finish() + client.log( + "Left {} (gid:{}) and closed {} ongoing study sessions.".format(guild.name, guild.id, len(sessions)), + context="SESSION_TRACKER" + ) + + +async def join_guild_sessions(client, guild): + """ + `guild_join` hook. + Refresh all sessions for the guild when we rejoin. + """ + # Delete existing current sessions, which should have been closed when we left + # It is possible we were removed from the guild during an outage + current_sessions.delete_where(guildid=guild.id) + + untracked = untracked_channels.get(guild.id).data + members = [ + member + for channel in guild.voice_channels + for member in channel.members + if channel.members and channel.id not in untracked + ] + for member in members: + client.log( + "Starting new session for '{}' (uid: {}) in '{}' (cid: {}) of '{}' (gid: {})".format( + member.name, + member.id, + member.voice.channel.name, + member.voice.channel.id, + member.guild.name, + member.guild.id + ), + context="SESSION_TRACKER", + level=logging.INFO, + post=False + ) + Session.start(member, member.voice) + + # Log newly started sessions + client.log( + "Joined {} (gid:{}) and started {} new study sessions from current voice channel members.".format( + guild.name, + guild.id, + len(members) + ), + context="SESSION_TRACKER", + ) + + async def _init_session_tracker(client): """ Load ongoing saved study sessions into the session cache, @@ -405,6 +462,8 @@ async def _init_session_tracker(client): # Now that we are in a valid initial state, attach the session event handler client.add_after_event("voice_state_update", session_voice_tracker) + client.add_after_event("guild_remove", leave_guild_sessions) + client.add_after_event("guild_join", join_guild_sessions) @module.launch_task From 144ccf9e81effdf7160edabb1048476df93d2b17 Mon Sep 17 00:00:00 2001 From: Conatum Date: Sat, 4 Dec 2021 10:24:11 +0200 Subject: [PATCH 10/17] (Lion): Update to account for current session. Remove `time` pending and syncing logic. Update `time` and `coins` to account for current session. Add `Session.duration` for current session duration. Add `Session.coins_earned` for current session coins. --- bot/cmdClient | 2 +- bot/core/data.py | 7 ++-- bot/core/lion.py | 40 +++++++++++-------- bot/modules/study/tracking/session_tracker.py | 23 +++++++++++ 4 files changed, 50 insertions(+), 22 deletions(-) diff --git a/bot/cmdClient b/bot/cmdClient index 75410acc..6eb42690 160000 --- a/bot/cmdClient +++ b/bot/cmdClient @@ -1 +1 @@ -Subproject commit 75410acc120b456ff315ff468357b6fb22a0a406 +Subproject commit 6eb426903423d6be8439621eb0b906aa94957efd diff --git a/bot/core/data.py b/bot/core/data.py index cd3f850b..013339b8 100644 --- a/bot/core/data.py +++ b/bot/core/data.py @@ -57,7 +57,7 @@ lion_ranks = Table('member_ranks', attach_as='lion_ranks') def add_pending(pending): """ pending: - List of tuples of the form `(userid, pending_coins, pending_time)`. + List of tuples of the form `(guildid, userid, pending_coins)`. """ with lions.conn: cursor = lions.conn.cursor() @@ -66,12 +66,11 @@ def add_pending(pending): """ UPDATE members SET - coins = coins + t.coin_diff, - tracked_time = tracked_time + t.time_diff + coins = coins + t.coin_diff FROM (VALUES %s) AS - t (guildid, userid, coin_diff, time_diff) + t (guildid, userid, coin_diff) WHERE members.guildid = t.guildid AND diff --git a/bot/core/lion.py b/bot/core/lion.py index 15bed9e0..c803cff8 100644 --- a/bot/core/lion.py +++ b/bot/core/lion.py @@ -12,7 +12,7 @@ class Lion: Mostly acts as a transparent interface to the corresponding Row, but also adds some transaction caching logic to `coins` and `tracked_time`. """ - __slots__ = ('guildid', 'userid', '_pending_coins', '_pending_time', '_member') + __slots__ = ('guildid', 'userid', '_pending_coins', '_member') # Members with pending transactions _pending = {} # userid -> User @@ -25,7 +25,6 @@ class Lion: self.userid = userid self._pending_coins = 0 - self._pending_time = 0 self._member = None @@ -93,16 +92,33 @@ class Lion: @property def time(self): """ - Amount of time the user has spent studying, accounting for pending values. + Amount of time the user has spent studying, accounting for a current session. """ - return int(self.data.tracked_time + self._pending_time) + # Base time from cached member data + time = self.data.tracked_time + + # Add current session time if it exists + if session := self.session: + time += session.duration + + return int(time) @property def coins(self): """ - Number of coins the user has, accounting for the pending value. + Number of coins the user has, accounting for the pending value and current session. """ - return int(self.data.coins + self._pending_coins) + # Base coin amount from cached member data + coins = self.data.coins + + # Add pending coin amount + coins += self._pending_coins + + # Add current session coins if applicable + if session := self.session: + coins += session.coins_earned + + return int(coins) @property def session(self): @@ -176,15 +192,6 @@ class Lion: if flush: self.flush() - def addTime(self, amount, flush=True): - """ - Add time to a user (in seconds), optionally storing the transaction in pending. - """ - self._pending_time += amount - self._pending[self.key] = self - if flush: - self.flush() - def flush(self): """ Flush any pending transactions to the database. @@ -202,7 +209,7 @@ class Lion: if lions: # Build userid to pending coin map pending = [ - (lion.guildid, lion.userid, int(lion._pending_coins), int(lion._pending_time)) + (lion.guildid, lion.userid, int(lion._pending_coins)) for lion in lions ] @@ -212,5 +219,4 @@ class Lion: # Cleanup pending users for lion in lions: lion._pending_coins -= int(lion._pending_coins) - lion._pending_time -= int(lion._pending_time) cls._pending.pop(lion.key, None) diff --git a/bot/modules/study/tracking/session_tracker.py b/bot/modules/study/tracking/session_tracker.py index 24ffde49..24187f17 100644 --- a/bot/modules/study/tracking/session_tracker.py +++ b/bot/modules/study/tracking/session_tracker.py @@ -128,8 +128,31 @@ class Session: @property def data(self): + """ + Row of the `current_sessions` table corresponding to this session. + """ return current_sessions.fetch(self.key) + @property + def duration(self): + """ + Current duration of the session. + """ + return (utc_now() - self.data.start_time).total_seconds() + + @property + def coins_earned(self): + """ + Number of coins earned so far. + """ + data = self.data + + coins = self.duration * data.hourly_coins + coins += data.live_duration * data.hourly_live_coins + if data.live_start: + coins += (utc_now() - data.live_start).total_seconds() * data.hourly_live_coins + return coins // 3600 + def activate(self): """ Activate the study session. From bb181d90395f3168f1c5351fa94d2dd18f76534b Mon Sep 17 00:00:00 2001 From: Conatum Date: Sat, 4 Dec 2021 11:13:26 +0200 Subject: [PATCH 11/17] (badges): Update to account for session tracker. Include current sessions in new badge scans. Remove the voice update badge hook. Account for current session in the current badge view. --- bot/modules/study/badges/badge_tracker.py | 18 +++--------------- data/migration/v5-v6/migration.sql | 2 +- data/schema.sql | 2 +- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/bot/modules/study/badges/badge_tracker.py b/bot/modules/study/badges/badge_tracker.py index 34e97c3a..2c0d33fb 100644 --- a/bot/modules/study/badges/badge_tracker.py +++ b/bot/modules/study/badges/badge_tracker.py @@ -8,7 +8,6 @@ import discord from meta import client from data.conditions import GEQ -from core import Lion from core.data import lions from utils.lib import strfdur from settings import GuildSettings @@ -56,7 +55,8 @@ async def update_study_badges(full=False): # Retrieve member rows with out of date study badges if not full and client.appdata.last_study_badge_scan is not None: update_rows = new_study_badges.select_where( - _timestamp=GEQ(client.appdata.last_study_badge_scan or 0) + _timestamp=GEQ(client.appdata.last_study_badge_scan or 0), + _extra="OR session_start IS NOT NULL" ) else: update_rows = new_study_badges.select_where() @@ -287,7 +287,6 @@ async def study_badge_tracker(): """ while True: try: - Lion.sync() await update_study_badges() except Exception: # Unknown exception. Catch it so the loop doesn't die. @@ -304,11 +303,10 @@ async def study_badge_tracker(): await asyncio.sleep(60) -async def _update_member_studybadge(member): +async def update_member_studybadge(member): """ Checks and (if required) updates the study badge for a single member. """ - Lion.fetch(member.guild.id, member.id).flush() update_rows = new_study_badges.select_where( guildid=member.guild.id, userid=member.id @@ -332,16 +330,6 @@ async def _update_member_studybadge(member): await _update_guild_badges(member.guild, update_rows) -@client.add_after_event("voice_state_update") -async def voice_studybadge_updater(client, member, before, after): - if not client.is_ready(): - # The poll loop will pick it up - return - - if before.channel and not after.channel: - await _update_member_studybadge(member) - - @module.launch_task async def launch_study_badge_tracker(client): asyncio.create_task(study_badge_tracker()) diff --git a/data/migration/v5-v6/migration.sql b/data/migration/v5-v6/migration.sql index 3e64007a..99fcefbf 100644 --- a/data/migration/v5-v6/migration.sql +++ b/data/migration/v5-v6/migration.sql @@ -125,7 +125,7 @@ CREATE VIEW current_study_badges AS *, (SELECT r.badgeid FROM study_badges r - WHERE r.guildid = members_totals.guildid AND members_totals.tracked_time > r.required_time + WHERE r.guildid = members_totals.guildid AND members_totals.total_tracked_time > r.required_time ORDER BY r.required_time DESC LIMIT 1) AS current_study_badgeid FROM members_totals; diff --git a/data/schema.sql b/data/schema.sql index 9f821f01..230f462c 100644 --- a/data/schema.sql +++ b/data/schema.sql @@ -563,7 +563,7 @@ CREATE VIEW current_study_badges AS *, (SELECT r.badgeid FROM study_badges r - WHERE r.guildid = members_totals.guildid AND members_totals.tracked_time > r.required_time + WHERE r.guildid = members_totals.guildid AND members_totals.total_tracked_time > r.required_time ORDER BY r.required_time DESC LIMIT 1) AS current_study_badgeid FROM members_totals; From 179b6ebf4ea11830699f9866fb69f0fda760ed58 Mon Sep 17 00:00:00 2001 From: Conatum Date: Sat, 4 Dec 2021 11:36:52 +0200 Subject: [PATCH 12/17] (leaderboards): Update to support sessions. Use `member_totals` to generate leaderboards instead of `members`. Fix typo in data coin conversion. --- bot/modules/economy/cointop_cmd.py | 20 ++++++++------------ bot/modules/study/top_cmd.py | 24 +++++++++--------------- bot/modules/study/tracking/data.py | 3 +++ data/migration/v5-v6/migration.sql | 4 ++-- data/schema.sql | 4 ++-- 5 files changed, 24 insertions(+), 31 deletions(-) diff --git a/bot/modules/economy/cointop_cmd.py b/bot/modules/economy/cointop_cmd.py index cd90b537..9d1b9b2d 100644 --- a/bot/modules/economy/cointop_cmd.py +++ b/bot/modules/economy/cointop_cmd.py @@ -46,19 +46,15 @@ async def cmd_topcoin(ctx): exclude.update(ctx.client.objects['blacklisted_users']) exclude.update(ctx.client.objects['ignored_members'][ctx.guild.id]) + args = { + 'guildid': ctx.guild.id, + 'select_columns': ('userid', 'total_coins::INTEGER'), + '_extra': "AND total_coins > 0 ORDER BY total_coins DESC " + ("LIMIT 100" if top100 else "") + } if exclude: - user_data = tables.lions.select_where( - guildid=ctx.guild.id, - userid=data.NOT(list(exclude)), - select_columns=('userid', 'coins'), - _extra="AND coins > 0 ORDER BY coins DESC " + ("LIMIT 100" if top100 else "") - ) - else: - user_data = tables.lions.select_where( - guildid=ctx.guild.id, - select_columns=('userid', 'coins'), - _extra="AND coins > 0 ORDER BY coins DESC " + ("LIMIT 100" if top100 else "") - ) + args['userid'] = data.NOT(list(exclude)) + + user_data = tables.members_totals.select_where(**args) # Quit early if the leaderboard is empty if not user_data: diff --git a/bot/modules/study/top_cmd.py b/bot/modules/study/top_cmd.py index 774f0409..cb737ed8 100644 --- a/bot/modules/study/top_cmd.py +++ b/bot/modules/study/top_cmd.py @@ -38,27 +38,20 @@ async def cmd_top(ctx): ) top100 = (ctx.args == "100" or ctx.alias == "top100") - # Flush any pending coin transactions - Lion.sync() - # Fetch the leaderboard exclude = set(m.id for m in ctx.guild_settings.unranked_roles.members) exclude.update(ctx.client.objects['blacklisted_users']) exclude.update(ctx.client.objects['ignored_members'][ctx.guild.id]) + args = { + 'guildid': ctx.guild.id, + 'select_columns': ('userid', 'total_tracked_time::INTEGER'), + '_extra': "AND total_tracked_time > 0 ORDER BY total_tracked_time DESC " + ("LIMIT 100" if top100 else "") + } if exclude: - user_data = tables.lions.select_where( - guildid=ctx.guild.id, - userid=data.NOT(list(exclude)), - select_columns=('userid', 'tracked_time'), - _extra="AND tracked_time > 0 ORDER BY tracked_time DESC " + ("LIMIT 100" if top100 else "") - ) - else: - user_data = tables.lions.select_where( - guildid=ctx.guild.id, - select_columns=('userid', 'tracked_time'), - _extra="AND tracked_time > 0 ORDER BY tracked_time DESC " + ("LIMIT 100" if top100 else "") - ) + args['userid'] = data.NOT(list(exclude)) + + user_data = tables.members_totals.select_where(**args) # Quit early if the leaderboard is empty if not user_data: @@ -68,6 +61,7 @@ async def cmd_top(ctx): author_index = None entries = [] for i, (userid, time) in enumerate(user_data): + time = int(time) member = ctx.guild.get_member(userid) name = member.display_name if member else str(userid) name = name.replace('*', ' ').replace('_', ' ') diff --git a/bot/modules/study/tracking/data.py b/bot/modules/study/tracking/data.py index 5940bf63..d9dcae38 100644 --- a/bot/modules/study/tracking/data.py +++ b/bot/modules/study/tracking/data.py @@ -57,3 +57,6 @@ def study_time_since(guildid, userid, timestamp): cursor.callproc('study_time_since', (guildid, userid, timestamp)) rows = cursor.fetchall() return (rows[0][0] if rows else None) or 0 + + +members_totals = Table('members_totals') diff --git a/data/migration/v5-v6/migration.sql b/data/migration/v5-v6/migration.sql index 99fcefbf..0bf8f610 100644 --- a/data/migration/v5-v6/migration.sql +++ b/data/migration/v5-v6/migration.sql @@ -77,7 +77,7 @@ AS $$ ) SELECT guildid, userid, channelid, channel_type, start_time, total_duration, total_stream_duration, total_video_duration, total_live_duration, - (total_duration * hourly_coins + live_duration * hourly_live_coins) / 60 + (total_duration * hourly_coins + live_duration * hourly_live_coins) / 3600 FROM current_sesh RETURNING * ) @@ -108,7 +108,7 @@ CREATE VIEW members_totals AS *, sesh.start_time AS session_start, tracked_time + COALESCE(sesh.total_duration, 0) AS total_tracked_time, - coins + COALESCE((sesh.total_duration * sesh.hourly_coins + sesh.live_duration * sesh.hourly_live_coins) / 60, 0) AS total_coins + coins + COALESCE((sesh.total_duration * sesh.hourly_coins + sesh.live_duration * sesh.hourly_live_coins) / 3600, 0) AS total_coins FROM members LEFT JOIN current_sessions_totals sesh USING (guildid, userid); diff --git a/data/schema.sql b/data/schema.sql index 230f462c..1e44151a 100644 --- a/data/schema.sql +++ b/data/schema.sql @@ -514,7 +514,7 @@ AS $$ ) SELECT guildid, userid, channelid, channel_type, start_time, total_duration, total_stream_duration, total_video_duration, total_live_duration, - (total_duration * hourly_coins + live_duration * hourly_live_coins) / 60 + (total_duration * hourly_coins + live_duration * hourly_live_coins) / 3600 FROM current_sesh RETURNING * ) @@ -544,7 +544,7 @@ CREATE VIEW members_totals AS *, sesh.start_time AS session_start, tracked_time + COALESCE(sesh.total_duration, 0) AS total_tracked_time, - coins + COALESCE((sesh.total_duration * sesh.hourly_coins + sesh.live_duration * sesh.hourly_live_coins) / 60, 0) AS total_coins + coins + COALESCE((sesh.total_duration * sesh.hourly_coins + sesh.live_duration * sesh.hourly_live_coins) / 3600, 0) AS total_coins FROM members LEFT JOIN current_sessions_totals sesh USING (guildid, userid); From 971446ffb590092e9785481c5e53eed996ee6d77 Mon Sep 17 00:00:00 2001 From: Conatum Date: Mon, 6 Dec 2021 20:48:28 +0200 Subject: [PATCH 13/17] (stats): Rewrite to include session data. Complete `stats` command rewrite to include session data. Added `get_member_rank` query to get accurate time and coin ranks. --- bot/core/data.py | 31 +++- bot/modules/study/stats_cmd.py | 266 ++++++++++++++++++++++++++++----- 2 files changed, 255 insertions(+), 42 deletions(-) diff --git a/bot/core/data.py b/bot/core/data.py index 013339b8..eca2f4e6 100644 --- a/bot/core/data.py +++ b/bot/core/data.py @@ -50,8 +50,6 @@ lions = RowTable( attach_as='lions' ) -lion_ranks = Table('member_ranks', attach_as='lion_ranks') - @lions.save_query def add_pending(pending): @@ -83,6 +81,35 @@ def add_pending(pending): return lions._make_rows(*data) +lion_ranks = Table('member_ranks', attach_as='lion_ranks') + + +@lions.save_query +def get_member_rank(guildid, userid, untracked): + """ + Get the time and coin ranking for the given member, ignoring the provided untracked members. + """ + with lions.conn as conn: + with conn.cursor() as curs: + curs.execute( + """ + SELECT + time_rank, coin_rank + FROM ( + SELECT + userid, + row_number() OVER (ORDER BY total_tracked_time DESC, userid ASC) AS time_rank, + row_number() OVER (ORDER BY total_coins DESC, userid ASC) AS coin_rank + FROM members_totals + WHERE + guildid=%s AND userid NOT IN %s + ) AS guild_ranks WHERE userid=%s + """, + (guildid, tuple(untracked), userid) + ) + return curs.fetchone() or (None, None) + + global_guild_blacklist = Table('global_guild_blacklist') global_user_blacklist = Table('global_user_blacklist') ignored_members = Table('ignored_members') diff --git a/bot/modules/study/stats_cmd.py b/bot/modules/study/stats_cmd.py index 90768202..9b9329dc 100644 --- a/bot/modules/study/stats_cmd.py +++ b/bot/modules/study/stats_cmd.py @@ -1,18 +1,22 @@ -import datetime +from datetime import datetime, timedelta import discord from cmdClient.checks import in_guild -from utils.lib import strfdur +from utils.lib import strfdur, prop_tabulate, utc_now from data import tables +from data.conditions import LEQ from core import Lion +from .tracking.data import study_time_since, session_history + from .module import module @module.cmd( "stats", group="Statistics", - desc="View a summary of your study statistics!" + desc="View a summary of your study statistics!", + aliases=('profile',) ) @in_guild() async def cmd_stats(ctx): @@ -23,6 +27,7 @@ async def cmd_stats(ctx): Description: View the study statistics for yourself or the mentioned user. """ + # Identify the target if ctx.args: if not ctx.msg.mentions: return await ctx.error_reply("Please mention a user to view their statistics!") @@ -30,54 +35,235 @@ async def cmd_stats(ctx): else: target = ctx.author - # Collect the required target data + # System sync + Lion.sync() + + # Fetch the required data lion = Lion.fetch(ctx.guild.id, target.id) - rank_data = tables.lion_ranks.select_one_where( + + history = session_history.select_where( + guildid=ctx.guild.id, userid=target.id, - guildid=ctx.guild.id + select_columns=( + "start_time", + "(start_time + duration * interval '1 second') AS end_time" + ), + _extra="ORDER BY start_time DESC" ) - # Extract and format data - time = strfdur(lion.time) + # Current economy balance (accounting for current session) coins = lion.coins - workouts = lion.data.workout_count - if lion.data.last_study_badgeid: - badge_row = tables.study_badges.fetch(lion.data.last_study_badgeid) - league = "<@&{}>".format(badge_row.roleid) + season_time = lion.time + workout_total = lion.data.workout_count + + # Leaderboard ranks + exclude = set(m.id for m in ctx.guild_settings.unranked_roles.members) + exclude.update(ctx.client.objects['blacklisted_users']) + exclude.update(ctx.client.objects['ignored_members'][ctx.guild.id]) + if target.id in exclude: + time_rank = None + coin_rank = None else: - league = "No league yet!" + time_rank, coin_rank = tables.lions.queries.get_member_rank(ctx.guild.id, target.id, list(exclude or [0])) - time_lb_pos = rank_data['time_rank'] - coin_lb_pos = rank_data['coin_rank'] + # Study time + # First get the all/month/week/day timestamps + day_start = lion.day_start + period_timestamps = ( + datetime(1970, 1, 1), + day_start.replace(day=1), + day_start - timedelta(days=day_start.weekday()), + day_start + ) + study_times = [0, 0, 0, 0] + for i, timestamp in enumerate(period_timestamps): + study_time = tables.session_history.queries.study_time_since(ctx.guild.id, target.id, timestamp) + if not study_time: + # So we don't make unecessary database calls + break + study_times[i] = study_time - # Build embed + # Streak statistics + streak = 0 + current_streak = None + max_streak = 0 + + day_attended = True if 'sessions' in ctx.client.objects and lion.session else None + date = day_start + daydiff = timedelta(days=1) + + periods = [(row['start_time'], row['end_time']) for row in history] + + i = 0 + while i < len(periods): + row = periods[i] + i += 1 + if row[1] > date: + # They attended this day + day_attended = True + continue + elif day_attended is None: + # Didn't attend today, but don't break streak + day_attended = False + date -= daydiff + i -= 1 + continue + elif not day_attended: + # Didn't attend the day, streak broken + date -= daydiff + i -= 1 + pass + else: + # Attended the day + streak += 1 + + # Move window to the previous day and try the row again + day_attended = False + prev_date = date + date -= daydiff + i -= 1 + + # Special case, when the last session started in the previous day + # Then the day is already attended + if i > 1 and date < periods[i-2][0] <= prev_date: + day_attended = True + + continue + + max_streak = max(max_streak, streak) + if current_streak is None: + current_streak = streak + streak = 0 + + # Handle loop exit state, i.e. the last streak + if day_attended: + streak += 1 + max_streak = max(max_streak, streak) + if current_streak is None: + current_streak = streak + + # Accountability stats + accountability = tables.accountability_member_info.select_where( + userid=target.id, + start_at=LEQ(utc_now()), + select_columns=("*", "(duration > 0 OR last_joined_at IS NOT NULL) AS attended"), + _extra="ORDER BY start_at DESC" + ) + if len(accountability): + acc_duration = sum(row['duration'] for row in accountability) + + acc_attended = sum(row['attended'] for row in accountability) + acc_total = len(accountability) + acc_rate = (acc_attended * 100) / acc_total + else: + acc_duration = 0 + acc_rate = 0 + + # Study League + guild_badges = tables.study_badges.fetch_rows_where(guildid=ctx.guild.id) + if lion.data.last_study_badgeid: + current_badge = tables.study_badges.fetch(lion.data.last_study_badgeid) + else: + current_badge = None + + next_badge = min( + (badge for badge in guild_badges + if badge.required_time > (current_badge.required_time if current_badge else 0)), + key=lambda badge: badge.required_time, + default=None + ) + + # We have all the data + # Now start building the embed embed = discord.Embed( - colour=discord.Colour.blue(), - timestamp=datetime.datetime.utcnow(), - title="Revision Statistics" - ).set_footer(text=str(target), icon_url=target.avatar_url).set_thumbnail(url=target.avatar_url) - embed.add_field( - name="📚 Study Time", - value=time + colour=discord.Colour.orange(), + title="Study Profile for {}".format(str(target)) ) - embed.add_field( - name="🦁 Revision League", - value=league + embed.set_thumbnail(url=target.avatar_url) + + # Add studying since if they have studied + if history: + embed.set_footer(text="Studying Since") + embed.timestamp = history[-1]['start_time'] + + # Set the description based on season time and server rank + if season_time: + time_str = "**{}:{}**".format( + season_time // 3600, + (season_time // 60) % 60 + ) + if time_rank is None: + rank_str = None + elif time_rank == 1: + rank_str = "1st" + elif time_rank == 2: + rank_str = "2nd" + elif time_rank == 3: + rank_str = "3rd" + else: + time_rank = "{}th".format(time_rank) + + embed.description = "{} has studied for **{}**{}{}".format( + target.mention, + time_str, + " this season" if study_times[0] - season_time > 60 else "", + ", and is ranked **{}** in the server!".format(rank_str) if rank_str else "." + ) + else: + embed.description = "{} hasn't studied in this server yet!".format(target.mention) + + # Build the stats table + stats = {} + + stats['Coins Earned'] = "**{}** LC".format( + coins, + # "Rank `{}`".format(coin_rank) if coins and coin_rank else "Unranked" ) - embed.add_field( - name="🦁 LionCoins", - value=coins + if workout_total: + stats['Workouts'] = "**{}** sessions".format(workout_total) + if acc_duration: + stats['Accountability'] = "**{}** hours (`{:.0f}%` attended)".format( + acc_duration // 3600, + acc_rate + ) + stats['Study Streak'] = "**{}** days{}".format( + streak, + " (longest **{}** days)".format(max_streak) if max_streak else '' ) - embed.add_field( - name="🏆 Leaderboard Position", - value="Time: {}\n LC: {}".format(time_lb_pos, coin_lb_pos) - ) - embed.add_field( - name="💪 Workouts", - value=workouts - ) - embed.add_field( - name="📋 Attendence", - value="TBD" + + stats_table = prop_tabulate(*zip(*stats.items())) + + # Build the time table + time_table = prop_tabulate( + ('Daily', 'Weekly', 'Monthly', 'All Time'), + ["{:02}:{:02}".format(t // 3600, (t // 60) % 60) for t in reversed(study_times)] ) + + # The order they are added depends on the size of the stats table + if len(stats) >= 4: + embed.add_field(name="Statistics", value=stats_table) + embed.add_field(name="Study Time", value=time_table) + else: + embed.add_field(name="Study Time", value=time_table) + embed.add_field(name="Statistics", value=stats_table) + + # Add the study league field + if current_badge or next_badge: + current_str = ( + "You are currently in <@&{}>!".format(current_badge.roleid) if current_badge else "Not league yet!" + ) + if next_badge: + needed = max(next_badge.required_time - season_time, 0) + next_str = "Study for **{:02}:{:02}** more to achieve <@&{}>.".format( + needed // 3600, + (needed // 60) % 60, + next_badge.roleid + ) + else: + next_str = "You have reached the highest league! Congratulations!" + embed.add_field( + name="Study League", + value="{}\n{}".format(current_str, next_str), + inline=False + ) await ctx.reply(embed=embed) From d95faed02bc5453362d35bc72cc56f32340ecdb2 Mon Sep 17 00:00:00 2001 From: Conatum Date: Tue, 7 Dec 2021 11:02:40 +0200 Subject: [PATCH 14/17] fix (stats): Repair some formatting issues. --- bot/modules/study/stats_cmd.py | 10 +++++----- bot/modules/study/top_cmd.py | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/bot/modules/study/stats_cmd.py b/bot/modules/study/stats_cmd.py index 9b9329dc..dbd71302 100644 --- a/bot/modules/study/stats_cmd.py +++ b/bot/modules/study/stats_cmd.py @@ -2,12 +2,12 @@ from datetime import datetime, timedelta import discord from cmdClient.checks import in_guild -from utils.lib import strfdur, prop_tabulate, utc_now +from utils.lib import prop_tabulate, utc_now from data import tables from data.conditions import LEQ from core import Lion -from .tracking.data import study_time_since, session_history +from .tracking.data import session_history from .module import module @@ -15,7 +15,7 @@ from .module import module @module.cmd( "stats", group="Statistics", - desc="View a summary of your study statistics!", + desc="View your personal server study statistics!", aliases=('profile',) ) @in_guild() @@ -227,7 +227,7 @@ async def cmd_stats(ctx): acc_rate ) stats['Study Streak'] = "**{}** days{}".format( - streak, + current_streak, " (longest **{}** days)".format(max_streak) if max_streak else '' ) @@ -250,7 +250,7 @@ async def cmd_stats(ctx): # Add the study league field if current_badge or next_badge: current_str = ( - "You are currently in <@&{}>!".format(current_badge.roleid) if current_badge else "Not league yet!" + "You are currently in <@&{}>!".format(current_badge.roleid) if current_badge else "No league yet!" ) if next_badge: needed = max(next_badge.required_time - season_time, 0) diff --git a/bot/modules/study/top_cmd.py b/bot/modules/study/top_cmd.py index cb737ed8..cb4008f7 100644 --- a/bot/modules/study/top_cmd.py +++ b/bot/modules/study/top_cmd.py @@ -61,7 +61,6 @@ async def cmd_top(ctx): author_index = None entries = [] for i, (userid, time) in enumerate(user_data): - time = int(time) member = ctx.guild.get_member(userid) name = member.display_name if member else str(userid) name = name.replace('*', ' ').replace('_', ' ') From b273ae05969eadca9c1462e55a8d70de6841df8f Mon Sep 17 00:00:00 2001 From: Conatum Date: Tue, 7 Dec 2021 13:23:13 +0200 Subject: [PATCH 15/17] (data): Update version and migration. --- bot/constants.py | 2 +- data/migration/v5-v6/migration.sql | 16 +++++++++------- data/schema.sql | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bot/constants.py b/bot/constants.py index 132ce93d..e5eb789a 100644 --- a/bot/constants.py +++ b/bot/constants.py @@ -1,2 +1,2 @@ CONFIG_FILE = "config/bot.conf" -DATA_VERSION = 5 +DATA_VERSION = 6 diff --git a/data/migration/v5-v6/migration.sql b/data/migration/v5-v6/migration.sql index 0bf8f610..eae1f88b 100644 --- a/data/migration/v5-v6/migration.sql +++ b/data/migration/v5-v6/migration.sql @@ -1,16 +1,16 @@ -DROP TYPE IF EXISTS SessionChannelType CASCADE; -DROP TABLE IF EXISTS session_history CASCADE; -DROP TABLE IF EXISTS current_sessions CASCADE; -DROP FUNCTION IF EXISTS close_study_session; +-- DROP TYPE IF EXISTS SessionChannelType CASCADE; +-- DROP TABLE IF EXISTS session_history CASCADE; +-- DROP TABLE IF EXISTS current_sessions CASCADE; +-- DROP FUNCTION IF EXISTS close_study_session(_guildid BIGINT, _userid BIGINT); +-- DROP FUNCTION IF EXISTS study_time_since(_guildid BIGINT, _userid BIGINT, _timestamp TIMESTAMPTZ) + +-- DROP VIEW IF EXISTS current_sessions_totals CASCADE; -DROP VIEW IF EXISTS current_sessions_totals CASCADE; DROP VIEW IF EXISTS member_totals CASCADE; DROP VIEW IF EXISTS member_ranks CASCADE; DROP VIEW IF EXISTS current_study_badges CASCADE; DROP VIEW IF EXISTS new_study_badges CASCADE; -DROP FUNCTION IF EXISTS study_time_since; - CREATE TYPE SessionChannelType AS ENUM ( 'STANDARD', @@ -176,3 +176,5 @@ AS $$ $$ LANGUAGE PLPGSQL; ALTER TABLE guild_config ADD COLUMN daily_study_cap INTEGER; + +INSERT INTO VersionHistory (version, author) VALUES (6, 'v5-v6 Migration'); diff --git a/data/schema.sql b/data/schema.sql index 1e44151a..a7a4af31 100644 --- a/data/schema.sql +++ b/data/schema.sql @@ -4,7 +4,7 @@ CREATE TABLE VersionHistory( time TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL, author TEXT ); -INSERT INTO VersionHistory (version, author) VALUES (5, 'Initial Creation'); +INSERT INTO VersionHistory (version, author) VALUES (6, 'Initial Creation'); CREATE OR REPLACE FUNCTION update_timestamp_column() From 9d0fd93822192d14626616ab100740ee80699317 Mon Sep 17 00:00:00 2001 From: Conatum Date: Wed, 8 Dec 2021 11:44:15 +0200 Subject: [PATCH 16/17] fix (stats): Fix time formatting issue. --- bot/modules/study/stats_cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/modules/study/stats_cmd.py b/bot/modules/study/stats_cmd.py index dbd71302..67ecfa06 100644 --- a/bot/modules/study/stats_cmd.py +++ b/bot/modules/study/stats_cmd.py @@ -188,7 +188,7 @@ async def cmd_stats(ctx): # Set the description based on season time and server rank if season_time: - time_str = "**{}:{}**".format( + time_str = "**{}:{:02}**".format( season_time // 3600, (season_time // 60) % 60 ) From f18af33fb600c4fcd79ad8190877225f4ff2d1d1 Mon Sep 17 00:00:00 2001 From: Conatum Date: Wed, 8 Dec 2021 11:44:35 +0200 Subject: [PATCH 17/17] fix (sessions): Recognise accountability sessions. --- bot/modules/study/tracking/session_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/modules/study/tracking/session_tracker.py b/bot/modules/study/tracking/session_tracker.py index 24187f17..96262f04 100644 --- a/bot/modules/study/tracking/session_tracker.py +++ b/bot/modules/study/tracking/session_tracker.py @@ -78,7 +78,7 @@ class Session: # TODO: More reliable channel type determination if state.channel.id in tables.rented.row_cache: channel_type = SessionChannelType.RENTED - elif state.channel.id in tables.accountability_rooms.row_cache: + elif state.channel.category and state.channel.category.id == lion.guild_settings.accountability_category.data: channel_type = SessionChannelType.ACCOUNTABILITY else: channel_type = SessionChannelType.STANDARD