From e2c096f350e4f75470a836fa31e53e55ea1e47b8 Mon Sep 17 00:00:00 2001 From: Conatum Date: Wed, 29 Dec 2021 08:27:53 +0200 Subject: [PATCH] (todo): Add extra metrics for stats. v6 -> v7 data migration. Use soft deletion for tasks. Remove task expiry. Migrate `complete` field to `completed_at`. --- bot/modules/todo/Tasklist.py | 61 +++++++++++++++--------------- bot/modules/todo/data.py | 14 +------ data/migration/v6-v7/migration.sql | 18 +++++++++ data/schema.sql | 7 ++-- 4 files changed, 54 insertions(+), 46 deletions(-) create mode 100644 data/migration/v6-v7/migration.sql diff --git a/bot/modules/todo/Tasklist.py b/bot/modules/todo/Tasklist.py index dd6b6880..f1b2ebac 100644 --- a/bot/modules/todo/Tasklist.py +++ b/bot/modules/todo/Tasklist.py @@ -6,8 +6,9 @@ import asyncio from cmdClient.lib import SafeCancellation from meta import client from core import Lion +from data import NULL, NOTNULL from settings import GuildSettings -from utils.lib import parse_ranges +from utils.lib import parse_ranges, utc_now from . import data # from .module import module @@ -130,8 +131,8 @@ class Tasklist: """ self.tasklist = data.tasklist.fetch_rows_where( userid=self.member.id, - _extra=("AND last_updated_at > timezone('utc', NOW()) - INTERVAL '24h' " - "ORDER BY created_at ASC, taskid ASC") + deleted_at=NULL, + _extra="ORDER BY created_at ASC, taskid ASC" ) self._refreshed_at = datetime.datetime.utcnow() @@ -144,7 +145,7 @@ class Tasklist: "{num:>{numlen}}. [{mark}] {content}".format( num=i, numlen=((self.block_size * (i // self.block_size + 1) - 1) // 10) + 1, - mark=self.checkmark if task.complete else ' ', + mark=self.checkmark if task.completed_at else ' ', content=task.content ) for i, task in enumerate(self.tasklist) @@ -159,7 +160,7 @@ class Tasklist: # Formatting strings and data page_count = len(task_blocks) or 1 task_count = len(task_strings) - complete_count = len([task for task in self.tasklist if task.complete]) + complete_count = len([task for task in self.tasklist if task.completed_at]) if task_count > 0: title = "TODO list ({}/{} complete)".format( @@ -205,7 +206,7 @@ class Tasklist: # Calculate or adjust the current page number if self.current_page is None: # First page with incomplete task, or the first page - first_incomplete = next((i for i, task in enumerate(self.tasklist) if not task.complete), 0) + first_incomplete = next((i for i, task in enumerate(self.tasklist) if not task.completed_at), 0) self.current_page = first_incomplete // self.block_size elif self.current_page >= len(self.pages): self.current_page = len(self.pages) - 1 @@ -394,8 +395,14 @@ class Tasklist: Delete tasks from the task list """ taskids = [self.tasklist[i].taskid for i in indexes] - return data.tasklist.delete_where( - taskid=taskids + + now = utc_now() + return data.tasklist.update_where( + { + 'deleted_at': now, + 'last_updated_at': now + }, + taskid=taskids, ) def _edit_task(self, index, new_content): @@ -403,10 +410,12 @@ class Tasklist: Update the provided task with the new content """ taskid = self.tasklist[index].taskid + + now = utc_now() return data.tasklist.update_where( { 'content': new_content, - 'last_updated_at': datetime.datetime.utcnow() + 'last_updated_at': now }, taskid=taskid, ) @@ -416,13 +425,15 @@ class Tasklist: Mark provided tasks as complete """ taskids = [self.tasklist[i].taskid for i in indexes] + + now = utc_now() return data.tasklist.update_where( { - 'complete': True, - 'last_updated_at': datetime.datetime.utcnow() + 'completed_at': now, + 'last_updated_at': now }, taskid=taskids, - complete=False, + completed_at=NULL, ) def _uncheck_tasks(self, *indexes): @@ -430,13 +441,15 @@ class Tasklist: Mark provided tasks as incomplete """ taskids = [self.tasklist[i].taskid for i in indexes] + + now = utc_now() return data.tasklist.update_where( { - 'complete': False, - 'last_updated_at': datetime.datetime.utcnow() + 'completed_at': None, + 'last_updated_at': now }, taskid=taskids, - complete=True, + completed_at=NOTNULL, ) def _index_range_parser(self, userstr): @@ -466,7 +479,7 @@ class Tasklist: count = data.tasklist.select_one_where( select_columns=("COUNT(*)",), userid=self.member.id, - _extra="AND last_updated_at > timezone('utc', NOW()) - INTERVAL '24h'" + deleted_at=NOTNULL )[0] # Fetch maximum allowed count @@ -503,8 +516,8 @@ class Tasklist: # Parse provided ranges indexes = self._index_range_parser(userstr) - to_check = [index for index in indexes if not self.tasklist[index].complete] - to_uncheck = [index for index in indexes if self.tasklist[index].complete] + to_check = [index for index in indexes if not self.tasklist[index].completed_at] + to_uncheck = [index for index in indexes if self.tasklist[index].completed_at] if to_uncheck: self._uncheck_tasks(*to_uncheck) @@ -694,15 +707,3 @@ async def tasklist_message_handler(client, message): async def tasklist_reaction_add_handler(client, reaction, user): if user != client.user and reaction.message.id in Tasklist.messages: await Tasklist.messages[reaction.message.id].handle_reaction(reaction, user, True) - - -# @module.launch_task -# Commented because we don't actually need to expire these -async def tasklist_expiry_watchdog(client): - removed = data.tasklist.queries.expire_old_tasks() - if removed: - client.log( - "Remove {} stale todo tasks.".format(len(removed)), - context="TASKLIST_EXPIRY", - post=True - ) diff --git a/bot/modules/todo/data.py b/bot/modules/todo/data.py index f340197a..be2d8c65 100644 --- a/bot/modules/todo/data.py +++ b/bot/modules/todo/data.py @@ -2,23 +2,11 @@ from data import RowTable, Table tasklist = RowTable( 'tasklist', - ('taskid', 'userid', 'content', 'complete', 'rewarded', 'created_at', 'last_updated_at'), + ('taskid', 'userid', 'content', 'rewarded', 'created_at', 'completed_at', 'deleted_at', 'last_updated_at'), 'taskid' ) -@tasklist.save_query -def expire_old_tasks(): - with tasklist.conn: - with tasklist.conn.cursor() as curs: - curs.execute( - "DELETE FROM tasklist WHERE " - "last_updated_at < timezone('utc', NOW()) - INTERVAL '7d' " - "RETURNING *" - ) - return curs.fetchall() - - tasklist_channels = Table('tasklist_channels') tasklist_rewards = Table('tasklist_reward_history') diff --git a/data/migration/v6-v7/migration.sql b/data/migration/v6-v7/migration.sql new file mode 100644 index 00000000..04cc39da --- /dev/null +++ b/data/migration/v6-v7/migration.sql @@ -0,0 +1,18 @@ +-- Add deletion column to tasklist entries +-- Add completed_at column to the tasklist entries, replacing complete + + +ALTER TABLE tasklist + ADD COLUMN completed_at TIMESTAMPTZ, + ADD COLUMN deleted_at TIMESTAMPTZ, + ALTER COLUMN created_at TYPE TIMESTAMPTZ USING created_at AT TIME ZONE 'UTC', + ALTER COLUMN last_updated_at TYPE TIMESTAMPTZ USING created_at AT TIME ZONE 'UTC'; + +UPDATE tasklist SET deleted_at = NOW() WHERE last_updated_at < NOW() - INTERVAL '24h'; +UPDATE tasklist SET completed_at = last_updated_at WHERE complete; + +ALTER TABLE tasklist + DROP COLUMN complete; + + +-- Mark all tasklist entries older than a day as deleted diff --git a/data/schema.sql b/data/schema.sql index a7a4af31..cf960112 100644 --- a/data/schema.sql +++ b/data/schema.sql @@ -135,10 +135,11 @@ CREATE TABLE tasklist( taskid SERIAL PRIMARY KEY, userid BIGINT NOT NULL, content TEXT NOT NULL, - complete BOOL DEFAULT FALSE, rewarded BOOL DEFAULT FALSE, - created_at TIMESTAMP DEFAULT (now() at time zone 'utc'), - last_updated_at TIMESTAMP DEFAULT (now() at time zone 'utc') + deleted_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ, + last_updated_at TIMESTAMPTZ ); CREATE INDEX tasklist_users ON tasklist (userid);