(todo): Add extra metrics for stats.

v6 -> v7 data migration.
Use soft deletion for tasks.
Remove task expiry.
Migrate `complete` field to `completed_at`.

bot/modules/todo/Tasklist.py

@@ -6,8 +6,9 @@ import asyncio
 from cmdClient.lib import SafeCancellation
 from meta import client
 from core import Lion
+from data import NULL, NOTNULL
 from settings import GuildSettings
-from utils.lib import parse_ranges
+from utils.lib import parse_ranges, utc_now
 
 from . import data
 # from .module import module
@@ -130,8 +131,8 @@ class Tasklist:
         """
         self.tasklist = data.tasklist.fetch_rows_where(
             userid=self.member.id,
-            _extra=("AND last_updated_at > timezone('utc', NOW()) - INTERVAL '24h' "
+            deleted_at=NULL,
-                    "ORDER BY created_at ASC, taskid ASC")
+            _extra="ORDER BY created_at ASC, taskid ASC"
         )
         self._refreshed_at = datetime.datetime.utcnow()
 
@@ -144,7 +145,7 @@ class Tasklist:
             "{num:>{numlen}}. [{mark}] {content}".format(
                 num=i,
                 numlen=((self.block_size * (i // self.block_size + 1) - 1) // 10) + 1,
-                mark=self.checkmark if task.complete else ' ',
+                mark=self.checkmark if task.completed_at else ' ',
                 content=task.content
             )
             for i, task in enumerate(self.tasklist)
@@ -159,7 +160,7 @@ class Tasklist:
         # Formatting strings and data
         page_count = len(task_blocks) or 1
         task_count = len(task_strings)
-        complete_count = len([task for task in self.tasklist if task.complete])
+        complete_count = len([task for task in self.tasklist if task.completed_at])
 
         if task_count > 0:
             title = "TODO list ({}/{} complete)".format(
@@ -205,7 +206,7 @@ class Tasklist:
         # Calculate or adjust the current page number
         if self.current_page is None:
             # First page with incomplete task, or the first page
-            first_incomplete = next((i for i, task in enumerate(self.tasklist) if not task.complete), 0)
+            first_incomplete = next((i for i, task in enumerate(self.tasklist) if not task.completed_at), 0)
             self.current_page = first_incomplete // self.block_size
         elif self.current_page >= len(self.pages):
             self.current_page = len(self.pages) - 1
@@ -394,8 +395,14 @@ class Tasklist:
         Delete tasks from the task list
         """
         taskids = [self.tasklist[i].taskid for i in indexes]
-        return data.tasklist.delete_where(
+
-            taskid=taskids
+        now = utc_now()
+        return data.tasklist.update_where(
+            {
+                'deleted_at': now,
+                'last_updated_at': now
+            },
+            taskid=taskids,
         )
 
     def _edit_task(self, index, new_content):
@@ -403,10 +410,12 @@ class Tasklist:
         Update the provided task with the new content
         """
         taskid = self.tasklist[index].taskid
+
+        now = utc_now()
         return data.tasklist.update_where(
             {
                 'content': new_content,
-                'last_updated_at': datetime.datetime.utcnow()
+                'last_updated_at': now
             },
             taskid=taskid,
         )
@@ -416,13 +425,15 @@ class Tasklist:
         Mark provided tasks as complete
         """
         taskids = [self.tasklist[i].taskid for i in indexes]
+
+        now = utc_now()
         return data.tasklist.update_where(
             {
-                'complete': True,
+                'completed_at': now,
-                'last_updated_at': datetime.datetime.utcnow()
+                'last_updated_at': now
             },
             taskid=taskids,
-            complete=False,
+            completed_at=NULL,
         )
 
     def _uncheck_tasks(self, *indexes):
@@ -430,13 +441,15 @@ class Tasklist:
         Mark provided tasks as incomplete
         """
         taskids = [self.tasklist[i].taskid for i in indexes]
+
+        now = utc_now()
         return data.tasklist.update_where(
             {
-                'complete': False,
+                'completed_at': None,
-                'last_updated_at': datetime.datetime.utcnow()
+                'last_updated_at': now
             },
             taskid=taskids,
-            complete=True,
+            completed_at=NOTNULL,
         )
 
     def _index_range_parser(self, userstr):
@@ -466,7 +479,7 @@ class Tasklist:
         count = data.tasklist.select_one_where(
             select_columns=("COUNT(*)",),
             userid=self.member.id,
-            _extra="AND last_updated_at > timezone('utc', NOW()) - INTERVAL '24h'"
+            deleted_at=NOTNULL
         )[0]
 
         # Fetch maximum allowed count
@@ -503,8 +516,8 @@ class Tasklist:
         # Parse provided ranges
         indexes = self._index_range_parser(userstr)
 
-        to_check = [index for index in indexes if not self.tasklist[index].complete]
+        to_check = [index for index in indexes if not self.tasklist[index].completed_at]
-        to_uncheck = [index for index in indexes if self.tasklist[index].complete]
+        to_uncheck = [index for index in indexes if self.tasklist[index].completed_at]
 
         if to_uncheck:
             self._uncheck_tasks(*to_uncheck)
@@ -694,15 +707,3 @@ async def tasklist_message_handler(client, message):
 async def tasklist_reaction_add_handler(client, reaction, user):
     if user != client.user and reaction.message.id in Tasklist.messages:
         await Tasklist.messages[reaction.message.id].handle_reaction(reaction, user, True)
-
-
-# @module.launch_task
-# Commented because we don't actually need to expire these
-async def tasklist_expiry_watchdog(client):
-    removed = data.tasklist.queries.expire_old_tasks()
-    if removed:
-        client.log(
-            "Remove {} stale todo tasks.".format(len(removed)),
-            context="TASKLIST_EXPIRY",
-            post=True
-        )

bot/modules/todo/data.py

@@ -2,23 +2,11 @@ from data import RowTable, Table
 
 tasklist = RowTable(
     'tasklist',
-    ('taskid', 'userid', 'content', 'complete', 'rewarded', 'created_at', 'last_updated_at'),
+    ('taskid', 'userid', 'content', 'rewarded', 'created_at', 'completed_at', 'deleted_at', 'last_updated_at'),
     'taskid'
 )
 
 
-@tasklist.save_query
-def expire_old_tasks():
-    with tasklist.conn:
-        with tasklist.conn.cursor() as curs:
-            curs.execute(
-                "DELETE FROM tasklist WHERE "
-                "last_updated_at < timezone('utc', NOW()) - INTERVAL '7d' "
-                "RETURNING *"
-            )
-            return curs.fetchall()
-
-
 tasklist_channels = Table('tasklist_channels')
 
 tasklist_rewards = Table('tasklist_reward_history')

data/migration/v6-v7/migration.sql

@@ -0,0 +1,18 @@
+-- Add deletion column to tasklist entries
+-- Add completed_at column to the tasklist entries, replacing complete
+
+
+ALTER TABLE tasklist
+  ADD COLUMN completed_at TIMESTAMPTZ,
+  ADD COLUMN deleted_at TIMESTAMPTZ,
+  ALTER COLUMN created_at TYPE TIMESTAMPTZ USING created_at AT TIME ZONE 'UTC',
+  ALTER COLUMN last_updated_at TYPE TIMESTAMPTZ USING created_at AT TIME ZONE 'UTC';
+
+UPDATE tasklist SET deleted_at = NOW() WHERE last_updated_at < NOW() - INTERVAL '24h';
+UPDATE tasklist SET completed_at = last_updated_at WHERE complete;
+
+ALTER TABLE tasklist
+  DROP COLUMN complete;
+
+
+-- Mark all tasklist entries older than a day as deleted

data/schema.sql

@@ -135,10 +135,11 @@ CREATE TABLE tasklist(
   taskid SERIAL PRIMARY KEY,
   userid BIGINT NOT NULL,
   content TEXT NOT NULL,
-  complete BOOL DEFAULT FALSE,
   rewarded BOOL DEFAULT FALSE,
-  created_at TIMESTAMP DEFAULT (now() at time zone 'utc'),
+  deleted_at TIMESTAMPTZ,
-  last_updated_at TIMESTAMP DEFAULT (now() at time zone 'utc')
+  completed_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ,
+  last_updated_at TIMESTAMPTZ
 );
 CREATE INDEX tasklist_users ON tasklist (userid);