sharding (core): Add base sharding support.

Add `meta.args` for command line argument access.
Add command line argument support for shard number.
Add shard count to config file.
Add `meta.sharding` exposing shard properties.
Add shard number to logging methods.
Add shard number to data appid.

bot/data/__init__.py

@@ -1,5 +1,5 @@
+from .conditions import Condition, NOT, Constant, NULL, NOTNULL  # noqa
 from .connection import conn  # noqa
 from .formatters import UpdateValue, UpdateValueAdd  # noqa
 from .interfaces import Table, RowTable, Row, tables  # noqa
 from .queries import insert, insert_many, select_where, update_where, upsert, delete_where  # noqa
-from .conditions import Condition, NOT, Constant, NULL, NOTNULL  # noqa

bot/data/conditions.py

@@ -1,5 +1,7 @@
 from .connection import _replace_char
 
+from meta import sharding
+
 
 class Condition:
     """
@@ -82,5 +84,8 @@ class SHARDID(Condition):
         values.append(self.shardid)
 
 
+THIS_SHARD = SHARDID(sharding.shard_number, sharding.shard_count)
+
+
 NULL = Constant('IS NULL')
 NOTNULL = Constant('IS NOT NULL')

bot/main.py

@@ -1,4 +1,4 @@
-from meta import client, conf, log
+from meta import client, conf, log, sharding
 
 from data import tables
 
@@ -7,7 +7,12 @@ import core # noqa
 import modules  # noqa
 
 # Load and attach app specific data
-client.appdata = core.data.meta.fetch_or_create(conf.bot['data_appid'])
+if sharding.sharded:
+    appname = f"{conf.bot['data_appid']}_{sharding.shard_count}_{sharding.shard_number}"
+else:
+    appname = conf.bot['data_appid']
+client.appdata = core.data.meta.fetch_or_create(appname)
+
 client.data = tables
 
 # Initialise all modules

bot/meta/__init__.py

@@ -1,3 +1,5 @@
+from .logger import log, logger
 from .client import client
 from .config import conf
-from .logger import log, logger
+from .args import args
+from . import sharding

bot/meta/args.py

@@ -0,0 +1,19 @@
+import argparse
+
+from constants import CONFIG_FILE
+
+# ------------------------------
+# Parsed commandline arguments
+# ------------------------------
+parser = argparse.ArgumentParser()
+parser.add_argument('--conf',
+                    dest='config',
+                    default=CONFIG_FILE,
+                    help="Path to configuration file.")
+parser.add_argument('--shard',
+                    dest='shard',
+                    default=None,
+                    type=int,
+                    help="Shard number to run, if applicable.")
+
+args = parser.parse_args()

bot/meta/client.py

@@ -1,16 +1,19 @@
 from discord import Intents
 from cmdClient.cmdClient import cmdClient
 
-from .config import Conf
+from .config import conf
+from .sharding import shard_number, shard_count
 
-from constants import CONFIG_FILE
-
-# Initialise config
-conf = Conf(CONFIG_FILE)
 
 # Initialise client
 owners = [int(owner) for owner in conf.bot.getlist('owners')]
 intents = Intents.all()
 intents.presences = False
-client = cmdClient(prefix=conf.bot['prefix'], owners=owners, intents=intents)
+client = cmdClient(
+    prefix=conf.bot['prefix'],
+    owners=owners,
+    intents=intents,
+    shard_id=shard_number,
+    shard_count=shard_count
+)
 client.conf = conf

bot/meta/config.py

@@ -1,9 +1,6 @@
 import configparser as cfgp
 
-
-conf = None  # type: Conf
-
-CONF_FILE = "bot/bot.conf"
+from .args import args
 
 
 class Conf:
@@ -57,3 +54,6 @@ class Conf:
     def write(self):
         with open(self.configfile, 'w') as conffile:
             self.config.write(conffile)
+
+
+conf = Conf(args.config)

bot/meta/logger.py

@@ -9,11 +9,18 @@ from utils.lib import mail, split_text
 
 from .client import client
 from .config import conf
+from . import sharding
 
 
 # Setup the logger
 logger = logging.getLogger()
-log_fmt = logging.Formatter(fmt='[{asctime}][{levelname:^8}] {message}', datefmt='%d/%m | %H:%M:%S', style='{')
+log_fmt = logging.Formatter(
+    fmt=('[{asctime}][{levelname:^8}]' +
+         '[SHARD {}]'.format(sharding.shard_number) +
+         '{message}'),
+    datefmt='%d/%m | %H:%M:%S',
+    style='{'
+)
 # term_handler = logging.StreamHandler(sys.stdout)
 # term_handler.setFormatter(log_fmt)
 # logger.addHandler(term_handler)
@@ -77,7 +84,11 @@ async def live_log(message, context, level):
             log_chid = conf.bot.getint('log_channel')
 
         # Generate the log messages
-        header = "[{}][{}]".format(logging.getLevelName(level), str(context))
+        if sharding.sharded:
+            header = f"[{logging.getLevelName(level)}][SHARD {sharding.shard_number}][{context}]"
+        else:
+            header = f"[{logging.getLevelName(level)}][{context}]"
+
         if len(message) > 1900:
             blocks = split_text(message, blocksize=1900, code=False)
         else:

bot/meta/sharding.py

@@ -0,0 +1,9 @@
+from .args import args
+from .config import conf
+
+
+shard_number = args.shard or 0
+
+shard_count = conf.bot.getint('shard_count', 1)
+
+sharded = (shard_count > 0)

config/example-bot.conf

@@ -1,6 +1,7 @@
 [DEFAULT]
 log_file = bot.log
 log_channel = 
+error_channel =
 guild_log_channel = 
 
 prefix = !
@@ -10,4 +11,6 @@ owners = 413668234269818890, 389399222400712714
 database = dbname=lionbot
 data_appid = LionBot
 
+shard_count = 1
+
 lion_sync_period = 60