python - Discord.gateway warning "Shard ID None heartbeat blocked for more than 10 seconds." while using pandas

Question

So I've made a discord bot using discord.py in python and have been running it for some time. However, recently the bot has randomly started to die. So I added the logging library to my program to try and find out what was happening and I got this log this morning:

https://pastebin.com/s5yjQMs7

This error traceback goes on forever referencing multiple pandas files. My discord bot code:

# Import libraries
import asyncio
import random
import AO3
import pandas as pd
from discord.ext import commands
import logging


# Function to setup the dataframe
def dataframeSetup():
    # Create the dataframe
    df = pd.read_csv(
        "https://docs.google.com/spreadsheets/d/16QtBJEtvV5a5DheR78x5AsoVA5b2DpXD1mq-x3lCFiA/export?format=csv",
        names=["NaN", "Title", "Author", "Ship(s)", "Type", "Series", "Status", "Smut", "No of words", "No of chapters",
               "Link"])
    # Remove first two lines
    df = df.iloc[2:]
    # Remove the first column
    df.drop("NaN", axis=1, inplace=True)
    # Create variable to store the index of the first empty row
    firstEmptyRow = 0
    # Iterate over every row
    for index, row in df.iterrows():
        # Test if every cell is empty
        if row.isnull().all():
            # Set firstEmptyRow to the index (it is minus 2 because the index of the dataframe starts at 2)
            firstEmptyRow = index - 2
            break
    # Return the final dataframe
    return df.iloc[0:firstEmptyRow]

# Function to make random quotes
def quoteMaker(df):
    # Grab a random fic
    randomFic = df.iloc[random.randint(2, len(df))]
    # Create AO3 session
    ao3Session = AO3.Session("username", "password")
    # Create work object
    work = AO3.Work(AO3.utils.workid_from_url(randomFic["Link"]), ao3Session)
    # Get chapter amount
    chapterAmount = work.chapters
    # Get chapter text for a random chapter
    randomChapter = random.randint(1, chapterAmount)
    randomChapterText = work.get_chapter_text(randomChapter)
    # Convert the chapter text into a list
    textList = list(filter(None, randomChapterText.split("
")))
    # Return random string
    return textList[random.randint(0, len(textList) - 1)], work, randomChapter, ao3Session

# Function to create trivia
def triviaMaker(triviaDone):
    # Test if all trivia questions have been done
    if len(triviaDone) == len(df1):
        # They've all been done, so clear the list and start again
        triviaDone.clear()
    # Generate a random index and use that to get a random trivia question
    randomIndex = random.randint(0, len(df1)) - 1
    randomTrivia = df1.iloc[randomIndex]
    # Test if the selected trivia question has been done before
    while randomIndex in triviaDone:
        # Trivia has already been done recently so try another one
        randomTrivia = df.iloc[random.randint(0, len(df1))]
    # Add the selected trivia question's index to the list
    triviaDone.append(randomIndex)
    # Return the formatted string as well as the correct index to allow for validation
    return f'''{randomTrivia["Question"]}:
1. {randomTrivia["Option 1"]}
2. {randomTrivia["Option 2"]}
3. {randomTrivia["Option 3"]}
4. {randomTrivia["Option 4"]}''', randomTrivia, randomTrivia["Correct Option"]


def record(work):
    # Create initial array to store results
    ficResults = []
    # Open file and write existing results to ficResults
    with open("QuoteResults.txt", "r") as file:
        for line in file.readlines():
            ficResults.append(line)
    # Test if fic already exists in the results
    found = False
    for count, fic in enumerate(ficResults):
        if str(work.workid) in fic:
            # Fic already exists
            found = True
            break
    # Assign the new result
    if found == True:
        # Increment the result
        ficResults[count] = f"22561831, {int(ficResults[count][-2:]) + 1}
"
    else:
        # Create new result
        ficResults.append(f"{work.workid}, 1
")
    # Write to file
    with open("QuoteResults.txt", "w") as file:
        for result in ficResults:
            file.write(result)


def authorGrab(work, session):
    # Function to grab only the authors
    return session.request(work.url).findAll("h3", {"class": "byline heading"})[0].text.replace("
", "")


# Initialise discord variables
token = "discord token"
client = commands.Bot(command_prefix="!", case_insensitive=True)
# Initialise the dataframe
df = dataframeSetup()
# Initialise trivia variables
df1 = pd.read_csv("Trivia.txt", delimiter="/",
                  names=["Question", "Option 1", "Option 2", "Option 3", "Option 4", "Correct Option"])
# Initialise asked trivia questions list
triviaDone = []
# Initialise channel ID variables using a file
with open("IDs.txt", "r") as file:
    channelIDs = file.read().splitlines()

# Initialise logging
logger = logging.getLogger("discord")
logger.setLevel(logging.DEBUG)
handler = logging.FileHandler(filename="quoteBot.log", encoding="utf-8", mode="a")
handler.setFormatter(logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s'))
logger.addHandler(handler)

# Register !quote command
@client.command()
@commands.cooldown(1, 10, commands.BucketType.default)
async def quote(ctx):
    if ctx.channel.id == int(channelIDs[0][10:]):
        quote = ""
        # Test whether the quote is longer than 10 words
        while len(quote.split()) < 10:
            # Grab quote and related attributes
            quote, work, randomChapter, session = quoteMaker(df)
        # Grab authors
        authors = authorGrab(work, session)
        # Print quote and attributes
        await ctx.channel.send(quote)
        await ctx.channel.send(f"-{work.title} chapter {randomChapter} by {authors}. Link {work.url}")
        record(work)


# Register !trivia command
# This command can only be used once every 60 seconds server-wide
@client.command()
@commands.cooldown(1, 60, commands.BucketType.default)
async def trivia(ctx):
    shortenedIDString = channelIDs[1][11:]
    for id in shortenedIDString.split(", "):
        if ctx.channel.id == int(id):
            # Display trivia question
            triviaString, randomTrivia, correctIndex = triviaMaker(triviaDone)
            await ctx.channel.send(triviaString)

            # Function to check if an answer is correct
            def check(message):
                # Check if answer is correct
                if "!answer" in message.content:
                    return message.content == f"!answer {randomTrivia.iloc[int(correctIndex)]}" or message.content == f"!answer {int(correctIndex)}"

            # Try and except statement to catch timeout error
            try:
                # Wait for user response
                await client.wait_for("message", check=check, timeout=15)
                # User response is correct
                await ctx.channel.send("Correct answer")
            except asyncio.TimeoutError:
                # Time has run out
                await ctx.channel.send("Times up, better luck next time")


# Register empty !answer command
# This is only needed to stop an error being returned
@client.command()
async def answer(ctx):
    return None


# Register !cruzie command
@client.command()
@commands.cooldown(1, 5, commands.BucketType.default)
async def cruzie(ctx):
    # User has types !cruzie so do secret
    await ctx.channel.send("https://giphy.com/gifs/midland-l4FsJgbbeKQC8MGBy")


# Register !murica command
@client.command()
@commands.cooldown(1, 5, commands.BucketType.default)
async def murica(ctx):
    # User has typed !murica so play murica gif
    await ctx.channel.send("https://tenor.com/view/merica-gif-9091003")


# Register !gamer command
@client.command()
@commands.cooldown(1, 5, commands.BucketType.default)
async def gamer(ctx):
    # User has typed !gamer so play gamers gif
    await ctx.channel.send("https://tenor.com/view/hello-gamers-hello-hi-howdy-whats-up-gif-12988393")


# Register !stinky command
@client.command()
@commands.cooldown(1, 5, commands.BucketType.default)
async def stinky(ctx):
    # User has typed !stinky so play srinky gif
    await ctx.channel.send("https://tenor.com/view/monke-uh-oh-stinky-uh-oh-stinky-monke-gif-18263597")


# Run when discord bot has started
@client.event
async def on_ready():
    # Get channel ID for test channel
    channel = client.get_channel("debug channel")
    # Send message to user signalling that the bot is ready
    await channel.send("Running")


# Catch discord errors
@client.event
async def on_command_error(ctx, error):
    if isinstance(error, commands.CommandOnCooldown):
        # CommandOnCooldown error detected
        await ctx.channel.send(f"Command is on cooldown, try again in {round(error.retry_after, 2)} seconds")


# Start discord bot
client.run(token)

If anyone can figure out why this error occurs, that'll be greatly appreciated.

Answer 1

The warning essentially means that your code is blocking for more than x seconds, it blocks the heartbeat and triggers that warning (you can reproduce this with time.sleep(x)). To fix it, you have to run the blocking functions (the panda ones) in a non-blocking way:

import time # To reproduce the error
import typing # For typehinting 
import functools

def blocking_func(a, b, c=1):
    """A very blocking function"""
    time.sleep(a + b + c)
    return "some stuff"


async def run_blocking(blocking_func: typing.Callable, *args, **kwargs) -> typing.Any:
    """Runs a blocking function in a non-blocking way"""
    func = functools.partial(blocking_func, *args, **kwargs) # `run_in_executor` doesn't support kwargs, `functools.partial` does
    return await client.loop.run_in_executor(None, func)


@client.command()
async def test(ctx):
    r = await run_blocking(blocking_func, 1, 2, c=3) # Pass the args and kwargs here
    print(r) # -> "some stuff"
    await ctx.send(r) 

You should run all the blocking functions this way

Another (easier) way would be to simply create a decorator

import functools
import typing
import asyncio


def to_thread(func: typing.Callable) -> typing.Coroutine:
    @functools.wraps(func)
    async def wrapper(*args, **kwargs):
        return await asyncio.to_thread(func, *args, **kwargs)
    return wrapper


@to_thread
def blocking_func(a, b, c=1):
    time.sleep(a + b + c)
    return "some stuff"


await blocking_func(1, 2, 3)

If you're using python <3.9 you should use loop.run_in_executor instead of asyncio.to_thread

def to_thread(func: typing.Callable) -> typing.Coroutine:
    @functools.wraps(func)
    async def wrapper(*args, **kwargs):
        loop = asyncio.get_event_loop()
        wrapped = functools.partial(func, *args, **kwargs)
        return await loop.run_in_executor(None, func)
    return wrapper


@to_thread
def blocking_func(a, b, c=1):
    time.sleep(a + b + c)
    return "some stuff"


await blocking_func(1, 2, 3)