javascript - Troubleshooting delays in node.js fetching loop

Question

I'm experiencing delays and was wondering how to properly troubleshoot/identify issues.

FETCHES:

async function getWebsite() {
    var html = await fetch('https://www.website.com').then(r => r.text());
    const doc = new JSDOM(html).window.document;
    const links = Array.from(doc.querySelectorAll('.card-title a')).map(({ textContent, href }) => ({ textContent, href }));
    return links;
}

THIS READ/SAVE

function readItems() {
    try {
        if (fs.statSync(idFile)) {
            return JSON.parse(fs.readFileSync(idFile));
        }
    } catch {
        return [];
    }
}

function saveItems(items) {
    fs.writeFileSync(idFile, JSON.stringify(items));
}

COMPARE:

setInterval(async function () {
    const links = await getWebsite();
    const linkIDs = links.map(l => {
        parts = l.href.split('/');
        parts.pop();
        return parts.pop();
    });
    var knownItems = readItems();
    const newItems = links.filter((link, i) => {
        return !knownItems.includes(linkIDs[i]);
    });
    console.log("new items found: " + newItems.length);
    if (newItems.length) {
...

How would I add a timestamp on each time this is output AND log it in a file

console.log("new items found: " + newItems.length);   //add timestamp

Basically, I want to verify if the intermittent delays are simply caused by a code issue or a server issue (throttled/firewalled)

question from:https://stackoverflow.com/questions/65948013/troubleshooting-delays-in-node-js-fetching-loop

Answer 1

Q: How to add a timestamp?

A: Date.now()

Q: How to log into a file?

A: The most simple way is using packages, like winstonjs.

Q: How to verify if the intermittent delays?

A: Well, you can try to simulate timeout for fetch API:

const timeout = 200;
try{
    let timer = setTimeout(() => {
        throw new Error('Timeout');
    }, timeout);
    const html = await fetch(...).then(r => r.text());
    clearTimeout(timer);
    return html;
} catch (e) {
    // if timeout, code goes here
    return false;
}

EDIT:

Here is an example for you:

// edit your getWebsite function
async function getWebsite() {
    const timeout = 200;
    try{
        let timer = setTimeout(() => {
            throw new Error('Timeout');
        }, timeout);
        const html = await fetch(...).then(r => r.text());
        clearTimeout(timer);
        const doc = new JSDOM(html).window.document;
        const links = Array.from(doc.querySelectorAll('.card-title a')).map(({ textContent, href }) => ({ textContent, href }));
        return links;
    } catch (e) {
        // if timeout, code goes here
        return false;
    }
}

// and your compare logic should be
async function compare() {
    const links = await getWebsite();
    if (links === false) {
        // if code goes here, there is something went wrong in getWebsite function, like timeout or code issue
        // you can log server being throttled/firewalled here
        compare();  // compare infinitely
        return;
    }

    const linkIDs = links.map(l => {
        parts = l.href.split('/');
        parts.pop();
        return parts.pop();
    });
...
    // compare infinitely
    compare();
}