Automate PageSpeed Insights site testing with Node JS

I was debating whether to post this, given the name of my blog—but in the end, I thought what the hell. It’s my blog, I’ll use it to write about everything and anything I make, not just Arduino projects.

For those of you who don’t know, when I’m not tinkering I work as an SEO, optimizing sites to rank higher in Google. As part of this day job—and as a huge fan of all-things automation—I write my own SEO tools and scripts to make life easier on myself.

In this post, I’m going to share one of these scripts that I use to automate Web Core Vitals testing for sites using the PageSpeed Insights API.

The Code

This isn’t intended as a programming tutorial, so I’m not going to put much effort into explaining how it works beyond giving a simple, high-level overview of what it does.

Essentially, it grabs all of the URLs in your sitemap using sitemap-xml-parser, runs each of these through the Pagespeed Insights API twice, once for each strategy type (Desktop/Mobile), using Axios, and creates an excel workbook of the results using excel4node.

For anyone that’s interested, I’ve included the code below. If you don’t care much about that and just want to use it, jump to the “How to use it” section.

import SitemapXMLParser from 'sitemap-xml-parser'
import axios from 'axios'
import xl from 'excel4node'

const apiKey = 'YOUR_API_KEY'
const siteMap = 'YOUR_SITEMAP_URL'

function getDateString() {
    const date = new Date();
    const year = date.getFullYear();
    const month = `${date.getMonth() + 1}`.padStart(2, '0');
    const day =`${date.getDate()}`.padStart(2, '0');
    return `${year}${month}${day}`
}

function exportData(data) {
    const domain = (new URL(data[0][0].URL)).hostname.replace('www.','');
    var wb = new xl.Workbook();
    var ws = wb.addWorksheet(domain);
    const header = wb.createStyle({
        font: { bold: true }
    });
    const good = wb.createStyle({
        fill: {
            type: 'pattern',
            patternType: 'solid',
            bgColor: '#92d050',
            fgColor: '#92d050',
        }
    });
    const needsImprovement = wb.createStyle({
        fill: {
            type: 'pattern',
            patternType: 'solid',
            bgColor: '#ffc000',
            fgColor: '#ffc000',
        }
    });
    const poor = wb.createStyle({
        fill: {
            type: 'pattern',
            patternType: 'solid',
            bgColor: '#ff0000',
            fgColor: '#ff0000',
        }
    });
    ws.cell(2, 1).string("URL").style(header)
    ws.cell(1, 2, 1, 3, 1, 4, 1, 5, true).string("Mobile").style(header)
    ws.cell(1, 6, 1, 7, 1, 8, 1, 9, true).string("Desktop").style(header)
    ws.cell(2, 2).string("FCP").style(header)
    ws.cell(2, 3).string("LCP").style(header)
    ws.cell(2, 4).string("FID").style(header)
    ws.cell(2, 5).string("CLS").style(header)
    ws.cell(2, 6).string("FCP").style(header)
    ws.cell(2, 7).string("LCP").style(header)
    ws.cell(2, 8).string("FID").style(header)
    ws.cell(2, 9).string("CLS").style(header)
    for (const [i,row] of data.entries()) {
        ws.cell((i+3), 1).string(row[0].URL)
        if(row[0].Mobile[0].FCP < 1.8) {
            ws.cell((i+3), 2).number(row[0].Mobile[0].FCP).style(good)
        }
        else if(row[0].Mobile[0].FCP >= 1.8 && row[0].Mobile[0].FCP < 3) {
            ws.cell((i+3), 2).number(row[0].Mobile[0].FCP).style(needsImprovement)
        }
        else if(row[0].Mobile[0].FCP > 3) {
            ws.cell((i+3), 2).number(row[0].Mobile[0].FCP).style(poor)
        }
        if(row[0].Mobile[0].LCP < 2.5) {
            ws.cell((i+3), 3).number(row[0].Mobile[0].LCP).style(good)
        }
        else if(row[0].Mobile[0].LCP >= 2.5 && row[0].Mobile[0].LCP <= 4) {
            ws.cell((i+3), 3).number(row[0].Mobile[0].LCP).style(needsImprovement)
        }
        else if(row[0].Mobile[0].LCP > 4) {
            ws.cell((i+3), 3).number(row[0].Mobile[0].LCP).style(poor)
        }
        if(row[0].Mobile[0].FID < 100) {
            ws.cell((i+3), 4).number(row[0].Mobile[0].FID).style(good)
        }
        else if(row[0].Mobile[0].FID >= 100 && row[0].Mobile[0].FID <= 300) {
            ws.cell((i+3), 4).number(row[0].Mobile[0].FID).style(needsImprovement)
        }
        else if(row[0].Mobile[0].FID > 300) {
            ws.cell((i+3), 4).number(row[0].Mobile[0].FID).style(poor)
        }
        if(row[0].Mobile[0].CLS < 0.1) {
            ws.cell((i+3), 5).number(row[0].Mobile[0].CLS).style(good)
        }
        else if(row[0].Mobile[0].CLS >= 0.1 && row[0].Mobile[0].CLS <= 0.25) {
            ws.cell((i+3), 5).number(row[0].Mobile[0].CLS).style(needsImprovement)
        }
        else if(row[0].Mobile[0].CLS > 0.25) {
            ws.cell((i+3), 5).number(row[0].Mobile[0].CLS).style(poor)
        }
        if(row[0].Desktop[0].FCP < 1.8) {
            ws.cell((i+3), 6).number(row[0].Desktop[0].FCP).style(good)
        }
        else if(row[0].Desktop[0].FCP >= 1.8 && row[0].Mobile[0].FCP < 3) {
            ws.cell((i+3), 6).number(row[0].Desktop[0].FCP).style(needsImprovement)
        }
        else if(row[0].Desktop[0].FCP > 3) {
            ws.cell((i+3), 6).number(row[0].Desktop[0].FCP).style(poor)
        }
        if(row[0].Desktop[0].LCP < 2.5) {
            ws.cell((i+3), 7).number(row[0].Desktop[0].LCP).style(good)
        }
        else if(row[0].Desktop[0].LCP >= 2.5 && row[0].Desktop[0].LCP <= 4) {
            ws.cell((i+3), 7).number(row[0].Desktop[0].LCP).style(needsImprovement)
        }
        else if(row[0].Desktop[0].LCP > 4) {
            ws.cell((i+3), 7).number(row[0].Desktop[0].LCP).style(poor)
        }
        if(row[0].Desktop[0].FID < 100) {
            ws.cell((i+3), 8).number(row[0].Desktop[0].FID).style(good)
        }
        else if(row[0].Desktop[0].FID >= 100 && row[0].Desktop[0].FID <= 300) {
            ws.cell((i+3), 8).number(row[0].Desktop[0].FID).style(needsImprovement)
        }
        else if(row[0].Desktop[0].FID > 300) {
            ws.cell((i+3), 8).number(row[0].Desktop[0].FID).style(poor)
        }
        if(row[0].Desktop[0].CLS < 0.1) {
            ws.cell((i+3), 9).number(row[0].Desktop[0].CLS).style(good)
        }
        else if(row[0].Desktop[0].CLS >= 0.1 && row[0].Desktop[0].CLS <= 0.25) {
            ws.cell((i+3), 9).number(row[0].Desktop[0].CLS).style(needsImprovement)
        }
        else if(row[0].Desktop[0].CLS > 0.25) {
            ws.cell((i+3), 9).number(row[0].Desktop[0].CLS).style(poor)
        }
    }
    let shortfilename = `${domain}_${getDateString()}`.substring(0,31);
    wb.write(`${shortfilename}.xlsx`);
    console.log(`Process Completed!`)
}

function getData(url) {
    return new Promise(resolve => {
        const dataMobile = axios.get('https://www.googleapis.com/pagespeedonline/v5/runPagespeed', {
            params: { url: url[0], key: apiKey, strategy: 'mobile' }   
        })
        const dataDesktop = axios.get('https://www.googleapis.com/pagespeedonline/v5/runPagespeed', {
            params: { url: url[0], key: apiKey, strategy: 'desktop' }  
        })
        axios.all([dataMobile, dataDesktop]).then(axios.spread((...responses) => {
            let dataArray = []
            let fcpMobile = parseFloat(((responses[0].data.lighthouseResult.audits["first-contentful-paint"].numericValue)*0.001).toFixed(2))
            let lcpMobile = parseFloat(((responses[0].data.lighthouseResult.audits["largest-contentful-paint"].numericValue)*0.001).toFixed(2))
            let fidMobile = responses[0].data.lighthouseResult.audits["max-potential-fid"].numericValue
            let clsMobile = parseFloat((responses[0].data.lighthouseResult.audits["cumulative-layout-shift"].numericValue).toFixed(2))
            let fcpDesktop = parseFloat(((responses[1].data.lighthouseResult.audits["first-contentful-paint"].numericValue)*0.001).toFixed(2))
            let lcpDesktop = parseFloat(((responses[1].data.lighthouseResult.audits["largest-contentful-paint"].numericValue)*0.001).toFixed(2))
            let fidDesktop = responses[1].data.lighthouseResult.audits["max-potential-fid"].numericValue
            let clsDesktop = parseFloat((responses[1].data.lighthouseResult.audits["cumulative-layout-shift"].numericValue).toFixed(2))
            dataArray.push({"URL": url, "Mobile": [{"FCP": fcpMobile, "LCP": lcpMobile, "FID": fidMobile, "CLS": clsMobile}], "Desktop": [{"FCP": fcpDesktop, "LCP": lcpDesktop, "FID": fidDesktop, "CLS": clsDesktop}]})
            resolve(dataArray)
          })).catch(errors => {
            console.log(`error ${errors}`)
          })
    })
  }

async function getURLs(siteMap) {
    return new Promise(resolve => {
    const options = {
        delay: 3000,
        limit: 5
    }
    let pageArray = []
      const sitemapXMLParser = new SitemapXMLParser(siteMap, options)
      sitemapXMLParser.fetch().then(result => {
        for(var i = 0; i < result.length; i++) {
          var obj = result[i]
          pageArray.push(obj.loc)
        }
        resolve(pageArray)
        })
    })
}
  
async function start() {
    console.log(`Process started...`)
    const pageUrls = await getURLs(siteMap)
    const promises = [];
    for (const [i,url] of pageUrls.entries()) {
        console.log(`Processing ${i+1} of ${pageUrls.length}`)
        await new Promise(resolve => setTimeout(resolve, 2000));
        promises.push(getData(url));
    }
    Promise.all(promises)
    .then(function(results) {
        exportData(results)
    })
}

start()

How to use it

While it might look a little daunting to anybody with no coding experience, it’s really not that difficult, being pretty easy to get up and running. I’ve outlined the steps you’ll need to follow below:

1. If you don’t already have Node on your machine, you’ll need to install it. You can grab the installers on the download page of the official NodeJS website.
   
   
   
   
2. You’ll need to get a Key for the PageSpeed Insights API. You can create one by visiting the “Get Started” guide (you’ll also need to create a Google account, if you don’t have one already).
   
   
   
   
   
3. Download the “index.js” and “package.json” files for the script here. Create a new directory on your machine and copy these two files to it.
   
   
4. Open “index.js” in a text editor and enter your own “apiKey” and “siteMap“, then save and close the file.
   
   

   const apiKey = 'YOUR_API_KEY'
   const siteMap = 'YOUR_SITEMAP_URL'

5. Open a terminal, navigate to the directory you just created, and enter “npm install” to install the dependencies.
   
   
6. Once the dependencies are installed, enter “npm start” to run the script.
   
   
7. After the script has finished, you’ll find an excel workbook (.xlsx) in the root of the directory, the format of which will look like the image below:
   
   
   
   

Performance

The PageSpeed Insights API has a rate limit of 240 queries per minute, and a quota of 25,000 calls per day, which means you should be able to process one URL per second.

In practice, I ran into rate limit issues, so slowed this down to one every 2 seconds. If you want to play around with that, you can change the setTimeout in start() function—code snippet below for reference.

await new Promise(resolve => setTimeout(resolve, 2000));

To put that into context, I’m able to process 100 page URLs in around 3-4 minutes with this script.

Summing it up and next steps

While this project is functional, it isn’t yet complete. The next steps for me are to add the ability to email reports from it and get it running on a Raspberry Pi, which I can schedule to run automatically and forget about it in the corner of my desk.

This is much easier said than done at the moment, given that you can’t buy them anywhere! I’ve had to order a clone from AliExpress—once it’s here and I’ve got it setup and running, I’ll likely drop an update, so watch out for that if you’re interested.

If you’ve got any questions, comments, or need a little help getting it working, drop it in the comments and I’ll do my best to respond. I hope someone found this useful, and thanks for reading!