Userscript to Loop over several HTTP Requests and combine the results?

Question

I now know how to load columns, of a table, from an external webpage.

Now I want to expand on that and:

• Fetch tabular data from several pages (rankings by player position).
• Merge it into one master table.

This is the URL (http:...fantasysports.yahoo.com...pos=QB) that the script currently fetches. The columns are the team name and the team's rank for the various positions.

I want to have it iterate over other positions (i.e. WR, RB, TE). This is done by just changing the last 2 letters of the URL to its respective value. I then want to have all this data in a single array where first column is the team name, second column is the ranking # for the QB position, third column is ranking # for WR position, and so on.

My plan was to put the GM_xmlhttpRequest in a for loop that iterates over the different position names.
My code is able to display separate tables for the positions but for some reason they are not in order.

Another issue I ran into was the variable scope. Currently newStatTable is defined inside the parseResponse function but I am unable to access it from outside the function. I tried removing var from var newStatTable to make it a global but it did not work.

Here is a sample output array that I am looking for (Note: I just randomly selected these ranks):

TeamName                QB   WR   TE    and so on...
---                     --   --   --
Jacksonville Jaguars    1    6    28
Sanfrancisco 49ers      4    2    32
Seattle Seahawks        31   5    10

Here is my attempt.

// ==UserScript==
// @name        _Grab stuff of a *static*, third-party web site.
// @include  http://football.fantasysports.yahoo.com/*
// @include  https://football.fantasysports.yahoo.com/*
// @require     http://ajax.googleapis.com/ajax/libs/jquery/2.1.0/jquery.min.js
// @grant       GM_xmlhttpRequest
// ==/UserScript==

pos = ["QB", "WR", "RB", "TE", "K", "DEF"];
for (x in pos) {
    GM_xmlhttpRequest ( {
        method:     "GET",
        url:        "http://football.fantasysports.yahoo.com/f1/326198/pointsagainst?pos=" + pos[x],
        onload:     parseResponse,
        onerror:    function (e) { console.error ('**** error ', e); },
        onabort:    function (e) { console.error ('**** abort ', e); },
        ontimeout:  function (e) { console.error ('**** timeout ', e); }
    } );
}
function parseResponse (response) {
    var parser  = new DOMParser ();
    var ajaxDoc         = parser.parseFromString (response.responseText, "text/html");
    var statRows        = ajaxDoc.querySelectorAll ("#statTable0 > tbody > tr");
    var newStatTable    = $(statRows).map ( function () {
        var tblRow      = $(this);
        var teamRank    = parseInt (tblRow.find (".rank-indicator").text().trim(), 10);
        var teamName    = tblRow.find ("td:eq(1)").text().trim().split(" vs")[0];

        return [ [teamName, teamRank] ]; //Return Teamname, Rank #
    } ).get ();

    console.log (newStatTable);
}

Answer 1

AJAX, is asynchronous by the very definition. This means that the returned data will come back in just about any order, and the data will not be available outside the callback function(s) unless you explicitly copy it out.

So, for multi-page mashups, you must have some way to collate the data as necessary. (In your case you can collate on team name.)

A general approach is to:

1. Create an Associative Array to hold the data.
2. Pass a context in each AJAX call.
3. Use that context to control how the AJAX callback function parses the data and collates it into the overall array.
4. Monitor the status of the AJAX calls and only do final processing when they are all complete.

Here's what it looks like in a userscript:

// ==UserScript==
// @name        _Mash up tables from several *static*, third-party, web pages.
// @match       *://football.fantasysports.yahoo.com/*
// @require     http://ajax.googleapis.com/ajax/libs/jquery/2.1.0/jquery.min.js
// @grant       GM_xmlhttpRequest
// ==/UserScript==
const playerPositions   = ["QB", "WR", "RB", "TE", "K", "DEF"];
const numPositions      = playerPositions.length;
const baseURL           = "http://football.fantasysports.yahoo.com/f1/326198/pointsagainst?pos=";
var rankingsTable       = {};  //-- This will be filled by the AJAX parser.
var numPagesFetched     = 0;

for (var J in playerPositions) {
    GM_xmlhttpRequest ( {
        method:     "GET",
        url:        baseURL + playerPositions[J],
        context:    playerPositions[J],
        onload:     parseResponse,
        onerror:    function (e) { console.error ('**** error ', e); },
        onabort:    function (e) { console.error ('**** abort ', e); },
        ontimeout:  function (e) { console.error ('**** timeout ', e); }
    } );
}
function parseResponse (response) {
    var playerPosition  = response.context;
    var parser          = new DOMParser ();
    var ajaxDoc         = parser.parseFromString (response.responseText, "text/html");
    var statRows        = ajaxDoc.querySelectorAll ("#statTable0 > tbody > tr");
    var newStatTable    = $(statRows).map ( function () {
        var tblRow          = $(this);
        var teamRank        = parseInt (tblRow.find (".rank-indicator").text().trim(), 10);
        var teamName        = tblRow.find ("td:eq(1)").text().trim().split(" vs")[0];

        return [ [teamName, teamRank] ];
    } ).get ();

    numPagesFetched++;
    console.log ('Fetched page ' + numPagesFetched + ' of ' + numPositions + '.');

    /*--- Now loop over the fetched rows and collate them into the master table, depending
          on playerPosition.
    */
    var columnIdx       = playerPositions.indexOf (playerPosition);

    for (var K in newStatTable) {
        var teamName        = newStatTable[K][0];
        var teamRank        = newStatTable[K][1];
        var teamStats       = rankingsTable[teamName]  ||  new Array (numPositions);

        teamStats[columnIdx]    = teamRank;
        rankingsTable[teamName] = teamStats;
    }

    if (numPagesFetched === numPositions) {
        displayFinalResult ();
    }
}

function displayFinalResult () {
    var sortedTeamNames = Object.keys (rankingsTable).sort ( function (zA, zB) {
        return zA.localeCompare (zB);
    } );

    const namePadStr    = new Array (25).join (' ');
    console.log (
        'Team                       Ranks QB, WR, RB, TE, K, DEF\n' +
        '------------------------   ------------------------------'
    );
    for (var J in sortedTeamNames) {
        var teamName    = sortedTeamNames[J];
        if (rankingsTable.hasOwnProperty (teamName) ) {
            console.log (
                (teamName + namePadStr).slice (0, 24) + '  ', rankingsTable[teamName]
            );
        }
    }
}

We're going to get a large cut of your FF winnings, right? ;)