0

I am working in a program that has below steps:

  1. fetch user data from Database(it has more than 2000 users
  2. get list of roles for all users( each user might have roles)
  3. send mail to those users which has more roles

Running them synchronously will take lot of time. It takes more than one hour. I am making multiple axios calls to fetch data from server. I want the logic to wait unit all these jobs are completed before moving into next method to trigger mail.

I have tried Promise.all, but I can pass only one parameter with same name. In my case I am making axios calls in for loop with same variable name.

const axios = require("axios");
const https = require("https");

async function fetch_finaldata(req, user_records, token) {

    try {
        // get list of available users from system
        let res_usercount = await getuser_count(req);

        do {
            // get users 200 each to save time
            let res_user = getdemousers(req, user_records);

            // get list of roles for each user
            let res_catalogs = getusercatalogs(req, user_records, res1);

            // increase the counter to read next 200 records    
            skip = skip + 200;
        } while (skip < res_usercount.data);

        // get user count
        async function getuser_count(req) {
            let usercount = {
                method: "GET",
                url: "*****************",
                headers: {
                    Authorization: "***************",
                },
            };

            let usercount_res = axios(usercount);
            return usercount_res;
        }

        // get users
        async function getdemousers(req, user_records) {
            let demoUserParameters = {
                method: "GET",
                url: `*******$skip=${skip}&$top=200&$orderby=UserName asc&$select=UserID,UserName,FirstName,LastName,EmailAddress,to_AssignedBusinessRoles&$expand=to_AssignedBusinessRoles&saml2=disabled&$format=json`,
                headers: {
                    Authorization: "**********************",
                },
            };

            let demoUsersResponse = await axios(demoUserParameters);
            return demoUsersResponse;
        }

        //get user role information
        async function getusercatalogs(req, user_records, res_user) {
            try {
                // loop through each user and fetch relavant details
                for (let i = 0; i < filterusers.length; i++) {
                    // console.log(filterusers[i]).UserName;
                    let user = filterusers[i];
                    let roles = user.to_AssignedBusinessRoles.results;
                    try {
                        for (let k = 0; k < roles.length; k++) {
                            let catalogparams = {
                                method: "GET",
                                url: `******************`,
                                headers: {
                                    Authorization: "************",
                                },
                            };
                            try {
                                // make an axios call to fetch relevant details
                                getcatalogs = axios(catalogparams);
                                getcatalogs.then((getcatalogs) => {
                                    console.log("Catalog stage reached ");
                                    console.log(getcatalogs.data.d.results);
                                    let catalogs = getcatalogs.data.d.results;

                                    try {
                                        for (let t = 0; t < catalogs.length; t++) {
                                            let tileparams = {
                                                method: "GET",
                                                url: `************************`,
                                                headers: {
                                                    Authorization: "**************************",
                                                },
                                            };

                                            try {
                                                // make an axios call to fetch relevant details
                                                let tiles_res = axios(tileparams);
                                                tiles_res.then((tiles_res) => {
                                                    if (tiles_res.data > 0) {
                                                        console.log("Tile count is " + tiles_res.data);
                                                    }
                                                });
                                            } catch (error) {
                                                console.log(error.message);
                                            }
                                        }
                                    } finally {
                                    }
                                });
                            } catch (error) {
                                console.log(error.message);
                            }
                        }
                    } catch (error) {
                        console.log(error.message);
                    }
                }
            } 
    }
    } catch (error) {
        console.log(error.message);
    }
}

exports.fetch_finaldata = fetch_finaldata;

Can you help me solve this?

Krzysztof Madej
  • 13,194
  • 6
  • 28
  • 50
  • For that many requests, you will want to run N operations in parallel at a time where N is some smallish number, far less than all your requests. This will get you some parallelization and therefore improvement in overall throughput, but will not over tax resources locally on on the target servers by attempting launch a zillion requests all in flight at the same time. My goto way to do that is `mapConcurrent()` [here](https://stackoverflow.com/questions/46654265/promise-all-consumes-all-my-ram/46654592#46654592). – jfriend00 Jul 28 '20 at 05:07
  • Thanks for your reply, i will try it – syed imran Jul 28 '20 at 21:19
  • 1
    Wait, is this a clientside program? It seems rather weird to talk to the database using a paginated http api, and to not let the database do the cross-joining and filtering of the data. It seems that your whole code could be written in about 5 lines of SQL. – Bergi Jul 28 '20 at 21:27

0 Answers0