Using promise.all to make external api calls in node js

Question

I am using the request-promise npm package to make multiple api calls in nodeJS. In order for me to get the data I require, I would have to cycle through all categories and then in each category cycle through the array of products to get the product info. Sample category response is as below

{
        "success": true,
        "response": [
          {
            "category_id": 9,
            "name": "Leather Wheeled luggage",
            "products": [
              "TL141911",
              "TL141888"
            ],
            "parent_category_id": 34
          },
          {
            "category_id": 10,
            "name": "Leather Luggage Weekender Duffles bags",
            "products": [
              "TL141794",
              "TL141658"

            ],
            "parent_category_id": 34
          }
        }

Because I have to loop through and make api calls, I am trying to use Promise.all, but it is not producing the correct results, it is throwing me a 404 not found error, can someone please help me where i am going wrong here? below is what i have tried

const rp = require('request-promise');

const requestUrl = "https://stage.tuscanyleather.it/api/v1/";
const categoryRequestUrl = requestUrl + 'categories';
let tuscanApiOptions = {
    uri: categoryRequestUrl,
    headers: {
        'Authorization': 'Bearer asd343'
    },
    method: 'Get',
    json: true 
};

rp(tuscanApiOptions)
    .then((categResponse) => {
        //res.end(categResponse);  
        let ps = [];
        categResponse.response.forEach((category) => {
            if (category.products !== undefined) {
                category.products.forEach((product) =>{
                    let productApiOptions = {
                        uri: requestUrl + `product-info?code=${product}`,
                        headers: {
                            'Authorization': 'Bearer asd343'
                        },
                        method: 'Get',
                        json: true 
                    };
                    ps.push(rp(productApiOptions));
                });
                Promise.all(ps)
                    .then((values) => {
                        console.log(values); //This is where things are going wrong
                  })
                    .catch((err) =>{
                        console.log(JSON.stringify(err,null,2));
                    })
            }

        })

    })
    .catch((error) => {
        console.log(error);
        //res.status(error.statusCode).send(error.error.error_description);
    });

Answer 1

This code should help you to debug the 404s:

const rp = require('request-promise');

const requestUrl = "https://stage.tuscanyleather.it/api/v1/";
const categoryRequestUrl = `${requestUrl}categories`;
const tuscanApiOptions = {
  uri: categoryRequestUrl,
  headers: {
    'Authorization': 'Bearer asd343'
  },
  method: 'Get',
  json: true 
};

// Promise of flat array of products (loses category data)
const productsP = rp(tuscanApiOptions))
  .then(({response}) => response
    .map(category => category.products) // array of arrays
      .flat() // flatten to array of products
      .filter(p => !!p) // remove empty 
  )

const prodUrl = `${requestUrl}product-info?code=`

const productDataP = productsP.then(products => 
  products.map(product => 
    rp({
      ...tuscanApiOptions,
      uri: `${prodUrl}${product}` 
    }).catch(e => `Error fetching ${prodUrl}${product} ${e.message}`)
  )
)

// Wait for all requests to resolve with data or error. 
// resultsP is an array of Promise<productdata|error>
const resultsP = Promise.all(productsDataP)

resultsP.then(console.log)             

A couple of suggestions, take 'em or leave 'em:

Take Array.forEach out of your programming vocabulary. It will cause you more problems than it solves, because it is side-effective (essentially a loop). Use Array.map to return a new array, then operate on that. Array.map separates the state machine from the data transformation, forEach mixes them together.

Don't use let or var for things that don't mutate. That tells other programmers and the machine not to rely on the value. Use const.

See this article for a more in-depth explanation.

It might be easier to do this with async/await, at least to debug. You have data structure traversal and async monads (Promise) you are grappling with here, and a 404 you need to debug. awaiting a promise extends the value out of the monad context, so it removes one layer of complexity.

When you have an array of Promises, you await Promise.all(arrayP) to get an array of the unwrapped values.

const rp = require('request-promise');

const requestUrl = "https://stage.tuscanyleather.it/api/v1/";
const categoryRequestUrl = `${requestUrl}categories`;
const tuscanApiOptions = {
  uri: categoryRequestUrl,
  headers: {
    'Authorization': 'Bearer asd343'
  },
  method: 'Get',
  json: true 
};

async function main() {
  const categories = await rp(tuscanApiOptions)
  const products = categories.response
    .map(category => category.products) 
    .flat() 
    .filter(p => !!p) // remove undefined

  console.log(products) // should be flat array of products

  const prodUrl = `${requestUrl}product-info?code=`

  const requestsP = products
    .map(product => 
      rp({
        ...tuscanApiOptions,
        uri: `${prodUrl}${product}` 
      }).catch(e => `Error fetching ${prodUrl}${product} ${e.message}`)
  )

  const results = await Promise.all(requestsP)

  console.log(results)
}

main()