Asynchronously generate list of reverse geocoded addresses without API rate limiting

Question

I'm generating test/development dummy data with a node script in my Mongo database (using Mongoose) which includes Geolocation coordinates. (sets of lat/lon). Schema follows:

    location: {
      type: {
        type: String,
        enum: ["Point"], // 'location.type' must be 'Point'
        default: "Point",
      },
      coordinates: {
        type: [Number],
        required: true,
      },
      geocoded: {
        type: String, // this has to be done with an external API
      },
    },

For that reason, I have an external (paid) Reverse Geocoding API which I want/need to call for each document/set of coordinates. The Geocoding API though has a rate limiter so I'm hitting 429 - too many requests. I'm looking for a clean and simple solution to run my requests sequentially and add a throttling/waiting time ( for a specified number of milliseconds ) after each HTTP request.

messageSchema.pre("insertMany", async function save(next, docs) {
      docs.map(async (doc) => { // now I understand I should replace map with for ... of or for ... in
        [err, response] = await to(
            reverseGeocode(
              doc.location.coordinates[0],
              doc.location.coordinates[1]
            )
        );
        if (err) {
          next(err);
        }
        doc.location.geocoded = response;
      });
    });

The reverseGeocode signature:

  reverseGeocode: (lon, lat) =>
    axios({
      baseURL: "https://eu1.locationiq.com/",
      url: "v1/reverse.php",
      params: {
        lat,
        lon,
        key: geocodeKey,
      },
    }).then((response) => response.data),

Answer 1

I use this library for throttling requests. You simply tell it what the rate limit of the API is, and you can call it as fast as you want and it will automatically space the requests out over time for you.

If you don't want another dependency, then to make your solution work you need to use a for loop. map will always execute as fast as it possibly can.

const wait = (time) => {
  return new Promise((resolve) => {
    setTimeout(resolve, time);
  });
}

messageSchema.pre("insertMany", async function(next, docs) {
  for(let i in docs) {
    const doc = docs[i];
    await wait(3000); // milliseconds to space requests out by.
    const response = await reverseGeocode(
      doc.location.coordinates[0],
      doc.location.coordinates[1]
    );
  }
  console.log(this);
});

Answer 2

There's no need to "fork" the process as nodejs has native async promise support.

The best solution is actually to add delays to your code to match the limit. Assuming this is too cumbersome and the fact that this is only for development purposes here is a quick brute force example:

Note that waiting on a promise does not block the process as synchronous code would.

async function getGeoCode(doc) {
    return reverseGeocode(
        doc.location.coordinates[0],
        doc.location.coordinates[1]
    )
}

const randomSleep = [1000, 2000, 3000, 4000, 5000, 6000];

messageSchema.pre("insertMany", async function save(next, docs) {
    let sleep = false;
    docs.map(async (doc) => {
        
        
        while (sleep) {
            const randomSleep = randomSleep[Math.floor(Math.random() * randomSleep.length)];

            await new Promise(function (resolve) {
                setTimeout(resolve, 2000 + randomSleep)
            });
        }
        
        let [err, response] = await getGeoCode(doc);
        if (err) {
            if (err.statusCode !== 429) {
                throw new Error(err);
            }

            sleep = true;

            while (err && err.statusCode === 429) {
                const randomSleep = randomSleep[Math.floor(Math.random() * randomSleep.length)];

                await new Promise(function (resolve) {
                    setTimeout(resolve, 2000 + randomSleep)
                });
                
                [err, response] = await getGeoCode(doc);
            }
            sleep = false;
        }

    });
    console.log(this);
});