How to determine number of dimensions in a JSON object and put all data in an array in javascript?

Question

I am pulling data from an API that is going to return JSON objects of varying lengths and content. It is a mishmash of objects and arrays that can be several levels deep.

I'm trying to get all of the pairs of data and get them into a single, 2D array that I can then put into google sheets using google sheets script.

Here's the code I'm starting with:

function parseTheAPI(rawResponse){
  var theparsedJSONdata = JSON.parse(rawResponse)
  console.log(theparsedJSONdata)
  return theparsedJSONdata
};

Here's the response from the console:

{ '0xabcxyz': 
   { products: [ [Object] ],
     meta: [ [Object], [Object], [Object] ] } }

There's lots more actual data where it says Object. I understand how to get at individual pieces of data once I know the contents of the object. If the contents of the object change, however, the code would break.

I want to dynamically get at all the various information pairs and use them.

I've looked at SO questions like this and this and tried to figure out if I can just see that data. Here's my attempt at using recursion to figure what's inside:

function logJsonLevelOne(parsedJson){
  for(var k in parsedJson){
    if(parsedJson[k] instanceof Object){
      console.log("parsedJsonOne = "+ parsedJson[k])
      var n = Array.isArray(logJsonLevelOne[k])
      logJsonLevelOne(parsedJson[k])
    } else {
      var n = logJsonLevelOne[k] instanceof Array;
      console.log(n)
    }
  }
};

I get some of the data printed, but I don't know how to get to the next level. I don't understand what's coming out of each part of the "if" test. Here's an example of one of the nested objects printed in the console:

{ type: 'claimable',
  category: 'claimable',
  address: '0x00000000123456lkjlkj',
  symbol: 'WMATIC',
  decimals: 18,
  label: 'Claimable WMATIC',
  img: 'networks/polygon/0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270.png',
  price: 1.64,
  balance: 0.03567595026086894,
  balanceRaw: '35675950260868942',
  balanceUSD: 0.05850855842782506 }

How do I figure out what is in the JSON data, at any depth, see if it's an array or object, and put the extracted pairs into a single array?

EDIT:

I'm trying to get this data into two rows in a google sheet. so I can track individual pieces every day. The "price", the "balanceRaw"...

The issue is the user may be participating in different AAVE pools from day to day; i.e. this JSON object might have lots of different information every day.

Here's the raw data from the API:

   {"0x0000123456abcdef":{"products":[{"label":"Aave V2","assets":[{"type":"interest-bearing","category":"deposit","address":"0x27F8D03b3a2196956ED754baDc28D73be8830A6e","symbol":"DAI","decimals":18,"label":"DAI in Aave","img":"networks/polygon/0x8f3cf7ad23cd3cadbd9735aff958023239c6a063.png","protocol":"aave","protocolDisplay":"Aave","protocolSymbol":"AAVE","price":0.999254,"apy":0.027310184786005925,"balanceRaw":"2668910745526108687981","balance":2668.910745526109,"balanceUSD":2666.9197381099466},{"type":"claimable","category":"claimable","address":"0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270","symbol":"WMATIC","decimals":18,"label":"Claimable WMATIC","img":"networks/polygon/0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270.png","price":1.65,"balance":0.042818503811087726,"balanceRaw":"42818503811087730","balanceUSD":0.07065053128829474}],"meta":[]}],"meta":[{"label":"Total","value":2666.990388641235,"type":"dollar"},{"label":"Assets","value":2666.990388641235,"type":"dollar"},{"label":"Debt","value":0,"type":"dollar"}]}}

EDIT 2

I've tried the this code from this SO question:

function flattenJsonObject (parsedJson){
  console.log("test first")
      Object.flatten = function(parsedJson) {
          var result = {};
          function recurse (cur, prop) {
            console.log("test")
              if (Object(cur) !== cur) {
                  result[prop] = cur;
              } else if (Array.isArray(cur)) {
                  for(var i=0, l=cur.length; i<l; i++)
                      recurse(cur[i], prop + "[" + i + "]");
                  if (l == 0)
                     result[prop] = [];
              } else {
                  var isEmpty = true;
                  for (var p in cur) {
                      isEmpty = false;
                      recurse(cur[p], prop ? prop+"."+p : p);
                  }
                  if (isEmpty && prop)
                    result[prop] = {};
              }
          }
          recurse(parsedJson, "");
          console.log(result)
          return result;
      }
}

Something is not working because the second and third console.log are not being printed in the console.

It sounds like you want to flatten your JSON structure before writing the results to a sheet. It really depends on what you want the results to look like (e.g. just the innermost name for each value; or the concatenation of names down through the hierarchy...). If this is what you are interested in, take a look at questions like [this](https://stackoverflow.com/questions/19098797/fastest-way-to-flatten-un-flatten-nested-json-objects) and several others like it on SO. Or, if I have missed the purpose of the question, maybe you can add some example "before" and "after" data? — andrewJames, Jun 04 '21 at 17:08
I see the data sample - thank you. The "flatten" code I linked to does work on your data: `let targetData = JSON.flatten(sourceData);` Does that meet your needs? — andrewJames, Jun 04 '21 at 18:11
@andrewjames I'm getting this error `TypeError: JSON.flatten is not a function`. Does this mean I have to create the function using the SO answer you referenced? — DBWeinstein, Jun 04 '21 at 18:14
Yes - add the code from that answer to your script: `JSON.flatten = function(data) { ... }` - and then you can call that function. — andrewJames, Jun 04 '21 at 18:30
@andrewjames Thank you. I've tried to look up and learn about the part `Object.flatten = function`. What do I search for to read about it. Last question: I assume the variable `data` is an argument that would accept by json data? — DBWeinstein, Jun 04 '21 at 18:39
If your API returns a JSON string containing `{"0x0000123456abcdef"...}`, then you would first need to parse that string to a JavaScript object: `JSON.parse(rawJsonString);`. In this case, `JSON.parse()` is provided for you [by JavaScript](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse). So there is no "extra" code you need to add. So, it becomes: `targetData = JSON.flatten(JSON.parse(yourSourceJsonString));`. — andrewJames, Jun 04 '21 at 18:56
That function from the other question basically does what your question's starting point was: it recursively drills down into each layer of the nested data you provide. It checks whether each new part is an object `{ }`, or an array `[ ]` and so on, and then processes that section accordingly. The [JavaScript reference](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference) plus tutorials are helpful. I have to look at each piece separately before I can begin to grasp the whole thing! — andrewJames, Jun 04 '21 at 19:02
You are welcome to post your own answer, showing how you used that "flatten" function, if you like (assuming it works for you, of course). That could be helpful to future readers wanting to do something similar in the context of Sheets. — andrewJames, Jun 04 '21 at 19:04
@andrewjames will do. I’m struggling with it. If I figure it out, I will post. — DBWeinstein, Jun 04 '21 at 19:26
In order to correctly understand about your goal, using the sample input value of `Here's the raw data from the API:`, can you provide the sample output value you expect? — Tanaike, Jun 04 '21 at 23:26

score 0 · Answer 1 · answered Jun 04 '21 at 20:51

Here is a demo of the flattening function. Its starting point is a string of text, which is the response from your API call:

let sourceData = '{"0x0000123456abcdef":{"products":[{"label":"Aave V2","assets":[{"type":"interest-bearing","category":"deposit","address":"0x27F8D03b3a2196956ED754baDc28D73be8830A6e","symbol":"DAI","decimals":18,"label":"DAI in Aave","img":"networks/polygon/0x8f3cf7ad23cd3cadbd9735aff958023239c6a063.png","protocol":"aave","protocolDisplay":"Aave","protocolSymbol":"AAVE","price":0.999254,"apy":0.027310184786005925,"balanceRaw":"2668910745526108687981","balance":2668.910745526109,"balanceUSD":2666.9197381099466},{"type":"claimable","category":"claimable","address":"0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270","symbol":"WMATIC","decimals":18,"label":"Claimable WMATIC","img":"networks/polygon/0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270.png","price":1.65,"balance":0.042818503811087726,"balanceRaw":"42818503811087730","balanceUSD":0.07065053128829474}],"meta":[]}],"meta":[{"label":"Total","value":2666.990388641235,"type":"dollar"},{"label":"Assets","value":2666.990388641235,"type":"dollar"},{"label":"Debt","value":0,"type":"dollar"}]}}';

JSON.flatten = function(data) {
    var result = {};
    function recurse (cur, prop) {
        if (Object(cur) !== cur) {
            result[prop] = cur;
        } else if (Array.isArray(cur)) {
             for(var i=0, l=cur.length; i<l; i++)
                 recurse(cur[i], prop ? prop+"."+i : ""+i);
            if (l == 0)
                result[prop] = [];
        } else {
            var isEmpty = true;
            for (var p in cur) {
                isEmpty = false;
                recurse(cur[p], prop ? prop+"."+p : p);
            }
            if (isEmpty)
                result[prop] = {};
        }
    }
    recurse(data, "");
    return result;
}

let targetData = JSON.flatten(JSON.parse(sourceData));

console.log(targetData);

The result of running this snippet is data (JS objects) like the following (summarised for brevity):

"0x0000123456abcdef.products.0.assets.0.type": "interest-bearing"
"0x0000123456abcdef.products.0.assets.0.symbol": "DAI"
"0x0000123456abcdef.products.0.assets.0.price": 0.999254
"0x0000123456abcdef.products.0.assets.0.balanceRaw": "2668910745526108687981"

and:

"0x0000123456abcdef.products.0.assets.1.type": "claimable"
"0x0000123456abcdef.products.0.assets.1.symbol": "WMATIC"
"0x0000123456abcdef.products.0.assets.1.price": 1.65
"0x0000123456abcdef.products.0.assets.1.balanceRaw": "42818503811087730"

You can see how the flattening process has built names which show you where in the hierarchy the data came from.

This may not be what you want (they are somewhat cumbersome). So, additional manipulation may be needed to format the names strings into something more friendly.

Than you. I'm screwing this up somehow. When I check the the parsed data by using this function: `function parseTheAPI(rawResponse){ var theparsedJSONdata = JSON.parse(rawResponse) console.log(theparsedJSONdata) return theparsedJSONdata };` I get the following in the console: `{ '0x0000000004': { products: [ [Object] ], meta: [ [Object], [Object], [Object] ] } }`. Prolly something wrong with my API call, correct? — DBWeinstein, Jun 04 '21 at 21:39
Maybe your raw response does not need to be parsed. Try with just `rawResponse` instead of `JSON.parse(rawResponse)`. — andrewJames, Jun 04 '21 at 22:18
doesn't work. I need to re-write everything from scratch to make sure I've got all the steps. I'm able to get at the data with stuff like this `var neededArray = parsedJson[walletAddress].products[0].assets`. I was just hoping to avoid specifics like this and dynamically figure out what data was coming through. — DBWeinstein, Jun 04 '21 at 22:34

Mike Steelson · Answer 2 · 2021-06-05T01:13:09.373

0

Here is what I use to get all pairs :

let result = []; 
function getData(jsonString){
  var data = JSON.parse(jsonString)
  getAllPairs(eval(data),'data')
  return result
}
function getAllPairs(obj,id) {
  const regex = new RegExp('[^0-9]+');
  for (let p in obj) {
    var newid = (regex.test(p)) ? id + '.' + p : id + '[' + p + ']';
    if (obj[p]!=null){
      if (typeof obj[p] != 'object' && typeof obj[p] != 'function'){
        result.push([p, obj[p]]);
      }
      if (typeof obj[p] == 'object') {
        getAllPairs( obj[p], newid );
      }
    }
  }
}

https://docs.google.com/spreadsheets/d/1_didPNsEswVHWygI3DNL3KGlRJgsmxYNqpOJNFLN-xI/copy

edited Jun 05 '21 at 01:13

answered Jun 05 '21 at 01:05

Mike Steelson

14,650
2
5
20

in `getAllPairs` what is the argument `id`? – DBWeinstein Jun 05 '21 at 20:34
it is in fact the key – Mike Steelson Jun 06 '21 at 02:42
id is only use in the recursive internal function, use only getData – Mike Steelson Jun 06 '21 at 03:24

How to determine number of dimensions in a JSON object and put all data in an array in javascript?

2 Answers2