How to recursively find specific key in nested JSON?

Question

I'm trying to pull nested values from a json file. I want to print out each of the values for every "id" key. I think I'm close but can't figure out why the obj type changes from a dict to a list, and then why I'm unable to parse that list. Here is a link to the json I'm working with: http://hastebin.com/ratevimixa.tex

and here is my current code:

#!/usr/bin/env python
#-*- coding: utf-8 -*-

import json

json_data = open('JubJubProductions.json', 'r+')
jdata = json.loads(json_data.read().decode("utf-8"))

def recursion(dict):

    for key, value in dict.items():

        if type(value) == type(dict):
            if key != "paging":
                for key, value in value.items():
                    if isinstance (value,list):
                        print key
                        # place where I need to enter list comprehension?
                if type(value) == type(dict):
                    if key == "id":
                        print " id found " + value
                    if key != "id":
                        print key + " 1st level"
                if key == "id":
                    print key
        else:
            if key == "id":
                print "id found " + value       
if __name__ == '__main__':
    recursion(jdata)

-------------------------------------------------------------------------------------------update

This is now what I'm working with and it'll return a single id value, but not all of them:

#!/usr/bin/env python
#-*- coding: utf-8 -*-

import json

json_data = open('jubjubProductions', 'r+')
jdata = json.loads(json_data.read().decode("utf-8"))

def id_generator(d):
    for k, v in d.items():
        if k == "id":
            yield v
        elif isinstance(v, dict):
            for id_val in id_generator(v):
                yield id_val

if __name__ == '__main__':
    for _ in id_generator(jdata):
        print (_)

Answer 1

The JSON might contain a list of objects, which needs to be searched:

Python 2.7 version:

def item_generator(json_input, lookup_key):
    if isinstance(json_input, dict):
        for k, v in json_input.iteritems():
            if k == lookup_key:
                yield v
            else:
                for child_val in item_generator(v, lookup_key):
                    yield child_val
    elif isinstance(json_input, list):
        for item in json_input:
            for item_val in item_generator(item, lookup_key):
                yield item_val

Python 3.x version:

def item_generator(json_input, lookup_key):
    if isinstance(json_input, dict):
        for k, v in json_input.items():
            if k == lookup_key:
                yield v
            else:
                yield from item_generator(v, lookup_key)
    elif isinstance(json_input, list):
        for item in json_input:
            yield from item_generator(item, lookup_key)

Answer 2

def id_generator(dict_var):
    for k, v in dict_var.items():
        if k == "id":
            yield v
        elif isinstance(v, dict):
            for id_val in id_generator(v):
                yield id_val

This will create an iterator which will yield every value on any level under key "id". Example usage (printing all of those values):

for _ in id_generator(some_json_dict):
    print(_)

Answer 3

A little bit cleaner code (in python 3.x).

def parse_json_recursively(json_object, target_key):
    if type(json_object) is dict and json_object:
        for key in json_object:
            if key == target_key:
                print("{}: {}".format(target_key, json_object[key]))
            parse_json_recursively(json_object[key], target_key)

    elif type(json_object) is list and json_object:
        for item in json_object:
            parse_json_recursively(item, target_key)


json_object = {"key1": "val1", "key2": [{"key3":"val3", "key4": "val4"}, 123, "abc"]}
target_key = "key3"
parse_json_recursively(json_object, target_key) # Ouput key3: val3

Answer 4

Here is a simple recursive function to collect all values from a json document for a given key. Values can be json documents as well. The corresponding values appended to search_result.

def json_full_search(lookup_key, json_dict, search_result = []):
    if type(json_dict) == dict:
        for key, value in  json_dict.items():
            if key == lookup_key:
                search_result.append(value)
            json_full_search(lookup_key, value, search_result)
    elif type(json_dict) == list:
        for element in json_dict:
            json_full_search(lookup_key, element, search_result)
    return search_result

Answer 5

Extension to python 3.x answer: If nested json has similar keys under different list or dictionaries and you want to take first value of it.. below is the generic way:

 def get_value_from_generator(json_input, lookup_key):
        value = list(item_generator(json_input, lookup_key))
        val = value[0] if value else None
        print(f'lookup_key -> value : {val}')
        return val
def item_generator(json_input, lookup_key):
    if isinstance(json_input, dict):
        for k, v in json_input.items():
            print(f'{k} -- {v}')
            if k == lookup_key:
                yield v
            else:
                yield from item_generator(v, lookup_key)
    elif isinstance(json_input, list):
        for item in json_input:
            yield from item_generator(item, lookup_key)

Answer 6

def get_val(j, s, v=None):
for k in j:
    if v == None and k == s:
        return j[k]
    elif v != None and k == s and v == j[k]:
        return True
    elif v != None and k == s and v != j[k]:
        return False
    elif isinstance(j[k], dict):
        return get_val(j[k], s, v)

You can use with for a json list l below,

for l in j:
    r = get_val(l, 'key')
    print(r)

---

for l in j:
    r = get_val(l, 'mac', '00-60-2F-5A-04-51')
    print(r)