How to implement insert for OrderedDict in python 3

Question

I want to insert an item into an OrderedDict at a certain position. Using the gist of this SO answer i have the problem that it doesn't work on python 3.

This is the implementation used

from collections import OrderedDict

class ListDict(OrderedDict):

    def __init__(self, *args, **kwargs):
        super(ListDict, self).__init__(*args, **kwargs)

    def __insertion(self, link_prev, key_value):
        key, value = key_value
        if link_prev[2] != key:
            if key in self:
                del self[key]
            link_next = link_prev[1]
            self._OrderedDict__map[key] = link_prev[1] = link_next[0] = [link_prev, link_next, key]
        dict.__setitem__(self, key, value)

    def insert_after(self, existing_key, key_value):
        self.__insertion(self._OrderedDict__map[existing_key], key_value)

    def insert_before(self, existing_key, key_value):
        self.__insertion(self._OrderedDict__map[existing_key][0], key_value)

Using it like

ld = ListDict([(1,1), (2,2), (3,3)])
ld.insert_before(2, (1.5, 1.5))

gives

File "...", line 35, in insert_before
    self.__insertion(self._OrderedDict__map[existing_key][0], key_value)
AttributeError: 'ListDict' object has no attribute '_OrderedDict__map'

It works with python 2.7. What is the reason that it fails in python 3? Checking the source code of the OrderedDict implementation shows that self.__map is used instead of self._OrderedDict__map. Changing the code to the usage of self.__map gives

AttributeError: 'ListDict' object has no attribute '_ListDict__map'

How come? And how can i make this work in python 3? OrderedDict uses the internal __map attribute to store a doubly linked list. So how can i access this attribute properly?

Answer 1

I'm not sure you wouldn't be better served just keeping up with a separate list and dict in your code, but here is a stab at a pure Python implementation of such an object. This will be an order of magnitude slower than an actual OrderedDict in Python 3.5, which as I pointed out in my comment has been rewritten in C.

"""
A list/dict hybrid; like OrderedDict with insert_before and insert_after
"""
import collections.abc


class MutableOrderingDict(collections.abc.MutableMapping):
    def __init__(self, iterable_or_mapping=None, **kw):
        # This mimics dict's initialization and accepts the same arguments
        # Of course, you have to pass an ordered iterable or mapping unless you
        # want the order to be arbitrary. Garbage in, garbage out and all :)
        self.__data = {}
        self.__keys = []
        if iterable_or_mapping is not None:
            try:
                iterable = iterable_or_mapping.items()
            except AttributeError:
                iterable = iterable_or_mapping
            for key, value in iterable:
                self.__keys.append(key)
                self.__data[key] = value
        for key, value in kw.items():
            self.__keys.append(key)
            self.__data[key] = value

    def insert_before(self, key, new_key, value):
        try:
            self.__keys.insert(self.__keys.index(key), new_key)
        except ValueError:
            raise KeyError(key) from ValueError
        else:
            self.__data[new_key] = value

    def insert_after(self, key, new_key, value):
        try:
            self.__keys.insert(self.__keys.index(key) + 1, new_key)
        except ValueError:
            raise KeyError(key) from ValueError
        else:
            self.__data[new_key] = value

    def __getitem__(self, key):
        return self.__data[key]

    def __setitem__(self, key, value):
        self.__keys.append(key)
        self.__data[key] = value

    def __delitem__(self, key):
        del self.__data[key]
        self.__keys.remove(key)

    def __iter__(self):
        return iter(self.__keys)

    def __len__(self):
        return len(self.__keys)

    def __contains__(self, key):
        return key in self.__keys

    def __eq__(self, other):
        try:
            return (self.__data == dict(other.items()) and
                    self.__keys == list(other.keys()))
        except AttributeError:
            return False

    def keys(self):
        for key in self.__keys:
            yield key

    def items(self):
        for key in self.__keys:
            yield key, self.__data[key]

    def values(self):
        for key in self.__keys:
            yield self.__data[key]

    def get(self, key, default=None):
        try:
            return self.__data[key]
        except KeyError:
            return default

    def pop(self, key, default=None):
        value = self.get(key, default)
        self.__delitem__(key)
        return value

    def popitem(self):
        try:
            return self.__data.pop(self.__keys.pop())
        except IndexError:
            raise KeyError('%s is empty' % self.__class__.__name__)


    def clear(self):
        self.__keys = []
        self.__data = {}

    def update(self, mapping):
        for key, value in mapping.items():
            self.__keys.append(key)
            self.__data[key] = value

    def setdefault(self, key, default):
        try:
            return self[key]
        except KeyError:
            self[key] = default
            return self[key]

    def __repr__(self):
        return 'MutableOrderingDict(%s)' % ', '.join(('%r: %r' % (k, v)
                                                      for k, v in self.items()))

I ended up implementing the whole collections.abc.MutableMapping contract because none of the methods were very long, but you probably won't use all of them. In particular, __eq__ and popitem are a little arbitrary. I changed your signature on the insert_* methods to a 4-argument one that feels a little more natural to me. Final note: Only tested on Python 3.5. Certainly will not work on Python 2 without some (minor) changes.

Answer 2

Trying out the new dict object in 3.7 and thought I'd try to implement what Two-Bit Alchemist had done with his answer but just overriding the native dict class because in 3.7 dict's are ordered.

''' Script that extends python3.7 dictionary to include insert_before and insert_after methods. '''
from sys import exit as sExit

class MutableDict(dict):
    ''' Class that extends python3.7 dictionary to include insert_before and insert_after methods. '''

    def insert_before(self, key, newKey, val):
        ''' Insert newKey:value into dict before key'''
        try:
            __keys = list(self.keys())
            __vals = list(self.values())

            insertAt = __keys.index(key)

            __keys.insert(insertAt, newKey)
            __vals.insert(insertAt, val)

            self.clear()
            self.update({x: __vals[i] for i, x in enumerate(__keys)})

        except ValueError as e:
            sExit(e)

    def insert_after(self, key, newKey, val):
        ''' Insert newKey:value into dict after key'''
        try:
            __keys = list(self.keys())
            __vals = list(self.values())

            insertAt = __keys.index(key) + 1

            if __keys[-1] != key:
                __keys.insert(insertAt, newKey)
                __vals.insert(insertAt, val)
                self.clear()
                self.update({x: __vals[i] for i, x in enumerate(__keys)})
            else:
                self.update({newKey: val})

        except ValueError as e:
            sExit(e)

A little testing:

 In: v = MutableDict([('a', 1), ('b', 2), ('c', 3)])
Out: {'a': 1, 'b': 2, 'c': 3}

 In: v.insert_before('a', 'g', 5)
Out: {'g': 5, 'a': 1, 'b': 2, 'c': 3}

 In: v.insert_after('b', 't', 5)
Out: {'g': 5, 'a': 1, 'b': 2, 't': 5, 'c': 3}

Edit: I decided to do a little benchmark test to see what kind of performance hit this would take. I will use from timeit import timeit

Get a baseline. Create a dict with arbitrary values.

 In: timeit('{x: ord(x) for x in string.ascii_lowercase[:27]}', setup='import string', number=1000000)
Out: 1.8214202160015702

See how much longer it would take to initialize the MutableDict with the same arbitrary values as before.

 In: timeit('MD({x: ord(x) for x in string.ascii_lowercase[:27]})', setup='import string; from MutableDict import MutableDict as MD', number=1000000)
Out: 2.382507269998314

1.82 / 2.38 = 0.76. So if I'm thinking about this right MutableDict is 24% slower on creation.

Lets see how long it takes to do an insert. For this test I'll use the insert_after method as it is slightly bigger. Will also look for a key close to the end for insertion. 't' in this case.

 In: timeit('v.insert_after("t", "zzrr", ord("z"))', setup='import string; from MutableDict import MutableDict as MD; v = MD({x: ord(x) for x in string.ascii_lowercase[:27]})' ,number=1000000)
Out: 3.9161406760104

2.38 / 3.91 = 0.60, 40% slower inserting_after than it's initialization. Not bad on a small test of 1 million loops. For a comparison in time relation we'll test this:

 In: timeit('"-".join(map(str, range(100)))', number=1000000)
Out: 10.342204540997045

Not quite an apples to apples comparison but I hope these tests will aid you in your(reader not necessarily OP) decision to use or not use this class in your 3.7 projects.

Answer 3

Since Python 3.2, move_to_end can be used to move items around in an OrderedDict. The following code will implement the insert functionality by moving all items after the provided index to the end.

Note that this isn't very efficient and should be used sparingly (if at all).

def ordered_dict_insert(ordered_dict, index, key, value):
    if key in ordered_dict:
        raise KeyError("Key already exists")
    if index < 0 or index > len(ordered_dict):
        raise IndexError("Index out of range")

    keys = list(ordered_dict.keys())[index:]
    ordered_dict[key] = value
    for k in keys:
        ordered_dict.move_to_end(k)

There are obvious optimizations and improvements that could be made, but that's the general idea.

Answer 4

from collections import OrderedDict

od1 = OrderedDict([
    ('a', 1),
    ('b', 2),
    ('d', 4),
])

items = od1.items()
items.insert(2, ('c', 3))
od2 = OrderedDict(items)

print(od2)  # OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 4)])