Update protocol tests and handle null values in sparse map serialization

awscli/botocore/parsers.py

@@ -709,7 +709,12 @@ def _handle_map(self, shape, value):
         value_shape = shape.value
         for key, value in value.items():
             actual_key = self._parse_shape(key_shape, key)
-            actual_value = self._parse_shape(value_shape, value)
+            # Treat all maps as sparse during parsing to safely handle null
+            # values that may be present in service responses.
+            if value is None:
+                actual_value = None
+            else:
+                actual_value = self._parse_shape(value_shape, value)
             parsed[actual_key] = actual_value
         return parsed
 
@@ -891,8 +896,7 @@ def _parse_map(self, stream, additional_info):
     def _parse_key_value_pair(self, stream, items):
         key = self.parse_data_item(stream)
         value = self.parse_data_item(stream)
-        if value is not None:
-            items[key] = value
+        items[key] = value
 
     # Major type 6 is tags.  The only tag we currently support is tag 1 for unix
     # timestamps

awscli/botocore/serialize.py

@@ -416,19 +416,27 @@ def _serialize_type_map(self, serialized, value, shape, key):
         map_obj = self.MAP_TYPE()
         serialized[key] = map_obj
         for sub_key, sub_value in value.items():
-            self._serialize(map_obj, sub_value, shape.value, sub_key)
+            if sub_value is None:
+                map_obj[sub_key] = None
+            else:
+                self._serialize(map_obj, sub_value, shape.value, sub_key)
 
     def _serialize_type_list(self, serialized, value, shape, key):
         list_obj = []
         serialized[key] = list_obj
         for list_item in value:
-            wrapper = {}
-            # The JSON list serialization is the only case where we aren't
-            # setting a key on a dict.  We handle this by using
-            # a __current__ key on a wrapper dict to serialize each
-            # list item before appending it to the serialized list.
-            self._serialize(wrapper, list_item, shape.member, "__current__")
-            list_obj.append(wrapper["__current__"])
+            if list_item is None:
+                list_obj.append(None)
+            else:
+                wrapper = {}
+                # The JSON list serialization is the only case where we aren't
+                # setting a key on a dict.  We handle this by using
+                # a __current__ key on a wrapper dict to serialize each
+                # list item before appending it to the serialized list.
+                self._serialize(
+                    wrapper, list_item, shape.member, "__current__"
+                )
+                list_obj.append(wrapper["__current__"])
 
     def _default_serialize(self, serialized, value, shape, key):
         serialized[key] = value
@@ -539,7 +547,14 @@ def _serialize_type_list(self, serialized, value, shape, key):
         else:
             serialized.extend(initial_byte + length.to_bytes(num_bytes, "big"))
         for item in value:
-            self._serialize_data_item(serialized, item, shape.member)
+            if item is None:
+                serialized.extend(
+                    self._get_initial_byte(
+                        self.FLOAT_AND_SIMPLE_MAJOR_TYPE, 22
+                    )
+                )
+            else:
+                self._serialize_data_item(serialized, item, shape.member)
 
     def _serialize_type_map(self, serialized, value, shape, key):
         length = len(value)
@@ -555,7 +570,14 @@ def _serialize_type_map(self, serialized, value, shape, key):
             serialized.extend(initial_byte + length.to_bytes(num_bytes, "big"))
         for key_item, item in value.items():
             self._serialize_data_item(serialized, key_item, shape.key)
-            self._serialize_data_item(serialized, item, shape.value)
+            if item is None:
+                serialized.extend(
+                    self._get_initial_byte(
+                        self.FLOAT_AND_SIMPLE_MAJOR_TYPE, 22
+                    )
+                )
+            else:
+                self._serialize_data_item(serialized, item, shape.value)
 
     def _serialize_type_structure(self, serialized, value, shape, key):
         if key is not None: