omitNullValues: response serialization/deserialization, Preference he…

…aders handling.

src/Microsoft.OData.Core/JsonLight/ODataJsonLightPropertySerializer.cs

@@ -90,13 +90,34 @@ internal void WriteTopLevelProperty(ODataProperty property)
                         null /*owningType*/,
                         true /* isTopLevel */,
                         false /* allowStreamProperty */,
+                        false /*omitNullValues */,
                         this.CreateDuplicatePropertyNameChecker());
                     this.JsonLightValueSerializer.AssertRecursionDepthIsZero();
 
                     this.JsonWriter.EndObjectScope();
                 });
         }
 
+        /// <summary>
+        /// This is the backward-compatible method to write property names and value pairs
+        /// without omitting null property values.
+        /// </summary>
+        /// <param name="owningType">The <see cref="IEdmStructuredType"/> of the resource (or null if not metadata is available).</param>
+        /// <param name="properties">The enumeration of properties to write out.</param>
+        /// <param name="isComplexValue">
+        /// Whether the properties are being written for complex value. Also used for detecting whether stream properties
+        /// are allowed as named stream properties should only be defined on ODataResource instances
+        /// </param>
+        /// <param name="duplicatePropertyNameChecker">The DuplicatePropertyNameChecker to use.</param>
+        internal void WriteProperties(
+            IEdmStructuredType owningType,
+            IEnumerable<ODataProperty> properties,
+            bool isComplexValue,
+            IDuplicatePropertyNameChecker duplicatePropertyNameChecker)
+        {
+            this.WriteProperties(owningType, properties, isComplexValue, /*omitNullValues*/ false, duplicatePropertyNameChecker);
+        }
+
         /// <summary>
         /// Writes property names and value pairs.
         /// </summary>
@@ -106,11 +127,13 @@ internal void WriteTopLevelProperty(ODataProperty property)
         /// Whether the properties are being written for complex value. Also used for detecting whether stream properties
         /// are allowed as named stream properties should only be defined on ODataResource instances
         /// </param>
+        /// <param name="omitNullValues">Whether to omit null property values.</param>
         /// <param name="duplicatePropertyNameChecker">The DuplicatePropertyNameChecker to use.</param>
         internal void WriteProperties(
             IEdmStructuredType owningType,
             IEnumerable<ODataProperty> properties,
             bool isComplexValue,
+            bool omitNullValues,
             IDuplicatePropertyNameChecker duplicatePropertyNameChecker)
         {
             if (properties == null)
@@ -125,6 +148,7 @@ internal void WriteProperties(
                     owningType,
                     false /* isTopLevel */,
                     !isComplexValue,
+                    omitNullValues,
                     duplicatePropertyNameChecker);
             }
         }
@@ -177,6 +201,7 @@ private void WriteProperty(
             IEdmStructuredType owningType,
             bool isTopLevel,
             bool allowStreamProperty,
+            bool omitNullValues,
             IDuplicatePropertyNameChecker duplicatePropertyNameChecker)
         {
             WriterValidationUtils.ValidatePropertyNotNull(property);
@@ -235,7 +260,7 @@ private void WriteProperty(
 
             if (value is ODataNullValue || value == null)
             {
-                this.WriteNullProperty(property);
+                this.WriteNullProperty(property, omitNullValues);
                 return;
             }
 
@@ -363,7 +388,8 @@ private void WriteStreamReferenceProperty(string propertyName, ODataStreamRefere
         /// </summary>
         /// <param name="property">The property to write out.</param>
         private void WriteNullProperty(
-            ODataProperty property)
+            ODataProperty property,
+            bool omitNullValues)
         {
             this.WriterValidator.ValidateNullPropertyValue(
                 this.currentPropertyInfo.MetadataType.TypeReference, property.Name, this.Model);
@@ -373,7 +399,7 @@ private void WriteNullProperty(
                 // TODO: Enable updating top-level properties to null #645
                 throw new ODataException("A null top-level property is not allowed to be serialized.");
             }
-            else if (!this.MessageWriterSettings.IgnoreNullValues)
+            else if (!omitNullValues)
             {
                 this.JsonWriter.WriteName(property.Name);
                 this.JsonLightValueSerializer.WriteNullValue();

src/Microsoft.OData.Core/JsonLight/ODataJsonLightReader.cs

@@ -2232,6 +2232,9 @@ private void EndEntry()
 
             if (this.State == ODataReaderState.ResourceStart)
             {
+                // For non-delta response, need to restore omitted null values as required.
+                RestoreOmittedNullValues();
+
                 this.EndEntry(
                     new JsonLightResourceScope(
                         ODataReaderState.ResourceEnd,
@@ -2256,6 +2259,91 @@ private void EndEntry()
             }
         }
 
+        /// <summary>
+        /// For response de-serialization, restore the null values for properties that are omitted
+        /// by the omit-values=nulls preference header.
+        /// </summary>
+        private void RestoreOmittedNullValues()
+        {
+            // Restore omitted null value only when processing response and when
+            // Preference header omit-values=nulls is specified.
+            if (this.jsonLightInputContext.ReadingResponse
+                && this.jsonLightInputContext.MessageReaderSettings.NullValuesOmitted)
+            {
+                IODataJsonLightReaderResourceState resourceState = this.CurrentResourceState;
+                IEdmStructuredType edmStructuredType = resourceState.ResourceType;
+
+                if (resourceState.SelectedProperties == SelectedPropertiesNode.Empty)
+                {
+                    return;
+                }
+                else
+                {
+                    if (resourceState.Resource != null && edmStructuredType != null)
+                    {
+                        ODataResourceBase resource = resourceState.Resource;
+
+                        IEnumerable<IEdmProperty> selectedProperties;
+                        if (resourceState.SelectedProperties == SelectedPropertiesNode.EntireSubtree)
+                        {
+                            selectedProperties = edmStructuredType.DeclaredProperties;
+                        }
+                        else
+                        { // Partial subtree. Combine navigation properties and selected properties at the node with distinct.
+                            selectedProperties =
+                                resourceState.SelectedProperties.GetSelectedNavigationProperties(edmStructuredType)
+                                as IEnumerable<IEdmProperty>;
+                            selectedProperties = selectedProperties.Concat(
+                                resourceState.SelectedProperties.GetSelectedProperties(edmStructuredType).Values)
+                                .Distinct();
+                        }
+
+                        foreach (IEdmProperty currentProperty in selectedProperties)
+                        {
+                            Debug.Assert(currentProperty.Type != null, "currentProperty.Type != null");
+                            if (!currentProperty.Type.IsNullable)
+                            {
+                                // Skip declared properties that are not null-able types.
+                                continue;
+                            }
+
+                            // Response should be generated using case sensitive, matching EDM defined by metadata.
+                            // In order to find potential omitted properties, need to check
+                            // resource properties read and navigation properties read.
+                            if (!resource.Properties.Any(
+                                p => p.Name.Equals(currentProperty.Name, StringComparison.Ordinal))
+                                && !resourceState.NavigationPropertiesRead.Contains(currentProperty.Name))
+                            {
+                                // Add null value to omitted property declared in the type
+                                ODataProperty property = new ODataProperty
+                                {
+                                    Name = currentProperty.Name,
+                                    Value = new ODataNullValue()
+                                };
+
+                                // Mark as processed, will throw if duplicate is detected.
+                                resourceState.PropertyAndAnnotationCollector.MarkPropertyAsProcessed(property.Name);
+
+                                ReadOnlyEnumerable<ODataProperty> readonlyEnumerable =
+                                    resource.Properties as ReadOnlyEnumerable<ODataProperty>;
+                                if (readonlyEnumerable != null)
+                                {
+                                    // For non-entity resource, add the property underneath the read-only enumerable.
+                                    resource.Properties.ConcatToReadOnlyEnumerable("Properties", property);
+                                }
+                                else
+                                {
+                                    // For entity resource, concatenate the property to original enumerable.
+                                    resource.Properties =
+                                        resource.Properties.Concat(new List<ODataProperty>() {property});
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
         /// <summary>
         /// Add info resolved from context url to current scope.
         /// </summary>

src/Microsoft.OData.Core/JsonLight/ODataJsonLightWriter.cs

@@ -331,6 +331,7 @@ protected override void StartResource(ODataResource resource)
                 this.ResourceType,
                 resource.Properties,
                 false /* isComplexValue */,
+                this.ShouldOmitNullValues(),
                 this.DuplicatePropertyNameChecker);
             this.jsonLightResourceSerializer.JsonLightValueSerializer.AssertRecursionDepthIsZero();
 
@@ -1226,6 +1227,7 @@ private void WriteDeltaResourceProperties(IEnumerable<ODataProperty> properties)
                 this.ResourceType,
                 properties,
                 false /* isComplexValue */,
+                false /* omitNullValues */,
                 this.DuplicatePropertyNameChecker);
             this.jsonLightResourceSerializer.JsonLightValueSerializer.AssertRecursionDepthIsZero();
         }

src/Microsoft.OData.Core/Microsoft.OData.Core.cs

@@ -271,6 +271,9 @@ internal sealed class TextRes {
         internal const string HttpHeaderValueLexer_FailedToReadTokenOrQuotedString = "HttpHeaderValueLexer_FailedToReadTokenOrQuotedString";
         internal const string HttpHeaderValueLexer_InvalidSeparatorAfterQuotedString = "HttpHeaderValueLexer_InvalidSeparatorAfterQuotedString";
         internal const string HttpHeaderValueLexer_EndOfFileAfterSeparator = "HttpHeaderValueLexer_EndOfFileAfterSeparator";
+
+        internal const string HttpPreferenceHeader_InvalidValueForToken = "HttpPreferenceHeader_InvalidValueForToken";
+
         internal const string MediaType_EncodingNotSupported = "MediaType_EncodingNotSupported";
         internal const string MediaTypeUtils_DidNotFindMatchingMediaType = "MediaTypeUtils_DidNotFindMatchingMediaType";
         internal const string MediaTypeUtils_CannotDetermineFormatFromContentType = "MediaTypeUtils_CannotDetermineFormatFromContentType";

src/Microsoft.OData.Core/Microsoft.OData.Core.txt

@@ -241,6 +241,8 @@ HttpHeaderValueLexer_FailedToReadTokenOrQuotedString=An error occurred when pars
 HttpHeaderValueLexer_InvalidSeparatorAfterQuotedString=An error occurred when parsing the HTTP header '{0}'. The header value '{1}' is incorrect at position '{2}' because '{3}' is not a valid separator after a quoted-string.
 HttpHeaderValueLexer_EndOfFileAfterSeparator=An error occurred when parsing the HTTP header '{0}'. The header value '{1}' is incorrect at position '{2}' because the header value should not end with the separator '{3}'.
 
+HttpPreferenceHeader_InvalidValueForToken=The value '{0}' is not a valid value for OData token '{1}' in HTTP Prefer or Preference-Applied headers.
+
 MediaType_EncodingNotSupported=The character set '{0}' is not supported.
 
 MediaTypeUtils_DidNotFindMatchingMediaType=A supported MIME type could not be found that matches the acceptable MIME types for the request. The supported type(s) '{0}' do not match any of the acceptable MIME types '{1}'.

src/Microsoft.OData.Core/ODataConstants.cs

@@ -199,6 +199,19 @@ internal static class ODataInternalConstants
 
         /// <summary>The $deletedLink token indicates delta deleted link.</summary>
         internal const string ContextUriDeletedLink = UriSegmentSeparator + DeletedLink;
+
         #endregion Context URL
+
+        #region Preference Headers
+            /// <summary>
+            /// Valid value for <code>OmitValuesPreferenceToken</code> indicating nulls are omitted.
+            /// </summary>
+            internal const string OmitValuesNulls = "nulls";
+
+            /// <summary>
+            /// Valid value for <code>OmitValuesPreferenceToken</code> indicating defaults are omitted.
+            /// </summary>
+            internal const string OmitValuesDefaults = "defaults";
+        #endregion Preference Headers
     }
 }

src/Microsoft.OData.Core/ODataMessageReader.cs

@@ -169,6 +169,17 @@ public ODataMessageReader(IODataResponseMessage responseMessage, ODataMessageRea
             this.model = model ?? GetModel(this.container);
             this.edmTypeResolver = new EdmTypeReaderResolver(this.model, this.settings.ClientCustomTypeResolver);
 
+            // Whether null values were omitted in the response.
+            this.settings.NullValuesOmitted = false;
+            string omitValuePreferenceApplied = responseMessage.PreferenceAppliedHeader().OmitValues;
+            if (omitValuePreferenceApplied != null)
+            {
+                // If the Preference-Applied header's omit-values parameter is present in the response, its value should
+                // be applied to the reader's setting.
+                this.settings.NullValuesOmitted =
+                    omitValuePreferenceApplied.Equals(ODataConstants.OmitValuesNulls, StringComparison.OrdinalIgnoreCase);
+            }
+
             // If the Preference-Applied header on the response message contains an annotation filter, we set the filter
             // to the reader settings if it's not already set, so that we would only read annotations that satisfy the filter.
             string annotationFilter = responseMessage.PreferenceAppliedHeader().AnnotationFilter;

src/Microsoft.OData.Core/ODataMessageReaderSettings.cs

@@ -196,6 +196,11 @@ public ODataMessageQuotas MessageQuotas
         /// </summary>
         internal bool ThrowOnUndeclaredPropertyForNonOpenType { get; private set; }
 
+        /// <summary>
+        /// True if null values are omitted, per the omit-values parameter in Preference header of the message.
+        /// </summary>
+        internal bool NullValuesOmitted { get; set; }
+
         /// <summary>
         /// Creates a shallow copy of this <see cref="ODataMessageReaderSettings"/>.
         /// </summary>
@@ -253,6 +258,7 @@ private void CopyFrom(ODataMessageReaderSettings other)
             this.MaxProtocolVersion = other.MaxProtocolVersion;
             this.ReadUntypedAsString = other.ReadUntypedAsString;
             this.ShouldIncludeAnnotation = other.ShouldIncludeAnnotation;
+            this.NullValuesOmitted = other.NullValuesOmitted;
             this.validations = other.validations;
             this.ThrowOnDuplicatePropertyNames = other.ThrowOnDuplicatePropertyNames;
             this.ThrowIfTypeConflictsWithMetadata = other.ThrowIfTypeConflictsWithMetadata;

src/Microsoft.OData.Core/ODataMessageWriter.cs

@@ -151,6 +151,14 @@ public ODataMessageWriter(IODataResponseMessage responseMessage, ODataMessageWri
             WriterValidationUtils.ValidateMessageWriterSettings(this.settings, this.writingResponse);
             this.message = new ODataResponseMessage(responseMessage, /*writing*/ true, this.settings.EnableMessageStreamDisposal, /*maxMessageSize*/ -1);
 
+            // Set the Preference-Applied header's parameter omit-values=nulls per writer settings.
+            // For response writing, source of specifying omit-null-values preference is the settings object
+            // from caller(the OData service).
+            if (this.settings.OmitNullValues)
+            {
+                responseMessage.PreferenceAppliedHeader().OmitValues = ODataConstants.OmitValuesNulls;
+            }
+
             // If the Preference-Applied header on the response message contains an annotation filter, we set the filter
             // to the writer settings so that we would only write annotations that satisfy the filter.
             string annotationFilter = responseMessage.PreferenceAppliedHeader().AnnotationFilter;

src/Microsoft.OData.Core/ODataMessageWriterSettings.cs

@@ -178,12 +178,12 @@ public ODataUri ODataUri
         internal bool ThrowOnDuplicatePropertyNames { get; private set; }
 
         /// <summary>
-        /// Don't serialize null values
+        /// Whether to omit null values when writing response.
         /// </summary>
         /// <remarks>
-        /// Default valus is false, that means serialize null values.
+        /// Default value is false, that means serialize null values.
         /// </remarks>
-        public bool IgnoreNullValues { get; set; }
+        public bool OmitNullValues { get; set; }
 
         /// <summary>
         /// Returns whether ThrowOnUndeclaredPropertyForNonOpenType validation setting is enabled.
@@ -406,7 +406,7 @@ private void CopyFrom(ODataMessageWriterSettings other)
             this.shouldIncludeAnnotation = other.shouldIncludeAnnotation;
             this.useFormat = other.useFormat;
             this.Version = other.Version;
-            this.IgnoreNullValues = other.IgnoreNullValues;
+            this.OmitNullValues = other.OmitNullValues;
 
             this.validations = other.validations;
             this.ThrowIfTypeConflictsWithMetadata = other.ThrowIfTypeConflictsWithMetadata;