Best effort paging (#840)

* Adds support for best-effort paging links (in case the total number of resources is unknown). This strategy is also applied on all secondary resources, as it is quite complex to determine total resource count.

Fixed: exception on pass-through query string parameter in secondary request
Optimization: skip query execution for primary resource if total count is zero
Fixed: self link was calculated twice (in different ways) when paging active
Fixed: nested page size was lost when rendering paging links
Fixed: invalid paging links on secondary endpoints
Fixed: cases where self link was different from the actual URL being requested (we now leave page number/size in, even when they match the default)

* Migrated PaginationLinkTests

Author: Bart Koelman

src/JsonApiDotNetCore/Queries/IPaginationContext.cs

@@ -19,6 +19,12 @@ public interface IPaginationContext
         /// </summary>
         PageSize PageSize { get; set; }
 
+        /// <summary>
+        /// Indicates whether the number of resources on the current page equals the page size.
+        /// When <c>true</c>, a subsequent page might exist (assuming <see cref="TotalResourceCount"/> is unknown).
+        /// </summary>
+        bool IsPageFull { get; set; }
+
         /// <summary>
         /// The total number of resources.
         /// <c>null</c> when <see cref="IJsonApiOptions.IncludeTotalResourceCount"/> is set to <c>false</c>.

src/JsonApiDotNetCore/Queries/PaginationContext.cs

@@ -12,6 +12,9 @@ internal sealed class PaginationContext : IPaginationContext
         /// <inheritdoc />
         public PageSize PageSize { get; set; }
 
+        /// <inheritdoc />
+        public bool IsPageFull { get; set; }
+
         /// <inheritdoc />
         public int? TotalResourceCount { get; set; }
 

src/JsonApiDotNetCore/QueryStrings/Internal/ResourceDefinitionQueryableParameterReader.cs

@@ -46,15 +46,17 @@ public virtual void Read(string parameterName, StringValues parameterValue)
 
         private object GetQueryableHandler(string parameterName)
         {
-            if (_request.Kind != EndpointKind.Primary)
+            var resourceType = _request.PrimaryResource.ResourceType;
+            var handler = _resourceDefinitionAccessor.GetQueryableHandlerForQueryStringParameter(resourceType, parameterName);
+
+            if (handler != null && _request.Kind != EndpointKind.Primary)
             {
                 throw new InvalidQueryStringParameterException(parameterName,
                     "Custom query string parameters cannot be used on nested resource endpoints.",
                     $"Query string parameter '{parameterName}' cannot be used on a nested resource endpoint.");
             }
 
-            var resourceType = _request.PrimaryResource.ResourceType;
-            return _resourceDefinitionAccessor.GetQueryableHandlerForQueryStringParameter(resourceType, parameterName);
+            return handler;
         }
 
         /// <inheritdoc />

src/JsonApiDotNetCore/Serialization/Building/LinkBuilder.cs

@@ -5,6 +5,8 @@
 using JsonApiDotNetCore.Configuration;
 using JsonApiDotNetCore.Middleware;
 using JsonApiDotNetCore.Queries;
+using JsonApiDotNetCore.Queries.Expressions;
+using JsonApiDotNetCore.Queries.Internal.Parsing;
 using JsonApiDotNetCore.QueryStrings;
 using JsonApiDotNetCore.Resources;
 using JsonApiDotNetCore.Resources.Annotations;
@@ -15,6 +17,9 @@ namespace JsonApiDotNetCore.Serialization.Building
 {
     public class LinkBuilder : ILinkBuilder
     {
+        private const string _pageSizeParameterName = "page[size]";
+        private const string _pageNumberParameterName = "page[number]";
+
         private readonly IResourceContextProvider _provider;
         private readonly IRequestQueryStringAccessor _queryStringAccessor;
         private readonly IJsonApiOptions _options;
@@ -42,10 +47,10 @@ public TopLevelLinks GetTopLevelLinks()
             TopLevelLinks topLevelLinks = null;
             if (ShouldAddTopLevelLink(resourceContext, LinkTypes.Self))
             {
-                topLevelLinks = new TopLevelLinks { Self = GetSelfTopLevelLink(resourceContext) };
+                topLevelLinks = new TopLevelLinks {Self = GetSelfTopLevelLink(resourceContext, null)};
             }
 
-            if (ShouldAddTopLevelLink(resourceContext, LinkTypes.Paging) && _paginationContext.PageSize != null)
+            if (ShouldAddTopLevelLink(resourceContext, LinkTypes.Paging) && _paginationContext.PageSize != null && _request.IsCollection)
             {   
                 SetPageLinks(resourceContext, topLevelLinks ??= new TopLevelLinks());
             }
@@ -70,36 +75,38 @@ private bool ShouldAddTopLevelLink(ResourceContext resourceContext, LinkTypes li
 
         private void SetPageLinks(ResourceContext resourceContext, TopLevelLinks links)
         {
-            if (_paginationContext.PageNumber.OneBasedValue > 1)
+            links.First = GetPageLink(resourceContext, 1, _paginationContext.PageSize);
+
+            if (_paginationContext.TotalPageCount > 0)
             {
-                links.Prev = GetPageLink(resourceContext, _paginationContext.PageNumber.OneBasedValue - 1, _paginationContext.PageSize);
+                links.Last = GetPageLink(resourceContext, _paginationContext.TotalPageCount.Value, _paginationContext.PageSize);
             }
 
-            if (_paginationContext.PageNumber.OneBasedValue < _paginationContext.TotalPageCount)
+            if (_paginationContext.PageNumber.OneBasedValue > 1)
             {
-                links.Next = GetPageLink(resourceContext, _paginationContext.PageNumber.OneBasedValue + 1, _paginationContext.PageSize);
+                links.Prev = GetPageLink(resourceContext, _paginationContext.PageNumber.OneBasedValue - 1, _paginationContext.PageSize);
             }
 
-            if (_paginationContext.TotalPageCount > 0)
+            bool hasNextPage = _paginationContext.PageNumber.OneBasedValue < _paginationContext.TotalPageCount;
+            bool possiblyHasNextPage = _paginationContext.TotalPageCount == null && _paginationContext.IsPageFull;
+
+            if (hasNextPage || possiblyHasNextPage)
             {
-                links.Self = GetPageLink(resourceContext, _paginationContext.PageNumber.OneBasedValue, _paginationContext.PageSize);
-                links.First = GetPageLink(resourceContext, 1, _paginationContext.PageSize);
-                links.Last = GetPageLink(resourceContext, _paginationContext.TotalPageCount.Value, _paginationContext.PageSize);
+                links.Next = GetPageLink(resourceContext, _paginationContext.PageNumber.OneBasedValue + 1, _paginationContext.PageSize);
             }
         }
 
-        private string GetSelfTopLevelLink(ResourceContext resourceContext)
+        private string GetSelfTopLevelLink(ResourceContext resourceContext, Action<Dictionary<string, string>> queryStringUpdateAction)
         {
             var builder = new StringBuilder();
             builder.Append(_request.BasePath);
             builder.Append("/");
             builder.Append(resourceContext.PublicName);
 
-            string resourceId = _request.PrimaryId;
-            if (resourceId != null)
+            if (_request.PrimaryId != null)
             {
                 builder.Append("/");
-                builder.Append(resourceId);
+                builder.Append(_request.PrimaryId);
             }
 
             if (_request.Relationship != null)
@@ -108,49 +115,102 @@ private string GetSelfTopLevelLink(ResourceContext resourceContext)
                 builder.Append(_request.Relationship.PublicName);
             }
 
-            builder.Append(DecodeSpecialCharacters(_queryStringAccessor.QueryString.Value));
+            string queryString = BuildQueryString(queryStringUpdateAction);
+            builder.Append(queryString);
 
             return builder.ToString();
         }
 
+        private string BuildQueryString(Action<Dictionary<string, string>> updateAction)
+        {
+            var parameters = _queryStringAccessor.Query.ToDictionary(pair => pair.Key, pair => pair.Value.ToString());
+            updateAction?.Invoke(parameters);
+            string queryString = QueryString.Create(parameters).Value;
+
+            return DecodeSpecialCharacters(queryString);
+        }
+
+        private static string DecodeSpecialCharacters(string uri)
+        {
+            return uri.Replace("%5B", "[").Replace("%5D", "]").Replace("%27", "'").Replace("%3A", ":");
+        }
+
         private string GetPageLink(ResourceContext resourceContext, int pageOffset, PageSize pageSize)
         {
-            string queryString = BuildQueryString(parameters =>
+            return GetSelfTopLevelLink(resourceContext, parameters =>
             {
-                if (pageSize == null || pageSize.Equals(_options.DefaultPageSize))
+                var existingPageSizeParameterValue = parameters.ContainsKey(_pageSizeParameterName)
+                    ? parameters[_pageSizeParameterName]
+                    : null;
+
+                PageSize newTopPageSize = Equals(pageSize, _options.DefaultPageSize) ? null : pageSize;
+
+                string newPageSizeParameterValue = ChangeTopPageSize(existingPageSizeParameterValue, newTopPageSize);
+                if (newPageSizeParameterValue == null)
                 {
-                    parameters.Remove("page[size]");
+                    parameters.Remove(_pageSizeParameterName);
                 }
                 else
                 {
-                    parameters["page[size]"] = pageSize.ToString();
+                    parameters[_pageSizeParameterName] = newPageSizeParameterValue;
                 }
 
                 if (pageOffset == 1)
                 {
-                    parameters.Remove("page[number]");
+                    parameters.Remove(_pageNumberParameterName);
                 }
                 else
                 {
-                    parameters["page[number]"] = pageOffset.ToString();
+                    parameters[_pageNumberParameterName] = pageOffset.ToString();
                 }
             });
-
-            return $"{_request.BasePath}/{resourceContext.PublicName}" + queryString;
         }
 
-        private string BuildQueryString(Action<Dictionary<string, string>> updateAction)
+        private string ChangeTopPageSize(string pageSizeParameterValue, PageSize topPageSize)
         {
-            var parameters = _queryStringAccessor.Query.ToDictionary(pair => pair.Key, pair => pair.Value.ToString());
-            updateAction(parameters);
-            string queryString = QueryString.Create(parameters).Value;
+            var elements = ParsePageSizeExpression(pageSizeParameterValue);
+            var elementInTopScopeIndex = elements.FindIndex(expression => expression.Scope == null);
 
-            return DecodeSpecialCharacters(queryString);
+            if (topPageSize != null)
+            {
+                var topPageSizeElement = new PaginationElementQueryStringValueExpression(null, topPageSize.Value);
+
+                if (elementInTopScopeIndex != -1)
+                {
+                    elements[elementInTopScopeIndex] = topPageSizeElement;
+                }
+                else
+                {
+                    elements.Insert(0, topPageSizeElement);
+                }
+            }
+            else
+            {
+                if (elementInTopScopeIndex != -1)
+                {
+                    elements.RemoveAt(elementInTopScopeIndex);
+                }
+            }
+
+            var parameterValue = string.Join(',',
+                elements.Select(expression => expression.Scope == null ? expression.Value.ToString() : $"{expression.Scope}:{expression.Value}"));
+
+            return parameterValue == string.Empty ? null : parameterValue;
         }
 
-        private static string DecodeSpecialCharacters(string uri)
+        private List<PaginationElementQueryStringValueExpression> ParsePageSizeExpression(string pageSizeParameterValue)
         {
-            return uri.Replace("%5B", "[").Replace("%5D", "]").Replace("%27", "'");
+            if (pageSizeParameterValue == null)
+            {
+                return new List<PaginationElementQueryStringValueExpression>();
+            }
+
+            var requestResource = _request.SecondaryResource ?? _request.PrimaryResource;
+
+            var parser = new PaginationParser(_provider);
+            var paginationExpression = parser.Parse(pageSizeParameterValue, requestResource);
+
+            return new List<PaginationElementQueryStringValueExpression>(paginationExpression.Elements);
         }
 
         /// <inheritdoc />

src/JsonApiDotNetCore/Services/JsonApiResourceService.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
@@ -119,6 +120,11 @@ public virtual async Task<IReadOnlyCollection<TResource>> GetAsync()
             {
                 var topFilter = _queryLayerComposer.GetTopFilter();
                 _paginationContext.TotalResourceCount = await _repository.CountAsync(topFilter);
+
+                if (_paginationContext.TotalResourceCount == 0)
+                {
+                    return Array.Empty<TResource>();
+                }
             }
 
             var queryLayer = _queryLayerComposer.Compose(_request.PrimaryResource);
@@ -130,6 +136,11 @@ public virtual async Task<IReadOnlyCollection<TResource>> GetAsync()
                 return _hookExecutor.OnReturn(resources, ResourcePipeline.Get).ToArray();
             }
 
+            if (queryLayer.Pagination?.PageSize != null && queryLayer.Pagination.PageSize.Value == resources.Count)
+            {
+                _paginationContext.IsPageFull = true;
+            }
+
             return resources;
         }
 
@@ -233,6 +244,14 @@ public virtual async Task<object> GetSecondaryAsync(TId id, string relationshipN
             var secondaryLayer = _queryLayerComposer.Compose(_request.SecondaryResource);
             var primaryLayer = _queryLayerComposer.WrapLayerForSecondaryEndpoint(secondaryLayer, _request.PrimaryResource, id, _request.Relationship);
 
+            if (_request.IsCollection && _options.IncludeTotalResourceCount)
+            {
+                // TODO: Consider support for pagination links on secondary resource collection. This requires to call Count() on the inverse relationship (which may not exist).
+                // For /blogs/1/articles we need to execute Count(Articles.Where(article => article.Blog.Id == 1 && article.Blog.existingFilter))) to determine TotalResourceCount.
+                // This also means we need to invoke ResourceRepository<Article>.CountAsync() from ResourceService<Blog>.
+                // And we should call BlogResourceDefinition.OnApplyFilter to filter out soft-deleted blogs and translate from equals('IsDeleted','false') to equals('Blog.IsDeleted','false')
+            }
+
             var primaryResources = await _repository.GetAsync(primaryLayer);
 
             var primaryResource = primaryResources.SingleOrDefault();
@@ -244,7 +263,15 @@ public virtual async Task<object> GetSecondaryAsync(TId id, string relationshipN
                 primaryResource = _hookExecutor.OnReturn(AsList(primaryResource), ResourcePipeline.GetRelationship).Single();
             }
 
-            return _request.Relationship.GetValue(primaryResource);
+            var secondaryResource = _request.Relationship.GetValue(primaryResource);
+
+            if (secondaryResource is ICollection secondaryResources && 
+                secondaryLayer.Pagination?.PageSize != null && secondaryLayer.Pagination.PageSize.Value == secondaryResources.Count)
+            {
+                _paginationContext.IsPageFull = true;
+            }
+
+            return secondaryResource;
         }
 
         /// <inheritdoc />