From 2644543382c1b3f114c068bb10a413e64d26715b Mon Sep 17 00:00:00 2001 From: Alexey Kudelko Date: Wed, 21 Feb 2024 00:35:10 +0300 Subject: [PATCH 1/2] fixed no limit and no offset bug --- feed/helpers.py | 13 +++++++++++-- feed/pagination.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/feed/helpers.py b/feed/helpers.py index 3c58e552..21785665 100644 --- a/feed/helpers.py +++ b/feed/helpers.py @@ -29,9 +29,16 @@ def paginate_serialize_feed( result = [] pages_count = 0 - offset = int(request.query_params.get("offset", 0)) + offset = request.query_params.get("offset", None) request.query_params._mutable = True - request.query_params["offset"] = int(request.query_params["offset"]) + + if offset == "" or offset is None: + offset = 0 + elif offset is not None: + offset = int(offset) + request.query_params["offset"] = offset + + offset_numbers = randomize_offset(offset, len(model_data.keys())) for i in range(len(model_data.keys())): @@ -45,6 +52,8 @@ def paginate_serialize_feed( shuffle(result) limit = int(request.query_params.get("limit", LIMIT_PAGINATION_CONSTANT)) + if limit == "": + limit = LIMIT_PAGINATION_CONSTANT return result[:limit], pages_count diff --git a/feed/pagination.py b/feed/pagination.py index 3335339b..9e88fc83 100644 --- a/feed/pagination.py +++ b/feed/pagination.py @@ -25,7 +25,7 @@ def custom_paginate_queryset( if self.count == 0 or self.offset > self.count: return {"queryset_ready": [], "count": self.count} - queryset_ready = queryset[self.offset: self.offset + self.limit] + queryset_ready = queryset[self.offset : self.offset + self.limit] return { "queryset_ready": queryset_ready, "count": self.count, From 5abf8cb166029353ce42d3a2c59bd2992ec694f5 Mon Sep 17 00:00:00 2001 From: Alexey Kudelko Date: Wed, 21 Feb 2024 02:46:58 +0300 Subject: [PATCH 2/2] fix of fix of feed --- feed/helpers.py | 75 ++++++++++++++++++++++++++++++++++------------ feed/pagination.py | 4 +-- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/feed/helpers.py b/feed/helpers.py index 21785665..090f222e 100644 --- a/feed/helpers.py +++ b/feed/helpers.py @@ -5,7 +5,11 @@ from rest_framework.views import APIView from feed import constants -from feed.constants import SupportedModel, SupportedQuerySet, LIMIT_PAGINATION_CONSTANT +from feed.constants import ( + SupportedModel, + SupportedQuerySet, + LIMIT_PAGINATION_CONSTANT +) from feed.pagination import FeedPagination from feed.serializers import FeedItemSerializer from news.models import News @@ -29,42 +33,74 @@ def paginate_serialize_feed( result = [] pages_count = 0 - offset = request.query_params.get("offset", None) + if len(model_data) == 0: + return [], 0 + + offset = request.query_params.get("offset", 0) request.query_params._mutable = True - if offset == "" or offset is None: - offset = 0 - elif offset is not None: + if isinstance(offset, str) and offset.isdigit(): offset = int(offset) - request.query_params["offset"] = offset + else: + offset = 0 + request.query_params["offset"] = offset - offset_numbers = randomize_offset(offset, len(model_data.keys())) + models_counts = { + model_name: model_data[model_name].count() for model_name in model_data.keys() + } + offset_numbers = offset_distribution(offset, models_counts) + + for model_name in model_data.keys(): + request.query_params["offset"] = offset_numbers[model_name] - for i in range(len(model_data.keys())): - request.query_params["offset"] = offset_numbers[i] paginated_part: dict = paginate_serialize_feed_queryset( - model_data, paginator, request, list(model_data.keys())[i], view + model_data, paginator, request, model_name, models_counts[model_name], view ) + result += paginated_part["paginated_data"] pages_count += paginated_part["page_count"] - shuffle(result) + limit = request.query_params.get("limit", LIMIT_PAGINATION_CONSTANT) - limit = int(request.query_params.get("limit", LIMIT_PAGINATION_CONSTANT)) if limit == "": limit = LIMIT_PAGINATION_CONSTANT + else: + limit = int(limit) + + shuffle(result) return result[:limit], pages_count -def randomize_offset(offset: int, quantity_models: int) -> list[int]: - full_division = offset // quantity_models - extra_items = offset % quantity_models +def offset_distribution(offset: int, models_counts: dict) -> dict: + common_key_list = list(models_counts.keys()) + quantity_of_models = len(list(models_counts.keys())) + + full_division = offset // quantity_of_models + extra_items = offset % quantity_of_models + + distributed_not_ready = {model_name: full_division for model_name in common_key_list} + distributed = dict( + sorted(distributed_not_ready.items(), key=lambda item: models_counts[item[0]]) + ) + + last_key = common_key_list[-1] + distributed[last_key] += extra_items - pagination_numbers = [full_division] * quantity_models - pagination_numbers[-1] += extra_items + new_keys_list = list(distributed.keys()) + for i, key in enumerate( + new_keys_list + ): # распределяем переполненные значения от маленьких моделей к большим + offset_value = distributed[key] + model_count = models_counts[key] + if offset_value > model_count: + diff = offset_value - model_count + distributed[key] = model_count + if i + 1 < len(new_keys_list): + next_key = new_keys_list[i + 1] + distributed[next_key] += diff - return pagination_numbers + return distributed def paginate_serialize_feed_queryset( @@ -72,10 +108,11 @@ def paginate_serialize_feed_queryset( paginator: FeedPagination, request: Request, model: SupportedModel, + count: int, view: APIView, ) -> dict: paginated_info = paginator.custom_paginate_queryset( - model_data[model], request, view=view + model_data[model], request, count, view=view ) paginated_data = paginated_info["queryset_ready"] num_pages = paginated_info["count"] diff --git a/feed/pagination.py b/feed/pagination.py index 9e88fc83..eea7ff91 100644 --- a/feed/pagination.py +++ b/feed/pagination.py @@ -10,13 +10,13 @@ class FeedPagination(pagination.LimitOffsetPagination): offset_query_param = "offset" def custom_paginate_queryset( - self, queryset: SupportedQuerySet, request: Request, view=None + self, queryset: SupportedQuerySet, request: Request, count: int, view=None ) -> dict: self.limit = self.get_limit(request) if self.limit is None: return None - self.count = self.get_count(queryset) + self.count = count self.offset = self.get_offset(request) self.request = request if self.count > self.limit and self.template is not None: