From 4cc6f744c626a92c15f0403d28283f97243db5fe Mon Sep 17 00:00:00 2001 From: Thijmen Date: Tue, 30 Apr 2024 21:58:42 +0200 Subject: [PATCH 1/4] fix to address Picnic API update, specifically new url for search endpoint which returns a new format --- python_picnic_api/client.py | 17 +++++++++-------- python_picnic_api/helper.py | 28 ++++++++++++++++++++++++++-- tests/test_client.py | 6 +++--- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/python_picnic_api/client.py b/python_picnic_api/client.py index 83ab1ec..0415588 100644 --- a/python_picnic_api/client.py +++ b/python_picnic_api/client.py @@ -1,6 +1,6 @@ from hashlib import md5 -from .helper import _tree_generator, _url_generator, _get_category_name +from .helper import _tree_generator, _url_generator, _get_category_name, _extract_search_results from .session import PicnicAPISession, PicnicAuthError DEFAULT_URL = "https://storefront-prod.{}.picnicinternational.com/api/{}" @@ -23,7 +23,7 @@ def __init__( # Login if not authenticated if not self.session.authenticated and username and password: self.login(username, password) - + self.high_level_categories = None def initialize_high_level_categories(self): @@ -36,8 +36,8 @@ def _get(self, path: str, add_picnic_headers=False): # Make the request, add special picnic headers if needed headers = { - "x-picnic-agent": "30100;1.15.183-14941;", - "x-picnic-did": "00DE6414C744E7CB" + "x-picnic-agent": "30100;1.15.232-15154;", + "x-picnic-did": "3C417201548B2E3B" } if add_picnic_headers else None response = self.session.get(url, headers=headers).json() @@ -77,8 +77,9 @@ def get_user(self): return self._get("/user") def search(self, term: str): - path = "/search?search_term=" + term - return self._get(path) + path = f"/pages/search-page-results?search_term={term}" + raw_results = self._get(path, add_picnic_headers=True) + return _extract_search_results(raw_results) def get_lists(self, list_id: str = None): if list_id: @@ -101,7 +102,7 @@ def get_sublist(self, list_id: str, sublist_id: str) -> list: def get_cart(self): return self._get("/cart") - + def get_article(self, article_id: str, add_category_name=False): path = "/articles/" + article_id article = self._get(path) @@ -111,7 +112,7 @@ def get_article(self, article_id: str, add_category_name=False): category_name=_get_category_name(article['category_link'], self.high_level_categories) ) return article - + def get_article_category(self, article_id: str): path = "/articles/" + article_id + "/category" return self._get(path) diff --git a/python_picnic_api/helper.py b/python_picnic_api/helper.py index b28fb1e..0fc7dce 100644 --- a/python_picnic_api/helper.py +++ b/python_picnic_api/helper.py @@ -1,3 +1,4 @@ +import json import re # prefix components: @@ -45,8 +46,8 @@ def _get_category_id_from_link(category_link: str) -> str: return result else: return None - - + + def _get_category_name(category_link: str, categories: list) -> str: category_id = _get_category_id_from_link(category_link) if category_id: @@ -76,3 +77,26 @@ def get_image(id: str, size="regular", suffix="webp"): ) return f"{IMAGE_BASE_URL}/{id}/{size}.{suffix}" + +def _extract_search_results(raw_results: dict) -> list: + search_results = [] + sole_article_id_pattern = re.compile(r"sole_article_id=([0-9]+)") + + # Iterate over the nested structure of raw_results + for child1 in raw_results.get("body", {}).get("children", []): + for child2 in child1.get("children", []): + content = child2.get("content") + if content and "selling_unit" in content: + # Extracting the sole_article_id from the serialized JSON of pml + sole_article_ids = sole_article_id_pattern.findall( + json.dumps(child2.get("pml", {})) + ) + if sole_article_ids: + sole_article_id = sole_article_ids[0] + # Create and append the result entry + result_entry = { + **content["selling_unit"], + "sole_article_id": sole_article_id, + } + search_results.append(result_entry) + return search_results diff --git a/tests/test_client.py b/tests/test_client.py index 359082c..975a699 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -6,7 +6,7 @@ from python_picnic_api.session import PicnicAuthError PICNIC_HEADERS = { - "x-picnic-agent": "30100;1.15.77-10293", + "x-picnic-agent": "30100;1.15.232-15154", "x-picnic-did": "3C417201548B2E3B", } @@ -34,7 +34,7 @@ def test_login_credentials(self): PicnicAPI(username='test@test.nl', password='test') self.session_mock().post.assert_called_with( self.expected_base_url + '/user/login', - json={'key': 'test@test.nl', 'secret': '098f6bcd4621d373cade4e832627b4f6', "client_id": 1} + json={'key': 'test@test.nl', 'secret': '098f6bcd4621d373cade4e832627b4f6', "client_id": 30100} ) def test_login_auth_token(self): @@ -83,7 +83,7 @@ def test_get_user(self): def test_search(self): self.client.search("test-product") self.session_mock().get.assert_called_with( - self.expected_base_url + "/search?search_term=test-product", headers=None + self.expected_base_url + "/pages/search-page-results?search_term=test-product", headers=PICNIC_HEADERS ) def test_get_lists(self): From d09382e43936fe0613662e34e4024778cc8264fb Mon Sep 17 00:00:00 2001 From: Thijmen Date: Mon, 6 May 2024 22:34:38 +0200 Subject: [PATCH 2/4] Previous commit unnecessarily excluded a number of search results. This version seems to yield all expected search results. --- python_picnic_api/helper.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/python_picnic_api/helper.py b/python_picnic_api/helper.py index 0fc7dce..eff6e41 100644 --- a/python_picnic_api/helper.py +++ b/python_picnic_api/helper.py @@ -11,6 +11,7 @@ IMAGE_SIZES = ["small", "medium", "regular", "large", "extra-large"] IMAGE_BASE_URL = "https://storefront-prod.nl.picnicinternational.com/static/images" + def _tree_generator(response: list, prefix: str = ""): """A recursive tree generator, will yield a visual tree structure line by line @@ -25,7 +26,7 @@ def _tree_generator(response: list, prefix: str = ""): pre = f"{item['unit_quantity']} " after = "" if "display_price" in item.keys(): - after = f" €{int(item['display_price'])/100.0:.2f}" + after = f" €{int(item['display_price']) / 100.0:.2f}" yield prefix + pointer + pre + item["name"] + after if "items" in item: # extend the prefix and recurse: @@ -59,6 +60,7 @@ def _get_category_name(category_link: str, categories: list) -> str: else: return None + def get_recipe_image(id: str, size="regular"): sizes = IMAGE_SIZES + ["1250x1250"] assert size in sizes, "size must be one of: " + ", ".join(sizes) @@ -73,30 +75,17 @@ def get_image(id: str, size="regular", suffix="webp"): sizes = IMAGE_SIZES + [f"tile-{size}" for size in IMAGE_SIZES] assert size in sizes, ( - "size must be one of: " + ", ".join(sizes) + "size must be one of: " + ", ".join(sizes) ) return f"{IMAGE_BASE_URL}/{id}/{size}.{suffix}" def _extract_search_results(raw_results: dict) -> list: - search_results = [] - sole_article_id_pattern = re.compile(r"sole_article_id=([0-9]+)") - - # Iterate over the nested structure of raw_results - for child1 in raw_results.get("body", {}).get("children", []): - for child2 in child1.get("children", []): - content = child2.get("content") + parsed_results = [] + for parent in raw_results.get("body", {}).get("children", []): + for child in parent.get("children", []): + content = child.get("content") if content and "selling_unit" in content: - # Extracting the sole_article_id from the serialized JSON of pml - sole_article_ids = sole_article_id_pattern.findall( - json.dumps(child2.get("pml", {})) - ) - if sole_article_ids: - sole_article_id = sole_article_ids[0] - # Create and append the result entry - result_entry = { - **content["selling_unit"], - "sole_article_id": sole_article_id, - } - search_results.append(result_entry) - return search_results + parsed_results.append(content["selling_unit"]) + + return parsed_results From 9dc1f50961010c5e4cdf2d7f7054f6c95e5f5244 Mon Sep 17 00:00:00 2001 From: Thijmen Date: Tue, 7 May 2024 18:50:43 +0200 Subject: [PATCH 3/4] Revert "Previous commit unnecessarily excluded a number of search results. This version seems to yield all expected search results." This reverts commit d09382e43936fe0613662e34e4024778cc8264fb. --- python_picnic_api/helper.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/python_picnic_api/helper.py b/python_picnic_api/helper.py index eff6e41..0fc7dce 100644 --- a/python_picnic_api/helper.py +++ b/python_picnic_api/helper.py @@ -11,7 +11,6 @@ IMAGE_SIZES = ["small", "medium", "regular", "large", "extra-large"] IMAGE_BASE_URL = "https://storefront-prod.nl.picnicinternational.com/static/images" - def _tree_generator(response: list, prefix: str = ""): """A recursive tree generator, will yield a visual tree structure line by line @@ -26,7 +25,7 @@ def _tree_generator(response: list, prefix: str = ""): pre = f"{item['unit_quantity']} " after = "" if "display_price" in item.keys(): - after = f" €{int(item['display_price']) / 100.0:.2f}" + after = f" €{int(item['display_price'])/100.0:.2f}" yield prefix + pointer + pre + item["name"] + after if "items" in item: # extend the prefix and recurse: @@ -60,7 +59,6 @@ def _get_category_name(category_link: str, categories: list) -> str: else: return None - def get_recipe_image(id: str, size="regular"): sizes = IMAGE_SIZES + ["1250x1250"] assert size in sizes, "size must be one of: " + ", ".join(sizes) @@ -75,17 +73,30 @@ def get_image(id: str, size="regular", suffix="webp"): sizes = IMAGE_SIZES + [f"tile-{size}" for size in IMAGE_SIZES] assert size in sizes, ( - "size must be one of: " + ", ".join(sizes) + "size must be one of: " + ", ".join(sizes) ) return f"{IMAGE_BASE_URL}/{id}/{size}.{suffix}" def _extract_search_results(raw_results: dict) -> list: - parsed_results = [] - for parent in raw_results.get("body", {}).get("children", []): - for child in parent.get("children", []): - content = child.get("content") - if content and "selling_unit" in content: - parsed_results.append(content["selling_unit"]) + search_results = [] + sole_article_id_pattern = re.compile(r"sole_article_id=([0-9]+)") - return parsed_results + # Iterate over the nested structure of raw_results + for child1 in raw_results.get("body", {}).get("children", []): + for child2 in child1.get("children", []): + content = child2.get("content") + if content and "selling_unit" in content: + # Extracting the sole_article_id from the serialized JSON of pml + sole_article_ids = sole_article_id_pattern.findall( + json.dumps(child2.get("pml", {})) + ) + if sole_article_ids: + sole_article_id = sole_article_ids[0] + # Create and append the result entry + result_entry = { + **content["selling_unit"], + "sole_article_id": sole_article_id, + } + search_results.append(result_entry) + return search_results From 7e0796716d8383d7dc3e6c13eb89585be0e1de01 Mon Sep 17 00:00:00 2001 From: Thijmen Date: Tue, 7 May 2024 19:01:18 +0200 Subject: [PATCH 4/4] Some results were missing from search due to the "if sole_article_ids" condition. This commit includes items that don't have a sole article id, and just sets the sole article id to None. Also includes some minor reformatting and refactor (updated type hint Optional for functions that can return a None). --- python_picnic_api/helper.py | 61 +++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/python_picnic_api/helper.py b/python_picnic_api/helper.py index 0fc7dce..618e897 100644 --- a/python_picnic_api/helper.py +++ b/python_picnic_api/helper.py @@ -1,5 +1,6 @@ import json import re +from typing import List, Dict, Any, Optional # prefix components: space = " " @@ -11,6 +12,9 @@ IMAGE_SIZES = ["small", "medium", "regular", "large", "extra-large"] IMAGE_BASE_URL = "https://storefront-prod.nl.picnicinternational.com/static/images" +SOLE_ARTICLE_ID_PATTERN = re.compile(r"sole_article_id=([0-9]+)") + + def _tree_generator(response: list, prefix: str = ""): """A recursive tree generator, will yield a visual tree structure line by line @@ -38,8 +42,8 @@ def _url_generator(url: str, country_code: str, api_version: str): return url.format(country_code.lower(), api_version) -def _get_category_id_from_link(category_link: str) -> str: - pattern = r'categories/(\d+)' +def _get_category_id_from_link(category_link: str) -> Optional[str]: + pattern = r"categories/(\d+)" first_number = re.search(pattern, category_link) if first_number: result = str(first_number.group(1)) @@ -48,10 +52,12 @@ def _get_category_id_from_link(category_link: str) -> str: return None -def _get_category_name(category_link: str, categories: list) -> str: +def _get_category_name(category_link: str, categories: list) -> Optional[str]: category_id = _get_category_id_from_link(category_link) if category_id: - category = next((item for item in categories if item["id"] == category_id), None) + category = next( + (item for item in categories if item["id"] == category_id), None + ) if category: return category["name"] else: @@ -59,6 +65,7 @@ def _get_category_name(category_link: str, categories: list) -> str: else: return None + def get_recipe_image(id: str, size="regular"): sizes = IMAGE_SIZES + ["1250x1250"] assert size in sizes, "size must be one of: " + ", ".join(sizes) @@ -66,37 +73,33 @@ def get_recipe_image(id: str, size="regular"): def get_image(id: str, size="regular", suffix="webp"): - assert "tile" in size if suffix == "webp" else True, ( - "webp format only supports tile sizes" - ) + assert ( + "tile" in size if suffix == "webp" else True + ), "webp format only supports tile sizes" assert suffix in ["webp", "png"], "suffix must be webp or png" sizes = IMAGE_SIZES + [f"tile-{size}" for size in IMAGE_SIZES] - assert size in sizes, ( - "size must be one of: " + ", ".join(sizes) - ) + assert size in sizes, "size must be one of: " + ", ".join(sizes) return f"{IMAGE_BASE_URL}/{id}/{size}.{suffix}" -def _extract_search_results(raw_results: dict) -> list: +def _extract_search_results(raw_results: Dict[str, Any]) -> List[Dict[str, Any]]: + """Extract search results from a nested dictionary structure returned by Picnic search.""" search_results = [] - sole_article_id_pattern = re.compile(r"sole_article_id=([0-9]+)") - - # Iterate over the nested structure of raw_results - for child1 in raw_results.get("body", {}).get("children", []): - for child2 in child1.get("children", []): - content = child2.get("content") - if content and "selling_unit" in content: - # Extracting the sole_article_id from the serialized JSON of pml - sole_article_ids = sole_article_id_pattern.findall( - json.dumps(child2.get("pml", {})) + + for section in raw_results.get("body", {}).get("children", []): + for item in section.get("children", []): + content = item.get("content", {}) + if "selling_unit" in content: + sole_article_ids = SOLE_ARTICLE_ID_PATTERN.findall( + json.dumps(item.get("pml", {})) ) - if sole_article_ids: - sole_article_id = sole_article_ids[0] - # Create and append the result entry - result_entry = { - **content["selling_unit"], - "sole_article_id": sole_article_id, - } - search_results.append(result_entry) + sole_article_id = sole_article_ids[0] if sole_article_ids else None + + result_entry = { + **content["selling_unit"], + "sole_article_id": sole_article_id, + } + search_results.append(result_entry) + return search_results