Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 4 additions & 8 deletions database/mongodb_store_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,14 +103,10 @@ async def get_collection(self, collection_suffix: str) -> AsyncIOMotorCollection
return db[collection_name]

async def save_or_update(self, collection_suffix: str, query: Dict, data: Dict) -> bool:
"""Save or update data (upsert)"""
try:
collection = await self.get_collection(collection_suffix)
await collection.update_one(query, {"$set": data}, upsert=True)
return True
except Exception as e:
utils.logger.error(f"[MongoDBStoreBase] Save failed ({self.collection_prefix}_{collection_suffix}): {e}")
return False
"""Upsert data; propagate failures so callers cannot report false success."""
collection = await self.get_collection(collection_suffix)
await collection.update_one(query, {"$set": data}, upsert=True)
return True

async def find_one(self, collection_suffix: str, query: Dict) -> Optional[Dict]:
"""Query a single record"""
Expand Down
5 changes: 5 additions & 0 deletions docs/mongodb-write-errors.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# MongoDB 写入失败处理

MongoDB upsert 成功仍返回 True;写入失败现在抛出原始异常。
平台 store 只有在写入成功后才会执行后续成功日志。自定义调用方应处理异常或让任务失败,
不要继续依赖 False 表示失败。本次没有自动重试,以免隐藏持续性配置/权限错误。
54 changes: 54 additions & 0 deletions tests/test_mongodb_errors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_mongodb_errors.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
# 3. 不得进行大规模爬取或对平台造成运营干扰。
# 4. 应合理控制请求频率,避免给目标平台带来不必要的负担。
# 5. 不得用于任何非法或不当的用途。
#
# 详细许可条款请参阅项目根目录下的LICENSE文件。
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。

from unittest.mock import AsyncMock, Mock

import pytest

from database.mongodb_store_base import MongoDBStoreBase


@pytest.mark.asyncio
async def test_mongodb_failed_write_reaches_the_caller(monkeypatch):
store = MongoDBStoreBase("xhs")
collection = Mock(update_one=AsyncMock(side_effect=OSError("database unavailable")))
monkeypatch.setattr(store, "get_collection", AsyncMock(return_value=collection))
with pytest.raises(OSError, match="database unavailable"):
await store.save_or_update("contents", {"note_id": "1"}, {"title": "test"})


@pytest.mark.asyncio
async def test_mongodb_successful_upsert_retains_its_contract(monkeypatch):
store = MongoDBStoreBase("xhs")
collection = Mock(update_one=AsyncMock())
monkeypatch.setattr(store, "get_collection", AsyncMock(return_value=collection))
assert await store.save_or_update("contents", {"note_id": "1"}, {"title": "test"}) is True
collection.update_one.assert_awaited_once_with({"note_id": "1"}, {"$set": {"title": "test"}}, upsert=True)


@pytest.mark.asyncio
async def test_mongodb_platform_store_does_not_log_false_success(monkeypatch):
from store.xhs._store_impl import XhsMongoStoreImplement
from tools import utils
store = XhsMongoStoreImplement()
monkeypatch.setattr(store.mongo_store, "get_collection", AsyncMock(side_effect=OSError("offline")))
log = Mock()
monkeypatch.setattr(utils.logger, "info", log)
with pytest.raises(OSError, match="offline"):
await store.store_content({"note_id": "1", "title": "example"})
assert not any("Saved note" in str(call) for call in log.call_args_list)