fix(plugin_manager): prevent permanent ERROR state after update timeout

src/plugin_system/plugin_manager.py

@@ -677,6 +677,44 @@ def _get_plugin_update_interval(self, plugin_id: str, plugin_instance: Any) -> O
         # Default: 60 seconds
         return 60.0
 
+    def _record_update_failure(
+        self,
+        plugin_id: str,
+        exc: Optional[Exception] = None,
+    ) -> None:
+        """Apply the standard failure-recovery path for a plugin update.
+
+        Stamps plugin_last_update with the actual failure time so the full
+        configured interval elapses before the next retry, then transitions
+        the plugin back to ENABLED (not ERROR) with structured error context
+        so automatic recovery happens on the next scheduled cycle.
+
+        Args:
+            plugin_id: Plugin identifier
+            exc: The exception that caused the failure, if any.  When None a
+                 synthetic ExecutionFailure exception is constructed from the
+                 timeout/executor-error path.
+        """
+        failure_time = time.time()
+        if exc is not None:
+            err: Exception = exc
+            error_type = type(exc).__name__
+        else:
+            err = Exception(f"Plugin {plugin_id} execution failed (timeout or executor error)")
+            error_type = 'ExecutionFailure'
+
+        error_info = {
+            'error': str(err),
+            'error_type': error_type,
+            'timestamp': failure_time,
+            'recoverable': True,
+        }
+        self.logger.warning("Plugin %s update() failed; will retry after interval", plugin_id)
+        self.plugin_last_update[plugin_id] = failure_time
+        self.state_manager.set_state_with_error(plugin_id, PluginState.ENABLED, error_info, error=err)
+        if self.health_tracker:
+            self.health_tracker.record_failure(plugin_id, err)
+
     def run_scheduled_updates(self, current_time: Optional[float] = None) -> None:
         """
         Trigger plugin updates based on their defined update intervals.
@@ -734,16 +772,10 @@ def monitored_update():
                         if self.health_tracker:
                             self.health_tracker.record_success(plugin_id)
                     else:
-                        # Execution failed (timeout or error)
-                        self.state_manager.set_state(plugin_id, PluginState.ERROR)
-                        if self.health_tracker:
-                            self.health_tracker.record_failure(plugin_id, Exception("Plugin execution failed"))
+                        self._record_update_failure(plugin_id)
                 except Exception as exc:  # pylint: disable=broad-except
                     self.logger.exception("Error updating plugin %s: %s", plugin_id, exc)
-                    self.state_manager.set_state(plugin_id, PluginState.ERROR, error=exc)
-                    # Record failure
-                    if self.health_tracker:
-                        self.health_tracker.record_failure(plugin_id, exc)
+                    self._record_update_failure(plugin_id, exc=exc)
 
     def update_all_plugins(self) -> None:
         """
@@ -769,14 +801,12 @@ def update_all_plugins(self) -> None:
                 if success:
                     self.plugin_last_update[plugin_id] = time.time()
                     self.state_manager.record_update(plugin_id)
-                    # Update state back to ENABLED
                     self.state_manager.set_state(plugin_id, PluginState.ENABLED)
                 else:
-                    # Execution failed
-                    self.state_manager.set_state(plugin_id, PluginState.ERROR)
+                    self._record_update_failure(plugin_id)
             except Exception as exc:  # pylint: disable=broad-except
                 self.logger.exception("Error updating plugin %s: %s", plugin_id, exc)
-                self.state_manager.set_state(plugin_id, PluginState.ERROR, error=exc)
+                self._record_update_failure(plugin_id, exc=exc)
 
     def get_plugin_health_metrics(self) -> Dict[str, Any]:
         """

src/plugin_system/plugin_state.py

@@ -5,6 +5,7 @@
 with state transitions and queries.
 """
 
+import threading
 from enum import Enum
 from typing import Optional, Dict, Any
 from datetime import datetime
@@ -34,6 +35,7 @@ def __init__(self, logger: Optional[logging.Logger] = None) -> None:
             logger: Optional logger instance
         """
         self.logger = logger or get_logger(__name__)
+        self._lock = threading.RLock()
         self._states: Dict[str, PluginState] = {}
         self._state_history: Dict[str, list] = {}
         self._error_info: Dict[str, Dict[str, Any]] = {}
@@ -48,44 +50,44 @@ def set_state(
     ) -> None:
         """
         Set plugin state and record transition.
-        
+
         Args:
             plugin_id: Plugin identifier
             state: New state
             error: Optional error if transitioning to ERROR state
         """
-        old_state = self._states.get(plugin_id, PluginState.UNLOADED)
-        self._states[plugin_id] = state
-
-        # Record state transition
-        if plugin_id not in self._state_history:
-            self._state_history[plugin_id] = []
-
-        transition = {
-            'timestamp': datetime.now(),
-            'from': old_state.value,
-            'to': state.value,
-            'error': str(error) if error else None
-        }
-        self._state_history[plugin_id].append(transition)
-
-        # Store error info if transitioning to ERROR state
-        if state == PluginState.ERROR and error:
-            self._error_info[plugin_id] = {
-                'error': str(error),
-                'error_type': type(error).__name__,
-                'timestamp': datetime.now()
+        with self._lock:
+            old_state = self._states.get(plugin_id, PluginState.UNLOADED)
+            self._states[plugin_id] = state
+
+            if plugin_id not in self._state_history:
+                self._state_history[plugin_id] = []
+
+            transition = {
+                'timestamp': datetime.now(),
+                'from': old_state.value,
+                'to': state.value,
+                'error': str(error) if error else None
             }
-        elif state != PluginState.ERROR:
-            # Clear error info when leaving ERROR state
-            self._error_info.pop(plugin_id, None)
-
-        self.logger.debug(
-            "Plugin %s state transition: %s → %s",
-            plugin_id,
-            old_state.value,
-            state.value
-        )
+            self._state_history[plugin_id].append(transition)
+
+            # Store error info if transitioning to ERROR state
+            if state == PluginState.ERROR and error:
+                self._error_info[plugin_id] = {
+                    'error': str(error),
+                    'error_type': type(error).__name__,
+                    'timestamp': datetime.now()
+                }
+            elif state != PluginState.ERROR:
+                # Clear error info when leaving ERROR state
+                self._error_info.pop(plugin_id, None)
+
+            self.logger.debug(
+                "Plugin %s state transition: %s → %s",
+                plugin_id,
+                old_state.value,
+                state.value
+            )
 
     def get_state(self, plugin_id: str) -> PluginState:
         """
@@ -136,17 +138,82 @@ def get_state_history(self, plugin_id: str) -> list:
         """
         return self._state_history.get(plugin_id, [])
 
+    def set_error_info(self, plugin_id: str, error_info: Dict[str, Any]) -> None:
+        """
+        Persist structured error context without changing plugin state.
+
+        Used for recoverable failures (e.g. update timeout) where the plugin
+        stays ENABLED but the error details should remain queryable.
+
+        Args:
+            plugin_id: Plugin identifier
+            error_info: Arbitrary dict describing the error
+        """
+        with self._lock:
+            self._error_info[plugin_id] = dict(error_info)
+
+    def set_state_with_error(
+        self,
+        plugin_id: str,
+        state: PluginState,
+        error_info: Dict[str, Any],
+        error: Optional[Exception] = None,
+    ) -> None:
+        """Set plugin state and persist error context atomically.
+
+        Unlike calling set_state() then set_error_info() separately, this
+        method holds ``_lock`` for both writes so no reader can observe the
+        new state without the accompanying error context.
+
+        Intentionally does not clear ``_error_info`` the way set_state() does
+        for non-ERROR transitions — this is the recoverable-failure path where
+        the error dict is the entire point.
+
+        Args:
+            plugin_id: Plugin identifier
+            state: New state
+            error_info: Structured error dict to persist alongside the state
+            error: Optional exception recorded in the transition history
+        """
+        with self._lock:
+            old_state = self._states.get(plugin_id, PluginState.UNLOADED)
+            self._states[plugin_id] = state
+
+            if plugin_id not in self._state_history:
+                self._state_history[plugin_id] = []
+            self._state_history[plugin_id].append({
+                'timestamp': datetime.now(),
+                'from': old_state.value,
+                'to': state.value,
+                'error': str(error) if error else None,
+            })
+
+            self._error_info[plugin_id] = dict(error_info)
+
+            self.logger.debug(
+                "Plugin %s state transition: %s → %s (recoverable error stored)",
+                plugin_id,
+                old_state.value,
+                state.value,
+            )
+
     def get_error_info(self, plugin_id: str) -> Optional[Dict[str, Any]]:
         """
-        Get error information for a plugin in ERROR state.
-
+        Get error information for a plugin.
+
+        Returns the stored error dict whether the plugin is in ERROR state or
+        still ENABLED after a recoverable failure. Returns a shallow copy so
+        callers cannot mutate the stored snapshot.
+
         Args:
             plugin_id: Plugin identifier
-            
+
         Returns:
-            Error information dict or None
+            Copy of the error information dict, or None
         """
-        return self._error_info.get(plugin_id)
+        with self._lock:
+            info = self._error_info.get(plugin_id)
+            return dict(info) if info is not None else None
 
     def record_update(self, plugin_id: str) -> None:
         """Record that plugin update() was called."""