PERF: Optimize execute() hot path: soft reset, prepare caching, and guarded diagnostics

benchmarks/perf-benchmarking.py

@@ -49,6 +49,11 @@ def _init_conn_strings():
  CONN_STR_PYODBC = f"Driver={{ODBC Driver 18 for SQL Server}};{CONN_STR}"
  else:
  CONN_STR_PYODBC = CONN_STR
+ # mssql-python manages its own driver and rejects Driver= in the
+ # connection string. Strip it so both drivers can share one env var.
+ parts = [p for p in CONN_STR.split(";") if not p.strip().lower().startswith("driver=")]
+ CONN_STR = ";".join(parts)
+
 
 
 class BenchmarkResult:

mssql_python/cursor.py

@@ -12,6 +12,7 @@
 # pylint: disable=too-many-lines # Large file due to comprehensive DB-API 2.0 implementation
 
 import decimal
+import logging
 import uuid
 import datetime
 import warnings
@@ -747,6 +748,24 @@ def _reset_cursor(self) -> None:
 
  # Reinitialize the statement handle
  self._initialize_cursor()
+ self.is_stmt_prepared = [False]
+
+ def _soft_reset_cursor(self) -> None:
+ """Lightweight reset: close cursor and unbind params without freeing the HSTMT.
+
+ Preserves the prepared statement plan on the server so repeated
+ executions of the same SQL skip SQLPrepare entirely.
+ """
+ if self.hstmt:
+ ret = ddbc_bindings.DDBCSQLResetStmt(self.hstmt)
+ try:
+ check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret)
+ except Exception:
+ logger.warning("_soft_reset_cursor failed; falling back to full reset")
+ self._reset_cursor()
+ self.last_executed_stmt = ""
+ return
+ self._clear_rownumber()
 
  def close(self) -> None:
  """
@@ -1363,8 +1382,10 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state
 
  self._check_closed() # Check if the cursor is closed
  if reset_cursor:
- logger.debug("execute: Resetting cursor state")
- self._reset_cursor()
+ if self.hstmt:
+ self._soft_reset_cursor()
+ else:
+ self._reset_cursor()
  else:
  # Close just the ODBC cursor (not the statement handle) so the
  # prepared plan can be reused. SQLFreeStmt(SQL_CLOSE) releases
@@ -1409,20 +1430,22 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state
  # Check if single parameter is a nested container that should be unwrapped
  # e.g., execute("SELECT ?", (value,)) vs execute("SELECT ?, ?", ((1, 2),))
  if isinstance(parameters, tuple) and len(parameters) == 1:
- # Could be either (value,) for single param or ((tuple),) for nested
- # Check if it's a nested container
  if isinstance(parameters[0], (tuple, list, dict)):
  actual_params = parameters[0]
  else:
  actual_params = parameters
  else:
  actual_params = parameters
 
- # Convert parameters based on detected style
- operation, converted_params = detect_and_convert_parameters(operation, actual_params)
-
- # Convert back to list format expected by the binding code
- parameters = list(converted_params)
+ # Skip detect_and_convert_parameters when re-executing the same SQL —
+ # the parameter style (qmark vs pyformat) won't change between calls.
+ if operation == self.last_executed_stmt and isinstance(actual_params, (tuple, list)):
+ parameters = list(actual_params)
+ else:
+ operation, converted_params = detect_and_convert_parameters(
+ operation, actual_params
+ )
+ parameters = list(converted_params)
  else:
  parameters = []
 
@@ -1450,35 +1473,36 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state
  paraminfo = self._create_parameter_types_list(param, param_info, parameters, i)
  parameters_type.append(paraminfo)
 
- # TODO: Use a more sophisticated string compare that handles redundant spaces etc.
- # Also consider storing last query's hash instead of full query string. This will help
- # in low-memory conditions
- # (Ex: huge number of parallel queries with huge query string sizes)
- if operation != self.last_executed_stmt:
- # Executing a new statement. Reset is_stmt_prepared to false
+ # Prepare caching: skip SQLPrepare when re-executing the same SQL
+ # with parameters. The HSTMT is reused via _soft_reset_cursor, so the
+ # server-side plan from the previous SQLPrepare is still valid.
+ same_sql = parameters and operation == self.last_executed_stmt and self.is_stmt_prepared[0]
+ if not same_sql:
  self.is_stmt_prepared = [False]
+ effective_use_prepare = use_prepare and not same_sql
 
- for i, param in enumerate(parameters):
- logger.debug(
- """Parameter number: %s, Parameter: %s,
- Param Python Type: %s, ParamInfo: %s, %s, %s, %s, %s""",
- i + 1,
- param,
- str(type(param)),
- parameters_type[i].paramSQLType,
- parameters_type[i].paramCType,
- parameters_type[i].columnSize,
- parameters_type[i].decimalDigits,
- parameters_type[i].inputOutputType,
- )
+ if logger.isEnabledFor(logging.DEBUG):
+ for i, param in enumerate(parameters):
+ logger.debug(
+ """Parameter number: %s, Parameter: %s,
+ Param Python Type: %s, ParamInfo: %s, %s, %s, %s, %s""",
+ i + 1,
+ param,
+ str(type(param)),
+ parameters_type[i].paramSQLType,
+ parameters_type[i].paramCType,
+ parameters_type[i].columnSize,
+ parameters_type[i].decimalDigits,
+ parameters_type[i].inputOutputType,
+ )
 
  ret = ddbc_bindings.DDBCSQLExecute(
  self.hstmt,
  operation,
  parameters,
  parameters_type,
  self.is_stmt_prepared,
-use_prepare,
+effective_use_prepare,
  encoding_settings,
  )
  # Check return code
@@ -1491,8 +1515,12 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state
  self._reset_cursor()
  raise
 
- # Capture any diagnostic messages (SQL_SUCCESS_WITH_INFO, etc.)
- if self.hstmt:
+ # Capture diagnostic messages only on SQL_SUCCESS_WITH_INFO.
+ # SQL_SUCCESS has no records — calling DDBCSQLGetAllDiagRecords on it
+ # costs ~10ms/call (driver scans internal state to find nothing).
+ # SQL_ERROR is already handled by check_error() above which extracts
+ # diagnostics and raises.
+ if ret == ddbc_sql_const.SQL_SUCCESS_WITH_INFO.value and self.hstmt:
  self.messages.extend(ddbc_bindings.DDBCSQLGetAllDiagRecords(self.hstmt))
 
  self.last_executed_stmt = operation

mssql_python/pybind/ddbc_bindings.cpp

@@ -1378,6 +1378,32 @@ void SqlHandle::close_cursor() {
  }
 }
 
+SQLRETURN SQLResetStmt_wrap(SqlHandlePtr statementHandle) {
+ if (!statementHandle || !statementHandle->get()) {
+ return SQL_INVALID_HANDLE;
+ }
+ if (statementHandle->isImplicitlyFreed()) {
+ return SQL_INVALID_HANDLE;
+ }
+ if (!SQLFreeStmt_ptr) {
+ DriverLoader::getInstance().loadDriver();
+ }
+ SQLHANDLE hStmt = statementHandle->get();
+
+ SQLRETURN rc;
+ {
+ py::gil_scoped_release release;
+ rc = SQLFreeStmt_ptr(hStmt, SQL_CLOSE);
+ if (SQL_SUCCEEDED(rc)) {
+ rc = SQLFreeStmt_ptr(hStmt, SQL_RESET_PARAMS);
+ }
+ if (SQL_SUCCEEDED(rc) && SQLSetStmtAttr_ptr) {
+ rc = SQLSetStmtAttr_ptr(hStmt, SQL_ATTR_PARAMSET_SIZE, (SQLPOINTER)1, 0);
+ }
+ }
+ return rc;
+}
+
 SQLRETURN SQLGetTypeInfo_Wrapper(SqlHandlePtr StatementHandle, SQLSMALLINT DataType) {
  if (!SQLGetTypeInfo_ptr) {
  ThrowStdException("SQLGetTypeInfo function not loaded");
@@ -5800,6 +5826,7 @@ PYBIND11_MODULE(ddbc_bindings, m) {
  py::arg("wcharEncoding") = "utf-16le");
  m.def("DDBCSQLFetchArrowBatch", &FetchArrowBatch_wrap, "Fetch an arrow batch of given length from the result set");
  m.def("DDBCSQLFreeHandle", &SQLFreeHandle_wrap, "Free a handle");
+ m.def("DDBCSQLResetStmt", &SQLResetStmt_wrap, "Close cursor and unbind params without freeing HSTMT");
  m.def("DDBCSQLCheckError", &SQLCheckError_Wrap, "Check for driver errors");
  m.def("DDBCSQLGetAllDiagRecords", &SQLGetAllDiagRecords,
  "Get all diagnostic records for a handle", py::arg("handle"));

mssql_python/pybind/ddbc_bindings.h

@@ -379,6 +379,7 @@ class SqlHandle {
  SQLSMALLINT type() const;
  void free();
  void close_cursor();
+ bool isImplicitlyFreed() const { return _implicitly_freed; }
 
  // Mark this handle as implicitly freed (freed by parent handle)
  // This prevents double-free attempts when the ODBC driver automatically