diff --git a/mssql_python/pybind/connection/connection.cpp b/mssql_python/pybind/connection/connection.cpp index aaf172b9..711ae2fb 100644 --- a/mssql_python/pybind/connection/connection.cpp +++ b/mssql_python/pybind/connection/connection.cpp @@ -191,7 +191,12 @@ void Connection::commit() { } updateLastUsed(); LOG("Committing transaction"); - SQLRETURN ret = SQLEndTran_ptr(SQL_HANDLE_DBC, _dbcHandle->get(), SQL_COMMIT); + SQLRETURN ret; + { + // Release the GIL during the blocking SQLEndTran network round-trip. + py::gil_scoped_release release; + ret = SQLEndTran_ptr(SQL_HANDLE_DBC, _dbcHandle->get(), SQL_COMMIT); + } checkError(ret); } @@ -201,7 +206,12 @@ void Connection::rollback() { } updateLastUsed(); LOG("Rolling back transaction"); - SQLRETURN ret = SQLEndTran_ptr(SQL_HANDLE_DBC, _dbcHandle->get(), SQL_ROLLBACK); + SQLRETURN ret; + { + // Release the GIL during the blocking SQLEndTran network round-trip. + py::gil_scoped_release release; + ret = SQLEndTran_ptr(SQL_HANDLE_DBC, _dbcHandle->get(), SQL_ROLLBACK); + } checkError(ret); } diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 47a2a255..d6759cfc 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1383,6 +1383,8 @@ SQLRETURN SQLGetTypeInfo_Wrapper(SqlHandlePtr StatementHandle, SQLSMALLINT DataT ThrowStdException("SQLGetTypeInfo function not loaded"); } + // Release the GIL during the blocking ODBC catalog call (issue #540). + py::gil_scoped_release release; return SQLGetTypeInfo_ptr(StatementHandle->get(), DataType); } @@ -1405,6 +1407,8 @@ SQLRETURN SQLProcedures_wrap(SqlHandlePtr StatementHandle, const py::object& cat std::vector schemaBuf = WStringToSQLWCHAR(schema); std::vector procedureBuf = WStringToSQLWCHAR(procedure); + // Release the GIL during the blocking ODBC catalog call (issue #540). + py::gil_scoped_release release; return SQLProcedures_ptr( StatementHandle->get(), catalog.empty() ? nullptr : catalogBuf.data(), catalog.empty() ? 0 : SQL_NTS, schema.empty() ? nullptr : schemaBuf.data(), @@ -1412,6 +1416,7 @@ SQLRETURN SQLProcedures_wrap(SqlHandlePtr StatementHandle, const py::object& cat procedure.empty() ? 0 : SQL_NTS); #else // Windows implementation + py::gil_scoped_release release; return SQLProcedures_ptr( StatementHandle->get(), catalog.empty() ? nullptr : (SQLWCHAR*)catalog.c_str(), catalog.empty() ? 0 : SQL_NTS, schema.empty() ? nullptr : (SQLWCHAR*)schema.c_str(), @@ -1450,6 +1455,8 @@ SQLRETURN SQLForeignKeys_wrap(SqlHandlePtr StatementHandle, const py::object& pk std::vector fkSchemaBuf = WStringToSQLWCHAR(fkSchema); std::vector fkTableBuf = WStringToSQLWCHAR(fkTable); + // Release the GIL during the blocking ODBC catalog call (issue #540). + py::gil_scoped_release release; return SQLForeignKeys_ptr( StatementHandle->get(), pkCatalog.empty() ? nullptr : pkCatalogBuf.data(), pkCatalog.empty() ? 0 : SQL_NTS, pkSchema.empty() ? nullptr : pkSchemaBuf.data(), @@ -1460,6 +1467,7 @@ SQLRETURN SQLForeignKeys_wrap(SqlHandlePtr StatementHandle, const py::object& pk fkTable.empty() ? 0 : SQL_NTS); #else // Windows implementation + py::gil_scoped_release release; return SQLForeignKeys_ptr( StatementHandle->get(), pkCatalog.empty() ? nullptr : (SQLWCHAR*)pkCatalog.c_str(), pkCatalog.empty() ? 0 : SQL_NTS, pkSchema.empty() ? nullptr : (SQLWCHAR*)pkSchema.c_str(), @@ -1487,6 +1495,8 @@ SQLRETURN SQLPrimaryKeys_wrap(SqlHandlePtr StatementHandle, const py::object& ca std::vector schemaBuf = WStringToSQLWCHAR(schema); std::vector tableBuf = WStringToSQLWCHAR(table); + // Release the GIL during the blocking ODBC catalog call (issue #540). + py::gil_scoped_release release; return SQLPrimaryKeys_ptr( StatementHandle->get(), catalog.empty() ? nullptr : catalogBuf.data(), catalog.empty() ? 0 : SQL_NTS, schema.empty() ? nullptr : schemaBuf.data(), @@ -1494,6 +1504,7 @@ SQLRETURN SQLPrimaryKeys_wrap(SqlHandlePtr StatementHandle, const py::object& ca table.empty() ? 0 : SQL_NTS); #else // Windows implementation + py::gil_scoped_release release; return SQLPrimaryKeys_ptr( StatementHandle->get(), catalog.empty() ? nullptr : (SQLWCHAR*)catalog.c_str(), catalog.empty() ? 0 : SQL_NTS, schema.empty() ? nullptr : (SQLWCHAR*)schema.c_str(), @@ -1519,6 +1530,8 @@ SQLRETURN SQLStatistics_wrap(SqlHandlePtr StatementHandle, const py::object& cat std::vector schemaBuf = WStringToSQLWCHAR(schema); std::vector tableBuf = WStringToSQLWCHAR(table); + // Release the GIL during the blocking ODBC catalog call (issue #540). + py::gil_scoped_release release; return SQLStatistics_ptr( StatementHandle->get(), catalog.empty() ? nullptr : catalogBuf.data(), catalog.empty() ? 0 : SQL_NTS, schema.empty() ? nullptr : schemaBuf.data(), @@ -1526,6 +1539,7 @@ SQLRETURN SQLStatistics_wrap(SqlHandlePtr StatementHandle, const py::object& cat table.empty() ? 0 : SQL_NTS, unique, reserved); #else // Windows implementation + py::gil_scoped_release release; return SQLStatistics_ptr( StatementHandle->get(), catalog.empty() ? nullptr : (SQLWCHAR*)catalog.c_str(), catalog.empty() ? 0 : SQL_NTS, schema.empty() ? nullptr : (SQLWCHAR*)schema.c_str(), @@ -1554,6 +1568,8 @@ SQLRETURN SQLColumns_wrap(SqlHandlePtr StatementHandle, const py::object& catalo std::vector tableBuf = WStringToSQLWCHAR(tableStr); std::vector columnBuf = WStringToSQLWCHAR(columnStr); + // Release the GIL during the blocking ODBC catalog call (issue #540). + py::gil_scoped_release release; return SQLColumns_ptr( StatementHandle->get(), catalogStr.empty() ? nullptr : catalogBuf.data(), catalogStr.empty() ? 0 : SQL_NTS, schemaStr.empty() ? nullptr : schemaBuf.data(), @@ -1562,6 +1578,7 @@ SQLRETURN SQLColumns_wrap(SqlHandlePtr StatementHandle, const py::object& catalo columnStr.empty() ? 0 : SQL_NTS); #else // Windows implementation + py::gil_scoped_release release; return SQLColumns_ptr( StatementHandle->get(), catalogStr.empty() ? nullptr : (SQLWCHAR*)catalogStr.c_str(), catalogStr.empty() ? 0 : SQL_NTS, @@ -1702,7 +1719,14 @@ SQLRETURN SQLExecDirect_wrap(SqlHandlePtr StatementHandle, const std::wstring& Q #else queryPtr = const_cast(Query.c_str()); #endif - SQLRETURN ret = SQLExecDirect_ptr(StatementHandle->get(), queryPtr, SQL_NTS); + SQLRETURN ret; + { + // Release the GIL during the blocking ODBC call so that other Python + // threads (e.g. asyncio event loop, heartbeat threads) can run while + // SQL Server executes the query. See issue #540. + py::gil_scoped_release release; + ret = SQLExecDirect_ptr(StatementHandle->get(), queryPtr, SQL_NTS); + } if (!SQL_SUCCEEDED(ret)) { LOG("SQLExecDirect: Query execution failed - SQLRETURN=%d", ret); } @@ -1774,8 +1798,13 @@ SQLRETURN SQLTables_wrap(SqlHandlePtr StatementHandle, const std::wstring& catal } #endif - SQLRETURN ret = SQLTables_ptr(StatementHandle->get(), catalogPtr, catalogLen, schemaPtr, - schemaLen, tablePtr, tableLen, tableTypePtr, tableTypeLen); + SQLRETURN ret; + { + // Release the GIL during the blocking ODBC catalog call (issue #540). + py::gil_scoped_release release; + ret = SQLTables_ptr(StatementHandle->get(), catalogPtr, catalogLen, schemaPtr, + schemaLen, tablePtr, tableLen, tableTypePtr, tableTypeLen); + } LOG("SQLTables: Catalog metadata query %s - SQLRETURN=%d", SQL_SUCCEEDED(ret) ? "succeeded" : "failed", ret); @@ -1832,7 +1861,11 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, // is the fastest way to submit a SQL statement for one-time execution // according to DDBC documentation - // https://learn.microsoft.com/en-us/sql/odbc/reference/syntax/sqlexecdirect-function?view=sql-server-ver16 - rc = SQLExecDirect_ptr(hStmt, queryPtr, SQL_NTS); + { + // Release the GIL during the blocking ODBC call (issue #540). + py::gil_scoped_release release; + rc = SQLExecDirect_ptr(hStmt, queryPtr, SQL_NTS); + } if (!SQL_SUCCEEDED(rc) && rc != SQL_NO_DATA) { LOG("SQLExecute: Direct execution failed (non-parameterized query) " "- SQLRETURN=%d", @@ -1846,7 +1879,11 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, // element assert(isStmtPrepared.size() == 1); if (usePrepare) { - rc = SQLPrepare_ptr(hStmt, queryPtr, SQL_NTS); + { + // Release the GIL during the blocking SQLPrepare network call. + py::gil_scoped_release release; + rc = SQLPrepare_ptr(hStmt, queryPtr, SQL_NTS); + } if (!SQL_SUCCEEDED(rc)) { LOG("SQLExecute: SQLPrepare failed - SQLRETURN=%d, " "statement_handle=%p", @@ -1878,12 +1915,27 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, return rc; } - rc = SQLExecute_ptr(hStmt); + { + // Release the GIL during the blocking SQLExecute network call. + py::gil_scoped_release release; + rc = SQLExecute_ptr(hStmt); + } if (rc == SQL_NEED_DATA) { LOG("SQLExecute: SQL_NEED_DATA received - Starting DAE " "(Data-At-Execution) loop for large parameter streaming"); SQLPOINTER paramToken = nullptr; - while ((rc = SQLParamData_ptr(hStmt, ¶mToken)) == SQL_NEED_DATA) { + // For DAE, release the GIL only around individual ODBC calls; + // Python type inspection of the parameter happens between calls + // and requires the GIL. + auto paramData = [&](SQLPOINTER* tok) { + py::gil_scoped_release release; + return SQLParamData_ptr(hStmt, tok); + }; + auto putData = [&](SQLPOINTER data, SQLLEN len) { + py::gil_scoped_release release; + return SQLPutData_ptr(hStmt, data, len); + }; + while ((rc = paramData(¶mToken)) == SQL_NEED_DATA) { // Finding the paramInfo that matches the returned token const ParamInfo* matchedInfo = nullptr; for (auto& info : paramInfos) { @@ -1897,7 +1949,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, } const py::object& pyObj = matchedInfo->dataPtr; if (pyObj.is_none()) { - SQLPutData_ptr(hStmt, nullptr, 0); + putData(nullptr, 0); continue; } if (py::isinstance(pyObj)) { @@ -1923,8 +1975,8 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, ThrowStdException("Chunk size exceeds maximum " "allowed by SQLLEN"); } - rc = SQLPutData_ptr(hStmt, (SQLPOINTER)(dataPtr + offset), - static_cast(lenBytes)); + rc = putData((SQLPOINTER)(dataPtr + offset), + static_cast(lenBytes)); if (!SQL_SUCCEEDED(rc)) { LOG("SQLExecute: SQLPutData failed for " "SQL_C_WCHAR chunk - offset=%zu", @@ -1958,8 +2010,8 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, while (offset < totalBytes) { size_t len = std::min(chunkBytes, totalBytes - offset); - rc = SQLPutData_ptr(hStmt, (SQLPOINTER)(dataPtr + offset), - static_cast(len)); + rc = putData((SQLPOINTER)(dataPtr + offset), + static_cast(len)); if (!SQL_SUCCEEDED(rc)) { LOG("SQLExecute: SQLPutData failed for " "SQL_C_CHAR chunk - offset=%zu", @@ -1980,8 +2032,8 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, const size_t chunkSize = DAE_CHUNK_SIZE; for (size_t offset = 0; offset < totalBytes; offset += chunkSize) { size_t len = std::min(chunkSize, totalBytes - offset); - rc = SQLPutData_ptr(hStmt, (SQLPOINTER)(dataPtr + offset), - static_cast(len)); + rc = putData((SQLPOINTER)(dataPtr + offset), + static_cast(len)); if (!SQL_SUCCEEDED(rc)) { LOG("SQLExecute: SQLPutData failed for " "binary/bytes chunk - offset=%zu", @@ -2706,7 +2758,12 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::wst queryPtr = const_cast(query.c_str()); LOG("SQLExecuteMany: Using wide string query directly"); #endif - RETCODE rc = SQLPrepare_ptr(hStmt, queryPtr, SQL_NTS); + RETCODE rc; + { + // Release the GIL during the blocking SQLPrepare network call. + py::gil_scoped_release release; + rc = SQLPrepare_ptr(hStmt, queryPtr, SQL_NTS); + } if (!SQL_SUCCEEDED(rc)) { LOG("SQLExecuteMany: SQLPrepare failed - rc=%d", rc); return rc; @@ -2747,7 +2804,11 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::wst } LOG("SQLExecuteMany: PARAMSET_SIZE set to %zu", paramSetSize); - rc = SQLExecute_ptr(hStmt); + { + // Release the GIL during the blocking SQLExecute network call. + py::gil_scoped_release release; + rc = SQLExecute_ptr(hStmt); + } LOG("SQLExecuteMany: SQLExecute completed - rc=%d", rc); return rc; } else { @@ -2767,12 +2828,20 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::wst } LOG("SQLExecuteMany: Parameters bound for row %zu", rowIndex); - rc = SQLExecute_ptr(hStmt); + { + // Release the GIL during the blocking SQLExecute network call. + py::gil_scoped_release release; + rc = SQLExecute_ptr(hStmt); + } LOG("SQLExecuteMany: SQLExecute for row %zu - initial_rc=%d", rowIndex, rc); size_t dae_chunk_count = 0; while (rc == SQL_NEED_DATA) { SQLPOINTER token; - rc = SQLParamData_ptr(hStmt, &token); + { + // Release the GIL around the blocking SQLParamData call. + py::gil_scoped_release release; + rc = SQLParamData_ptr(hStmt, &token); + } LOG("SQLExecuteMany: SQLParamData called - chunk=%zu, rc=%d, " "token=%p", dae_chunk_count, rc, token); @@ -2795,7 +2864,10 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::wst LOG("SQLExecuteMany: Sending string DAE data - chunk=%zu, " "length=%lld", dae_chunk_count, static_cast(data_len)); - rc = SQLPutData_ptr(hStmt, (SQLPOINTER)data.c_str(), data_len); + rc = [&] { + py::gil_scoped_release release; + return SQLPutData_ptr(hStmt, (SQLPOINTER)data.c_str(), data_len); + }(); if (!SQL_SUCCEEDED(rc) && rc != SQL_NEED_DATA) { LOG("SQLExecuteMany: SQLPutData(string) failed - " "chunk=%zu, rc=%d", @@ -2808,7 +2880,10 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const std::wst LOG("SQLExecuteMany: Sending bytes/bytearray DAE data - " "chunk=%zu, length=%lld", dae_chunk_count, static_cast(data_len)); - rc = SQLPutData_ptr(hStmt, (SQLPOINTER)data.c_str(), data_len); + rc = [&] { + py::gil_scoped_release release; + return SQLPutData_ptr(hStmt, (SQLPOINTER)data.c_str(), data_len); + }(); if (!SQL_SUCCEEDED(rc) && rc != SQL_NEED_DATA) { LOG("SQLExecuteMany: SQLPutData(bytes) failed - " "chunk=%zu, rc=%d", @@ -2917,6 +2992,8 @@ SQLRETURN SQLSpecialColumns_wrap(SqlHandlePtr StatementHandle, SQLSMALLINT ident std::vector schemaBuf = WStringToSQLWCHAR(schema); std::vector tableBuf = WStringToSQLWCHAR(table); + // Release the GIL during the blocking ODBC catalog call (issue #540). + py::gil_scoped_release release; return SQLSpecialColumns_ptr( StatementHandle->get(), identifierType, catalog.empty() ? nullptr : catalogBuf.data(), catalog.empty() ? 0 : SQL_NTS, schema.empty() ? nullptr : schemaBuf.data(), @@ -2924,6 +3001,7 @@ SQLRETURN SQLSpecialColumns_wrap(SqlHandlePtr StatementHandle, SQLSMALLINT ident table.empty() ? 0 : SQL_NTS, scope, nullable); #else // Windows implementation + py::gil_scoped_release release; return SQLSpecialColumns_ptr( StatementHandle->get(), identifierType, catalog.empty() ? nullptr : (SQLWCHAR*)catalog.c_str(), catalog.empty() ? 0 : SQL_NTS, @@ -2941,6 +3019,8 @@ SQLRETURN SQLFetch_wrap(SqlHandlePtr StatementHandle) { DriverLoader::getInstance().loadDriver(); // Load the driver } + // Release the GIL during the blocking ODBC call (issue #540). + py::gil_scoped_release release; return SQLFetch_ptr(StatementHandle->get()); } @@ -2955,7 +3035,11 @@ py::object FetchLobColumnData(SQLHSTMT hStmt, SQLUSMALLINT colIndex, SQLSMALLINT ++loopCount; std::vector chunk(DAE_CHUNK_SIZE, 0); SQLLEN actualRead = 0; - ret = SQLGetData_ptr(hStmt, colIndex, cType, chunk.data(), DAE_CHUNK_SIZE, &actualRead); + { + // Release the GIL during blocking SQLGetData LOB streaming (issue #540). + py::gil_scoped_release release; + ret = SQLGetData_ptr(hStmt, colIndex, cType, chunk.data(), DAE_CHUNK_SIZE, &actualRead); + } if (ret == SQL_ERROR || !SQL_SUCCEEDED(ret) && ret != SQL_SUCCESS_WITH_INFO) { std::ostringstream oss; @@ -3715,7 +3799,12 @@ SQLRETURN SQLFetchScroll_wrap(SqlHandlePtr StatementHandle, SQLSMALLINT FetchOri SQLFreeStmt_ptr(StatementHandle->get(), SQL_UNBIND); // Perform scroll operation - SQLRETURN ret = SQLFetchScroll_ptr(StatementHandle->get(), FetchOrientation, FetchOffset); + SQLRETURN ret; + { + // Release the GIL during the blocking ODBC fetch (issue #540). + py::gil_scoped_release release; + ret = SQLFetchScroll_ptr(StatementHandle->get(), FetchOrientation, FetchOffset); + } // If successful and caller wants data, retrieve it if (SQL_SUCCEEDED(ret) && row_data.size() == 0) { @@ -3903,7 +3992,12 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum const std::vector& lobColumns, const std::string& charEncoding = "utf-8") { LOG("FetchBatchData: Fetching data in batches"); - SQLRETURN ret = SQLFetchScroll_ptr(hStmt, SQL_FETCH_NEXT, 0); + SQLRETURN ret; + { + // Release the GIL during the blocking ODBC fetch (issue #540). + py::gil_scoped_release release; + ret = SQLFetchScroll_ptr(hStmt, SQL_FETCH_NEXT, 0); + } if (ret == SQL_NO_DATA) { LOG("FetchBatchData: No data to fetch"); return ret; @@ -4378,7 +4472,11 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch "SQLGetData path", lobColumns.size()); while (numRowsFetched < (SQLULEN)fetchSize) { - ret = SQLFetch_ptr(hStmt); + { + // Release GIL during the blocking fetch (issue #540). + py::gil_scoped_release release; + ret = SQLFetch_ptr(hStmt); + } if (ret == SQL_NO_DATA) break; if (!SQL_SUCCEEDED(ret)) @@ -4763,7 +4861,11 @@ SQLRETURN FetchArrowBatch_wrap( // Adjust fetch size for final batch to avoid overfetching fetchStateGuard.setRowArraySize(spaceLeftInArrowBatch); } - ret = SQLFetch_ptr(hStmt); + { + // Release GIL during the blocking ODBC fetch (issue #540). + py::gil_scoped_release release; + ret = SQLFetch_ptr(hStmt); + } if (ret == SQL_NO_DATA) { ret = SQL_SUCCESS; // Normal completion break; @@ -5515,7 +5617,11 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows, "SQLGetData path", lobColumns.size()); while (true) { - ret = SQLFetch_ptr(hStmt); + { + // Release GIL during the blocking fetch (issue #540). + py::gil_scoped_release release; + ret = SQLFetch_ptr(hStmt); + } if (ret == SQL_NO_DATA) break; if (!SQL_SUCCEEDED(ret)) @@ -5630,7 +5736,11 @@ SQLRETURN FetchOne_wrap(SqlHandlePtr StatementHandle, py::list& row, SQLFreeStmt_ptr(hStmt, SQL_UNBIND); // Assume hStmt is already allocated and a query has been executed - ret = SQLFetch_ptr(hStmt); + { + // Release the GIL during the blocking ODBC fetch (issue #540). + py::gil_scoped_release release; + ret = SQLFetch_ptr(hStmt); + } if (SQL_SUCCEEDED(ret)) { // Retrieve column count SQLSMALLINT colCount = SQLNumResultCols_wrap(StatementHandle); @@ -5654,6 +5764,8 @@ SQLRETURN SQLMoreResults_wrap(SqlHandlePtr StatementHandle) { DriverLoader::getInstance().loadDriver(); // Load the driver } + // Release the GIL during the blocking ODBC call (issue #540). + py::gil_scoped_release release; return SQLMoreResults_ptr(StatementHandle->get()); }