Execute Stored Procedures Directly (#2154)

* Initial implementation for exec cstmt directly

* Regression fix, continue to do extra metadata lookup call to fetch SP param name info

* Fixes p1

* Fixes p2

* Fixes p3

* Fixes p4

* Updated test to test erroring out on retrieving out param value

* Revert "Fixes p4"

This reverts commit b19dfb0.

* UDF fix

* Accounted for premature closing of statements

* Datetime out of range fix

* Cstmts executed internally by the driver for XA transactions send out params as non-PLP over the wire for nvarchar and binary types

* Accounted for zero param sproc

* Fixed setting/registering of out of order params

* Datetime fractional seconds correction

* DTC tests

* Zero param sproc test; Error test for casting of sproc return value

* Test shuffled ordering of registering and setting cstmt params

* UDF test

* Skip return values from RPC on statement close if there are any

* Test mix of index a param names for cstmt

* Removed static callRpcDirectly variable

* Test cleanup

* Fixed invalid param test

* Fixed test for JDK 8

* Formatting; Removed comma counting

* Removed getter/setter enum

* Removed comments regarding Yukon in IOBuffer

* Changed returnValueIsAccessed to isReturnValueAccessed

* Fixed intellij wildcard imports

* Renamed DTC test class and test

* Revert "Removed getter/setter enum"

This reverts commit 2b35e53.

* New connection string property to toggle sp_sproc_columns calls

* Deleted DTC test

* Removed non-driver error from TestResource

* Fixed switch case for date/time types in dtv class

* SQLServerConnTest and RequestBoundaryMethodTest update

* Removed DTC test group

* Fixed docs/comments for new CS prop

* Fixed setting/registering of when using only named parameters

* RPC call check considers whether the call is an internal encryption query call

* Switched from useFastCallableStatements to useFlexibleCallableStatements

* Fixed typo in CS prop setter/getter

* Include enclave package in RPC call

* Code Review Changes; Fixed threading issue

* Merge conflict fix p2.

* Update src/main/java/com/microsoft/sqlserver/jdbc/ISQLServerDataSource.java

Co-authored-by: David Engel <[email protected]>

* Update src/main/java/com/microsoft/sqlserver/jdbc/SQLServerCallableStatement.java

Co-authored-by: David Engel <[email protected]>

* Update src/main/java/com/microsoft/sqlserver/jdbc/SQLServerStatement.java

Co-authored-by: David Engel <[email protected]>

* Update src/main/java/com/microsoft/sqlserver/jdbc/SQLServerPreparedStatement.java

Co-authored-by: David Engel <[email protected]>

* PR review; initial named parameter and index parameter restrictions

---------

Co-authored-by: David Engel <[email protected]>

Author: tkyc

pom.xml

@@ -43,7 +43,7 @@
 			xAzureSQLDW - - - - For tests not compatible with Azure Data Warehouse - 
 			xAzureSQLMI - - - - For tests not compatible with Azure SQL Managed Instance 
 			NTLM  - - - - - - - For tests using NTLM Authentication mode (excluded by default)
-			kerberos -  - - - - For tests using Kerberos authentication (excluded by default)
+			kerberos  - - - - - For tests using Kerberos authentication (excluded by default)
 			reqExternalSetup - For tests requiring external setup (excluded by default)
 			clientCertAuth - - For tests requiring client certificate authentication 
 			setup (excluded by default) - - - - - - - - - - - - - - - - - - - - - - - 

src/main/java/com/microsoft/sqlserver/jdbc/IOBuffer.java

@@ -466,6 +466,7 @@ static final String getEncryptionLevel(int level) {
     final static int COLINFO_STATUS_DIFFERENT_NAME = 0x20;
 
     final static int MAX_FRACTIONAL_SECONDS_SCALE = 7;
+    final static int DEFAULT_FRACTIONAL_SECONDS_SCALE = 3;
 
     final static Timestamp MAX_TIMESTAMP = Timestamp.valueOf("2079-06-06 23:59:59");
     final static Timestamp MIN_TIMESTAMP = Timestamp.valueOf("1900-01-01 00:00:00");
@@ -4788,7 +4789,7 @@ void writeVMaxHeader(long headerLength, boolean isNull, SQLCollation collation)
      * Utility for internal writeRPCString calls
      */
     void writeRPCStringUnicode(String sValue) throws SQLServerException {
-        writeRPCStringUnicode(null, sValue, false, null);
+        writeRPCStringUnicode(null, sValue, false, null, false);
     }
 
     /**
@@ -4803,8 +4804,8 @@ void writeRPCStringUnicode(String sValue) throws SQLServerException {
      * @param collation
      *        the collation of the data value
      */
-    void writeRPCStringUnicode(String sName, String sValue, boolean bOut,
-            SQLCollation collation) throws SQLServerException {
+    void writeRPCStringUnicode(String sName, String sValue, boolean bOut, SQLCollation collation,
+            boolean isNonPLP) throws SQLServerException {
         boolean bValueNull = (sValue == null);
         int nValueLen = bValueNull ? 0 : (2 * sValue.length());
         // Textual RPC requires a collation. If none is provided, as is the case when
@@ -4816,10 +4817,9 @@ void writeRPCStringUnicode(String sName, String sValue, boolean bOut,
          * Use PLP encoding if either OUT params were specified or if the user query exceeds
          * DataTypes.SHORT_VARTYPE_MAX_BYTES
          */
-        if (nValueLen > DataTypes.SHORT_VARTYPE_MAX_BYTES || bOut) {
+        if ((nValueLen > DataTypes.SHORT_VARTYPE_MAX_BYTES || bOut) && !isNonPLP) {
             writeRPCNameValType(sName, bOut, TDSType.NVARCHAR);
 
-            // Handle Yukon v*max type header here.
             writeVMaxHeader(nValueLen, // Length
                     bValueNull, // Is null?
                     collation);
@@ -5418,7 +5418,6 @@ void writeTVPColumnMetaData(TVP value) throws SQLServerException {
                     // Use PLP encoding on Yukon and later with long values
                     if (!isShortValue) // PLP
                     {
-                        // Handle Yukon v*max type header here.
                         writeShort((short) 0xFFFF);
                         con.getDatabaseCollation().writeCollation(this);
                     } else // non PLP
@@ -5436,7 +5435,6 @@ void writeTVPColumnMetaData(TVP value) throws SQLServerException {
                     isShortValue = pair.getValue().precision <= DataTypes.SHORT_VARTYPE_MAX_BYTES;
                     // Use PLP encoding on Yukon and later with long values
                     if (!isShortValue) // PLP
-                        // Handle Yukon v*max type header here.
                         writeShort((short) 0xFFFF);
                     else // non PLP
                         writeShort((short) DataTypes.SHORT_VARTYPE_MAX_BYTES);
@@ -5571,8 +5569,8 @@ void writeCryptoMetaData() throws SQLServerException {
         writeByte(cryptoMeta.normalizationRuleVersion);
     }
 
-    void writeRPCByteArray(String sName, byte[] bValue, boolean bOut, JDBCType jdbcType,
-            SQLCollation collation) throws SQLServerException {
+    void writeRPCByteArray(String sName, byte[] bValue, boolean bOut, JDBCType jdbcType, SQLCollation collation,
+            boolean isNonPLP) throws SQLServerException {
         boolean bValueNull = (bValue == null);
         int nValueLen = bValueNull ? 0 : bValue.length;
         boolean isShortValue = (nValueLen <= DataTypes.SHORT_VARTYPE_MAX_BYTES);
@@ -5618,8 +5616,7 @@ void writeRPCByteArray(String sName, byte[] bValue, boolean bOut, JDBCType jdbcT
 
         writeRPCNameValType(sName, bOut, tdsType);
 
-        if (usePLP) {
-            // Handle Yukon v*max type header here.
+        if (usePLP && !isNonPLP) {
             writeVMaxHeader(nValueLen, bValueNull, collation);
 
             // Send the data.
@@ -6400,7 +6397,6 @@ void writeRPCInputStream(String sName, InputStream stream, long streamLength, bo
 
             writeRPCNameValType(sName, bOut, jdbcType.isTextual() ? TDSType.BIGVARCHAR : TDSType.BIGVARBINARY);
 
-            // Handle Yukon v*max type header here.
             writeVMaxHeader(streamLength, false, jdbcType.isTextual() ? collation : null);
         }
 
@@ -6540,7 +6536,6 @@ void writeRPCReaderUnicode(String sName, Reader re, long reLength, boolean bOut,
 
             writeRPCNameValType(sName, bOut, TDSType.NVARCHAR);
 
-            // Handle Yukon v*max type header here.
             writeVMaxHeader(
                     (DataTypes.UNKNOWN_STREAM_LENGTH == reLength) ? DataTypes.UNKNOWN_STREAM_LENGTH : 2 * reLength, // Length
                                                                                                                     // (in
@@ -6995,6 +6990,35 @@ final short peekStatusFlag() {
         return 0;
     }
 
+    final int peekReturnValueStatus() throws SQLServerException {
+        // Ensure that we have a packet to read from.
+        if (!ensurePayload()) {
+            throwInvalidTDS();
+        }
+
+        // In order to parse the 'status' value, we need to skip over the following properties in the TDS packet
+        // payload: TDS token type (1 byte value), ordinal/length (2 byte value), parameter name length value (1 byte value) and
+        // the number of bytes that make the parameter name (need to be calculated).
+        //
+        // 'offset' starts at 4 because tdsTokenType + ordinal/length + parameter name length value is 4 bytes. So, we
+        // skip 4 bytes immediateley.
+        int offset = 4;
+        int paramNameLength = currentPacket.payload[payloadOffset + 3];
+
+        // Check if parameter name is set. If it's set, it should be > 0. In which case, we add the
+        // additional bytes to skip.
+        if (paramNameLength > 0) {
+            // Each character in unicode is 2 bytes
+            offset += 2 * paramNameLength;
+        }
+
+        if (payloadOffset + offset <= currentPacket.payloadLength) {
+            return currentPacket.payload[payloadOffset + offset] & 0xFF;
+        }
+
+        return -1;
+    }
+
     final int readUnsignedByte() throws SQLServerException {
         // Ensure that we have a packet to read from.
         if (!ensurePayload())

src/main/java/com/microsoft/sqlserver/jdbc/ISQLServerConnection.java

@@ -496,5 +496,5 @@ CallableStatement prepareCall(String sql, int nType, int nConcur, int nHold,
      * @param calcBigDecimalScale
      *        A boolean that indicates if the driver should calculate scale from inputted big decimal values.
      */
-    void setCalcBigDecimalScale(boolean computeBigDecimal);
+    void setCalcBigDecimalScale(boolean calcBigDecimalScale);
 }

src/main/java/com/microsoft/sqlserver/jdbc/ISQLServerDataSource.java

@@ -609,6 +609,25 @@ public interface ISQLServerDataSource extends javax.sql.CommonDataSource {
      */
     boolean getUseDefaultGSSCredential();
 
+    /**
+     * Sets whether or not sp_sproc_columns will be used for parameter name lookup.
+     *
+     * @param useFlexibleCallableStatements
+     *        When set to false, sp_sproc_columns is not used for parameter name lookup
+     *        in callable statements. This eliminates a round trip to the server but imposes limitations
+     *        on how parameters are set. When set to false, applications must either reference
+     *        parameters by name or by index, not both. Parameters must also be set in the same
+     *        order as the stored procedure definition.
+     */
+    void setUseFlexibleCallableStatements(boolean useFlexibleCallableStatements);
+
+    /**
+     * Returns whether or not sp_sproc_columns is being used for parameter name lookup.
+     *
+     * @return useFlexibleCallableStatements
+     */
+    boolean getUseFlexibleCallableStatements();
+
     /**
      * Sets the GSSCredential.
      *

src/main/java/com/microsoft/sqlserver/jdbc/Parameter.java

@@ -37,6 +37,8 @@ final class Parameter {
     // For unencrypted parameters cryptometa will be null. For encrypted parameters it will hold encryption metadata.
     CryptoMetadata cryptoMeta = null;
 
+    boolean isNonPLP = false;
+
     TypeInfo getTypeInfo() {
         return typeInfo;
     }
@@ -48,6 +50,7 @@ final CryptoMetadata getCryptoMetadata() {
     private boolean shouldHonorAEForParameter = false;
     private boolean userProvidesPrecision = false;
     private boolean userProvidesScale = false;
+    private boolean isReturnValue = false;
 
     // The parameter type definition
     private String typeDefinition = null;
@@ -70,6 +73,60 @@ boolean isOutput() {
         return null != registeredOutDTV;
     }
 
+    /**
+     * Returns true/false if the parameter is of return type
+     *
+     * @return isReturnValue
+     */
+    boolean isReturnValue() {
+        return isReturnValue;
+    }
+
+    /**
+     * Sets the parameter to be of return type
+     *
+     * @param isReturnValue
+     */
+    void setReturnValue(boolean isReturnValue) {
+        this.isReturnValue = isReturnValue;
+    }
+
+    /**
+     * Sets the name of the parameter
+     *
+     * @param name
+     */
+    void setName(String name) {
+        this.name = name;
+    }
+
+    /**
+     * Retrieve the name of the parameter
+     *
+     * @return
+     */
+    String getName() {
+        return this.name;
+    }
+
+    /**
+     * Returns the `registeredOutDTV` instance of the parameter
+     *
+     * @return registeredOutDTV
+     */
+    DTV getRegisteredOutDTV() {
+        return this.registeredOutDTV;
+    }
+
+    /**
+     * Returns the `inputDTV` instance of the parameter
+     *
+     * @return inputDTV
+     */
+    DTV getInputDTV() {
+        return this.inputDTV;
+    }
+
     // Since a parameter can have only one type definition for both sending its value to the server (IN)
     // and getting its value from the server (OUT), we use the JDBC type of the IN parameter value if there
     // is one; otherwise we use the registered OUT param JDBC type.
@@ -246,7 +303,7 @@ void setFromReturnStatus(int returnStatus, SQLServerConnection con) throws SQLSe
         if (null == getterDTV)
             getterDTV = new DTV();
 
-        getterDTV.setValue(null, JDBCType.INTEGER, returnStatus, JavaType.INTEGER, null, null, null, con,
+        getterDTV.setValue(null, this.getJdbcType(), returnStatus, JavaType.INTEGER, null, null, null, con,
                 getForceEncryption());
     }
 
@@ -387,10 +444,14 @@ boolean isValueGotten() {
 
     Object getValue(JDBCType jdbcType, InputStreamGetterArgs getterArgs, Calendar cal, TDSReader tdsReader,
             SQLServerStatement statement) throws SQLServerException {
-        if (null == getterDTV)
+        if (null == getterDTV) {
             getterDTV = new DTV();
+        }
+
+        if (null != tdsReader) {
+            deriveTypeInfo(tdsReader);
+        }
 
-        deriveTypeInfo(tdsReader);
         // If the parameter is not encrypted or column encryption is turned off (either at connection or
         // statement level), cryptoMeta would be null.
         return getterDTV.getValue(jdbcType, outScale, getterArgs, cal, typeInfo, cryptoMeta, tdsReader, statement);
@@ -1206,15 +1267,17 @@ String getTypeDefinition(SQLServerConnection con, TDSReader tdsReader) throws SQ
         return typeDefinition;
     }
 
-    void sendByRPC(TDSWriter tdsWriter, SQLServerStatement statement) throws SQLServerException {
+    void sendByRPC(TDSWriter tdsWriter, boolean callRPCDirectly,
+            SQLServerStatement statement) throws SQLServerException {
         assert null != inputDTV : "Parameter was neither set nor registered";
         SQLServerConnection conn = statement.connection;
 
         try {
+            inputDTV.isNonPLP = isNonPLP;
             inputDTV.sendCryptoMetaData(this.cryptoMeta, tdsWriter);
             inputDTV.setJdbcTypeSetByUser(getJdbcTypeSetByUser(), getValueLength());
-            inputDTV.sendByRPC(name, null, conn.getDatabaseCollation(), valueLength, isOutput() ? outScale : scale,
-                    isOutput(), tdsWriter, statement);
+            inputDTV.sendByRPC(callRPCDirectly ? name : null, null, conn.getDatabaseCollation(), valueLength,
+                    isOutput() ? outScale : scale, isOutput(), tdsWriter, statement);
         } finally {
             // reset the cryptoMeta in IOBuffer
             inputDTV.sendCryptoMetaData(null, tdsWriter);