8286526: Improve NTLM support

Reviewed-by: mbaesken
Backport-of: 2d1bc2e55c7ed5f123f3ab5d505b7866a28ba4c7

Author: martinuy

src/java.base/share/classes/com/sun/security/ntlm/Client.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -117,9 +117,10 @@ public byte[] type1() {
      * {@code nonce} is null for NTLM v1.
      */
     public byte[] type3(byte[] type2, byte[] nonce) throws NTLMException {
-        if (type2 == null || (v != Version.NTLM && nonce == null)) {
+        if (type2 == null || (v != Version.NTLM && nonce == null) ||
+                (nonce != null && nonce.length != 8)) {
             throw new NTLMException(NTLMException.PROTOCOL,
-                    "type2 and nonce cannot be null");
+                    "type2 cannot be null, and nonce must be 8-byte long");
         }
         debug("NTLM Client: Type 2 received\n");
         debug(type2);

src/java.base/share/classes/com/sun/security/ntlm/NTLM.java

@@ -226,23 +226,27 @@ void writeBytes(int offset, byte[] data) {
             System.arraycopy(data, 0, internal, offset, data.length);
         }
 
-        void writeSecurityBuffer(int offset, byte[] data) {
+        void writeSecurityBuffer(int offset, byte[] data) throws NTLMException {
             if (data == null) {
-                writeShort(offset+4, current);
+                writeInt(offset+4, current);
             } else {
                 int len = data.length;
+                if (len > 65535) {
+                    throw new NTLMException(NTLMException.INVALID_INPUT,
+                            "Invalid data length " + len);
+                }
                 if (current + len > internal.length) {
                     internal = Arrays.copyOf(internal, current + len + 256);
                 }
                 writeShort(offset, len);
                 writeShort(offset+2, len);
-                writeShort(offset+4, current);
+                writeInt(offset+4, current);
                 System.arraycopy(data, 0, internal, current, len);
                 current += len;
             }
         }
 
-        void writeSecurityBuffer(int offset, String str, boolean unicode) {
+        void writeSecurityBuffer(int offset, String str, boolean unicode) throws NTLMException {
             writeSecurityBuffer(offset, str == null ? null : str.getBytes(
                     unicode ? StandardCharsets.UTF_16LE
                             : StandardCharsets.ISO_8859_1));

src/java.base/share/classes/com/sun/security/ntlm/NTLMException.java

@@ -65,6 +65,11 @@ public final class NTLMException extends GeneralSecurityException {
      */
     public static final int PROTOCOL = 6;
 
+    /**
+     * If an invalid input is provided.
+     */
+    public static final int INVALID_INPUT = 7;
+
     private int errorCode;
 
     /**

src/java.base/share/classes/com/sun/security/ntlm/Server.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -85,9 +85,9 @@ public Server(String version, String domain) throws NTLMException {
      * {@code nonce} is null.
      */
     public byte[] type2(byte[] type1, byte[] nonce) throws NTLMException {
-        if (nonce == null) {
+        if (nonce == null || nonce.length != 8) {
             throw new NTLMException(NTLMException.PROTOCOL,
-                    "nonce cannot be null");
+                    "nonce must be 8-byte long");
         }
         debug("NTLM Server: Type 1 received\n");
         if (type1 != null) debug(type1);

test/jdk/sun/net/www/protocol/http/NULLTargetInfoTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,7 +23,7 @@
 
 /*
  * @test
- * @bug 8151788
+ * @bug 8151788 8286526
  * @summary NullPointerException from ntlm.Client.type3
  * @modules java.base/com.sun.security.ntlm
  * @run main NULLTargetInfoTest
@@ -42,7 +42,7 @@ public static void main(String[] args) throws Exception {
                 "4E 54 4C 4D 53 53 50 00 02 00 00 00 00 00 00 00"
                 + "00 00 00 00 05 82 89 00 0B 87 81 B6 2D 6E 8B C1"
                 + "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00");
-        byte[] nonce = new byte[10];
+        byte[] nonce = new byte[8];
         c.type3(type2, nonce);
     }