Fix outgoing maximum-packet-size limit check.

Closes #3503. Thanks to Julian Graf.

Author: ralight

ChangeLog.txt

@@ -6,6 +6,7 @@
   of that client. Closes #3487.
 - Fix password length not being passed to MOSQ_EVT_BASIC_AUTH events.
   Closes #3490.
+- Fix incorrect maximum-packet-size restriction. Closes #3503.
 
 # Common lib:
 - Fix potential crash if reading a file in restricted mode and the group id

lib/packet_mosq.c

@@ -238,7 +238,7 @@ int packet__check_oversize(struct mosquitto *mosq, uint32_t remaining_length)
 		return MOSQ_ERR_SUCCESS;
 	}
 
-	len = remaining_length + mosquitto_varint_bytes(remaining_length);
+	len = 1 + remaining_length + mosquitto_varint_bytes(remaining_length);
 	if(len > mosq->maximum_packet_size){
 		return MOSQ_ERR_OVERSIZE_PACKET;
 	}else{

test/broker/12-prop-maximum-packet-size-publish-qos1.py

@@ -17,13 +17,18 @@ def do_test(start_broker):
     suback_packet = mosq_test.gen_suback(mid, 1, proto_ver=5)
 
     mid=1
-    publish1_packet = mosq_test.gen_publish(topic="12/max/publish/qos1/test/topic", mid=mid, qos=1, payload="12345678901234567890", proto_ver=5)
+    publish1_packet = mosq_test.gen_publish(topic="12/max/publish/qos1/test/topic", mid=mid, qos=1, payload="1234", proto_ver=5)
     puback1_packet = mosq_test.gen_puback(mid, proto_ver=5)
 
     mid=2
-    publish2_packet = mosq_test.gen_publish(topic="12/max/publish/qos1/test/topic", mid=mid, qos=1, payload="7890", proto_ver=5)
+    props = mqtt5_props.gen_byte_prop(mqtt5_props.PAYLOAD_FORMAT_INDICATOR, 1)
+    publish2_packet = mosq_test.gen_publish(topic="12/max/publish/qos1/test/topic", mid=mid, qos=1, payload="56", proto_ver=5, properties=props)
     puback2_packet = mosq_test.gen_puback(mid, proto_ver=5)
 
+    mid=3
+    publish3_packet = mosq_test.gen_publish(topic="12/max/publish/qos1/test/topic", mid=mid, qos=1, payload="789", proto_ver=5)
+    puback3_packet = mosq_test.gen_puback(mid, proto_ver=5)
+
     port = mosq_test.get_port()
     if start_broker:
         broker = mosq_test.start_broker(filename=os.path.basename(__file__), port=port)
@@ -33,12 +38,15 @@ def do_test(start_broker):
         mosq_test.do_send_receive(sock, subscribe_packet, suback_packet)
 
         mosq_test.do_send_receive(sock, publish1_packet, puback1_packet, "puback 1")
+        # We shouldn't receive the publish here because it is > MAXIMUM_PACKET_SIZE
+        mosq_test.do_ping(sock)
 
+        mosq_test.do_send_receive(sock, publish2_packet, puback2_packet, "puback 2")
         # We shouldn't receive the publish here because it is > MAXIMUM_PACKET_SIZE
         mosq_test.do_ping(sock)
 
-        sock.send(publish2_packet)
-        mosq_test.receive_unordered(sock, puback2_packet, publish2_packet, "puback 2/publish2")
+        sock.send(publish3_packet)
+        mosq_test.receive_unordered(sock, puback3_packet, publish3_packet, "puback 3/publish3")
         rc = 0
     except mosq_test.TestError:
         pass

test/broker/12-prop-maximum-packet-size-publish-qos2.py

@@ -17,13 +17,13 @@ def do_test(start_broker):
     suback_packet = mosq_test.gen_suback(mid, 2, proto_ver=5)
 
     mid=1
-    publish1_packet = mosq_test.gen_publish(topic="12/max/publish/qos2/test/topic", mid=mid, qos=2, payload="12345678901234567890", proto_ver=5)
+    publish1_packet = mosq_test.gen_publish(topic="12/max/publish/qos2/test/topic", mid=mid, qos=2, payload="1234", proto_ver=5)
     pubrec1_packet = mosq_test.gen_pubrec(mid, proto_ver=5)
     pubrel1_packet = mosq_test.gen_pubrel(mid, proto_ver=5)
     pubcomp1_packet = mosq_test.gen_pubcomp(mid, proto_ver=5)
 
     mid=2
-    publish2_packet = mosq_test.gen_publish(topic="12/max/publish/qos2/test/topic", mid=mid, qos=2, payload="7890", proto_ver=5)
+    publish2_packet = mosq_test.gen_publish(topic="12/max/publish/qos2/test/topic", mid=mid, qos=2, payload="789", proto_ver=5)
     pubrec2_packet = mosq_test.gen_pubrec(mid, proto_ver=5)
     pubrel2_packet = mosq_test.gen_pubrel(mid, proto_ver=5)
     pubcomp2_packet = mosq_test.gen_pubcomp(mid, proto_ver=5)