Land rapid7#5427, check payload compatability for set payload fix

Author: MSadek-r7

lib/msf/core/exploit.rb

@@ -698,6 +698,20 @@ def target_arch
     (target and target.arch) ? target.arch : (arch == []) ? nil : arch
   end
 
+
+  #
+  # Returns whether the requested payload is compatible with the module.
+  #
+  # @param [String] payload_name The payload name
+  # @return [TrueClass] Payload is compatible.
+  # @return [FalseClass] Payload is not compatible.
+  #
+  def is_payload_compatible?(payload_name)
+    payload_names = compatible_payloads.collect { |entry| entry[0] }
+
+    payload_names.include?(payload_name)
+  end
+
   #
   # Returns a list of compatible payloads based on platform, architecture,
   # and size requirements.

lib/msf/ui/console/driver.rb

@@ -571,6 +571,8 @@ def on_variable_set(glob, var, val)
 
         if (framework and framework.payloads.valid?(val) == false)
           return false
+        elsif active_module.type == 'exploit' && !active_module.is_payload_compatible?(val)
+          return false
         elsif (active_module)
           active_module.datastore.clear_non_user_defined
         elsif (framework)