Decode the badchars string correctly.

lib/msf/core/payload_generator.rb

@@ -364,7 +364,9 @@ module Msf
       iterations.times do |x|
         shellcode = encoder_module.encode(shellcode.dup, badchars, nil, platform_list)
         cli_print "#{encoder_module.refname} succeeded with size #{shellcode.length} (iteration=#{x})"
-        raise EncoderSpaceViolation, "encoder has made a buffer that is too big" if shellcode.length > space
+        if shellcode.length > space
+          raise EncoderSpaceViolation, "encoder has made a buffer that is too big"
+        end
       end
       shellcode
     end

msfvenom

@@ -99,7 +99,7 @@ require 'msf/core/payload_generator'
     end
 
     opt.on('-b', '--bad-chars  <list>', String, 'The list of characters to avoid example: \'\x00\xff\'') do |b|
-      opts[:badchars] = b
+      opts[:badchars] = Rex::Text.hex_to_raw(b)
     end
 
     opt.on('-i', '--iterations <count>', Integer, 'The number of times to encode the payload') do |i|