Land #10110, Use JSON instead of PSTORE for module metadata

lib/msf/core/modules/metadata/cache.rb

@@ -146,6 +146,7 @@ class Cache
   end
 
   def initialize
+    super
     @mutex = Mutex.new
     @module_metadata_cache = {}
     @store_loaded = false

lib/msf/core/modules/metadata/obj.rb

@@ -1,4 +1,5 @@
 require 'msf/core/modules/metadata'
+require 'json'
 
 #
 # Simple object for storing a modules metadata.
@@ -26,7 +27,12 @@ class Obj
   attr_reader :is_install_path
   attr_reader :ref_name
 
-  def initialize(module_instance)
+  def initialize(module_instance, obj_hash = nil)
+    unless obj_hash.nil?
+      init_from_hash(obj_hash)
+      return
+    end
+
     @name = module_instance.name
     @full_name = module_instance.fullname
     @disclosure_date = module_instance.disclosure_date
@@ -37,7 +43,11 @@ class Obj
     @references = module_instance.references.map{|x| [x.ctx_id, x.ctx_val].join("-") }
     @is_server = (module_instance.respond_to?(:stance) and module_instance.stance == "aggressive")
     @is_client = (module_instance.respond_to?(:stance) and module_instance.stance == "passive")
+
     @platform = module_instance.platform_to_s
+    # Done to ensure that differences do not show up for the same array grouping
+    sort_platform_string
+
     @arch = module_instance.arch_to_s
     @rport = module_instance.datastore['RPORT'].to_s
     @path = module_instance.file_path
@@ -52,6 +62,42 @@ class Obj
     if module_instance.respond_to?(:targets) and module_instance.targets
       @targets = module_instance.targets.map{|x| x.name}
     end
+
+    # Due to potentially non-standard ASCII we force UTF-8 to ensure no problem with JSON serialization
+    force_encoding(Encoding::UTF_8)
+  end
+
+  #
+  # Returns the JSON representation of the module metadata
+  #
+  def to_json(*args)
+    {
+      'name' => @name,
+      'full_name' => @full_name,
+      'rank' => @rank,
+      'disclosure_date' => @disclosure_date.nil? ? nil : @disclosure_date.to_s,
+      'type' => @type,
+      'author' => @author,
+      'description' => @description,
+      'references' => @references,
+      'is_server' => @is_server,
+      'is_client' => @is_client,
+      'platform' => @platform,
+      'arch' => @arch,
+      'rport' => @rport,
+      'targets' => @targets,
+      'mod_time' => @mod_time.to_s,
+      'path' => @path,
+      'is_install_path' => @is_install_path,
+      'ref_name' => @ref_name
+    }.to_json(*args)
+  end
+
+  #
+  # Initialize this object from a hash
+  #
+  def self.from_hash(obj_hash)
+    return Obj.new(nil, obj_hash)
   end
 
   def update_mod_time(mod_time)
@@ -65,6 +111,49 @@ class Obj
 
     @path
   end
+
+  #######
+  private
+  #######
+
+  def init_from_hash(obj_hash)
+    @name = obj_hash['name']
+    @full_name = obj_hash['full_name']
+    @disclosure_date = obj_hash['disclosure_date'].nil? ? nil : Time.parse(obj_hash['disclosure_date'])
+    @rank = obj_hash['rank']
+    @type = obj_hash['type']
+    @description = obj_hash['description']
+    @author = obj_hash['author'].nil? ? [] : obj_hash['author']
+    @references = obj_hash['references']
+    @is_server = obj_hash['is_server']
+    @is_client = obj_hash['is_client']
+    @platform = obj_hash['platform']
+    @arch = obj_hash['arch']
+    @rport = obj_hash['rport']
+    @mod_time = Time.parse(obj_hash['mod_time'])
+    @ref_name = obj_hash['ref_name']
+    @path = obj_hash['path']
+    @is_install_path = obj_hash['is_install_path']
+    @targets = obj_hash['targets'].nil? ? [] : obj_hash['targets']
+  end
+
+  def sort_platform_string
+    arr = @platform.split(',')
+    unless arr.empty?
+      arr.each {|value| value.strip!}
+      if arr.length > 1
+        @platform = arr.sort.join(',')
+      else
+        @platform = arr[0]
+      end
+    end
+  end
+
+  def force_encoding(encoding)
+    @description.force_encoding(encoding)
+    @author.each {|a| a.force_encoding(encoding)}
+  end
+
 end
 end
 end

lib/msf/core/modules/metadata/store.rb

@@ -1,4 +1,4 @@
-require 'pstore'
+require 'json'
 require 'msf/core/modules/metadata'
 
 #
@@ -9,8 +9,12 @@ require 'msf/core/modules/metadata'
 #
 module Msf::Modules::Metadata::Store
 
-  BaseMetaDataFile = 'modules_metadata_base.pstore'
-  UserMetaDataFile = 'modules_metadata.pstore'
+  def initialize
+    @update_mutex = Mutex.new
+  end
+
+  BaseMetaDataFile = 'modules_metadata_base.json'
+  UserMetaDataFile = 'modules_metadata.json'
 
   #
   # Initializes from user store (under ~/store/.msf4) if it exists. else base file (under $INSTALL_ROOT/db) is copied and loaded.
@@ -28,10 +32,13 @@ module Msf::Modules::Metadata::Store
   #
   def update_store
     begin
-      @store.transaction do
-        @store[:module_metadata] = @module_metadata_cache
-      end
-    rescue Excepion => e
+      @update_mutex.synchronize {
+        json_map = @module_metadata_cache.sort.to_h
+        File.open(@path_to_user_metadata, "w") do |f|
+          f.write(JSON.pretty_generate(json_map))
+        end
+      }
+    rescue => e
       elog("Unable to update metadata store: #{e.message}")
     end
   end
@@ -40,8 +47,7 @@ module Msf::Modules::Metadata::Store
     begin
       retries ||= 0
       copied = configure_user_store
-      @store = PStore.new(@path_to_user_metadata, true)
-      @module_metadata_cache = @store.transaction(true) { @store[:module_metadata]}
+      load_cache_from_file_store
       validate_data(copied) if (!@module_metadata_cache.nil? && @module_metadata_cache.size > 0)
       @module_metadata_cache = {} if @module_metadata_cache.nil?
     rescue Exception => e
@@ -108,5 +114,16 @@ module Msf::Modules::Metadata::Store
     return ::File.join(store_dir, UserMetaDataFile)
   end
 
+  def load_cache_from_file_store
+    cache_map = JSON.parse(File.read(@path_to_user_metadata))
+    cache_map.each {|k,v|
+      begin
+        @module_metadata_cache[k] =  Msf::Modules::Metadata::Obj.from_hash(v)
+      rescue => e
+        elog("Unable to load module metadata object with key: #{k}")
+      end
+    }
+  end
+
 end