metasploit-gs/lib/msf/core/exploit/http/client.rb

# -*- coding: binary -*-

require 'uri'
require 'digest'
module Msf

###
#
# This module provides methods for acting as an HTTP client when
# exploiting an HTTP server.
#
###
module Exploit::Remote::HttpClient
  include Msf::Auxiliary::Report

  #
  # Initializes an exploit module that exploits a vulnerability in an HTTP
  # server.
  #
  def initialize(info = {})
    super

    register_options(
      [
        Opt::RHOST,
        Opt::RPORT(80),
        OptString.new('VHOST', [ false, "HTTP server virtual host" ]),
        OptBool.new('SSL', [ false, 'Negotiate SSL/TLS for outgoing connections', false]),
        Opt::Proxies
      ], self.class
    )

    register_advanced_options(
      [
        OptString.new('UserAgent', [false, 'The User-Agent header to use for all requests',
          Rex::Proto::Http::Client::DefaultUserAgent
          ]),
        OptString.new('HttpUsername', [false, 'The HTTP username to specify for authentication', '']),
        OptString.new('HttpPassword', [false, 'The HTTP password to specify for authentication', '']),
        OptPath.new('HttpRawHeaders', [false, 'Path to ERB-templatized raw headers to append to existing headers']),
        OptBool.new('DigestAuthIIS', [false, 'Conform to IIS, should work for most servers. Only set to false for non-IIS servers', true]),
        Opt::SSLVersion,
        OptBool.new('FingerprintCheck', [ false, 'Conduct a pre-exploit fingerprint verification', true]),
        OptString.new('DOMAIN', [ true, 'The domain to use for windows authentification', 'WORKSTATION']),
        OptFloat.new('HttpClientTimeout', [false, 'HTTP connection and receive timeout']),
        OptBool.new('HttpPartialResponses', [false, 'Return partial HTTP responses despite timeouts', false]),
        OptBool.new('HttpTrace', [false, 'Show the raw HTTP requests and responses', false])
      ], self.class
    )

    register_evasion_options(
      [
        OptEnum.new('HTTP::uri_encode_mode', [false, 'Enable URI encoding', 'hex-normal', ['none', 'hex-normal', 'hex-noslashes', 'hex-random', 'hex-all', 'u-normal', 'u-all', 'u-random']]),
        OptBool.new('HTTP::uri_full_url', [false, 'Use the full URL for all HTTP requests', false]),
        OptInt.new('HTTP::pad_method_uri_count', [false, 'How many whitespace characters to use between the method and uri', 1]),
        OptInt.new('HTTP::pad_uri_version_count', [false, 'How many whitespace characters to use between the uri and version', 1]),
        OptEnum.new('HTTP::pad_method_uri_type', [false, 'What type of whitespace to use between the method and uri', 'space', ['space', 'tab', 'apache']]),
        OptEnum.new('HTTP::pad_uri_version_type', [false, 'What type of whitespace to use between the uri and version', 'space', ['space', 'tab', 'apache']]),
        OptBool.new('HTTP::method_random_valid', [false, 'Use a random, but valid, HTTP method for request', false]),
        OptBool.new('HTTP::method_random_invalid', [false, 'Use a random invalid, HTTP method for request', false]),
        OptBool.new('HTTP::method_random_case', [false, 'Use random casing for the HTTP method', false]),
        OptBool.new('HTTP::version_random_valid', [false, 'Use a random, but valid, HTTP version for request', false]),
        OptBool.new('HTTP::version_random_invalid', [false, 'Use a random invalid, HTTP version for request', false]),
        OptBool.new('HTTP::uri_dir_self_reference', [false, 'Insert self-referential directories into the uri', false]),
        OptBool.new('HTTP::uri_dir_fake_relative', [false, 'Insert fake relative directories into the uri', false]),
        OptBool.new('HTTP::uri_use_backslashes', [false, 'Use back slashes instead of forward slashes in the uri ', false]),
        OptBool.new('HTTP::pad_fake_headers', [false, 'Insert random, fake headers into the HTTP request', false]),
        OptInt.new('HTTP::pad_fake_headers_count', [false, 'How many fake headers to insert into the HTTP request', 0]),
        OptBool.new('HTTP::pad_get_params', [false, 'Insert random, fake query string variables into the request', false]),
        OptInt.new('HTTP::pad_get_params_count', [false, 'How many fake query string variables to insert into the request', 16]),
        OptBool.new('HTTP::pad_post_params', [false, 'Insert random, fake post variables into the request', false]),
        OptInt.new('HTTP::pad_post_params_count', [false, 'How many fake post variables to insert into the request', 16]),
        OptBool.new('HTTP::uri_fake_end', [false, 'Add a fake end of URI (eg: /%20HTTP/1.0/../../)', false]),
        OptBool.new('HTTP::uri_fake_params_start', [false, 'Add a fake start of params to the URI (eg: /%3fa=b/../)', false]),
        OptBool.new('HTTP::header_folding', [false, 'Enable folding of HTTP headers', false])
#
# Remaining evasions to implement
#
#        OptBool.new('HTTP::chunked', [false, 'Enable chunking of HTTP request via "Transfer-Encoding: chunked"', false]),
#        OptInt.new('HTTP::junk_pipeline', [true, 'Insert the specified number of junk pipeline requests', 0]),
      ], self.class
    )
    register_autofilter_ports([ 80, 8080, 443, 8000, 8888, 8880, 8008, 3000, 8443 ])
    register_autofilter_services(%W{ http https })
  end

  def deregister_http_client_options
    deregister_options('RHOST', 'RPORT', 'VHOST', 'SSL', 'Proxies')
  end

  #
  # For HTTP Client exploits, we often want to verify that the server info matches some regex before
  # firing a giant binary exploit blob at it. We override setup() here to accomplish that.
  #
  def setup
    validate_fingerprint
    super
  end


  #
  # This method is meant to be overriden in the exploit module to specify a set of regexps to
  # attempt to match against. A failure to match any of them results in a RuntimeError exception
  # being raised.
  #
  def validate_fingerprint()
    # Don't bother checking if there's no database active.
    if (framework.db.active and
        datastore['FingerprintCheck'] and
        self.class.const_defined?('HttpFingerprint'))
      # Get the module-specific config
      opts = self.class.const_get('HttpFingerprint')
      #
      # XXX: Ideally we could have more structured matches, but doing that requires
      # a more structured response cache.
      #
      info = http_fingerprint(opts)
      if info and opts[:pattern]
        opts[:pattern].each do |re|
          if not re.match(info)
            err = "The target server fingerprint \"#{info}\" does not match \"#{re.to_s}\", use 'set FingerprintCheck false' to disable this check."
            fail_with(::Msf::Module::Failure::NotFound, err)
          end
        end
      elsif info.nil?
        err = "The target server did not respond to fingerprinting, use 'set FingerprintCheck false' to disable this check."
        fail_with(::Msf::Module::Failure::Unreachable, err)
      end
    end
  end


  #
  # Connects to an HTTP server.
  #
  def connect(opts={})
    dossl = false
    if(opts.has_key?('SSL'))
      dossl = opts['SSL']
    else
      dossl = ssl
    end

    client_username = opts['username'] || datastore['HttpUsername'] || ''
    client_password = opts['password'] || datastore['HttpPassword'] || ''

    nclient = Rex::Proto::Http::Client.new(
      opts['rhost'] || rhost,
      (opts['rport'] || rport).to_i,
      {
        'Msf'        => framework,
        'MsfExploit' => self,
      },
      dossl,
      ssl_version,
      proxies,
      client_username,
      client_password
    )

    # Configure the HTTP client with the supplied parameter
    nclient.set_config(
      'vhost' => opts['vhost'] || opts['rhost'] || self.vhost(),
      'agent' => datastore['UserAgent'],
      'partial' => opts['partial'] || datastore['HttpPartialResponses'],
      'uri_encode_mode'        => datastore['HTTP::uri_encode_mode'],
      'uri_full_url'           => datastore['HTTP::uri_full_url'],
      'pad_method_uri_count'   => datastore['HTTP::pad_method_uri_count'],
      'pad_uri_version_count'  => datastore['HTTP::pad_uri_version_count'],
      'pad_method_uri_type'    => datastore['HTTP::pad_method_uri_type'],
      'pad_uri_version_type'   => datastore['HTTP::pad_uri_version_type'],
      'method_random_valid'    => datastore['HTTP::method_random_valid'],
      'method_random_invalid'  => datastore['HTTP::method_random_invalid'],
      'method_random_case'     => datastore['HTTP::method_random_case'],
      'version_random_valid'   => datastore['HTTP::version_random_valid'],
      'version_random_invalid' => datastore['HTTP::version_random_invalid'],
      'uri_dir_self_reference' => datastore['HTTP::uri_dir_self_reference'],
      'uri_dir_fake_relative'  => datastore['HTTP::uri_dir_fake_relative'],
      'uri_use_backslashes'    => datastore['HTTP::uri_use_backslashes'],
      'pad_fake_headers'       => datastore['HTTP::pad_fake_headers'],
      'pad_fake_headers_count' => datastore['HTTP::pad_fake_headers_count'],
      'pad_get_params'         => datastore['HTTP::pad_get_params'],
      'pad_get_params_count'   => datastore['HTTP::pad_get_params_count'],
      'pad_post_params'        => datastore['HTTP::pad_post_params'],
      'pad_post_params_count'  => datastore['HTTP::pad_post_params_count'],
      'uri_fake_end'           => datastore['HTTP::uri_fake_end'],
      'uri_fake_params_start'  => datastore['HTTP::uri_fake_params_start'],
      'header_folding'         => datastore['HTTP::header_folding'],
      'domain'                 => datastore['DOMAIN'],
      'DigestAuthIIS'          => datastore['DigestAuthIIS']
    )

    # NOTE: Please use opts['headers'] to programmatically set headers
    if datastore['HttpRawHeaders'] && File.readable?(datastore['HttpRawHeaders'])
      # Templatize with ERB
      headers = ERB.new(File.read(datastore['HttpRawHeaders'])).result(binding)

      # Append templatized headers to existing headers
      nclient.set_config('raw_headers' => headers)
    end

    # If this connection is global, persist it
    # Required for findsock on these sockets
    if (opts['global'])
      if (self.client)
        disconnect
      end
    end

    self.client = nclient

    return nclient
  end

  #
  # Converts datastore options into configuration parameters for the
  # Metasploit::LoginScanner::Http class. Any parameters passed into
  # this method will override the defaults.
  #
  def configure_http_login_scanner(conf)
    {
      host:                          rhost,
      port:                          rport,
      ssl:                           ssl,
      ssl_version:                   ssl_version,
      proxies:                       datastore['PROXIES'],
      framework:                     framework,
      framework_module:              self,
      vhost:                         vhost,
      user_agent:                    datastore['UserAgent'],
      evade_uri_encode_mode:         datastore['HTTP::uri_encode_mode'],
      evade_uri_full_url:            datastore['HTTP::uri_full_url'],
      evade_pad_method_uri_count:    datastore['HTTP::pad_method_uri_count'],
      evade_pad_uri_version_count:   datastore['HTTP::pad_uri_version_count'],
      evade_pad_method_uri_type:     datastore['HTTP::pad_method_uri_type'],
      evade_pad_uri_version_type:    datastore['HTTP::pad_uri_version_type'],
      evade_method_random_valid:     datastore['HTTP::method_random_valid'],
      evade_method_random_invalid:   datastore['HTTP::method_random_invalid'],
      evade_method_random_case:      datastore['HTTP::method_random_case'],
      evade_version_random_valid:    datastore['HTTP::version_random_valid'],
      evade_version_random_invalid:  datastore['HTTP::version_random_invalid'],
      evade_uri_dir_self_reference:  datastore['HTTP::uri_dir_self_reference'],
      evade_uri_dir_fake_relative:   datastore['HTTP::uri_dir_fake_relative'],
      evade_uri_use_backslashes:     datastore['HTTP::uri_use_backslashes'],
      evade_pad_fake_headers:        datastore['HTTP::pad_fake_headers'],
      evade_pad_fake_headers_count:  datastore['HTTP::pad_fake_headers_count'],
      evade_pad_get_params:          datastore['HTTP::pad_get_params'],
      evade_pad_get_params_count:    datastore['HTTP::pad_get_params_count'],
      evade_pad_post_params:         datastore['HTTP::pad_post_params'],
      evade_pad_post_params_count:   datastore['HTTP::pad_post_params_count'],
      evade_uri_fake_end:            datastore['HTTP::uri_fake_end'],
      evade_uri_fake_params_start:   datastore['HTTP::uri_fake_params_start'],
      evade_header_folding:          datastore['HTTP::header_folding'],
      ntlm_domain:                   datastore['DOMAIN'],
      digest_auth_iis:               datastore['DigestAuthIIS']
    }.merge(conf)
  end

  #
  # Passes the client connection down to the handler to see if it's of any
  # use.
  #
  def handler(nsock = nil)
    # If no socket was provided, try the global one.
    if ((!nsock) and (self.client))
      nsock = self.client.conn
    end

    # If the parent claims the socket associated with the HTTP client, then
    # we rip the socket out from under the HTTP client.
    if (((rv = super(nsock)) == Handler::Claimed) and
        (self.client) and
        (nsock == self.client.conn))
      self.client.conn = nil
    end

    rv
  end

  #
  # Disconnects the HTTP client
  #
  def disconnect(nclient = self.client)
    if (nclient)
      nclient.close
    end

    if (nclient == self.client)
      if self.client.respond_to?(:close)
        self.client.close
      end

      self.client = nil
    end
  end

  #
  # Performs cleanup as necessary, disconnecting the HTTP client if it's
  # still established.
  #
  def cleanup
    super
    disconnect
  end

  #
  # Connects to the server, creates a request, sends the request, reads the response
  #
  # Passes +opts+ through directly to Rex::Proto::Http::Client#request_raw.
  #
  def send_request_raw(opts={}, timeout = 20, disconnect = false)
    if datastore['HttpClientTimeout'] && datastore['HttpClientTimeout'] > 0
      actual_timeout = datastore['HttpClientTimeout']
    else
      actual_timeout =  opts[:timeout] || timeout
    end

    begin
      c = connect(opts)
      r = c.request_raw(opts)

      if datastore['HttpTrace']
        print_line('#' * 20)
        print_line('# Request:')
        print_line('#' * 20)
        print_line(r.to_s)
      end

      res = c.send_recv(r, actual_timeout)

      if datastore['HttpTrace']
        print_line('#' * 20)
        print_line('# Response:')
        print_line('#' * 20)
        if res.nil?
          print_line("No response received")
        else
          print_line(res.to_terminal_output)
        end
      end

      disconnect(c) if disconnect

      res
    rescue ::Errno::EPIPE, ::Timeout::Error => e
      print_line(e.message) if datastore['HttpTrace']
      nil
    rescue Rex::ConnectionError => e
      vprint_error(e.to_s)
      nil
    rescue ::Exception => e
      print_line(e.message) if datastore['HttpTrace']
      raise e
    end
  end


  # Connects to the server, creates a request, sends the request,
  # reads the response
  #
  # Passes `opts` through directly to {Rex::Proto::Http::Client#request_cgi}.
  #
  # @return (see Rex::Proto::Http::Client#send_recv))
  def send_request_cgi(opts={}, timeout = 20, disconnect = true)
    if datastore['HttpClientTimeout'] && datastore['HttpClientTimeout'] > 0
      actual_timeout = datastore['HttpClientTimeout']
    else
      actual_timeout =  opts[:timeout] || timeout
    end

    print_line("*" * 20) if datastore['HttpTrace']

    begin
      c = connect(opts)
      r = c.request_cgi(opts)

      if datastore['HttpTrace']
        print_line('#' * 20)
        print_line('# Request:')
        print_line('#' * 20)
        print_line(r.to_s)
      end

      res = c.send_recv(r, actual_timeout)

      if datastore['HttpTrace']
        print_line('#' * 20)
        print_line('# Response:')
        print_line('#' * 20)
        if res.nil?
          print_line("No response received")
        else
          print_line(res.to_terminal_output)
        end
      end

      disconnect(c) if disconnect

      res
    rescue ::Errno::EPIPE, ::Timeout::Error => e
      print_line(e.message) if datastore['HttpTrace']
      nil
    rescue Rex::ConnectionError => e
      vprint_error(e.to_s)
      nil
    rescue ::Exception => e
      print_line(e.message) if datastore['HttpTrace']
      raise e
    end
  end

  # Connects to the server, creates a request, sends the request, reads the
  # response if a redirect (HTTP 30x response) is received it will attempt to
  # follow the direct and retrieve that URI.
  #
  # @note `opts` will be updated to the updated location and
  #   `opts['redirect_uri']` will contain the full URI.
  #
  # @return (see #send_request_cgi)
  def send_request_cgi!(opts={}, timeout = 20, redirect_depth = 1)
    if datastore['HttpClientTimeout'] && datastore['HttpClientTimeout'] > 0
      actual_timeout = datastore['HttpClientTimeout']
    else
      actual_timeout =  opts[:timeout] || timeout
    end

    res = send_request_cgi(opts, actual_timeout)
    return res unless res && res.redirect? && redirect_depth > 0

    redirect_depth -= 1
    return res if res.redirection.nil?

    reconfig_redirect_opts!(res, opts)
    send_request_cgi!(opts, actual_timeout, redirect_depth)
  end


  # Modifies the HTTP request options for a redirection.
  #
  # @param res [Rex::Proto::HTTP::Response] HTTP Response.
  # @param opts [Hash] The HTTP request options to modify.
  # @return [void]
  def reconfig_redirect_opts!(res, opts)
    location = res.redirection

    if location.relative?
      parent_path = File.dirname(opts['uri'].to_s)
      parent_path = '/' if parent_path == '.'
      new_redirect_uri = normalize_uri(parent_path, location.path.gsub(/^\./, ''))
      opts['redirect_uri'] = new_redirect_uri
      opts['uri'] = new_redirect_uri
      opts['rhost'] = datastore['RHOST']
      opts['vhost'] = opts['vhost'] || opts['rhost'] || self.vhost()
      opts['rport'] = datastore['RPORT']

      opts['SSL'] = ssl
    else
      disconnect

      opts['redirect_uri'] = location
      opts['uri'] = location.path
      opts['rhost'] = location.host
      opts['vhost'] = location.host
      opts['rport'] = location.port

      if location.scheme == 'https'
        opts['SSL'] = true
      else
        opts['SSL'] = false
      end
    end
  end

  #
  # Combine the user/pass into an auth string for the HTTP Client
  #
  def basic_auth(username, password)
    auth_str = Rex::Text.encode_base64("#{username}:#{password}")
    "Basic #{auth_str}"
  end

  ##
  #
  # Wrappers for getters
  #
  ##

  #
  # Returns the target URI
  #
  def target_uri
    begin
      # In case TARGETURI is empty, at least we default to '/'
      u = datastore['TARGETURI']
      u = "/" if u.nil? or u.empty?
      URI(u)
    rescue ::URI::InvalidURIError
      print_error "Invalid URI: #{datastore['TARGETURI'].inspect}"
      raise Msf::OptionValidateError.new(['TARGETURI'])
    end
  end

  # Returns the complete URI as string including the scheme, port and host
  def full_uri(custom_uri = nil, vhost_uri: false)
    uri_scheme = ssl ? 'https' : 'http'

    if (rport == 80 && !ssl) || (rport == 443 && ssl)
      uri_port = ''
    else
      uri_port = ":#{rport}"
    end

    uri = normalize_uri(custom_uri || target_uri.to_s)

    if vhost_uri && datastore['VHOST']
      uri_host = datastore['VHOST']
    elsif Rex::Socket.is_ipv6?(rhost)
      uri_host = "[#{rhost}]"
    else
      uri_host = rhost
    end

    "#{uri_scheme}://#{uri_host}#{uri_port}#{uri}"
  end

  #
  # Returns a modified version of the URI that:
  # 1. Always has a starting slash
  # 2. Removes all the double slashes
  #
  def normalize_uri(*strs)
    new_str = strs * "/"

    new_str = new_str.gsub!("//", "/") while new_str.index("//")

    # Makes sure there's a starting slash
    unless new_str[0,1] == '/'
      new_str = '/' + new_str
    end

    new_str
  end

  # Returns the Path+Query from a full URI String, nil on error
  def path_from_uri(uri)
    begin
      temp = URI(uri)
      ret_uri = temp.path
      ret_uri << "?#{temp.query}" unless temp.query.nil? or temp.query.empty?
      return ret_uri
    rescue URI::Error
      print_error "Invalid URI: #{uri}"
      return nil
    end
  end

  #
  # Returns a hash of request opts from a URL string
  def request_opts_from_url(url)
    # verify and extract components from the URL
    begin
      tgt = URI.parse(url)
      raise 'Invalid URL' unless tgt.scheme =~ %r{https?}
      raise 'Invalid URL' if tgt.host.to_s.eql? ''
    rescue => e
      print_error "Could not parse URL: #{e}"
      return nil
    end

    opts = { 'rhost' => tgt.host, 'rport' => tgt.port, 'uri' => tgt.request_uri }
    opts['SSL'] = true if tgt.scheme == 'https'
    if tgt.query and tgt.query.size > 13
      # Assming that this is going to be mostly used for GET requests as string -> req
      opts['vars_get'] = {}
      tgt.query.split('&').each do |pair|
        k,v = pair.split('=',2)
        opts['vars_get'][k] = v
      end
    end
    return opts
  end

  #
  # Returns response from a simple URL call
  def request_url(url, keepalive = false)
    opts = request_opts_from_url(url)
    return nil if opts.nil?
    res = send_request_raw(opts)
    disconnect unless keepalive
    return res
  end

  #
  # Downloads a URL
  def download(url)
    print_status "Downloading '#{url}'"

    begin
      target = URI.parse url
      raise 'Invalid URL' unless target.scheme =~ /https?/
      raise 'Invalid URL' if target.host.to_s.eql? ''
    rescue => e
      print_error "Could not parse URL: #{e}"
      return nil
    end

    res = request_url(url)

    unless res
      print_error 'Connection failed'
      return nil
    end

    print_status "- HTTP #{res.code} - #{res.body.length} bytes"

    res.code == 200 ? res.body : nil
  end

  # removes HTML tags from a provided string.
  # The string is html-unescaped before the tags are removed
  # Leading whitespaces and double linebreaks are removed too
  def strip_tags(html)
    Rex::Text.html_decode(html).gsub(/<\/?[^>]*>/, '').gsub(/^\s+/, '').strip
  end

  #
  # Returns the target host
  #
  def rhost
    datastore['RHOST']
  end

  #
  # Returns the remote port
  #
  def rport
    datastore['RPORT']
  end

  #
  # Returns the Host and Port as a string
  #
  def peer
    "#{rhost}:#{rport}"
  end

  #
  # Returns the VHOST of the HTTP server.
  #
  def vhost
    datastore['VHOST'] || datastore['RHOST']
  end

  #
  # Returns the boolean indicating SSL
  #
  def ssl
    ((datastore.default?('SSL') and [443,3790].include?(rport.to_i)) or datastore['SSL'])
  end

  #
  # Returns the string indicating SSL version
  #
  def ssl_version
    datastore['SSLVersion']
  end

  #
  # Returns the configured proxy list
  #
  def proxies
    datastore['Proxies']
  end


  #
  # Lookup HTTP fingerprints from the database that match the current
  # destination host and port. This method falls back to using the old
  # service.info field to represent the HTTP Server header.
  #
  # @option opts [String] :uri ('/') An HTTP URI to request in order to generate
  #   a fingerprint
  # @option opts [String] :method ('GET') An HTTP method to use in the fingerprint
  #   request
  def lookup_http_fingerprints(opts={})
    uri     = opts[:uri] || '/'
    method  = opts[:method] || 'GET'
    fprints = []

    return fprints unless framework.db.active

    ::ActiveRecord::Base.connection_pool.with_connection {
      wspace = datastore['WORKSPACE'] ?
        framework.db.find_workspace(datastore['WORKSPACE']) : framework.db.workspace

      service = framework.db.get_service(wspace, rhost, 'tcp', rport)
      return fprints unless service

      # Order by note_id descending so the first value is the most recent
      service.notes.where(:ntype => 'http.fingerprint').order("notes.id DESC").each do |n|
        next unless n.data && n.data.kind_of?(::Hash)
        next unless n.data[:uri] == uri && n.data[:method] == method
        # Append additional fingerprints to the results as found
        fprints.unshift n.data.dup
      end
    }

    fprints
  end

  #
  # Record various things about an HTTP server that we can glean from the
  # response to a single request.  If this method is passed a response, it
  # will use it directly, otherwise it will check the database for a previous
  # fingerprint.  Failing that, it will make a request for /.
  #
  # Other options are passed directly to {#connect} if :response is not given
  #
  # @option opts [Rex::Proto::Http::Packet] :response The return value from any
  #   of the send_* methods
  # @option opts [String] :uri ('/') An HTTP URI to request in order to generate
  #   a fingerprint
  # @option opts [String] :method ('GET') An HTTP method to use in the fingerprint
  #   request
  # @option opts [Boolean] :full (false) Request the full HTTP fingerprint, not
  #   just the signature
  #
  # @return [String]
  def http_fingerprint(opts={})
    res    = nil
    uri    = opts[:uri] || '/'
    method = opts[:method] || 'GET'

    # Short-circuit the fingerprint lookup and HTTP request if a response has
    # already been provided by the caller.
    if opts[:response]
      res = opts[:response]
    else
      fprints = lookup_http_fingerprints(opts)

      if fprints.length > 0

        # Grab the most recent fingerprint available for this service, uri, and method
        fprint = fprints.last

        # Return the full HTTP fingerprint if requested by the caller
        return fprint if opts[:full]

        # Otherwise just return the signature string for compatibility
        return fprint[:signature]
      end

      # Go ahead and send a request to the target for fingerprinting
      connect(opts)
      res = send_request_raw(
        {
          'uri'     => uri,
          'method'  => method
        }) rescue nil
    end

    # Bail if the request did not receive a readable response
    return if not res

    # This section handles a few simple cases of pattern matching and service
    # classification. This logic should be deprecated in favor of Recog-based
    # fingerprint databases, but has been left in place for backward compat.

    extras = []

    if res.headers['Set-Cookie'] =~ /^vmware_soap_session/
      extras << "VMWare Web Services"
    end

    if (res.headers['X-Powered-By'])
      extras << "Powered by " + res.headers['X-Powered-By']
    end

    if (res.headers['Via'])
      extras << "Via-" + res.headers['Via']
    end

    if (res.headers['X-AspNet-Version'])
      extras << "AspNet-Version-" + res.headers['X-AspNet-Version']
    end

    case res.body
      when nil
        # Nothing
      when /openAboutWindow.*\>DD\-WRT ([^\<]+)\<|Authorization.*please note that the default username is \"root\" in/
        extras << "DD-WRT #{$1.to_s.strip}".strip

      when /ID_ESX_Welcome/, /ID_ESX_VIClientDesc/
        extras << "VMware ESX Server"

      when /Test Page for.*Fedora/
        extras << "Fedora Default Page"

      when /Placeholder page/
        extras << "Debian Default Page"

      when /Welcome to Windows Small Business Server (\d+)/
        extras << "Windows SBS #{$1}"

      when /Asterisk@Home/
        extras << "Asterisk"

      when /swfs\/Shell\.html/
        extras << "BPS-1000"
    end

    if datastore['RPORT'].to_i == 3790
      if res.code == 302 and res.headers and res.headers['Location'] =~ /[\x5c\x2f](login|setup)$/n
        if res['Server'] =~ /^(thin.*No Hup)|(nginx[\x5c\x2f][\d\.]+)$/n
          extras << "Metasploit"
        end
      end
    end

    #
    # This HTTP response code tracking is used by a few modules and the MSP logic
    # to identify and bruteforce certain types of servers. In the long run we
    # should deprecate this and use the http.fingerprint fields instead.
    #
    case res.code
    when 301,302
      extras << "#{res.code}-#{res.headers['Location']}"
    when 401
      extras << "#{res.code}-#{res.headers['WWW-Authenticate']}"
    when 403
      extras << "#{res.code}-#{res.headers['WWW-Authenticate']||res.message}"
    when 500 .. 599
      extras << "#{res.code}-#{res.message}"
    end

    # Build a human-readable string to store in service.info and http.fingerprint[:signature]
    info = res.headers['Server'].to_s.dup
    info << " ( #{extras.join(", ")} )" if extras.length > 0

    # Create a new fingerprint structure to track this response
    fprint = {
      :uri => uri, :method => method, :server_port => rport,
      :code => res.code.to_s, :message => res.message.to_s,
      :signature => info
    }

    res.headers.each_pair do |k,v|
      hname = k.to_s.downcase.tr('-', '_').gsub(/[^a-z0-9_]+/, '')
      next unless hname.length > 0

      # Set-Cookie >        :header_set_cookie       => JSESSIONID=AAASD23423452
      # Server >            :header_server           => Apache/1.3.37
      # WWW-Authenticate >  :header_www_authenticate => basic realm='www'

      fprint["header_#{hname}".intern] = v
    end

    # Store the first 64k of the HTTP body as well
    fprint[:content] = res.body.to_s[0,65535]

    # Report a new http.fingerprint note
    report_note(
      :host   => rhost,
      :port   => rport,
      :proto  => 'tcp',
      :ntype  => 'http.fingerprint',
      :data   => fprint,
      # Limit reporting to one stored note per host/service combination
      :update => :unique
    )

    # Report here even if info is empty since the fact that we didn't
    # return early means we at least got a connection and the service is up
    report_web_site(:host => rhost, :port => rport, :ssl => ssl, :vhost => vhost, :info => info.dup)

    # Return the full HTTP fingerprint if requested by the caller
    return fprint if opts[:full]

    # Otherwise just return the signature string for compatibility
    fprint[:signature]
  end

  def service_details
    {
      origin_type: :service,
      protocol: 'tcp',
      service_name: (ssl ? 'https' : 'http'),
      address: rhost,
      port: rport
    }
  end

protected

  attr_accessor :client

end

end