Skip to content

Commit

Permalink
Fixed an issue that caused errors with HTTPS sites with bad certificates
Browse files Browse the repository at this point in the history
  • Loading branch information
@urbanadventurer
urbanadventurer committed May 1, 2016
1 parent baff22c commit 29ba77a
Show file tree
Hide file tree
Showing 4 changed files with 27 additions and 152 deletions.
153 changes: 17 additions & 136 deletions lib/extend-http_ruby1.9.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ def initialize(address, port = nil)
@enable_post_connection_check = true
@compression = nil
@sspi_enabled = false

#$DEBUG=true

if defined?(SSL_ATTRIBUTES)
SSL_ATTRIBUTES.each do |name|
instance_variable_set "@#{name}", nil
Expand All @@ -62,7 +65,7 @@ def initialize(address, port = nil)
# ExtendedHTTP :: raw
# added def raw
def raw
@raw
@raw
end

# ExtendedHTTP :: raw
Expand All @@ -81,15 +84,17 @@ def connect
ssl_parameters[name] = value
end
end
@ssl_context = OpenSSL::SSL::SSLContext.new
@ssl_context = OpenSSL::SSL::SSLContext.new
@ssl_context.set_params(ssl_parameters)

s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context)
s.sync_close = true
end
@socket = BufferedIO.new(s)
@socket.read_timeout = @read_timeout
@socket.continue_timeout = @continue_timeout
@socket.debug_output = @debug_output

if use_ssl?
begin
if proxy?
Expand All @@ -104,16 +109,16 @@ def connect
@socket.writeline ''

# whatweb
#HTTPResponse.read_new(@socket).value
x,raw=ExtendedHTTPResponse.read_new(@socket)
@raw = raw
res=x.value
# HTTPResponse.read_new(@socket).value
x,raw = ExtendedHTTPResponse.read_new(@socket)
@raw = raw
res = x.value

end
# Server Name Indication (SNI) RFC 3546
s.hostname = @address if s.respond_to? :hostname=
s.hostname = @address if s.respond_to? :hostname=
timeout(@open_timeout) { s.connect }
if @ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE
if @ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE
s.post_connection_check(@address)
end
rescue => exception
Expand All @@ -122,116 +127,12 @@ def connect
raise exception
end
end
on_connect
on_connect
end
private :connect



=begin
def connect
@raw=[]
if RUBY_VERSION =~ /^1\.8/
D "opening connection to #{conn_address()}..."
s = Timeout::timeout(@open_timeout) { TCPSocket.open(conn_address(), conn_port()) }
D "opened"
if use_ssl?
unless @ssl_context.verify_mode
warn "warning: peer certificate won't be verified in this SSL session"
@ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE
end
s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context)
s.sync_close = true
end
@socket = BufferedIO.new(s)
@socket.read_timeout = @read_timeout
@socket.debug_output = @debug_output
if use_ssl?
if proxy?
@socket.writeline sprintf('CONNECT %s:%s HTTP/%s',
@address, @port, HTTPVersion)
@socket.writeline "Host: #{@address}:#{@port}"
if proxy_user
credential = ["#{proxy_user}:#{proxy_pass}"].pack('m')
credential.delete!("\r\n")
@socket.writeline "Proxy-Authorization: Basic #{credential}"
end
@socket.writeline ''
# added this
x,raw=ExtendedHTTPResponse.read_new(@socket)
@raw = raw
res=x.value
end
s.connect
if @ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE
s.post_connection_check(@address)
end
end
on_connect
end
if RUBY_VERSION =~ /^1\.9/ || RUBY_VERSION =~ /^2\./
D "opening connection to #{conn_address()}..."
s = Timeout::timeout(@open_timeout) { TCPSocket.open(conn_address(), conn_port()) }
D "opened"
if use_ssl?
ssl_parameters = Hash.new
iv_list = instance_variables
SSL_ATTRIBUTES.each do |name|
ivname = "@#{name}".intern
if iv_list.include?(ivname) and
value = instance_variable_get(ivname)
ssl_parameters[name] = value
end
end
@ssl_context = OpenSSL::SSL::SSLContext.new
@ssl_context.set_params(ssl_parameters)
@ssl_context.set_params({:ssl_version=>:TLSv1})
s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context)
s.sync_close = true
end
@socket = BufferedIO.new(s)
@socket.read_timeout = @read_timeout
@socket.debug_output = @debug_output
if use_ssl?
begin
if proxy?
@socket.writeline sprintf('CONNECT %s:%s HTTP/%s',
@address, @port, HTTPVersion)
@socket.writeline "Host: #{@address}:#{@port}"
if proxy_user
credential = ["#{proxy_user}:#{proxy_pass}"].pack('m')
credential.delete!("\r\n")
@socket.writeline "Proxy-Authorization: Basic #{credential}"
end
@socket.writeline ''
# HTTPResponse.read_new(@socket).value
# added this
x,raw=ExtendedHTTPResponse.read_new(@socket)
@raw = raw
res=x.value
end
Timeout::timeout(@open_timeout) { s.connect }
if @ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE
s.post_connection_check(@address)
end
rescue => exception
D "Conn close because of connect error #{exception}"
@socket.close if @socket and not @socket.closed?
raise exception
end
end
on_connect
end
end
private :connect
=end


def transport_request(req)
begin_transport req
Expand Down Expand Up @@ -275,8 +176,9 @@ def read_new(sock) #:nodoc: internal use only
each_response_header(sock) do |k,v|
res.add_field k, v
end
# added for whatweb
real = @rawlines
# added for whatweb
real = @rawlines
#pp real
[res,real]
end

Expand All @@ -290,7 +192,6 @@ def read_status_line(sock)
end



def each_response_header(sock)
key = value = nil
while true
Expand All @@ -313,27 +214,7 @@ def each_response_header(sock)
end
end

=begin
def each_response_header(sock)
while true
line = sock.readuntil("\n", true)
# headers are interpreted here
# added this
@rawlines << line unless line.nil?
line.sub!(/\s+\z/, '')
break if line.empty?
m = /\A([^:]+):\s*/.match(line) or
raise HTTPBadResponse, 'wrong header line format'
yield m[1], m.post_match
end
end
end
=end

###################
public
# include HTTPHeader

def initialize(httpv, code, msg) #:nodoc: internal use only
@http_version = httpv
Expand Down
5 changes: 3 additions & 2 deletions lib/extend-http_ruby2.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
class ExtendedHTTP < Net::HTTP #:nodoc:
include Net


# Creates a new Net::HTTP object for the specified server address,
# without opening the TCP connection or initializing the HTTP session.
# The +address+ should be a DNS hostname or IP address.
Expand Down Expand Up @@ -50,6 +49,7 @@ def initialize(address, port = nil)
@ssl_session = nil
@enable_post_connection_check = true
@sspi_enabled = false

SSL_IVNAMES.each do |ivname|
instance_variable_set ivname, nil
end
Expand Down Expand Up @@ -93,6 +93,7 @@ def connect
end
@ssl_context = OpenSSL::SSL::SSLContext.new
@ssl_context.set_params(ssl_parameters)

D "starting SSL for #{conn_address}:#{conn_port}..."
s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context)
s.sync_close = true
Expand Down Expand Up @@ -154,7 +155,7 @@ def transport_request(req)
# added for whatweb
#res = HTTPResponse.read_new(@socket)
res, y = ExtendedHTTPResponse.read_new(@socket)
@raw << y
@raw << y
#
res.decode_content = req.decode_content
end while res.kind_of?(HTTPContinue)
Expand Down
20 changes: 6 additions & 14 deletions lib/target.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,38 +158,30 @@ def open_url(options)
req=nil

if options[:method] == "GET"
req=Net::HTTP::Get.new(getthis, $CUSTOM_HEADERS)
req=ExtendedHTTP::Get.new(getthis, $CUSTOM_HEADERS)
end
if options[:method] == "HEAD"
req=Net::HTTP::Head.new(getthis, $CUSTOM_HEADERS)
req=ExtendedHTTP::Head.new(getthis, $CUSTOM_HEADERS)
end
if options[:method] == "POST"
req=Net::HTTP::Post.new(getthis, $CUSTOM_HEADERS)
req=ExtendedHTTP::Post.new(getthis, $CUSTOM_HEADERS)
req.set_form_data(options[:data])
end

if $BASIC_AUTH_USER
req.basic_auth $BASIC_AUTH_USER, $BASIC_AUTH_PASS
end

res=http.request(req)

@raw_headers=http.raw.join("\n")
@headers={}; res.each_header { |x, y| @headers[x]=y }
@headers["set-cookie"] = res.get_fields('set-cookie').join("\n") unless @headers["set-cookie"].nil?
require 'open-uri'

@body=Net::HTTP.get(@uri)
@body=res.body
@status=res.code.to_i
puts @uri.to_s + " [#{status}]" if $verbose > 1

=begin
if @raw_headers =~ /^Server:.*^Server:/m
puts "raw_headers-"*20
puts @raw_headers
exit
end
puts @raw_headers+"\n"+"*"*40
=end

rescue SocketError => err
error(@target + " ERROR: Socket error #{err}")
return
Expand Down
1 change: 1 addition & 0 deletions whatweb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ require 'tempfile'
require 'rbconfig' # detect environment, e.g. windows or linux
require 'resolv'
require 'resolv-replace' # asynchronous DNS
require 'open-uri'



Expand Down

0 comments on commit 29ba77a

Please sign in to comment.