Added support for chunked transfer-encoding response.

Implemented correctly the redirection support for NET_HTTP_CLIENT...
Added the possibility to use HTTP/1.0 .
Splitted the manual tests that were using during development.
First step to redesign and clean the new code.
This commit is contained in:
2015-09-10 23:05:56 +02:00
parent 9cd0f0b117
commit 29c4931dc0
8 changed files with 425 additions and 202 deletions

View File

@@ -20,15 +20,22 @@ feature {NONE} -- Initialization
current_redirects := 0
request_method := a_request_method
session := a_session
initialize (a_url, ctx)
ensure
context_set: context = ctx
ctx_header_set: ctx /= Void implies across ctx.headers as ctx_h all attached headers.item (ctx_h.key) as v and then v.same_string (ctx_h.item) end
end
initialize (a_url: READABLE_STRING_8; ctx: like context)
do
url := a_url
headers := session.headers.twin
if ctx /= Void then
context := ctx
import (ctx)
else
context := Void
end
ensure
context_set: context = ctx
ctx_header_set: ctx /= Void implies across ctx.headers as ctx_h all attached headers.item (ctx_h.key) as v and then v.same_string (ctx_h.item) end
end
feature {NONE} -- Internal

View File

@@ -81,7 +81,10 @@ feature -- Access
-- Optional output file to get downloaded content and header
output_content_file: detachable FILE
-- Optional output file to get downloaded content
-- Optional output file to get downloaded content
http_version: detachable IMMUTABLE_STRING_8
-- Overwrite default http version if set.
feature -- Status report
@@ -209,6 +212,17 @@ feature -- Element change
output_content_file := f
end
set_http_version (v: detachable READABLE_STRING_8)
require
valid_version: v = Void or else v.starts_with_general ("HTTP/")
do
if v = Void then
http_version := Void
else
create http_version.make_from_string (v)
end
end
feature -- Status setting
set_proxy (a_host: detachable READABLE_STRING_8; a_port: INTEGER)
@@ -264,7 +278,7 @@ feature {NONE} -- Implementation
end
note
copyright: "2011-2013, Jocelyn Fiat, Javier Velilla, Eiffel Software and others"
copyright: "2011-2015, Jocelyn Fiat, Javier Velilla, Eiffel Software and others"
license: "Eiffel Forum License v2 (see http://www.eiffel.com/licensing/forum.txt)"
source: "[
Eiffel Software

View File

@@ -61,6 +61,7 @@ feature -- Execution
l_upload_data: detachable READABLE_STRING_8
l_upload_filename: detachable READABLE_STRING_GENERAL
l_headers: like headers
l_is_http_1_0: BOOLEAN
do
if not retried then
curl := session.curl
@@ -70,6 +71,10 @@ feature -- Execution
ctx := context
if ctx /= Void then
l_is_http_1_0 := attached ctx.http_version as l_http_version and then l_http_version.same_string ("HTTP/1.0")
end
--| Configure cURL session
initialize_curl_session (ctx, curl, curl_easy, curl_handle)
@@ -84,7 +89,11 @@ feature -- Execution
io.put_new_line
end
curl_easy.setopt_string (curl_handle, {CURL_OPT_CONSTANTS}.curlopt_url, l_url)
if l_is_http_1_0 then
curl_easy.setopt_integer (curl_handle, {CURL_OPT_CONSTANTS}.curlopt_http_version, {CURL_OPT_CONSTANTS}.curl_http_version_1_0)
else
curl_easy.setopt_integer (curl_handle, {CURL_OPT_CONSTANTS}.curlopt_http_version, {CURL_OPT_CONSTANTS}.curl_http_version_none)
end
l_headers := headers
-- Context

View File

@@ -31,6 +31,7 @@ feature -- Access
response: HTTP_CLIENT_RESPONSE
-- <Precursor>
local
redirection_response: detachable like response
l_uri: URI
l_host: READABLE_STRING_8
l_request_uri: STRING
@@ -43,7 +44,6 @@ feature -- Access
l_location: detachable READABLE_STRING_8
l_port: INTEGER
l_authorization: HTTP_AUTHORIZATION
l_session: NET_HTTP_CLIENT_SESSION
l_platform: STRING
l_useragent: STRING
l_upload_data: detachable READABLE_STRING_8
@@ -55,9 +55,14 @@ feature -- Access
l_mime_type_mapping: HTTP_FILE_EXTENSION_MIME_MAPPING
l_mime_type: STRING
l_fn_extension: READABLE_STRING_GENERAL
l_i: INTEGER
l_prev_header: READABLE_STRING_8
l_boundary: READABLE_STRING_8
l_is_http_1_0: BOOLEAN
do
ctx := context
if ctx /= Void then
l_is_http_1_0 := attached ctx.http_version as l_http_version and then l_http_version.same_string ("HTTP/1.0")
end
create Result.make (url)
create l_form_string.make_empty
@@ -129,22 +134,23 @@ feature -- Access
l_form_data := ctx.form_parameters
check non_empty_form_data: not l_form_data.is_empty end
if l_upload_data = Void and l_upload_filename = Void then
-- Send as form-urlencoded
-- Send as form-urlencoded
headers.extend ("application/x-www-form-urlencoded", "Content-Type")
l_upload_data := ctx.form_parameters_to_url_encoded_string
headers.extend (l_upload_data.count.out, "Content-Length")
headers.force (l_upload_data.count.out, "Content-Length")
else
-- create form
headers.extend ("multipart/form-data; boundary=----------------------------5eadfcf3bb3e", "Content-Type")
if attached l_form_data then
-- create form
l_boundary := new_mime_boundary
headers.extend ("multipart/form-data; boundary=" + l_boundary, "Content-Type")
if l_form_data /= Void then
headers.extend ("*/*", "Accept")
from
l_form_data.start
until
l_form_data.after
loop
l_form_string.append ("------------------------------5eadfcf3bb3e")
l_form_string.append (l_boundary)
l_form_string.append (http_end_of_header_line)
l_form_string.append ("Content-Disposition: form-data; name=")
l_form_string.append ("%"" + l_form_data.key_for_iteration + "%"")
@@ -155,8 +161,8 @@ feature -- Access
l_form_data.forth
end
if l_upload_filename /= Void then
-- get file extension, otherwise set default
if l_upload_filename /= Void then
-- get file extension, otherwise set default
l_mime_type := "application/octet-stream"
create l_mime_type_mapping.make_default
l_fn_extension := l_upload_filename.tail (l_upload_filename.count - l_upload_filename.last_index_of ('.', l_upload_filename.count))
@@ -164,7 +170,7 @@ feature -- Access
l_mime_type := mime
end
l_form_string.append ("------------------------------5eadfcf3bb3e")
l_form_string.append (l_boundary)
l_form_string.append (http_end_of_header_line)
l_form_string.append ("Content-Disposition: form-data; name=%"" + l_upload_filename.as_string_32 + "%"")
l_form_string.append ("; filename=%"" + l_upload_filename + "%"")
@@ -182,7 +188,7 @@ feature -- Access
end
l_form_string.append (http_end_of_header_line)
end
l_form_string.append ("------------------------------5eadfcf3bb3e--")
l_form_string.append (l_boundary + "--") --| end
l_upload_data := l_form_string
headers.extend (l_upload_data.count.out, "Content-Length")
@@ -212,7 +218,7 @@ feature -- Access
end
end
-- handle put requests
-- handle put requests
if request_method.is_case_insensitive_equal ("PUT") then
if ctx /= Void then
if ctx.has_upload_filename then
@@ -227,11 +233,9 @@ feature -- Access
end
end
end
-- Connect
-- Connect
create socket.make_client_by_port (l_port, l_host)
socket.set_timeout (timeout)
socket.set_connect_timeout (connect_timeout)
@@ -241,7 +245,11 @@ feature -- Access
s.append_character (' ')
s.append (l_request_uri)
s.append_character (' ')
s.append (Http_version)
if l_is_http_1_0 then
s.append ("HTTP/1.0")
else
s.append ("HTTP/1.1")
end
s.append (Http_end_of_header_line)
s.append (Http_host_header)
@@ -275,7 +283,7 @@ feature -- Access
until
l_upload_file.after
loop
l_upload_file.read_line
l_upload_file.read_line_thread_aware
s.append (l_upload_file.last_string)
end
end
@@ -283,65 +291,29 @@ feature -- Access
socket.put_string (s)
-- Get header message
from
l_content_length := -1
create l_message.make_empty
socket.read_line
s := socket.last_string
until
s.same_string ("%R") or not socket.is_readable
loop
l_message.append (s)
l_message.append_character ('%N')
-- Search for Content-Length is not yet set.
if
l_content_length = -1 and then -- Content-Length is not yet found.
s.starts_with_general ("Content-Length:")
then
i := s.index_of (':', 1)
check has_colon: i > 0 end
s.remove_head (i)
s.right_adjust -- Remove trailing %R
if s.is_integer then
l_content_length := s.to_integer
end
elseif
l_location = Void and then
s.starts_with_general ("Location:")
then
i := s.index_of (':', 1)
check has_colon: i > 0 end
s.remove_head (i)
s.left_adjust -- Remove startung spaces
s.right_adjust -- Remove trailing %R
l_location := s
elseif s.starts_with_general ("Set-Cookie:") then
i := s.index_of (':', 1)
s.remove_head (i)
s.left_adjust
s.right_adjust
session.set_cookie (s)
end
-- Next iteration
socket.read_line
s := socket.last_string
--| Get response.
--| Get header message
create l_message.make_empty
read_header_from_socket (socket, l_message)
l_prev_header := Result.raw_header
Result.set_raw_header (l_message.string)
l_content_length := -1
if attached Result.header ("Content-Length") as s_len and then s_len.is_integer then
l_content_length := s_len.to_integer
end
l_location := Result.header ("Location")
if attached Result.header ("Set-Cookies") as s_cookies then
session.set_cookie (s_cookies)
end
l_message.append (http_end_of_header_line)
-- Get content if any.
if
l_content_length > 0 and
socket.is_readable
then
socket.read_stream_thread_aware (l_content_length)
if socket.bytes_read /= l_content_length then
check full_content_read: False end
end
l_message.append (socket.last_string)
end
append_socket_content_to (Result, socket, l_content_length, l_message)
-- Restore previous header
Result.set_raw_header (l_prev_header)
-- Set message
Result.set_response_message (l_message, context)
-- Get status code.
if attached Result.status_line as l_status_line then
if l_status_line.starts_with ("HTTP/") then
@@ -364,12 +336,15 @@ feature -- Access
end
end
-- follow redirect
-- follow redirect
if l_location /= Void then
if current_redirects < max_redirects then
current_redirects := current_redirects + 1
url := l_location
Result := response
initialize (l_location, ctx)
redirection_response := response
redirection_response.add_redirection (Result.status_line, Result.raw_header, Result.body)
Result := redirection_response
-- Result.add_redirection (redirection_response.status_line, redirection_response.raw_header, redirection_response.body)
end
end
@@ -378,6 +353,108 @@ feature -- Access
Result.set_error_message ("Could not connect")
end
end
feature {NONE} -- Helpers
read_header_from_socket (a_socket: NETWORK_STREAM_SOCKET; a_output: STRING)
-- Get header from `a_socket' into `a_output'.
local
s: READABLE_STRING_8
do
if a_socket.is_readable then
from
s := ""
until
s.same_string ("%R") or not a_socket.is_readable
loop
a_socket.read_line_thread_aware
s := a_socket.last_string
if s.same_string ("%R") then
-- Reach end of header
-- a_output.append (http_end_of_header_line)
else
a_output.append (s)
a_output.append_character ('%N')
end
end
end
end
append_socket_content_to (a_response: HTTP_CLIENT_RESPONSE; a_socket: NETWORK_STREAM_SOCKET; a_len: INTEGER; a_buffer: STRING)
-- Get content from `a_socket' and append it to `a_buffer'.
-- If `a_len' is negative, try to get as much as possible,
-- this is probably HTTP/1.0 without any Content-Length.
local
s: STRING_8
n,pos: INTEGER
hexa2int: HEXADECIMAL_STRING_TO_INTEGER_CONVERTER
do
if a_socket.is_readable then
if a_len >= 0 then
a_socket.read_stream_thread_aware (a_len)
s := a_socket.last_string
check full_content_read: a_socket.bytes_read = a_len end
a_buffer.append (s)
else
if attached a_response.header ("Transfer-Encoding") as l_enc and then l_enc.is_case_insensitive_equal ("chunked") then
from
create hexa2int.make
n := 1
until
n = 0 or not a_socket.is_readable
loop
a_socket.read_line_thread_aware -- Read chunk info
s := a_socket.last_string
s.right_adjust
pos := s.index_of (';', 1)
if pos > 0 then
s.keep_head (pos - 1)
end
if s.is_empty then
n := 0
else
hexa2int.parse_string_with_type (s, hexa2int.type_integer)
if hexa2int.parse_successful then
n := hexa2int.parsed_integer
else
n := 0
end
end
if n > 0 then
a_socket.read_stream_thread_aware (n)
check a_socket.bytes_read = n end
a_buffer.append (a_socket.last_string)
a_socket.read_character
check a_socket.last_character = '%R' end
a_socket.read_character
check a_socket.last_character = '%N' end
end
end
else
-- HTTP/1.0
from
n := 1_024
until
n < 1_024 or not a_socket.is_readable
loop
a_socket.read_stream_thread_aware (1_024)
s := a_socket.last_string
n := a_socket.bytes_read
a_buffer.append (s)
end
end
end
end
end
new_mime_boundary: STRING
-- New MIME boundary.
do
-- FIXME: better boundary creation
Result := "----------------------------5eadfcf3bb3e"
end
invariant
note
copyright: "2011-2015, Jocelyn Fiat, Javier Velilla, Eiffel Software and others"
license: "Eiffel Forum License v2 (see http://www.eiffel.com/licensing/forum.txt)"

View File

@@ -16,122 +16,175 @@ feature {NONE} -- Initialization
make
-- Run application.
do
requestbin_path := "/15u47xi2"
test_1
test_2
test_3
test_4
test_5
test_6
test_7
end
requestbin_path: STRING
feature -- Tests
test_1
local
sess: NET_HTTP_CLIENT_SESSION
h: STRING_8
l_ctx: HTTP_CLIENT_REQUEST_CONTEXT
l_test_case: INTEGER
l_requestbin_path: STRING
do
l_requestbin_path := "/15u47xi2"
create h.make_empty
l_test_case := 1 -- select which test to execute
inspect l_test_case
when 1 then
-- URL ENCODED POST REQUEST
-- check requestbin to ensure the "Hello World" has been received in the raw body
-- also check that User-Agent was sent
create sess.make("http://requestb.in")
if attached sess.post (l_requestbin_path, Void, "Hello World").headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
create h.make_empty
create sess.make("http://requestb.in")
if attached sess.post (requestbin_path, Void, "Hello World").headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
print (h)
when 2 then
end
print (h)
end
test_2
local
sess: NET_HTTP_CLIENT_SESSION
h: STRING_8
l_ctx: HTTP_CLIENT_REQUEST_CONTEXT
do
-- POST REQUEST WITH FORM DATA
-- check requestbin to ensure the form parameters are correctly received
create sess.make("http://requestb.in")
create l_ctx.make
l_ctx.form_parameters.extend ("First Value", "First Key")
l_ctx.form_parameters.extend ("Second Value", "Second Key")
create sess.make("http://requestb.in")
if attached sess.post (l_requestbin_path, l_ctx, "").headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
create sess.make("http://requestb.in")
create l_ctx.make
l_ctx.form_parameters.extend ("First Value", "First Key")
l_ctx.form_parameters.extend ("Second Value", "Second Key")
create sess.make("http://requestb.in")
create h.make_empty
if attached sess.post (requestbin_path, l_ctx, "").headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
end
print (h)
end
when 3 then
test_3
local
sess: NET_HTTP_CLIENT_SESSION
h: STRING_8
l_ctx: HTTP_CLIENT_REQUEST_CONTEXT
do
-- POST REQUEST WITH A FILE
-- check requestbin to ensure the form parameters are correctly received
-- set filename to a local file
create sess.make("http://requestb.in")
create l_ctx.make
l_ctx.set_upload_filename ("C:\temp\test.txt")
if attached sess.post (l_requestbin_path, l_ctx, "").headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
create sess.make("http://requestb.in")
create l_ctx.make
l_ctx.set_upload_filename ("C:\temp\test.txt")
create h.make_empty
if attached sess.post (requestbin_path, l_ctx, "").headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
end
print (h)
end
when 4 then
test_4
local
sess: NET_HTTP_CLIENT_SESSION
h: STRING_8
l_ctx: HTTP_CLIENT_REQUEST_CONTEXT
do
-- PUT REQUEST WITH A FILE
-- check requestbin to ensure the file is correctly received
-- set filename to a local file
create sess.make("http://requestb.in")
create l_ctx.make
l_ctx.set_upload_filename ("C:\temp\test.txt")
if attached sess.put (l_requestbin_path, l_ctx, "").headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
create sess.make("http://requestb.in")
create l_ctx.make
l_ctx.set_upload_filename ("C:\temp\test.txt")
create h.make_empty
if attached sess.put (requestbin_path, l_ctx, "").headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
end
print (h)
end
when 5 then
test_5
local
sess: NET_HTTP_CLIENT_SESSION
h: STRING_8
l_ctx: HTTP_CLIENT_REQUEST_CONTEXT
do
-- POST REQUEST WITH A FILE AND FORM DATA
-- check requestbin to ensure the file and form parameters are correctly received
-- set filename to a local file
create sess.make("http://requestb.in")
create l_ctx.make
l_ctx.set_upload_filename ("C:\temp\logo.png")
l_ctx.form_parameters.extend ("First Value", "First Key")
l_ctx.form_parameters.extend ("Second Value", "Second Key")
if attached sess.post (l_requestbin_path, l_ctx, "").headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
create sess.make("http://requestb.in")
create l_ctx.make
l_ctx.set_upload_filename ("C:\temp\logo.png")
l_ctx.form_parameters.extend ("First Value", "First Key")
l_ctx.form_parameters.extend ("Second Value", "Second Key")
create h.make_empty
if attached sess.post (requestbin_path, l_ctx, "").headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
end
print (h)
end
when 6 then
test_6
local
sess: NET_HTTP_CLIENT_SESSION
h: STRING_8
l_ctx: HTTP_CLIENT_REQUEST_CONTEXT
do
-- GET REQUEST, Forwarding (google's first answer is a forward)
-- check headers received (printed in console)
create sess.make("http://google.com")
if attached sess.get ("/", Void).headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
create sess.make("http://google.com")
create h.make_empty
if attached sess.get ("/", Void).headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
print (h)
when 7 then
-- GET REQUEST WITH AUTHENTICATION, see http://browserspy.dk/password.php
-- check header WWW-Authendicate is received (authentication successful)
create sess.make("http://test:test@browserspy.dk")
if attached sess.get ("/password-ok.php", Void).headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
end
print (h)
else
end
print (h)
end
test_7
local
sess: NET_HTTP_CLIENT_SESSION
h: STRING_8
l_ctx: HTTP_CLIENT_REQUEST_CONTEXT
do
-- GET REQUEST WITH AUTHENTICATION, see http://browserspy.dk/password.php
-- check header WWW-Authenticate is received (authentication successful)
create sess.make("http://test:test@browserspy.dk")
create h.make_empty
if attached sess.get ("/password-ok.php", Void).headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
end
print (h)
end
end
end

View File

@@ -7,49 +7,27 @@ note
revision: "$Revision$"
testing: "type/manual"
class
TEST_HTTP_CLIENT
deferred class
TEST_HTTP_CLIENT_I
inherit
EQA_TEST_SET
feature -- Factory
new_session (a_url: READABLE_STRING_8): HTTP_CLIENT_SESSION
deferred
end
feature -- Test routines
test_libcurl_http_client
test_http_client
-- New test routine
local
sess: LIBCURL_HTTP_CLIENT_SESSION
sess: like new_session
h: STRING_8
do
create sess.make ("http://www.google.com")
if attached sess.get ("/search?q=eiffel", Void) as res then
assert ("Get returned without error", not res.error_occurred)
create h.make_empty
if attached res.headers as hds then
across
hds as c
loop
h.append (c.item.name + ": " + c.item.value + "%R%N")
end
end
if attached res.body as l_body then
assert ("body not empty", not l_body.is_empty)
else
assert ("missing body", False)
end
assert ("same headers", h.same_string (res.raw_header))
else
assert ("Not found", False)
end
end
test_socket_http_client
-- New test routine
local
sess: NET_HTTP_CLIENT_SESSION
h: STRING_8
do
create sess.make ("http://www.google.com")
sess := new_session ("http://www.google.com")
if attached sess.get ("/search?q=eiffel", Void) as res then
assert ("Get returned without error", not res.error_occurred)
create h.make_empty
@@ -71,13 +49,13 @@ feature -- Test routines
end
end
test_socket_http_client_requestbin
test_http_client_requestbin
local
sess: NET_HTTP_CLIENT_SESSION
sess: like new_session
h: STRING_8
do
--| Add your code here
create sess.make("http://requestb.in")
sess := new_session ("http://requestb.in")
create h.make_empty
if attached sess.get ("/1a0q2h61", Void).headers as hds then
across

View File

@@ -0,0 +1,43 @@
note
description: "[
Eiffel tests that can be executed by testing tool.
]"
author: "EiffelStudio test wizard"
date: "$Date$"
revision: "$Revision$"
testing: "type/manual"
class
TEST_LIBCURL_HTTP_CLIENT
inherit
TEST_HTTP_CLIENT_I
feature -- Factory
new_session (a_url: READABLE_STRING_8): HTTP_CLIENT_SESSION
do
create {LIBCURL_HTTP_CLIENT_SESSION} Result.make (a_url)
end
feature -- Tests
test_libcurl_http_client
do
test_http_client
end
test_libcurl_http_client_requestbin
do
test_http_client_requestbin
end
test_libcurl_headers
do
test_headers
end
end

View File

@@ -0,0 +1,42 @@
note
description: "[
Eiffel tests that can be executed by testing tool.
]"
author: "EiffelStudio test wizard"
date: "$Date$"
revision: "$Revision$"
testing: "type/manual"
class
TEST_NET_HTTP_CLIENT
inherit
TEST_HTTP_CLIENT_I
feature -- Factory
new_session (a_url: READABLE_STRING_8): HTTP_CLIENT_SESSION
do
create {NET_HTTP_CLIENT_SESSION} Result.make (a_url)
end
feature -- Tests
test_net_http_client
do
test_http_client
end
test_net_http_client_requestbin
do
test_http_client_requestbin
end
test_net_headers
do
test_headers
end
end