From a547cbaeb19ee6bf8ad7169843d1bfd305361f9c Mon Sep 17 00:00:00 2001 From: Jocelyn Fiat Date: Thu, 11 Apr 2013 15:53:46 +0200 Subject: [PATCH] Fixed HTTP_CLIENT_RESPONSE when dealing with redirection before it was storing some header in the body. now we added redirections: .. which is a list of redirection informations: - status line - header - and eventual redirection body (but at least by default, libcurl does not cache body) Enhanced the http_client library to be able to write directly the downloaded data into a file (or as convenient thanks to agent). --- .../src/http_client_request_context.e | 37 ++++- .../http_client/src/http_client_response.e | 130 ++++++++++++++++++ .../spec/libcurl/libcurl_custom_function.e | 105 ++++++++++++++ .../libcurl/libcurl_http_client_request.e | 102 +++++++++----- .../libcurl_upload_file_read_function.e | 3 + 5 files changed, 341 insertions(+), 36 deletions(-) create mode 100644 library/network/http_client/src/spec/libcurl/libcurl_custom_function.e diff --git a/library/network/http_client/src/http_client_request_context.e b/library/network/http_client/src/http_client_request_context.e index 7aef5e19..a02ba6b3 100644 --- a/library/network/http_client/src/http_client_request_context.e +++ b/library/network/http_client/src/http_client_request_context.e @@ -71,7 +71,17 @@ feature -- Access upload_filename: detachable READABLE_STRING_8 -- Upload data read from `upload_filename' - --| Note: make sure to precise the Content-Type header + --| Note: make sure to precise the Content-Type header + + write_agent: detachable PROCEDURE [ANY, TUPLE [READABLE_STRING_8]] + -- Use this agent to hook the write of the response. + --| could be used to save the response directly in a file + + output_file: detachable FILE + -- Optional output file to get downloaded content and header + + output_content_file: detachable FILE + -- Optional output file to get downloaded content feature -- Status report @@ -90,6 +100,12 @@ feature -- Status report Result := attached upload_filename as fn and then not fn.is_empty end + has_write_option: BOOLEAN + -- Has non default write behavior? + do + Result := write_agent /= Void or output_file /= Void or output_content_file /= Void + end + feature -- Element change add_header (k: READABLE_STRING_8; v: READABLE_STRING_8) @@ -126,6 +142,25 @@ feature -- Element change upload_filename := a_fn end + set_write_agent (agt: like write_agent) + do + write_agent := agt + end + + set_output_file (f: FILE) + require + f_is_open_write: f.is_open_write + do + output_file := f + end + + set_output_content_file (f: FILE) + require + f_is_open_write: f.is_open_write + do + output_content_file := f + end + feature -- Status setting set_proxy (a_host: detachable READABLE_STRING_8; a_port: INTEGER) diff --git a/library/network/http_client/src/http_client_response.e b/library/network/http_client/src/http_client_response.e index f16a181a..aa6e547f 100644 --- a/library/network/http_client/src/http_client_response.e +++ b/library/network/http_client/src/http_client_response.e @@ -52,9 +52,14 @@ feature -- Access status: INTEGER assign set_status -- Status code of the response. + status_line: detachable READABLE_STRING_8 + raw_header: READABLE_STRING_8 -- Raw http header of the response. + redirections: detachable ARRAYED_LIST [TUPLE [status_line: detachable READABLE_STRING_8; raw_header: READABLE_STRING_8; body: detachable READABLE_STRING_8]] + -- Header of previous redirection if any. + header (a_name: READABLE_STRING_8): detachable READABLE_STRING_8 -- Header entry value related to `a_name' -- if multiple entries, just concatenate them using comma character @@ -150,6 +155,39 @@ feature -- Access body: detachable READABLE_STRING_8 assign set_body -- Content of the response + response_message_source (a_include_redirection: BOOLEAN): STRING_8 + -- Full message source including redirection if any + do + create Result.make (1_024) + if + a_include_redirection and then + attached redirections as lst + then + across + lst as c + loop + if attached c.item.status_line as s then + Result.append (s) + Result.append ("%R%N") + end + Result.append (c.item.raw_header) + Result.append ("%R%N") + if attached c.item.body as l_body then + Result.append (l_body) + end + end + end + if attached status_line as s then + Result.append (s) + Result.append ("%R%N") + end + Result.append (raw_header) + Result.append ("%R%N") + if attached body as l_body then + Result.append (l_body) + end + end + feature -- Change set_status (s: INTEGER) @@ -158,6 +196,85 @@ feature -- Change status := s end + set_response_message (a_source: READABLE_STRING_8; ctx: detachable HTTP_CLIENT_REQUEST_CONTEXT) + -- Parse `a_source' response message + -- and set `header' and `body'. + --| ctx is the context associated with the request + --| it might be useful to deal with redirection customization... + local + i, j, pos: INTEGER + l_has_location: BOOLEAN + l_content_length: INTEGER + s: READABLE_STRING_8 + l_status_line,h: detachable STRING_8 + do + from + i := 1 + j := 1 + pos := 1 + + i := a_source.substring_index ("%R%N", i) + until + i = 0 or i > a_source.count + loop + s := a_source.substring (j, i - 1) + if s.starts_with ("HTTP/") then + --| Skip first line which is the status line + --| ex: HTTP/1.1 200 OK%R%N + j := i + 2 + l_status_line := s + pos := j + elseif s.is_empty then + -- End of header %R%N%R%N + if attached raw_header as l_raw_header and then not l_raw_header.is_empty then + add_redirection (status_line, l_raw_header, body) + end + + h := a_source.substring (pos, i - 1) + + j := i + 2 + pos := j + status_line := l_status_line + set_raw_header (h) + +-- libcURL does not cache redirection content. +-- FIXME: check if this is customizable +-- if l_has_location then +-- if l_content_length > 0 then +-- j := pos + l_content_length - 1 +-- l_body := a_source.substring (pos, j) +-- pos := j +-- else +-- l_body := Void +-- end +-- set_body (l_body) +-- end + if not l_has_location then + i := 0 -- exit loop + end + l_content_length := 0 + l_status_line := Void + l_has_location := False + else + if s.starts_with ("Location:") then + l_has_location := True + elseif s.starts_with ("Content-Length:") then + s := s.substring (16, s.count) + if s.is_integer then + l_content_length := s.to_integer + end + end + j := i + 2 + end + if i > 0 then + i := a_source.substring_index ("%R%N", j) + end + end + set_body (a_source.substring (pos, a_source.count)) + ensure + parsed: response_message_source (True).count = a_source.count + end + set_raw_header (h: READABLE_STRING_8) -- Set http header `raw_header' to `h' do @@ -166,6 +283,19 @@ feature -- Change internal_headers := Void end + add_redirection (s: detachable READABLE_STRING_8; h: READABLE_STRING_8; a_body: detachable READABLE_STRING_8) + -- Add redirection with status line `s' and raw header `h' and body `a_body' if any + local + lst: like redirections + do + lst := redirections + if lst = Void then + create lst.make (1) + redirections := lst + end + lst.force ([s,h, a_body]) + end + set_body (s: like body) -- Set `body' message to `s' do diff --git a/library/network/http_client/src/spec/libcurl/libcurl_custom_function.e b/library/network/http_client/src/spec/libcurl/libcurl_custom_function.e new file mode 100644 index 00000000..f8e50a33 --- /dev/null +++ b/library/network/http_client/src/spec/libcurl/libcurl_custom_function.e @@ -0,0 +1,105 @@ +note + description: "[ + LIBCURL_CUSTOM_FUNCTION is used to custom the input and output libcurl execution + + ]" + date: "$Date$" + revision: "$Revision$" + +class + LIBCURL_CUSTOM_FUNCTION + +inherit + LIBCURL_DEFAULT_FUNCTION + redefine + read_function, + write_function + end + +create + make + +feature -- Access + + write_procedure: detachable PROCEDURE [ANY, TUPLE [READABLE_STRING_8]] + -- File for sending data + + file_to_read: detachable FILE + -- File for sending data + +feature -- Change + + set_write_procedure (proc: like write_procedure) + do + write_procedure := proc + end + + set_file_to_read (f: like file_to_read) + do + file_to_read := f + end + +feature -- Basic operation + + write_function (a_data_pointer: POINTER; a_size, a_nmemb: INTEGER; a_object_id: POINTER): INTEGER + -- Redefine + local + l_c_string: C_STRING + s: STRING + do + if attached write_procedure as agt then + Result := a_size * a_nmemb + create l_c_string.make_shared_from_pointer_and_count (a_data_pointer, Result) + s := l_c_string.substring (1, Result) + agt.call ([s]) + else + Result := Precursor (a_data_pointer, a_size, a_nmemb, a_object_id) + end + end + + read_function (a_data_pointer: POINTER; a_size, a_nmemb: INTEGER_32; a_object_id: POINTER): INTEGER_32 + -- + local + l_pointer: MANAGED_POINTER + l_max_transfer, l_byte_transfered: INTEGER + do + if attached file_to_read as l_file then + if not l_file.after then + l_max_transfer := a_size * a_nmemb + if l_max_transfer > l_file.count - l_file.position then + l_max_transfer := l_file.count - l_file.position + end + create l_pointer.share_from_pointer (a_data_pointer, l_max_transfer) + + from + until + l_file.after or l_byte_transfered >= l_max_transfer + loop + l_file.read_character + l_pointer.put_character (l_file.last_character, l_byte_transfered) + + l_byte_transfered := l_byte_transfered + 1 + end + + Result := l_max_transfer + else + -- Result is 0 means stop file transfer + Result := 0 + end + else + Result := Precursor (a_data_pointer, a_size, a_nmemb, a_object_id) + end + end + + +note + copyright: "2011-2012, Jocelyn Fiat, Javier Velilla, Eiffel Software and others" + license: "Eiffel Forum License v2 (see http://www.eiffel.com/licensing/forum.txt)" + source: "[ + Eiffel Software + 5949 Hollister Ave., Goleta, CA 93117 USA + Telephone 805-685-1006, Fax 805-685-6869 + Website http://www.eiffel.com + Customer support http://support.eiffel.com + ]" +end diff --git a/library/network/http_client/src/spec/libcurl/libcurl_http_client_request.e b/library/network/http_client/src/spec/libcurl/libcurl_http_client_request.e index 8382b77f..87d3d4ee 100644 --- a/library/network/http_client/src/spec/libcurl/libcurl_http_client_request.e +++ b/library/network/http_client/src/spec/libcurl/libcurl_http_client_request.e @@ -47,12 +47,12 @@ feature -- Execution execute: HTTP_CLIENT_RESPONSE local l_result: INTEGER - l_curl_string: CURL_STRING + l_curl_string: detachable CURL_STRING l_url: READABLE_STRING_8 l_form: detachable CURL_FORM l_last: CURL_FORM l_upload_file: detachable RAW_FILE - l_uploade_file_read_function: detachable LIBCURL_UPLOAD_FILE_READ_FUNCTION + l_custom_function: detachable LIBCURL_CUSTOM_FUNCTION curl: detachable CURL_EXTERNALS curl_easy: detachable CURL_EASY_EXTERNALS curl_handle: POINTER @@ -178,10 +178,12 @@ feature -- Execution curl_easy.setopt_integer (curl_handle, {CURL_OPT_CONSTANTS}.curlopt_infilesize, l_upload_file.count) -- specify callback read function for upload file - create l_uploade_file_read_function.make_with_file (l_upload_file) + if l_custom_function = Void then + create l_custom_function.make + end + l_custom_function.set_file_to_read (l_upload_file) l_upload_file.open_read - curl_easy.set_curl_function (l_uploade_file_read_function) - curl_easy.set_read_function (curl_handle) + curl_easy.set_curl_function (l_custom_function) end else check no_upload_data: l_upload_data = Void and l_upload_filename = Void end @@ -197,6 +199,7 @@ feature -- Execution p_slist := curl.slist_append (p_slist, "Expect:") curl_easy.setopt_slist (curl_handle, {CURL_OPT_CONSTANTS}.curlopt_httpheader, p_slist) + --| Execution curl_easy.set_read_function (curl_handle) curl_easy.set_write_function (curl_handle) @@ -204,8 +207,29 @@ feature -- Execution curl_easy.set_debug_function (curl_handle) curl_easy.setopt_integer (curl_handle, {CURL_OPT_CONSTANTS}.curlopt_verbose, 1) end - create l_curl_string.make_empty - curl_easy.setopt_curl_string (curl_handle, {CURL_OPT_CONSTANTS}.curlopt_writedata, l_curl_string) + + --| Write options + + if ctx /= Void and then ctx.has_write_option then + if l_custom_function = Void then + create l_custom_function.make + end + if attached ctx.write_agent as l_write_agent then + l_custom_function.set_write_procedure (l_write_agent) + elseif attached ctx.output_content_file as l_output_content_file then + create l_curl_string.make_empty + l_custom_function.set_write_procedure (new_write_content_data_to_file_agent (l_output_content_file, l_curl_string)) + -- l_curl_string will contain the raw header, used to fill `Result' + elseif attached ctx.output_file as l_output_file then + create l_curl_string.make_empty + l_custom_function.set_write_procedure (new_write_data_to_file_agent (l_output_file, l_curl_string)) + -- l_curl_string will contain the raw header, used to fill `Result' + end + curl_easy.set_curl_function (l_custom_function) + else + create l_curl_string.make_empty + curl_easy.setopt_curl_string (curl_handle, {CURL_OPT_CONSTANTS}.curlopt_writedata, l_curl_string) + end create Result.make (l_url) l_result := curl_easy.perform (curl_handle) @@ -213,7 +237,9 @@ feature -- Execution --| Result if l_result = {CURL_CODES}.curle_ok then Result.status := response_status_code (curl_easy, curl_handle) - set_header_and_body_to (l_curl_string.string, Result) + if l_curl_string /= Void then + Result.set_response_message (l_curl_string.string, ctx) + end else Result.set_error_message ("Error: cURL Error[" + l_result.out + "]") Result.status := response_status_code (curl_easy, curl_handle) @@ -326,35 +352,41 @@ feature {NONE} -- Implementation end end - - set_header_and_body_to (a_source: READABLE_STRING_8; res: HTTP_CLIENT_RESPONSE) - -- Parse `a_source' response - -- and set `header' and `body' from HTTP_CLIENT_RESPONSE `res' - local - pos, l_start : INTEGER + new_write_data_to_file_agent (f: FILE; h: detachable STRING): PROCEDURE [ANY, TUPLE [READABLE_STRING_8]] + -- Write all downloaded header and content data into `f' + -- and write raw header into `h' if attached. do - l_start := a_source.substring_index ("%R%N", 1) - if l_start > 0 then - --| Skip first line which is the status line - --| ex: HTTP/1.1 200 OK%R%N - l_start := l_start + 2 - end - if l_start = 0 or else - (l_start < a_source.count and then - a_source[l_start] = '%R' and a_source[l_start + 1] = '%N' - ) - then - res.set_body (a_source) - else - pos := a_source.substring_index ("%R%N%R%N", l_start) - if pos > 0 then - res.set_raw_header (a_source.substring (l_start, pos + 1)) --| Keep the last %R%N - res.set_body (a_source.substring (pos + 4, a_source.count)) - else - res.set_body (a_source) - end - end + Result := agent (s: READABLE_STRING_8; ia_header: detachable STRING; ia_file: FILE; ia_header_fetched: CELL [BOOLEAN]) + do + ia_file.put_string (s) + if ia_header /= Void and not ia_header_fetched.item then + ia_header.append (s) + if s.starts_with ("%R%N") then + ia_header_fetched.replace (True) + end + end + end (?, h, f, create {CELL [BOOLEAN]}.put (False)) end + + new_write_content_data_to_file_agent (f: FILE; h: STRING): PROCEDURE [ANY, TUPLE [READABLE_STRING_8]] + -- Write all downloaded content data into `f' (without raw header) + -- and write raw header into `h' if attached. + do + Result := agent (s: READABLE_STRING_8; ia_header: detachable STRING; ia_file: FILE; ia_header_fetched: CELL [BOOLEAN]) + do + if ia_header_fetched.item then + ia_file.put_string (s) + else + if ia_header /= Void then + ia_header.append (s) + end + if s.starts_with ("%R%N") then + ia_header_fetched.replace (True) + end + end + end (?, h, f, create {CELL [BOOLEAN]}.put (False)) + end + note copyright: "2011-2012, Jocelyn Fiat, Javier Velilla, Eiffel Software and others" license: "Eiffel Forum License v2 (see http://www.eiffel.com/licensing/forum.txt)" diff --git a/library/network/http_client/src/spec/libcurl/libcurl_upload_file_read_function.e b/library/network/http_client/src/spec/libcurl/libcurl_upload_file_read_function.e index 317be974..2a81b132 100644 --- a/library/network/http_client/src/spec/libcurl/libcurl_upload_file_read_function.e +++ b/library/network/http_client/src/spec/libcurl/libcurl_upload_file_read_function.e @@ -8,6 +8,9 @@ note class LIBCURL_UPLOAD_FILE_READ_FUNCTION +obsolete + "Use LIBCURL_CUSTOM_FUNCTION [2013-apr-04]" + inherit LIBCURL_DEFAULT_FUNCTION redefine