:- [common].
cvs_id('$Id: download.pl,v 1.3 2004/10/16 08:47:49 sandro Exp $').
/*

   See also curl.pl for something which does roughtly the same thing.
  
*/


:- ensure_loaded(library(debug)).
:- debug(download).


%    http_get('http://www.w3.org/', _G319, [to(atom(_G311))])

%    http_get('http://www.w3.org/', Result, [to(atom), after_dns(format(user_error, 'dns done (~w)~n')), after_connect(format(user_error, 'after connect', []))]).

%    http_client:do_connect('www.w3.org':80, In, Out, [to(atom), after_dns(format(user_error, 'dns done (~w)~n')), after_connect(format(user_error, 'after connect', []))]).

%   socket:tcp_socket('128.30.52.24':80)      FAIL?!

%  pass in the result of an earlier url:parse_url structure?
%  timeout keep-alives???

%              Socket error: Host not found
%              Socket error: Connection refused

:- multi_dynamic(transfer_in_progress/2).

prolog_download(Ret, Address, HeadersFile, DataFile, Accept, LastModifiedString, ReturnHeader, RemainingRedirects) :-
	(   RemainingRedirects =< 0
	->  throw(too_many_redirects(Address))
	;   true
	),
	open(DataFile, write, Out, [lock(write), buffer(full)]),
	assert(transfer_in_progress(Ret, DataFile)),
	ret_set_status(Ret, retr:lookingUpHostname),
	call_cleanup(
		     http_get(Address,
			      _Reply,
			      [  connection('Close'),   % Keep-Alive bugs?
				 user_agent('SemWalker/Ontaria/SWI-Prolog'),
				 request_header('Accept'=Accept),
				 request_header('If-Modified-Since'=LastModifiedString),
				 reply_header(Header),
				 to(stream(Out)),
				 after_dns(after_dns(Ret)),
				 after_connect(after_connect(Ret)),
				 on_content_length(on_content_length(Ret))
				 ]
			     ),
		     (	 close(Out),
			 retractall(transfer_in_progress(Ret, DataFile)),
			 ret_set_status(Ret, transferComplete)
		     )
		    ),
	update_transfer_status(Ret, DataFile),
	open(HeadersFile, write, H, [lock(write), buffer(full)]),
	get_time(Now),
	set_prolog_flag(float_format, '	%.16g'),   % is this per thread?!
	format(H, 'retrieval(~q, ~q, ~w, ~q).~n', [Address, Ret, Now, Header]),
	close(H),
	(   redirected(Header, NewAddress, RePred)
	->  atom_concat(HeadersFile, '.redir', RedirHeaderFile),
	    RRR is RemainingRedirects - 1,
	    ret_assert_triple(Ret, Address, RePred, NewAddress),
	    prolog_download(Ret, NewAddress, RedirHeaderFile, DataFile, Accept, LastModifiedString, SubHeader, RRR),
	    ReturnHeader = SubHeader
	;   ret_set(Ret, retr:finalResource, Address),
	    ReturnHeader = Header
	).

redirected(Header, NewAddress, GPred) :-
	memberchk(status(Code, _), Header),
	% see http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
	% (techinically, some of these have different cachability....   I guess
	% it's a bug -- we just use the cachability of the final result)
	memberchk(p(Code, Pred),
		  [p(moved, retr:movedTo),
		   p(302, retr:foundAt),
		   p(303, retr:seeOther),
		   p(307, retr:temporarilyMovedTo)]),
	rdf_global_id(Pred, GPred),
	memberchk(location(NewAddressPlus), Header),
	%  Some sites include a fragment in the
	%  redirection "Location" header.   I think this is a bug
	%  at their end, but we can work around it easily enough.
	%
	% bash$ HEAD -U -S http://purl.org/dc/elements/1.1/title
        % ...
	% HEAD http://purl.org/dc/elements/1.1/title --> 302 Found
        % HEAD http://dublincore.org/2003/03/24/dces#title --> 200 OK
        %
	uri_stem(NewAddressPlus, NewAddress).

after_dns(Ret, IP) :-
	ret_set_status(Ret, retr:connecting),
	IP=ip(X1, X2, X3, X4),
	concat_atom([X1, X2, X3, X4], '.', IP_Atom),
	ret_set(Ret, retr:hostIP, literal(IP_Atom)).

after_connect(Ret) :-
	ret_set_status(Ret, retr:exchangingHeaders).

on_content_length(Ret, Bytes) :-
	ret_set(Ret, retr:contentLength, literal(Bytes)).

start_to_monitor_transfers :-
	thread_create(monitor_transfers, _, [detached(true),
					  alias(monitor_transfers),
					  global(20),
					  local(20),
					  trail(20),
					  argument(20),
					  stack(1024)
					 ]).

monitor_transfers :-
	repeat,
	sleep(0.2),
	forall(transfer_in_progress(Ret, DataFile),
	       update_transfer_status(Ret, DataFile)
	      ),
	fail.

update_transfer_status(Ret, DataFile) :-
	size_file(DataFile, Size),
	retractall(ret_length_so_far(Ret, _)),
	assert(ret_length_so_far(Ret, Size)).


/*

Exit: (9) http_client:address([protocol(http), host('www.w3.org'), path(/)], 'www.w3.org':80, [to(atom(_G311))]) ? creep
   Call: (9) http_client:do_connect('www.w3.org':80, _L231, _L232) ? creep
   Call: (10) socket:tcp_socket(_L244) ?

  
socket:tcp_connect is given the hostname; C code does the DNS

*/