%
%  Implement the Robot Exclusion Protocol
%
%  We do a nice trick: when we load a robots.txt file, we turn it into
%  prolog code consisting of robot_guidance/4 clauses.  Here just call
%  retrieve/4, but it will use retrieve.sh for post-processing, which
%  will call robots2pl which uses a robots.txt grammar to create
%  robot_guidance/4 rules.   Our normal caching logic should
%  minimize those recompiles.
%

%
%  TODO:  %-escaping in robots.txt files
  
:- [cvs, unit_test].
cvs_id('$Id: robot_excl.pl,v 1.9 2004/05/26 21:14:22 sandro Exp $').
:- ensure_loaded(retrieve).

:- dynamic(robot_guidance/4).

%% robot_allowed(+Retrieval)
%%
%%      Succeeds if the retrieval should go ahead, given the
%%      robot exclusion protocol information we can obtain.
%%
robot_allowed(Ret) :-
	(   rdf(Ret, retr:resource, URI, retdata),
	    rdf(Ret, retr:host, literal(Host), retdata),
	    rdf(Ret, retr:robotName, literal(RobotName), retdata)
	->  (	ensure_loaded_robot_info(Host),
		uri_path(URI, Path),
		\+ robot_guidance(Host, RobotName, Path, disallow)
	    )
	; true
	).

%% ensure_loaded_robot_info(+Host)
%%
%%      Like ensure_loaded, but for the rules generated from the
%%      given Host.   The rules should be robot_guidance/4 clauses.
%%
ensure_loaded_robot_info(Host) :-
	concat_atom(['http://', Host, '/robots.txt'], RobotsURI),
	% DefaultMaxAge is 7 * 24 * 3600,
	% http://www.robotstxt.org/wc/norobots-rfc.html says
	% default to seven days, but I think that's nuts; if
	% you had a null robots.txt, you'd have to wait a week
	% for the new one to take effect?  An hour seems more
	% like it...
	DefaultMaxAge is 3600,   %
	% user agent?...?
	retrieve(RobotsURI, [], RobotsRet, [default_max_age(DefaultMaxAge)]),
	!,
	rdf(RobotsRet, retr:dir, LDir, retdata),
	literal(Dir) = LDir,
	concat_atom([Dir, '/robot_rules.pl'], File),
	(  exists_file(File)
	-> load_files([File],  [if(changed)])
	;  true
	).


/*

convoluted manual test:

 retrieve('http://www.google.com/search?q=how+to+lock+unix+nfs',[],Ret, [robot(test)]).

and it should throw "robot excluded"

and the second time it should have robots.txt cached and throw it
without even going to the net.

then try

 retrieve('http://www.google.com/sdfsdfsdfsdfssearch?q=how+to+lock+unix+nfs',[],Ret, [robot(test)]).

and it should try to fetch it.

  WORKS.

*/
