
:- [cvs, unit_test].
cvs_id('$Id: robots2pl.pl,v 1.4 2004/06/01 18:58:18 sandro Exp $').

:- ensure_loaded('grammars/robotstxt').
:- ensure_loaded(uri).

robot_rules(URI, InFile) :-
	format(':- multifile(robot_guidance/4).~n~n'),
	read_file_to_codes(InFile, Codes, []),
	uri_host_part(URI, Host),
	( robotstxt_text(Structure, Codes, "")
	; format('% robots.txt parse failed~n', [])
	  % should we disallow use?!?!
	),
	!,
	format('% ~q~n~n', [Structure]),
	checklist(visit(Host), Structure).

visit(Host, record(Agents, Rules)) :-
	out(Host, Agents, Rules).

out(Host, [A|As], Rs) :-
	outx(Host, A, Rs),
	out(Host, As, Rs).
out(_Host, [], _).

outx(Host, A, [R|Rs]) :-
	out1(Host, A, R),
	outx(Host, A, Rs).
outx(_Host, _, []).

out1(Host, '*', allow(Prefix)) :-
	format('robot_guidance(~q, _, URI, allow) :- atom_prefix(URI, ~q).~n', [Host, Prefix]).
out1(Host, '*', disallow(Prefix)) :-
	format('robot_guidance(~q, _, URI, disallow) :- atom_prefix(URI, ~q).~n', [Host, Prefix]).
out1(Host, '*', extension(Token, Value)) :-
	format('% extension  ~q _ ~q ~q', [Host, Token, Value]).
out1(Host, Agent, allow(Prefix)) :-
	format('robot_guidance(~q, ~q, URI, allow) :- atom_prefix(URI, ~q).~n', [Host, Agent, Prefix]).
out1(Host, Agent, disallow(Prefix)) :-
	format('robot_guidance(~q, ~q, URI, disallow) :- atom_prefix(URI, ~q).~n', [Host, Agent, Prefix]).
out1(Host, Agent, extension(Token, Value)) :-
	format('% extension  ~q ~q ~q ~q', [Host, Agent, Token, Value]).


/*

User-agent:	foo
User-agent:	bar
Disallow:	/cgi-bin
User-agent:	baz
User-agent:	bux
User-agent:	boom
Disallow:	/a
Allow:		/b
Disallow:	/c


gives us


robot_guidance('example.com', foo, URI, disallow) :- atom_prefix(URI, '/cgi-bin').
robot_guidance('example.com', bar, URI, disallow) :- atom_prefix(URI, '/cgi-bin').
robot_guidance('example.com', baz, URI, disallow) :- atom_prefix(URI, '/a').
robot_guidance('example.com', baz, URI, allow) :- atom_prefix(URI, '/b').
robot_guidance('example.com', baz, URI, disallow) :- atom_prefix(URI, '/c').
robot_guidance('example.com', bux, URI, disallow) :- atom_prefix(URI, '/a').
robot_guidance('example.com', bux, URI, allow) :- atom_prefix(URI, '/b').
robot_guidance('example.com', bux, URI, disallow) :- atom_prefix(URI, '/c').
robot_guidance('example.com', boom, URI, disallow) :- atom_prefix(URI, '/a').
robot_guidance('example.com', boom, URI, allow) :- atom_prefix(URI, '/b').
robot_guidance('example.com', boom, URI, disallow) :- atom_prefix(URI, '/c').


*/
