
:- ['../cvs', '../unit_test'].
cvs_id('$Id: robotstxt.pl,v 1.3 2004/04/29 20:37:06 sandro Exp $').


%% robotstxt(-Structure, +Li, ?Lo)
%%
%%      DCG for parsing robots.txt file into a structure like
%%
%%      [ record([A1, A2, ...], [R1, R2, ... ]), ... ]
  
% http://www.robotstxt.org/wc/norobots-rfc.html
%
% and then changed a lot

robotstxt_text([Record|Rest]) --> record(Record), robotstxt_text(Rest).
robotstxt_text([]) --> opt_ignored.

ignored_line --> optws, (eol; comment_line).

ignored_lines --> ignored_line, (""; ignored_lines).

opt_ignored --> (ignored_lines ; "").

unit_test((  opt_ignored("", "")  )).

unit_test((  comment_line("#foo\n", "")   )).
unit_test((  \+comment_line("#foo", "")   )).
unit_test((  \+comment_line("#foo\n\n", "")   )).

unit_test((  blank_line("\n", "")   )).
unit_test((  \+blank_line("\n\n", "")   )).
unit_test((  \+blank_line("", "")   )).

blank_line --> eol.
blank_line --> ws, eol.

rest_of_line --> eol.
rest_of_line --> any_char_but_eol, rest_of_line.

comment_line --> "#", rest_of_line.

eol --> [10], !.
eol --> [13], !, ([10]; ""), !.
%eol --> [10, 13].
% eol --> eostream.

eostream([], []).

any_char_but_eol -->
	[X], { X \= 10, X \= 13 }.

optws --> ws, !; "".	       % cut makes it take all spaces, even if
				% following thing wants them too. 

ws --> ws_char, ws.
ws --> ws_char.

ws_char --> " ".
ws_char --> "\t".
	 


record(record(Agents, Rules)) -->
	agent_lines(Agents),
	rule_lines(Rules),
	{ Agents \= [],
	  Rules \= [] }.

agent_lines([Agent|Rest]) --> opt_ignored, agent_line(Agent), !, agent_lines(Rest).
agent_lines([]) --> opt_ignored.

rule_lines([Rule|Rest]) --> opt_ignored, rule_line(Rule), !, rule_lines(Rest).
rule_lines([]) --> opt_ignored.

agent_line(Agent) -->
	"User-agent:", optws,
	atom_without("#\n\r", Agent),   % force spaces INTO value
	(comment_line ; eol).

rule_line(allow(Prefix)) -->
	("allow";"Allow:"), !, optws,
	atom_without(" #\n\r",Prefix),     % force spaces out of value
	(comment_line ; eol).
rule_line(disallow(Prefix)) -->
	("disallow:";"Disallow:"), !, optws,
	atom_without(" #\n\r",Prefix),   % force spaces out of value
	(comment_line ; eol).
rule_line(extension(Token, Value)) -->
	token(Token),
	{ Token \= 'User-agent' },
	":", optws,
	atom_without("#\n\r", Value),     % force spaces INTO value
	(comment_line ; eol).

token(Text) -->
	atom_without(" :#\n\r\t", Text).

atom_without(Not, Atom) -->
	rstring_without(Not, Codes),
	{ atom_codes(Atom, Codes) }.

% borrowed from JanW (SWI) 's dcg_basics.
rstring_without(Not, [C|T]) -->
	[C],
	{ \+ memberchk(C, Not)
	}, !,
	rstring_without(Not, T).
rstring_without(_, []) -->
	[].


unit_test((   atom_without("789", '012345', "01234580", "80")    )).

unit_test((
	   read_file_to_codes('test/googlex-robots.txt', Input, []),
	   robotstxt_text(Out, Input, ""),
	   Out = [record([*], [disallow('/search'), disallow('/groups'), disallow('/images')])]
	   )).

/*
pl -L64M
[grammars/robotstxt].
guitracer.
trace.

read_file_to_codes('test/whitehouse-robots.txt', Input, []), robotstxt_text(Out, Input, "").

% 332,673 inferences, 0.13 CPU in 0.14 seconds (96% CPU, 2559023 Lips)

wc whitehouse-robots.txt
   1812    3622   70826 whitehouse-robots.txt

   nothing stupid, I think.

read_file_to_codes('test/whitehouse-robots.txt', Input, []), time(robotstxt_text(Out, Input, "")).

*/

