:- ensure_loaded(common).
cvs_id('$Id: search.pl,v 1.27 2005/04/14 22:17:40 sandro Exp $').
/*

   Stuff related to the search features....

*/


search_form(Focus, form(class=searchForm, method=get, action=basic, Parts)) :-
	Parts = [
		 span(id=x, 'Search: '),
		 input(name='old', type=hidden, value=Focus),
		 input(name='focus', size='60', type='text', value=Focus),
		 %input(name='timeout', type='hidden', value='2'),
		 ' ',
		 input(type='submit', value='Go'),
		 ' ',
		 /*
		   input(type='button', value='Clear',
		   onclick='document.searchForm.focus.value='''''),
		   ' ',    i've lost the dom naming rules
		 */
		 a(href='static/help-search-form', help),
		 ' ',
		 a(href='static/help-privacy', privacy)
		 %p(class=privacyNotice, 'Search text and viewed web contents may be logged and made public.', a(href='static/help-privacy', 'Why?'))

		].

generate_search_uri(ReqId, N) :-
	flag(search, N, N+1),
	flag(aboutThisRun, Run, Run),
	atom_number(AtomN, N),
	atom_concat('search_', AtomN, Local),
	rdfpage_sub_uri(Run, Local, ReqId).

%%  search(Pattern, Results)
:- dynamic(search_results/2).    % HACK - should be in RDF?
/*
search(Pattern, SearchId, Results) :-
	generate_search_uri(SearchId, _Number),
	findall(Subject,
		(
		  %     word(Word), string(String), qname(...) ?

		  %  find one subject
		  rdf(Subject, _, literal(word(Word), _), _),

		  %  make sure this subject matches all the Pattern criteria
		  forall(   member(text(Word), Pattern), 
			    rdf(Subject, _, literal(word(Word), _), _)
			)
		), UnSubjects),
	sort(UnSubjects, Subjects),
	debug(reply, 'found it in ~q', [Subjects]),
	% assert(search_results(SearchId, Pattern, Subjects)).
	Subjects = Results.
*/

:- debug(search).

/*
     This is written with insider knowledge of persistence.pl
     and index.pl to get some performance advantage.
*/

search_term(word(_)).       % all others ignored for now!
search_clause(WSR3, WSR1, W2C, word(PWord), ranked(Rank, Clause)) :-
	downcase_atom(PWord, Word),
	(   stopword(Word)
	->  Rank=0,
	    Clause = stopword_search(Word)
	;   (   db_get(W2C, Word, Count)
	    ->  true
	    ;   Count=0
	    ),
	    Rank is Count,
	    Clause = db_getx(WSR3, WSR1, Word)
	).

db_getx(WSR3, _WSR1, Word, Usage) :-
	var(Usage),
	db_get(WSR3, Word, Usage).
db_getx(_WSR3, WSR1, Word, Usage) :-
	ground(Usage),
	atom_number(UsageAtom, Usage),
	concat_atom([Word, '/', UsageAtom], Combined),
	db_get(WSR1, Combined, _).

stopword_search(_Word, _Usage) :-
	% TODO - page it in and search the text
	true.

conjoin([Clause], Clause).
conjoin([Clause|Clauses], (Clause, X)) :-
	Clauses \= [],
	conjoin(Clauses, X).
conjoin([], true).

qtest:-
	conjoin([a,b,c,d], Out),
	Out = (a, b, c, d).    % parens stop make/0 from giving a warning
qtest:-
	conjoin([], Out),
	Out = true.
qtest:-
	conjoin([a], Out),
	Out = (a).
qtest:-
	conjoin([a,b], Out),
	Out = (a, b).
	

unpack_usage(Usage, sr(Subj, Graph)) :-
	persistent_term_index(T, Usage),
	!,
	T = values(Subj, Graph).

clause_only(ranked(_, C), C).

add_an(A, I1, I2) :-
	I1 =.. L1,
	append(L1, [A], L2),
	I2 =.. L2.

build_rule(Pattern, RuleName, SearchSize) :-
	persistence_handle(word_subj_graph, 3, WSR3),
	persistence_handle(word_subj_graph, 1, WSR1),
	persistence_handle(word_count, 2, W2C),
	gensym(search_rule_, RuleName),
	sublist(search_term, Pattern, PatternGood),
	maplist(search_clause(WSR3, WSR1, W2C), PatternGood, RankedClauses),
	sort(RankedClauses, SortedClauses),
	(   SortedClauses = [ ranked(SearchSize, _) | _]
	->  true
	;   SearchSize = 0
	),
	debug(search, 'SearchSize: ~q', [SearchSize]),
	debug(search, 'RankedClauses: ~q', [RankedClauses]),
	debug(search, 'SortClauses: ~q', [SortedClauses]),
	maplist(clause_only, SortedClauses, SortedPlainClauses),
	debug(search, 'ClauseOnly: ~q', [SortedPlainClauses]),
	maplist(add_an(X), SortedPlainClauses, C5),
	maplist(add_an(X), SortedPlainClauses, C5),
	debug(search, 'C5: ~q', [C5]),
	C5 = [C5Head | C5Tail],
	% increment a counter (whose name happens to be the same
	% as the rule name) each time the first clause succeeds,
	% so we can estimate how we're doing.
	C6 = [C5Head, flag(RuleName, Counter, Counter+1) | C5Tail ],
	Head =.. [RuleName, X],
	conjoin(C6, Body),
	Rule=( Head :- Body ),
	assert(Rule),
	debug(search, 'Rule: ~q', [Rule]).
	       
search(Pattern, SearchId, Results, Max, Done) :-
	generate_search_uri(SearchId, _Number),
	build_rule(Pattern, RuleName, SearchSpaceSize),
	debug(search, 'running findsome', []),
	flag(RuleName, _, 0),
	setofsome(Max,
		  Done,
		 Usage,
		call(RuleName, Usage),
		Usages),
	debug(search, 'ran findall', []),
	maplist(unpack_usage, Usages, Results),
	length(Results, NumResults),
	!,
	flag(RuleName, SearchSpaceUsed, SearchSpaceUsed),
	(   SearchSpaceSize > 0,
	    SearchSpaceUsed > 0,
	    \+ Done
	->  Fraction is SearchSpaceUsed / SearchSpaceSize,
	    Estimate is NumResults / Fraction,
	    debug(search, 'Used ~w, Size ~w, Frac ~w, Est ~w',
		  [SearchSpaceUsed,  SearchSpaceSize, Fraction, Estimate])
	;   true
	),
	debug(search, '~w matches.', [NumResults]).

search_reply(Q, ReqId, Focus, FocusFlags) :-
	memberchk(starting(Starting), Q),
	Max is 9 + Starting,
	search(FocusFlags, _SearchId, Results, Max, Done),
	debug(reply, 'Giving search results: ~q', [Results]),
	debug(search, 'Starting at ~q', [Starting]),
	write_page(ReqId, search_results(Q, Focus, Results, Starting, Done), Focus).



%tab(1000, Q, Focus, 'Search Results', +search_results(Q, Focus, List)) :-
%	search_results(Focus, _Pattern, List).

indexing_warning(H) :-
	(   index_queue(_,_)
	->  H='   Note that some retrieved triples are not yet indexed.'
	;   H=''
	).

search_results(_Q, _Focus, [], _Starting, _, p('No results found.', +indexing_warning)).
	
search_results(Q, _Focus, List, Starting, true, div(%p(int(Count), ' results: '),
						    Entries, p('End of results.'), +indexing_warning)) :-
	%length(Entries, Count),
	search_result_entry(Q, Starting, 1, List, Entries).

search_results(Q, Focus, List, Starting, fail,
	       div(%p(int(Count), ' results: '),
		   Entries, More)
	      ) :-
	%length(Entries, Count),
	search_result_entry(Q, Starting, 1, List, Entries),
	NewStarting is Starting + 10,
	More = p(a(href= \self_uri(basic, Q, [focus=Focus, starting=NewStarting]), 'More results', +indexing_warning)).

% search_result_entry(Match, Text)
search_result_entry(_, _, _, [], []).
search_result_entry(Q, Starting, Count, [_E|Es], [Ts]) :-
	Count < Starting,
	C2 is Count+1,
	search_result_entry(Q, Starting, C2, Es, Ts).
search_result_entry(Q, Starting, Count, [E|Es], [T|Ts]) :-
	Count >= Starting,
	E=sr(Subject, Graph),
	pagein_graph(Graph),
	% pagein_around(Subject)    <-- too slow for eg rdf:type
	%     use pagein_rdf for that kind of thing, I guess.
	T=div(class=searchResult, p('Result ', int(Count)),
	      +para([linkParaHeading|Q], Subject)),
	C2 is Count+1,
	search_result_entry(Q, Starting, C2, Es, Ts).

/*

rdf(Subject, Predicate, Value, Source), Value=literal(Lit), atom(Lit).

Lits = ['www.w3.org', 'http://www.w3.org/People/Charles/foaf.rdf', 'max-age=21600', 'Tue, 14 Oct 2003 15:05:26 GMT', 'application/rdf+xml; qs=0.9', '128.2.199.68', 'http://128.2.199.68/ontologies/eWallet/locationProfile/v1/locationProfile-ont', 'Thu, 03 Oct 2002 01:07:26 GMT', 'text/plain'|...]

*/