/*

  An rdf "page" is a smallish database holding an RDF graph,
  identified by a URI, and often web accessible.  We store the page in
  the semweb/rdf_db, with the page URI in the fourth column.  We save
  and reload to disk.  We don't serve it on the web, but we make it
  easy if you want to.
  
  Used by the webcache, and maybe other things....

  Don't use the Page->FileSystem mapping outside; we might not keep
  that. 
  
*/

:- [cvs, unit_test].
cvs_id('$Id: rdfpage.pl,v 1.22 2004/10/21 15:29:03 sandro Exp $').

:- ensure_loaded(library(debug)).
:- debug(rdfpage).
:- ensure_loaded(shell).
:- ensure_loaded(mkdir).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%
%   Settings
%
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

rdfpage_set_uri_prefix(Old, New) :-
	( rdfpage_uri_prefix(Old) ; true ),
	retractall(rdfpage_uri_prefix(_)),
	assert(rdfpage_uri_prefix(New)).

rdfpage_set_storage_prefix(Old, New) :-
	( rdfpage_storage_prefix(Old) ; true ),
	retractall(rdfpage_storage_prefix(_)),
	assert(rdfpage_storage_prefix(New)).

rdfpage_set_uri_delimiter(Old, New) :-
	(New = '#' ; New = '/'),   % this is all rdf_split_url allows
	( rdfpage_uri_delimiter(Old) ; true ),
	retractall(rdfpage_uri_delimiter(_)),
	assert(rdfpage_uri_delimiter(New)).

:- dynamic rdfpage_uri_prefix/1.
:- dynamic rdfpage_storage_prefix/1.
:- dynamic rdfpage_uri_delimiter/1.
:- (   rdfpage_uri_prefix(_)
   ->  true
   ;   assert(rdfpage_uri_prefix('http://example.org/not_configured/'))
   ).
:- (   rdfpage_storage_prefix(_)
   ->  true
   ;   assert(rdfpage_storage_prefix('/tmp/rdfpage-default/'))
   ).
:- (   rdfpage_uri_delimiter(_)
   ->  true
   ;   assert(rdfpage_uri_delimiter('#'))
   ).

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%
%   Basic URI Manipulation
%
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 100us
rdfpage_gen_uri(Page) :-
	rdfpage_gen_uri(page, Page).

% at this point, the "Group" is ONLY for URI readability; we don't
% keep separate counters for different groups.
%
% Actually -- Group is broken!   We have no way to reconstruct the
% group from the filename.....
rdfpage_gen_uri(Group, Page) :-
	rdfpage__take_next_index(Index),
	rdfpage__group_index(Page, Group, Index).

rdfpage_uri(Page) :-
	rdfpage_uri_prefix(UriPrefix),
	sub_atom(Page, 0, _Len, _After, UriPrefix).

rdfpage_sub_uri(Page, Local, Full) :-
	var(Full),
	rdfpage_uri_delimiter(Delim),
	concat_atom([Page, Delim, Local], Full).
rdfpage_sub_uri(Page, Local, Full) :-
	var(Local),
	rdf_split_url(Page2, Local, Full),
	rdfpage_uri_delimiter(Delim),
	atom_concat(Page, Delim, Page2),
	rdfpage_uri(Page).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%
%   Storage
%
%
%   We put an entry called "__sentry" there just to know something
%   is there, because users expect rdfpage_exists_storage to be
%   traversable.   Hrm, maybe not -- maybe only if something WAS
%   actually stored there...
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% 200 usec
rdfpage_make_storage(Page) :-
	rdfpage__fileprefix(Page, Fileprefix),
	make_parent_directories_if_missing(Fileprefix),
	atom_concat(Fileprefix, '__sentry', Sentry),
	open(Sentry, write, SentryStream, []),
	close(SentryStream).

% 40000 usec (obviously a slow impl!)
rdfpage_remove_storage(Page) :-
	rdfpage__fileprefix(Page, Fileprefix),
	shell_format('rm -rf ~a*', [Fileprefix]).

rdfpage_exists_storage(Page) :-
	atom(Page),
	rdfpage__fileprefix(Page, Fileprefix),
	atom_concat(Fileprefix, '__sentry', Sentry),
	exists_file(Sentry).
rdfpage_exists_storage(Page) :-
	var(Page),
	rdfpage_storage_prefix(D),
	atom_concat(D, '*/*/*__sentry', Pat),    /* reset emacs */ 
	expand_file_name(Pat, Matches),
	member(Sentry, Matches),
	atom_concat(Fileprefix, '__sentry', Sentry),
        rdfpage__fileprefix(Page, Fileprefix).

rdfpage_filename(Page, Local, Filename) :-
	rdfpage__fileprefix(Page, Fileprefix),
	concat_atom([Fileprefix, '-', Local], Filename).
	
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%
%   Load the page data into semweb/rdf_db, or save it back again.
%
%   Uses the Page (URI) as the source name (fourth column).
%
%   
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

rdfpage_load_safe(Page) :-
	rdf_statistics(triples(T0)),
	rdfpage_load(Page),
	rdf_statistics(triples(T1)),
	findall(x, rdf(_,_,_,Page), X),
	length(X, OnPage),
	T is T1 - T0,
	(   T = OnPage
	->  true
	;   print_message(warning, format('rdfpage_load(~q) added ~w triples, but ~w in page', [Page, T, OnPage])),
	    throw(foo)
	).

rdfpage_load(Page) :-
	% rdfpage_unload(Page),
	(   rdfpage_filename(Page, 'save.rdfdb', Filename),
	    exists_file(Filename),
	    % format(':- ~q.', [rdf_load_db(Filename)]),
	    % format(user_error, '~q.~n', [rdf_load_db(Filename)]),
	    rdf_load_db(Filename)
	;   rdfpage_filename(Page, 'save.rdf', Filename),
	    exists_file(Filename),
	    open(Filename, read, Stream, [lock(shared)]),
	    rdf_load(Stream, [base_uri(Page)]),	% @@ needs convert_typed_literal(pred)
	    close(Stream)
	), !.

rdfpage_unload(Page) :-
	rdf_transaction(rdf_retractall(_S,_P,_O,Page)).

rdfpage_qsave(Page) :-
	rdfpage_make_storage(Page),
	rdfpage_filename(Page, 'save.rdfdb', Filename1),
	%  nested locks -- is deadlock possible?   I don't think so.
	%with_read_lock(rdf,
		       with_mutex(rdfpage_save_mutex,
				  rdf_save_db(Filename1, Page))
		%       )
         .


rdfpage_save(Page) :-
	rdfpage_make_storage(Page),
	rdfpage_filename(Page, 'save.rdfdb', Filename1),
	with_mutex(rdfpage_save_mutex, rdf_save_db(Filename1, Page)),
	% store the long version too...?
	%
	%     @@ bug: not sure this saves all the right triples...
	%
	rdfpage_filename(Page, 'save.rdf', Filename),
	open(Filename, write, Stream, [lock(exclusive)]),
	rdfpage_write(Page, Stream),
	close(Stream).

rdfpage_write(Page, Stream) :-
	rdf_save_header(Stream, [db(Page)]),
	findall(Subject, 
		rdf(Subject, _, _, Page),
		SubjectsWithDups),
	sort(SubjectsWithDups, Subjects),
	forall(  member(Subject, Subjects),
		 rdf_save_subject(Stream, Subject, Page)
	      ),
	rdf_save_footer(Stream).

	

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%
%   Convenient access to semweb/rdf_db   
%
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%  like rdf(S,P,O), but requires the data be source'd from the Page
%%  associated with Subj.
%
%   requires rdfpage_load having been done
%
rdfpage(Subj, Pred, Obj) :-
	rdf(Subj, Pred, Obj, Page),
	rdfpage_sub_uri(Page, _, Subj).

user:goal_expansion(rdfpage(Subj0, Pred0, Obj0),
		    rdfpage(Subj, Pred, Obj)) :-
	rdf_global_id(Subj0, Subj),
	rdf_global_id(Pred0, Pred),
	rdf_global_id(Obj0, Obj).

%   be sure to rdfpage_save afterwords!
%
%   we could set a flag, and have a rdfpage_save_all_modified/0.
%
rdfpage_assert(Subj, Pred, Obj) :-
	rdfpage_sub_uri(Page, _, Subj),
	!,   % not needed if rdfpage_uri were more descriminating
	rdf_transaction(rdf_assert(Subj, Pred, Obj, Page)).
rdfpage_assert(Subj, Pred, Obj) :-
	Subj = Page,
	rdfpage_uri(Page),
	!,
	rdf_transaction(rdf_assert(Subj, Pred, Obj, Page)).
rdfpage_assert(Subj, _Pred, _Obj) :-
	throw(not_an_rdfpage_subject(Subj)).

user:goal_expansion(rdfpage_assert(Subj0, Pred0, Obj0),
		    rdfpage_assert(Subj, Pred, Obj)) :-
	rdf_global_id(Subj0, Subj),
	rdf_global_id(Pred0, Pred),
	rdf_global_id(Obj0, Obj).


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
%
%   Internals...   
%
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


mkdir_open(SrcDest, Mode, Stream, Options) :-
	catch(
	      ( open(SrcDest, Mode, Stream, Options), Status=good ),
	      _Error,
	      Status=bad
	     ),
	(   Status=bad
	->  make_parent_directories_if_missing(SrcDest),
	    open(SrcDest, Mode, Stream, Options)
	;   true
	).

%  80us runtime
rdfpage__take_next_index(Index) :-
	rdfpage_storage_prefix(StoragePrefix),
	%  We have to lock a different file, because I don't know how
	%  to get an exclusive lock on a file open for reading.  I get
	%  (Bad file descriptor) when I try.
	atom_concat(StoragePrefix, '.lock', LockFileName),
	mkdir_open(LockFileName, write, LockStream, [lock(exclusive)]),
	atom_concat(StoragePrefix, 'highest-used', FileName),
	(   exists_file(FileName)
	->  open(FileName, read, Stream1, []),
	    read(Stream1, Old),
            atom_number(Old, OldNumber),
	    Index is OldNumber + 1,
	    close(Stream1)
	;   Index = 1
	),
	open(FileName, write, Stream2, []),
	format(Stream2, '\'~a\'.~n', [Index]),
	close(Stream2),
	close(LockStream).
	

index_fileprefix(Index, Fileprefix) :-
	number(Index),
	index_suffix(Index, Suffix),
	rdfpage_storage_prefix(StoragePrefix),
	atom_concat(StoragePrefix, Suffix, Fileprefix),
	!.
index_fileprefix(Index, Fileprefix) :-
	atom(Fileprefix),
	rdfpage_storage_prefix(StoragePrefix),
	atom_concat(StoragePrefix, Suffix, Fileprefix),
	index_suffix(Index, Suffix),
	!.

index_suffix(Index, Suffix) :-
	number(Index),
	P1 is Index // 100000,
	P2 is (Index // 100) mod 100,
	P3 is Index mod 100,
	sformat(Str, '~w~|/~`0t~w~4+/~`0t~w~3+', [P1, P2, P3]),
	string_to_atom(Str, Suffix).
index_suffix(Index, Suffix) :-
	var(Index),
	concat_atom([AP1, AP2, AP3], '/', Suffix),
	atom_number(AP1, P1),
	atom_number(AP2, P2),
	atom_number(AP3, P3),
	Index is P1 * 100000 + P2 * 100 + P3.

unit_test(index_suffix(0, '0/000/00')).
unit_test(index_suffix(1002, '0/010/02')).
unit_test(index_suffix(12345678, '123/056/78')).
unit_test(index_suffix(1234567890, '12345/078/90')).
% this next one unless your maxint is > 2^31
% unit_test(index_suffix(777712345678, '777712/234/678')).
unit_test((index_suffix(I, '0/000/00'), I=0)).
unit_test((index_suffix(I, '0/010/02'), I=1002)).
unit_test((index_suffix(I, '123/456/78'), I=12345678)).
unit_test((index_suffix(I, '12345/678/90'), I=1234567890)).



rdfpage__group_index(Page, Group, Index) :-
	var(Index),
	rdfpage_uri_prefix(SiteURI),
	atom_concat(SiteURI, Rest, Page),
	sub_atom(Rest, GroupLen, 1, _After1, '/'),
	GroupLenPlus1 is GroupLen + 1,
	sub_atom(Rest, GroupLenPlus1, _Len, 0, IndexAtom),
	sub_atom(Rest, 0, GroupLen, _After2, Group),
	/*
	atom_concat(Group, '/', GroupSlash),
	atom_concat(GroupSlash, IndexAtom, Rest),
	*/
	atom_number(IndexAtom, Index).
rdfpage__group_index(Page, Group,Index) :-
	var(Page),
	rdfpage_uri_prefix(SiteURI),
	atom_number(IndexAtom, Index),
	concat_atom([Group, '/', IndexAtom], Rest),
	atom_concat(SiteURI, Rest, Page).

rdfpage__fileprefix(Page, Fileprefix) :-
	atom(Page),
	rdfpage__group_index(Page, _Group, Index),
	index_fileprefix(Index, Fileprefix),
	!.
rdfpage__fileprefix(Page, Fileprefix) :-
	atom(Fileprefix),
	index_fileprefix(Index, Fileprefix),
	% @@@@ Bug: using "page" here as the Group, because the group
	% name isn't encoded anywhere.
	rdfpage__group_index(Page, page,Index),
	!.

/*


delete_full_directory(D) :-
	shell_format('rm -rf ~a', [D]).

  
	debug(rdfpage, 'deleting directory ~a', [D]),
	atom_concat(D, '/.??*', Pattern),
	debug(rdfpage, 'pattern: ~a', [Pattern]),
	expand_file_name(Pattern, Entries1),
	debug(rdfpage, 'entries: ~q', [Entries1]),
	atom_concat(D, '/*', Pattern2),                  */
	debug(rdfpage, 'pattern: ~a', [Pattern2]),
	expand_file_name(Pattern, Entries2),
	debug(rdfpage, 'entries: ~q', [Entries2]),
	(   ( member(Entry, Entries1) ; member(Entry, Entries2) ),
	    (   exists_file(Entry)
	    ->  debug(rdfpage, 'deleting file ~a', [Entry]),
		delete_file(Entry)
	    ;   (   exists_directory(Entry)
		->  debug(rdfpage, 'recursively delete directory ~a', [Entry]),
		    delete_full_directory(Entry)
		;   print_message(error, format('cant delete ~a', [Entry]))
		)
	    ),
	    fail
	;   true
	),
	delete_directory(D).

*/

