Annotation of libwww/Robot/src/HTRobot.c, revision 1.54
1.1 frystyk 1: /* HTRobot.c
2: ** W3C MINI ROBOT
3: **
4: ** (c) COPRIGHT MIT 1995.
5: ** Please first read the full copyright statement in the file COPYRIGH.
6: **
7: ** This program illustrates how to travers links using the Anchor object
8: **
9: ** Authors:
10: ** HFN Henrik Frystyk Nielsen, (frystyk@w3.org)
11: **
12: ** History:
13: ** Dec 04 95 First version
14: */
15:
16: #include "WWWLib.h" /* Global Library Include file */
17: #include "WWWApp.h" /* Application stuff */
1.17 frystyk 18: #include "WWWTrans.h"
1.10 frystyk 19: #include "WWWInit.h"
1.9 frystyk 20:
1.4 frystyk 21: #include "HText.h"
1.1 frystyk 22:
23: #include "HTRobot.h" /* Implemented here */
24:
1.14 frystyk 25: #ifndef W3C_VERSION
1.33 eric 26: #define W3C_VERSION "Unspecified"
1.1 frystyk 27: #endif
28:
29: #define APP_NAME "W3CRobot"
1.14 frystyk 30: #define APP_VERSION W3C_VERSION
1.1 frystyk 31:
32: #define DEFAULT_OUTPUT_FILE "robot.out"
33: #define DEFAULT_RULE_FILE "robot.conf"
34: #define DEFAULT_LOG_FILE "robot.log"
1.51 frystyk 35: #define DEFAULT_MEMLOG "robot.mem"
1.7 frystyk 36: #define DEFAULT_DEPTH 0
1.53 frystyk 37: #define DEFAULT_DELAY 50 /* Write delay in ms */
1.1 frystyk 38:
1.51 frystyk 39: #if 0
1.53 frystyk 40: #define HT_MEMLOG /* May be expensive in performance! */
1.51 frystyk 41: #endif
42:
1.46 eric 43: /* #define SHOW_MSG (WWWTRACE || HTAlert_interactive()) */
44: #define SHOW_MSG (!(mr->flags & MR_QUIET))
1.1 frystyk 45:
1.40 frystyk 46: #define DEFAULT_TIMEOUT 10000 /* timeout in millis */
1.1 frystyk 47:
48: #if defined(__svr4__)
49: #define CATCH_SIG
50: #endif
51:
52: typedef enum _MRFlags {
1.45 frystyk 53: MR_IMG = 0x1,
54: MR_LINK = 0x2,
55: MR_PREEMPTIVE = 0x4,
56: MR_TIME = 0x8,
1.46 eric 57: MR_SAVE = 0x10,
1.48 frystyk 58: MR_QUIET = 0x20,
59: MR_VALIDATE = 0x40,
60: MR_END_VALIDATE = 0x80
1.1 frystyk 61: } MRFlags;
62:
63: typedef struct _Robot {
1.2 frystyk 64: int depth; /* How deep is our tree */
1.30 frystyk 65: int cnt; /* Count of requests */
1.2 frystyk 66: HTList * hyperdoc; /* List of our HyperDoc Objects */
1.4 frystyk 67: HTList * htext; /* List of our HText Objects */
1.34 eric 68: HTList * fingers;
1.40 frystyk 69: int timer;
1.1 frystyk 70: char * cwd; /* Current dir URL */
71: char * rules;
72: char * logfile;
73: char * outputfile;
74: FILE * output;
75: MRFlags flags;
76: } Robot;
1.34 eric 77:
78: typedef struct _Finger {
79: Robot * robot;
80: HTRequest * request;
81: HTParentAnchor * dest;
82: } Finger;
83:
1.1 frystyk 84: typedef enum _LoadState {
85: L_INVALID = -2,
86: L_LOADING = -1,
87: L_SUCCESS = 0,
88: L_ERROR
89: } LoadState;
90:
91: /*
92: ** The HyperDoc object is bound to the anchor and contains information about
93: ** where we are in the search for recursive searches
94: */
95: typedef struct _HyperDoc {
96: HTParentAnchor * anchor;
97: LoadState state;
98: int depth;
99: } HyperDoc;
100:
101: /*
102: ** This is the HText object that is created every time we start parsing a
103: ** HTML object
104: */
1.4 frystyk 105: struct _HText {
1.1 frystyk 106: HTRequest * request;
1.4 frystyk 107: };
1.1 frystyk 108:
109: PUBLIC HText * HTMainText = NULL;
110: PUBLIC HTParentAnchor * HTMainAnchor = NULL;
111: PUBLIC HTStyleSheet * styleSheet = NULL;
112:
113: /* ------------------------------------------------------------------------- */
114:
1.13 eric 115: /* Standard (non-error) Output
116: ** ---------------------------
117: */
118: PUBLIC int OutputData(const char * fmt, ...)
119: {
120: int ret;
121: va_list pArgs;
122: va_start(pArgs, fmt);
123: ret = vfprintf(stdout, fmt, pArgs);
124: va_end(pArgs);
125: return ret;
126: }
127:
128: /* ------------------------------------------------------------------------- */
129:
1.2 frystyk 130: /* Create a "HyperDoc" object
131: ** --------------------------
132: ** A HyperDoc object contains information about whether we have already
133: ** started checking the anchor and the depth in our search
134: */
135: PRIVATE HyperDoc * HyperDoc_new (Robot * mr,HTParentAnchor * anchor, int depth)
136: {
137: HyperDoc * hd;
1.14 frystyk 138: if ((hd = (HyperDoc *) HT_CALLOC(1, sizeof(HyperDoc))) == NULL)
139: HT_OUTOFMEM("HyperDoc_new");
1.2 frystyk 140: hd->state = L_INVALID;
141: hd->depth = depth;
142:
143: /* Bind the HyperDoc object together with the Anchor Object */
144: hd->anchor = anchor;
145: HTAnchor_setDocument(anchor, (void *) hd);
146:
147: /* Add this HyperDoc object to our list */
148: if (!mr->hyperdoc) mr->hyperdoc = HTList_new();
149: HTList_addObject(mr->hyperdoc, (void *) hd);
150: return hd;
151: }
152:
153: /* Delete a "HyperDoc" object
154: ** --------------------------
155: */
156: PRIVATE BOOL HyperDoc_delete (HyperDoc * hd)
157: {
158: if (hd) {
1.11 frystyk 159: HT_FREE (hd);
1.2 frystyk 160: return YES;
161: }
162: return NO;
163: }
164:
1.1 frystyk 165: /* Create a Command Line Object
166: ** ----------------------------
167: */
168: PRIVATE Robot * Robot_new (void)
169: {
170: Robot * me;
1.41 frystyk 171: if ((me = (Robot *) HT_CALLOC(1, sizeof(Robot))) == NULL)
1.14 frystyk 172: HT_OUTOFMEM("Robot_new");
1.2 frystyk 173: me->hyperdoc = HTList_new();
1.4 frystyk 174: me->htext = HTList_new();
1.40 frystyk 175: me->timer = DEFAULT_TIMEOUT;
1.25 frystyk 176: me->cwd = HTGetCurrentDirectoryURL();
1.1 frystyk 177: me->output = OUTPUT;
1.35 eric 178: me->cnt = 0;
1.34 eric 179: me->fingers = HTList_new();
1.1 frystyk 180: return me;
181: }
182:
183: /* Delete a Command Line Object
184: ** ----------------------------
185: */
186: PRIVATE BOOL Robot_delete (Robot * me)
187: {
188: if (me) {
1.34 eric 189: HTList_delete(me->fingers);
1.2 frystyk 190: if (me->hyperdoc) {
191: HTList * cur = me->hyperdoc;
192: HyperDoc * pres;
193: while ((pres = (HyperDoc *) HTList_nextObject(cur)))
194: HyperDoc_delete(pres);
195: HTList_delete(me->hyperdoc);
196: }
1.4 frystyk 197: if (me->htext) {
198: HTList * cur = me->htext;
199: HText * pres;
200: while ((pres = (HText *) HTList_nextObject(cur)))
201: HText_free(pres);
202: HTList_delete(me->htext);
203: }
1.1 frystyk 204: if (me->logfile) HTLog_close();
205: if (me->output && me->output != STDOUT) fclose(me->output);
1.12 frystyk 206: if (me->flags & MR_TIME) {
207: time_t local = time(NULL);
1.13 eric 208: HTTrace("Robot terminated %s\n",HTDateTimeStr(&local,YES));
1.12 frystyk 209: }
1.11 frystyk 210: HT_FREE(me->cwd);
211: HT_FREE(me);
1.1 frystyk 212: return YES;
213: }
214: return NO;
215: }
216:
1.2 frystyk 217: /*
1.34 eric 218: ** This function creates a new finger object and initializes it with a new request
1.2 frystyk 219: */
1.34 eric 220: PRIVATE Finger * Finger_new (Robot * robot, HTParentAnchor * dest, HTMethod method)
1.2 frystyk 221: {
1.34 eric 222: Finger * me;
223: HTRequest * request = HTRequest_new();
224: if ((me = (Finger *) HT_CALLOC(1, sizeof(Finger))) == NULL)
225: HT_OUTOFMEM("Finger_new");
226: me->robot = robot;
227: me->request = request;
228: me->dest = dest;
229: HTList_addObject(robot->fingers, (void *)me);
230:
1.48 frystyk 231: /* Set the context for this request */
1.34 eric 232: HTRequest_setContext (request, me);
1.48 frystyk 233:
234: /* Check the various flags to customize the request */
235: if (robot->flags & MR_PREEMPTIVE)
236: HTRequest_setPreemptive(request, YES);
237: if (robot->flags & MR_VALIDATE)
238: HTRequest_setReloadMode(request, HT_CACHE_VALIDATE);
239: if (robot->flags & MR_END_VALIDATE)
240: HTRequest_setReloadMode(request, HT_CACHE_END_VALIDATE);
241:
242: /* We wanna make sure that we are sending a Host header (default) */
1.34 eric 243: HTRequest_addRqHd(request, HT_C_HOST);
1.48 frystyk 244:
245: /* Set the method for this request */
1.34 eric 246: HTRequest_setMethod(request, method);
247: robot->cnt++;
248: return me;
1.2 frystyk 249: }
250:
1.34 eric 251: PRIVATE int Finger_delete (Finger * me)
1.2 frystyk 252: {
1.34 eric 253: HTList_removeObject(me->robot->fingers, (void *)me);
254: me->robot->cnt--;
1.37 frystyk 255:
256: /*
257: ** If we are down at one request then flush the output buffer
258: */
259: if (me->request) {
260: if (me->robot->cnt == 1) HTRequest_forceFlush(me->request);
1.34 eric 261: HTRequest_delete(me->request);
1.37 frystyk 262: }
263:
264: /*
265: ** Delete the request and free myself
266: */
1.34 eric 267: HT_FREE(me);
268: return YES;
1.2 frystyk 269: }
270:
271: /*
272: ** Cleanup and make sure we close all connections including the persistent
273: ** ones
274: */
1.1 frystyk 275: PRIVATE void Cleanup (Robot * me, int status)
276: {
277: Robot_delete(me);
1.29 eric 278: HTProfile_delete();
1.50 frystyk 279: #ifdef HT_MEMLOG
1.39 eric 280: HTMemLog_close();
1.47 frystyk 281: #endif
282:
1.1 frystyk 283: #ifdef VMS
284: exit(status ? status : 1);
285: #else
286: exit(status ? status : 0);
287: #endif
288: }
289:
290: #ifdef CATCH_SIG
291: #include <signal.h>
292: /* SetSignal
293: ** This function sets up signal handlers. This might not be necessary to
294: ** call if the application has its own handlers (lossage on SVR4)
295: */
296: PRIVATE void SetSignal (void)
297: {
298: /* On some systems (SYSV) it is necessary to catch the SIGPIPE signal
299: ** when attemting to connect to a remote host where you normally should
300: ** get `connection refused' back
301: */
302: if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) {
1.13 eric 303: if (PROT_TRACE) HTTrace("HTSignal.... Can't catch SIGPIPE\n");
1.1 frystyk 304: } else {
1.13 eric 305: if (PROT_TRACE) HTTrace("HTSignal.... Ignoring SIGPIPE\n");
1.1 frystyk 306: }
1.47 frystyk 307:
1.50 frystyk 308: #ifdef HT_MEMLOG
1.44 eric 309: HTMemLog_flush();
1.47 frystyk 310: #endif
311:
1.1 frystyk 312: }
313: #endif /* CATCH_SIG */
314:
315: PRIVATE void VersionInfo (void)
316: {
1.13 eric 317: OutputData("\n\nW3C Reference Software\n\n");
318: OutputData("\tW3C Mini Robot (%s) version %s.\n",
1.1 frystyk 319: APP_NAME, APP_VERSION);
1.13 eric 320: OutputData("\tW3C Reference Library version %s.\n\n",HTLib_version());
321: OutputData("Please send feedback to <libwww@w3.org>\n");
1.1 frystyk 322: }
323:
324: /* terminate_handler
325: ** -----------------
1.2 frystyk 326: ** This function is registered to handle the result of the request.
327: ** If no more requests are pending then terminate program
1.1 frystyk 328: */
1.32 frystyk 329: PRIVATE int terminate_handler (HTRequest * request, HTResponse * response,
330: void * param, int status)
1.1 frystyk 331: {
1.34 eric 332: Finger * finger = (Finger *) HTRequest_context(request);
1.46 eric 333: Robot * mr = finger->robot;
1.34 eric 334: if (SHOW_MSG) HTTrace("Robot....... done with %s\n", HTAnchor_physical(finger->dest));
335: Finger_delete(finger);
1.46 eric 336: if (mr->cnt <= 0) {
1.34 eric 337: if (SHOW_MSG) HTTrace(" Everything is finished...\n");
1.46 eric 338: Cleanup(mr, 0); /* No way back from here */
1.30 frystyk 339: }
1.37 frystyk 340:
1.46 eric 341: if (SHOW_MSG) HTTrace(" %d outstanding request%s\n", mr->cnt, mr->cnt == 1 ? "" : "s");
1.1 frystyk 342: return HT_OK;
343: }
344:
345: /* ------------------------------------------------------------------------- */
346: /* HTEXT INTERFACE */
347: /* ------------------------------------------------------------------------- */
348:
349: PUBLIC HText * HText_new2 (HTRequest * request, HTParentAnchor * anchor,
350: HTStream * stream)
351: {
352: HText * me;
1.34 eric 353: Finger * finger = (Finger *) HTRequest_context(request);
354: Robot * mr = finger->robot;
1.14 frystyk 355: if ((me = (HText *) HT_CALLOC(1, sizeof(HText))) == NULL)
356: HT_OUTOFMEM("HText_new2");
1.4 frystyk 357:
358: /* Bind the HText object together with the Request Object */
1.1 frystyk 359: me->request = request;
1.4 frystyk 360:
361: /* Add this HyperDoc object to our list */
362: if (!mr->htext) mr->htext = HTList_new();
363: HTList_addObject(mr->htext, (void *) me);
1.1 frystyk 364: return me;
365: }
366:
1.4 frystyk 367: PUBLIC void HText_free (HText * me) {
1.11 frystyk 368: if (me) HT_FREE (me);
1.4 frystyk 369: }
370:
1.1 frystyk 371: PUBLIC void HText_beginAnchor (HText * text, HTChildAnchor * anchor)
372: {
373: if (text && anchor) {
1.34 eric 374: Finger * finger = (Finger *) HTRequest_context(text->request);
375: Robot * mr = finger->robot;
1.1 frystyk 376: HTAnchor * dest = HTAnchor_followMainLink((HTAnchor *) anchor);
377: HTParentAnchor * dest_parent = HTAnchor_parent(dest);
1.7 frystyk 378: char * uri = HTAnchor_address((HTAnchor *) dest_parent);
1.1 frystyk 379: HyperDoc * hd = HTAnchor_document(dest_parent);
380:
1.13 eric 381: if (SHOW_MSG) HTTrace("Robot....... Found `%s\' - ", uri ? uri : "NULL");
1.7 frystyk 382:
1.2 frystyk 383: /* Test whether we already have a hyperdoc for this document */
384: if (mr->flags & MR_LINK && dest_parent && !hd) {
1.1 frystyk 385: HTParentAnchor * parent = HTRequest_parent(text->request);
386: HyperDoc * last = HTAnchor_document(parent);
387: int depth = last ? last->depth+1 : 0;
1.34 eric 388: Finger * newfinger = Finger_new(mr, dest_parent, METHOD_GET);
389: HTRequest * newreq = newfinger->request;
1.2 frystyk 390: HyperDoc_new(mr, dest_parent, depth);
1.7 frystyk 391: HTRequest_setParent(newreq, HTRequest_anchor(text->request));
392: if (depth >= mr->depth) {
393: if (SHOW_MSG)
1.13 eric 394: HTTrace("loading at depth %d using HEAD\n", depth);
1.7 frystyk 395: HTRequest_setMethod(newreq, METHOD_HEAD);
1.30 frystyk 396: HTRequest_setOutputFormat(newreq, WWW_DEBUG);
1.7 frystyk 397: } else {
1.13 eric 398: if (SHOW_MSG) HTTrace("loading at depth %d\n", depth);
1.2 frystyk 399: }
400: if (HTLoadAnchor((HTAnchor *) dest_parent, newreq) != YES) {
1.13 eric 401: if (SHOW_MSG) HTTrace("not tested!\n");
1.34 eric 402: Finger_delete(newfinger);
1.2 frystyk 403: }
1.7 frystyk 404: } else {
1.18 frystyk 405: if (SHOW_MSG) HTTrace("duplicate or max depth reached\n");
1.2 frystyk 406: }
1.11 frystyk 407: HT_FREE(uri);
1.2 frystyk 408: }
409: }
410:
411: PUBLIC void HText_appendImage (HText * text, HTChildAnchor * anchor,
1.14 frystyk 412: const char *alt, const char * align, BOOL isMap)
1.2 frystyk 413: {
414: if (text && anchor) {
1.34 eric 415: Finger * finger = (Finger *) HTRequest_context(text->request);
416: Robot * mr = finger->robot;
1.2 frystyk 417: HTParentAnchor * dest = (HTParentAnchor *)
418: HTAnchor_followMainLink((HTAnchor *) anchor);
419: HyperDoc * hd = HTAnchor_document(dest);
1.1 frystyk 420:
1.2 frystyk 421: /* Test whether we already have a hyperdoc for this document */
422: if (mr->flags & MR_IMG && dest && !hd) {
423: HTParentAnchor * parent = HTRequest_parent(text->request);
424: HyperDoc * last = HTAnchor_document(parent);
425: int depth = last ? last->depth+1 : 0;
1.45 frystyk 426: Finger * newfinger = Finger_new(mr, dest,
427: mr->flags & MR_SAVE ?
428: METHOD_GET : METHOD_HEAD);
1.34 eric 429: HTRequest * newreq = newfinger->request;
1.2 frystyk 430: HyperDoc_new(mr, dest, depth);
431: if (SHOW_MSG) {
432: char * uri = HTAnchor_address((HTAnchor *) dest);
1.13 eric 433: HTTrace("Robot....... Checking Image `%s\'\n", uri);
1.11 frystyk 434: HT_FREE(uri);
1.2 frystyk 435: }
436: if (HTLoadAnchor((HTAnchor *) dest, newreq) != YES) {
437: if (SHOW_MSG)
1.13 eric 438: HTTrace("Robot....... Image not tested!\n");
1.34 eric 439: Finger_delete(newfinger);
1.1 frystyk 440: }
441: }
442: }
443: }
444:
445: PUBLIC void HText_endAnchor (HText * text) {}
1.14 frystyk 446: PUBLIC void HText_appendText (HText * text, const char * str) {}
1.1 frystyk 447: PUBLIC void HText_appendCharacter (HText * text, char ch) {}
448: PUBLIC void HText_endAppend (HText * text) {}
449: PUBLIC void HText_setStyle (HText * text, HTStyle * style) {}
450: PUBLIC void HText_beginAppend (HText * text) {}
451: PUBLIC void HText_appendParagraph (HText * text) {}
452:
1.48 frystyk 453: PRIVATE int RobotTrace (const char * fmt, va_list pArgs)
454: {
455: return (vfprintf(stderr, fmt, pArgs));
456: }
457:
1.1 frystyk 458: /* ------------------------------------------------------------------------- */
459: /* MAIN PROGRAM */
460: /* ------------------------------------------------------------------------- */
461:
462: int main (int argc, char ** argv)
463: {
1.48 frystyk 464: int status = 0;
1.1 frystyk 465: int arg;
1.48 frystyk 466: BOOL cache = NO; /* Use persistent cache */
467: BOOL flush = NO; /* flush the persistent cache */
1.54 ! frystyk 468: char * cache_root = NULL;
1.1 frystyk 469: HTChunk * keywords = NULL; /* From command line */
470: int keycnt = 0;
1.12 frystyk 471: Robot * mr = NULL;
1.43 frystyk 472: Finger * finger = NULL;
473: HTParentAnchor * startAnchor = NULL;
1.1 frystyk 474:
475: /* Starts Mac GUSI socket library */
476: #ifdef GUSI
477: GUSISetup(GUSIwithSIOUXSockets);
478: GUSISetup(GUSIwithInternetSockets);
479: #endif
480:
481: #ifdef __MWERKS__ /* STR */
482: InitGraf((Ptr) &qd.thePort);
483: InitFonts();
484: InitWindows();
485: InitMenus(); TEInit();
486: InitDialogs(nil);
487: InitCursor();
488: SIOUXSettings.asktosaveonclose = false;
489: argc=ccommand(&argv);
1.50 frystyk 490: #endif /* __MWERKS__ */
1.1 frystyk 491:
1.50 frystyk 492: #ifdef HT_MEMLOG
1.51 frystyk 493: HTMemLog_open(DEFAULT_MEMLOG, 8192, YES);
1.47 frystyk 494: #endif
1.46 eric 495:
1.27 frystyk 496: /* Initiate W3C Reference Library with a robot profile */
497: HTProfile_newRobot(APP_NAME, APP_VERSION);
1.48 frystyk 498: HTTrace_setCallback(RobotTrace);
1.27 frystyk 499:
500: /* Add the default HTML parser to the set of converters */
501: {
502: HTList * converters = HTFormat_conversion();
503: HTMLInit(converters);
504: }
1.1 frystyk 505:
1.12 frystyk 506: /* Build a new robot object */
507: mr = Robot_new();
508:
1.1 frystyk 509: /* Scan command Line for parameters */
510: for (arg=1; arg<argc; arg++) {
511: if (*argv[arg] == '-') {
512:
513: /* non-interactive */
1.17 frystyk 514: if (!strcmp(argv[arg], "-n")) {
1.1 frystyk 515: HTAlert_setInteractive(NO);
516:
517: /* log file */
518: } else if (!strcmp(argv[arg], "-l")) {
519: mr->logfile = (arg+1 < argc && *argv[arg+1] != '-') ?
520: argv[++arg] : DEFAULT_LOG_FILE;
521:
522: /* rule file */
523: } else if (!strcmp(argv[arg], "-r")) {
524: mr->rules = (arg+1 < argc && *argv[arg+1] != '-') ?
525: argv[++arg] : DEFAULT_RULE_FILE;
526:
527: /* output filename */
528: } else if (!strcmp(argv[arg], "-o")) {
529: mr->outputfile = (arg+1 < argc && *argv[arg+1] != '-') ?
530: argv[++arg] : DEFAULT_OUTPUT_FILE;
531:
532: /* timeout -- Change the default request timeout */
533: } else if (!strcmp(argv[arg], "-timeout")) {
534: int timeout = (arg+1 < argc && *argv[arg+1] != '-') ?
535: atoi(argv[++arg]) : DEFAULT_TIMEOUT;
1.40 frystyk 536: if (timeout > 0) mr->timer = timeout;
1.1 frystyk 537:
1.54 ! frystyk 538: /* Force no pipelined requests */
! 539: } else if (!strcmp(argv[arg], "-nopipe")) {
! 540: HTTP_setConnectionMode(HTTP_NO_PIPELINING);
! 541:
1.48 frystyk 542: /* Start the persistent cache */
543: } else if (!strcmp(argv[arg], "-cache")) {
544: cache = YES;
545:
1.54 ! frystyk 546: /* Determine the cache root */
! 547: } else if (!strcmp(argv[arg], "-cacheroot")) {
! 548: cache_root = (arg+1 < argc && *argv[arg+1] != '-') ?
! 549: argv[++arg] : NULL;
1.51 frystyk 550:
1.52 frystyk 551: /* Stream write flush delay in ms */
552: } else if (!strcmp(argv[arg], "-delay")) {
553: int delay = (arg+1 < argc && *argv[arg+1] != '-') ?
554: atoi(argv[++arg]) : DEFAULT_DELAY;
555: HTHost_setDefaultWriteDelay(delay);
556:
1.48 frystyk 557: /* Persistent cache flush */
558: } else if (!strcmp(argv[arg], "-flush")) {
559: flush = YES;
560:
561: /* Do a cache validation */
562: } else if (!strcmp(argv[arg], "-validate")) {
563: mr->flags |= MR_VALIDATE;
564:
565: /* Do an end-to-end cache-validation */
566: } else if (!strcmp(argv[arg], "-endvalidate")) {
567: mr->flags |= MR_END_VALIDATE;
568:
1.7 frystyk 569: /* preemptive or non-preemptive access */
1.1 frystyk 570: } else if (!strcmp(argv[arg], "-single")) {
1.7 frystyk 571: mr->flags |= MR_PREEMPTIVE;
1.2 frystyk 572:
573: /* test inlined images */
574: } else if (!strcmp(argv[arg], "-img")) {
575: mr->flags |= MR_IMG;
1.45 frystyk 576:
577: /* load inlined images */
578: } else if (!strcmp(argv[arg], "-saveimg")) {
579: mr->flags |= (MR_IMG | MR_SAVE);
1.2 frystyk 580:
581: /* load anchors */
582: } else if (!strcmp(argv[arg], "-link")) {
583: mr->flags |= MR_LINK;
1.7 frystyk 584: mr->depth = (arg+1 < argc && *argv[arg+1] != '-') ?
585: atoi(argv[++arg]) : DEFAULT_DEPTH;
1.2 frystyk 586:
1.12 frystyk 587: /* Output start and end time */
588: } else if (!strcmp(argv[arg], "-ss")) {
589: time_t local = time(NULL);
1.13 eric 590: HTTrace("Robot started on %s\n",
1.12 frystyk 591: HTDateTimeStr(&local, YES));
592: mr->flags |= MR_TIME;
593:
1.1 frystyk 594: /* print version and exit */
595: } else if (!strcmp(argv[arg], "-version")) {
596: VersionInfo();
597: Cleanup(mr, 0);
1.46 eric 598:
599: /* run in quiet mode */
600: } else if (!strcmp(argv[arg], "-q")) {
601: mr->flags |= MR_QUIET;
1.1 frystyk 602:
603: #ifdef WWWTRACE
604: /* trace flags */
605: } else if (!strncmp(argv[arg], "-v", 2)) {
1.24 frystyk 606: HTSetTraceMessageMask(argv[arg]+2);
1.1 frystyk 607: #endif
608:
609: } else {
1.13 eric 610: if (SHOW_MSG) HTTrace("Bad Argument (%s)\n", argv[arg]);
1.1 frystyk 611: }
1.17 frystyk 612: } else { /* If no leading `-' then check for URL or keywords */
1.1 frystyk 613: if (!keycnt) {
614: char * ref = HTParse(argv[arg], mr->cwd, PARSE_ALL);
1.34 eric 615: startAnchor = (HTParentAnchor *) HTAnchor_findAddress(ref);
616: HyperDoc_new(mr, startAnchor, 0);
1.1 frystyk 617: keycnt = 1;
1.11 frystyk 618: HT_FREE(ref);
1.1 frystyk 619: } else { /* Check for successive keyword arguments */
620: char *escaped = HTEscape(argv[arg], URL_XALPHAS);
621: if (keycnt++ <= 1)
1.5 frystyk 622: keywords = HTChunk_new(128);
1.1 frystyk 623: else
1.5 frystyk 624: HTChunk_putc(keywords, ' ');
625: HTChunk_puts(keywords, HTStrip(escaped));
1.11 frystyk 626: HT_FREE(escaped);
1.1 frystyk 627: }
628: }
629: }
630:
631: #ifdef CATCH_SIG
632: SetSignal();
633: #endif
634:
635: if (!keycnt) {
1.13 eric 636: if (SHOW_MSG) HTTrace("Please specify URL to check.\n");
1.1 frystyk 637: Cleanup(mr, -1);
638: }
639:
1.23 manoli 640: /* Testing that HTTrace is working */
1.47 frystyk 641: if (SHOW_MSG) HTTrace ("Welcome to the W3C mini Robot\n");
1.23 manoli 642:
1.1 frystyk 643: /* Rule file specified? */
644: if (mr->rules) {
645: char * rules = HTParse(mr->rules, mr->cwd, PARSE_ALL);
1.27 frystyk 646: if (!HTLoadRules(rules))
1.13 eric 647: if (SHOW_MSG) HTTrace("Can't access rules\n");
1.11 frystyk 648: HT_FREE(rules);
1.1 frystyk 649: }
650:
651: /* Output file specified? */
652: if (mr->outputfile) {
653: if ((mr->output = fopen(mr->outputfile, "wb")) == NULL) {
1.13 eric 654: if (SHOW_MSG) HTTrace("Can't open `%s'\n", mr->outputfile);
1.1 frystyk 655: mr->output = OUTPUT;
656: }
657: }
658:
1.48 frystyk 659: /* Should we use persistent cache? */
660: if (cache) {
1.54 ! frystyk 661: HTCacheInit(cache_root, 20);
1.49 frystyk 662: HTNet_addBefore(HTCacheFilter, "http://*", NULL, HT_FILTER_MIDDLE);
663: HTNet_addAfter(HTCacheUpdateFilter, "http://*", NULL,
664: HT_NOT_MODIFIED, HT_FILTER_MIDDLE);
1.48 frystyk 665:
666: /* Should we start by flushing? */
667: if (flush) HTCache_flushAll();
668: }
669:
1.1 frystyk 670: /* Log file specifed? */
671: if (mr->logfile) HTLog_open(mr->logfile, YES, YES);
672:
1.27 frystyk 673: /* Register our own someterminater filter */
1.32 frystyk 674: HTNet_addAfter(terminate_handler, NULL, NULL, HT_ALL, HT_FILTER_LAST);
1.40 frystyk 675:
676: /* Setting event timeout */
677: HTHost_setEventTimeout(mr->timer);
1.37 frystyk 678:
1.34 eric 679: /* Start the request */
680: finger = Finger_new(mr, startAnchor, METHOD_GET);
1.43 frystyk 681:
682: /*
683: ** Make sure that the first request is flushed immediately and not
684: ** buffered in the output buffer
685: */
686: HTRequest_setFlush(finger->request, YES);
687:
688: /*
1.48 frystyk 689: ** Check whether we should do some kind of cache validation on
690: ** the load
691: */
692: if (mr->flags & MR_VALIDATE)
693: HTRequest_setReloadMode(finger->request, HT_CACHE_VALIDATE);
694: if (mr->flags & MR_END_VALIDATE)
695: HTRequest_setReloadMode(finger->request, HT_CACHE_END_VALIDATE);
696:
697: /*
1.43 frystyk 698: ** Now do the load
699: */
1.34 eric 700: if (mr->flags & MR_PREEMPTIVE)
701: HTRequest_setPreemptive(finger->request, YES);
1.1 frystyk 702:
703: if (keywords) /* Search */
1.34 eric 704: status = HTSearchAnchor(keywords, (HTAnchor *)startAnchor, finger->request);
1.1 frystyk 705: else
1.34 eric 706: status = HTLoadAnchor((HTAnchor *)startAnchor, finger->request);
1.1 frystyk 707:
1.5 frystyk 708: if (keywords) HTChunk_delete(keywords);
1.1 frystyk 709: if (status != YES) {
1.13 eric 710: if (SHOW_MSG) HTTrace("Can't access resource\n");
1.1 frystyk 711: Cleanup(mr, -1);
712: }
713:
714: /* Go into the event loop... */
1.34 eric 715: HTEventList_loop(finger->request);
1.1 frystyk 716:
717: /* Only gets here if event loop fails */
718: Cleanup(mr, 0);
719: return 0;
720: }
Webmaster
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to HTRobot.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Public] / libwww / Robot / src