Annotation of libwww/Robot/src/HTRobot.c, revision 1.47
1.1 frystyk 1: /* HTRobot.c
2: ** W3C MINI ROBOT
3: **
4: ** (c) COPRIGHT MIT 1995.
5: ** Please first read the full copyright statement in the file COPYRIGH.
6: **
7: ** This program illustrates how to travers links using the Anchor object
8: **
9: ** Authors:
10: ** HFN Henrik Frystyk Nielsen, (frystyk@w3.org)
11: **
12: ** History:
13: ** Dec 04 95 First version
14: */
15:
16: #include "WWWLib.h" /* Global Library Include file */
17: #include "WWWApp.h" /* Application stuff */
1.17 frystyk 18: #include "WWWTrans.h"
1.10 frystyk 19: #include "WWWInit.h"
1.9 frystyk 20:
1.4 frystyk 21: #include "HText.h"
1.1 frystyk 22:
1.39 eric 23: #include "HTMemLog.h"
1.1 frystyk 24: #include "HTRobot.h" /* Implemented here */
25:
1.14 frystyk 26: #ifndef W3C_VERSION
1.33 eric 27: #define W3C_VERSION "Unspecified"
1.1 frystyk 28: #endif
29:
30: #define APP_NAME "W3CRobot"
1.14 frystyk 31: #define APP_VERSION W3C_VERSION
1.1 frystyk 32:
33: #define DEFAULT_OUTPUT_FILE "robot.out"
34: #define DEFAULT_RULE_FILE "robot.conf"
35: #define DEFAULT_LOG_FILE "robot.log"
1.7 frystyk 36: #define DEFAULT_DEPTH 0
1.1 frystyk 37:
1.46 eric 38: /* #define SHOW_MSG (WWWTRACE || HTAlert_interactive()) */
39: #define SHOW_MSG (!(mr->flags & MR_QUIET))
1.1 frystyk 40:
1.40 frystyk 41: #define DEFAULT_TIMEOUT 10000 /* timeout in millis */
1.1 frystyk 42:
43: #if defined(__svr4__)
44: #define CATCH_SIG
45: #endif
46:
47: typedef enum _MRFlags {
1.45 frystyk 48: MR_IMG = 0x1,
49: MR_LINK = 0x2,
50: MR_PREEMPTIVE = 0x4,
51: MR_TIME = 0x8,
1.46 eric 52: MR_SAVE = 0x10,
53: MR_QUIET = 0x20
1.1 frystyk 54: } MRFlags;
55:
56: typedef struct _Robot {
1.2 frystyk 57: int depth; /* How deep is our tree */
1.30 frystyk 58: int cnt; /* Count of requests */
1.2 frystyk 59: HTList * hyperdoc; /* List of our HyperDoc Objects */
1.4 frystyk 60: HTList * htext; /* List of our HText Objects */
1.34 eric 61: HTList * fingers;
1.40 frystyk 62: int timer;
1.1 frystyk 63: char * cwd; /* Current dir URL */
64: char * rules;
65: char * logfile;
66: char * outputfile;
67: FILE * output;
68: MRFlags flags;
69: } Robot;
1.34 eric 70:
71: typedef struct _Finger {
72: Robot * robot;
73: HTRequest * request;
74: HTParentAnchor * dest;
75: } Finger;
76:
1.1 frystyk 77: typedef enum _LoadState {
78: L_INVALID = -2,
79: L_LOADING = -1,
80: L_SUCCESS = 0,
81: L_ERROR
82: } LoadState;
83:
84: /*
85: ** The HyperDoc object is bound to the anchor and contains information about
86: ** where we are in the search for recursive searches
87: */
88: typedef struct _HyperDoc {
89: HTParentAnchor * anchor;
90: LoadState state;
91: int depth;
92: } HyperDoc;
93:
94: /*
95: ** This is the HText object that is created every time we start parsing a
96: ** HTML object
97: */
1.4 frystyk 98: struct _HText {
1.1 frystyk 99: HTRequest * request;
1.4 frystyk 100: };
1.1 frystyk 101:
102: PUBLIC HText * HTMainText = NULL;
103: PUBLIC HTParentAnchor * HTMainAnchor = NULL;
104: PUBLIC HTStyleSheet * styleSheet = NULL;
105:
106: /* ------------------------------------------------------------------------- */
107:
1.13 eric 108: /* Standard (non-error) Output
109: ** ---------------------------
110: */
111: PUBLIC int OutputData(const char * fmt, ...)
112: {
113: int ret;
114: va_list pArgs;
115: va_start(pArgs, fmt);
116: ret = vfprintf(stdout, fmt, pArgs);
117: va_end(pArgs);
118: return ret;
119: }
120:
121: /* ------------------------------------------------------------------------- */
122:
1.2 frystyk 123: /* Create a "HyperDoc" object
124: ** --------------------------
125: ** A HyperDoc object contains information about whether we have already
126: ** started checking the anchor and the depth in our search
127: */
128: PRIVATE HyperDoc * HyperDoc_new (Robot * mr,HTParentAnchor * anchor, int depth)
129: {
130: HyperDoc * hd;
1.14 frystyk 131: if ((hd = (HyperDoc *) HT_CALLOC(1, sizeof(HyperDoc))) == NULL)
132: HT_OUTOFMEM("HyperDoc_new");
1.2 frystyk 133: hd->state = L_INVALID;
134: hd->depth = depth;
135:
136: /* Bind the HyperDoc object together with the Anchor Object */
137: hd->anchor = anchor;
138: HTAnchor_setDocument(anchor, (void *) hd);
139:
140: /* Add this HyperDoc object to our list */
141: if (!mr->hyperdoc) mr->hyperdoc = HTList_new();
142: HTList_addObject(mr->hyperdoc, (void *) hd);
143: return hd;
144: }
145:
146: /* Delete a "HyperDoc" object
147: ** --------------------------
148: */
149: PRIVATE BOOL HyperDoc_delete (HyperDoc * hd)
150: {
151: if (hd) {
1.11 frystyk 152: HT_FREE (hd);
1.2 frystyk 153: return YES;
154: }
155: return NO;
156: }
157:
1.1 frystyk 158: /* Create a Command Line Object
159: ** ----------------------------
160: */
161: PRIVATE Robot * Robot_new (void)
162: {
163: Robot * me;
1.41 frystyk 164: if ((me = (Robot *) HT_CALLOC(1, sizeof(Robot))) == NULL)
1.14 frystyk 165: HT_OUTOFMEM("Robot_new");
1.2 frystyk 166: me->hyperdoc = HTList_new();
1.4 frystyk 167: me->htext = HTList_new();
1.40 frystyk 168: me->timer = DEFAULT_TIMEOUT;
1.25 frystyk 169: me->cwd = HTGetCurrentDirectoryURL();
1.1 frystyk 170: me->output = OUTPUT;
1.35 eric 171: me->cnt = 0;
1.34 eric 172: me->fingers = HTList_new();
1.1 frystyk 173: return me;
174: }
175:
176: /* Delete a Command Line Object
177: ** ----------------------------
178: */
179: PRIVATE BOOL Robot_delete (Robot * me)
180: {
181: if (me) {
1.34 eric 182: HTList_delete(me->fingers);
1.2 frystyk 183: if (me->hyperdoc) {
184: HTList * cur = me->hyperdoc;
185: HyperDoc * pres;
186: while ((pres = (HyperDoc *) HTList_nextObject(cur)))
187: HyperDoc_delete(pres);
188: HTList_delete(me->hyperdoc);
189: }
1.4 frystyk 190: if (me->htext) {
191: HTList * cur = me->htext;
192: HText * pres;
193: while ((pres = (HText *) HTList_nextObject(cur)))
194: HText_free(pres);
195: HTList_delete(me->htext);
196: }
1.1 frystyk 197: if (me->logfile) HTLog_close();
198: if (me->output && me->output != STDOUT) fclose(me->output);
1.12 frystyk 199: if (me->flags & MR_TIME) {
200: time_t local = time(NULL);
1.13 eric 201: HTTrace("Robot terminated %s\n",HTDateTimeStr(&local,YES));
1.12 frystyk 202: }
1.11 frystyk 203: HT_FREE(me->cwd);
204: HT_FREE(me);
1.1 frystyk 205: return YES;
206: }
207: return NO;
208: }
209:
1.2 frystyk 210: /*
1.34 eric 211: ** This function creates a new finger object and initializes it with a new request
1.2 frystyk 212: */
1.34 eric 213: PRIVATE Finger * Finger_new (Robot * robot, HTParentAnchor * dest, HTMethod method)
1.2 frystyk 214: {
1.34 eric 215: Finger * me;
216: HTRequest * request = HTRequest_new();
217: if ((me = (Finger *) HT_CALLOC(1, sizeof(Finger))) == NULL)
218: HT_OUTOFMEM("Finger_new");
219: me->robot = robot;
220: me->request = request;
221: me->dest = dest;
222: HTList_addObject(robot->fingers, (void *)me);
223:
224: HTRequest_setContext (request, me);
225: if (robot->flags & MR_PREEMPTIVE) HTRequest_setPreemptive(request, YES);
226: HTRequest_addRqHd(request, HT_C_HOST);
227: HTRequest_setMethod(request, method);
228: robot->cnt++;
229: return me;
1.2 frystyk 230: }
231:
1.34 eric 232: PRIVATE int Finger_delete (Finger * me)
1.2 frystyk 233: {
1.34 eric 234: HTList_removeObject(me->robot->fingers, (void *)me);
235: me->robot->cnt--;
1.37 frystyk 236:
237: /*
238: ** If we are down at one request then flush the output buffer
239: */
240: if (me->request) {
241: if (me->robot->cnt == 1) HTRequest_forceFlush(me->request);
1.34 eric 242: HTRequest_delete(me->request);
1.37 frystyk 243: }
244:
245: /*
246: ** Delete the request and free myself
247: */
1.34 eric 248: HT_FREE(me);
249: return YES;
1.2 frystyk 250: }
251:
252: /*
253: ** Cleanup and make sure we close all connections including the persistent
254: ** ones
255: */
1.1 frystyk 256: PRIVATE void Cleanup (Robot * me, int status)
257: {
258: Robot_delete(me);
1.29 eric 259: HTProfile_delete();
1.47 ! frystyk 260: #ifdef MEMLOG
1.39 eric 261: HTMemLog_close();
1.47 ! frystyk 262: #endif
! 263:
1.1 frystyk 264: #ifdef VMS
265: exit(status ? status : 1);
266: #else
267: exit(status ? status : 0);
268: #endif
269: }
270:
271: #ifdef CATCH_SIG
272: #include <signal.h>
273: /* SetSignal
274: ** This function sets up signal handlers. This might not be necessary to
275: ** call if the application has its own handlers (lossage on SVR4)
276: */
277: PRIVATE void SetSignal (void)
278: {
279: /* On some systems (SYSV) it is necessary to catch the SIGPIPE signal
280: ** when attemting to connect to a remote host where you normally should
281: ** get `connection refused' back
282: */
283: if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) {
1.13 eric 284: if (PROT_TRACE) HTTrace("HTSignal.... Can't catch SIGPIPE\n");
1.1 frystyk 285: } else {
1.13 eric 286: if (PROT_TRACE) HTTrace("HTSignal.... Ignoring SIGPIPE\n");
1.1 frystyk 287: }
1.47 ! frystyk 288:
! 289: #ifdef MEMLOG
1.44 eric 290: HTMemLog_flush();
1.47 ! frystyk 291: #endif
! 292:
1.1 frystyk 293: }
294: #endif /* CATCH_SIG */
295:
296: PRIVATE void VersionInfo (void)
297: {
1.13 eric 298: OutputData("\n\nW3C Reference Software\n\n");
299: OutputData("\tW3C Mini Robot (%s) version %s.\n",
1.1 frystyk 300: APP_NAME, APP_VERSION);
1.13 eric 301: OutputData("\tW3C Reference Library version %s.\n\n",HTLib_version());
302: OutputData("Please send feedback to <libwww@w3.org>\n");
1.1 frystyk 303: }
304:
305: /* terminate_handler
306: ** -----------------
1.2 frystyk 307: ** This function is registered to handle the result of the request.
308: ** If no more requests are pending then terminate program
1.1 frystyk 309: */
1.32 frystyk 310: PRIVATE int terminate_handler (HTRequest * request, HTResponse * response,
311: void * param, int status)
1.1 frystyk 312: {
1.34 eric 313: Finger * finger = (Finger *) HTRequest_context(request);
1.46 eric 314: Robot * mr = finger->robot;
1.34 eric 315: if (SHOW_MSG) HTTrace("Robot....... done with %s\n", HTAnchor_physical(finger->dest));
316: Finger_delete(finger);
1.46 eric 317: if (mr->cnt <= 0) {
1.34 eric 318: if (SHOW_MSG) HTTrace(" Everything is finished...\n");
1.46 eric 319: Cleanup(mr, 0); /* No way back from here */
1.30 frystyk 320: }
1.37 frystyk 321:
1.46 eric 322: if (SHOW_MSG) HTTrace(" %d outstanding request%s\n", mr->cnt, mr->cnt == 1 ? "" : "s");
1.1 frystyk 323: return HT_OK;
324: }
325:
326: /* ------------------------------------------------------------------------- */
327: /* HTEXT INTERFACE */
328: /* ------------------------------------------------------------------------- */
329:
330: PUBLIC HText * HText_new2 (HTRequest * request, HTParentAnchor * anchor,
331: HTStream * stream)
332: {
333: HText * me;
1.34 eric 334: Finger * finger = (Finger *) HTRequest_context(request);
335: Robot * mr = finger->robot;
1.14 frystyk 336: if ((me = (HText *) HT_CALLOC(1, sizeof(HText))) == NULL)
337: HT_OUTOFMEM("HText_new2");
1.4 frystyk 338:
339: /* Bind the HText object together with the Request Object */
1.1 frystyk 340: me->request = request;
1.4 frystyk 341:
342: /* Add this HyperDoc object to our list */
343: if (!mr->htext) mr->htext = HTList_new();
344: HTList_addObject(mr->htext, (void *) me);
1.1 frystyk 345: return me;
346: }
347:
1.4 frystyk 348: PUBLIC void HText_free (HText * me) {
1.11 frystyk 349: if (me) HT_FREE (me);
1.4 frystyk 350: }
351:
1.1 frystyk 352: PUBLIC void HText_beginAnchor (HText * text, HTChildAnchor * anchor)
353: {
354: if (text && anchor) {
1.34 eric 355: Finger * finger = (Finger *) HTRequest_context(text->request);
356: Robot * mr = finger->robot;
1.1 frystyk 357: HTAnchor * dest = HTAnchor_followMainLink((HTAnchor *) anchor);
358: HTParentAnchor * dest_parent = HTAnchor_parent(dest);
1.7 frystyk 359: char * uri = HTAnchor_address((HTAnchor *) dest_parent);
1.1 frystyk 360: HyperDoc * hd = HTAnchor_document(dest_parent);
361:
1.13 eric 362: if (SHOW_MSG) HTTrace("Robot....... Found `%s\' - ", uri ? uri : "NULL");
1.7 frystyk 363:
1.2 frystyk 364: /* Test whether we already have a hyperdoc for this document */
365: if (mr->flags & MR_LINK && dest_parent && !hd) {
1.1 frystyk 366: HTParentAnchor * parent = HTRequest_parent(text->request);
367: HyperDoc * last = HTAnchor_document(parent);
368: int depth = last ? last->depth+1 : 0;
1.34 eric 369: Finger * newfinger = Finger_new(mr, dest_parent, METHOD_GET);
370: HTRequest * newreq = newfinger->request;
1.2 frystyk 371: HyperDoc_new(mr, dest_parent, depth);
1.7 frystyk 372: HTRequest_setParent(newreq, HTRequest_anchor(text->request));
373: if (depth >= mr->depth) {
374: if (SHOW_MSG)
1.13 eric 375: HTTrace("loading at depth %d using HEAD\n", depth);
1.7 frystyk 376: HTRequest_setMethod(newreq, METHOD_HEAD);
1.30 frystyk 377: HTRequest_setOutputFormat(newreq, WWW_DEBUG);
1.7 frystyk 378: } else {
1.13 eric 379: if (SHOW_MSG) HTTrace("loading at depth %d\n", depth);
1.2 frystyk 380: }
381: if (HTLoadAnchor((HTAnchor *) dest_parent, newreq) != YES) {
1.13 eric 382: if (SHOW_MSG) HTTrace("not tested!\n");
1.34 eric 383: Finger_delete(newfinger);
1.2 frystyk 384: }
1.7 frystyk 385: } else {
1.18 frystyk 386: if (SHOW_MSG) HTTrace("duplicate or max depth reached\n");
1.2 frystyk 387: }
1.11 frystyk 388: HT_FREE(uri);
1.2 frystyk 389: }
390: }
391:
392: PUBLIC void HText_appendImage (HText * text, HTChildAnchor * anchor,
1.14 frystyk 393: const char *alt, const char * align, BOOL isMap)
1.2 frystyk 394: {
395: if (text && anchor) {
1.34 eric 396: Finger * finger = (Finger *) HTRequest_context(text->request);
397: Robot * mr = finger->robot;
1.2 frystyk 398: HTParentAnchor * dest = (HTParentAnchor *)
399: HTAnchor_followMainLink((HTAnchor *) anchor);
400: HyperDoc * hd = HTAnchor_document(dest);
1.1 frystyk 401:
1.2 frystyk 402: /* Test whether we already have a hyperdoc for this document */
403: if (mr->flags & MR_IMG && dest && !hd) {
404: HTParentAnchor * parent = HTRequest_parent(text->request);
405: HyperDoc * last = HTAnchor_document(parent);
406: int depth = last ? last->depth+1 : 0;
1.45 frystyk 407: Finger * newfinger = Finger_new(mr, dest,
408: mr->flags & MR_SAVE ?
409: METHOD_GET : METHOD_HEAD);
1.34 eric 410: HTRequest * newreq = newfinger->request;
1.2 frystyk 411: HyperDoc_new(mr, dest, depth);
412: if (SHOW_MSG) {
413: char * uri = HTAnchor_address((HTAnchor *) dest);
1.13 eric 414: HTTrace("Robot....... Checking Image `%s\'\n", uri);
1.11 frystyk 415: HT_FREE(uri);
1.2 frystyk 416: }
417: if (HTLoadAnchor((HTAnchor *) dest, newreq) != YES) {
418: if (SHOW_MSG)
1.13 eric 419: HTTrace("Robot....... Image not tested!\n");
1.34 eric 420: Finger_delete(newfinger);
1.1 frystyk 421: }
422: }
423: }
424: }
425:
426: PUBLIC void HText_endAnchor (HText * text) {}
1.14 frystyk 427: PUBLIC void HText_appendText (HText * text, const char * str) {}
1.1 frystyk 428: PUBLIC void HText_appendCharacter (HText * text, char ch) {}
429: PUBLIC void HText_endAppend (HText * text) {}
430: PUBLIC void HText_setStyle (HText * text, HTStyle * style) {}
431: PUBLIC void HText_beginAppend (HText * text) {}
432: PUBLIC void HText_appendParagraph (HText * text) {}
433:
434: /* ------------------------------------------------------------------------- */
435: /* MAIN PROGRAM */
436: /* ------------------------------------------------------------------------- */
437:
438: int main (int argc, char ** argv)
439: {
440: int status = 0;
441: int arg;
442: HTChunk * keywords = NULL; /* From command line */
443: int keycnt = 0;
1.12 frystyk 444: Robot * mr = NULL;
1.43 frystyk 445: Finger * finger = NULL;
446: HTParentAnchor * startAnchor = NULL;
1.1 frystyk 447:
448: /* Starts Mac GUSI socket library */
449: #ifdef GUSI
450: GUSISetup(GUSIwithSIOUXSockets);
451: GUSISetup(GUSIwithInternetSockets);
452: #endif
453:
454: #ifdef __MWERKS__ /* STR */
455: InitGraf((Ptr) &qd.thePort);
456: InitFonts();
457: InitWindows();
458: InitMenus(); TEInit();
459: InitDialogs(nil);
460: InitCursor();
461: SIOUXSettings.asktosaveonclose = false;
462: argc=ccommand(&argv);
463: #endif
464:
1.47 ! frystyk 465: #ifdef MEMLOG
1.44 eric 466: HTMemLog_open("data.log", 8192, YES);
1.47 ! frystyk 467: #endif
1.46 eric 468:
1.27 frystyk 469: /* Initiate W3C Reference Library with a robot profile */
470: HTProfile_newRobot(APP_NAME, APP_VERSION);
471:
472: /* Add the default HTML parser to the set of converters */
473: {
474: HTList * converters = HTFormat_conversion();
475: HTMLInit(converters);
476: }
1.1 frystyk 477:
1.12 frystyk 478: /* Build a new robot object */
479: mr = Robot_new();
480:
1.1 frystyk 481: /* Scan command Line for parameters */
482: for (arg=1; arg<argc; arg++) {
483: if (*argv[arg] == '-') {
484:
485: /* non-interactive */
1.17 frystyk 486: if (!strcmp(argv[arg], "-n")) {
1.1 frystyk 487: HTAlert_setInteractive(NO);
488:
489: /* log file */
490: } else if (!strcmp(argv[arg], "-l")) {
491: mr->logfile = (arg+1 < argc && *argv[arg+1] != '-') ?
492: argv[++arg] : DEFAULT_LOG_FILE;
493:
494: /* rule file */
495: } else if (!strcmp(argv[arg], "-r")) {
496: mr->rules = (arg+1 < argc && *argv[arg+1] != '-') ?
497: argv[++arg] : DEFAULT_RULE_FILE;
498:
499: /* output filename */
500: } else if (!strcmp(argv[arg], "-o")) {
501: mr->outputfile = (arg+1 < argc && *argv[arg+1] != '-') ?
502: argv[++arg] : DEFAULT_OUTPUT_FILE;
503:
504: /* timeout -- Change the default request timeout */
505: } else if (!strcmp(argv[arg], "-timeout")) {
506: int timeout = (arg+1 < argc && *argv[arg+1] != '-') ?
507: atoi(argv[++arg]) : DEFAULT_TIMEOUT;
1.40 frystyk 508: if (timeout > 0) mr->timer = timeout;
1.1 frystyk 509:
1.7 frystyk 510: /* preemptive or non-preemptive access */
1.1 frystyk 511: } else if (!strcmp(argv[arg], "-single")) {
1.7 frystyk 512: mr->flags |= MR_PREEMPTIVE;
1.2 frystyk 513:
514: /* test inlined images */
515: } else if (!strcmp(argv[arg], "-img")) {
516: mr->flags |= MR_IMG;
1.45 frystyk 517:
518: /* load inlined images */
519: } else if (!strcmp(argv[arg], "-saveimg")) {
520: mr->flags |= (MR_IMG | MR_SAVE);
1.2 frystyk 521:
522: /* load anchors */
523: } else if (!strcmp(argv[arg], "-link")) {
524: mr->flags |= MR_LINK;
1.7 frystyk 525: mr->depth = (arg+1 < argc && *argv[arg+1] != '-') ?
526: atoi(argv[++arg]) : DEFAULT_DEPTH;
1.2 frystyk 527:
1.12 frystyk 528: /* Output start and end time */
529: } else if (!strcmp(argv[arg], "-ss")) {
530: time_t local = time(NULL);
1.13 eric 531: HTTrace("Robot started on %s\n",
1.12 frystyk 532: HTDateTimeStr(&local, YES));
533: mr->flags |= MR_TIME;
534:
1.1 frystyk 535: /* print version and exit */
536: } else if (!strcmp(argv[arg], "-version")) {
537: VersionInfo();
538: Cleanup(mr, 0);
1.46 eric 539:
540: /* run in quiet mode */
541: } else if (!strcmp(argv[arg], "-q")) {
542: mr->flags |= MR_QUIET;
1.1 frystyk 543:
544: #ifdef WWWTRACE
545: /* trace flags */
546: } else if (!strncmp(argv[arg], "-v", 2)) {
1.24 frystyk 547: HTSetTraceMessageMask(argv[arg]+2);
1.1 frystyk 548: #endif
549:
550: } else {
1.13 eric 551: if (SHOW_MSG) HTTrace("Bad Argument (%s)\n", argv[arg]);
1.1 frystyk 552: }
1.17 frystyk 553: } else { /* If no leading `-' then check for URL or keywords */
1.1 frystyk 554: if (!keycnt) {
555: char * ref = HTParse(argv[arg], mr->cwd, PARSE_ALL);
1.34 eric 556: startAnchor = (HTParentAnchor *) HTAnchor_findAddress(ref);
557: HyperDoc_new(mr, startAnchor, 0);
1.1 frystyk 558: keycnt = 1;
1.11 frystyk 559: HT_FREE(ref);
1.1 frystyk 560: } else { /* Check for successive keyword arguments */
561: char *escaped = HTEscape(argv[arg], URL_XALPHAS);
562: if (keycnt++ <= 1)
1.5 frystyk 563: keywords = HTChunk_new(128);
1.1 frystyk 564: else
1.5 frystyk 565: HTChunk_putc(keywords, ' ');
566: HTChunk_puts(keywords, HTStrip(escaped));
1.11 frystyk 567: HT_FREE(escaped);
1.1 frystyk 568: }
569: }
570: }
571:
572: #ifdef CATCH_SIG
573: SetSignal();
574: #endif
575:
576: if (!keycnt) {
1.13 eric 577: if (SHOW_MSG) HTTrace("Please specify URL to check.\n");
1.1 frystyk 578: Cleanup(mr, -1);
579: }
580:
1.23 manoli 581: /* Testing that HTTrace is working */
1.47 ! frystyk 582: if (SHOW_MSG) HTTrace ("Welcome to the W3C mini Robot\n");
1.23 manoli 583:
1.1 frystyk 584: /* Rule file specified? */
585: if (mr->rules) {
586: char * rules = HTParse(mr->rules, mr->cwd, PARSE_ALL);
1.27 frystyk 587: if (!HTLoadRules(rules))
1.13 eric 588: if (SHOW_MSG) HTTrace("Can't access rules\n");
1.11 frystyk 589: HT_FREE(rules);
1.1 frystyk 590: }
591:
592: /* Output file specified? */
593: if (mr->outputfile) {
594: if ((mr->output = fopen(mr->outputfile, "wb")) == NULL) {
1.13 eric 595: if (SHOW_MSG) HTTrace("Can't open `%s'\n", mr->outputfile);
1.1 frystyk 596: mr->output = OUTPUT;
597: }
598: }
599:
600: /* Log file specifed? */
601: if (mr->logfile) HTLog_open(mr->logfile, YES, YES);
602:
1.27 frystyk 603: /* Register our own someterminater filter */
1.32 frystyk 604: HTNet_addAfter(terminate_handler, NULL, NULL, HT_ALL, HT_FILTER_LAST);
1.40 frystyk 605:
606: /* Setting event timeout */
607: HTHost_setEventTimeout(mr->timer);
1.37 frystyk 608:
1.34 eric 609: /* Start the request */
610: finger = Finger_new(mr, startAnchor, METHOD_GET);
1.43 frystyk 611:
612: /*
613: ** Make sure that the first request is flushed immediately and not
614: ** buffered in the output buffer
615: */
616: HTRequest_setFlush(finger->request, YES);
617:
618: /*
619: ** Now do the load
620: */
1.34 eric 621: if (mr->flags & MR_PREEMPTIVE)
622: HTRequest_setPreemptive(finger->request, YES);
1.1 frystyk 623:
624: if (keywords) /* Search */
1.34 eric 625: status = HTSearchAnchor(keywords, (HTAnchor *)startAnchor, finger->request);
1.1 frystyk 626: else
1.34 eric 627: status = HTLoadAnchor((HTAnchor *)startAnchor, finger->request);
1.1 frystyk 628:
1.5 frystyk 629: if (keywords) HTChunk_delete(keywords);
1.1 frystyk 630: if (status != YES) {
1.13 eric 631: if (SHOW_MSG) HTTrace("Can't access resource\n");
1.1 frystyk 632: Cleanup(mr, -1);
633: }
634:
635: /* Go into the event loop... */
1.34 eric 636: HTEventList_loop(finger->request);
1.1 frystyk 637:
638: /* Only gets here if event loop fails */
639: Cleanup(mr, 0);
640: return 0;
641: }
Webmaster
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to HTRobot.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Public] / libwww / Robot / src