Annotation of libwww/Robot/src/HTRobot.c, revision 1.77
1.75 frystyk 1: /*
1.77 ! frystyk 2: ** @(#) $Id: HTRobot.c,v 1.76 1998/10/27 17:42:53 frystyk Exp $
1.75 frystyk 3: **
4: ** W3C Webbot can be found at "http://www.w3.org/Robot/"
5: **
6: ** Copyright 1995-1998 World Wide Web Consortium, (Massachusetts
7: ** Institute of Technology, Institut National de Recherche en
8: ** Informatique et en Automatique, Keio University). All Rights
9: ** Reserved. This program is distributed under the W3C's Software
10: ** Intellectual Property License. This program is distributed in the hope
11: ** that it will be useful, but WITHOUT ANY WARRANTY; without even the
12: ** implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
13: ** PURPOSE. See W3C License http://www.w3.org/Consortium/Legal/ for more
14: ** details.
1.1 frystyk 15: **
16: ** Authors:
17: ** HFN Henrik Frystyk Nielsen, (frystyk@w3.org)
1.75 frystyk 18: ** BR Bob Racko
19: ** JP John Punin
1.1 frystyk 20: **
21: ** History:
22: ** Dec 04 95 First version
1.75 frystyk 23: ** Oct 1998 Split into separate files
1.1 frystyk 24: */
25:
1.75 frystyk 26: #include "HTRobMan.h"
27: #include "HTQueue.h"
28: #include "HTAncMan.h"
1.51 frystyk 29:
1.62 frystyk 30: #define SHOW_QUIET(mr) ((mr) && !((mr)->flags & MR_QUIET))
31: #define SHOW_REAL_QUIET(mr) ((mr) && !((mr)->flags & MR_REAL_QUIET))
1.1 frystyk 32:
1.75 frystyk 33: PRIVATE HTErrorMessage HTErrors[HTERR_ELEMENTS] = {HTERR_ENGLISH_INITIALIZER};
1.58 frystyk 34:
35: /*
36: ** Some sorting algorithms
37: */
1.63 frystyk 38: PRIVATE HTComparer HitSort, FormatSort, LastModifiedSort, TitleSort;
1.58 frystyk 39:
1.1 frystyk 40: PUBLIC HText * HTMainText = NULL;
41: PUBLIC HTParentAnchor * HTMainAnchor = NULL;
42: PUBLIC HTStyleSheet * styleSheet = NULL;
43:
44: /* ------------------------------------------------------------------------- */
45:
1.13 eric 46: /* Standard (non-error) Output
47: ** ---------------------------
48: */
49: PUBLIC int OutputData(const char * fmt, ...)
50: {
51: int ret;
52: va_list pArgs;
53: va_start(pArgs, fmt);
54: ret = vfprintf(stdout, fmt, pArgs);
55: va_end(pArgs);
56: return ret;
57: }
58:
59: /* ------------------------------------------------------------------------- */
60:
1.2 frystyk 61: /* Create a "HyperDoc" object
62: ** --------------------------
63: ** A HyperDoc object contains information about whether we have already
64: ** started checking the anchor and the depth in our search
65: */
1.75 frystyk 66: PUBLIC HyperDoc * HyperDoc_new (Robot * mr,HTParentAnchor * anchor, int depth)
1.2 frystyk 67: {
68: HyperDoc * hd;
1.14 frystyk 69: if ((hd = (HyperDoc *) HT_CALLOC(1, sizeof(HyperDoc))) == NULL)
70: HT_OUTOFMEM("HyperDoc_new");
1.2 frystyk 71: hd->depth = depth;
1.55 frystyk 72: hd->hits = 1;
1.75 frystyk 73:
74: hd->code = -1;
75: hd->index = ++mr->cindex;
76:
1.2 frystyk 77: /* Bind the HyperDoc object together with the Anchor Object */
78: hd->anchor = anchor;
79: HTAnchor_setDocument(anchor, (void *) hd);
80:
81: /* Add this HyperDoc object to our list */
82: if (!mr->hyperdoc) mr->hyperdoc = HTList_new();
83: HTList_addObject(mr->hyperdoc, (void *) hd);
84: return hd;
85: }
86:
87: /* Delete a "HyperDoc" object
88: ** --------------------------
89: */
1.75 frystyk 90: PUBLIC BOOL HyperDoc_delete (HyperDoc * hd)
1.2 frystyk 91: {
92: if (hd) {
1.11 frystyk 93: HT_FREE (hd);
1.2 frystyk 94: return YES;
95: }
96: return NO;
97: }
98:
1.55 frystyk 99: /*
100: ** Sort the anchor array and log reference count
101: */
102: PRIVATE BOOL calculate_hits (Robot * mr, HTArray * array)
103: {
104: if (mr && array) {
105: HTLog * log = HTLog_open(mr->hitfile, YES, YES);
106: if (log) {
107: void ** data = NULL;
108: HTParentAnchor * anchor = NULL;
109: HTArray_sort(array, HitSort);
110: anchor = (HTParentAnchor *) HTArray_firstObject(array, data);
111: while (anchor) {
112: char * uri = HTAnchor_address((HTAnchor *) anchor);
113: HyperDoc * hd = (HyperDoc *) HTAnchor_document(anchor);
1.63 frystyk 114: if (uri && hd) HTLog_addText(log, "%8d %s\n", hd->hits, uri);
1.55 frystyk 115: HT_FREE(uri);
116: anchor = (HTParentAnchor *) HTArray_nextObject(array, data);
117: }
118: }
119: HTLog_close(log);
120: return YES;
121: }
122: return NO;
123: }
124:
125: PRIVATE int HitSort (const void * a, const void * b)
126: {
127: HyperDoc * aa = HTAnchor_document(*(HTParentAnchor **) a);
128: HyperDoc * bb = HTAnchor_document(*(HTParentAnchor **) b);
129: if (aa && bb) return (bb->hits - aa->hits);
130: return bb - aa;
131: }
132:
1.58 frystyk 133: /*
1.64 frystyk 134: ** Sort the anchor array and log link relations
135: */
136: PRIVATE BOOL calculate_linkRelations (Robot * mr, HTArray * array)
137: {
138: if (mr && array) {
1.68 frystyk 139: HTLog * log = mr->relfile ? HTLog_open(mr->relfile, YES, YES) : NULL;
140: void ** data = NULL;
141: HTParentAnchor * anchor = NULL;
142: anchor = (HTParentAnchor *) HTArray_firstObject(array, data);
143: while (anchor) {
144:
145: /*
146: ** If we have a specific link relation to look for then do this.
147: ** Otherwise look for all link relations.
148: */
149: if (mr->relation) {
150: HTLink * link = HTAnchor_findLinkType((HTAnchor *) anchor, mr->relation);
151: if (link) {
152: HTParentAnchor * dest = HTAnchor_parent(HTLink_destination(link));
153: char * src_uri = HTAnchor_address((HTAnchor *) anchor);
154: char * dest_uri = HTAnchor_address((HTAnchor *) dest);
155: if (src_uri && dest_uri) {
156: #ifdef HT_MYSQL
157: if (mr->sqllog) {
158: HTSQLLog_addLinkRelationship (mr->sqllog,
159: src_uri, dest_uri,
160: HTAtom_name(mr->relation),
161: NULL);
162: }
163: #endif
164: if (log) {
165: HTFormat format = HTAnchor_format(dest);
166: HTLog_addText(log, "%s %s %s --> %s\n",
167: HTAtom_name(mr->relation),
168: format != WWW_UNKNOWN ?
169: HTAtom_name(format) : "<unknown>",
170: src_uri, dest_uri);
171: }
172:
173: /* Cleanup */
174: HT_FREE(src_uri);
175: HT_FREE(dest_uri);
176: }
177: }
178: } else {
179: HTLink * link = HTAnchor_mainLink((HTAnchor *) anchor);
180: HTList * sublinks = HTAnchor_subLinks((HTAnchor *) anchor);
181: char * src_uri = HTAnchor_address((HTAnchor *) anchor);
182: HTLinkType linktype;
183:
184: /* First look in the main link */
185: if (link && (linktype = HTLink_type(link))) {
186: HTParentAnchor * dest = HTAnchor_parent(HTLink_destination(link));
187: char * dest_uri = HTAnchor_address((HTAnchor *) dest);
188: if (src_uri && dest_uri) {
189: #ifdef HT_MYSQL
190: if (mr->sqllog) {
191: HTSQLLog_addLinkRelationship (mr->sqllog,
192: src_uri, dest_uri,
193: HTAtom_name(linktype),
194: NULL);
195: }
196: #endif
197: if (log) {
198: HTFormat format = HTAnchor_format(dest);
199: HTLog_addText(log, "%s %s %s --> %s\n",
200: HTAtom_name(linktype),
201: format != WWW_UNKNOWN ?
202: HTAtom_name(format) : "<unknown>",
203: src_uri, dest_uri);
204: }
205: }
206: HT_FREE(dest_uri);
207: }
208:
209: /* and then in any sublinks */
210: if (sublinks) {
211: HTLink * pres;
212: while ((pres = (HTLink *) HTList_nextObject(sublinks))) {
213: if ((linktype = HTLink_type(pres))) {
214: HTParentAnchor * dest = HTAnchor_parent(HTLink_destination(pres));
1.64 frystyk 215: char * dest_uri = HTAnchor_address((HTAnchor *) dest);
1.68 frystyk 216: if (src_uri && dest_uri) {
217: #ifdef HT_MYSQL
218: if (mr->sqllog) {
219: HTSQLLog_addLinkRelationship (mr->sqllog,
220: src_uri, dest_uri,
221: HTAtom_name(linktype),
222: NULL);
223: }
224: #endif
225: if (log) {
226: HTFormat format = HTAnchor_format(dest);
227: HTLog_addText(log, "%s %s %s --> %s\n",
228: HTAtom_name(linktype),
229: format != WWW_UNKNOWN ?
230: HTAtom_name(format) : "<unknown>",
231: src_uri, dest_uri);
232: }
1.64 frystyk 233: HT_FREE(dest_uri);
234: }
235: }
236: }
237: }
1.68 frystyk 238:
239: /* Cleanup */
240: HT_FREE(src_uri);
241: }
242: anchor = (HTParentAnchor *) HTArray_nextObject(array, data);
1.64 frystyk 243: }
1.68 frystyk 244: if (log) HTLog_close(log);
1.64 frystyk 245: return YES;
246: }
247: return NO;
248: }
249:
250: /*
1.63 frystyk 251: ** Sort the anchor array and log last modified date
252: */
253: PRIVATE BOOL calculate_lm (Robot * mr, HTArray * array)
254: {
255: if (mr && array) {
256: HTLog * log = HTLog_open(mr->lmfile, YES, YES);
257: if (log) {
258: void ** data = NULL;
259: HTParentAnchor * anchor = NULL;
260: HTArray_sort(array, LastModifiedSort);
261: anchor = (HTParentAnchor *) HTArray_firstObject(array, data);
262: while (anchor) {
263: char * uri = HTAnchor_address((HTAnchor *) anchor);
264: time_t lm = HTAnchor_lastModified(anchor);
265: if (uri && lm > 0)
266: HTLog_addText(log, "%s %s\n", HTDateTimeStr(&lm, NO), uri);
267: HT_FREE(uri);
268: anchor = (HTParentAnchor *) HTArray_nextObject(array, data);
269: }
270: }
271: HTLog_close(log);
272: return YES;
273: }
274: return NO;
275: }
276:
277: PRIVATE int LastModifiedSort (const void * a, const void * b)
278: {
279: time_t aa = HTAnchor_lastModified(*(HTParentAnchor **) a);
280: time_t bb = HTAnchor_lastModified(*(HTParentAnchor **) b);
281: return bb - aa;
282: }
283:
284: /*
285: ** Sort the anchor array and log the document title
286: */
287: PRIVATE BOOL calculate_title (Robot * mr, HTArray * array)
288: {
289: if (mr && array) {
290: HTLog * log = HTLog_open(mr->titlefile, YES, YES);
291: if (log) {
292: void ** data = NULL;
293: HTParentAnchor * anchor = NULL;
294: HTArray_sort(array, TitleSort);
295: anchor = (HTParentAnchor *) HTArray_firstObject(array, data);
296: while (anchor) {
297: char * uri = HTAnchor_address((HTAnchor *) anchor);
298: const char * title = HTAnchor_title(anchor);
299: HTCharset charset = HTAnchor_charset(anchor);
300: if (uri) HTLog_addText(log, "%s `%s\' %s\n",
301: charset ? HTAtom_name(charset) : "<none>",
302: title ? title : "<none>",
303: uri);
304: HT_FREE(uri);
305: anchor = (HTParentAnchor *) HTArray_nextObject(array, data);
306: }
307: }
308: HTLog_close(log);
309: return YES;
310: }
311: return NO;
312: }
313:
314: PRIVATE int TitleSort (const void * a, const void * b)
315: {
316: const char * aa = HTAnchor_title(*(HTParentAnchor **) a);
317: const char * bb = HTAnchor_title(*(HTParentAnchor **) b);
318: return strcasecomp(bb?bb:"", aa?aa:"");
319: }
320:
321: /*
1.58 frystyk 322: ** Calculate distributions for media types. The same mechanism
323: ** can be used for other characteristics with relatively
324: ** few outcomes.
325: */
326: PRIVATE HTList * mediatype_distribution (HTArray * array)
327: {
328: if (array) {
329: HTList * mt = HTList_new();
330: MetaDist * pres = NULL;
331: void ** data = NULL;
332: HTParentAnchor * anchor = NULL;
333: anchor = (HTParentAnchor *) HTArray_firstObject(array, data);
334: while (anchor) {
335: HTFormat format = HTAnchor_format(anchor);
336: if (format && format != WWW_UNKNOWN) {
337: HTList * cur = mt;
338:
339: /* If found then increase counter */
340: while ((pres = (MetaDist *) HTList_nextObject(cur))) {
341: if (pres->name == format) {
342: pres->hits++;
343: break;
344: }
345: }
346:
347: /* If not found then add new format to list */
348: if (!pres) {
349: if ((pres = (MetaDist *) HT_CALLOC(1, sizeof(MetaDist))) == NULL)
350: HT_OUTOFMEM("mediatype_distribution");
351: pres->name = format;
352: pres->hits = 1;
353: HTList_addObject(mt, pres);
354: HTList_insertionSort(mt, FormatSort);
355: }
356: }
357:
358: /* Find next anchor in array */
359: anchor = (HTParentAnchor *) HTArray_nextObject(array, data);
360: }
361: return mt;
362: }
363: return NULL;
364: }
365:
1.60 frystyk 366: /*
367: ** Calculate distributions for charsets. The same mechanism
368: ** can be used for other characteristics with relatively
369: ** few outcomes.
370: */
371: PRIVATE HTList * charset_distribution (HTArray * array)
372: {
373: if (array) {
374: HTList * cs = HTList_new();
375: MetaDist * pres = NULL;
376: void ** data = NULL;
377: HTParentAnchor * anchor = NULL;
378: anchor = (HTParentAnchor *) HTArray_firstObject(array, data);
379: while (anchor) {
380: HTCharset charset = HTAnchor_charset(anchor);
381: if (charset) {
382: HTList * cur = cs;
383:
384: /* If found then increase counter */
385: while ((pres = (MetaDist *) HTList_nextObject(cur))) {
386: if (pres->name == charset) {
387: pres->hits++;
388: break;
389: }
390: }
391:
392: /* If not found then add new format to list */
393: if (!pres) {
394: if ((pres = (MetaDist *) HT_CALLOC(1, sizeof(MetaDist))) == NULL)
395: HT_OUTOFMEM("charset_distribution");
396: pres->name = charset;
397: pres->hits = 1;
398: HTList_addObject(cs, pres);
399: HTList_insertionSort(cs, FormatSort);
400: }
401: }
402:
403: /* Find next anchor in array */
404: anchor = (HTParentAnchor *) HTArray_nextObject(array, data);
405: }
406: return cs;
407: }
408: return NULL;
409: }
410:
1.58 frystyk 411: PRIVATE int FormatSort (const void * a, const void * b)
412: {
413: MetaDist * aa = (MetaDist *) a;
414: MetaDist * bb = (MetaDist *) b;
415: return strcmp(HTAtom_name(bb->name), HTAtom_name(aa->name));
416: }
417:
418: PRIVATE BOOL log_meta_distribution (const char * logfile, HTList * distribution)
419: {
420: if (logfile && distribution) {
421: HTLog * log = HTLog_open(logfile, YES, YES);
422: if (log) {
423: HTList * cur = distribution;
424: MetaDist * pres;
425: while ((pres = (MetaDist *) HTList_nextObject(cur))) {
426: if (pres->name) {
1.60 frystyk 427: HTLog_addText(log, "%8d %s\n", pres->hits, HTAtom_name(pres->name));
1.58 frystyk 428: }
429: }
430: HTLog_close(log);
431: }
432: }
433: return NO;
434: }
435:
436: PRIVATE BOOL delete_meta_distribution (HTList * distribution)
437: {
438: if (distribution) {
439: HTList * cur = distribution;
440: MetaDist * pres;
441: while ((pres = (MetaDist *) HTList_nextObject(cur)))
442: HT_FREE(pres);
443: HTList_delete(distribution);
444: return YES;
445: }
446: return NO;
447: }
448:
449:
1.55 frystyk 450: /* Statistics
451: ** ----------
452: ** Calculates a bunch of statistics for the anchors traversed
453: */
454: PRIVATE BOOL calculate_statistics (Robot * mr)
455: {
1.59 frystyk 456: long total_docs = mr->get_docs + mr->head_docs + mr->other_docs;
1.55 frystyk 457: if (!mr) return NO;
458:
459: /* Calculate efficiency */
1.59 frystyk 460: if (mr->time > 0) {
1.56 frystyk 461: ms_t t = HTGetTimeInMillis() - mr->time;
462: if (t > 0) {
1.60 frystyk 463: double loadfactor = (mr->get_bytes / (t * 0.001));
464: double reqprsec = (total_docs / (t * 0.001));
1.56 frystyk 465: double secs = t / 1000.0;
1.55 frystyk 466: char bytes[50];
1.62 frystyk 467: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 468: HTTrace("\nAccessed %ld documents in %.2f seconds (%.2f requests pr sec)\n",
1.62 frystyk 469: total_docs, secs, reqprsec);
1.59 frystyk 470:
471: HTNumToStr(mr->get_bytes, bytes, 50);
1.62 frystyk 472: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 473: HTTrace("\tDid a GET on %ld document(s) and downloaded %s bytes of document bodies (%2.1f bytes/sec)\n",
1.62 frystyk 474: mr->get_docs, bytes, loadfactor);
1.59 frystyk 475:
476: HTNumToStr(mr->head_bytes, bytes, 50);
1.62 frystyk 477: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 478: HTTrace("\tDid a HEAD on %ld document(s) with a total of %s bytes\n",
1.62 frystyk 479: mr->head_docs, bytes);
1.55 frystyk 480: }
481: }
482:
483: /* Create an array of existing anchors */
1.59 frystyk 484: if (total_docs > 1) {
485: HTArray * array = HTAnchor_getArray(total_docs);
1.55 frystyk 486: if (array) {
487:
1.63 frystyk 488: /* Distributions */
489: if (mr->flags & MR_DISTRIBUTIONS) {
1.64 frystyk 490: if (SHOW_REAL_QUIET(mr)) HTTrace("\nDistributions:\n");
1.63 frystyk 491: }
492:
1.55 frystyk 493: /* Sort after hit counts */
1.63 frystyk 494: if (mr->hitfile) {
495: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 496: HTTrace("\tLogged hit count distribution in file `%s\'\n",
1.63 frystyk 497: mr->hitfile);
498: calculate_hits(mr, array);
499: }
500:
1.64 frystyk 501: /* Sort after link relations */
1.68 frystyk 502: #ifdef HT_MYSQL
503: if (mr->relfile || mr->sqllog) {
1.69 frystyk 504: #else
505: if (mr->relfile) {
506: #endif
1.68 frystyk 507: if (mr->relfile && SHOW_REAL_QUIET(mr))
1.64 frystyk 508: HTTrace("\tLogged link relationship distribution in file `%s\'\n",
509: mr->relfile);
510: calculate_linkRelations(mr, array);
511: }
512:
1.63 frystyk 513: /* Sort after modified date */
514: if (mr->lmfile) {
515: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 516: HTTrace("\tLogged last modified distribution in file `%s\'\n",
1.63 frystyk 517: mr->lmfile);
518: calculate_lm(mr, array);
519: }
520:
521: /* Sort after title */
522: if (mr->titlefile) {
523: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 524: HTTrace("\tLogged title distribution in file `%s\'\n",
1.63 frystyk 525: mr->titlefile);
526: calculate_title(mr, array);
527: }
1.55 frystyk 528:
1.58 frystyk 529: /* Find mediatype distribution */
530: if (mr->mtfile) {
531: HTList * mtdist = mediatype_distribution(array);
532: if (mtdist) {
1.63 frystyk 533: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 534: HTTrace("\tLogged media type distribution in file `%s\'\n",
1.63 frystyk 535: mr->mtfile);
1.58 frystyk 536: log_meta_distribution(mr->mtfile, mtdist);
537: delete_meta_distribution(mtdist);
538: }
539: }
1.55 frystyk 540:
1.60 frystyk 541: /* Find charset distribution */
542: if (mr->charsetfile) {
543: HTList * charsetdist = charset_distribution(array);
544: if (charsetdist) {
1.63 frystyk 545: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 546: HTTrace("\tLogged charset distribution in file `%s\'\n",
1.63 frystyk 547: mr->charsetfile);
1.60 frystyk 548: log_meta_distribution(mr->charsetfile, charsetdist);
549: delete_meta_distribution(charsetdist);
550: }
551: }
552:
1.55 frystyk 553: /* Add as may other stats here as you like */
1.60 frystyk 554: /* ... */
1.58 frystyk 555:
556: /* Delete the array */
1.55 frystyk 557: HTArray_delete(array);
558: }
559: }
560: return YES;
561: }
562:
1.75 frystyk 563: PRIVATE HTParentAnchor *
564: get_last_parent(HTParentAnchor *anchor)
565: {
566: HTAnchor *anc;
567: HTList *sources = anchor->sources;
568:
569: while((anc = (HTAnchor *) HTList_nextObject(sources)) != NULL)
570: {
571: HTParentAnchor *panchor = HTAnchor_parent(anc);
572: return panchor;
573: }
574: return NULL;
575: }
576:
577: PRIVATE void
578: set_error_state_hyperdoc(HyperDoc * hd, HTRequest *request)
579: {
580: HTList * cur = HTRequest_error(request);
581: HTError *pres;
582:
583: while((pres = (HTError *) HTList_nextObject(cur)) != NULL)
584: {
585: int code =HTErrors[HTError_index(pres)].code;
586:
587: hd->code = code;
588: }
589: }
590:
591:
592: PRIVATE int
593: test_for_blank_spaces(char *uri)
594: {
595: char *ptr = uri;
596: for(;*ptr!='\0';ptr++)
597: if(*ptr == ' ')
598: return 1;
599: return 0;
600: }
601:
602:
1.1 frystyk 603: /* Create a Command Line Object
604: ** ----------------------------
605: */
1.75 frystyk 606: PUBLIC Robot * Robot_new (void)
1.1 frystyk 607: {
608: Robot * me;
1.41 frystyk 609: if ((me = (Robot *) HT_CALLOC(1, sizeof(Robot))) == NULL)
1.14 frystyk 610: HT_OUTOFMEM("Robot_new");
1.2 frystyk 611: me->hyperdoc = HTList_new();
1.4 frystyk 612: me->htext = HTList_new();
1.74 frystyk 613: me->timer = DEFAULT_TIMEOUT*MILLIES;
1.75 frystyk 614: me->waits = 0;
1.25 frystyk 615: me->cwd = HTGetCurrentDirectoryURL();
1.1 frystyk 616: me->output = OUTPUT;
1.35 eric 617: me->cnt = 0;
1.75 frystyk 618: me->ndoc = -1;
1.34 eric 619: me->fingers = HTList_new();
1.75 frystyk 620:
621: /* This is new */
622: me->queue = HTQueue_new();
623: me->cq = 0;
624: me->furl = NULL;
625:
1.1 frystyk 626: return me;
627: }
628:
629: /* Delete a Command Line Object
630: ** ----------------------------
631: */
1.62 frystyk 632: PRIVATE BOOL Robot_delete (Robot * mr)
1.1 frystyk 633: {
1.62 frystyk 634: if (mr) {
635: HTList_delete(mr->fingers);
1.55 frystyk 636:
637: /* Calculate statistics */
1.62 frystyk 638: calculate_statistics(mr);
1.55 frystyk 639:
1.62 frystyk 640: if (mr->hyperdoc) {
641: HTList * cur = mr->hyperdoc;
1.2 frystyk 642: HyperDoc * pres;
643: while ((pres = (HyperDoc *) HTList_nextObject(cur)))
644: HyperDoc_delete(pres);
1.62 frystyk 645: HTList_delete(mr->hyperdoc);
1.2 frystyk 646: }
1.62 frystyk 647: if (mr->htext) {
648: HTList * cur = mr->htext;
1.4 frystyk 649: HText * pres;
650: while ((pres = (HText *) HTList_nextObject(cur)))
651: HText_free(pres);
1.62 frystyk 652: HTList_delete(mr->htext);
1.4 frystyk 653: }
1.62 frystyk 654:
655: /* Close all the log files */
1.63 frystyk 656: if (mr->flags & MR_LOGGING) {
1.64 frystyk 657: if (SHOW_REAL_QUIET(mr)) HTTrace("\nRaw Log files:\n");
1.63 frystyk 658: }
659:
1.62 frystyk 660: if (mr->log) {
661: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 662: HTTrace("\tLogged %5d entries in general log file `%s\'\n",
1.62 frystyk 663: HTLog_accessCount(mr->log), mr->logfile);
664: HTLog_close(mr->log);
665: }
666: if (mr->ref) {
667: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 668: HTTrace("\tLogged %5d entries in referer log file `%s\'\n",
1.62 frystyk 669: HTLog_accessCount(mr->ref), mr->reffile);
670: HTLog_close(mr->ref);
671: }
672: if (mr->reject) {
673: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 674: HTTrace("\tLogged %5d entries in rejected log file `%s\'\n",
1.62 frystyk 675: HTLog_accessCount(mr->reject), mr->rejectfile);
676: HTLog_close(mr->reject);
677: }
678: if (mr->notfound) {
679: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 680: HTTrace("\tLogged %5d entries in not found log file `%s\'\n",
1.62 frystyk 681: HTLog_accessCount(mr->notfound), mr->notfoundfile);
682: HTLog_close(mr->notfound);
683: }
684: if (mr->conneg) {
685: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 686: HTTrace("\tLogged %5d entries in content negotiation log file `%s\'\n",
1.62 frystyk 687: HTLog_accessCount(mr->conneg), mr->connegfile);
688: HTLog_close(mr->conneg);
689: }
690: if (mr->noalttag) {
691: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 692: HTTrace("\tLogged %5d entries in missing alt tag log file `%s\'\n",
1.62 frystyk 693: HTLog_accessCount(mr->noalttag), mr->noalttagfile);
694: HTLog_close(mr->noalttag);
695: }
696:
697: if (mr->output && mr->output != STDOUT) fclose(mr->output);
698:
699: if (mr->flags & MR_TIME) {
1.12 frystyk 700: time_t local = time(NULL);
1.62 frystyk 701: if (SHOW_REAL_QUIET(mr))
1.64 frystyk 702: HTTrace("\nRobot terminated %s\n", HTDateTimeStr(&local, YES));
1.12 frystyk 703: }
1.55 frystyk 704:
1.75 frystyk 705: /* This is new */
706: if(mr->cdepth)
707: HT_FREE(mr->cdepth);
708: if(mr->furl) HT_FREE(mr->furl);
709:
1.58 frystyk 710: #ifdef HT_POSIX_REGEX
1.62 frystyk 711: if (mr->include) {
712: regfree(mr->include);
713: HT_FREE(mr->include);
714: }
715: if (mr->exclude) {
716: regfree(mr->exclude);
717: HT_FREE(mr->exclude);
718: }
1.75 frystyk 719: if (mr->exc_robot) {
720: regfree(mr->exc_robot);
721: HT_FREE(mr->exc_robot);
722: }
1.62 frystyk 723: if (mr->check) {
724: regfree(mr->check);
725: HT_FREE(mr->check);
1.58 frystyk 726: }
727: #endif
728:
1.68 frystyk 729: #ifdef HT_MYSQL
730: if (mr->sqllog) {
731: HTSQLLog_close(mr->sqllog);
732: mr->sqllog = NULL;
733: }
734: #endif
735:
1.62 frystyk 736: HT_FREE(mr->cwd);
737: HT_FREE(mr->prefix);
738: HT_FREE(mr->img_prefix);
739: HT_FREE(mr);
1.1 frystyk 740: return YES;
741: }
742: return NO;
743: }
744:
1.2 frystyk 745: /*
1.34 eric 746: ** This function creates a new finger object and initializes it with a new request
1.2 frystyk 747: */
1.75 frystyk 748: PUBLIC Finger * Finger_new (Robot * robot, HTParentAnchor * dest, HTMethod method)
1.2 frystyk 749: {
1.34 eric 750: Finger * me;
751: HTRequest * request = HTRequest_new();
752: if ((me = (Finger *) HT_CALLOC(1, sizeof(Finger))) == NULL)
753: HT_OUTOFMEM("Finger_new");
754: me->robot = robot;
755: me->request = request;
756: me->dest = dest;
757: HTList_addObject(robot->fingers, (void *)me);
758:
1.48 frystyk 759: /* Set the context for this request */
1.34 eric 760: HTRequest_setContext (request, me);
1.48 frystyk 761:
762: /* Check the various flags to customize the request */
763: if (robot->flags & MR_PREEMPTIVE)
764: HTRequest_setPreemptive(request, YES);
765: if (robot->flags & MR_VALIDATE)
766: HTRequest_setReloadMode(request, HT_CACHE_VALIDATE);
767: if (robot->flags & MR_END_VALIDATE)
768: HTRequest_setReloadMode(request, HT_CACHE_END_VALIDATE);
769:
770: /* We wanna make sure that we are sending a Host header (default) */
1.34 eric 771: HTRequest_addRqHd(request, HT_C_HOST);
1.48 frystyk 772:
773: /* Set the method for this request */
1.34 eric 774: HTRequest_setMethod(request, method);
775: robot->cnt++;
776: return me;
1.2 frystyk 777: }
778:
1.34 eric 779: PRIVATE int Finger_delete (Finger * me)
1.2 frystyk 780: {
1.34 eric 781: HTList_removeObject(me->robot->fingers, (void *)me);
782: me->robot->cnt--;
1.37 frystyk 783:
784: /*
785: ** If we are down at one request then flush the output buffer
786: */
787: if (me->request) {
788: if (me->robot->cnt == 1) HTRequest_forceFlush(me->request);
1.34 eric 789: HTRequest_delete(me->request);
1.37 frystyk 790: }
791:
792: /*
793: ** Delete the request and free myself
794: */
1.34 eric 795: HT_FREE(me);
796: return YES;
1.2 frystyk 797: }
798:
799: /*
800: ** Cleanup and make sure we close all connections including the persistent
801: ** ones
802: */
1.75 frystyk 803: PUBLIC void Cleanup (Robot * me, int status)
1.1 frystyk 804: {
805: Robot_delete(me);
1.29 eric 806: HTProfile_delete();
1.50 frystyk 807: #ifdef HT_MEMLOG
1.39 eric 808: HTMemLog_close();
1.47 frystyk 809: #endif
810:
1.1 frystyk 811: #ifdef VMS
812: exit(status ? status : 1);
813: #else
814: exit(status ? status : 0);
815: #endif
816: }
817:
818: #ifdef CATCH_SIG
819: #include <signal.h>
820: /* SetSignal
821: ** This function sets up signal handlers. This might not be necessary to
822: ** call if the application has its own handlers (lossage on SVR4)
823: */
1.75 frystyk 824: PUBLIC void SetSignal (void)
1.1 frystyk 825: {
826: /* On some systems (SYSV) it is necessary to catch the SIGPIPE signal
827: ** when attemting to connect to a remote host where you normally should
828: ** get `connection refused' back
829: */
830: if (signal(SIGPIPE, SIG_IGN) == SIG_ERR) {
1.13 eric 831: if (PROT_TRACE) HTTrace("HTSignal.... Can't catch SIGPIPE\n");
1.1 frystyk 832: } else {
1.13 eric 833: if (PROT_TRACE) HTTrace("HTSignal.... Ignoring SIGPIPE\n");
1.1 frystyk 834: }
1.47 frystyk 835:
1.50 frystyk 836: #ifdef HT_MEMLOG
1.44 eric 837: HTMemLog_flush();
1.47 frystyk 838: #endif
839:
1.1 frystyk 840: }
841: #endif /* CATCH_SIG */
842:
1.58 frystyk 843: #ifdef HT_POSIX_REGEX
844: PRIVATE char * get_regerror (int errcode, regex_t * compiled)
845: {
846: size_t length = regerror (errcode, compiled, NULL, 0);
847: char * str = NULL;
848: if ((str = (char *) HT_MALLOC(length+1)) == NULL)
849: HT_OUTOFMEM("get_regerror");
850: (void) regerror (errcode, compiled, str, length);
851: return str;
852: }
853:
1.75 frystyk 854: PUBLIC regex_t * get_regtype (Robot * mr, const char * regex_str, int cflags)
1.58 frystyk 855: {
856: regex_t * regex = NULL;
857: if (regex_str && *regex_str) {
858: int status;
859: if ((regex = (regex_t *) HT_CALLOC(1, sizeof(regex_t))) == NULL)
860: HT_OUTOFMEM("get_regtype");
1.60 frystyk 861: if ((status = regcomp(regex, regex_str, cflags))) {
1.58 frystyk 862: char * err_msg = get_regerror(status, regex);
1.62 frystyk 863: if (SHOW_REAL_QUIET(mr))
864: HTTrace("Regular expression error: %s\n", err_msg);
1.58 frystyk 865: HT_FREE(err_msg);
866: Cleanup(mr, -1);
867: }
868: }
869: return regex;
870: }
871: #endif
872:
1.75 frystyk 873: PUBLIC void VersionInfo (void)
1.1 frystyk 874: {
1.75 frystyk 875: OutputData("\nW3C OpenSource Software");
876: OutputData("\n-----------------------\n\n");
877: OutputData("\tWebbot version %s\n", APP_VERSION);
878: OutputData("\tusing the W3C libwww library version %s.\n\n",HTLib_version());
879: OutputData("\tSee \"%s\" for help\n", COMMAND_LINE);
880: OutputData("\tSee \"http://www.w3.org/Robot/User/\" for user information\n");
881: OutputData("\tSee \"http://www.w3.org/Robot/\" for general information\n\n");
882: OutputData("\tPlease send feedback to the <www-lib@w3.org> mailing list,\n");
883: OutputData("\tsee \"http://www.w3.org/Library/#Forums\" for details\n\n");
1.1 frystyk 884: }
885:
886: /* terminate_handler
887: ** -----------------
1.2 frystyk 888: ** This function is registered to handle the result of the request.
889: ** If no more requests are pending then terminate program
1.1 frystyk 890: */
1.75 frystyk 891: PUBLIC int terminate_handler (HTRequest * request, HTResponse * response,
1.32 frystyk 892: void * param, int status)
1.1 frystyk 893: {
1.34 eric 894: Finger * finger = (Finger *) HTRequest_context(request);
1.46 eric 895: Robot * mr = finger->robot;
1.62 frystyk 896: if (SHOW_QUIET(mr)) HTTrace("Robot....... done with %s\n", HTAnchor_physical(finger->dest));
1.55 frystyk 897:
1.68 frystyk 898: #ifdef HT_MYSQL
899: if (mr->sqllog) HTSQLLog_addEntry(mr->sqllog, request, status);
900: #endif
901:
1.58 frystyk 902: /* Check if negotiated resource and whether we should log that*/
903: if (mr->conneg) {
904: HTAssocList * cur = HTResponse_variant(response);
905: if (cur) {
906: BOOL first = YES;
907: HTChunk * buffer = HTChunk_new(128);
908: char * uri = HTAnchor_address((HTAnchor *) finger->dest);
909: HTAssoc * pres;
1.60 frystyk 910: HTChunk_puts(buffer, uri);
1.58 frystyk 911: while ((pres = (HTAssoc *) HTAssocList_nextObject(cur))) {
912: char * value = HTAssoc_value(pres);
913: if (first) {
1.60 frystyk 914: HTChunk_puts(buffer, "\t(");
1.58 frystyk 915: first = NO;
916: } else
917: HTChunk_puts(buffer, ", ");
918:
919: /* Output the name */
920: HTChunk_puts(buffer, HTAssoc_name(pres));
921:
922: /* Only output the value if not empty string */
1.60 frystyk 923: if (value && *value) {
1.58 frystyk 924: HTChunk_puts(buffer, "=");
925: HTChunk_puts(buffer, value);
926: }
927: }
1.60 frystyk 928: if (!first) HTChunk_puts(buffer, ")");
929: HTLog_addLine(mr->conneg, HTChunk_data(buffer));
1.58 frystyk 930: HTChunk_delete(buffer);
931: HT_FREE(uri);
932: }
933: }
934:
1.55 frystyk 935: /* Count the amount of body data that we have read */
1.59 frystyk 936: if (HTRequest_method(request) == METHOD_GET) {
937: int length = HTAnchor_length(HTRequest_anchor(request));
938: if (length > 0) mr->get_bytes += length;
939: mr->get_docs++;
940: } else if (HTRequest_method(request) == METHOD_HEAD) {
1.56 frystyk 941: int length = HTAnchor_length(HTRequest_anchor(request));
1.59 frystyk 942: if (length > 0) mr->head_bytes += length;
943: mr->head_docs++;
944: } else {
945: mr->other_docs++;
1.55 frystyk 946: }
947:
1.75 frystyk 948: if (SHOW_QUIET(mr)) HTTrace(" %d outstanding request%s\n", mr->cnt, mr->cnt == 1 ? "" : "s");
949: return HT_OK;
950:
951: }
952: PUBLIC int my_terminate_handler (HTRequest * request, HTResponse * response,
953: void * param, int status)
954: {
955: Finger * finger = (Finger *) HTRequest_context(request);
956: Robot * mr = finger->robot;
957: HTParentAnchor * dest = finger->dest;
958: HyperDoc * hd = HTAnchor_document(dest);
959: int depth = (hd ? hd->depth : -1);
960:
961: if (hd) set_error_state_hyperdoc(hd,request);
962:
963: if(hd && (HTRequest_method(request)== METHOD_HEAD) &&
964: (depth < mr->depth))
965: {
966: hd->method = METHOD_GET;
967: HTQueue_append(mr->queue, (void *)hd); (mr->cq)++;
968: }
1.58 frystyk 969:
1.34 eric 970: Finger_delete(finger);
1.55 frystyk 971:
1.75 frystyk 972: if(!(mr->flags & MR_PREEMPTIVE))
973: Serving_queue(mr);
974:
975: return HT_OK;
976: }
977:
978:
979: PUBLIC void Serving_queue(Robot *mr)
980: {
981: BOOL abort = NO;
982: Finger *nfinger;
983:
984: while(!abort)
985: {
986: if(!HTQueue_isEmpty(mr->queue))
987: {
988: HTRequest *newreq;
989:
990: HyperDoc *nhd = (HyperDoc *)HTQueue_headOfQueue(mr->queue);
991:
992: if(nhd)
993: {
994: char *uri = HTAnchor_address((HTAnchor *)nhd->anchor);
995: HTQueue_dequeue(mr->queue); (mr->cq)--;
996:
997: nfinger = Finger_new(mr, nhd->anchor, nhd->method);
998:
999: newreq = nfinger->request;
1000:
1001: if(SHOW_QUIET(mr)) HTTrace("Request from QUEUE %s\n",uri);
1002: HT_FREE(uri);
1003: if(SHOW_QUIET(mr)) HTTrace("%d elements in queue \n", mr->cq);
1004:
1005: HTRequest_setParent(newreq,get_last_parent(nhd->anchor));
1006:
1.76 frystyk 1007: /* @@@ Should be done using a timer and not sleep! @@@ */
1008: #if 0
1.75 frystyk 1009: if(mr->waits)
1010: sleep(mr->waits);
1.76 frystyk 1011: #endif
1.75 frystyk 1012:
1013: if (HTLoadAnchor((HTAnchor *)nhd->anchor , newreq) != YES)
1014: {
1015: if (SHOW_QUIET(mr)) HTTrace("not tested!\n");
1016: Finger_delete(nfinger);
1017: }
1018: }
1019: else
1020: abort = YES;
1021: }
1022: else
1023: abort = YES;
1024: }
1025:
1026: if(SHOW_QUIET(mr)) HTTrace("Queue size: %d \n", mr->cq);
1027:
1028: if (mr->cnt <= 0 || (abort && (mr->flags & MR_PREEMPTIVE)))
1029: {
1030: if(mr->cnt > 0)
1031: if(SHOW_QUIET(mr)) HTTrace("%d requests were not served\n", mr->cnt);
1032:
1.62 frystyk 1033: if (SHOW_QUIET(mr)) HTTrace(" Everything is finished...\n");
1.46 eric 1034: Cleanup(mr, 0); /* No way back from here */
1.75 frystyk 1035: }
1.1 frystyk 1036: }
1037:
1038: /* ------------------------------------------------------------------------- */
1039: /* HTEXT INTERFACE */
1040: /* ------------------------------------------------------------------------- */
1041:
1042: PUBLIC HText * HText_new2 (HTRequest * request, HTParentAnchor * anchor,
1043: HTStream * stream)
1044: {
1045: HText * me;
1.34 eric 1046: Finger * finger = (Finger *) HTRequest_context(request);
1047: Robot * mr = finger->robot;
1.65 frystyk 1048: char * robots = NULL;
1049:
1.14 frystyk 1050: if ((me = (HText *) HT_CALLOC(1, sizeof(HText))) == NULL)
1051: HT_OUTOFMEM("HText_new2");
1.4 frystyk 1052:
1053: /* Bind the HText object together with the Request Object */
1.1 frystyk 1054: me->request = request;
1.65 frystyk 1055: me->follow = YES;
1056:
1057: /* Check to see if we have any meta tags */
1.77 ! frystyk 1058: if (!(mr->flags & MR_NOMETATAGS) && (robots = HTAnchor_robots(anchor)) != NULL) {
1.65 frystyk 1059: char * strval = NULL;
1060: char * ptr = NULL;
1061: char * token = NULL;
1062: StrAllocCopy(strval, robots);
1063: ptr = strval;
1064: while ((token = HTNextField(&ptr)) != NULL) {
1065: if (!strcasecomp(token, "nofollow")) {
1066: me->follow = NO;
1067: break;
1068: }
1069: }
1070: HT_FREE(strval);
1071: }
1.4 frystyk 1072:
1073: /* Add this HyperDoc object to our list */
1074: if (!mr->htext) mr->htext = HTList_new();
1075: HTList_addObject(mr->htext, (void *) me);
1.1 frystyk 1076: return me;
1077: }
1078:
1.4 frystyk 1079: PUBLIC void HText_free (HText * me) {
1.11 frystyk 1080: if (me) HT_FREE (me);
1.4 frystyk 1081: }
1082:
1.1 frystyk 1083: PUBLIC void HText_beginAnchor (HText * text, HTChildAnchor * anchor)
1084: {
1085: if (text && anchor) {
1.34 eric 1086: Finger * finger = (Finger *) HTRequest_context(text->request);
1087: Robot * mr = finger->robot;
1.1 frystyk 1088: HTAnchor * dest = HTAnchor_followMainLink((HTAnchor *) anchor);
1089: HTParentAnchor * dest_parent = HTAnchor_parent(dest);
1.7 frystyk 1090: char * uri = HTAnchor_address((HTAnchor *) dest_parent);
1.1 frystyk 1091: HyperDoc * hd = HTAnchor_document(dest_parent);
1.60 frystyk 1092: HTParentAnchor * referer = HTRequest_anchor(text->request);
1.65 frystyk 1093: BOOL match = text->follow;
1.58 frystyk 1094: BOOL check = NO;
1.1 frystyk 1095:
1.75 frystyk 1096: /* These are new variables */
1097: HyperDoc * nhd = NULL;
1098: BOOL follow = YES;
1099:
1100: /* These three variables were moved */
1101: /*HTParentAnchor * last_anchor = HTRequest_parent(text->request);*/
1102: HTParentAnchor * last_anchor = HTRequest_anchor(text->request);
1103: HyperDoc * last_doc = HTAnchor_document(last_anchor);
1104: int depth = last_doc ? last_doc->depth+1 : 0;
1105:
1.55 frystyk 1106: if (!uri) return;
1.75 frystyk 1107: if (SHOW_QUIET(mr)) HTTrace("Robot....... Found `%s\' - \n", uri ? uri : "NULL\n");
1.55 frystyk 1108:
1109: if (hd) {
1.75 frystyk 1110: if (SHOW_QUIET(mr)) HTTrace("............ Already checked\n");
1.55 frystyk 1111: hd->hits++;
1.68 frystyk 1112: #ifdef HT_MYSQL
1113: if (mr->sqllog) {
1114: char * ref_addr = HTAnchor_address((HTAnchor *) referer);
1115: if (ref_addr) {
1116: HTSQLLog_addLinkRelationship(mr->sqllog, ref_addr, uri,
1117: "referer", NULL);
1118: HT_FREE(ref_addr);
1119: }
1120: }
1121: #endif
1.58 frystyk 1122: HT_FREE(uri);
1123: return;
1124: }
1.70 frystyk 1125:
1.58 frystyk 1126: /* Check for prefix match */
1.65 frystyk 1127: if (match && mr->prefix) {
1128: match = HTStrMatch(mr->prefix, uri) ? YES : NO;
1129: }
1.58 frystyk 1130:
1131: #ifdef HT_POSIX_REGEX
1.69 frystyk 1132: /*
1133: ** Check for any regular expression. The include may override
1134: ** the prefix matching
1135: */
1136: if (mr->include) {
1.58 frystyk 1137: match = regexec(mr->include, uri, 0, NULL, 0) ? NO : YES;
1138: }
1.75 frystyk 1139: if (match && mr->exc_robot) {
1140: match = regexec(mr->exc_robot, uri, 0, NULL, 0) ? YES : NO;
1141: }
1.58 frystyk 1142: if (match && mr->exclude) {
1143: match = regexec(mr->exclude, uri, 0, NULL, 0) ? YES : NO;
1144: }
1145: if (match && mr->check) {
1146: check = regexec(mr->check, uri, 0, NULL, 0) ? NO : YES;
1147: }
1148: #endif
1.75 frystyk 1149: if(uri && test_for_blank_spaces(uri))
1150: follow = NO;
1151: else if (mr->ndoc == 0) /* Number of Documents is reached */
1152: follow = NO;
1153:
1154: /* Test whether we already have a hyperdoc for this document */
1155: if(!hd && dest_parent)
1156: {
1157: nhd = HyperDoc_new(mr, dest_parent, depth);
1158: mr->cdepth[depth]++;
1159: }
1.58 frystyk 1160:
1161: /* Test whether we already have a hyperdoc for this document */
1.75 frystyk 1162: if (mr->flags & MR_LINK && match && dest_parent
1163: && follow && !hd ) {
1164:
1165: nhd->method = METHOD_HEAD;
1166: HTQueue_enqueue(mr->queue, (void *)nhd); (mr->cq)++;
1167: if(mr->ndoc > 0) mr->ndoc--;
1168:
1.7 frystyk 1169: } else {
1.75 frystyk 1170: if (SHOW_QUIET(mr)) HTTrace("............ does not fulfill constraints\n");
1.68 frystyk 1171: #ifdef HT_MYSQL
1172: if (mr->reject || mr->sqllog) {
1173: #else
1.60 frystyk 1174: if (mr->reject) {
1.68 frystyk 1175: #endif
1.60 frystyk 1176: if (referer) {
1177: char * ref_addr = HTAnchor_address((HTAnchor *) referer);
1.68 frystyk 1178: if (mr->reject && ref_addr)
1179: HTLog_addText(mr->reject, "%s --> %s\n", ref_addr, uri);
1180: #ifdef HT_MYSQL
1181: if (mr->sqllog && mr->sqlexternals && ref_addr)
1182: HTSQLLog_addLinkRelationship(mr->sqllog,
1183: ref_addr, uri,
1184: "referer", NULL);
1185: #endif
1186:
1.60 frystyk 1187: HT_FREE(ref_addr);
1188: }
1189: }
1.2 frystyk 1190: }
1.11 frystyk 1191: HT_FREE(uri);
1.2 frystyk 1192: }
1193: }
1194:
1195: PUBLIC void HText_appendImage (HText * text, HTChildAnchor * anchor,
1.14 frystyk 1196: const char *alt, const char * align, BOOL isMap)
1.2 frystyk 1197: {
1198: if (text && anchor) {
1.34 eric 1199: Finger * finger = (Finger *) HTRequest_context(text->request);
1200: Robot * mr = finger->robot;
1.75 frystyk 1201:
1.59 frystyk 1202: if (mr->flags & MR_IMG) {
1.60 frystyk 1203: HTAnchor * dest = HTAnchor_followMainLink((HTAnchor *) anchor);
1204: HTParentAnchor * dest_parent = HTAnchor_parent(dest);
1205: char * uri = HTAnchor_address((HTAnchor *) dest_parent);
1206: HyperDoc * hd = HTAnchor_document(dest_parent);
1207: HTParentAnchor * referer = HTRequest_anchor(text->request);
1.59 frystyk 1208: BOOL match = YES;
1209:
1.72 frystyk 1210: if (!uri) return;
1.59 frystyk 1211: if (hd) {
1.75 frystyk 1212: if (SHOW_QUIET(mr)) HTTrace("............ Already checked\n");
1.59 frystyk 1213: hd->hits++;
1.68 frystyk 1214: #ifdef HT_MYSQL
1215: if (mr->sqllog) {
1216: char * ref_addr = HTAnchor_address((HTAnchor *) referer);
1217: if (ref_addr) {
1218: HTSQLLog_addLinkRelationship(mr->sqllog,
1219: ref_addr, uri,
1220: "image", alt);
1221: HT_FREE(ref_addr);
1222: }
1223: }
1224: #endif
1.11 frystyk 1225: HT_FREE(uri);
1.59 frystyk 1226: return;
1.2 frystyk 1227: }
1.59 frystyk 1228:
1229: /* Check for prefix match */
1230: if (mr->img_prefix) match = HTStrMatch(mr->img_prefix, uri) ? YES : NO;
1231:
1232: /* Test whether we already have a hyperdoc for this document */
1233: if (match && dest) {
1.60 frystyk 1234: Finger * newfinger = Finger_new(mr, dest_parent,
1.59 frystyk 1235: mr->flags & MR_SAVE ?
1236: METHOD_GET : METHOD_HEAD);
1237: HTRequest * newreq = newfinger->request;
1.60 frystyk 1238: HyperDoc_new(mr, dest_parent, 1);
1239: HTRequest_setParent(newreq, referer);
1240:
1241: /* Check whether we should report missing ALT tags */
1242: if (mr->noalttag && (alt==NULL || *alt=='\0')) {
1243: if (referer) {
1244: char * ref_addr = HTAnchor_address((HTAnchor *) referer);
1245: if (ref_addr) HTLog_addText(mr->noalttag, "%s --> %s\n", ref_addr, uri);
1246: HT_FREE(ref_addr);
1247: }
1248: }
1249:
1.62 frystyk 1250: if (SHOW_QUIET(mr)) HTTrace("Robot....... Checking Image `%s\'\n", uri);
1.59 frystyk 1251: if (HTLoadAnchor((HTAnchor *) dest, newreq) != YES) {
1.62 frystyk 1252: if (SHOW_QUIET(mr)) HTTrace("Robot....... Image not tested!\n");
1.59 frystyk 1253: Finger_delete(newfinger);
1254: }
1255: } else {
1.75 frystyk 1256: if (SHOW_QUIET(mr)) HTTrace("............ does not fulfill constraints\n");
1.68 frystyk 1257: #ifdef HT_MYSQL
1258: if (mr->reject || mr->sqllog) {
1259: #else
1.60 frystyk 1260: if (mr->reject) {
1.68 frystyk 1261: #endif
1.60 frystyk 1262: if (referer) {
1263: char * ref_addr = HTAnchor_address((HTAnchor *) referer);
1.68 frystyk 1264: if (mr->reject && ref_addr)
1265: HTLog_addText(mr->reject, "%s --> %s\n", ref_addr, uri);
1266: #ifdef HT_MYSQL
1267: if (mr->sqllog && mr->sqlexternals && ref_addr)
1268: HTSQLLog_addLinkRelationship(mr->sqllog,
1269: ref_addr, uri,
1270: "image", alt);
1271: #endif
1272:
1.60 frystyk 1273: HT_FREE(ref_addr);
1274: }
1275: }
1.1 frystyk 1276: }
1.59 frystyk 1277: HT_FREE(uri);
1.72 frystyk 1278: }
1279: }
1280: }
1281:
1282: PUBLIC void HText_appendLink (HText * text, HTChildAnchor * anchor,
1283: const BOOL * present, const char ** value)
1284: {
1285: if (text && anchor) {
1286: Finger * finger = (Finger *) HTRequest_context(text->request);
1287: Robot * mr = finger->robot;
1288: if (SHOW_QUIET(mr))
1289: HTTrace("Robot....... Received Link element with anchor %p\n", anchor);
1290: HText_beginAnchor(text, anchor);
1291: }
1292: }
1293:
1294: PUBLIC void HText_appendObject (HText * text, int element_number,
1295: const BOOL * present, const char ** value)
1296: {
1297: /* Here we can look for frames, link tags, meta tags etc. */
1298: if (text && text->request) {
1299: Finger * finger = (Finger *) HTRequest_context(text->request);
1300: Robot * mr = finger->robot;
1301:
1302: if (SHOW_QUIET(mr))
1303: HTTrace("Robot....... HText Object %p called with HTML element number %d\n",
1304: text, element_number);
1305:
1306: switch (element_number) {
1307:
1308: case HTML_FRAME:
1309: {
1310: HTChildAnchor * source = HTAnchor_findChildAndLink(
1311: HTRequest_anchor(text->request), /* Parent */
1312: NULL, /* Tag */
1313: present[HTML_FRAME_SRC] ? value[HTML_FRAME_SRC] : NULL, /* Addresss */
1314: NULL); /* Rels */
1315: HText_beginAnchor(text, source);
1316: }
1317: break;
1318:
1319: case HTML_BODY:
1320: {
1321: HTChildAnchor * source = HTAnchor_findChildAndLink(
1322: HTRequest_anchor(text->request), /* Parent */
1323: NULL, /* Tag */
1324: present[HTML_BODY_BACKGROUND] ? value[HTML_BODY_BACKGROUND] : NULL, /* Addresss */
1325: NULL); /* Rels */
1326: HText_appendImage(text, source, NULL, NULL, NO);
1327: }
1328: break;
1329:
1330: default:
1331: break;
1.1 frystyk 1332: }
1333: }
1334: }
1335:
1336: PUBLIC void HText_endAnchor (HText * text) {}
1.14 frystyk 1337: PUBLIC void HText_appendText (HText * text, const char * str) {}
1.1 frystyk 1338: PUBLIC void HText_appendCharacter (HText * text, char ch) {}
1339: PUBLIC void HText_endAppend (HText * text) {}
1340: PUBLIC void HText_setStyle (HText * text, HTStyle * style) {}
1341: PUBLIC void HText_beginAppend (HText * text) {}
1342: PUBLIC void HText_appendParagraph (HText * text) {}
1343:
1.75 frystyk 1344:
1345: PUBLIC char *
1346: get_robots_txt(char *uri)
1.48 frystyk 1347: {
1.75 frystyk 1348: char *str = NULL;
1349: HTChunk * chunk;
1350: HTParentAnchor *anchor = HTAnchor_parent(HTAnchor_findAddress(uri));
1351: HTRequest *request = HTRequest_new();
1352: HTRequest_setOutputFormat(request, WWW_SOURCE);
1353: HTRequest_setPreemptive(request, YES);
1354: HTRequest_setMethod(request, METHOD_GET);
1355: chunk = HTLoadAnchorToChunk ((HTAnchor *)anchor, request);
1356: str = HTChunk_toCString(chunk);
1357: HTRequest_delete(request);
1358: return str;
1.48 frystyk 1359: }
1360:
Webmaster
![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to HTRobot.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Public] / libwww / Robot / src