Subversion Repositories wimsdev

Rev

Rev 15379 | Rev 15393 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | RSS feed

  1. /*    Copyright (C) 1998-2003 XIAO, Gang of Universite de Nice - Sophia Antipolis
  2.  *
  3.  *  This program is free software; you can redistribute it and/or modify
  4.  *  it under the terms of the GNU General Public License as published by
  5.  *  the Free Software Foundation; either version 2 of the License, or
  6.  *  (at your option) any later version.
  7.  *
  8.  *  This program is distributed in the hope that it will be useful,
  9.  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  10.  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11.  *  GNU General Public License for more details.
  12.  *
  13.  *  You should have received a copy of the GNU General Public License
  14.  *  along with this program; if not, write to the Free Software
  15.  *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  16.  */
  17.  
  18. /*  This is an internal program,
  19.  * used to index modules for search engine.
  20.  */
  21.  
  22. #include "../Lib/libwims.h"
  23. #include "translator_.h"
  24. #include "suffix.h"
  25.  
  26. #define MAX_LANGS    MAX_LANGUAGES
  27. #define MAX_MODULES    65536
  28. char *moduledir=    "public_html/modules";
  29. char *sheetdir=     "public_html/bases/sheet";
  30. char *glossarydir=  "public_html/scripts/data/glossary";
  31. char *dicdir=       "public_html/bases";
  32. char *outdir=       "public_html/bases/site2";
  33. char *sheetoutdir=  "public_html/bases/sheet/index";
  34. char *glossaryoutdir=  "public_html/scripts/data/glossary/index";
  35. char *maindic=      "sys/words";
  36. char *groupdic=     "sys/wgrp/wgrp";
  37. char *suffixdic=    "sys/suffix";
  38. char *domaindic=    "sys/domaindic";
  39. char *ignoredic=    "sys/indignore";
  40. char *conffile=     "log/wims.conf";
  41. char *mlistbase=    "lists";
  42.  
  43. char lang[MAX_LANGS][4]={
  44.     "en","fr","cn","es","it","nl","si","ca","pt"
  45. };
  46. #define DEFAULT_LANGCNT    6
  47. char allang[MAX_LANGS][4]={
  48.     "en","fr","cn","es","it","nl","de","si","ca","pt"
  49. };
  50. #define allangcnt 8
  51. char ignore[MAX_LANGS][MAX_LINELEN+1];
  52. char mlistfile[MAX_LANGS][256];
  53. int langcnt;
  54. FILE *langf, *titf, *descf, *weightf, *robotf, *indf, *listf, *addrf, *serialf, *authorf, *versionf, *remf, *titlef;
  55.  
  56. struct cat {
  57.     char *name;
  58.     char typ;
  59. } cat[]={
  60.     {"all_types", 'A'},
  61.     {"exercise",  'X'},
  62.     {"oef",       'O'},
  63.     {"tool",      'T'},
  64.     {"recreation",'R'},
  65.     {"reference", 'Y'},
  66.     {"document",  'D'},
  67.     {"popup",     'P'},
  68.     {"datamodule",'M'}
  69. };
  70. #define catno (sizeof(cat)/sizeof(cat[0]))
  71.  
  72. struct mod {
  73.     char *name;
  74.     unsigned char langs[MAX_LANGS];
  75.     int counts[MAX_LANGS];
  76.     int  langcnt;
  77. } mod[MAX_MODULES];
  78. int modcnt;
  79.  
  80. char *mlist;
  81.  
  82. /*  fold known accented letters to unaccented, other strange characters to space
  83.  *  apostrophe is among the exceptions to be kept (important for multi-word expressions)
  84.  */
  85. void deaccent2(char *p)
  86. {
  87.   char *sp;
  88.   char *v;
  89.   for(sp=p;*sp;sp++) {
  90.   if(*sp<0 && (v=strchr(acctab,*sp))!=NULL)
  91.     *sp=*(deatab+(v-acctab));
  92.   if(!isalnum(*sp) && strchr(",.&$+*",*sp)==0) *sp=' ';
  93.   else *sp=tolower(*sp);
  94.   }
  95. }
  96.  
  97. /*  translate everything non-alphanumeric into space */
  98. void towords(char *p)
  99. {
  100.   char *pp;
  101.   for(pp=p;*pp;pp++) if(!isalnum(*pp) && strchr("&$+*",*pp)==0) *pp=' ';
  102. }
  103.  
  104. /*  Find first occurrence of word */
  105. char *wordchr2(char *p, char *w)
  106. {
  107.   char *r;
  108.  
  109.   for(r=strstr(p,w);r!=NULL &&
  110.     ( (r>p && !isspace(*(r-1))) || (!isspace(*(r+strlen(w))) && *(r+strlen(w))!=0) );
  111.   r=strstr(r+1,w));
  112.   return r;
  113. }
  114.  
  115. char *find_tag_end(char *p)
  116. {
  117.   char *pp;
  118.   pp=p; if(*pp=='<') pp++;
  119.   for(; *pp && *pp!='>'; pp++) {
  120.     if(*pp=='<') {
  121.       pp=find_tag_end(pp)-1; continue;
  122.     }
  123.     if(*pp=='"') {
  124.       pp=strchr(pp+1,'"');
  125.       if(pp==NULL) return p+strlen(p); else continue;
  126.     }
  127.     if(*pp=='\'') {
  128.       pp=strchr(pp+1,'\'');
  129.       if(pp==NULL) return p+strlen(p); else continue;
  130.     }
  131.   }
  132.   if(*pp=='>') pp++;
  133.   return pp;
  134. }
  135.  
  136. char *find_tag(char *p, char *tag)
  137. {
  138.   char *pp;
  139.   int len;
  140.   len=strlen(tag);
  141.   for(pp=strchr(p,'<'); pp!=NULL && *pp; pp=strchr(pp+1,'<')) {
  142.     if(strncasecmp(pp+1,tag,len)==0 && !isalnum(*(pp+1+len))) return pp;
  143.   }
  144.   return p+strlen(p);
  145. }
  146.  
  147. /*  remove all html tags */
  148. void detag(char *p)
  149. {
  150.   char *pp, *p2;
  151.   for(pp=strchr(p,'<'); pp!=NULL; pp=strchr(pp,'<')) {
  152.     p2=find_tag_end(pp);
  153.     if(*p2==0) {*pp=0; return; }
  154.     ovlstrcpy(pp,p2);
  155.   }
  156. }
  157.  
  158. /* add a space after comma to see end of words */
  159.  
  160. void comma(char *p)
  161. {
  162.   char *pp;
  163.   for(pp=strchr(p,','); pp; pp=strchr(pp+1,','))
  164.     string_modify3(p,pp,pp+1,", ");
  165. }
  166.  
  167. /* _getdef from lines.c except the error msg*/
  168. void _getdef(char buf[], char *name, char value[])
  169. {
  170.   char *p1, *p2, *p3, *p4;
  171.  
  172.   if(*name==0) goto nothing;      /* this would create segfault. */
  173.   for(p1=strstr(buf,name); p1!=NULL; p1=strstr(p1+1,name)) {
  174.     p2=find_word_start(p1+strlen(name));
  175.     if((p1>buf && !isspace(*(p1-1))) || *p2!='=') continue;
  176.     p3=p1; while(p3>buf && *(p3-1)!='\n') p3--;
  177.     p3=find_word_start(p3);
  178.     if(p3<p1 && *p3!='!') continue;
  179.     if(p3<p1) {
  180.       p3++; p4=find_word_end(p3);
  181.       if(find_word_start(p4)!=p1) continue;
  182.       if(p4-p3!=3 || (strncmp(p3,"set",3)!=0 &&
  183.            strncmp(p3,"let",3)!=0 &&
  184.            strncmp(p3,"def",3)!=0)) {
  185.         if(p4-p3!=6 || strncmp(p3,"define",6)!=0) continue;
  186.       }
  187.     }
  188.     p2++;p3=strchr(p2,'\n'); if(p3==NULL) p3=p2+strlen(p2);
  189.     p2=find_word_start(p2);
  190.     if(p2>p3) goto nothing;
  191.     /*if(p3-p2>=MAX_LINELEN) user_error("cmd_output_too_long");*/
  192.     memmove(value,p2,p3-p2); value[p3-p2]=0;
  193.     strip_trailing_spaces(value); return;
  194.   }
  195. nothing:
  196.   value[0]=0; return;
  197. }
  198.  
  199. /*  Get variable definition from a file.
  200.  * Result stored in buffer value of length MAX_LINELEN.
  201.  */
  202. void getdef(char *fname, char *name, char value[])
  203. {
  204.   FILE *f;
  205.   char *buf;
  206.   int l;
  207.  
  208.   value[0]=0;
  209.   f=fopen(fname,"r"); if(f==NULL) return;
  210.   fseek(f,0,SEEK_END); l=ftell(f); fseek(f,0,SEEK_SET);
  211.   buf=xmalloc(l+256); l=fread(buf,1,l,f);
  212.   fclose(f);
  213.   if(l<=0) return; else buf[l]=0;
  214.   _getdef(buf,name,value);
  215.   free(buf);
  216. }
  217.  
  218. char *mdicbuf, *gdicbuf, *ddicbuf, *gentry, *mentry, *dentry;
  219.  
  220. int gentrycount, mentrycount, dentrycount;
  221.  
  222. /*  Preparation of data */
  223. void prep(void)
  224. {
  225.   char buf[MAX_LINELEN+1];
  226.   char *p1,*p2,*s,*old;
  227.   int i,l,thislang,t;
  228.   FILE *f;
  229.  
  230.   s=getenv("modind_outdir"); if(s!=NULL && *s!=0) outdir=s;
  231.   s=getenv("modind_sheetdir"); if(s!=NULL && *s!=0) sheetdir=s;
  232.   s=getenv("modind_sheetoutdir"); if(s!=NULL && *s!=0) sheetoutdir=s;
  233.   snprintf(buf,sizeof(buf),"%s/addr",outdir);
  234.   addrf=fopen(buf,"w");
  235.   if(!addrf) { fprintf(stderr,"modind: error creating output files addr.\n"); exit(1);}
  236.   snprintf(buf,sizeof(buf),"%s/serial",outdir);
  237.   serialf=fopen(buf,"w");
  238.   if(!serialf) { fprintf(stderr,"modind: error creating output files serial.\n"); exit(1);}
  239.   modcnt=langcnt=0;
  240. /* take the langs declared in conffile */
  241.   getdef(conffile,"site_languages",buf);
  242.   for(p1=buf;*p1;p1++) if(!isalnum(*p1)) *p1=' ';
  243.   for(p1=find_word_start(buf); *p1 && langcnt<MAX_LANGS; p1=find_word_start(p2)) {
  244.     p2=find_word_end(p1);
  245.     if(p2!=p1+2 || !isalpha(*p1) || !isalpha(*(p1+1))) continue;
  246.     memmove(lang[langcnt],p1,2); lang[langcnt++][2]=0;
  247.   }
  248.   if(langcnt==0) {/*  default languages */
  249.     langcnt=DEFAULT_LANGCNT;
  250.   }
  251.   s=getenv("mlist"); if(s==NULL) exit(1);
  252.   l=strlen(s); if(l<0 || l>100*MAX_LINELEN) exit(1);
  253.   mlist=xmalloc(l+16); ovlstrcpy(mlist,s); old="";
  254.   for(i=0;i<langcnt;i++) {
  255.     snprintf(buf,sizeof(buf),"%s/%s.%s",dicdir,ignoredic,lang[i]);
  256.     f=fopen(buf,"r"); if(f==NULL) continue;
  257.     l=fread(ignore[i],1,MAX_LINELEN,f);fclose(f);
  258.     if(l<0 || l>=MAX_LINELEN) l=0;
  259.     ignore[i][l]=0;
  260.   }
  261.   for(t=0, p1=find_word_start(mlist); *p1 && modcnt<MAX_MODULES;
  262.         p1=find_word_start(p2), t++) {
  263.     p2=find_word_end(p1);
  264.     l=p2-p1; if(*p2) *p2++=0;
  265.     fprintf(addrf,"%d:%s\n",t,p1);
  266.     fprintf(serialf,"%s:%d\n",p1,t);
  267.     thislang=-1;
  268. /* language is taken from the address */
  269.     if(l>3 && p1[l-3]=='.') {
  270.       for(i=0;i<langcnt;i++) if(strcasecmp(lang[i],p1+l-2)==0) break;
  271.       if(i<langcnt) {p1[l-3]=0; thislang=i;}
  272.       else {/*  unknown language, not referenced */
  273.         continue;
  274.       }
  275.     }
  276.     if(modcnt>0 && strcmp(old,p1)==0 && thislang>=0) {
  277.       if(mod[modcnt-1].langcnt<langcnt) {
  278.         mod[modcnt-1].langs[mod[modcnt-1].langcnt]=thislang;
  279.         mod[modcnt-1].counts[mod[modcnt-1].langcnt]=t;
  280.         (mod[modcnt-1].langcnt)++;
  281.       }
  282.     }
  283.     else {
  284.       mod[modcnt].name=old=p1;
  285.       if(thislang>=0) {
  286.         mod[modcnt].langs[0]=thislang;
  287.         mod[modcnt].langcnt=1;
  288.       }
  289.       else mod[modcnt].langcnt=0;
  290.       mod[modcnt].counts[0]=t;
  291.       modcnt++;
  292.     }
  293.   }
  294.   snprintf(buf,sizeof(buf),"%s/language",outdir);
  295.   langf=fopen(buf,"w");
  296.   snprintf(buf,sizeof(buf),"%s/title",outdir);
  297.   titf=fopen(buf,"w");
  298.   snprintf(buf,sizeof(buf),"%s/description",outdir);
  299.   descf=fopen(buf,"w");
  300.   snprintf(buf,sizeof(buf),"%s/author",outdir);
  301.   authorf=fopen(buf,"w");
  302.   snprintf(buf,sizeof(buf),"%s/version",outdir);
  303.   versionf=fopen(buf,"w");
  304.   snprintf(buf,sizeof(buf),"%s/%s/robot.phtml",outdir,mlistbase);
  305.   robotf=fopen(buf,"w");
  306.   fclose(addrf); fclose(serialf);
  307.   if(!robotf || !versionf || !authorf || !descf || !titf || !langf) {
  308.     fprintf(stderr,"modind: error creating output files.\n");
  309.     exit(1);
  310.   }
  311. }
  312.  
  313. void sprep(void)
  314. {
  315.   char *p1,*p2,*s;
  316.   int i,l,thislang;
  317.  
  318.   modcnt=0;
  319.   s=getenv("slist"); if(s==NULL) return;
  320.   l=strlen(s); if(l<0 || l>100*MAX_LINELEN) return;
  321.   mlist=xmalloc(l+16); ovlstrcpy(mlist,s);
  322.   for(p1=find_word_start(mlist); *p1 && modcnt<MAX_MODULES; p1=find_word_start(p2)) {
  323.     p2=find_word_end(p1);
  324.     l=p2-p1; if(*p2) *p2++=0;
  325.     for(i=0;i<langcnt;i++) if(strncasecmp(lang[i],p1,2)==0) break;
  326.     if(i<langcnt) thislang=i; else continue;
  327.     mod[modcnt].name=p1;
  328.     mod[modcnt].langs[0]=thislang;
  329.     mod[modcnt].langcnt=1;
  330.     modcnt++;
  331.   }
  332. }
  333.  
  334. void gprep(void)
  335. {
  336.   char *p1,*p2,*s;
  337.   int l,i;
  338.   modcnt=0;
  339.   s=getenv("glist"); if(s==NULL) return;
  340.   l=strlen(s); if(l<0 || l>100*MAX_LINELEN) return;
  341.   mlist=xmalloc(l+16); ovlstrcpy(mlist,s);
  342.   for(p1=find_word_start(mlist); *p1 && modcnt<MAX_MODULES; p1=find_word_start(p2)) {
  343.     p2=find_word_end(p1);
  344.     if(*p2) *p2++=0;
  345.     s=strchr(p1,'/');
  346.     if(s != NULL) s=strchr(s+1,'/');
  347.     if(s==NULL) {
  348.       fprintf(stderr,"modind: no language %s\n",p1); exit(1);
  349.     }
  350.     s++;
  351.     for(i=0;i<langcnt;i++) if(strncasecmp(lang[i],s,2)==0) break;
  352.     if(i==langcnt) continue;
  353.     mod[modcnt].name=p1;
  354.     mod[modcnt].langs[0]=i;
  355.     mod[modcnt].langcnt=1;
  356.     modcnt++;
  357.   }
  358. }
  359.  
  360. void clean(void)
  361. {
  362.   fclose(langf); fclose(titf); fclose(descf); fclose(robotf);
  363.   fclose(authorf); fclose(versionf);
  364. }
  365.  
  366. char *sheetindex[]={
  367.   "title", "description",
  368.   "duration", "severity",
  369.   "level", "domain",
  370.   "keywords", "reserved1", "reserved2", "information"
  371. };
  372. /* correspond to the order of sheetindex */
  373. char *glindex[]={
  374.   "gl_title", "gl_description",
  375.   "", "",
  376.   "gl_level", "gl_domain",
  377.   "gl_keywords","","",""};
  378.  
  379. #define SHEETINDEX_NO (sizeof(sheetindex)/sizeof(sheetindex[0]))
  380. char gsindbuf[SHEETINDEX_NO][MAX_LINELEN+1];
  381.  
  382. /* do not modify the order, correspond to the order in the sheet file */
  383. enum{s_title, s_description,
  384.       s_duration, s_severity,
  385.       s_level, s_domain,
  386.       s_keywords, s_reserved1, s_reserved2,
  387.       s_information
  388. };
  389.  
  390. char *modindex[]={
  391.   "title", "description",
  392.   "author", "address", "copyright",
  393.   "version", "wims_version", "language",
  394.   "category", "level", "domain", "keywords",
  395.   "keywords_ca", "keywords_en", "keywords_fr", "keywords_it", "keywords_nl",
  396.   "title_ca", "title_en", "title_fr", "title_it", "title_nl",
  397.   "require"
  398. };
  399. #define MODINDEX_NO (sizeof(modindex)/sizeof(modindex[0]))
  400. char indbuf[MODINDEX_NO][MAX_LINELEN+1];
  401. enum{i_title, i_description,
  402.   i_author,i_address,i_copyright,
  403.   i_version,i_wims_version,i_language,
  404.   i_category,i_level,i_domain,i_keywords,
  405.   i_keywords_ca,i_keywords_en,i_keywords_fr,i_keywords_it,i_keywords_nl,
  406.   i_title_ca,i_title_en,i_title_fr,i_title_it,i_title_nl,
  407.   i_require
  408. };
  409.  
  410. char *module_special_file[]={
  411.   "intro","help","about"
  412. };
  413. #define MODSPEC_NO (sizeof(module_special_file)/sizeof(module_special_file[0]))
  414. char module_language[4];
  415.  
  416. /*  read and treat module's INDEX file */
  417. int module_index(const char *name)
  418. {
  419.   char *p, fbuf[MAX_LINELEN+1], ibuf[MAX_LINELEN+1];
  420.   FILE *indf;
  421.   int i,l;
  422.  
  423.   snprintf(fbuf,sizeof(fbuf),"%s/%s/INDEX",moduledir,name);
  424.   indf=fopen(fbuf,"r");
  425.   if(indf==NULL) {
  426.     fprintf(stderr,"modind: INDEX of %s not found\n",fbuf); return -1;
  427.   }
  428.   l=fread(ibuf,1,MAX_LINELEN,indf); fclose(indf);
  429.   if(l>0 && l<MAX_LINELEN) ibuf[l]=0; else return -1;
  430. /* treate all fields in *modindex */
  431.   for(i=0;i<MODINDEX_NO;i++) {
  432.     _getdef(ibuf,modindex[i],indbuf[i]);
  433. /*  compatibility precaution */
  434.     if(indbuf[i][0]==':') indbuf[i][0]='.';
  435.   }
  436.   p=find_word_start(indbuf[i_language]);
  437.   if(isalpha(*p) && isalpha(*(p+1))) {
  438.     memmove(module_language,p,2); module_language[2]=0;
  439.   }
  440.   else ovlstrcpy(module_language,"en");
  441.   return 0;
  442. }
  443.  
  444. int sheet_index(int serial)
  445. {
  446.   char *p1, *p2, fbuf[MAX_LINELEN+1], ibuf[MAX_LINELEN+1];
  447.   FILE *indf;
  448.   int i,l;
  449.  
  450.   snprintf(fbuf,sizeof(fbuf),"%s/%s.def",sheetdir,mod[serial].name);
  451.   indf=fopen(fbuf,"r"); if(indf==NULL) return -1;
  452.   l=fread(ibuf,1,MAX_LINELEN,indf); fclose(indf);
  453.   if(l>0 && l<MAX_LINELEN) ibuf[l]=0; else return -1;
  454.   for(i=0;i<SHEETINDEX_NO;i++) gsindbuf[i][0]=0;
  455.   for(i=0,p1=find_word_start(ibuf);
  456.       i<SHEETINDEX_NO-1 && *p1!=':' && *p1!=0;
  457.       i++,p1=p2) {
  458.     p2=strchr(p1,'\n');
  459.     if(p2!=NULL) *p2++=0; else p2=p1+strlen(p1);
  460.     p1=find_word_start(p1); strip_trailing_spaces2(p1);
  461.     snprintf(gsindbuf[i],MAX_LINELEN,"%s",p1);
  462.   }
  463.   p2=strstr(p1,"\n:"); if(p2==NULL) p2=p1+strlen(p1);
  464.   else *p2=0;
  465.   p1=find_word_start(p1); strip_trailing_spaces2(p1);
  466.   for(p2=p1;*p2;p2++) if(*p2=='\n') *p2=' ';
  467.   ovlstrcpy(gsindbuf[s_information],p1);
  468.   return 0;
  469. }
  470.  
  471. int glossary_index(int serial)
  472. {
  473.   char fbuf[MAX_LINELEN+1], ibuf[MAX_LINELEN+1];
  474.   FILE *indf;
  475.   int i,l;
  476.   snprintf(fbuf,sizeof(fbuf),"%s/%s",glossarydir,mod[serial].name);
  477.   indf=fopen(fbuf,"r");
  478.   l=fread(ibuf,1,MAX_LINELEN,indf); fclose(indf);
  479.   if(l>0 && l<MAX_LINELEN) ibuf[l]=0; else return -1;
  480.   for(i=0;i<SHEETINDEX_NO;i++) {
  481.     _getdef(ibuf,glindex[i],gsindbuf[i]);
  482.   }
  483.   return 0;
  484. }
  485.  
  486. unsigned char categories[16];
  487. char taken[MAX_LINELEN+1];
  488. int catcnt, takenlen, tweight;
  489.  
  490. void appenditem(char *word, int lind, int serial, int weight, char *l)
  491. {
  492.   char nbuf[MAX_LINELEN+1], buf[MAX_LINELEN+1];
  493.   int i, ll;
  494.   char *p;
  495.   FILE *f;
  496.  
  497.   if(!isalnum(*word) || (ll=strlen(word))<2 ||
  498.      wordchr2(taken,word)!=NULL ||
  499.      wordchr2(ignore[lind],word)!=NULL ||
  500.      takenlen>=MAX_LINELEN-ll-16)
  501.     return;
  502.   if(ll==2 && (!isdigit(word[0]) || !isalpha(word[1]))) return;
  503.   for(p=word;*p;p++) if(!isalnum(*p) && *p!=' ') return;
  504.   taken[takenlen++]=' '; taken[takenlen++]=' ';
  505.   ovlstrcpy(taken+takenlen,word);
  506.   takenlen+=ll; tweight+=weight;
  507.   snprintf(buf,sizeof(buf),"%s:%d?%d\n",word,serial,weight);
  508.   for(i=0;i<catcnt;i++) {
  509.     snprintf(nbuf,sizeof(nbuf),"%s/%c.%s",
  510.        outdir,categories[i],lang[lind]);
  511.     f=fopen(nbuf,"a");
  512.     if(f!=NULL) {fputs(buf,f); fclose(f);}
  513.   }
  514. }
  515.  
  516. void appenditem1 (char *buf, int lind, int serial, int weight, char *l )
  517. {
  518.   char *p1, *p2 ;
  519.   for(p1=find_word_start(buf); *p1;
  520.     p1=find_word_start(p2)) {
  521.     p2=strchr(p1,',');
  522.     if(p2!=NULL) *p2++=0; else p2=p1+strlen(p1);
  523.     if(strlen(p1)<=0) continue;
  524.     appenditem(p1,lind,serial,weight,module_language);
  525.   }
  526. }
  527. void appenditem2 (char *buf, int lind, int serial, int weight, char *l )
  528. {
  529.   char *p1, *p2 ;
  530.   for(p1=find_word_start(buf);*p1;
  531.       p1=find_word_start(p2)) {
  532.     p2=find_word_end(p1); if(*p2) *p2++=0;
  533.     appenditem(p1,lind,serial,weight,module_language);
  534.   }
  535. }
  536. void onemodule(const char *name, int serial, int lind)
  537. {
  538.   int i;
  539.   unsigned char trlist[]={
  540.   i_title,i_description,i_category,i_domain,i_keywords,
  541.   i_require,i_author,
  542.   i_keywords_ca,i_keywords_en,i_keywords_fr,i_keywords_it,i_keywords_nl,
  543.   i_title_ca,i_title_en,i_title_fr,i_title_it,i_title_nl
  544.   };
  545.   int trcnt=sizeof(trlist)/sizeof(trlist[0]);
  546.   char *p1, *p2, *pp, *q, buf[MAX_LINELEN+1], lbuf[16];
  547.   FILE *f;
  548.  
  549.   if(module_index(name)) return;
  550.   towords(indbuf[i_category]);
  551. /*   list the categories (among A=all,X=eXercise,O,D,...) corresponding
  552.  *   to this module
  553.  */
  554.   for(i=catcnt=0;i<catno && catcnt<16;i++) {
  555.     if(wordchr2(indbuf[i_category],cat[i].name)!=NULL)
  556.       categories[catcnt++]=cat[i].typ;
  557.   }
  558.   if(catcnt==0) return;
  559.   if(categories[0]!=cat[0].typ)
  560.     categories[catcnt++]=cat[0].typ;
  561. /*  write module's name in the category.language files, for instance lists/X.fr
  562.  * for french exercises
  563.  */
  564.   for(i=0;i<catcnt;i++) {
  565.     snprintf(buf,sizeof(buf),"%s/%s/%c.%s",
  566.        outdir,mlistbase,categories[i],lang[lind]);
  567.     f=fopen(buf,"a");
  568.     if(f!=NULL) {fprintf(f,"%s\n",name); fclose(f);}
  569.   }
  570. /*   add serial number and language (resp.title, ...) to corresponding file  */
  571.   fprintf(langf,"%d:%s\n",serial,module_language);
  572.   fprintf(titf,"%d:%s\n",serial,indbuf[i_title]);
  573.   fprintf(descf,"%d:%s\n",serial,indbuf[i_description]);
  574.   fprintf(authorf,"%d:%s\n",serial,indbuf[i_author]);
  575.   fprintf(versionf,"%d:%s\n",serial,indbuf[i_version]);
  576.  
  577. /*   add module's information in html page for robots  */
  578.   snprintf(buf,sizeof(buf),"%s",indbuf[i_description]);
  579.   for(pp=strchr(buf,','); pp; pp=strchr(pp,','))
  580.     string_modify3(buf,pp,pp+1,"&#44;");
  581.   if(strcmp(module_language,lang[lind])==0)
  582.     fprintf(robotf,"%s ,%s,%s,%s,%s\n",name,module_language,name,
  583.         indbuf[i_title], buf);
  584.  
  585. /*   Normalize the information of trlist, using dictionary
  586.  *  -- bases/sys/domain.xx without suffix translation (--> english version)
  587.  */
  588.   entrycount=dentrycount; dicbuf=ddicbuf;
  589.   memmove(entry,dentry,dentrycount*sizeof(entry[0]));
  590.   unknown_type=unk_leave;
  591.   for(i=0;i<trcnt;i++) {
  592.     detag(indbuf[trlist[i]]);
  593.     deaccent2(indbuf[trlist[i]]);
  594.     comma(indbuf[trlist[i]]);
  595.     singlespace2(indbuf[trlist[i]]);
  596.     translate(indbuf[trlist[i]]);
  597.   }
  598. /*   Normalize the information, using dictionary
  599.  *   bases/sys/words.xx with suffix translation
  600.  */
  601.   entrycount=mentrycount; dicbuf=mdicbuf;
  602.   memmove(entry,mentry,mentrycount*sizeof(entry[0]));
  603.   unknown_type=unk_leave;/*  used in translator_.c */
  604.   for(i=0;i<trcnt;i++) {
  605.   suffix_translate(indbuf[trlist[i]]);
  606.   translate(indbuf[trlist[i]]);
  607.   }
  608.  
  609. /* taken contains all words already seen in the module index */
  610.   taken[0]=0; takenlen=tweight=0;
  611. /*  append words of title  */
  612.   ovlstrcpy(buf,indbuf[i_title]); towords(buf);
  613.   appenditem2(buf,lind,serial,4,module_language);
  614.  
  615. /*  extract words of every other information except level */
  616.   snprintf(buf,sizeof(buf),"%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s",
  617.     indbuf[i_description],indbuf[i_keywords],
  618.     indbuf[i_keywords_ca],indbuf[i_keywords_en],indbuf[i_keywords_fr],
  619.     indbuf[i_keywords_it],indbuf[i_keywords_nl],
  620.     indbuf[i_title_ca],indbuf[i_title_en],indbuf[i_title_fr],
  621.     indbuf[i_title_it],indbuf[i_title_nl],
  622.     indbuf[i_domain],indbuf[i_require],indbuf[i_author]);
  623.   towords(buf);
  624.   appenditem2(buf,lind,serial,2,module_language);
  625.  
  626. /*   this time the dictionary is the group dictionary  sys/wgrp/wgrp
  627.  *   with a g (groupdic), not an m (maindic) . see below main, suffix, group.
  628.  *   and delete unknown ?? and translate
  629.  */
  630.   entrycount=gentrycount; dicbuf=gdicbuf;
  631.   memmove(entry,gentry,gentrycount*sizeof(entry[0]));
  632.  
  633. /* append words of every title information  */
  634.   ovlstrcpy(buf,indbuf[i_title]);
  635.   unknown_type=unk_delete;
  636.   translate(buf);
  637.   appenditem1(buf,lind,serial,2,module_language);
  638.  
  639. /* append words of information of description except level  */
  640.   snprintf(buf,sizeof(buf),"%s", indbuf[i_description]);
  641.   unknown_type=unk_delete;
  642.   translate(buf);
  643.   appenditem1(buf,lind,serial,4,module_language);
  644.  
  645. /* append words (or group of words) of keywords and domain  */
  646.   snprintf(buf,sizeof(buf),"%s, %s, %s, %s, %s, %s, %s",
  647.     indbuf[i_domain],indbuf[i_keywords],
  648.     indbuf[i_keywords_ca], indbuf[i_keywords_en],indbuf[i_keywords_fr],
  649.     indbuf[i_keywords_it], indbuf[i_keywords_nl]);
  650.   unknown_type=unk_leave;
  651.   translate(buf);
  652.   appenditem1(buf,lind,serial,2,module_language);
  653.  
  654. /* append level information, with weight 2 */
  655.   snprintf(buf,sizeof(buf),"%s",indbuf[i_level]);
  656.   ovlstrcpy(lbuf,"level");
  657.   for(p1=buf; *p1; p1++) if(!isalnum(*p1)) *p1=' ';
  658.   q=buf+strlen(buf);
  659.   for(p1=find_word_start(buf); (*p1) && (p1 < q) ; p1=find_word_start(p2)) {
  660.     p2=find_word_end(p1);
  661.     if(p2!=NULL) *p2++=0; else p2=p1+strlen(p1);
  662.     if(strncmp(p1, "Lang" , p2-p1) &&
  663.      (!isalpha(*p1) ||
  664.      (!isdigit(*(p1+1)) && *(p1+1)!=0) ||
  665.      (*(p1+1)!=0 && *(p1+2)!=0)))
  666.        continue;
  667.     *p1=tolower(*p1);
  668.     ovlstrcpy(lbuf+strlen("level"),p1);
  669.     appenditem(lbuf,lind,serial,2,module_language);
  670.   }
  671. /*   append total weight of module to weight file site2/weight.xx  */
  672.   fprintf(weightf,"%d:%d\n",serial,tweight);
  673. }
  674.  
  675. void modules(void)
  676. {
  677.   int i,j,k,d;
  678.   char namebuf[MAX_LINELEN+1];
  679.   char mdic[MAX_LINELEN+1], sdic[MAX_LINELEN+1], gdic[MAX_LINELEN+1], ddic[MAX_LINELEN+1];
  680.  
  681.   for(j=0;j<langcnt;j++) {
  682.     snprintf(namebuf,sizeof(namebuf),"%s/weight.%s",outdir,lang[j]);
  683.     weightf=fopen(namebuf,"w");
  684.     snprintf(mdic,sizeof(mdic),"%s/%s.%s",dicdir,maindic,lang[j]);
  685.     snprintf(sdic,sizeof(sdic),"%s/%s.%s",dicdir,suffixdic,lang[j]);
  686.     snprintf(gdic,sizeof(gdic),"%s/%s.%s",dicdir,groupdic,lang[j]);
  687.     snprintf(ddic,sizeof(ddic),"%s/%s.%s",dicdir,domaindic,lang[j]);
  688.     suffix_dic(sdic); prepare_dic(gdic);
  689.     gdicbuf=dicbuf; gentrycount=entrycount;
  690.     memmove(gentry,entry,gentrycount*sizeof(entry[0]));
  691.     prepare_dic(mdic);
  692.     mdicbuf=dicbuf; mentrycount=entrycount;
  693.     memmove(mentry,entry,mentrycount*sizeof(entry[0]));
  694.     prepare_dic(ddic);
  695.     ddicbuf=dicbuf; dentrycount=entrycount;
  696.     memmove(dentry,entry,dentrycount*sizeof(entry[0]));
  697.     unknown_type=unk_leave; translate(ignore[j]);
  698.     for(i=0;i<modcnt;i++) {
  699.       if(mod[i].langcnt>0) {
  700.       /* look for another language */
  701.         for(d=k=0;k<mod[i].langcnt;k++)
  702.           if(mod[i].langs[k]<mod[i].langs[d]) d=k;
  703.         for(k=0;k<mod[i].langcnt && mod[i].langs[k]!=j;k++);
  704.         if(k>=mod[i].langcnt) k=d;
  705.         snprintf(namebuf,MAX_LINELEN,"%s.%s",mod[i].name,
  706.            lang[mod[i].langs[k]]);
  707.         onemodule(namebuf,mod[i].counts[k],j);
  708.       }
  709.       else {
  710.         onemodule(mod[i].name,mod[i].counts[0],j);
  711.       }
  712.     }
  713.     if(mentrycount>0) free(mdicbuf);
  714.     if(gentrycount>0) free(gdicbuf);
  715.     if(suffixcnt>0) free(sufbuf);
  716.     if(dentrycount>0) free(ddicbuf);
  717.     if(weightf) fclose(weightf);
  718.   }
  719. }
  720.  
  721. /* FIXME ? differences with appenditem - use fprintf instead of  snprintf */
  722. void sappenditem(char *word, int lind, int serial, int weight)
  723. {
  724.   int ll;
  725.   char *p;
  726.  
  727.   if(!isalnum(*word) || (ll=strlen(word))<2 ||
  728.      wordchr2(taken,word)!=NULL ||
  729.      wordchr2(ignore[lind],word)!=NULL ||
  730.      takenlen>=MAX_LINELEN-ll-16)
  731.     return;
  732.   if(ll==2 && (!isdigit(word[0]) || !isalpha(word[1]))) return;
  733.   for(p=word;*p;p++) if(!isalnum(*p) && *p!=' ') return;
  734.   taken[takenlen++]=' ';taken[takenlen++]=' ';
  735.   ovlstrcpy(taken+takenlen,word);
  736.   takenlen+=ll; tweight+=weight;
  737.   fprintf(indf,"%s:%d?%d\n",word,serial,weight);
  738. }
  739. /* onesg / onemodule are similar */
  740. void onesg(int serial, int lind, int index(int))
  741. {
  742.   int i;
  743.   unsigned char trlist[]={
  744.     s_title,s_description,s_domain,s_keywords,s_information
  745.   };
  746.   int trcnt=sizeof(trlist)/sizeof(trlist[0]);
  747.   char *p1, *p2, *q, buf[MAX_LINELEN+1], lbuf[16];
  748.  
  749.   if(index(serial)) return;
  750.   fprintf(listf,"%s\n",mod[serial].name);
  751.   fprintf(titf,"%d:%s\n",serial,gsindbuf[s_title]);
  752.   fprintf(descf,"%d:%s\n",serial,gsindbuf[s_description]);
  753.   fprintf(remf,"%d:%s\n",serial,gsindbuf[s_information]);
  754.   fprintf(titlef,"%s:%s\n",mod[serial].name,gsindbuf[s_title]);
  755.  
  756. /*   Normalize the information of trlist, using dictionary
  757.  *  -- bases/sys/domain.xx without suffix translation (--> english version)
  758.  */
  759.   entrycount=dentrycount; dicbuf=ddicbuf;
  760.   memmove(entry,dentry,dentrycount*sizeof(entry[0]));
  761.   unknown_type=unk_leave;
  762.   for(i=0;i<trcnt;i++) {
  763.     detag(gsindbuf[trlist[i]]);
  764.     deaccent2(gsindbuf[trlist[i]]);
  765.     comma(gsindbuf[trlist[i]]);
  766.     singlespace2(gsindbuf[trlist[i]]);
  767.     translate(gsindbuf[trlist[i]]);
  768.   }
  769. /*   Normalize the information, using dictionary
  770.  *   bases/sys/words.xx with suffix translation
  771.  */
  772.   entrycount=mentrycount; dicbuf=mdicbuf;
  773.   memmove(entry,mentry,mentrycount*sizeof(entry[0]));
  774.   unknown_type=unk_leave;/*  used in translator_.c */
  775.   for(i=0;i<trcnt;i++) {
  776.     suffix_translate(gsindbuf[trlist[i]]);
  777.     translate(gsindbuf[trlist[i]]);
  778.   }
  779.  
  780. /* taken contains all words already seen in the module index */
  781.   taken[0]=0; takenlen=tweight=0;
  782. /*  append words of title  */
  783.   ovlstrcpy(buf,gsindbuf[s_title]); towords(buf);
  784.   for(p1=find_word_start(buf);*p1;
  785.       p1=find_word_start(p2)) {
  786.     p2=find_word_end(p1); if(*p2) *p2++=0;
  787.     sappenditem(p1,lind,serial,4);
  788.   }
  789.  
  790. /*  extract words of every other information except level */
  791.   snprintf(buf,sizeof(buf),"%s %s %s %s",
  792.          gsindbuf[s_description],gsindbuf[s_keywords],
  793.          gsindbuf[s_domain],gsindbuf[s_information]);
  794.   towords(buf);
  795.   for(p1=find_word_start(buf);*p1;p1=find_word_start(p2)) {
  796.     p2=find_word_end(p1); if(*p2) *p2++=0;
  797.     sappenditem(p1,lind,serial,2);
  798.   }
  799. /*   this time the dictionary is the group dictionary  sys/wgrp/wgrp
  800.  *   with a g (groupdic), not an m (maindic) . see below main, suffix, group.
  801.  *   and delete unknown ?? and translate
  802.  */
  803.   entrycount=gentrycount; dicbuf=gdicbuf;
  804.   memmove(entry,gentry,gentrycount*sizeof(entry[0]));
  805.  
  806. /*  append words of every title information  */
  807.   ovlstrcpy(buf,gsindbuf[s_title]);
  808.   unknown_type=unk_delete;
  809.   translate(buf);
  810.   for(p1=find_word_start(buf); *p1; p1=find_word_start(p2)) {
  811.     p2=strchr(p1,',');
  812.     if(p2!=NULL) *p2++=0; else p2=p1+strlen(p1);
  813.     if(strlen(p1)<=0) continue;
  814.     sappenditem(p1,lind,serial,4);
  815.   }
  816.  
  817. /*  append words (or group of words) of keywords and domain  */
  818.   snprintf(buf,sizeof(buf),"%s, %s",
  819.        gsindbuf[s_keywords],
  820.        gsindbuf[s_domain]);
  821.   unknown_type=unk_leave;
  822.   translate(buf);
  823.   for(p1=find_word_start(buf); *p1; p1=find_word_start(p2)) {
  824.     p2=strchr(p1,',');
  825.     if(p2!=NULL) *p2++=0; else p2=p1+strlen(p1);
  826.     if(strlen(p1)<=0) continue;
  827.     sappenditem(p1,lind,serial,2);
  828.   }
  829.  
  830. /*   append level information, with weight 2 */
  831.   snprintf(buf,sizeof(buf),"%s",gsindbuf[s_level]);
  832.   ovlstrcpy(lbuf,"level");
  833.   for(p1=buf; *p1; p1++) if(!isalnum(*p1)) *p1=' ';
  834.   q=buf+strlen(buf);
  835.   for(p1=find_word_start(buf); (*p1) && (p1 < q) ;
  836.   p1=find_word_start(p2)) {
  837.     p2=find_word_end(p1);
  838.     if(p2!=NULL) *p2++=0; else p2=p1+strlen(p1);
  839.     if(strncmp(p1, "Lang" , p2-p1) &&
  840.         (!isalpha(*p1) || (!isdigit(*(p1+1))) ||
  841.         (*(p1+1)!=0 && *(p1+2)!=0)))
  842.       continue;
  843.     *p1=tolower(*p1);
  844.     ovlstrcpy(lbuf+strlen("level"),p1);
  845.     sappenditem(lbuf,lind,serial,2);
  846.   }
  847. /*   append total weight of module to weight file site2/weight.xx  */
  848.   fprintf(weightf,"%d:%d\n",serial,tweight);
  849. }
  850.  
  851. void sgs(char *outdir, int index(int))
  852. {
  853.   int i,j;
  854.   char mdic[MAX_LINELEN+1], sdic[MAX_LINELEN+1], gdic[MAX_LINELEN+1], ddic[MAX_LINELEN+1];
  855.   char buf[MAX_LINELEN+1];
  856.  
  857.   for(j=0;j<langcnt;j++) {
  858.     snprintf(buf,sizeof(buf),"%s/title.%s",outdir,lang[j]);
  859.     titf=fopen(buf,"w");
  860.     snprintf(buf,sizeof(buf),"%s/description.%s",outdir,lang[j]);
  861.     descf=fopen(buf,"w");
  862.     snprintf(buf,sizeof(buf),"%s/%s",outdir,lang[j]);
  863.     indf=fopen(buf,"w");
  864.     snprintf(buf,sizeof(buf),"%s/list.%s",outdir,lang[j]);
  865.     listf=fopen(buf,"w");
  866.     snprintf(buf,sizeof(buf),"%s/weight.%s",outdir,lang[j]);
  867.     weightf=fopen(buf,"w");
  868.     snprintf(buf,sizeof(buf),"%s/addr.%s",outdir,lang[j]);
  869.     addrf=fopen(buf,"w");
  870.     snprintf(buf,sizeof(buf),"%s/information.%s",outdir,lang[j]);
  871.     remf=fopen(buf,"w");
  872.     snprintf(buf,sizeof(buf),"%s/serial.%s",outdir,lang[j]);
  873.     serialf=fopen(buf,"w");
  874.     snprintf(buf,sizeof(buf),"%s/tit.%s",outdir,lang[j]);
  875.     titlef=fopen(buf,"w");
  876.     if(!titlef || !serialf || !remf || !addrf || !weightf || !listf
  877.       || !indf || !descf || !titf ) {
  878.     fprintf(stderr,"modind: error creating output files for %s.\n",outdir); exit(1);
  879.     }
  880.     snprintf(mdic,sizeof(mdic),"%s/%s.%s",dicdir,maindic,lang[j]);
  881.     snprintf(sdic,sizeof(sdic),"%s/%s.%s",dicdir,suffixdic,lang[j]);
  882.     snprintf(gdic,sizeof(gdic),"%s/%s.%s",dicdir,groupdic,lang[j]);
  883.     snprintf(ddic,sizeof(ddic),"%s/%s.%s",dicdir,domaindic,lang[j]);
  884.     suffix_dic(sdic); prepare_dic(gdic);
  885.     gdicbuf=dicbuf; gentrycount=entrycount;
  886.     memmove(gentry,entry,gentrycount*sizeof(entry[0]));
  887.     prepare_dic(mdic);
  888.     mdicbuf=dicbuf; mentrycount=entrycount;
  889.     memmove(mentry,entry,mentrycount*sizeof(entry[0]));
  890.     prepare_dic(ddic);
  891.     ddicbuf=dicbuf; dentrycount=entrycount;
  892.     memmove(dentry,entry,dentrycount*sizeof(entry[0]));
  893.     unknown_type=unk_leave; translate(ignore[j]);
  894.     for(i=0;i<modcnt;i++) {
  895.       if(mod[i].langs[0]!=j) continue;
  896.       fprintf(addrf,"%d:%s\n",i,mod[i].name);
  897.       fprintf(serialf,"%s:%d\n",mod[i].name,i);
  898.       onesg(i,j,index);
  899.     }
  900.     if(mentrycount>0) free(mdicbuf);
  901.     if(gentrycount>0) free(gdicbuf);
  902.     if(suffixcnt>0) free(sufbuf);
  903.     if(dentrycount>0) free(ddicbuf);
  904.     fclose(titf); fclose(descf); fclose(indf); fclose(listf);
  905.     fclose(weightf); fclose(addrf); fclose(serialf);
  906.   }
  907. }
  908.  
  909. int main()
  910. {
  911.   gentry=xmalloc(entry_size);
  912.   dentry=xmalloc(entry_size);
  913.   mentry=xmalloc(entry_size);
  914.   prep();
  915.   if(modcnt>0) modules();
  916.   clean();
  917.   sprep();
  918.   if(modcnt>0) sgs(sheetoutdir,sheet_index);
  919.   gprep();
  920.   if(modcnt>0) sgs(glossaryoutdir,glossary_index);
  921.   return 0;
  922. }
  923.