Subversion Repositories wimsdev

/*    Copyright (C) 1998-2003 XIAO, Gang of Universite de Nice - Sophia Antipolis

 *

 *  This program is free software; you can redistribute it and/or modify

 *  it under the terms of the GNU General Public License as published by

 *  the Free Software Foundation; either version 2 of the License, or

 *  (at your option) any later version.

 *

 *  This program is distributed in the hope that it will be useful,

 *  but WITHOUT ANY WARRANTY; without even the implied warranty of

 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 *  GNU General Public License for more details.

 *

 *  You should have received a copy of the GNU General Public License

 *  along with this program; if not, write to the Free Software

 *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

 */

/*  This is an internal program,

 * used to index modules for search engine.

 */

#include "../Lib/libwims.h"

#include "translator_.h"

#include "suffix.h"

#define MAX_LANGS    MAX_LANGUAGES

#define MAX_MODULES    65536

char *moduledir=    "public_html/modules";

char *sheetdir=     "public_html/bases/sheet";

char *glossarydir=  "public_html/scripts/data/glossary";

char *dicdir=       "public_html/bases";

char *outdir=       "public_html/bases/site2";

char *sheetoutdir=  "public_html/bases/sheet/index";

char *glossaryoutdir=  "public_html/scripts/data/glossary/index";

char *maindic=      "sys/words";

char *groupdic=     "sys/wgrp/wgrp";

char *suffixdic=    "sys/suffix";

char *domaindic=    "sys/domaindic";

char *ignoredic=    "sys/indignore";

char *conffile=     "log/wims.conf";

char *mlistbase=    "lists";

char lang[MAX_LANGS][4]={

    "en","fr","cn","es","it","nl","si","ca","pt"

};

#define DEFAULT_LANGCNT    6

char allang[MAX_LANGS][4]={

    "en","fr","cn","es","it","nl","de","si","ca","pt"

};

#define allangcnt 8

char ignore[MAX_LANGS][MAX_LINELEN+1];

char mlistfile[MAX_LANGS][256];

int langcnt;

FILE *langf, *titf, *descf, *weightf, *robotf, *indf, *listf, *addrf, *serialf, *authorf, *versionf, *remf;

FILE *titf_ca,*titf_en,*titf_es,*titf_fr,*titf_it,*titf_nl;

struct cat {

    char *name;

    char typ;

} cat[]={

    {"all_types", 'A'},

    {"exercise",  'X'},

    {"oef",       'O'},

    {"tool",      'T'},

    {"recreation",'R'},

    {"reference", 'Y'},

    {"document",  'D'},

    {"popup",     'P'},

    {"datamodule",'M'}

};

#define catno (sizeof(cat)/sizeof(cat[0]))

struct mod {

    char *name;

    unsigned char langs[MAX_LANGS];

    int counts[MAX_LANGS];

    int langcnt;

} mod[MAX_MODULES];

// serial-> the name of the module indexed by serial, lang and its classe

struct revmod {

    char name[MAX_MODULELEN+1];

    int lang;

    int imod;

    char keywords[MAX_FNAME];

} revmod[MAX_MODULES];

int modcnt;

char *mlist;

char *sheetindex[]={

  "title", "description",

  "duration", "severity",

  "level", "domain",

  "keywords", "reserved1", "reserved2", "information"

};

/* correspond to the order of sheetindex */

char *glindex[]={

  "gl_title", "gl_description",

  "", "",

  "gl_level", "gl_domain",

  "gl_keywords","","",""};

#define SHEETINDEX_NO (sizeof(sheetindex)/sizeof(sheetindex[0]))

char gsindbuf[SHEETINDEX_NO+1][MAX_LINELEN+1];

/* do not modify the order, correspond to the order in the sheet file */

enum{s_title, s_description,

      s_duration, s_severity,

      s_level, s_domain,

      s_keywords, s_reserved1, s_reserved2,

      s_information

};

char *modindex[]={

  "title", "description",

  "author", "address", "copyright",

  "version", "wims_version", "language",

  "category", "level", "domain", "keywords",

  "keywords_ca", "keywords_en", "keywords_fr", "keywords_it", "keywords_nl",

  "title_ca", "title_en", "title_es", "title_fr", "title_it", "title_nl",

  "require"

};

#define MODINDEX_NO (sizeof(modindex)/sizeof(modindex[0]))

char indbuf[MODINDEX_NO][MAX_LINELEN+1];

enum{i_title, i_description,

  i_author,i_address,i_copyright,

  i_version,i_wims_version,i_language,

  i_category,i_level,i_domain,i_keywords,

  i_keywords_ca,i_keywords_en,i_keywords_fr,i_keywords_it,i_keywords_nl,

  i_title_ca,i_title_en,i_title_es,i_title_fr,i_title_it,i_title_nl,

  i_require

};

char *module_special_file[]={

  "intro","help","about"

};

#define MODSPEC_NO (sizeof(module_special_file)/sizeof(module_special_file[0]))

char module_language[4];

char *mdicbuf, *gdicbuf, *ddicbuf, *gentry, *mentry, *dentry;

int gentrycount, mentrycount, dentrycount;

/*  fold known accented letters to unaccented, other strange characters to space

 *  apostrophe is among the exceptions to be kept (important for multi-word expressions)

 */

void deaccent2(char *p)

{

  char *sp;

  char *v;

  for(sp=p;*sp;sp++) {

  if(*sp<0 && (v=strchr(acctab,*sp))!=NULL)

    *sp=*(deatab+(v-acctab));

  if(!isalnum(*sp) && strchr(",.&$+*",*sp)==0) *sp=' ';

  else *sp=tolower(*sp);

  }

}

/*  translate everything non-alphanumeric into space */

void towords(char *p)

{

  char *pp;

  for(pp=p;*pp;pp++) if(!isalnum(*pp) && strchr("&$+*",*pp)==0) *pp=' ';

}

/*  Find first occurrence of word */

char *wordchr2(char *p, char *w)

{

  char *r;

  for(r=strstr(p,w);r!=NULL &&

    ( (r>p && !isspace(*(r-1))) || (!isspace(*(r+strlen(w))) && *(r+strlen(w))!=0) );

  r=strstr(r+1,w));

  return r;

}

char *find_tag_end(char *p)

{

  char *pp;

  pp=p; if(*pp=='<') pp++;

  for(; *pp && *pp!='>'; pp++) {

    if(*pp=='<') {

      pp=find_tag_end(pp)-1; continue;

    }

    if(*pp=='"') {

      pp=strchr(pp+1,'"');

      if(pp==NULL) return p+strlen(p); else continue;

    }

    if(*pp=='\'') {

      pp=strchr(pp+1,'\'');

      if(pp==NULL) return p+strlen(p); else continue;

    }

  }

  if(*pp=='>') pp++;

  return pp;

}

char *find_tag(char *p, char *tag)

{

  char *pp;

  int len;

  len=strlen(tag);

  for(pp=strchr(p,'<'); pp!=NULL && *pp; pp=strchr(pp+1,'<')) {

    if(strncasecmp(pp+1,tag,len)==0 && !isalnum(*(pp+1+len))) return pp;

  }

  return p+strlen(p);

}

/*  remove all html tags */

void detag(char *p)

{

  char *pp, *p2;

  for(pp=strchr(p,'<'); pp!=NULL; pp=strchr(pp,'<')) {

    p2=find_tag_end(pp);

    if(*p2==0) {*pp=0; return; }

    ovlstrcpy(pp,p2);

  }

}

/* add a space after comma to see end of words */

void comma(char *p)

{

  char *pp;

  for(pp=strchr(p,','); pp; pp=strchr(pp+1,','))

    string_modify3(p,pp,pp+1,", ");

}

/* replace / by , */

void backslash(char *p)

{

  char *pp;

  for(pp=strchr(p,'/'); pp; pp=strchr(pp+1,'/'))

    string_modify3(p,pp,pp+1,",");

}

/* _getdef from lines.c except the error msg*/

void _getdef(char buf[], char *name, char value[])

{

  char *p1, *p2, *p3, *p4;

  if(*name==0) goto nothing;      /* this would create segfault. */

  for(p1=strstr(buf,name); p1!=NULL; p1=strstr(p1+1,name)) {

    p2=find_word_start(p1+strlen(name));

    if((p1>buf && !isspace(*(p1-1))) || *p2!='=') continue;

    p3=p1; while(p3>buf && *(p3-1)!='\n') p3--;

    p3=find_word_start(p3);

    if(p3<p1 && *p3!='!') continue;

    if(p3<p1) {

      p3++; p4=find_word_end(p3);

      if(find_word_start(p4)!=p1) continue;

      if(p4-p3!=3 || (strncmp(p3,"set",3)!=0 &&

           strncmp(p3,"let",3)!=0 &&

           strncmp(p3,"def",3)!=0)) {

        if(p4-p3!=6 || strncmp(p3,"define",6)!=0) continue;

      }

    }

    p2++;p3=strchr(p2,'\n'); if(p3==NULL) p3=p2+strlen(p2);

    p2=find_word_start(p2);

    if(p2>p3) goto nothing;

    /*if(p3-p2>=MAX_LINELEN) user_error("cmd_output_too_long");*/

    memmove(value,p2,p3-p2); value[p3-p2]=0;

    strip_trailing_spaces(value); return;

  }

nothing:

  value[0]=0;

}

/*  Get variable definition from a file.

 * Result stored in buffer value of length MAX_LINELEN.

 */

void getdef(char *fname, char *name, char value[])

{

  FILE *f;

  char *buf;

  int l;

  value[0]=0;

  f=fopen(fname,"r"); if(f==NULL) return;

  fseek(f,0,SEEK_END); l=ftell(f); fseek(f,0,SEEK_SET);

  buf=xmalloc(l+256); l=fread(buf,1,l,f);

  fclose(f);

  if(l<=0) return; else buf[l]=0;

  _getdef(buf,name,value);

  free(buf);

}

void init(void)

{

  char buf[MAX_LINELEN+1];

  char *p1,*p2,*s;

  int i,l;

  FILE *f;

  s=getenv("modind_outdir"); if(s!=NULL && *s!=0) outdir=s;

  s=getenv("modind_sheetdir"); if(s!=NULL && *s!=0) sheetdir=s;

  s=getenv("modind_sheetoutdir"); if(s!=NULL && *s!=0) sheetoutdir=s;

  s=getenv("modind_glossaryoutdir"); if(s!=NULL && *s!=0) glossaryoutdir=s;

/* take the langs declared in conffile */

  getdef(conffile,"site_languages",buf);

  langcnt=0;

  for(p1=buf;*p1;p1++) if(!isalnum(*p1)) *p1=' ';

  for(p1=find_word_start(buf); *p1 && langcnt<MAX_LANGS; p1=find_word_start(p2)) {

    p2=find_word_end(p1);

    if(p2!=p1+2 || !isalpha(*p1) || !isalpha(*(p1+1))) continue;

    memmove(lang[langcnt],p1,2); lang[langcnt++][2]=0;

  }

  if(langcnt==0) {/*  default languages */

    langcnt=DEFAULT_LANGCNT;

  }

  for(i=0;i<langcnt;i++) {

    snprintf(buf,sizeof(buf),"%s/%s.%s",dicdir,ignoredic,lang[i]);

    f=fopen(buf,"r"); if(f==NULL) continue;

    l=fread(ignore[i],1,MAX_LINELEN,f);fclose(f);

    if(l<0 || l>=MAX_LINELEN) l=0;

    ignore[i][l]=0;

  }

}

/*  Preparation of data */

void prep(void)

{

  char buf[MAX_LINELEN+1];

  char *p1,*p2,*s,*old;

  int i,l,thislang,t;

  modcnt=0; old="";

  snprintf(buf,sizeof(buf),"%s/addr",outdir);

  addrf=fopen(buf,"w");

  if(!addrf) { fprintf(stderr,"modind: error creating output files addr.\n"); exit(1);}

  snprintf(buf,sizeof(buf),"%s/serial",outdir);

  serialf=fopen(buf,"w");

  if(!serialf) { fprintf(stderr,"modind: error creating output files serial.\n"); exit(1);}

  s=getenv("mlist"); if(s==NULL) exit(1);

  l=strlen(s); if(l<0 || l>100*MAX_LINELEN) exit(1);

  mlist=xmalloc(l+16); ovlstrcpy(mlist,s);

  for(t=0, p1=find_word_start(mlist); *p1 && modcnt<MAX_MODULES;

        p1=find_word_start(p2), t++) {

    p2=find_word_end(p1);

    l=p2-p1; if(*p2) *p2++=0;

    fprintf(addrf,"%d:%s\n",t,p1);

    fprintf(serialf,"%s:%d\n",p1,t);

    thislang=-1;

/* language is taken from the address */

    if(l>3 && p1[l-3]=='.') {

      for(i=0;i<langcnt;i++) if(strcasecmp(lang[i],p1+l-2)==0) break;

      if(i<langcnt) {p1[l-3]=0; thislang=i;}

      else {/*  unknown language, not referenced */

        continue;

      }

    }

    if(modcnt>0 && strcmp(old,p1)==0 && thislang>=0) {

      if(mod[modcnt-1].langcnt<langcnt) {

        mod[modcnt-1].langs[mod[modcnt-1].langcnt]=thislang;

        mod[modcnt-1].counts[mod[modcnt-1].langcnt]=t;

        (mod[modcnt-1].langcnt)++;

      }

    }

    else {

      mod[modcnt].name=old=p1;

      if(thislang>=0) {

        mod[modcnt].langs[0]=thislang;

        mod[modcnt].langcnt=1;

      }

      else mod[modcnt].langcnt=0;

      mod[modcnt].counts[0]=t;

      modcnt++;

    }

  }

  snprintf(buf,sizeof(buf),"%s/language",outdir);

  langf=fopen(buf,"w");

  snprintf(buf,sizeof(buf),"%s/title",outdir);

  titf=fopen(buf,"w");

  snprintf(buf,sizeof(buf),"%s/title_ca",outdir);

  titf_ca=fopen(buf,"w");

  snprintf(buf,sizeof(buf),"%s/title_en",outdir);

  titf_en=fopen(buf,"w");

  snprintf(buf,sizeof(buf),"%s/title_es",outdir);

  titf_es=fopen(buf,"w");

  snprintf(buf,sizeof(buf),"%s/title_fr",outdir);

  titf_fr=fopen(buf,"w");

  snprintf(buf,sizeof(buf),"%s/title_it",outdir);

  titf_it=fopen(buf,"w");

  snprintf(buf,sizeof(buf),"%s/title_nl",outdir);

  titf_nl=fopen(buf,"w");

  snprintf(buf,sizeof(buf),"%s/description",outdir);

  descf=fopen(buf,"w");

  snprintf(buf,sizeof(buf),"%s/author",outdir);

  authorf=fopen(buf,"w");

  snprintf(buf,sizeof(buf),"%s/version",outdir);

  versionf=fopen(buf,"w");

  snprintf(buf,sizeof(buf),"%s/%s/robot.phtml",outdir,mlistbase);

  robotf=fopen(buf,"w");

  fclose(addrf); fclose(serialf);

  if(!robotf || !versionf || !authorf || !descf || !titf

    || !titf_ca || !titf_en || !titf_es || !titf_fr || !titf_it || !titf_nl

    || !langf) {

    fprintf(stderr,"modind: error creating output files.\n");

    exit(1);

  }

}

void sprep(void)

{

  char buf[MAX_LINELEN+1];

  char *p1,*p2,*s;

  int i,l,t,thislang;

  modcnt=0;

  snprintf(buf,sizeof(buf),"%s/addr",sheetoutdir);

  addrf=fopen(buf,"w");

  if(!addrf) { fprintf(stderr,"modind: error creating output files addr.\n"); exit(1);}

  snprintf(buf,sizeof(buf),"%s/serial",sheetoutdir);

  serialf=fopen(buf,"w");

  s=getenv("slist"); if(s==NULL) return;

  l=strlen(s); if(l<0 || l>100*MAX_LINELEN) return;

  mlist=xmalloc(l+16); ovlstrcpy(mlist,s);

  for(t=0,p1=find_word_start(mlist); *p1 && modcnt<MAX_MODULES; p1=find_word_start(p2),t++) {

    p2=find_word_end(p1);

    l=p2-p1; if(*p2) *p2++=0;

    for(i=0;i<langcnt;i++) if(strncasecmp(lang[i],p1,2)==0) break;

    if(i<langcnt) thislang=i; else continue;

    ovlstrcpy(revmod[t].name,p1);

    revmod[t].lang=thislang;

    mod[modcnt].name=p1;

    mod[modcnt].langs[0]=thislang;

    mod[modcnt].langcnt=1;

    revmod[t].imod=modcnt;

    fprintf(addrf,"%d:%s\n",modcnt,p1);

    fprintf(serialf,"%s:%d\n",p1,modcnt);

    modcnt++;

  }

 fclose(addrf); fclose(serialf);

}

void gprep(void)

{

  char buf[MAX_LINELEN+1];

  char *p1,*p2,*s,*old;

  int l,i,t,thislang;

  modcnt=0; old="";

  snprintf(buf,sizeof(buf),"%s/addr",glossaryoutdir);

  addrf=fopen(buf,"w");

  if(!addrf) { fprintf(stderr,"modind: error creating output files addr.\n"); exit(1);}

  snprintf(buf,sizeof(buf),"%s/serial",glossaryoutdir);

  serialf=fopen(buf,"w");

  s=getenv("glist"); if(s==NULL) return;

  l=strlen(s); if(l<0 || l>100*MAX_LINELEN) return;

  mlist=xmalloc(l+16); ovlstrcpy(mlist,s);

  for(t=0,p1=find_word_start(mlist); *p1 && modcnt<MAX_MODULES; p1=find_word_start(p2),t++) {

    p2=find_word_end(p1);

    if(*p2) *p2++=0;

    fprintf(addrf,"%d:%s\n",t,p1);

    fprintf(serialf,"%s:%d\n",p1,t);

    ovlstrcpy(revmod[t].name,p1);

    ovlstrcpy(revmod[t].keywords,p1);

    s=strchr(p1,'/');

    if(s != NULL) s=strchr(s+1,'/');

    if(s==NULL) {

      fprintf(stderr,"modind: no language %s\n",p1); exit(1);

    }

    revmod[t].keywords[s-p1]=0;

    s++;

    for(i=0;i<langcnt;i++) if(strncasecmp(lang[i],s,2)==0) break;

    thislang = i<langcnt ? i : -1;

    revmod[t].lang=i;

    s[0]=s[1]='x';

    if(modcnt>0 && strcmp(old,p1)==0 && thislang >= 0) {

      if(mod[modcnt-1].langcnt<langcnt) {

        mod[modcnt-1].langs[mod[modcnt-1].langcnt]=thislang;

        mod[modcnt-1].counts[mod[modcnt-1].langcnt]=t;

        (mod[modcnt-1].langcnt)++;

      }

      revmod[t].imod=modcnt-1;

    }

    else {

      mod[modcnt].name=old=p1;

      if(thislang>=0) {

        mod[modcnt].langs[0]=thislang;

        mod[modcnt].langcnt=1;

      }

      else mod[modcnt].langcnt=0;

      mod[modcnt].counts[0]=t;

      revmod[t].imod=modcnt;

      modcnt++;

    }

  }

  fclose(addrf); fclose(serialf);

}

/*  read and treat module's INDEX file */

int module_index(const char *name)

{

  char *p, fbuf[MAX_LINELEN+1], ibuf[MAX_LINELEN+1];

  FILE *indf;

  int i,l;

  snprintf(fbuf,sizeof(fbuf),"%s/%s/INDEX",moduledir,name);

  indf=fopen(fbuf,"r");

  if(indf==NULL) {

    fprintf(stderr,"modind: INDEX of %s not found\n",fbuf); return -1;

  }

  l=fread(ibuf,1,MAX_LINELEN,indf); fclose(indf);

  if(l>0 && l<MAX_LINELEN) ibuf[l]=0; else return -1;

/* treate all fields in *modindex */

  for(i=0;i<MODINDEX_NO;i++) {

    _getdef(ibuf,modindex[i],indbuf[i]);

/*  compatibility precaution */

    if(indbuf[i][0]==':') indbuf[i][0]='.';

  }

  p=find_word_start(indbuf[i_language]);

  if(isalpha(*p) && isalpha(*(p+1))) {

    memmove(module_language,p,2); module_language[2]=0;

  }

  else ovlstrcpy(module_language,"en");

  return 0;

}

int sheet_index(int serial)

{

  char *p1, *p2, fbuf[MAX_LINELEN+1], ibuf[MAX_LINELEN+1];

  FILE *indf;

  int i,l;

  snprintf(fbuf,sizeof(fbuf),"%s/%s.def",sheetdir,mod[serial].name);

  indf=fopen(fbuf,"r"); if(indf==NULL) return -1;

  l=fread(ibuf,1,MAX_LINELEN,indf); fclose(indf);

  if(l>0 && l<MAX_LINELEN) ibuf[l]=0; else return -1;

  for(i=0;i<SHEETINDEX_NO;i++) gsindbuf[i][0]=0;

  for(i=0,p1=find_word_start(ibuf);

      i<SHEETINDEX_NO-1 && *p1!=':' && *p1!=0;

      i++,p1=p2) {

    p2=strchr(p1,'\n');

    if(p2!=NULL) *p2++=0; else p2=p1+strlen(p1);

    p1=find_word_start(p1); strip_trailing_spaces2(p1);

    snprintf(gsindbuf[i],MAX_LINELEN,"%s",p1);

  }

  p2=strstr(p1,"\n:"); if(p2==NULL) p2=p1+strlen(p1);

  else *p2=0;

  p1=find_word_start(p1); strip_trailing_spaces2(p1);

  for(p2=p1;*p2;p2++) if(*p2=='\n') *p2=' ';

  ovlstrcpy(gsindbuf[s_information],p1);

  ovlstrcpy(gsindbuf[SHEETINDEX_NO],revmod[serial].name);

  return 0;

}

int glossary_index(int serial)

{

  char nbuf[MAX_LINELEN+1],fbuf[MAX_LINELEN+1], ibuf[MAX_LINELEN+1],*p,*s;

  FILE *indf;

  int i,l;

  s=lang[revmod[serial].lang];

  p=strchr(nbuf,'/');

  if(p != NULL) p=strchr(p+1,'/');

  if(p != NULL) {p[1]=s[0];p[2]=s[1];}

  snprintf(fbuf,sizeof(fbuf),"%s/%s",glossarydir,revmod[serial].name);

  indf=fopen(fbuf,"r");

  l=fread(ibuf,1,MAX_LINELEN,indf); fclose(indf);

  if(l>0 && l<MAX_LINELEN) ibuf[l]=0; else return -1;

  for(i=0;i<SHEETINDEX_NO;i++) {

    _getdef(ibuf,glindex[i],gsindbuf[i]);

  }

  s=gsindbuf[s_keywords]+strlen(gsindbuf[s_keywords]);

  *s++ = ',';

  ovlstrcpy(s,revmod[serial].keywords);

  backslash(revmod[serial].keywords);

  ovlstrcpy(gsindbuf[SHEETINDEX_NO],nbuf);

  return 0;

}

unsigned char categories[16];

char taken[MAX_LINELEN+1];