Subversion Repositories wimsdev

Rev

Rev 3718 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | RSS feed

  1. /*    Copyright (C) 1998-2003 XIAO, Gang of Universite de Nice - Sophia Antipolis
  2.  *
  3.  *  This program is free software; you can redistribute it and/or modify
  4.  *  it under the terms of the GNU General Public License as published by
  5.  *  the Free Software Foundation; either version 2 of the License, or
  6.  *  (at your option) any later version.
  7.  *
  8.  *  This program is distributed in the hope that it will be useful,
  9.  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  10.  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11.  *  GNU General Public License for more details.
  12.  *
  13.  *  You should have received a copy of the GNU General Public License
  14.  *  along with this program; if not, write to the Free Software
  15.  *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  16.  */
  17.  
  18.         /* Sort dictionary */
  19.  
  20. /*************** Customization: change values hereafter ****************/
  21.  
  22.         /* limit of dictionary entries */
  23. #define entrylim 512*1024
  24.         /* limit of dictionary length */
  25. #define diclim  32*1024*1024
  26.         /* separation character */
  27. char sepchar=':', grpchar=0;
  28.  
  29. /***************** Nothing should need change hereafter *****************/
  30.  
  31. #include "../wims.h"
  32.  
  33. char inpbuf[MAX_LINELEN+1], outbuf[2*MAX_LINELEN+2];
  34. char *dicbuf;
  35. char dicname[1024], suffixname[1024];
  36.  
  37. struct entry {
  38.     char *original;
  39.     char *replace;
  40. } entry[entrylim];
  41. int entrycount;
  42.  
  43. int nocase=0, hassuffix=0, leaveline=0;
  44. int entrycount, ocount;
  45.  
  46. void *xmalloc(size_t n)
  47. {
  48.     void *p;
  49.     p=malloc(n);
  50.     if(p==NULL) exit(1);
  51.     return p;
  52. }
  53.  
  54.         /* Points to the end of the word */
  55. char *find_word_end(char *p)
  56. {
  57.     int i;
  58.     for(i=0;!isspace(*p) && *p!=0 && i<MAX_LINELEN; p++,i++);
  59.     return p;
  60. }
  61.  
  62.         /* Strips leading spaces */
  63. char *find_word_start(char *p)
  64. {
  65.     int i;
  66.     for(i=0; isspace(*p) && i<MAX_LINELEN; p++,i++);
  67.     return p;
  68. }
  69.  
  70.         /* strip trailing spaces; return string end. */
  71. char *strip_trailing_spaces(char *p)
  72. {
  73.     char *pp;
  74.     if(*p==0) return p;
  75.     for(pp=p+strlen(p)-1; pp>=p && isspace(*pp); *(pp--)=0);
  76.     return pp;
  77. }
  78.  
  79. int compare(const void *s1, const void *s2)
  80. {
  81.     const struct entry *p1, *p2;
  82.     p1=s1; p2=s2;
  83.     if(nocase) return strcasecmp(p1->original,p2->original);
  84.     else return strcmp(p1->original,p2->original);
  85. }
  86.  
  87. void sortdic(void)
  88. {
  89.     qsort(entry,entrycount,sizeof(entry[0]),compare);
  90. }
  91.  
  92.         /* modify a string. Bufferlen must be ast least MAX_LINELEN */
  93. void string_modify(char *start, char *bad_beg, char *bad_end, char *good,...)
  94. {
  95.     char buf[MAX_LINELEN+1];
  96.     va_list vp;
  97.    
  98.     va_start(vp,good);
  99.     vsnprintf(buf,sizeof(buf),good,vp); va_end(vp);
  100.     if(strlen(start)-(bad_end-bad_beg)+strlen(buf)>=MAX_LINELEN)
  101.       return;   /* this is an error situation. */
  102.     strcat(buf,bad_end);
  103.     strcpy(bad_beg,buf);
  104. }
  105.  
  106.         /* change all spaces into ' ', and collapse multiple occurences */
  107. void singlespace(char *p)
  108. {
  109.     char *pp, *p2;
  110.     for(pp=p;*pp;pp++) {
  111.         if(!isspace(*pp)) continue;
  112.         if(leaveline) {
  113.             if(*pp==13) strcpy(pp,pp+1);
  114.             if(*pp=='\n') {
  115.                 pp++;
  116.                 gopt: for(p2=pp; isspace(*p2) && *p2!='\n'; p2++);
  117.                 if(p2>pp) strcpy(pp,p2); pp--;
  118.             }
  119.             else {
  120.                 pp++; if(!isspace(*pp) || *pp=='\n') continue;
  121.                 goto gopt;
  122.             }
  123.         }
  124.         else {
  125.             if(*pp!=' ') *pp=' ';
  126.             pp++; if(!isspace(*pp)) continue;
  127.             for(p2=pp;isspace(*p2);p2++);
  128.             strcpy(pp,p2); pp--;
  129.         }
  130.     }
  131. }
  132.  
  133.         /* Prepare dictionary */
  134. void prepare_dic(void)
  135. {
  136.     int i;
  137.     FILE *dicf;
  138.     char *p1, *p2, *pp;
  139.     long int flen;
  140.  
  141.     entrycount=0;
  142.     dicf=fopen(dicname,"r"); if(dicf==NULL) return;
  143.     fseek(dicf,0,SEEK_END);flen=ftell(dicf); fseek(dicf,0,SEEK_SET);
  144.     if(flen>diclim) return;
  145.     dicbuf=xmalloc(2*flen+1024);flen=fread(dicbuf,1,flen,dicf);
  146.     fclose(dicf);
  147.     if(flen>0 && flen<diclim) dicbuf[flen]=0;
  148.     else return;
  149.     for(i=0,p1=dicbuf;p1!=NULL && *p1!=0 && i<entrylim;p1=p2) {
  150.         p2=strchr(p1+1,'\n'); if(p2>p1) *p2++=0;
  151.         pp=strchr(p1,sepchar); if(pp==NULL) continue;
  152.         *pp++=0;
  153.         strip_trailing_spaces(p1); strip_trailing_spaces(pp);
  154.         singlespace(p1);
  155.         p1=find_word_start(p1); pp=find_word_start(pp);
  156.         if(*p1==0) continue;
  157.         entry[i].original=p1; entry[i].replace=pp; i++;
  158.     }
  159.     entrycount=i;
  160. }
  161.  
  162. #include "suffix.c"
  163.  
  164. void output(void)
  165. {
  166.     int i;
  167.     FILE *f;
  168.  
  169.     ocount=0;
  170.     strcat(dicname,".sorted");
  171.     f=fopen(dicname,"w"); if(f==NULL) return;
  172.     for(i=0;i<entrycount;i++) {
  173.         if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0
  174.            && strcmp(entry[i].replace,entry[i-1].replace)==0)
  175.           continue;
  176.         if(grpchar!=0) {
  177.             if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0)
  178.               fprintf(f,"%c%s",grpchar, entry[i].replace);
  179.             else {
  180.                 if(i>0) fprintf(f,"\n");
  181.                 fprintf(f,"%s%c%s",entry[i].original,sepchar,entry[i].replace);
  182.                 ocount++;
  183.             }
  184.            
  185.         }
  186.         else {
  187.             fprintf(f,"%s%c%s\n",entry[i].original,sepchar,entry[i].replace);
  188.             ocount++;
  189.         }
  190.     }
  191.     if(grpchar!=0) fprintf(f,"\n");
  192.     fclose(f);
  193. }
  194.  
  195. int main(int argc, char *argv[])
  196. {
  197.     char *ss, *gr;
  198.     if(argc<2) return -1;
  199.    
  200.     ss=getenv("dicsort_separator");
  201.     if(ss!=NULL && *ss!=0) sepchar=*ss;
  202.     gr=getenv("dicsort_grouping");
  203.     if(gr!=NULL && *gr!=0) grpchar=*gr;
  204.     snprintf(dicname,sizeof(dicname)-128,"%s",argv[1]); prepare_dic();
  205.     if(argc>2) {
  206.         snprintf(suffixname,sizeof(suffixname),"%s",argv[2]);
  207.         suffix_dic(suffixname); hassuffix=1;
  208.     }
  209.     else suffixname[0]=hassuffix=0;
  210.     sortdic(); output();
  211.     printf("%s: sorted %d entries.\n",dicname, ocount);
  212.     return 0;
  213. }
  214.  
  215.