Subversion Repositories wimsdev

Rev

Rev 11132 | Blame | Compare with Previous | Last modification | View Log | RSS feed

  1. /*    Copyright (C) 1998-2003 XIAO, Gang of Universite de Nice - Sophia Antipolis
  2.  *
  3.  *  This program is free software; you can redistribute it and/or modify
  4.  *  it under the terms of the GNU General Public License as published by
  5.  *  the Free Software Foundation; either version 2 of the License, or
  6.  *  (at your option) any later version.
  7.  *
  8.  *  This program is distributed in the hope that it will be useful,
  9.  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  10.  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11.  *  GNU General Public License for more details.
  12.  *
  13.  *  You should have received a copy of the GNU General Public License
  14.  *  along with this program; if not, write to the Free Software
  15.  *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  16.  */
  17.  
  18. /* Sort dictionary */
  19.  
  20. /*************** Customization: change values hereafter ****************/
  21.  
  22. /* limit of dictionary entries */
  23. #define entrylim 512*1024
  24. /* limit of dictionary length */
  25. #define diclim 32*1024*1024
  26. /* separation character */
  27. char sepchar=':', grpchar=0;
  28.  
  29. /***************** Nothing should need change hereafter *****************/
  30.  
  31. #include "../Lib/libwims.h"
  32. #include "suffix.h"
  33.  
  34. char inpbuf[MAX_LINELEN+1], outbuf[2*MAX_LINELEN+2];
  35. char *dicbuf;
  36. char dicname[1024], suffixname[1024];
  37.  
  38. struct entry {
  39.   char *original;
  40.   char *replace;
  41. } entry[entrylim];
  42. int entrycount;
  43.  
  44. int nocase=0, hassuffix=0, leaveline=0;
  45. int entrycount, ocount;
  46.  
  47. int compare(const void *s1, const void *s2)
  48. {
  49.   const struct entry *p1, *p2;
  50.   p1=s1; p2=s2;
  51.   if(nocase) return strcasecmp(p1->original,p2->original);
  52.   else return strcmp(p1->original,p2->original);
  53. }
  54.  
  55. void sortdic(void)
  56. {
  57.   qsort(entry,entrycount,sizeof(entry[0]),compare);
  58. }
  59.  
  60. /* change all spaces into ' ', and collapse multiple occurences */
  61. void singlespace2(char *p)
  62. {
  63.   char *pp, *p2;
  64.   for(pp=p;*pp;pp++) {
  65.     if(!isspace(*pp)) continue;
  66.     if(leaveline) {
  67.       if(*pp==13) ovlstrcpy(pp,pp+1);
  68.       if(*pp=='\n') {
  69.         pp++;
  70.         gopt: for(p2=pp; isspace(*p2) && *p2!='\n'; p2++);
  71.         if(p2>pp) ovlstrcpy(pp,p2);
  72.         pp--;
  73.       }
  74.       else {
  75.         pp++;
  76.         if(!isspace(*pp) || *pp=='\n') continue;
  77.         goto gopt;
  78.       }
  79.     }
  80.     else {
  81.       if(*pp!=' ') *pp=' ';
  82.       pp++; if(!isspace(*pp)) continue;
  83.       for(p2=pp;isspace(*p2);p2++);
  84.       ovlstrcpy(pp,p2); pp--;
  85.     }
  86.   }
  87. }
  88.  
  89. /* Prepare dictionary */
  90. static
  91. void prepare_dics(void)
  92. {
  93.   int i;
  94.   FILE *dicf;
  95.   char *p1, *p2, *pp;
  96.   long int flen;
  97.  
  98.   entrycount=0;
  99.   dicf=fopen(dicname,"r"); if(dicf==NULL) return;
  100.   fseek(dicf,0,SEEK_END);flen=ftell(dicf); fseek(dicf,0,SEEK_SET);
  101.   if(flen>diclim) return;
  102.   dicbuf=xmalloc(2*flen+1024);flen=fread(dicbuf,1,flen,dicf);
  103.   fclose(dicf);
  104.   if(flen>0 && flen<diclim) dicbuf[flen]=0;
  105.   else return;
  106.   for(i=0,p1=dicbuf;p1!=NULL && *p1!=0 && i<entrylim;p1=p2) {
  107.     p2=strchr(p1+1,'\n'); if(p2>p1) *p2++=0;
  108.     pp=strchr(p1,sepchar); if(pp==NULL) continue;
  109.     *pp++=0;
  110.     strip_trailing_spaces2(p1); strip_trailing_spaces2(pp);
  111.     singlespace2(p1);
  112.     p1=find_word_start(p1); pp=find_word_start(pp);
  113.     if(*p1==0) continue;
  114.     entry[i].original=p1; entry[i].replace=pp; i++;
  115.   }
  116.   entrycount=i;
  117. }
  118.  
  119. void output(void)
  120. {
  121.   int i;
  122.   FILE *f;
  123.  
  124.   ocount=0;
  125.   strcat(dicname,".sorted");
  126.   f=fopen(dicname,"w"); if(f==NULL) return;
  127.   for(i=0;i<entrycount;i++) {
  128.     if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0
  129.        && strcmp(entry[i].replace,entry[i-1].replace)==0)
  130.       continue;
  131.     if(grpchar!=0) {
  132.       if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0)
  133.         fprintf(f,"%c%s",grpchar, entry[i].replace);
  134.       else {
  135.         if(i>0) fprintf(f,"\n");
  136.         fprintf(f,"%s%c%s",entry[i].original,sepchar,entry[i].replace);
  137.         ocount++;
  138.       }
  139.     }
  140.     else {
  141.       fprintf(f,"%s%c%s\n",entry[i].original,sepchar,entry[i].replace);
  142.       ocount++;
  143.     }
  144.   }
  145.   if(grpchar!=0) fprintf(f,"\n");
  146.   fclose(f);
  147. }
  148.  
  149. int main(int argc, char *argv[])
  150. {
  151.   char *ss, *gr;
  152.   if(argc<2) return -1;
  153.  
  154.   ss=getenv("dicsort_separator");
  155.   if(ss!=NULL && *ss!=0) sepchar=*ss;
  156.   gr=getenv("dicsort_grouping");
  157.   if(gr!=NULL && *gr!=0) grpchar=*gr;
  158.   snprintf(dicname,sizeof(dicname)-128,"%s",argv[1]); prepare_dics();
  159.   if(argc>2) {
  160.     snprintf(suffixname,sizeof(suffixname),"%s",argv[2]);
  161.     suffix_dic(suffixname); hassuffix=1;
  162.   }
  163.   else suffixname[0]=hassuffix=0;
  164.   sortdic(); output();
  165.   printf("%s: sorted %d entries.\n",dicname, ocount);
  166.   return 0;
  167. }
  168.  
  169.