Subversion Repositories wimsdev

Rev

Rev 11132 | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
10 reyssat 1
/*    Copyright (C) 1998-2003 XIAO, Gang of Universite de Nice - Sophia Antipolis
2
 *
3
 *  This program is free software; you can redistribute it and/or modify
4
 *  it under the terms of the GNU General Public License as published by
5
 *  the Free Software Foundation; either version 2 of the License, or
6
 *  (at your option) any later version.
7
 *
8
 *  This program is distributed in the hope that it will be useful,
9
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
10
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
 *  GNU General Public License for more details.
12
 *
13
 *  You should have received a copy of the GNU General Public License
14
 *  along with this program; if not, write to the Free Software
15
 *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
16
 */
17
 
7676 bpr 18
/* Sort dictionary */
10 reyssat 19
 
20
/*************** Customization: change values hereafter ****************/
21
 
7676 bpr 22
/* limit of dictionary entries */
10 reyssat 23
#define entrylim 512*1024
7676 bpr 24
/* limit of dictionary length */
25
#define diclim 32*1024*1024
26
/* separation character */
10 reyssat 27
char sepchar=':', grpchar=0;
28
 
29
/***************** Nothing should need change hereafter *****************/
30
 
8185 bpr 31
#include "../Lib/libwims.h"
8123 bpr 32
#include "suffix.h"
10 reyssat 33
 
34
char inpbuf[MAX_LINELEN+1], outbuf[2*MAX_LINELEN+2];
35
char *dicbuf;
36
char dicname[1024], suffixname[1024];
37
 
38
struct entry {
12248 bpr 39
  char *original;
40
  char *replace;
10 reyssat 41
} entry[entrylim];
42
int entrycount;
43
 
44
int nocase=0, hassuffix=0, leaveline=0;
45
int entrycount, ocount;
46
 
47
int compare(const void *s1, const void *s2)
48
{
12248 bpr 49
  const struct entry *p1, *p2;
50
  p1=s1; p2=s2;
51
  if(nocase) return strcasecmp(p1->original,p2->original);
52
  else return strcmp(p1->original,p2->original);
10 reyssat 53
}
54
 
55
void sortdic(void)
56
{
12248 bpr 57
  qsort(entry,entrycount,sizeof(entry[0]),compare);
10 reyssat 58
}
59
 
7676 bpr 60
/* change all spaces into ' ', and collapse multiple occurences */
8100 bpr 61
void singlespace2(char *p)
10 reyssat 62
{
12248 bpr 63
  char *pp, *p2;
64
  for(pp=p;*pp;pp++) {
65
    if(!isspace(*pp)) continue;
66
    if(leaveline) {
67
      if(*pp==13) ovlstrcpy(pp,pp+1);
68
      if(*pp=='\n') {
69
        pp++;
70
        gopt: for(p2=pp; isspace(*p2) && *p2!='\n'; p2++);
71
        if(p2>pp) ovlstrcpy(pp,p2);
72
        pp--;
7676 bpr 73
      }
74
      else {
12248 bpr 75
        pp++;
76
        if(!isspace(*pp) || *pp=='\n') continue;
77
        goto gopt;
7676 bpr 78
      }
10 reyssat 79
    }
12248 bpr 80
    else {
81
      if(*pp!=' ') *pp=' ';
82
      pp++; if(!isspace(*pp)) continue;
83
      for(p2=pp;isspace(*p2);p2++);
84
      ovlstrcpy(pp,p2); pp--;
85
    }
86
  }
10 reyssat 87
}
88
 
7676 bpr 89
/* Prepare dictionary */
8898 bpr 90
static
91
void prepare_dics(void)
10 reyssat 92
{
12248 bpr 93
  int i;
94
  FILE *dicf;
95
  char *p1, *p2, *pp;
96
  long int flen;
10 reyssat 97
 
12248 bpr 98
  entrycount=0;
99
  dicf=fopen(dicname,"r"); if(dicf==NULL) return;
100
  fseek(dicf,0,SEEK_END);flen=ftell(dicf); fseek(dicf,0,SEEK_SET);
101
  if(flen>diclim) return;
102
  dicbuf=xmalloc(2*flen+1024);flen=fread(dicbuf,1,flen,dicf);
103
  fclose(dicf);
104
  if(flen>0 && flen<diclim) dicbuf[flen]=0;
105
  else return;
106
  for(i=0,p1=dicbuf;p1!=NULL && *p1!=0 && i<entrylim;p1=p2) {
107
    p2=strchr(p1+1,'\n'); if(p2>p1) *p2++=0;
108
    pp=strchr(p1,sepchar); if(pp==NULL) continue;
109
    *pp++=0;
110
    strip_trailing_spaces2(p1); strip_trailing_spaces2(pp);
111
    singlespace2(p1);
112
    p1=find_word_start(p1); pp=find_word_start(pp);
113
    if(*p1==0) continue;
114
    entry[i].original=p1; entry[i].replace=pp; i++;
115
  }
116
  entrycount=i;
10 reyssat 117
}
118
 
119
void output(void)
120
{
12248 bpr 121
  int i;
122
  FILE *f;
10 reyssat 123
 
12248 bpr 124
  ocount=0;
125
  strcat(dicname,".sorted");
126
  f=fopen(dicname,"w"); if(f==NULL) return;
127
  for(i=0;i<entrycount;i++) {
128
    if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0
129
       && strcmp(entry[i].replace,entry[i-1].replace)==0)
130
      continue;
131
    if(grpchar!=0) {
132
      if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0)
133
        fprintf(f,"%c%s",grpchar, entry[i].replace);
7676 bpr 134
      else {
12248 bpr 135
        if(i>0) fprintf(f,"\n");
136
        fprintf(f,"%s%c%s",entry[i].original,sepchar,entry[i].replace);
137
        ocount++;
7676 bpr 138
      }
10 reyssat 139
    }
12248 bpr 140
    else {
141
      fprintf(f,"%s%c%s\n",entry[i].original,sepchar,entry[i].replace);
142
      ocount++;
143
    }
144
  }
145
  if(grpchar!=0) fprintf(f,"\n");
146
  fclose(f);
10 reyssat 147
}
148
 
149
int main(int argc, char *argv[])
150
{
12248 bpr 151
  char *ss, *gr;
152
  if(argc<2) return -1;
7676 bpr 153
 
12248 bpr 154
  ss=getenv("dicsort_separator");
155
  if(ss!=NULL && *ss!=0) sepchar=*ss;
156
  gr=getenv("dicsort_grouping");
157
  if(gr!=NULL && *gr!=0) grpchar=*gr;
158
  snprintf(dicname,sizeof(dicname)-128,"%s",argv[1]); prepare_dics();
159
  if(argc>2) {
160
    snprintf(suffixname,sizeof(suffixname),"%s",argv[2]);
161
    suffix_dic(suffixname); hassuffix=1;
162
  }
163
  else suffixname[0]=hassuffix=0;
164
  sortdic(); output();
165
  printf("%s: sorted %d entries.\n",dicname, ocount);
166
  return 0;
10 reyssat 167
}
168