Rev 10 | Rev 7676 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
10 | reyssat | 1 | /* Copyright (C) 1998-2003 XIAO, Gang of Universite de Nice - Sophia Antipolis |
2 | * |
||
3 | * This program is free software; you can redistribute it and/or modify |
||
4 | * it under the terms of the GNU General Public License as published by |
||
5 | * the Free Software Foundation; either version 2 of the License, or |
||
6 | * (at your option) any later version. |
||
7 | * |
||
8 | * This program is distributed in the hope that it will be useful, |
||
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
||
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||
11 | * GNU General Public License for more details. |
||
12 | * |
||
13 | * You should have received a copy of the GNU General Public License |
||
14 | * along with this program; if not, write to the Free Software |
||
15 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
||
16 | */ |
||
17 | |||
18 | /* Sort dictionary */ |
||
19 | |||
20 | /*************** Customization: change values hereafter ****************/ |
||
21 | |||
22 | /* limit of dictionary entries */ |
||
23 | #define entrylim 512*1024 |
||
24 | /* limit of dictionary length */ |
||
25 | #define diclim 32*1024*1024 |
||
26 | /* separation character */ |
||
27 | char sepchar=':', grpchar=0; |
||
28 | |||
29 | /***************** Nothing should need change hereafter *****************/ |
||
30 | |||
31 | #include "../wims.h" |
||
3718 | reyssat | 32 | #include "../Lib/basicstr.c" |
10 | reyssat | 33 | |
34 | char inpbuf[MAX_LINELEN+1], outbuf[2*MAX_LINELEN+2]; |
||
35 | char *dicbuf; |
||
36 | char dicname[1024], suffixname[1024]; |
||
37 | |||
38 | struct entry { |
||
39 | char *original; |
||
40 | char *replace; |
||
41 | } entry[entrylim]; |
||
42 | int entrycount; |
||
43 | |||
44 | int nocase=0, hassuffix=0, leaveline=0; |
||
45 | int entrycount, ocount; |
||
46 | |||
47 | void *xmalloc(size_t n) |
||
48 | { |
||
49 | void *p; |
||
50 | p=malloc(n); |
||
51 | if(p==NULL) exit(1); |
||
52 | return p; |
||
53 | } |
||
54 | |||
55 | /* Points to the end of the word */ |
||
56 | char *find_word_end(char *p) |
||
57 | { |
||
58 | int i; |
||
59 | for(i=0;!isspace(*p) && *p!=0 && i<MAX_LINELEN; p++,i++); |
||
60 | return p; |
||
61 | } |
||
62 | |||
63 | /* Strips leading spaces */ |
||
64 | char *find_word_start(char *p) |
||
65 | { |
||
66 | int i; |
||
67 | for(i=0; isspace(*p) && i<MAX_LINELEN; p++,i++); |
||
68 | return p; |
||
69 | } |
||
70 | |||
71 | /* strip trailing spaces; return string end. */ |
||
72 | char *strip_trailing_spaces(char *p) |
||
73 | { |
||
74 | char *pp; |
||
75 | if(*p==0) return p; |
||
76 | for(pp=p+strlen(p)-1; pp>=p && isspace(*pp); *(pp--)=0); |
||
77 | return pp; |
||
78 | } |
||
79 | |||
80 | int compare(const void *s1, const void *s2) |
||
81 | { |
||
82 | const struct entry *p1, *p2; |
||
83 | p1=s1; p2=s2; |
||
84 | if(nocase) return strcasecmp(p1->original,p2->original); |
||
85 | else return strcmp(p1->original,p2->original); |
||
86 | } |
||
87 | |||
88 | void sortdic(void) |
||
89 | { |
||
90 | qsort(entry,entrycount,sizeof(entry[0]),compare); |
||
91 | } |
||
92 | |||
93 | /* modify a string. Bufferlen must be ast least MAX_LINELEN */ |
||
94 | void string_modify(char *start, char *bad_beg, char *bad_end, char *good,...) |
||
95 | { |
||
96 | char buf[MAX_LINELEN+1]; |
||
97 | va_list vp; |
||
98 | |||
99 | va_start(vp,good); |
||
100 | vsnprintf(buf,sizeof(buf),good,vp); va_end(vp); |
||
101 | if(strlen(start)-(bad_end-bad_beg)+strlen(buf)>=MAX_LINELEN) |
||
102 | return; /* this is an error situation. */ |
||
103 | strcat(buf,bad_end); |
||
3718 | reyssat | 104 | ovlstrcpy(bad_beg,buf); |
10 | reyssat | 105 | } |
106 | |||
107 | /* change all spaces into ' ', and collapse multiple occurences */ |
||
108 | void singlespace(char *p) |
||
109 | { |
||
110 | char *pp, *p2; |
||
111 | for(pp=p;*pp;pp++) { |
||
112 | if(!isspace(*pp)) continue; |
||
113 | if(leaveline) { |
||
3718 | reyssat | 114 | if(*pp==13) ovlstrcpy(pp,pp+1); |
10 | reyssat | 115 | if(*pp=='\n') { |
116 | pp++; |
||
117 | gopt: for(p2=pp; isspace(*p2) && *p2!='\n'; p2++); |
||
3718 | reyssat | 118 | if(p2>pp) ovlstrcpy(pp,p2); pp--; |
10 | reyssat | 119 | } |
120 | else { |
||
121 | pp++; if(!isspace(*pp) || *pp=='\n') continue; |
||
122 | goto gopt; |
||
123 | } |
||
124 | } |
||
125 | else { |
||
126 | if(*pp!=' ') *pp=' '; |
||
127 | pp++; if(!isspace(*pp)) continue; |
||
128 | for(p2=pp;isspace(*p2);p2++); |
||
3718 | reyssat | 129 | ovlstrcpy(pp,p2); pp--; |
10 | reyssat | 130 | } |
131 | } |
||
132 | } |
||
133 | |||
134 | /* Prepare dictionary */ |
||
135 | void prepare_dic(void) |
||
136 | { |
||
137 | int i; |
||
138 | FILE *dicf; |
||
139 | char *p1, *p2, *pp; |
||
140 | long int flen; |
||
141 | |||
142 | entrycount=0; |
||
143 | dicf=fopen(dicname,"r"); if(dicf==NULL) return; |
||
144 | fseek(dicf,0,SEEK_END);flen=ftell(dicf); fseek(dicf,0,SEEK_SET); |
||
145 | if(flen>diclim) return; |
||
146 | dicbuf=xmalloc(2*flen+1024);flen=fread(dicbuf,1,flen,dicf); |
||
147 | fclose(dicf); |
||
148 | if(flen>0 && flen<diclim) dicbuf[flen]=0; |
||
149 | else return; |
||
150 | for(i=0,p1=dicbuf;p1!=NULL && *p1!=0 && i<entrylim;p1=p2) { |
||
151 | p2=strchr(p1+1,'\n'); if(p2>p1) *p2++=0; |
||
152 | pp=strchr(p1,sepchar); if(pp==NULL) continue; |
||
153 | *pp++=0; |
||
154 | strip_trailing_spaces(p1); strip_trailing_spaces(pp); |
||
155 | singlespace(p1); |
||
156 | p1=find_word_start(p1); pp=find_word_start(pp); |
||
157 | if(*p1==0) continue; |
||
158 | entry[i].original=p1; entry[i].replace=pp; i++; |
||
159 | } |
||
160 | entrycount=i; |
||
161 | } |
||
162 | |||
163 | #include "suffix.c" |
||
164 | |||
165 | void output(void) |
||
166 | { |
||
167 | int i; |
||
168 | FILE *f; |
||
169 | |||
170 | ocount=0; |
||
171 | strcat(dicname,".sorted"); |
||
172 | f=fopen(dicname,"w"); if(f==NULL) return; |
||
173 | for(i=0;i<entrycount;i++) { |
||
174 | if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0 |
||
175 | && strcmp(entry[i].replace,entry[i-1].replace)==0) |
||
176 | continue; |
||
177 | if(grpchar!=0) { |
||
178 | if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0) |
||
179 | fprintf(f,"%c%s",grpchar, entry[i].replace); |
||
180 | else { |
||
181 | if(i>0) fprintf(f,"\n"); |
||
182 | fprintf(f,"%s%c%s",entry[i].original,sepchar,entry[i].replace); |
||
183 | ocount++; |
||
184 | } |
||
185 | |||
186 | } |
||
187 | else { |
||
188 | fprintf(f,"%s%c%s\n",entry[i].original,sepchar,entry[i].replace); |
||
189 | ocount++; |
||
190 | } |
||
191 | } |
||
192 | if(grpchar!=0) fprintf(f,"\n"); |
||
193 | fclose(f); |
||
194 | } |
||
195 | |||
196 | int main(int argc, char *argv[]) |
||
197 | { |
||
198 | char *ss, *gr; |
||
199 | if(argc<2) return -1; |
||
200 | |||
201 | ss=getenv("dicsort_separator"); |
||
202 | if(ss!=NULL && *ss!=0) sepchar=*ss; |
||
203 | gr=getenv("dicsort_grouping"); |
||
204 | if(gr!=NULL && *gr!=0) grpchar=*gr; |
||
205 | snprintf(dicname,sizeof(dicname)-128,"%s",argv[1]); prepare_dic(); |
||
206 | if(argc>2) { |
||
207 | snprintf(suffixname,sizeof(suffixname),"%s",argv[2]); |
||
208 | suffix_dic(suffixname); hassuffix=1; |
||
209 | } |
||
210 | else suffixname[0]=hassuffix=0; |
||
211 | sortdic(); output(); |
||
212 | printf("%s: sorted %d entries.\n",dicname, ocount); |
||
213 | return 0; |
||
214 | } |
||
215 |