Rev 8149 | Rev 8863 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
10 | reyssat | 1 | /* Copyright (C) 1998-2003 XIAO, Gang of Universite de Nice - Sophia Antipolis |
2 | * |
||
3 | * This program is free software; you can redistribute it and/or modify |
||
4 | * it under the terms of the GNU General Public License as published by |
||
5 | * the Free Software Foundation; either version 2 of the License, or |
||
6 | * (at your option) any later version. |
||
7 | * |
||
8 | * This program is distributed in the hope that it will be useful, |
||
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
||
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||
11 | * GNU General Public License for more details. |
||
12 | * |
||
13 | * You should have received a copy of the GNU General Public License |
||
14 | * along with this program; if not, write to the Free Software |
||
15 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
||
16 | */ |
||
17 | |||
7676 | bpr | 18 | /* Sort dictionary */ |
10 | reyssat | 19 | |
20 | /*************** Customization: change values hereafter ****************/ |
||
21 | |||
7676 | bpr | 22 | /* limit of dictionary entries */ |
10 | reyssat | 23 | #define entrylim 512*1024 |
7676 | bpr | 24 | /* limit of dictionary length */ |
25 | #define diclim 32*1024*1024 |
||
26 | /* separation character */ |
||
10 | reyssat | 27 | char sepchar=':', grpchar=0; |
28 | |||
29 | /***************** Nothing should need change hereafter *****************/ |
||
30 | |||
8185 | bpr | 31 | #include "../Lib/libwims.h" |
8123 | bpr | 32 | #include "suffix.h" |
10 | reyssat | 33 | |
34 | char inpbuf[MAX_LINELEN+1], outbuf[2*MAX_LINELEN+2]; |
||
35 | char *dicbuf; |
||
36 | char dicname[1024], suffixname[1024]; |
||
37 | |||
38 | struct entry { |
||
39 | char *original; |
||
40 | char *replace; |
||
41 | } entry[entrylim]; |
||
42 | int entrycount; |
||
43 | |||
44 | int nocase=0, hassuffix=0, leaveline=0; |
||
45 | int entrycount, ocount; |
||
46 | |||
7676 | bpr | 47 | /* strip trailing spaces; return string end. */ |
8100 | bpr | 48 | char *strip_trailing_spaces2(char *p) |
10 | reyssat | 49 | { |
50 | char *pp; |
||
51 | if(*p==0) return p; |
||
52 | for(pp=p+strlen(p)-1; pp>=p && isspace(*pp); *(pp--)=0); |
||
53 | return pp; |
||
54 | } |
||
55 | |||
56 | int compare(const void *s1, const void *s2) |
||
57 | { |
||
58 | const struct entry *p1, *p2; |
||
59 | p1=s1; p2=s2; |
||
60 | if(nocase) return strcasecmp(p1->original,p2->original); |
||
61 | else return strcmp(p1->original,p2->original); |
||
62 | } |
||
63 | |||
64 | void sortdic(void) |
||
65 | { |
||
66 | qsort(entry,entrycount,sizeof(entry[0]),compare); |
||
67 | } |
||
68 | |||
8100 | bpr | 69 | /* modify a string. Bufferlen must be at least MAX_LINELEN */ |
70 | void string_modify3(char *start, char *bad_beg, char *bad_end, char *good,...) |
||
10 | reyssat | 71 | { |
72 | char buf[MAX_LINELEN+1]; |
||
73 | va_list vp; |
||
7676 | bpr | 74 | |
10 | reyssat | 75 | va_start(vp,good); |
76 | vsnprintf(buf,sizeof(buf),good,vp); va_end(vp); |
||
77 | if(strlen(start)-(bad_end-bad_beg)+strlen(buf)>=MAX_LINELEN) |
||
7676 | bpr | 78 | return; /* this is an error situation. */ |
10 | reyssat | 79 | strcat(buf,bad_end); |
3718 | reyssat | 80 | ovlstrcpy(bad_beg,buf); |
10 | reyssat | 81 | } |
82 | |||
7676 | bpr | 83 | /* change all spaces into ' ', and collapse multiple occurences */ |
8100 | bpr | 84 | void singlespace2(char *p) |
10 | reyssat | 85 | { |
86 | char *pp, *p2; |
||
87 | for(pp=p;*pp;pp++) { |
||
7676 | bpr | 88 | if(!isspace(*pp)) continue; |
89 | if(leaveline) { |
||
90 | if(*pp==13) ovlstrcpy(pp,pp+1); |
||
91 | if(*pp=='\n') { |
||
92 | pp++; |
||
93 | gopt: for(p2=pp; isspace(*p2) && *p2!='\n'; p2++); |
||
94 | if(p2>pp) ovlstrcpy(pp,p2); pp--; |
||
95 | } |
||
96 | else { |
||
97 | pp++; if(!isspace(*pp) || *pp=='\n') continue; |
||
98 | goto gopt; |
||
99 | } |
||
100 | } |
||
101 | else { |
||
102 | if(*pp!=' ') *pp=' '; |
||
103 | pp++; if(!isspace(*pp)) continue; |
||
104 | for(p2=pp;isspace(*p2);p2++); |
||
105 | ovlstrcpy(pp,p2); pp--; |
||
106 | } |
||
10 | reyssat | 107 | } |
108 | } |
||
109 | |||
7676 | bpr | 110 | /* Prepare dictionary */ |
10 | reyssat | 111 | void prepare_dic(void) |
112 | { |
||
113 | int i; |
||
114 | FILE *dicf; |
||
115 | char *p1, *p2, *pp; |
||
116 | long int flen; |
||
117 | |||
118 | entrycount=0; |
||
119 | dicf=fopen(dicname,"r"); if(dicf==NULL) return; |
||
120 | fseek(dicf,0,SEEK_END);flen=ftell(dicf); fseek(dicf,0,SEEK_SET); |
||
121 | if(flen>diclim) return; |
||
122 | dicbuf=xmalloc(2*flen+1024);flen=fread(dicbuf,1,flen,dicf); |
||
123 | fclose(dicf); |
||
124 | if(flen>0 && flen<diclim) dicbuf[flen]=0; |
||
125 | else return; |
||
126 | for(i=0,p1=dicbuf;p1!=NULL && *p1!=0 && i<entrylim;p1=p2) { |
||
7676 | bpr | 127 | p2=strchr(p1+1,'\n'); if(p2>p1) *p2++=0; |
128 | pp=strchr(p1,sepchar); if(pp==NULL) continue; |
||
129 | *pp++=0; |
||
8100 | bpr | 130 | strip_trailing_spaces2(p1); strip_trailing_spaces2(pp); |
131 | singlespace2(p1); |
||
7676 | bpr | 132 | p1=find_word_start(p1); pp=find_word_start(pp); |
133 | if(*p1==0) continue; |
||
134 | entry[i].original=p1; entry[i].replace=pp; i++; |
||
10 | reyssat | 135 | } |
136 | entrycount=i; |
||
137 | } |
||
138 | |||
139 | void output(void) |
||
140 | { |
||
141 | int i; |
||
142 | FILE *f; |
||
143 | |||
144 | ocount=0; |
||
145 | strcat(dicname,".sorted"); |
||
146 | f=fopen(dicname,"w"); if(f==NULL) return; |
||
147 | for(i=0;i<entrycount;i++) { |
||
7676 | bpr | 148 | if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0 |
149 | && strcmp(entry[i].replace,entry[i-1].replace)==0) |
||
150 | continue; |
||
151 | if(grpchar!=0) { |
||
152 | if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0) |
||
153 | fprintf(f,"%c%s",grpchar, entry[i].replace); |
||
154 | else { |
||
155 | if(i>0) fprintf(f,"\n"); |
||
156 | fprintf(f,"%s%c%s",entry[i].original,sepchar,entry[i].replace); |
||
157 | ocount++; |
||
158 | } |
||
159 | |||
160 | } |
||
161 | else { |
||
162 | fprintf(f,"%s%c%s\n",entry[i].original,sepchar,entry[i].replace); |
||
163 | ocount++; |
||
164 | } |
||
10 | reyssat | 165 | } |
166 | if(grpchar!=0) fprintf(f,"\n"); |
||
167 | fclose(f); |
||
168 | } |
||
169 | |||
170 | int main(int argc, char *argv[]) |
||
171 | { |
||
172 | char *ss, *gr; |
||
173 | if(argc<2) return -1; |
||
7676 | bpr | 174 | |
10 | reyssat | 175 | ss=getenv("dicsort_separator"); |
176 | if(ss!=NULL && *ss!=0) sepchar=*ss; |
||
177 | gr=getenv("dicsort_grouping"); |
||
178 | if(gr!=NULL && *gr!=0) grpchar=*gr; |
||
179 | snprintf(dicname,sizeof(dicname)-128,"%s",argv[1]); prepare_dic(); |
||
180 | if(argc>2) { |
||
7676 | bpr | 181 | snprintf(suffixname,sizeof(suffixname),"%s",argv[2]); |
182 | suffix_dic(suffixname); hassuffix=1; |
||
10 | reyssat | 183 | } |
184 | else suffixname[0]=hassuffix=0; |
||
185 | sortdic(); output(); |
||
186 | printf("%s: sorted %d entries.\n",dicname, ocount); |
||
187 | return 0; |
||
188 | } |
||
189 |