Rev 11132 | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
10 | reyssat | 1 | /* Copyright (C) 1998-2003 XIAO, Gang of Universite de Nice - Sophia Antipolis |
2 | * |
||
3 | * This program is free software; you can redistribute it and/or modify |
||
4 | * it under the terms of the GNU General Public License as published by |
||
5 | * the Free Software Foundation; either version 2 of the License, or |
||
6 | * (at your option) any later version. |
||
7 | * |
||
8 | * This program is distributed in the hope that it will be useful, |
||
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
||
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||
11 | * GNU General Public License for more details. |
||
12 | * |
||
13 | * You should have received a copy of the GNU General Public License |
||
14 | * along with this program; if not, write to the Free Software |
||
15 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
||
16 | */ |
||
17 | |||
7676 | bpr | 18 | /* Sort dictionary */ |
10 | reyssat | 19 | |
20 | /*************** Customization: change values hereafter ****************/ |
||
21 | |||
7676 | bpr | 22 | /* limit of dictionary entries */ |
10 | reyssat | 23 | #define entrylim 512*1024 |
7676 | bpr | 24 | /* limit of dictionary length */ |
25 | #define diclim 32*1024*1024 |
||
26 | /* separation character */ |
||
10 | reyssat | 27 | char sepchar=':', grpchar=0; |
28 | |||
29 | /***************** Nothing should need change hereafter *****************/ |
||
30 | |||
8185 | bpr | 31 | #include "../Lib/libwims.h" |
8123 | bpr | 32 | #include "suffix.h" |
10 | reyssat | 33 | |
34 | char inpbuf[MAX_LINELEN+1], outbuf[2*MAX_LINELEN+2]; |
||
35 | char *dicbuf; |
||
36 | char dicname[1024], suffixname[1024]; |
||
37 | |||
38 | struct entry { |
||
12248 | bpr | 39 | char *original; |
40 | char *replace; |
||
10 | reyssat | 41 | } entry[entrylim]; |
42 | int entrycount; |
||
43 | |||
44 | int nocase=0, hassuffix=0, leaveline=0; |
||
45 | int entrycount, ocount; |
||
46 | |||
47 | int compare(const void *s1, const void *s2) |
||
48 | { |
||
12248 | bpr | 49 | const struct entry *p1, *p2; |
50 | p1=s1; p2=s2; |
||
51 | if(nocase) return strcasecmp(p1->original,p2->original); |
||
52 | else return strcmp(p1->original,p2->original); |
||
10 | reyssat | 53 | } |
54 | |||
55 | void sortdic(void) |
||
56 | { |
||
12248 | bpr | 57 | qsort(entry,entrycount,sizeof(entry[0]),compare); |
10 | reyssat | 58 | } |
59 | |||
7676 | bpr | 60 | /* change all spaces into ' ', and collapse multiple occurences */ |
8100 | bpr | 61 | void singlespace2(char *p) |
10 | reyssat | 62 | { |
12248 | bpr | 63 | char *pp, *p2; |
64 | for(pp=p;*pp;pp++) { |
||
65 | if(!isspace(*pp)) continue; |
||
66 | if(leaveline) { |
||
67 | if(*pp==13) ovlstrcpy(pp,pp+1); |
||
68 | if(*pp=='\n') { |
||
69 | pp++; |
||
70 | gopt: for(p2=pp; isspace(*p2) && *p2!='\n'; p2++); |
||
71 | if(p2>pp) ovlstrcpy(pp,p2); |
||
72 | pp--; |
||
7676 | bpr | 73 | } |
74 | else { |
||
12248 | bpr | 75 | pp++; |
76 | if(!isspace(*pp) || *pp=='\n') continue; |
||
77 | goto gopt; |
||
7676 | bpr | 78 | } |
10 | reyssat | 79 | } |
12248 | bpr | 80 | else { |
81 | if(*pp!=' ') *pp=' '; |
||
82 | pp++; if(!isspace(*pp)) continue; |
||
83 | for(p2=pp;isspace(*p2);p2++); |
||
84 | ovlstrcpy(pp,p2); pp--; |
||
85 | } |
||
86 | } |
||
10 | reyssat | 87 | } |
88 | |||
7676 | bpr | 89 | /* Prepare dictionary */ |
8898 | bpr | 90 | static |
91 | void prepare_dics(void) |
||
10 | reyssat | 92 | { |
12248 | bpr | 93 | int i; |
94 | FILE *dicf; |
||
95 | char *p1, *p2, *pp; |
||
96 | long int flen; |
||
10 | reyssat | 97 | |
12248 | bpr | 98 | entrycount=0; |
99 | dicf=fopen(dicname,"r"); if(dicf==NULL) return; |
||
100 | fseek(dicf,0,SEEK_END);flen=ftell(dicf); fseek(dicf,0,SEEK_SET); |
||
101 | if(flen>diclim) return; |
||
102 | dicbuf=xmalloc(2*flen+1024);flen=fread(dicbuf,1,flen,dicf); |
||
103 | fclose(dicf); |
||
104 | if(flen>0 && flen<diclim) dicbuf[flen]=0; |
||
105 | else return; |
||
106 | for(i=0,p1=dicbuf;p1!=NULL && *p1!=0 && i<entrylim;p1=p2) { |
||
107 | p2=strchr(p1+1,'\n'); if(p2>p1) *p2++=0; |
||
108 | pp=strchr(p1,sepchar); if(pp==NULL) continue; |
||
109 | *pp++=0; |
||
110 | strip_trailing_spaces2(p1); strip_trailing_spaces2(pp); |
||
111 | singlespace2(p1); |
||
112 | p1=find_word_start(p1); pp=find_word_start(pp); |
||
113 | if(*p1==0) continue; |
||
114 | entry[i].original=p1; entry[i].replace=pp; i++; |
||
115 | } |
||
116 | entrycount=i; |
||
10 | reyssat | 117 | } |
118 | |||
119 | void output(void) |
||
120 | { |
||
12248 | bpr | 121 | int i; |
122 | FILE *f; |
||
10 | reyssat | 123 | |
12248 | bpr | 124 | ocount=0; |
125 | strcat(dicname,".sorted"); |
||
126 | f=fopen(dicname,"w"); if(f==NULL) return; |
||
127 | for(i=0;i<entrycount;i++) { |
||
128 | if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0 |
||
129 | && strcmp(entry[i].replace,entry[i-1].replace)==0) |
||
130 | continue; |
||
131 | if(grpchar!=0) { |
||
132 | if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0) |
||
133 | fprintf(f,"%c%s",grpchar, entry[i].replace); |
||
7676 | bpr | 134 | else { |
12248 | bpr | 135 | if(i>0) fprintf(f,"\n"); |
136 | fprintf(f,"%s%c%s",entry[i].original,sepchar,entry[i].replace); |
||
137 | ocount++; |
||
7676 | bpr | 138 | } |
10 | reyssat | 139 | } |
12248 | bpr | 140 | else { |
141 | fprintf(f,"%s%c%s\n",entry[i].original,sepchar,entry[i].replace); |
||
142 | ocount++; |
||
143 | } |
||
144 | } |
||
145 | if(grpchar!=0) fprintf(f,"\n"); |
||
146 | fclose(f); |
||
10 | reyssat | 147 | } |
148 | |||
149 | int main(int argc, char *argv[]) |
||
150 | { |
||
12248 | bpr | 151 | char *ss, *gr; |
152 | if(argc<2) return -1; |
||
7676 | bpr | 153 | |
12248 | bpr | 154 | ss=getenv("dicsort_separator"); |
155 | if(ss!=NULL && *ss!=0) sepchar=*ss; |
||
156 | gr=getenv("dicsort_grouping"); |
||
157 | if(gr!=NULL && *gr!=0) grpchar=*gr; |
||
158 | snprintf(dicname,sizeof(dicname)-128,"%s",argv[1]); prepare_dics(); |
||
159 | if(argc>2) { |
||
160 | snprintf(suffixname,sizeof(suffixname),"%s",argv[2]); |
||
161 | suffix_dic(suffixname); hassuffix=1; |
||
162 | } |
||
163 | else suffixname[0]=hassuffix=0; |
||
164 | sortdic(); output(); |
||
165 | printf("%s: sorted %d entries.\n",dicname, ocount); |
||
166 | return 0; |
||
10 | reyssat | 167 | } |
168 |