Rev 8863 | Rev 11124 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
10 | reyssat | 1 | /* Copyright (C) 1998-2003 XIAO, Gang of Universite de Nice - Sophia Antipolis |
2 | * |
||
3 | * This program is free software; you can redistribute it and/or modify |
||
4 | * it under the terms of the GNU General Public License as published by |
||
5 | * the Free Software Foundation; either version 2 of the License, or |
||
6 | * (at your option) any later version. |
||
7 | * |
||
8 | * This program is distributed in the hope that it will be useful, |
||
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
||
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||
11 | * GNU General Public License for more details. |
||
12 | * |
||
13 | * You should have received a copy of the GNU General Public License |
||
14 | * along with this program; if not, write to the Free Software |
||
15 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
||
16 | */ |
||
17 | |||
7676 | bpr | 18 | /* Sort dictionary */ |
10 | reyssat | 19 | |
20 | /*************** Customization: change values hereafter ****************/ |
||
21 | |||
7676 | bpr | 22 | /* limit of dictionary entries */ |
10 | reyssat | 23 | #define entrylim 512*1024 |
7676 | bpr | 24 | /* limit of dictionary length */ |
25 | #define diclim 32*1024*1024 |
||
26 | /* separation character */ |
||
10 | reyssat | 27 | char sepchar=':', grpchar=0; |
28 | |||
29 | /***************** Nothing should need change hereafter *****************/ |
||
30 | |||
8185 | bpr | 31 | #include "../Lib/libwims.h" |
8123 | bpr | 32 | #include "suffix.h" |
10 | reyssat | 33 | |
34 | char inpbuf[MAX_LINELEN+1], outbuf[2*MAX_LINELEN+2]; |
||
35 | char *dicbuf; |
||
36 | char dicname[1024], suffixname[1024]; |
||
37 | |||
38 | struct entry { |
||
39 | char *original; |
||
40 | char *replace; |
||
41 | } entry[entrylim]; |
||
42 | int entrycount; |
||
43 | |||
44 | int nocase=0, hassuffix=0, leaveline=0; |
||
45 | int entrycount, ocount; |
||
46 | |||
47 | int compare(const void *s1, const void *s2) |
||
48 | { |
||
49 | const struct entry *p1, *p2; |
||
50 | p1=s1; p2=s2; |
||
51 | if(nocase) return strcasecmp(p1->original,p2->original); |
||
52 | else return strcmp(p1->original,p2->original); |
||
53 | } |
||
54 | |||
55 | void sortdic(void) |
||
56 | { |
||
57 | qsort(entry,entrycount,sizeof(entry[0]),compare); |
||
58 | } |
||
59 | |||
7676 | bpr | 60 | /* change all spaces into ' ', and collapse multiple occurences */ |
8100 | bpr | 61 | void singlespace2(char *p) |
10 | reyssat | 62 | { |
63 | char *pp, *p2; |
||
64 | for(pp=p;*pp;pp++) { |
||
7676 | bpr | 65 | if(!isspace(*pp)) continue; |
66 | if(leaveline) { |
||
67 | if(*pp==13) ovlstrcpy(pp,pp+1); |
||
68 | if(*pp=='\n') { |
||
69 | pp++; |
||
70 | gopt: for(p2=pp; isspace(*p2) && *p2!='\n'; p2++); |
||
71 | if(p2>pp) ovlstrcpy(pp,p2); pp--; |
||
72 | } |
||
73 | else { |
||
74 | pp++; if(!isspace(*pp) || *pp=='\n') continue; |
||
75 | goto gopt; |
||
76 | } |
||
77 | } |
||
78 | else { |
||
79 | if(*pp!=' ') *pp=' '; |
||
80 | pp++; if(!isspace(*pp)) continue; |
||
81 | for(p2=pp;isspace(*p2);p2++); |
||
82 | ovlstrcpy(pp,p2); pp--; |
||
83 | } |
||
10 | reyssat | 84 | } |
85 | } |
||
86 | |||
7676 | bpr | 87 | /* Prepare dictionary */ |
8898 | bpr | 88 | static |
89 | void prepare_dics(void) |
||
10 | reyssat | 90 | { |
91 | int i; |
||
92 | FILE *dicf; |
||
93 | char *p1, *p2, *pp; |
||
94 | long int flen; |
||
95 | |||
96 | entrycount=0; |
||
97 | dicf=fopen(dicname,"r"); if(dicf==NULL) return; |
||
98 | fseek(dicf,0,SEEK_END);flen=ftell(dicf); fseek(dicf,0,SEEK_SET); |
||
99 | if(flen>diclim) return; |
||
100 | dicbuf=xmalloc(2*flen+1024);flen=fread(dicbuf,1,flen,dicf); |
||
101 | fclose(dicf); |
||
102 | if(flen>0 && flen<diclim) dicbuf[flen]=0; |
||
103 | else return; |
||
104 | for(i=0,p1=dicbuf;p1!=NULL && *p1!=0 && i<entrylim;p1=p2) { |
||
7676 | bpr | 105 | p2=strchr(p1+1,'\n'); if(p2>p1) *p2++=0; |
106 | pp=strchr(p1,sepchar); if(pp==NULL) continue; |
||
107 | *pp++=0; |
||
8100 | bpr | 108 | strip_trailing_spaces2(p1); strip_trailing_spaces2(pp); |
109 | singlespace2(p1); |
||
7676 | bpr | 110 | p1=find_word_start(p1); pp=find_word_start(pp); |
111 | if(*p1==0) continue; |
||
112 | entry[i].original=p1; entry[i].replace=pp; i++; |
||
10 | reyssat | 113 | } |
114 | entrycount=i; |
||
115 | } |
||
116 | |||
117 | void output(void) |
||
118 | { |
||
119 | int i; |
||
120 | FILE *f; |
||
121 | |||
122 | ocount=0; |
||
123 | strcat(dicname,".sorted"); |
||
124 | f=fopen(dicname,"w"); if(f==NULL) return; |
||
125 | for(i=0;i<entrycount;i++) { |
||
7676 | bpr | 126 | if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0 |
127 | && strcmp(entry[i].replace,entry[i-1].replace)==0) |
||
128 | continue; |
||
129 | if(grpchar!=0) { |
||
130 | if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0) |
||
131 | fprintf(f,"%c%s",grpchar, entry[i].replace); |
||
132 | else { |
||
133 | if(i>0) fprintf(f,"\n"); |
||
134 | fprintf(f,"%s%c%s",entry[i].original,sepchar,entry[i].replace); |
||
135 | ocount++; |
||
136 | } |
||
137 | |||
138 | } |
||
139 | else { |
||
140 | fprintf(f,"%s%c%s\n",entry[i].original,sepchar,entry[i].replace); |
||
141 | ocount++; |
||
142 | } |
||
10 | reyssat | 143 | } |
144 | if(grpchar!=0) fprintf(f,"\n"); |
||
145 | fclose(f); |
||
146 | } |
||
147 | |||
148 | int main(int argc, char *argv[]) |
||
149 | { |
||
150 | char *ss, *gr; |
||
151 | if(argc<2) return -1; |
||
7676 | bpr | 152 | |
10 | reyssat | 153 | ss=getenv("dicsort_separator"); |
154 | if(ss!=NULL && *ss!=0) sepchar=*ss; |
||
155 | gr=getenv("dicsort_grouping"); |
||
156 | if(gr!=NULL && *gr!=0) grpchar=*gr; |
||
8898 | bpr | 157 | snprintf(dicname,sizeof(dicname)-128,"%s",argv[1]); prepare_dics(); |
10 | reyssat | 158 | if(argc>2) { |
7676 | bpr | 159 | snprintf(suffixname,sizeof(suffixname),"%s",argv[2]); |
160 | suffix_dic(suffixname); hassuffix=1; |
||
10 | reyssat | 161 | } |
162 | else suffixname[0]=hassuffix=0; |
||
163 | sortdic(); output(); |
||
164 | printf("%s: sorted %d entries.\n",dicname, ocount); |
||
165 | return 0; |
||
166 | } |
||
167 |