Subversion Repositories wimsdev

Rev

Rev 8123 | Rev 8185 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
10 reyssat 1
/*    Copyright (C) 1998-2003 XIAO, Gang of Universite de Nice - Sophia Antipolis
2
 *
3
 *  This program is free software; you can redistribute it and/or modify
4
 *  it under the terms of the GNU General Public License as published by
5
 *  the Free Software Foundation; either version 2 of the License, or
6
 *  (at your option) any later version.
7
 *
8
 *  This program is distributed in the hope that it will be useful,
9
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
10
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11
 *  GNU General Public License for more details.
12
 *
13
 *  You should have received a copy of the GNU General Public License
14
 *  along with this program; if not, write to the Free Software
15
 *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
16
 */
17
 
7676 bpr 18
/* Sort dictionary */
10 reyssat 19
 
20
/*************** Customization: change values hereafter ****************/
21
 
7676 bpr 22
/* limit of dictionary entries */
10 reyssat 23
#define entrylim 512*1024
7676 bpr 24
/* limit of dictionary length */
25
#define diclim 32*1024*1024
26
/* separation character */
10 reyssat 27
char sepchar=':', grpchar=0;
28
 
29
/***************** Nothing should need change hereafter *****************/
30
 
8123 bpr 31
#include "suffix.h"
10 reyssat 32
#include "../wims.h"
8100 bpr 33
#include "../Lib/libwims.h"
10 reyssat 34
 
35
char inpbuf[MAX_LINELEN+1], outbuf[2*MAX_LINELEN+2];
36
char *dicbuf;
37
char dicname[1024], suffixname[1024];
38
 
39
struct entry {
40
    char *original;
41
    char *replace;
42
} entry[entrylim];
43
int entrycount;
44
 
45
int nocase=0, hassuffix=0, leaveline=0;
46
int entrycount, ocount;
47
 
7676 bpr 48
/* strip trailing spaces; return string end. */
8100 bpr 49
char *strip_trailing_spaces2(char *p)
10 reyssat 50
{
51
    char *pp;
52
    if(*p==0) return p;
53
    for(pp=p+strlen(p)-1; pp>=p && isspace(*pp); *(pp--)=0);
54
    return pp;
55
}
56
 
57
int compare(const void *s1, const void *s2)
58
{
59
    const struct entry *p1, *p2;
60
    p1=s1; p2=s2;
61
    if(nocase) return strcasecmp(p1->original,p2->original);
62
    else return strcmp(p1->original,p2->original);
63
}
64
 
65
void sortdic(void)
66
{
67
    qsort(entry,entrycount,sizeof(entry[0]),compare);
68
}
69
 
8100 bpr 70
/* modify a string. Bufferlen must be at least MAX_LINELEN */
71
void string_modify3(char *start, char *bad_beg, char *bad_end, char *good,...)
10 reyssat 72
{
73
    char buf[MAX_LINELEN+1];
74
    va_list vp;
7676 bpr 75
 
10 reyssat 76
    va_start(vp,good);
77
    vsnprintf(buf,sizeof(buf),good,vp); va_end(vp);
78
    if(strlen(start)-(bad_end-bad_beg)+strlen(buf)>=MAX_LINELEN)
7676 bpr 79
      return; /* this is an error situation. */
10 reyssat 80
    strcat(buf,bad_end);
3718 reyssat 81
    ovlstrcpy(bad_beg,buf);
10 reyssat 82
}
83
 
7676 bpr 84
/* change all spaces into ' ', and collapse multiple occurences */
8100 bpr 85
void singlespace2(char *p)
10 reyssat 86
{
87
    char *pp, *p2;
88
    for(pp=p;*pp;pp++) {
7676 bpr 89
      if(!isspace(*pp)) continue;
90
      if(leaveline) {
91
          if(*pp==13) ovlstrcpy(pp,pp+1);
92
          if(*pp=='\n') {
93
            pp++;
94
            gopt: for(p2=pp; isspace(*p2) && *p2!='\n'; p2++);
95
            if(p2>pp) ovlstrcpy(pp,p2); pp--;
96
          }
97
          else {
98
            pp++; if(!isspace(*pp) || *pp=='\n') continue;
99
            goto gopt;
100
          }
101
      }
102
      else {
103
          if(*pp!=' ') *pp=' ';
104
          pp++; if(!isspace(*pp)) continue;
105
          for(p2=pp;isspace(*p2);p2++);
106
          ovlstrcpy(pp,p2); pp--;
107
      }
10 reyssat 108
    }
109
}
110
 
7676 bpr 111
/* Prepare dictionary */
10 reyssat 112
void prepare_dic(void)
113
{
114
    int i;
115
    FILE *dicf;
116
    char *p1, *p2, *pp;
117
    long int flen;
118
 
119
    entrycount=0;
120
    dicf=fopen(dicname,"r"); if(dicf==NULL) return;
121
    fseek(dicf,0,SEEK_END);flen=ftell(dicf); fseek(dicf,0,SEEK_SET);
122
    if(flen>diclim) return;
123
    dicbuf=xmalloc(2*flen+1024);flen=fread(dicbuf,1,flen,dicf);
124
    fclose(dicf);
125
    if(flen>0 && flen<diclim) dicbuf[flen]=0;
126
    else return;
127
    for(i=0,p1=dicbuf;p1!=NULL && *p1!=0 && i<entrylim;p1=p2) {
7676 bpr 128
      p2=strchr(p1+1,'\n'); if(p2>p1) *p2++=0;
129
      pp=strchr(p1,sepchar); if(pp==NULL) continue;
130
      *pp++=0;
8100 bpr 131
      strip_trailing_spaces2(p1); strip_trailing_spaces2(pp);
132
      singlespace2(p1);
7676 bpr 133
      p1=find_word_start(p1); pp=find_word_start(pp);
134
      if(*p1==0) continue;
135
      entry[i].original=p1; entry[i].replace=pp; i++;
10 reyssat 136
    }
137
    entrycount=i;
138
}
139
 
140
void output(void)
141
{
142
    int i;
143
    FILE *f;
144
 
145
    ocount=0;
146
    strcat(dicname,".sorted");
147
    f=fopen(dicname,"w"); if(f==NULL) return;
148
    for(i=0;i<entrycount;i++) {
7676 bpr 149
      if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0
150
         && strcmp(entry[i].replace,entry[i-1].replace)==0)
151
        continue;
152
      if(grpchar!=0) {
153
          if(i>0 && strcmp(entry[i].original,entry[i-1].original)==0)
154
            fprintf(f,"%c%s",grpchar, entry[i].replace);
155
          else {
156
            if(i>0) fprintf(f,"\n");
157
            fprintf(f,"%s%c%s",entry[i].original,sepchar,entry[i].replace);
158
            ocount++;
159
          }
160
 
161
      }
162
      else {
163
          fprintf(f,"%s%c%s\n",entry[i].original,sepchar,entry[i].replace);
164
          ocount++;
165
      }
10 reyssat 166
    }
167
    if(grpchar!=0) fprintf(f,"\n");
168
    fclose(f);
169
}
170
 
171
int main(int argc, char *argv[])
172
{
173
    char *ss, *gr;
174
    if(argc<2) return -1;
7676 bpr 175
 
10 reyssat 176
    ss=getenv("dicsort_separator");
177
    if(ss!=NULL && *ss!=0) sepchar=*ss;
178
    gr=getenv("dicsort_grouping");
179
    if(gr!=NULL && *gr!=0) grpchar=*gr;
180
    snprintf(dicname,sizeof(dicname)-128,"%s",argv[1]); prepare_dic();
181
    if(argc>2) {
7676 bpr 182
      snprintf(suffixname,sizeof(suffixname),"%s",argv[2]);
183
      suffix_dic(suffixname); hassuffix=1;
10 reyssat 184
    }
185
    else suffixname[0]=hassuffix=0;
186
    sortdic(); output();
187
    printf("%s: sorted %d entries.\n",dicname, ocount);
188
    return 0;
189
}
190