1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
This patch removes the usage of tolower() in fts code, which is not locale
neutral and causes problem in some locales such as Turkish.
See http://crbug.com/15261 for details.
An upstream ticket was also created for this issue:
http://www.sqlite.org/src/tktview/991789d9f3136a0460dc83a33e815c1aa9757c26
Contains backport for upstream http://www.sqlite.org/src/ci/b8b465ed2c.
Index: ext/fts3/fts3.c
===================================================================
--- ext/fts3/fts3.c 2009-09-04 13:37:41.000000000 -0700
+++ ext/fts3/fts3.c 2009-09-14 18:17:45.000000000 -0700
@@ -326,7 +326,7 @@
return (c&0x80)==0 ? isspace(c) : 0;
}
static int safe_tolower(char c){
- return (c&0x80)==0 ? tolower(c) : c;
+ return (c>='A' && c<='Z') ? (c-'A'+'a') : c;
}
static int safe_isalnum(char c){
return (c&0x80)==0 ? isalnum(c) : 0;
Index: ext/fts3/fts3_expr.c
===================================================================
--- ext/fts3/fts3_expr.c
+++ ext/fts3/fts3_expr.c
@@ -58,7 +58,6 @@ int sqlite3_fts3_enable_parentheses = 0;
#include "fts3_expr.h"
#include "sqlite3.h"
-#include <ctype.h>
#include <string.h>
#include <assert.h>
@@ -84,7 +83,7 @@ struct ParseContext {
** negative values).
*/
static int fts3isspace(char c){
- return (c&0x80)==0 ? isspace(c) : 0;
+ return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f';
}
/*
Index: ext/fts3/fts3_porter.c
===================================================================
--- ext/fts3/fts3_porter.c
+++ ext/fts3/fts3_porter.c
@@ -29,7 +29,6 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
-#include <ctype.h>
#include "fts3_tokenizer.h"
Index: ext/fts3/fts3_tokenizer1.c
===================================================================
--- ext/fts3/fts3_tokenizer1.c
+++ ext/fts3/fts3_tokenizer1.c
@@ -29,7 +29,6 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
-#include <ctype.h>
#include "fts3_tokenizer.h"
@@ -55,6 +54,9 @@ static const sqlite3_tokenizer_module simpleTokenizerModule;
static int simpleDelim(simple_tokenizer *t, unsigned char c){
return c<0x80 && t->delim[c];
}
+static int fts3_isalnum(int x){
+ return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z');
+}
/*
** Create a new tokenizer instance.
@@ -89,7 +91,7 @@ static int simpleCreate(
/* Mark non-alphanumeric ASCII characters as delimiters */
int i;
for(i=1; i<0x80; i++){
- t->delim[i] = !isalnum(i);
+ t->delim[i] = !fts3_isalnum(i);
}
}
@@ -191,7 +193,7 @@ static int simpleNext(
** case-insensitivity.
*/
unsigned char ch = p[iStartOffset+i];
- c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
+ c->pToken[i] = (ch>='A' && ch<='Z') ? ch-'A'+'a' : ch;
}
*ppToken = c->pToken;
*pnBytes = n;
Index: ext/fts2/fts2.c
===================================================================
--- ext/fts2/fts2.c 2009-09-04 13:37:41.000000000 -0700
+++ ext/fts2/fts2.c 2009-09-14 18:17:02.000000000 -0700
@@ -372,7 +372,7 @@
return (c&0x80)==0 ? isspace(c) : 0;
}
static int safe_tolower(char c){
- return (c&0x80)==0 ? tolower(c) : c;
+ return (c>='A' && c<='Z') ? (c-'A'+'a') : c;
}
static int safe_isalnum(char c){
return (c&0x80)==0 ? isalnum(c) : 0;
Index: ext/fts2/fts2_tokenizer1.c
===================================================================
--- ext/fts2/fts2_tokenizer1.c 2009-09-03 13:32:06.000000000 -0700
+++ ext/fts2/fts2_tokenizer1.c 2009-09-02 11:40:21.000000000 -0700
@@ -191,7 +191,7 @@
** case-insensitivity.
*/
unsigned char ch = p[iStartOffset+i];
- c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
+ c->pToken[i] = (ch>='A' && ch<='Z') ? (ch-'A'+'a') : ch;
}
*ppToken = c->pToken;
*pnBytes = n;
|
