Bug Summary

File:root/firefox-clang/third_party/sqlite3/ext/fts5.c
Warning:line 7287, column 25
Access to field 'pRoot' results in a dereference of a null pointer (loaded from variable 'p')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O2 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name fts5.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/sqlite3/ext -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/sqlite3/ext -resource-dir /usr/lib/llvm-23/lib/clang/23 -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG=1 -D SQLITE_VEC_ENABLE_DISKANN=0 -I /root/firefox-clang/third_party/sqlite3/ext -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/sqlite3/ext -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/llvm-23/lib/clang/23/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/15/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=tautological-type-limit-compare -Wno-range-loop-analysis -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-unknown-warning-option -Wno-character-conversion -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -fdiagnostics-absolute-paths -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -mllvm -dwarf-linkage-names=Abstract -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-07-16-093543-1626485-1 -x c /root/firefox-clang/third_party/sqlite3/ext/fts5.c
1
2/*
3** This, the "fts5.c" source file, is a composite file that is itself
4** assembled from the following files:
5**
6** fts5.h
7** fts5Int.h
8** fts5parse.h <--- Generated from fts5parse.y by Lemon
9** fts5parse.c <--- Generated from fts5parse.y by Lemon
10** fts5_aux.c
11** fts5_buffer.c
12** fts5_config.c
13** fts5_expr.c
14** fts5_hash.c
15** fts5_index.c
16** fts5_main.c
17** fts5_storage.c
18** fts5_tokenize.c
19** fts5_unicode2.c
20** fts5_varint.c
21** fts5_vocab.c
22*/
23#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5)
24
25#if !defined(NDEBUG1) && !defined(SQLITE_DEBUG)
26# define NDEBUG1 1
27#endif
28#if defined(NDEBUG1) && defined(SQLITE_DEBUG)
29# undef NDEBUG1
30#endif
31
32#ifdef HAVE_STDINT_H1
33#include <stdint.h>
34#endif
35#ifdef HAVE_INTTYPES_H1
36#include <inttypes.h>
37#endif
38#line 1 "fts5.h"
39/*
40** 2014 May 31
41**
42** The author disclaims copyright to this source code. In place of
43** a legal notice, here is a blessing:
44**
45** May you do good and not evil.
46** May you find forgiveness for yourself and forgive others.
47** May you share freely, never taking more than you give.
48**
49******************************************************************************
50**
51** Interfaces to extend FTS5. Using the interfaces defined in this file,
52** FTS5 may be extended with:
53**
54** * custom tokenizers, and
55** * custom auxiliary functions.
56*/
57
58
59#ifndef _FTS5_H
60#define _FTS5_H
61
62#include "sqlite3.h"
63
64#ifdef __cplusplus
65extern "C" {
66#endif
67
68/*************************************************************************
69** CUSTOM AUXILIARY FUNCTIONS
70**
71** Virtual table implementations may overload SQL functions by implementing
72** the sqlite3_module.xFindFunction() method.
73*/
74
75typedef struct Fts5ExtensionApi Fts5ExtensionApi;
76typedef struct Fts5Context Fts5Context;
77typedef struct Fts5PhraseIter Fts5PhraseIter;
78
79typedef void (*fts5_extension_function)(
80 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
81 Fts5Context *pFts, /* First arg to pass to pApi functions */
82 sqlite3_context *pCtx, /* Context for returning result/error */
83 int nVal, /* Number of values in apVal[] array */
84 sqlite3_value **apVal /* Array of trailing arguments */
85);
86
87struct Fts5PhraseIter {
88 const unsigned char *a;
89 const unsigned char *b;
90};
91
92/*
93** EXTENSION API FUNCTIONS
94**
95** xUserData(pFts):
96** Return a copy of the pUserData pointer passed to the xCreateFunction()
97** API when the extension function was registered.
98**
99** xColumnTotalSize(pFts, iCol, pnToken):
100** If parameter iCol is less than zero, set output variable *pnToken
101** to the total number of tokens in the FTS5 table. Or, if iCol is
102** non-negative but less than the number of columns in the table, return
103** the total number of tokens in column iCol, considering all rows in
104** the FTS5 table.
105**
106** If parameter iCol is greater than or equal to the number of columns
107** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
108** an OOM condition or IO error), an appropriate SQLite error code is
109** returned.
110**
111** xColumnCount(pFts):
112** Return the number of columns in the table.
113**
114** xColumnSize(pFts, iCol, pnToken):
115** If parameter iCol is less than zero, set output variable *pnToken
116** to the total number of tokens in the current row. Or, if iCol is
117** non-negative but less than the number of columns in the table, set
118** *pnToken to the number of tokens in column iCol of the current row.
119**
120** If parameter iCol is greater than or equal to the number of columns
121** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
122** an OOM condition or IO error), an appropriate SQLite error code is
123** returned.
124**
125** This function may be quite inefficient if used with an FTS5 table
126** created with the "columnsize=0" option.
127**
128** xColumnText:
129** If parameter iCol is less than zero, or greater than or equal to the
130** number of columns in the table, SQLITE_RANGE is returned.
131**
132** Otherwise, this function attempts to retrieve the text of column iCol of
133** the current document. If successful, (*pz) is set to point to a buffer
134** containing the text in utf-8 encoding, (*pn) is set to the size in bytes
135** (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
136** if an error occurs, an SQLite error code is returned and the final values
137** of (*pz) and (*pn) are undefined.
138**
139** xPhraseCount:
140** Returns the number of phrases in the current query expression.
141**
142** xPhraseSize:
143** If parameter iCol is less than zero, or greater than or equal to the
144** number of phrases in the current query, as returned by xPhraseCount,
145** 0 is returned. Otherwise, this function returns the number of tokens in
146** phrase iPhrase of the query. Phrases are numbered starting from zero.
147**
148** xInstCount:
149** Set *pnInst to the total number of occurrences of all phrases within
150** the query within the current row. Return SQLITE_OK if successful, or
151** an error code (i.e. SQLITE_NOMEM) if an error occurs.
152**
153** This API can be quite slow if used with an FTS5 table created with the
154** "detail=none" or "detail=column" option. If the FTS5 table is created
155** with either "detail=none" or "detail=column" and "content=" option
156** (i.e. if it is a contentless table), then this API always returns 0.
157**
158** xInst:
159** Query for the details of phrase match iIdx within the current row.
160** Phrase matches are numbered starting from zero, so the iIdx argument
161** should be greater than or equal to zero and smaller than the value
162** output by xInstCount(). If iIdx is less than zero or greater than
163** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.
164**
165** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol
166** to the column in which it occurs and *piOff the token offset of the
167** first token of the phrase. SQLITE_OK is returned if successful, or an
168** error code (i.e. SQLITE_NOMEM) if an error occurs.
169**
170** This API can be quite slow if used with an FTS5 table created with the
171** "detail=none" or "detail=column" option.
172**
173** xRowid:
174** Returns the rowid of the current row.
175**
176** xTokenize:
177** Tokenize text using the tokenizer belonging to the FTS5 table.
178**
179** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
180** This API function is used to query the FTS table for phrase iPhrase
181** of the current query. Specifically, a query equivalent to:
182**
183** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid
184**
185** with $p set to a phrase equivalent to the phrase iPhrase of the
186** current query is executed. Any column filter that applies to
187** phrase iPhrase of the current query is included in $p. For each
188** row visited, the callback function passed as the fourth argument
189** is invoked. The context and API objects passed to the callback
190** function may be used to access the properties of each matched row.
191** Invoking Api.xUserData() returns a copy of the pointer passed as
192** the third argument to pUserData.
193**
194** If parameter iPhrase is less than zero, or greater than or equal to
195** the number of phrases in the query, as returned by xPhraseCount(),
196** this function returns SQLITE_RANGE.
197**
198** If the callback function returns any value other than SQLITE_OK, the
199** query is abandoned and the xQueryPhrase function returns immediately.
200** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
201** Otherwise, the error code is propagated upwards.
202**
203** If the query runs to completion without incident, SQLITE_OK is returned.
204** Or, if some error occurs before the query completes or is aborted by
205** the callback, an SQLite error code is returned.
206**
207**
208** xSetAuxdata(pFts5, pAux, xDelete)
209**
210** Save the pointer passed as the second argument as the extension function's
211** "auxiliary data". The pointer may then be retrieved by the current or any
212** future invocation of the same fts5 extension function made as part of
213** the same MATCH query using the xGetAuxdata() API.
214**
215** Each extension function is allocated a single auxiliary data slot for
216** each FTS query (MATCH expression). If the extension function is invoked
217** more than once for a single FTS query, then all invocations share a
218** single auxiliary data context.
219**
220** If there is already an auxiliary data pointer when this function is
221** invoked, then it is replaced by the new pointer. If an xDelete callback
222** was specified along with the original pointer, it is invoked at this
223** point.
224**
225** The xDelete callback, if one is specified, is also invoked on the
226** auxiliary data pointer after the FTS5 query has finished.
227**
228** If an error (e.g. an OOM condition) occurs within this function,
229** the auxiliary data is set to NULL and an error code returned. If the
230** xDelete parameter was not NULL, it is invoked on the auxiliary data
231** pointer before returning.
232**
233**
234** xGetAuxdata(pFts5, bClear)
235**
236** Returns the current auxiliary data pointer for the fts5 extension
237** function. See the xSetAuxdata() method for details.
238**
239** If the bClear argument is non-zero, then the auxiliary data is cleared
240** (set to NULL) before this function returns. In this case the xDelete,
241** if any, is not invoked.
242**
243**
244** xRowCount(pFts5, pnRow)
245**
246** This function is used to retrieve the total number of rows in the table.
247** In other words, the same value that would be returned by:
248**
249** SELECT count(*) FROM ftstable;
250**
251** xPhraseFirst()
252** This function is used, along with type Fts5PhraseIter and the xPhraseNext
253** method, to iterate through all instances of a single query phrase within
254** the current row. This is the same information as is accessible via the
255** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient
256** to use, this API may be faster under some circumstances. To iterate
257** through instances of phrase iPhrase, use the following code:
258**
259** Fts5PhraseIter iter;
260** int iCol, iOff;
261** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
262** iCol>=0;
263** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
264** ){
265** // An instance of phrase iPhrase at offset iOff of column iCol
266** }
267**
268** The Fts5PhraseIter structure is defined above. Applications should not
269** modify this structure directly - it should only be used as shown above
270** with the xPhraseFirst() and xPhraseNext() API methods (and by
271** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).
272**
273** This API can be quite slow if used with an FTS5 table created with the
274** "detail=none" or "detail=column" option. If the FTS5 table is created
275** with either "detail=none" or "detail=column" and "content=" option
276** (i.e. if it is a contentless table), then this API always iterates
277** through an empty set (all calls to xPhraseFirst() set iCol to -1).
278**
279** In all cases, matches are visited in (column ASC, offset ASC) order.
280** i.e. all those in column 0, sorted by offset, followed by those in
281** column 1, etc.
282**
283** xPhraseNext()
284** See xPhraseFirst above.
285**
286** xPhraseFirstColumn()
287** This function and xPhraseNextColumn() are similar to the xPhraseFirst()
288** and xPhraseNext() APIs described above. The difference is that instead
289** of iterating through all instances of a phrase in the current row, these
290** APIs are used to iterate through the set of columns in the current row
291** that contain one or more instances of a specified phrase. For example:
292**
293** Fts5PhraseIter iter;
294** int iCol;
295** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
296** iCol>=0;
297** pApi->xPhraseNextColumn(pFts, &iter, &iCol)
298** ){
299** // Column iCol contains at least one instance of phrase iPhrase
300** }
301**
302** This API can be quite slow if used with an FTS5 table created with the
303** "detail=none" option. If the FTS5 table is created with either
304** "detail=none" "content=" option (i.e. if it is a contentless table),
305** then this API always iterates through an empty set (all calls to
306** xPhraseFirstColumn() set iCol to -1).
307**
308** The information accessed using this API and its companion
309** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext
310** (or xInst/xInstCount). The chief advantage of this API is that it is
311** significantly more efficient than those alternatives when used with
312** "detail=column" tables.
313**
314** xPhraseNextColumn()
315** See xPhraseFirstColumn above.
316**
317** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
318** This is used to access token iToken of phrase iPhrase of the current
319** query. Before returning, output parameter *ppToken is set to point
320** to a buffer containing the requested token, and *pnToken to the
321** size of this buffer in bytes.
322**
323** If iPhrase or iToken are less than zero, or if iPhrase is greater than
324** or equal to the number of phrases in the query as reported by
325** xPhraseCount(), or if iToken is equal to or greater than the number of
326** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken
327 are both zeroed.
328**
329** The output text is not a copy of the query text that specified the
330** token. It is the output of the tokenizer module. For tokendata=1
331** tables, this includes any embedded 0x00 and trailing data.
332**
333** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
334** This is used to access token iToken of phrase hit iIdx within the
335** current row. If iIdx is less than zero or greater than or equal to the
336** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise,
337** output variable (*ppToken) is set to point to a buffer containing the
338** matching document token, and (*pnToken) to the size of that buffer in
339** bytes.
340**
341** The output text is not a copy of the document text that was tokenized.
342** It is the output of the tokenizer module. For tokendata=1 tables, this
343** includes any embedded 0x00 and trailing data.
344**
345** This API may be slow in some cases if the token identified by parameters
346** iIdx and iToken matched a prefix token in the query. In most cases, the
347** first call to this API for each prefix token in the query is forced
348** to scan the portion of the full-text index that matches the prefix
349** token to collect the extra data required by this API. If the prefix
350** token matches a large number of token instances in the document set,
351** this may be a performance problem.
352**
353** If the user knows in advance that a query may use this API for a
354** prefix token, FTS5 may be configured to collect all required data as part
355** of the initial querying of the full-text index, avoiding the second scan
356** entirely. This also causes prefix queries that do not use this API to
357** run more slowly and use more memory. FTS5 may be configured in this way
358** either on a per-table basis using the [FTS5 insttoken | 'insttoken']
359** option, or on a per-query basis using the
360** [fts5_insttoken | fts5_insttoken()] user function.
361**
362** This API can be quite slow if used with an FTS5 table created with the
363** "detail=none" or "detail=column" option.
364**
365** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale)
366** If parameter iCol is less than zero, or greater than or equal to the
367** number of columns in the table, SQLITE_RANGE is returned.
368**
369** Otherwise, this function attempts to retrieve the locale associated
370** with column iCol of the current row. Usually, there is no associated
371** locale, and output parameters (*pzLocale) and (*pnLocale) are set
372** to NULL and 0, respectively. However, if the fts5_locale() function
373** was used to associate a locale with the value when it was inserted
374** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated
375** buffer containing the name of the locale in utf-8 encoding. (*pnLocale)
376** is set to the size in bytes of the buffer, not including the
377** nul-terminator.
378**
379** If successful, SQLITE_OK is returned. Or, if an error occurs, an
380** SQLite error code is returned. The final value of the output parameters
381** is undefined in this case.
382**
383** xTokenize_v2:
384** Tokenize text using the tokenizer belonging to the FTS5 table. This
385** API is the same as the xTokenize() API, except that it allows a tokenizer
386** locale to be specified.
387*/
388struct Fts5ExtensionApi {
389 int iVersion; /* Currently always set to 4 */
390
391 void *(*xUserData)(Fts5Context*);
392
393 int (*xColumnCount)(Fts5Context*);
394 int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
395 int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);
396
397 int (*xTokenize)(Fts5Context*,
398 const char *pText, int nText, /* Text to tokenize */
399 void *pCtx, /* Context passed to xToken() */
400 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
401 );
402
403 int (*xPhraseCount)(Fts5Context*);
404 int (*xPhraseSize)(Fts5Context*, int iPhrase);
405
406 int (*xInstCount)(Fts5Context*, int *pnInst);
407 int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff);
408
409 sqlite3_int64 (*xRowid)(Fts5Context*);
410 int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
411 int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
412
413 int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
414 int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
415 );
416 int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
417 void *(*xGetAuxdata)(Fts5Context*, int bClear);
418
419 int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
420 void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);
421
422 int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
423 void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
424
425 /* Below this point are iVersion>=3 only */
426 int (*xQueryToken)(Fts5Context*,
427 int iPhrase, int iToken,
428 const char **ppToken, int *pnToken
429 );
430 int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
431
432 /* Below this point are iVersion>=4 only */
433 int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn);
434 int (*xTokenize_v2)(Fts5Context*,
435 const char *pText, int nText, /* Text to tokenize */
436 const char *pLocale, int nLocale, /* Locale to pass to tokenizer */
437 void *pCtx, /* Context passed to xToken() */
438 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
439 );
440};
441
442/*
443** CUSTOM AUXILIARY FUNCTIONS
444*************************************************************************/
445
446/*************************************************************************
447** CUSTOM TOKENIZERS
448**
449** Applications may also register custom tokenizer types. A tokenizer
450** is registered by providing fts5 with a populated instance of the
451** following structure. All structure methods must be defined, setting
452** any member of the fts5_tokenizer struct to NULL leads to undefined
453** behaviour. The structure methods are expected to function as follows:
454**
455** xCreate:
456** This function is used to allocate and initialize a tokenizer instance.
457** A tokenizer instance is required to actually tokenize text.
458**
459** The first argument passed to this function is a copy of the (void*)
460** pointer provided by the application when the fts5_tokenizer_v2 object
461** was registered with FTS5 (the third argument to xCreateTokenizer()).
462** The second and third arguments are an array of nul-terminated strings
463** containing the tokenizer arguments, if any, specified following the
464** tokenizer name as part of the CREATE VIRTUAL TABLE statement used
465** to create the FTS5 table.
466**
467** The final argument is an output variable. If successful, (*ppOut)
468** should be set to point to the new tokenizer handle and SQLITE_OK
469** returned. If an error occurs, some value other than SQLITE_OK should
470** be returned. In this case, fts5 assumes that the final value of *ppOut
471** is undefined.
472**
473** xDelete:
474** This function is invoked to delete a tokenizer handle previously
475** allocated using xCreate(). Fts5 guarantees that this function will
476** be invoked exactly once for each successful call to xCreate().
477**
478** xTokenize:
479** This function is expected to tokenize the nText byte string indicated
480** by argument pText. pText may or may not be nul-terminated. The first
481** argument passed to this function is a pointer to an Fts5Tokenizer object
482** returned by an earlier call to xCreate().
483**
484** The third argument indicates the reason that FTS5 is requesting
485** tokenization of the supplied text. This is always one of the following
486** four values:
487**
488** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into
489** or removed from the FTS table. The tokenizer is being invoked to
490** determine the set of tokens to add to (or delete from) the
491** FTS index.
492**
493** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed
494** against the FTS index. The tokenizer is being called to tokenize
495** a bareword or quoted string specified as part of the query.
496**
497** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as
498** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is
499** followed by a "*" character, indicating that the last token
500** returned by the tokenizer will be treated as a token prefix.
501**
502** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to
503** satisfy an fts5_api.xTokenize() request made by an auxiliary
504** function. Or an fts5_api.xColumnSize() request made by the same
505** on a columnsize=0 database.
506** </ul>
507**
508** The sixth and seventh arguments passed to xTokenize() - pLocale and
509** nLocale - are a pointer to a buffer containing the locale to use for
510** tokenization (e.g. "en_US") and its size in bytes, respectively. The
511** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in
512** which case nLocale is always 0) to indicate that the tokenizer should
513** use its default locale.
514**
515** For each token in the input string, the supplied callback xToken() must
516** be invoked. The first argument to it should be a copy of the pointer
517** passed as the second argument to xTokenize(). The third and fourth
518** arguments are a pointer to a buffer containing the token text, and the
519** size of the token in bytes. The 4th and 5th arguments are the byte offsets
520** of the first byte of and first byte immediately following the text from
521** which the token is derived within the input.
522**
523** The second argument passed to the xToken() callback ("tflags") should
524** normally be set to 0. The exception is if the tokenizer supports
525** synonyms. In this case see the discussion below for details.
526**
527** FTS5 assumes the xToken() callback is invoked for each token in the
528** order that they occur within the input text.
529**
530** If an xToken() callback returns any value other than SQLITE_OK, then
531** the tokenization should be abandoned and the xTokenize() method should
532** immediately return a copy of the xToken() return value. Or, if the
533** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally,
534** if an error occurs with the xTokenize() implementation itself, it
535** may abandon the tokenization and return any error code other than
536** SQLITE_OK or SQLITE_DONE.
537**
538** If the tokenizer is registered using an fts5_tokenizer_v2 object,
539** then the xTokenize() method has two additional arguments - pLocale
540** and nLocale. These specify the locale that the tokenizer should use
541** for the current request. If pLocale and nLocale are both 0, then the
542** tokenizer should use its default locale. Otherwise, pLocale points to
543** an nLocale byte buffer containing the name of the locale to use as utf-8
544** text. pLocale is not nul-terminated.
545**
546** FTS5_TOKENIZER
547**
548** There is also an fts5_tokenizer object. This is an older, deprecated,
549** version of fts5_tokenizer_v2. It is similar except that:
550**
551** <ul>
552** <li> There is no "iVersion" field, and
553** <li> The xTokenize() method does not take a locale argument.
554** </ul>
555**
556** Legacy fts5_tokenizer tokenizers must be registered using the
557** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2().
558**
559** Tokenizer implementations registered using either API may be retrieved
560** using both xFindTokenizer() and xFindTokenizer_v2().
561**
562** SYNONYM SUPPORT
563**
564** Custom tokenizers may also support synonyms. Consider a case in which a
565** user wishes to query for a phrase such as "first place". Using the
566** built-in tokenizers, the FTS5 query 'first + place' will match instances
567** of "first place" within the document set, but not alternative forms
568** such as "1st place". In some applications, it would be better to match
569** all instances of "first place" or "1st place" regardless of which form
570** the user specified in the MATCH query text.
571**
572** There are several ways to approach this in FTS5:
573**
574** <ol><li> By mapping all synonyms to a single token. In this case, using
575** the above example, this means that the tokenizer returns the
576** same token for inputs "first" and "1st". Say that token is in
577** fact "first", so that when the user inserts the document "I won
578** 1st place" entries are added to the index for tokens "i", "won",
579** "first" and "place". If the user then queries for '1st + place',
580** the tokenizer substitutes "first" for "1st" and the query works
581** as expected.
582**
583** <li> By querying the index for all synonyms of each query term
584** separately. In this case, when tokenizing query text, the
585** tokenizer may provide multiple synonyms for a single term
586** within the document. FTS5 then queries the index for each
587** synonym individually. For example, faced with the query:
588**
589** <codeblock>
590** ... MATCH 'first place'</codeblock>
591**
592** the tokenizer offers both "1st" and "first" as synonyms for the
593** first token in the MATCH query and FTS5 effectively runs a query
594** similar to:
595**
596** <codeblock>
597** ... MATCH '(first OR 1st) place'</codeblock>
598**
599** except that, for the purposes of auxiliary functions, the query
600** still appears to contain just two phrases - "(first OR 1st)"
601** being treated as a single phrase.
602**
603** <li> By adding multiple synonyms for a single term to the FTS index.
604** Using this method, when tokenizing document text, the tokenizer
605** provides multiple synonyms for each token. So that when a
606** document such as "I won first place" is tokenized, entries are
607** added to the FTS index for "i", "won", "first", "1st" and
608** "place".
609**
610** This way, even if the tokenizer does not provide synonyms
611** when tokenizing query text (it should not - to do so would be
612** inefficient), it doesn't matter if the user queries for
613** 'first + place' or '1st + place', as there are entries in the
614** FTS index corresponding to both forms of the first token.
615** </ol>
616**
617** Whether it is parsing document or query text, any call to xToken that
618** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit
619** is considered to supply a synonym for the previous token. For example,
620** when parsing the document "I won first place", a tokenizer that supports
621** synonyms would call xToken() 5 times, as follows:
622**
623** <codeblock>
624** xToken(pCtx, 0, "i", 1, 0, 1);
625** xToken(pCtx, 0, "won", 3, 2, 5);
626** xToken(pCtx, 0, "first", 5, 6, 11);
627** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11);
628** xToken(pCtx, 0, "place", 5, 12, 17);
629**</codeblock>
630**
631** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time
632** xToken() is called. Multiple synonyms may be specified for a single token
633** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence.
634** There is no limit to the number of synonyms that may be provided for a
635** single token.
636**
637** In many cases, method (1) above is the best approach. It does not add
638** extra data to the FTS index or require FTS5 to query for multiple terms,
639** so it is efficient in terms of disk space and query speed. However, it
640** does not support prefix queries very well. If, as suggested above, the
641** token "first" is substituted for "1st" by the tokenizer, then the query:
642**
643** <codeblock>
644** ... MATCH '1s*'</codeblock>
645**
646** will not match documents that contain the token "1st" (as the tokenizer
647** will probably not map "1s" to any prefix of "first").
648**
649** For full prefix support, method (3) may be preferred. In this case,
650** because the index contains entries for both "first" and "1st", prefix
651** queries such as 'fi*' or '1s*' will match correctly. However, because
652** extra entries are added to the FTS index, this method uses more space
653** within the database.
654**
655** Method (2) offers a midpoint between (1) and (3). Using this method,
656** a query such as '1s*' will match documents that contain the literal
657** token "1st", but not "first" (assuming the tokenizer is not able to
658** provide synonyms for prefixes). However, a non-prefix query like '1st'
659** will match against "1st" and "first". This method does not require
660** extra disk space, as no extra entries are added to the FTS index.
661** On the other hand, it may require more CPU cycles to run MATCH queries,
662** as separate queries of the FTS index are required for each synonym.
663**
664** When using methods (2) or (3), it is important that the tokenizer only
665** provide synonyms when tokenizing document text (method (3)) or query
666** text (method (2)), not both. Doing so will not cause any errors, but is
667** inefficient.
668*/
669typedef struct Fts5Tokenizer Fts5Tokenizer;
670typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2;
671struct fts5_tokenizer_v2 {
672 int iVersion; /* Currently always 2 */
673
674 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
675 void (*xDelete)(Fts5Tokenizer*);
676 int (*xTokenize)(Fts5Tokenizer*,
677 void *pCtx,
678 int flags, /* Mask of FTS5_TOKENIZE_* flags */
679 const char *pText, int nText,
680 const char *pLocale, int nLocale,
681 int (*xToken)(
682 void *pCtx, /* Copy of 2nd argument to xTokenize() */
683 int tflags, /* Mask of FTS5_TOKEN_* flags */
684 const char *pToken, /* Pointer to buffer containing token */
685 int nToken, /* Size of token in bytes */
686 int iStart, /* Byte offset of token within input text */
687 int iEnd /* Byte offset of end of token within input text */
688 )
689 );
690};
691
692/*
693** New code should use the fts5_tokenizer_v2 type to define tokenizer
694** implementations. The following type is included for legacy applications
695** that still use it.
696*/
697typedef struct fts5_tokenizer fts5_tokenizer;
698struct fts5_tokenizer {
699 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
700 void (*xDelete)(Fts5Tokenizer*);
701 int (*xTokenize)(Fts5Tokenizer*,
702 void *pCtx,
703 int flags, /* Mask of FTS5_TOKENIZE_* flags */
704 const char *pText, int nText,
705 int (*xToken)(
706 void *pCtx, /* Copy of 2nd argument to xTokenize() */
707 int tflags, /* Mask of FTS5_TOKEN_* flags */
708 const char *pToken, /* Pointer to buffer containing token */
709 int nToken, /* Size of token in bytes */
710 int iStart, /* Byte offset of token within input text */
711 int iEnd /* Byte offset of end of token within input text */
712 )
713 );
714};
715
716
717/* Flags that may be passed as the third argument to xTokenize() */
718#define FTS5_TOKENIZE_QUERY0x0001 0x0001
719#define FTS5_TOKENIZE_PREFIX0x0002 0x0002
720#define FTS5_TOKENIZE_DOCUMENT0x0004 0x0004
721#define FTS5_TOKENIZE_AUX0x0008 0x0008
722
723/* Flags that may be passed by the tokenizer implementation back to FTS5
724** as the third argument to the supplied xToken callback. */
725#define FTS5_TOKEN_COLOCATED0x0001 0x0001 /* Same position as prev. token */
726
727/*
728** END OF CUSTOM TOKENIZERS
729*************************************************************************/
730
731/*************************************************************************
732** FTS5 EXTENSION REGISTRATION API
733*/
734typedef struct fts5_api fts5_api;
735struct fts5_api {
736 int iVersion; /* Currently always set to 3 */
737
738 /* Create a new tokenizer */
739 int (*xCreateTokenizer)(
740 fts5_api *pApi,
741 const char *zName,
742 void *pUserData,
743 fts5_tokenizer *pTokenizer,
744 void (*xDestroy)(void*)
745 );
746
747 /* Find an existing tokenizer */
748 int (*xFindTokenizer)(
749 fts5_api *pApi,
750 const char *zName,
751 void **ppUserData,
752 fts5_tokenizer *pTokenizer
753 );
754
755 /* Create a new auxiliary function */
756 int (*xCreateFunction)(
757 fts5_api *pApi,
758 const char *zName,
759 void *pUserData,
760 fts5_extension_function xFunction,
761 void (*xDestroy)(void*)
762 );
763
764 /* APIs below this point are only available if iVersion>=3 */
765
766 /* Create a new tokenizer */
767 int (*xCreateTokenizer_v2)(
768 fts5_api *pApi,
769 const char *zName,
770 void *pUserData,
771 fts5_tokenizer_v2 *pTokenizer,
772 void (*xDestroy)(void*)
773 );
774
775 /* Find an existing tokenizer */
776 int (*xFindTokenizer_v2)(
777 fts5_api *pApi,
778 const char *zName,
779 void **ppUserData,
780 fts5_tokenizer_v2 **ppTokenizer
781 );
782};
783
784/*
785** END OF REGISTRATION API
786*************************************************************************/
787
788#ifdef __cplusplus
789} /* end of the 'extern "C"' block */
790#endif
791
792#endif /* _FTS5_H */
793
794#line 1 "fts5Int.h"
795/*
796** 2014 May 31
797**
798** The author disclaims copyright to this source code. In place of
799** a legal notice, here is a blessing:
800**
801** May you do good and not evil.
802** May you find forgiveness for yourself and forgive others.
803** May you share freely, never taking more than you give.
804**
805******************************************************************************
806**
807*/
808#ifndef _FTS5INT_H
809#define _FTS5INT_H
810
811/* #include "fts5.h" */
812#include "sqlite3ext.h"
813SQLITE_EXTENSION_INIT1const sqlite3_api_routines *sqlite3_api=0;
814
815#include <string.h>
816#include <assert.h>
817#include <stddef.h>
818
819#ifndef SQLITE_AMALGAMATION
820
821typedef unsigned char u8;
822typedef unsigned int u32;
823typedef unsigned short u16;
824typedef short i16;
825typedef sqlite3_int64 i64;
826typedef sqlite3_uint64 u64;
827
828#ifndef ArraySize
829# define ArraySize(x)((int)(sizeof(x) / sizeof(x[0]))) ((int)(sizeof(x) / sizeof(x[0])))
830#endif
831
832#define testcase(x)
833
834#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
835# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1
836#endif
837#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS)
838# define ALWAYS(X)(X) (1)
839# define NEVER(X)(X) (0)
840#elif !defined(NDEBUG1)
841# define ALWAYS(X)(X) ((X)?1:(assert(0)((void) (0)),0))
842# define NEVER(X)(X) ((X)?(assert(0)((void) (0)),1):0)
843#else
844# define ALWAYS(X)(X) (X)
845# define NEVER(X)(X) (X)
846#endif
847
848#define MIN(x,y)(((x) < (y)) ? (x) : (y)) (((x) < (y)) ? (x) : (y))
849#define MAX(x,y)(((x) > (y)) ? (x) : (y)) (((x) > (y)) ? (x) : (y))
850
851/*
852** Constants for the largest and smallest possible 64-bit signed integers.
853*/
854# define LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)) (0xffffffff|(((i64)0x7fffffff)<<32))
855# define SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32))) (((i64)-1) - LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)))
856
857/*
858** This macro is used in a single assert() within fts5 to check that an
859** allocation is aligned to an 8-byte boundary. But it is a complicated
860** macro to get right for multiple platforms without generating warnings.
861** So instead of reproducing the entire definition from sqliteInt.h, we
862** just do without this assert() for the rare non-amalgamation builds.
863*/
864#define EIGHT_BYTE_ALIGNMENT(x)1 1
865
866/*
867** Macros needed to provide flexible arrays in a portable way
868*/
869#ifndef offsetof
870# define offsetof(ST,M)__builtin_offsetof(ST, M) ((size_t)((char*)&((ST*)0)->M - (char*)0))
871#endif
872#if defined(__STDC_VERSION__201710L) && (__STDC_VERSION__201710L >= 199901L)
873# define FLEXARRAY
874#else
875# define FLEXARRAY 1
876#endif
877
878#endif /* SQLITE_AMALGAMATION */
879
880/*
881** Constants for the largest and smallest possible 32-bit signed integers.
882*/
883# define LARGEST_INT32((int)(0x7fffffff)) ((int)(0x7fffffff))
884# define SMALLEST_INT32((int)((-1) - ((int)(0x7fffffff)))) ((int)((-1) - LARGEST_INT32((int)(0x7fffffff))))
885
886/* Truncate very long tokens to this many bytes. Hard limit is
887** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
888** field that occurs at the start of each leaf page (see fts5_index.c). */
889#define FTS5_MAX_TOKEN_SIZE32768 32768
890
891/*
892** Maximum number of prefix indexes on single FTS5 table. This must be
893** less than 32. If it is set to anything large than that, an #error
894** directive in fts5_index.c will cause the build to fail.
895*/
896#define FTS5_MAX_PREFIX_INDEXES31 31
897
898/*
899** Maximum segments permitted in a single index
900*/
901#define FTS5_MAX_SEGMENT2000 2000
902
903#define FTS5_DEFAULT_NEARDIST10 10
904#define FTS5_DEFAULT_RANK"bm25" "bm25"
905
906/* Name of rank and rowid columns */
907#define FTS5_RANK_NAME"rank" "rank"
908#define FTS5_ROWID_NAME"rowid" "rowid"
909
910#ifdef SQLITE_DEBUG
911# define FTS5_CORRUPT(11 | (1<<8)) sqlite3Fts5Corrupt()
912static int sqlite3Fts5Corrupt(void);
913#else
914# define FTS5_CORRUPT(11 | (1<<8)) SQLITE_CORRUPT_VTAB(11 | (1<<8))
915#endif
916
917/*
918** The assert_nc() macro is similar to the assert() macro, except that it
919** is used for assert() conditions that are true only if it can be
920** guranteed that the database is not corrupt.
921*/
922#ifdef SQLITE_DEBUG
923extern int sqlite3_fts5_may_be_corrupt;
924# define assert_nc(x)((void) (0)) assert(sqlite3_fts5_may_be_corrupt || (x))((void) (0))
925#else
926# define assert_nc(x)((void) (0)) assert(x)((void) (0))
927#endif
928
929/*
930** A version of memcmp() that does not cause asan errors if one of the pointer
931** parameters is NULL and the number of bytes to compare is zero.
932*/
933#define fts5Memcmp(s1, s2, n)((n)<=0 ? 0 : memcmp((s1), (s2), (n))) ((n)<=0 ? 0 : memcmp((s1), (s2), (n)))
934
935/* Mark a function parameter as unused, to suppress nuisance compiler
936** warnings. */
937#ifndef UNUSED_PARAM
938# define UNUSED_PARAM(X)(void)(X) (void)(X)
939#endif
940
941#ifndef UNUSED_PARAM2
942# define UNUSED_PARAM2(X, Y)(void)(X), (void)(Y) (void)(X), (void)(Y)
943#endif
944
945typedef struct Fts5Global Fts5Global;
946typedef struct Fts5Colset Fts5Colset;
947
948/* If a NEAR() clump or phrase may only match a specific set of columns,
949** then an object of the following type is used to record the set of columns.
950** Each entry in the aiCol[] array is a column that may be matched.
951**
952** This object is used by fts5_expr.c and fts5_index.c.
953*/
954struct Fts5Colset {
955 int nCol;
956 int aiCol[FLEXARRAY];
957};
958
959/* Size (int bytes) of a complete Fts5Colset object with N columns. */
960#define SZ_FTS5COLSET(N)(sizeof(i64)*((N+2)/2)) (sizeof(i64)*((N+2)/2))
961
962/**************************************************************************
963** Interface to code in fts5_config.c. fts5_config.c contains contains code
964** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
965*/
966
967typedef struct Fts5Config Fts5Config;
968typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;
969
970struct Fts5TokenizerConfig {
971 Fts5Tokenizer *pTok;
972 fts5_tokenizer_v2 *pApi2;
973 fts5_tokenizer *pApi1;
974 const char **azArg;
975 int nArg;
976 int ePattern; /* FTS_PATTERN_XXX constant */
977 const char *pLocale; /* Current locale to use */
978 int nLocale; /* Size of pLocale in bytes */
979};
980
981/*
982** An instance of the following structure encodes all information that can
983** be gleaned from the CREATE VIRTUAL TABLE statement.
984**
985** And all information loaded from the %_config table.
986**
987** nAutomerge:
988** The minimum number of segments that an auto-merge operation should
989** attempt to merge together. A value of 1 sets the object to use the
990** compile time default. Zero disables auto-merge altogether.
991**
992** bContentlessDelete:
993** True if the contentless_delete option was present in the CREATE
994** VIRTUAL TABLE statement.
995**
996** zContent:
997**
998** zContentRowid:
999** The value of the content_rowid= option, if one was specified. Or
1000** the string "rowid" otherwise. This text is not quoted - if it is
1001** used as part of an SQL statement it needs to be quoted appropriately.
1002**
1003** zContentExprlist:
1004**
1005** pzErrmsg:
1006** This exists in order to allow the fts5_index.c module to return a
1007** decent error message if it encounters a file-format version it does
1008** not understand.
1009**
1010** bColumnsize:
1011** True if the %_docsize table is created.
1012**
1013** bPrefixIndex:
1014** This is only used for debugging. If set to false, any prefix indexes
1015** are ignored. This value is configured using:
1016**
1017** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
1018**
1019** bLocale:
1020** Set to true if locale=1 was specified when the table was created.
1021*/
1022struct Fts5Config {
1023 sqlite3 *db; /* Database handle */
1024 Fts5Global *pGlobal; /* Global fts5 object for handle db */
1025 char *zDb; /* Database holding FTS index (e.g. "main") */
1026 char *zName; /* Name of FTS index */
1027 int nCol; /* Number of columns */
1028 char **azCol; /* Column names */
1029 u8 *abUnindexed; /* True for unindexed columns */
1030 int nPrefix; /* Number of prefix indexes */
1031 int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */
1032 int eContent; /* An FTS5_CONTENT value */
1033 int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */
1034 int bContentlessUnindexed; /* "contentless_unindexed=" option (dflt=0) */
1035 char *zContent; /* content table */
1036 char *zContentRowid; /* "content_rowid=" option value */
1037 int bColumnsize; /* "columnsize=" option value (dflt==1) */
1038 int bTokendata; /* "tokendata=" option value (dflt==0) */
1039 int bLocale; /* "locale=" option value (dflt==0) */
1040 int eDetail; /* FTS5_DETAIL_XXX value */
1041 char *zContentExprlist;
1042 Fts5TokenizerConfig t;
1043 int bLock; /* True when table is preparing statement */
1044
1045
1046 /* Values loaded from the %_config table */
1047 int iVersion; /* fts5 file format 'version' */
1048 int iCookie; /* Incremented when %_config is modified */
1049 int pgsz; /* Approximate page size used in %_data */
1050 int nAutomerge; /* 'automerge' setting */
1051 int nCrisisMerge; /* Maximum allowed segments per level */
1052 int nUsermerge; /* 'usermerge' setting */
1053 int nHashSize; /* Bytes of memory for in-memory hash */
1054 char *zRank; /* Name of rank function */
1055 char *zRankArgs; /* Arguments to rank function */
1056 int bSecureDelete; /* 'secure-delete' */
1057 int nDeleteMerge; /* 'deletemerge' */
1058 int bPrefixInsttoken; /* 'prefix-insttoken' */
1059
1060 /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
1061 char **pzErrmsg;
1062
1063#ifdef SQLITE_DEBUG
1064 int bPrefixIndex; /* True to use prefix-indexes */
1065#endif
1066};
1067
1068/* Current expected value of %_config table 'version' field. And
1069** the expected version if the 'secure-delete' option has ever been
1070** set on the table. */
1071#define FTS5_CURRENT_VERSION4 4
1072#define FTS5_CURRENT_VERSION_SECUREDELETE5 5
1073
1074#define FTS5_CONTENT_NORMAL0 0
1075#define FTS5_CONTENT_NONE1 1
1076#define FTS5_CONTENT_EXTERNAL2 2
1077#define FTS5_CONTENT_UNINDEXED3 3
1078
1079#define FTS5_DETAIL_FULL0 0
1080#define FTS5_DETAIL_NONE1 1
1081#define FTS5_DETAIL_COLUMNS2 2
1082
1083#define FTS5_PATTERN_NONE0 0
1084#define FTS5_PATTERN_LIKE65 65 /* matches SQLITE_INDEX_CONSTRAINT_LIKE */
1085#define FTS5_PATTERN_GLOB66 66 /* matches SQLITE_INDEX_CONSTRAINT_GLOB */
1086
1087static int sqlite3Fts5ConfigParse(
1088 Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
1089);
1090static void sqlite3Fts5ConfigFree(Fts5Config*);
1091
1092static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
1093
1094static int sqlite3Fts5Tokenize(
1095 Fts5Config *pConfig, /* FTS5 Configuration object */
1096 int flags, /* FTS5_TOKENIZE_* flags */
1097 const char *pText, int nText, /* Text to tokenize */
1098 void *pCtx, /* Context passed to xToken() */
1099 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
1100);
1101
1102static void sqlite3Fts5Dequote(char *z);
1103
1104/* Load the contents of the %_config table */
1105static int sqlite3Fts5ConfigLoad(Fts5Config*, int);
1106
1107/* Set the value of a single config attribute */
1108static int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
1109
1110static int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
1111
1112static void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...);
1113
1114/*
1115** End of interface to code in fts5_config.c.
1116**************************************************************************/
1117
1118/**************************************************************************
1119** Interface to code in fts5_buffer.c.
1120*/
1121
1122/*
1123** Buffer object for the incremental building of string data.
1124*/
1125typedef struct Fts5Buffer Fts5Buffer;
1126struct Fts5Buffer {
1127 u8 *p;
1128 int n;
1129 int nSpace;
1130};
1131
1132static int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
1133static void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
1134static void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
1135static void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
1136static void sqlite3Fts5BufferFree(Fts5Buffer*);
1137static void sqlite3Fts5BufferZero(Fts5Buffer*);
1138static void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
1139static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
1140
1141static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
1142
1143#define fts5BufferZero(x)sqlite3Fts5BufferZero(x) sqlite3Fts5BufferZero(x)
1144#define fts5BufferAppendVarint(a,b,c)sqlite3Fts5BufferAppendVarint(a,b,(i64)c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c)
1145#define fts5BufferFree(a)sqlite3Fts5BufferFree(a) sqlite3Fts5BufferFree(a)
1146#define fts5BufferAppendBlob(a,b,c,d)sqlite3Fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
1147#define fts5BufferSet(a,b,c,d)sqlite3Fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d)
1148
1149#define fts5BufferGrow(pRc,pBuf,nn)( (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace
) ? 0 : sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n)
) ( \
1150 (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
1151 sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
1152)
1153
1154/* Write and decode big-endian 32-bit integer values */
1155static void sqlite3Fts5Put32(u8*, int);
1156static int sqlite3Fts5Get32(const u8*);
1157
1158#define FTS5_POS2COLUMN(iPos)(int)((iPos >> 32) & 0x7FFFFFFF) (int)((iPos >> 32) & 0x7FFFFFFF)
1159#define FTS5_POS2OFFSET(iPos)(int)(iPos & 0x7FFFFFFF) (int)(iPos & 0x7FFFFFFF)
1160
1161typedef struct Fts5PoslistReader Fts5PoslistReader;
1162struct Fts5PoslistReader {
1163 /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
1164 const u8 *a; /* Position list to iterate through */
1165 int n; /* Size of buffer at a[] in bytes */
1166 int i; /* Current offset in a[] */
1167
1168 u8 bFlag; /* For client use (any custom purpose) */
1169
1170 /* Output variables */
1171 u8 bEof; /* Set to true at EOF */
1172 i64 iPos; /* (iCol<<32) + iPos */
1173};
1174static int sqlite3Fts5PoslistReaderInit(
1175 const u8 *a, int n, /* Poslist buffer to iterate through */
1176 Fts5PoslistReader *pIter /* Iterator object to initialize */
1177);
1178static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
1179
1180typedef struct Fts5PoslistWriter Fts5PoslistWriter;
1181struct Fts5PoslistWriter {
1182 i64 iPrev;
1183};
1184static int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
1185static void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
1186
1187static int sqlite3Fts5PoslistNext64(
1188 const u8 *a, int n, /* Buffer containing poslist */
1189 int *pi, /* IN/OUT: Offset within a[] */
1190 i64 *piOff /* IN/OUT: Current offset */
1191);
1192
1193/* Malloc utility */
1194static void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte);
1195static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
1196
1197/* Character set tests (like isspace(), isalpha() etc.) */
1198static int sqlite3Fts5IsBareword(char t);
1199
1200
1201/* Bucket of terms object used by the integrity-check in offsets=0 mode. */
1202typedef struct Fts5Termset Fts5Termset;
1203static int sqlite3Fts5TermsetNew(Fts5Termset**);
1204static int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
1205static void sqlite3Fts5TermsetFree(Fts5Termset*);
1206
1207/*
1208** End of interface to code in fts5_buffer.c.
1209**************************************************************************/
1210
1211/**************************************************************************
1212** Interface to code in fts5_index.c. fts5_index.c contains contains code
1213** to access the data stored in the %_data table.
1214*/
1215
1216typedef struct Fts5Index Fts5Index;
1217typedef struct Fts5IndexIter Fts5IndexIter;
1218
1219struct Fts5IndexIter {
1220 i64 iRowid;
1221 const u8 *pData;
1222 int nData;
1223 u8 bEof;
1224};
1225
1226#define sqlite3Fts5IterEof(x)((x)->bEof) ((x)->bEof)
1227
1228/*
1229** Values used as part of the flags argument passed to IndexQuery().
1230*/
1231#define FTS5INDEX_QUERY_PREFIX0x0001 0x0001 /* Prefix query */
1232#define FTS5INDEX_QUERY_DESC0x0002 0x0002 /* Docs in descending rowid order */
1233#define FTS5INDEX_QUERY_TEST_NOIDX0x0004 0x0004 /* Do not use prefix index */
1234#define FTS5INDEX_QUERY_SCAN0x0008 0x0008 /* Scan query (fts5vocab) */
1235
1236/* The following are used internally by the fts5_index.c module. They are
1237** defined here only to make it easier to avoid clashes with the flags
1238** above. */
1239#define FTS5INDEX_QUERY_SKIPEMPTY0x0010 0x0010
1240#define FTS5INDEX_QUERY_NOOUTPUT0x0020 0x0020
1241#define FTS5INDEX_QUERY_SKIPHASH0x0040 0x0040
1242#define FTS5INDEX_QUERY_NOTOKENDATA0x0080 0x0080
1243#define FTS5INDEX_QUERY_SCANONETERM0x0100 0x0100
1244
1245/*
1246** Create/destroy an Fts5Index object.
1247*/
1248static int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
1249static int sqlite3Fts5IndexClose(Fts5Index *p);
1250
1251/*
1252** Return a simple checksum value based on the arguments.
1253*/
1254static u64 sqlite3Fts5IndexEntryCksum(
1255 i64 iRowid,
1256 int iCol,
1257 int iPos,
1258 int iIdx,
1259 const char *pTerm,
1260 int nTerm
1261);
1262
1263/*
1264** Argument p points to a buffer containing utf-8 text that is n bytes in
1265** size. Return the number of bytes in the nChar character prefix of the
1266** buffer, or 0 if there are less than nChar characters in total.
1267*/
1268static int sqlite3Fts5IndexCharlenToBytelen(
1269 const char *p,
1270 int nByte,
1271 int nChar
1272);
1273
1274/*
1275** Open a new iterator to iterate though all rowids that match the
1276** specified token or token prefix.
1277*/
1278static int sqlite3Fts5IndexQuery(
1279 Fts5Index *p, /* FTS index to query */
1280 const char *pToken, int nToken, /* Token (or prefix) to query for */
1281 int flags, /* Mask of FTS5INDEX_QUERY_X flags */
1282 Fts5Colset *pColset, /* Match these columns only */
1283 Fts5IndexIter **ppIter /* OUT: New iterator object */
1284);
1285
1286/*
1287** The various operations on open token or token prefix iterators opened
1288** using sqlite3Fts5IndexQuery().
1289*/
1290static int sqlite3Fts5IterNext(Fts5IndexIter*);
1291static int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
1292
1293/*
1294** Close an iterator opened by sqlite3Fts5IndexQuery().
1295*/
1296static void sqlite3Fts5IterClose(Fts5IndexIter*);
1297
1298/*
1299** Close the reader blob handle, if it is open.
1300*/
1301static void sqlite3Fts5IndexCloseReader(Fts5Index*);
1302
1303/*
1304** This interface is used by the fts5vocab module.
1305*/
1306static const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
1307static int sqlite3Fts5IterNextScan(Fts5IndexIter*);
1308static void *sqlite3Fts5StructureRef(Fts5Index*);
1309static void sqlite3Fts5StructureRelease(void*);
1310static int sqlite3Fts5StructureTest(Fts5Index*, void*);
1311
1312/*
1313** Used by xInstToken():
1314*/
1315static int sqlite3Fts5IterToken(
1316 Fts5IndexIter *pIndexIter,
1317 const char *pToken, int nToken,
1318 i64 iRowid,
1319 int iCol,
1320 int iOff,
1321 const char **ppOut, int *pnOut
1322);
1323
1324/*
1325** Insert or remove data to or from the index. Each time a document is
1326** added to or removed from the index, this function is called one or more
1327** times.
1328**
1329** For an insert, it must be called once for each token in the new document.
1330** If the operation is a delete, it must be called (at least) once for each
1331** unique token in the document with an iCol value less than zero. The iPos
1332** argument is ignored for a delete.
1333*/
1334static int sqlite3Fts5IndexWrite(
1335 Fts5Index *p, /* Index to write to */
1336 int iCol, /* Column token appears in (-ve -> delete) */
1337 int iPos, /* Position of token within column */
1338 const char *pToken, int nToken /* Token to add or remove to or from index */
1339);
1340
1341/*
1342** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
1343** document iDocid.
1344*/
1345static int sqlite3Fts5IndexBeginWrite(
1346 Fts5Index *p, /* Index to write to */
1347 int bDelete, /* True if current operation is a delete */
1348 i64 iDocid /* Docid to add or remove data from */
1349);
1350
1351/*
1352** Flush any data stored in the in-memory hash tables to the database.
1353** Also close any open blob handles.
1354*/
1355static int sqlite3Fts5IndexSync(Fts5Index *p);
1356
1357/*
1358** Discard any data stored in the in-memory hash tables. Do not write it
1359** to the database. Additionally, assume that the contents of the %_data
1360** table may have changed on disk. So any in-memory caches of %_data
1361** records must be invalidated.
1362*/
1363static int sqlite3Fts5IndexRollback(Fts5Index *p);
1364
1365/*
1366** Get or set the "averages" values.
1367*/
1368static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
1369static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
1370
1371/*
1372** Functions called by the storage module as part of integrity-check.
1373*/
1374static int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum, int bUseCksum);
1375
1376/*
1377** Called during virtual module initialization to register UDF
1378** fts5_decode() with SQLite
1379*/
1380static int sqlite3Fts5IndexInit(sqlite3*);
1381
1382static int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
1383
1384/*
1385** Return the total number of entries read from the %_data table by
1386** this connection since it was created.
1387*/
1388static int sqlite3Fts5IndexReads(Fts5Index *p);
1389
1390static int sqlite3Fts5IndexReinit(Fts5Index *p);
1391static int sqlite3Fts5IndexOptimize(Fts5Index *p);
1392static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
1393static int sqlite3Fts5IndexReset(Fts5Index *p);
1394
1395static int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
1396
1397static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
1398static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
1399
1400static void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
1401
1402/* Used to populate hash tables for xInstToken in detail=none/column mode. */
1403static int sqlite3Fts5IndexIterWriteTokendata(
1404 Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff
1405);
1406
1407/*
1408** End of interface to code in fts5_index.c.
1409**************************************************************************/
1410
1411/**************************************************************************
1412** Interface to code in fts5_varint.c.
1413*/
1414static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
1415static int sqlite3Fts5GetVarintLen(u32 iVal);
1416static u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
1417static int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
1418
1419#define fts5GetVarint32(a,b)sqlite3Fts5GetVarint32(a,(u32*)&(b)) sqlite3Fts5GetVarint32(a,(u32*)&(b))
1420#define fts5GetVarintsqlite3Fts5GetVarint sqlite3Fts5GetVarint
1421
1422#define fts5FastGetVarint32(a, iOff, nVal){ nVal = (a)[iOff++]; if( nVal & 0x80 ){ iOff--; iOff += sqlite3Fts5GetVarint32
(&(a)[iOff],(u32*)&(nVal)); } } { \
1423 nVal = (a)[iOff++]; \
1424 if( nVal & 0x80 ){ \
1425 iOff--; \
1426 iOff += fts5GetVarint32(&(a)[iOff], nVal)sqlite3Fts5GetVarint32(&(a)[iOff],(u32*)&(nVal)); \
1427 } \
1428}
1429
1430
1431/*
1432** End of interface to code in fts5_varint.c.
1433**************************************************************************/
1434
1435
1436/**************************************************************************
1437** Interface to code in fts5_main.c.
1438*/
1439
1440/*
1441** Virtual-table object.
1442*/
1443typedef struct Fts5Table Fts5Table;
1444struct Fts5Table {
1445 sqlite3_vtab base; /* Base class used by SQLite core */
1446 Fts5Config *pConfig; /* Virtual table configuration */
1447 Fts5Index *pIndex; /* Full-text index */
1448};
1449
1450static int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig);
1451
1452static Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
1453
1454static int sqlite3Fts5FlushToDisk(Fts5Table*);
1455
1456static void sqlite3Fts5ClearLocale(Fts5Config *pConfig);
1457static void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc);
1458
1459static int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal);
1460static int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal,
1461 const char **ppText, int *pnText, const char **ppLoc, int *pnLoc
1462);
1463
1464/*
1465** End of interface to code in fts5.c.
1466**************************************************************************/
1467
1468/**************************************************************************
1469** Interface to code in fts5_hash.c.
1470*/
1471typedef struct Fts5Hash Fts5Hash;
1472
1473/*
1474** Create a hash table, free a hash table.
1475*/
1476static int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
1477static void sqlite3Fts5HashFree(Fts5Hash*);
1478
1479static int sqlite3Fts5HashWrite(
1480 Fts5Hash*,
1481 i64 iRowid, /* Rowid for this entry */
1482 int iCol, /* Column token appears in (-ve -> delete) */
1483 int iPos, /* Position of token within column */
1484 char bByte,
1485 const char *pToken, int nToken /* Token to add or remove to or from index */
1486);
1487
1488/*
1489** Empty (but do not delete) a hash table.
1490*/
1491static void sqlite3Fts5HashClear(Fts5Hash*);
1492
1493/*
1494** Return true if the hash is empty, false otherwise.
1495*/
1496static int sqlite3Fts5HashIsEmpty(Fts5Hash*);
1497
1498static int sqlite3Fts5HashQuery(
1499 Fts5Hash*, /* Hash table to query */
1500 int nPre,
1501 const char *pTerm, int nTerm, /* Query term */
1502 void **ppObj, /* OUT: Pointer to doclist for pTerm */
1503 int *pnDoclist /* OUT: Size of doclist in bytes */
1504);
1505
1506static int sqlite3Fts5HashScanInit(
1507 Fts5Hash*, /* Hash table to query */
1508 const char *pTerm, int nTerm /* Query prefix */
1509);
1510static void sqlite3Fts5HashScanNext(Fts5Hash*);
1511static int sqlite3Fts5HashScanEof(Fts5Hash*);
1512static void sqlite3Fts5HashScanEntry(Fts5Hash *,
1513 const char **pzTerm, /* OUT: term (nul-terminated) */
1514 int *pnTerm, /* OUT: Size of term in bytes */
1515 const u8 **ppDoclist, /* OUT: pointer to doclist */
1516 int *pnDoclist /* OUT: size of doclist in bytes */
1517);
1518
1519
1520
1521/*
1522** End of interface to code in fts5_hash.c.
1523**************************************************************************/
1524
1525/**************************************************************************
1526** Interface to code in fts5_storage.c. fts5_storage.c contains contains
1527** code to access the data stored in the %_content and %_docsize tables.
1528*/
1529
1530#define FTS5_STMT_SCAN_ASC0 0 /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
1531#define FTS5_STMT_SCAN_DESC1 1 /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
1532#define FTS5_STMT_LOOKUP2 2 /* SELECT rowid, * FROM ... WHERE rowid=? */
1533
1534typedef struct Fts5Storage Fts5Storage;
1535
1536static int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
1537static int sqlite3Fts5StorageClose(Fts5Storage *p);
1538static int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
1539
1540static int sqlite3Fts5DropAll(Fts5Config*);
1541static int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
1542
1543static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int);
1544static int sqlite3Fts5StorageContentInsert(Fts5Storage *p, int, sqlite3_value**, i64*);
1545static int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
1546
1547static int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg);
1548
1549static int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
1550static void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
1551
1552static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
1553static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
1554static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
1555
1556static int sqlite3Fts5StorageSync(Fts5Storage *p);
1557static int sqlite3Fts5StorageRollback(Fts5Storage *p);
1558
1559static int sqlite3Fts5StorageConfigValue(
1560 Fts5Storage *p, const char*, sqlite3_value*, int
1561);
1562
1563static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
1564static int sqlite3Fts5StorageRebuild(Fts5Storage *p);
1565static int sqlite3Fts5StorageOptimize(Fts5Storage *p);
1566static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
1567static int sqlite3Fts5StorageReset(Fts5Storage *p);
1568
1569static void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*);
1570static int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel);
1571
1572/*
1573** End of interface to code in fts5_storage.c.
1574**************************************************************************/
1575
1576
1577/**************************************************************************
1578** Interface to code in fts5_expr.c.
1579*/
1580typedef struct Fts5Expr Fts5Expr;
1581typedef struct Fts5ExprNode Fts5ExprNode;
1582typedef struct Fts5Parse Fts5Parse;
1583typedef struct Fts5Token Fts5Token;
1584typedef struct Fts5ExprPhrase Fts5ExprPhrase;
1585typedef struct Fts5ExprNearset Fts5ExprNearset;
1586
1587struct Fts5Token {
1588 const char *p; /* Token text (not NULL terminated) */
1589 int n; /* Size of buffer p in bytes */
1590};
1591
1592/* Parse a MATCH expression. */
1593static int sqlite3Fts5ExprNew(
1594 Fts5Config *pConfig,
1595 int bPhraseToAnd,
1596 int iCol, /* Column on LHS of MATCH operator */
1597 const char *zExpr,
1598 Fts5Expr **ppNew,
1599 char **pzErr
1600);
1601static int sqlite3Fts5ExprPattern(
1602 Fts5Config *pConfig,
1603 int bGlob,
1604 int iCol,
1605 const char *zText,
1606 Fts5Expr **pp
1607);
1608
1609/*
1610** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
1611** rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
1612** rc = sqlite3Fts5ExprNext(pExpr)
1613** ){
1614** // The document with rowid iRowid matches the expression!
1615** i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
1616** }
1617*/
1618static int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, i64, int bDesc);
1619static int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
1620static int sqlite3Fts5ExprEof(Fts5Expr*);
1621static i64 sqlite3Fts5ExprRowid(Fts5Expr*);
1622
1623static void sqlite3Fts5ExprFree(Fts5Expr*);
1624static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2);
1625
1626/* Called during startup to register a UDF with SQLite */
1627static int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
1628
1629static int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
1630static int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
1631static int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
1632
1633typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
1634static Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
1635static int sqlite3Fts5ExprPopulatePoslists(
1636 Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
1637);
1638static void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
1639
1640static int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
1641
1642static int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
1643
1644static int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
1645static int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
1646static void sqlite3Fts5ExprClearTokens(Fts5Expr*);
1647
1648/*******************************************
1649** The fts5_expr.c API above this point is used by the other hand-written
1650** C code in this module. The interfaces below this point are called by
1651** the parser code in fts5parse.y. */
1652
1653static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
1654
1655static Fts5ExprNode *sqlite3Fts5ParseNode(
1656 Fts5Parse *pParse,
1657 int eType,
1658 Fts5ExprNode *pLeft,
1659 Fts5ExprNode *pRight,
1660 Fts5ExprNearset *pNear
1661);
1662
1663static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
1664 Fts5Parse *pParse,
1665 Fts5ExprNode *pLeft,
1666 Fts5ExprNode *pRight
1667);
1668
1669static Fts5ExprPhrase *sqlite3Fts5ParseTerm(
1670 Fts5Parse *pParse,
1671 Fts5ExprPhrase *pPhrase,
1672 Fts5Token *pToken,
1673 int bPrefix
1674);
1675
1676static void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
1677
1678static Fts5ExprNearset *sqlite3Fts5ParseNearset(
1679 Fts5Parse*,
1680 Fts5ExprNearset*,
1681 Fts5ExprPhrase*
1682);
1683
1684static Fts5Colset *sqlite3Fts5ParseColset(
1685 Fts5Parse*,
1686 Fts5Colset*,
1687 Fts5Token *
1688);
1689
1690static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
1691static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
1692static void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
1693
1694static void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
1695static void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
1696static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
1697static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
1698static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
1699
1700/*
1701** End of interface to code in fts5_expr.c.
1702**************************************************************************/
1703
1704
1705
1706/**************************************************************************
1707** Interface to code in fts5_aux.c.
1708*/
1709
1710static int sqlite3Fts5AuxInit(fts5_api*);
1711/*
1712** End of interface to code in fts5_aux.c.
1713**************************************************************************/
1714
1715/**************************************************************************
1716** Interface to code in fts5_tokenizer.c.
1717*/
1718
1719static int sqlite3Fts5TokenizerInit(fts5_api*);
1720static int sqlite3Fts5TokenizerPattern(
1721 int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
1722 Fts5Tokenizer *pTok
1723);
1724static int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*);
1725/*
1726** End of interface to code in fts5_tokenizer.c.
1727**************************************************************************/
1728
1729/**************************************************************************
1730** Interface to code in fts5_vocab.c.
1731*/
1732
1733static int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
1734
1735/*
1736** End of interface to code in fts5_vocab.c.
1737**************************************************************************/
1738
1739
1740/**************************************************************************
1741** Interface to automatically generated code in fts5_unicode2.c.
1742*/
1743static int sqlite3Fts5UnicodeIsdiacritic(int c);
1744static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
1745
1746static int sqlite3Fts5UnicodeCatParse(const char*, u8*);
1747static int sqlite3Fts5UnicodeCategory(u32 iCode);
1748static void sqlite3Fts5UnicodeAscii(u8*, u8*);
1749/*
1750** End of interface to code in fts5_unicode2.c.
1751**************************************************************************/
1752
1753#endif
1754
1755#line 1 "fts5parse.h"
1756#define FTS5_OR1 1
1757#define FTS5_AND2 2
1758#define FTS5_NOT3 3
1759#define FTS5_TERM4 4
1760#define FTS5_COLON5 5
1761#define FTS5_MINUS6 6
1762#define FTS5_LCP7 7
1763#define FTS5_RCP8 8
1764#define FTS5_STRING9 9
1765#define FTS5_LP10 10
1766#define FTS5_RP11 11
1767#define FTS5_CARET12 12
1768#define FTS5_COMMA13 13
1769#define FTS5_PLUS14 14
1770#define FTS5_STAR15 15
1771
1772#line 1 "fts5parse.c"
1773/* This file is automatically generated by Lemon from input grammar
1774** source file "fts5parse.y".
1775*/
1776/*
1777** 2000-05-29
1778**
1779** The author disclaims copyright to this source code. In place of
1780** a legal notice, here is a blessing:
1781**
1782** May you do good and not evil.
1783** May you find forgiveness for yourself and forgive others.
1784** May you share freely, never taking more than you give.
1785**
1786*************************************************************************
1787** Driver template for the LEMON parser generator.
1788**
1789** The "lemon" program processes an LALR(1) input grammar file, then uses
1790** this template to construct a parser. The "lemon" program inserts text
1791** at each "%%" line. Also, any "P-a-r-s-e" identifier prefix (without the
1792** interstitial "-" characters) contained in this template is changed into
1793** the value of the %name directive from the grammar. Otherwise, the content
1794** of this template is copied straight through into the generate parser
1795** source file.
1796**
1797** The following is the concatenation of all %include directives from the
1798** input grammar file:
1799*/
1800/************ Begin %include sections from the grammar ************************/
1801#line 47 "fts5parse.y"
1802
1803/* #include "fts5Int.h" */
1804/* #include "fts5parse.h" */
1805
1806/*
1807** Disable all error recovery processing in the parser push-down
1808** automaton.
1809*/
1810#define fts5YYNOERRORRECOVERY1 1
1811
1812/*
1813** Make fts5yytestcase() the same as testcase()
1814*/
1815#define fts5yytestcase(X) testcase(X)
1816
1817/*
1818** Indicate that sqlite3ParserFree() will never be called with a null
1819** pointer.
1820*/
1821#define fts5YYPARSEFREENOTNULL1 1
1822
1823/*
1824** Alternative datatype for the argument to the malloc() routine passed
1825** into sqlite3ParserAlloc(). The default is size_t.
1826*/
1827#define fts5YYMALLOCARGTYPEu64 u64
1828
1829#line 58 "fts5parse.c"
1830/**************** End of %include directives **********************************/
1831/* These constants specify the various numeric values for terminal symbols.
1832***************** Begin token definitions *************************************/
1833#ifndef FTS5_OR1
1834#define FTS5_OR1 1
1835#define FTS5_AND2 2
1836#define FTS5_NOT3 3
1837#define FTS5_TERM4 4
1838#define FTS5_COLON5 5
1839#define FTS5_MINUS6 6
1840#define FTS5_LCP7 7
1841#define FTS5_RCP8 8
1842#define FTS5_STRING9 9
1843#define FTS5_LP10 10
1844#define FTS5_RP11 11
1845#define FTS5_CARET12 12
1846#define FTS5_COMMA13 13
1847#define FTS5_PLUS14 14
1848#define FTS5_STAR15 15
1849#endif
1850/**************** End token definitions ***************************************/
1851
1852/* The next sections is a series of control #defines.
1853** various aspects of the generated parser.
1854** fts5YYCODETYPE is the data type used to store the integer codes
1855** that represent terminal and non-terminal symbols.
1856** "unsigned char" is used if there are fewer than
1857** 256 symbols. Larger types otherwise.
1858** fts5YYNOCODE is a number of type fts5YYCODETYPE that is not used for
1859** any terminal or nonterminal symbol.
1860** fts5YYFALLBACK If defined, this indicates that one or more tokens
1861** (also known as: "terminal symbols") have fall-back
1862** values which should be used if the original symbol
1863** would not parse. This permits keywords to sometimes
1864** be used as identifiers, for example.
1865** fts5YYACTIONTYPE is the data type used for "action codes" - numbers
1866** that indicate what to do in response to the next
1867** token.
1868** sqlite3Fts5ParserFTS5TOKENTYPE is the data type used for minor type for terminal
1869** symbols. Background: A "minor type" is a semantic
1870** value associated with a terminal or non-terminal
1871** symbols. For example, for an "ID" terminal symbol,
1872** the minor type might be the name of the identifier.
1873** Each non-terminal can have a different minor type.
1874** Terminal symbols all have the same minor type, though.
1875** This macros defines the minor type for terminal
1876** symbols.
1877** fts5YYMINORTYPE is the data type used for all minor types.
1878** This is typically a union of many types, one of
1879** which is sqlite3Fts5ParserFTS5TOKENTYPE. The entry in the union
1880** for terminal symbols is called "fts5yy0".
1881** fts5YYSTACKDEPTH is the maximum depth of the parser's stack. If
1882** zero the stack is dynamically sized using realloc()
1883** sqlite3Fts5ParserARG_SDECL A static variable declaration for the %extra_argument
1884** sqlite3Fts5ParserARG_PDECL A parameter declaration for the %extra_argument
1885** sqlite3Fts5ParserARG_PARAM Code to pass %extra_argument as a subroutine parameter
1886** sqlite3Fts5ParserARG_STORE Code to store %extra_argument into fts5yypParser
1887** sqlite3Fts5ParserARG_FETCH Code to extract %extra_argument from fts5yypParser
1888** sqlite3Fts5ParserCTX_* As sqlite3Fts5ParserARG_ except for %extra_context
1889** fts5YYREALLOC Name of the realloc() function to use
1890** fts5YYFREE Name of the free() function to use
1891** fts5YYDYNSTACK True if stack space should be extended on heap
1892** fts5YYERRORSYMBOL is the code number of the error symbol. If not
1893** defined, then do no error processing.
1894** fts5YYNSTATE the combined number of states.
1895** fts5YYNRULE the number of rules in the grammar
1896** fts5YYNFTS5TOKEN Number of terminal symbols
1897** fts5YY_MAX_SHIFT Maximum value for shift actions
1898** fts5YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions
1899** fts5YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions
1900** fts5YY_ERROR_ACTION The fts5yy_action[] code for syntax error
1901** fts5YY_ACCEPT_ACTION The fts5yy_action[] code for accept
1902** fts5YY_NO_ACTION The fts5yy_action[] code for no-op
1903** fts5YY_MIN_REDUCE Minimum value for reduce actions
1904** fts5YY_MAX_REDUCE Maximum value for reduce actions
1905** fts5YY_MIN_DSTRCTR Minimum symbol value that has a destructor
1906** fts5YY_MAX_DSTRCTR Maximum symbol value that has a destructor
1907*/
1908#ifndef INTERFACE1
1909# define INTERFACE1 1
1910#endif
1911/************* Begin control #defines *****************************************/
1912#define fts5YYCODETYPEunsigned char unsigned char
1913#define fts5YYNOCODE27 27
1914#define fts5YYACTIONTYPEunsigned char unsigned char
1915#define sqlite3Fts5ParserFTS5TOKENTYPEFts5Token Fts5Token
1916typedef union {
1917 int fts5yyinit;
1918 sqlite3Fts5ParserFTS5TOKENTYPEFts5Token fts5yy0;
1919 int fts5yy4;
1920 Fts5Colset* fts5yy11;
1921 Fts5ExprNode* fts5yy24;
1922 Fts5ExprNearset* fts5yy46;
1923 Fts5ExprPhrase* fts5yy53;
1924} fts5YYMINORTYPE;
1925#ifndef fts5YYSTACKDEPTH100
1926#define fts5YYSTACKDEPTH100 100
1927#endif
1928#define sqlite3Fts5ParserARG_SDECLFts5Parse *pParse; Fts5Parse *pParse;
1929#define sqlite3Fts5ParserARG_PDECL,Fts5Parse *pParse ,Fts5Parse *pParse
1930#define sqlite3Fts5ParserARG_PARAM,pParse ,pParse
1931#define sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse; Fts5Parse *pParse=fts5yypParser->pParse;
1932#define sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse; fts5yypParser->pParse=pParse;
1933#undef fts5YYREALLOCrealloc
1934#define fts5YYREALLOCrealloc realloc
1935#undef fts5YYFREEfree
1936#define fts5YYFREEfree free
1937#undef fts5YYDYNSTACK0
1938#define fts5YYDYNSTACK0 0
1939#undef fts5YYSIZELIMIT
1940#define sqlite3Fts5ParserCTX(P)0 0
1941#define sqlite3Fts5ParserCTX_SDECL
1942#define sqlite3Fts5ParserCTX_PDECL
1943#define sqlite3Fts5ParserCTX_PARAM
1944#define sqlite3Fts5ParserCTX_FETCH
1945#define sqlite3Fts5ParserCTX_STORE
1946#undef fts5YYERRORSYMBOL
1947#undef fts5YYERRSYMDT
1948#undef fts5YYFALLBACK
1949#define fts5YYNSTATE35 35
1950#define fts5YYNRULE28 28
1951#define fts5YYNRULE_WITH_ACTION28 28
1952#define fts5YYNFTS5TOKEN16 16
1953#define fts5YY_MAX_SHIFT34 34
1954#define fts5YY_MIN_SHIFTREDUCE52 52
1955#define fts5YY_MAX_SHIFTREDUCE79 79
1956#define fts5YY_ERROR_ACTION80 80
1957#define fts5YY_ACCEPT_ACTION81 81
1958#define fts5YY_NO_ACTION82 82
1959#define fts5YY_MIN_REDUCE83 83
1960#define fts5YY_MAX_REDUCE110 110
1961#define fts5YY_MIN_DSTRCTR16 16
1962#define fts5YY_MAX_DSTRCTR24 24
1963/************* End control #defines *******************************************/
1964#define fts5YY_NLOOKAHEAD((int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0]))) ((int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0])))
1965
1966/* Define the fts5yytestcase() macro to be a no-op if is not already defined
1967** otherwise.
1968**
1969** Applications can choose to define fts5yytestcase() in the %include section
1970** to a macro that can assist in verifying code coverage. For production
1971** code the fts5yytestcase() macro should be turned off. But it is useful
1972** for testing.
1973*/
1974#ifndef fts5yytestcase
1975# define fts5yytestcase(X)
1976#endif
1977
1978/* Macro to determine if stack space has the ability to grow using
1979** heap memory.
1980*/
1981#if fts5YYSTACKDEPTH100<=0 || fts5YYDYNSTACK0
1982# define fts5YYGROWABLESTACK0 1
1983#else
1984# define fts5YYGROWABLESTACK0 0
1985#endif
1986
1987/* Guarantee a minimum number of initial stack slots.
1988*/
1989#if fts5YYSTACKDEPTH100<=0
1990# undef fts5YYSTACKDEPTH100
1991# define fts5YYSTACKDEPTH100 2 /* Need a minimum stack size */
1992#endif
1993
1994
1995/* Next are the tables used to determine what action to take based on the
1996** current state and lookahead token. These tables are used to implement
1997** functions that take a state number and lookahead value and return an
1998** action integer.
1999**
2000** Suppose the action integer is N. Then the action is determined as
2001** follows
2002**
2003** 0 <= N <= fts5YY_MAX_SHIFT Shift N. That is, push the lookahead
2004** token onto the stack and goto state N.
2005**
2006** N between fts5YY_MIN_SHIFTREDUCE Shift to an arbitrary state then
2007** and fts5YY_MAX_SHIFTREDUCE reduce by rule N-fts5YY_MIN_SHIFTREDUCE.
2008**
2009** N == fts5YY_ERROR_ACTION A syntax error has occurred.
2010**
2011** N == fts5YY_ACCEPT_ACTION The parser accepts its input.
2012**
2013** N == fts5YY_NO_ACTION No such action. Denotes unused
2014** slots in the fts5yy_action[] table.
2015**
2016** N between fts5YY_MIN_REDUCE Reduce by rule N-fts5YY_MIN_REDUCE
2017** and fts5YY_MAX_REDUCE
2018**
2019** The action table is constructed as a single large table named fts5yy_action[].
2020** Given state S and lookahead X, the action is computed as either:
2021**
2022** (A) N = fts5yy_action[ fts5yy_shift_ofst[S] + X ]
2023** (B) N = fts5yy_default[S]
2024**
2025** The (A) formula is preferred. The B formula is used instead if
2026** fts5yy_lookahead[fts5yy_shift_ofst[S]+X] is not equal to X.
2027**
2028** The formulas above are for computing the action when the lookahead is
2029** a terminal symbol. If the lookahead is a non-terminal (as occurs after
2030** a reduce action) then the fts5yy_reduce_ofst[] array is used in place of
2031** the fts5yy_shift_ofst[] array.
2032**
2033** The following are the tables generated in this section:
2034**
2035** fts5yy_action[] A single table containing all actions.
2036** fts5yy_lookahead[] A table containing the lookahead for each entry in
2037** fts5yy_action. Used to detect hash collisions.
2038** fts5yy_shift_ofst[] For each state, the offset into fts5yy_action for
2039** shifting terminals.
2040** fts5yy_reduce_ofst[] For each state, the offset into fts5yy_action for
2041** shifting non-terminals after a reduce.
2042** fts5yy_default[] Default action for each state.
2043**
2044*********** Begin parsing tables **********************************************/
2045#define fts5YY_ACTTAB_COUNT(105) (105)
2046static const fts5YYACTIONTYPEunsigned char fts5yy_action[] = {
2047 /* 0 */ 81, 20, 96, 6, 28, 99, 98, 26, 26, 18,
2048 /* 10 */ 96, 6, 28, 17, 98, 56, 26, 19, 96, 6,
2049 /* 20 */ 28, 14, 98, 14, 26, 31, 92, 96, 6, 28,
2050 /* 30 */ 108, 98, 25, 26, 21, 96, 6, 28, 78, 98,
2051 /* 40 */ 58, 26, 29, 96, 6, 28, 107, 98, 22, 26,
2052 /* 50 */ 24, 16, 12, 11, 1, 13, 13, 24, 16, 23,
2053 /* 60 */ 11, 33, 34, 13, 97, 8, 27, 32, 98, 7,
2054 /* 70 */ 26, 3, 4, 5, 3, 4, 5, 3, 83, 4,
2055 /* 80 */ 5, 3, 63, 5, 3, 62, 12, 2, 86, 13,
2056 /* 90 */ 9, 30, 10, 10, 54, 57, 75, 78, 78, 53,
2057 /* 100 */ 57, 15, 82, 82, 71,
2058};
2059static const fts5YYCODETYPEunsigned char fts5yy_lookahead[] = {
2060 /* 0 */ 16, 17, 18, 19, 20, 22, 22, 24, 24, 17,
2061 /* 10 */ 18, 19, 20, 7, 22, 9, 24, 17, 18, 19,
2062 /* 20 */ 20, 9, 22, 9, 24, 13, 17, 18, 19, 20,
2063 /* 30 */ 26, 22, 24, 24, 17, 18, 19, 20, 15, 22,
2064 /* 40 */ 9, 24, 17, 18, 19, 20, 26, 22, 21, 24,
2065 /* 50 */ 6, 7, 9, 9, 10, 12, 12, 6, 7, 21,
2066 /* 60 */ 9, 24, 25, 12, 18, 5, 20, 14, 22, 5,
2067 /* 70 */ 24, 3, 1, 2, 3, 1, 2, 3, 0, 1,
2068 /* 80 */ 2, 3, 11, 2, 3, 11, 9, 10, 5, 12,
2069 /* 90 */ 23, 24, 10, 10, 8, 9, 9, 15, 15, 8,
2070 /* 100 */ 9, 9, 27, 27, 11, 27, 27, 27, 27, 27,
2071 /* 110 */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
2072 /* 120 */ 27,
2073};
2074#define fts5YY_SHIFT_COUNT(34) (34)
2075#define fts5YY_SHIFT_MIN(0) (0)
2076#define fts5YY_SHIFT_MAX(93) (93)
2077static const unsigned char fts5yy_shift_ofst[] = {
2078 /* 0 */ 44, 44, 44, 44, 44, 44, 51, 77, 43, 12,
2079 /* 10 */ 14, 83, 82, 14, 23, 23, 31, 31, 71, 74,
2080 /* 20 */ 78, 81, 86, 91, 6, 53, 53, 60, 64, 68,
2081 /* 30 */ 53, 87, 92, 53, 93,
2082};
2083#define fts5YY_REDUCE_COUNT(17) (17)
2084#define fts5YY_REDUCE_MIN(-17) (-17)
2085#define fts5YY_REDUCE_MAX(67) (67)
2086static const signed char fts5yy_reduce_ofst[] = {
2087 /* 0 */ -16, -8, 0, 9, 17, 25, 46, -17, -17, 37,
2088 /* 10 */ 67, 4, 4, 8, 4, 20, 27, 38,
2089};
2090static const fts5YYACTIONTYPEunsigned char fts5yy_default[] = {
2091 /* 0 */ 80, 80, 80, 80, 80, 80, 95, 80, 80, 105,
2092 /* 10 */ 80, 110, 110, 80, 110, 110, 80, 80, 80, 80,
2093 /* 20 */ 80, 91, 80, 80, 80, 101, 100, 80, 80, 90,
2094 /* 30 */ 103, 80, 80, 104, 80,
2095};
2096/********** End of lemon-generated parsing tables *****************************/
2097
2098/* The next table maps tokens (terminal symbols) into fallback tokens.
2099** If a construct like the following:
2100**
2101** %fallback ID X Y Z.
2102**
2103** appears in the grammar, then ID becomes a fallback token for X, Y,
2104** and Z. Whenever one of the tokens X, Y, or Z is input to the parser
2105** but it does not parse, the type of the token is changed to ID and
2106** the parse is retried before an error is thrown.
2107**
2108** This feature can be used, for example, to cause some keywords in a language
2109** to revert to identifiers if they keyword does not apply in the context where
2110** it appears.
2111*/
2112#ifdef fts5YYFALLBACK
2113static const fts5YYCODETYPEunsigned char fts5yyFallback[] = {
2114};
2115#endif /* fts5YYFALLBACK */
2116
2117/* The following structure represents a single element of the
2118** parser's stack. Information stored includes:
2119**
2120** + The state number for the parser at this level of the stack.
2121**
2122** + The value of the token stored at this level of the stack.
2123** (In other words, the "major" token.)
2124**
2125** + The semantic value stored at this level of the stack. This is
2126** the information used by the action routines in the grammar.
2127** It is sometimes called the "minor" token.
2128**
2129** After the "shift" half of a SHIFTREDUCE action, the stateno field
2130** actually contains the reduce action for the second half of the
2131** SHIFTREDUCE.
2132*/
2133struct fts5yyStackEntry {
2134 fts5YYACTIONTYPEunsigned char stateno; /* The state-number, or reduce action in SHIFTREDUCE */
2135 fts5YYCODETYPEunsigned char major; /* The major token value. This is the code
2136 ** number for the token at this stack level */
2137 fts5YYMINORTYPE minor; /* The user-supplied minor token value. This
2138 ** is the value of the token */
2139};
2140typedef struct fts5yyStackEntry fts5yyStackEntry;
2141
2142/* The state of the parser is completely contained in an instance of
2143** the following structure */
2144struct fts5yyParser {
2145 fts5yyStackEntry *fts5yytos; /* Pointer to top element of the stack */
2146#ifdef fts5YYTRACKMAXSTACKDEPTH
2147 int fts5yyhwm; /* High-water mark of the stack */
2148#endif
2149#ifndef fts5YYNOERRORRECOVERY1
2150 int fts5yyerrcnt; /* Shifts left before out of the error */
2151#endif
2152 sqlite3Fts5ParserARG_SDECLFts5Parse *pParse; /* A place to hold %extra_argument */
2153 sqlite3Fts5ParserCTX_SDECL /* A place to hold %extra_context */
2154 fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */
2155 fts5yyStackEntry *fts5yystack; /* The parser stack */
2156 fts5yyStackEntry fts5yystk0[fts5YYSTACKDEPTH100]; /* Initial stack space */
2157};
2158typedef struct fts5yyParser fts5yyParser;
2159
2160#include <assert.h>
2161#ifndef NDEBUG1
2162#include <stdio.h>
2163static FILE *fts5yyTraceFILE = 0;
2164static char *fts5yyTracePrompt = 0;
2165#endif /* NDEBUG */
2166
2167#ifndef NDEBUG1
2168/*
2169** Turn parser tracing on by giving a stream to which to write the trace
2170** and a prompt to preface each trace message. Tracing is turned off
2171** by making either argument NULL
2172**
2173** Inputs:
2174** <ul>
2175** <li> A FILE* to which trace output should be written.
2176** If NULL, then tracing is turned off.
2177** <li> A prefix string written at the beginning of every
2178** line of trace output. If NULL, then tracing is
2179** turned off.
2180** </ul>
2181**
2182** Outputs:
2183** None.
2184*/
2185static void sqlite3Fts5ParserTrace(FILE *TraceFILE, char *zTracePrompt){
2186 fts5yyTraceFILE = TraceFILE;
2187 fts5yyTracePrompt = zTracePrompt;
2188 if( fts5yyTraceFILE==0 ) fts5yyTracePrompt = 0;
2189 else if( fts5yyTracePrompt==0 ) fts5yyTraceFILE = 0;
2190}
2191#endif /* NDEBUG */
2192
2193#if defined(fts5YYCOVERAGE) || !defined(NDEBUG1)
2194/* For tracing shifts, the names of all terminals and nonterminals
2195** are required. The following table supplies these names */
2196static const char *const fts5yyTokenName[] = {
2197 /* 0 */ "$",
2198 /* 1 */ "OR",
2199 /* 2 */ "AND",
2200 /* 3 */ "NOT",
2201 /* 4 */ "TERM",
2202 /* 5 */ "COLON",
2203 /* 6 */ "MINUS",
2204 /* 7 */ "LCP",
2205 /* 8 */ "RCP",
2206 /* 9 */ "STRING",
2207 /* 10 */ "LP",
2208 /* 11 */ "RP",
2209 /* 12 */ "CARET",
2210 /* 13 */ "COMMA",
2211 /* 14 */ "PLUS",
2212 /* 15 */ "STAR",
2213 /* 16 */ "input",
2214 /* 17 */ "expr",
2215 /* 18 */ "cnearset",
2216 /* 19 */ "exprlist",
2217 /* 20 */ "colset",
2218 /* 21 */ "colsetlist",
2219 /* 22 */ "nearset",
2220 /* 23 */ "nearphrases",
2221 /* 24 */ "phrase",
2222 /* 25 */ "neardist_opt",
2223 /* 26 */ "star_opt",
2224};
2225#endif /* defined(fts5YYCOVERAGE) || !defined(NDEBUG) */
2226
2227#ifndef NDEBUG1
2228/* For tracing reduce actions, the names of all rules are required.
2229*/
2230static const char *const fts5yyRuleName[] = {
2231 /* 0 */ "input ::= expr",
2232 /* 1 */ "colset ::= MINUS LCP colsetlist RCP",
2233 /* 2 */ "colset ::= LCP colsetlist RCP",
2234 /* 3 */ "colset ::= STRING",
2235 /* 4 */ "colset ::= MINUS STRING",
2236 /* 5 */ "colsetlist ::= colsetlist STRING",
2237 /* 6 */ "colsetlist ::= STRING",
2238 /* 7 */ "expr ::= expr AND expr",
2239 /* 8 */ "expr ::= expr OR expr",
2240 /* 9 */ "expr ::= expr NOT expr",
2241 /* 10 */ "expr ::= colset COLON LP expr RP",
2242 /* 11 */ "expr ::= LP expr RP",
2243 /* 12 */ "expr ::= exprlist",
2244 /* 13 */ "exprlist ::= cnearset",
2245 /* 14 */ "exprlist ::= exprlist cnearset",
2246 /* 15 */ "cnearset ::= nearset",
2247 /* 16 */ "cnearset ::= colset COLON nearset",
2248 /* 17 */ "nearset ::= phrase",
2249 /* 18 */ "nearset ::= CARET phrase",
2250 /* 19 */ "nearset ::= STRING LP nearphrases neardist_opt RP",
2251 /* 20 */ "nearphrases ::= phrase",
2252 /* 21 */ "nearphrases ::= nearphrases phrase",
2253 /* 22 */ "neardist_opt ::=",
2254 /* 23 */ "neardist_opt ::= COMMA STRING",
2255 /* 24 */ "phrase ::= phrase PLUS STRING star_opt",
2256 /* 25 */ "phrase ::= STRING star_opt",
2257 /* 26 */ "star_opt ::= STAR",
2258 /* 27 */ "star_opt ::=",
2259};
2260#endif /* NDEBUG */
2261
2262
2263#if fts5YYGROWABLESTACK0
2264/*
2265** Try to increase the size of the parser stack. Return the number
2266** of errors. Return 0 on success.
2267*/
2268static int fts5yyGrowStack(fts5yyParser *p)1{
2269 int oldSize = 1 + (int)(p->fts5yystackEnd - p->fts5yystack);
2270 int newSize;
2271 int idx;
2272 fts5yyStackEntry *pNew;
2273#ifdef fts5YYSIZELIMIT
2274 int nLimit = fts5YYSIZELIMIT(sqlite3Fts5ParserCTX(p)0);
2275#endif
2276
2277 newSize = oldSize*2 + 100;
2278#ifdef fts5YYSIZELIMIT
2279 if( newSize>nLimit ){
2280 newSize = nLimit;
2281 if( newSize<=oldSize ) return 1;
2282 }
2283#endif
2284 idx = (int)(p->fts5yytos - p->fts5yystack);
2285 if( p->fts5yystack==p->fts5yystk0 ){
2286 pNew = fts5YYREALLOCrealloc(0, newSize*sizeof(pNew[0]), sqlite3Fts5ParserCTX(p)0);
2287 if( pNew==0 ) return 1;
2288 memcpy(pNew, p->fts5yystack, oldSize*sizeof(pNew[0]));
2289 }else{
2290 pNew = fts5YYREALLOCrealloc(p->fts5yystack, newSize*sizeof(pNew[0]), sqlite3Fts5ParserCTX(p)0);
2291 if( pNew==0 ) return 1;
2292 }
2293 p->fts5yystack = pNew;
2294 p->fts5yytos = &p->fts5yystack[idx];
2295#ifndef NDEBUG1
2296 if( fts5yyTraceFILE ){
2297 fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n",
2298 fts5yyTracePrompt, oldSize, newSize);
2299 }
2300#endif
2301 p->fts5yystackEnd = &p->fts5yystack[newSize-1];
2302 return 0;
2303}
2304#endif /* fts5YYGROWABLESTACK */
2305
2306#if !fts5YYGROWABLESTACK0
2307/* For builds that do no have a growable stack, fts5yyGrowStack always
2308** returns an error.
2309*/
2310# define fts5yyGrowStack(X)1 1
2311#endif
2312
2313/* Datatype of the argument to the memory allocated passed as the
2314** second argument to sqlite3Fts5ParserAlloc() below. This can be changed by
2315** putting an appropriate #define in the %include section of the input
2316** grammar.
2317*/
2318#ifndef fts5YYMALLOCARGTYPEu64
2319# define fts5YYMALLOCARGTYPEu64 size_t
2320#endif
2321
2322/* Initialize a new parser that has already been allocated.
2323*/
2324static void sqlite3Fts5ParserInit(void *fts5yypRawParser sqlite3Fts5ParserCTX_PDECL){
2325 fts5yyParser *fts5yypParser = (fts5yyParser*)fts5yypRawParser;
2326 sqlite3Fts5ParserCTX_STORE
2327#ifdef fts5YYTRACKMAXSTACKDEPTH
2328 fts5yypParser->fts5yyhwm = 0;
2329#endif
2330 fts5yypParser->fts5yystack = fts5yypParser->fts5yystk0;
2331 fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH100-1];
2332#ifndef fts5YYNOERRORRECOVERY1
2333 fts5yypParser->fts5yyerrcnt = -1;
2334#endif
2335 fts5yypParser->fts5yytos = fts5yypParser->fts5yystack;
2336 fts5yypParser->fts5yystack[0].stateno = 0;
2337 fts5yypParser->fts5yystack[0].major = 0;
2338}
2339
2340#ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK
2341/*
2342** This function allocates a new parser.
2343** The only argument is a pointer to a function which works like
2344** malloc.
2345**
2346** Inputs:
2347** A pointer to the function used to allocate memory.
2348**
2349** Outputs:
2350** A pointer to a parser. This pointer is used in subsequent calls
2351** to sqlite3Fts5Parser and sqlite3Fts5ParserFree.
2352*/
2353static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(fts5YYMALLOCARGTYPEu64) sqlite3Fts5ParserCTX_PDECL){
2354 fts5yyParser *fts5yypParser;
2355 fts5yypParser = (fts5yyParser*)(*mallocProc)( (fts5YYMALLOCARGTYPEu64)sizeof(fts5yyParser) );
2356 if( fts5yypParser ){
2357 sqlite3Fts5ParserCTX_STORE
2358 sqlite3Fts5ParserInit(fts5yypParser sqlite3Fts5ParserCTX_PARAM);
2359 }
2360 return (void*)fts5yypParser;
2361}
2362#endif /* sqlite3Fts5Parser_ENGINEALWAYSONSTACK */
2363
2364
2365/* The following function deletes the "minor type" or semantic value
2366** associated with a symbol. The symbol can be either a terminal
2367** or nonterminal. "fts5yymajor" is the symbol code, and "fts5yypminor" is
2368** a pointer to the value to be deleted. The code used to do the
2369** deletions is derived from the %destructor and/or %token_destructor
2370** directives of the input grammar.
2371*/
2372static void fts5yy_destructor(
2373 fts5yyParser *fts5yypParser, /* The parser */
2374 fts5YYCODETYPEunsigned char fts5yymajor, /* Type code for object to destroy */
2375 fts5YYMINORTYPE *fts5yypminor /* The object to be destroyed */
2376){
2377 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
2378 sqlite3Fts5ParserCTX_FETCH
2379 switch( fts5yymajor ){
2380 /* Here is inserted the actions which take place when a
2381 ** terminal or non-terminal is destroyed. This can happen
2382 ** when the symbol is popped from the stack during a
2383 ** reduce or during error processing or when a parser is
2384 ** being destroyed before it is finished parsing.
2385 **
2386 ** Note: during a reduce, the only symbols destroyed are those
2387 ** which appear on the RHS of the rule, but which are *not* used
2388 ** inside the C code.
2389 */
2390/********* Begin destructor definitions ***************************************/
2391 case 16: /* input */
2392{
2393#line 83 "fts5parse.y"
2394 (void)pParse;
2395#line 624 "fts5parse.c"
2396}
2397 break;
2398 case 17: /* expr */
2399 case 18: /* cnearset */
2400 case 19: /* exprlist */
2401{
2402#line 89 "fts5parse.y"
2403 sqlite3Fts5ParseNodeFree((fts5yypminor->fts5yy24));
2404#line 633 "fts5parse.c"
2405}
2406 break;
2407 case 20: /* colset */
2408 case 21: /* colsetlist */
2409{
2410#line 93 "fts5parse.y"
2411 sqlite3_freesqlite3_api->free((fts5yypminor->fts5yy11));
2412#line 641 "fts5parse.c"
2413}
2414 break;
2415 case 22: /* nearset */
2416 case 23: /* nearphrases */
2417{
2418#line 148 "fts5parse.y"
2419 sqlite3Fts5ParseNearsetFree((fts5yypminor->fts5yy46));
2420#line 649 "fts5parse.c"
2421}
2422 break;
2423 case 24: /* phrase */
2424{
2425#line 183 "fts5parse.y"
2426 sqlite3Fts5ParsePhraseFree((fts5yypminor->fts5yy53));
2427#line 656 "fts5parse.c"
2428}
2429 break;
2430/********* End destructor definitions *****************************************/
2431 default: break; /* If no destructor action specified: do nothing */
2432 }
2433}
2434
2435/*
2436** Pop the parser's stack once.
2437**
2438** If there is a destructor routine associated with the token which
2439** is popped from the stack, then call it.
2440*/
2441static void fts5yy_pop_parser_stack(fts5yyParser *pParser){
2442 fts5yyStackEntry *fts5yytos;
2443 assert( pParser->fts5yytos!=0 )((void) (0));
2444 assert( pParser->fts5yytos > pParser->fts5yystack )((void) (0));
2445 fts5yytos = pParser->fts5yytos--;
2446#ifndef NDEBUG1
2447 if( fts5yyTraceFILE ){
2448 fprintf(fts5yyTraceFILE,"%sPopping %s\n",
2449 fts5yyTracePrompt,
2450 fts5yyTokenName[fts5yytos->major]);
2451 }
2452#endif
2453 fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor);
2454}
2455
2456/*
2457** Clear all secondary memory allocations from the parser
2458*/
2459static void sqlite3Fts5ParserFinalize(void *p){
2460 fts5yyParser *pParser = (fts5yyParser*)p;
2461
2462 /* In-lined version of calling fts5yy_pop_parser_stack() for each
2463 ** element left in the stack */
2464 fts5yyStackEntry *fts5yytos = pParser->fts5yytos;
2465 while( fts5yytos>pParser->fts5yystack ){
2466#ifndef NDEBUG1
2467 if( fts5yyTraceFILE ){
2468 fprintf(fts5yyTraceFILE,"%sPopping %s\n",
2469 fts5yyTracePrompt,
2470 fts5yyTokenName[fts5yytos->major]);
2471 }
2472#endif
2473 if( fts5yytos->major>=fts5YY_MIN_DSTRCTR16 ){
2474 fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor);
2475 }
2476 fts5yytos--;
2477 }
2478
2479#if fts5YYGROWABLESTACK0
2480 if( pParser->fts5yystack!=pParser->fts5yystk0 ){
2481 fts5YYFREEfree(pParser->fts5yystack, sqlite3Fts5ParserCTX(pParser)0);
2482 }
2483#endif
2484}
2485
2486#ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK
2487/*
2488** Deallocate and destroy a parser. Destructors are called for
2489** all stack elements before shutting the parser down.
2490**
2491** If the fts5YYPARSEFREENEVERNULL macro exists (for example because it
2492** is defined in a %include section of the input grammar) then it is
2493** assumed that the input pointer is never NULL.
2494*/
2495static void sqlite3Fts5ParserFree(
2496 void *p, /* The parser to be deleted */
2497 void (*freeProc)(void*) /* Function used to reclaim memory */
2498){
2499#ifndef fts5YYPARSEFREENEVERNULL
2500 if( p==0 ) return;
2501#endif
2502 sqlite3Fts5ParserFinalize(p);
2503 (*freeProc)(p);
2504}
2505#endif /* sqlite3Fts5Parser_ENGINEALWAYSONSTACK */
2506
2507/*
2508** Return the peak depth of the stack for a parser.
2509*/
2510#ifdef fts5YYTRACKMAXSTACKDEPTH
2511static int sqlite3Fts5ParserStackPeak(void *p){
2512 fts5yyParser *pParser = (fts5yyParser*)p;
2513 return pParser->fts5yyhwm;
2514}
2515#endif
2516
2517/* This array of booleans keeps track of the parser statement
2518** coverage. The element fts5yycoverage[X][Y] is set when the parser
2519** is in state X and has a lookahead token Y. In a well-tested
2520** systems, every element of this matrix should end up being set.
2521*/
2522#if defined(fts5YYCOVERAGE)
2523static unsigned char fts5yycoverage[fts5YYNSTATE35][fts5YYNFTS5TOKEN16];
2524#endif
2525
2526/*
2527** Write into out a description of every state/lookahead combination that
2528**
2529** (1) has not been used by the parser, and
2530** (2) is not a syntax error.
2531**
2532** Return the number of missed state/lookahead combinations.
2533*/
2534#if defined(fts5YYCOVERAGE)
2535static int sqlite3Fts5ParserCoverage(FILE *out){
2536 int stateno, iLookAhead, i;
2537 int nMissed = 0;
2538 for(stateno=0; stateno<fts5YYNSTATE35; stateno++){
2539 i = fts5yy_shift_ofst[stateno];
2540 for(iLookAhead=0; iLookAhead<fts5YYNFTS5TOKEN16; iLookAhead++){
2541 if( fts5yy_lookahead[i+iLookAhead]!=iLookAhead ) continue;
2542 if( fts5yycoverage[stateno][iLookAhead]==0 ) nMissed++;
2543 if( out ){
2544 fprintf(out,"State %d lookahead %s %s\n", stateno,
2545 fts5yyTokenName[iLookAhead],
2546 fts5yycoverage[stateno][iLookAhead] ? "ok" : "missed");
2547 }
2548 }
2549 }
2550 return nMissed;
2551}
2552#endif
2553
2554/*
2555** Find the appropriate action for a parser given the terminal
2556** look-ahead token iLookAhead.
2557*/
2558static fts5YYACTIONTYPEunsigned char fts5yy_find_shift_action(
2559 fts5YYCODETYPEunsigned char iLookAhead, /* The look-ahead token */
2560 fts5YYACTIONTYPEunsigned char stateno /* Current state number */
2561){
2562 int i;
2563
2564 if( stateno>fts5YY_MAX_SHIFT34 ) return stateno;
2565 assert( stateno <= fts5YY_SHIFT_COUNT )((void) (0));
2566#if defined(fts5YYCOVERAGE)
2567 fts5yycoverage[stateno][iLookAhead] = 1;
2568#endif
2569 do{
2570 i = fts5yy_shift_ofst[stateno];
2571 assert( i>=0 )((void) (0));
2572 assert( i<=fts5YY_ACTTAB_COUNT )((void) (0));
2573 assert( i+fts5YYNFTS5TOKEN<=(int)fts5YY_NLOOKAHEAD )((void) (0));
2574 assert( iLookAhead!=fts5YYNOCODE )((void) (0));
2575 assert( iLookAhead < fts5YYNFTS5TOKEN )((void) (0));
2576 i += iLookAhead;
2577 assert( i<(int)fts5YY_NLOOKAHEAD )((void) (0));
2578 if( fts5yy_lookahead[i]!=iLookAhead ){
2579#ifdef fts5YYFALLBACK
2580 fts5YYCODETYPEunsigned char iFallback; /* Fallback token */
2581 assert( iLookAhead<sizeof(fts5yyFallback)/sizeof(fts5yyFallback[0]) )((void) (0));
2582 iFallback = fts5yyFallback[iLookAhead];
2583 if( iFallback!=0 ){
2584#ifndef NDEBUG1
2585 if( fts5yyTraceFILE ){
2586 fprintf(fts5yyTraceFILE, "%sFALLBACK %s => %s\n",
2587 fts5yyTracePrompt, fts5yyTokenName[iLookAhead], fts5yyTokenName[iFallback]);
2588 }
2589#endif
2590 assert( fts5yyFallback[iFallback]==0 )((void) (0)); /* Fallback loop must terminate */
2591 iLookAhead = iFallback;
2592 continue;
2593 }
2594#endif
2595#ifdef fts5YYWILDCARD
2596 {
2597 int j = i - iLookAhead + fts5YYWILDCARD;
2598 assert( j<(int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0])) )((void) (0));
2599 if( fts5yy_lookahead[j]==fts5YYWILDCARD && iLookAhead>0 ){
2600#ifndef NDEBUG1
2601 if( fts5yyTraceFILE ){
2602 fprintf(fts5yyTraceFILE, "%sWILDCARD %s => %s\n",
2603 fts5yyTracePrompt, fts5yyTokenName[iLookAhead],
2604 fts5yyTokenName[fts5YYWILDCARD]);
2605 }
2606#endif /* NDEBUG */
2607 return fts5yy_action[j];
2608 }
2609 }
2610#endif /* fts5YYWILDCARD */
2611 return fts5yy_default[stateno];
2612 }else{
2613 assert( i>=0 && i<(int)(sizeof(fts5yy_action)/sizeof(fts5yy_action[0])) )((void) (0));
2614 return fts5yy_action[i];
2615 }
2616 }while(1);
2617}
2618
2619/*
2620** Find the appropriate action for a parser given the non-terminal
2621** look-ahead token iLookAhead.
2622*/
2623static fts5YYACTIONTYPEunsigned char fts5yy_find_reduce_action(
2624 fts5YYACTIONTYPEunsigned char stateno, /* Current state number */
2625 fts5YYCODETYPEunsigned char iLookAhead /* The look-ahead token */
2626){
2627 int i;
2628#ifdef fts5YYERRORSYMBOL
2629 if( stateno>fts5YY_REDUCE_COUNT(17) ){
2630 return fts5yy_default[stateno];
2631 }
2632#else
2633 assert( stateno<=fts5YY_REDUCE_COUNT )((void) (0));
2634#endif
2635 i = fts5yy_reduce_ofst[stateno];
2636 assert( iLookAhead!=fts5YYNOCODE )((void) (0));
2637 i += iLookAhead;
2638#ifdef fts5YYERRORSYMBOL
2639 if( i<0 || i>=fts5YY_ACTTAB_COUNT(105) || fts5yy_lookahead[i]!=iLookAhead ){
2640 return fts5yy_default[stateno];
2641 }
2642#else
2643 assert( i>=0 && i<fts5YY_ACTTAB_COUNT )((void) (0));
2644 assert( fts5yy_lookahead[i]==iLookAhead )((void) (0));
2645#endif
2646 return fts5yy_action[i];
2647}
2648
2649/*
2650** The following routine is called if the stack overflows.
2651*/
2652static void fts5yyStackOverflow(fts5yyParser *fts5yypParser){
2653 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
2654 sqlite3Fts5ParserCTX_FETCH
2655#ifndef NDEBUG1
2656 if( fts5yyTraceFILE ){
2657 fprintf(fts5yyTraceFILE,"%sStack Overflow!\n",fts5yyTracePrompt);
2658 }
2659#endif
2660 while( fts5yypParser->fts5yytos>fts5yypParser->fts5yystack ) fts5yy_pop_parser_stack(fts5yypParser);
2661 /* Here code is inserted which will execute if the parser
2662 ** stack every overflows */
2663/******** Begin %stack_overflow code ******************************************/
2664#line 36 "fts5parse.y"
2665
2666 sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow");
2667#line 896 "fts5parse.c"
2668/******** End %stack_overflow code ********************************************/
2669 sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse; /* Suppress warning about unused %extra_argument var */
2670 sqlite3Fts5ParserCTX_STORE
2671}
2672
2673/*
2674** Print tracing information for a SHIFT action
2675*/
2676#ifndef NDEBUG1
2677static void fts5yyTraceShift(fts5yyParser *fts5yypParser, int fts5yyNewState, const char *zTag){
2678 if( fts5yyTraceFILE ){
2679 if( fts5yyNewState<fts5YYNSTATE35 ){
2680 fprintf(fts5yyTraceFILE,"%s%s '%s', go to state %d\n",
2681 fts5yyTracePrompt, zTag, fts5yyTokenName[fts5yypParser->fts5yytos->major],
2682 fts5yyNewState);
2683 }else{
2684 fprintf(fts5yyTraceFILE,"%s%s '%s', pending reduce %d\n",
2685 fts5yyTracePrompt, zTag, fts5yyTokenName[fts5yypParser->fts5yytos->major],
2686 fts5yyNewState - fts5YY_MIN_REDUCE83);
2687 }
2688 }
2689}
2690#else
2691# define fts5yyTraceShift(X,Y,Z)
2692#endif
2693
2694/*
2695** Perform a shift action.
2696*/
2697static void fts5yy_shift(
2698 fts5yyParser *fts5yypParser, /* The parser to be shifted */
2699 fts5YYACTIONTYPEunsigned char fts5yyNewState, /* The new state to shift in */
2700 fts5YYCODETYPEunsigned char fts5yyMajor, /* The major token to shift in */
2701 sqlite3Fts5ParserFTS5TOKENTYPEFts5Token fts5yyMinor /* The minor token to shift in */
2702){
2703 fts5yyStackEntry *fts5yytos;
2704 fts5yypParser->fts5yytos++;
2705#ifdef fts5YYTRACKMAXSTACKDEPTH
2706 if( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)>fts5yypParser->fts5yyhwm ){
2707 fts5yypParser->fts5yyhwm++;
2708 assert( fts5yypParser->fts5yyhwm == (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack) )((void) (0));
2709 }
2710#endif
2711 fts5yytos = fts5yypParser->fts5yytos;
2712 if( fts5yytos>fts5yypParser->fts5yystackEnd ){
2713 if( fts5yyGrowStack(fts5yypParser)1 ){
2714 fts5yypParser->fts5yytos--;
2715 fts5yyStackOverflow(fts5yypParser);
2716 return;
2717 }
2718 fts5yytos = fts5yypParser->fts5yytos;
2719 assert( fts5yytos <= fts5yypParser->fts5yystackEnd )((void) (0));
2720 }
2721 if( fts5yyNewState > fts5YY_MAX_SHIFT34 ){
2722 fts5yyNewState += fts5YY_MIN_REDUCE83 - fts5YY_MIN_SHIFTREDUCE52;
2723 }
2724 fts5yytos->stateno = fts5yyNewState;
2725 fts5yytos->major = fts5yyMajor;
2726 fts5yytos->minor.fts5yy0 = fts5yyMinor;
2727 fts5yyTraceShift(fts5yypParser, fts5yyNewState, "Shift");
2728}
2729
2730/* For rule J, fts5yyRuleInfoLhs[J] contains the symbol on the left-hand side
2731** of that rule */
2732static const fts5YYCODETYPEunsigned char fts5yyRuleInfoLhs[] = {
2733 16, /* (0) input ::= expr */
2734 20, /* (1) colset ::= MINUS LCP colsetlist RCP */
2735 20, /* (2) colset ::= LCP colsetlist RCP */
2736 20, /* (3) colset ::= STRING */
2737 20, /* (4) colset ::= MINUS STRING */
2738 21, /* (5) colsetlist ::= colsetlist STRING */
2739 21, /* (6) colsetlist ::= STRING */
2740 17, /* (7) expr ::= expr AND expr */
2741 17, /* (8) expr ::= expr OR expr */
2742 17, /* (9) expr ::= expr NOT expr */
2743 17, /* (10) expr ::= colset COLON LP expr RP */
2744 17, /* (11) expr ::= LP expr RP */
2745 17, /* (12) expr ::= exprlist */
2746 19, /* (13) exprlist ::= cnearset */
2747 19, /* (14) exprlist ::= exprlist cnearset */
2748 18, /* (15) cnearset ::= nearset */
2749 18, /* (16) cnearset ::= colset COLON nearset */
2750 22, /* (17) nearset ::= phrase */
2751 22, /* (18) nearset ::= CARET phrase */
2752 22, /* (19) nearset ::= STRING LP nearphrases neardist_opt RP */
2753 23, /* (20) nearphrases ::= phrase */
2754 23, /* (21) nearphrases ::= nearphrases phrase */
2755 25, /* (22) neardist_opt ::= */
2756 25, /* (23) neardist_opt ::= COMMA STRING */
2757 24, /* (24) phrase ::= phrase PLUS STRING star_opt */
2758 24, /* (25) phrase ::= STRING star_opt */
2759 26, /* (26) star_opt ::= STAR */
2760 26, /* (27) star_opt ::= */
2761};
2762
2763/* For rule J, fts5yyRuleInfoNRhs[J] contains the negative of the number
2764** of symbols on the right-hand side of that rule. */
2765static const signed char fts5yyRuleInfoNRhs[] = {
2766 -1, /* (0) input ::= expr */
2767 -4, /* (1) colset ::= MINUS LCP colsetlist RCP */
2768 -3, /* (2) colset ::= LCP colsetlist RCP */
2769 -1, /* (3) colset ::= STRING */
2770 -2, /* (4) colset ::= MINUS STRING */
2771 -2, /* (5) colsetlist ::= colsetlist STRING */
2772 -1, /* (6) colsetlist ::= STRING */
2773 -3, /* (7) expr ::= expr AND expr */
2774 -3, /* (8) expr ::= expr OR expr */
2775 -3, /* (9) expr ::= expr NOT expr */
2776 -5, /* (10) expr ::= colset COLON LP expr RP */
2777 -3, /* (11) expr ::= LP expr RP */
2778 -1, /* (12) expr ::= exprlist */
2779 -1, /* (13) exprlist ::= cnearset */
2780 -2, /* (14) exprlist ::= exprlist cnearset */
2781 -1, /* (15) cnearset ::= nearset */
2782 -3, /* (16) cnearset ::= colset COLON nearset */
2783 -1, /* (17) nearset ::= phrase */
2784 -2, /* (18) nearset ::= CARET phrase */
2785 -5, /* (19) nearset ::= STRING LP nearphrases neardist_opt RP */
2786 -1, /* (20) nearphrases ::= phrase */
2787 -2, /* (21) nearphrases ::= nearphrases phrase */
2788 0, /* (22) neardist_opt ::= */
2789 -2, /* (23) neardist_opt ::= COMMA STRING */
2790 -4, /* (24) phrase ::= phrase PLUS STRING star_opt */
2791 -2, /* (25) phrase ::= STRING star_opt */
2792 -1, /* (26) star_opt ::= STAR */
2793 0, /* (27) star_opt ::= */
2794};
2795
2796static void fts5yy_accept(fts5yyParser*); /* Forward Declaration */
2797
2798/*
2799** Perform a reduce action and the shift that must immediately
2800** follow the reduce.
2801**
2802** The fts5yyLookahead and fts5yyLookaheadToken parameters provide reduce actions
2803** access to the lookahead token (if any). The fts5yyLookahead will be fts5YYNOCODE
2804** if the lookahead token has already been consumed. As this procedure is
2805** only called from one place, optimizing compilers will in-line it, which
2806** means that the extra parameters have no performance impact.
2807*/
2808static fts5YYACTIONTYPEunsigned char fts5yy_reduce(
2809 fts5yyParser *fts5yypParser, /* The parser */
2810 unsigned int fts5yyruleno, /* Number of the rule by which to reduce */
2811 int fts5yyLookahead, /* Lookahead token, or fts5YYNOCODE if none */
2812 sqlite3Fts5ParserFTS5TOKENTYPEFts5Token fts5yyLookaheadToken /* Value of the lookahead token */
2813 sqlite3Fts5ParserCTX_PDECL /* %extra_context */
2814){
2815 int fts5yygoto; /* The next state */
2816 fts5YYACTIONTYPEunsigned char fts5yyact; /* The next action */
2817 fts5yyStackEntry *fts5yymsp; /* The top of the parser's stack */
2818 int fts5yysize; /* Amount to pop the stack */
2819 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
2820 (void)fts5yyLookahead;
2821 (void)fts5yyLookaheadToken;
2822 fts5yymsp = fts5yypParser->fts5yytos;
2823
2824 switch( fts5yyruleno ){
2825 /* Beginning here are the reduction cases. A typical example
2826 ** follows:
2827 ** case 0:
2828 ** #line <lineno> <grammarfile>
2829 ** { ... } // User supplied code
2830 ** #line <lineno> <thisfile>
2831 ** break;
2832 */
2833/********** Begin reduce actions **********************************************/
2834 fts5YYMINORTYPE fts5yylhsminor;
2835 case 0: /* input ::= expr */
2836#line 82 "fts5parse.y"
2837{ sqlite3Fts5ParseFinished(pParse, fts5yymsp[0].minor.fts5yy24); }
2838#line 1067 "fts5parse.c"
2839 break;
2840 case 1: /* colset ::= MINUS LCP colsetlist RCP */
2841#line 97 "fts5parse.y"
2842{
2843 fts5yymsp[-3].minor.fts5yy11 = sqlite3Fts5ParseColsetInvert(pParse, fts5yymsp[-1].minor.fts5yy11);
2844}
2845#line 1074 "fts5parse.c"
2846 break;
2847 case 2: /* colset ::= LCP colsetlist RCP */
2848#line 100 "fts5parse.y"
2849{ fts5yymsp[-2].minor.fts5yy11 = fts5yymsp[-1].minor.fts5yy11; }
2850#line 1079 "fts5parse.c"
2851 break;
2852 case 3: /* colset ::= STRING */
2853#line 101 "fts5parse.y"
2854{
2855 fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0);
2856}
2857#line 1086 "fts5parse.c"
2858 fts5yymsp[0].minor.fts5yy11 = fts5yylhsminor.fts5yy11;
2859 break;
2860 case 4: /* colset ::= MINUS STRING */
2861#line 104 "fts5parse.y"
2862{
2863 fts5yymsp[-1].minor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0);
2864 fts5yymsp[-1].minor.fts5yy11 = sqlite3Fts5ParseColsetInvert(pParse, fts5yymsp[-1].minor.fts5yy11);
2865}
2866#line 1095 "fts5parse.c"
2867 break;
2868 case 5: /* colsetlist ::= colsetlist STRING */
2869#line 109 "fts5parse.y"
2870{
2871 fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, fts5yymsp[-1].minor.fts5yy11, &fts5yymsp[0].minor.fts5yy0); }
2872#line 1101 "fts5parse.c"
2873 fts5yymsp[-1].minor.fts5yy11 = fts5yylhsminor.fts5yy11;
2874 break;
2875 case 6: /* colsetlist ::= STRING */
2876#line 111 "fts5parse.y"
2877{
2878 fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0);
2879}
2880#line 1109 "fts5parse.c"
2881 fts5yymsp[0].minor.fts5yy11 = fts5yylhsminor.fts5yy11;
2882 break;
2883 case 7: /* expr ::= expr AND expr */
2884#line 115 "fts5parse.y"
2885{
2886 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_AND2, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0);
2887}
2888#line 1117 "fts5parse.c"
2889 fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2890 break;
2891 case 8: /* expr ::= expr OR expr */
2892#line 118 "fts5parse.y"
2893{
2894 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_OR1, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0);
2895}
2896#line 1125 "fts5parse.c"
2897 fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2898 break;
2899 case 9: /* expr ::= expr NOT expr */
2900#line 121 "fts5parse.y"
2901{
2902 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_NOT3, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0);
2903}
2904#line 1133 "fts5parse.c"
2905 fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2906 break;
2907 case 10: /* expr ::= colset COLON LP expr RP */
2908#line 125 "fts5parse.y"
2909{
2910 sqlite3Fts5ParseSetColset(pParse, fts5yymsp[-1].minor.fts5yy24, fts5yymsp[-4].minor.fts5yy11);
2911 fts5yylhsminor.fts5yy24 = fts5yymsp[-1].minor.fts5yy24;
2912}
2913#line 1142 "fts5parse.c"
2914 fts5yymsp[-4].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2915 break;
2916 case 11: /* expr ::= LP expr RP */
2917#line 129 "fts5parse.y"
2918{fts5yymsp[-2].minor.fts5yy24 = fts5yymsp[-1].minor.fts5yy24;}
2919#line 1148 "fts5parse.c"
2920 break;
2921 case 12: /* expr ::= exprlist */
2922 case 13: /* exprlist ::= cnearset */ fts5yytestcase(fts5yyruleno==13);
2923#line 130 "fts5parse.y"
2924{fts5yylhsminor.fts5yy24 = fts5yymsp[0].minor.fts5yy24;}
2925#line 1154 "fts5parse.c"
2926 fts5yymsp[0].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2927 break;
2928 case 14: /* exprlist ::= exprlist cnearset */
2929#line 133 "fts5parse.y"
2930{
2931 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseImplicitAnd(pParse, fts5yymsp[-1].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24);
2932}
2933#line 1162 "fts5parse.c"
2934 fts5yymsp[-1].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2935 break;
2936 case 15: /* cnearset ::= nearset */
2937#line 137 "fts5parse.y"
2938{
2939 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_STRING9, 0, 0, fts5yymsp[0].minor.fts5yy46);
2940}
2941#line 1170 "fts5parse.c"
2942 fts5yymsp[0].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2943 break;
2944 case 16: /* cnearset ::= colset COLON nearset */
2945#line 140 "fts5parse.y"
2946{
2947 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_STRING9, 0, 0, fts5yymsp[0].minor.fts5yy46);
2948 sqlite3Fts5ParseSetColset(pParse, fts5yylhsminor.fts5yy24, fts5yymsp[-2].minor.fts5yy11);
2949}
2950#line 1179 "fts5parse.c"
2951 fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2952 break;
2953 case 17: /* nearset ::= phrase */
2954#line 151 "fts5parse.y"
2955{ fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53); }
2956#line 1185 "fts5parse.c"
2957 fts5yymsp[0].minor.fts5yy46 = fts5yylhsminor.fts5yy46;
2958 break;
2959 case 18: /* nearset ::= CARET phrase */
2960#line 152 "fts5parse.y"
2961{
2962 sqlite3Fts5ParseSetCaret(fts5yymsp[0].minor.fts5yy53);
2963 fts5yymsp[-1].minor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53);
2964}
2965#line 1194 "fts5parse.c"
2966 break;
2967 case 19: /* nearset ::= STRING LP nearphrases neardist_opt RP */
2968#line 156 "fts5parse.y"
2969{
2970 sqlite3Fts5ParseNear(pParse, &fts5yymsp[-4].minor.fts5yy0);
2971 sqlite3Fts5ParseSetDistance(pParse, fts5yymsp[-2].minor.fts5yy46, &fts5yymsp[-1].minor.fts5yy0);
2972 fts5yylhsminor.fts5yy46 = fts5yymsp[-2].minor.fts5yy46;
2973}
2974#line 1203 "fts5parse.c"
2975 fts5yymsp[-4].minor.fts5yy46 = fts5yylhsminor.fts5yy46;
2976 break;
2977 case 20: /* nearphrases ::= phrase */
2978#line 162 "fts5parse.y"
2979{
2980 fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53);
2981}
2982#line 1211 "fts5parse.c"
2983 fts5yymsp[0].minor.fts5yy46 = fts5yylhsminor.fts5yy46;
2984 break;
2985 case 21: /* nearphrases ::= nearphrases phrase */
2986#line 165 "fts5parse.y"
2987{
2988 fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, fts5yymsp[-1].minor.fts5yy46, fts5yymsp[0].minor.fts5yy53);
2989}
2990#line 1219 "fts5parse.c"
2991 fts5yymsp[-1].minor.fts5yy46 = fts5yylhsminor.fts5yy46;
2992 break;
2993 case 22: /* neardist_opt ::= */
2994#line 172 "fts5parse.y"
2995{ fts5yymsp[1].minor.fts5yy0.p = 0; fts5yymsp[1].minor.fts5yy0.n = 0; }
2996#line 1225 "fts5parse.c"
2997 break;
2998 case 23: /* neardist_opt ::= COMMA STRING */
2999#line 173 "fts5parse.y"
3000{ fts5yymsp[-1].minor.fts5yy0 = fts5yymsp[0].minor.fts5yy0; }
3001#line 1230 "fts5parse.c"
3002 break;
3003 case 24: /* phrase ::= phrase PLUS STRING star_opt */
3004#line 185 "fts5parse.y"
3005{
3006 fts5yylhsminor.fts5yy53 = sqlite3Fts5ParseTerm(pParse, fts5yymsp[-3].minor.fts5yy53, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy4);
3007}
3008#line 1237 "fts5parse.c"
3009 fts5yymsp[-3].minor.fts5yy53 = fts5yylhsminor.fts5yy53;
3010 break;
3011 case 25: /* phrase ::= STRING star_opt */
3012#line 188 "fts5parse.y"
3013{
3014 fts5yylhsminor.fts5yy53 = sqlite3Fts5ParseTerm(pParse, 0, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy4);
3015}
3016#line 1245 "fts5parse.c"
3017 fts5yymsp[-1].minor.fts5yy53 = fts5yylhsminor.fts5yy53;
3018 break;
3019 case 26: /* star_opt ::= STAR */
3020#line 196 "fts5parse.y"
3021{ fts5yymsp[0].minor.fts5yy4 = 1; }
3022#line 1251 "fts5parse.c"
3023 break;
3024 case 27: /* star_opt ::= */
3025#line 197 "fts5parse.y"
3026{ fts5yymsp[1].minor.fts5yy4 = 0; }
3027#line 1256 "fts5parse.c"
3028 break;
3029 default:
3030 break;
3031/********** End reduce actions ************************************************/
3032 };
3033 assert( fts5yyruleno<sizeof(fts5yyRuleInfoLhs)/sizeof(fts5yyRuleInfoLhs[0]) )((void) (0));
3034 fts5yygoto = fts5yyRuleInfoLhs[fts5yyruleno];
3035 fts5yysize = fts5yyRuleInfoNRhs[fts5yyruleno];
3036 fts5yyact = fts5yy_find_reduce_action(fts5yymsp[fts5yysize].stateno,(fts5YYCODETYPEunsigned char)fts5yygoto);
3037
3038 /* There are no SHIFTREDUCE actions on nonterminals because the table
3039 ** generator has simplified them to pure REDUCE actions. */
3040 assert( !(fts5yyact>fts5YY_MAX_SHIFT && fts5yyact<=fts5YY_MAX_SHIFTREDUCE) )((void) (0));
3041
3042 /* It is not possible for a REDUCE to be followed by an error */
3043 assert( fts5yyact!=fts5YY_ERROR_ACTION )((void) (0));
3044
3045 fts5yymsp += fts5yysize+1;
3046 fts5yypParser->fts5yytos = fts5yymsp;
3047 fts5yymsp->stateno = (fts5YYACTIONTYPEunsigned char)fts5yyact;
3048 fts5yymsp->major = (fts5YYCODETYPEunsigned char)fts5yygoto;
3049 fts5yyTraceShift(fts5yypParser, fts5yyact, "... then shift");
3050 return fts5yyact;
3051}
3052
3053/*
3054** The following code executes when the parse fails
3055*/
3056#ifndef fts5YYNOERRORRECOVERY1
3057static void fts5yy_parse_failed(
3058 fts5yyParser *fts5yypParser /* The parser */
3059){
3060 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
3061 sqlite3Fts5ParserCTX_FETCH
3062#ifndef NDEBUG1
3063 if( fts5yyTraceFILE ){
3064 fprintf(fts5yyTraceFILE,"%sFail!\n",fts5yyTracePrompt);
3065 }
3066#endif
3067 while( fts5yypParser->fts5yytos>fts5yypParser->fts5yystack ) fts5yy_pop_parser_stack(fts5yypParser);
3068 /* Here code is inserted which will be executed whenever the
3069 ** parser fails */
3070/************ Begin %parse_failure code ***************************************/
3071/************ End %parse_failure code *****************************************/
3072 sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse; /* Suppress warning about unused %extra_argument variable */
3073 sqlite3Fts5ParserCTX_STORE
3074}
3075#endif /* fts5YYNOERRORRECOVERY */
3076
3077/*
3078** The following code executes when a syntax error first occurs.
3079*/
3080static void fts5yy_syntax_error(
3081 fts5yyParser *fts5yypParser, /* The parser */
3082 int fts5yymajor, /* The major type of the error token */
3083 sqlite3Fts5ParserFTS5TOKENTYPEFts5Token fts5yyminor /* The minor type of the error token */
3084){
3085 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
3086 sqlite3Fts5ParserCTX_FETCH
3087#define FTS5TOKENfts5yyminor fts5yyminor
3088/************ Begin %syntax_error code ****************************************/
3089#line 30 "fts5parse.y"
3090
3091 UNUSED_PARAM(fts5yymajor)(void)(fts5yymajor); /* Silence a compiler warning */
3092 sqlite3Fts5ParseError(
3093 pParse, "fts5: syntax error near \"%.*s\"",FTS5TOKENfts5yyminor.n,FTS5TOKENfts5yyminor.p
3094 );
3095#line 1324 "fts5parse.c"
3096/************ End %syntax_error code ******************************************/
3097 sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse; /* Suppress warning about unused %extra_argument variable */
3098 sqlite3Fts5ParserCTX_STORE
3099}
3100
3101/*
3102** The following is executed when the parser accepts
3103*/
3104static void fts5yy_accept(
3105 fts5yyParser *fts5yypParser /* The parser */
3106){
3107 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
3108 sqlite3Fts5ParserCTX_FETCH
3109#ifndef NDEBUG1
3110 if( fts5yyTraceFILE ){
3111 fprintf(fts5yyTraceFILE,"%sAccept!\n",fts5yyTracePrompt);
3112 }
3113#endif
3114#ifndef fts5YYNOERRORRECOVERY1
3115 fts5yypParser->fts5yyerrcnt = -1;
3116#endif
3117 assert( fts5yypParser->fts5yytos==fts5yypParser->fts5yystack )((void) (0));
3118 /* Here code is inserted which will be executed whenever the
3119 ** parser accepts */
3120/*********** Begin %parse_accept code *****************************************/
3121/*********** End %parse_accept code *******************************************/
3122 sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse; /* Suppress warning about unused %extra_argument variable */
3123 sqlite3Fts5ParserCTX_STORE
3124}
3125
3126/* The main parser program.
3127** The first argument is a pointer to a structure obtained from
3128** "sqlite3Fts5ParserAlloc" which describes the current state of the parser.
3129** The second argument is the major token number. The third is
3130** the minor token. The fourth optional argument is whatever the
3131** user wants (and specified in the grammar) and is available for
3132** use by the action routines.
3133**
3134** Inputs:
3135** <ul>
3136** <li> A pointer to the parser (an opaque structure.)
3137** <li> The major token number.
3138** <li> The minor token number.
3139** <li> An option argument of a grammar-specified type.
3140** </ul>
3141**
3142** Outputs:
3143** None.
3144*/
3145static void sqlite3Fts5Parser(
3146 void *fts5yyp, /* The parser */
3147 int fts5yymajor, /* The major token code number */
3148 sqlite3Fts5ParserFTS5TOKENTYPEFts5Token fts5yyminor /* The value for the token */
3149 sqlite3Fts5ParserARG_PDECL,Fts5Parse *pParse /* Optional %extra_argument parameter */
3150){
3151 fts5YYMINORTYPE fts5yyminorunion;
3152 fts5YYACTIONTYPEunsigned char fts5yyact; /* The parser action. */
3153#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY1)
3154 int fts5yyendofinput; /* True if we are at the end of input */
3155#endif
3156#ifdef fts5YYERRORSYMBOL
3157 int fts5yyerrorhit = 0; /* True if fts5yymajor has invoked an error */
3158#endif
3159 fts5yyParser *fts5yypParser = (fts5yyParser*)fts5yyp; /* The parser */
3160 sqlite3Fts5ParserCTX_FETCH
3161 sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse;
3162
3163 assert( fts5yypParser->fts5yytos!=0 )((void) (0));
3164#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY1)
3165 fts5yyendofinput = (fts5yymajor==0);
3166#endif
3167
3168 fts5yyact = fts5yypParser->fts5yytos->stateno;
3169#ifndef NDEBUG1
3170 if( fts5yyTraceFILE ){
3171 if( fts5yyact < fts5YY_MIN_REDUCE83 ){
3172 fprintf(fts5yyTraceFILE,"%sInput '%s' in state %d\n",
3173 fts5yyTracePrompt,fts5yyTokenName[fts5yymajor],fts5yyact);
3174 }else{
3175 fprintf(fts5yyTraceFILE,"%sInput '%s' with pending reduce %d\n",
3176 fts5yyTracePrompt,fts5yyTokenName[fts5yymajor],fts5yyact-fts5YY_MIN_REDUCE83);
3177 }
3178 }
3179#endif
3180
3181 while(1){ /* Exit by "break" */
3182 assert( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystack )((void) (0));
3183 assert( fts5yyact==fts5yypParser->fts5yytos->stateno )((void) (0));
3184 fts5yyact = fts5yy_find_shift_action((fts5YYCODETYPEunsigned char)fts5yymajor,fts5yyact);
3185 if( fts5yyact >= fts5YY_MIN_REDUCE83 ){
3186 unsigned int fts5yyruleno = fts5yyact - fts5YY_MIN_REDUCE83; /* Reduce by this rule */
3187#ifndef NDEBUG1
3188 assert( fts5yyruleno<(int)(sizeof(fts5yyRuleName)/sizeof(fts5yyRuleName[0])) )((void) (0));
3189 if( fts5yyTraceFILE ){
3190 int fts5yysize = fts5yyRuleInfoNRhs[fts5yyruleno];
3191 if( fts5yysize ){
3192 fprintf(fts5yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n",
3193 fts5yyTracePrompt,
3194 fts5yyruleno, fts5yyRuleName[fts5yyruleno],
3195 fts5yyruleno<fts5YYNRULE_WITH_ACTION28 ? "" : " without external action",
3196 fts5yypParser->fts5yytos[fts5yysize].stateno);
3197 }else{
3198 fprintf(fts5yyTraceFILE, "%sReduce %d [%s]%s.\n",
3199 fts5yyTracePrompt, fts5yyruleno, fts5yyRuleName[fts5yyruleno],
3200 fts5yyruleno<fts5YYNRULE_WITH_ACTION28 ? "" : " without external action");
3201 }
3202 }
3203#endif /* NDEBUG */
3204
3205 /* Check that the stack is large enough to grow by a single entry
3206 ** if the RHS of the rule is empty. This ensures that there is room
3207 ** enough on the stack to push the LHS value */
3208 if( fts5yyRuleInfoNRhs[fts5yyruleno]==0 ){
3209#ifdef fts5YYTRACKMAXSTACKDEPTH
3210 if( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)>fts5yypParser->fts5yyhwm ){
3211 fts5yypParser->fts5yyhwm++;
3212 assert( fts5yypParser->fts5yyhwm ==((void) (0))
3213 (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack))((void) (0));
3214 }
3215#endif
3216 if( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystackEnd ){
3217 if( fts5yyGrowStack(fts5yypParser)1 ){
3218 fts5yyStackOverflow(fts5yypParser);
3219 break;
3220 }
3221 }
3222 }
3223 fts5yyact = fts5yy_reduce(fts5yypParser,fts5yyruleno,fts5yymajor,fts5yyminor sqlite3Fts5ParserCTX_PARAM);
3224 }else if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE79 ){
3225 fts5yy_shift(fts5yypParser,fts5yyact,(fts5YYCODETYPEunsigned char)fts5yymajor,fts5yyminor);
3226#ifndef fts5YYNOERRORRECOVERY1
3227 fts5yypParser->fts5yyerrcnt--;
3228#endif
3229 break;
3230 }else if( fts5yyact==fts5YY_ACCEPT_ACTION81 ){
3231 fts5yypParser->fts5yytos--;
3232 fts5yy_accept(fts5yypParser);
3233 return;
3234 }else{
3235 assert( fts5yyact == fts5YY_ERROR_ACTION )((void) (0));
3236 fts5yyminorunion.fts5yy0 = fts5yyminor;
3237#ifdef fts5YYERRORSYMBOL
3238 int fts5yymx;
3239#endif
3240#ifndef NDEBUG1
3241 if( fts5yyTraceFILE ){
3242 fprintf(fts5yyTraceFILE,"%sSyntax Error!\n",fts5yyTracePrompt);
3243 }
3244#endif
3245#ifdef fts5YYERRORSYMBOL
3246 /* A syntax error has occurred.
3247 ** The response to an error depends upon whether or not the
3248 ** grammar defines an error token "ERROR".
3249 **
3250 ** This is what we do if the grammar does define ERROR:
3251 **
3252 ** * Call the %syntax_error function.
3253 **
3254 ** * Begin popping the stack until we enter a state where
3255 ** it is legal to shift the error symbol, then shift
3256 ** the error symbol.
3257 **
3258 ** * Set the error count to three.
3259 **
3260 ** * Begin accepting and shifting new tokens. No new error
3261 ** processing will occur until three tokens have been
3262 ** shifted successfully.
3263 **
3264 */
3265 if( fts5yypParser->fts5yyerrcnt<0 ){
3266 fts5yy_syntax_error(fts5yypParser,fts5yymajor,fts5yyminor);
3267 }
3268 fts5yymx = fts5yypParser->fts5yytos->major;
3269 if( fts5yymx==fts5YYERRORSYMBOL || fts5yyerrorhit ){
3270#ifndef NDEBUG1
3271 if( fts5yyTraceFILE ){
3272 fprintf(fts5yyTraceFILE,"%sDiscard input token %s\n",
3273 fts5yyTracePrompt,fts5yyTokenName[fts5yymajor]);
3274 }
3275#endif
3276 fts5yy_destructor(fts5yypParser, (fts5YYCODETYPEunsigned char)fts5yymajor, &fts5yyminorunion);
3277 fts5yymajor = fts5YYNOCODE27;
3278 }else{
3279 while( fts5yypParser->fts5yytos > fts5yypParser->fts5yystack ){
3280 fts5yyact = fts5yy_find_reduce_action(fts5yypParser->fts5yytos->stateno,
3281 fts5YYERRORSYMBOL);
3282 if( fts5yyact<=fts5YY_MAX_SHIFTREDUCE79 ) break;
3283 fts5yy_pop_parser_stack(fts5yypParser);
3284 }
3285 if( fts5yypParser->fts5yytos <= fts5yypParser->fts5yystack || fts5yymajor==0 ){
3286 fts5yy_destructor(fts5yypParser,(fts5YYCODETYPEunsigned char)fts5yymajor,&fts5yyminorunion);
3287 fts5yy_parse_failed(fts5yypParser);
3288#ifndef fts5YYNOERRORRECOVERY1
3289 fts5yypParser->fts5yyerrcnt = -1;
3290#endif
3291 fts5yymajor = fts5YYNOCODE27;
3292 }else if( fts5yymx!=fts5YYERRORSYMBOL ){
3293 fts5yy_shift(fts5yypParser,fts5yyact,fts5YYERRORSYMBOL,fts5yyminor);
3294 }
3295 }
3296 fts5yypParser->fts5yyerrcnt = 3;
3297 fts5yyerrorhit = 1;
3298 if( fts5yymajor==fts5YYNOCODE27 ) break;
3299 fts5yyact = fts5yypParser->fts5yytos->stateno;
3300#elif defined(fts5YYNOERRORRECOVERY1)
3301 /* If the fts5YYNOERRORRECOVERY macro is defined, then do not attempt to
3302 ** do any kind of error recovery. Instead, simply invoke the syntax
3303 ** error routine and continue going as if nothing had happened.
3304 **
3305 ** Applications can set this macro (for example inside %include) if
3306 ** they intend to abandon the parse upon the first syntax error seen.
3307 */
3308 fts5yy_syntax_error(fts5yypParser,fts5yymajor, fts5yyminor);
3309 fts5yy_destructor(fts5yypParser,(fts5YYCODETYPEunsigned char)fts5yymajor,&fts5yyminorunion);
3310 break;
3311#else /* fts5YYERRORSYMBOL is not defined */
3312 /* This is what we do if the grammar does not define ERROR:
3313 **
3314 ** * Report an error message, and throw away the input token.
3315 **
3316 ** * If the input token is $, then fail the parse.
3317 **
3318 ** As before, subsequent error messages are suppressed until
3319 ** three input tokens have been successfully shifted.
3320 */
3321 if( fts5yypParser->fts5yyerrcnt<=0 ){
3322 fts5yy_syntax_error(fts5yypParser,fts5yymajor, fts5yyminor);
3323 }
3324 fts5yypParser->fts5yyerrcnt = 3;
3325 fts5yy_destructor(fts5yypParser,(fts5YYCODETYPEunsigned char)fts5yymajor,&fts5yyminorunion);
3326 if( fts5yyendofinput ){
3327 fts5yy_parse_failed(fts5yypParser);
3328#ifndef fts5YYNOERRORRECOVERY1
3329 fts5yypParser->fts5yyerrcnt = -1;
3330#endif
3331 }
3332 break;
3333#endif
3334 }
3335 }
3336#ifndef NDEBUG1
3337 if( fts5yyTraceFILE ){
3338 fts5yyStackEntry *i;
3339 char cDiv = '[';
3340 fprintf(fts5yyTraceFILE,"%sReturn. Stack=",fts5yyTracePrompt);
3341 for(i=&fts5yypParser->fts5yystack[1]; i<=fts5yypParser->fts5yytos; i++){
3342 fprintf(fts5yyTraceFILE,"%c%s", cDiv, fts5yyTokenName[i->major]);
3343 cDiv = ' ';
3344 }
3345 fprintf(fts5yyTraceFILE,"]\n");
3346 }
3347#endif
3348 return;
3349}
3350
3351/*
3352** Return the fallback token corresponding to canonical token iToken, or
3353** 0 if iToken has no fallback.
3354*/
3355static int sqlite3Fts5ParserFallback(int iToken){
3356#ifdef fts5YYFALLBACK
3357 assert( iToken<(int)(sizeof(fts5yyFallback)/sizeof(fts5yyFallback[0])) )((void) (0));
3358 return fts5yyFallback[iToken];
3359#else
3360 (void)iToken;
3361 return 0;
3362#endif
3363}
3364
3365#line 1 "fts5_aux.c"
3366/*
3367** 2014 May 31
3368**
3369** The author disclaims copyright to this source code. In place of
3370** a legal notice, here is a blessing:
3371**
3372** May you do good and not evil.
3373** May you find forgiveness for yourself and forgive others.
3374** May you share freely, never taking more than you give.
3375**
3376******************************************************************************
3377*/
3378
3379
3380/* #include "fts5Int.h" */
3381#include <math.h> /* amalgamator: keep */
3382
3383/*
3384** Object used to iterate through all "coalesced phrase instances" in
3385** a single column of the current row. If the phrase instances in the
3386** column being considered do not overlap, this object simply iterates
3387** through them. Or, if they do overlap (share one or more tokens in
3388** common), each set of overlapping instances is treated as a single
3389** match. See documentation for the highlight() auxiliary function for
3390** details.
3391**
3392** Usage is:
3393**
3394** for(rc = fts5CInstIterNext(pApi, pFts, iCol, &iter);
3395** (rc==SQLITE_OK && 0==fts5CInstIterEof(&iter);
3396** rc = fts5CInstIterNext(&iter)
3397** ){
3398** printf("instance starts at %d, ends at %d\n", iter.iStart, iter.iEnd);
3399** }
3400**
3401*/
3402typedef struct CInstIter CInstIter;
3403struct CInstIter {
3404 const Fts5ExtensionApi *pApi; /* API offered by current FTS version */
3405 Fts5Context *pFts; /* First arg to pass to pApi functions */
3406 int iCol; /* Column to search */
3407 int iInst; /* Next phrase instance index */
3408 int nInst; /* Total number of phrase instances */
3409
3410 /* Output variables */
3411 int iStart; /* First token in coalesced phrase instance */
3412 int iEnd; /* Last token in coalesced phrase instance */
3413};
3414
3415/*
3416** Advance the iterator to the next coalesced phrase instance. Return
3417** an SQLite error code if an error occurs, or SQLITE_OK otherwise.
3418*/
3419static int fts5CInstIterNext(CInstIter *pIter){
3420 int rc = SQLITE_OK0;
3421 pIter->iStart = -1;
3422 pIter->iEnd = -1;
3423
3424 while( rc==SQLITE_OK0 && pIter->iInst<pIter->nInst ){
3425 int ip; int ic; int io;
3426 rc = pIter->pApi->xInst(pIter->pFts, pIter->iInst, &ip, &ic, &io);
3427 if( rc==SQLITE_OK0 ){
3428 if( ic==pIter->iCol ){
3429 int iEnd = io - 1 + pIter->pApi->xPhraseSize(pIter->pFts, ip);
3430 if( pIter->iStart<0 ){
3431 pIter->iStart = io;
3432 pIter->iEnd = iEnd;
3433 }else if( io<=pIter->iEnd ){
3434 if( iEnd>pIter->iEnd ) pIter->iEnd = iEnd;
3435 }else{
3436 break;
3437 }
3438 }
3439 pIter->iInst++;
3440 }
3441 }
3442
3443 return rc;
3444}
3445
3446/*
3447** Initialize the iterator object indicated by the final parameter to
3448** iterate through coalesced phrase instances in column iCol.
3449*/
3450static int fts5CInstIterInit(
3451 const Fts5ExtensionApi *pApi,
3452 Fts5Context *pFts,
3453 int iCol,
3454 CInstIter *pIter
3455){
3456 int rc;
3457
3458 memset(pIter, 0, sizeof(CInstIter));
3459 pIter->pApi = pApi;
3460 pIter->pFts = pFts;
3461 pIter->iCol = iCol;
3462 rc = pApi->xInstCount(pFts, &pIter->nInst);
3463
3464 if( rc==SQLITE_OK0 ){
3465 rc = fts5CInstIterNext(pIter);
3466 }
3467
3468 return rc;
3469}
3470
3471
3472
3473/*************************************************************************
3474** Start of highlight() implementation.
3475*/
3476typedef struct HighlightContext HighlightContext;
3477struct HighlightContext {
3478 /* Constant parameters to fts5HighlightCb() */
3479 int iRangeStart; /* First token to include */
3480 int iRangeEnd; /* If non-zero, last token to include */
3481 const char *zOpen; /* Opening highlight */
3482 const char *zClose; /* Closing highlight */
3483 const char *zIn; /* Input text */
3484 int nIn; /* Size of input text in bytes */
3485
3486 /* Variables modified by fts5HighlightCb() */
3487 CInstIter iter; /* Coalesced Instance Iterator */
3488 int iPos; /* Current token offset in zIn[] */
3489 int iOff; /* Have copied up to this offset in zIn[] */
3490 int bOpen; /* True if highlight is open */
3491 char *zOut; /* Output value */
3492};
3493
3494/*
3495** Append text to the HighlightContext output string - p->zOut. Argument
3496** z points to a buffer containing n bytes of text to append. If n is
3497** negative, everything up until the first '\0' is appended to the output.
3498**
3499** If *pRc is set to any value other than SQLITE_OK when this function is
3500** called, it is a no-op. If an error (i.e. an OOM condition) is encountered,
3501** *pRc is set to an error code before returning.
3502*/
3503static void fts5HighlightAppend(
3504 int *pRc,
3505 HighlightContext *p,
3506 const char *z, int n
3507){
3508 if( *pRc==SQLITE_OK0 && z ){
3509 if( n<0 ) n = (int)strlen(z);
3510 p->zOut = sqlite3_mprintfsqlite3_api->mprintf("%z%.*s", p->zOut, n, z);
3511 if( p->zOut==0 ) *pRc = SQLITE_NOMEM7;
3512 }
3513}
3514
3515/*
3516** Tokenizer callback used by implementation of highlight() function.
3517*/
3518static int fts5HighlightCb(
3519 void *pContext, /* Pointer to HighlightContext object */
3520 int tflags, /* Mask of FTS5_TOKEN_* flags */
3521 const char *pToken, /* Buffer containing token */
3522 int nToken, /* Size of token in bytes */
3523 int iStartOff, /* Start byte offset of token */
3524 int iEndOff /* End byte offset of token */
3525){
3526 HighlightContext *p = (HighlightContext*)pContext;
3527 int rc = SQLITE_OK0;
3528 int iPos;
3529
3530 UNUSED_PARAM2(pToken, nToken)(void)(pToken), (void)(nToken);
3531
3532 if( tflags & FTS5_TOKEN_COLOCATED0x0001 ) return SQLITE_OK0;
3533 iPos = p->iPos++;
3534
3535 if( p->iRangeEnd>=0 ){
3536 if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK0;
3537 if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
3538 }
3539
3540 /* If the parenthesis is open, and this token is not part of the current
3541 ** phrase, and the starting byte offset of this token is past the point
3542 ** that has currently been copied into the output buffer, close the
3543 ** parenthesis. */
3544 if( p->bOpen
3545 && (iPos<=p->iter.iStart || p->iter.iStart<0)
3546 && iStartOff>p->iOff
3547 ){
3548 fts5HighlightAppend(&rc, p, p->zClose, -1);
3549 p->bOpen = 0;
3550 }
3551
3552 /* If this is the start of a new phrase, and the highlight is not open:
3553 **
3554 ** * copy text from the input up to the start of the phrase, and
3555 ** * open the highlight.
3556 */
3557 if( iPos==p->iter.iStart && p->bOpen==0 ){
3558 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
3559 fts5HighlightAppend(&rc, p, p->zOpen, -1);
3560 p->iOff = iStartOff;
3561 p->bOpen = 1;
3562 }
3563
3564 if( iPos==p->iter.iEnd ){
3565 if( p->bOpen==0 ){
3566 assert( p->iRangeEnd>=0 )((void) (0));
3567 fts5HighlightAppend(&rc, p, p->zOpen, -1);
3568 p->bOpen = 1;
3569 }
3570 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
3571 p->iOff = iEndOff;
3572
3573 if( rc==SQLITE_OK0 ){
3574 rc = fts5CInstIterNext(&p->iter);
3575 }
3576 }
3577
3578 if( iPos==p->iRangeEnd ){
3579 if( p->bOpen ){
3580 if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){
3581 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
3582 p->iOff = iEndOff;
3583 }
3584 fts5HighlightAppend(&rc, p, p->zClose, -1);
3585 p->bOpen = 0;
3586 }
3587 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
3588 p->iOff = iEndOff;
3589 }
3590
3591 return rc;
3592}
3593
3594
3595/*
3596** Implementation of highlight() function.
3597*/
3598static void fts5HighlightFunction(
3599 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
3600 Fts5Context *pFts, /* First arg to pass to pApi functions */
3601 sqlite3_context *pCtx, /* Context for returning result/error */
3602 int nVal, /* Number of values in apVal[] array */
3603 sqlite3_value **apVal /* Array of trailing arguments */
3604){
3605 HighlightContext ctx;
3606 int rc;
3607 int iCol;
3608
3609 if( nVal!=3 ){
3610 const char *zErr = "wrong number of arguments to function highlight()";
3611 sqlite3_result_errorsqlite3_api->result_error(pCtx, zErr, -1);
3612 return;
3613 }
3614
3615 iCol = sqlite3_value_intsqlite3_api->value_int(apVal[0]);
3616 memset(&ctx, 0, sizeof(HighlightContext));
3617 ctx.zOpen = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[1]);
3618 ctx.zClose = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[2]);
3619 ctx.iRangeEnd = -1;
3620 rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);
3621 if( rc==SQLITE_RANGE25 ){
3622 sqlite3_result_textsqlite3_api->result_text(pCtx, "", -1, SQLITE_STATIC((sqlite3_destructor_type)0));
3623 rc = SQLITE_OK0;
3624 }else if( ctx.zIn ){
3625 const char *pLoc = 0; /* Locale of column iCol */
3626 int nLoc = 0; /* Size of pLoc in bytes */
3627 if( rc==SQLITE_OK0 ){
3628 rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
3629 }
3630
3631 if( rc==SQLITE_OK0 ){
3632 rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc);
3633 }
3634 if( rc==SQLITE_OK0 ){
3635 rc = pApi->xTokenize_v2(
3636 pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb
3637 );
3638 }
3639 if( ctx.bOpen ){
3640 fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
3641 }
3642 fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
3643
3644 if( rc==SQLITE_OK0 ){
3645 sqlite3_result_textsqlite3_api->result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
3646 }
3647 sqlite3_freesqlite3_api->free(ctx.zOut);
3648 }
3649 if( rc!=SQLITE_OK0 ){
3650 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
3651 }
3652}
3653/*
3654** End of highlight() implementation.
3655**************************************************************************/
3656
3657/*
3658** Context object passed to the fts5SentenceFinderCb() function.
3659*/
3660typedef struct Fts5SFinder Fts5SFinder;
3661struct Fts5SFinder {
3662 int iPos; /* Current token position */
3663 int nFirstAlloc; /* Allocated size of aFirst[] */
3664 int nFirst; /* Number of entries in aFirst[] */
3665 int *aFirst; /* Array of first token in each sentence */
3666 const char *zDoc; /* Document being tokenized */
3667};
3668
3669/*
3670** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if
3671** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an
3672** error occurs.
3673*/
3674static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
3675 if( p->nFirstAlloc==p->nFirst ){
3676 int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
3677 int *aNew;
3678
3679 aNew = (int*)sqlite3_realloc64sqlite3_api->realloc64(p->aFirst, nNew*sizeof(int));
3680 if( aNew==0 ) return SQLITE_NOMEM7;
3681 p->aFirst = aNew;
3682 p->nFirstAlloc = nNew;
3683 }
3684 p->aFirst[p->nFirst++] = iAdd;
3685 return SQLITE_OK0;
3686}
3687
3688/*
3689** This function is an xTokenize() callback used by the auxiliary snippet()
3690** function. Its job is to identify tokens that are the first in a sentence.
3691** For each such token, an entry is added to the SFinder.aFirst[] array.
3692*/
3693static int fts5SentenceFinderCb(
3694 void *pContext, /* Pointer to HighlightContext object */
3695 int tflags, /* Mask of FTS5_TOKEN_* flags */
3696 const char *pToken, /* Buffer containing token */
3697 int nToken, /* Size of token in bytes */
3698 int iStartOff, /* Start offset of token */
3699 int iEndOff /* End offset of token */
3700){
3701 int rc = SQLITE_OK0;
3702
3703 UNUSED_PARAM2(pToken, nToken)(void)(pToken), (void)(nToken);
3704 UNUSED_PARAM(iEndOff)(void)(iEndOff);
3705
3706 if( (tflags & FTS5_TOKEN_COLOCATED0x0001)==0 ){
3707 Fts5SFinder *p = (Fts5SFinder*)pContext;
3708 if( p->iPos>0 ){
3709 int i;
3710 char c = 0;
3711 for(i=iStartOff-1; i>=0; i--){
3712 c = p->zDoc[i];
3713 if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break;
3714 }
3715 if( i!=iStartOff-1 && (c=='.' || c==':') ){
3716 rc = fts5SentenceFinderAdd(p, p->iPos);
3717 }
3718 }else{
3719 rc = fts5SentenceFinderAdd(p, 0);
3720 }
3721 p->iPos++;
3722 }
3723 return rc;
3724}
3725
3726static int fts5SnippetScore(
3727 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
3728 Fts5Context *pFts, /* First arg to pass to pApi functions */
3729 int nDocsize, /* Size of column in tokens */
3730 unsigned char *aSeen, /* Array with one element per query phrase */
3731 int iCol, /* Column to score */
3732 int iPos, /* Starting offset to score */
3733 int nToken, /* Max tokens per snippet */
3734 int *pnScore, /* OUT: Score */
3735 int *piPos /* OUT: Adjusted offset */
3736){
3737 int rc;
3738 int i;
3739 int ip = 0;
3740 int ic = 0;
3741 int iOff = 0;
3742 int iFirst = -1;
3743 int nInst;
3744 int nScore = 0;
3745 int iLast = 0;
3746 sqlite3_int64 iEnd = (sqlite3_int64)iPos + nToken;
3747
3748 rc = pApi->xInstCount(pFts, &nInst);
3749 for(i=0; i<nInst && rc==SQLITE_OK0; i++){
3750 rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
3751 if( rc==SQLITE_OK0 && ic==iCol && iOff>=iPos && iOff<iEnd ){
3752 nScore += (aSeen[ip] ? 1 : 1000);
3753 aSeen[ip] = 1;
3754 if( iFirst<0 ) iFirst = iOff;
3755 iLast = iOff + pApi->xPhraseSize(pFts, ip);
3756 }
3757 }
3758
3759 *pnScore = nScore;
3760 if( piPos ){
3761 sqlite3_int64 iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
3762 if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
3763 if( iAdj<0 ) iAdj = 0;
3764 *piPos = (int)iAdj;
3765 }
3766
3767 return rc;
3768}
3769
3770/*
3771** Return the value in pVal interpreted as utf-8 text. Except, if pVal
3772** contains a NULL value, return a pointer to a static string zero
3773** bytes in length instead of a NULL pointer.
3774*/
3775static const char *fts5ValueToText(sqlite3_value *pVal){
3776 const char *zRet = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
3777 return zRet ? zRet : "";
3778}
3779
3780/*
3781** Implementation of snippet() function.
3782*/
3783static void fts5SnippetFunction(
3784 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
3785 Fts5Context *pFts, /* First arg to pass to pApi functions */
3786 sqlite3_context *pCtx, /* Context for returning result/error */
3787 int nVal, /* Number of values in apVal[] array */
3788 sqlite3_value **apVal /* Array of trailing arguments */
3789){
3790 HighlightContext ctx;
3791 int rc = SQLITE_OK0; /* Return code */
3792 int iCol; /* 1st argument to snippet() */
3793 const char *zEllips; /* 4th argument to snippet() */
3794 int nToken; /* 5th argument to snippet() */
3795 int nInst = 0; /* Number of instance matches this row */
3796 int i; /* Used to iterate through instances */
3797 int nPhrase; /* Number of phrases in query */
3798 unsigned char *aSeen; /* Array of "seen instance" flags */
3799 int iBestCol; /* Column containing best snippet */
3800 int iBestStart = 0; /* First token of best snippet */
3801 int nBestScore = 0; /* Score of best snippet */
3802 int nColSize = 0; /* Total size of iBestCol in tokens */
3803 Fts5SFinder sFinder; /* Used to find the beginnings of sentences */
3804 int nCol;
3805
3806 if( nVal!=5 ){
3807 const char *zErr = "wrong number of arguments to function snippet()";
3808 sqlite3_result_errorsqlite3_api->result_error(pCtx, zErr, -1);
3809 return;
3810 }
3811
3812 nCol = pApi->xColumnCount(pFts);
3813 memset(&ctx, 0, sizeof(HighlightContext));
3814 iCol = sqlite3_value_intsqlite3_api->value_int(apVal[0]);
3815 ctx.zOpen = fts5ValueToText(apVal[1]);
3816 ctx.zClose = fts5ValueToText(apVal[2]);
3817 ctx.iRangeEnd = -1;
3818 zEllips = fts5ValueToText(apVal[3]);
3819 nToken = sqlite3_value_intsqlite3_api->value_int(apVal[4]);
3820
3821 iBestCol = (iCol>=0 ? iCol : 0);
3822 nPhrase = pApi->xPhraseCount(pFts);
3823 aSeen = sqlite3_malloc64sqlite3_api->malloc64(nPhrase);
3824 if( aSeen==0 ){
3825 rc = SQLITE_NOMEM7;
3826 }
3827 if( rc==SQLITE_OK0 ){
3828 rc = pApi->xInstCount(pFts, &nInst);
3829 }
3830
3831 memset(&sFinder, 0, sizeof(Fts5SFinder));
3832 for(i=0; i<nCol; i++){
3833 if( iCol<0 || iCol==i ){
3834 const char *pLoc = 0; /* Locale of column iCol */
3835 int nLoc = 0; /* Size of pLoc in bytes */
3836 int nDoc;
3837 int nDocsize;
3838 int ii;
3839 sFinder.iPos = 0;
3840 sFinder.nFirst = 0;
3841 rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
3842 if( rc!=SQLITE_OK0 ) break;
3843 rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc);
3844 if( rc!=SQLITE_OK0 ) break;
3845 rc = pApi->xTokenize_v2(pFts,
3846 sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb
3847 );
3848 if( rc!=SQLITE_OK0 ) break;
3849 rc = pApi->xColumnSize(pFts, i, &nDocsize);
3850 if( rc!=SQLITE_OK0 ) break;
3851
3852 for(ii=0; rc==SQLITE_OK0 && ii<nInst; ii++){
3853 int ip, ic, io;
3854 int iAdj;
3855 int nScore;
3856 int jj;
3857
3858 rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
3859 if( ic!=i ) continue;
3860 if( io>nDocsize ) rc = FTS5_CORRUPT(11 | (1<<8));
3861 if( rc!=SQLITE_OK0 ) continue;
3862 memset(aSeen, 0, nPhrase);
3863 rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
3864 io, nToken, &nScore, &iAdj
3865 );
3866 if( rc==SQLITE_OK0 && nScore>nBestScore ){
3867 nBestScore = nScore;
3868 iBestCol = i;
3869 iBestStart = iAdj;
3870 nColSize = nDocsize;
3871 }
3872
3873 if( rc==SQLITE_OK0 && sFinder.nFirst && nDocsize>nToken ){
3874 for(jj=0; jj<(sFinder.nFirst-1); jj++){
3875 if( sFinder.aFirst[jj+1]>io ) break;
3876 }
3877
3878 if( sFinder.aFirst[jj]<io ){
3879 memset(aSeen, 0, nPhrase);
3880 rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
3881 sFinder.aFirst[jj], nToken, &nScore, 0
3882 );
3883
3884 nScore += (sFinder.aFirst[jj]==0 ? 120 : 100);
3885 if( rc==SQLITE_OK0 && nScore>nBestScore ){
3886 nBestScore = nScore;
3887 iBestCol = i;
3888 iBestStart = sFinder.aFirst[jj];
3889 nColSize = nDocsize;
3890 }
3891 }
3892 }
3893 }
3894 }
3895 }
3896
3897 if( rc==SQLITE_OK0 ){
3898 rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
3899 }
3900 if( rc==SQLITE_OK0 && nColSize==0 ){
3901 rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
3902 }
3903 if( ctx.zIn ){
3904 const char *pLoc = 0; /* Locale of column iBestCol */
3905 int nLoc = 0; /* Bytes in pLoc */
3906
3907 if( rc==SQLITE_OK0 ){
3908 rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
3909 }
3910
3911 ctx.iRangeStart = iBestStart;
3912 ctx.iRangeEnd = iBestStart + nToken - 1;
3913
3914 if( iBestStart>0 ){
3915 fts5HighlightAppend(&rc, &ctx, zEllips, -1);
3916 }
3917
3918 /* Advance iterator ctx.iter so that it points to the first coalesced
3919 ** phrase instance at or following position iBestStart. */
3920 while( ctx.iter.iStart>=0 && ctx.iter.iStart<iBestStart && rc==SQLITE_OK0 ){
3921 rc = fts5CInstIterNext(&ctx.iter);
3922 }
3923
3924 if( rc==SQLITE_OK0 ){
3925 rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc);
3926 }
3927 if( rc==SQLITE_OK0 ){
3928 rc = pApi->xTokenize_v2(
3929 pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb
3930 );
3931 }
3932 if( ctx.bOpen ){
3933 fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
3934 }
3935 if( ctx.iRangeEnd>=(nColSize-1) ){
3936 fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
3937 }else{
3938 fts5HighlightAppend(&rc, &ctx, zEllips, -1);
3939 }
3940 }
3941 if( rc==SQLITE_OK0 ){
3942 sqlite3_result_textsqlite3_api->result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
3943 }else{
3944 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
3945 }
3946 sqlite3_freesqlite3_api->free(ctx.zOut);
3947 sqlite3_freesqlite3_api->free(aSeen);
3948 sqlite3_freesqlite3_api->free(sFinder.aFirst);
3949}
3950
3951/************************************************************************/
3952
3953/*
3954** The first time the bm25() function is called for a query, an instance
3955** of the following structure is allocated and populated.
3956*/
3957typedef struct Fts5Bm25Data Fts5Bm25Data;
3958struct Fts5Bm25Data {
3959 int nPhrase; /* Number of phrases in query */
3960 double avgdl; /* Average number of tokens in each row */
3961 double *aIDF; /* IDF for each phrase */
3962 double *aFreq; /* Array used to calculate phrase freq. */
3963};
3964
3965/*
3966** Callback used by fts5Bm25GetData() to count the number of rows in the
3967** table matched by each individual phrase within the query.
3968*/
3969static int fts5CountCb(
3970 const Fts5ExtensionApi *pApi,
3971 Fts5Context *pFts,
3972 void *pUserData /* Pointer to sqlite3_int64 variable */
3973){
3974 sqlite3_int64 *pn = (sqlite3_int64*)pUserData;
3975 UNUSED_PARAM2(pApi, pFts)(void)(pApi), (void)(pFts);
3976 (*pn)++;
3977 return SQLITE_OK0;
3978}
3979
3980/*
3981** Set *ppData to point to the Fts5Bm25Data object for the current query.
3982** If the object has not already been allocated, allocate and populate it
3983** now.
3984*/
3985static int fts5Bm25GetData(
3986 const Fts5ExtensionApi *pApi,
3987 Fts5Context *pFts,
3988 Fts5Bm25Data **ppData /* OUT: bm25-data object for this query */
3989){
3990 int rc = SQLITE_OK0; /* Return code */
3991 Fts5Bm25Data *p; /* Object to return */
3992
3993 p = (Fts5Bm25Data*)pApi->xGetAuxdata(pFts, 0);
3994 if( p==0 ){
3995 int nPhrase; /* Number of phrases in query */
3996 sqlite3_int64 nRow = 0; /* Number of rows in table */
3997 sqlite3_int64 nToken = 0; /* Number of tokens in table */
3998 sqlite3_int64 nByte; /* Bytes of space to allocate */
3999 int i;
4000
4001 /* Allocate the Fts5Bm25Data object */
4002 nPhrase = pApi->xPhraseCount(pFts);
4003 nByte = sizeof(Fts5Bm25Data) + nPhrase*2*sizeof(double);
4004 p = (Fts5Bm25Data*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
4005 if( p==0 ){
4006 rc = SQLITE_NOMEM7;
4007 }else{
4008 memset(p, 0, (size_t)nByte);
4009 p->nPhrase = nPhrase;
4010 p->aIDF = (double*)&p[1];
4011 p->aFreq = &p->aIDF[nPhrase];
4012 }
4013
4014 /* Calculate the average document length for this FTS5 table */
4015 if( rc==SQLITE_OK0 ) rc = pApi->xRowCount(pFts, &nRow);
4016 assert( rc!=SQLITE_OK || nRow>0 )((void) (0));
4017 if( rc==SQLITE_OK0 ) rc = pApi->xColumnTotalSize(pFts, -1, &nToken);
4018 if( rc==SQLITE_OK0 ) p->avgdl = (double)nToken / (double)nRow;
4019
4020 /* Calculate an IDF for each phrase in the query */
4021 for(i=0; rc==SQLITE_OK0 && i<nPhrase; i++){
4022 sqlite3_int64 nHit = 0;
4023 rc = pApi->xQueryPhrase(pFts, i, (void*)&nHit, fts5CountCb);
4024 if( rc==SQLITE_OK0 ){
4025 /* Calculate the IDF (Inverse Document Frequency) for phrase i.
4026 ** This is done using the standard BM25 formula as found on wikipedia:
4027 **
4028 ** IDF = log( (N - nHit + 0.5) / (nHit + 0.5) )
4029 **
4030 ** where "N" is the total number of documents in the set and nHit
4031 ** is the number that contain at least one instance of the phrase
4032 ** under consideration.
4033 **
4034 ** The problem with this is that if (N < 2*nHit), the IDF is
4035 ** negative. Which is undesirable. So the minimum allowable IDF is
4036 ** (1e-6) - roughly the same as a term that appears in just over
4037 ** half of set of 5,000,000 documents. */
4038 double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) );
4039 if( idf<=0.0 ) idf = 1e-6;
4040 p->aIDF[i] = idf;
4041 }
4042 }
4043
4044 if( rc!=SQLITE_OK0 ){
4045 sqlite3_freesqlite3_api->free(p);
4046 }else{
4047 rc = pApi->xSetAuxdata(pFts, p, sqlite3_freesqlite3_api->free);
4048 }
4049 if( rc!=SQLITE_OK0 ) p = 0;
4050 }
4051 *ppData = p;
4052 return rc;
4053}
4054
4055/*
4056** Implementation of bm25() function.
4057*/
4058static void fts5Bm25Function(
4059 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
4060 Fts5Context *pFts, /* First arg to pass to pApi functions */
4061 sqlite3_context *pCtx, /* Context for returning result/error */
4062 int nVal, /* Number of values in apVal[] array */
4063 sqlite3_value **apVal /* Array of trailing arguments */
4064){
4065 const double k1 = 1.2; /* Constant "k1" from BM25 formula */
4066 const double b = 0.75; /* Constant "b" from BM25 formula */
4067 int rc; /* Error code */
4068 double score = 0.0; /* SQL function return value */
4069 Fts5Bm25Data *pData; /* Values allocated/calculated once only */
4070 int i; /* Iterator variable */
4071 int nInst = 0; /* Value returned by xInstCount() */
4072 double D = 0.0; /* Total number of tokens in row */
4073 double *aFreq = 0; /* Array of phrase freq. for current row */
4074
4075 /* Calculate the phrase frequency (symbol "f(qi,D)" in the documentation)
4076 ** for each phrase in the query for the current row. */
4077 rc = fts5Bm25GetData(pApi, pFts, &pData);
4078 if( rc==SQLITE_OK0 ){
4079 aFreq = pData->aFreq;
4080 memset(aFreq, 0, sizeof(double) * pData->nPhrase);
4081 rc = pApi->xInstCount(pFts, &nInst);
4082 }
4083 for(i=0; rc==SQLITE_OK0 && i<nInst; i++){
4084 int ip; int ic; int io;
4085 rc = pApi->xInst(pFts, i, &ip, &ic, &io);
4086 if( rc==SQLITE_OK0 ){
4087 double w = (nVal > ic) ? sqlite3_value_doublesqlite3_api->value_double(apVal[ic]) : 1.0;
4088 aFreq[ip] += w;
4089 }
4090 }
4091
4092 /* Figure out the total size of the current row in tokens. */
4093 if( rc==SQLITE_OK0 ){
4094 int nTok;
4095 rc = pApi->xColumnSize(pFts, -1, &nTok);
4096 D = (double)nTok;
4097 }
4098
4099 /* Determine and return the BM25 score for the current row. Or, if an
4100 ** error has occurred, throw an exception. */
4101 if( rc==SQLITE_OK0 ){
4102 for(i=0; i<pData->nPhrase; i++){
4103 score += pData->aIDF[i] * (
4104 ( aFreq[i] * (k1 + 1.0) ) /
4105 ( aFreq[i] + k1 * (1 - b + b * D / pData->avgdl) )
4106 );
4107 }
4108 sqlite3_result_doublesqlite3_api->result_double(pCtx, -1.0 * score);
4109 }else{
4110 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
4111 }
4112}
4113
4114/*
4115** Implementation of fts5_get_locale() function.
4116*/
4117static void fts5GetLocaleFunction(
4118 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
4119 Fts5Context *pFts, /* First arg to pass to pApi functions */
4120 sqlite3_context *pCtx, /* Context for returning result/error */
4121 int nVal, /* Number of values in apVal[] array */
4122 sqlite3_value **apVal /* Array of trailing arguments */
4123){
4124 int iCol = 0;
4125 int eType = 0;
4126 int rc = SQLITE_OK0;
4127 const char *zLocale = 0;
4128 int nLocale = 0;
4129
4130 /* xColumnLocale() must be available */
4131 assert( pApi->iVersion>=4 )((void) (0));
4132
4133 if( nVal!=1 ){
4134 const char *z = "wrong number of arguments to function fts5_get_locale()";
4135 sqlite3_result_errorsqlite3_api->result_error(pCtx, z, -1);
4136 return;
4137 }
4138
4139 eType = sqlite3_value_numeric_typesqlite3_api->value_numeric_type(apVal[0]);
4140 if( eType!=SQLITE_INTEGER1 ){
4141 const char *z = "non-integer argument passed to function fts5_get_locale()";
4142 sqlite3_result_errorsqlite3_api->result_error(pCtx, z, -1);
4143 return;
4144 }
4145
4146 iCol = sqlite3_value_intsqlite3_api->value_int(apVal[0]);
4147 if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){
4148 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, SQLITE_RANGE25);
4149 return;
4150 }
4151
4152 rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale);
4153 if( rc!=SQLITE_OK0 ){
4154 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
4155 return;
4156 }
4157
4158 sqlite3_result_textsqlite3_api->result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
4159}
4160
4161static int sqlite3Fts5AuxInit(fts5_api *pApi){
4162 struct Builtin {
4163 const char *zFunc; /* Function name (nul-terminated) */
4164 void *pUserData; /* User-data pointer */
4165 fts5_extension_function xFunc;/* Callback function */
4166 void (*xDestroy)(void*); /* Destructor function */
4167 } aBuiltin [] = {
4168 { "snippet", 0, fts5SnippetFunction, 0 },
4169 { "highlight", 0, fts5HighlightFunction, 0 },
4170 { "bm25", 0, fts5Bm25Function, 0 },
4171 { "fts5_get_locale", 0, fts5GetLocaleFunction, 0 },
4172 };
4173 int rc = SQLITE_OK0; /* Return code */
4174 int i; /* To iterate through builtin functions */
4175
4176 for(i=0; rc==SQLITE_OK0 && i<ArraySize(aBuiltin)((int)(sizeof(aBuiltin) / sizeof(aBuiltin[0]))); i++){
4177 rc = pApi->xCreateFunction(pApi,
4178 aBuiltin[i].zFunc,
4179 aBuiltin[i].pUserData,
4180 aBuiltin[i].xFunc,
4181 aBuiltin[i].xDestroy
4182 );
4183 }
4184
4185 return rc;
4186}
4187
4188#line 1 "fts5_buffer.c"
4189/*
4190** 2014 May 31
4191**
4192** The author disclaims copyright to this source code. In place of
4193** a legal notice, here is a blessing:
4194**
4195** May you do good and not evil.
4196** May you find forgiveness for yourself and forgive others.
4197** May you share freely, never taking more than you give.
4198**
4199******************************************************************************
4200*/
4201
4202
4203
4204/* #include "fts5Int.h" */
4205
4206static int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
4207 if( (u32)pBuf->nSpace<nByte ){
4208 u64 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
4209 u8 *pNew;
4210 while( nNew<nByte ){
4211 nNew = nNew * 2;
4212 }
4213 pNew = sqlite3_realloc64sqlite3_api->realloc64(pBuf->p, nNew);
4214 if( pNew==0 ){
4215 *pRc = SQLITE_NOMEM7;
4216 return 1;
4217 }else{
4218 pBuf->nSpace = (int)nNew;
4219 pBuf->p = pNew;
4220 }
4221 }
4222 return 0;
4223}
4224
4225
4226/*
4227** Encode value iVal as an SQLite varint and append it to the buffer object
4228** pBuf. If an OOM error occurs, set the error code in p.
4229*/
4230static void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
4231 if( fts5BufferGrow(pRc, pBuf, 9)( (u32)((pBuf)->n) + (u32)(9) <= (u32)((pBuf)->nSpace
) ? 0 : sqlite3Fts5BufferSize((pRc),(pBuf),(9)+(pBuf)->n) ) ) return;
4232 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal);
4233}
4234
4235static void sqlite3Fts5Put32(u8 *aBuf, int iVal){
4236 aBuf[0] = (iVal>>24) & 0x00FF;
4237 aBuf[1] = (iVal>>16) & 0x00FF;
4238 aBuf[2] = (iVal>> 8) & 0x00FF;
4239 aBuf[3] = (iVal>> 0) & 0x00FF;
4240}
4241
4242static int sqlite3Fts5Get32(const u8 *aBuf){
4243 return (int)((((u32)aBuf[0])<<24) + (aBuf[1]<<16) + (aBuf[2]<<8) + aBuf[3]);
4244}
4245
4246/*
4247** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set
4248** the error code in p. If an error has already occurred when this function
4249** is called, it is a no-op.
4250*/
4251static void sqlite3Fts5BufferAppendBlob(
4252 int *pRc,
4253 Fts5Buffer *pBuf,
4254 u32 nData,
4255 const u8 *pData
4256){
4257 if( nData ){
4258 if( fts5BufferGrow(pRc, pBuf, nData)( (u32)((pBuf)->n) + (u32)(nData) <= (u32)((pBuf)->nSpace
) ? 0 : sqlite3Fts5BufferSize((pRc),(pBuf),(nData)+(pBuf)->
n) ) ) return;
4259 assert( pBuf->p!=0 )((void) (0));
4260 memcpy(&pBuf->p[pBuf->n], pData, nData);
4261 pBuf->n += nData;
4262 }
4263}
4264
4265/*
4266** Append the nul-terminated string zStr to the buffer pBuf. This function
4267** ensures that the byte following the buffer data is set to 0x00, even
4268** though this byte is not included in the pBuf->n count.
4269*/
4270static void sqlite3Fts5BufferAppendString(
4271 int *pRc,
4272 Fts5Buffer *pBuf,
4273 const char *zStr
4274){
4275 int nStr = (int)strlen(zStr);
4276 sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr+1, (const u8*)zStr);
4277 pBuf->n--;
4278}
4279
4280/*
4281** Argument zFmt is a printf() style format string. This function performs
4282** the printf() style processing, then appends the results to buffer pBuf.
4283**
4284** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte
4285** following the buffer data is set to 0x00, even though this byte is not
4286** included in the pBuf->n count.
4287*/
4288static void sqlite3Fts5BufferAppendPrintf(
4289 int *pRc,
4290 Fts5Buffer *pBuf,
4291 char *zFmt, ...
4292){
4293 if( *pRc==SQLITE_OK0 ){
4294 char *zTmp;
4295 va_list ap;
4296 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
4297 zTmp = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
4298 va_end(ap)__builtin_va_end(ap);
4299
4300 if( zTmp==0 ){
4301 *pRc = SQLITE_NOMEM7;
4302 }else{
4303 sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp);
4304 sqlite3_freesqlite3_api->free(zTmp);
4305 }
4306 }
4307}
4308
4309static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...){
4310 char *zRet = 0;
4311 if( *pRc==SQLITE_OK0 ){
4312 va_list ap;
4313 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
4314 zRet = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
4315 va_end(ap)__builtin_va_end(ap);
4316 if( zRet==0 ){
4317 *pRc = SQLITE_NOMEM7;
4318 }
4319 }
4320 return zRet;
4321}
4322
4323
4324/*
4325** Free any buffer allocated by pBuf. Zero the structure before returning.
4326*/
4327static void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){
4328 sqlite3_freesqlite3_api->free(pBuf->p);
4329 memset(pBuf, 0, sizeof(Fts5Buffer));
4330}
4331
4332/*
4333** Zero the contents of the buffer object. But do not free the associated
4334** memory allocation.
4335*/
4336static void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){
4337 pBuf->n = 0;
4338}
4339
4340/*
4341** Set the buffer to contain nData/pData. If an OOM error occurs, leave an
4342** the error code in p. If an error has already occurred when this function
4343** is called, it is a no-op.
4344*/
4345static void sqlite3Fts5BufferSet(
4346 int *pRc,
4347 Fts5Buffer *pBuf,
4348 int nData,
4349 const u8 *pData
4350){
4351 pBuf->n = 0;
4352 sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData);
4353}
4354
4355static int sqlite3Fts5PoslistNext64(
4356 const u8 *a, int n, /* Buffer containing poslist */
4357 int *pi, /* IN/OUT: Offset within a[] */
4358 i64 *piOff /* IN/OUT: Current offset */
4359){
4360 int i = *pi;
4361 assert( a!=0 || i==0 )((void) (0));
4362 if( i>=n ){
4363 /* EOF */
4364 *piOff = -1;
4365 return 1;
4366 }else{
4367 i64 iOff = *piOff;
4368 u32 iVal;
4369 assert( a!=0 )((void) (0));
4370 fts5FastGetVarint32(a, i, iVal){ iVal = (a)[i++]; if( iVal & 0x80 ){ i--; i += sqlite3Fts5GetVarint32
(&(a)[i],(u32*)&(iVal)); } };
4371 if( iVal<=1 ){
4372 if( iVal==0 ){
4373 *pi = i;
4374 return 0;
4375 }
4376 fts5FastGetVarint32(a, i, iVal){ iVal = (a)[i++]; if( iVal & 0x80 ){ i--; i += sqlite3Fts5GetVarint32
(&(a)[i],(u32*)&(iVal)); } };
4377 iOff = ((i64)iVal) << 32;
4378 assert( iOff>=0 )((void) (0));
4379 fts5FastGetVarint32(a, i, iVal){ iVal = (a)[i++]; if( iVal & 0x80 ){ i--; i += sqlite3Fts5GetVarint32
(&(a)[i],(u32*)&(iVal)); } };
4380 if( iVal<2 ){
4381 /* This is a corrupt record. So stop parsing it here. */
4382 *piOff = -1;
4383 return 1;
4384 }
4385 *piOff = iOff + ((iVal-2) & 0x7FFFFFFF);
4386 }else{
4387 *piOff = (iOff & (i64)0x7FFFFFFF<<32)+((iOff + (iVal-2)) & 0x7FFFFFFF);
4388 }
4389 *pi = i;
4390 assert_nc( *piOff>=iOff )((void) (0));
4391 return 0;
4392 }
4393}
4394
4395
4396/*
4397** Advance the iterator object passed as the only argument. Return true
4398** if the iterator reaches EOF, or false otherwise.
4399*/
4400static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){
4401 if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) ){
4402 pIter->bEof = 1;
4403 }
4404 return pIter->bEof;
4405}
4406
4407static int sqlite3Fts5PoslistReaderInit(
4408 const u8 *a, int n, /* Poslist buffer to iterate through */
4409 Fts5PoslistReader *pIter /* Iterator object to initialize */
4410){
4411 memset(pIter, 0, sizeof(*pIter));
4412 pIter->a = a;
4413 pIter->n = n;
4414 sqlite3Fts5PoslistReaderNext(pIter);
4415 return pIter->bEof;
4416}
4417
4418/*
4419** Append position iPos to the position list being accumulated in buffer
4420** pBuf, which must be already be large enough to hold the new data.
4421** The previous position written to this list is *piPrev. *piPrev is set
4422** to iPos before returning.
4423*/
4424static void sqlite3Fts5PoslistSafeAppend(
4425 Fts5Buffer *pBuf,
4426 i64 *piPrev,
4427 i64 iPos
4428){
4429 if( iPos>=*piPrev ){
4430 static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
4431 if( (iPos & colmask) != (*piPrev & colmask) ){
4432 pBuf->p[pBuf->n++] = 1;
4433 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
4434 *piPrev = (iPos & colmask);
4435 }
4436 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
4437 *piPrev = iPos;
4438 }
4439}
4440
4441static int sqlite3Fts5PoslistWriterAppend(
4442 Fts5Buffer *pBuf,
4443 Fts5PoslistWriter *pWriter,
4444 i64 iPos
4445){
4446 int rc = 0; /* Initialized only to suppress erroneous warning from Clang */
4447 if( fts5BufferGrow(&rc, pBuf, 5+5+5)( (u32)((pBuf)->n) + (u32)(5+5+5) <= (u32)((pBuf)->nSpace
) ? 0 : sqlite3Fts5BufferSize((&rc),(pBuf),(5+5+5)+(pBuf)
->n) ) ) return rc;
4448 sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
4449 return SQLITE_OK0;
4450}
4451
4452static void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte){
4453 void *pRet = 0;
4454 if( *pRc==SQLITE_OK0 ){
4455 pRet = sqlite3_malloc64sqlite3_api->malloc64(nByte);
4456 if( pRet==0 ){
4457 if( nByte>0 ) *pRc = SQLITE_NOMEM7;
4458 }else{
4459 memset(pRet, 0, (size_t)nByte);
4460 }
4461 }
4462 return pRet;
4463}
4464
4465/*
4466** Return a nul-terminated copy of the string indicated by pIn. If nIn
4467** is non-negative, then it is the length of the string in bytes. Otherwise,
4468** the length of the string is determined using strlen().
4469**
4470** It is the responsibility of the caller to eventually free the returned
4471** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned.
4472*/
4473static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){
4474 char *zRet = 0;
4475 if( *pRc==SQLITE_OK0 ){
4476 if( nIn<0 ){
4477 nIn = (int)strlen(pIn);
4478 }
4479 zRet = (char*)sqlite3_malloc64sqlite3_api->malloc64((i64)nIn+1);
4480 if( zRet ){
4481 memcpy(zRet, pIn, nIn);
4482 zRet[nIn] = '\0';
4483 }else{
4484 *pRc = SQLITE_NOMEM7;
4485 }
4486 }
4487 return zRet;
4488}
4489
4490
4491/*
4492** Return true if character 't' may be part of an FTS5 bareword, or false
4493** otherwise. Characters that may be part of barewords:
4494**
4495** * All non-ASCII characters,
4496** * The 52 upper and lower case ASCII characters, and
4497** * The 10 integer ASCII characters.
4498** * The underscore character "_" (0x5F).
4499** * The unicode "substitute" character (0x1A).
4500*/
4501static int sqlite3Fts5IsBareword(char t){
4502 u8 aBareword[128] = {
4503 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */
4504 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */
4505 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */
4506 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */
4507 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */
4508 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */
4509 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */
4510 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */
4511 };
4512
4513 return (t & 0x80) || aBareword[(int)t];
4514}
4515
4516
4517/*************************************************************************
4518*/
4519typedef struct Fts5TermsetEntry Fts5TermsetEntry;
4520struct Fts5TermsetEntry {
4521 char *pTerm;
4522 int nTerm;
4523 int iIdx; /* Index (main or aPrefix[] entry) */
4524 Fts5TermsetEntry *pNext;
4525};
4526
4527struct Fts5Termset {
4528 Fts5TermsetEntry *apHash[512];
4529};
4530
4531static int sqlite3Fts5TermsetNew(Fts5Termset **pp){
4532 int rc = SQLITE_OK0;
4533 *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset));
4534 return rc;
4535}
4536
4537static int sqlite3Fts5TermsetAdd(
4538 Fts5Termset *p,
4539 int iIdx,
4540 const char *pTerm, int nTerm,
4541 int *pbPresent
4542){
4543 int rc = SQLITE_OK0;
4544 *pbPresent = 0;
4545 if( p ){
4546 int i;
4547 u32 hash = 13;
4548 Fts5TermsetEntry *pEntry;
4549
4550 /* Calculate a hash value for this term. This is the same hash checksum
4551 ** used by the fts5_hash.c module. This is not important for correct
4552 ** operation of the module, but is necessary to ensure that some tests
4553 ** designed to produce hash table collisions really do work. */
4554 for(i=nTerm-1; i>=0; i--){
4555 hash = (hash << 3) ^ hash ^ pTerm[i];
4556 }
4557 hash = (hash << 3) ^ hash ^ iIdx;
4558 hash = hash % ArraySize(p->apHash)((int)(sizeof(p->apHash) / sizeof(p->apHash[0])));
4559
4560 for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
4561 if( pEntry->iIdx==iIdx
4562 && pEntry->nTerm==nTerm
4563 && memcmp(pEntry->pTerm, pTerm, nTerm)==0
4564 ){
4565 *pbPresent = 1;
4566 break;
4567 }
4568 }
4569
4570 if( pEntry==0 ){
4571 pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm);
4572 if( pEntry ){
4573 pEntry->pTerm = (char*)&pEntry[1];
4574 pEntry->nTerm = nTerm;
4575 pEntry->iIdx = iIdx;
4576 memcpy(pEntry->pTerm, pTerm, nTerm);
4577 pEntry->pNext = p->apHash[hash];
4578 p->apHash[hash] = pEntry;
4579 }
4580 }
4581 }
4582
4583 return rc;
4584}
4585
4586static void sqlite3Fts5TermsetFree(Fts5Termset *p){
4587 if( p ){
4588 u32 i;
4589 for(i=0; i<ArraySize(p->apHash)((int)(sizeof(p->apHash) / sizeof(p->apHash[0]))); i++){
4590 Fts5TermsetEntry *pEntry = p->apHash[i];
4591 while( pEntry ){
4592 Fts5TermsetEntry *pDel = pEntry;
4593 pEntry = pEntry->pNext;
4594 sqlite3_freesqlite3_api->free(pDel);
4595 }
4596 }
4597 sqlite3_freesqlite3_api->free(p);
4598 }
4599}
4600
4601#line 1 "fts5_config.c"
4602/*
4603** 2014 Jun 09
4604**
4605** The author disclaims copyright to this source code. In place of
4606** a legal notice, here is a blessing:
4607**
4608** May you do good and not evil.
4609** May you find forgiveness for yourself and forgive others.
4610** May you share freely, never taking more than you give.
4611**
4612******************************************************************************
4613**
4614** This is an SQLite module implementing full-text search.
4615*/
4616
4617
4618/* #include "fts5Int.h" */
4619
4620#define FTS5_DEFAULT_PAGE_SIZE4050 4050
4621#define FTS5_DEFAULT_AUTOMERGE4 4
4622#define FTS5_DEFAULT_USERMERGE4 4
4623#define FTS5_DEFAULT_CRISISMERGE16 16
4624#define FTS5_DEFAULT_HASHSIZE(1024*1024) (1024*1024)
4625
4626#define FTS5_DEFAULT_DELETE_AUTOMERGE10 10 /* default 10% */
4627
4628/* Maximum allowed page size */
4629#define FTS5_MAX_PAGE_SIZE(64*1024) (64*1024)
4630
4631static int fts5_iswhitespace(char x){
4632 return (x==' ');
4633}
4634
4635static int fts5_isopenquote(char x){
4636 return (x=='"' || x=='\'' || x=='[' || x=='`');
4637}
4638
4639/*
4640** Argument pIn points to a character that is part of a nul-terminated
4641** string. Return a pointer to the first character following *pIn in
4642** the string that is not a white-space character.
4643*/
4644static const char *fts5ConfigSkipWhitespace(const char *pIn){
4645 const char *p = pIn;
4646 if( p ){
4647 while( fts5_iswhitespace(*p) ){ p++; }
4648 }
4649 return p;
4650}
4651
4652/*
4653** Argument pIn points to a character that is part of a nul-terminated
4654** string. Return a pointer to the first character following *pIn in
4655** the string that is not a "bareword" character.
4656*/
4657static const char *fts5ConfigSkipBareword(const char *pIn){
4658 const char *p = pIn;
4659 while ( sqlite3Fts5IsBareword(*p) ) p++;
4660 if( p==pIn ) p = 0;
4661 return p;
4662}
4663
4664static int fts5_isdigit(char a){
4665 return (a>='0' && a<='9');
4666}
4667
4668
4669
4670static const char *fts5ConfigSkipLiteral(const char *pIn){
4671 const char *p = pIn;
4672 switch( *p ){
4673 case 'n': case 'N':
4674 if( sqlite3_strnicmpsqlite3_api->strnicmp("null", p, 4)==0 ){
4675 p = &p[4];
4676 }else{
4677 p = 0;
4678 }
4679 break;
4680
4681 case 'x': case 'X':
4682 p++;
4683 if( *p=='\'' ){
4684 p++;
4685 while( (*p>='a' && *p<='f')
4686 || (*p>='A' && *p<='F')
4687 || (*p>='0' && *p<='9')
4688 ){
4689 p++;
4690 }
4691 if( *p=='\'' && 0==((p-pIn)%2) ){
4692 p++;
4693 }else{
4694 p = 0;
4695 }
4696 }else{
4697 p = 0;
4698 }
4699 break;
4700
4701 case '\'':
4702 p++;
4703 while( p ){
4704 if( *p=='\'' ){
4705 p++;
4706 if( *p!='\'' ) break;
4707 }
4708 p++;
4709 if( *p==0 ) p = 0;
4710 }
4711 break;
4712
4713 default:
4714 /* maybe a number */
4715 if( *p=='+' || *p=='-' ) p++;
4716 while( fts5_isdigit(*p) ) p++;
4717
4718 /* At this point, if the literal was an integer, the parse is
4719 ** finished. Or, if it is a floating point value, it may continue
4720 ** with either a decimal point or an 'E' character. */
4721 if( *p=='.' && fts5_isdigit(p[1]) ){
4722 p += 2;
4723 while( fts5_isdigit(*p) ) p++;
4724 }
4725 if( p==pIn ) p = 0;
4726
4727 break;
4728 }
4729
4730 return p;
4731}
4732
4733/*
4734** The first character of the string pointed to by argument z is guaranteed
4735** to be an open-quote character (see function fts5_isopenquote()).
4736**
4737** This function searches for the corresponding close-quote character within
4738** the string and, if found, dequotes the string in place and adds a new
4739** nul-terminator byte.
4740**
4741** If the close-quote is found, the value returned is the byte offset of
4742** the character immediately following it. Or, if the close-quote is not
4743** found, -1 is returned. If -1 is returned, the buffer is left in an
4744** undefined state.
4745*/
4746static int fts5Dequote(char *z){
4747 char q;
4748 int iIn = 1;
4749 int iOut = 0;
4750 q = z[0];
4751
4752 /* Set stack variable q to the close-quote character */
4753 assert( q=='[' || q=='\'' || q=='"' || q=='`' )((void) (0));
4754 if( q=='[' ) q = ']';
4755
4756 while( z[iIn] ){
4757 if( z[iIn]==q ){
4758 if( z[iIn+1]!=q ){
4759 /* Character iIn was the close quote. */
4760 iIn++;
4761 break;
4762 }else{
4763 /* Character iIn and iIn+1 form an escaped quote character. Skip
4764 ** the input cursor past both and copy a single quote character
4765 ** to the output buffer. */
4766 iIn += 2;
4767 z[iOut++] = q;
4768 }
4769 }else{
4770 z[iOut++] = z[iIn++];
4771 }
4772 }
4773
4774 z[iOut] = '\0';
4775 return iIn;
4776}
4777
4778/*
4779** Convert an SQL-style quoted string into a normal string by removing
4780** the quote characters. The conversion is done in-place. If the
4781** input does not begin with a quote character, then this routine
4782** is a no-op.
4783**
4784** Examples:
4785**
4786** "abc" becomes abc
4787** 'xyz' becomes xyz
4788** [pqr] becomes pqr
4789** `mno` becomes mno
4790*/
4791static void sqlite3Fts5Dequote(char *z){
4792 char quote; /* Quote character (if any ) */
4793
4794 assert( 0==fts5_iswhitespace(z[0]) )((void) (0));
4795 quote = z[0];
4796 if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
4797 fts5Dequote(z);
4798 }
4799}
4800
4801
4802struct Fts5Enum {
4803 const char *zName;
4804 int eVal;
4805};
4806typedef struct Fts5Enum Fts5Enum;
4807
4808static int fts5ConfigSetEnum(
4809 const Fts5Enum *aEnum,
4810 const char *zEnum,
4811 int *peVal
4812){
4813 int nEnum = (int)strlen(zEnum);
4814 int i;
4815 int iVal = -1;
4816
4817 for(i=0; aEnum[i].zName; i++){
4818 if( sqlite3_strnicmpsqlite3_api->strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
4819 if( iVal>=0 ) return SQLITE_ERROR1;
4820 iVal = aEnum[i].eVal;
4821 }
4822 }
4823
4824 *peVal = iVal;
4825 return iVal<0 ? SQLITE_ERROR1 : SQLITE_OK0;
4826}
4827
4828/*
4829** Parse a "special" CREATE VIRTUAL TABLE directive and update
4830** configuration object pConfig as appropriate.
4831**
4832** If successful, object pConfig is updated and SQLITE_OK returned. If
4833** an error occurs, an SQLite error code is returned and an error message
4834** may be left in *pzErr. It is the responsibility of the caller to
4835** eventually free any such error message using sqlite3_free().
4836*/
4837static int fts5ConfigParseSpecial(
4838 Fts5Config *pConfig, /* Configuration object to update */
4839 const char *zCmd, /* Special command to parse */
4840 const char *zArg, /* Argument to parse */
4841 char **pzErr /* OUT: Error message */
4842){
4843 int rc = SQLITE_OK0;
4844 int nCmd = (int)strlen(zCmd);
4845
4846 if( sqlite3_strnicmpsqlite3_api->strnicmp("prefix", zCmd, nCmd)==0 ){
4847 const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES31;
4848 const char *p;
4849 int bFirst = 1;
4850 if( pConfig->aPrefix==0 ){
4851 pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
4852 if( rc ) return rc;
4853 }
4854
4855 p = zArg;
4856 while( 1 ){
4857 int nPre = 0;
4858
4859 while( p[0]==' ' ) p++;
4860 if( bFirst==0 && p[0]==',' ){
4861 p++;
4862 while( p[0]==' ' ) p++;
4863 }else if( p[0]=='\0' ){
4864 break;
4865 }
4866 if( p[0]<'0' || p[0]>'9' ){
4867 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed prefix=... directive");
4868 rc = SQLITE_ERROR1;
4869 break;
4870 }
4871
4872 if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES31 ){
4873 *pzErr = sqlite3_mprintfsqlite3_api->mprintf(
4874 "too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES31
4875 );
4876 rc = SQLITE_ERROR1;
4877 break;
4878 }
4879
4880 while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
4881 nPre = nPre*10 + (p[0] - '0');
4882 p++;
4883 }
4884
4885 if( nPre<=0 || nPre>=1000 ){
4886 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("prefix length out of range (max 999)");
4887 rc = SQLITE_ERROR1;
4888 break;
4889 }
4890
4891 pConfig->aPrefix[pConfig->nPrefix] = nPre;
4892 pConfig->nPrefix++;
4893 bFirst = 0;
4894 }
4895 assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES )((void) (0));
4896 return rc;
4897 }
4898
4899 if( sqlite3_strnicmpsqlite3_api->strnicmp("tokenize", zCmd, nCmd)==0 ){
4900 const char *p = (const char*)zArg;
4901 sqlite3_int64 nArg = strlen(zArg) + 1;
4902 char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg);
4903
4904 if( azArg ){
4905 char *pSpace = (char*)&azArg[nArg];
4906 if( pConfig->t.azArg ){
4907 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("multiple tokenize=... directives");
4908 rc = SQLITE_ERROR1;
4909 }else{
4910 for(nArg=0; p && *p; nArg++){
4911 const char *p2 = fts5ConfigSkipWhitespace(p);
4912 if( *p2=='\'' ){
4913 p = fts5ConfigSkipLiteral(p2);
4914 }else{
4915 p = fts5ConfigSkipBareword(p2);
4916 }
4917 if( p ){
4918 memcpy(pSpace, p2, p-p2);
4919 azArg[nArg] = pSpace;
4920 sqlite3Fts5Dequote(pSpace);
4921 pSpace += (p - p2) + 1;
4922 p = fts5ConfigSkipWhitespace(p);
4923 }
4924 }
4925 if( p==0 ){
4926 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("parse error in tokenize directive");
4927 rc = SQLITE_ERROR1;
4928 }else{
4929 pConfig->t.azArg = (const char**)azArg;
4930 pConfig->t.nArg = nArg;
4931 azArg = 0;
4932 }
4933 }
4934 }
4935 sqlite3_freesqlite3_api->free(azArg);
4936
4937 return rc;
4938 }
4939
4940 if( sqlite3_strnicmpsqlite3_api->strnicmp("content", zCmd, nCmd)==0 ){
4941 if( pConfig->eContent!=FTS5_CONTENT_NORMAL0 ){
4942 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("multiple content=... directives");
4943 rc = SQLITE_ERROR1;
4944 }else{
4945 if( zArg[0] ){
4946 pConfig->eContent = FTS5_CONTENT_EXTERNAL2;
4947 pConfig->zContent = sqlite3Fts5Mprintf(&rc, "%Q.%Q", pConfig->zDb,zArg);
4948 }else{
4949 pConfig->eContent = FTS5_CONTENT_NONE1;
4950 }
4951 }
4952 return rc;
4953 }
4954
4955 if( sqlite3_strnicmpsqlite3_api->strnicmp("contentless_delete", zCmd, nCmd)==0 ){
4956 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
4957 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed contentless_delete=... directive");
4958 rc = SQLITE_ERROR1;
4959 }else{
4960 pConfig->bContentlessDelete = (zArg[0]=='1');
4961 }
4962 return rc;
4963 }
4964
4965 if( sqlite3_strnicmpsqlite3_api->strnicmp("contentless_unindexed", zCmd, nCmd)==0 ){
4966 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
4967 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed contentless_delete=... directive");
4968 rc = SQLITE_ERROR1;
4969 }else{
4970 pConfig->bContentlessUnindexed = (zArg[0]=='1');
4971 }
4972 return rc;
4973 }
4974
4975 if( sqlite3_strnicmpsqlite3_api->strnicmp("content_rowid", zCmd, nCmd)==0 ){
4976 if( pConfig->zContentRowid ){
4977 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("multiple content_rowid=... directives");
4978 rc = SQLITE_ERROR1;
4979 }else{
4980 pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1);
4981 }
4982 return rc;
4983 }
4984
4985 if( sqlite3_strnicmpsqlite3_api->strnicmp("columnsize", zCmd, nCmd)==0 ){
4986 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
4987 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed columnsize=... directive");
4988 rc = SQLITE_ERROR1;
4989 }else{
4990 pConfig->bColumnsize = (zArg[0]=='1');
4991 }
4992 return rc;
4993 }
4994
4995 if( sqlite3_strnicmpsqlite3_api->strnicmp("locale", zCmd, nCmd)==0 ){
4996 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
4997 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed locale=... directive");
4998 rc = SQLITE_ERROR1;
4999 }else{
5000 pConfig->bLocale = (zArg[0]=='1');
5001 }
5002 return rc;
5003 }
5004
5005 if( sqlite3_strnicmpsqlite3_api->strnicmp("detail", zCmd, nCmd)==0 ){
5006 const Fts5Enum aDetail[] = {
5007 { "none", FTS5_DETAIL_NONE1 },
5008 { "full", FTS5_DETAIL_FULL0 },
5009 { "columns", FTS5_DETAIL_COLUMNS2 },
5010 { 0, 0 }
5011 };
5012
5013 if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){
5014 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed detail=... directive");
5015 }
5016 return rc;
5017 }
5018
5019 if( sqlite3_strnicmpsqlite3_api->strnicmp("tokendata", zCmd, nCmd)==0 ){
5020 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
5021 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed tokendata=... directive");
5022 rc = SQLITE_ERROR1;
5023 }else{
5024 pConfig->bTokendata = (zArg[0]=='1');
5025 }
5026 return rc;
5027 }
5028
5029 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
5030 return SQLITE_ERROR1;
5031}
5032
5033/*
5034** Gobble up the first bareword or quoted word from the input buffer zIn.
5035** Return a pointer to the character immediately following the last in
5036** the gobbled word if successful, or a NULL pointer otherwise (failed
5037** to find close-quote character).
5038**
5039** Before returning, set pzOut to point to a new buffer containing a
5040** nul-terminated, dequoted copy of the gobbled word. If the word was
5041** quoted, *pbQuoted is also set to 1 before returning.
5042**
5043** If *pRc is other than SQLITE_OK when this function is called, it is
5044** a no-op (NULL is returned). Otherwise, if an OOM occurs within this
5045** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not*
5046** set if a parse error (failed to find close quote) occurs.
5047*/
5048static const char *fts5ConfigGobbleWord(
5049 int *pRc, /* IN/OUT: Error code */
5050 const char *zIn, /* Buffer to gobble string/bareword from */
5051 char **pzOut, /* OUT: malloc'd buffer containing str/bw */
5052 int *pbQuoted /* OUT: Set to true if dequoting required */
5053){
5054 const char *zRet = 0;
5055
5056 sqlite3_int64 nIn = strlen(zIn);
5057 char *zOut = sqlite3_malloc64sqlite3_api->malloc64(nIn+1);
5058
5059 assert( *pRc==SQLITE_OK )((void) (0));
5060 *pbQuoted = 0;
5061 *pzOut = 0;
5062
5063 if( zOut==0 ){
5064 *pRc = SQLITE_NOMEM7;
5065 }else{
5066 memcpy(zOut, zIn, (size_t)(nIn+1));
5067 if( fts5_isopenquote(zOut[0]) ){
5068 int ii = fts5Dequote(zOut);
5069 zRet = &zIn[ii];
5070 *pbQuoted = 1;
5071 }else{
5072 zRet = fts5ConfigSkipBareword(zIn);
5073 if( zRet ){
5074 zOut[zRet-zIn] = '\0';
5075 }
5076 }
5077 }
5078
5079 if( zRet==0 ){
5080 sqlite3_freesqlite3_api->free(zOut);
5081 }else{
5082 *pzOut = zOut;
5083 }
5084
5085 return zRet;
5086}
5087
5088static int fts5ConfigParseColumn(
5089 Fts5Config *p,
5090 char *zCol,
5091 char *zArg,
5092 char **pzErr,
5093 int *pbUnindexed
5094){
5095 int rc = SQLITE_OK0;
5096 if( 0==sqlite3_stricmpsqlite3_api->stricmp(zCol, FTS5_RANK_NAME"rank")
5097 || 0==sqlite3_stricmpsqlite3_api->stricmp(zCol, FTS5_ROWID_NAME"rowid")
5098 ){
5099 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("reserved fts5 column name: %s", zCol);
5100 rc = SQLITE_ERROR1;
5101 }else if( zArg ){
5102 if( 0==sqlite3_stricmpsqlite3_api->stricmp(zArg, "unindexed") ){
5103 p->abUnindexed[p->nCol] = 1;
5104 *pbUnindexed = 1;
5105 }else{
5106 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("unrecognized column option: %s", zArg);
5107 rc = SQLITE_ERROR1;
5108 }
5109 }
5110
5111 p->azCol[p->nCol++] = zCol;
5112 return rc;
5113}
5114
5115/*
5116** Populate the Fts5Config.zContentExprlist string.
5117*/
5118static int fts5ConfigMakeExprlist(Fts5Config *p){
5119 int i;
5120 int rc = SQLITE_OK0;
5121 Fts5Buffer buf = {0, 0, 0};
5122
5123 sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid);
5124 if( p->eContent!=FTS5_CONTENT_NONE1 ){
5125 assert( p->eContent==FTS5_CONTENT_EXTERNAL((void) (0))
5126 || p->eContent==FTS5_CONTENT_NORMAL((void) (0))
5127 || p->eContent==FTS5_CONTENT_UNINDEXED((void) (0))
5128 )((void) (0));
5129 for(i=0; i<p->nCol; i++){
5130 if( p->eContent==FTS5_CONTENT_EXTERNAL2 ){
5131 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]);
5132 }else if( p->eContent==FTS5_CONTENT_NORMAL0 || p->abUnindexed[i] ){
5133 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i);
5134 }else{
5135 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
5136 }
5137 }
5138 }
5139 if( p->eContent==FTS5_CONTENT_NORMAL0 && p->bLocale ){
5140 for(i=0; i<p->nCol; i++){
5141 if( p->abUnindexed[i]==0 ){
5142 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i);
5143 }else{
5144 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
5145 }
5146 }
5147 }
5148
5149 assert( p->zContentExprlist==0 )((void) (0));
5150 p->zContentExprlist = (char*)buf.p;
5151 return rc;
5152}
5153
5154/*
5155** Arguments nArg/azArg contain the string arguments passed to the xCreate
5156** or xConnect method of the virtual table. This function attempts to
5157** allocate an instance of Fts5Config containing the results of parsing
5158** those arguments.
5159**
5160** If successful, SQLITE_OK is returned and *ppOut is set to point to the
5161** new Fts5Config object. If an error occurs, an SQLite error code is
5162** returned, *ppOut is set to NULL and an error message may be left in
5163** *pzErr. It is the responsibility of the caller to eventually free any
5164** such error message using sqlite3_free().
5165*/
5166static int sqlite3Fts5ConfigParse(
5167 Fts5Global *pGlobal,
5168 sqlite3 *db,
5169 int nArg, /* Number of arguments */
5170 const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */
5171 Fts5Config **ppOut, /* OUT: Results of parse */
5172 char **pzErr /* OUT: Error message */
5173){
5174 int rc = SQLITE_OK0; /* Return code */
5175 Fts5Config *pRet; /* New object to return */
5176 int i;
5177 sqlite3_int64 nByte;
5178 int bUnindexed = 0; /* True if there are one or more UNINDEXED */
5179
5180 *ppOut = pRet = (Fts5Config*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5Config));
5181 if( pRet==0 ) return SQLITE_NOMEM7;
5182 memset(pRet, 0, sizeof(Fts5Config));
5183 pRet->pGlobal = pGlobal;
5184 pRet->db = db;
5185 pRet->iCookie = -1;
5186
5187 nByte = nArg * (sizeof(char*) + sizeof(u8));
5188 pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
5189 pRet->abUnindexed = pRet->azCol ? (u8*)&pRet->azCol[nArg] : 0;
5190 pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
5191 pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
5192 pRet->bColumnsize = 1;
5193 pRet->eDetail = FTS5_DETAIL_FULL0;
5194#ifdef SQLITE_DEBUG
5195 pRet->bPrefixIndex = 1;
5196#endif
5197 if( rc==SQLITE_OK0 && sqlite3_stricmpsqlite3_api->stricmp(pRet->zName, FTS5_RANK_NAME"rank")==0 ){
5198 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("reserved fts5 table name: %s", pRet->zName);
5199 rc = SQLITE_ERROR1;
5200 }
5201
5202 assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK )((void) (0));
5203 for(i=3; rc==SQLITE_OK0 && i<nArg; i++){
5204 const char *zOrig = azArg[i];
5205 const char *z;
5206 char *zOne = 0;
5207 char *zTwo = 0;
5208 int bOption = 0;
5209 int bMustBeCol = 0;
5210
5211 z = fts5ConfigGobbleWord(&rc, zOrig, &zOne, &bMustBeCol);
5212 z = fts5ConfigSkipWhitespace(z);
5213 if( z && *z=='=' ){
5214 bOption = 1;
5215 assert( zOne!=0 )((void) (0));
5216 z++;
5217 if( bMustBeCol ) z = 0;
5218 }
5219 z = fts5ConfigSkipWhitespace(z);
5220 if( z && z[0] ){
5221 int bDummy;
5222 z = fts5ConfigGobbleWord(&rc, z, &zTwo, &bDummy);
5223 if( z && z[0] ) z = 0;
5224 }
5225
5226 if( rc==SQLITE_OK0 ){
5227 if( z==0 ){
5228 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("parse error in \"%s\"", zOrig);
5229 rc = SQLITE_ERROR1;
5230 }else{
5231 if( bOption ){
5232 rc = fts5ConfigParseSpecial(pRet,
5233 ALWAYS(zOne)(zOne)?zOne:"",
5234 zTwo?zTwo:"",
5235 pzErr
5236 );
5237 }else{
5238 rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr, &bUnindexed);
5239 zOne = 0;
5240 }
5241 }
5242 }
5243
5244 sqlite3_freesqlite3_api->free(zOne);
5245 sqlite3_freesqlite3_api->free(zTwo);
5246 }
5247
5248 /* We only allow contentless_delete=1 if the table is indeed contentless. */
5249 if( rc==SQLITE_OK0
5250 && pRet->bContentlessDelete
5251 && pRet->eContent!=FTS5_CONTENT_NONE1
5252 ){
5253 *pzErr = sqlite3_mprintfsqlite3_api->mprintf(
5254 "contentless_delete=1 requires a contentless table"
5255 );
5256 rc = SQLITE_ERROR1;
5257 }
5258
5259 /* We only allow contentless_delete=1 if columnsize=0 is not present.
5260 **
5261 ** This restriction may be removed at some point.
5262 */
5263 if( rc==SQLITE_OK0 && pRet->bContentlessDelete && pRet->bColumnsize==0 ){
5264 *pzErr = sqlite3_mprintfsqlite3_api->mprintf(
5265 "contentless_delete=1 is incompatible with columnsize=0"
5266 );
5267 rc = SQLITE_ERROR1;
5268 }
5269
5270 /* We only allow contentless_unindexed=1 if the table is actually a
5271 ** contentless one.
5272 */
5273 if( rc==SQLITE_OK0
5274 && pRet->bContentlessUnindexed
5275 && pRet->eContent!=FTS5_CONTENT_NONE1
5276 ){
5277 *pzErr = sqlite3_mprintfsqlite3_api->mprintf(
5278 "contentless_unindexed=1 requires a contentless table"
5279 );
5280 rc = SQLITE_ERROR1;
5281 }
5282
5283 /* If no zContent option was specified, fill in the default values. */
5284 if( rc==SQLITE_OK0 && pRet->zContent==0 ){
5285 const char *zTail = 0;
5286 assert( pRet->eContent==FTS5_CONTENT_NORMAL((void) (0))
5287 || pRet->eContent==FTS5_CONTENT_NONE((void) (0))
5288 )((void) (0));
5289 if( pRet->eContent==FTS5_CONTENT_NORMAL0 ){
5290 zTail = "content";
5291 }else if( bUnindexed && pRet->bContentlessUnindexed ){
5292 pRet->eContent = FTS5_CONTENT_UNINDEXED3;
5293 zTail = "content";
5294 }else if( pRet->bColumnsize ){
5295 zTail = "docsize";
5296 }
5297
5298 if( zTail ){
5299 pRet->zContent = sqlite3Fts5Mprintf(
5300 &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail
5301 );
5302 }
5303 }
5304
5305 if( rc==SQLITE_OK0 && pRet->zContentRowid==0 ){
5306 pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1);
5307 }
5308
5309 /* Formulate the zContentExprlist text */
5310 if( rc==SQLITE_OK0 ){
5311 rc = fts5ConfigMakeExprlist(pRet);
5312 }
5313
5314 if( rc!=SQLITE_OK0 ){
5315 sqlite3Fts5ConfigFree(pRet);
5316 *ppOut = 0;
5317 }
5318 return rc;
5319}
5320
5321/*
5322** Free the configuration object passed as the only argument.
5323*/
5324static void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
5325 if( pConfig ){
5326 int i;
5327 if( pConfig->t.pTok ){
5328 if( pConfig->t.pApi1 ){
5329 pConfig->t.pApi1->xDelete(pConfig->t.pTok);
5330 }else{
5331 pConfig->t.pApi2->xDelete(pConfig->t.pTok);
5332 }
5333 }
5334 sqlite3_freesqlite3_api->free((char*)pConfig->t.azArg);
5335 sqlite3_freesqlite3_api->free(pConfig->zDb);
5336 sqlite3_freesqlite3_api->free(pConfig->zName);
5337 for(i=0; i<pConfig->nCol; i++){
5338 sqlite3_freesqlite3_api->free(pConfig->azCol[i]);
5339 }
5340 sqlite3_freesqlite3_api->free(pConfig->azCol);
5341 sqlite3_freesqlite3_api->free(pConfig->aPrefix);
5342 sqlite3_freesqlite3_api->free(pConfig->zRank);
5343 sqlite3_freesqlite3_api->free(pConfig->zRankArgs);
5344 sqlite3_freesqlite3_api->free(pConfig->zContent);
5345 sqlite3_freesqlite3_api->free(pConfig->zContentRowid);
5346 sqlite3_freesqlite3_api->free(pConfig->zContentExprlist);
5347 sqlite3_freesqlite3_api->free(pConfig);
5348 }
5349}
5350
5351/*
5352** Call sqlite3_declare_vtab() based on the contents of the configuration
5353** object passed as the only argument. Return SQLITE_OK if successful, or
5354** an SQLite error code if an error occurs.
5355*/
5356static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){
5357 int i;
5358 int rc = SQLITE_OK0;
5359 char *zSql;
5360
5361 zSql = sqlite3Fts5Mprintf(&rc, "CREATE TABLE x(");
5362 for(i=0; zSql && i<pConfig->nCol; i++){
5363 const char *zSep = (i==0?"":", ");
5364 zSql = sqlite3Fts5Mprintf(&rc, "%z%s%Q", zSql, zSep, pConfig->azCol[i]);
5365 }
5366 zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)",
5367 zSql, pConfig->zName, FTS5_RANK_NAME"rank"
5368 );
5369
5370 assert( zSql || rc==SQLITE_NOMEM )((void) (0));
5371 if( zSql ){
5372 rc = sqlite3_declare_vtabsqlite3_api->declare_vtab(pConfig->db, zSql);
5373 sqlite3_freesqlite3_api->free(zSql);
5374 }
5375
5376 return rc;
5377}
5378
5379/*
5380** Tokenize the text passed via the second and third arguments.
5381**
5382** The callback is invoked once for each token in the input text. The
5383** arguments passed to it are, in order:
5384**
5385** void *pCtx // Copy of 4th argument to sqlite3Fts5Tokenize()
5386** const char *pToken // Pointer to buffer containing token
5387** int nToken // Size of token in bytes
5388** int iStart // Byte offset of start of token within input text
5389** int iEnd // Byte offset of end of token within input text
5390** int iPos // Position of token in input (first token is 0)
5391**
5392** If the callback returns a non-zero value the tokenization is abandoned
5393** and no further callbacks are issued.
5394**
5395** This function returns SQLITE_OK if successful or an SQLite error code
5396** if an error occurs. If the tokenization was abandoned early because
5397** the callback returned SQLITE_DONE, this is not an error and this function
5398** still returns SQLITE_OK. Or, if the tokenization was abandoned early
5399** because the callback returned another non-zero value, it is assumed
5400** to be an SQLite error code and returned to the caller.
5401*/
5402static int sqlite3Fts5Tokenize(
5403 Fts5Config *pConfig, /* FTS5 Configuration object */
5404 int flags, /* FTS5_TOKENIZE_* flags */
5405 const char *pText, int nText, /* Text to tokenize */
5406 void *pCtx, /* Context passed to xToken() */
5407 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
5408){
5409 int rc = SQLITE_OK0;
5410 if( pText ){
5411 if( pConfig->t.pTok==0 ){
5412 rc = sqlite3Fts5LoadTokenizer(pConfig);
5413 }
5414 if( rc==SQLITE_OK0 ){
5415 if( pConfig->t.pApi1 ){
5416 rc = pConfig->t.pApi1->xTokenize(
5417 pConfig->t.pTok, pCtx, flags, pText, nText, xToken
5418 );
5419 }else{
5420 rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags,
5421 pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken
5422 );
5423 }
5424 }
5425 }
5426 return rc;
5427}
5428
5429/*
5430** Argument pIn points to the first character in what is expected to be
5431** a comma-separated list of SQL literals followed by a ')' character.
5432** If it actually is this, return a pointer to the ')'. Otherwise, return
5433** NULL to indicate a parse error.
5434*/
5435static const char *fts5ConfigSkipArgs(const char *pIn){
5436 const char *p = pIn;
5437
5438 while( 1 ){
5439 p = fts5ConfigSkipWhitespace(p);
5440 p = fts5ConfigSkipLiteral(p);
5441 p = fts5ConfigSkipWhitespace(p);
5442 if( p==0 || *p==')' ) break;
5443 if( *p!=',' ){
5444 p = 0;
5445 break;
5446 }
5447 p++;
5448 }
5449
5450 return p;
5451}
5452
5453/*
5454** Parameter zIn contains a rank() function specification. The format of
5455** this is:
5456**
5457** + Bareword (function name)
5458** + Open parenthesis - "("
5459** + Zero or more SQL literals in a comma separated list
5460** + Close parenthesis - ")"
5461*/
5462static int sqlite3Fts5ConfigParseRank(
5463 const char *zIn, /* Input string */
5464 char **pzRank, /* OUT: Rank function name */
5465 char **pzRankArgs /* OUT: Rank function arguments */
5466){
5467 const char *p = zIn;
5468 const char *pRank;
5469 char *zRank = 0;
5470 char *zRankArgs = 0;
5471 int rc = SQLITE_OK0;
5472
5473 *pzRank = 0;
5474 *pzRankArgs = 0;
5475
5476 if( p==0 ){
5477 rc = SQLITE_ERROR1;
5478 }else{
5479 p = fts5ConfigSkipWhitespace(p);
5480 pRank = p;
5481 p = fts5ConfigSkipBareword(p);
5482
5483 if( p ){
5484 zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
5485 if( zRank ) memcpy(zRank, pRank, p-pRank);
5486 }else{
5487 rc = SQLITE_ERROR1;
5488 }
5489
5490 if( rc==SQLITE_OK0 ){
5491 p = fts5ConfigSkipWhitespace(p);
5492 if( *p!='(' ) rc = SQLITE_ERROR1;
5493 p++;
5494 }
5495 if( rc==SQLITE_OK0 ){
5496 const char *pArgs;
5497 p = fts5ConfigSkipWhitespace(p);
5498 pArgs = p;
5499 if( *p!=')' ){
5500 p = fts5ConfigSkipArgs(p);
5501 if( p==0 ){
5502 rc = SQLITE_ERROR1;
5503 }else{
5504 zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
5505 if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
5506 }
5507 }
5508 }
5509 }
5510
5511 if( rc!=SQLITE_OK0 ){
5512 sqlite3_freesqlite3_api->free(zRank);
5513 assert( zRankArgs==0 )((void) (0));
5514 }else{
5515 *pzRank = zRank;
5516 *pzRankArgs = zRankArgs;
5517 }
5518 return rc;
5519}
5520
5521static int sqlite3Fts5ConfigSetValue(
5522 Fts5Config *pConfig,
5523 const char *zKey,
5524 sqlite3_value *pVal,
5525 int *pbBadkey
5526){
5527 int rc = SQLITE_OK0;
5528
5529 if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "pgsz") ){
5530 int pgsz = 0;
5531 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5532 pgsz = sqlite3_value_intsqlite3_api->value_int(pVal);
5533 }
5534 if( pgsz<32 || pgsz>FTS5_MAX_PAGE_SIZE(64*1024) ){
5535 *pbBadkey = 1;
5536 }else{
5537 pConfig->pgsz = pgsz;
5538 }
5539 }
5540
5541 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "hashsize") ){
5542 int nHashSize = -1;
5543 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5544 nHashSize = sqlite3_value_intsqlite3_api->value_int(pVal);
5545 }
5546 if( nHashSize<=0 ){
5547 *pbBadkey = 1;
5548 }else{
5549 pConfig->nHashSize = nHashSize;
5550 }
5551 }
5552
5553 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "automerge") ){
5554 int nAutomerge = -1;
5555 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5556 nAutomerge = sqlite3_value_intsqlite3_api->value_int(pVal);
5557 }
5558 if( nAutomerge<0 || nAutomerge>64 ){
5559 *pbBadkey = 1;
5560 }else{
5561 if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE4;
5562 pConfig->nAutomerge = nAutomerge;
5563 }
5564 }
5565
5566 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "usermerge") ){
5567 int nUsermerge = -1;
5568 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5569 nUsermerge = sqlite3_value_intsqlite3_api->value_int(pVal);
5570 }
5571 if( nUsermerge<2 || nUsermerge>16 ){
5572 *pbBadkey = 1;
5573 }else{
5574 pConfig->nUsermerge = nUsermerge;
5575 }
5576 }
5577
5578 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "crisismerge") ){
5579 int nCrisisMerge = -1;
5580 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5581 nCrisisMerge = sqlite3_value_intsqlite3_api->value_int(pVal);
5582 }
5583 if( nCrisisMerge<0 ){
5584 *pbBadkey = 1;
5585 }else{
5586 if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE16;
5587 if( nCrisisMerge>=FTS5_MAX_SEGMENT2000 ) nCrisisMerge = FTS5_MAX_SEGMENT2000-1;
5588 pConfig->nCrisisMerge = nCrisisMerge;
5589 }
5590 }
5591
5592 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "deletemerge") ){
5593 int nVal = -1;
5594 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5595 nVal = sqlite3_value_intsqlite3_api->value_int(pVal);
5596 }else{
5597 *pbBadkey = 1;
5598 }
5599 if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE10;
5600 if( nVal>100 ) nVal = 0;
5601 pConfig->nDeleteMerge = nVal;
5602 }
5603
5604 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "rank") ){
5605 const char *zIn = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
5606 char *zRank;
5607 char *zRankArgs;
5608 rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs);
5609 if( rc==SQLITE_OK0 ){
5610 sqlite3_freesqlite3_api->free(pConfig->zRank);
5611 sqlite3_freesqlite3_api->free(pConfig->zRankArgs);
5612 pConfig->zRank = zRank;
5613 pConfig->zRankArgs = zRankArgs;
5614 }else if( rc==SQLITE_ERROR1 ){
5615 rc = SQLITE_OK0;
5616 *pbBadkey = 1;
5617 }
5618 }
5619
5620 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "secure-delete") ){
5621 int bVal = -1;
5622 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5623 bVal = sqlite3_value_intsqlite3_api->value_int(pVal);
5624 }
5625 if( bVal<0 ){
5626 *pbBadkey = 1;
5627 }else{
5628 pConfig->bSecureDelete = (bVal ? 1 : 0);
5629 }
5630 }
5631
5632 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "insttoken") ){
5633 int bVal = -1;
5634 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5635 bVal = sqlite3_value_intsqlite3_api->value_int(pVal);
5636 }
5637 if( bVal<0 ){
5638 *pbBadkey = 1;
5639 }else{
5640 pConfig->bPrefixInsttoken = (bVal ? 1 : 0);
5641 }
5642
5643 }else{
5644 *pbBadkey = 1;
5645 }
5646 return rc;
5647}
5648
5649/*
5650** Load the contents of the %_config table into memory.
5651*/
5652static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
5653 const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
5654 char *zSql;
5655 sqlite3_stmt *p = 0;
5656 int rc = SQLITE_OK0;
5657 int iVersion = 0;
5658
5659 /* Set default values */
5660 pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE4050;
5661 pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE4;
5662 pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE4;
5663 pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE16;
5664 pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE(1024*1024);
5665 pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE10;
5666
5667 zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
5668 if( zSql ){
5669 rc = sqlite3_prepare_v2sqlite3_api->prepare_v2(pConfig->db, zSql, -1, &p, 0);
5670 sqlite3_freesqlite3_api->free(zSql);
5671 }
5672
5673 assert( rc==SQLITE_OK || p==0 )((void) (0));
5674 if( rc==SQLITE_OK0 ){
5675 while( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(p) ){
5676 const char *zK = (const char*)sqlite3_column_textsqlite3_api->column_text(p, 0);
5677 sqlite3_value *pVal = sqlite3_column_valuesqlite3_api->column_value(p, 1);
5678 if( 0==sqlite3_stricmpsqlite3_api->stricmp(zK, "version") ){
5679 iVersion = sqlite3_value_intsqlite3_api->value_int(pVal);
5680 }else{
5681 int bDummy = 0;
5682 sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, &bDummy);
5683 }
5684 }
5685 rc = sqlite3_finalizesqlite3_api->finalize(p);
5686 }
5687
5688 if( rc==SQLITE_OK0
5689 && iVersion!=FTS5_CURRENT_VERSION4
5690 && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE5
5691 ){
5692 rc = SQLITE_ERROR1;
5693 sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format "
5694 "(found %d, expected %d or %d) - run 'rebuild'",
5695 iVersion, FTS5_CURRENT_VERSION4, FTS5_CURRENT_VERSION_SECUREDELETE5
5696 );
5697 }else{
5698 pConfig->iVersion = iVersion;
5699 }
5700
5701 if( rc==SQLITE_OK0 ){
5702 pConfig->iCookie = iCookie;
5703 }
5704 return rc;
5705}
5706
5707/*
5708** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer
5709** containing the error message created using printf() style formatting
5710** string zFmt and its trailing arguments.
5711*/
5712static void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){
5713 va_list ap; /* ... printf arguments */
5714 char *zMsg = 0;
5715
5716 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
5717 zMsg = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
5718 if( pConfig->pzErrmsg ){
5719 assert( *pConfig->pzErrmsg==0 )((void) (0));
5720 *pConfig->pzErrmsg = zMsg;
5721 }else{
5722 sqlite3_freesqlite3_api->free(zMsg);
5723 }
5724
5725 va_end(ap)__builtin_va_end(ap);
5726}
5727
5728#line 1 "fts5_expr.c"
5729/*
5730** 2014 May 31
5731**
5732** The author disclaims copyright to this source code. In place of
5733** a legal notice, here is a blessing:
5734**
5735** May you do good and not evil.
5736** May you find forgiveness for yourself and forgive others.
5737** May you share freely, never taking more than you give.
5738**
5739******************************************************************************
5740**
5741*/
5742
5743
5744
5745/* #include "fts5Int.h" */
5746/* #include "fts5parse.h" */
5747
5748#ifndef SQLITE_FTS5_MAX_EXPR_DEPTH256
5749# define SQLITE_FTS5_MAX_EXPR_DEPTH256 256
5750#endif
5751
5752/*
5753** All token types in the generated fts5parse.h file are greater than 0.
5754*/
5755#define FTS5_EOF0 0
5756
5757#define FTS5_LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)) (0xffffffff|(((i64)0x7fffffff)<<32))
5758
5759typedef struct Fts5ExprTerm Fts5ExprTerm;
5760
5761/*
5762** Functions generated by lemon from fts5parse.y.
5763*/
5764static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(u64));
5765static void sqlite3Fts5ParserFree(void*, void (*freeProc)(void*));
5766static void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);
5767#ifndef NDEBUG1
5768#include <stdio.h>
5769static void sqlite3Fts5ParserTrace(FILE*, char*);
5770#endif
5771static int sqlite3Fts5ParserFallback(int);
5772
5773
5774struct Fts5Expr {
5775 Fts5Index *pIndex;
5776 Fts5Config *pConfig;
5777 Fts5ExprNode *pRoot;
5778 int bDesc; /* Iterate in descending rowid order */
5779 int nPhrase; /* Number of phrases in expression */
5780 Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */
5781};
5782
5783/*
5784** eType:
5785** Expression node type. Usually one of:
5786**
5787** FTS5_AND (nChild, apChild valid)
5788** FTS5_OR (nChild, apChild valid)
5789** FTS5_NOT (nChild, apChild valid)
5790** FTS5_STRING (pNear valid)
5791** FTS5_TERM (pNear valid)
5792**
5793** An expression node with eType==0 may also exist. It always matches zero
5794** rows. This is created when a phrase containing no tokens is parsed.
5795** e.g. "".
5796**
5797** iHeight:
5798** Distance from this node to furthest leaf. This is always 0 for nodes
5799** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one
5800** greater than the largest child value.
5801*/
5802struct Fts5ExprNode {
5803 int eType; /* Node type */
5804 int bEof; /* True at EOF */
5805 int bNomatch; /* True if entry is not a match */
5806 int iHeight; /* Distance to tree leaf nodes */
5807
5808 /* Next method for this node. */
5809 int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64);
5810
5811 i64 iRowid; /* Current rowid */
5812 Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */
5813
5814 /* Child nodes. For a NOT node, this array always contains 2 entries. For
5815 ** AND or OR nodes, it contains 2 or more entries. */
5816 int nChild; /* Number of child nodes */
5817 Fts5ExprNode *apChild[FLEXARRAY]; /* Array of child nodes */
5818};
5819
5820/* Size (in bytes) of an Fts5ExprNode object that holds up to N children */
5821#define SZ_FTS5EXPRNODE(N)(__builtin_offsetof(Fts5ExprNode, apChild) + (N)*sizeof(Fts5ExprNode
*)) \
5822 (offsetof(Fts5ExprNode,apChild)__builtin_offsetof(Fts5ExprNode, apChild) + (N)*sizeof(Fts5ExprNode*))
5823
5824#define Fts5NodeIsString(p)((p)->eType==4 || (p)->eType==9) ((p)->eType==FTS5_TERM4 || (p)->eType==FTS5_STRING9)
5825
5826/*
5827** Invoke the xNext method of an Fts5ExprNode object. This macro should be
5828** used as if it has the same signature as the xNext() methods themselves.
5829*/
5830#define fts5ExprNodeNext(a,b,c,d)(b)->xNext((a), (b), (c), (d)) (b)->xNext((a), (b), (c), (d))
5831
5832/*
5833** An instance of the following structure represents a single search term
5834** or term prefix.
5835*/
5836struct Fts5ExprTerm {
5837 u8 bPrefix; /* True for a prefix term */
5838 u8 bFirst; /* True if token must be first in column */
5839 char *pTerm; /* Term data */
5840 int nQueryTerm; /* Effective size of term in bytes */
5841 int nFullTerm; /* Size of term in bytes incl. tokendata */
5842 Fts5IndexIter *pIter; /* Iterator for this term */
5843 Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */
5844};
5845
5846/*
5847** A phrase. One or more terms that must appear in a contiguous sequence
5848** within a document for it to match.
5849*/
5850struct Fts5ExprPhrase {
5851 Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */
5852 Fts5Buffer poslist; /* Current position list */
5853 int nTerm; /* Number of entries in aTerm[] */
5854 Fts5ExprTerm aTerm[FLEXARRAY]; /* Terms that make up this phrase */
5855};
5856
5857/* Size (in bytes) of an Fts5ExprPhrase object that holds up to N terms */
5858#define SZ_FTS5EXPRPHRASE(N)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (N)*sizeof(Fts5ExprTerm
)) \
5859 (offsetof(Fts5ExprPhrase,aTerm)__builtin_offsetof(Fts5ExprPhrase, aTerm) + (N)*sizeof(Fts5ExprTerm))
5860
5861/*
5862** One or more phrases that must appear within a certain token distance of
5863** each other within each matching document.
5864*/
5865struct Fts5ExprNearset {
5866 int nNear; /* NEAR parameter */
5867 Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */
5868 int nPhrase; /* Number of entries in aPhrase[] array */
5869 Fts5ExprPhrase *apPhrase[FLEXARRAY]; /* Array of phrase pointers */
5870};
5871
5872/* Size (in bytes) of an Fts5ExprNearset object covering up to N phrases */
5873#define SZ_FTS5EXPRNEARSET(N)(__builtin_offsetof(Fts5ExprNearset, apPhrase)+(N)*sizeof(Fts5ExprPhrase
*)) \
5874 (offsetof(Fts5ExprNearset,apPhrase)__builtin_offsetof(Fts5ExprNearset, apPhrase)+(N)*sizeof(Fts5ExprPhrase*))
5875
5876/*
5877** Parse context.
5878*/
5879struct Fts5Parse {
5880 Fts5Config *pConfig;
5881 char *zErr;
5882 int rc;
5883 int nPhrase; /* Size of apPhrase array */
5884 Fts5ExprPhrase **apPhrase; /* Array of all phrases */
5885 Fts5ExprNode *pExpr; /* Result of a successful parse */
5886 int bPhraseToAnd; /* Convert "a+b" to "a AND b" */
5887};
5888
5889/*
5890** Check that the Fts5ExprNode.iHeight variables are set correctly in
5891** the expression tree passed as the only argument.
5892*/
5893#ifndef NDEBUG1
5894static void assert_expr_depth_ok(int rc, Fts5ExprNode *p){
5895 if( rc==SQLITE_OK0 ){
5896 if( p->eType==FTS5_TERM4 || p->eType==FTS5_STRING9 || p->eType==0 ){
5897 assert( p->iHeight==0 )((void) (0));
5898 }else{
5899 int ii;
5900 int iMaxChild = 0;
5901 for(ii=0; ii<p->nChild; ii++){
5902 Fts5ExprNode *pChild = p->apChild[ii];
5903 iMaxChild = MAX(iMaxChild, pChild->iHeight)(((iMaxChild) > (pChild->iHeight)) ? (iMaxChild) : (pChild
->iHeight));
5904 assert_expr_depth_ok(SQLITE_OK, pChild);
5905 }
5906 assert( p->iHeight==iMaxChild+1 )((void) (0));
5907 }
5908 }
5909}
5910#else
5911# define assert_expr_depth_ok(rc, p)
5912#endif
5913
5914static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
5915 va_list ap;
5916 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
5917 if( pParse->rc==SQLITE_OK0 ){
5918 assert( pParse->zErr==0 )((void) (0));
5919 pParse->zErr = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
5920 pParse->rc = SQLITE_ERROR1;
5921 }
5922 va_end(ap)__builtin_va_end(ap);
5923}
5924
5925static int fts5ExprIsspace(char t){
5926 return t==' ' || t=='\t' || t=='\n' || t=='\r';
5927}
5928
5929/*
5930** Read the first token from the nul-terminated string at *pz.
5931*/
5932static int fts5ExprGetToken(
5933 Fts5Parse *pParse,
5934 const char **pz, /* IN/OUT: Pointer into buffer */
5935 Fts5Token *pToken
5936){
5937 const char *z = *pz;
5938 int tok;
5939
5940 /* Skip past any whitespace */
5941 while( fts5ExprIsspace(*z) ) z++;
5942
5943 pToken->p = z;
5944 pToken->n = 1;
5945 switch( *z ){
5946 case '(': tok = FTS5_LP10; break;
5947 case ')': tok = FTS5_RP11; break;
5948 case '{': tok = FTS5_LCP7; break;
5949 case '}': tok = FTS5_RCP8; break;
5950 case ':': tok = FTS5_COLON5; break;
5951 case ',': tok = FTS5_COMMA13; break;
5952 case '+': tok = FTS5_PLUS14; break;
5953 case '*': tok = FTS5_STAR15; break;
5954 case '-': tok = FTS5_MINUS6; break;
5955 case '^': tok = FTS5_CARET12; break;
5956 case '\0': tok = FTS5_EOF0; break;
5957
5958 case '"': {
5959 const char *z2;
5960 tok = FTS5_STRING9;
5961
5962 for(z2=&z[1]; 1; z2++){
5963 if( z2[0]=='"' ){
5964 z2++;
5965 if( z2[0]!='"' ) break;
5966 }
5967 if( z2[0]=='\0' ){
5968 sqlite3Fts5ParseError(pParse, "unterminated string");
5969 return FTS5_EOF0;
5970 }
5971 }
5972 pToken->n = (z2 - z);
5973 break;
5974 }
5975
5976 default: {
5977 const char *z2;
5978 if( sqlite3Fts5IsBareword(z[0])==0 ){
5979 sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z);
5980 return FTS5_EOF0;
5981 }
5982 tok = FTS5_STRING9;
5983 for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++);
5984 pToken->n = (z2 - z);
5985 if( pToken->n==2 && memcmp(pToken->p, "OR", 2)==0 ) tok = FTS5_OR1;
5986 if( pToken->n==3 && memcmp(pToken->p, "NOT", 3)==0 ) tok = FTS5_NOT3;
5987 if( pToken->n==3 && memcmp(pToken->p, "AND", 3)==0 ) tok = FTS5_AND2;
5988 break;
5989 }
5990 }
5991
5992 *pz = &pToken->p[pToken->n];
5993 return tok;
5994}
5995
5996static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc64sqlite3_api->malloc64((sqlite3_int64)t);}
5997static void fts5ParseFree(void *p){ sqlite3_freesqlite3_api->free(p); }
5998
5999static int sqlite3Fts5ExprNew(
6000 Fts5Config *pConfig, /* FTS5 Configuration */
6001 int bPhraseToAnd,
6002 int iCol,
6003 const char *zExpr, /* Expression text */
6004 Fts5Expr **ppNew,
6005 char **pzErr
6006){
6007 Fts5Parse sParse;
6008 Fts5Token token;
6009 const char *z = zExpr;
6010 int t; /* Next token type */
6011 void *pEngine;
6012 Fts5Expr *pNew;
6013
6014 *ppNew = 0;
6015 *pzErr = 0;
6016 memset(&sParse, 0, sizeof(sParse));
6017 sParse.bPhraseToAnd = bPhraseToAnd;
6018 pEngine = sqlite3Fts5ParserAlloc(fts5ParseAlloc);
6019 if( pEngine==0 ){ return SQLITE_NOMEM7; }
6020 sParse.pConfig = pConfig;
6021
6022 do {
6023 t = fts5ExprGetToken(&sParse, &z, &token);
6024 sqlite3Fts5Parser(pEngine, t, token, &sParse);
6025 }while( sParse.rc==SQLITE_OK0 && t!=FTS5_EOF0 );
6026 sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
6027
6028 assert( sParse.pExpr || sParse.rc!=SQLITE_OK )((void) (0));
6029 assert_expr_depth_ok(sParse.rc, sParse.pExpr);
6030
6031 /* If the LHS of the MATCH expression was a user column, apply the
6032 ** implicit column-filter. */
6033 if( sParse.rc==SQLITE_OK0 && iCol<pConfig->nCol ){
6034 int n = SZ_FTS5COLSET(1)(sizeof(i64)*((1 +2)/2));
6035 Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
6036 if( pColset ){
6037 pColset->nCol = 1;
6038 pColset->aiCol[0] = iCol;
6039 sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset);
6040 }
6041 }
6042
6043 assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 )((void) (0));
6044 if( sParse.rc==SQLITE_OK0 ){
6045 *ppNew = pNew = sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5Expr));
6046 if( pNew==0 ){
6047 sParse.rc = SQLITE_NOMEM7;
6048 sqlite3Fts5ParseNodeFree(sParse.pExpr);
6049 }else{
6050 pNew->pRoot = sParse.pExpr;
6051 pNew->pIndex = 0;
6052 pNew->pConfig = pConfig;
6053 pNew->apExprPhrase = sParse.apPhrase;
6054 pNew->nPhrase = sParse.nPhrase;
6055 pNew->bDesc = 0;
6056 sParse.apPhrase = 0;
6057 }
6058 }else{
6059 sqlite3Fts5ParseNodeFree(sParse.pExpr);
6060 }
6061
6062 sqlite3_freesqlite3_api->free(sParse.apPhrase);
6063 if( 0==*pzErr ){
6064 *pzErr = sParse.zErr;
6065 }else{
6066 sqlite3_freesqlite3_api->free(sParse.zErr);
6067 }
6068 return sParse.rc;
6069}
6070
6071/*
6072** Assuming that buffer z is at least nByte bytes in size and contains a
6073** valid utf-8 string, return the number of characters in the string.
6074*/
6075static int fts5ExprCountChar(const char *z, int nByte){
6076 int nRet = 0;
6077 int ii;
6078 for(ii=0; ii<nByte; ii++){
6079 if( (z[ii] & 0xC0)!=0x80 ) nRet++;
6080 }
6081 return nRet;
6082}
6083
6084/*
6085** This function is only called when using the special 'trigram' tokenizer.
6086** Argument zText contains the text of a LIKE or GLOB pattern matched
6087** against column iCol. This function creates and compiles an FTS5 MATCH
6088** expression that will match a superset of the rows matched by the LIKE or
6089** GLOB. If successful, SQLITE_OK is returned. Otherwise, an SQLite error
6090** code.
6091*/
6092static int sqlite3Fts5ExprPattern(
6093 Fts5Config *pConfig, int bGlob, int iCol, const char *zText, Fts5Expr **pp
6094){
6095 i64 nText = strlen(zText);
6096 char *zExpr = (char*)sqlite3_malloc64sqlite3_api->malloc64(nText*4 + 1);
6097 int rc = SQLITE_OK0;
6098
6099 if( zExpr==0 ){
6100 rc = SQLITE_NOMEM7;
6101 }else{
6102 char aSpec[3];
6103 int iOut = 0;
6104 int i = 0;
6105 int iFirst = 0;
6106
6107 if( bGlob==0 ){
6108 aSpec[0] = '_';
6109 aSpec[1] = '%';
6110 aSpec[2] = 0;
6111 }else{
6112 aSpec[0] = '*';
6113 aSpec[1] = '?';
6114 aSpec[2] = '[';
6115 }
6116
6117 while( i<=nText ){
6118 if( i==nText
6119 || zText[i]==aSpec[0] || zText[i]==aSpec[1] || zText[i]==aSpec[2]
6120 ){
6121
6122 if( fts5ExprCountChar(&zText[iFirst], i-iFirst)>=3 ){
6123 int jj;
6124 zExpr[iOut++] = '"';
6125 for(jj=iFirst; jj<i; jj++){
6126 zExpr[iOut++] = zText[jj];
6127 if( zText[jj]=='"' ) zExpr[iOut++] = '"';
6128 }
6129 zExpr[iOut++] = '"';
6130 zExpr[iOut++] = ' ';
6131 }
6132 if( zText[i]==aSpec[2] ){
6133 i += 2;
6134 if( zText[i-1]=='^' ) i++;
6135 while( i<nText && zText[i]!=']' ) i++;
6136 }
6137 iFirst = i+1;
6138 }
6139 i++;
6140 }
6141 if( iOut>0 ){
6142 int bAnd = 0;
6143 if( pConfig->eDetail!=FTS5_DETAIL_FULL0 ){
6144 bAnd = 1;
6145 if( pConfig->eDetail==FTS5_DETAIL_NONE1 ){
6146 iCol = pConfig->nCol;
6147 }
6148 }
6149 zExpr[iOut] = '\0';
6150 rc = sqlite3Fts5ExprNew(pConfig, bAnd, iCol, zExpr, pp,pConfig->pzErrmsg);
6151 }else{
6152 *pp = 0;
6153 }
6154 sqlite3_freesqlite3_api->free(zExpr);
6155 }
6156
6157 return rc;
6158}
6159
6160/*
6161** Free the expression node object passed as the only argument.
6162*/
6163static void sqlite3Fts5ParseNodeFree(Fts5ExprNode *p){
6164 if( p ){
6165 int i;
6166 for(i=0; i<p->nChild; i++){
6167 sqlite3Fts5ParseNodeFree(p->apChild[i]);
6168 }
6169 sqlite3Fts5ParseNearsetFree(p->pNear);
6170 sqlite3_freesqlite3_api->free(p);
6171 }
6172}
6173
6174/*
6175** Free the expression object passed as the only argument.
6176*/
6177static void sqlite3Fts5ExprFree(Fts5Expr *p){
6178 if( p ){
6179 sqlite3Fts5ParseNodeFree(p->pRoot);
6180 sqlite3_freesqlite3_api->free(p->apExprPhrase);
6181 sqlite3_freesqlite3_api->free(p);
6182 }
6183}
6184
6185static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
6186 Fts5Parse sParse;
6187 memset(&sParse, 0, sizeof(sParse));
6188
6189 if( *pp1 && p2 ){
6190 Fts5Expr *p1 = *pp1;
6191 int nPhrase = p1->nPhrase + p2->nPhrase;
6192
6193 p1->pRoot = sqlite3Fts5ParseNode(&sParse, FTS5_AND2, p1->pRoot, p2->pRoot,0);
6194 p2->pRoot = 0;
6195
6196 if( sParse.rc==SQLITE_OK0 ){
6197 Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc64sqlite3_api->realloc64(
6198 p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*)
6199 );
6200 if( ap==0 ){
6201 sParse.rc = SQLITE_NOMEM7;
6202 }else{
6203 int i;
6204 memmove(&ap[p2->nPhrase], ap, p1->nPhrase*sizeof(Fts5ExprPhrase*));
6205 for(i=0; i<p2->nPhrase; i++){
6206 ap[i] = p2->apExprPhrase[i];
6207 }
6208 p1->nPhrase = nPhrase;
6209 p1->apExprPhrase = ap;
6210 }
6211 }
6212 sqlite3_freesqlite3_api->free(p2->apExprPhrase);
6213 sqlite3_freesqlite3_api->free(p2);
6214 }else if( p2 ){
6215 *pp1 = p2;
6216 }
6217
6218 return sParse.rc;
6219}
6220
6221/*
6222** Argument pTerm must be a synonym iterator. Return the current rowid
6223** that it points to.
6224*/
6225static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
6226 i64 iRet = 0;
6227 int bRetValid = 0;
6228 Fts5ExprTerm *p;
6229
6230 assert( pTerm )((void) (0));
6231 assert( pTerm->pSynonym )((void) (0));
6232 assert( bDesc==0 || bDesc==1 )((void) (0));
6233 for(p=pTerm; p; p=p->pSynonym){
6234 if( 0==sqlite3Fts5IterEof(p->pIter)((p->pIter)->bEof) ){
6235 i64 iRowid = p->pIter->iRowid;
6236 if( bRetValid==0 || (bDesc!=(iRowid<iRet)) ){
6237 iRet = iRowid;
6238 bRetValid = 1;
6239 }
6240 }
6241 }
6242
6243 if( pbEof && bRetValid==0 ) *pbEof = 1;
6244 return iRet;
6245}
6246
6247/*
6248** Argument pTerm must be a synonym iterator.
6249*/
6250static int fts5ExprSynonymList(
6251 Fts5ExprTerm *pTerm,
6252 i64 iRowid,
6253 Fts5Buffer *pBuf, /* Use this buffer for space if required */
6254 u8 **pa, int *pn
6255){
6256 Fts5PoslistReader aStatic[4];
6257 Fts5PoslistReader *aIter = aStatic;
6258 int nIter = 0;
6259 int nAlloc = 4;
6260 int rc = SQLITE_OK0;
6261 Fts5ExprTerm *p;
6262
6263 assert( pTerm->pSynonym )((void) (0));
6264 for(p=pTerm; p; p=p->pSynonym){
6265 Fts5IndexIter *pIter = p->pIter;
6266 if( sqlite3Fts5IterEof(pIter)((pIter)->bEof)==0 && pIter->iRowid==iRowid ){
6267 if( pIter->nData==0 ) continue;
6268 if( nIter==nAlloc ){
6269 sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
6270 Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
6271 if( aNew==0 ){
6272 rc = SQLITE_NOMEM7;
6273 goto synonym_poslist_out;
6274 }
6275 memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter);
6276 nAlloc = nAlloc*2;
6277 if( aIter!=aStatic ) sqlite3_freesqlite3_api->free(aIter);
6278 aIter = aNew;
6279 }
6280 sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]);
6281 assert( aIter[nIter].bEof==0 )((void) (0));
6282 nIter++;
6283 }
6284 }
6285
6286 if( nIter==1 ){
6287 *pa = (u8*)aIter[0].a;
6288 *pn = aIter[0].n;
6289 }else{
6290 Fts5PoslistWriter writer = {0};
6291 i64 iPrev = -1;
6292 fts5BufferZero(pBuf)sqlite3Fts5BufferZero(pBuf);
6293 while( 1 ){
6294 int i;
6295 i64 iMin = FTS5_LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32));
6296 for(i=0; i<nIter; i++){
6297 if( aIter[i].bEof==0 ){
6298 if( aIter[i].iPos==iPrev ){
6299 if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) continue;
6300 }
6301 if( aIter[i].iPos<iMin ){
6302 iMin = aIter[i].iPos;
6303 }
6304 }
6305 }
6306 if( iMin==FTS5_LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)) || rc!=SQLITE_OK0 ) break;
6307 rc = sqlite3Fts5PoslistWriterAppend(pBuf, &writer, iMin);
6308 iPrev = iMin;
6309 }
6310 if( rc==SQLITE_OK0 ){
6311 *pa = pBuf->p;
6312 *pn = pBuf->n;
6313 }
6314 }
6315
6316 synonym_poslist_out:
6317 if( aIter!=aStatic ) sqlite3_freesqlite3_api->free(aIter);
6318 return rc;
6319}
6320
6321
6322/*
6323** All individual term iterators in pPhrase are guaranteed to be valid and
6324** pointing to the same rowid when this function is called. This function
6325** checks if the current rowid really is a match, and if so populates
6326** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch
6327** is set to true if this is really a match, or false otherwise.
6328**
6329** SQLITE_OK is returned if an error occurs, or an SQLite error code
6330** otherwise. It is not considered an error code if the current rowid is
6331** not a match.
6332*/
6333static int fts5ExprPhraseIsMatch(
6334 Fts5ExprNode *pNode, /* Node pPhrase belongs to */
6335 Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */
6336 int *pbMatch /* OUT: Set to true if really a match */
6337){
6338 Fts5PoslistWriter writer = {0};
6339 Fts5PoslistReader aStatic[4];
6340 Fts5PoslistReader *aIter = aStatic;
6341 int i;
6342 int rc = SQLITE_OK0;
6343 int bFirst = pPhrase->aTerm[0].bFirst;
6344
6345 fts5BufferZero(&pPhrase->poslist)sqlite3Fts5BufferZero(&pPhrase->poslist);
6346
6347 /* If the aStatic[] array is not large enough, allocate a large array
6348 ** using sqlite3_malloc(). This approach could be improved upon. */
6349 if( pPhrase->nTerm>ArraySize(aStatic)((int)(sizeof(aStatic) / sizeof(aStatic[0]))) ){
6350 sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
6351 aIter = (Fts5PoslistReader*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
6352 if( !aIter ) return SQLITE_NOMEM7;
6353 }
6354 memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm);
6355
6356 /* Initialize a term iterator for each term in the phrase */
6357 for(i=0; i<pPhrase->nTerm; i++){
6358 Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
6359 int n = 0;
6360 int bFlag = 0;
6361 u8 *a = 0;
6362 if( pTerm->pSynonym ){
6363 Fts5Buffer buf = {0, 0, 0};
6364 rc = fts5ExprSynonymList(pTerm, pNode->iRowid, &buf, &a, &n);
6365 if( rc ){
6366 sqlite3_freesqlite3_api->free(a);
6367 goto ismatch_out;
6368 }
6369 if( a==buf.p ) bFlag = 1;
6370 }else{
6371 a = (u8*)pTerm->pIter->pData;
6372 n = pTerm->pIter->nData;
6373 }
6374 sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
6375 aIter[i].bFlag = (u8)bFlag;
6376 if( aIter[i].bEof ) goto ismatch_out;
6377 }
6378
6379 while( 1 ){
6380 int bMatch;
6381 i64 iPos = aIter[0].iPos;
6382 do {
6383 bMatch = 1;
6384 for(i=0; i<pPhrase->nTerm; i++){
6385 Fts5PoslistReader *pPos = &aIter[i];
6386 i64 iAdj = iPos + i;
6387 if( pPos->iPos!=iAdj ){
6388 bMatch = 0;
6389 while( pPos->iPos<iAdj ){
6390 if( sqlite3Fts5PoslistReaderNext(pPos) ) goto ismatch_out;
6391 }
6392 if( pPos->iPos>iAdj ) iPos = pPos->iPos-i;
6393 }
6394 }
6395 }while( bMatch==0 );
6396
6397 /* Append position iPos to the output */
6398 if( bFirst==0 || FTS5_POS2OFFSET(iPos)(int)(iPos & 0x7FFFFFFF)==0 ){
6399 rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
6400 if( rc!=SQLITE_OK0 ) goto ismatch_out;
6401 }
6402
6403 for(i=0; i<pPhrase->nTerm; i++){
6404 if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
6405 }
6406 }
6407
6408 ismatch_out:
6409 *pbMatch = (pPhrase->poslist.n>0);
6410 for(i=0; i<pPhrase->nTerm; i++){
6411 if( aIter[i].bFlag ) sqlite3_freesqlite3_api->free((u8*)aIter[i].a);
6412 }
6413 if( aIter!=aStatic ) sqlite3_freesqlite3_api->free(aIter);
6414 return rc;
6415}
6416
6417typedef struct Fts5LookaheadReader Fts5LookaheadReader;
6418struct Fts5LookaheadReader {
6419 const u8 *a; /* Buffer containing position list */
6420 int n; /* Size of buffer a[] in bytes */
6421 int i; /* Current offset in position list */
6422 i64 iPos; /* Current position */
6423 i64 iLookahead; /* Next position */
6424};
6425
6426#define FTS5_LOOKAHEAD_EOF(((i64)1) << 62) (((i64)1) << 62)
6427
6428static int fts5LookaheadReaderNext(Fts5LookaheadReader *p){
6429 p->iPos = p->iLookahead;
6430 if( sqlite3Fts5PoslistNext64(p->a, p->n, &p->i, &p->iLookahead) ){
6431 p->iLookahead = FTS5_LOOKAHEAD_EOF(((i64)1) << 62);
6432 }
6433 return (p->iPos==FTS5_LOOKAHEAD_EOF(((i64)1) << 62));
6434}
6435
6436static int fts5LookaheadReaderInit(
6437 const u8 *a, int n, /* Buffer to read position list from */
6438 Fts5LookaheadReader *p /* Iterator object to initialize */
6439){
6440 memset(p, 0, sizeof(Fts5LookaheadReader));
6441 p->a = a;
6442 p->n = n;
6443 fts5LookaheadReaderNext(p);
6444 return fts5LookaheadReaderNext(p);
6445}
6446
6447typedef struct Fts5NearTrimmer Fts5NearTrimmer;
6448struct Fts5NearTrimmer {
6449 Fts5LookaheadReader reader; /* Input iterator */
6450 Fts5PoslistWriter writer; /* Writer context */
6451 Fts5Buffer *pOut; /* Output poslist */
6452};
6453
6454/*
6455** The near-set object passed as the first argument contains more than
6456** one phrase. All phrases currently point to the same row. The
6457** Fts5ExprPhrase.poslist buffers are populated accordingly. This function
6458** tests if the current row contains instances of each phrase sufficiently
6459** close together to meet the NEAR constraint. Non-zero is returned if it
6460** does, or zero otherwise.
6461**
6462** If in/out parameter (*pRc) is set to other than SQLITE_OK when this
6463** function is called, it is a no-op. Or, if an error (e.g. SQLITE_NOMEM)
6464** occurs within this function (*pRc) is set accordingly before returning.
6465** The return value is undefined in both these cases.
6466**
6467** If no error occurs and non-zero (a match) is returned, the position-list
6468** of each phrase object is edited to contain only those entries that
6469** meet the constraint before returning.
6470*/
6471static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){
6472 Fts5NearTrimmer aStatic[4];
6473 Fts5NearTrimmer *a = aStatic;
6474 Fts5ExprPhrase **apPhrase = pNear->apPhrase;
6475
6476 int i;
6477 int rc = *pRc;
6478 int bMatch;
6479
6480 assert( pNear->nPhrase>1 )((void) (0));
6481
6482 /* If the aStatic[] array is not large enough, allocate a large array
6483 ** using sqlite3_malloc(). This approach could be improved upon. */
6484 if( pNear->nPhrase>ArraySize(aStatic)((int)(sizeof(aStatic) / sizeof(aStatic[0]))) ){
6485 sqlite3_int64 nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
6486 a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte);
6487 }else{
6488 memset(aStatic, 0, sizeof(aStatic));
6489 }
6490 if( rc!=SQLITE_OK0 ){
6491 *pRc = rc;
6492 return 0;
6493 }
6494
6495 /* Initialize a lookahead iterator for each phrase. After passing the
6496 ** buffer and buffer size to the lookaside-reader init function, zero
6497 ** the phrase poslist buffer. The new poslist for the phrase (containing
6498 ** the same entries as the original with some entries removed on account
6499 ** of the NEAR constraint) is written over the original even as it is
6500 ** being read. This is safe as the entries for the new poslist are a
6501 ** subset of the old, so it is not possible for data yet to be read to
6502 ** be overwritten. */
6503 for(i=0; i<pNear->nPhrase; i++){
6504 Fts5Buffer *pPoslist = &apPhrase[i]->poslist;
6505 fts5LookaheadReaderInit(pPoslist->p, pPoslist->n, &a[i].reader);
6506 pPoslist->n = 0;
6507 a[i].pOut = pPoslist;
6508 }
6509
6510 while( 1 ){
6511 int iAdv;
6512 i64 iMin;
6513 i64 iMax;
6514
6515 /* This block advances the phrase iterators until they point to a set of
6516 ** entries that together comprise a match. */
6517 iMax = a[0].reader.iPos;
6518 do {
6519 bMatch = 1;
6520 for(i=0; i<pNear->nPhrase; i++){
6521 Fts5LookaheadReader *pPos = &a[i].reader;
6522 iMin = iMax - pNear->apPhrase[i]->nTerm - pNear->nNear;
6523 if( pPos->iPos<iMin || pPos->iPos>iMax ){
6524 bMatch = 0;
6525 while( pPos->iPos<iMin ){
6526 if( fts5LookaheadReaderNext(pPos) ) goto ismatch_out;
6527 }
6528 if( pPos->iPos>iMax ) iMax = pPos->iPos;
6529 }
6530 }
6531 }while( bMatch==0 );
6532
6533 /* Add an entry to each output position list */
6534 for(i=0; i<pNear->nPhrase; i++){
6535 i64 iPos = a[i].reader.iPos;
6536 Fts5PoslistWriter *pWriter = &a[i].writer;
6537 if( a[i].pOut->n==0 || iPos!=pWriter->iPrev ){
6538 sqlite3Fts5PoslistWriterAppend(a[i].pOut, pWriter, iPos);
6539 }
6540 }
6541
6542 iAdv = 0;
6543 iMin = a[0].reader.iLookahead;
6544 for(i=0; i<pNear->nPhrase; i++){
6545 if( a[i].reader.iLookahead < iMin ){
6546 iMin = a[i].reader.iLookahead;
6547 iAdv = i;
6548 }
6549 }
6550 if( fts5LookaheadReaderNext(&a[iAdv].reader) ) goto ismatch_out;
6551 }
6552
6553 ismatch_out: {
6554 int bRet = a[0].pOut->n>0;
6555 *pRc = rc;
6556 if( a!=aStatic ) sqlite3_freesqlite3_api->free(a);
6557 return bRet;
6558 }
6559}
6560
6561/*
6562** Advance iterator pIter until it points to a value equal to or laster
6563** than the initial value of *piLast. If this means the iterator points
6564** to a value laster than *piLast, update *piLast to the new lastest value.
6565**
6566** If the iterator reaches EOF, set *pbEof to true before returning. If
6567** an error occurs, set *pRc to an error code. If either *pbEof or *pRc
6568** are set, return a non-zero value. Otherwise, return zero.
6569*/
6570static int fts5ExprAdvanceto(
6571 Fts5IndexIter *pIter, /* Iterator to advance */
6572 int bDesc, /* True if iterator is "rowid DESC" */
6573 i64 *piLast, /* IN/OUT: Lastest rowid seen so far */
6574 int *pRc, /* OUT: Error code */
6575 int *pbEof /* OUT: Set to true if EOF */
6576){
6577 i64 iLast = *piLast;
6578 i64 iRowid;
6579
6580 iRowid = pIter->iRowid;
6581 if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
6582 int rc = sqlite3Fts5IterNextFrom(pIter, iLast);
6583 if( rc || sqlite3Fts5IterEof(pIter)((pIter)->bEof) ){
6584 *pRc = rc;
6585 *pbEof = 1;
6586 return 1;
6587 }
6588 iRowid = pIter->iRowid;
6589 assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) )((void) (0));
6590 }
6591 *piLast = iRowid;
6592
6593 return 0;
6594}
6595
6596static int fts5ExprSynonymAdvanceto(
6597 Fts5ExprTerm *pTerm, /* Term iterator to advance */
6598 int bDesc, /* True if iterator is "rowid DESC" */
6599 i64 *piLast, /* IN/OUT: Lastest rowid seen so far */
6600 int *pRc /* OUT: Error code */
6601){
6602 int rc = SQLITE_OK0;
6603 i64 iLast = *piLast;
6604 Fts5ExprTerm *p;
6605 int bEof = 0;
6606
6607 for(p=pTerm; rc==SQLITE_OK0 && p; p=p->pSynonym){
6608 if( sqlite3Fts5IterEof(p->pIter)((p->pIter)->bEof)==0 ){
6609 i64 iRowid = p->pIter->iRowid;
6610 if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
6611 rc = sqlite3Fts5IterNextFrom(p->pIter, iLast);
6612 }
6613 }
6614 }
6615
6616 if( rc!=SQLITE_OK0 ){
6617 *pRc = rc;
6618 bEof = 1;
6619 }else{
6620 *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof);
6621 }
6622 return bEof;
6623}
6624
6625
6626static int fts5ExprNearTest(
6627 int *pRc,
6628 Fts5Expr *pExpr, /* Expression that pNear is a part of */
6629 Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */
6630){
6631 Fts5ExprNearset *pNear = pNode->pNear;
6632 int rc = *pRc;
6633
6634 if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL0 ){
6635 Fts5ExprTerm *pTerm;
6636 Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
6637 pPhrase->poslist.n = 0;
6638 for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
6639 Fts5IndexIter *pIter = pTerm->pIter;
6640 if( sqlite3Fts5IterEof(pIter)((pIter)->bEof)==0 ){
6641 if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){
6642 pPhrase->poslist.n = 1;
6643 }
6644 }
6645 }
6646 return pPhrase->poslist.n;
6647 }else{
6648 int i;
6649
6650 /* Check that each phrase in the nearset matches the current row.
6651 ** Populate the pPhrase->poslist buffers at the same time. If any
6652 ** phrase is not a match, break out of the loop early. */
6653 for(i=0; rc==SQLITE_OK0 && i<pNear->nPhrase; i++){
6654 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
6655 if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym
6656 || pNear->pColset || pPhrase->aTerm[0].bFirst
6657 ){
6658 int bMatch = 0;
6659 rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
6660 if( bMatch==0 ) break;
6661 }else{
6662 Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
6663 fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData)sqlite3Fts5BufferSet(&rc,&pPhrase->poslist,pIter->
nData,pIter->pData);
6664 }
6665 }
6666
6667 *pRc = rc;
6668 if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
6669 return 1;
6670 }
6671 return 0;
6672 }
6673}
6674
6675
6676/*
6677** Initialize all term iterators in the pNear object. If any term is found
6678** to match no documents at all, return immediately without initializing any
6679** further iterators.
6680**
6681** If an error occurs, return an SQLite error code. Otherwise, return
6682** SQLITE_OK. It is not considered an error if some term matches zero
6683** documents.
6684*/
6685static int fts5ExprNearInitAll(
6686 Fts5Expr *pExpr,
6687 Fts5ExprNode *pNode
6688){
6689 Fts5ExprNearset *pNear = pNode->pNear;
6690 int i;
6691
6692 assert( pNode->bNomatch==0 )((void) (0));
6693 for(i=0; i<pNear->nPhrase; i++){
6694 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
6695 if( pPhrase->nTerm==0 ){
6696 pNode->bEof = 1;
6697 return SQLITE_OK0;
6698 }else{
6699 int j;
6700 for(j=0; j<pPhrase->nTerm; j++){
6701 Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
6702 Fts5ExprTerm *p;
6703 int bHit = 0;
6704
6705 for(p=pTerm; p; p=p->pSynonym){
6706 int rc;
6707 if( p->pIter ){
6708 sqlite3Fts5IterClose(p->pIter);
6709 p->pIter = 0;
6710 }
6711 rc = sqlite3Fts5IndexQuery(
6712 pExpr->pIndex, p->pTerm, p->nQueryTerm,
6713 (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX0x0001 : 0) |
6714 (pExpr->bDesc ? FTS5INDEX_QUERY_DESC0x0002 : 0),
6715 pNear->pColset,
6716 &p->pIter
6717 );
6718 assert( (rc==SQLITE_OK)==(p->pIter!=0) )((void) (0));
6719 if( rc!=SQLITE_OK0 ) return rc;
6720 if( 0==sqlite3Fts5IterEof(p->pIter)((p->pIter)->bEof) ){
6721 bHit = 1;
6722 }
6723 }
6724
6725 if( bHit==0 ){
6726 pNode->bEof = 1;
6727 return SQLITE_OK0;
6728 }
6729 }
6730 }
6731 }
6732
6733 pNode->bEof = 0;
6734 return SQLITE_OK0;
6735}
6736
6737/*
6738** If pExpr is an ASC iterator, this function returns a value with the
6739** same sign as:
6740**
6741** (iLhs - iRhs)
6742**
6743** Otherwise, if this is a DESC iterator, the opposite is returned:
6744**
6745** (iRhs - iLhs)
6746*/
6747static int fts5RowidCmp(
6748 Fts5Expr *pExpr,
6749 i64 iLhs,
6750 i64 iRhs
6751){
6752 assert( pExpr->bDesc==0 || pExpr->bDesc==1 )((void) (0));
6753 if( pExpr->bDesc==0 ){
6754 if( iLhs<iRhs ) return -1;
6755 return (iLhs > iRhs);
6756 }else{
6757 if( iLhs>iRhs ) return -1;
6758 return (iLhs < iRhs);
6759 }
6760}
6761
6762static void fts5ExprSetEof(Fts5ExprNode *pNode){
6763 int i;
6764 pNode->bEof = 1;
6765 pNode->bNomatch = 0;
6766 for(i=0; i<pNode->nChild; i++){
6767 fts5ExprSetEof(pNode->apChild[i]);
6768 }
6769}
6770
6771static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){
6772 if( pNode->eType==FTS5_STRING9 || pNode->eType==FTS5_TERM4 ){
6773 Fts5ExprNearset *pNear = pNode->pNear;
6774 int i;
6775 for(i=0; i<pNear->nPhrase; i++){
6776 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
6777 pPhrase->poslist.n = 0;
6778 }
6779 }else{
6780 int i;
6781 for(i=0; i<pNode->nChild; i++){
6782 fts5ExprNodeZeroPoslist(pNode->apChild[i]);
6783 }
6784 }
6785}
6786
6787
6788
6789/*
6790** Compare the values currently indicated by the two nodes as follows:
6791**
6792** res = (*p1) - (*p2)
6793**
6794** Nodes that point to values that come later in the iteration order are
6795** considered to be larger. Nodes at EOF are the largest of all.
6796**
6797** This means that if the iteration order is ASC, then numerically larger
6798** rowids are considered larger. Or if it is the default DESC, numerically
6799** smaller rowids are larger.
6800*/
6801static int fts5NodeCompare(
6802 Fts5Expr *pExpr,
6803 Fts5ExprNode *p1,
6804 Fts5ExprNode *p2
6805){
6806 if( p2->bEof ) return -1;
6807 if( p1->bEof ) return +1;
6808 return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid);
6809}
6810
6811/*
6812** All individual term iterators in pNear are guaranteed to be valid when
6813** this function is called. This function checks if all term iterators
6814** point to the same rowid, and if not, advances them until they do.
6815** If an EOF is reached before this happens, *pbEof is set to true before
6816** returning.
6817**
6818** SQLITE_OK is returned if an error occurs, or an SQLite error code
6819** otherwise. It is not considered an error code if an iterator reaches
6820** EOF.
6821*/
6822static int fts5ExprNodeTest_STRING(
6823 Fts5Expr *pExpr, /* Expression pPhrase belongs to */
6824 Fts5ExprNode *pNode
6825){
6826 Fts5ExprNearset *pNear = pNode->pNear;
6827 Fts5ExprPhrase *pLeft = pNear->apPhrase[0];
6828 int rc = SQLITE_OK0;
6829 i64 iLast; /* Lastest rowid any iterator points to */
6830 int i, j; /* Phrase and token index, respectively */
6831 int bMatch; /* True if all terms are at the same rowid */
6832 const int bDesc = pExpr->bDesc;
6833
6834 /* Check that this node should not be FTS5_TERM */
6835 assert( pNear->nPhrase>1((void) (0))
6836 || pNear->apPhrase[0]->nTerm>1((void) (0))
6837 || pNear->apPhrase[0]->aTerm[0].pSynonym((void) (0))
6838 || pNear->apPhrase[0]->aTerm[0].bFirst((void) (0))
6839 )((void) (0));
6840
6841 /* Initialize iLast, the "lastest" rowid any iterator points to. If the
6842 ** iterator skips through rowids in the default ascending order, this means
6843 ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
6844 ** means the minimum rowid. */
6845 if( pLeft->aTerm[0].pSynonym ){
6846 iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0);
6847 }else{
6848 iLast = pLeft->aTerm[0].pIter->iRowid;
6849 }
6850
6851 do {
6852 bMatch = 1;
6853 for(i=0; i<pNear->nPhrase; i++){
6854 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
6855 for(j=0; j<pPhrase->nTerm; j++){
6856 Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
6857 if( pTerm->pSynonym ){
6858 i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
6859 if( iRowid==iLast ) continue;
6860 bMatch = 0;
6861 if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
6862 pNode->bNomatch = 0;
6863 pNode->bEof = 1;
6864 return rc;
6865 }
6866 }else{
6867 Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
6868 if( pIter->iRowid==iLast ) continue;
6869 bMatch = 0;
6870 if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
6871 return rc;
6872 }
6873 }
6874 }
6875 }
6876 }while( bMatch==0 );
6877
6878 pNode->iRowid = iLast;
6879 pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK0);
6880 assert( pNode->bEof==0 || pNode->bNomatch==0 )((void) (0));
6881
6882 return rc;
6883}
6884
6885/*
6886** Advance the first term iterator in the first phrase of pNear. Set output
6887** variable *pbEof to true if it reaches EOF or if an error occurs.
6888**
6889** Return SQLITE_OK if successful, or an SQLite error code if an error
6890** occurs.
6891*/
6892static int fts5ExprNodeNext_STRING(
6893 Fts5Expr *pExpr, /* Expression pPhrase belongs to */
6894 Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */
6895 int bFromValid,
6896 i64 iFrom
6897){
6898 Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0];
6899 int rc = SQLITE_OK0;
6900
6901 pNode->bNomatch = 0;
6902 if( pTerm->pSynonym ){
6903 int bEof = 1;
6904 Fts5ExprTerm *p;
6905
6906 /* Find the firstest rowid any synonym points to. */
6907 i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0);
6908
6909 /* Advance each iterator that currently points to iRowid. Or, if iFrom
6910 ** is valid - each iterator that points to a rowid before iFrom. */
6911 for(p=pTerm; p; p=p->pSynonym){
6912 if( sqlite3Fts5IterEof(p->pIter)((p->pIter)->bEof)==0 ){
6913 i64 ii = p->pIter->iRowid;
6914 if( ii==iRowid
6915 || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc)
6916 ){
6917 if( bFromValid ){
6918 rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom);
6919 }else{
6920 rc = sqlite3Fts5IterNext(p->pIter);
6921 }
6922 if( rc!=SQLITE_OK0 ) break;
6923 if( sqlite3Fts5IterEof(p->pIter)((p->pIter)->bEof)==0 ){
6924 bEof = 0;
6925 }
6926 }else{
6927 bEof = 0;
6928 }
6929 }
6930 }
6931
6932 /* Set the EOF flag if either all synonym iterators are at EOF or an
6933 ** error has occurred. */
6934 pNode->bEof = (rc || bEof);
6935 }else{
6936 Fts5IndexIter *pIter = pTerm->pIter;
6937
6938 assert( Fts5NodeIsString(pNode) )((void) (0));
6939 if( bFromValid ){
6940 rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
6941 }else{
6942 rc = sqlite3Fts5IterNext(pIter);
6943 }
6944
6945 pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)((pIter)->bEof));
6946 }
6947
6948 if( pNode->bEof==0 ){
6949 assert( rc==SQLITE_OK )((void) (0));
6950 rc = fts5ExprNodeTest_STRING(pExpr, pNode);
6951 }
6952
6953 return rc;
6954}
6955
6956
6957static int fts5ExprNodeTest_TERM(
6958 Fts5Expr *pExpr, /* Expression that pNear is a part of */
6959 Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */
6960){
6961 /* As this "NEAR" object is actually a single phrase that consists
6962 ** of a single term only, grab pointers into the poslist managed by the
6963 ** fts5_index.c iterator object. This is much faster than synthesizing
6964 ** a new poslist the way we have to for more complicated phrase or NEAR
6965 ** expressions. */
6966 Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0];
6967 Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
6968
6969 assert( pNode->eType==FTS5_TERM )((void) (0));
6970 assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 )((void) (0));
6971 assert( pPhrase->aTerm[0].pSynonym==0 )((void) (0));
6972
6973 pPhrase->poslist.n = pIter->nData;
6974 if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL0 ){
6975 pPhrase->poslist.p = (u8*)pIter->pData;
6976 }
6977 pNode->iRowid = pIter->iRowid;
6978 pNode->bNomatch = (pPhrase->poslist.n==0);
6979 return SQLITE_OK0;
6980}
6981
6982/*
6983** xNext() method for a node of type FTS5_TERM.
6984*/
6985static int fts5ExprNodeNext_TERM(
6986 Fts5Expr *pExpr,
6987 Fts5ExprNode *pNode,
6988 int bFromValid,
6989 i64 iFrom
6990){
6991 int rc;
6992 Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter;
6993
6994 assert( pNode->bEof==0 )((void) (0));
6995 if( bFromValid ){
6996 rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
6997 }else{
6998 rc = sqlite3Fts5IterNext(pIter);
6999 }
7000 if( rc==SQLITE_OK0 && sqlite3Fts5IterEof(pIter)((pIter)->bEof)==0 ){
7001 rc = fts5ExprNodeTest_TERM(pExpr, pNode);
7002 }else{
7003 pNode->bEof = 1;
7004 pNode->bNomatch = 0;
7005 }
7006 return rc;
7007}
7008
7009static void fts5ExprNodeTest_OR(
7010 Fts5Expr *pExpr, /* Expression of which pNode is a part */
7011 Fts5ExprNode *pNode /* Expression node to test */
7012){
7013 Fts5ExprNode *pNext = pNode->apChild[0];
7014 int i;
7015
7016 for(i=1; i<pNode->nChild; i++){
7017 Fts5ExprNode *pChild = pNode->apChild[i];
7018 int cmp = fts5NodeCompare(pExpr, pNext, pChild);
7019 if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){
7020 pNext = pChild;
7021 }
7022 }
7023 pNode->iRowid = pNext->iRowid;
7024 pNode->bEof = pNext->bEof;
7025 pNode->bNomatch = pNext->bNomatch;
7026}
7027
7028static int fts5ExprNodeNext_OR(
7029 Fts5Expr *pExpr,
7030 Fts5ExprNode *pNode,
7031 int bFromValid,
7032 i64 iFrom
7033){
7034 int i;
7035 i64 iLast = pNode->iRowid;
7036
7037 for(i=0; i<pNode->nChild; i++){
7038 Fts5ExprNode *p1 = pNode->apChild[i];
7039 assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 )((void) (0));
7040 if( p1->bEof==0 ){
7041 if( (p1->iRowid==iLast)
7042 || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
7043 ){
7044 int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom)(p1)->xNext((pExpr), (p1), (bFromValid), (iFrom));
7045 if( rc!=SQLITE_OK0 ){
7046 pNode->bNomatch = 0;
7047 return rc;
7048 }
7049 }
7050 }
7051 }
7052
7053 fts5ExprNodeTest_OR(pExpr, pNode);
7054 return SQLITE_OK0;
7055}
7056
7057/*
7058** Argument pNode is an FTS5_AND node.
7059*/
7060static int fts5ExprNodeTest_AND(
7061 Fts5Expr *pExpr, /* Expression pPhrase belongs to */
7062 Fts5ExprNode *pAnd /* FTS5_AND node to advance */
7063){
7064 int iChild;
7065 i64 iLast = pAnd->iRowid;
7066 int rc = SQLITE_OK0;
7067 int bMatch;
7068
7069 assert( pAnd->bEof==0 )((void) (0));
7070 do {
7071 pAnd->bNomatch = 0;
7072 bMatch = 1;
7073 for(iChild=0; iChild<pAnd->nChild; iChild++){
7074 Fts5ExprNode *pChild = pAnd->apChild[iChild];
7075 int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid);
7076 if( cmp>0 ){
7077 /* Advance pChild until it points to iLast or laster */
7078 rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast)(pChild)->xNext((pExpr), (pChild), (1), (iLast));
7079 if( rc!=SQLITE_OK0 ){
7080 pAnd->bNomatch = 0;
7081 return rc;
7082 }
7083 }
7084
7085 /* If the child node is now at EOF, so is the parent AND node. Otherwise,
7086 ** the child node is guaranteed to have advanced at least as far as
7087 ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the
7088 ** new lastest rowid seen so far. */
7089 assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 )((void) (0));
7090 if( pChild->bEof ){
7091 fts5ExprSetEof(pAnd);
7092 bMatch = 1;
7093 break;
7094 }else if( iLast!=pChild->iRowid ){
7095 bMatch = 0;
7096 iLast = pChild->iRowid;
7097 }
7098
7099 if( pChild->bNomatch ){
7100 pAnd->bNomatch = 1;
7101 }
7102 }
7103 }while( bMatch==0 );
7104
7105 if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){
7106 fts5ExprNodeZeroPoslist(pAnd);
7107 }
7108 pAnd->iRowid = iLast;
7109 return SQLITE_OK0;
7110}
7111
7112static int fts5ExprNodeNext_AND(
7113 Fts5Expr *pExpr,
7114 Fts5ExprNode *pNode,
7115 int bFromValid,
7116 i64 iFrom
7117){
7118 int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom)(pNode->apChild[0])->xNext((pExpr), (pNode->apChild[
0]), (bFromValid), (iFrom));
7119 if( rc==SQLITE_OK0 ){
7120 rc = fts5ExprNodeTest_AND(pExpr, pNode);
7121 }else{
7122 pNode->bNomatch = 0;
7123 }
7124 return rc;
7125}
7126
7127static int fts5ExprNodeTest_NOT(
7128 Fts5Expr *pExpr, /* Expression pPhrase belongs to */
7129 Fts5ExprNode *pNode /* FTS5_NOT node to advance */
7130){
7131 int rc = SQLITE_OK0;
7132 Fts5ExprNode *p1 = pNode->apChild[0];
7133 Fts5ExprNode *p2 = pNode->apChild[1];
7134 assert( pNode->nChild==2 )((void) (0));
7135
7136 while( rc==SQLITE_OK0 && p1->bEof==0 ){
7137 int cmp = fts5NodeCompare(pExpr, p1, p2);
7138 if( cmp>0 ){
7139 rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid)(p2)->xNext((pExpr), (p2), (1), (p1->iRowid));
7140 cmp = fts5NodeCompare(pExpr, p1, p2);
7141 }
7142 assert( rc!=SQLITE_OK || cmp<=0 )((void) (0));
7143 if( cmp || p2->bNomatch ) break;
7144 rc = fts5ExprNodeNext(pExpr, p1, 0, 0)(p1)->xNext((pExpr), (p1), (0), (0));
7145 }
7146 pNode->bEof = p1->bEof;
7147 pNode->bNomatch = p1->bNomatch;
7148 pNode->iRowid = p1->iRowid;
7149 if( p1->bEof ){
7150 fts5ExprNodeZeroPoslist(p2);
7151 }
7152 return rc;
7153}
7154
7155static int fts5ExprNodeNext_NOT(
7156 Fts5Expr *pExpr,
7157 Fts5ExprNode *pNode,
7158 int bFromValid,
7159 i64 iFrom
7160){
7161 int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom)(pNode->apChild[0])->xNext((pExpr), (pNode->apChild[
0]), (bFromValid), (iFrom));
7162 if( rc==SQLITE_OK0 ){
7163 rc = fts5ExprNodeTest_NOT(pExpr, pNode);
7164 }
7165 if( rc!=SQLITE_OK0 ){
7166 pNode->bNomatch = 0;
7167 }
7168 return rc;
7169}
7170
7171/*
7172** If pNode currently points to a match, this function returns SQLITE_OK
7173** without modifying it. Otherwise, pNode is advanced until it does point
7174** to a match or EOF is reached.
7175*/
7176static int fts5ExprNodeTest(
7177 Fts5Expr *pExpr, /* Expression of which pNode is a part */
7178 Fts5ExprNode *pNode /* Expression node to test */
7179){
7180 int rc = SQLITE_OK0;
7181 if( pNode->bEof==0 ){
7182 switch( pNode->eType ){
7183
7184 case FTS5_STRING9: {
7185 rc = fts5ExprNodeTest_STRING(pExpr, pNode);
7186 break;
7187 }
7188
7189 case FTS5_TERM4: {
7190 rc = fts5ExprNodeTest_TERM(pExpr, pNode);
7191 break;
7192 }
7193
7194 case FTS5_AND2: {
7195 rc = fts5ExprNodeTest_AND(pExpr, pNode);
7196 break;
7197 }
7198
7199 case FTS5_OR1: {
7200 fts5ExprNodeTest_OR(pExpr, pNode);
7201 break;
7202 }
7203
7204 default: assert( pNode->eType==FTS5_NOT )((void) (0)); {
7205 rc = fts5ExprNodeTest_NOT(pExpr, pNode);
7206 break;
7207 }
7208 }
7209 }
7210 return rc;
7211}
7212
7213
7214/*
7215** Set node pNode, which is part of expression pExpr, to point to the first
7216** match. If there are no matches, set the Node.bEof flag to indicate EOF.
7217**
7218** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise.
7219** It is not an error if there are no matches.
7220*/
7221static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
7222 int rc = SQLITE_OK0;
7223 pNode->bEof = 0;
7224 pNode->bNomatch = 0;
7225
7226 if( Fts5NodeIsString(pNode)((pNode)->eType==4 || (pNode)->eType==9) ){
7227 /* Initialize all term iterators in the NEAR object. */
7228 rc = fts5ExprNearInitAll(pExpr, pNode);
7229 }else if( pNode->xNext==0 ){
7230 pNode->bEof = 1;
7231 }else{
7232 int i;
7233 int nEof = 0;
7234 for(i=0; i<pNode->nChild && rc==SQLITE_OK0; i++){
7235 Fts5ExprNode *pChild = pNode->apChild[i];
7236 rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]);
7237 assert( pChild->bEof==0 || pChild->bEof==1 )((void) (0));
7238 nEof += pChild->bEof;
7239 }
7240 pNode->iRowid = pNode->apChild[0]->iRowid;
7241
7242 switch( pNode->eType ){
7243 case FTS5_AND2:
7244 if( nEof>0 ) fts5ExprSetEof(pNode);
7245 break;
7246
7247 case FTS5_OR1:
7248 if( pNode->nChild==nEof ) fts5ExprSetEof(pNode);
7249 break;
7250
7251 default:
7252 assert( pNode->eType==FTS5_NOT )((void) (0));
7253 pNode->bEof = pNode->apChild[0]->bEof;
7254 break;
7255 }
7256 }
7257
7258 if( rc==SQLITE_OK0 ){
7259 rc = fts5ExprNodeTest(pExpr, pNode);
7260 }
7261 return rc;
7262}
7263
7264
7265/*
7266** Begin iterating through the set of documents in index pIdx matched by
7267** the MATCH expression passed as the first argument. If the "bDesc"
7268** parameter is passed a non-zero value, iteration is in descending rowid
7269** order. Or, if it is zero, in ascending order.
7270**
7271** If iterating in ascending rowid order (bDesc==0), the first document
7272** visited is that with the smallest rowid that is larger than or equal
7273** to parameter iFirst. Or, if iterating in ascending order (bDesc==1),
7274** then the first document visited must have a rowid smaller than or
7275** equal to iFirst.
7276**
7277** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
7278** is not considered an error if the query does not match any documents.
7279*/
7280static int sqlite3Fts5ExprFirst(
7281 Fts5Expr *p,
7282 Fts5Index *pIdx,
7283 i64 iFirst,
7284 i64 iLast,
7285 int bDesc
7286){
7287 Fts5ExprNode *pRoot = p->pRoot;
27
←
Access to field 'pRoot' results in a dereference of a null pointer (loaded from variable 'p')
7288 int rc; /* Return code */
7289
7290 p->pIndex = pIdx;
7291 p->bDesc = bDesc;
7292 rc = fts5ExprNodeFirst(p, pRoot);
7293
7294 /* If not at EOF but the current rowid occurs earlier than iFirst in
7295 ** the iteration order, move to document iFirst or later. */
7296 if( rc==SQLITE_OK0
7297 && 0==pRoot->bEof
7298 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0
7299 ){
7300 rc = fts5ExprNodeNext(p, pRoot, 1, iFirst)(pRoot)->xNext((p), (pRoot), (1), (iFirst));
7301 }
7302
7303 /* If the iterator is not at a real match, skip forward until it is. */
7304 while( pRoot->bNomatch && rc==SQLITE_OK0 ){
7305 assert( pRoot->bEof==0 )((void) (0));
7306 rc = fts5ExprNodeNext(p, pRoot, 0, 0)(pRoot)->xNext((p), (pRoot), (0), (0));
7307 }
7308 if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){
7309 pRoot->bEof = 1;
7310 }
7311 return rc;
7312}
7313
7314/*
7315** Move to the next document
7316**
7317** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
7318** is not considered an error if the query does not match any documents.
7319*/
7320static int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){
7321 int rc;
7322 Fts5ExprNode *pRoot = p->pRoot;
7323 assert( pRoot->bEof==0 && pRoot->bNomatch==0 )((void) (0));
7324 do {
7325 rc = fts5ExprNodeNext(p, pRoot, 0, 0)(pRoot)->xNext((p), (pRoot), (0), (0));
7326 assert( pRoot->bNomatch==0 || (rc==SQLITE_OK && pRoot->bEof==0) )((void) (0));
7327 }while( pRoot->bNomatch );
7328 if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){
7329 pRoot->bEof = 1;
7330 }
7331 return rc;
7332}
7333
7334static int sqlite3Fts5ExprEof(Fts5Expr *p){
7335 return p->pRoot->bEof;
7336}
7337
7338static i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
7339 return p->pRoot->iRowid;
7340}
7341
7342static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
7343 int rc = SQLITE_OK0;
7344 *pz = sqlite3Fts5Strndup(&rc, pToken->p, pToken->n);
7345 return rc;
7346}
7347
7348/*
7349** Free the phrase object passed as the only argument.
7350*/
7351static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
7352 if( pPhrase ){
7353 int i;
7354 for(i=0; i<pPhrase->nTerm; i++){
7355 Fts5ExprTerm *pSyn;
7356 Fts5ExprTerm *pNext;
7357 Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
7358 sqlite3_freesqlite3_api->free(pTerm->pTerm);
7359 sqlite3Fts5IterClose(pTerm->pIter);
7360 for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
7361 pNext = pSyn->pSynonym;
7362 sqlite3Fts5IterClose(pSyn->pIter);
7363 fts5BufferFree((Fts5Buffer*)&pSyn[1])sqlite3Fts5BufferFree((Fts5Buffer*)&pSyn[1]);
7364 sqlite3_freesqlite3_api->free(pSyn);
7365 }
7366 }
7367 if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist)sqlite3Fts5BufferFree(&pPhrase->poslist);
7368 sqlite3_freesqlite3_api->free(pPhrase);
7369 }
7370}
7371
7372/*
7373** Set the "bFirst" flag on the first token of the phrase passed as the
7374** only argument.
7375*/
7376static void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){
7377 if( pPhrase && pPhrase->nTerm ){
7378 pPhrase->aTerm[0].bFirst = 1;
7379 }
7380}
7381
7382/*
7383** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
7384** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
7385** appended to it and the results returned.
7386**
7387** If an OOM error occurs, both the pNear and pPhrase objects are freed and
7388** NULL returned.
7389*/
7390static Fts5ExprNearset *sqlite3Fts5ParseNearset(
7391 Fts5Parse *pParse, /* Parse context */
7392 Fts5ExprNearset *pNear, /* Existing nearset, or NULL */
7393 Fts5ExprPhrase *pPhrase /* Recently parsed phrase */
7394){
7395 const int SZALLOC = 8;
7396 Fts5ExprNearset *pRet = 0;
7397
7398 if( pParse->rc==SQLITE_OK0 ){
7399 if( pNear==0 ){
7400 sqlite3_int64 nByte;
7401 nByte = SZ_FTS5EXPRNEARSET(SZALLOC+1)(__builtin_offsetof(Fts5ExprNearset, apPhrase)+(SZALLOC+1)*sizeof
(Fts5ExprPhrase*));
7402 pRet = sqlite3_malloc64sqlite3_api->malloc64(nByte);
7403 if( pRet==0 ){
7404 pParse->rc = SQLITE_NOMEM7;
7405 }else{
7406 memset(pRet, 0, (size_t)nByte);
7407 }
7408 }else if( (pNear->nPhrase % SZALLOC)==0 ){
7409 int nNew = pNear->nPhrase + SZALLOC;
7410 sqlite3_int64 nByte;
7411
7412 nByte = SZ_FTS5EXPRNEARSET(nNew+1)(__builtin_offsetof(Fts5ExprNearset, apPhrase)+(nNew+1)*sizeof
(Fts5ExprPhrase*));
7413 pRet = (Fts5ExprNearset*)sqlite3_realloc64sqlite3_api->realloc64(pNear, nByte);
7414 if( pRet==0 ){
7415 pParse->rc = SQLITE_NOMEM7;
7416 }
7417 }else{
7418 pRet = pNear;
7419 }
7420 }
7421
7422 if( pRet==0 ){
7423 assert( pParse->rc!=SQLITE_OK )((void) (0));
7424 sqlite3Fts5ParseNearsetFree(pNear);
7425 sqlite3Fts5ParsePhraseFree(pPhrase);
7426 }else{
7427 if( pRet->nPhrase>0 ){
7428 Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1];
7429 assert( pParse!=0 )((void) (0));
7430 assert( pParse->apPhrase!=0 )((void) (0));
7431 assert( pParse->nPhrase>=2 )((void) (0));
7432 assert( pLast==pParse->apPhrase[pParse->nPhrase-2] )((void) (0));
7433 if( pPhrase->nTerm==0 ){
7434 fts5ExprPhraseFree(pPhrase);
7435 pRet->nPhrase--;
7436 pParse->nPhrase--;
7437 pPhrase = pLast;
7438 }else if( pLast->nTerm==0 ){
7439 fts5ExprPhraseFree(pLast);
7440 pParse->apPhrase[pParse->nPhrase-2] = pPhrase;
7441 pParse->nPhrase--;
7442 pRet->nPhrase--;
7443 }
7444 }
7445 pRet->apPhrase[pRet->nPhrase++] = pPhrase;
7446 }
7447 return pRet;
7448}
7449
7450typedef struct TokenCtx TokenCtx;
7451struct TokenCtx {
7452 Fts5ExprPhrase *pPhrase;
7453 Fts5Config *pConfig;
7454 int rc;
7455};
7456
7457/*
7458** Callback for tokenizing terms used by ParseTerm().
7459*/
7460static int fts5ParseTokenize(
7461 void *pContext, /* Pointer to Fts5InsertCtx object */
7462 int tflags, /* Mask of FTS5_TOKEN_* flags */
7463 const char *pToken, /* Buffer containing token */
7464 int nToken, /* Size of token in bytes */
7465 int iUnused1, /* Start offset of token */
7466 int iUnused2 /* End offset of token */
7467){
7468 int rc = SQLITE_OK0;
7469 const int SZALLOC = 8;
7470 TokenCtx *pCtx = (TokenCtx*)pContext;
7471 Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
7472
7473 UNUSED_PARAM2(iUnused1, iUnused2)(void)(iUnused1), (void)(iUnused2);
7474
7475 /* If an error has already occurred, this is a no-op */
7476 if( pCtx->rc!=SQLITE_OK0 ) return pCtx->rc;
7477 if( nToken>FTS5_MAX_TOKEN_SIZE32768 ) nToken = FTS5_MAX_TOKEN_SIZE32768;
7478
7479 if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED0x0001) ){
7480 Fts5ExprTerm *pSyn;
7481 sqlite3_int64 nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
7482 pSyn = (Fts5ExprTerm*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
7483 if( pSyn==0 ){
7484 rc = SQLITE_NOMEM7;
7485 }else{
7486 memset(pSyn, 0, (size_t)nByte);
7487 pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
7488 pSyn->nFullTerm = pSyn->nQueryTerm = nToken;
7489 if( pCtx->pConfig->bTokendata ){
7490 pSyn->nQueryTerm = (int)strlen(pSyn->pTerm);
7491 }
7492 memcpy(pSyn->pTerm, pToken, nToken);
7493 pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
7494 pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
7495 }
7496 }else{
7497 Fts5ExprTerm *pTerm;
7498 if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
7499 Fts5ExprPhrase *pNew;
7500 int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0);
7501
7502 pNew = (Fts5ExprPhrase*)sqlite3_realloc64sqlite3_api->realloc64(pPhrase,
7503 SZ_FTS5EXPRPHRASE(nNew+1)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (nNew+1)*sizeof(
Fts5ExprTerm))
7504 );
7505 if( pNew==0 ){
7506 rc = SQLITE_NOMEM7;
7507 }else{
7508 if( pPhrase==0 ) memset(pNew, 0, SZ_FTS5EXPRPHRASE(1)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (1)*sizeof(Fts5ExprTerm
)));
7509 pCtx->pPhrase = pPhrase = pNew;
7510 pNew->nTerm = nNew - SZALLOC;
7511 }
7512 }
7513
7514 if( rc==SQLITE_OK0 ){
7515 pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
7516 memset(pTerm, 0, sizeof(Fts5ExprTerm));
7517 pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
7518 pTerm->nFullTerm = pTerm->nQueryTerm = nToken;
7519 if( pCtx->pConfig->bTokendata && rc==SQLITE_OK0 ){
7520 pTerm->nQueryTerm = (int)strlen(pTerm->pTerm);
7521 }
7522 }
7523 }
7524
7525 pCtx->rc = rc;
7526 return rc;
7527}
7528
7529
7530/*
7531** Free the phrase object passed as the only argument.
7532*/
7533static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase *pPhrase){
7534 fts5ExprPhraseFree(pPhrase);
7535}
7536
7537/*
7538** Free the phrase object passed as the second argument.
7539*/
7540static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset *pNear){
7541 if( pNear ){
7542 int i;
7543 for(i=0; i<pNear->nPhrase; i++){
7544 fts5ExprPhraseFree(pNear->apPhrase[i]);
7545 }
7546 sqlite3_freesqlite3_api->free(pNear->pColset);
7547 sqlite3_freesqlite3_api->free(pNear);
7548 }
7549}
7550
7551static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p){
7552 assert( pParse->pExpr==0 )((void) (0));
7553 pParse->pExpr = p;
7554}
7555
7556static int parseGrowPhraseArray(Fts5Parse *pParse){
7557 if( (pParse->nPhrase % 8)==0 ){
7558 sqlite3_int64 nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
7559 Fts5ExprPhrase **apNew;
7560 apNew = (Fts5ExprPhrase**)sqlite3_realloc64sqlite3_api->realloc64(pParse->apPhrase, nByte);
7561 if( apNew==0 ){
7562 pParse->rc = SQLITE_NOMEM7;
7563 return SQLITE_NOMEM7;
7564 }
7565 pParse->apPhrase = apNew;
7566 }
7567 return SQLITE_OK0;
7568}
7569
7570/*
7571** This function is called by the parser to process a string token. The
7572** string may or may not be quoted. In any case it is tokenized and a
7573** phrase object consisting of all tokens returned.
7574*/
7575static Fts5ExprPhrase *sqlite3Fts5ParseTerm(
7576 Fts5Parse *pParse, /* Parse context */
7577 Fts5ExprPhrase *pAppend, /* Phrase to append to */
7578 Fts5Token *pToken, /* String to tokenize */
7579 int bPrefix /* True if there is a trailing "*" */
7580){
7581 Fts5Config *pConfig = pParse->pConfig;
7582 TokenCtx sCtx; /* Context object passed to callback */
7583 int rc; /* Tokenize return code */
7584 char *z = 0;
7585
7586 memset(&sCtx, 0, sizeof(TokenCtx));
7587 sCtx.pPhrase = pAppend;
7588 sCtx.pConfig = pConfig;
7589
7590 rc = fts5ParseStringFromToken(pToken, &z);
7591 if( rc==SQLITE_OK0 ){
7592 int flags = FTS5_TOKENIZE_QUERY0x0001 | (bPrefix ? FTS5_TOKENIZE_PREFIX0x0002 : 0);
7593 int n;
7594 sqlite3Fts5Dequote(z);
7595 n = (int)strlen(z);
7596 rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
7597 }
7598 sqlite3_freesqlite3_api->free(z);
7599 if( rc || (rc = sCtx.rc) ){
7600 pParse->rc = rc;
7601 fts5ExprPhraseFree(sCtx.pPhrase);
7602 sCtx.pPhrase = 0;
7603 }else{
7604
7605 if( pAppend==0 ){
7606 if( parseGrowPhraseArray(pParse) ){
7607 fts5ExprPhraseFree(sCtx.pPhrase);
7608 return 0;
7609 }
7610 pParse->nPhrase++;
7611 }
7612
7613 if( sCtx.pPhrase==0 ){
7614 /* This happens when parsing a token or quoted phrase that contains
7615 ** no token characters at all. (e.g ... MATCH '""'). */
7616 sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, SZ_FTS5EXPRPHRASE(1)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (1)*sizeof(Fts5ExprTerm
)));
7617 }else if( sCtx.pPhrase->nTerm ){
7618 sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
7619 }
7620 assert( pParse->apPhrase!=0 )((void) (0));
7621 pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
7622 }
7623
7624 return sCtx.pPhrase;
7625}
7626
7627/*
7628** Create a new FTS5 expression by cloning phrase iPhrase of the
7629** expression passed as the second argument.
7630*/
7631static int sqlite3Fts5ExprClonePhrase(
7632 Fts5Expr *pExpr,
7633 int iPhrase,
7634 Fts5Expr **ppNew
7635){
7636 int rc = SQLITE_OK0; /* Return code */
7637 Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */
7638 Fts5Expr *pNew = 0; /* Expression to return via *ppNew */
7639 TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */
7640 if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){
3
←
Assuming 'pExpr' is non-null
→
4
←
Assuming 'iPhrase' is >= 0
→
5
←
Assuming 'iPhrase' is < field 'nPhrase'
→
6
←
Taking false branch
→
7641 rc = SQLITE_RANGE25;
7642 }else{
7643 pOrig = pExpr->apExprPhrase[iPhrase];
7644 pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
7645 }
7646 if( rc
6.1
'rc' is equal to SQLITE_OK
==SQLITE_OK0 ){
7
←
Taking true branch
→
7647 pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc,
7648 sizeof(Fts5ExprPhrase*));
7649 }
7650 if( rc
7.1
'rc' is equal to SQLITE_OK
==SQLITE_OK0 ){
8
←
Taking true branch
→
7651 pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRNODE(1)(__builtin_offsetof(Fts5ExprNode, apChild) + (1)*sizeof(Fts5ExprNode
*)));
7652 }
7653 if( rc
8.1
'rc' is equal to SQLITE_OK
==SQLITE_OK0 ){
9
←
Taking true branch
→
7654 pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc,
7655 SZ_FTS5EXPRNEARSET(2)(__builtin_offsetof(Fts5ExprNearset, apPhrase)+(2)*sizeof(Fts5ExprPhrase
*)));
7656 }
7657 if( rc
9.1
'rc' is equal to SQLITE_OK
==SQLITE_OK0 && ALWAYS(pOrig!=0)(pOrig!=0) ){
10
←
Assuming 'pOrig' is not equal to null
→
11
←
Taking true branch
→
7658 Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset;
7659 if( pColsetOrig ){
12
←
Assuming 'pColsetOrig' is null
→
13
←
Taking false branch
→
7660 sqlite3_int64 nByte;
7661 Fts5Colset *pColset;
7662 nByte = SZ_FTS5COLSET(pColsetOrig->nCol)(sizeof(i64)*((pColsetOrig->nCol+2)/2));
7663 pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte);
7664 if( pColset ){
7665 memcpy(pColset, pColsetOrig, (size_t)nByte);
7666 }
7667 pNew->pRoot->pNear->pColset = pColset;
7668 }
7669 }
7670
7671 if( rc
13.1
'rc' is equal to SQLITE_OK
==SQLITE_OK0 ){
14
←
Taking true branch
→
7672 if( pOrig->nTerm ){
15
←
Assuming field 'nTerm' is not equal to 0
→
16
←
Taking true branch
→
7673 int i; /* Used to iterate through phrase terms */
7674 sCtx.pConfig = pExpr->pConfig;
7675 for(i=0; rc
16.1
'rc' is equal to SQLITE_OK
==SQLITE_OK0 && i<pOrig->nTerm; i++){
17
←
Assuming 'i' is >= field 'nTerm'
→
7676 int tflags = 0;
7677 Fts5ExprTerm *p;
7678 for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK0; p=p->pSynonym){
7679 rc = fts5ParseTokenize((void*)&sCtx,tflags,p->pTerm,p->nFullTerm,0,0);
7680 tflags = FTS5_TOKEN_COLOCATED0x0001;
7681 }
7682 if( rc==SQLITE_OK0 ){
7683 sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
7684 sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
7685 }
7686 }
7687 }else{
7688 /* This happens when parsing a token or quoted phrase that contains
7689 ** no token characters at all. (e.g ... MATCH '""'). */
7690 sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRPHRASE(1)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (1)*sizeof(Fts5ExprTerm
)));
7691 }
7692 }
7693
7694 if( rc
17.1
'rc' is equal to SQLITE_OK
==SQLITE_OK0 && ALWAYS(sCtx.pPhrase)(sCtx.pPhrase) ){
18
←
Taking false branch
→
7695 /* All the allocations succeeded. Put the expression object together. */
7696 pNew->pIndex = pExpr->pIndex;
7697 pNew->pConfig = pExpr->pConfig;
7698 pNew->nPhrase = 1;
7699 pNew->apExprPhrase[0] = sCtx.pPhrase;
7700 pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
7701 pNew->pRoot->pNear->nPhrase = 1;
7702 sCtx.pPhrase->pNode = pNew->pRoot;
7703
7704 if( pOrig->nTerm==1
7705 && pOrig->aTerm[0].pSynonym==0
7706 && pOrig->aTerm[0].bFirst==0
7707 ){
7708 pNew->pRoot->eType = FTS5_TERM4;
7709 pNew->pRoot->xNext = fts5ExprNodeNext_TERM;
7710 }else{
7711 pNew->pRoot->eType = FTS5_STRING9;
7712 pNew->pRoot->xNext = fts5ExprNodeNext_STRING;
7713 }
7714 }else{
7715 sqlite3Fts5ExprFree(pNew);
7716 fts5ExprPhraseFree(sCtx.pPhrase);
7717 pNew = 0;
19
←
Null pointer value stored to 'pNew'
→
7718 }
7719
7720 *ppNew = pNew;
20
←
Null pointer value stored to field 'pExpr'
→
7721 return rc;
7722}
7723
7724
7725/*
7726** Token pTok has appeared in a MATCH expression where the NEAR operator
7727** is expected. If token pTok does not contain "NEAR", store an error
7728** in the pParse object.
7729*/
7730static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token *pTok){
7731 if( pTok->n!=4 || memcmp("NEAR", pTok->p, 4) ){
7732 sqlite3Fts5ParseError(
7733 pParse, "fts5: syntax error near \"%.*s\"", pTok->n, pTok->p
7734 );
7735 }
7736}
7737
7738static void sqlite3Fts5ParseSetDistance(
7739 Fts5Parse *pParse,
7740 Fts5ExprNearset *pNear,
7741 Fts5Token *p
7742){
7743 if( pNear ){
7744 int nNear = 0;
7745 int i;
7746 if( p->n ){
7747 for(i=0; i<p->n; i++){
7748 char c = (char)p->p[i];
7749 if( c<'0' || c>'9' ){
7750 sqlite3Fts5ParseError(
7751 pParse, "expected integer, got \"%.*s\"", p->n, p->p
7752 );
7753 return;
7754 }
7755 if( nNear<214748363 ) nNear = nNear * 10 + (p->p[i] - '0');
7756 /* ^^^^^^^^^^^^^^^--- Prevent integer overflow */
7757 }
7758 }else{
7759 nNear = FTS5_DEFAULT_NEARDIST10;
7760 }
7761 pNear->nNear = nNear;
7762 }
7763}
7764
7765/*
7766** The second argument passed to this function may be NULL, or it may be
7767** an existing Fts5Colset object. This function returns a pointer to
7768** a new colset object containing the contents of (p) with new value column
7769** number iCol appended.
7770**
7771** If an OOM error occurs, store an error code in pParse and return NULL.
7772** The old colset object (if any) is not freed in this case.
7773*/
7774static Fts5Colset *fts5ParseColset(
7775 Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */
7776 Fts5Colset *p, /* Existing colset object */
7777 int iCol /* New column to add to colset object */
7778){
7779 int nCol = p ? p->nCol : 0; /* Num. columns already in colset object */
7780 Fts5Colset *pNew; /* New colset object to return */
7781
7782 assert( pParse->rc==SQLITE_OK )((void) (0));
7783 assert( iCol>=0 && iCol<pParse->pConfig->nCol )((void) (0));
7784
7785 pNew = sqlite3_realloc64sqlite3_api->realloc64(p, SZ_FTS5COLSET(nCol+1)(sizeof(i64)*((nCol+1 +2)/2)));
7786 if( pNew==0 ){
7787 pParse->rc = SQLITE_NOMEM7;
7788 }else{
7789 int *aiCol = pNew->aiCol;
7790 int i, j;
7791 for(i=0; i<nCol; i++){
7792 if( aiCol[i]==iCol ) return pNew;
7793 if( aiCol[i]>iCol ) break;
7794 }
7795 for(j=nCol; j>i; j--){
7796 aiCol[j] = aiCol[j-1];
7797 }
7798 aiCol[i] = iCol;
7799 pNew->nCol = nCol+1;
7800
7801#ifndef NDEBUG1
7802 /* Check that the array is in order and contains no duplicate entries. */
7803 for(i=1; i<pNew->nCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] )((void) (0));
7804#endif
7805 }
7806
7807 return pNew;
7808}
7809
7810/*
7811** Allocate and return an Fts5Colset object specifying the inverse of
7812** the colset passed as the second argument. Free the colset passed
7813** as the second argument before returning.
7814*/
7815static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){
7816 Fts5Colset *pRet;
7817 int nCol = pParse->pConfig->nCol;
7818
7819 pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc,
7820 SZ_FTS5COLSET(nCol+1)(sizeof(i64)*((nCol+1 +2)/2))
7821 );
7822 if( pRet ){
7823 int i;
7824 int iOld = 0;
7825 for(i=0; i<nCol; i++){
7826 if( iOld>=p->nCol || p->aiCol[iOld]!=i ){
7827 pRet->aiCol[pRet->nCol++] = i;
7828 }else{
7829 iOld++;
7830 }
7831 }
7832 }
7833
7834 sqlite3_freesqlite3_api->free(p);
7835 return pRet;
7836}
7837
7838static Fts5Colset *sqlite3Fts5ParseColset(
7839 Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */
7840 Fts5Colset *pColset, /* Existing colset object */
7841 Fts5Token *p
7842){
7843 Fts5Colset *pRet = 0;
7844 int iCol;
7845 char *z; /* Dequoted copy of token p */
7846
7847 z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n);
7848 if( pParse->rc==SQLITE_OK0 ){
7849 Fts5Config *pConfig = pParse->pConfig;
7850 sqlite3Fts5Dequote(z);
7851 for(iCol=0; iCol<pConfig->nCol; iCol++){
7852 if( 0==sqlite3_stricmpsqlite3_api->stricmp(pConfig->azCol[iCol], z) ) break;
7853 }
7854 if( iCol==pConfig->nCol ){
7855 sqlite3Fts5ParseError(pParse, "no such column: %s", z);
7856 }else{
7857 pRet = fts5ParseColset(pParse, pColset, iCol);
7858 }
7859 sqlite3_freesqlite3_api->free(z);
7860 }
7861
7862 if( pRet==0 ){
7863 assert( pParse->rc!=SQLITE_OK )((void) (0));
7864 sqlite3_freesqlite3_api->free(pColset);
7865 }
7866
7867 return pRet;
7868}
7869
7870/*
7871** If argument pOrig is NULL, or if (*pRc) is set to anything other than
7872** SQLITE_OK when this function is called, NULL is returned.
7873**
7874** Otherwise, a copy of (*pOrig) is made into memory obtained from
7875** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
7876** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
7877*/
7878static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
7879 Fts5Colset *pRet;
7880 if( pOrig ){
7881 sqlite3_int64 nByte = SZ_FTS5COLSET(pOrig->nCol)(sizeof(i64)*((pOrig->nCol+2)/2));
7882 pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
7883 if( pRet ){
7884 memcpy(pRet, pOrig, (size_t)nByte);
7885 }
7886 }else{
7887 pRet = 0;
7888 }
7889 return pRet;
7890}
7891
7892/*
7893** Remove from colset pColset any columns that are not also in colset pMerge.
7894*/
7895static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
7896 int iIn = 0; /* Next input in pColset */
7897 int iMerge = 0; /* Next input in pMerge */
7898 int iOut = 0; /* Next output slot in pColset */
7899
7900 while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
7901 int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
7902 if( iDiff==0 ){
7903 pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
7904 iMerge++;
7905 iIn++;
7906 }else if( iDiff>0 ){
7907 iMerge++;
7908 }else{
7909 iIn++;
7910 }
7911 }
7912 pColset->nCol = iOut;
7913}
7914
7915/*
7916** Recursively apply colset pColset to expression node pNode and all of
7917** its decendents. If (*ppFree) is not NULL, it contains a spare copy
7918** of pColset. This function may use the spare copy and set (*ppFree) to
7919** zero, or it may create copies of pColset using fts5CloneColset().
7920*/
7921static void fts5ParseSetColset(
7922 Fts5Parse *pParse,
7923 Fts5ExprNode *pNode,
7924 Fts5Colset *pColset,
7925 Fts5Colset **ppFree
7926){
7927 if( pParse->rc==SQLITE_OK0 ){
7928 assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING((void) (0))
7929 || pNode->eType==FTS5_AND || pNode->eType==FTS5_OR((void) (0))
7930 || pNode->eType==FTS5_NOT || pNode->eType==FTS5_EOF((void) (0))
7931 )((void) (0));
7932 if( pNode->eType==FTS5_STRING9 || pNode->eType==FTS5_TERM4 ){
7933 Fts5ExprNearset *pNear = pNode->pNear;
7934 if( pNear->pColset ){
7935 fts5MergeColset(pNear->pColset, pColset);
7936 if( pNear->pColset->nCol==0 ){
7937 pNode->eType = FTS5_EOF0;
7938 pNode->xNext = 0;
7939 }
7940 }else if( *ppFree ){
7941 pNear->pColset = pColset;
7942 *ppFree = 0;
7943 }else{
7944 pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
7945 }
7946 }else{
7947 int i;
7948 assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 )((void) (0));
7949 for(i=0; i<pNode->nChild; i++){
7950 fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
7951 }
7952 }
7953 }
7954}
7955
7956/*
7957** Apply colset pColset to expression node pExpr and all of its descendents.
7958*/
7959static void sqlite3Fts5ParseSetColset(
7960 Fts5Parse *pParse,
7961 Fts5ExprNode *pExpr,
7962 Fts5Colset *pColset
7963){
7964 Fts5Colset *pFree = pColset;
7965 if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE1 ){
7966 sqlite3Fts5ParseError(pParse,
7967 "fts5: column queries are not supported (detail=none)"
7968 );
7969 }else{
7970 fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
7971 }
7972 sqlite3_freesqlite3_api->free(pFree);
7973}
7974
7975static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
7976 switch( pNode->eType ){
7977 case FTS5_STRING9: {
7978 Fts5ExprNearset *pNear = pNode->pNear;
7979 if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1
7980 && pNear->apPhrase[0]->aTerm[0].pSynonym==0
7981 && pNear->apPhrase[0]->aTerm[0].bFirst==0
7982 ){
7983 pNode->eType = FTS5_TERM4;
7984 pNode->xNext = fts5ExprNodeNext_TERM;
7985 }else{
7986 pNode->xNext = fts5ExprNodeNext_STRING;
7987 }
7988 break;
7989 };
7990
7991 case FTS5_OR1: {
7992 pNode->xNext = fts5ExprNodeNext_OR;
7993 break;
7994 };
7995
7996 case FTS5_AND2: {
7997 pNode->xNext = fts5ExprNodeNext_AND;
7998 break;
7999 };
8000
8001 default: assert( pNode->eType==FTS5_NOT )((void) (0)); {
8002 pNode->xNext = fts5ExprNodeNext_NOT;
8003 break;
8004 };
8005 }
8006}
8007
8008/*
8009** Add pSub as a child of p.
8010*/
8011static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
8012 int ii = p->nChild;
8013 if( p->eType!=FTS5_NOT3 && pSub->eType==p->eType ){
8014 int nByte = sizeof(Fts5ExprNode*) * pSub->nChild;
8015 memcpy(&p->apChild[p->nChild], pSub->apChild, nByte);
8016 p->nChild += pSub->nChild;
8017 sqlite3_freesqlite3_api->free(pSub);
8018 }else{
8019 p->apChild[p->nChild++] = pSub;
8020 }
8021 for( ; ii<p->nChild; ii++){
8022 p->iHeight = MAX(p->iHeight, p->apChild[ii]->iHeight + 1)(((p->iHeight) > (p->apChild[ii]->iHeight + 1)) ?
(p->iHeight) : (p->apChild[ii]->iHeight + 1));
8023 }
8024}
8025
8026/*
8027** This function is used when parsing LIKE or GLOB patterns against
8028** trigram indexes that specify either detail=column or detail=none.
8029** It converts a phrase:
8030**
8031** abc + def + ghi
8032**
8033** into an AND tree:
8034**
8035** abc AND def AND ghi
8036*/
8037static Fts5ExprNode *fts5ParsePhraseToAnd(
8038 Fts5Parse *pParse,
8039 Fts5ExprNearset *pNear
8040){
8041 int nTerm = pNear->apPhrase[0]->nTerm;
8042 int ii;
8043 int nByte;
8044 Fts5ExprNode *pRet;
8045
8046 assert( pNear->nPhrase==1 )((void) (0));
8047 assert( pParse->bPhraseToAnd )((void) (0));
8048
8049 nByte = SZ_FTS5EXPRNODE(nTerm+1)(__builtin_offsetof(Fts5ExprNode, apChild) + (nTerm+1)*sizeof
(Fts5ExprNode*));
8050 pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte);
8051 if( pRet ){
8052 pRet->eType = FTS5_AND2;
8053 pRet->nChild = nTerm;
8054 pRet->iHeight = 1;
8055 fts5ExprAssignXNext(pRet);
8056 pParse->nPhrase--;
8057 for(ii=0; ii<nTerm; ii++){
8058 Fts5ExprPhrase *pPhrase = (Fts5ExprPhrase*)sqlite3Fts5MallocZero(
8059 &pParse->rc, SZ_FTS5EXPRPHRASE(1)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (1)*sizeof(Fts5ExprTerm
))
8060 );
8061 if( pPhrase ){
8062 if( parseGrowPhraseArray(pParse) ){
8063 fts5ExprPhraseFree(pPhrase);
8064 }else{
8065 Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii];
8066 Fts5ExprTerm *pTo = &pPhrase->aTerm[0];
8067 pParse->apPhrase[pParse->nPhrase++] = pPhrase;
8068 pPhrase->nTerm = 1;
8069 pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm);
8070 pTo->nQueryTerm = p->nQueryTerm;
8071 pTo->nFullTerm = p->nFullTerm;
8072 pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING9,
8073 0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase)
8074 );
8075 }
8076 }
8077 }
8078
8079 if( pParse->rc ){
8080 sqlite3Fts5ParseNodeFree(pRet);
8081 pRet = 0;
8082 }else{
8083 sqlite3Fts5ParseNearsetFree(pNear);
8084 }
8085 }
8086
8087 return pRet;
8088}
8089
8090/*
8091** Allocate and return a new expression object. If anything goes wrong (i.e.
8092** OOM error), leave an error code in pParse and return NULL.
8093*/
8094static Fts5ExprNode *sqlite3Fts5ParseNode(
8095 Fts5Parse *pParse, /* Parse context */
8096 int eType, /* FTS5_STRING, AND, OR or NOT */
8097 Fts5ExprNode *pLeft, /* Left hand child expression */
8098 Fts5ExprNode *pRight, /* Right hand child expression */
8099 Fts5ExprNearset *pNear /* For STRING expressions, the near cluster */
8100){
8101 Fts5ExprNode *pRet = 0;
8102
8103 if( pParse->rc==SQLITE_OK0 ){
8104 int nChild = 0; /* Number of children of returned node */
8105 sqlite3_int64 nByte; /* Bytes of space to allocate for this node */
8106
8107 assert( (eType!=FTS5_STRING && !pNear)((void) (0))
8108 || (eType==FTS5_STRING && !pLeft && !pRight)((void) (0))
8109 )((void) (0));
8110 if( eType==FTS5_STRING9 && pNear==0 ) return 0;
8111 if( eType!=FTS5_STRING9 && pLeft==0 ) return pRight;
8112 if( eType!=FTS5_STRING9 && pRight==0 ) return pLeft;
8113
8114 if( eType==FTS5_STRING9
8115 && pParse->bPhraseToAnd
8116 && pNear->apPhrase[0]->nTerm>1
8117 ){
8118 pRet = fts5ParsePhraseToAnd(pParse, pNear);
8119 }else{
8120 if( eType==FTS5_NOT3 ){
8121 nChild = 2;
8122 }else if( eType==FTS5_AND2 || eType==FTS5_OR1 ){
8123 nChild = 2;
8124 if( pLeft->eType==eType ) nChild += pLeft->nChild-1;
8125 if( pRight->eType==eType ) nChild += pRight->nChild-1;
8126 }
8127
8128 nByte = SZ_FTS5EXPRNODE(nChild)(__builtin_offsetof(Fts5ExprNode, apChild) + (nChild)*sizeof(
Fts5ExprNode*));
8129 pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte);
8130
8131 if( pRet ){
8132 pRet->eType = eType;
8133 pRet->pNear = pNear;
8134 fts5ExprAssignXNext(pRet);
8135 if( eType==FTS5_STRING9 ){
8136 int iPhrase;
8137 for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
8138 pNear->apPhrase[iPhrase]->pNode = pRet;
8139 if( pNear->apPhrase[iPhrase]->nTerm==0 ){
8140 pRet->xNext = 0;
8141 pRet->eType = FTS5_EOF0;
8142 }
8143 }
8144
8145 if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL0 ){
8146 Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
8147 if( pNear->nPhrase!=1
8148 || pPhrase->nTerm>1
8149 || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst)
8150 ){
8151 sqlite3Fts5ParseError(pParse,
8152 "fts5: %s queries are not supported (detail!=full)",
8153 pNear->nPhrase==1 ? "phrase": "NEAR"
8154 );
8155 sqlite3Fts5ParseNodeFree(pRet);
8156 pRet = 0;
8157 pNear = 0;
8158 assert( pLeft==0 && pRight==0 )((void) (0));
8159 }
8160 }
8161 }else{
8162 assert( pNear==0 )((void) (0));
8163 fts5ExprAddChildren(pRet, pLeft);
8164 fts5ExprAddChildren(pRet, pRight);
8165 pLeft = pRight = 0;
8166 if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH256 ){
8167 sqlite3Fts5ParseError(pParse,
8168 "fts5 expression tree is too large (maximum depth %d)",
8169 SQLITE_FTS5_MAX_EXPR_DEPTH256
8170 );
8171 sqlite3Fts5ParseNodeFree(pRet);
8172 pRet = 0;
8173 }
8174 }
8175 }
8176 }
8177 }
8178
8179 if( pRet==0 ){
8180 assert( pParse->rc!=SQLITE_OK )((void) (0));
8181 sqlite3Fts5ParseNodeFree(pLeft);
8182 sqlite3Fts5ParseNodeFree(pRight);
8183 sqlite3Fts5ParseNearsetFree(pNear);
8184 }
8185 return pRet;
8186}
8187
8188static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
8189 Fts5Parse *pParse, /* Parse context */
8190 Fts5ExprNode *pLeft, /* Left hand child expression */
8191 Fts5ExprNode *pRight /* Right hand child expression */
8192){
8193 Fts5ExprNode *pRet = 0;
8194 Fts5ExprNode *pPrev;
8195
8196 if( pParse->rc ){
8197 sqlite3Fts5ParseNodeFree(pLeft);
8198 sqlite3Fts5ParseNodeFree(pRight);
8199 }else{
8200
8201 assert( pLeft->eType==FTS5_STRING((void) (0))
8202 || pLeft->eType==FTS5_TERM((void) (0))
8203 || pLeft->eType==FTS5_EOF((void) (0))
8204 || pLeft->eType==FTS5_AND((void) (0))
8205 )((void) (0));
8206 assert( pRight->eType==FTS5_STRING((void) (0))
8207 || pRight->eType==FTS5_TERM((void) (0))
8208 || pRight->eType==FTS5_EOF((void) (0))
8209 || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd)((void) (0))
8210 )((void) (0));
8211
8212 if( pLeft->eType==FTS5_AND2 ){
8213 pPrev = pLeft->apChild[pLeft->nChild-1];
8214 }else{
8215 pPrev = pLeft;
8216 }
8217 assert( pPrev->eType==FTS5_STRING((void) (0))
8218 || pPrev->eType==FTS5_TERM((void) (0))
8219 || pPrev->eType==FTS5_EOF((void) (0))
8220 )((void) (0));
8221
8222 if( pRight->eType==FTS5_EOF0 ){
8223 assert( pParse->apPhrase!=0 )((void) (0));
8224 assert( pParse->nPhrase>0 )((void) (0));
8225 assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] )((void) (0));
8226 sqlite3Fts5ParseNodeFree(pRight);
8227 pRet = pLeft;
8228 pParse->nPhrase--;
8229 }
8230 else if( pPrev->eType==FTS5_EOF0 ){
8231 Fts5ExprPhrase **ap;
8232
8233 if( pPrev==pLeft ){
8234 pRet = pRight;
8235 }else{
8236 pLeft->apChild[pLeft->nChild-1] = pRight;
8237 pRet = pLeft;
8238 }
8239
8240 ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase];
8241 assert( ap[0]==pPrev->pNear->apPhrase[0] )((void) (0));
8242 memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase);
8243 pParse->nPhrase--;
8244
8245 sqlite3Fts5ParseNodeFree(pPrev);
8246 }
8247 else{
8248 pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND2, pLeft, pRight, 0);
8249 }
8250 }
8251
8252 return pRet;
8253}
8254
8255#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
8256static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
8257 sqlite3_int64 nByte = 0;
8258 Fts5ExprTerm *p;
8259 char *zQuoted;
8260
8261 /* Determine the maximum amount of space required. */
8262 for(p=pTerm; p; p=p->pSynonym){
8263 nByte += pTerm->nQueryTerm * 2 + 3 + 2;
8264 }
8265 zQuoted = sqlite3_malloc64sqlite3_api->malloc64(nByte);
8266
8267 if( zQuoted ){
8268 int i = 0;
8269 for(p=pTerm; p; p=p->pSynonym){
8270 char *zIn = p->pTerm;
8271 char *zEnd = &zIn[p->nQueryTerm];
8272 zQuoted[i++] = '"';
8273 while( zIn<zEnd ){
8274 if( *zIn=='"' ) zQuoted[i++] = '"';
8275 zQuoted[i++] = *zIn++;
8276 }
8277 zQuoted[i++] = '"';
8278 if( p->pSynonym ) zQuoted[i++] = '|';
8279 }
8280 if( pTerm->bPrefix ){
8281 zQuoted[i++] = ' ';
8282 zQuoted[i++] = '*';
8283 }
8284 zQuoted[i++] = '\0';
8285 }
8286 return zQuoted;
8287}
8288
8289static char *fts5PrintfAppend(char *zApp, const char *zFmt, ...){
8290 char *zNew;
8291 va_list ap;
8292 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
8293 zNew = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
8294 va_end(ap)__builtin_va_end(ap);
8295 if( zApp && zNew ){
8296 char *zNew2 = sqlite3_mprintfsqlite3_api->mprintf("%s%s", zApp, zNew);
8297 sqlite3_freesqlite3_api->free(zNew);
8298 zNew = zNew2;
8299 }
8300 sqlite3_freesqlite3_api->free(zApp);
8301 return zNew;
8302}
8303
8304/*
8305** Compose a tcl-readable representation of expression pExpr. Return a
8306** pointer to a buffer containing that representation. It is the
8307** responsibility of the caller to at some point free the buffer using
8308** sqlite3_free().
8309*/
8310static char *fts5ExprPrintTcl(
8311 Fts5Config *pConfig,
8312 const char *zNearsetCmd,
8313 Fts5ExprNode *pExpr
8314){
8315 char *zRet = 0;
8316 if( pExpr->eType==FTS5_STRING9 || pExpr->eType==FTS5_TERM4 ){
8317 Fts5ExprNearset *pNear = pExpr->pNear;
8318 int i;
8319 int iTerm;
8320
8321 zRet = fts5PrintfAppend(zRet, "%s ", zNearsetCmd);
8322 if( zRet==0 ) return 0;
8323 if( pNear->pColset ){
8324 int *aiCol = pNear->pColset->aiCol;
8325 int nCol = pNear->pColset->nCol;
8326 if( nCol==1 ){
8327 zRet = fts5PrintfAppend(zRet, "-col %d ", aiCol[0]);
8328 }else{
8329 zRet = fts5PrintfAppend(zRet, "-col {%d", aiCol[0]);
8330 for(i=1; i<pNear->pColset->nCol; i++){
8331 zRet = fts5PrintfAppend(zRet, " %d", aiCol[i]);
8332 }
8333 zRet = fts5PrintfAppend(zRet, "} ");
8334 }
8335 if( zRet==0 ) return 0;
8336 }
8337
8338 if( pNear->nPhrase>1 ){
8339 zRet = fts5PrintfAppend(zRet, "-near %d ", pNear->nNear);
8340 if( zRet==0 ) return 0;
8341 }
8342
8343 zRet = fts5PrintfAppend(zRet, "--");
8344 if( zRet==0 ) return 0;
8345
8346 for(i=0; i<pNear->nPhrase; i++){
8347 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
8348
8349 zRet = fts5PrintfAppend(zRet, " {");
8350 for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
8351 Fts5ExprTerm *p = &pPhrase->aTerm[iTerm];
8352 zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ",
8353 p->nQueryTerm, p->pTerm
8354 );
8355 if( pPhrase->aTerm[iTerm].bPrefix ){
8356 zRet = fts5PrintfAppend(zRet, "*");
8357 }
8358 }
8359
8360 if( zRet ) zRet = fts5PrintfAppend(zRet, "}");
8361 if( zRet==0 ) return 0;
8362 }
8363
8364 }else if( pExpr->eType==0 ){
8365 zRet = sqlite3_mprintfsqlite3_api->mprintf("{}");
8366 }else{
8367 char const *zOp = 0;
8368 int i;
8369 switch( pExpr->eType ){
8370 case FTS5_AND2: zOp = "AND"; break;
8371 case FTS5_NOT3: zOp = "NOT"; break;
8372 default:
8373 assert( pExpr->eType==FTS5_OR )((void) (0));
8374 zOp = "OR";
8375 break;
8376 }
8377
8378 zRet = sqlite3_mprintfsqlite3_api->mprintf("%s", zOp);
8379 for(i=0; zRet && i<pExpr->nChild; i++){
8380 char *z = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->apChild[i]);
8381 if( !z ){
8382 sqlite3_freesqlite3_api->free(zRet);
8383 zRet = 0;
8384 }else{
8385 zRet = fts5PrintfAppend(zRet, " [%z]", z);
8386 }
8387 }
8388 }
8389
8390 return zRet;
8391}
8392
8393static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){
8394 char *zRet = 0;
8395 if( pExpr->eType==0 ){
8396 return sqlite3_mprintfsqlite3_api->mprintf("\"\"");
8397 }else
8398 if( pExpr->eType==FTS5_STRING9 || pExpr->eType==FTS5_TERM4 ){
8399 Fts5ExprNearset *pNear = pExpr->pNear;
8400 int i;
8401 int iTerm;
8402
8403 if( pNear->pColset ){
8404 int ii;
8405 Fts5Colset *pColset = pNear->pColset;
8406 if( pColset->nCol>1 ) zRet = fts5PrintfAppend(zRet, "{");
8407 for(ii=0; ii<pColset->nCol; ii++){
8408 zRet = fts5PrintfAppend(zRet, "%s%s",
8409 pConfig->azCol[pColset->aiCol[ii]], ii==pColset->nCol-1 ? "" : " "
8410 );
8411 }
8412 if( zRet ){
8413 zRet = fts5PrintfAppend(zRet, "%s : ", pColset->nCol>1 ? "}" : "");
8414 }
8415 if( zRet==0 ) return 0;
8416 }
8417
8418 if( pNear->nPhrase>1 ){
8419 zRet = fts5PrintfAppend(zRet, "NEAR(");
8420 if( zRet==0 ) return 0;
8421 }
8422
8423 for(i=0; i<pNear->nPhrase; i++){
8424 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
8425 if( i!=0 ){
8426 zRet = fts5PrintfAppend(zRet, " ");
8427 if( zRet==0 ) return 0;
8428 }
8429 for(iTerm=0; iTerm<pPhrase->nTerm; iTerm++){
8430 char *zTerm = fts5ExprTermPrint(&pPhrase->aTerm[iTerm]);
8431 if( zTerm ){
8432 zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" + ", zTerm);
8433 sqlite3_freesqlite3_api->free(zTerm);
8434 }
8435 if( zTerm==0 || zRet==0 ){
8436 sqlite3_freesqlite3_api->free(zRet);
8437 return 0;
8438 }
8439 }
8440 }
8441
8442 if( pNear->nPhrase>1 ){
8443 zRet = fts5PrintfAppend(zRet, ", %d)", pNear->nNear);
8444 if( zRet==0 ) return 0;
8445 }
8446
8447 }else{
8448 char const *zOp = 0;
8449 int i;
8450
8451 switch( pExpr->eType ){
8452 case FTS5_AND2: zOp = " AND "; break;
8453 case FTS5_NOT3: zOp = " NOT "; break;
8454 default:
8455 assert( pExpr->eType==FTS5_OR )((void) (0));
8456 zOp = " OR ";
8457 break;
8458 }
8459
8460 for(i=0; i<pExpr->nChild; i++){
8461 char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]);
8462 if( z==0 ){
8463 sqlite3_freesqlite3_api->free(zRet);
8464 zRet = 0;
8465 }else{
8466 int e = pExpr->apChild[i]->eType;
8467 int b = (e!=FTS5_STRING9 && e!=FTS5_TERM4 && e!=FTS5_EOF0);
8468 zRet = fts5PrintfAppend(zRet, "%s%s%z%s",
8469 (i==0 ? "" : zOp),
8470 (b?"(":""), z, (b?")":"")
8471 );
8472 }
8473 if( zRet==0 ) break;
8474 }
8475 }
8476
8477 return zRet;
8478}
8479
8480/*
8481** The implementation of user-defined scalar functions fts5_expr() (bTcl==0)
8482** and fts5_expr_tcl() (bTcl!=0).
8483*/
8484static void fts5ExprFunction(
8485 sqlite3_context *pCtx, /* Function call context */
8486 int nArg, /* Number of args */
8487 sqlite3_value **apVal, /* Function arguments */
8488 int bTcl
8489){
8490 Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_datasqlite3_api->user_data(pCtx);
8491 sqlite3 *db = sqlite3_context_db_handlesqlite3_api->context_db_handle(pCtx);
8492 const char *zExpr = 0;
8493 char *zErr = 0;
8494 Fts5Expr *pExpr = 0;
8495 int rc;
8496 int i;
8497
8498 const char **azConfig; /* Array of arguments for Fts5Config */
8499 const char *zNearsetCmd = "nearset";
8500 int nConfig; /* Size of azConfig[] */
8501 Fts5Config *pConfig = 0;
8502 int iArg = 1;
8503
8504 if( nArg<1 ){
8505 zErr = sqlite3_mprintfsqlite3_api->mprintf("wrong number of arguments to function %s",
8506 bTcl ? "fts5_expr_tcl" : "fts5_expr"
8507 );
8508 sqlite3_result_errorsqlite3_api->result_error(pCtx, zErr, -1);
8509 sqlite3_freesqlite3_api->free(zErr);
8510 return;
8511 }
8512
8513 if( bTcl && nArg>1 ){
8514 zNearsetCmd = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[1]);
8515 iArg = 2;
8516 }
8517
8518 nConfig = 3 + (nArg-iArg);
8519 azConfig = (const char**)sqlite3_malloc64sqlite3_api->malloc64(sizeof(char*) * nConfig);
8520 if( azConfig==0 ){
8521 sqlite3_result_error_nomemsqlite3_api->result_error_nomem(pCtx);
8522 return;
8523 }
8524 azConfig[0] = 0;
8525 azConfig[1] = "main";
8526 azConfig[2] = "tbl";
8527 for(i=3; iArg<nArg; iArg++){
8528 const char *z = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[iArg]);
8529 azConfig[i++] = (z ? z : "");
8530 }
8531
8532 zExpr = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[0]);
8533 if( zExpr==0 ) zExpr = "";
8534
8535 rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
8536 if( rc==SQLITE_OK0 ){
8537 rc = sqlite3Fts5ExprNew(pConfig, 0, pConfig->nCol, zExpr, &pExpr, &zErr);
8538 }
8539 if( rc==SQLITE_OK0 ){
8540 char *zText;
8541 if( pExpr->pRoot->xNext==0 ){
8542 zText = sqlite3_mprintfsqlite3_api->mprintf("");
8543 }else if( bTcl ){
8544 zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
8545 }else{
8546 zText = fts5ExprPrint(pConfig, pExpr->pRoot);
8547 }
8548 if( zText==0 ){
8549 rc = SQLITE_NOMEM7;
8550 }else{
8551 sqlite3_result_textsqlite3_api->result_text(pCtx, zText, -1, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
8552 sqlite3_freesqlite3_api->free(zText);
8553 }
8554 }
8555
8556 if( rc!=SQLITE_OK0 ){
8557 if( zErr ){
8558 sqlite3_result_errorsqlite3_api->result_error(pCtx, zErr, -1);
8559 sqlite3_freesqlite3_api->free(zErr);
8560 }else{
8561 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
8562 }
8563 }
8564 sqlite3_freesqlite3_api->free((void *)azConfig);
8565 sqlite3Fts5ConfigFree(pConfig);
8566 sqlite3Fts5ExprFree(pExpr);
8567}
8568
8569static void fts5ExprFunctionHr(
8570 sqlite3_context *pCtx, /* Function call context */
8571 int nArg, /* Number of args */
8572 sqlite3_value **apVal /* Function arguments */
8573){
8574 fts5ExprFunction(pCtx, nArg, apVal, 0);
8575}
8576static void fts5ExprFunctionTcl(
8577 sqlite3_context *pCtx, /* Function call context */
8578 int nArg, /* Number of args */
8579 sqlite3_value **apVal /* Function arguments */
8580){
8581 fts5ExprFunction(pCtx, nArg, apVal, 1);
8582}
8583
8584/*
8585** The implementation of an SQLite user-defined-function that accepts a
8586** single integer as an argument. If the integer is an alpha-numeric
8587** unicode code point, 1 is returned. Otherwise 0.
8588*/
8589static void fts5ExprIsAlnum(
8590 sqlite3_context *pCtx, /* Function call context */
8591 int nArg, /* Number of args */
8592 sqlite3_value **apVal /* Function arguments */
8593){
8594 int iCode;
8595 u8 aArr[32];
8596 if( nArg!=1 ){
8597 sqlite3_result_errorsqlite3_api->result_error(pCtx,
8598 "wrong number of arguments to function fts5_isalnum", -1
8599 );
8600 return;
8601 }
8602 memset(aArr, 0, sizeof(aArr));
8603 sqlite3Fts5UnicodeCatParse("L*", aArr);
8604 sqlite3Fts5UnicodeCatParse("N*", aArr);
8605 sqlite3Fts5UnicodeCatParse("Co", aArr);
8606 iCode = sqlite3_value_intsqlite3_api->value_int(apVal[0]);
8607 sqlite3_result_intsqlite3_api->result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory((u32)iCode)]);
8608}
8609
8610static void fts5ExprFold(
8611 sqlite3_context *pCtx, /* Function call context */
8612 int nArg, /* Number of args */
8613 sqlite3_value **apVal /* Function arguments */
8614){
8615 if( nArg!=1 && nArg!=2 ){
8616 sqlite3_result_errorsqlite3_api->result_error(pCtx,
8617 "wrong number of arguments to function fts5_fold", -1
8618 );
8619 }else{
8620 int iCode;
8621 int bRemoveDiacritics = 0;
8622 iCode = sqlite3_value_intsqlite3_api->value_int(apVal[0]);
8623 if( nArg==2 ) bRemoveDiacritics = sqlite3_value_intsqlite3_api->value_int(apVal[1]);
8624 sqlite3_result_intsqlite3_api->result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics));
8625 }
8626}
8627#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */
8628
8629/*
8630** This is called during initialization to register the fts5_expr() scalar
8631** UDF with the SQLite handle passed as the only argument.
8632*/
8633static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
8634#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
8635 struct Fts5ExprFunc {
8636 const char *z;
8637 void (*x)(sqlite3_context*,int,sqlite3_value**);
8638 } aFunc[] = {
8639 { "fts5_expr", fts5ExprFunctionHr },
8640 { "fts5_expr_tcl", fts5ExprFunctionTcl },
8641 { "fts5_isalnum", fts5ExprIsAlnum },
8642 { "fts5_fold", fts5ExprFold },
8643 };
8644 int i;
8645 int rc = SQLITE_OK0;
8646 void *pCtx = (void*)pGlobal;
8647
8648 for(i=0; rc==SQLITE_OK0 && i<ArraySize(aFunc)((int)(sizeof(aFunc) / sizeof(aFunc[0]))); i++){
8649 struct Fts5ExprFunc *p = &aFunc[i];
8650 rc = sqlite3_create_functionsqlite3_api->create_function(db, p->z, -1, SQLITE_UTF81, pCtx, p->x, 0, 0);
8651 }
8652#else
8653 int rc = SQLITE_OK0;
8654 UNUSED_PARAM2(pGlobal,db)(void)(pGlobal), (void)(db);
8655#endif
8656
8657 /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and
8658 ** sqlite3Fts5ParserFallback() are unused */
8659#ifndef NDEBUG1
8660 (void)sqlite3Fts5ParserTrace;
8661#endif
8662 (void)sqlite3Fts5ParserFallback;
8663
8664 return rc;
8665}
8666
8667/*
8668** Return the number of phrases in expression pExpr.
8669*/
8670static int sqlite3Fts5ExprPhraseCount(Fts5Expr *pExpr){
8671 return (pExpr ? pExpr->nPhrase : 0);
8672}
8673
8674/*
8675** Return the number of terms in the iPhrase'th phrase in pExpr.
8676*/
8677static int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){
8678 if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0;
8679 return pExpr->apExprPhrase[iPhrase]->nTerm;
8680}
8681
8682/*
8683** This function is used to access the current position list for phrase
8684** iPhrase.
8685*/
8686static int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){
8687 int nRet;
8688 Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
8689 Fts5ExprNode *pNode = pPhrase->pNode;
8690 if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){
8691 *pa = pPhrase->poslist.p;
8692 nRet = pPhrase->poslist.n;
8693 }else{
8694 *pa = 0;
8695 nRet = 0;
8696 }
8697 return nRet;
8698}
8699
8700struct Fts5PoslistPopulator {
8701 Fts5PoslistWriter writer;
8702 int bOk; /* True if ok to populate */
8703 int bMiss;
8704};
8705
8706/*
8707** Clear the position lists associated with all phrases in the expression
8708** passed as the first argument. Argument bLive is true if the expression
8709** might be pointing to a real entry, otherwise it has just been reset.
8710**
8711** At present this function is only used for detail=col and detail=none
8712** fts5 tables. This implies that all phrases must be at most 1 token
8713** in size, as phrase matches are not supported without detail=full.
8714*/
8715static Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){
8716 Fts5PoslistPopulator *pRet;
8717 pRet = sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
8718 if( pRet ){
8719 int i;
8720 memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
8721 for(i=0; i<pExpr->nPhrase; i++){
8722 Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
8723 Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
8724 assert( pExpr->apExprPhrase[i]->nTerm<=1 )((void) (0));
8725 if( bLive &&
8726 (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof)
8727 ){
8728 pRet[i].bMiss = 1;
8729 }else{
8730 pBuf->n = 0;
8731 }
8732 }
8733 }
8734 return pRet;
8735}
8736
8737struct Fts5ExprCtx {
8738 Fts5Expr *pExpr;
8739 Fts5PoslistPopulator *aPopulator;
8740 i64 iOff;
8741};
8742typedef struct Fts5ExprCtx Fts5ExprCtx;
8743
8744/*
8745** TODO: Make this more efficient!
8746*/
8747static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
8748 int i;
8749 for(i=0; i<pColset->nCol; i++){
8750 if( pColset->aiCol[i]==iCol ) return 1;
8751 }
8752 return 0;
8753}
8754
8755/*
8756** pToken is a buffer nToken bytes in size that may or may not contain
8757** an embedded 0x00 byte. If it does, return the number of bytes in
8758** the buffer before the 0x00. If it does not, return nToken.
8759*/
8760static int fts5QueryTerm(const char *pToken, int nToken){
8761 int ii;
8762 for(ii=0; ii<nToken && pToken[ii]; ii++){}
8763 return ii;
8764}
8765
8766static int fts5ExprPopulatePoslistsCb(
8767 void *pCtx, /* Copy of 2nd argument to xTokenize() */
8768 int tflags, /* Mask of FTS5_TOKEN_* flags */
8769 const char *pToken, /* Pointer to buffer containing token */
8770 int nToken, /* Size of token in bytes */
8771 int iUnused1, /* Byte offset of token within input text */
8772 int iUnused2 /* Byte offset of end of token within input text */
8773){
8774 Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
8775 Fts5Expr *pExpr = p->pExpr;
8776 int i;
8777 int nQuery = nToken;
8778 i64 iRowid = pExpr->pRoot->iRowid;
8779
8780 UNUSED_PARAM2(iUnused1, iUnused2)(void)(iUnused1), (void)(iUnused2);
8781
8782 if( nQuery>FTS5_MAX_TOKEN_SIZE32768 ) nQuery = FTS5_MAX_TOKEN_SIZE32768;
8783 if( pExpr->pConfig->bTokendata ){
8784 nQuery = fts5QueryTerm(pToken, nQuery);
8785 }
8786 if( (tflags & FTS5_TOKEN_COLOCATED0x0001)==0 ) p->iOff++;
8787 for(i=0; i<pExpr->nPhrase; i++){
8788 Fts5ExprTerm *pT;
8789 if( p->aPopulator[i].bOk==0 ) continue;
8790 for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
8791 if( (pT->nQueryTerm==nQuery || (pT->nQueryTerm<nQuery && pT->bPrefix))
8792 && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0
8793 ){
8794 int rc = sqlite3Fts5PoslistWriterAppend(
8795 &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
8796 );
8797 if( rc==SQLITE_OK0 && (pExpr->pConfig->bTokendata || pT->bPrefix) ){
8798 int iCol = p->iOff>>32;
8799 int iTokOff = p->iOff & 0x7FFFFFFF;
8800 rc = sqlite3Fts5IndexIterWriteTokendata(
8801 pT->pIter, pToken, nToken, iRowid, iCol, iTokOff
8802 );
8803 }
8804 if( rc ) return rc;
8805 break;
8806 }
8807 }
8808 }
8809 return SQLITE_OK0;
8810}
8811
8812static int sqlite3Fts5ExprPopulatePoslists(
8813 Fts5Config *pConfig,
8814 Fts5Expr *pExpr,
8815 Fts5PoslistPopulator *aPopulator,
8816 int iCol,
8817 const char *z, int n
8818){
8819 int i;
8820 Fts5ExprCtx sCtx;
8821 sCtx.pExpr = pExpr;
8822 sCtx.aPopulator = aPopulator;
8823 sCtx.iOff = (((i64)iCol) << 32) - 1;
8824
8825 for(i=0; i<pExpr->nPhrase; i++){
8826 Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
8827 Fts5Colset *pColset = pNode->pNear->pColset;
8828 if( (pColset && 0==fts5ExprColsetTest(pColset, iCol))
8829 || aPopulator[i].bMiss
8830 ){
8831 aPopulator[i].bOk = 0;
8832 }else{
8833 aPopulator[i].bOk = 1;
8834 }
8835 }
8836
8837 return sqlite3Fts5Tokenize(pConfig,
8838 FTS5_TOKENIZE_DOCUMENT0x0004, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
8839 );
8840}
8841
8842static void fts5ExprClearPoslists(Fts5ExprNode *pNode){
8843 if( pNode->eType==FTS5_TERM4 || pNode->eType==FTS5_STRING9 ){
8844 pNode->pNear->apPhrase[0]->poslist.n = 0;
8845 }else{
8846 int i;
8847 for(i=0; i<pNode->nChild; i++){
8848 fts5ExprClearPoslists(pNode->apChild[i]);
8849 }
8850 }
8851}
8852
8853static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
8854 pNode->iRowid = iRowid;
8855 pNode->bEof = 0;
8856 switch( pNode->eType ){
8857 case 0:
8858 case FTS5_TERM4:
8859 case FTS5_STRING9:
8860 return (pNode->pNear->apPhrase[0]->poslist.n>0);
8861
8862 case FTS5_AND2: {
8863 int i;
8864 for(i=0; i<pNode->nChild; i++){
8865 if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
8866 fts5ExprClearPoslists(pNode);
8867 return 0;
8868 }
8869 }
8870 break;
8871 }
8872
8873 case FTS5_OR1: {
8874 int i;
8875 int bRet = 0;
8876 for(i=0; i<pNode->nChild; i++){
8877 if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
8878 bRet = 1;
8879 }
8880 }
8881 return bRet;
8882 }
8883
8884 default: {
8885 assert( pNode->eType==FTS5_NOT )((void) (0));
8886 if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
8887 || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
8888 ){
8889 fts5ExprClearPoslists(pNode);
8890 return 0;
8891 }
8892 break;
8893 }
8894 }
8895 return 1;
8896}
8897
8898static void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
8899 fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
8900}
8901
8902/*
8903** This function is only called for detail=columns tables.
8904*/
8905static int sqlite3Fts5ExprPhraseCollist(
8906 Fts5Expr *pExpr,
8907 int iPhrase,
8908 const u8 **ppCollist,
8909 int *pnCollist
8910){
8911 Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
8912 Fts5ExprNode *pNode = pPhrase->pNode;
8913 int rc = SQLITE_OK0;
8914
8915 assert( iPhrase>=0 && iPhrase<pExpr->nPhrase )((void) (0));
8916 assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS )((void) (0));
8917
8918 if( pNode->bEof==0
8919 && pNode->iRowid==pExpr->pRoot->iRowid
8920 && pPhrase->poslist.n>0
8921 ){
8922 Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
8923 if( pTerm->pSynonym ){
8924 Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1];
8925 rc = fts5ExprSynonymList(
8926 pTerm, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist
8927 );
8928 }else{
8929 *ppCollist = pPhrase->aTerm[0].pIter->pData;
8930 *pnCollist = pPhrase->aTerm[0].pIter->nData;
8931 }
8932 }else{
8933 *ppCollist = 0;
8934 *pnCollist = 0;
8935 }
8936
8937 return rc;
8938}
8939
8940/*
8941** Does the work of the fts5_api.xQueryToken() API method.
8942*/
8943static int sqlite3Fts5ExprQueryToken(
8944 Fts5Expr *pExpr,
8945 int iPhrase,
8946 int iToken,
8947 const char **ppOut,
8948 int *pnOut
8949){
8950 Fts5ExprPhrase *pPhrase = 0;
8951
8952 if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
8953 return SQLITE_RANGE25;
8954 }
8955 pPhrase = pExpr->apExprPhrase[iPhrase];
8956 if( iToken<0 || iToken>=pPhrase->nTerm ){
8957 return SQLITE_RANGE25;
8958 }
8959
8960 *ppOut = pPhrase->aTerm[iToken].pTerm;
8961 *pnOut = pPhrase->aTerm[iToken].nFullTerm;
8962 return SQLITE_OK0;
8963}
8964
8965/*
8966** Does the work of the fts5_api.xInstToken() API method.
8967*/
8968static int sqlite3Fts5ExprInstToken(
8969 Fts5Expr *pExpr,
8970 i64 iRowid,
8971 int iPhrase,
8972 int iCol,
8973 int iOff,
8974 int iToken,
8975 const char **ppOut,
8976 int *pnOut
8977){
8978 Fts5ExprPhrase *pPhrase = 0;
8979 Fts5ExprTerm *pTerm = 0;
8980 int rc = SQLITE_OK0;
8981
8982 if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
8983 return SQLITE_RANGE25;
8984 }
8985 pPhrase = pExpr->apExprPhrase[iPhrase];
8986 if( iToken<0 || iToken>=pPhrase->nTerm ){
8987 return SQLITE_RANGE25;
8988 }
8989 pTerm = &pPhrase->aTerm[iToken];
8990 if( pExpr->pConfig->bTokendata || pTerm->bPrefix ){
8991 rc = sqlite3Fts5IterToken(
8992 pTerm->pIter, pTerm->pTerm, pTerm->nQueryTerm,
8993 iRowid, iCol, iOff+iToken, ppOut, pnOut
8994 );
8995 }else{
8996 *ppOut = pTerm->pTerm;
8997 *pnOut = pTerm->nFullTerm;
8998 }
8999 return rc;
9000}
9001
9002/*
9003** Clear the token mappings for all Fts5IndexIter objects managed by
9004** the expression passed as the only argument.
9005*/
9006static void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){
9007 int ii;
9008 for(ii=0; ii<pExpr->nPhrase; ii++){
9009 Fts5ExprTerm *pT;
9010 for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){
9011 sqlite3Fts5IndexIterClearTokendata(pT->pIter);
9012 }
9013 }
9014}
9015
9016#line 1 "fts5_hash.c"
9017/*
9018** 2014 August 11
9019**
9020** The author disclaims copyright to this source code. In place of
9021** a legal notice, here is a blessing:
9022**
9023** May you do good and not evil.
9024** May you find forgiveness for yourself and forgive others.
9025** May you share freely, never taking more than you give.
9026**
9027******************************************************************************
9028**
9029*/
9030
9031
9032
9033/* #include "fts5Int.h" */
9034
9035typedef struct Fts5HashEntry Fts5HashEntry;
9036
9037/*
9038** This file contains the implementation of an in-memory hash table used
9039** to accumulate "term -> doclist" content before it is flushed to a level-0
9040** segment.
9041*/
9042
9043
9044struct Fts5Hash {
9045 int eDetail; /* Copy of Fts5Config.eDetail */
9046 int *pnByte; /* Pointer to bytes counter */
9047 int nEntry; /* Number of entries currently in hash */
9048 int nSlot; /* Size of aSlot[] array */
9049 Fts5HashEntry *pScan; /* Current ordered scan item */
9050 Fts5HashEntry **aSlot; /* Array of hash slots */
9051};
9052
9053/*
9054** Each entry in the hash table is represented by an object of the
9055** following type. Each object, its key, and its current data are stored
9056** in a single memory allocation. The key immediately follows the object
9057** in memory. The position list data immediately follows the key data
9058** in memory.
9059**
9060** The key is Fts5HashEntry.nKey bytes in size. It consists of a single
9061** byte identifying the index (either the main term index or a prefix-index),
9062** followed by the term data. For example: "0token". There is no
9063** nul-terminator - in this case nKey=6.
9064**
9065** The data that follows the key is in a similar, but not identical format
9066** to the doclist data stored in the database. It is:
9067**
9068** * Rowid, as a varint
9069** * Position list, without 0x00 terminator.
9070** * Size of previous position list and rowid, as a 4 byte
9071** big-endian integer.
9072**
9073** iRowidOff:
9074** Offset of last rowid written to data area. Relative to first byte of
9075** structure.
9076**
9077** nData:
9078** Bytes of data written since iRowidOff.
9079*/
9080struct Fts5HashEntry {
9081 Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */
9082 Fts5HashEntry *pScanNext; /* Next entry in sorted order */
9083
9084 int nAlloc; /* Total size of allocation */
9085 int iSzPoslist; /* Offset of space for 4-byte poslist size */
9086 int nData; /* Total bytes of data (incl. structure) */
9087 int nKey; /* Length of key in bytes */
9088 u8 bDel; /* Set delete-flag @ iSzPoslist */
9089 u8 bContent; /* Set content-flag (detail=none mode) */
9090 i16 iCol; /* Column of last value written */
9091 int iPos; /* Position of last value written */
9092 i64 iRowid; /* Rowid of last value written */
9093};
9094
9095/*
9096** Equivalent to:
9097**
9098** char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; }
9099*/
9100#define fts5EntryKey(p)( ((char *)(&(p)[1])) ) ( ((char *)(&(p)[1])) )
9101
9102
9103/*
9104** Allocate a new hash table.
9105*/
9106static int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){
9107 int rc = SQLITE_OK0;
9108 Fts5Hash *pNew;
9109
9110 *ppNew = pNew = (Fts5Hash*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5Hash));
9111 if( pNew==0 ){
9112 rc = SQLITE_NOMEM7;
9113 }else{
9114 sqlite3_int64 nByte;
9115 memset(pNew, 0, sizeof(Fts5Hash));
9116 pNew->pnByte = pnByte;
9117 pNew->eDetail = pConfig->eDetail;
9118
9119 pNew->nSlot = 1024;
9120 nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
9121 pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc64sqlite3_api->malloc64(nByte);
9122 if( pNew->aSlot==0 ){
9123 sqlite3_freesqlite3_api->free(pNew);
9124 *ppNew = 0;
9125 rc = SQLITE_NOMEM7;
9126 }else{
9127 memset(pNew->aSlot, 0, (size_t)nByte);
9128 }
9129 }
9130 return rc;
9131}
9132
9133/*
9134** Free a hash table object.
9135*/
9136static void sqlite3Fts5HashFree(Fts5Hash *pHash){
9137 if( pHash ){
9138 sqlite3Fts5HashClear(pHash);
9139 sqlite3_freesqlite3_api->free(pHash->aSlot);
9140 sqlite3_freesqlite3_api->free(pHash);
9141 }
9142}
9143
9144/*
9145** Empty (but do not delete) a hash table.
9146*/
9147static void sqlite3Fts5HashClear(Fts5Hash *pHash){
9148 int i;
9149 for(i=0; i<pHash->nSlot; i++){
9150 Fts5HashEntry *pNext;
9151 Fts5HashEntry *pSlot;
9152 for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){
9153 pNext = pSlot->pHashNext;
9154 sqlite3_freesqlite3_api->free(pSlot);
9155 }
9156 }
9157 memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*));
9158 pHash->nEntry = 0;
9159}
9160
9161static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){
9162 int i;
9163 unsigned int h = 13;
9164 for(i=n-1; i>=0; i--){
9165 h = (h << 3) ^ h ^ p[i];
9166 }
9167 return (h % nSlot);
9168}
9169
9170static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){
9171 int i;
9172 unsigned int h = 13;
9173 for(i=n-1; i>=0; i--){
9174 h = (h << 3) ^ h ^ p[i];
9175 }
9176 h = (h << 3) ^ h ^ b;
9177 return (h % nSlot);
9178}
9179
9180/*
9181** Resize the hash table by doubling the number of slots.
9182*/
9183static int fts5HashResize(Fts5Hash *pHash){
9184 int nNew = pHash->nSlot*2;
9185 int i;
9186 Fts5HashEntry **apNew;
9187 Fts5HashEntry **apOld = pHash->aSlot;
9188
9189 apNew = (Fts5HashEntry**)sqlite3_malloc64sqlite3_api->malloc64(nNew*sizeof(Fts5HashEntry*));
9190 if( !apNew ) return SQLITE_NOMEM7;
9191 memset(apNew, 0, nNew*sizeof(Fts5HashEntry*));
9192
9193 for(i=0; i<pHash->nSlot; i++){
9194 while( apOld[i] ){
9195 unsigned int iHash;
9196 Fts5HashEntry *p = apOld[i];
9197 apOld[i] = p->pHashNext;
9198 iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p)( ((char *)(&(p)[1])) ), p->nKey);
9199 p->pHashNext = apNew[iHash];
9200 apNew[iHash] = p;
9201 }
9202 }
9203
9204 sqlite3_freesqlite3_api->free(apOld);
9205 pHash->nSlot = nNew;
9206 pHash->aSlot = apNew;
9207 return SQLITE_OK0;
9208}
9209
9210static int fts5HashAddPoslistSize(
9211 Fts5Hash *pHash,
9212 Fts5HashEntry *p,
9213 Fts5HashEntry *p2
9214){
9215 int nRet = 0;
9216 if( p->iSzPoslist ){
9217 u8 *pPtr = p2 ? (u8*)p2 : (u8*)p;
9218 int nData = p->nData;
9219 if( pHash->eDetail==FTS5_DETAIL_NONE1 ){
9220 assert( nData==p->iSzPoslist )((void) (0));
9221 if( p->bDel ){
9222 pPtr[nData++] = 0x00;
9223 if( p->bContent ){
9224 pPtr[nData++] = 0x00;
9225 }
9226 }
9227 }else{
9228 int nSz = (nData - p->iSzPoslist - 1); /* Size in bytes */
9229 int nPos = nSz*2 + p->bDel; /* Value of nPos field */
9230
9231 assert( p->bDel==0 || p->bDel==1 )((void) (0));
9232 if( nPos<=127 ){
9233 pPtr[p->iSzPoslist] = (u8)nPos;
9234 }else{
9235 int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
9236 memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
9237 sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
9238 nData += (nByte-1);
9239 }
9240 }
9241
9242 nRet = nData - p->nData;
9243 if( p2==0 ){
9244 p->iSzPoslist = 0;
9245 p->bDel = 0;
9246 p->bContent = 0;
9247 p->nData = nData;
9248 }
9249 }
9250 return nRet;
9251}
9252
9253/*
9254** Add an entry to the in-memory hash table. The key is the concatenation
9255** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos).
9256**
9257** (bByte || pToken) -> (iRowid,iCol,iPos)
9258**
9259** Or, if iCol is negative, then the value is a delete marker.
9260*/
9261static int sqlite3Fts5HashWrite(
9262 Fts5Hash *pHash,
9263 i64 iRowid, /* Rowid for this entry */
9264 int iCol, /* Column token appears in (-ve -> delete) */
9265 int iPos, /* Position of token within column */
9266 char bByte, /* First byte of token */
9267 const char *pToken, int nToken /* Token to add or remove to or from index */
9268){
9269 unsigned int iHash;
9270 Fts5HashEntry *p;
9271 u8 *pPtr;
9272 int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */
9273 int bNew; /* If non-delete entry should be written */
9274
9275 bNew = (pHash->eDetail==FTS5_DETAIL_FULL0);
9276
9277 /* Attempt to locate an existing hash entry */
9278 iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
9279 for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
9280 char *zKey = fts5EntryKey(p)( ((char *)(&(p)[1])) );
9281 if( zKey[0]==bByte
9282 && p->nKey==nToken+1
9283 && memcmp(&zKey[1], pToken, nToken)==0
9284 ){
9285 break;
9286 }
9287 }
9288
9289 /* If an existing hash entry cannot be found, create a new one. */
9290 if( p==0 ){
9291 /* Figure out how much space to allocate */
9292 char *zKey;
9293 sqlite3_int64 nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64;
9294 if( nByte<128 ) nByte = 128;
9295
9296 /* Grow the Fts5Hash.aSlot[] array if necessary. */
9297 if( (pHash->nEntry*2)>=pHash->nSlot ){
9298 int rc = fts5HashResize(pHash);
9299 if( rc!=SQLITE_OK0 ) return rc;
9300 iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
9301 }
9302
9303 /* Allocate new Fts5HashEntry and add it to the hash table. */
9304 p = (Fts5HashEntry*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
9305 if( !p ) return SQLITE_NOMEM7;
9306 memset(p, 0, sizeof(Fts5HashEntry));
9307 p->nAlloc = (int)nByte;
9308 zKey = fts5EntryKey(p)( ((char *)(&(p)[1])) );
9309 zKey[0] = bByte;
9310 memcpy(&zKey[1], pToken, nToken);
9311 assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) )((void) (0));
9312 p->nKey = nToken+1;
9313 zKey[nToken+1] = '\0';
9314 p->nData = nToken+1 + sizeof(Fts5HashEntry);
9315 p->pHashNext = pHash->aSlot[iHash];
9316 pHash->aSlot[iHash] = p;
9317 pHash->nEntry++;
9318
9319 /* Add the first rowid field to the hash-entry */
9320 p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
9321 p->iRowid = iRowid;
9322
9323 p->iSzPoslist = p->nData;
9324 if( pHash->eDetail!=FTS5_DETAIL_NONE1 ){
9325 p->nData += 1;
9326 p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL0 ? 0 : -1);
9327 }
9328
9329 }else{
9330
9331 /* Appending to an existing hash-entry. Check that there is enough
9332 ** space to append the largest possible new entry. Worst case scenario
9333 ** is:
9334 **
9335 ** + 9 bytes for a new rowid,
9336 ** + 4 byte reserved for the "poslist size" varint.
9337 ** + 1 byte for a "new column" byte,
9338 ** + 3 bytes for a new column number (16-bit max) as a varint,
9339 ** + 5 bytes for the new position offset (32-bit max).
9340 */
9341 if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
9342 sqlite3_int64 nNew = p->nAlloc * 2;
9343 Fts5HashEntry *pNew;
9344 Fts5HashEntry **pp;
9345 pNew = (Fts5HashEntry*)sqlite3_realloc64sqlite3_api->realloc64(p, nNew);
9346 if( pNew==0 ) return SQLITE_NOMEM7;
9347 pNew->nAlloc = (int)nNew;
9348 for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
9349 *pp = pNew;
9350 p = pNew;
9351 }
9352 nIncr -= p->nData;
9353 }
9354 assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) )((void) (0));
9355
9356 pPtr = (u8*)p;
9357
9358 /* If this is a new rowid, append the 4-byte size field for the previous
9359 ** entry, and the new rowid for this entry. */
9360 if( iRowid!=p->iRowid ){
9361 u64 iDiff = (u64)iRowid - (u64)p->iRowid;
9362 fts5HashAddPoslistSize(pHash, p, 0);
9363 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iDiff);
9364 p->iRowid = iRowid;
9365 bNew = 1;
9366 p->iSzPoslist = p->nData;
9367 if( pHash->eDetail!=FTS5_DETAIL_NONE1 ){
9368 p->nData += 1;
9369 p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL0 ? 0 : -1);
9370 p->iPos = 0;
9371 }
9372 }
9373
9374 if( iCol>=0 ){
9375 if( pHash->eDetail==FTS5_DETAIL_NONE1 ){
9376 p->bContent = 1;
9377 }else{
9378 /* Append a new column value, if necessary */
9379 assert_nc( iCol>=p->iCol )((void) (0));
9380 if( iCol!=p->iCol ){
9381 if( pHash->eDetail==FTS5_DETAIL_FULL0 ){
9382 pPtr[p->nData++] = 0x01;
9383 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
9384 p->iCol = (i16)iCol;
9385 p->iPos = 0;
9386 }else{
9387 bNew = 1;
9388 p->iCol = (i16)(iPos = iCol);
9389 }
9390 }
9391
9392 /* Append the new position offset, if necessary */
9393 if( bNew ){
9394 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
9395 p->iPos = iPos;
9396 }
9397 }
9398 }else{
9399 /* This is a delete. Set the delete flag. */
9400 p->bDel = 1;
9401 }
9402
9403 nIncr += p->nData;
9404 *pHash->pnByte += nIncr;
9405 return SQLITE_OK0;
9406}
9407
9408
9409/*
9410** Arguments pLeft and pRight point to linked-lists of hash-entry objects,
9411** each sorted in key order. This function merges the two lists into a
9412** single list and returns a pointer to its first element.
9413*/
9414static Fts5HashEntry *fts5HashEntryMerge(
9415 Fts5HashEntry *pLeft,
9416 Fts5HashEntry *pRight
9417){
9418 Fts5HashEntry *p1 = pLeft;
9419 Fts5HashEntry *p2 = pRight;
9420 Fts5HashEntry *pRet = 0;
9421 Fts5HashEntry **ppOut = &pRet;
9422
9423 while( p1 || p2 ){
9424 if( p1==0 ){
9425 *ppOut = p2;
9426 p2 = 0;
9427 }else if( p2==0 ){
9428 *ppOut = p1;
9429 p1 = 0;
9430 }else{
9431 char *zKey1 = fts5EntryKey(p1)( ((char *)(&(p1)[1])) );
9432 char *zKey2 = fts5EntryKey(p2)( ((char *)(&(p2)[1])) );
9433 int nMin = MIN(p1->nKey, p2->nKey)(((p1->nKey) < (p2->nKey)) ? (p1->nKey) : (p2->
nKey));
9434
9435 int cmp = memcmp(zKey1, zKey2, nMin);
9436 if( cmp==0 ){
9437 cmp = p1->nKey - p2->nKey;
9438 }
9439 assert( cmp!=0 )((void) (0));
9440
9441 if( cmp>0 ){
9442 /* p2 is smaller */
9443 *ppOut = p2;
9444 ppOut = &p2->pScanNext;
9445 p2 = p2->pScanNext;
9446 }else{
9447 /* p1 is smaller */
9448 *ppOut = p1;
9449 ppOut = &p1->pScanNext;
9450 p1 = p1->pScanNext;
9451 }
9452 *ppOut = 0;
9453 }
9454 }
9455
9456 return pRet;
9457}
9458
9459/*
9460** Link all tokens from hash table iHash into a list in sorted order. The
9461** tokens are not removed from the hash table.
9462*/
9463static int fts5HashEntrySort(
9464 Fts5Hash *pHash,
9465 const char *pTerm, int nTerm, /* Query prefix, if any */
9466 Fts5HashEntry **ppSorted
9467){
9468 const int nMergeSlot = 32;
9469 Fts5HashEntry **ap;
9470 Fts5HashEntry *pList;
9471 int iSlot;
9472 int i;
9473
9474 *ppSorted = 0;
9475 ap = sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5HashEntry*) * nMergeSlot);
9476 if( !ap ) return SQLITE_NOMEM7;
9477 memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot);
9478
9479 for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
9480 Fts5HashEntry *pIter;
9481 for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
9482 if( pTerm==0
9483 || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter)( ((char *)(&(pIter)[1])) ), pTerm, nTerm))
9484 ){
9485 Fts5HashEntry *pEntry = pIter;
9486 pEntry->pScanNext = 0;
9487 for(i=0; ap[i]; i++){
9488 pEntry = fts5HashEntryMerge(pEntry, ap[i]);
9489 ap[i] = 0;
9490 }
9491 ap[i] = pEntry;
9492 }
9493 }
9494 }
9495
9496 pList = 0;
9497 for(i=0; i<nMergeSlot; i++){
9498 pList = fts5HashEntryMerge(pList, ap[i]);
9499 }
9500
9501 sqlite3_freesqlite3_api->free(ap);
9502 *ppSorted = pList;
9503 return SQLITE_OK0;
9504}
9505
9506/*
9507** Query the hash table for a doclist associated with term pTerm/nTerm.
9508*/
9509static int sqlite3Fts5HashQuery(
9510 Fts5Hash *pHash, /* Hash table to query */
9511 int nPre,
9512 const char *pTerm, int nTerm, /* Query term */
9513 void **ppOut, /* OUT: Pointer to new object */
9514 int *pnDoclist /* OUT: Size of doclist in bytes */
9515){
9516 unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
9517 char *zKey = 0;
9518 Fts5HashEntry *p;
9519
9520 for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
9521 zKey = fts5EntryKey(p)( ((char *)(&(p)[1])) );
9522 if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break;
9523 }
9524
9525 if( p ){
9526 int nHashPre = sizeof(Fts5HashEntry) + nTerm;
9527 int nList = p->nData - nHashPre;
9528 u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64sqlite3_api->malloc64(nPre + nList + 10));
9529 if( pRet ){
9530 Fts5HashEntry *pFaux = (Fts5HashEntry*)&pRet[nPre-nHashPre];
9531 memcpy(&pRet[nPre], &((u8*)p)[nHashPre], nList);
9532 nList += fts5HashAddPoslistSize(pHash, p, pFaux);
9533 *pnDoclist = nList;
9534 }else{
9535 *pnDoclist = 0;
9536 return SQLITE_NOMEM7;
9537 }
9538 }else{
9539 *ppOut = 0;
9540 *pnDoclist = 0;
9541 }
9542
9543 return SQLITE_OK0;
9544}
9545
9546static int sqlite3Fts5HashScanInit(
9547 Fts5Hash *p, /* Hash table to query */
9548 const char *pTerm, int nTerm /* Query prefix */
9549){
9550 return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
9551}
9552
9553#ifdef SQLITE_DEBUG
9554static int fts5HashCount(Fts5Hash *pHash){
9555 int nEntry = 0;
9556 int ii;
9557 for(ii=0; ii<pHash->nSlot; ii++){
9558 Fts5HashEntry *p = 0;
9559 for(p=pHash->aSlot[ii]; p; p=p->pHashNext){
9560 nEntry++;
9561 }
9562 }
9563 return nEntry;
9564}
9565#endif
9566
9567/*
9568** Return true if the hash table is empty, false otherwise.
9569*/
9570static int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){
9571 assert( pHash->nEntry==fts5HashCount(pHash) )((void) (0));
9572 return pHash->nEntry==0;
9573}
9574
9575static void sqlite3Fts5HashScanNext(Fts5Hash *p){
9576 assert( !sqlite3Fts5HashScanEof(p) )((void) (0));
9577 p->pScan = p->pScan->pScanNext;
9578}
9579
9580static int sqlite3Fts5HashScanEof(Fts5Hash *p){
9581 return (p->pScan==0);
9582}
9583
9584static void sqlite3Fts5HashScanEntry(
9585 Fts5Hash *pHash,
9586 const char **pzTerm, /* OUT: term (nul-terminated) */
9587 int *pnTerm, /* OUT: Size of term in bytes */
9588 const u8 **ppDoclist, /* OUT: pointer to doclist */
9589 int *pnDoclist /* OUT: size of doclist in bytes */
9590){
9591 Fts5HashEntry *p;
9592 if( (p = pHash->pScan) ){
9593 char *zKey = fts5EntryKey(p)( ((char *)(&(p)[1])) );
9594 int nTerm = p->nKey;
9595 fts5HashAddPoslistSize(pHash, p, 0);
9596 *pzTerm = zKey;
9597 *pnTerm = nTerm;
9598 *ppDoclist = (const u8*)&zKey[nTerm];
9599 *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm);
9600 }else{
9601 *pzTerm = 0;
9602 *pnTerm = 0;
9603 *ppDoclist = 0;
9604 *pnDoclist = 0;
9605 }
9606}
9607
9608#line 1 "fts5_index.c"
9609/*
9610** 2014 May 31
9611**
9612** The author disclaims copyright to this source code. In place of
9613** a legal notice, here is a blessing:
9614**
9615** May you do good and not evil.
9616** May you find forgiveness for yourself and forgive others.
9617** May you share freely, never taking more than you give.
9618**
9619******************************************************************************
9620**
9621** Low level access to the FTS index stored in the database file. The
9622** routines in this file file implement all read and write access to the
9623** %_data table. Other parts of the system access this functionality via
9624** the interface defined in fts5Int.h.
9625*/
9626
9627
9628/* #include "fts5Int.h" */
9629
9630/*
9631** Overview:
9632**
9633** The %_data table contains all the FTS indexes for an FTS5 virtual table.
9634** As well as the main term index, there may be up to 31 prefix indexes.
9635** The format is similar to FTS3/4, except that:
9636**
9637** * all segment b-tree leaf data is stored in fixed size page records
9638** (e.g. 1000 bytes). A single doclist may span multiple pages. Care is
9639** taken to ensure it is possible to iterate in either direction through
9640** the entries in a doclist, or to seek to a specific entry within a
9641** doclist, without loading it into memory.
9642**
9643** * large doclists that span many pages have associated "doclist index"
9644** records that contain a copy of the first rowid on each page spanned by
9645** the doclist. This is used to speed up seek operations, and merges of
9646** large doclists with very small doclists.
9647**
9648** * extra fields in the "structure record" record the state of ongoing
9649** incremental merge operations.
9650**
9651*/
9652
9653
9654#define FTS5_OPT_WORK_UNIT1000 1000 /* Number of leaf pages per optimize step */
9655#define FTS5_WORK_UNIT64 64 /* Number of leaf pages in unit of work */
9656
9657#define FTS5_MIN_DLIDX_SIZE4 4 /* Add dlidx if this many empty pages */
9658
9659#define FTS5_MAIN_PREFIX'0' '0'
9660
9661#if FTS5_MAX_PREFIX_INDEXES31 > 31
9662# error "FTS5_MAX_PREFIX_INDEXES is too large"
9663#endif
9664
9665#define FTS5_MAX_LEVEL64 64
9666
9667/*
9668** There are two versions of the format used for the structure record:
9669**
9670** 1. the legacy format, that may be read by all fts5 versions, and
9671**
9672** 2. the V2 format, which is used by contentless_delete=1 databases.
9673**
9674** Both begin with a 4-byte "configuration cookie" value. Then, a legacy
9675** format structure record contains a varint - the number of levels in
9676** the structure. Whereas a V2 structure record contains the constant
9677** 4 bytes [0xff 0x00 0x00 0x01]. This is unambiguous as the value of a
9678** varint has to be at least 16256 to begin with "0xFF". And the default
9679** maximum number of levels is 64.
9680**
9681** See below for more on structure record formats.
9682*/
9683#define FTS5_STRUCTURE_V2"\xFF\x00\x00\x01" "\xFF\x00\x00\x01"
9684
9685/*
9686** Details:
9687**
9688** The %_data table managed by this module,
9689**
9690** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB);
9691**
9692** , contains the following 6 types of records. See the comments surrounding
9693** the FTS5_*_ROWID macros below for a description of how %_data rowids are
9694** assigned to each fo them.
9695**
9696** 1. Structure Records:
9697**
9698** The set of segments that make up an index - the index structure - are
9699** recorded in a single record within the %_data table. The record consists
9700** of a single 32-bit configuration cookie value followed by a list of
9701** SQLite varints.
9702**
9703** If the structure record is a V2 record, the configuration cookie is
9704** followed by the following 4 bytes: [0xFF 0x00 0x00 0x01].
9705**
9706** Next, the record continues with three varints:
9707**
9708** + number of levels,
9709** + total number of segments on all levels,
9710** + value of write counter.
9711**
9712** Then, for each level from 0 to nMax:
9713**
9714** + number of input segments in ongoing merge.
9715** + total number of segments in level.
9716** + for each segment from oldest to newest:
9717** + segment id (always > 0)
9718** + first leaf page number (often 1, always greater than 0)
9719** + final leaf page number
9720**
9721** Then, for V2 structures only:
9722**
9723** + lower origin counter value,
9724** + upper origin counter value,
9725** + the number of tombstone hash pages.
9726**
9727** 2. The Averages Record:
9728**
9729** A single record within the %_data table. The data is a list of varints.
9730** The first value is the number of rows in the index. Then, for each column
9731** from left to right, the total number of tokens in the column for all
9732** rows of the table.
9733**
9734** 3. Segment leaves:
9735**
9736** TERM/DOCLIST FORMAT:
9737**
9738** Most of each segment leaf is taken up by term/doclist data. The
9739** general format of term/doclist, starting with the first term
9740** on the leaf page, is:
9741**
9742** varint : size of first term
9743** blob: first term data
9744** doclist: first doclist
9745** zero-or-more {
9746** varint: number of bytes in common with previous term
9747** varint: number of bytes of new term data (nNew)
9748** blob: nNew bytes of new term data
9749** doclist: next doclist
9750** }
9751**
9752** doclist format:
9753**
9754** varint: first rowid
9755** poslist: first poslist
9756** zero-or-more {
9757** varint: rowid delta (always > 0)
9758** poslist: next poslist
9759** }
9760**
9761** poslist format:
9762**
9763** varint: size of poslist in bytes multiplied by 2, not including
9764** this field. Plus 1 if this entry carries the "delete" flag.
9765** collist: collist for column 0
9766** zero-or-more {
9767** 0x01 byte
9768** varint: column number (I)
9769** collist: collist for column I
9770** }
9771**
9772** collist format:
9773**
9774** varint: first offset + 2
9775** zero-or-more {
9776** varint: offset delta + 2
9777** }
9778**
9779** PAGE FORMAT
9780**
9781** Each leaf page begins with a 4-byte header containing 2 16-bit
9782** unsigned integer fields in big-endian format. They are:
9783**
9784** * The byte offset of the first rowid on the page, if it exists
9785** and occurs before the first term (otherwise 0).
9786**
9787** * The byte offset of the start of the page footer. If the page
9788** footer is 0 bytes in size, then this field is the same as the
9789** size of the leaf page in bytes.
9790**
9791** The page footer consists of a single varint for each term located
9792** on the page. Each varint is the byte offset of the current term
9793** within the page, delta-compressed against the previous value. In
9794** other words, the first varint in the footer is the byte offset of
9795** the first term, the second is the byte offset of the second less that
9796** of the first, and so on.
9797**
9798** The term/doclist format described above is accurate if the entire
9799** term/doclist data fits on a single leaf page. If this is not the case,
9800** the format is changed in two ways:
9801**
9802** + if the first rowid on a page occurs before the first term, it
9803** is stored as a literal value:
9804**
9805** varint: first rowid
9806**
9807** + the first term on each page is stored in the same way as the
9808** very first term of the segment:
9809**
9810** varint : size of first term
9811** blob: first term data
9812**
9813** 5. Segment doclist indexes:
9814**
9815** Doclist indexes are themselves b-trees, however they usually consist of
9816** a single leaf record only. The format of each doclist index leaf page
9817** is:
9818**
9819** * Flags byte. Bits are:
9820** 0x01: Clear if leaf is also the root page, otherwise set.
9821**
9822** * Page number of fts index leaf page. As a varint.
9823**
9824** * First rowid on page indicated by previous field. As a varint.
9825**
9826** * A list of varints, one for each subsequent termless page. A
9827** positive delta if the termless page contains at least one rowid,
9828** or an 0x00 byte otherwise.
9829**
9830** Internal doclist index nodes are:
9831**
9832** * Flags byte. Bits are:
9833** 0x01: Clear for root page, otherwise set.
9834**
9835** * Page number of first child page. As a varint.
9836**
9837** * Copy of first rowid on page indicated by previous field. As a varint.
9838**
9839** * A list of delta-encoded varints - the first rowid on each subsequent
9840** child page.
9841**
9842** 6. Tombstone Hash Page
9843**
9844** These records are only ever present in contentless_delete=1 tables.
9845** There are zero or more of these associated with each segment. They
9846** are used to store the tombstone rowids for rows contained in the
9847** associated segments.
9848**
9849** The set of nHashPg tombstone hash pages associated with a single
9850** segment together form a single hash table containing tombstone rowids.
9851** To find the page of the hash on which a key might be stored:
9852**
9853** iPg = (rowid % nHashPg)
9854**
9855** Then, within page iPg, which has nSlot slots:
9856**
9857** iSlot = (rowid / nHashPg) % nSlot
9858**
9859** Each tombstone hash page begins with an 8 byte header:
9860**
9861** 1-byte: Key-size (the size in bytes of each slot). Either 4 or 8.
9862** 1-byte: rowid-0-tombstone flag. This flag is only valid on the
9863** first tombstone hash page for each segment (iPg=0). If set,
9864** the hash table contains rowid 0. If clear, it does not.
9865** Rowid 0 is handled specially.
9866** 2-bytes: unused.
9867** 4-bytes: Big-endian integer containing number of entries on page.
9868**
9869** Following this are nSlot 4 or 8 byte slots (depending on the key-size
9870** in the first byte of the page header). The number of slots may be
9871** determined based on the size of the page record and the key-size:
9872**
9873** nSlot = (nByte - 8) / key-size
9874*/
9875
9876/*
9877** Rowids for the averages and structure records in the %_data table.
9878*/
9879#define FTS5_AVERAGES_ROWID1 1 /* Rowid used for the averages record */
9880#define FTS5_STRUCTURE_ROWID10 10 /* The structure record */
9881
9882/*
9883** Macros determining the rowids used by segment leaves and dlidx leaves
9884** and nodes. All nodes and leaves are stored in the %_data table with large
9885** positive rowids.
9886**
9887** Each segment has a unique non-zero 16-bit id.
9888**
9889** The rowid for each segment leaf is found by passing the segment id and
9890** the leaf page number to the FTS5_SEGMENT_ROWID macro. Leaves are numbered
9891** sequentially starting from 1.
9892*/
9893#define FTS5_DATA_ID_B16 16 /* Max seg id number 65535 */
9894#define FTS5_DATA_DLI_B1 1 /* Doclist-index flag (1 bit) */
9895#define FTS5_DATA_HEIGHT_B5 5 /* Max dlidx tree height of 32 */
9896#define FTS5_DATA_PAGE_B31 31 /* Max page number of 2147483648 */
9897
9898#define fts5_dri(segid, dlidx, height, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(dlidx) <<
(31 + 5)) + ((i64)(height) << (31)) + ((i64)(pgno)) ) ( \
9899 ((i64)(segid) << (FTS5_DATA_PAGE_B31+FTS5_DATA_HEIGHT_B5+FTS5_DATA_DLI_B1)) + \
9900 ((i64)(dlidx) << (FTS5_DATA_PAGE_B31 + FTS5_DATA_HEIGHT_B5)) + \
9901 ((i64)(height) << (FTS5_DATA_PAGE_B31)) + \
9902 ((i64)(pgno)) \
9903)
9904
9905#define FTS5_SEGMENT_ROWID(segid, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pgno)) ) fts5_dri(segid, 0, 0, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pgno)) )
9906#define FTS5_DLIDX_ROWID(segid, height, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(1) << (31
+ 5)) + ((i64)(height) << (31)) + ((i64)(pgno)) ) fts5_dri(segid, 1, height, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(1) << (31
+ 5)) + ((i64)(height) << (31)) + ((i64)(pgno)) )
9907#define FTS5_TOMBSTONE_ROWID(segid,ipg)( ((i64)(segid+(1<<16)) << (31 +5 +1)) + ((i64)(0
) << (31 + 5)) + ((i64)(0) << (31)) + ((i64)(ipg)
) ) fts5_dri(segid+(1<<16), 0, 0, ipg)( ((i64)(segid+(1<<16)) << (31 +5 +1)) + ((i64)(0
) << (31 + 5)) + ((i64)(0) << (31)) + ((i64)(ipg)
) )
9908
9909#ifdef SQLITE_DEBUG
9910static int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB(11 | (1<<8)); }
9911#endif
9912
9913
9914/*
9915** Each time a blob is read from the %_data table, it is padded with this
9916** many zero bytes. This makes it easier to decode the various record formats
9917** without overreading if the records are corrupt.
9918*/
9919#define FTS5_DATA_ZERO_PADDING8 8
9920#define FTS5_DATA_PADDING20 20
9921
9922typedef struct Fts5Data Fts5Data;
9923typedef struct Fts5DlidxIter Fts5DlidxIter;
9924typedef struct Fts5DlidxLvl Fts5DlidxLvl;
9925typedef struct Fts5DlidxWriter Fts5DlidxWriter;
9926typedef struct Fts5Iter Fts5Iter;
9927typedef struct Fts5PageWriter Fts5PageWriter;
9928typedef struct Fts5SegIter Fts5SegIter;
9929typedef struct Fts5DoclistIter Fts5DoclistIter;
9930typedef struct Fts5SegWriter Fts5SegWriter;
9931typedef struct Fts5Structure Fts5Structure;
9932typedef struct Fts5StructureLevel Fts5StructureLevel;
9933typedef struct Fts5StructureSegment Fts5StructureSegment;
9934typedef struct Fts5TokenDataIter Fts5TokenDataIter;
9935typedef struct Fts5TokenDataMap Fts5TokenDataMap;
9936typedef struct Fts5TombstoneArray Fts5TombstoneArray;
9937
9938struct Fts5Data {
9939 u8 *p; /* Pointer to buffer containing record */
9940 int nn; /* Size of record in bytes */
9941 int szLeaf; /* Size of leaf without page-index */
9942};
9943
9944/*
9945** One object per %_data table.
9946**
9947** nContentlessDelete:
9948** The number of contentless delete operations since the most recent
9949** call to fts5IndexFlush() or fts5IndexDiscardData(). This is tracked
9950** so that extra auto-merge work can be done by fts5IndexFlush() to
9951** account for the delete operations.
9952*/
9953struct Fts5Index {
9954 Fts5Config *pConfig; /* Virtual table configuration */
9955 char *zDataTbl; /* Name of %_data table */
9956 int nWorkUnit; /* Leaf pages in a "unit" of work */
9957
9958 /*
9959 ** Variables related to the accumulation of tokens and doclists within the
9960 ** in-memory hash tables before they are flushed to disk.
9961 */
9962 Fts5Hash *pHash; /* Hash table for in-memory data */
9963 int nPendingData; /* Current bytes of pending data */
9964 i64 iWriteRowid; /* Rowid for current doc being written */
9965 int bDelete; /* Current write is a delete */
9966 int nContentlessDelete; /* Number of contentless delete ops */
9967 int nPendingRow; /* Number of INSERT in hash table */
9968
9969 /* Error state. */
9970 int rc; /* Current error code */
9971 int flushRc;
9972
9973 /* State used by the fts5DataXXX() functions. */
9974 sqlite3_blob *pReader; /* RO incr-blob open on %_data table */
9975 sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */
9976 sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */
9977 sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */
9978 sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=?" */
9979 sqlite3_stmt *pIdxSelect;
9980 sqlite3_stmt *pIdxNextSelect;
9981 int nRead; /* Total number of blocks read */
9982
9983 sqlite3_stmt *pDeleteFromIdx;
9984
9985 sqlite3_stmt *pDataVersion;
9986 i64 iStructVersion; /* data_version when pStruct read */
9987 Fts5Structure *pStruct; /* Current db structure (or NULL) */
9988};
9989
9990struct Fts5DoclistIter {
9991 u8 *aEof; /* Pointer to 1 byte past end of doclist */
9992
9993 /* Output variables. aPoslist==0 at EOF */
9994 i64 iRowid;
9995 u8 *aPoslist;
9996 int nPoslist;
9997 int nSize;
9998};
9999
10000/*
10001** The contents of the "structure" record for each index are represented
10002** using an Fts5Structure record in memory. Which uses instances of the
10003** other Fts5StructureXXX types as components.
10004**
10005** nOriginCntr:
10006** This value is set to non-zero for structure records created for
10007** contentlessdelete=1 tables only. In that case it represents the
10008** origin value to apply to the next top-level segment created.
10009*/
10010struct Fts5StructureSegment {
10011 int iSegid; /* Segment id */
10012 int pgnoFirst; /* First leaf page number in segment */
10013 int pgnoLast; /* Last leaf page number in segment */
10014
10015 /* contentlessdelete=1 tables only: */
10016 u64 iOrigin1;
10017 u64 iOrigin2;
10018 int nPgTombstone; /* Number of tombstone hash table pages */
10019 u64 nEntryTombstone; /* Number of tombstone entries that "count" */
10020 u64 nEntry; /* Number of rows in this segment */
10021};
10022struct Fts5StructureLevel {
10023 int nMerge; /* Number of segments in incr-merge */
10024 int nSeg; /* Total number of segments on level */
10025 Fts5StructureSegment *aSeg; /* Array of segments. aSeg[0] is oldest. */
10026};
10027struct Fts5Structure {
10028 int nRef; /* Object reference count */
10029 u64 nWriteCounter; /* Total leaves written to level 0 */
10030 u64 nOriginCntr; /* Origin value for next top-level segment */
10031 int nSegment; /* Total segments in this structure */
10032 int nLevel; /* Number of levels in this index */
10033 Fts5StructureLevel aLevel[FLEXARRAY]; /* Array of nLevel level objects */
10034};
10035
10036/* Size (in bytes) of an Fts5Structure object holding up to N levels */
10037#define SZ_FTS5STRUCTURE(N)(__builtin_offsetof(Fts5Structure, aLevel) + (N)*sizeof(Fts5StructureLevel
)) \
10038 (offsetof(Fts5Structure,aLevel)__builtin_offsetof(Fts5Structure, aLevel) + (N)*sizeof(Fts5StructureLevel))
10039
10040/*
10041** An object of type Fts5SegWriter is used to write to segments.
10042*/
10043struct Fts5PageWriter {
10044 int pgno; /* Page number for this page */
10045 int iPrevPgidx; /* Previous value written into pgidx */
10046 Fts5Buffer buf; /* Buffer containing leaf data */
10047 Fts5Buffer pgidx; /* Buffer containing page-index */
10048 Fts5Buffer term; /* Buffer containing previous term on page */
10049};
10050struct Fts5DlidxWriter {
10051 int pgno; /* Page number for this page */
10052 int bPrevValid; /* True if iPrev is valid */
10053 i64 iPrev; /* Previous rowid value written to page */
10054 Fts5Buffer buf; /* Buffer containing page data */
10055};
10056struct Fts5SegWriter {
10057 int iSegid; /* Segid to write to */
10058 Fts5PageWriter writer; /* PageWriter object */
10059 i64 iPrevRowid; /* Previous rowid written to current leaf */
10060 u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */
10061 u8 bFirstRowidInPage; /* True if next rowid is first in page */
10062 /* TODO1: Can use (writer.pgidx.n==0) instead of bFirstTermInPage */
10063 u8 bFirstTermInPage; /* True if next term will be first in leaf */
10064 int nLeafWritten; /* Number of leaf pages written */
10065 int nEmpty; /* Number of contiguous term-less nodes */
10066
10067 int nDlidx; /* Allocated size of aDlidx[] array */
10068 Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */
10069
10070 /* Values to insert into the %_idx table */
10071 Fts5Buffer btterm; /* Next term to insert into %_idx table */
10072 int iBtPage; /* Page number corresponding to btterm */
10073};
10074
10075typedef struct Fts5CResult Fts5CResult;
10076struct Fts5CResult {
10077 u16 iFirst; /* aSeg[] index of firstest iterator */
10078 u8 bTermEq; /* True if the terms are equal */
10079};
10080
10081/*
10082** Object for iterating through a single segment, visiting each term/rowid
10083** pair in the segment.
10084**
10085** pSeg:
10086** The segment to iterate through.
10087**
10088** iLeafPgno:
10089** Current leaf page number within segment.
10090**
10091** iLeafOffset:
10092** Byte offset within the current leaf that is the first byte of the
10093** position list data (one byte passed the position-list size field).
10094**
10095** pLeaf:
10096** Buffer containing current leaf page data. Set to NULL at EOF.
10097**
10098** iTermLeafPgno, iTermLeafOffset:
10099** Leaf page number containing the last term read from the segment. And
10100** the offset immediately following the term data.
10101**
10102** flags:
10103** Mask of FTS5_SEGITER_XXX values. Interpreted as follows:
10104**
10105** FTS5_SEGITER_ONETERM:
10106** If set, set the iterator to point to EOF after the current doclist
10107** has been exhausted. Do not proceed to the next term in the segment.
10108**
10109** FTS5_SEGITER_REVERSE:
10110** This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If
10111** it is set, iterate through rowid in descending order instead of the
10112** default ascending order.
10113**
10114** iRowidOffset/nRowidOffset/aRowidOffset:
10115** These are used if the FTS5_SEGITER_REVERSE flag is set.
10116**
10117** For each rowid on the page corresponding to the current term, the
10118** corresponding aRowidOffset[] entry is set to the byte offset of the
10119** start of the "position-list-size" field within the page.
10120**
10121** iTermIdx:
10122** Index of current term on iTermLeafPgno.
10123**
10124** apTombstone/nTombstone:
10125** These are used for contentless_delete=1 tables only. When the cursor
10126** is first allocated, the apTombstone[] array is allocated so that it
10127** is large enough for all tombstones hash pages associated with the
10128** segment. The pages themselves are loaded lazily from the database as
10129** they are required.
10130*/
10131struct Fts5SegIter {
10132 Fts5StructureSegment *pSeg; /* Segment to iterate through */
10133 int flags; /* Mask of configuration flags */
10134 int iLeafPgno; /* Current leaf page number */
10135 Fts5Data *pLeaf; /* Current leaf data */
10136 Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */
10137 i64 iLeafOffset; /* Byte offset within current leaf */
10138 Fts5TombstoneArray *pTombArray; /* Array of tombstone pages */
10139
10140 /* Next method */
10141 void (*xNext)(Fts5Index*, Fts5SegIter*, int*);
10142
10143 /* The page and offset from which the current term was read. The offset
10144 ** is the offset of the first rowid in the current doclist. */
10145 int iTermLeafPgno;
10146 int iTermLeafOffset;
10147
10148 int iPgidxOff; /* Next offset in pgidx */
10149 int iEndofDoclist;
10150
10151 /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */
10152 int iRowidOffset; /* Current entry in aRowidOffset[] */
10153 int nRowidOffset; /* Allocated size of aRowidOffset[] array */
10154 int *aRowidOffset; /* Array of offset to rowid fields */
10155
10156 Fts5DlidxIter *pDlidx; /* If there is a doclist-index */
10157
10158 /* Variables populated based on current entry. */
10159 Fts5Buffer term; /* Current term */
10160 i64 iRowid; /* Current rowid */
10161 int nPos; /* Number of bytes in current position list */
10162 u8 bDel; /* True if the delete flag is set */
10163};
10164
10165static int fts5IndexCorruptRowid(Fts5Index *pIdx, i64 iRowid){
10166 pIdx->rc = FTS5_CORRUPT(11 | (1<<8));
10167 sqlite3Fts5ConfigErrmsg(pIdx->pConfig,
10168 "fts5: corruption found reading blob %lld from table \"%s\"",
10169 iRowid, pIdx->pConfig->zName
10170 );
10171 return SQLITE_CORRUPT_VTAB(11 | (1<<8));
10172}
10173#define FTS5_CORRUPT_ROWID(pIdx, iRowid)fts5IndexCorruptRowid(pIdx, iRowid) fts5IndexCorruptRowid(pIdx, iRowid)
10174
10175static int fts5IndexCorruptIter(Fts5Index *pIdx, Fts5SegIter *pIter){
10176 pIdx->rc = FTS5_CORRUPT(11 | (1<<8));
10177 sqlite3Fts5ConfigErrmsg(pIdx->pConfig,
10178 "fts5: corruption on page %d, segment %d, table \"%s\"",
10179 pIter->iLeafPgno, pIter->pSeg->iSegid, pIdx->pConfig->zName
10180 );
10181 return SQLITE_CORRUPT_VTAB(11 | (1<<8));
10182}
10183#define FTS5_CORRUPT_ITER(pIdx, pIter)fts5IndexCorruptIter(pIdx, pIter) fts5IndexCorruptIter(pIdx, pIter)
10184
10185static int fts5IndexCorruptIdx(Fts5Index *pIdx){
10186 pIdx->rc = FTS5_CORRUPT(11 | (1<<8));
10187 sqlite3Fts5ConfigErrmsg(pIdx->pConfig,
10188 "fts5: corruption in table \"%s\"", pIdx->pConfig->zName
10189 );
10190 return SQLITE_CORRUPT_VTAB(11 | (1<<8));
10191}
10192#define FTS5_CORRUPT_IDX(pIdx)fts5IndexCorruptIdx(pIdx) fts5IndexCorruptIdx(pIdx)
10193
10194
10195/*
10196** Array of tombstone pages. Reference counted.
10197*/
10198struct Fts5TombstoneArray {
10199 int nRef; /* Number of pointers to this object */
10200 int nTombstone;
10201 Fts5Data *apTombstone[FLEXARRAY]; /* Array of tombstone pages */
10202};
10203
10204/* Size (in bytes) of an Fts5TombstoneArray holding up to N tombstones */
10205#define SZ_FTS5TOMBSTONEARRAY(N)(__builtin_offsetof(Fts5TombstoneArray, apTombstone)+(N)*sizeof
(Fts5Data*)) \
10206 (offsetof(Fts5TombstoneArray,apTombstone)__builtin_offsetof(Fts5TombstoneArray, apTombstone)+(N)*sizeof(Fts5Data*))
10207
10208/*
10209** Argument is a pointer to an Fts5Data structure that contains a
10210** leaf page.
10211*/
10212#define ASSERT_SZLEAF_OK(x)((void) (0)) assert( \((void) (0))
10213 (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \((void) (0))
10214)((void) (0))
10215
10216#define FTS5_SEGITER_ONETERM0x01 0x01
10217#define FTS5_SEGITER_REVERSE0x02 0x02
10218
10219/*
10220** Argument is a pointer to an Fts5Data structure that contains a leaf
10221** page. This macro evaluates to true if the leaf contains no terms, or
10222** false if it contains at least one term.
10223*/
10224#define fts5LeafIsTermless(x)((x)->szLeaf >= (x)->nn) ((x)->szLeaf >= (x)->nn)
10225
10226#define fts5LeafTermOff(x, i)(fts5GetU16(&(x)->p[(x)->szLeaf + (i)*2])) (fts5GetU16(&(x)->p[(x)->szLeaf + (i)*2]))
10227
10228#define fts5LeafFirstRowidOff(x)(fts5GetU16((x)->p)) (fts5GetU16((x)->p))
10229
10230/*
10231** Object for iterating through the merged results of one or more segments,
10232** visiting each term/rowid pair in the merged data.
10233**
10234** nSeg is always a power of two greater than or equal to the number of
10235** segments that this object is merging data from. Both the aSeg[] and
10236** aFirst[] arrays are sized at nSeg entries. The aSeg[] array is padded
10237** with zeroed objects - these are handled as if they were iterators opened
10238** on empty segments.
10239**
10240** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an
10241** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the
10242** comparison in this context is the index of the iterator that currently
10243** points to the smaller term/rowid combination. Iterators at EOF are
10244** considered to be greater than all other iterators.
10245**
10246** aFirst[1] contains the index in aSeg[] of the iterator that points to
10247** the smallest key overall. aFirst[0] is unused.
10248**
10249** poslist:
10250** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
10251** There is no way to tell if this is populated or not.
10252**
10253** pColset:
10254** If not NULL, points to an object containing a set of column indices.
10255** Only matches that occur in one of these columns will be returned.
10256** The Fts5Iter does not own the Fts5Colset object, and so it is not
10257** freed when the iterator is closed - it is owned by the upper layer.
10258*/
10259struct Fts5Iter {
10260 Fts5IndexIter base; /* Base class containing output vars */
10261 Fts5TokenDataIter *pTokenDataIter;
10262
10263 Fts5Index *pIndex; /* Index that owns this iterator */
10264 Fts5Buffer poslist; /* Buffer containing current poslist */
10265 Fts5Colset *pColset; /* Restrict matches to these columns */
10266
10267 /* Invoked to set output variables. */
10268 void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*);
10269
10270 int nSeg; /* Size of aSeg[] array */
10271 int bRev; /* True to iterate in reverse order */
10272 u8 bSkipEmpty; /* True to skip deleted entries */
10273
10274 i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */
10275 Fts5CResult *aFirst; /* Current merge state (see above) */
10276 Fts5SegIter aSeg[FLEXARRAY]; /* Array of segment iterators */
10277};
10278
10279/* Size (in bytes) of an Fts5Iter object holding up to N segment iterators */
10280#define SZ_FTS5ITER(N)(__builtin_offsetof(Fts5Iter, aSeg)+(N)*sizeof(Fts5SegIter)) (offsetof(Fts5Iter,aSeg)__builtin_offsetof(Fts5Iter, aSeg)+(N)*sizeof(Fts5SegIter))
10281
10282/*
10283** An instance of the following type is used to iterate through the contents
10284** of a doclist-index record.
10285**
10286** pData:
10287** Record containing the doclist-index data.
10288**
10289** bEof:
10290** Set to true once iterator has reached EOF.
10291**
10292** iOff:
10293** Set to the current offset within record pData.
10294*/
10295struct Fts5DlidxLvl {
10296 Fts5Data *pData; /* Data for current page of this level */
10297 int iOff; /* Current offset into pData */
10298 int bEof; /* At EOF already */
10299 int iFirstOff; /* Used by reverse iterators */
10300
10301 /* Output variables */
10302 int iLeafPgno; /* Page number of current leaf page */
10303 i64 iRowid; /* First rowid on leaf iLeafPgno */
10304};
10305struct Fts5DlidxIter {
10306 int nLvl;
10307 int iSegid;
10308 Fts5DlidxLvl aLvl[FLEXARRAY];
10309};
10310
10311/* Size (in bytes) of an Fts5DlidxIter object with up to N levels */
10312#define SZ_FTS5DLIDXITER(N)(__builtin_offsetof(Fts5DlidxIter, aLvl)+(N)*sizeof(Fts5DlidxLvl
)) \
10313 (offsetof(Fts5DlidxIter,aLvl)__builtin_offsetof(Fts5DlidxIter, aLvl)+(N)*sizeof(Fts5DlidxLvl))
10314
10315static void fts5PutU16(u8 *aOut, u16 iVal){
10316 aOut[0] = (iVal>>8);
10317 aOut[1] = (iVal&0xFF);
10318}
10319
10320static u16 fts5GetU16(const u8 *aIn){
10321 return ((u16)aIn[0] << 8) + aIn[1];
10322}
10323
10324/*
10325** The only argument points to a buffer at least 8 bytes in size. This
10326** function interprets the first 8 bytes of the buffer as a 64-bit big-endian
10327** unsigned integer and returns the result.
10328*/
10329static u64 fts5GetU64(u8 *a){
10330 return ((u64)a[0] << 56)
10331 + ((u64)a[1] << 48)
10332 + ((u64)a[2] << 40)
10333 + ((u64)a[3] << 32)
10334 + ((u64)a[4] << 24)
10335 + ((u64)a[5] << 16)
10336 + ((u64)a[6] << 8)
10337 + ((u64)a[7] << 0);
10338}
10339
10340/*
10341** The only argument points to a buffer at least 4 bytes in size. This
10342** function interprets the first 4 bytes of the buffer as a 32-bit big-endian
10343** unsigned integer and returns the result.
10344*/
10345static u32 fts5GetU32(const u8 *a){
10346 return ((u32)a[0] << 24)
10347 + ((u32)a[1] << 16)
10348 + ((u32)a[2] << 8)
10349 + ((u32)a[3] << 0);
10350}
10351
10352/*
10353** Write iVal, formated as a 64-bit big-endian unsigned integer, to the
10354** buffer indicated by the first argument.
10355*/
10356static void fts5PutU64(u8 *a, u64 iVal){
10357 a[0] = ((iVal >> 56) & 0xFF);
10358 a[1] = ((iVal >> 48) & 0xFF);
10359 a[2] = ((iVal >> 40) & 0xFF);
10360 a[3] = ((iVal >> 32) & 0xFF);
10361 a[4] = ((iVal >> 24) & 0xFF);
10362 a[5] = ((iVal >> 16) & 0xFF);
10363 a[6] = ((iVal >> 8) & 0xFF);
10364 a[7] = ((iVal >> 0) & 0xFF);
10365}
10366
10367/*
10368** Write iVal, formated as a 32-bit big-endian unsigned integer, to the
10369** buffer indicated by the first argument.
10370*/
10371static void fts5PutU32(u8 *a, u32 iVal){
10372 a[0] = ((iVal >> 24) & 0xFF);
10373 a[1] = ((iVal >> 16) & 0xFF);
10374 a[2] = ((iVal >> 8) & 0xFF);
10375 a[3] = ((iVal >> 0) & 0xFF);
10376}
10377
10378/*
10379** Allocate and return a buffer at least nByte bytes in size.
10380**
10381** If an OOM error is encountered, return NULL and set the error code in
10382** the Fts5Index handle passed as the first argument.
10383*/
10384static void *fts5IdxMalloc(Fts5Index *p, sqlite3_int64 nByte){
10385 return sqlite3Fts5MallocZero(&p->rc, nByte);
10386}
10387
10388/*
10389** Compare the contents of the pLeft buffer with the pRight/nRight blob.
10390**
10391** Return -ve if pLeft is smaller than pRight, 0 if they are equal or
10392** +ve if pRight is smaller than pLeft. In other words:
10393**
10394** res = *pLeft - *pRight
10395*/
10396#ifdef SQLITE_DEBUG
10397static int fts5BufferCompareBlob(
10398 Fts5Buffer *pLeft, /* Left hand side of comparison */
10399 const u8 *pRight, int nRight /* Right hand side of comparison */
10400){
10401 int nCmp = MIN(pLeft->n, nRight)(((pLeft->n) < (nRight)) ? (pLeft->n) : (nRight));
10402 int res = memcmp(pLeft->p, pRight, nCmp);
10403 return (res==0 ? (pLeft->n - nRight) : res);
10404}
10405#endif
10406
10407/*
10408** Compare the contents of the two buffers using memcmp(). If one buffer
10409** is a prefix of the other, it is considered the lesser.
10410**
10411** Return -ve if pLeft is smaller than pRight, 0 if they are equal or
10412** +ve if pRight is smaller than pLeft. In other words:
10413**
10414** res = *pLeft - *pRight
10415*/
10416static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){
10417 int nCmp, res;
10418 nCmp = MIN(pLeft->n, pRight->n)(((pLeft->n) < (pRight->n)) ? (pLeft->n) : (pRight
->n));
10419 assert( nCmp<=0 || pLeft->p!=0 )((void) (0));
10420 assert( nCmp<=0 || pRight->p!=0 )((void) (0));
10421 res = fts5Memcmp(pLeft->p, pRight->p, nCmp)((nCmp)<=0 ? 0 : memcmp((pLeft->p), (pRight->p), (nCmp
)));
10422 return (res==0 ? (pLeft->n - pRight->n) : res);
10423}
10424
10425static int fts5LeafFirstTermOff(Fts5Data *pLeaf){
10426 int ret;
10427 fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret)sqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(ret));
10428 return ret;
10429}
10430
10431/*
10432** Close the read-only blob handle, if it is open.
10433*/
10434static void fts5IndexCloseReader(Fts5Index *p){
10435 if( p->pReader ){
10436 int rc;
10437 sqlite3_blob *pReader = p->pReader;
10438 p->pReader = 0;
10439 rc = sqlite3_blob_closesqlite3_api->blob_close(pReader);
10440 if( p->rc==SQLITE_OK0 ) p->rc = rc;
10441 }
10442}
10443
10444/*
10445** Retrieve a record from the %_data table.
10446**
10447** If an error occurs, NULL is returned and an error left in the
10448** Fts5Index object.
10449*/
10450static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
10451 Fts5Data *pRet = 0;
10452 if( p->rc==SQLITE_OK0 ){
10453 int rc = SQLITE_OK0;
10454
10455 if( p->pReader ){
10456 /* This call may return SQLITE_ABORT if there has been a savepoint
10457 ** rollback since it was last used. In this case a new blob handle
10458 ** is required. */
10459 sqlite3_blob *pBlob = p->pReader;
10460 p->pReader = 0;
10461 rc = sqlite3_blob_reopensqlite3_api->blob_reopen(pBlob, iRowid);
10462 assert( p->pReader==0 )((void) (0));
10463 p->pReader = pBlob;
10464 if( rc!=SQLITE_OK0 ){
10465 fts5IndexCloseReader(p);
10466 }
10467 if( rc==SQLITE_ABORT4 ) rc = SQLITE_OK0;
10468 }
10469
10470 /* If the blob handle is not open at this point, open it and seek
10471 ** to the requested entry. */
10472 if( p->pReader==0 && rc==SQLITE_OK0 ){
10473 Fts5Config *pConfig = p->pConfig;
10474 rc = sqlite3_blob_opensqlite3_api->blob_open(pConfig->db,
10475 pConfig->zDb, p->zDataTbl, "block", iRowid, 0, &p->pReader
10476 );
10477 }
10478
10479 /* If either of the sqlite3_blob_open() or sqlite3_blob_reopen() calls
10480 ** above returned SQLITE_ERROR, return SQLITE_CORRUPT_VTAB instead.
10481 ** All the reasons those functions might return SQLITE_ERROR - missing
10482 ** table, missing row, non-blob/text in block column - indicate
10483 ** backing store corruption. */
10484 if( rc==SQLITE_ERROR1 ) rc = FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
10485
10486 if( rc==SQLITE_OK0 ){
10487 u8 *aOut = 0; /* Read blob data into this buffer */
10488 i64 nByte = sqlite3_blob_bytessqlite3_api->blob_bytes(p->pReader);
10489 i64 szData = (sizeof(Fts5Data) + 7) & ~7;
10490 i64 nAlloc = szData + nByte + FTS5_DATA_PADDING20;
10491 pRet = (Fts5Data*)sqlite3_malloc64sqlite3_api->malloc64(nAlloc);
10492 if( pRet ){
10493 pRet->nn = nByte;
10494 aOut = pRet->p = (u8*)pRet + szData;
10495 }else{
10496 rc = SQLITE_NOMEM7;
10497 }
10498
10499 if( rc==SQLITE_OK0 ){
10500 rc = sqlite3_blob_readsqlite3_api->blob_read(p->pReader, aOut, nByte, 0);
10501 }
10502 if( rc!=SQLITE_OK0 ){
10503 sqlite3_freesqlite3_api->free(pRet);
10504 pRet = 0;
10505 }else{
10506 /* TODO1: Fix this */
10507 pRet->p[nByte] = 0x00;
10508 pRet->p[nByte+1] = 0x00;
10509 pRet->szLeaf = fts5GetU16(&pRet->p[2]);
10510 }
10511 }
10512 p->rc = rc;
10513 p->nRead++;
10514 }
10515
10516 assert( (pRet==0)==(p->rc!=SQLITE_OK) )((void) (0));
10517 assert( pRet==0 || EIGHT_BYTE_ALIGNMENT( pRet->p ) )((void) (0));
10518 return pRet;
10519}
10520
10521
10522/*
10523** Release a reference to data record returned by an earlier call to
10524** fts5DataRead().
10525*/
10526static void fts5DataRelease(Fts5Data *pData){
10527 sqlite3_freesqlite3_api->free(pData);
10528}
10529
10530static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){
10531 Fts5Data *pRet = fts5DataRead(p, iRowid);
10532 if( pRet ){
10533 if( pRet->szLeaf<4 || pRet->szLeaf>pRet->nn ){
10534 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
10535 fts5DataRelease(pRet);
10536 pRet = 0;
10537 }
10538 }
10539 return pRet;
10540}
10541
10542static int fts5IndexPrepareStmt(
10543 Fts5Index *p,
10544 sqlite3_stmt **ppStmt,
10545 char *zSql
10546){
10547 if( p->rc==SQLITE_OK0 ){
10548 if( zSql ){
10549 int rc = sqlite3_prepare_v3sqlite3_api->prepare_v3(p->pConfig->db, zSql, -1,
10550 SQLITE_PREPARE_PERSISTENT0x01|SQLITE_PREPARE_NO_VTAB0x04,
10551 ppStmt, 0);
10552 /* If this prepare() call fails with SQLITE_ERROR, then one of the
10553 ** %_idx or %_data tables has been removed or modified. Call this
10554 ** corruption. */
10555 p->rc = (rc==SQLITE_ERROR1 ? SQLITE_CORRUPT11 : rc);
10556 }else{
10557 p->rc = SQLITE_NOMEM7;
10558 }
10559 }
10560 sqlite3_freesqlite3_api->free(zSql);
10561 return p->rc;
10562}
10563
10564
10565/*
10566** INSERT OR REPLACE a record into the %_data table.
10567*/
10568static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){
10569 if( p->rc!=SQLITE_OK0 ) return;
10570
10571 if( p->pWriter==0 ){
10572 Fts5Config *pConfig = p->pConfig;
10573 fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintfsqlite3_api->mprintf(
10574 "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)",
10575 pConfig->zDb, pConfig->zName
10576 ));
10577 if( p->rc ) return;
10578 }
10579
10580 sqlite3_bind_int64sqlite3_api->bind_int64(p->pWriter, 1, iRowid);
10581 sqlite3_bind_blobsqlite3_api->bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC((sqlite3_destructor_type)0));
10582 sqlite3_stepsqlite3_api->step(p->pWriter);
10583 p->rc = sqlite3_resetsqlite3_api->reset(p->pWriter);
10584 sqlite3_bind_nullsqlite3_api->bind_null(p->pWriter, 2);
10585}
10586
10587/*
10588** Execute the following SQL:
10589**
10590** DELETE FROM %_data WHERE id BETWEEN $iFirst AND $iLast
10591*/
10592static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){
10593 if( p->rc!=SQLITE_OK0 ) return;
10594
10595 if( p->pDeleter==0 ){
10596 Fts5Config *pConfig = p->pConfig;
10597 char *zSql = sqlite3_mprintfsqlite3_api->mprintf(
10598 "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?",
10599 pConfig->zDb, pConfig->zName
10600 );
10601 if( fts5IndexPrepareStmt(p, &p->pDeleter, zSql) ) return;
10602 }
10603
10604 sqlite3_bind_int64sqlite3_api->bind_int64(p->pDeleter, 1, iFirst);
10605 sqlite3_bind_int64sqlite3_api->bind_int64(p->pDeleter, 2, iLast);
10606 sqlite3_stepsqlite3_api->step(p->pDeleter);
10607 p->rc = sqlite3_resetsqlite3_api->reset(p->pDeleter);
10608}
10609
10610/*
10611** Remove all records associated with segment iSegid.
10612*/
10613static void fts5DataRemoveSegment(Fts5Index *p, Fts5StructureSegment *pSeg){
10614 int iSegid = pSeg->iSegid;
10615 i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(0)) );
10616 i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)( ((i64)(iSegid+1) << (31 +5 +1)) + ((i64)(0) << (
31 + 5)) + ((i64)(0) << (31)) + ((i64)(0)) )-1;
10617 fts5DataDelete(p, iFirst, iLast);
10618
10619 if( pSeg->nPgTombstone ){
10620 i64 iTomb1 = FTS5_TOMBSTONE_ROWID(iSegid, 0)( ((i64)(iSegid+(1<<16)) << (31 +5 +1)) + ((i64)(
0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64)(0))
);
10621 i64 iTomb2 = FTS5_TOMBSTONE_ROWID(iSegid, pSeg->nPgTombstone-1)( ((i64)(iSegid+(1<<16)) << (31 +5 +1)) + ((i64)(
0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64)(pSeg
->nPgTombstone-1)) );
10622 fts5DataDelete(p, iTomb1, iTomb2);
10623 }
10624 if( p->pIdxDeleter==0 ){
10625 Fts5Config *pConfig = p->pConfig;
10626 fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintfsqlite3_api->mprintf(
10627 "DELETE FROM '%q'.'%q_idx' WHERE segid=?",
10628 pConfig->zDb, pConfig->zName
10629 ));
10630 }
10631 if( p->rc==SQLITE_OK0 ){
10632 sqlite3_bind_intsqlite3_api->bind_int(p->pIdxDeleter, 1, iSegid);
10633 sqlite3_stepsqlite3_api->step(p->pIdxDeleter);
10634 p->rc = sqlite3_resetsqlite3_api->reset(p->pIdxDeleter);
10635 }
10636}
10637
10638/*
10639** Release a reference to an Fts5Structure object returned by an earlier
10640** call to fts5StructureRead() or fts5StructureDecode().
10641*/
10642static void fts5StructureRelease(Fts5Structure *pStruct){
10643 if( pStruct && 0>=(--pStruct->nRef) ){
10644 int i;
10645 assert( pStruct->nRef==0 )((void) (0));
10646 for(i=0; i<pStruct->nLevel; i++){
10647 sqlite3_freesqlite3_api->free(pStruct->aLevel[i].aSeg);
10648 }
10649 sqlite3_freesqlite3_api->free(pStruct);
10650 }
10651}
10652
10653static void fts5StructureRef(Fts5Structure *pStruct){
10654 pStruct->nRef++;
10655}
10656
10657static void *sqlite3Fts5StructureRef(Fts5Index *p){
10658 fts5StructureRef(p->pStruct);
10659 return (void*)p->pStruct;
10660}
10661static void sqlite3Fts5StructureRelease(void *p){
10662 if( p ){
10663 fts5StructureRelease((Fts5Structure*)p);
10664 }
10665}
10666static int sqlite3Fts5StructureTest(Fts5Index *p, void *pStruct){
10667 if( p->pStruct!=(Fts5Structure*)pStruct ){
10668 return SQLITE_ABORT4;
10669 }
10670 return SQLITE_OK0;
10671}
10672
10673/*
10674** Ensure that structure object (*pp) is writable.
10675**
10676** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. If
10677** an error occurs, (*pRc) is set to an SQLite error code before returning.
10678*/
10679static void fts5StructureMakeWritable(int *pRc, Fts5Structure **pp){
10680 Fts5Structure *p = *pp;
10681 if( *pRc==SQLITE_OK0 && p->nRef>1 ){
10682 i64 nByte = SZ_FTS5STRUCTURE(p->nLevel)(__builtin_offsetof(Fts5Structure, aLevel) + (p->nLevel)*sizeof
(Fts5StructureLevel));
10683 Fts5Structure *pNew;
10684 pNew = (Fts5Structure*)sqlite3Fts5MallocZero(pRc, nByte);
10685 if( pNew ){
10686 int i;
10687 memcpy(pNew, p, nByte);
10688 for(i=0; i<p->nLevel; i++) pNew->aLevel[i].aSeg = 0;
10689 for(i=0; i<p->nLevel; i++){
10690 Fts5StructureLevel *pLvl = &pNew->aLevel[i];
10691 nByte = sizeof(Fts5StructureSegment) * pNew->aLevel[i].nSeg;
10692 pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(pRc, nByte);
10693 if( pLvl->aSeg==0 ){
10694 for(i=0; i<p->nLevel; i++){
10695 sqlite3_freesqlite3_api->free(pNew->aLevel[i].aSeg);
10696 }
10697 sqlite3_freesqlite3_api->free(pNew);
10698 return;
10699 }
10700 memcpy(pLvl->aSeg, p->aLevel[i].aSeg, nByte);
10701 }
10702 p->nRef--;
10703 pNew->nRef = 1;
10704 }
10705 *pp = pNew;
10706 }
10707}
10708
10709/*
10710** Deserialize and return the structure record currently stored in serialized
10711** form within buffer pData/nData.
10712**
10713** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
10714** are over-allocated by one slot. This allows the structure contents
10715** to be more easily edited.
10716**
10717** If an error occurs, *ppOut is set to NULL and an SQLite error code
10718** returned. Otherwise, *ppOut is set to point to the new object and
10719** SQLITE_OK returned.
10720*/
10721static int fts5StructureDecode(
10722 const u8 *pData, /* Buffer containing serialized structure */
10723 int nData, /* Size of buffer pData in bytes */
10724 int *piCookie, /* Configuration cookie value */
10725 Fts5Structure **ppOut /* OUT: Deserialized object */
10726){
10727 int rc = SQLITE_OK0;
10728 int i = 0;
10729 int iLvl;
10730 int nLevel = 0;
10731 int nSegment = 0;
10732 sqlite3_int64 nByte; /* Bytes of space to allocate at pRet */
10733 Fts5Structure *pRet = 0; /* Structure object to return */
10734 int bStructureV2 = 0; /* True for FTS5_STRUCTURE_V2 */
10735 u64 nOriginCntr = 0; /* Largest origin value seen so far */
10736
10737 /* Grab the cookie value */
10738 if( piCookie ) *piCookie = sqlite3Fts5Get32(pData);
10739 i = 4;
10740
10741 /* Check if this is a V2 structure record. Set bStructureV2 if it is. */
10742 if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2"\xFF\x00\x00\x01", 4) ){
10743 i += 4;
10744 bStructureV2 = 1;
10745 }
10746
10747 /* Read the total number of levels and segments from the start of the
10748 ** structure record. */
10749 i += fts5GetVarint32(&pData[i], nLevel)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(nLevel));
10750 i += fts5GetVarint32(&pData[i], nSegment)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(nSegment));
10751 if( nLevel>FTS5_MAX_SEGMENT2000 || nLevel<0
10752 || nSegment>FTS5_MAX_SEGMENT2000 || nSegment<0
10753 ){
10754 return FTS5_CORRUPT(11 | (1<<8));
10755 }
10756 nByte = SZ_FTS5STRUCTURE(nLevel)(__builtin_offsetof(Fts5Structure, aLevel) + (nLevel)*sizeof(
Fts5StructureLevel));
10757 pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte);
10758
10759 if( pRet ){
10760 pRet->nRef = 1;
10761 pRet->nLevel = nLevel;
10762 pRet->nSegment = nSegment;
10763 i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter);
10764
10765 for(iLvl=0; rc==SQLITE_OK0 && iLvl<nLevel; iLvl++){
10766 Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl];
10767 int nTotal = 0;
10768 int iSeg;
10769
10770 if( i>=nData ){
10771 rc = FTS5_CORRUPT(11 | (1<<8));
10772 }else{
10773 i += fts5GetVarint32(&pData[i], pLvl->nMerge)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(pLvl->nMerge
));
10774 i += fts5GetVarint32(&pData[i], nTotal)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(nTotal));
10775 if( nTotal<pLvl->nMerge ) rc = FTS5_CORRUPT(11 | (1<<8));
10776 pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc,
10777 nTotal * sizeof(Fts5StructureSegment)
10778 );
10779 nSegment -= nTotal;
10780 }
10781
10782 if( rc==SQLITE_OK0 ){
10783 pLvl->nSeg = nTotal;
10784 for(iSeg=0; iSeg<nTotal; iSeg++){
10785 Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
10786 if( i>=nData ){
10787 rc = FTS5_CORRUPT(11 | (1<<8));
10788 break;
10789 }
10790 assert( pSeg!=0 )((void) (0));
10791 i += fts5GetVarint32(&pData[i], pSeg->iSegid)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(pSeg->iSegid
));
10792 i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(pSeg->pgnoFirst
));
10793 i += fts5GetVarint32(&pData[i], pSeg->pgnoLast)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(pSeg->pgnoLast
));
10794 if( bStructureV2 ){
10795 i += fts5GetVarintsqlite3Fts5GetVarint(&pData[i], &pSeg->iOrigin1);
10796 i += fts5GetVarintsqlite3Fts5GetVarint(&pData[i], &pSeg->iOrigin2);
10797 i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(pSeg->nPgTombstone
));
10798 i += fts5GetVarintsqlite3Fts5GetVarint(&pData[i], &pSeg->nEntryTombstone);
10799 i += fts5GetVarintsqlite3Fts5GetVarint(&pData[i], &pSeg->nEntry);
10800 nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2)(((nOriginCntr) > (pSeg->iOrigin2)) ? (nOriginCntr) : (
pSeg->iOrigin2));
10801 }
10802 if( pSeg->pgnoLast<pSeg->pgnoFirst ){
10803 rc = FTS5_CORRUPT(11 | (1<<8));
10804 break;
10805 }
10806 }
10807 if( iLvl>0 && pLvl[-1].nMerge && nTotal==0 ) rc = FTS5_CORRUPT(11 | (1<<8));
10808 if( iLvl==nLevel-1 && pLvl->nMerge ) rc = FTS5_CORRUPT(11 | (1<<8));
10809 }
10810 }
10811 if( nSegment!=0 && rc==SQLITE_OK0 ) rc = FTS5_CORRUPT(11 | (1<<8));
10812 if( bStructureV2 ){
10813 pRet->nOriginCntr = nOriginCntr+1;
10814 }
10815
10816 if( rc!=SQLITE_OK0 ){
10817 fts5StructureRelease(pRet);
10818 pRet = 0;
10819 }
10820 }
10821
10822 *ppOut = pRet;
10823 return rc;
10824}
10825
10826/*
10827** Add a level to the Fts5Structure.aLevel[] array of structure object
10828** (*ppStruct).
10829*/
10830static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){
10831 fts5StructureMakeWritable(pRc, ppStruct);
10832 assert( (ppStruct!=0 && (*ppStruct)!=0) || (*pRc)!=SQLITE_OK )((void) (0));
10833 if( *pRc==SQLITE_OK0 ){
10834 Fts5Structure *pStruct = *ppStruct;
10835 int nLevel = pStruct->nLevel;
10836 sqlite3_int64 nByte = SZ_FTS5STRUCTURE(nLevel+2)(__builtin_offsetof(Fts5Structure, aLevel) + (nLevel+2)*sizeof
(Fts5StructureLevel));
10837
10838 pStruct = sqlite3_realloc64sqlite3_api->realloc64(pStruct, nByte);
10839 if( pStruct ){
10840 memset(&pStruct->aLevel[nLevel], 0, sizeof(Fts5StructureLevel));
10841 pStruct->nLevel++;
10842 *ppStruct = pStruct;
10843 }else{
10844 *pRc = SQLITE_NOMEM7;
10845 }
10846 }
10847}
10848
10849/*
10850** Extend level iLvl so that there is room for at least nExtra more
10851** segments.
10852*/
10853static void fts5StructureExtendLevel(
10854 int *pRc,
10855 Fts5Structure *pStruct,
10856 int iLvl,
10857 int nExtra,
10858 int bInsert
10859){
10860 if( *pRc==SQLITE_OK0 ){
10861 Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
10862 Fts5StructureSegment *aNew;
10863 sqlite3_int64 nByte;
10864
10865 nByte = (pLvl->nSeg + nExtra) * sizeof(Fts5StructureSegment);
10866 aNew = sqlite3_realloc64sqlite3_api->realloc64(pLvl->aSeg, nByte);
10867 if( aNew ){
10868 if( bInsert==0 ){
10869 memset(&aNew[pLvl->nSeg], 0, sizeof(Fts5StructureSegment) * nExtra);
10870 }else{
10871 int nMove = pLvl->nSeg * sizeof(Fts5StructureSegment);
10872 memmove(&aNew[nExtra], aNew, nMove);
10873 memset(aNew, 0, sizeof(Fts5StructureSegment) * nExtra);
10874 }
10875 pLvl->aSeg = aNew;
10876 }else{
10877 *pRc = SQLITE_NOMEM7;
10878 }
10879 }
10880}
10881
10882static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){
10883 Fts5Structure *pRet = 0;
10884 Fts5Config *pConfig = p->pConfig;
10885 int iCookie; /* Configuration cookie */
10886 Fts5Data *pData;
10887
10888 pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID10);
10889 if( p->rc==SQLITE_OK0 ){
10890 /* TODO: Do we need this if the leaf-index is appended? Probably... */
10891 memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING20);
10892 p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
10893 if( p->rc==SQLITE_OK0 ){
10894 if( (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){
10895 p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
10896 }
10897 }else if( p->rc==SQLITE_CORRUPT_VTAB(11 | (1<<8)) ){
10898 sqlite3Fts5ConfigErrmsg(p->pConfig,
10899 "fts5: corrupt structure record for table \"%s\"", p->pConfig->zName
10900 );
10901 }
10902 fts5DataRelease(pData);
10903 if( p->rc!=SQLITE_OK0 ){
10904 fts5StructureRelease(pRet);
10905 pRet = 0;
10906 }
10907 }
10908
10909 return pRet;
10910}
10911
10912static i64 fts5IndexDataVersion(Fts5Index *p){
10913 i64 iVersion = 0;
10914
10915 if( p->rc==SQLITE_OK0 ){
10916 if( p->pDataVersion==0 ){
10917 p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion,
10918 sqlite3_mprintfsqlite3_api->mprintf("PRAGMA %Q.data_version", p->pConfig->zDb)
10919 );
10920 if( p->rc ) return 0;
10921 }
10922
10923 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(p->pDataVersion) ){
10924 iVersion = sqlite3_column_int64sqlite3_api->column_int64(p->pDataVersion, 0);
10925 }
10926 p->rc = sqlite3_resetsqlite3_api->reset(p->pDataVersion);
10927 }
10928
10929 return iVersion;
10930}
10931
10932/*
10933** Read, deserialize and return the structure record.
10934**
10935** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
10936** are over-allocated as described for function fts5StructureDecode()
10937** above.
10938**
10939** If an error occurs, NULL is returned and an error code left in the
10940** Fts5Index handle. If an error has already occurred when this function
10941** is called, it is a no-op.
10942*/
10943static Fts5Structure *fts5StructureRead(Fts5Index *p){
10944
10945 if( p->pStruct==0 ){
10946 p->iStructVersion = fts5IndexDataVersion(p);
10947 if( p->rc==SQLITE_OK0 ){
10948 p->pStruct = fts5StructureReadUncached(p);
10949 }
10950 }
10951
10952#if 0
10953 else{
10954 Fts5Structure *pTest = fts5StructureReadUncached(p);
10955 if( pTest ){
10956 int i, j;
10957 assert_nc( p->pStruct->nSegment==pTest->nSegment )((void) (0));
10958 assert_nc( p->pStruct->nLevel==pTest->nLevel )((void) (0));
10959 for(i=0; i<pTest->nLevel; i++){
10960 assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge )((void) (0));
10961 assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg )((void) (0));
10962 for(j=0; j<pTest->aLevel[i].nSeg; j++){
10963 Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j];
10964 Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j];
10965 assert_nc( p1->iSegid==p2->iSegid )((void) (0));
10966 assert_nc( p1->pgnoFirst==p2->pgnoFirst )((void) (0));
10967 assert_nc( p1->pgnoLast==p2->pgnoLast )((void) (0));
10968 }
10969 }
10970 fts5StructureRelease(pTest);
10971 }
10972 }
10973#endif
10974
10975 if( p->rc!=SQLITE_OK0 ) return 0;
10976 assert( p->iStructVersion!=0 )((void) (0));
10977 assert( p->pStruct!=0 )((void) (0));
10978 fts5StructureRef(p->pStruct);
10979 return p->pStruct;
10980}
10981
10982static void fts5StructureInvalidate(Fts5Index *p){
10983 if( p->pStruct ){
10984 fts5StructureRelease(p->pStruct);
10985 p->pStruct = 0;
10986 }
10987}
10988
10989/*
10990** Return the total number of segments in index structure pStruct. This
10991** function is only ever used as part of assert() conditions.
10992*/
10993#ifdef SQLITE_DEBUG
10994static int fts5StructureCountSegments(Fts5Structure *pStruct){
10995 int nSegment = 0; /* Total number of segments */
10996 if( pStruct ){
10997 int iLvl; /* Used to iterate through levels */
10998 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
10999 nSegment += pStruct->aLevel[iLvl].nSeg;
11000 }
11001 }
11002
11003 return nSegment;
11004}
11005#endif
11006
11007#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob){ ((void) (0)); memcpy(&(pBuf)->p[(pBuf)->n], pBlob
, nBlob); (pBuf)->n += nBlob; } { \
11008 assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) )((void) (0)); \
11009 memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \
11010 (pBuf)->n += nBlob; \
11011}
11012
11013#define fts5BufferSafeAppendVarint(pBuf, iVal){ (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf
)->n], (iVal)); ((void) (0)); } { \
11014 (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \
11015 assert( (pBuf)->nSpace>=(pBuf)->n )((void) (0)); \
11016}
11017
11018
11019/*
11020** Serialize and store the "structure" record.
11021**
11022** If an error occurs, leave an error code in the Fts5Index object. If an
11023** error has already occurred, this function is a no-op.
11024*/
11025static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
11026 if( p->rc==SQLITE_OK0 ){
11027 Fts5Buffer buf; /* Buffer to serialize record into */
11028 int iLvl; /* Used to iterate through levels */
11029 int iCookie; /* Cookie value to store */
11030 int nHdr = (pStruct->nOriginCntr>0 ? (4+4+9+9+9) : (4+9+9));
11031
11032 assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) )((void) (0));
11033 memset(&buf, 0, sizeof(Fts5Buffer));
11034
11035 /* Append the current configuration cookie */
11036 iCookie = p->pConfig->iCookie;
11037 if( iCookie<0 ) iCookie = 0;
11038
11039 if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){
11040 sqlite3Fts5Put32(buf.p, iCookie);
11041 buf.n = 4;
11042 if( pStruct->nOriginCntr>0 ){
11043 fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4){ ((void) (0)); memcpy(&(&buf)->p[(&buf)->n
], "\xFF\x00\x00\x01", 4); (&buf)->n += 4; };
11044 }
11045 fts5BufferSafeAppendVarint(&buf, pStruct->nLevel){ (&buf)->n += sqlite3Fts5PutVarint(&(&buf)->
p[(&buf)->n], (pStruct->nLevel)); ((void) (0)); };
11046 fts5BufferSafeAppendVarint(&buf, pStruct->nSegment){ (&buf)->n += sqlite3Fts5PutVarint(&(&buf)->
p[(&buf)->n], (pStruct->nSegment)); ((void) (0)); };
11047 fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter){ (&buf)->n += sqlite3Fts5PutVarint(&(&buf)->
p[(&buf)->n], ((i64)pStruct->nWriteCounter)); ((void
) (0)); };
11048 }
11049
11050 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
11051 int iSeg; /* Used to iterate through segments */
11052 Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
11053 fts5BufferAppendVarint(&p->rc, &buf, pLvl->nMerge)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pLvl
->nMerge);
11054 fts5BufferAppendVarint(&p->rc, &buf, pLvl->nSeg)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pLvl
->nSeg);
11055 assert( pLvl->nMerge<=pLvl->nSeg )((void) (0));
11056
11057 for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
11058 Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
11059 fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->iSegid);
11060 fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->pgnoFirst);
11061 fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->pgnoLast);
11062 if( pStruct->nOriginCntr>0 ){
11063 fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->iOrigin1);
11064 fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->iOrigin2);
11065 fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->nPgTombstone);
11066 fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->nEntryTombstone);
11067 fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->nEntry);
11068 }
11069 }
11070 }
11071
11072 fts5DataWrite(p, FTS5_STRUCTURE_ROWID10, buf.p, buf.n);
11073 fts5BufferFree(&buf)sqlite3Fts5BufferFree(&buf);
11074 }
11075}
11076
11077#if 0
11078static void fts5DebugStructure(int*,Fts5Buffer*,Fts5Structure*);
11079static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){
11080 int rc = SQLITE_OK0;
11081 Fts5Buffer buf;
11082 memset(&buf, 0, sizeof(buf));
11083 fts5DebugStructure(&rc, &buf, pStruct);
11084 fprintf(stdout, "%s: %s\n", zCaption, buf.p);
11085 fflush(stdout);
11086 fts5BufferFree(&buf)sqlite3Fts5BufferFree(&buf);
11087}
11088#else
11089# define fts5PrintStructure(x,y)
11090#endif
11091
11092static int fts5SegmentSize(Fts5StructureSegment *pSeg){
11093 return 1 + pSeg->pgnoLast - pSeg->pgnoFirst;
11094}
11095
11096/*
11097** Return a copy of index structure pStruct. Except, promote as many
11098** segments as possible to level iPromote. If an OOM occurs, NULL is
11099** returned.
11100*/
11101static void fts5StructurePromoteTo(
11102 Fts5Index *p,
11103 int iPromote,
11104 int szPromote,
11105 Fts5Structure *pStruct
11106){
11107 int il, is;
11108 Fts5StructureLevel *pOut = &pStruct->aLevel[iPromote];
11109
11110 if( pOut->nMerge==0 ){
11111 for(il=iPromote+1; il<pStruct->nLevel; il++){
11112 Fts5StructureLevel *pLvl = &pStruct->aLevel[il];
11113 if( pLvl->nMerge ) return;
11114 for(is=pLvl->nSeg-1; is>=0; is--){
11115 int sz = fts5SegmentSize(&pLvl->aSeg[is]);
11116 if( sz>szPromote ) return;
11117 fts5StructureExtendLevel(&p->rc, pStruct, iPromote, 1, 1);
11118 if( p->rc ) return;
11119 memcpy(pOut->aSeg, &pLvl->aSeg[is], sizeof(Fts5StructureSegment));
11120 pOut->nSeg++;
11121 pLvl->nSeg--;
11122 }
11123 }
11124 }
11125}
11126
11127/*
11128** A new segment has just been written to level iLvl of index structure
11129** pStruct. This function determines if any segments should be promoted
11130** as a result. Segments are promoted in two scenarios:
11131**
11132** a) If the segment just written is smaller than one or more segments
11133** within the previous populated level, it is promoted to the previous
11134** populated level.
11135**
11136** b) If the segment just written is larger than the newest segment on
11137** the next populated level, then that segment, and any other adjacent
11138** segments that are also smaller than the one just written, are
11139** promoted.
11140**
11141** If one or more segments are promoted, the structure object is updated
11142** to reflect this.
11143*/
11144static void fts5StructurePromote(
11145 Fts5Index *p, /* FTS5 backend object */
11146 int iLvl, /* Index level just updated */
11147 Fts5Structure *pStruct /* Index structure */
11148){
11149 if( p->rc==SQLITE_OK0 ){
11150 int iTst;
11151 int iPromote = -1;
11152 int szPromote = 0; /* Promote anything this size or smaller */
11153 Fts5StructureSegment *pSeg; /* Segment just written */
11154 int szSeg; /* Size of segment just written */
11155 int nSeg = pStruct->aLevel[iLvl].nSeg;
11156
11157 if( nSeg==0 ) return;
11158 pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1];
11159 szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst);
11160
11161 /* Check for condition (a) */
11162 for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--);
11163 if( iTst>=0 ){
11164 int i;
11165 int szMax = 0;
11166 Fts5StructureLevel *pTst = &pStruct->aLevel[iTst];
11167 assert( pTst->nMerge==0 )((void) (0));
11168 for(i=0; i<pTst->nSeg; i++){
11169 int sz = pTst->aSeg[i].pgnoLast - pTst->aSeg[i].pgnoFirst + 1;
11170 if( sz>szMax ) szMax = sz;
11171 }
11172 if( szMax>=szSeg ){
11173 /* Condition (a) is true. Promote the newest segment on level
11174 ** iLvl to level iTst. */
11175 iPromote = iTst;
11176 szPromote = szMax;
11177 }
11178 }
11179
11180 /* If condition (a) is not met, assume (b) is true. StructurePromoteTo()
11181 ** is a no-op if it is not. */
11182 if( iPromote<0 ){
11183 iPromote = iLvl;
11184 szPromote = szSeg;
11185 }
11186 fts5StructurePromoteTo(p, iPromote, szPromote, pStruct);
11187 }
11188}
11189
11190
11191/*
11192** Advance the iterator passed as the only argument. If the end of the
11193** doclist-index page is reached, return non-zero.
11194*/
11195static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){
11196 Fts5Data *pData = pLvl->pData;
11197
11198 if( pLvl->iOff==0 ){
11199 assert( pLvl->bEof==0 )((void) (0));
11200 pLvl->iOff = 1;
11201 pLvl->iOff += fts5GetVarint32(&pData->p[1], pLvl->iLeafPgno)sqlite3Fts5GetVarint32(&pData->p[1],(u32*)&(pLvl->
iLeafPgno));
11202 pLvl->iOff += fts5GetVarintsqlite3Fts5GetVarint(&pData->p[pLvl->iOff], (u64*)&pLvl->iRowid);
11203 pLvl->iFirstOff = pLvl->iOff;
11204 }else{
11205 int iOff;
11206 for(iOff=pLvl->iOff; iOff<pData->nn; iOff++){
11207 if( pData->p[iOff] ) break;
11208 }
11209
11210 if( iOff<pData->nn ){
11211 u64 iVal;
11212 pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1;
11213 iOff += fts5GetVarintsqlite3Fts5GetVarint(&pData->p[iOff], &iVal);
11214 pLvl->iRowid += iVal;
11215 pLvl->iOff = iOff;
11216 }else{
11217 pLvl->bEof = 1;
11218 }
11219 }
11220
11221 return pLvl->bEof;
11222}
11223
11224/*
11225** Advance the iterator passed as the only argument.
11226*/
11227static int fts5DlidxIterNextR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){
11228 Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl];
11229
11230 assert( iLvl<pIter->nLvl )((void) (0));
11231 if( fts5DlidxLvlNext(pLvl) ){
11232 if( (iLvl+1) < pIter->nLvl ){
11233 fts5DlidxIterNextR(p, pIter, iLvl+1);
11234 if( pLvl[1].bEof==0 ){
11235 fts5DataRelease(pLvl->pData);
11236 memset(pLvl, 0, sizeof(Fts5DlidxLvl));
11237 pLvl->pData = fts5DataRead(p,
11238 FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno)( ((i64)(pIter->iSegid) << (31 +5 +1)) + ((i64)(1) <<
(31 + 5)) + ((i64)(iLvl) << (31)) + ((i64)(pLvl[1].iLeafPgno
)) )
11239 );
11240 if( pLvl->pData ) fts5DlidxLvlNext(pLvl);
11241 }
11242 }
11243 }
11244
11245 return pIter->aLvl[0].bEof;
11246}
11247static int fts5DlidxIterNext(Fts5Index *p, Fts5DlidxIter *pIter){
11248 return fts5DlidxIterNextR(p, pIter, 0);
11249}
11250
11251/*
11252** The iterator passed as the first argument has the following fields set
11253** as follows. This function sets up the rest of the iterator so that it
11254** points to the first rowid in the doclist-index.
11255**
11256** pData:
11257** pointer to doclist-index record,
11258**
11259** When this function is called pIter->iLeafPgno is the page number the
11260** doclist is associated with (the one featuring the term).
11261*/
11262static int fts5DlidxIterFirst(Fts5DlidxIter *pIter){
11263 int i;
11264 for(i=0; i<pIter->nLvl; i++){
11265 fts5DlidxLvlNext(&pIter->aLvl[i]);
11266 }
11267 return pIter->aLvl[0].bEof;
11268}
11269
11270
11271static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){
11272 return p->rc!=SQLITE_OK0 || pIter->aLvl[0].bEof;
11273}
11274
11275static void fts5DlidxIterLast(Fts5Index *p, Fts5DlidxIter *pIter){
11276 int i;
11277
11278 /* Advance each level to the last entry on the last page */
11279 for(i=pIter->nLvl-1; p->rc==SQLITE_OK0 && i>=0; i--){
11280 Fts5DlidxLvl *pLvl = &pIter->aLvl[i];
11281 while( fts5DlidxLvlNext(pLvl)==0 );
11282 pLvl->bEof = 0;
11283
11284 if( i>0 ){
11285 Fts5DlidxLvl *pChild = &pLvl[-1];
11286 fts5DataRelease(pChild->pData);
11287 memset(pChild, 0, sizeof(Fts5DlidxLvl));
11288 pChild->pData = fts5DataRead(p,
11289 FTS5_DLIDX_ROWID(pIter->iSegid, i-1, pLvl->iLeafPgno)( ((i64)(pIter->iSegid) << (31 +5 +1)) + ((i64)(1) <<
(31 + 5)) + ((i64)(i-1) << (31)) + ((i64)(pLvl->iLeafPgno
)) )
11290 );
11291 }
11292 }
11293}
11294
11295/*
11296** Move the iterator passed as the only argument to the previous entry.
11297*/
11298static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){
11299 int iOff = pLvl->iOff;
11300
11301 assert( pLvl->bEof==0 )((void) (0));
11302 if( iOff<=pLvl->iFirstOff ){
11303 pLvl->bEof = 1;
11304 }else{
11305 u8 *a = pLvl->pData->p;
11306
11307 pLvl->iOff = 0;
11308 fts5DlidxLvlNext(pLvl);
11309 while( 1 ){
11310 int nZero = 0;
11311 int ii = pLvl->iOff;
11312 u64 delta = 0;
11313
11314 while( a[ii]==0 ){
11315 nZero++;
11316 ii++;
11317 }
11318 ii += sqlite3Fts5GetVarint(&a[ii], &delta);
11319
11320 if( ii>=iOff ) break;
11321 pLvl->iLeafPgno += nZero+1;
11322 pLvl->iRowid += delta;
11323 pLvl->iOff = ii;
11324 }
11325 }
11326
11327 return pLvl->bEof;
11328}
11329
11330static int fts5DlidxIterPrevR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){
11331 Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl];
11332
11333 assert( iLvl<pIter->nLvl )((void) (0));
11334 if( fts5DlidxLvlPrev(pLvl) ){
11335 if( (iLvl+1) < pIter->nLvl ){
11336 fts5DlidxIterPrevR(p, pIter, iLvl+1);
11337 if( pLvl[1].bEof==0 ){
11338 fts5DataRelease(pLvl->pData);
11339 memset(pLvl, 0, sizeof(Fts5DlidxLvl));
11340 pLvl->pData = fts5DataRead(p,
11341 FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno)( ((i64)(pIter->iSegid) << (31 +5 +1)) + ((i64)(1) <<
(31 + 5)) + ((i64)(iLvl) << (31)) + ((i64)(pLvl[1].iLeafPgno
)) )
11342 );
11343 if( pLvl->pData ){
11344 while( fts5DlidxLvlNext(pLvl)==0 );
11345 pLvl->bEof = 0;
11346 }
11347 }
11348 }
11349 }
11350
11351 return pIter->aLvl[0].bEof;
11352}
11353static int fts5DlidxIterPrev(Fts5Index *p, Fts5DlidxIter *pIter){
11354 return fts5DlidxIterPrevR(p, pIter, 0);
11355}
11356
11357/*
11358** Free a doclist-index iterator object allocated by fts5DlidxIterInit().
11359*/
11360static void fts5DlidxIterFree(Fts5DlidxIter *pIter){
11361 if( pIter ){
11362 int i;
11363 for(i=0; i<pIter->nLvl; i++){
11364 fts5DataRelease(pIter->aLvl[i].pData);
11365 }
11366 sqlite3_freesqlite3_api->free(pIter);
11367 }
11368}
11369
11370static Fts5DlidxIter *fts5DlidxIterInit(
11371 Fts5Index *p, /* Fts5 Backend to iterate within */
11372 int bRev, /* True for ORDER BY ASC */
11373 int iSegid, /* Segment id */
11374 int iLeafPg /* Leaf page number to load dlidx for */
11375){
11376 Fts5DlidxIter *pIter = 0;
11377 int i;
11378 int bDone = 0;
11379
11380 for(i=0; p->rc==SQLITE_OK0 && bDone==0; i++){
11381 sqlite3_int64 nByte = SZ_FTS5DLIDXITER(i+1)(__builtin_offsetof(Fts5DlidxIter, aLvl)+(i+1)*sizeof(Fts5DlidxLvl
));
11382 Fts5DlidxIter *pNew;
11383
11384 pNew = (Fts5DlidxIter*)sqlite3_realloc64sqlite3_api->realloc64(pIter, nByte);
11385 if( pNew==0 ){
11386 p->rc = SQLITE_NOMEM7;
11387 }else{
11388 i64 iRowid = FTS5_DLIDX_ROWID(iSegid, i, iLeafPg)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(1) << (31
+ 5)) + ((i64)(i) << (31)) + ((i64)(iLeafPg)) );
11389 Fts5DlidxLvl *pLvl = &pNew->aLvl[i];
11390 pIter = pNew;
11391 memset(pLvl, 0, sizeof(Fts5DlidxLvl));
11392 pLvl->pData = fts5DataRead(p, iRowid);
11393 if( pLvl->pData && (pLvl->pData->p[0] & 0x0001)==0 ){
11394 bDone = 1;
11395 }
11396 pIter->nLvl = i+1;
11397 }
11398 }
11399
11400 if( p->rc==SQLITE_OK0 ){
11401 pIter->iSegid = iSegid;
11402 if( bRev==0 ){
11403 fts5DlidxIterFirst(pIter);
11404 }else{
11405 fts5DlidxIterLast(p, pIter);
11406 }
11407 }
11408
11409 if( p->rc!=SQLITE_OK0 ){
11410 fts5DlidxIterFree(pIter);
11411 pIter = 0;
11412 }
11413
11414 return pIter;
11415}
11416
11417static i64 fts5DlidxIterRowid(Fts5DlidxIter *pIter){
11418 return pIter->aLvl[0].iRowid;
11419}
11420static int fts5DlidxIterPgno(Fts5DlidxIter *pIter){
11421 return pIter->aLvl[0].iLeafPgno;
11422}
11423
11424/*
11425** Load the next leaf page into the segment iterator.
11426*/
11427static void fts5SegIterNextPage(
11428 Fts5Index *p, /* FTS5 backend object */
11429 Fts5SegIter *pIter /* Iterator to advance to next page */
11430){
11431 Fts5Data *pLeaf;
11432 Fts5StructureSegment *pSeg = pIter->pSeg;
11433 fts5DataRelease(pIter->pLeaf);
11434 pIter->iLeafPgno++;
11435 if( pIter->pNextLeaf ){
11436 pIter->pLeaf = pIter->pNextLeaf;
11437 pIter->pNextLeaf = 0;
11438 }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
11439 pIter->pLeaf = fts5LeafRead(p,
11440 FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno)( ((i64)(pSeg->iSegid) << (31 +5 +1)) + ((i64)(0) <<
(31 + 5)) + ((i64)(0) << (31)) + ((i64)(pIter->iLeafPgno
)) )
11441 );
11442 }else{
11443 pIter->pLeaf = 0;
11444 }
11445 pLeaf = pIter->pLeaf;
11446
11447 if( pLeaf ){
11448 pIter->iPgidxOff = pLeaf->szLeaf;
11449 if( fts5LeafIsTermless(pLeaf)((pLeaf)->szLeaf >= (pLeaf)->nn) ){
11450 pIter->iEndofDoclist = pLeaf->nn+1;
11451 }else{
11452 pIter->iPgidxOff += fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff],sqlite3Fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff],
(u32*)&(pIter->iEndofDoclist))
11453 pIter->iEndofDoclistsqlite3Fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff],
(u32*)&(pIter->iEndofDoclist))
11454 )sqlite3Fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff],
(u32*)&(pIter->iEndofDoclist));
11455 }
11456 }
11457}
11458
11459/*
11460** Argument p points to a buffer containing a varint to be interpreted as a
11461** position list size field. Read the varint and return the number of bytes
11462** read. Before returning, set *pnSz to the number of bytes in the position
11463** list, and *pbDel to true if the delete flag is set, or false otherwise.
11464*/
11465static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){
11466 int nSz;
11467 int n = 0;
11468 fts5FastGetVarint32(p, n, nSz){ nSz = (p)[n++]; if( nSz & 0x80 ){ n--; n += sqlite3Fts5GetVarint32
(&(p)[n],(u32*)&(nSz)); } };
11469 assert_nc( nSz>=0 )((void) (0));
11470 *pnSz = nSz/2;
11471 *pbDel = nSz & 0x0001;
11472 return n;
11473}
11474
11475/*
11476** Fts5SegIter.iLeafOffset currently points to the first byte of a
11477** position-list size field. Read the value of the field and store it
11478** in the following variables:
11479**
11480** Fts5SegIter.nPos
11481** Fts5SegIter.bDel
11482**
11483** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the
11484** position list content (if any).
11485*/
11486static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
11487 if( p->rc==SQLITE_OK0 ){
11488 int iOff = pIter->iLeafOffset; /* Offset to read at */
11489 ASSERT_SZLEAF_OK(pIter->pLeaf)((void) (0));
11490 if( p->pConfig->eDetail==FTS5_DETAIL_NONE1 ){
11491 int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf)(((pIter->iEndofDoclist) < (pIter->pLeaf->szLeaf)
) ? (pIter->iEndofDoclist) : (pIter->pLeaf->szLeaf));
11492 pIter->bDel = 0;
11493 pIter->nPos = 1;
11494 if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){
11495 pIter->bDel = 1;
11496 iOff++;
11497 if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){
11498 pIter->nPos = 1;
11499 iOff++;
11500 }else{
11501 pIter->nPos = 0;
11502 }
11503 }
11504 }else{
11505 int nSz;
11506 fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz){ nSz = (pIter->pLeaf->p)[iOff++]; if( nSz & 0x80 )
{ iOff--; iOff += sqlite3Fts5GetVarint32(&(pIter->pLeaf
->p)[iOff],(u32*)&(nSz)); } };
11507 pIter->bDel = (nSz & 0x0001);
11508 pIter->nPos = nSz>>1;
11509 assert_nc( pIter->nPos>=0 )((void) (0));
11510 }
11511 pIter->iLeafOffset = iOff;
11512 }
11513}
11514
11515static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
11516 u8 *a = pIter->pLeaf->p; /* Buffer to read data from */
11517 i64 iOff = pIter->iLeafOffset;
11518
11519 ASSERT_SZLEAF_OK(pIter->pLeaf)((void) (0));
11520 while( iOff>=pIter->pLeaf->szLeaf ){
11521 fts5SegIterNextPage(p, pIter);
11522 if( pIter->pLeaf==0 ){
11523 if( p->rc==SQLITE_OK0 ) FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
11524 return;
11525 }
11526 iOff = 4;
11527 a = pIter->pLeaf->p;
11528 }
11529 iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
11530 pIter->iLeafOffset = iOff;
11531}
11532
11533/*
11534** Fts5SegIter.iLeafOffset currently points to the first byte of the
11535** "nSuffix" field of a term. Function parameter nKeep contains the value
11536** of the "nPrefix" field (if there was one - it is passed 0 if this is
11537** the first term in the segment).
11538**
11539** This function populates:
11540**
11541** Fts5SegIter.term
11542** Fts5SegIter.rowid
11543**
11544** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the content of
11545** the first position list. The position list belonging to document
11546** (Fts5SegIter.iRowid).
11547*/
11548static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){
11549 u8 *a = pIter->pLeaf->p; /* Buffer to read data from */
11550 i64 iOff = pIter->iLeafOffset; /* Offset to read at */
11551 int nNew; /* Bytes of new data */
11552
11553 iOff += fts5GetVarint32(&a[iOff], nNew)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nNew));
11554 if( iOff+nNew>pIter->pLeaf->szLeaf || nKeep>pIter->term.n || nNew==0 ){
11555 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
11556 return;
11557 }
11558 pIter->term.n = nKeep;
11559 fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff])sqlite3Fts5BufferAppendBlob(&p->rc,&pIter->term
,nNew,&a[iOff]);
11560 assert( pIter->term.n<=pIter->term.nSpace )((void) (0));
11561 iOff += nNew;
11562 pIter->iTermLeafOffset = iOff;
11563 pIter->iTermLeafPgno = pIter->iLeafPgno;
11564 pIter->iLeafOffset = iOff;
11565
11566 if( pIter->iPgidxOff>=pIter->pLeaf->nn ){
11567 pIter->iEndofDoclist = pIter->pLeaf->nn+1;
11568 }else{
11569 int nExtra;
11570 pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra)sqlite3Fts5GetVarint32(&a[pIter->iPgidxOff],(u32*)&
(nExtra));
11571 pIter->iEndofDoclist += nExtra;
11572 }
11573
11574 fts5SegIterLoadRowid(p, pIter);
11575}
11576
11577static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*);
11578static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*);
11579static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*);
11580
11581static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){
11582 if( pIter->flags & FTS5_SEGITER_REVERSE0x02 ){
11583 pIter->xNext = fts5SegIterNext_Reverse;
11584 }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE1 ){
11585 pIter->xNext = fts5SegIterNext_None;
11586 }else{
11587 pIter->xNext = fts5SegIterNext;
11588 }
11589}
11590
11591/*
11592** Allocate a tombstone hash page array object (pIter->pTombArray) for
11593** the iterator passed as the second argument. If an OOM error occurs,
11594** leave an error in the Fts5Index object.
11595*/
11596static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){
11597 const i64 nTomb = (i64)pIter->pSeg->nPgTombstone;
11598 if( nTomb>0 ){
11599 i64 nByte = SZ_FTS5TOMBSTONEARRAY(nTomb+1)(__builtin_offsetof(Fts5TombstoneArray, apTombstone)+(nTomb+1
)*sizeof(Fts5Data*));
11600 Fts5TombstoneArray *pNew;
11601 pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte);
11602 if( pNew ){
11603 pNew->nTombstone = nTomb;
11604 pNew->nRef = 1;
11605 pIter->pTombArray = pNew;
11606 }
11607 }
11608}
11609
11610/*
11611** Initialize the iterator object pIter to iterate through the entries in
11612** segment pSeg. The iterator is left pointing to the first entry when
11613** this function returns.
11614**
11615** If an error occurs, Fts5Index.rc is set to an appropriate error code. If
11616** an error has already occurred when this function is called, it is a no-op.
11617*/
11618static void fts5SegIterInit(
11619 Fts5Index *p, /* FTS index object */
11620 Fts5StructureSegment *pSeg, /* Description of segment */
11621 Fts5SegIter *pIter /* Object to populate */
11622){
11623 if( pSeg->pgnoFirst==0 ){
11624 /* This happens if the segment is being used as an input to an incremental
11625 ** merge and all data has already been "trimmed". See function
11626 ** fts5TrimSegments() for details. In this case leave the iterator empty.
11627 ** The caller will see the (pIter->pLeaf==0) and assume the iterator is
11628 ** at EOF already. */
11629 assert( pIter->pLeaf==0 )((void) (0));
11630 return;
11631 }
11632
11633 if( p->rc==SQLITE_OK0 ){
11634 memset(pIter, 0, sizeof(*pIter));
11635 fts5SegIterSetNext(p, pIter);
11636 pIter->pSeg = pSeg;
11637 pIter->iLeafPgno = pSeg->pgnoFirst-1;
11638 do {
11639 fts5SegIterNextPage(p, pIter);
11640 }while( p->rc==SQLITE_OK0 && pIter->pLeaf && pIter->pLeaf->nn==4 );
11641 }
11642
11643 if( p->rc==SQLITE_OK0 && pIter->pLeaf ){
11644 pIter->iLeafOffset = 4;
11645 assert( pIter->pLeaf!=0 )((void) (0));
11646 assert_nc( pIter->pLeaf->nn>4 )((void) (0));
11647 assert_nc( fts5LeafFirstTermOff(pIter->pLeaf)==4 )((void) (0));
11648 pIter->iPgidxOff = pIter->pLeaf->szLeaf+1;
11649 fts5SegIterLoadTerm(p, pIter, 0);
11650 fts5SegIterLoadNPos(p, pIter);
11651 fts5SegIterAllocTombstone(p, pIter);
11652 }
11653}
11654
11655/*
11656** This function is only ever called on iterators created by calls to
11657** Fts5IndexQuery() with the FTS5INDEX_QUERY_DESC flag set.
11658**
11659** The iterator is in an unusual state when this function is called: the
11660** Fts5SegIter.iLeafOffset variable is set to the offset of the start of
11661** the position-list size field for the first relevant rowid on the page.
11662** Fts5SegIter.rowid is set, but nPos and bDel are not.
11663**
11664** This function advances the iterator so that it points to the last
11665** relevant rowid on the page and, if necessary, initializes the
11666** aRowidOffset[] and iRowidOffset variables. At this point the iterator
11667** is in its regular state - Fts5SegIter.iLeafOffset points to the first
11668** byte of the position list content associated with said rowid.
11669*/
11670static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){
11671 int eDetail = p->pConfig->eDetail;
11672 int n = pIter->pLeaf->szLeaf;
11673 int i = pIter->iLeafOffset;
11674 u8 *a = pIter->pLeaf->p;
11675 int iRowidOffset = 0;
11676
11677 if( n>pIter->iEndofDoclist ){
11678 n = pIter->iEndofDoclist;
11679 }
11680
11681 ASSERT_SZLEAF_OK(pIter->pLeaf)((void) (0));
11682 while( 1 ){
11683 u64 iDelta = 0;
11684
11685 if( i>=n ) break;
11686 if( eDetail==FTS5_DETAIL_NONE1 ){
11687 /* todo */
11688 if( i<n && a[i]==0 ){
11689 i++;
11690 if( i<n && a[i]==0 ) i++;
11691 }
11692 }else{
11693 int nPos;
11694 int bDummy;
11695 i += fts5GetPoslistSize(&a[i], &nPos, &bDummy);
11696 i += nPos;
11697 }
11698 if( i>=n ) break;
11699 i += fts5GetVarintsqlite3Fts5GetVarint(&a[i], &iDelta);
11700 pIter->iRowid += iDelta;
11701
11702 /* If necessary, grow the pIter->aRowidOffset[] array. */
11703 if( iRowidOffset>=pIter->nRowidOffset ){
11704 i64 nNew = pIter->nRowidOffset + 8;
11705 int *aNew = (int*)sqlite3_realloc64sqlite3_api->realloc64(pIter->aRowidOffset,nNew*sizeof(int));
11706 if( aNew==0 ){
11707 p->rc = SQLITE_NOMEM7;
11708 break;
11709 }
11710 pIter->aRowidOffset = aNew;
11711 pIter->nRowidOffset = nNew;
11712 }
11713
11714 pIter->aRowidOffset[iRowidOffset++] = pIter->iLeafOffset;
11715 pIter->iLeafOffset = i;
11716 }
11717 pIter->iRowidOffset = iRowidOffset;
11718 fts5SegIterLoadNPos(p, pIter);
11719}
11720
11721/*
11722**
11723*/
11724static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
11725 assert( pIter->flags & FTS5_SEGITER_REVERSE )((void) (0));
11726 assert( pIter->flags & FTS5_SEGITER_ONETERM )((void) (0));
11727
11728 fts5DataRelease(pIter->pLeaf);
11729 pIter->pLeaf = 0;
11730 while( p->rc==SQLITE_OK0 && pIter->iLeafPgno>pIter->iTermLeafPgno ){
11731 Fts5Data *pNew;
11732 pIter->iLeafPgno--;
11733 pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(( ((i64)(pIter->pSeg->iSegid) << (31 +5 +1)) + ((
i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64
)(pIter->iLeafPgno)) )
11734 pIter->pSeg->iSegid, pIter->iLeafPgno( ((i64)(pIter->pSeg->iSegid) << (31 +5 +1)) + ((
i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64
)(pIter->iLeafPgno)) )
11735 )( ((i64)(pIter->pSeg->iSegid) << (31 +5 +1)) + ((
i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64
)(pIter->iLeafPgno)) ));
11736 if( pNew ){
11737 /* iTermLeafOffset may be equal to szLeaf if the term is the last
11738 ** thing on the page - i.e. the first rowid is on the following page.
11739 ** In this case leave pIter->pLeaf==0, this iterator is at EOF. */
11740 if( pIter->iLeafPgno==pIter->iTermLeafPgno ){
11741 assert( pIter->pLeaf==0 )((void) (0));
11742 if( pIter->iTermLeafOffset<pNew->szLeaf ){
11743 pIter->pLeaf = pNew;
11744 pIter->iLeafOffset = pIter->iTermLeafOffset;
11745 }
11746 }else{
11747 int iRowidOff;
11748 iRowidOff = fts5LeafFirstRowidOff(pNew)(fts5GetU16((pNew)->p));
11749 if( iRowidOff ){
11750 if( iRowidOff>=pNew->szLeaf ){
11751 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
11752 }else{
11753 pIter->pLeaf = pNew;
11754 pIter->iLeafOffset = iRowidOff;
11755 }
11756 }
11757 }
11758
11759 if( pIter->pLeaf ){
11760 u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset];
11761 pIter->iLeafOffset += fts5GetVarintsqlite3Fts5GetVarint(a, (u64*)&pIter->iRowid);
11762 break;
11763 }else{
11764 fts5DataRelease(pNew);
11765 }
11766 }
11767 }
11768
11769 if( pIter->pLeaf ){
11770 pIter->iEndofDoclist = pIter->pLeaf->nn+1;
11771 fts5SegIterReverseInitPage(p, pIter);
11772 }
11773}
11774
11775/*
11776** Return true if the iterator passed as the second argument currently
11777** points to a delete marker. A delete marker is an entry with a 0 byte
11778** position-list.
11779*/
11780static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){
11781 Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
11782 return (p->rc==SQLITE_OK0 && pSeg->pLeaf && pSeg->nPos==0);
11783}
11784
11785/*
11786** Advance iterator pIter to the next entry.
11787**
11788** This version of fts5SegIterNext() is only used by reverse iterators.
11789*/
11790static void fts5SegIterNext_Reverse(
11791 Fts5Index *p, /* FTS5 backend object */
11792 Fts5SegIter *pIter, /* Iterator to advance */
11793 int *pbUnused /* Unused */
11794){
11795 assert( pIter->flags & FTS5_SEGITER_REVERSE )((void) (0));
11796 assert( pIter->pNextLeaf==0 )((void) (0));
11797 UNUSED_PARAM(pbUnused)(void)(pbUnused);
11798
11799 if( pIter->iRowidOffset>0 ){
11800 u8 *a = pIter->pLeaf->p;
11801 int iOff;
11802 u64 iDelta;
11803
11804 pIter->iRowidOffset--;
11805 pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset];
11806 fts5SegIterLoadNPos(p, pIter);
11807 iOff = pIter->iLeafOffset;
11808 if( p->pConfig->eDetail!=FTS5_DETAIL_NONE1 ){
11809 iOff += pIter->nPos;
11810 }
11811 fts5GetVarintsqlite3Fts5GetVarint(&a[iOff], &iDelta);
11812 pIter->iRowid -= iDelta;
11813 }else{
11814 fts5SegIterReverseNewPage(p, pIter);
11815 }
11816}
11817
11818/*
11819** Advance iterator pIter to the next entry.
11820**
11821** This version of fts5SegIterNext() is only used if detail=none and the
11822** iterator is not a reverse direction iterator.
11823*/
11824static void fts5SegIterNext_None(
11825 Fts5Index *p, /* FTS5 backend object */
11826 Fts5SegIter *pIter, /* Iterator to advance */
11827 int *pbNewTerm /* OUT: Set for new term */
11828){
11829 int iOff;
11830
11831 assert( p->rc==SQLITE_OK )((void) (0));
11832 assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 )((void) (0));
11833 assert( p->pConfig->eDetail==FTS5_DETAIL_NONE )((void) (0));
11834
11835 ASSERT_SZLEAF_OK(pIter->pLeaf)((void) (0));
11836 iOff = pIter->iLeafOffset;
11837
11838 /* Next entry is on the next page */
11839 while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
11840 fts5SegIterNextPage(p, pIter);
11841 if( p->rc || pIter->pLeaf==0 ) return;
11842 pIter->iRowid = 0;
11843 iOff = 4;
11844 }
11845
11846 if( iOff<pIter->iEndofDoclist ){
11847 /* Next entry is on the current page */
11848 u64 iDelta;
11849 iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta);
11850 pIter->iLeafOffset = iOff;
11851 pIter->iRowid += iDelta;
11852 }else if( (pIter->flags & FTS5_SEGITER_ONETERM0x01)==0 ){
11853 if( pIter->pSeg ){
11854 int nKeep = 0;
11855 if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){
11856 iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep)sqlite3Fts5GetVarint32(&pIter->pLeaf->p[iOff],(u32*
)&(nKeep));
11857 }
11858 pIter->iLeafOffset = iOff;
11859 fts5SegIterLoadTerm(p, pIter, nKeep);
11860 }else{
11861 const u8 *pList = 0;
11862 const char *zTerm = 0;
11863 int nTerm = 0;
11864 int nList;
11865 sqlite3Fts5HashScanNext(p->pHash);
11866 sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList);
11867 if( pList==0 ) goto next_none_eof;
11868 pIter->pLeaf->p = (u8*)pList;
11869 pIter->pLeaf->nn = nList;
11870 pIter->pLeaf->szLeaf = nList;
11871 pIter->iEndofDoclist = nList;
11872 sqlite3Fts5BufferSet(&p->rc,&pIter->term, nTerm, (u8*)zTerm);
11873 pIter->iLeafOffset = fts5GetVarintsqlite3Fts5GetVarint(pList, (u64*)&pIter->iRowid);
11874 }
11875
11876 if( pbNewTerm ) *pbNewTerm = 1;
11877 }else{
11878 goto next_none_eof;
11879 }
11880
11881 fts5SegIterLoadNPos(p, pIter);
11882
11883 return;
11884 next_none_eof:
11885 fts5DataRelease(pIter->pLeaf);
11886 pIter->pLeaf = 0;
11887}
11888
11889
11890/*
11891** Advance iterator pIter to the next entry.
11892**
11893** If an error occurs, Fts5Index.rc is set to an appropriate error code. It
11894** is not considered an error if the iterator reaches EOF. If an error has
11895** already occurred when this function is called, it is a no-op.
11896*/
11897static void fts5SegIterNext(
11898 Fts5Index *p, /* FTS5 backend object */
11899 Fts5SegIter *pIter, /* Iterator to advance */
11900 int *pbNewTerm /* OUT: Set for new term */
11901){
11902 Fts5Data *pLeaf = pIter->pLeaf;
11903 int iOff;
11904 int bNewTerm = 0;
11905 int nKeep = 0;
11906 u8 *a;
11907 int n;
11908
11909 assert( pbNewTerm==0 || *pbNewTerm==0 )((void) (0));
11910 assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE )((void) (0));
11911
11912 /* Search for the end of the position list within the current page. */
11913 a = pLeaf->p;
11914 n = pLeaf->szLeaf;
11915
11916 ASSERT_SZLEAF_OK(pLeaf)((void) (0));
11917 iOff = pIter->iLeafOffset + pIter->nPos;
11918
11919 if( iOff<n ){
11920 /* The next entry is on the current page. */
11921 assert_nc( iOff<=pIter->iEndofDoclist )((void) (0));
11922 if( iOff>=pIter->iEndofDoclist ){
11923 bNewTerm = 1;
11924 if( iOff!=fts5LeafFirstTermOff(pLeaf) ){
11925 iOff += fts5GetVarint32(&a[iOff], nKeep)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nKeep));
11926 }
11927 }else{
11928 u64 iDelta;
11929 iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta);
11930 pIter->iRowid += iDelta;
11931 assert_nc( iDelta>0 )((void) (0));
11932 }
11933 pIter->iLeafOffset = iOff;
11934
11935 }else if( pIter->pSeg==0 ){
11936 const u8 *pList = 0;
11937 const char *zTerm = 0;
11938 int nTerm = 0;
11939 int nList = 0;
11940 assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm )((void) (0));
11941 if( 0==(pIter->flags & FTS5_SEGITER_ONETERM0x01) ){
11942 sqlite3Fts5HashScanNext(p->pHash);
11943 sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList);
11944 }
11945 if( pList==0 ){
11946 fts5DataRelease(pIter->pLeaf);
11947 pIter->pLeaf = 0;
11948 }else{
11949 pIter->pLeaf->p = (u8*)pList;
11950 pIter->pLeaf->nn = nList;
11951 pIter->pLeaf->szLeaf = nList;
11952 pIter->iEndofDoclist = nList+1;
11953 sqlite3Fts5BufferSet(&p->rc, &pIter->term, nTerm, (u8*)zTerm);
11954 pIter->iLeafOffset = fts5GetVarintsqlite3Fts5GetVarint(pList, (u64*)&pIter->iRowid);
11955 *pbNewTerm = 1;
11956 }
11957 }else{
11958 iOff = 0;
11959 /* Next entry is not on the current page */
11960 while( iOff==0 ){
11961 fts5SegIterNextPage(p, pIter);
11962 pLeaf = pIter->pLeaf;
11963 if( pLeaf==0 ) break;
11964 ASSERT_SZLEAF_OK(pLeaf)((void) (0));
11965 if( (iOff = fts5LeafFirstRowidOff(pLeaf)(fts5GetU16((pLeaf)->p))) && iOff<pLeaf->szLeaf ){
11966 iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
11967 pIter->iLeafOffset = iOff;
11968
11969 if( pLeaf->nn>pLeaf->szLeaf ){
11970 pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(sqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(pIter->iEndofDoclist))
11971 &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclistsqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(pIter->iEndofDoclist))
11972 )sqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(pIter->iEndofDoclist));
11973 }
11974 }
11975 else if( pLeaf->nn>pLeaf->szLeaf ){
11976 pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(sqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(iOff))
11977 &pLeaf->p[pLeaf->szLeaf], iOffsqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(iOff))
11978 )sqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(iOff));
11979 pIter->iLeafOffset = iOff;
11980 pIter->iEndofDoclist = iOff;
11981 bNewTerm = 1;
11982 }
11983 assert_nc( iOff<pLeaf->szLeaf )((void) (0));
11984 if( iOff>pLeaf->szLeaf ){
11985 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
11986 return;
11987 }
11988 }
11989 }
11990
11991 /* Check if the iterator is now at EOF. If so, return early. */
11992 if( pIter->pLeaf ){
11993 if( bNewTerm ){
11994 if( pIter->flags & FTS5_SEGITER_ONETERM0x01 ){
11995 fts5DataRelease(pIter->pLeaf);
11996 pIter->pLeaf = 0;
11997 }else{
11998 fts5SegIterLoadTerm(p, pIter, nKeep);
11999 fts5SegIterLoadNPos(p, pIter);
12000 if( pbNewTerm ) *pbNewTerm = 1;
12001 }
12002 }else{
12003 /* The following could be done by calling fts5SegIterLoadNPos(). But
12004 ** this block is particularly performance critical, so equivalent
12005 ** code is inlined. */
12006 int nSz;
12007 assert_nc( pIter->iLeafOffset<=pIter->pLeaf->nn )((void) (0));
12008 fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz){ nSz = (pIter->pLeaf->p)[pIter->iLeafOffset++]; if(
nSz & 0x80 ){ pIter->iLeafOffset--; pIter->iLeafOffset
+= sqlite3Fts5GetVarint32(&(pIter->pLeaf->p)[pIter
->iLeafOffset],(u32*)&(nSz)); } };
12009 pIter->bDel = (nSz & 0x0001);
12010 pIter->nPos = nSz>>1;
12011 assert_nc( pIter->nPos>=0 )((void) (0));
12012 }
12013 }
12014}
12015
12016#define SWAPVAL(T, a, b){ T tmp; tmp=a; a=b; b=tmp; } { T tmp; tmp=a; a=b; b=tmp; }
12017
12018#define fts5IndexSkipVarint(a, iOff){ int iEnd = iOff+9; while( (a[iOff++] & 0x80) &&
iOff<iEnd ); } { \
12019 int iEnd = iOff+9; \
12020 while( (a[iOff++] & 0x80) && iOff<iEnd ); \
12021}
12022
12023/*
12024** Iterator pIter currently points to the first rowid in a doclist. This
12025** function sets the iterator up so that iterates in reverse order through
12026** the doclist.
12027*/
12028static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
12029 Fts5DlidxIter *pDlidx = pIter->pDlidx;
12030 Fts5Data *pLast = 0;
12031 int pgnoLast = 0;
12032
12033 if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION4 ){
12034 int iSegid = pIter->pSeg->iSegid;
12035 pgnoLast = fts5DlidxIterPgno(pDlidx);
12036 pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pgnoLast)) ));
12037 }else{
12038 Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */
12039
12040 /* Currently, Fts5SegIter.iLeafOffset points to the first byte of
12041 ** position-list content for the current rowid. Back it up so that it
12042 ** points to the start of the position-list size field. */
12043 int iPoslist;
12044 if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
12045 iPoslist = pIter->iTermLeafOffset;
12046 }else{
12047 iPoslist = 4;
12048 }
12049 fts5IndexSkipVarint(pLeaf->p, iPoslist){ int iEnd = iPoslist+9; while( (pLeaf->p[iPoslist++] &
0x80) && iPoslist<iEnd ); };
12050 pIter->iLeafOffset = iPoslist;
12051
12052 /* If this condition is true then the largest rowid for the current
12053 ** term may not be stored on the current page. So search forward to
12054 ** see where said rowid really is. */
12055 if( pIter->iEndofDoclist>=pLeaf->szLeaf ){
12056 int pgno;
12057 Fts5StructureSegment *pSeg = pIter->pSeg;
12058
12059 /* The last rowid in the doclist may not be on the current page. Search
12060 ** forward to find the page containing the last rowid. */
12061 for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){
12062 i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno)( ((i64)(pSeg->iSegid) << (31 +5 +1)) + ((i64)(0) <<
(31 + 5)) + ((i64)(0) << (31)) + ((i64)(pgno)) );
12063 Fts5Data *pNew = fts5LeafRead(p, iAbs);
12064 if( pNew ){
12065 int iRowid, bTermless;
12066 iRowid = fts5LeafFirstRowidOff(pNew)(fts5GetU16((pNew)->p));
12067 bTermless = fts5LeafIsTermless(pNew)((pNew)->szLeaf >= (pNew)->nn);
12068 if( iRowid ){
12069 SWAPVAL(Fts5Data*, pNew, pLast){ Fts5Data* tmp; tmp=pNew; pNew=pLast; pLast=tmp; };
12070 pgnoLast = pgno;
12071 }
12072 fts5DataRelease(pNew);
12073 if( bTermless==0 ) break;
12074 }
12075 }
12076 }
12077 }
12078
12079 /* If pLast is NULL at this point, then the last rowid for this doclist
12080 ** lies on the page currently indicated by the iterator. In this case
12081 ** pIter->iLeafOffset is already set to point to the position-list size
12082 ** field associated with the first relevant rowid on the page.
12083 **
12084 ** Or, if pLast is non-NULL, then it is the page that contains the last
12085 ** rowid. In this case configure the iterator so that it points to the
12086 ** first rowid on this page.
12087 */
12088 if( pLast ){
12089 int iOff;
12090 fts5DataRelease(pIter->pLeaf);
12091 pIter->pLeaf = pLast;
12092 pIter->iLeafPgno = pgnoLast;
12093 if( p->rc==SQLITE_OK0 ){
12094 iOff = fts5LeafFirstRowidOff(pLast)(fts5GetU16((pLast)->p));
12095 if( iOff>pLast->szLeaf ){
12096 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12097 return;
12098 }
12099 iOff += fts5GetVarintsqlite3Fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid);
12100 pIter->iLeafOffset = iOff;
12101
12102 if( fts5LeafIsTermless(pLast)((pLast)->szLeaf >= (pLast)->nn) ){
12103 pIter->iEndofDoclist = pLast->nn+1;
12104 }else{
12105 pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast);
12106 }
12107 }
12108 }
12109
12110 fts5SegIterReverseInitPage(p, pIter);
12111}
12112
12113/*
12114** Iterator pIter currently points to the first rowid of a doclist.
12115** There is a doclist-index associated with the final term on the current
12116** page. If the current term is the last term on the page, load the
12117** doclist-index from disk and initialize an iterator at (pIter->pDlidx).
12118*/
12119static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){
12120 int iSeg = pIter->pSeg->iSegid;
12121 int bRev = (pIter->flags & FTS5_SEGITER_REVERSE0x02);
12122 Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */
12123
12124 assert( pIter->flags & FTS5_SEGITER_ONETERM )((void) (0));
12125 assert( pIter->pDlidx==0 )((void) (0));
12126
12127 /* Check if the current doclist ends on this page. If it does, return
12128 ** early without loading the doclist-index (as it belongs to a different
12129 ** term. */
12130 if( pIter->iTermLeafPgno==pIter->iLeafPgno
12131 && pIter->iEndofDoclist<pLeaf->szLeaf
12132 ){
12133 return;
12134 }
12135
12136 pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
12137}
12138
12139/*
12140** The iterator object passed as the second argument currently contains
12141** no valid values except for the Fts5SegIter.pLeaf member variable. This
12142** function searches the leaf page for a term matching (pTerm/nTerm).
12143**
12144** If the specified term is found on the page, then the iterator is left
12145** pointing to it. If argument bGe is zero and the term is not found,
12146** the iterator is left pointing at EOF.
12147**
12148** If bGe is non-zero and the specified term is not found, then the
12149** iterator is left pointing to the smallest term in the segment that
12150** is larger than the specified term, even if this term is not on the
12151** current page.
12152*/
12153static void fts5LeafSeek(
12154 Fts5Index *p, /* Leave any error code here */
12155 int bGe, /* True for a >= search */
12156 Fts5SegIter *pIter, /* Iterator to seek */
12157 const u8 *pTerm, int nTerm /* Term to search for */
12158){
12159 u32 iOff;
12160 const u8 *a = pIter->pLeaf->p;
12161 u32 n = (u32)pIter->pLeaf->nn;
12162
12163 u32 nMatch = 0;
12164 u32 nKeep = 0;
12165 u32 nNew = 0;
12166 u32 iTermOff;
12167 u32 iPgidx; /* Current offset in pgidx */
12168 int bEndOfPage = 0;
12169
12170 assert( p->rc==SQLITE_OK )((void) (0));
12171
12172 iPgidx = (u32)pIter->pLeaf->szLeaf;
12173 iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff)sqlite3Fts5GetVarint32(&a[iPgidx],(u32*)&(iTermOff));
12174 iOff = iTermOff;
12175 if( iOff>n ){
12176 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12177 return;
12178 }
12179
12180 while( 1 ){
12181
12182 /* Figure out how many new bytes are in this term */
12183 fts5FastGetVarint32(a, iOff, nNew){ nNew = (a)[iOff++]; if( nNew & 0x80 ){ iOff--; iOff += sqlite3Fts5GetVarint32
(&(a)[iOff],(u32*)&(nNew)); } };
12184 if( nKeep<nMatch ){
12185 goto search_failed;
12186 }
12187 if( (iOff+nNew)>n ){
12188 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12189 return;
12190 }
12191
12192 assert( nKeep>=nMatch )((void) (0));
12193 if( nKeep==nMatch ){
12194 u32 nCmp;
12195 u32 i;
12196 nCmp = (u32)MIN(nNew, nTerm-nMatch)(((nNew) < (nTerm-nMatch)) ? (nNew) : (nTerm-nMatch));
12197 for(i=0; i<nCmp; i++){
12198 if( a[iOff+i]!=pTerm[nMatch+i] ) break;
12199 }
12200 nMatch += i;
12201
12202 if( (u32)nTerm==nMatch ){
12203 if( i==nNew ){
12204 goto search_success;
12205 }else{
12206 goto search_failed;
12207 }
12208 }else if( i<nNew && a[iOff+i]>pTerm[nMatch] ){
12209 goto search_failed;
12210 }
12211 }
12212
12213 if( iPgidx>=n ){
12214 bEndOfPage = 1;
12215 break;
12216 }
12217
12218 iPgidx += fts5GetVarint32(&a[iPgidx], nKeep)sqlite3Fts5GetVarint32(&a[iPgidx],(u32*)&(nKeep));
12219 iTermOff += nKeep;
12220 iOff = iTermOff;
12221
12222 if( iOff>=n ){
12223 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12224 return;
12225 }
12226
12227 /* Read the nKeep field of the next term. */
12228 fts5FastGetVarint32(a, iOff, nKeep){ nKeep = (a)[iOff++]; if( nKeep & 0x80 ){ iOff--; iOff +=
sqlite3Fts5GetVarint32(&(a)[iOff],(u32*)&(nKeep)); }
};
12229 }
12230
12231 search_failed:
12232 if( bGe==0 ){
12233 fts5DataRelease(pIter->pLeaf);
12234 pIter->pLeaf = 0;
12235 return;
12236 }else if( bEndOfPage ){
12237 do {
12238 fts5SegIterNextPage(p, pIter);
12239 if( pIter->pLeaf==0 ) return;
12240 a = pIter->pLeaf->p;
12241 if( fts5LeafIsTermless(pIter->pLeaf)((pIter->pLeaf)->szLeaf >= (pIter->pLeaf)->nn)==0 ){
12242 iPgidx = (u32)pIter->pLeaf->szLeaf;
12243 iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff)sqlite3Fts5GetVarint32(&pIter->pLeaf->p[iPgidx],(u32
*)&(iOff));
12244 if( iOff<4 || (i64)iOff>=pIter->pLeaf->szLeaf ){
12245 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12246 return;
12247 }else{
12248 nKeep = 0;
12249 iTermOff = iOff;
12250 n = (u32)pIter->pLeaf->nn;
12251 iOff += fts5GetVarint32(&a[iOff], nNew)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nNew));
12252 break;
12253 }
12254 }
12255 }while( 1 );
12256 }
12257
12258 search_success:
12259 if( (i64)iOff+nNew>n || nNew<1 ){
12260 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12261 return;
12262 }
12263 pIter->iLeafOffset = iOff + nNew;
12264 pIter->iTermLeafOffset = pIter->iLeafOffset;
12265 pIter->iTermLeafPgno = pIter->iLeafPgno;
12266
12267 fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm)sqlite3Fts5BufferSet(&p->rc,&pIter->term,nKeep,
pTerm);
12268 fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff])sqlite3Fts5BufferAppendBlob(&p->rc,&pIter->term
,nNew,&a[iOff]);
12269
12270 if( iPgidx>=n ){
12271 pIter->iEndofDoclist = pIter->pLeaf->nn+1;
12272 }else{
12273 int nExtra;
12274 iPgidx += fts5GetVarint32(&a[iPgidx], nExtra)sqlite3Fts5GetVarint32(&a[iPgidx],(u32*)&(nExtra));
12275 pIter->iEndofDoclist = iTermOff + nExtra;
12276 }
12277 pIter->iPgidxOff = iPgidx;
12278
12279 fts5SegIterLoadRowid(p, pIter);
12280 fts5SegIterLoadNPos(p, pIter);
12281}
12282
12283static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){
12284 if( p->pIdxSelect==0 ){
12285 Fts5Config *pConfig = p->pConfig;
12286 fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintfsqlite3_api->mprintf(
12287 "SELECT pgno FROM '%q'.'%q_idx' WHERE "
12288 "segid=? AND term<=? ORDER BY term DESC LIMIT 1",
12289 pConfig->zDb, pConfig->zName
12290 ));
12291 }
12292 return p->pIdxSelect;
12293}
12294
12295/*
12296** Initialize the object pIter to point to term pTerm/nTerm within segment
12297** pSeg. If there is no such term in the index, the iterator is set to EOF.
12298**
12299** If an error occurs, Fts5Index.rc is set to an appropriate error code. If
12300** an error has already occurred when this function is called, it is a no-op.
12301*/
12302static void fts5SegIterSeekInit(
12303 Fts5Index *p, /* FTS5 backend */
12304 const u8 *pTerm, int nTerm, /* Term to seek to */
12305 int flags, /* Mask of FTS5INDEX_XXX flags */
12306 Fts5StructureSegment *pSeg, /* Description of segment */
12307 Fts5SegIter *pIter /* Object to populate */
12308){
12309 int iPg = 1;
12310 int bGe = (flags & FTS5INDEX_QUERY_SCAN0x0008);
12311 int bDlidx = 0; /* True if there is a doclist-index */
12312 sqlite3_stmt *pIdxSelect = 0;
12313
12314 assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 )((void) (0));
12315 assert( pTerm && nTerm )((void) (0));
12316 memset(pIter, 0, sizeof(*pIter));
12317 pIter->pSeg = pSeg;
12318
12319 /* This block sets stack variable iPg to the leaf page number that may
12320 ** contain term (pTerm/nTerm), if it is present in the segment. */
12321 pIdxSelect = fts5IdxSelectStmt(p);
12322 if( p->rc ) return;
12323 sqlite3_bind_intsqlite3_api->bind_int(pIdxSelect, 1, pSeg->iSegid);
12324 sqlite3_bind_blobsqlite3_api->bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC((sqlite3_destructor_type)0));
12325 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pIdxSelect) ){
12326 i64 val = sqlite3_column_intsqlite3_api->column_int(pIdxSelect, 0);
12327 iPg = (int)(val>>1);
12328 bDlidx = (val & 0x0001);
12329 }
12330 p->rc = sqlite3_resetsqlite3_api->reset(pIdxSelect);
12331 sqlite3_bind_nullsqlite3_api->bind_null(pIdxSelect, 2);
12332
12333 if( iPg<pSeg->pgnoFirst ){
12334 iPg = pSeg->pgnoFirst;
12335 bDlidx = 0;
12336 }
12337
12338 pIter->iLeafPgno = iPg - 1;
12339 fts5SegIterNextPage(p, pIter);
12340
12341 if( pIter->pLeaf ){
12342 fts5LeafSeek(p, bGe, pIter, pTerm, nTerm);
12343 }
12344
12345 if( p->rc==SQLITE_OK0 && (bGe==0 || (flags & FTS5INDEX_QUERY_SCANONETERM0x0100)) ){
12346 pIter->flags |= FTS5_SEGITER_ONETERM0x01;
12347 if( pIter->pLeaf ){
12348 if( flags & FTS5INDEX_QUERY_DESC0x0002 ){
12349 pIter->flags |= FTS5_SEGITER_REVERSE0x02;
12350 }
12351 if( bDlidx ){
12352 fts5SegIterLoadDlidx(p, pIter);
12353 }
12354 if( flags & FTS5INDEX_QUERY_DESC0x0002 ){
12355 fts5SegIterReverse(p, pIter);
12356 }
12357 }
12358 }
12359
12360 fts5SegIterSetNext(p, pIter);
12361 if( 0==(flags & FTS5INDEX_QUERY_SCANONETERM0x0100) ){
12362 fts5SegIterAllocTombstone(p, pIter);
12363 }
12364
12365 /* Either:
12366 **
12367 ** 1) an error has occurred, or
12368 ** 2) the iterator points to EOF, or
12369 ** 3) the iterator points to an entry with term (pTerm/nTerm), or
12370 ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points
12371 ** to an entry with a term greater than or equal to (pTerm/nTerm).
12372 */
12373 assert_nc( p->rc!=SQLITE_OK /* 1 */((void) (0))
12374 || pIter->pLeaf==0 /* 2 */((void) (0))
12375 || fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)==0 /* 3 */((void) (0))
12376 || (bGe && fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0) /* 4 */((void) (0))
12377 )((void) (0));
12378}
12379
12380
12381/*
12382** SQL used by fts5SegIterNextInit() to find the page to open.
12383*/
12384static sqlite3_stmt *fts5IdxNextStmt(Fts5Index *p){
12385 if( p->pIdxNextSelect==0 ){
12386 Fts5Config *pConfig = p->pConfig;
12387 fts5IndexPrepareStmt(p, &p->pIdxNextSelect, sqlite3_mprintfsqlite3_api->mprintf(
12388 "SELECT pgno FROM '%q'.'%q_idx' WHERE "
12389 "segid=? AND term>? ORDER BY term ASC LIMIT 1",
12390 pConfig->zDb, pConfig->zName
12391 ));
12392
12393 }
12394 return p->pIdxNextSelect;
12395}
12396
12397/*
12398** This is similar to fts5SegIterSeekInit(), except that it initializes
12399** the segment iterator to point to the first term following the page
12400** with pToken/nToken on it.
12401*/
12402static void fts5SegIterNextInit(
12403 Fts5Index *p,
12404 const char *pTerm, int nTerm,
12405 Fts5StructureSegment *pSeg, /* Description of segment */
12406 Fts5SegIter *pIter /* Object to populate */
12407){
12408 int iPg = -1; /* Page of segment to open */
12409 int bDlidx = 0;
12410 sqlite3_stmt *pSel = 0; /* SELECT to find iPg */
12411
12412 pSel = fts5IdxNextStmt(p);
12413 if( pSel ){
12414 assert( p->rc==SQLITE_OK )((void) (0));
12415 sqlite3_bind_intsqlite3_api->bind_int(pSel, 1, pSeg->iSegid);
12416 sqlite3_bind_blobsqlite3_api->bind_blob(pSel, 2, pTerm, nTerm, SQLITE_STATIC((sqlite3_destructor_type)0));
12417
12418 if( sqlite3_stepsqlite3_api->step(pSel)==SQLITE_ROW100 ){
12419 i64 val = sqlite3_column_int64sqlite3_api->column_int64(pSel, 0);
12420 iPg = (int)(val>>1);
12421 bDlidx = (val & 0x0001);
12422 }
12423 p->rc = sqlite3_resetsqlite3_api->reset(pSel);
12424 sqlite3_bind_nullsqlite3_api->bind_null(pSel, 2);
12425 if( p->rc ) return;
12426 }
12427
12428 memset(pIter, 0, sizeof(*pIter));
12429 pIter->pSeg = pSeg;
12430 pIter->flags |= FTS5_SEGITER_ONETERM0x01;
12431 if( iPg>=0 ){
12432 pIter->iLeafPgno = iPg - 1;
12433 fts5SegIterNextPage(p, pIter);
12434 fts5SegIterSetNext(p, pIter);
12435 }
12436 if( pIter->pLeaf ){
12437 const u8 *a = pIter->pLeaf->p;
12438 int iTermOff = 0;
12439
12440 pIter->iPgidxOff = pIter->pLeaf->szLeaf;
12441 pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], iTermOff)sqlite3Fts5GetVarint32(&a[pIter->iPgidxOff],(u32*)&
(iTermOff));
12442 pIter->iLeafOffset = iTermOff;
12443 fts5SegIterLoadTerm(p, pIter, 0);
12444 fts5SegIterLoadNPos(p, pIter);
12445 if( bDlidx ) fts5SegIterLoadDlidx(p, pIter);
12446
12447 assert( p->rc!=SQLITE_OK ||((void) (0))
12448 fts5BufferCompareBlob(&pIter->term, (const u8*)pTerm, nTerm)>0((void) (0))
12449 )((void) (0));
12450 }
12451}
12452
12453/*
12454** Initialize the object pIter to point to term pTerm/nTerm within the
12455** in-memory hash table. If there is no such term in the hash-table, the
12456** iterator is set to EOF.
12457**
12458** If an error occurs, Fts5Index.rc is set to an appropriate error code. If
12459** an error has already occurred when this function is called, it is a no-op.
12460*/
12461static void fts5SegIterHashInit(
12462 Fts5Index *p, /* FTS5 backend */
12463 const u8 *pTerm, int nTerm, /* Term to seek to */
12464 int flags, /* Mask of FTS5INDEX_XXX flags */
12465 Fts5SegIter *pIter /* Object to populate */
12466){
12467 int nList = 0;
12468 const u8 *z = 0;
12469 int n = 0;
12470 Fts5Data *pLeaf = 0;
12471
12472 assert( p->pHash )((void) (0));
12473 assert( p->rc==SQLITE_OK )((void) (0));
12474
12475 if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN0x0008) ){
12476 const u8 *pList = 0;
12477
12478 p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm);
12479 sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &n, &pList, &nList);
12480 if( pList ){
12481 pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data));
12482 if( pLeaf ){
12483 pLeaf->p = (u8*)pList;
12484 }
12485 }
12486
12487 /* The call to sqlite3Fts5HashScanInit() causes the hash table to
12488 ** fill the size field of all existing position lists. This means they
12489 ** can no longer be appended to. Since the only scenario in which they
12490 ** can be appended to is if the previous operation on this table was
12491 ** a DELETE, by clearing the Fts5Index.bDelete flag we can avoid this
12492 ** possibility altogether. */
12493 p->bDelete = 0;
12494 }else{
12495 p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data),
12496 (const char*)pTerm, nTerm, (void**)&pLeaf, &nList
12497 );
12498 if( pLeaf ){
12499 pLeaf->p = (u8*)&pLeaf[1];
12500 }
12501 z = pTerm;
12502 n = nTerm;
12503 pIter->flags |= FTS5_SEGITER_ONETERM0x01;
12504 }
12505
12506 if( pLeaf ){
12507 sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z);
12508 pLeaf->nn = pLeaf->szLeaf = nList;
12509 pIter->pLeaf = pLeaf;
12510 pIter->iLeafOffset = fts5GetVarintsqlite3Fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid);
12511 pIter->iEndofDoclist = pLeaf->nn;
12512
12513 if( flags & FTS5INDEX_QUERY_DESC0x0002 ){
12514 pIter->flags |= FTS5_SEGITER_REVERSE0x02;
12515 fts5SegIterReverseInitPage(p, pIter);
12516 }else{
12517 fts5SegIterLoadNPos(p, pIter);
12518 }
12519 }
12520
12521 fts5SegIterSetNext(p, pIter);
12522}
12523
12524/*
12525** Array ap[] contains n elements. Release each of these elements using
12526** fts5DataRelease(). Then free the array itself using sqlite3_free().
12527*/
12528static void fts5IndexFreeArray(Fts5Data **ap, int n){
12529 if( ap ){
12530 int ii;
12531 for(ii=0; ii<n; ii++){
12532 fts5DataRelease(ap[ii]);
12533 }
12534 sqlite3_freesqlite3_api->free(ap);
12535 }
12536}
12537
12538/*
12539** Decrement the ref-count of the object passed as the only argument. If it
12540** reaches 0, free it and its contents.
12541*/
12542static void fts5TombstoneArrayDelete(Fts5TombstoneArray *p){
12543 if( p ){
12544 p->nRef--;
12545 if( p->nRef<=0 ){
12546 int ii;
12547 for(ii=0; ii<p->nTombstone; ii++){
12548 fts5DataRelease(p->apTombstone[ii]);
12549 }
12550 sqlite3_freesqlite3_api->free(p);
12551 }
12552 }
12553}
12554
12555/*
12556** Zero the iterator passed as the only argument.
12557*/
12558static void fts5SegIterClear(Fts5SegIter *pIter){
12559 fts5BufferFree(&pIter->term)sqlite3Fts5BufferFree(&pIter->term);
12560 fts5DataRelease(pIter->pLeaf);
12561 fts5DataRelease(pIter->pNextLeaf);
12562 fts5TombstoneArrayDelete(pIter->pTombArray);
12563 fts5DlidxIterFree(pIter->pDlidx);
12564 sqlite3_freesqlite3_api->free(pIter->aRowidOffset);
12565 memset(pIter, 0, sizeof(Fts5SegIter));
12566}
12567
12568#ifdef SQLITE_DEBUG
12569
12570/*
12571** This function is used as part of the big assert() procedure implemented by
12572** fts5AssertMultiIterSetup(). It ensures that the result currently stored
12573** in *pRes is the correct result of comparing the current positions of the
12574** two iterators.
12575*/
12576static void fts5AssertComparisonResult(
12577 Fts5Iter *pIter,
12578 Fts5SegIter *p1,
12579 Fts5SegIter *p2,
12580 Fts5CResult *pRes
12581){
12582 int i1 = p1 - pIter->aSeg;
12583 int i2 = p2 - pIter->aSeg;
12584
12585 if( p1->pLeaf || p2->pLeaf ){
12586 if( p1->pLeaf==0 ){
12587 assert( pRes->iFirst==i2 )((void) (0));
12588 }else if( p2->pLeaf==0 ){
12589 assert( pRes->iFirst==i1 )((void) (0));
12590 }else{
12591 int nMin = MIN(p1->term.n, p2->term.n)(((p1->term.n) < (p2->term.n)) ? (p1->term.n) : (
p2->term.n));
12592 int res = fts5Memcmp(p1->term.p, p2->term.p, nMin)((nMin)<=0 ? 0 : memcmp((p1->term.p), (p2->term.p), (
nMin)));
12593 if( res==0 ) res = p1->term.n - p2->term.n;
12594
12595 if( res==0 ){
12596 assert( pRes->bTermEq==1 )((void) (0));
12597 assert( p1->iRowid!=p2->iRowid )((void) (0));
12598 res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : 1;
12599 }else{
12600 assert( pRes->bTermEq==0 )((void) (0));
12601 }
12602
12603 if( res<0 ){
12604 assert( pRes->iFirst==i1 )((void) (0));
12605 }else{
12606 assert( pRes->iFirst==i2 )((void) (0));
12607 }
12608 }
12609 }
12610}
12611
12612/*
12613** This function is a no-op unless SQLITE_DEBUG is defined when this module
12614** is compiled. In that case, this function is essentially an assert()
12615** statement used to verify that the contents of the pIter->aFirst[] array
12616** are correct.
12617*/
12618static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){
12619 if( p->rc==SQLITE_OK0 ){
12620 Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
12621 int i;
12622
12623 assert( (pFirst->pLeaf==0)==pIter->base.bEof )((void) (0));
12624
12625 /* Check that pIter->iSwitchRowid is set correctly. */
12626 for(i=0; i<pIter->nSeg; i++){
12627 Fts5SegIter *p1 = &pIter->aSeg[i];
12628 assert( p1==pFirst((void) (0))
12629 || p1->pLeaf==0((void) (0))
12630 || fts5BufferCompare(&pFirst->term, &p1->term)((void) (0))
12631 || p1->iRowid==pIter->iSwitchRowid((void) (0))
12632 || (p1->iRowid<pIter->iSwitchRowid)==pIter->bRev((void) (0))
12633 )((void) (0));
12634 }
12635
12636 for(i=0; i<pIter->nSeg; i+=2){
12637 Fts5SegIter *p1 = &pIter->aSeg[i];
12638 Fts5SegIter *p2 = &pIter->aSeg[i+1];
12639 Fts5CResult *pRes = &pIter->aFirst[(pIter->nSeg + i) / 2];
12640 fts5AssertComparisonResult(pIter, p1, p2, pRes);
12641 }
12642
12643 for(i=1; i<(pIter->nSeg / 2); i+=2){
12644 Fts5SegIter *p1 = &pIter->aSeg[ pIter->aFirst[i*2].iFirst ];
12645 Fts5SegIter *p2 = &pIter->aSeg[ pIter->aFirst[i*2+1].iFirst ];
12646 Fts5CResult *pRes = &pIter->aFirst[i];
12647 fts5AssertComparisonResult(pIter, p1, p2, pRes);
12648 }
12649 }
12650}
12651#else
12652# define fts5AssertMultiIterSetup(x,y)
12653#endif
12654
12655/*
12656** Do the comparison necessary to populate pIter->aFirst[iOut].
12657**
12658** If the returned value is non-zero, then it is the index of an entry
12659** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing
12660** to a key that is a duplicate of another, higher priority,
12661** segment-iterator in the pSeg->aSeg[] array.
12662*/
12663static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
12664 int i1; /* Index of left-hand Fts5SegIter */
12665 int i2; /* Index of right-hand Fts5SegIter */
12666 int iRes;
12667 Fts5SegIter *p1; /* Left-hand Fts5SegIter */
12668 Fts5SegIter *p2; /* Right-hand Fts5SegIter */
12669 Fts5CResult *pRes = &pIter->aFirst[iOut];
12670
12671 assert( iOut<pIter->nSeg && iOut>0 )((void) (0));
12672 assert( pIter->bRev==0 || pIter->bRev==1 )((void) (0));
12673
12674 if( iOut>=(pIter->nSeg/2) ){
12675 i1 = (iOut - pIter->nSeg/2) * 2;
12676 i2 = i1 + 1;
12677 }else{
12678 i1 = pIter->aFirst[iOut*2].iFirst;
12679 i2 = pIter->aFirst[iOut*2+1].iFirst;
12680 }
12681 p1 = &pIter->aSeg[i1];
12682 p2 = &pIter->aSeg[i2];
12683
12684 pRes->bTermEq = 0;
12685 if( p1->pLeaf==0 ){ /* If p1 is at EOF */
12686 iRes = i2;
12687 }else if( p2->pLeaf==0 ){ /* If p2 is at EOF */
12688 iRes = i1;
12689 }else{
12690 int res = fts5BufferCompare(&p1->term, &p2->term);
12691 if( res==0 ){
12692 assert_nc( i2>i1 )((void) (0));
12693 assert_nc( i2!=0 )((void) (0));
12694 pRes->bTermEq = 1;
12695 if( p1->iRowid==p2->iRowid ){
12696 return i2;
12697 }
12698 res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1;
12699 }
12700 assert( res!=0 )((void) (0));
12701 if( res<0 ){
12702 iRes = i1;
12703 }else{
12704 iRes = i2;
12705 }
12706 }
12707
12708 pRes->iFirst = (u16)iRes;
12709 return 0;
12710}
12711
12712/*
12713** Move the seg-iter so that it points to the first rowid on page iLeafPgno.
12714** It is an error if leaf iLeafPgno does not exist. Unless the db is
12715** a 'secure-delete' db, if it contains no rowids then this is also an error.
12716*/
12717static void fts5SegIterGotoPage(
12718 Fts5Index *p, /* FTS5 backend object */
12719 Fts5SegIter *pIter, /* Iterator to advance */
12720 int iLeafPgno
12721){
12722 assert( iLeafPgno>pIter->iLeafPgno )((void) (0));
12723
12724 if( iLeafPgno>pIter->pSeg->pgnoLast ){
12725 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
12726 }else{
12727 fts5DataRelease(pIter->pNextLeaf);
12728 pIter->pNextLeaf = 0;
12729 pIter->iLeafPgno = iLeafPgno-1;
12730
12731 while( p->rc==SQLITE_OK0 ){
12732 int iOff;
12733 fts5SegIterNextPage(p, pIter);
12734 if( pIter->pLeaf==0 ) break;
12735 iOff = fts5LeafFirstRowidOff(pIter->pLeaf)(fts5GetU16((pIter->pLeaf)->p));
12736 if( iOff>0 ){
12737 u8 *a = pIter->pLeaf->p;
12738 int n = pIter->pLeaf->szLeaf;
12739 if( iOff<4 || iOff>=n ){
12740 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
12741 }else{
12742 iOff += fts5GetVarintsqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
12743 pIter->iLeafOffset = iOff;
12744 fts5SegIterLoadNPos(p, pIter);
12745 }
12746 break;
12747 }
12748 }
12749 }
12750}
12751
12752/*
12753** Advance the iterator passed as the second argument until it is at or
12754** past rowid iFrom. Regardless of the value of iFrom, the iterator is
12755** always advanced at least once.
12756*/
12757static void fts5SegIterNextFrom(
12758 Fts5Index *p, /* FTS5 backend object */
12759 Fts5SegIter *pIter, /* Iterator to advance */
12760 i64 iMatch /* Advance iterator at least this far */
12761){
12762 int bRev = (pIter->flags & FTS5_SEGITER_REVERSE0x02);
12763 Fts5DlidxIter *pDlidx = pIter->pDlidx;
12764 int iLeafPgno = pIter->iLeafPgno;
12765 int bMove = 1;
12766
12767 assert( pIter->flags & FTS5_SEGITER_ONETERM )((void) (0));
12768 assert( pIter->pDlidx )((void) (0));
12769 assert( pIter->pLeaf )((void) (0));
12770
12771 if( bRev==0 ){
12772 while( !fts5DlidxIterEof(p, pDlidx) && iMatch>fts5DlidxIterRowid(pDlidx) ){
12773 iLeafPgno = fts5DlidxIterPgno(pDlidx);
12774 fts5DlidxIterNext(p, pDlidx);
12775 }
12776 assert_nc( iLeafPgno>=pIter->iLeafPgno || p->rc )((void) (0));
12777 if( iLeafPgno>pIter->iLeafPgno ){
12778 fts5SegIterGotoPage(p, pIter, iLeafPgno);
12779 bMove = 0;
12780 }
12781 }else{
12782 assert( pIter->pNextLeaf==0 )((void) (0));
12783 assert( iMatch<pIter->iRowid )((void) (0));
12784 while( !fts5DlidxIterEof(p, pDlidx) && iMatch<fts5DlidxIterRowid(pDlidx) ){
12785 fts5DlidxIterPrev(p, pDlidx);
12786 }
12787 iLeafPgno = fts5DlidxIterPgno(pDlidx);
12788
12789 assert( fts5DlidxIterEof(p, pDlidx) || iLeafPgno<=pIter->iLeafPgno )((void) (0));
12790
12791 if( iLeafPgno<pIter->iLeafPgno ){
12792 pIter->iLeafPgno = iLeafPgno+1;
12793 fts5SegIterReverseNewPage(p, pIter);
12794 bMove = 0;
12795 }
12796 }
12797
12798 do{
12799 if( bMove && p->rc==SQLITE_OK0 ) pIter->xNext(p, pIter, 0);
12800 if( pIter->pLeaf==0 ) break;
12801 if( bRev==0 && pIter->iRowid>=iMatch ) break;
12802 if( bRev!=0 && pIter->iRowid<=iMatch ) break;
12803 bMove = 1;
12804 }while( p->rc==SQLITE_OK0 );
12805}
12806
12807/*
12808** Free the iterator object passed as the second argument.
12809*/
12810static void fts5MultiIterFree(Fts5Iter *pIter){
12811 if( pIter ){
12812 int i;
12813 for(i=0; i<pIter->nSeg; i++){
12814 fts5SegIterClear(&pIter->aSeg[i]);
12815 }
12816 fts5BufferFree(&pIter->poslist)sqlite3Fts5BufferFree(&pIter->poslist);
12817 sqlite3_freesqlite3_api->free(pIter);
12818 }
12819}
12820
12821static void fts5MultiIterAdvanced(
12822 Fts5Index *p, /* FTS5 backend to iterate within */
12823 Fts5Iter *pIter, /* Iterator to update aFirst[] array for */
12824 int iChanged, /* Index of sub-iterator just advanced */
12825 int iMinset /* Minimum entry in aFirst[] to set */
12826){
12827 int i;
12828 for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK0; i=i/2){
12829 int iEq;
12830 if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){
12831 Fts5SegIter *pSeg = &pIter->aSeg[iEq];
12832 assert( p->rc==SQLITE_OK )((void) (0));
12833 pSeg->xNext(p, pSeg, 0);
12834 i = pIter->nSeg + iEq;
12835 }
12836 }
12837}
12838
12839/*
12840** Sub-iterator iChanged of iterator pIter has just been advanced. It still
12841** points to the same term though - just a different rowid. This function
12842** attempts to update the contents of the pIter->aFirst[] accordingly.
12843** If it does so successfully, 0 is returned. Otherwise 1.
12844**
12845** If non-zero is returned, the caller should call fts5MultiIterAdvanced()
12846** on the iterator instead. That function does the same as this one, except
12847** that it deals with more complicated cases as well.
12848*/
12849static int fts5MultiIterAdvanceRowid(
12850 Fts5Iter *pIter, /* Iterator to update aFirst[] array for */
12851 int iChanged, /* Index of sub-iterator just advanced */
12852 Fts5SegIter **ppFirst
12853){
12854 Fts5SegIter *pNew = &pIter->aSeg[iChanged];
12855
12856 if( pNew->iRowid==pIter->iSwitchRowid
12857 || (pNew->iRowid<pIter->iSwitchRowid)==pIter->bRev
12858 ){
12859 int i;
12860 Fts5SegIter *pOther = &pIter->aSeg[iChanged ^ 0x0001];
12861 pIter->iSwitchRowid = pIter->bRev ? SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32))) : LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32));
12862 for(i=(pIter->nSeg+iChanged)/2; 1; i=i/2){
12863 Fts5CResult *pRes = &pIter->aFirst[i];
12864
12865 assert( pNew->pLeaf )((void) (0));
12866 assert( pRes->bTermEq==0 || pOther->pLeaf )((void) (0));
12867
12868 if( pRes->bTermEq ){
12869 if( pNew->iRowid==pOther->iRowid ){
12870 return 1;
12871 }else if( (pOther->iRowid>pNew->iRowid)==pIter->bRev ){
12872 pIter->iSwitchRowid = pOther->iRowid;
12873 pNew = pOther;
12874 }else if( (pOther->iRowid>pIter->iSwitchRowid)==pIter->bRev ){
12875 pIter->iSwitchRowid = pOther->iRowid;
12876 }
12877 }
12878 pRes->iFirst = (u16)(pNew - pIter->aSeg);
12879 if( i==1 ) break;
12880
12881 pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ];
12882 }
12883 }
12884
12885 *ppFirst = pNew;
12886 return 0;
12887}
12888
12889/*
12890** Set the pIter->bEof variable based on the state of the sub-iterators.
12891*/
12892static void fts5MultiIterSetEof(Fts5Iter *pIter){
12893 Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
12894 pIter->base.bEof = pSeg->pLeaf==0;
12895 pIter->iSwitchRowid = pSeg->iRowid;
12896}
12897
12898/*
12899** The argument to this macro must be an Fts5Data structure containing a
12900** tombstone hash page. This macro returns the key-size of the hash-page.
12901*/
12902#define TOMBSTONE_KEYSIZE(pPg)(pPg->p[0]==4 ? 4 : 8) (pPg->p[0]==4 ? 4 : 8)
12903
12904#define TOMBSTONE_NSLOT(pPg)((pPg->nn > 16) ? ((pPg->nn-8) / (pPg->p[0]==4 ? 4
: 8)) : 1) \
12905 ((pPg->nn > 16) ? ((pPg->nn-8) / TOMBSTONE_KEYSIZE(pPg)(pPg->p[0]==4 ? 4 : 8)) : 1)
12906
12907/*
12908** Query a single tombstone hash table for rowid iRowid. Return true if
12909** it is found or false otherwise. The tombstone hash table is one of
12910** nHashTable tables.
12911*/
12912static int fts5IndexTombstoneQuery(
12913 Fts5Data *pHash, /* Hash table page to query */
12914 int nHashTable, /* Number of pages attached to segment */
12915 u64 iRowid /* Rowid to query hash for */
12916){
12917 const int szKey = TOMBSTONE_KEYSIZE(pHash)(pHash->p[0]==4 ? 4 : 8);
12918 const int nSlot = TOMBSTONE_NSLOT(pHash)((pHash->nn > 16) ? ((pHash->nn-8) / (pHash->p[0]
==4 ? 4 : 8)) : 1);
12919 int iSlot = (iRowid / nHashTable) % nSlot;
12920 int nCollide = nSlot;
12921
12922 if( iRowid==0 ){
12923 return pHash->p[1];
12924 }else if( szKey==4 ){
12925 u32 *aSlot = (u32*)&pHash->p[8];
12926 while( aSlot[iSlot] ){
12927 if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1;
12928 if( nCollide--==0 ) break;
12929 iSlot = (iSlot+1)%nSlot;
12930 }
12931 }else{
12932 u64 *aSlot = (u64*)&pHash->p[8];
12933 while( aSlot[iSlot] ){
12934 if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1;
12935 if( nCollide--==0 ) break;
12936 iSlot = (iSlot+1)%nSlot;
12937 }
12938 }
12939
12940 return 0;
12941}
12942
12943/*
12944** Return true if the iterator passed as the only argument points
12945** to an segment entry for which there is a tombstone. Return false
12946** if there is no tombstone or if the iterator is already at EOF.
12947*/
12948static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
12949 int iFirst = pIter->aFirst[1].iFirst;
12950 Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
12951 Fts5TombstoneArray *pArray = pSeg->pTombArray;
12952
12953 if( pSeg->pLeaf && pArray ){
12954 /* Figure out which page the rowid might be present on. */
12955 int iPg = ((u64)pSeg->iRowid) % pArray->nTombstone;
12956 assert( iPg>=0 )((void) (0));
12957
12958 /* If tombstone hash page iPg has not yet been loaded from the
12959 ** database, load it now. */
12960 if( pArray->apTombstone[iPg]==0 ){
12961 pArray->apTombstone[iPg] = fts5DataRead(pIter->pIndex,
12962 FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg)( ((i64)(pSeg->pSeg->iSegid+(1<<16)) << (31
+5 +1)) + ((i64)(0) << (31 + 5)) + ((i64)(0) << (
31)) + ((i64)(iPg)) )
12963 );
12964 if( pArray->apTombstone[iPg]==0 ) return 0;
12965 }
12966
12967 return fts5IndexTombstoneQuery(
12968 pArray->apTombstone[iPg],
12969 pArray->nTombstone,
12970 pSeg->iRowid
12971 );
12972 }
12973
12974 return 0;
12975}
12976
12977/*
12978** Move the iterator to the next entry.
12979**
12980** If an error occurs, an error code is left in Fts5Index.rc. It is not
12981** considered an error if the iterator reaches EOF, or if it is already at
12982** EOF when this function is called.
12983*/
12984static void fts5MultiIterNext(
12985 Fts5Index *p,
12986 Fts5Iter *pIter,
12987 int bFrom, /* True if argument iFrom is valid */
12988 i64 iFrom /* Advance at least as far as this */
12989){
12990 int bUseFrom = bFrom;
12991 assert( pIter->base.bEof==0 )((void) (0));
12992 while( p->rc==SQLITE_OK0 ){
12993 int iFirst = pIter->aFirst[1].iFirst;
12994 int bNewTerm = 0;
12995 Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
12996 assert( p->rc==SQLITE_OK )((void) (0));
12997 if( bUseFrom && pSeg->pDlidx ){
12998 fts5SegIterNextFrom(p, pSeg, iFrom);
12999 }else{
13000 pSeg->xNext(p, pSeg, &bNewTerm);
13001 }
13002
13003 if( pSeg->pLeaf==0 || bNewTerm
13004 || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg)
13005 ){
13006 fts5MultiIterAdvanced(p, pIter, iFirst, 1);
13007 fts5MultiIterSetEof(pIter);
13008 pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
13009 if( pSeg->pLeaf==0 ) return;
13010 }
13011
13012 fts5AssertMultiIterSetup(p, pIter);
13013 assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf )((void) (0));
13014 if( (pIter->bSkipEmpty==0 || pSeg->nPos)
13015 && 0==fts5MultiIterIsDeleted(pIter)
13016 ){
13017 pIter->xSetOutputs(pIter, pSeg);
13018 return;
13019 }
13020 bUseFrom = 0;
13021 }
13022}
13023
13024static void fts5MultiIterNext2(
13025 Fts5Index *p,
13026 Fts5Iter *pIter,
13027 int *pbNewTerm /* OUT: True if *might* be new term */
13028){
13029 assert( pIter->bSkipEmpty )((void) (0));
13030 if( p->rc==SQLITE_OK0 ){
13031 *pbNewTerm = 0;
13032 do{
13033 int iFirst = pIter->aFirst[1].iFirst;
13034 Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
13035 int bNewTerm = 0;
13036
13037 assert( p->rc==SQLITE_OK )((void) (0));
13038 pSeg->xNext(p, pSeg, &bNewTerm);
13039 if( pSeg->pLeaf==0 || bNewTerm
13040 || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg)
13041 ){
13042 fts5MultiIterAdvanced(p, pIter, iFirst, 1);
13043 fts5MultiIterSetEof(pIter);
13044 *pbNewTerm = 1;
13045 }
13046 fts5AssertMultiIterSetup(p, pIter);
13047
13048 }while( (fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter))
13049 && (p->rc==SQLITE_OK0)
13050 );
13051 }
13052}
13053
13054static void fts5IterSetOutputs_Noop(Fts5Iter *pUnused1, Fts5SegIter *pUnused2){
13055 UNUSED_PARAM2(pUnused1, pUnused2)(void)(pUnused1), (void)(pUnused2);
13056}
13057
13058static Fts5Iter *fts5MultiIterAlloc(
13059 Fts5Index *p, /* FTS5 backend to iterate within */
13060 int nSeg
13061){
13062 Fts5Iter *pNew;
13063 i64 nSlot; /* Power of two >= nSeg */
13064
13065 for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
13066 pNew = fts5IdxMalloc(p,
13067 SZ_FTS5ITER(nSlot)(__builtin_offsetof(Fts5Iter, aSeg)+(nSlot)*sizeof(Fts5SegIter
)) + /* pNew + pNew->aSeg[] */
13068 sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */
13069 );
13070 if( pNew ){
13071 pNew->nSeg = nSlot;
13072 pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot];
13073 pNew->pIndex = p;
13074 pNew->xSetOutputs = fts5IterSetOutputs_Noop;
13075 }
13076 return pNew;
13077}
13078
13079static void fts5PoslistCallback(
13080 Fts5Index *pUnused,
13081 void *pContext,
13082 const u8 *pChunk, int nChunk
13083){
13084 UNUSED_PARAM(pUnused)(void)(pUnused);
13085 assert_nc( nChunk>=0 )((void) (0));
13086 if( nChunk>0 ){
13087 fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk){ ((void) (0)); memcpy(&((Fts5Buffer*)pContext)->p[((Fts5Buffer
*)pContext)->n], pChunk, nChunk); ((Fts5Buffer*)pContext)->
n += nChunk; };
13088 }
13089}
13090
13091typedef struct PoslistCallbackCtx PoslistCallbackCtx;
13092struct PoslistCallbackCtx {
13093 Fts5Buffer *pBuf; /* Append to this buffer */
13094 Fts5Colset *pColset; /* Restrict matches to this column */
13095 int eState; /* See above */
13096};
13097
13098typedef struct PoslistOffsetsCtx PoslistOffsetsCtx;
13099struct PoslistOffsetsCtx {
13100 Fts5Buffer *pBuf; /* Append to this buffer */
13101 Fts5Colset *pColset; /* Restrict matches to this column */
13102 int iRead;
13103 int iWrite;
13104};
13105
13106/*
13107** TODO: Make this more efficient!
13108*/
13109static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){
13110 int i;
13111 for(i=0; i<pColset->nCol; i++){
13112 if( pColset->aiCol[i]==iCol ) return 1;
13113 }
13114 return 0;
13115}
13116
13117static void fts5PoslistOffsetsCallback(
13118 Fts5Index *pUnused,
13119 void *pContext,
13120 const u8 *pChunk, int nChunk
13121){
13122 PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext;
13123 UNUSED_PARAM(pUnused)(void)(pUnused);
13124 assert_nc( nChunk>=0 )((void) (0));
13125 if( nChunk>0 ){
13126 int i = 0;
13127 while( i<nChunk ){
13128 int iVal;
13129 i += fts5GetVarint32(&pChunk[i], iVal)sqlite3Fts5GetVarint32(&pChunk[i],(u32*)&(iVal));
13130 iVal += pCtx->iRead - 2;
13131 pCtx->iRead = iVal;
13132 if( fts5IndexColsetTest(pCtx->pColset, iVal) ){
13133 fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite){ (pCtx->pBuf)->n += sqlite3Fts5PutVarint(&(pCtx->
pBuf)->p[(pCtx->pBuf)->n], (iVal + 2 - pCtx->iWrite
)); ((void) (0)); };
13134 pCtx->iWrite = iVal;
13135 }
13136 }
13137 }
13138}
13139
13140static void fts5PoslistFilterCallback(
13141 Fts5Index *pUnused,
13142 void *pContext,
13143 const u8 *pChunk, int nChunk
13144){
13145 PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext;
13146 UNUSED_PARAM(pUnused)(void)(pUnused);
13147 assert_nc( nChunk>=0 )((void) (0));
13148 if( nChunk>0 ){
13149 /* Search through to find the first varint with value 1. This is the
13150 ** start of the next columns hits. */
13151 int i = 0;
13152 int iStart = 0;
13153
13154 if( pCtx->eState==2 ){
13155 int iCol;
13156 fts5FastGetVarint32(pChunk, i, iCol){ iCol = (pChunk)[i++]; if( iCol & 0x80 ){ i--; i += sqlite3Fts5GetVarint32
(&(pChunk)[i],(u32*)&(iCol)); } };
13157 if( fts5IndexColsetTest(pCtx->pColset, iCol) ){
13158 pCtx->eState = 1;
13159 fts5BufferSafeAppendVarint(pCtx->pBuf, 1){ (pCtx->pBuf)->n += sqlite3Fts5PutVarint(&(pCtx->
pBuf)->p[(pCtx->pBuf)->n], (1)); ((void) (0)); };
13160 }else{
13161 pCtx->eState = 0;
13162 }
13163 }
13164
13165 do {
13166 while( i<nChunk && pChunk[i]!=0x01 ){
13167 fts5IndexSkipVarint(pChunk, i){ int iEnd = i+9; while( (pChunk[i++] & 0x80) && i
<iEnd ); };
13168 }
13169 if( pCtx->eState ){
13170 fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart){ ((void) (0)); memcpy(&(pCtx->pBuf)->p[(pCtx->pBuf
)->n], &pChunk[iStart], i-iStart); (pCtx->pBuf)->
n += i-iStart; };
13171 }
13172 if( i<nChunk ){
13173 int iCol;
13174 iStart = i;
13175 i++;
13176 if( i>=nChunk ){
13177 pCtx->eState = 2;
13178 }else{
13179 fts5FastGetVarint32(pChunk, i, iCol){ iCol = (pChunk)[i++]; if( iCol & 0x80 ){ i--; i += sqlite3Fts5GetVarint32
(&(pChunk)[i],(u32*)&(iCol)); } };
13180 pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol);
13181 if( pCtx->eState ){
13182 fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart){ ((void) (0)); memcpy(&(pCtx->pBuf)->p[(pCtx->pBuf
)->n], &pChunk[iStart], i-iStart); (pCtx->pBuf)->
n += i-iStart; };
13183 iStart = i;
13184 }
13185 }
13186 }
13187 }while( i<nChunk );
13188 }
13189}
13190
13191static void fts5ChunkIterate(
13192 Fts5Index *p, /* Index object */
13193 Fts5SegIter *pSeg, /* Poslist of this iterator */
13194 void *pCtx, /* Context pointer for xChunk callback */
13195 void (*xChunk)(Fts5Index*, void*, const u8*, int)
13196){
13197 int nRem = pSeg->nPos; /* Number of bytes still to come */
13198 Fts5Data *pData = 0;
13199 u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset];
13200 int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset)(((nRem) < (pSeg->pLeaf->szLeaf - pSeg->iLeafOffset
)) ? (nRem) : (pSeg->pLeaf->szLeaf - pSeg->iLeafOffset
));
13201 int pgno = pSeg->iLeafPgno;
13202 int pgnoSave = 0;
13203
13204 /* This function does not work with detail=none databases. */
13205 assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE )((void) (0));
13206
13207 if( (pSeg->flags & FTS5_SEGITER_REVERSE0x02)==0 ){
13208 pgnoSave = pgno+1;
13209 }
13210
13211 while( 1 ){
13212 xChunk(p, pCtx, pChunk, nChunk);
13213 nRem -= nChunk;
13214 fts5DataRelease(pData);
13215 if( nRem<=0 ){
13216 break;
13217 }else if( pSeg->pSeg==0 ){
13218 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
13219 return;
13220 }else{
13221 pgno++;
13222 pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)( ((i64)(pSeg->pSeg->iSegid) << (31 +5 +1)) + ((i64
)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64)(pgno
)) ));
13223 if( pData==0 ) break;
13224 pChunk = &pData->p[4];
13225 nChunk = MIN(nRem, pData->szLeaf - 4)(((nRem) < (pData->szLeaf - 4)) ? (nRem) : (pData->szLeaf
- 4));
13226 if( pgno==pgnoSave ){
13227 assert( pSeg->pNextLeaf==0 )((void) (0));
13228 pSeg->pNextLeaf = pData;
13229 pData = 0;
13230 }
13231 }
13232 }
13233}
13234
13235/*
13236** Iterator pIter currently points to a valid entry (not EOF). This
13237** function appends the position list data for the current entry to
13238** buffer pBuf. It does not make a copy of the position-list size
13239** field.
13240*/
13241static void fts5SegiterPoslist(
13242 Fts5Index *p,
13243 Fts5SegIter *pSeg,
13244 Fts5Colset *pColset,
13245 Fts5Buffer *pBuf
13246){
13247 assert( pBuf!=0 )((void) (0));
13248 assert( pSeg!=0 )((void) (0));
13249 if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+FTS5_DATA_ZERO_PADDING)( (u32)((pBuf)->n) + (u32)(pSeg->nPos+8) <= (u32)((pBuf
)->nSpace) ? 0 : sqlite3Fts5BufferSize((&p->rc),(pBuf
),(pSeg->nPos+8)+(pBuf)->n) ) ){
13250 assert( pBuf->p!=0 )((void) (0));
13251 assert( pBuf->nSpace >= pBuf->n+pSeg->nPos+FTS5_DATA_ZERO_PADDING )((void) (0));
13252 memset(&pBuf->p[pBuf->n+pSeg->nPos], 0, FTS5_DATA_ZERO_PADDING8);
13253 if( pColset==0 ){
13254 fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
13255 }else{
13256 if( p->pConfig->eDetail==FTS5_DETAIL_FULL0 ){
13257 PoslistCallbackCtx sCtx;
13258 sCtx.pBuf = pBuf;
13259 sCtx.pColset = pColset;
13260 sCtx.eState = fts5IndexColsetTest(pColset, 0);
13261 assert( sCtx.eState==0 || sCtx.eState==1 )((void) (0));
13262 fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
13263 }else{
13264 PoslistOffsetsCtx sCtx;
13265 memset(&sCtx, 0, sizeof(sCtx));
13266 sCtx.pBuf = pBuf;
13267 sCtx.pColset = pColset;
13268 fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
13269 }
13270 }
13271 }
13272}
13273
13274/*
13275** Parameter pPos points to a buffer containing a position list, size nPos.
13276** This function filters it according to pColset (which must be non-NULL)
13277** and sets pIter->base.pData/nData to point to the new position list.
13278** If memory is required for the new position list, use buffer pIter->poslist.
13279** Or, if the new position list is a contiguous subset of the input, set
13280** pIter->base.pData/nData to point directly to it.
13281**
13282** This function is a no-op if *pRc is other than SQLITE_OK when it is
13283** called. If an OOM error is encountered, *pRc is set to SQLITE_NOMEM
13284** before returning.
13285*/
13286static void fts5IndexExtractColset(
13287 int *pRc,
13288 Fts5Colset *pColset, /* Colset to filter on */
13289 const u8 *pPos, int nPos, /* Position list */
13290 Fts5Iter *pIter
13291){
13292 if( *pRc==SQLITE_OK0 ){
13293 const u8 *p = pPos;
13294 const u8 *aCopy = p;
13295 const u8 *pEnd = &p[nPos]; /* One byte past end of position list */
13296 int i = 0;
13297 int iCurrent = 0;
13298
13299 if( pColset->nCol>1 && sqlite3Fts5BufferSize(pRc, &pIter->poslist, nPos) ){
13300 return;
13301 }
13302
13303 while( 1 ){
13304 while( pColset->aiCol[i]<iCurrent ){
13305 i++;
13306 if( i==pColset->nCol ){
13307 pIter->base.pData = pIter->poslist.p;
13308 pIter->base.nData = pIter->poslist.n;
13309 return;
13310 }
13311 }
13312
13313 /* Advance pointer p until it points to pEnd or an 0x01 byte that is
13314 ** not part of a varint */
13315 while( p<pEnd && *p!=0x01 ){
13316 while( p<pEnd && (*p++ & 0x80) );
13317 }
13318
13319 if( pColset->aiCol[i]==iCurrent ){
13320 if( pColset->nCol==1 ){
13321 pIter->base.pData = aCopy;
13322 pIter->base.nData = p-aCopy;
13323 return;
13324 }
13325 fts5BufferSafeAppendBlob(&pIter->poslist, aCopy, p-aCopy){ ((void) (0)); memcpy(&(&pIter->poslist)->p[(&
pIter->poslist)->n], aCopy, p-aCopy); (&pIter->poslist
)->n += p-aCopy; };
13326 }
13327 if( p>=pEnd ){
13328 pIter->base.pData = pIter->poslist.p;
13329 pIter->base.nData = pIter->poslist.n;
13330 return;
13331 }
13332 aCopy = p++;
13333 iCurrent = *p++;
13334 if( iCurrent & 0x80 ){
13335 p--;
13336 p += fts5GetVarint32(p, iCurrent)sqlite3Fts5GetVarint32(p,(u32*)&(iCurrent));
13337 }
13338 }
13339 }
13340
13341}
13342
13343/*
13344** xSetOutputs callback used by detail=none tables.
13345*/
13346static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
13347 assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE )((void) (0));
13348 pIter->base.iRowid = pSeg->iRowid;
13349 pIter->base.nData = pSeg->nPos;
13350}
13351
13352/*
13353** xSetOutputs callback used by detail=full and detail=col tables when no
13354** column filters are specified.
13355*/
13356static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
13357 pIter->base.iRowid = pSeg->iRowid;
13358 pIter->base.nData = pSeg->nPos;
13359
13360 assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE )((void) (0));
13361 assert( pIter->pColset==0 )((void) (0));
13362
13363 if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
13364 /* All data is stored on the current page. Populate the output
13365 ** variables to point into the body of the page object. */
13366 pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset];
13367 }else{
13368 /* The data is distributed over two or more pages. Copy it into the
13369 ** Fts5Iter.poslist buffer and then set the output pointer to point
13370 ** to this buffer. */
13371 fts5BufferZero(&pIter->poslist)sqlite3Fts5BufferZero(&pIter->poslist);
13372 fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
13373 pIter->base.pData = pIter->poslist.p;
13374 }
13375}
13376
13377/*
13378** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match
13379** against no columns at all).
13380*/
13381static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){
13382 UNUSED_PARAM(pSeg)(void)(pSeg);
13383 pIter->base.nData = 0;
13384}
13385
13386/*
13387** xSetOutputs callback used by detail=col when there is a column filter
13388** and there are 100 or more columns. Also called as a fallback from
13389** fts5IterSetOutputs_Col100 if the column-list spans more than one page.
13390*/
13391static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
13392 fts5BufferZero(&pIter->poslist)sqlite3Fts5BufferZero(&pIter->poslist);
13393 fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist);
13394 pIter->base.iRowid = pSeg->iRowid;
13395 pIter->base.pData = pIter->poslist.p;
13396 pIter->base.nData = pIter->poslist.n;
13397}
13398
13399/*
13400** xSetOutputs callback used when:
13401**
13402** * detail=col,
13403** * there is a column filter, and
13404** * the table contains 100 or fewer columns.
13405**
13406** The last point is to ensure all column numbers are stored as
13407** single-byte varints.
13408*/
13409static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){
13410
13411 assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS )((void) (0));
13412 assert( pIter->pColset )((void) (0));
13413 assert( pIter->poslist.nSpace>=pIter->pIndex->pConfig->nCol )((void) (0));
13414
13415 if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf
13416 || pSeg->nPos>pIter->pIndex->pConfig->nCol
13417 ){
13418 fts5IterSetOutputs_Col(pIter, pSeg);
13419 }else{
13420 u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
13421 u8 *pEnd = (u8*)&a[pSeg->nPos];
13422 int iPrev = 0;
13423 int *aiCol = pIter->pColset->aiCol;
13424 int *aiColEnd = &aiCol[pIter->pColset->nCol];
13425
13426 u8 *aOut = pIter->poslist.p;
13427 int iPrevOut = 0;
13428
13429 pIter->base.iRowid = pSeg->iRowid;
13430
13431 while( a<pEnd ){
13432 iPrev += (int)a++[0] - 2;
13433 while( *aiCol<iPrev ){
13434 aiCol++;
13435 if( aiCol==aiColEnd ) goto setoutputs_col_out;
13436 }
13437 if( *aiCol==iPrev ){
13438 *aOut++ = (u8)((iPrev - iPrevOut) + 2);
13439 iPrevOut = iPrev;
13440 }
13441 }
13442
13443setoutputs_col_out:
13444 pIter->base.pData = pIter->poslist.p;
13445 pIter->base.nData = aOut - pIter->poslist.p;
13446 }
13447}
13448
13449/*
13450** xSetOutputs callback used by detail=full when there is a column filter.
13451*/
13452static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
13453 Fts5Colset *pColset = pIter->pColset;
13454 pIter->base.iRowid = pSeg->iRowid;
13455
13456 assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL )((void) (0));
13457 assert( pColset )((void) (0));
13458
13459 if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
13460 /* All data is stored on the current page. Populate the output
13461 ** variables to point into the body of the page object. */
13462 const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
13463 int *pRc = &pIter->pIndex->rc;
13464 fts5BufferZero(&pIter->poslist)sqlite3Fts5BufferZero(&pIter->poslist);
13465 fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, pIter);
13466 }else{
13467 /* The data is distributed over two or more pages. Copy it into the
13468 ** Fts5Iter.poslist buffer and then set the output pointer to point
13469 ** to this buffer. */
13470 fts5BufferZero(&pIter->poslist)sqlite3Fts5BufferZero(&pIter->poslist);
13471 fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
13472 pIter->base.pData = pIter->poslist.p;
13473 pIter->base.nData = pIter->poslist.n;
13474 }
13475}
13476
13477static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){
13478 assert( pIter!=0 || (*pRc)!=SQLITE_OK )((void) (0));
13479 if( *pRc==SQLITE_OK0 ){
13480 Fts5Config *pConfig = pIter->pIndex->pConfig;
13481 if( pConfig->eDetail==FTS5_DETAIL_NONE1 ){
13482 pIter->xSetOutputs = fts5IterSetOutputs_None;
13483 }
13484
13485 else if( pIter->pColset==0 ){
13486 pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
13487 }
13488
13489 else if( pIter->pColset->nCol==0 ){
13490 pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset;
13491 }
13492
13493 else if( pConfig->eDetail==FTS5_DETAIL_FULL0 ){
13494 pIter->xSetOutputs = fts5IterSetOutputs_Full;
13495 }
13496
13497 else{
13498 assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS )((void) (0));
13499 if( pConfig->nCol<=100 ){
13500 pIter->xSetOutputs = fts5IterSetOutputs_Col100;
13501 sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol);
13502 }else{
13503 pIter->xSetOutputs = fts5IterSetOutputs_Col;
13504 }
13505 }
13506 }
13507}
13508
13509/*
13510** All the component segment-iterators of pIter have been set up. This
13511** functions finishes setup for iterator pIter itself.
13512*/
13513static void fts5MultiIterFinishSetup(Fts5Index *p, Fts5Iter *pIter){
13514 int iIter;
13515 for(iIter=pIter->nSeg-1; iIter>0; iIter--){
13516 int iEq;
13517 if( (iEq = fts5MultiIterDoCompare(pIter, iIter)) ){
13518 Fts5SegIter *pSeg = &pIter->aSeg[iEq];
13519 if( p->rc==SQLITE_OK0 ) pSeg->xNext(p, pSeg, 0);
13520 fts5MultiIterAdvanced(p, pIter, iEq, iIter);
13521 }
13522 }
13523 fts5MultiIterSetEof(pIter);
13524 fts5AssertMultiIterSetup(p, pIter);
13525
13526 if( (pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter))
13527 || fts5MultiIterIsDeleted(pIter)
13528 ){
13529 fts5MultiIterNext(p, pIter, 0, 0);
13530 }else if( pIter->base.bEof==0 ){
13531 Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
13532 pIter->xSetOutputs(pIter, pSeg);
13533 }
13534}
13535
13536/*
13537** Allocate a new Fts5Iter object.
13538**
13539** The new object will be used to iterate through data in structure pStruct.
13540** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel
13541** is zero or greater, data from the first nSegment segments on level iLevel
13542** is merged.
13543**
13544** The iterator initially points to the first term/rowid entry in the
13545** iterated data.
13546*/
13547static void fts5MultiIterNew(
13548 Fts5Index *p, /* FTS5 backend to iterate within */
13549 Fts5Structure *pStruct, /* Structure of specific index */
13550 int flags, /* FTS5INDEX_QUERY_XXX flags */
13551 Fts5Colset *pColset, /* Colset to filter on (or NULL) */
13552 const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */
13553 int iLevel, /* Level to iterate (-1 for all) */
13554 int nSegment, /* Number of segments to merge (iLevel>=0) */
13555 Fts5Iter **ppOut /* New object */
13556){
13557 int nSeg = 0; /* Number of segment-iters in use */
13558 int iIter = 0; /* */
13559 int iSeg; /* Used to iterate through segments */
13560 Fts5StructureLevel *pLvl;
13561 Fts5Iter *pNew;
13562
13563 assert( (pTerm==0 && nTerm==0) || iLevel<0 )((void) (0));
13564
13565 /* Allocate space for the new multi-seg-iterator. */
13566 if( p->rc==SQLITE_OK0 ){
13567 if( iLevel<0 ){
13568 assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) )((void) (0));
13569 nSeg = pStruct->nSegment;
13570 nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH0x0040));
13571 }else{
13572 nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment)(((pStruct->aLevel[iLevel].nSeg) < (nSegment)) ? (pStruct
->aLevel[iLevel].nSeg) : (nSegment));
13573 }
13574 }
13575 *ppOut = pNew = fts5MultiIterAlloc(p, nSeg);
13576 if( pNew==0 ){
13577 assert( p->rc!=SQLITE_OK )((void) (0));
13578 goto fts5MultiIterNew_post_check;
13579 }
13580 pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC0x0002));
13581 pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY0x0010));
13582 pNew->pColset = pColset;
13583 if( (flags & FTS5INDEX_QUERY_NOOUTPUT0x0020)==0 ){
13584 fts5IterSetOutputCb(&p->rc, pNew);
13585 }
13586
13587 /* Initialize each of the component segment iterators. */
13588 if( p->rc==SQLITE_OK0 ){
13589 if( iLevel<0 ){
13590 Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
13591 if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH0x0040) ){
13592 /* Add a segment iterator for the current contents of the hash table. */
13593 Fts5SegIter *pIter = &pNew->aSeg[iIter++];
13594 fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
13595 }
13596 for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){
13597 for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){
13598 Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
13599 Fts5SegIter *pIter = &pNew->aSeg[iIter++];
13600 if( pTerm==0 ){
13601 fts5SegIterInit(p, pSeg, pIter);
13602 }else{
13603 fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter);
13604 }
13605 }
13606 }
13607 }else{
13608 pLvl = &pStruct->aLevel[iLevel];
13609 for(iSeg=nSeg-1; iSeg>=0; iSeg--){
13610 fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]);
13611 }
13612 }
13613 assert( iIter==nSeg )((void) (0));
13614 }
13615
13616 /* If the above was successful, each component iterator now points
13617 ** to the first entry in its segment. In this case initialize the
13618 ** aFirst[] array. Or, if an error has occurred, free the iterator
13619 ** object and set the output variable to NULL. */
13620 if( p->rc==SQLITE_OK0 ){
13621 fts5MultiIterFinishSetup(p, pNew);
13622 }else{
13623 fts5MultiIterFree(pNew);
13624 *ppOut = 0;
13625 }
13626
13627fts5MultiIterNew_post_check:
13628 assert( (*ppOut)!=0 || p->rc!=SQLITE_OK )((void) (0));
13629 return;
13630}
13631
13632/*
13633** Create an Fts5Iter that iterates through the doclist provided
13634** as the second argument.
13635*/
13636static void fts5MultiIterNew2(
13637 Fts5Index *p, /* FTS5 backend to iterate within */
13638 Fts5Data *pData, /* Doclist to iterate through */
13639 int bDesc, /* True for descending rowid order */
13640 Fts5Iter **ppOut /* New object */
13641){
13642 Fts5Iter *pNew;
13643 pNew = fts5MultiIterAlloc(p, 2);
13644 if( pNew ){
13645 Fts5SegIter *pIter = &pNew->aSeg[1];
13646 pIter->flags = FTS5_SEGITER_ONETERM0x01;
13647 if( pData->szLeaf>0 ){
13648 pIter->pLeaf = pData;
13649 pIter->iLeafOffset = fts5GetVarintsqlite3Fts5GetVarint(pData->p, (u64*)&pIter->iRowid);
13650 pIter->iEndofDoclist = pData->nn;
13651 pNew->aFirst[1].iFirst = 1;
13652 if( bDesc ){
13653 pNew->bRev = 1;
13654 pIter->flags |= FTS5_SEGITER_REVERSE0x02;
13655 fts5SegIterReverseInitPage(p, pIter);
13656 }else{
13657 fts5SegIterLoadNPos(p, pIter);
13658 }
13659 pData = 0;
13660 }else{
13661 pNew->base.bEof = 1;
13662 }
13663 fts5SegIterSetNext(p, pIter);
13664
13665 *ppOut = pNew;
13666 }
13667
13668 fts5DataRelease(pData);
13669}
13670
13671/*
13672** Return true if the iterator is at EOF or if an error has occurred.
13673** False otherwise.
13674*/
13675static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){
13676 assert( pIter!=0 || p->rc!=SQLITE_OK )((void) (0));
13677 assert( p->rc!=SQLITE_OK((void) (0))
13678 || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof((void) (0))
13679 )((void) (0));
13680 return (p->rc || pIter->base.bEof);
13681}
13682
13683/*
13684** Return the rowid of the entry that the iterator currently points
13685** to. If the iterator points to EOF when this function is called the
13686** results are undefined.
13687*/
13688static i64 fts5MultiIterRowid(Fts5Iter *pIter){
13689 assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf )((void) (0));
13690 return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid;
13691}
13692
13693/*
13694** Move the iterator to the next entry at or following iMatch.
13695*/
13696static void fts5MultiIterNextFrom(
13697 Fts5Index *p,
13698 Fts5Iter *pIter,
13699 i64 iMatch
13700){
13701 while( 1 ){
13702 i64 iRowid;
13703 fts5MultiIterNext(p, pIter, 1, iMatch);
13704 if( fts5MultiIterEof(p, pIter) ) break;
13705 iRowid = fts5MultiIterRowid(pIter);
13706 if( pIter->bRev==0 && iRowid>=iMatch ) break;
13707 if( pIter->bRev!=0 && iRowid<=iMatch ) break;
13708 }
13709}
13710
13711/*
13712** Return a pointer to a buffer containing the term associated with the
13713** entry that the iterator currently points to.
13714*/
13715static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){
13716 Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
13717 *pn = p->term.n;
13718 return p->term.p;
13719}
13720
13721/*
13722** Allocate a new segment-id for the structure pStruct. The new segment
13723** id must be between 1 and 65335 inclusive, and must not be used by
13724** any currently existing segment. If a free segment id cannot be found,
13725** SQLITE_FULL is returned.
13726**
13727** If an error has already occurred, this function is a no-op. 0 is
13728** returned in this case.
13729*/
13730static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
13731 int iSegid = 0;
13732
13733 if( p->rc==SQLITE_OK0 ){
13734 if( pStruct->nSegment>=FTS5_MAX_SEGMENT2000 ){
13735 p->rc = SQLITE_FULL13;
13736 }else{
13737 /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following
13738 ** array is 63 elements, or 252 bytes, in size. */
13739 u32 aUsed[(FTS5_MAX_SEGMENT2000+31) / 32];
13740 int iLvl, iSeg;
13741 int i;
13742 u32 mask;
13743 memset(aUsed, 0, sizeof(aUsed));
13744 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
13745 for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
13746 int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid;
13747 if( iId<=FTS5_MAX_SEGMENT2000 && iId>0 ){
13748 aUsed[(iId-1) / 32] |= (u32)1 << ((iId-1) % 32);
13749 }
13750 }
13751 }
13752
13753 for(i=0; aUsed[i]==0xFFFFFFFF; i++);
13754 mask = aUsed[i];
13755 for(iSegid=0; mask & ((u32)1 << iSegid); iSegid++);
13756 iSegid += 1 + i*32;
13757
13758#ifdef SQLITE_DEBUG
13759 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
13760 for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
13761 assert_nc( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid )((void) (0));
13762 }
13763 }
13764 assert_nc( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT )((void) (0));
13765
13766 {
13767 sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p);
13768 if( p->rc==SQLITE_OK0 ){
13769 u8 aBlob[2] = {0xff, 0xff};
13770 sqlite3_bind_intsqlite3_api->bind_int(pIdxSelect, 1, iSegid);
13771 sqlite3_bind_blobsqlite3_api->bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC((sqlite3_destructor_type)0));
13772 assert_nc( sqlite3_step(pIdxSelect)!=SQLITE_ROW )((void) (0));
13773 p->rc = sqlite3_resetsqlite3_api->reset(pIdxSelect);
13774 sqlite3_bind_nullsqlite3_api->bind_null(pIdxSelect, 2);
13775 }
13776 }
13777#endif
13778 }
13779 }
13780
13781 return iSegid;
13782}
13783
13784/*
13785** Discard all data currently cached in the hash-tables.
13786*/
13787static void fts5IndexDiscardData(Fts5Index *p){
13788 assert( p->pHash || p->nPendingData==0 )((void) (0));
13789 if( p->pHash ){
13790 sqlite3Fts5HashClear(p->pHash);
13791 p->nPendingData = 0;
13792 p->nPendingRow = 0;
13793 p->flushRc = SQLITE_OK0;
13794 }
13795 p->nContentlessDelete = 0;
13796}
13797
13798/*
13799** Return the size of the prefix, in bytes, that buffer
13800** (pNew/<length-unknown>) shares with buffer (pOld/nOld).
13801**
13802** Buffer (pNew/<length-unknown>) is guaranteed to be greater
13803** than buffer (pOld/nOld).
13804*/
13805static int fts5PrefixCompress(int nOld, const u8 *pOld, const u8 *pNew){
13806 int i;
13807 for(i=0; i<nOld; i++){
13808 if( pOld[i]!=pNew[i] ) break;
13809 }
13810 return i;
13811}
13812
13813static void fts5WriteDlidxClear(
13814 Fts5Index *p,
13815 Fts5SegWriter *pWriter,
13816 int bFlush /* If true, write dlidx to disk */
13817){
13818 int i;
13819 assert( bFlush==0 || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n>0) )((void) (0));
13820 for(i=0; i<pWriter->nDlidx; i++){
13821 Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i];
13822 if( pDlidx->buf.n==0 ) break;
13823 if( bFlush ){
13824 assert( pDlidx->pgno!=0 )((void) (0));
13825 fts5DataWrite(p,
13826 FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno)( ((i64)(pWriter->iSegid) << (31 +5 +1)) + ((i64)(1)
<< (31 + 5)) + ((i64)(i) << (31)) + ((i64)(pDlidx
->pgno)) ),
13827 pDlidx->buf.p, pDlidx->buf.n
13828 );
13829 }
13830 sqlite3Fts5BufferZero(&pDlidx->buf);
13831 pDlidx->bPrevValid = 0;
13832 }
13833}
13834
13835/*
13836** Grow the pWriter->aDlidx[] array to at least nLvl elements in size.
13837** Any new array elements are zeroed before returning.
13838*/
13839static int fts5WriteDlidxGrow(
13840 Fts5Index *p,
13841 Fts5SegWriter *pWriter,
13842 int nLvl
13843){
13844 if( p->rc==SQLITE_OK0 && nLvl>=pWriter->nDlidx ){
13845 Fts5DlidxWriter *aDlidx = (Fts5DlidxWriter*)sqlite3_realloc64sqlite3_api->realloc64(
13846 pWriter->aDlidx, sizeof(Fts5DlidxWriter) * nLvl
13847 );
13848 if( aDlidx==0 ){
13849 p->rc = SQLITE_NOMEM7;
13850 }else{
13851 size_t nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx);
13852 memset(&aDlidx[pWriter->nDlidx], 0, nByte);
13853 pWriter->aDlidx = aDlidx;
13854 pWriter->nDlidx = nLvl;
13855 }
13856 }
13857 return p->rc;
13858}
13859
13860/*
13861** If the current doclist-index accumulating in pWriter->aDlidx[] is large
13862** enough, flush it to disk and return 1. Otherwise discard it and return
13863** zero.
13864*/
13865static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){
13866 int bFlag = 0;
13867
13868 /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written
13869 ** to the database, also write the doclist-index to disk. */
13870 if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE4 ){
13871 bFlag = 1;
13872 }
13873 fts5WriteDlidxClear(p, pWriter, bFlag);
13874 pWriter->nEmpty = 0;
13875 return bFlag;
13876}
13877
13878/*
13879** This function is called whenever processing of the doclist for the
13880** last term on leaf page (pWriter->iBtPage) is completed.
13881**
13882** The doclist-index for that term is currently stored in-memory within the
13883** Fts5SegWriter.aDlidx[] array. If it is large enough, this function
13884** writes it out to disk. Or, if it is too small to bother with, discards
13885** it.
13886**
13887** Fts5SegWriter.btterm currently contains the first term on page iBtPage.
13888*/
13889static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){
13890 int bFlag;
13891
13892 assert( pWriter->iBtPage || pWriter->nEmpty==0 )((void) (0));
13893 if( pWriter->iBtPage==0 ) return;
13894 bFlag = fts5WriteFlushDlidx(p, pWriter);
13895
13896 if( p->rc==SQLITE_OK0 ){
13897 const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:"");
13898 /* The following was already done in fts5WriteInit(): */
13899 /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */
13900 sqlite3_bind_blobsqlite3_api->bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC((sqlite3_destructor_type)0));
13901 sqlite3_bind_int64sqlite3_api->bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1));
13902 sqlite3_stepsqlite3_api->step(p->pIdxWriter);
13903 p->rc = sqlite3_resetsqlite3_api->reset(p->pIdxWriter);
13904 sqlite3_bind_nullsqlite3_api->bind_null(p->pIdxWriter, 2);
13905 }
13906 pWriter->iBtPage = 0;
13907}
13908
13909/*
13910** This is called once for each leaf page except the first that contains
13911** at least one term. Argument (nTerm/pTerm) is the split-key - a term that
13912** is larger than all terms written to earlier leaves, and equal to or
13913** smaller than the first term on the new leaf.
13914**
13915** If an error occurs, an error code is left in Fts5Index.rc. If an error
13916** has already occurred when this function is called, it is a no-op.
13917*/
13918static void fts5WriteBtreeTerm(
13919 Fts5Index *p, /* FTS5 backend object */
13920 Fts5SegWriter *pWriter, /* Writer object */
13921 int nTerm, const u8 *pTerm /* First term on new page */
13922){
13923 fts5WriteFlushBtree(p, pWriter);
13924 if( p->rc==SQLITE_OK0 ){
13925 fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm)sqlite3Fts5BufferSet(&p->rc,&pWriter->btterm,nTerm
,pTerm);
13926 pWriter->iBtPage = pWriter->writer.pgno;
13927 }
13928}
13929
13930/*
13931** This function is called when flushing a leaf page that contains no
13932** terms at all to disk.
13933*/
13934static void fts5WriteBtreeNoTerm(
13935 Fts5Index *p, /* FTS5 backend object */
13936 Fts5SegWriter *pWriter /* Writer object */
13937){
13938 /* If there were no rowids on the leaf page either and the doclist-index
13939 ** has already been started, append an 0x00 byte to it. */
13940 if( pWriter->bFirstRowidInPage && pWriter->aDlidx[0].buf.n>0 ){
13941 Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[0];
13942 assert( pDlidx->bPrevValid )((void) (0));
13943 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, 0);
13944 }
13945
13946 /* Increment the "number of sequential leaves without a term" counter. */
13947 pWriter->nEmpty++;
13948}
13949
13950static i64 fts5DlidxExtractFirstRowid(Fts5Buffer *pBuf){
13951 i64 iRowid;
13952 int iOff;
13953
13954 iOff = 1 + fts5GetVarintsqlite3Fts5GetVarint(&pBuf->p[1], (u64*)&iRowid);
13955 fts5GetVarintsqlite3Fts5GetVarint(&pBuf->p[iOff], (u64*)&iRowid);
13956 return iRowid;
13957}
13958
13959/*
13960** Rowid iRowid has just been appended to the current leaf page. It is the
13961** first on the page. This function appends an appropriate entry to the current
13962** doclist-index.
13963*/
13964static void fts5WriteDlidxAppend(
13965 Fts5Index *p,
13966 Fts5SegWriter *pWriter,
13967 i64 iRowid
13968){
13969 int i;
13970 int bDone = 0;
13971
13972 for(i=0; p->rc==SQLITE_OK0 && bDone==0; i++){
13973 i64 iVal;
13974 Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i];
13975
13976 if( pDlidx->buf.n>=p->pConfig->pgsz ){
13977 /* The current doclist-index page is full. Write it to disk and push
13978 ** a copy of iRowid (which will become the first rowid on the next
13979 ** doclist-index leaf page) up into the next level of the b-tree
13980 ** hierarchy. If the node being flushed is currently the root node,
13981 ** also push its first rowid upwards. */
13982 pDlidx->buf.p[0] = 0x01; /* Not the root node */
13983 fts5DataWrite(p,
13984 FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno)( ((i64)(pWriter->iSegid) << (31 +5 +1)) + ((i64)(1)
<< (31 + 5)) + ((i64)(i) << (31)) + ((i64)(pDlidx
->pgno)) ),
13985 pDlidx->buf.p, pDlidx->buf.n
13986 );
13987 fts5WriteDlidxGrow(p, pWriter, i+2);
13988 pDlidx = &pWriter->aDlidx[i];
13989 if( p->rc==SQLITE_OK0 && pDlidx[1].buf.n==0 ){
13990 i64 iFirst = fts5DlidxExtractFirstRowid(&pDlidx->buf);
13991
13992 /* This was the root node. Push its first rowid up to the new root. */
13993 pDlidx[1].pgno = pDlidx->pgno;
13994 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, 0);
13995 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, pDlidx->pgno);
13996 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, iFirst);
13997 pDlidx[1].bPrevValid = 1;
13998 pDlidx[1].iPrev = iFirst;
13999 }
14000
14001 sqlite3Fts5BufferZero(&pDlidx->buf);
14002 pDlidx->bPrevValid = 0;
14003 pDlidx->pgno++;
14004 }else{
14005 bDone = 1;
14006 }
14007
14008 if( pDlidx->bPrevValid ){
14009 iVal = (u64)iRowid - (u64)pDlidx->iPrev;
14010 }else{
14011 i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno);
14012 assert( pDlidx->buf.n==0 )((void) (0));
14013 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone);
14014 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno);
14015 iVal = iRowid;
14016 }
14017
14018 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iVal);
14019 pDlidx->bPrevValid = 1;
14020 pDlidx->iPrev = iRowid;
14021 }
14022}
14023
14024static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
14025 static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
14026 Fts5PageWriter *pPage = &pWriter->writer;
14027 i64 iRowid;
14028
14029 assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) )((void) (0));
14030
14031 /* Set the szLeaf header field. */
14032 assert( 0==fts5GetU16(&pPage->buf.p[2]) )((void) (0));
14033 fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n);
14034
14035 if( pWriter->bFirstTermInPage ){
14036 /* No term was written to this page. */
14037 assert( pPage->pgidx.n==0 )((void) (0));
14038 fts5WriteBtreeNoTerm(p, pWriter);
14039 }else{
14040 /* Append the pgidx to the page buffer. Set the szLeaf header field. */
14041 fts5BufferAppendBlob(&p->rc, &pPage->buf, pPage->pgidx.n, pPage->pgidx.p)sqlite3Fts5BufferAppendBlob(&p->rc,&pPage->buf,
pPage->pgidx.n,pPage->pgidx.p);
14042 }
14043
14044 /* Write the page out to disk */
14045 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, pPage->pgno)( ((i64)(pWriter->iSegid) << (31 +5 +1)) + ((i64)(0)
<< (31 + 5)) + ((i64)(0) << (31)) + ((i64)(pPage
->pgno)) );
14046 fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n);
14047
14048 /* Initialize the next page. */
14049 fts5BufferZero(&pPage->buf)sqlite3Fts5BufferZero(&pPage->buf);
14050 fts5BufferZero(&pPage->pgidx)sqlite3Fts5BufferZero(&pPage->pgidx);
14051 fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero)sqlite3Fts5BufferAppendBlob(&p->rc,&pPage->buf,
4,zero);
14052 pPage->iPrevPgidx = 0;
14053 pPage->pgno++;
14054
14055 /* Increase the leaves written counter */
14056 pWriter->nLeafWritten++;
14057
14058 /* The new leaf holds no terms or rowids */
14059 pWriter->bFirstTermInPage = 1;
14060 pWriter->bFirstRowidInPage = 1;
14061}
14062
14063/*
14064** Append term pTerm/nTerm to the segment being written by the writer passed
14065** as the second argument.
14066**
14067** If an error occurs, set the Fts5Index.rc error code. If an error has
14068** already occurred, this function is a no-op.
14069*/
14070static void fts5WriteAppendTerm(
14071 Fts5Index *p,
14072 Fts5SegWriter *pWriter,
14073 int nTerm, const u8 *pTerm
14074){
14075 int nPrefix; /* Bytes of prefix compression for term */
14076 Fts5PageWriter *pPage = &pWriter->writer;
14077 Fts5Buffer *pPgidx = &pWriter->writer.pgidx;
14078 int nMin = MIN(pPage->term.n, nTerm)(((pPage->term.n) < (nTerm)) ? (pPage->term.n) : (nTerm
));
14079
14080 assert( p->rc==SQLITE_OK )((void) (0));
14081 assert( pPage->buf.n>=4 )((void) (0));
14082 assert( pPage->buf.n>4 || pWriter->bFirstTermInPage )((void) (0));
14083
14084 /* If the current leaf page is full, flush it to disk. */
14085 if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){
14086 if( pPage->buf.n>4 ){
14087 fts5WriteFlushLeaf(p, pWriter);
14088 if( p->rc!=SQLITE_OK0 ) return;
14089 }
14090 fts5BufferGrow(&p->rc, &pPage->buf, nTerm+FTS5_DATA_PADDING)( (u32)((&pPage->buf)->n) + (u32)(nTerm+20) <= (
u32)((&pPage->buf)->nSpace) ? 0 : sqlite3Fts5BufferSize
((&p->rc),(&pPage->buf),(nTerm+20)+(&pPage->
buf)->n) );
14091 }
14092
14093 /* TODO1: Updating pgidx here. */
14094 pPgidx->n += sqlite3Fts5PutVarint(
14095 &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx
14096 );
14097 pPage->iPrevPgidx = pPage->buf.n;
14098#if 0
14099 fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n);
14100 pPgidx->n += 2;
14101#endif
14102
14103 if( pWriter->bFirstTermInPage ){
14104 nPrefix = 0;
14105 if( pPage->pgno!=1 ){
14106 /* This is the first term on a leaf that is not the leftmost leaf in
14107 ** the segment b-tree. In this case it is necessary to add a term to
14108 ** the b-tree hierarchy that is (a) larger than the largest term
14109 ** already written to the segment and (b) smaller than or equal to
14110 ** this term. In other words, a prefix of (pTerm/nTerm) that is one
14111 ** byte longer than the longest prefix (pTerm/nTerm) shares with the
14112 ** previous term.
14113 **
14114 ** Usually, the previous term is available in pPage->term. The exception
14115 ** is if this is the first term written in an incremental-merge step.
14116 ** In this case the previous term is not available, so just write a
14117 ** copy of (pTerm/nTerm) into the parent node. This is slightly
14118 ** inefficient, but still correct. */
14119 int n = nTerm;
14120 if( pPage->term.n ){
14121 n = 1 + fts5PrefixCompress(nMin, pPage->term.p, pTerm);
14122 }
14123 fts5WriteBtreeTerm(p, pWriter, n, pTerm);
14124 if( p->rc!=SQLITE_OK0 ) return;
14125 pPage = &pWriter->writer;
14126 }
14127 }else{
14128 nPrefix = fts5PrefixCompress(nMin, pPage->term.p, pTerm);
14129 fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix)sqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)nPrefix);
14130 }
14131
14132 /* Append the number of bytes of new data, then the term data itself
14133 ** to the page. */
14134 fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix)sqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)nTerm - nPrefix);
14135 fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix])sqlite3Fts5BufferAppendBlob(&p->rc,&pPage->buf,
nTerm - nPrefix,&pTerm[nPrefix]);
14136
14137 /* Update the Fts5PageWriter.term field. */
14138 fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm)sqlite3Fts5BufferSet(&p->rc,&pPage->term,nTerm,
pTerm);
14139 pWriter->bFirstTermInPage = 0;
14140
14141 pWriter->bFirstRowidInPage = 0;
14142 pWriter->bFirstRowidInDoclist = 1;
14143
14144 assert( p->rc || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n==0) )((void) (0));
14145 pWriter->aDlidx[0].pgno = pPage->pgno;
14146}
14147
14148/*
14149** Append a rowid and position-list size field to the writers output.
14150*/
14151static void fts5WriteAppendRowid(
14152 Fts5Index *p,
14153 Fts5SegWriter *pWriter,
14154 i64 iRowid
14155){
14156 if( p->rc==SQLITE_OK0 ){
14157 Fts5PageWriter *pPage = &pWriter->writer;
14158
14159 if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){
14160 fts5WriteFlushLeaf(p, pWriter);
14161 }
14162
14163 /* If this is to be the first rowid written to the page, set the
14164 ** rowid-pointer in the page-header. Also append a value to the dlidx
14165 ** buffer, in case a doclist-index is required. */
14166 if( pWriter->bFirstRowidInPage ){
14167 fts5PutU16(pPage->buf.p, (u16)pPage->buf.n);
14168 fts5WriteDlidxAppend(p, pWriter, iRowid);
14169 }
14170
14171 /* Write the rowid. */
14172 if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){
14173 fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid)sqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)iRowid);
14174 }else{
14175 assert_nc( p->rc || iRowid>pWriter->iPrevRowid )((void) (0));
14176 fts5BufferAppendVarint(&p->rc, &pPage->buf,sqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)(u64)iRowid - (u64)pWriter->iPrevRowid)
14177 (u64)iRowid - (u64)pWriter->iPrevRowidsqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)(u64)iRowid - (u64)pWriter->iPrevRowid)
14178 )sqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)(u64)iRowid - (u64)pWriter->iPrevRowid);
14179 }
14180 pWriter->iPrevRowid = iRowid;
14181 pWriter->bFirstRowidInDoclist = 0;
14182 pWriter->bFirstRowidInPage = 0;
14183 }
14184}
14185
14186static void fts5WriteAppendPoslistData(
14187 Fts5Index *p,
14188 Fts5SegWriter *pWriter,
14189 const u8 *aData,
14190 int nData
14191){
14192 Fts5PageWriter *pPage = &pWriter->writer;
14193 const u8 *a = aData;
14194 int n = nData;
14195
14196 assert( p->pConfig->pgsz>0 || p->rc!=SQLITE_OK )((void) (0));
14197 while( p->rc==SQLITE_OK0
14198 && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz
14199 ){
14200 int nReq = p->pConfig->pgsz - pPage->buf.n - pPage->pgidx.n;
14201 int nCopy = 0;
14202 while( nCopy<nReq ){
14203 i64 dummy;
14204 nCopy += fts5GetVarintsqlite3Fts5GetVarint(&a[nCopy], (u64*)&dummy);
14205 }
14206 fts5BufferAppendBlob(&p->rc, &pPage->buf, nCopy, a)sqlite3Fts5BufferAppendBlob(&p->rc,&pPage->buf,
nCopy,a);
14207 a += nCopy;
14208 n -= nCopy;
14209 fts5WriteFlushLeaf(p, pWriter);
14210 }
14211 if( n>0 ){
14212 fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a)sqlite3Fts5BufferAppendBlob(&p->rc,&pPage->buf,
n,a);
14213 }
14214}
14215
14216/*
14217** Flush any data cached by the writer object to the database. Free any
14218** allocations associated with the writer.
14219*/
14220static void fts5WriteFinish(
14221 Fts5Index *p,
14222 Fts5SegWriter *pWriter, /* Writer object */
14223 int *pnLeaf /* OUT: Number of leaf pages in b-tree */
14224){
14225 int i;
14226 Fts5PageWriter *pLeaf = &pWriter->writer;
14227 if( p->rc==SQLITE_OK0 ){
14228 assert( pLeaf->pgno>=1 )((void) (0));
14229 if( pLeaf->buf.n>4 ){
14230 fts5WriteFlushLeaf(p, pWriter);
14231 }
14232 *pnLeaf = pLeaf->pgno-1;
14233 if( pLeaf->pgno>1 ){
14234 fts5WriteFlushBtree(p, pWriter);
14235 }
14236 }
14237 fts5BufferFree(&pLeaf->term)sqlite3Fts5BufferFree(&pLeaf->term);
14238 fts5BufferFree(&pLeaf->buf)sqlite3Fts5BufferFree(&pLeaf->buf);
14239 fts5BufferFree(&pLeaf->pgidx)sqlite3Fts5BufferFree(&pLeaf->pgidx);
14240 fts5BufferFree(&pWriter->btterm)sqlite3Fts5BufferFree(&pWriter->btterm);
14241
14242 for(i=0; i<pWriter->nDlidx; i++){
14243 sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf);
14244 }
14245 sqlite3_freesqlite3_api->free(pWriter->aDlidx);
14246}
14247
14248static void fts5WriteInit(
14249 Fts5Index *p,
14250 Fts5SegWriter *pWriter,
14251 int iSegid
14252){
14253 const int nBuffer = p->pConfig->pgsz + FTS5_DATA_PADDING20;
14254
14255 memset(pWriter, 0, sizeof(Fts5SegWriter));
14256 pWriter->iSegid = iSegid;
14257
14258 fts5WriteDlidxGrow(p, pWriter, 1);
14259 pWriter->writer.pgno = 1;
14260 pWriter->bFirstTermInPage = 1;
14261 pWriter->iBtPage = 1;
14262
14263 assert( pWriter->writer.buf.n==0 )((void) (0));
14264 assert( pWriter->writer.pgidx.n==0 )((void) (0));
14265
14266 /* Grow the two buffers to pgsz + padding bytes in size. */
14267 sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.pgidx, nBuffer);
14268 sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.buf, nBuffer);
14269
14270 if( p->pIdxWriter==0 ){
14271 Fts5Config *pConfig = p->pConfig;
14272 fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintfsqlite3_api->mprintf(
14273 "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)",
14274 pConfig->zDb, pConfig->zName
14275 ));
14276 }
14277
14278 if( p->rc==SQLITE_OK0 ){
14279 /* Initialize the 4-byte leaf-page header to 0x00. */
14280 memset(pWriter->writer.buf.p, 0, 4);
14281 pWriter->writer.buf.n = 4;
14282
14283 /* Bind the current output segment id to the index-writer. This is an
14284 ** optimization over binding the same value over and over as rows are
14285 ** inserted into %_idx by the current writer. */
14286 sqlite3_bind_intsqlite3_api->bind_int(p->pIdxWriter, 1, pWriter->iSegid);
14287 }
14288}
14289
14290/*
14291** Iterator pIter was used to iterate through the input segments of on an
14292** incremental merge operation. This function is called if the incremental
14293** merge step has finished but the input has not been completely exhausted.
14294*/
14295static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){
14296 int i;
14297 Fts5Buffer buf;
14298 memset(&buf, 0, sizeof(Fts5Buffer));
14299 for(i=0; i<pIter->nSeg && p->rc==SQLITE_OK0; i++){
14300 Fts5SegIter *pSeg = &pIter->aSeg[i];
14301 if( pSeg->pSeg==0 ){
14302 /* no-op */
14303 }else if( pSeg->pLeaf==0 ){
14304 /* All keys from this input segment have been transfered to the output.
14305 ** Set both the first and last page-numbers to 0 to indicate that the
14306 ** segment is now empty. */
14307 pSeg->pSeg->pgnoLast = 0;
14308 pSeg->pSeg->pgnoFirst = 0;
14309 }else{
14310 int iOff = pSeg->iTermLeafOffset; /* Offset on new first leaf page */
14311 i64 iLeafRowid;
14312 Fts5Data *pData;
14313 int iId = pSeg->pSeg->iSegid;
14314 u8 aHdr[4] = {0x00, 0x00, 0x00, 0x00};
14315
14316 iLeafRowid = FTS5_SEGMENT_ROWID(iId, pSeg->iTermLeafPgno)( ((i64)(iId) << (31 +5 +1)) + ((i64)(0) << (31 +
5)) + ((i64)(0) << (31)) + ((i64)(pSeg->iTermLeafPgno
)) );
14317 pData = fts5LeafRead(p, iLeafRowid);
14318 if( pData ){
14319 if( iOff>pData->szLeaf ){
14320 /* This can occur if the pages that the segments occupy overlap - if
14321 ** a single page has been assigned to more than one segment. In
14322 ** this case a prior iteration of this loop may have corrupted the
14323 ** segment currently being trimmed. */
14324 FTS5_CORRUPT_ROWID(p, iLeafRowid)fts5IndexCorruptRowid(p, iLeafRowid);
14325 }else{
14326 fts5BufferZero(&buf)sqlite3Fts5BufferZero(&buf);
14327 fts5BufferGrow(&p->rc, &buf, pData->nn)( (u32)((&buf)->n) + (u32)(pData->nn) <= (u32)((
&buf)->nSpace) ? 0 : sqlite3Fts5BufferSize((&p->
rc),(&buf),(pData->nn)+(&buf)->n) );
14328 fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr)sqlite3Fts5BufferAppendBlob(&p->rc,&buf,sizeof(aHdr
),aHdr);
14329 fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->term.n);
14330 fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p)sqlite3Fts5BufferAppendBlob(&p->rc,&buf,pSeg->term
.n,pSeg->term.p);
14331 fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff])sqlite3Fts5BufferAppendBlob(&p->rc,&buf,pData->
szLeaf-iOff,&pData->p[iOff]);
14332 if( p->rc==SQLITE_OK0 ){
14333 /* Set the szLeaf field */
14334 fts5PutU16(&buf.p[2], (u16)buf.n);
14335 }
14336
14337 /* Set up the new page-index array */
14338 fts5BufferAppendVarint(&p->rc, &buf, 4)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)4);
14339 if( pSeg->iLeafPgno==pSeg->iTermLeafPgno
14340 && pSeg->iEndofDoclist<pData->szLeaf
14341 && pSeg->iPgidxOff<=pData->nn
14342 ){
14343 int nDiff = pData->szLeaf - pSeg->iEndofDoclist;
14344 fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)buf
.n - 1 - nDiff - 4);
14345 fts5BufferAppendBlob(&p->rc, &buf,sqlite3Fts5BufferAppendBlob(&p->rc,&buf,pData->
nn - pSeg->iPgidxOff,&pData->p[pSeg->iPgidxOff])
14346 pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff]sqlite3Fts5BufferAppendBlob(&p->rc,&buf,pData->
nn - pSeg->iPgidxOff,&pData->p[pSeg->iPgidxOff])
14347 )sqlite3Fts5BufferAppendBlob(&p->rc,&buf,pData->
nn - pSeg->iPgidxOff,&pData->p[pSeg->iPgidxOff]);
14348 }
14349
14350 pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno;
14351 fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 1)( ((i64)(iId) << (31 +5 +1)) + ((i64)(0) << (31 +
5)) + ((i64)(0) << (31)) + ((i64)(1)) ), iLeafRowid);
14352 fts5DataWrite(p, iLeafRowid, buf.p, buf.n);
14353 }
14354 fts5DataRelease(pData);
14355 }
14356 }
14357 }
14358 fts5BufferFree(&buf)sqlite3Fts5BufferFree(&buf);
14359}
14360
14361static void fts5MergeChunkCallback(
14362 Fts5Index *p,
14363 void *pCtx,
14364 const u8 *pChunk, int nChunk
14365){
14366 Fts5SegWriter *pWriter = (Fts5SegWriter*)pCtx;
14367 fts5WriteAppendPoslistData(p, pWriter, pChunk, nChunk);
14368}
14369
14370/*
14371**
14372*/
14373static void fts5IndexMergeLevel(
14374 Fts5Index *p, /* FTS5 backend object */
14375 Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */
14376 int iLvl, /* Level to read input from */
14377 int *pnRem /* Write up to this many output leaves */
14378){
14379 Fts5Structure *pStruct = *ppStruct;
14380 Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
14381 Fts5StructureLevel *pLvlOut;
14382 Fts5Iter *pIter = 0; /* Iterator to read input data */
14383 int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */
14384 int nInput; /* Number of input segments */
14385 Fts5SegWriter writer; /* Writer object */
14386 Fts5StructureSegment *pSeg; /* Output segment */
14387 Fts5Buffer term;
14388 int bOldest; /* True if the output segment is the oldest */
14389 int eDetail = p->pConfig->eDetail;
14390 const int flags = FTS5INDEX_QUERY_NOOUTPUT0x0020;
14391 int bTermWritten = 0; /* True if current term already output */
14392
14393 assert( iLvl<pStruct->nLevel )((void) (0));
14394 assert( pLvl->nMerge<=pLvl->nSeg )((void) (0));
14395
14396 memset(&writer, 0, sizeof(Fts5SegWriter));
14397 memset(&term, 0, sizeof(Fts5Buffer));
14398 if( pLvl->nMerge ){
14399 pLvlOut = &pStruct->aLevel[iLvl+1];
14400 assert( pLvlOut->nSeg>0 )((void) (0));
14401 nInput = pLvl->nMerge;
14402 pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1];
14403
14404 fts5WriteInit(p, &writer, pSeg->iSegid);
14405 writer.writer.pgno = pSeg->pgnoLast+1;
14406 writer.iBtPage = 0;
14407 }else{
14408 int iSegid = fts5AllocateSegid(p, pStruct);
14409
14410 /* Extend the Fts5Structure object as required to ensure the output
14411 ** segment exists. */
14412 if( iLvl==pStruct->nLevel-1 ){
14413 fts5StructureAddLevel(&p->rc, ppStruct);
14414 pStruct = *ppStruct;
14415 }
14416 fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0);
14417 if( p->rc ) return;
14418 pLvl = &pStruct->aLevel[iLvl];
14419 pLvlOut = &pStruct->aLevel[iLvl+1];
14420
14421 fts5WriteInit(p, &writer, iSegid);
14422
14423 /* Add the new segment to the output level */
14424 pSeg = &pLvlOut->aSeg[pLvlOut->nSeg];
14425 pLvlOut->nSeg++;
14426 pSeg->pgnoFirst = 1;
14427 pSeg->iSegid = iSegid;
14428 pStruct->nSegment++;
14429
14430 /* Read input from all segments in the input level */
14431 nInput = pLvl->nSeg;
14432
14433 /* Set the range of origins that will go into the output segment. */
14434 if( pStruct->nOriginCntr>0 ){
14435 pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1;
14436 pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2;
14437 }
14438 }
14439 bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2);
14440
14441 assert( iLvl>=0 )((void) (0));
14442 for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter);
14443 fts5MultiIterEof(p, pIter)==0;
14444 fts5MultiIterNext(p, pIter, 0, 0)
14445 ){
14446 Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
14447 int nPos; /* position-list size field value */
14448 int nTerm;
14449 const u8 *pTerm;
14450
14451 pTerm = fts5MultiIterTerm(pIter, &nTerm);
14452 if( nTerm!=term.n || fts5Memcmp(pTerm, term.p, nTerm)((nTerm)<=0 ? 0 : memcmp((pTerm), (term.p), (nTerm))) ){
14453 if( pnRem && writer.nLeafWritten>nRem ){
14454 break;
14455 }
14456 fts5BufferSet(&p->rc, &term, nTerm, pTerm)sqlite3Fts5BufferSet(&p->rc,&term,nTerm,pTerm);
14457 bTermWritten =0;
14458 }
14459
14460 /* Check for key annihilation. */
14461 if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue;
14462
14463 if( p->rc==SQLITE_OK0 && bTermWritten==0 ){
14464 /* This is a new term. Append a term to the output segment. */
14465 fts5WriteAppendTerm(p, &writer, nTerm, pTerm);
14466 bTermWritten = 1;
14467 }
14468
14469 /* Append the rowid to the output */
14470 /* WRITEPOSLISTSIZE */
14471 fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter));
14472
14473 if( eDetail==FTS5_DETAIL_NONE1 ){
14474 if( pSegIter->bDel ){
14475 fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0)sqlite3Fts5BufferAppendVarint(&p->rc,&writer.writer
.buf,(i64)0);
14476 if( pSegIter->nPos>0 ){
14477 fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0)sqlite3Fts5BufferAppendVarint(&p->rc,&writer.writer
.buf,(i64)0);
14478 }
14479 }
14480 }else{
14481 /* Append the position-list data to the output */
14482 nPos = pSegIter->nPos*2 + pSegIter->bDel;
14483 fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos)sqlite3Fts5BufferAppendVarint(&p->rc,&writer.writer
.buf,(i64)nPos);
14484 fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback);
14485 }
14486 }
14487
14488 /* Flush the last leaf page to disk. Set the output segment b-tree height
14489 ** and last leaf page number at the same time. */
14490 fts5WriteFinish(p, &writer, &pSeg->pgnoLast);
14491
14492 assert( pIter!=0 || p->rc!=SQLITE_OK )((void) (0));
14493 if( fts5MultiIterEof(p, pIter) ){
14494 int i;
14495
14496 /* Remove the redundant segments from the %_data table */
14497 assert( pSeg->nEntry==0 )((void) (0));
14498 for(i=0; i<nInput; i++){
14499 Fts5StructureSegment *pOld = &pLvl->aSeg[i];
14500 pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone);
14501 fts5DataRemoveSegment(p, pOld);
14502 }
14503
14504 /* Remove the redundant segments from the input level */
14505 if( pLvl->nSeg!=nInput ){
14506 int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment);
14507 memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove);
14508 }
14509 pStruct->nSegment -= nInput;
14510 pLvl->nSeg -= nInput;
14511 pLvl->nMerge = 0;
14512 if( pSeg->pgnoLast==0 ){
14513 pLvlOut->nSeg--;
14514 pStruct->nSegment--;
14515 }
14516 }else{
14517 assert( pSeg->pgnoLast>0 )((void) (0));
14518 fts5TrimSegments(p, pIter);
14519 pLvl->nMerge = nInput;
14520 }
14521
14522 fts5MultiIterFree(pIter);
14523 fts5BufferFree(&term)sqlite3Fts5BufferFree(&term);
14524 if( pnRem ) *pnRem -= writer.nLeafWritten;
14525}
14526
14527/*
14528** If this is not a contentless_delete=1 table, or if the 'deletemerge'
14529** configuration option is set to 0, then this function always returns -1.
14530** Otherwise, it searches the structure object passed as the second argument
14531** for a level suitable for merging due to having a large number of
14532** tombstones in the tombstone hash. If one is found, its index is returned.
14533** Otherwise, if there is no suitable level, -1.
14534*/
14535static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){
14536 Fts5Config *pConfig = p->pConfig;
14537 int iRet = -1;
14538 if( pConfig->bContentlessDelete && pConfig->nDeleteMerge>0 ){
14539 int ii;
14540 int nBest = 0;
14541
14542 for(ii=0; ii<pStruct->nLevel; ii++){
14543 Fts5StructureLevel *pLvl = &pStruct->aLevel[ii];
14544 i64 nEntry = 0;
14545 i64 nTomb = 0;
14546 int iSeg;
14547 for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
14548 nEntry += pLvl->aSeg[iSeg].nEntry;
14549 nTomb += pLvl->aSeg[iSeg].nEntryTombstone;
14550 }
14551 assert_nc( nEntry>0 || pLvl->nSeg==0 )((void) (0));
14552 if( nEntry>0 ){
14553 int nPercent = (nTomb * 100) / nEntry;
14554 if( nPercent>=pConfig->nDeleteMerge && nPercent>nBest ){
14555 iRet = ii;
14556 nBest = nPercent;
14557 }
14558 }
14559
14560 /* If pLvl is already the input level to an ongoing merge, look no
14561 ** further for a merge candidate. The caller should be allowed to
14562 ** continue merging from pLvl first. */
14563 if( pLvl->nMerge ) break;
14564 }
14565 }
14566 return iRet;
14567}
14568
14569/*
14570** Do up to nPg pages of automerge work on the index.
14571**
14572** Return true if any changes were actually made, or false otherwise.
14573*/
14574static int fts5IndexMerge(
14575 Fts5Index *p, /* FTS5 backend object */
14576 Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */
14577 int nPg, /* Pages of work to do */
14578 int nMin /* Minimum number of segments to merge */
14579){
14580 int nRem = nPg;
14581 int bRet = 0;
14582 Fts5Structure *pStruct = *ppStruct;
14583 while( nRem>0 && p->rc==SQLITE_OK0 ){
14584 int iLvl; /* To iterate through levels */
14585 int iBestLvl = 0; /* Level offering the most input segments */
14586 int nBest = 0; /* Number of input segments on best level */
14587
14588 /* Set iBestLvl to the level to read input segments from. Or to -1 if
14589 ** there is no level suitable to merge segments from. */
14590 assert( pStruct->nLevel>0 )((void) (0));
14591 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
14592 Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
14593 if( pLvl->nMerge ){
14594 if( pLvl->nMerge>nBest ){
14595 iBestLvl = iLvl;
14596 nBest = nMin;
14597 }
14598 break;
14599 }
14600 if( pLvl->nSeg>nBest ){
14601 nBest = pLvl->nSeg;
14602 iBestLvl = iLvl;
14603 }
14604 }
14605 if( nBest<nMin ){
14606 iBestLvl = fts5IndexFindDeleteMerge(p, pStruct);
14607 }
14608
14609 if( iBestLvl<0 ) break;
14610 bRet = 1;
14611 fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
14612 if( p->rc==SQLITE_OK0 && pStruct->aLevel[iBestLvl].nMerge==0 ){
14613 fts5StructurePromote(p, iBestLvl+1, pStruct);
14614 }
14615
14616 if( nMin==1 ) nMin = 2;
14617 }
14618 *ppStruct = pStruct;
14619 return bRet;
14620}
14621
14622/*
14623** A total of nLeaf leaf pages of data has just been flushed to a level-0
14624** segment. This function updates the write-counter accordingly and, if
14625** necessary, performs incremental merge work.
14626**
14627** If an error occurs, set the Fts5Index.rc error code. If an error has
14628** already occurred, this function is a no-op.
14629*/
14630static void fts5IndexAutomerge(
14631 Fts5Index *p, /* FTS5 backend object */
14632 Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */
14633 int nLeaf /* Number of output leaves just written */
14634){
14635 if( p->rc==SQLITE_OK0 && p->pConfig->nAutomerge>0 && ALWAYS((*ppStruct)!=0)((*ppStruct)!=0) ){
14636 Fts5Structure *pStruct = *ppStruct;
14637 u64 nWrite; /* Initial value of write-counter */
14638 int nWork; /* Number of work-quanta to perform */
14639 int nRem; /* Number of leaf pages left to write */
14640
14641 /* Update the write-counter. While doing so, set nWork. */
14642 nWrite = pStruct->nWriteCounter;
14643 nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit));
14644 pStruct->nWriteCounter += nLeaf;
14645 nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel);
14646
14647 fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge);
14648 }
14649}
14650
14651static void fts5IndexCrisismerge(
14652 Fts5Index *p, /* FTS5 backend object */
14653 Fts5Structure **ppStruct /* IN/OUT: Current structure of index */
14654){
14655 const int nCrisis = p->pConfig->nCrisisMerge;
14656 Fts5Structure *pStruct = *ppStruct;
14657 if( pStruct && pStruct->nLevel>0 ){
14658 int iLvl = 0;
14659 while( p->rc==SQLITE_OK0 && pStruct->aLevel[iLvl].nSeg>=nCrisis ){
14660 fts5IndexMergeLevel(p, &pStruct, iLvl, 0);
14661 assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) )((void) (0));
14662 fts5StructurePromote(p, iLvl+1, pStruct);
14663 iLvl++;
14664 }
14665 *ppStruct = pStruct;
14666 }
14667}
14668
14669static int fts5IndexReturn(Fts5Index *p){
14670 int rc = p->rc;
14671 p->rc = SQLITE_OK0;
14672 return rc;
14673}
14674
14675/*
14676** Close the read-only blob handle, if it is open.
14677*/
14678static void sqlite3Fts5IndexCloseReader(Fts5Index *p){
14679 fts5IndexCloseReader(p);
14680 fts5IndexReturn(p);
14681}
14682
14683typedef struct Fts5FlushCtx Fts5FlushCtx;
14684struct Fts5FlushCtx {
14685 Fts5Index *pIdx;
14686 Fts5SegWriter writer;
14687};
14688
14689/*
14690** Buffer aBuf[] contains a list of varints, all small enough to fit
14691** in a 32-bit integer. Return the size of the largest prefix of this
14692** list nMax bytes or less in size.
14693*/
14694static int fts5PoslistPrefix(const u8 *aBuf, int nMax){
14695 int ret;
14696 u32 dummy;
14697 ret = fts5GetVarint32(aBuf, dummy)sqlite3Fts5GetVarint32(aBuf,(u32*)&(dummy));
14698 if( ret<nMax ){
14699 while( 1 ){
14700 int i = fts5GetVarint32(&aBuf[ret], dummy)sqlite3Fts5GetVarint32(&aBuf[ret],(u32*)&(dummy));
14701 if( (ret + i) > nMax ) break;
14702 ret += i;
14703 }
14704 }
14705 return ret;
14706}
14707
14708/*
14709** Execute the SQL statement:
14710**
14711** DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno);
14712**
14713** This is used when a secure-delete operation removes the last term
14714** from a segment leaf page. In that case the %_idx entry is removed
14715** too. This is done to ensure that if all instances of a token are
14716** removed from an fts5 database in secure-delete mode, no trace of
14717** the token itself remains in the database.
14718*/
14719static void fts5SecureDeleteIdxEntry(
14720 Fts5Index *p, /* FTS5 backend object */
14721 int iSegid, /* Id of segment to delete entry for */
14722 int iPgno /* Page number within segment */
14723){
14724 if( iPgno!=1 ){
14725 assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE )((void) (0));
14726 if( p->pDeleteFromIdx==0 ){
14727 fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintfsqlite3_api->mprintf(
14728 "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)",
14729 p->pConfig->zDb, p->pConfig->zName
14730 ));
14731 }
14732 if( p->rc==SQLITE_OK0 ){
14733 sqlite3_bind_intsqlite3_api->bind_int(p->pDeleteFromIdx, 1, iSegid);
14734 sqlite3_bind_intsqlite3_api->bind_int(p->pDeleteFromIdx, 2, iPgno);
14735 sqlite3_stepsqlite3_api->step(p->pDeleteFromIdx);
14736 p->rc = sqlite3_resetsqlite3_api->reset(p->pDeleteFromIdx);
14737 }
14738 }
14739}
14740
14741/*
14742** This is called when a secure-delete operation removes a position-list
14743** that overflows onto segment page iPgno of segment pSeg. This function
14744** rewrites node iPgno, and possibly one or more of its right-hand peers,
14745** to remove this portion of the position list.
14746**
14747** Output variable (*pbLastInDoclist) is set to true if the position-list
14748** removed is followed by a new term or the end-of-segment, or false if
14749** it is followed by another rowid/position list.
14750*/
14751static void fts5SecureDeleteOverflow(
14752 Fts5Index *p,
14753 Fts5StructureSegment *pSeg,
14754 int iPgno,
14755 int *pbLastInDoclist
14756){
14757 const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE1);
14758 int pgno;
14759 Fts5Data *pLeaf = 0;
14760 assert( iPgno!=1 )((void) (0));
14761
14762 *pbLastInDoclist = 1;
14763 for(pgno=iPgno; p->rc==SQLITE_OK0 && pgno<=pSeg->pgnoLast; pgno++){
14764 i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno)( ((i64)(pSeg->iSegid) << (31 +5 +1)) + ((i64)(0) <<
(31 + 5)) + ((i64)(0) << (31)) + ((i64)(pgno)) );
14765 int iNext = 0;
14766 u8 *aPg = 0;
14767
14768 pLeaf = fts5DataRead(p, iRowid);
14769 if( pLeaf==0 ) break;
14770 aPg = pLeaf->p;
14771
14772 iNext = fts5GetU16(&aPg[0]);
14773 if( iNext!=0 ){
14774 *pbLastInDoclist = 0;
14775 }
14776 if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){
14777 fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext)sqlite3Fts5GetVarint32(&aPg[pLeaf->szLeaf],(u32*)&
(iNext));
14778 }
14779
14780 if( iNext==0 ){
14781 /* The page contains no terms or rowids. Replace it with an empty
14782 ** page and move on to the right-hand peer. */
14783 const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04};
14784 assert_nc( bDetailNone==0 || pLeaf->nn==4 )((void) (0));
14785 if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
14786 fts5DataRelease(pLeaf);
14787 pLeaf = 0;
14788 }else if( bDetailNone ){
14789 break;
14790 }else if( iNext>=pLeaf->szLeaf || pLeaf->nn<pLeaf->szLeaf || iNext<4 ){
14791 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
14792 break;
14793 }else{
14794 int nShift = iNext - 4;
14795 int nPg;
14796
14797 int nIdx = 0;
14798 u8 *aIdx = 0;
14799
14800 /* Unless the current page footer is 0 bytes in size (in which case
14801 ** the new page footer will be as well), allocate and populate a
14802 ** buffer containing the new page footer. Set stack variables aIdx
14803 ** and nIdx accordingly. */
14804 if( pLeaf->nn>pLeaf->szLeaf ){
14805 int iFirst = 0;
14806 int i1 = pLeaf->szLeaf;
14807 int i2 = 0;
14808
14809 i1 += fts5GetVarint32(&aPg[i1], iFirst)sqlite3Fts5GetVarint32(&aPg[i1],(u32*)&(iFirst));
14810 if( iFirst<iNext ){
14811 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
14812 break;
14813 }
14814 aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2);
14815 if( aIdx==0 ) break;
14816 i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift);
14817 if( i1<pLeaf->nn ){
14818 memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1);
14819 i2 += (pLeaf->nn-i1);
14820 }
14821 nIdx = i2;
14822 }
14823
14824 /* Modify the contents of buffer aPg[]. Set nPg to the new size
14825 ** in bytes. The new page is always smaller than the old. */
14826 nPg = pLeaf->szLeaf - nShift;
14827 memmove(&aPg[4], &aPg[4+nShift], nPg-4);
14828 fts5PutU16(&aPg[2], nPg);
14829 if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4);
14830 if( nIdx>0 ){
14831 memcpy(&aPg[nPg], aIdx, nIdx);
14832 nPg += nIdx;
14833 }
14834 sqlite3_freesqlite3_api->free(aIdx);
14835
14836 /* Write the new page to disk and exit the loop */
14837 assert( nPg>4 || fts5GetU16(aPg)==0 )((void) (0));
14838 fts5DataWrite(p, iRowid, aPg, nPg);
14839 break;
14840 }
14841 }
14842 fts5DataRelease(pLeaf);
14843}
14844
14845/*
14846** Completely remove the entry that pSeg currently points to from
14847** the database.
14848*/
14849static void fts5DoSecureDelete(
14850 Fts5Index *p,
14851 Fts5SegIter *pSeg
14852){
14853 const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE1);
14854 int iSegid = pSeg->pSeg->iSegid;
14855 u8 *aPg = pSeg->pLeaf->p;
14856 int nPg = pSeg->pLeaf->nn;
14857 int iPgIdx = pSeg->pLeaf->szLeaf; /* Offset of page footer */
14858
14859 u64 iDelta = 0;
14860 int iNextOff = 0;
14861 int iOff = 0;
14862 int nIdx = 0;
14863 u8 *aIdx = 0;
14864 int bLastInDoclist = 0;
14865 int iIdx = 0;
14866 int iStart = 0;
14867 int iDelKeyOff = 0; /* Offset of deleted key, if any */
14868
14869 nIdx = nPg-iPgIdx;
14870 aIdx = sqlite3Fts5MallocZero(&p->rc, ((i64)nIdx)+16);
14871 if( p->rc ) return;
14872 memcpy(aIdx, &aPg[iPgIdx], nIdx);
14873
14874 /* At this point segment iterator pSeg points to the entry
14875 ** this function should remove from the b-tree segment.
14876 **
14877 ** In detail=full or detail=column mode, pSeg->iLeafOffset is the
14878 ** offset of the first byte in the position-list for the entry to
14879 ** remove. Immediately before this comes two varints that will also
14880 ** need to be removed:
14881 **
14882 ** + the rowid or delta rowid value for the entry, and
14883 ** + the size of the position list in bytes.
14884 **
14885 ** Or, in detail=none mode, there is a single varint prior to
14886 ** pSeg->iLeafOffset - the rowid or delta rowid value.
14887 **
14888 ** This block sets the following variables:
14889 **
14890 ** iStart:
14891 ** The offset of the first byte of the rowid or delta-rowid
14892 ** value for the doclist entry being removed.
14893 **
14894 ** iDelta:
14895 ** The value of the rowid or delta-rowid value for the doclist
14896 ** entry being removed.
14897 **
14898 ** iNextOff:
14899 ** The offset of the next entry following the position list
14900 ** for the one being removed. If the position list for this
14901 ** entry overflows onto the next leaf page, this value will be
14902 ** greater than pLeaf->szLeaf.
14903 */
14904 {
14905 int iSOP; /* Start-Of-Position-list */
14906 if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
14907 iStart = pSeg->iTermLeafOffset;
14908 }else{
14909 iStart = fts5GetU16(&aPg[0]);
14910 }
14911 if( iStart>nPg ){
14912 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
14913 sqlite3_freesqlite3_api->free(aIdx);
14914 return;
14915 }
14916
14917 iSOP = iStart + fts5GetVarintsqlite3Fts5GetVarint(&aPg[iStart], &iDelta);
14918 assert_nc( iSOP<=pSeg->iLeafOffset )((void) (0));
14919
14920 if( bDetailNone ){
14921 while( iSOP<pSeg->iLeafOffset ){
14922 if( aPg[iSOP]==0x00 ) iSOP++;
14923 if( aPg[iSOP]==0x00 ) iSOP++;
14924 iStart = iSOP;
14925 iSOP = iStart + fts5GetVarintsqlite3Fts5GetVarint(&aPg[iStart], &iDelta);
14926 }
14927
14928 iNextOff = iSOP;
14929 if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
14930 if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
14931
14932 }else{
14933 int nPos = 0;
14934 iSOP += fts5GetVarint32(&aPg[iSOP], nPos)sqlite3Fts5GetVarint32(&aPg[iSOP],(u32*)&(nPos));
14935 while( iSOP<pSeg->iLeafOffset ){
14936 iStart = iSOP + (nPos/2);
14937 iSOP = iStart + fts5GetVarintsqlite3Fts5GetVarint(&aPg[iStart], &iDelta);
14938 iSOP += fts5GetVarint32(&aPg[iSOP], nPos)sqlite3Fts5GetVarint32(&aPg[iSOP],(u32*)&(nPos));
14939 }
14940 assert_nc( iSOP==pSeg->iLeafOffset )((void) (0));
14941 iNextOff = iSOP + pSeg->nPos;
14942 }
14943 }
14944
14945 iOff = iStart;
14946
14947 /* If the position-list for the entry being removed flows over past
14948 ** the end of this page, delete the portion of the position-list on the
14949 ** next page and beyond.
14950 **
14951 ** Set variable bLastInDoclist to true if this entry happens
14952 ** to be the last rowid in the doclist for its term. */
14953 if( iNextOff>=iPgIdx ){
14954 int pgno = pSeg->iLeafPgno+1;
14955 fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
14956 iNextOff = iPgIdx;
14957 }
14958
14959 if( pSeg->bDel==0 ){
14960 if( iNextOff!=iPgIdx ){
14961 /* Loop through the page-footer. If iNextOff (offset of the
14962 ** entry following the one we are removing) is equal to the
14963 ** offset of a key on this page, then the entry is the last
14964 ** in its doclist. */
14965 int iKeyOff = 0;
14966 for(iIdx=0; iIdx<nIdx; /* no-op */){
14967 u32 iVal = 0;
14968 iIdx += fts5GetVarint32(&aIdx[iIdx], iVal)sqlite3Fts5GetVarint32(&aIdx[iIdx],(u32*)&(iVal));
14969 iKeyOff += iVal;
14970 if( iKeyOff==iNextOff ){
14971 bLastInDoclist = 1;
14972 }
14973 }
14974 }
14975
14976 /* If this is (a) the first rowid on a page and (b) is not followed by
14977 ** another position list on the same page, set the "first-rowid" field
14978 ** of the header to 0. */
14979 if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist || iNextOff==iPgIdx) ){
14980 fts5PutU16(&aPg[0], 0);
14981 }
14982 }
14983
14984 if( pSeg->bDel ){
14985 iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta);
14986 aPg[iOff++] = 0x01;
14987 }else if( bLastInDoclist==0 ){
14988 if( iNextOff!=iPgIdx ){
14989 u64 iNextDelta = 0;
14990 iNextOff += fts5GetVarintsqlite3Fts5GetVarint(&aPg[iNextOff], &iNextDelta);
14991 iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
14992 }
14993 }else if(
14994 pSeg->iLeafPgno==pSeg->iTermLeafPgno
14995 && iStart==pSeg->iTermLeafOffset
14996 ){
14997 /* The entry being removed was the only position list in its
14998 ** doclist. Therefore the term needs to be removed as well. */
14999 int iKey = 0;
15000 int iKeyOff = 0;
15001
15002 /* Set iKeyOff to the offset of the term that will be removed - the
15003 ** last offset in the footer that is not greater than iStart. */
15004 for(iIdx=0; iIdx<nIdx; iKey++){
15005 u32 iVal = 0;
15006 iIdx += fts5GetVarint32(&aIdx[iIdx], iVal)sqlite3Fts5GetVarint32(&aIdx[iIdx],(u32*)&(iVal));
15007 if( (iKeyOff+iVal)>(u32)iStart ) break;
15008 iKeyOff += iVal;
15009 }
15010 assert_nc( iKey>=1 )((void) (0));
15011
15012 /* Set iDelKeyOff to the value of the footer entry to remove from
15013 ** the page. */
15014 iDelKeyOff = iOff = iKeyOff;
15015
15016 if( iNextOff!=iPgIdx ){
15017 /* This is the only position-list associated with the term, and there
15018 ** is another term following it on this page. So the subsequent term
15019 ** needs to be moved to replace the term associated with the entry
15020 ** being removed. */
15021 u64 nPrefix = 0;
15022 u64 nSuffix = 0;
15023 u64 nPrefix2 = 0;
15024 u64 nSuffix2 = 0;
15025
15026 iDelKeyOff = iNextOff;
15027 iNextOff += fts5GetVarintsqlite3Fts5GetVarint(&aPg[iNextOff], &nPrefix2);
15028 iNextOff += fts5GetVarintsqlite3Fts5GetVarint(&aPg[iNextOff], &nSuffix2);
15029
15030 if( iKey!=1 ){
15031 iKeyOff += fts5GetVarintsqlite3Fts5GetVarint(&aPg[iKeyOff], &nPrefix);
15032 }
15033 iKeyOff += fts5GetVarintsqlite3Fts5GetVarint(&aPg[iKeyOff], &nSuffix);
15034
15035 nPrefix = MIN(nPrefix, nPrefix2)(((nPrefix) < (nPrefix2)) ? (nPrefix) : (nPrefix2));
15036 nSuffix = (nPrefix2 + nSuffix2) - nPrefix;
15037
15038 if( (iKeyOff+nSuffix)>(u64)iPgIdx || (iNextOff+nSuffix2)>(u64)iPgIdx ){
15039 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
15040 }else{
15041 if( iKey!=1 ){
15042 iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix);
15043 }
15044 iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix);
15045 if( nPrefix2>(u64)pSeg->term.n ){
15046 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
15047 }else if( nPrefix2>nPrefix ){
15048 memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix);
15049 iOff += (nPrefix2-nPrefix);
15050 }
15051 memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2);
15052 iOff += nSuffix2;
15053 iNextOff += nSuffix2;
15054 }
15055 }
15056 }else if( iStart==4 ){
15057 int iPgno;
15058
15059 assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno )((void) (0));
15060 /* The entry being removed may be the only position list in
15061 ** its doclist. */
15062 for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
15063 Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(iPgno)) ));
15064 int bEmpty = (pPg && pPg->nn==4);
15065 fts5DataRelease(pPg);
15066 if( bEmpty==0 ) break;
15067 }
15068
15069 if( iPgno==pSeg->iTermLeafPgno ){
15070 i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pSeg->iTermLeafPgno
)) );
15071 Fts5Data *pTerm = fts5DataRead(p, iId);
15072 if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
15073 u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
15074 int nTermIdx = pTerm->nn - pTerm->szLeaf;
15075 int iTermIdx = 0;
15076 i64 iTermOff = 0;
15077
15078 while( 1 ){
15079 u32 iVal = 0;
15080 int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal)sqlite3Fts5GetVarint32(&aTermIdx[iTermIdx],(u32*)&(iVal
));
15081 iTermOff += iVal;
15082 if( (iTermIdx+nByte)>=nTermIdx ) break;
15083 iTermIdx += nByte;
15084 }
15085 nTermIdx = iTermIdx;
15086
15087 if( iTermOff>pTerm->szLeaf ){
15088 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
15089 }else{
15090 memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
15091 fts5PutU16(&pTerm->p[2], iTermOff);
15092 fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
15093 if( nTermIdx==0 ){
15094 fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
15095 }
15096 }
15097 }
15098 fts5DataRelease(pTerm);
15099 }
15100 }
15101
15102 /* Assuming no error has occurred, this block does final edits to the
15103 ** leaf page before writing it back to disk. Input variables are:
15104 **
15105 ** nPg: Total initial size of leaf page.
15106 ** iPgIdx: Initial offset of page footer.
15107 **
15108 ** iOff: Offset to move data to
15109 ** iNextOff: Offset to move data from
15110 */
15111 if( p->rc==SQLITE_OK0 ){
15112 const int nMove = nPg - iNextOff; /* Number of bytes to move */
15113 int nShift = iNextOff - iOff; /* Distance to move them */
15114
15115 int iPrevKeyOut = 0;
15116 int iKeyIn = 0;
15117
15118 if( nMove>0 ){
15119 memmove(&aPg[iOff], &aPg[iNextOff], nMove);
15120 }
15121 iPgIdx -= nShift;
15122 nPg = iPgIdx;
15123 fts5PutU16(&aPg[2], iPgIdx);
15124
15125 for(iIdx=0; iIdx<nIdx; /* no-op */){
15126 u32 iVal = 0;
15127 iIdx += fts5GetVarint32(&aIdx[iIdx], iVal)sqlite3Fts5GetVarint32(&aIdx[iIdx],(u32*)&(iVal));
15128 iKeyIn += iVal;
15129 if( iKeyIn!=iDelKeyOff ){
15130 int iKeyOut = (iKeyIn - (iKeyIn>iOff ? nShift : 0));
15131 nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOut - iPrevKeyOut);
15132 iPrevKeyOut = iKeyOut;
15133 }
15134 }
15135
15136 if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
15137 fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
15138 }
15139
15140 assert_nc( nPg>4 || fts5GetU16(aPg)==0 )((void) (0));
15141 fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pSeg->iLeafPgno
)) ), aPg, nPg);
15142 }
15143 sqlite3_freesqlite3_api->free(aIdx);
15144}
15145
15146/*
15147** This is called as part of flushing a delete to disk in 'secure-delete'
15148** mode. It edits the segments within the database described by argument
15149** pStruct to remove the entries for term zTerm, rowid iRowid.
15150**
15151** Return SQLITE_OK if successful, or an SQLite error code if an error
15152** has occurred. Any error code is also stored in the Fts5Index handle.
15153*/
15154static int fts5FlushSecureDelete(
15155 Fts5Index *p,
15156 Fts5Structure *pStruct,
15157 const char *zTerm,
15158 int nTerm,
15159 i64 iRowid
15160){
15161 const int f = FTS5INDEX_QUERY_SKIPHASH0x0040;
15162 Fts5Iter *pIter = 0; /* Used to find term instance */
15163
15164 /* If the version number has not been set to SECUREDELETE, do so now. */
15165 if( p->pConfig->iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE5 ){
15166 Fts5Config *pConfig = p->pConfig;
15167 sqlite3_stmt *pStmt = 0;
15168 fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintfsqlite3_api->mprintf(
15169 "REPLACE INTO %Q.'%q_config' VALUES ('version', %d)",
15170 pConfig->zDb, pConfig->zName, FTS5_CURRENT_VERSION_SECUREDELETE5
15171 ));
15172 if( p->rc==SQLITE_OK0 ){
15173 int rc;
15174 sqlite3_stepsqlite3_api->step(pStmt);
15175 rc = sqlite3_finalizesqlite3_api->finalize(pStmt);
15176 if( p->rc==SQLITE_OK0 ) p->rc = rc;
15177 pConfig->iCookie++;
15178 pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE5;
15179 }
15180 }
15181
15182 fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter);
15183 if( fts5MultiIterEof(p, pIter)==0 ){
15184 i64 iThis = fts5MultiIterRowid(pIter);
15185 if( iThis<iRowid ){
15186 fts5MultiIterNextFrom(p, pIter, iRowid);
15187 }
15188
15189 if( p->rc==SQLITE_OK0
15190 && fts5MultiIterEof(p, pIter)==0
15191 && iRowid==fts5MultiIterRowid(pIter)
15192 ){
15193 Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
15194 fts5DoSecureDelete(p, pSeg);
15195 }
15196 }
15197
15198 fts5MultiIterFree(pIter);
15199 return p->rc;
15200}
15201
15202
15203/*
15204** Flush the contents of in-memory hash table iHash to a new level-0
15205** segment on disk. Also update the corresponding structure record.
15206**
15207** If an error occurs, set the Fts5Index.rc error code. If an error has
15208** already occurred, this function is a no-op.
15209*/
15210static void fts5FlushOneHash(Fts5Index *p){
15211 Fts5Hash *pHash = p->pHash;
15212 Fts5Structure *pStruct;
15213 int iSegid;
15214 int pgnoLast = 0; /* Last leaf page number in segment */
15215
15216 /* Obtain a reference to the index structure and allocate a new segment-id
15217 ** for the new level-0 segment. */
15218 pStruct = fts5StructureRead(p);
15219 fts5StructureInvalidate(p);
15220
15221 if( sqlite3Fts5HashIsEmpty(pHash)==0 ){
15222 iSegid = fts5AllocateSegid(p, pStruct);
15223 if( iSegid ){
15224 const int pgsz = p->pConfig->pgsz;
15225 int eDetail = p->pConfig->eDetail;
15226 int bSecureDelete = p->pConfig->bSecureDelete;
15227 Fts5StructureSegment *pSeg; /* New segment within pStruct */
15228 Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */
15229 Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */
15230
15231 Fts5SegWriter writer;
15232 fts5WriteInit(p, &writer, iSegid);
15233
15234 pBuf = &writer.writer.buf;
15235 pPgidx = &writer.writer.pgidx;
15236
15237 /* fts5WriteInit() should have initialized the buffers to (most likely)
15238 ** the maximum space required. */
15239 assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) )((void) (0));
15240 assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) )((void) (0));
15241
15242 /* Begin scanning through hash table entries. This loop runs once for each
15243 ** term/doclist currently stored within the hash table. */
15244 if( p->rc==SQLITE_OK0 ){
15245 p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0);
15246 }
15247 while( p->rc==SQLITE_OK0 && 0==sqlite3Fts5HashScanEof(pHash) ){
15248 const char *zTerm; /* Buffer containing term */
15249 int nTerm; /* Size of zTerm in bytes */
15250 const u8 *pDoclist; /* Pointer to doclist for this term */
15251 int nDoclist; /* Size of doclist in bytes */
15252
15253 /* Get the term and doclist for this entry. */
15254 sqlite3Fts5HashScanEntry(pHash, &zTerm, &nTerm, &pDoclist, &nDoclist);
15255 if( bSecureDelete==0 ){
15256 fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
15257 if( p->rc!=SQLITE_OK0 ) break;
15258 assert( writer.bFirstRowidInPage==0 )((void) (0));
15259 }
15260
15261 if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
15262 /* The entire doclist will fit on the current leaf. */
15263 fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist){ ((void) (0)); memcpy(&(pBuf)->p[(pBuf)->n], pDoclist
, nDoclist); (pBuf)->n += nDoclist; };
15264 }else{
15265 int bTermWritten = !bSecureDelete;
15266 i64 iRowid = 0;
15267 i64 iPrev = 0;
15268 int iOff = 0;
15269
15270 /* The entire doclist will not fit on this leaf. The following
15271 ** loop iterates through the poslists that make up the current
15272 ** doclist. */
15273 while( p->rc==SQLITE_OK0 && iOff<nDoclist ){
15274 u64 iDelta = 0;
15275 iOff += fts5GetVarintsqlite3Fts5GetVarint(&pDoclist[iOff], &iDelta);
15276 iRowid += iDelta;
15277
15278 /* If in secure delete mode, and if this entry in the poslist is
15279 ** in fact a delete, then edit the existing segments directly
15280 ** using fts5FlushSecureDelete(). */
15281 if( bSecureDelete ){
15282 if( eDetail==FTS5_DETAIL_NONE1 ){
15283 if( iOff<nDoclist && pDoclist[iOff]==0x00
15284 && !fts5FlushSecureDelete(p, pStruct, zTerm, nTerm, iRowid)
15285 ){
15286 iOff++;
15287 if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
15288 iOff++;
15289 nDoclist = 0;
15290 }else{
15291 continue;
15292 }
15293 }
15294 }else if( (pDoclist[iOff] & 0x01)
15295 && !fts5FlushSecureDelete(p, pStruct, zTerm, nTerm, iRowid)
15296 ){
15297 if( p->rc!=SQLITE_OK0 || pDoclist[iOff]==0x01 ){
15298 iOff++;
15299 continue;
15300 }
15301 }
15302 }
15303
15304 if( p->rc==SQLITE_OK0 && bTermWritten==0 ){
15305 fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
15306 bTermWritten = 1;
15307 assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 )((void) (0));
15308 }
15309
15310 if( writer.bFirstRowidInPage ){
15311 fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */
15312 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
15313 writer.bFirstRowidInPage = 0;
15314 fts5WriteDlidxAppend(p, &writer, iRowid);
15315 }else{
15316 u64 iRowidDelta = (u64)iRowid - (u64)iPrev;
15317 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowidDelta);
15318 }
15319 if( p->rc!=SQLITE_OK0 ) break;
15320 assert( pBuf->n<=pBuf->nSpace )((void) (0));
15321 iPrev = iRowid;
15322
15323 if( eDetail==FTS5_DETAIL_NONE1 ){
15324 if( iOff<nDoclist && pDoclist[iOff]==0 ){
15325 pBuf->p[pBuf->n++] = 0;
15326 iOff++;
15327 if( iOff<nDoclist && pDoclist[iOff]==0 ){
15328 pBuf->p[pBuf->n++] = 0;
15329 iOff++;
15330 }
15331 }
15332 if( (pBuf->n + pPgidx->n)>=pgsz ){
15333 fts5WriteFlushLeaf(p, &writer);
15334 }
15335 }else{
15336 int bDel = 0;
15337 int nPos = 0;
15338 int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDel);
15339 if( bDel && bSecureDelete ){
15340 fts5BufferAppendVarint(&p->rc, pBuf, nPos*2)sqlite3Fts5BufferAppendVarint(&p->rc,pBuf,(i64)nPos*2);
15341 iOff += nCopy;
15342 nCopy = nPos;
15343 }else{
15344 nCopy += nPos;
15345 }
15346 if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
15347 /* The entire poslist will fit on the current leaf. So copy
15348 ** it in one go. */
15349 fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy){ ((void) (0)); memcpy(&(pBuf)->p[(pBuf)->n], &
pDoclist[iOff], nCopy); (pBuf)->n += nCopy; };
15350 }else{
15351 /* The entire poslist will not fit on this leaf. So it needs
15352 ** to be broken into sections. The only qualification being
15353 ** that each varint must be stored contiguously. */
15354 const u8 *pPoslist = &pDoclist[iOff];
15355 int iPos = 0;
15356 while( p->rc==SQLITE_OK0 ){
15357 int nSpace = pgsz - pBuf->n - pPgidx->n;
15358 int n = 0;
15359 if( (nCopy - iPos)<=nSpace ){
15360 n = nCopy - iPos;
15361 }else{
15362 n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
15363 }
15364 assert( n>0 )((void) (0));
15365 fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n){ ((void) (0)); memcpy(&(pBuf)->p[(pBuf)->n], &
pPoslist[iPos], n); (pBuf)->n += n; };
15366 iPos += n;
15367 if( (pBuf->n + pPgidx->n)>=pgsz ){
15368 fts5WriteFlushLeaf(p, &writer);
15369 }
15370 if( iPos>=nCopy ) break;
15371 }
15372 }
15373 iOff += nCopy;
15374 }
15375 }
15376 }
15377
15378 /* TODO2: Doclist terminator written here. */
15379 /* pBuf->p[pBuf->n++] = '\0'; */
15380 assert( pBuf->n<=pBuf->nSpace )((void) (0));
15381 if( p->rc==SQLITE_OK0 ) sqlite3Fts5HashScanNext(pHash);
15382 }
15383 fts5WriteFinish(p, &writer, &pgnoLast);
15384
15385 assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 )((void) (0));
15386 if( pgnoLast>0 ){
15387 /* Update the Fts5Structure. It is written back to the database by the
15388 ** fts5StructureRelease() call below. */
15389 if( pStruct->nLevel==0 ){
15390 fts5StructureAddLevel(&p->rc, &pStruct);
15391 }
15392 fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
15393 if( p->rc==SQLITE_OK0 ){
15394 pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
15395 pSeg->iSegid = iSegid;
15396 pSeg->pgnoFirst = 1;
15397 pSeg->pgnoLast = pgnoLast;
15398 if( pStruct->nOriginCntr>0 ){
15399 pSeg->iOrigin1 = pStruct->nOriginCntr;
15400 pSeg->iOrigin2 = pStruct->nOriginCntr;
15401 pSeg->nEntry = p->nPendingRow;
15402 pStruct->nOriginCntr++;
15403 }
15404 pStruct->nSegment++;
15405 }
15406 fts5StructurePromote(p, 0, pStruct);
15407 }
15408 }
15409 }
15410
15411 fts5IndexAutomerge(p, &pStruct, pgnoLast + p->nContentlessDelete);
15412 fts5IndexCrisismerge(p, &pStruct);
15413 fts5StructureWrite(p, pStruct);
15414 fts5StructureRelease(pStruct);
15415}
15416
15417/*
15418** Flush any data stored in the in-memory hash tables to the database.
15419*/
15420static void fts5IndexFlush(Fts5Index *p){
15421 /* Unless it is empty, flush the hash table to disk */
15422 if( p->flushRc ){
15423 p->rc = p->flushRc;
15424 return;
15425 }
15426 if( p->nPendingData || p->nContentlessDelete ){
15427 assert( p->pHash )((void) (0));
15428 fts5FlushOneHash(p);
15429 if( p->rc==SQLITE_OK0 ){
15430 sqlite3Fts5HashClear(p->pHash);
15431 p->nPendingData = 0;
15432 p->nPendingRow = 0;
15433 p->nContentlessDelete = 0;
15434 }else if( p->nPendingData || p->nContentlessDelete ){
15435 p->flushRc = p->rc;
15436 }
15437 }
15438}
15439
15440static Fts5Structure *fts5IndexOptimizeStruct(
15441 Fts5Index *p,
15442 Fts5Structure *pStruct
15443){
15444 Fts5Structure *pNew = 0;
15445 sqlite3_int64 nByte = SZ_FTS5STRUCTURE(1)(__builtin_offsetof(Fts5Structure, aLevel) + (1)*sizeof(Fts5StructureLevel
));
15446 int nSeg = pStruct->nSegment;
15447 int i;
15448
15449 /* Figure out if this structure requires optimization. A structure does
15450 ** not require optimization if either:
15451 **
15452 ** 1. it consists of fewer than two segments, or
15453 ** 2. all segments are on the same level, or
15454 ** 3. all segments except one are currently inputs to a merge operation.
15455 **
15456 ** In the first case, if there are no tombstone hash pages, return NULL. In
15457 ** the second, increment the ref-count on *pStruct and return a copy of the
15458 ** pointer to it.
15459 */
15460 if( nSeg==0 ) return 0;
15461 for(i=0; i<pStruct->nLevel; i++){
15462 int nThis = pStruct->aLevel[i].nSeg;
15463 int nMerge = pStruct->aLevel[i].nMerge;
15464 if( nThis>0 && (nThis==nSeg || (nThis==nSeg-1 && nMerge==nThis)) ){
15465 if( nSeg==1 && nThis==1 && pStruct->aLevel[i].aSeg[0].nPgTombstone==0 ){
15466 return 0;
15467 }
15468 fts5StructureRef(pStruct);
15469 return pStruct;
15470 }
15471 assert( pStruct->aLevel[i].nMerge<=nThis )((void) (0));
15472 }
15473
15474 nByte += (((i64)pStruct->nLevel)+1) * sizeof(Fts5StructureLevel);
15475 assert( nByte==(i64)SZ_FTS5STRUCTURE(pStruct->nLevel+2) )((void) (0));
15476 pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
15477
15478 if( pNew ){
15479 Fts5StructureLevel *pLvl;
15480 nByte = nSeg * sizeof(Fts5StructureSegment);
15481 pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL)(((pStruct->nLevel+1) < (64)) ? (pStruct->nLevel+1) :
(64));
15482 pNew->nRef = 1;
15483 pNew->nWriteCounter = pStruct->nWriteCounter;
15484 pNew->nOriginCntr = pStruct->nOriginCntr;
15485 pLvl = &pNew->aLevel[pNew->nLevel-1];
15486 pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
15487 if( pLvl->aSeg ){
15488 int iLvl, iSeg;
15489 int iSegOut = 0;
15490 /* Iterate through all segments, from oldest to newest. Add them to
15491 ** the new Fts5Level object so that pLvl->aSeg[0] is the oldest
15492 ** segment in the data structure. */
15493 for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){
15494 for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
15495 pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg];
15496 iSegOut++;
15497 }
15498 }
15499 pNew->nSegment = pLvl->nSeg = nSeg;
15500 }else{
15501 sqlite3_freesqlite3_api->free(pNew);
15502 pNew = 0;
15503 }
15504 }
15505
15506 return pNew;
15507}
15508
15509static int sqlite3Fts5IndexOptimize(Fts5Index *p){
15510 Fts5Structure *pStruct;
15511 Fts5Structure *pNew = 0;
15512
15513 assert( p->rc==SQLITE_OK )((void) (0));
15514 fts5IndexFlush(p);
15515 assert( p->rc!=SQLITE_OK || p->nContentlessDelete==0 )((void) (0));
15516 pStruct = fts5StructureRead(p);
15517 assert( p->rc!=SQLITE_OK || pStruct!=0 )((void) (0));
15518 fts5StructureInvalidate(p);
15519
15520 if( pStruct ){
15521 pNew = fts5IndexOptimizeStruct(p, pStruct);
15522 }
15523 fts5StructureRelease(pStruct);
15524
15525 assert( pNew==0 || pNew->nSegment>0 )((void) (0));
15526 if( pNew ){
15527 int iLvl;
15528 for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){}
15529 while( p->rc==SQLITE_OK0 && pNew->aLevel[iLvl].nSeg>0 ){
15530 int nRem = FTS5_OPT_WORK_UNIT1000;
15531 fts5IndexMergeLevel(p, &pNew, iLvl, &nRem);
15532 }
15533
15534 fts5StructureWrite(p, pNew);
15535 fts5StructureRelease(pNew);
15536 }
15537
15538 return fts5IndexReturn(p);
15539}
15540
15541/*
15542** This is called to implement the special "VALUES('merge', $nMerge)"
15543** INSERT command.
15544*/
15545static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
15546 Fts5Structure *pStruct = 0;
15547
15548 fts5IndexFlush(p);
15549 pStruct = fts5StructureRead(p);
15550 if( pStruct ){
15551 int nMin = p->pConfig->nUsermerge;
15552 fts5StructureInvalidate(p);
15553 if( nMerge<0 ){
15554 Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
15555 fts5StructureRelease(pStruct);
15556 pStruct = pNew;
15557 nMin = 1;
15558 nMerge = (nMerge==SMALLEST_INT32((int)((-1) - ((int)(0x7fffffff)))) ? LARGEST_INT32((int)(0x7fffffff)) : (nMerge*-1));
15559 }
15560 if( pStruct && pStruct->nLevel ){
15561 if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){
15562 fts5StructureWrite(p, pStruct);
15563 }
15564 }
15565 fts5StructureRelease(pStruct);
15566 }
15567 return fts5IndexReturn(p);
15568}
15569
15570static void fts5AppendRowid(
15571 Fts5Index *p,
15572 u64 iDelta,
15573 Fts5Iter *pUnused,
15574 Fts5Buffer *pBuf
15575){
15576 UNUSED_PARAM(pUnused)(void)(pUnused);
15577 fts5BufferAppendVarint(&p->rc, pBuf, iDelta)sqlite3Fts5BufferAppendVarint(&p->rc,pBuf,(i64)iDelta);
15578}
15579
15580static void fts5AppendPoslist(
15581 Fts5Index *p,
15582 u64 iDelta,
15583 Fts5Iter *pMulti,
15584 Fts5Buffer *pBuf
15585){
15586 int nData = pMulti->base.nData;
15587 int nByte = nData + 9 + 9 + FTS5_DATA_ZERO_PADDING8;
15588 assert( nData>0 )((void) (0));
15589 if( p->rc==SQLITE_OK0 && 0==fts5BufferGrow(&p->rc, pBuf, nByte)( (u32)((pBuf)->n) + (u32)(nByte) <= (u32)((pBuf)->nSpace
) ? 0 : sqlite3Fts5BufferSize((&p->rc),(pBuf),(nByte)+
(pBuf)->n) ) ){
15590 fts5BufferSafeAppendVarint(pBuf, iDelta){ (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf
)->n], (iDelta)); ((void) (0)); };
15591 fts5BufferSafeAppendVarint(pBuf, nData*2){ (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf
)->n], (nData*2)); ((void) (0)); };
15592 fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData){ ((void) (0)); memcpy(&(pBuf)->p[(pBuf)->n], pMulti
->base.pData, nData); (pBuf)->n += nData; };
15593 memset(&pBuf->p[pBuf->n], 0, FTS5_DATA_ZERO_PADDING8);
15594 }
15595}
15596
15597
15598static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
15599 u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;
15600
15601 assert( pIter->aPoslist || (p==0 && pIter->aPoslist==0) )((void) (0));
15602 if( p>=pIter->aEof ){
15603 pIter->aPoslist = 0;
15604 }else{
15605 i64 iDelta;
15606
15607 p += fts5GetVarintsqlite3Fts5GetVarint(p, (u64*)&iDelta);
15608 pIter->iRowid += iDelta;
15609
15610 /* Read position list size */
15611 if( p[0] & 0x80 ){
15612 int nPos;
15613 pIter->nSize = fts5GetVarint32(p, nPos)sqlite3Fts5GetVarint32(p,(u32*)&(nPos));
15614 pIter->nPoslist = (nPos>>1);
15615 }else{
15616 pIter->nPoslist = ((int)(p[0])) >> 1;
15617 pIter->nSize = 1;
15618 }
15619
15620 pIter->aPoslist = p;
15621 if( &pIter->aPoslist[pIter->nPoslist]>pIter->aEof ){
15622 pIter->aPoslist = 0;
15623 }
15624 }
15625}
15626
15627static void fts5DoclistIterInit(
15628 Fts5Buffer *pBuf,
15629 Fts5DoclistIter *pIter
15630){
15631 memset(pIter, 0, sizeof(*pIter));
15632 if( pBuf->n>0 ){
15633 pIter->aPoslist = pBuf->p;
15634 pIter->aEof = &pBuf->p[pBuf->n];
15635 fts5DoclistIterNext(pIter);
15636 }
15637}
15638
15639#if 0
15640/*
15641** Append a doclist to buffer pBuf.
15642**
15643** This function assumes that space within the buffer has already been
15644** allocated.
15645*/
15646static void fts5MergeAppendDocid({ ((void) (0)); { ((Fts5Buffer *pBuf))->n += sqlite3Fts5PutVarint
(&((Fts5Buffer *pBuf))->p[((Fts5Buffer *pBuf))->n],
((u64)(i64 iRowid) - (u64)(i64 *piLastRowid))); ((void) (0))
; }; (i64 *piLastRowid) = (i64 iRowid); }
15647 Fts5Buffer *pBuf, /* Buffer to write to */{ ((void) (0)); { ((Fts5Buffer *pBuf))->n += sqlite3Fts5PutVarint
(&((Fts5Buffer *pBuf))->p[((Fts5Buffer *pBuf))->n],
((u64)(i64 iRowid) - (u64)(i64 *piLastRowid))); ((void) (0))
; }; (i64 *piLastRowid) = (i64 iRowid); }
15648 i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */{ ((void) (0)); { ((Fts5Buffer *pBuf))->n += sqlite3Fts5PutVarint
(&((Fts5Buffer *pBuf))->p[((Fts5Buffer *pBuf))->n],
((u64)(i64 iRowid) - (u64)(i64 *piLastRowid))); ((void) (0))
; }; (i64 *piLastRowid) = (i64 iRowid); }
15649 i64 iRowid /* Rowid to append */{ ((void) (0)); { ((Fts5Buffer *pBuf))->n += sqlite3Fts5PutVarint
(&((Fts5Buffer *pBuf))->p[((Fts5Buffer *pBuf))->n],
((u64)(i64 iRowid) - (u64)(i64 *piLastRowid))); ((void) (0))
; }; (i64 *piLastRowid) = (i64 iRowid); }
15650){ ((void) (0)); { ((Fts5Buffer *pBuf))->n += sqlite3Fts5PutVarint
(&((Fts5Buffer *pBuf))->p[((Fts5Buffer *pBuf))->n],
((u64)(i64 iRowid) - (u64)(i64 *piLastRowid))); ((void) (0))
; }; (i64 *piLastRowid) = (i64 iRowid); }{
15651 assert( pBuf->n!=0 || (*piLastRowid)==0 )((void) (0));
15652 fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid){ (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf
)->n], (iRowid - *piLastRowid)); ((void) (0)); };
15653 *piLastRowid = iRowid;
15654}
15655#endif
15656
15657#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid){ ((void) (0)); { ((pBuf))->n += sqlite3Fts5PutVarint(&
((pBuf))->p[((pBuf))->n], ((u64)(iRowid) - (u64)(iLastRowid
))); ((void) (0)); }; (iLastRowid) = (iRowid); } { \
15658 assert( (pBuf)->n!=0 || (iLastRowid)==0 )((void) (0)); \
15659 fts5BufferSafeAppendVarint((pBuf), (u64)(iRowid) - (u64)(iLastRowid)){ ((pBuf))->n += sqlite3Fts5PutVarint(&((pBuf))->p[
((pBuf))->n], ((u64)(iRowid) - (u64)(iLastRowid))); ((void
) (0)); }; \
15660 (iLastRowid) = (iRowid); \
15661}
15662
15663/*
15664** Swap the contents of buffer *p1 with that of *p2.
15665*/
15666static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
15667 Fts5Buffer tmp = *p1;
15668 *p1 = *p2;
15669 *p2 = tmp;
15670}
15671
15672static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){
15673 int i = *piOff;
15674 if( i>=pBuf->n ){
15675 *piOff = -1;
15676 }else{
15677 u64 iVal;
15678 *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal);
15679 *piRowid += iVal;
15680 }
15681}
15682
15683/*
15684** This is the equivalent of fts5MergePrefixLists() for detail=none mode.
15685** In this case the buffers consist of a delta-encoded list of rowids only.
15686*/
15687static void fts5MergeRowidLists(
15688 Fts5Index *p, /* FTS5 backend object */
15689 Fts5Buffer *p1, /* First list to merge */
15690 int nBuf, /* Number of entries in apBuf[] */
15691 Fts5Buffer *aBuf /* Array of other lists to merge into p1 */
15692){
15693 int i1 = 0;
15694 int i2 = 0;
15695 i64 iRowid1 = 0;
15696 i64 iRowid2 = 0;
15697 i64 iOut = 0;
15698 Fts5Buffer *p2 = &aBuf[0];
15699 Fts5Buffer out;
15700
15701 (void)nBuf;
15702 memset(&out, 0, sizeof(out));
15703 assert( nBuf==1 )((void) (0));
15704 sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n);
15705 if( p->rc ) return;
15706
15707 fts5NextRowid(p1, &i1, &iRowid1);
15708 fts5NextRowid(p2, &i2, &iRowid2);
15709 while( i1>=0 || i2>=0 ){
15710 if( i1>=0 && (i2<0 || iRowid1<iRowid2) ){
15711 assert( iOut==0 || iRowid1>iOut )((void) (0));
15712 fts5BufferSafeAppendVarint(&out, iRowid1 - iOut){ (&out)->n += sqlite3Fts5PutVarint(&(&out)->
p[(&out)->n], (iRowid1 - iOut)); ((void) (0)); };
15713 iOut = iRowid1;
15714 fts5NextRowid(p1, &i1, &iRowid1);
15715 }else{
15716 assert( iOut==0 || iRowid2>iOut )((void) (0));
15717 fts5BufferSafeAppendVarint(&out, iRowid2 - iOut){ (&out)->n += sqlite3Fts5PutVarint(&(&out)->
p[(&out)->n], (iRowid2 - iOut)); ((void) (0)); };
15718 iOut = iRowid2;
15719 if( i1>=0 && iRowid1==iRowid2 ){
15720 fts5NextRowid(p1, &i1, &iRowid1);
15721 }
15722 fts5NextRowid(p2, &i2, &iRowid2);
15723 }
15724 }
15725
15726 fts5BufferSwap(&out, p1);
15727 fts5BufferFree(&out)sqlite3Fts5BufferFree(&out);
15728}
15729
15730typedef struct PrefixMerger PrefixMerger;
15731struct PrefixMerger {
15732 Fts5DoclistIter iter; /* Doclist iterator */
15733 i64 iPos; /* For iterating through a position list */
15734 int iOff;
15735 u8 *aPos;
15736 PrefixMerger *pNext; /* Next in docid/poslist order */
15737};
15738
15739static void fts5PrefixMergerInsertByRowid(
15740 PrefixMerger **ppHead,
15741 PrefixMerger *p
15742){
15743 if( p->iter.aPoslist ){
15744 PrefixMerger **pp = ppHead;
15745 while( *pp && p->iter.iRowid>(*pp)->iter.iRowid ){
15746 pp = &(*pp)->pNext;
15747 }
15748 p->pNext = *pp;
15749 *pp = p;
15750 }
15751}
15752
15753static void fts5PrefixMergerInsertByPosition(
15754 PrefixMerger **ppHead,
15755 PrefixMerger *p
15756){
15757 if( p->iPos>=0 ){
15758 PrefixMerger **pp = ppHead;
15759 while( *pp && p->iPos>(*pp)->iPos ){
15760 pp = &(*pp)->pNext;
15761 }
15762 p->pNext = *pp;
15763 *pp = p;
15764 }
15765}
15766
15767
15768/*
15769** Array aBuf[] contains nBuf doclists. These are all merged in with the
15770** doclist in buffer p1.
15771*/
15772static void fts5MergePrefixLists(
15773 Fts5Index *p, /* FTS5 backend object */
15774 Fts5Buffer *p1, /* First list to merge */
15775 int nBuf, /* Number of buffers in array aBuf[] */
15776 Fts5Buffer *aBuf /* Other lists to merge in */
15777){
15778#define fts5PrefixMergerNextPosition(p)sqlite3Fts5PoslistNext64((p)->aPos,(p)->iter.nPoslist,&
(p)->iOff,&(p)->iPos) \
15779 sqlite3Fts5PoslistNext64((p)->aPos,(p)->iter.nPoslist,&(p)->iOff,&(p)->iPos)
15780#define FTS5_MERGE_NLIST16 16
15781 PrefixMerger aMerger[FTS5_MERGE_NLIST16];
15782 PrefixMerger *pHead = 0;
15783 int i;
15784 int nOut = 0;
15785 Fts5Buffer out = {0, 0, 0};
15786 Fts5Buffer tmp = {0, 0, 0};
15787 i64 iLastRowid = 0;
15788
15789 /* Initialize a doclist-iterator for each input buffer. Arrange them in
15790 ** a linked-list starting at pHead in ascending order of rowid. Avoid
15791 ** linking any iterators already at EOF into the linked list at all. */
15792 assert( nBuf+1<=(int)(sizeof(aMerger)/sizeof(aMerger[0])) )((void) (0));
15793 memset(aMerger, 0, sizeof(PrefixMerger)*(nBuf+1));
15794 pHead = &aMerger[nBuf];
15795 fts5DoclistIterInit(p1, &pHead->iter);
15796 for(i=0; i<nBuf; i++){
15797 fts5DoclistIterInit(&aBuf[i], &aMerger[i].iter);
15798 fts5PrefixMergerInsertByRowid(&pHead, &aMerger[i]);
15799 nOut += aBuf[i].n;
15800 }
15801 if( nOut==0 ) return;
15802 nOut += p1->n + 9 + 10*nBuf;
15803
15804 /* The maximum size of the output is equal to the sum of the
15805 ** input sizes + 1 varint (9 bytes). The extra varint is because if the
15806 ** first rowid in one input is a large negative number, and the first in
15807 ** the other a non-negative number, the delta for the non-negative
15808 ** number will be larger on disk than the literal integer value
15809 ** was.
15810 **
15811 ** Or, if the input position-lists are corrupt, then the output might
15812 ** include up to (nBuf+1) extra 10-byte positions created by interpreting -1
15813 ** (the value PoslistNext64() uses for EOF) as a position and appending
15814 ** it to the output. This can happen at most once for each input
15815 ** position-list, hence (nBuf+1) 10 byte paddings. */
15816 if( sqlite3Fts5BufferSize(&p->rc, &out, nOut) ) return;
15817
15818 while( pHead ){
15819 fts5MergeAppendDocid(&out, iLastRowid, pHead->iter.iRowid){ ((void) (0)); { ((&out))->n += sqlite3Fts5PutVarint(
&((&out))->p[((&out))->n], ((u64)(pHead->
iter.iRowid) - (u64)(iLastRowid))); ((void) (0)); }; (iLastRowid
) = (pHead->iter.iRowid); };
15820
15821 if( pHead->pNext && iLastRowid==pHead->pNext->iter.iRowid ){
15822 /* Merge data from two or more poslists */
15823 i64 iPrev = 0;
15824 int nTmp = FTS5_DATA_ZERO_PADDING8;
15825 int nMerge = 0;
15826 PrefixMerger *pSave = pHead;
15827 PrefixMerger *pThis = 0;
15828 int nTail = 0;
15829
15830 pHead = 0;
15831 while( pSave && pSave->iter.iRowid==iLastRowid ){
15832 PrefixMerger *pNext = pSave->pNext;
15833 pSave->iOff = 0;
15834 pSave->iPos = 0;
15835 pSave->aPos = &pSave->iter.aPoslist[pSave->iter.nSize];
15836 fts5PrefixMergerNextPosition(pSave)sqlite3Fts5PoslistNext64((pSave)->aPos,(pSave)->iter.nPoslist
,&(pSave)->iOff,&(pSave)->iPos);
15837 nTmp += pSave->iter.nPoslist + 10;
15838 nMerge++;
15839 fts5PrefixMergerInsertByPosition(&pHead, pSave);
15840 pSave = pNext;
15841 }
15842
15843 if( pHead==0 || pHead->pNext==0 ){
15844 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
15845 break;
15846 }
15847
15848 /* See the earlier comment in this function for an explanation of why
15849 ** corrupt input position lists might cause the output to consume
15850 ** at most nMerge*10 bytes of unexpected space. */
15851 if( sqlite3Fts5BufferSize(&p->rc, &tmp, nTmp+nMerge*10) ){
15852 break;
15853 }
15854 fts5BufferZero(&tmp)sqlite3Fts5BufferZero(&tmp);
15855
15856 pThis = pHead;
15857 pHead = pThis->pNext;
15858 sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pThis->iPos);
15859 fts5PrefixMergerNextPosition(pThis)sqlite3Fts5PoslistNext64((pThis)->aPos,(pThis)->iter.nPoslist
,&(pThis)->iOff,&(pThis)->iPos);
15860 fts5PrefixMergerInsertByPosition(&pHead, pThis);
15861
15862 while( pHead->pNext ){
15863 pThis = pHead;
15864 if( pThis->iPos!=iPrev ){
15865 sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pThis->iPos);
15866 }
15867 fts5PrefixMergerNextPosition(pThis)sqlite3Fts5PoslistNext64((pThis)->aPos,(pThis)->iter.nPoslist
,&(pThis)->iOff,&(pThis)->iPos);
15868 pHead = pThis->pNext;
15869 fts5PrefixMergerInsertByPosition(&pHead, pThis);
15870 }
15871
15872 if( pHead->iPos!=iPrev ){
15873 sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pHead->iPos);
15874 }
15875 nTail = pHead->iter.nPoslist - pHead->iOff;
15876
15877 /* WRITEPOSLISTSIZE */
15878 assert_nc( tmp.n+nTail<=nTmp )((void) (0));
15879 assert( tmp.n+nTail<=nTmp+nMerge*10 )((void) (0));
15880 if( tmp.n+nTail>nTmp-FTS5_DATA_ZERO_PADDING8 ){
15881 if( p->rc==SQLITE_OK0 ) FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
15882 break;
15883 }
15884 fts5BufferSafeAppendVarint(&out, (tmp.n+nTail) * 2){ (&out)->n += sqlite3Fts5PutVarint(&(&out)->
p[(&out)->n], ((tmp.n+nTail) * 2)); ((void) (0)); };
15885 fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n){ ((void) (0)); memcpy(&(&out)->p[(&out)->n
], tmp.p, tmp.n); (&out)->n += tmp.n; };
15886 if( nTail>0 ){
15887 fts5BufferSafeAppendBlob(&out, &pHead->aPos[pHead->iOff], nTail){ ((void) (0)); memcpy(&(&out)->p[(&out)->n
], &pHead->aPos[pHead->iOff], nTail); (&out)->
n += nTail; };
15888 }
15889
15890 pHead = pSave;
15891 for(i=0; i<nBuf+1; i++){
15892 PrefixMerger *pX = &aMerger[i];
15893 if( pX->iter.aPoslist && pX->iter.iRowid==iLastRowid ){
15894 fts5DoclistIterNext(&pX->iter);
15895 fts5PrefixMergerInsertByRowid(&pHead, pX);
15896 }
15897 }
15898
15899 }else{
15900 /* Copy poslist from pHead to output */
15901 PrefixMerger *pThis = pHead;
15902 Fts5DoclistIter *pI = &pThis->iter;
15903 fts5BufferSafeAppendBlob(&out, pI->aPoslist, pI->nPoslist+pI->nSize){ ((void) (0)); memcpy(&(&out)->p[(&out)->n
], pI->aPoslist, pI->nPoslist+pI->nSize); (&out)
->n += pI->nPoslist+pI->nSize; };
15904 fts5DoclistIterNext(pI);
15905 pHead = pThis->pNext;
15906 fts5PrefixMergerInsertByRowid(&pHead, pThis);
15907 }
15908 }
15909
15910 fts5BufferFree(p1)sqlite3Fts5BufferFree(p1);
15911 fts5BufferFree(&tmp)sqlite3Fts5BufferFree(&tmp);
15912 memset(&out.p[out.n], 0, FTS5_DATA_ZERO_PADDING8);
15913 *p1 = out;
15914}
15915
15916
15917/*
15918** Iterate through a range of entries in the FTS index, invoking the xVisit
15919** callback for each of them.
15920**
15921** Parameter pToken points to an nToken buffer containing an FTS index term
15922** (i.e. a document term with the preceding 1 byte index identifier -
15923** FTS5_MAIN_PREFIX or similar). If bPrefix is true, then the call visits
15924** all entries for terms that have pToken/nToken as a prefix. If bPrefix
15925** is false, then only entries with pToken/nToken as the entire key are
15926** visited.
15927**
15928** If the current table is a tokendata=1 table, then if bPrefix is true then
15929** each index term is treated separately. However, if bPrefix is false, then
15930** all index terms corresponding to pToken/nToken are collapsed into a single
15931** term before the callback is invoked.
15932**
15933** The callback invoked for each entry visited is specified by paramter xVisit.
15934** Each time it is invoked, it is passed a pointer to the Fts5Index object,
15935** a copy of the 7th paramter to this function (pCtx) and a pointer to the
15936** iterator that indicates the current entry. If the current entry is the
15937** first with a new term (i.e. different from that of the previous entry,
15938** including the very first term), then the final two parameters are passed
15939** a pointer to the term and its size in bytes, respectively. If the current
15940** entry is not the first associated with its term, these two parameters
15941** are passed 0.
15942**
15943** If parameter pColset is not NULL, then it is used to filter entries before
15944** the callback is invoked.
15945*/
15946static int fts5VisitEntries(
15947 Fts5Index *p, /* Fts5 index object */
15948 Fts5Colset *pColset, /* Columns filter to apply, or NULL */
15949 u8 *pToken, /* Buffer containing token */
15950 int nToken, /* Size of buffer pToken in bytes */
15951 int bPrefix, /* True for a prefix scan */
15952 void (*xVisit)(Fts5Index*, void *pCtx, Fts5Iter *pIter, const u8*, int),
15953 void *pCtx /* Passed as second argument to xVisit() */
15954){
15955 const int flags = (bPrefix ? FTS5INDEX_QUERY_SCAN0x0008 : 0)
15956 | FTS5INDEX_QUERY_SKIPEMPTY0x0010
15957 | FTS5INDEX_QUERY_NOOUTPUT0x0020;
15958 Fts5Iter *p1 = 0; /* Iterator used to gather data from index */
15959 int bNewTerm = 1;
15960 Fts5Structure *pStruct = fts5StructureRead(p);
15961
15962 fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1);
15963 fts5IterSetOutputCb(&p->rc, p1);
15964 for( /* no-op */ ;
15965 fts5MultiIterEof(p, p1)==0;
15966 fts5MultiIterNext2(p, p1, &bNewTerm)
15967 ){
15968 Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ];
15969 int nNew = 0;
15970 const u8 *pNew = 0;
15971
15972 p1->xSetOutputs(p1, pSeg);
15973 if( p->rc ) break;
15974
15975 if( bNewTerm ){
15976 nNew = pSeg->term.n;
15977 pNew = pSeg->term.p;
15978 if( nNew<nToken || memcmp(pToken, pNew, nToken) ) break;
15979 }
15980
15981 xVisit(p, pCtx, p1, pNew, nNew);
15982 }
15983 fts5MultiIterFree(p1);
15984
15985 fts5StructureRelease(pStruct);
15986 return p->rc;
15987}
15988
15989
15990/*
15991** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an
15992** array of these for each row it visits (so all iRowid fields are the same).
15993** Or, for an iterator used by an "ORDER BY rank" query, it accumulates an
15994** array of these for the entire query (in which case iRowid fields may take
15995** a variety of values).
15996**
15997** Each instance in the array indicates the iterator (and therefore term)
15998** associated with position iPos of rowid iRowid. This is used by the
15999** xInstToken() API.
16000**
16001** iRowid:
16002** Rowid for the current entry.
16003**
16004** iPos:
16005** Position of current entry within row. In the usual ((iCol<<32)+iOff)
16006** format (e.g. see macros FTS5_POS2COLUMN() and FTS5_POS2OFFSET()).
16007**
16008** iIter:
16009** If the Fts5TokenDataIter iterator that the entry is part of is
16010** actually an iterator (i.e. with nIter>0, not just a container for
16011** Fts5TokenDataMap structures), then this variable is an index into
16012** the apIter[] array. The corresponding term is that which the iterator
16013** at apIter[iIter] currently points to.
16014**
16015** Or, if the Fts5TokenDataIter iterator is just a container object
16016** (nIter==0), then iIter is an index into the term.p[] buffer where
16017** the term is stored.
16018**
16019** nByte:
16020** In the case where iIter is an index into term.p[], this variable
16021** is the size of the term in bytes. If iIter is an index into apIter[],
16022** this variable is unused.
16023*/
16024struct Fts5TokenDataMap {
16025 i64 iRowid; /* Row this token is located in */
16026 i64 iPos; /* Position of token */
16027 int iIter; /* Iterator token was read from */
16028 int nByte; /* Length of token in bytes (or 0) */
16029};
16030
16031/*
16032** An object used to supplement Fts5Iter for tokendata=1 iterators.
16033**
16034** This object serves two purposes. The first is as a container for an array
16035** of Fts5TokenDataMap structures, which are used to find the token required
16036** when the xInstToken() API is used. This is done by the nMapAlloc, nMap and
16037** aMap[] variables.
16038*/
16039struct Fts5TokenDataIter {
16040 i64 nMapAlloc; /* Allocated size of aMap[] in entries */
16041 i64 nMap; /* Number of valid entries in aMap[] */
16042 Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */
16043
16044 /* The following are used for prefix-queries only. */
16045 Fts5Buffer terms;
16046
16047 /* The following are used for other full-token tokendata queries only. */
16048 i64 nIter;
16049 i64 nIterAlloc;
16050 Fts5PoslistReader *aPoslistReader;
16051 int *aPoslistToIter;
16052 Fts5Iter *apIter[FLEXARRAY];
16053};
16054
16055/* Size in bytes of an Fts5TokenDataIter object holding up to N iterators */
16056#define SZ_FTS5TOKENDATAITER(N)(__builtin_offsetof(Fts5TokenDataIter, apIter) + (N)*sizeof(Fts5Iter
)) \
16057 (offsetof(Fts5TokenDataIter,apIter)__builtin_offsetof(Fts5TokenDataIter, apIter) + (N)*sizeof(Fts5Iter))
16058
16059/*
16060** The two input arrays - a1[] and a2[] - are in sorted order. This function
16061** merges the two arrays together and writes the result to output array
16062** aOut[]. aOut[] is guaranteed to be large enough to hold the result.
16063**
16064** Duplicate entries are copied into the output. So the size of the output
16065** array is always (n1+n2) entries.
16066*/
16067static void fts5TokendataMerge(
16068 Fts5TokenDataMap *a1, int n1, /* Input array 1 */
16069 Fts5TokenDataMap *a2, int n2, /* Input array 2 */
16070 Fts5TokenDataMap *aOut /* Output array */
16071){
16072 int i1 = 0;
16073 int i2 = 0;
16074
16075 assert( n1>=0 && n2>=0 )((void) (0));
16076 while( i1<n1 || i2<n2 ){
16077 Fts5TokenDataMap *pOut = &aOut[i1+i2];
16078 if( i2>=n2 || (i1<n1 && (
16079 a1[i1].iRowid<a2[i2].iRowid
16080 || (a1[i1].iRowid==a2[i2].iRowid && a1[i1].iPos<=a2[i2].iPos)
16081 ))){
16082 memcpy(pOut, &a1[i1], sizeof(Fts5TokenDataMap));
16083 i1++;
16084 }else{
16085 memcpy(pOut, &a2[i2], sizeof(Fts5TokenDataMap));
16086 i2++;
16087 }
16088 }
16089}
16090
16091
16092/*
16093** Append a mapping to the token-map belonging to object pT.
16094*/
16095static void fts5TokendataIterAppendMap(
16096 Fts5Index *p,
16097 Fts5TokenDataIter *pT,
16098 int iIter,
16099 int nByte,
16100 i64 iRowid,
16101 i64 iPos
16102){
16103 if( p->rc==SQLITE_OK0 ){
16104 if( pT->nMap==pT->nMapAlloc ){
16105 i64 nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64;
16106 i64 nAlloc = nNew * sizeof(Fts5TokenDataMap);
16107 Fts5TokenDataMap *aNew;
16108
16109 aNew = (Fts5TokenDataMap*)sqlite3_realloc64sqlite3_api->realloc64(pT->aMap, nAlloc);
16110 if( aNew==0 ){
16111 p->rc = SQLITE_NOMEM7;
16112 return;
16113 }
16114
16115 pT->aMap = aNew;
16116 pT->nMapAlloc = nNew;
16117 }
16118
16119 pT->aMap[pT->nMap].iRowid = iRowid;
16120 pT->aMap[pT->nMap].iPos = iPos;
16121 pT->aMap[pT->nMap].iIter = iIter;
16122 pT->aMap[pT->nMap].nByte = nByte;
16123 pT->nMap++;
16124 }
16125}
16126
16127/*
16128** Sort the contents of the pT->aMap[] array.
16129**
16130** The sorting algorithm requires a malloc(). If this fails, an error code
16131** is left in Fts5Index.rc before returning.
16132*/
16133static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){
16134 Fts5TokenDataMap *aTmp = 0;
16135 i64 nByte = pT->nMap * sizeof(Fts5TokenDataMap);
16136
16137 aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte);
16138 if( aTmp ){
16139 Fts5TokenDataMap *a1 = pT->aMap;
16140 Fts5TokenDataMap *a2 = aTmp;
16141 i64 nHalf;
16142
16143 for(nHalf=1; nHalf<pT->nMap; nHalf=nHalf*2){
16144 int i1;
16145 for(i1=0; i1<pT->nMap; i1+=(nHalf*2)){
16146 int n1 = MIN(nHalf, pT->nMap-i1)(((nHalf) < (pT->nMap-i1)) ? (nHalf) : (pT->nMap-i1)
);
16147 int n2 = MIN(nHalf, pT->nMap-i1-n1)(((nHalf) < (pT->nMap-i1-n1)) ? (nHalf) : (pT->nMap-
i1-n1));
16148 fts5TokendataMerge(&a1[i1], n1, &a1[i1+n1], n2, &a2[i1]);
16149 }
16150 SWAPVAL(Fts5TokenDataMap*, a1, a2){ Fts5TokenDataMap* tmp; tmp=a1; a1=a2; a2=tmp; };
16151 }
16152
16153 if( a1!=pT->aMap ){
16154 memcpy(pT->aMap, a1, pT->nMap*sizeof(Fts5TokenDataMap));
16155 }
16156 sqlite3_freesqlite3_api->free(aTmp);
16157
16158#ifdef SQLITE_DEBUG
16159 {
16160 int ii;
16161 for(ii=1; ii<pT->nMap; ii++){
16162 Fts5TokenDataMap *p1 = &pT->aMap[ii-1];
16163 Fts5TokenDataMap *p2 = &pT->aMap[ii];
16164 assert( p1->iRowid<p2->iRowid((void) (0))
16165 || (p1->iRowid==p2->iRowid && p1->iPos<=p2->iPos)((void) (0))
16166 )((void) (0));
16167 }
16168 }
16169#endif
16170 }
16171}
16172
16173/*
16174** Delete an Fts5TokenDataIter structure and its contents.
16175*/
16176static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){
16177 if( pSet ){
16178 int ii;
16179 for(ii=0; ii<pSet->nIter; ii++){
16180 fts5MultiIterFree(pSet->apIter[ii]);
16181 }
16182 fts5BufferFree(&pSet->terms)sqlite3Fts5BufferFree(&pSet->terms);
16183 sqlite3_freesqlite3_api->free(pSet->aPoslistReader);
16184 sqlite3_freesqlite3_api->free(pSet->aMap);
16185 sqlite3_freesqlite3_api->free(pSet);
16186 }
16187}
16188
16189
16190/*
16191** fts5VisitEntries() context object used by fts5SetupPrefixIterTokendata()
16192** to pass data to prefixIterSetupTokendataCb().
16193*/
16194typedef struct TokendataSetupCtx TokendataSetupCtx;
16195struct TokendataSetupCtx {
16196 Fts5TokenDataIter *pT; /* Object being populated with mappings */
16197 int iTermOff; /* Offset of current term in terms.p[] */
16198 int nTermByte; /* Size of current term in bytes */
16199};
16200
16201/*
16202** fts5VisitEntries() callback used by fts5SetupPrefixIterTokendata(). This
16203** callback adds an entry to the Fts5TokenDataIter.aMap[] array for each
16204** position in the current position-list. It doesn't matter that some of
16205** these may be out of order - they will be sorted later.
16206*/
16207static void prefixIterSetupTokendataCb(
16208 Fts5Index *p,
16209 void *pCtx,
16210 Fts5Iter *p1,
16211 const u8 *pNew,
16212 int nNew
16213){
16214 TokendataSetupCtx *pSetup = (TokendataSetupCtx*)pCtx;
16215 int iPosOff = 0;
16216 i64 iPos = 0;
16217
16218 if( pNew ){
16219 pSetup->nTermByte = nNew-1;
16220 pSetup->iTermOff = pSetup->pT->terms.n;
16221 fts5BufferAppendBlob(&p->rc, &pSetup->pT->terms, nNew-1, pNew+1)sqlite3Fts5BufferAppendBlob(&p->rc,&pSetup->pT->
terms,nNew-1,pNew+1);
16222 }
16223
16224 while( 0==sqlite3Fts5PoslistNext64(
16225 p1->base.pData, p1->base.nData, &iPosOff, &iPos
16226 ) ){
16227 fts5TokendataIterAppendMap(p,
16228 pSetup->pT, pSetup->iTermOff, pSetup->nTermByte, p1->base.iRowid, iPos
16229 );
16230 }
16231}
16232
16233
16234/*
16235** Context object passed by fts5SetupPrefixIter() to fts5VisitEntries().
16236*/
16237typedef struct PrefixSetupCtx PrefixSetupCtx;
16238struct PrefixSetupCtx {
16239 void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*);
16240 void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*);
16241 i64 iLastRowid;
16242 int nMerge;
16243 Fts5Buffer *aBuf;
16244 int nBuf;
16245 Fts5Buffer doclist;
16246 TokendataSetupCtx *pTokendata;
16247};
16248
16249/*
16250** fts5VisitEntries() callback used by fts5SetupPrefixIter()
16251*/
16252static void prefixIterSetupCb(
16253 Fts5Index *p,
16254 void *pCtx,
16255 Fts5Iter *p1,
16256 const u8 *pNew,
16257 int nNew
16258){
16259 PrefixSetupCtx *pSetup = (PrefixSetupCtx*)pCtx;
16260 const int nMerge = pSetup->nMerge;
16261
16262 if( p1->base.nData>0 ){
16263 if( p1->base.iRowid<=pSetup->iLastRowid && pSetup->doclist.n>0 ){
16264 int i;
16265 for(i=0; p->rc==SQLITE_OK0 && pSetup->doclist.n; i++){
16266 int i1 = i*nMerge;
16267 int iStore;
16268 assert( i1+nMerge<=pSetup->nBuf )((void) (0));
16269 for(iStore=i1; iStore<i1+nMerge; iStore++){
16270 if( pSetup->aBuf[iStore].n==0 ){
16271 fts5BufferSwap(&pSetup->doclist, &pSetup->aBuf[iStore]);
16272 fts5BufferZero(&pSetup->doclist)sqlite3Fts5BufferZero(&pSetup->doclist);
16273 break;
16274 }
16275 }
16276 if( iStore==i1+nMerge ){
16277 pSetup->xMerge(p, &pSetup->doclist, nMerge, &pSetup->aBuf[i1]);
16278 for(iStore=i1; iStore<i1+nMerge; iStore++){
16279 fts5BufferZero(&pSetup->aBuf[iStore])sqlite3Fts5BufferZero(&pSetup->aBuf[iStore]);
16280 }
16281 }
16282 }
16283 pSetup->iLastRowid = 0;
16284 }
16285
16286 pSetup->xAppend(
16287 p, (u64)p1->base.iRowid-(u64)pSetup->iLastRowid, p1, &pSetup->doclist
16288 );
16289 pSetup->iLastRowid = p1->base.iRowid;
16290 }
16291
16292 if( pSetup->pTokendata ){
16293 prefixIterSetupTokendataCb(p, (void*)pSetup->pTokendata, p1, pNew, nNew);
16294 }
16295}
16296
16297static void fts5SetupPrefixIter(
16298 Fts5Index *p, /* Index to read from */
16299 int bDesc, /* True for "ORDER BY rowid DESC" */
16300 int iIdx, /* Index to scan for data */
16301 u8 *pToken, /* Buffer containing prefix to match */
16302 int nToken, /* Size of buffer pToken in bytes */
16303 Fts5Colset *pColset, /* Restrict matches to these columns */
16304 Fts5Iter **ppIter /* OUT: New iterator */
16305){
16306 Fts5Structure *pStruct;
16307 PrefixSetupCtx s;
16308 TokendataSetupCtx s2;
16309
16310 memset(&s, 0, sizeof(s));
16311 memset(&s2, 0, sizeof(s2));
16312
16313 s.nMerge = 1;
16314 s.iLastRowid = 0;
16315 s.nBuf = 32;
16316 if( iIdx==0
16317 && p->pConfig->eDetail==FTS5_DETAIL_FULL0
16318 && p->pConfig->bPrefixInsttoken
16319 ){
16320 s.pTokendata = &s2;
16321 s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, SZ_FTS5TOKENDATAITER(1)(__builtin_offsetof(Fts5TokenDataIter, apIter) + (1)*sizeof(Fts5Iter
)));
16322 }
16323
16324 if( p->pConfig->eDetail==FTS5_DETAIL_NONE1 ){
16325 s.xMerge = fts5MergeRowidLists;
16326 s.xAppend = fts5AppendRowid;
16327 }else{
16328 s.nMerge = FTS5_MERGE_NLIST16-1;
16329 s.nBuf = s.nMerge*8; /* Sufficient to merge (16^8)==(2^32) lists */
16330 s.xMerge = fts5MergePrefixLists;
16331 s.xAppend = fts5AppendPoslist;
16332 }
16333
16334 s.aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*s.nBuf);
16335 pStruct = fts5StructureRead(p);
16336 assert( p->rc!=SQLITE_OK || (s.aBuf && pStruct) )((void) (0));
16337
16338 if( p->rc==SQLITE_OK0 ){
16339 void *pCtx = (void*)&s;
16340 int i;
16341 Fts5Data *pData;
16342
16343 /* If iIdx is non-zero, then it is the number of a prefix-index for
16344 ** prefixes 1 character longer than the prefix being queried for. That
16345 ** index contains all the doclists required, except for the one
16346 ** corresponding to the prefix itself. That one is extracted from the
16347 ** main term index here. */
16348 if( iIdx!=0 ){
16349 pToken[0] = FTS5_MAIN_PREFIX'0';
16350 fts5VisitEntries(p, pColset, pToken, nToken, 0, prefixIterSetupCb, pCtx);
16351 }
16352
16353 pToken[0] = FTS5_MAIN_PREFIX'0' + iIdx;
16354 fts5VisitEntries(p, pColset, pToken, nToken, 1, prefixIterSetupCb, pCtx);
16355
16356 assert( (s.nBuf%s.nMerge)==0 )((void) (0));
16357 for(i=0; i<s.nBuf; i+=s.nMerge){
16358 int iFree;
16359 if( p->rc==SQLITE_OK0 ){
16360 s.xMerge(p, &s.doclist, s.nMerge, &s.aBuf[i]);
16361 }
16362 for(iFree=i; iFree<i+s.nMerge; iFree++){
16363 fts5BufferFree(&s.aBuf[iFree])sqlite3Fts5BufferFree(&s.aBuf[iFree]);
16364 }
16365 }
16366
16367 pData = fts5IdxMalloc(p, sizeof(*pData)
16368 + ((i64)s.doclist.n)+FTS5_DATA_ZERO_PADDING8);
16369 assert( pData!=0 || p->rc!=SQLITE_OK )((void) (0));
16370 if( pData ){
16371 pData->p = (u8*)&pData[1];
16372 pData->nn = pData->szLeaf = s.doclist.n;
16373 if( s.doclist.n ) memcpy(pData->p, s.doclist.p, s.doclist.n);
16374 fts5MultiIterNew2(p, pData, bDesc, ppIter);
16375 }
16376
16377 assert( (*ppIter)!=0 || p->rc!=SQLITE_OK )((void) (0));
16378 if( p->rc==SQLITE_OK0 && s.pTokendata ){
16379 fts5TokendataIterSortMap(p, s2.pT);
16380 (*ppIter)->pTokenDataIter = s2.pT;
16381 s2.pT = 0;
16382 }
16383 }
16384
16385 fts5TokendataIterDelete(s2.pT);
16386 fts5BufferFree(&s.doclist)sqlite3Fts5BufferFree(&s.doclist);
16387 fts5StructureRelease(pStruct);
16388 sqlite3_freesqlite3_api->free(s.aBuf);
16389}
16390
16391
16392/*
16393** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
16394** to the document with rowid iRowid.
16395*/
16396static int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
16397 assert( p->rc==SQLITE_OK )((void) (0));
16398
16399 /* Allocate the hash table if it has not already been allocated */
16400 if( p->pHash==0 ){
16401 p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData);
16402 }
16403
16404 /* Flush the hash table to disk if required */
16405 if( iRowid<p->iWriteRowid
16406 || (iRowid==p->iWriteRowid && p->bDelete==0)
16407 || (p->nPendingData > p->pConfig->nHashSize)
16408 ){
16409 fts5IndexFlush(p);
16410 }
16411
16412 p->iWriteRowid = iRowid;
16413 p->bDelete = bDelete;
16414 if( bDelete==0 ){
16415 p->nPendingRow++;
16416 }
16417 return fts5IndexReturn(p);
16418}
16419
16420/*
16421** Commit data to disk.
16422*/
16423static int sqlite3Fts5IndexSync(Fts5Index *p){
16424 assert( p->rc==SQLITE_OK )((void) (0));
16425 fts5IndexFlush(p);
16426 fts5IndexCloseReader(p);
16427 return fts5IndexReturn(p);
16428}
16429
16430/*
16431** Discard any data stored in the in-memory hash tables. Do not write it
16432** to the database. Additionally, assume that the contents of the %_data
16433** table may have changed on disk. So any in-memory caches of %_data
16434** records must be invalidated.
16435*/
16436static int sqlite3Fts5IndexRollback(Fts5Index *p){
16437 fts5IndexCloseReader(p);
16438 fts5IndexDiscardData(p);
16439 fts5StructureInvalidate(p);
16440 return fts5IndexReturn(p);
16441}
16442
16443/*
16444** The %_data table is completely empty when this function is called. This
16445** function populates it with the initial structure objects for each index,
16446** and the initial version of the "averages" record (a zero-byte blob).
16447*/
16448static int sqlite3Fts5IndexReinit(Fts5Index *p){
16449 Fts5Structure *pTmp;
16450 union {
16451 Fts5Structure sFts;
16452 u8 tmpSpace[SZ_FTS5STRUCTURE(1)(__builtin_offsetof(Fts5Structure, aLevel) + (1)*sizeof(Fts5StructureLevel
))];
16453 } uFts;
16454 fts5StructureInvalidate(p);
16455 fts5IndexDiscardData(p);
16456 pTmp = &uFts.sFts;
16457 memset(uFts.tmpSpace, 0, sizeof(uFts.tmpSpace));
16458 if( p->pConfig->bContentlessDelete ){
16459 pTmp->nOriginCntr = 1;
16460 }
16461 fts5DataWrite(p, FTS5_AVERAGES_ROWID1, (const u8*)"", 0);
16462 fts5StructureWrite(p, pTmp);
16463 return fts5IndexReturn(p);
16464}
16465
16466/*
16467** Open a new Fts5Index handle. If the bCreate argument is true, create
16468** and initialize the underlying %_data table.
16469**
16470** If successful, set *pp to point to the new object and return SQLITE_OK.
16471** Otherwise, set *pp to NULL and return an SQLite error code.
16472*/
16473static int sqlite3Fts5IndexOpen(
16474 Fts5Config *pConfig,
16475 int bCreate,
16476 Fts5Index **pp,
16477 char **pzErr
16478){
16479 int rc = SQLITE_OK0;
16480 Fts5Index *p; /* New object */
16481
16482 *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index));
16483 if( rc==SQLITE_OK0 ){
16484 p->pConfig = pConfig;
16485 p->nWorkUnit = FTS5_WORK_UNIT64;
16486 p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName);
16487 if( p->zDataTbl && bCreate ){
16488 rc = sqlite3Fts5CreateTable(
16489 pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
16490 );
16491 if( rc==SQLITE_OK0 ){
16492 rc = sqlite3Fts5CreateTable(pConfig, "idx",
16493 "segid, term, pgno, PRIMARY KEY(segid, term)",
16494 1, pzErr
16495 );
16496 }
16497 if( rc==SQLITE_OK0 ){
16498 rc = sqlite3Fts5IndexReinit(p);
16499 }
16500 }
16501 }
16502
16503 assert( rc!=SQLITE_OK || p->rc==SQLITE_OK )((void) (0));
16504 if( rc ){
16505 sqlite3Fts5IndexClose(p);
16506 *pp = 0;
16507 }
16508 return rc;
16509}
16510
16511/*
16512** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
16513*/
16514static int sqlite3Fts5IndexClose(Fts5Index *p){
16515 int rc = SQLITE_OK0;
16516 if( p ){
16517 assert( p->pReader==0 )((void) (0));
16518 fts5StructureInvalidate(p);
16519 sqlite3_finalizesqlite3_api->finalize(p->pWriter);
16520 sqlite3_finalizesqlite3_api->finalize(p->pDeleter);
16521 sqlite3_finalizesqlite3_api->finalize(p->pIdxWriter);
16522 sqlite3_finalizesqlite3_api->finalize(p->pIdxDeleter);
16523 sqlite3_finalizesqlite3_api->finalize(p->pIdxSelect);
16524 sqlite3_finalizesqlite3_api->finalize(p->pIdxNextSelect);
16525 sqlite3_finalizesqlite3_api->finalize(p->pDataVersion);
16526 sqlite3_finalizesqlite3_api->finalize(p->pDeleteFromIdx);
16527 sqlite3Fts5HashFree(p->pHash);
16528 sqlite3_freesqlite3_api->free(p->zDataTbl);
16529 sqlite3_freesqlite3_api->free(p);
16530 }
16531 return rc;
16532}
16533
16534/*
16535** Argument p points to a buffer containing utf-8 text that is n bytes in
16536** size. Return the number of bytes in the nChar character prefix of the
16537** buffer, or 0 if there are less than nChar characters in total.
16538*/
16539static int sqlite3Fts5IndexCharlenToBytelen(
16540 const char *p,
16541 int nByte,
16542 int nChar
16543){
16544 int n = 0;
16545 int i;
16546 for(i=0; i<nChar; i++){
16547 if( n>=nByte ) return 0; /* Input contains fewer than nChar chars */
16548 if( (unsigned char)p[n++]>=0xc0 ){
16549 if( n>=nByte ) return 0;
16550 while( (p[n] & 0xc0)==0x80 ){
16551 n++;
16552 if( n>=nByte ){
16553 if( i+1==nChar ) break;
16554 return 0;
16555 }
16556 }
16557 }
16558 }
16559 return n;
16560}
16561
16562/*
16563** pIn is a UTF-8 encoded string, nIn bytes in size. Return the number of
16564** unicode characters in the string.
16565*/
16566static int fts5IndexCharlen(const char *pIn, int nIn){
16567 int nChar = 0;
16568 int i = 0;
16569 while( i<nIn ){
16570 if( (unsigned char)pIn[i++]>=0xc0 ){
16571 while( i<nIn && (pIn[i] & 0xc0)==0x80 ) i++;
16572 }
16573 nChar++;
16574 }
16575 return nChar;
16576}
16577
16578/*
16579** Insert or remove data to or from the index. Each time a document is
16580** added to or removed from the index, this function is called one or more
16581** times.
16582**
16583** For an insert, it must be called once for each token in the new document.
16584** If the operation is a delete, it must be called (at least) once for each
16585** unique token in the document with an iCol value less than zero. The iPos
16586** argument is ignored for a delete.
16587*/
16588static int sqlite3Fts5IndexWrite(
16589 Fts5Index *p, /* Index to write to */
16590 int iCol, /* Column token appears in (-ve -> delete) */
16591 int iPos, /* Position of token within column */
16592 const char *pToken, int nToken /* Token to add or remove to or from index */
16593){
16594 int i; /* Used to iterate through indexes */
16595 int rc = SQLITE_OK0; /* Return code */
16596 Fts5Config *pConfig = p->pConfig;
16597
16598 assert( p->rc==SQLITE_OK )((void) (0));
16599 assert( (iCol<0)==p->bDelete )((void) (0));
16600
16601 /* Add the entry to the main terms index. */
16602 rc = sqlite3Fts5HashWrite(
16603 p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX'0', pToken, nToken
16604 );
16605
16606 for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK0; i++){
16607 const int nChar = pConfig->aPrefix[i];
16608 int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
16609 if( nByte ){
16610 rc = sqlite3Fts5HashWrite(p->pHash,
16611 p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX'0'+i+1), pToken,
16612 nByte
16613 );
16614 }
16615 }
16616
16617 return rc;
16618}
16619
16620/*
16621** pToken points to a buffer of size nToken bytes containing a search
16622** term, including the index number at the start, used on a tokendata=1
16623** table. This function returns true if the term in buffer pBuf matches
16624** token pToken/nToken.
16625*/
16626static int fts5IsTokendataPrefix(
16627 Fts5Buffer *pBuf,
16628 const u8 *pToken,
16629 int nToken
16630){
16631 return (
16632 pBuf->n>=nToken
16633 && 0==memcmp(pBuf->p, pToken, nToken)
16634 && (pBuf->n==nToken || pBuf->p[nToken]==0x00)
16635 );
16636}
16637
16638/*
16639** Ensure the segment-iterator passed as the only argument points to EOF.
16640*/
16641static void fts5SegIterSetEOF(Fts5SegIter *pSeg){
16642 fts5DataRelease(pSeg->pLeaf);
16643 pSeg->pLeaf = 0;
16644}
16645
16646static void fts5IterClose(Fts5IndexIter *pIndexIter){
16647 if( pIndexIter ){
16648 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
16649 Fts5Index *pIndex = pIter->pIndex;
16650 fts5TokendataIterDelete(pIter->pTokenDataIter);
16651 fts5MultiIterFree(pIter);
16652 fts5IndexCloseReader(pIndex);
16653 }
16654}
16655
16656/*
16657** This function appends iterator pAppend to Fts5TokenDataIter pIn and
16658** returns the result.
16659*/
16660static Fts5TokenDataIter *fts5AppendTokendataIter(
16661 Fts5Index *p, /* Index object (for error code) */
16662 Fts5TokenDataIter *pIn, /* Current Fts5TokenDataIter struct */
16663 Fts5Iter *pAppend /* Append this iterator */
16664){
16665 Fts5TokenDataIter *pRet = pIn;
16666
16667 if( p->rc==SQLITE_OK0 ){
16668 if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){
16669 i64 nAlloc = pIn ? pIn->nIterAlloc*2 : 16;
16670 i64 nByte = SZ_FTS5TOKENDATAITER(nAlloc+1)(__builtin_offsetof(Fts5TokenDataIter, apIter) + (nAlloc+1)*sizeof
(Fts5Iter));
16671 Fts5TokenDataIter *pNew;
16672 pNew = (Fts5TokenDataIter*)sqlite3_realloc64sqlite3_api->realloc64(pIn, nByte);
16673
16674 if( pNew==0 ){
16675 p->rc = SQLITE_NOMEM7;
16676 }else{
16677 if( pIn==0 ) memset(pNew, 0, nByte);
16678 pRet = pNew;
16679 pNew->nIterAlloc = nAlloc;
16680 }
16681 }
16682 }
16683 if( p->rc ){
16684 fts5IterClose((Fts5IndexIter*)pAppend);
16685 }else{
16686 pRet->apIter[pRet->nIter++] = pAppend;
16687 }
16688 assert( pRet==0 || pRet->nIter<=pRet->nIterAlloc )((void) (0));
16689
16690 return pRet;
16691}
16692
16693/*
16694** The iterator passed as the only argument must be a tokendata=1 iterator
16695** (pIter->pTokenDataIter!=0). This function sets the iterator output
16696** variables (pIter->base.*) according to the contents of the current
16697** row.
16698*/
16699static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){
16700 int ii;
16701 int nHit = 0;
16702 i64 iRowid = SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32)));
16703 int iMin = 0;
16704
16705 Fts5TokenDataIter *pT = pIter->pTokenDataIter;
16706
16707 pIter->base.nData = 0;
16708 pIter->base.pData = 0;
16709
16710 for(ii=0; ii<pT->nIter; ii++){
16711 Fts5Iter *p = pT->apIter[ii];
16712 if( p->base.bEof==0 ){
16713 if( nHit==0 || p->base.iRowid<iRowid ){
16714 iRowid = p->base.iRowid;
16715 nHit = 1;
16716 pIter->base.pData = p->base.pData;
16717 pIter->base.nData = p->base.nData;
16718 iMin = ii;
16719 }else if( p->base.iRowid==iRowid ){
16720 nHit++;
16721 }
16722 }
16723 }
16724
16725 if( nHit==0 ){
16726 pIter->base.bEof = 1;
16727 }else{
16728 int eDetail = pIter->pIndex->pConfig->eDetail;
16729 pIter->base.bEof = 0;
16730 pIter->base.iRowid = iRowid;
16731
16732 if( nHit==1 && eDetail==FTS5_DETAIL_FULL0 ){
16733 fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, 0, iRowid, -1);
16734 }else
16735 if( nHit>1 && eDetail!=FTS5_DETAIL_NONE1 ){
16736 int nReader = 0;
16737 int nByte = 0;
16738 i64 iPrev = 0;
16739
16740 /* Allocate array of iterators if they are not already allocated. */
16741 if( pT->aPoslistReader==0 ){
16742 pT->aPoslistReader = (Fts5PoslistReader*)sqlite3Fts5MallocZero(
16743 &pIter->pIndex->rc,
16744 pT->nIter * (sizeof(Fts5PoslistReader) + sizeof(int))
16745 );
16746 if( pT->aPoslistReader==0 ) return;
16747 pT->aPoslistToIter = (int*)&pT->aPoslistReader[pT->nIter];
16748 }
16749
16750 /* Populate an iterator for each poslist that will be merged */
16751 for(ii=0; ii<pT->nIter; ii++){
16752 Fts5Iter *p = pT->apIter[ii];
16753 if( iRowid==p->base.iRowid ){
16754 pT->aPoslistToIter[nReader] = ii;
16755 sqlite3Fts5PoslistReaderInit(
16756 p->base.pData, p->base.nData, &pT->aPoslistReader[nReader++]
16757 );
16758 nByte += p->base.nData;
16759 }
16760 }
16761
16762 /* Ensure the output buffer is large enough */
16763 if( fts5BufferGrow(&pIter->pIndex->rc, &pIter->poslist, nByte+nHit*10)( (u32)((&pIter->poslist)->n) + (u32)(nByte+nHit*10
) <= (u32)((&pIter->poslist)->nSpace) ? 0 : sqlite3Fts5BufferSize
((&pIter->pIndex->rc),(&pIter->poslist),(nByte
+nHit*10)+(&pIter->poslist)->n) ) ){
16764 return;
16765 }
16766
16767 /* Ensure the token-mapping is large enough */
16768 if( eDetail==FTS5_DETAIL_FULL0 && pT->nMapAlloc<(pT->nMap + nByte) ){
16769 i64 nNew = (pT->nMapAlloc + nByte) * 2;
16770 Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc64sqlite3_api->realloc64(
16771 pT->aMap, nNew*sizeof(Fts5TokenDataMap)
16772 );
16773 if( aNew==0 ){
16774 pIter->pIndex->rc = SQLITE_NOMEM7;
16775 return;
16776 }
16777 pT->aMap = aNew;
16778 pT->nMapAlloc = nNew;
16779 }
16780
16781 pIter->poslist.n = 0;
16782
16783 while( 1 ){
16784 i64 iMinPos = LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32));
16785
16786 /* Find smallest position */
16787 iMin = 0;
16788 for(ii=0; ii<nReader; ii++){
16789 Fts5PoslistReader *pReader = &pT->aPoslistReader[ii];
16790 if( pReader->bEof==0 ){
16791 if( pReader->iPos<iMinPos ){
16792 iMinPos = pReader->iPos;
16793 iMin = ii;
16794 }
16795 }
16796 }
16797
16798 /* If all readers were at EOF, break out of the loop. */
16799 if( iMinPos==LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)) ) break;
16800
16801 sqlite3Fts5PoslistSafeAppend(&pIter->poslist, &iPrev, iMinPos);
16802 sqlite3Fts5PoslistReaderNext(&pT->aPoslistReader[iMin]);
16803
16804 if( eDetail==FTS5_DETAIL_FULL0 ){
16805 pT->aMap[pT->nMap].iPos = iMinPos;
16806 pT->aMap[pT->nMap].iIter = pT->aPoslistToIter[iMin];
16807 pT->aMap[pT->nMap].iRowid = iRowid;
16808 pT->nMap++;
16809 }
16810 }
16811
16812 pIter->base.pData = pIter->poslist.p;
16813 pIter->base.nData = pIter->poslist.n;
16814 }
16815 }
16816}
16817
16818/*
16819** The iterator passed as the only argument must be a tokendata=1 iterator
16820** (pIter->pTokenDataIter!=0). This function advances the iterator. If
16821** argument bFrom is false, then the iterator is advanced to the next
16822** entry. Or, if bFrom is true, it is advanced to the first entry with
16823** a rowid of iFrom or greater.
16824*/
16825static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){
16826 int ii;
16827 Fts5TokenDataIter *pT = pIter->pTokenDataIter;
16828 Fts5Index *pIndex = pIter->pIndex;
16829
16830 for(ii=0; ii<pT->nIter; ii++){
16831 Fts5Iter *p = pT->apIter[ii];
16832 if( p->base.bEof==0
16833 && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowid<iFrom))
16834 ){
16835 fts5MultiIterNext(pIndex, p, bFrom, iFrom);
16836 while( bFrom && p->base.bEof==0
16837 && p->base.iRowid<iFrom
16838 && pIndex->rc==SQLITE_OK0
16839 ){
16840 fts5MultiIterNext(pIndex, p, 0, 0);
16841 }
16842 }
16843 }
16844
16845 if( pIndex->rc==SQLITE_OK0 ){
16846 fts5IterSetOutputsTokendata(pIter);
16847 }
16848}
16849
16850/*
16851** If the segment-iterator passed as the first argument is at EOF, then
16852** set pIter->term to a copy of buffer pTerm.
16853*/
16854static void fts5TokendataSetTermIfEof(Fts5Iter *pIter, Fts5Buffer *pTerm){
16855 if( pIter && pIter->aSeg[0].pLeaf==0 ){
16856 fts5BufferSet(&pIter->pIndex->rc, &pIter->aSeg[0].term, pTerm->n, pTerm->p)sqlite3Fts5BufferSet(&pIter->pIndex->rc,&pIter->
aSeg[0].term,pTerm->n,pTerm->p);
16857 }
16858}
16859
16860/*
16861** This function sets up an iterator to use for a non-prefix query on a
16862** tokendata=1 table.
16863*/
16864static Fts5Iter *fts5SetupTokendataIter(
16865 Fts5Index *p, /* FTS index to query */
16866 const u8 *pToken, /* Buffer containing query term */
16867 int nToken, /* Size of buffer pToken in bytes */
16868 Fts5Colset *pColset /* Colset to filter on */
16869){
16870 Fts5Iter *pRet = 0;
16871 Fts5TokenDataIter *pSet = 0;
16872 Fts5Structure *pStruct = 0;
16873 const int flags = FTS5INDEX_QUERY_SCANONETERM0x0100 | FTS5INDEX_QUERY_SCAN0x0008;
16874
16875 Fts5Buffer bSeek = {0, 0, 0};
16876 Fts5Buffer *pSmall = 0;
16877
16878 fts5IndexFlush(p);
16879 pStruct = fts5StructureRead(p);
16880
16881 while( p->rc==SQLITE_OK0 ){
16882 Fts5Iter *pPrev = pSet ? pSet->apIter[pSet->nIter-1] : 0;
16883 Fts5Iter *pNew = 0;
16884 Fts5SegIter *pNewIter = 0;
16885 Fts5SegIter *pPrevIter = 0;
16886
16887 int iLvl, iSeg, ii;
16888
16889 pNew = fts5MultiIterAlloc(p, pStruct->nSegment);
16890 if( pSmall ){
16891 fts5BufferSet(&p->rc, &bSeek, pSmall->n, pSmall->p)sqlite3Fts5BufferSet(&p->rc,&bSeek,pSmall->n,pSmall
->p);
16892 fts5BufferAppendBlob(&p->rc, &bSeek, 1, (const u8*)"\0")sqlite3Fts5BufferAppendBlob(&p->rc,&bSeek,1,(const
u8*)"\0");
16893 }else{
16894 fts5BufferSet(&p->rc, &bSeek, nToken, pToken)sqlite3Fts5BufferSet(&p->rc,&bSeek,nToken,pToken);
16895 }
16896 if( p->rc ){
16897 fts5IterClose((Fts5IndexIter*)pNew);
16898 break;
16899 }
16900
16901 pNewIter = &pNew->aSeg[0];
16902 pPrevIter = (pPrev ? &pPrev->aSeg[0] : 0);
16903 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
16904 for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){
16905 Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
16906 int bDone = 0;
16907
16908 if( pPrevIter ){
16909 if( fts5BufferCompare(pSmall, &pPrevIter->term) ){
16910 memcpy(pNewIter, pPrevIter, sizeof(Fts5SegIter));
16911 memset(pPrevIter, 0, sizeof(Fts5SegIter));
16912 bDone = 1;
16913 }else if( pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf ){
16914 fts5SegIterNextInit(p,(const char*)bSeek.p,bSeek.n-1,pSeg,pNewIter);
16915 bDone = 1;
16916 }
16917 }
16918
16919 if( bDone==0 ){
16920 fts5SegIterSeekInit(p, bSeek.p, bSeek.n, flags, pSeg, pNewIter);
16921 }
16922
16923 if( pPrevIter ){
16924 if( pPrevIter->pTombArray ){
16925 pNewIter->pTombArray = pPrevIter->pTombArray;
16926 pNewIter->pTombArray->nRef++;
16927 }
16928 }else{
16929 fts5SegIterAllocTombstone(p, pNewIter);
16930 }
16931
16932 pNewIter++;
16933 if( pPrevIter ) pPrevIter++;
16934 if( p->rc ) break;
16935 }
16936 }
16937 fts5TokendataSetTermIfEof(pPrev, pSmall);
16938
16939 pNew->bSkipEmpty = 1;
16940 pNew->pColset = pColset;
16941 fts5IterSetOutputCb(&p->rc, pNew);
16942
16943 /* Loop through all segments in the new iterator. Find the smallest
16944 ** term that any segment-iterator points to. Iterator pNew will be
16945 ** used for this term. Also, set any iterator that points to a term that
16946 ** does not match pToken/nToken to point to EOF */
16947 pSmall = 0;
16948 for(ii=0; ii<pNew->nSeg; ii++){
16949 Fts5SegIter *pII = &pNew->aSeg[ii];
16950 if( 0==fts5IsTokendataPrefix(&pII->term, pToken, nToken) ){
16951 fts5SegIterSetEOF(pII);
16952 }
16953 if( pII->pLeaf && (!pSmall || fts5BufferCompare(pSmall, &pII->term)>0) ){
16954 pSmall = &pII->term;
16955 }
16956 }
16957
16958 /* If pSmall is still NULL at this point, then the new iterator does
16959 ** not point to any terms that match the query. So delete it and break
16960 ** out of the loop - all required iterators have been collected. */
16961 if( pSmall==0 ){
16962 fts5IterClose((Fts5IndexIter*)pNew);
16963 break;
16964 }
16965
16966 /* Append this iterator to the set and continue. */
16967 pSet = fts5AppendTokendataIter(p, pSet, pNew);
16968 }
16969
16970 if( p->rc==SQLITE_OK0 && pSet ){
16971 int ii;
16972 for(ii=0; ii<pSet->nIter; ii++){
16973 Fts5Iter *pIter = pSet->apIter[ii];
16974 int iSeg;
16975 for(iSeg=0; iSeg<pIter->nSeg; iSeg++){
16976 pIter->aSeg[iSeg].flags |= FTS5_SEGITER_ONETERM0x01;
16977 }
16978 fts5MultiIterFinishSetup(p, pIter);
16979 }
16980 }
16981
16982 if( p->rc==SQLITE_OK0 ){
16983 pRet = fts5MultiIterAlloc(p, 0);
16984 }
16985 if( pRet ){
16986 pRet->nSeg = 0;
16987 pRet->pTokenDataIter = pSet;
16988 if( pSet ){
16989 fts5IterSetOutputsTokendata(pRet);
16990 }else{
16991 pRet->base.bEof = 1;
16992 }
16993 }else{
16994 fts5TokendataIterDelete(pSet);
16995 }
16996
16997 fts5StructureRelease(pStruct);
16998 fts5BufferFree(&bSeek)sqlite3Fts5BufferFree(&bSeek);
16999 return pRet;
17000}
17001
17002/*
17003** Open a new iterator to iterate though all rowid that match the
17004** specified token or token prefix.
17005*/
17006static int sqlite3Fts5IndexQuery(
17007 Fts5Index *p, /* FTS index to query */
17008 const char *pToken, int nToken, /* Token (or prefix) to query for */
17009 int flags, /* Mask of FTS5INDEX_QUERY_X flags */
17010 Fts5Colset *pColset, /* Match these columns only */
17011 Fts5IndexIter **ppIter /* OUT: New iterator object */
17012){
17013 Fts5Config *pConfig = p->pConfig;
17014 Fts5Iter *pRet = 0;
17015 Fts5Buffer buf = {0, 0, 0};
17016
17017 /* If the QUERY_SCAN flag is set, all other flags must be clear. */
17018 assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN )((void) (0));
17019
17020 if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
17021 int iIdx = 0; /* Index to search */
17022 int iPrefixIdx = 0; /* +1 prefix index */
17023 int bTokendata = pConfig->bTokendata;
17024 assert( buf.p!=0 )((void) (0));
17025 if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken);
17026
17027 /* The NOTOKENDATA flag is set when each token in a tokendata=1 table
17028 ** should be treated individually, instead of merging all those with
17029 ** a common prefix into a single entry. This is used, for example, by
17030 ** queries performed as part of an integrity-check, or by the fts5vocab
17031 ** module. */
17032 if( flags & (FTS5INDEX_QUERY_NOTOKENDATA0x0080|FTS5INDEX_QUERY_SCAN0x0008) ){
17033 bTokendata = 0;
17034 }
17035
17036 /* Figure out which index to search and set iIdx accordingly. If this
17037 ** is a prefix query for which there is no prefix index, set iIdx to
17038 ** greater than pConfig->nPrefix to indicate that the query will be
17039 ** satisfied by scanning multiple terms in the main index.
17040 **
17041 ** If the QUERY_TEST_NOIDX flag was specified, then this must be a
17042 ** prefix-query. Instead of using a prefix-index (if one exists),
17043 ** evaluate the prefix query using the main FTS index. This is used
17044 ** for internal sanity checking by the integrity-check in debug
17045 ** mode only. */
17046#ifdef SQLITE_DEBUG
17047 if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX0x0004) ){
17048 assert( flags & FTS5INDEX_QUERY_PREFIX )((void) (0));
17049 iIdx = 1+pConfig->nPrefix;
17050 }else
17051#endif
17052 if( flags & FTS5INDEX_QUERY_PREFIX0x0001 ){
17053 int nChar = fts5IndexCharlen(pToken, nToken);
17054 for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
17055 int nIdxChar = pConfig->aPrefix[iIdx-1];
17056 if( nIdxChar==nChar ) break;
17057 if( nIdxChar==nChar+1 ) iPrefixIdx = iIdx;
17058 }
17059 }
17060
17061 if( bTokendata && iIdx==0 ){
17062 buf.p[0] = FTS5_MAIN_PREFIX'0';
17063 pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset);
17064 }else if( iIdx<=pConfig->nPrefix ){
17065 /* Straight index lookup */
17066 Fts5Structure *pStruct = fts5StructureRead(p);
17067 buf.p[0] = (u8)(FTS5_MAIN_PREFIX'0' + iIdx);
17068 if( pStruct ){
17069 fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY0x0010,
17070 pColset, buf.p, nToken+1, -1, 0, &pRet
17071 );
17072 fts5StructureRelease(pStruct);
17073 }
17074 }else{
17075 /* Scan multiple terms in the main index for a prefix query. */
17076 int bDesc = (flags & FTS5INDEX_QUERY_DESC0x0002)!=0;
17077 fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet);
17078 if( pRet==0 ){
17079 assert( p->rc!=SQLITE_OK )((void) (0));
17080 }else{
17081 assert( pRet->pColset==0 )((void) (0));
17082 fts5IterSetOutputCb(&p->rc, pRet);
17083 if( p->rc==SQLITE_OK0 ){
17084 Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst];
17085 if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
17086 }
17087 }
17088 }
17089
17090 if( p->rc ){
17091 fts5IterClose((Fts5IndexIter*)pRet);
17092 pRet = 0;
17093 fts5IndexCloseReader(p);
17094 }
17095
17096 *ppIter = (Fts5IndexIter*)pRet;
17097 sqlite3Fts5BufferFree(&buf);
17098 }
17099 return fts5IndexReturn(p);
17100}
17101
17102/*
17103** Return true if the iterator passed as the only argument is at EOF.
17104*/
17105/*
17106** Move to the next matching rowid.
17107*/
17108static int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){
17109 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17110 assert( pIter->pIndex->rc==SQLITE_OK )((void) (0));
17111 if( pIter->nSeg==0 ){
17112 assert( pIter->pTokenDataIter )((void) (0));
17113 fts5TokendataIterNext(pIter, 0, 0);
17114 }else{
17115 fts5MultiIterNext(pIter->pIndex, pIter, 0, 0);
17116 }
17117 return fts5IndexReturn(pIter->pIndex);
17118}
17119
17120/*
17121** Move to the next matching term/rowid. Used by the fts5vocab module.
17122*/
17123static int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){
17124 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17125 Fts5Index *p = pIter->pIndex;
17126
17127 assert( pIter->pIndex->rc==SQLITE_OK )((void) (0));
17128
17129 fts5MultiIterNext(p, pIter, 0, 0);
17130 if( p->rc==SQLITE_OK0 ){
17131 Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
17132 if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX'0' ){
17133 fts5DataRelease(pSeg->pLeaf);
17134 pSeg->pLeaf = 0;
17135 pIter->base.bEof = 1;
17136 }
17137 }
17138
17139 return fts5IndexReturn(pIter->pIndex);
17140}
17141
17142/*
17143** Move to the next matching rowid that occurs at or after iMatch. The
17144** definition of "at or after" depends on whether this iterator iterates
17145** in ascending or descending rowid order.
17146*/
17147static int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){
17148 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17149 if( pIter->nSeg==0 ){
17150 assert( pIter->pTokenDataIter )((void) (0));
17151 fts5TokendataIterNext(pIter, 1, iMatch);
17152 }else{
17153 fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch);
17154 }
17155 return fts5IndexReturn(pIter->pIndex);
17156}
17157
17158/*
17159** Return the current term.
17160*/
17161static const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){
17162 int n;
17163 const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n);
17164 assert_nc( z || n<=1 )((void) (0));
17165 *pn = n-1;
17166 return (z ? &z[1] : 0);
17167}
17168
17169/*
17170** pIter is a prefix query. This function populates pIter->pTokenDataIter
17171** with an Fts5TokenDataIter object containing mappings for all rows
17172** matched by the query.
17173*/
17174static int fts5SetupPrefixIterTokendata(
17175 Fts5Iter *pIter,
17176 const char *pToken, /* Token prefix to search for */
17177 int nToken /* Size of pToken in bytes */
17178){
17179 Fts5Index *p = pIter->pIndex;
17180 Fts5Buffer token = {0, 0, 0};
17181 TokendataSetupCtx ctx;
17182
17183 memset(&ctx, 0, sizeof(ctx));
17184
17185 fts5BufferGrow(&p->rc, &token, nToken+1)( (u32)((&token)->n) + (u32)(nToken+1) <= (u32)((&
token)->nSpace) ? 0 : sqlite3Fts5BufferSize((&p->rc
),(&token),(nToken+1)+(&token)->n) );
17186 assert( token.p!=0 || p->rc!=SQLITE_OK )((void) (0));
17187 ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc,
17188 SZ_FTS5TOKENDATAITER(1)(__builtin_offsetof(Fts5TokenDataIter, apIter) + (1)*sizeof(Fts5Iter
)));
17189
17190 if( p->rc==SQLITE_OK0 ){
17191
17192 /* Fill in the token prefix to search for */
17193 token.p[0] = FTS5_MAIN_PREFIX'0';
17194 memcpy(&token.p[1], pToken, nToken);
17195 token.n = nToken+1;
17196
17197 fts5VisitEntries(
17198 p, 0, token.p, token.n, 1, prefixIterSetupTokendataCb, (void*)&ctx
17199 );
17200
17201 fts5TokendataIterSortMap(p, ctx.pT);
17202 }
17203
17204 if( p->rc==SQLITE_OK0 ){
17205 pIter->pTokenDataIter = ctx.pT;
17206 }else{
17207 fts5TokendataIterDelete(ctx.pT);
17208 }
17209 fts5BufferFree(&token)sqlite3Fts5BufferFree(&token);
17210
17211 return fts5IndexReturn(p);
17212}
17213
17214/*
17215** This is used by xInstToken() to access the token at offset iOff, column
17216** iCol of row iRowid. The token is returned via output variables *ppOut
17217** and *pnOut. The iterator passed as the first argument must be a tokendata=1
17218** iterator (pIter->pTokenDataIter!=0).
17219**
17220** pToken/nToken:
17221*/
17222static int sqlite3Fts5IterToken(
17223 Fts5IndexIter *pIndexIter,
17224 const char *pToken, int nToken,
17225 i64 iRowid,
17226 int iCol,
17227 int iOff,
17228 const char **ppOut, int *pnOut
17229){
17230 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17231 Fts5TokenDataIter *pT = pIter->pTokenDataIter;
17232 i64 iPos = (((i64)iCol)<<32) + iOff;
17233 Fts5TokenDataMap *aMap = 0;
17234 int i1 = 0;
17235 int i2 = 0;
17236 int iTest = 0;
17237
17238 assert( pT || (pToken && pIter->nSeg>0) )((void) (0));
17239 if( pT==0 ){
17240 int rc = fts5SetupPrefixIterTokendata(pIter, pToken, nToken);
17241 if( rc!=SQLITE_OK0 ) return rc;
17242 pT = pIter->pTokenDataIter;
17243 }
17244
17245 i2 = pT->nMap;
17246 aMap = pT->aMap;
17247
17248 while( i2>i1 ){
17249 iTest = (i1 + i2) / 2;
17250
17251 if( aMap[iTest].iRowid<iRowid ){
17252 i1 = iTest+1;
17253 }else if( aMap[iTest].iRowid>iRowid ){
17254 i2 = iTest;
17255 }else{
17256 if( aMap[iTest].iPos<iPos ){
17257 if( aMap[iTest].iPos<0 ){
17258 break;
17259 }
17260 i1 = iTest+1;
17261 }else if( aMap[iTest].iPos>iPos ){
17262 i2 = iTest;
17263 }else{
17264 break;
17265 }
17266 }
17267 }
17268
17269 if( i2>i1 ){
17270 if( pIter->nSeg==0 ){
17271 Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter];
17272 *ppOut = (const char*)pMap->aSeg[0].term.p+1;
17273 *pnOut = pMap->aSeg[0].term.n-1;
17274 }else{
17275 Fts5TokenDataMap *p = &aMap[iTest];
17276 *ppOut = (const char*)&pT->terms.p[p->iIter];
17277 *pnOut = aMap[iTest].nByte;
17278 }
17279 }
17280
17281 return SQLITE_OK0;
17282}
17283
17284/*
17285** Clear any existing entries from the token-map associated with the
17286** iterator passed as the only argument.
17287*/
17288static void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){
17289 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17290 if( pIter && pIter->pTokenDataIter
17291 && (pIter->nSeg==0 || pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL0)
17292 ){
17293 pIter->pTokenDataIter->nMap = 0;
17294 }
17295}
17296
17297/*
17298** Set a token-mapping for the iterator passed as the first argument. This
17299** is used in detail=column or detail=none mode when a token is requested
17300** using the xInstToken() API. In this case the caller tokenizers the
17301** current row and configures the token-mapping via multiple calls to this
17302** function.
17303*/
17304static int sqlite3Fts5IndexIterWriteTokendata(
17305 Fts5IndexIter *pIndexIter,
17306 const char *pToken, int nToken,
17307 i64 iRowid, int iCol, int iOff
17308){
17309 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17310 Fts5TokenDataIter *pT = pIter->pTokenDataIter;
17311 Fts5Index *p = pIter->pIndex;
17312 i64 iPos = (((i64)iCol)<<32) + iOff;
17313
17314 assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL )((void) (0));
17315 assert( pIter->pTokenDataIter || pIter->nSeg>0 )((void) (0));
17316 if( pIter->nSeg>0 ){
17317 /* This is a prefix term iterator. */
17318 if( pT==0 ){
17319 pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc,
17320 SZ_FTS5TOKENDATAITER(1)(__builtin_offsetof(Fts5TokenDataIter, apIter) + (1)*sizeof(Fts5Iter
)));
17321 pIter->pTokenDataIter = pT;
17322 }
17323 if( pT ){
17324 fts5TokendataIterAppendMap(p, pT, pT->terms.n, nToken, iRowid, iPos);
17325 fts5BufferAppendBlob(&p->rc, &pT->terms, nToken, (const u8*)pToken)sqlite3Fts5BufferAppendBlob(&p->rc,&pT->terms,nToken
,(const u8*)pToken);
17326 }
17327 }else{
17328 int ii;
17329 for(ii=0; ii<pT->nIter; ii++){
17330 Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term;
17331 if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break;
17332 }
17333 if( ii<pT->nIter ){
17334 fts5TokendataIterAppendMap(p, pT, ii, 0, iRowid, iPos);
17335 }
17336 }
17337 return fts5IndexReturn(p);
17338}
17339
17340/*
17341** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
17342*/
17343static void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){
17344 if( pIndexIter ){
17345 Fts5Index *pIndex = ((Fts5Iter*)pIndexIter)->pIndex;
17346 fts5IterClose(pIndexIter);
17347 fts5IndexReturn(pIndex);
17348 }
17349}
17350
17351/*
17352** Read and decode the "averages" record from the database.
17353**
17354** Parameter anSize must point to an array of size nCol, where nCol is
17355** the number of user defined columns in the FTS table.
17356*/
17357static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){
17358 int nCol = p->pConfig->nCol;
17359 Fts5Data *pData;
17360
17361 *pnRow = 0;
17362 memset(anSize, 0, sizeof(i64) * nCol);
17363 pData = fts5DataRead(p, FTS5_AVERAGES_ROWID1);
17364 if( p->rc==SQLITE_OK0 && pData->nn ){
17365 int i = 0;
17366 int iCol;
17367 i += fts5GetVarintsqlite3Fts5GetVarint(&pData->p[i], (u64*)pnRow);
17368 for(iCol=0; i<pData->nn && iCol<nCol; iCol++){
17369 i += fts5GetVarintsqlite3Fts5GetVarint(&pData->p[i], (u64*)&anSize[iCol]);
17370 }
17371 }
17372
17373 fts5DataRelease(pData);
17374 return fts5IndexReturn(p);
17375}
17376
17377/*
17378** Replace the current "averages" record with the contents of the buffer
17379** supplied as the second argument.
17380*/
17381static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){
17382 assert( p->rc==SQLITE_OK )((void) (0));
17383 fts5DataWrite(p, FTS5_AVERAGES_ROWID1, pData, nData);
17384 return fts5IndexReturn(p);
17385}
17386
17387/*
17388** Return the total number of blocks this module has read from the %_data
17389** table since it was created.
17390*/
17391static int sqlite3Fts5IndexReads(Fts5Index *p){
17392 return p->nRead;
17393}
17394
17395/*
17396** Set the 32-bit cookie value stored at the start of all structure
17397** records to the value passed as the second argument.
17398**
17399** Return SQLITE_OK if successful, or an SQLite error code if an error
17400** occurs.
17401*/
17402static int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){
17403 int rc; /* Return code */
17404 Fts5Config *pConfig = p->pConfig; /* Configuration object */
17405 u8 aCookie[4]; /* Binary representation of iNew */
17406 sqlite3_blob *pBlob = 0;
17407
17408 assert( p->rc==SQLITE_OK )((void) (0));
17409 sqlite3Fts5Put32(aCookie, iNew);
17410
17411 rc = sqlite3_blob_opensqlite3_api->blob_open(pConfig->db, pConfig->zDb, p->zDataTbl,
17412 "block", FTS5_STRUCTURE_ROWID10, 1, &pBlob
17413 );
17414 if( rc==SQLITE_OK0 ){
17415 sqlite3_blob_writesqlite3_api->blob_write(pBlob, aCookie, 4, 0);
17416 rc = sqlite3_blob_closesqlite3_api->blob_close(pBlob);
17417 }
17418
17419 return rc;
17420}
17421
17422static int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
17423 Fts5Structure *pStruct;
17424 pStruct = fts5StructureRead(p);
17425 fts5StructureRelease(pStruct);
17426 return fts5IndexReturn(p);
17427}
17428
17429/*
17430** Retrieve the origin value that will be used for the segment currently
17431** being accumulated in the in-memory hash table when it is flushed to
17432** disk. If successful, SQLITE_OK is returned and (*piOrigin) set to
17433** the queried value. Or, if an error occurs, an error code is returned
17434** and the final value of (*piOrigin) is undefined.
17435*/
17436static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){
17437 Fts5Structure *pStruct;
17438 pStruct = fts5StructureRead(p);
17439 if( pStruct ){
17440 *piOrigin = pStruct->nOriginCntr;
17441 fts5StructureRelease(pStruct);
17442 }
17443 return fts5IndexReturn(p);
17444}
17445
17446/*
17447** Buffer pPg contains a page of a tombstone hash table - one of nPg pages
17448** associated with the same segment. This function adds rowid iRowid to
17449** the hash table. The caller is required to guarantee that there is at
17450** least one free slot on the page.
17451**
17452** If parameter bForce is false and the hash table is deemed to be full
17453** (more than half of the slots are occupied), then non-zero is returned
17454** and iRowid not inserted. Or, if bForce is true or if the hash table page
17455** is not full, iRowid is inserted and zero returned.
17456*/
17457static int fts5IndexTombstoneAddToPage(
17458 Fts5Data *pPg,
17459 int bForce,
17460 int nPg,
17461 u64 iRowid
17462){
17463 const int szKey = TOMBSTONE_KEYSIZE(pPg)(pPg->p[0]==4 ? 4 : 8);
17464 const int nSlot = TOMBSTONE_NSLOT(pPg)((pPg->nn > 16) ? ((pPg->nn-8) / (pPg->p[0]==4 ? 4
: 8)) : 1);
17465 const int nElem = fts5GetU32(&pPg->p[4]);
17466 int iSlot = (iRowid / nPg) % nSlot;
17467 int nCollide = nSlot;
17468
17469 if( szKey==4 && iRowid>0xFFFFFFFF ) return 2;
17470 if( iRowid==0 ){
17471 pPg->p[1] = 0x01;
17472 return 0;
17473 }
17474
17475 if( bForce==0 && nElem>=(nSlot/2) ){
17476 return 1;
17477 }
17478
17479 fts5PutU32(&pPg->p[4], nElem+1);
17480 if( szKey==4 ){
17481 u32 *aSlot = (u32*)&pPg->p[8];
17482 while( aSlot[iSlot] ){
17483 iSlot = (iSlot + 1) % nSlot;
17484 if( nCollide--==0 ) return 0;
17485 }
17486 fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid);
17487 }else{
17488 u64 *aSlot = (u64*)&pPg->p[8];
17489 while( aSlot[iSlot] ){
17490 iSlot = (iSlot + 1) % nSlot;
17491 if( nCollide--==0 ) return 0;
17492 }
17493 fts5PutU64((u8*)&aSlot[iSlot], iRowid);
17494 }
17495
17496 return 0;
17497}
17498
17499/*
17500** This function attempts to build a new hash containing all the keys
17501** currently in the tombstone hash table for segment pSeg. The new
17502** hash will be stored in the nOut buffers passed in array apOut[].
17503** All pages of the new hash use key-size szKey (4 or 8).
17504**
17505** Return 0 if the hash is successfully rebuilt into the nOut pages.
17506** Or non-zero if it is not (because one page became overfull). In this
17507** case the caller should retry with a larger nOut parameter.
17508**
17509** Parameter pData1 is page iPg1 of the hash table being rebuilt.
17510*/
17511static int fts5IndexTombstoneRehash(
17512 Fts5Index *p,
17513 Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */
17514 Fts5Data *pData1, /* One page of current hash - or NULL */
17515 int iPg1, /* Which page of the current hash is pData1 */
17516 int szKey, /* 4 or 8, the keysize */
17517 int nOut, /* Number of output pages */
17518 Fts5Data **apOut /* Array of output hash pages */
17519){
17520 int ii;
17521 int res = 0;
17522
17523 /* Initialize the headers of all the output pages */
17524 for(ii=0; ii<nOut; ii++){
17525 apOut[ii]->p[0] = szKey;
17526 fts5PutU32(&apOut[ii]->p[4], 0);
17527 }
17528
17529 /* Loop through the current pages of the hash table. */
17530 for(ii=0; res==0 && ii<pSeg->nPgTombstone; ii++){
17531 Fts5Data *pData = 0; /* Page ii of the current hash table */
17532 Fts5Data *pFree = 0; /* Free this at the end of the loop */
17533
17534 if( iPg1==ii ){
17535 pData = pData1;
17536 }else{
17537 pFree = pData = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii)( ((i64)(pSeg->iSegid+(1<<16)) << (31 +5 +1)) +
((i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((
i64)(ii)) ));
17538 }
17539
17540 if( pData ){
17541 int szKeyIn = TOMBSTONE_KEYSIZE(pData)(pData->p[0]==4 ? 4 : 8);
17542 int nSlotIn = (pData->nn - 8) / szKeyIn;
17543 int iIn;
17544 for(iIn=0; iIn<nSlotIn; iIn++){
17545 u64 iVal = 0;
17546
17547 /* Read the value from slot iIn of the input page into iVal. */
17548 if( szKeyIn==4 ){
17549 u32 *aSlot = (u32*)&pData->p[8];
17550 if( aSlot[iIn] ) iVal = fts5GetU32((u8*)&aSlot[iIn]);
17551 }else{
17552 u64 *aSlot = (u64*)&pData->p[8];
17553 if( aSlot[iIn] ) iVal = fts5GetU64((u8*)&aSlot[iIn]);
17554 }
17555
17556 /* If iVal is not 0 at this point, insert it into the new hash table */
17557 if( iVal ){
17558 Fts5Data *pPg = apOut[(iVal % nOut)];
17559 res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal);
17560 if( res ) break;
17561 }
17562 }
17563
17564 /* If this is page 0 of the old hash, copy the rowid-0-flag from the
17565 ** old hash to the new. */
17566 if( ii==0 ){
17567 apOut[0]->p[1] = pData->p[1];
17568 }
17569 }
17570 fts5DataRelease(pFree);
17571 }
17572
17573 return res;
17574}
17575
17576/*
17577** This is called to rebuild the hash table belonging to segment pSeg.
17578** If parameter pData1 is not NULL, then one page of the existing hash table
17579** has already been loaded - pData1, which is page iPg1. The key-size for
17580** the new hash table is szKey (4 or 8).
17581**
17582** If successful, the new hash table is not written to disk. Instead,
17583** output parameter (*pnOut) is set to the number of pages in the new
17584** hash table, and (*papOut) to point to an array of buffers containing
17585** the new page data.
17586**
17587** If an error occurs, an error code is left in the Fts5Index object and
17588** both output parameters set to 0 before returning.
17589*/
17590static void fts5IndexTombstoneRebuild(
17591 Fts5Index *p,
17592 Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */
17593 Fts5Data *pData1, /* One page of current hash - or NULL */
17594 int iPg1, /* Which page of the current hash is pData1 */
17595 int szKey, /* 4 or 8, the keysize */
17596 int *pnOut, /* OUT: Number of output pages */
17597 Fts5Data ***papOut /* OUT: Output hash pages */
17598){
17599 const int MINSLOT = 32;
17600 int nSlotPerPage = MAX(MINSLOT, (p->pConfig->pgsz - 8) / szKey)(((MINSLOT) > ((p->pConfig->pgsz - 8) / szKey)) ? (MINSLOT
) : ((p->pConfig->pgsz - 8) / szKey));
17601 int nSlot = 0; /* Number of slots in each output page */
17602 int nOut = 0;
17603
17604 /* Figure out how many output pages (nOut) and how many slots per
17605 ** page (nSlot). There are three possibilities:
17606 **
17607 ** 1. The hash table does not yet exist. In this case the new hash
17608 ** table will consist of a single page with MINSLOT slots.
17609 **
17610 ** 2. The hash table exists but is currently a single page. In this
17611 ** case an attempt is made to grow the page to accommodate the new
17612 ** entry. The page is allowed to grow up to nSlotPerPage (see above)
17613 ** slots.
17614 **
17615 ** 3. The hash table already consists of more than one page, or of
17616 ** a single page already so large that it cannot be grown. In this
17617 ** case the new hash consists of (nPg*2+1) pages of nSlotPerPage
17618 ** slots each, where nPg is the current number of pages in the
17619 ** hash table.
17620 */
17621 if( pSeg->nPgTombstone==0 ){
17622 /* Case 1. */
17623 nOut = 1;
17624 nSlot = MINSLOT;
17625 }else if( pSeg->nPgTombstone==1 ){
17626 /* Case 2. */
17627 int nElem = (int)fts5GetU32(&pData1->p[4]);
17628 assert( pData1 && iPg1==0 )((void) (0));
17629 nOut = 1;
17630 nSlot = MAX(nElem*4, MINSLOT)(((nElem*4) > (MINSLOT)) ? (nElem*4) : (MINSLOT));
17631 if( nSlot>nSlotPerPage ) nOut = 0;
17632 }
17633 if( nOut==0 ){
17634 /* Case 3. */
17635 nOut = (pSeg->nPgTombstone * 2 + 1);
17636 nSlot = nSlotPerPage;
17637 }
17638
17639 /* Allocate the required array and output pages */
17640 while( 1 ){
17641 int res = 0;
17642 int ii = 0;
17643 int szPage = 0;
17644 Fts5Data **apOut = 0;
17645
17646 /* Allocate space for the new hash table */
17647 assert( nSlot>=MINSLOT )((void) (0));
17648 apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut);
17649 szPage = 8 + nSlot*szKey;
17650 for(ii=0; ii<nOut; ii++){
17651 Fts5Data *pNew = (Fts5Data*)sqlite3Fts5MallocZero(&p->rc,
17652 sizeof(Fts5Data)+szPage
17653 );
17654 if( pNew ){
17655 pNew->nn = szPage;
17656 pNew->p = (u8*)&pNew[1];
17657 apOut[ii] = pNew;
17658 }
17659 }
17660
17661 /* Rebuild the hash table. */
17662 if( p->rc==SQLITE_OK0 ){
17663 res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut);
17664 }
17665 if( res==0 ){
17666 if( p->rc ){
17667 fts5IndexFreeArray(apOut, nOut);
17668 apOut = 0;
17669 nOut = 0;
17670 }
17671 *pnOut = nOut;
17672 *papOut = apOut;
17673 break;
17674 }
17675
17676 /* If control flows to here, it was not possible to rebuild the hash
17677 ** table. Free all buffers and then try again with more pages. */
17678 assert( p->rc==SQLITE_OK )((void) (0));
17679 fts5IndexFreeArray(apOut, nOut);
17680 nSlot = nSlotPerPage;
17681 nOut = nOut*2 + 1;
17682 }
17683}
17684
17685
17686/*
17687** Add a tombstone for rowid iRowid to segment pSeg.
17688*/
17689static void fts5IndexTombstoneAdd(
17690 Fts5Index *p,
17691 Fts5StructureSegment *pSeg,
17692 u64 iRowid
17693){
17694 Fts5Data *pPg = 0;
17695 int iPg = -1;
17696 int szKey = 0;
17697 int nHash = 0;
17698 Fts5Data **apHash = 0;
17699
17700 p->nContentlessDelete++;
17701
17702 if( pSeg->nPgTombstone>0 ){
17703 iPg = iRowid % pSeg->nPgTombstone;
17704 pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg)( ((i64)(pSeg->iSegid+(1<<16)) << (31 +5 +1)) +
((i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((
i64)(iPg)) ));
17705 if( pPg==0 ){
17706 assert( p->rc!=SQLITE_OK )((void) (0));
17707 return;
17708 }
17709
17710 if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){
17711 fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg)( ((i64)(pSeg->iSegid+(1<<16)) << (31 +5 +1)) +
((i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((
i64)(iPg)) ), pPg->p, pPg->nn);
17712 fts5DataRelease(pPg);
17713 return;
17714 }
17715 }
17716
17717 /* Have to rebuild the hash table. First figure out the key-size (4 or 8). */
17718 szKey = pPg ? TOMBSTONE_KEYSIZE(pPg)(pPg->p[0]==4 ? 4 : 8) : 4;
17719 if( iRowid>0xFFFFFFFF ) szKey = 8;
17720
17721 /* Rebuild the hash table */
17722 fts5IndexTombstoneRebuild(p, pSeg, pPg, iPg, szKey, &nHash, &apHash);
17723 assert( p->rc==SQLITE_OK || (nHash==0 && apHash==0) )((void) (0));
17724
17725 /* If all has succeeded, write the new rowid into one of the new hash
17726 ** table pages, then write them all out to disk. */
17727 if( nHash ){
17728 int ii = 0;
17729 fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid);
17730 for(ii=0; ii<nHash; ii++){
17731 i64 iTombstoneRowid = FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii)( ((i64)(pSeg->iSegid+(1<<16)) << (31 +5 +1)) +
((i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((
i64)(ii)) );
17732 fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn);
17733 }
17734 pSeg->nPgTombstone = nHash;
17735 fts5StructureWrite(p, p->pStruct);
17736 }
17737
17738 fts5DataRelease(pPg);
17739 fts5IndexFreeArray(apHash, nHash);
17740}
17741
17742/*
17743** Add iRowid to the tombstone list of the segment or segments that contain
17744** rows from origin iOrigin. Return SQLITE_OK if successful, or an SQLite
17745** error code otherwise.
17746*/
17747static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){
17748 Fts5Structure *pStruct;
17749 pStruct = fts5StructureRead(p);
17750 if( pStruct ){
17751 int bFound = 0; /* True after pSeg->nEntryTombstone incr. */
17752 int iLvl;
17753 for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){
17754 int iSeg;
17755 for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){
17756 Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
17757 if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){
17758 if( bFound==0 ){
17759 pSeg->nEntryTombstone++;
17760 bFound = 1;
17761 }
17762 fts5IndexTombstoneAdd(p, pSeg, iRowid);
17763 }
17764 }
17765 }
17766 fts5StructureRelease(pStruct);
17767 }
17768 return fts5IndexReturn(p);
17769}
17770
17771/*************************************************************************
17772**************************************************************************
17773** Below this point is the implementation of the integrity-check
17774** functionality.
17775*/
17776
17777/*
17778** Return a simple checksum value based on the arguments.
17779*/
17780static u64 sqlite3Fts5IndexEntryCksum(
17781 i64 iRowid,
17782 int iCol,
17783 int iPos,
17784 int iIdx,
17785 const char *pTerm,
17786 int nTerm
17787){
17788 int i;
17789 u64 ret = iRowid;
17790 ret += (ret<<3) + iCol;
17791 ret += (ret<<3) + iPos;
17792 if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX'0' + iIdx);
17793 for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i];
17794 return ret;
17795}
17796
17797#ifdef SQLITE_DEBUG
17798/*
17799** This function is purely an internal test. It does not contribute to
17800** FTS functionality, or even the integrity-check, in any way.
17801**
17802** Instead, it tests that the same set of pgno/rowid combinations are
17803** visited regardless of whether the doclist-index identified by parameters
17804** iSegid/iLeaf is iterated in forwards or reverse order.
17805*/
17806static void fts5TestDlidxReverse(
17807 Fts5Index *p,
17808 int iSegid, /* Segment id to load from */
17809 int iLeaf /* Load doclist-index for this leaf */
17810){
17811 Fts5DlidxIter *pDlidx = 0;
17812 u64 cksum1 = 13;
17813 u64 cksum2 = 13;
17814
17815 for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iLeaf);
17816 fts5DlidxIterEof(p, pDlidx)==0;
17817 fts5DlidxIterNext(p, pDlidx)
17818 ){
17819 i64 iRowid = fts5DlidxIterRowid(pDlidx);
17820 int pgno = fts5DlidxIterPgno(pDlidx);
17821 assert( pgno>iLeaf )((void) (0));
17822 cksum1 += iRowid + ((i64)pgno<<32);
17823 }
17824 fts5DlidxIterFree(pDlidx);
17825 pDlidx = 0;
17826
17827 for(pDlidx=fts5DlidxIterInit(p, 1, iSegid, iLeaf);
17828 fts5DlidxIterEof(p, pDlidx)==0;
17829 fts5DlidxIterPrev(p, pDlidx)
17830 ){
17831 i64 iRowid = fts5DlidxIterRowid(pDlidx);
17832 int pgno = fts5DlidxIterPgno(pDlidx);
17833 assert( fts5DlidxIterPgno(pDlidx)>iLeaf )((void) (0));
17834 cksum2 += iRowid + ((i64)pgno<<32);
17835 }
17836 fts5DlidxIterFree(pDlidx);
17837 pDlidx = 0;
17838
17839 if( p->rc==SQLITE_OK0 && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT(11 | (1<<8));
17840}
17841
17842static int fts5QueryCksum(
17843 Fts5Index *p, /* Fts5 index object */
17844 int iIdx,
17845 const char *z, /* Index key to query for */
17846 int n, /* Size of index key in bytes */
17847 int flags, /* Flags for Fts5IndexQuery */
17848 u64 *pCksum /* IN/OUT: Checksum value */
17849){
17850 int eDetail = p->pConfig->eDetail;
17851 u64 cksum = *pCksum;
17852 Fts5IndexIter *pIter = 0;
17853 int rc = sqlite3Fts5IndexQuery(
17854 p, z, n, (flags | FTS5INDEX_QUERY_NOTOKENDATA0x0080), 0, &pIter
17855 );
17856
17857 while( rc==SQLITE_OK0 && ALWAYS(pIter!=0)(pIter!=0) && 0==sqlite3Fts5IterEof(pIter)((pIter)->bEof) ){
17858 i64 rowid = pIter->iRowid;
17859
17860 if( eDetail==FTS5_DETAIL_NONE1 ){
17861 cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n);
17862 }else{
17863 Fts5PoslistReader sReader;
17864 for(sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &sReader);
17865 sReader.bEof==0;
17866 sqlite3Fts5PoslistReaderNext(&sReader)
17867 ){
17868 int iCol = FTS5_POS2COLUMN(sReader.iPos)(int)((sReader.iPos >> 32) & 0x7FFFFFFF);
17869 int iOff = FTS5_POS2OFFSET(sReader.iPos)(int)(sReader.iPos & 0x7FFFFFFF);
17870 cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
17871 }
17872 }
17873 if( rc==SQLITE_OK0 ){
17874 rc = sqlite3Fts5IterNext(pIter);
17875 }
17876 }
17877 fts5IterClose(pIter);
17878
17879 *pCksum = cksum;
17880 return rc;
17881}
17882
17883/*
17884** Check if buffer z[], size n bytes, contains as series of valid utf-8
17885** encoded codepoints. If so, return 0. Otherwise, if the buffer does not
17886** contain valid utf-8, return non-zero.
17887*/
17888static int fts5TestUtf8(const char *z, int n){
17889 int i = 0;
17890 assert_nc( n>0 )((void) (0));
17891 while( i<n ){
17892 if( (z[i] & 0x80)==0x00 ){
17893 i++;
17894 }else
17895 if( (z[i] & 0xE0)==0xC0 ){
17896 if( i+1>=n || (z[i+1] & 0xC0)!=0x80 ) return 1;
17897 i += 2;
17898 }else
17899 if( (z[i] & 0xF0)==0xE0 ){
17900 if( i+2>=n || (z[i+1] & 0xC0)!=0x80 || (z[i+2] & 0xC0)!=0x80 ) return 1;
17901 i += 3;
17902 }else
17903 if( (z[i] & 0xF8)==0xF0 ){
17904 if( i+3>=n || (z[i+1] & 0xC0)!=0x80 || (z[i+2] & 0xC0)!=0x80 ) return 1;
17905 if( (z[i+2] & 0xC0)!=0x80 ) return 1;
17906 i += 3;
17907 }else{
17908 return 1;
17909 }
17910 }
17911
17912 return 0;
17913}
17914
17915/*
17916** This function is also purely an internal test. It does not contribute to
17917** FTS functionality, or even the integrity-check, in any way.
17918**
17919** This function sets output variable (*pbFail) to true if the test fails. Or
17920** leaves it unchanged if the test succeeds.
17921*/
17922static void fts5TestTerm(
17923 Fts5Index *p,
17924 Fts5Buffer *pPrev, /* Previous term */
17925 const char *z, int n, /* Possibly new term to test */
17926 u64 expected,
17927 u64 *pCksum,
17928 int *pbFail
17929){
17930 int rc = p->rc;
17931 if( pPrev->n==0 ){
17932 fts5BufferSet(&rc, pPrev, n, (const u8*)z)sqlite3Fts5BufferSet(&rc,pPrev,n,(const u8*)z);
17933 }else
17934 if( *pbFail==0
17935 && rc==SQLITE_OK0
17936 && (pPrev->n!=n || memcmp(pPrev->p, z, n))
17937 && (p->pHash==0 || p->pHash->nEntry==0)
17938 ){
17939 u64 cksum3 = *pCksum;
17940 const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */
17941 int nTerm = pPrev->n-1; /* Size of zTerm in bytes */
17942 int iIdx = (pPrev->p[0] - FTS5_MAIN_PREFIX'0');
17943 int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX0x0001);
17944 u64 ck1 = 0;
17945 u64 ck2 = 0;
17946
17947 /* Check that the results returned for ASC and DESC queries are
17948 ** the same. If not, call this corruption. */
17949 rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1);
17950 if( rc==SQLITE_OK0 ){
17951 int f = flags|FTS5INDEX_QUERY_DESC0x0002;
17952 rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
17953 }
17954 if( rc==SQLITE_OK0 && ck1!=ck2 ) rc = FTS5_CORRUPT(11 | (1<<8));
17955
17956 /* If this is a prefix query, check that the results returned if the
17957 ** the index is disabled are the same. In both ASC and DESC order.
17958 **
17959 ** This check may only be performed if the hash table is empty. This
17960 ** is because the hash table only supports a single scan query at
17961 ** a time, and the multi-iter loop from which this function is called
17962 ** is already performing such a scan.
17963 **
17964 ** Also only do this if buffer zTerm contains nTerm bytes of valid
17965 ** utf-8. Otherwise, the last part of the buffer contents might contain
17966 ** a non-utf-8 sequence that happens to be a prefix of a valid utf-8
17967 ** character stored in the main fts index, which will cause the
17968 ** test to fail. */
17969 if( p->nPendingData==0 && 0==fts5TestUtf8(zTerm, nTerm) ){
17970 if( iIdx>0 && rc==SQLITE_OK0 ){
17971 int f = flags|FTS5INDEX_QUERY_TEST_NOIDX0x0004;
17972 ck2 = 0;
17973 rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
17974 if( rc==SQLITE_OK0 && ck1!=ck2 ) rc = FTS5_CORRUPT(11 | (1<<8));
17975 }
17976 if( iIdx>0 && rc==SQLITE_OK0 ){
17977 int f = flags|FTS5INDEX_QUERY_TEST_NOIDX0x0004|FTS5INDEX_QUERY_DESC0x0002;
17978 ck2 = 0;
17979 rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
17980 if( rc==SQLITE_OK0 && ck1!=ck2 ) rc = FTS5_CORRUPT(11 | (1<<8));
17981 }
17982 }
17983
17984 cksum3 ^= ck1;
17985 fts5BufferSet(&rc, pPrev, n, (const u8*)z)sqlite3Fts5BufferSet(&rc,pPrev,n,(const u8*)z);
17986
17987 if( rc==SQLITE_OK0 && cksum3!=expected ){
17988 *pbFail = 1;
17989 }
17990 *pCksum = cksum3;
17991 }
17992 p->rc = rc;
17993}
17994
17995#else
17996# define fts5TestDlidxReverse(x,y,z)
17997# define fts5TestTerm(t,u,v,w,x,y,z)
17998#endif
17999
18000/*
18001** Check that:
18002**
18003** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and
18004** contain zero terms.
18005** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and
18006** contain zero rowids.
18007*/
18008static void fts5IndexIntegrityCheckEmpty(
18009 Fts5Index *p,
18010 Fts5StructureSegment *pSeg, /* Segment to check internal consistency */
18011 int iFirst,
18012 int iNoRowid,
18013 int iLast
18014){
18015 int i;
18016
18017 /* Now check that the iter.nEmpty leaves following the current leaf
18018 ** (a) exist and (b) contain no terms. */
18019 for(i=iFirst; p->rc==SQLITE_OK0 && i<=iLast; i++){
18020 Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)( ((i64)(pSeg->iSegid) << (31 +5 +1)) + ((i64)(0) <<
(31 + 5)) + ((i64)(0) << (31)) + ((i64)(i)) ));
18021 if( pLeaf ){
18022 if( !fts5LeafIsTermless(pLeaf)((pLeaf)->szLeaf >= (pLeaf)->nn)
18023 || (i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf)(fts5GetU16((pLeaf)->p)))
18024 ){
18025 FTS5_CORRUPT_ROWID(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i))fts5IndexCorruptRowid(p, ( ((i64)(pSeg->iSegid) << (
31 +5 +1)) + ((i64)(0) << (31 + 5)) + ((i64)(0) <<
(31)) + ((i64)(i)) ));
18026 }
18027 }
18028 fts5DataRelease(pLeaf);
18029 }
18030}
18031
18032static void fts5IntegrityCheckPgidx(Fts5Index *p, i64 iRowid, Fts5Data *pLeaf){
18033 i64 iTermOff = 0;
18034 int ii;
18035
18036 Fts5Buffer buf1 = {0,0,0};
18037 Fts5Buffer buf2 = {0,0,0};
18038
18039 ii = pLeaf->szLeaf;
18040 while( ii<pLeaf->nn && p->rc==SQLITE_OK0 ){
18041 int res;
18042 i64 iOff;
18043 int nIncr;
18044
18045 ii += fts5GetVarint32(&pLeaf->p[ii], nIncr)sqlite3Fts5GetVarint32(&pLeaf->p[ii],(u32*)&(nIncr
));
18046 iTermOff += nIncr;
18047 iOff = iTermOff;
18048
18049 if( iOff>=pLeaf->szLeaf ){
18050 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
18051 }else if( iTermOff==nIncr ){
18052 int nByte;
18053 iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte)sqlite3Fts5GetVarint32(&pLeaf->p[iOff],(u32*)&(nByte
));
18054 if( (iOff+nByte)>pLeaf->szLeaf ){
18055 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
18056 }else{
18057 fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff])sqlite3Fts5BufferSet(&p->rc,&buf1,nByte,&pLeaf
->p[iOff]);
18058 }
18059 }else{
18060 int nKeep, nByte;
18061 iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep)sqlite3Fts5GetVarint32(&pLeaf->p[iOff],(u32*)&(nKeep
));
18062 iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte)sqlite3Fts5GetVarint32(&pLeaf->p[iOff],(u32*)&(nByte
));
18063 if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){
18064 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
18065 }else{
18066 buf1.n = nKeep;
18067 fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff])sqlite3Fts5BufferAppendBlob(&p->rc,&buf1,nByte,&
pLeaf->p[iOff]);
18068 }
18069
18070 if( p->rc==SQLITE_OK0 ){
18071 res = fts5BufferCompare(&buf1, &buf2);
18072 if( res<=0 ) FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
18073 }
18074 }
18075 fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p)sqlite3Fts5BufferSet(&p->rc,&buf2,buf1.n,buf1.p);
18076 }
18077
18078 fts5BufferFree(&buf1)sqlite3Fts5BufferFree(&buf1);
18079 fts5BufferFree(&buf2)sqlite3Fts5BufferFree(&buf2);
18080}
18081
18082static void fts5IndexIntegrityCheckSegment(
18083 Fts5Index *p, /* FTS5 backend object */
18084 Fts5StructureSegment *pSeg /* Segment to check internal consistency */
18085){
18086 Fts5Config *pConfig = p->pConfig;
18087 int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE5);
18088 sqlite3_stmt *pStmt = 0;
18089 int rc2;
18090 int iIdxPrevLeaf = pSeg->pgnoFirst-1;
18091 int iDlidxPrevLeaf = pSeg->pgnoLast;
18092
18093 if( pSeg->pgnoFirst==0 ) return;
18094
18095 fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintfsqlite3_api->mprintf(
18096 "SELECT segid, term, (pgno>>1), (pgno&1) FROM %Q.'%q_idx' WHERE segid=%d "
18097 "ORDER BY 1, 2",
18098 pConfig->zDb, pConfig->zName, pSeg->iSegid
18099 ));
18100
18101 /* Iterate through the b-tree hierarchy. */
18102 while( p->rc==SQLITE_OK0 && SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pStmt) ){
18103 i64 iRow; /* Rowid for this leaf */
18104 Fts5Data *pLeaf; /* Data for this leaf */
18105
18106 const char *zIdxTerm = (const char*)sqlite3_column_blobsqlite3_api->column_blob(pStmt, 1);
18107 int nIdxTerm = sqlite3_column_bytessqlite3_api->column_bytes(pStmt, 1);
18108 int iIdxLeaf = sqlite3_column_intsqlite3_api->column_int(pStmt, 2);
18109 int bIdxDlidx = sqlite3_column_intsqlite3_api->column_int(pStmt, 3);
18110
18111 /* If the leaf in question has already been trimmed from the segment,
18112 ** ignore this b-tree entry. Otherwise, load it into memory. */
18113 if( iIdxLeaf<pSeg->pgnoFirst ) continue;
18114 iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf)( ((i64)(pSeg->iSegid) << (31 +5 +1)) + ((i64)(0) <<
(31 + 5)) + ((i64)(0) << (31)) + ((i64)(iIdxLeaf)) );
18115 pLeaf = fts5LeafRead(p, iRow);
18116 if( pLeaf==0 ) break;
18117
18118 /* Check that the leaf contains at least one term, and that it is equal
18119 ** to or larger than the split-key in zIdxTerm. Also check that if there
18120 ** is also a rowid pointer within the leaf page header, it points to a
18121 ** location before the term. */
18122 if( pLeaf->nn<=pLeaf->szLeaf ){
18123
18124 if( nIdxTerm==0
18125 && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE5
18126 && pLeaf->nn==pLeaf->szLeaf
18127 && pLeaf->nn==4
18128 ){
18129 /* special case - the very first page in a segment keeps its %_idx
18130 ** entry even if all the terms are removed from it by secure-delete
18131 ** operations. */
18132 }else{
18133 FTS5_CORRUPT_ROWID(p, iRow)fts5IndexCorruptRowid(p, iRow);
18134 }
18135
18136 }else{
18137 int iOff; /* Offset of first term on leaf */
18138 int iRowidOff; /* Offset of first rowid on leaf */
18139 int nTerm; /* Size of term on leaf in bytes */
18140 int res; /* Comparison of term and split-key */
18141
18142 iOff = fts5LeafFirstTermOff(pLeaf);
18143 iRowidOff = fts5LeafFirstRowidOff(pLeaf)(fts5GetU16((pLeaf)->p));
18144 if( iRowidOff>=iOff || iOff>=pLeaf->szLeaf ){
18145 FTS5_CORRUPT_ROWID(p, iRow)fts5IndexCorruptRowid(p, iRow);
18146 }else{
18147 iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm)sqlite3Fts5GetVarint32(&pLeaf->p[iOff],(u32*)&(nTerm
));
18148 res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm))(((((nTerm) < (nIdxTerm)) ? (nTerm) : (nIdxTerm)))<=0 ?
0 : memcmp((&pLeaf->p[iOff]), (zIdxTerm), ((((nTerm) <
(nIdxTerm)) ? (nTerm) : (nIdxTerm)))));
18149 if( res==0 ) res = nTerm - nIdxTerm;
18150 if( res<0 ) FTS5_CORRUPT_ROWID(p, iRow)fts5IndexCorruptRowid(p, iRow);
18151 }
18152
18153 fts5IntegrityCheckPgidx(p, iRow, pLeaf);
18154 }
18155 fts5DataRelease(pLeaf);
18156 if( p->rc ) break;
18157
18158 /* Now check that the iter.nEmpty leaves following the current leaf
18159 ** (a) exist and (b) contain no terms. */
18160 fts5IndexIntegrityCheckEmpty(
18161 p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1
18162 );
18163 if( p->rc ) break;
18164
18165 /* If there is a doclist-index, check that it looks right. */
18166 if( bIdxDlidx ){
18167 Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */
18168 int iPrevLeaf = iIdxLeaf;
18169 int iSegid = pSeg->iSegid;
18170 int iPg = 0;
18171 i64 iKey;
18172
18173 for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf);
18174 fts5DlidxIterEof(p, pDlidx)==0;
18175 fts5DlidxIterNext(p, pDlidx)
18176 ){
18177
18178 /* Check any rowid-less pages that occur before the current leaf. */
18179 for(iPg=iPrevLeaf+1; iPg<fts5DlidxIterPgno(pDlidx); iPg++){
18180 iKey = FTS5_SEGMENT_ROWID(iSegid, iPg)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(iPg)) );
18181 pLeaf = fts5DataRead(p, iKey);
18182 if( pLeaf ){
18183 if( fts5LeafFirstRowidOff(pLeaf)(fts5GetU16((pLeaf)->p))!=0 ) FTS5_CORRUPT_ROWID(p, iKey)fts5IndexCorruptRowid(p, iKey);
18184 fts5DataRelease(pLeaf);
18185 }
18186 }
18187 iPrevLeaf = fts5DlidxIterPgno(pDlidx);
18188
18189 /* Check that the leaf page indicated by the iterator really does
18190 ** contain the rowid suggested by the same. */
18191 iKey = FTS5_SEGMENT_ROWID(iSegid, iPrevLeaf)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(iPrevLeaf)) );
18192 pLeaf = fts5DataRead(p, iKey);
18193 if( pLeaf ){
18194 i64 iRowid;
18195 int iRowidOff = fts5LeafFirstRowidOff(pLeaf)(fts5GetU16((pLeaf)->p));
18196 ASSERT_SZLEAF_OK(pLeaf)((void) (0));
18197 if( iRowidOff>=pLeaf->szLeaf ){
18198 FTS5_CORRUPT_ROWID(p, iKey)fts5IndexCorruptRowid(p, iKey);
18199 }else if( bSecureDelete==0 || iRowidOff>0 ){
18200 i64 iDlRowid = fts5DlidxIterRowid(pDlidx);
18201 fts5GetVarintsqlite3Fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
18202 if( iRowid<iDlRowid || (bSecureDelete==0 && iRowid!=iDlRowid) ){
18203 FTS5_CORRUPT_ROWID(p, iKey)fts5IndexCorruptRowid(p, iKey);
18204 }
18205 }
18206 fts5DataRelease(pLeaf);
18207 }
18208 }
18209
18210 iDlidxPrevLeaf = iPg;
18211 fts5DlidxIterFree(pDlidx);
18212 fts5TestDlidxReverse(p, iSegid, iIdxLeaf);
18213 }else{
18214 iDlidxPrevLeaf = pSeg->pgnoLast;
18215 /* TODO: Check there is no doclist index */
18216 }
18217
18218 iIdxPrevLeaf = iIdxLeaf;
18219 }
18220
18221 rc2 = sqlite3_finalizesqlite3_api->finalize(pStmt);
18222 if( p->rc==SQLITE_OK0 ) p->rc = rc2;
18223
18224 /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */
18225#if 0
18226 if( p->rc==SQLITE_OK0 && iter.iLeaf!=pSeg->pgnoLast ){
18227 p->rc = FTS5_CORRUPT(11 | (1<<8));
18228 }
18229#endif
18230}
18231
18232
18233/*
18234** Run internal checks to ensure that the FTS index (a) is internally
18235** consistent and (b) contains entries for which the XOR of the checksums
18236** as calculated by sqlite3Fts5IndexEntryCksum() is cksum.
18237**
18238** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
18239** checksum does not match. Return SQLITE_OK if all checks pass without
18240** error, or some other SQLite error code if another error (e.g. OOM)
18241** occurs.
18242*/
18243static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){
18244 int eDetail = p->pConfig->eDetail;
18245 u64 cksum2 = 0; /* Checksum based on contents of indexes */
18246 Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */
18247 Fts5Iter *pIter; /* Used to iterate through entire index */
18248 Fts5Structure *pStruct; /* Index structure */
18249 int iLvl, iSeg;
18250
18251#ifdef SQLITE_DEBUG
18252 /* Used by extra internal tests only run if NDEBUG is not defined */
18253 u64 cksum3 = 0; /* Checksum based on contents of indexes */
18254 Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */
18255 int bTestFail = 0;
18256#endif
18257 const int flags = FTS5INDEX_QUERY_NOOUTPUT0x0020;
18258
18259 /* Load the FTS index structure */
18260 pStruct = fts5StructureRead(p);
18261 if( pStruct==0 ){
18262 assert( p->rc!=SQLITE_OK )((void) (0));
18263 return fts5IndexReturn(p);
18264 }
18265
18266 /* Check that the internal nodes of each segment match the leaves */
18267 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
18268 for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
18269 Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
18270 fts5IndexIntegrityCheckSegment(p, pSeg);
18271 }
18272 }
18273
18274 /* The cksum argument passed to this function is a checksum calculated
18275 ** based on all expected entries in the FTS index (including prefix index
18276 ** entries). This block checks that a checksum calculated based on the
18277 ** actual contents of FTS index is identical.
18278 **
18279 ** Two versions of the same checksum are calculated. The first (stack
18280 ** variable cksum2) based on entries extracted from the full-text index
18281 ** while doing a linear scan of each individual index in turn.
18282 **
18283 ** As each term visited by the linear scans, a separate query for the
18284 ** same term is performed. cksum3 is calculated based on the entries
18285 ** extracted by these queries.
18286 */
18287 for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, -1, 0, &pIter);
18288 fts5MultiIterEof(p, pIter)==0;
18289 fts5MultiIterNext(p, pIter, 0, 0)
18290 ){
18291 int n; /* Size of term in bytes */
18292 i64 iPos = 0; /* Position read from poslist */
18293 int iOff = 0; /* Offset within poslist */
18294 i64 iRowid = fts5MultiIterRowid(pIter);
18295 char *z = (char*)fts5MultiIterTerm(pIter, &n);
18296
18297 /* If this is a new term, query for it. Update cksum3 with the results. */
18298 fts5TestTerm(p, &term, z, n, cksum2, &cksum3, &bTestFail);
18299 if( p->rc ) break;
18300
18301 if( eDetail==FTS5_DETAIL_NONE1 ){
18302 if( 0==fts5MultiIterIsEmpty(p, pIter) ){
18303 cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n);
18304 }
18305 }else{
18306 poslist.n = 0;
18307 fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist);
18308 fts5BufferAppendBlob(&p->rc, &poslist, 4, (const u8*)"\0\0\0\0")sqlite3Fts5BufferAppendBlob(&p->rc,&poslist,4,(const
u8*)"\0\0\0\0");
18309 while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
18310 int iCol = FTS5_POS2COLUMN(iPos)(int)((iPos >> 32) & 0x7FFFFFFF);
18311 int iTokOff = FTS5_POS2OFFSET(iPos)(int)(iPos & 0x7FFFFFFF);
18312 cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
18313 }
18314 }
18315 }
18316 fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3, &bTestFail);
18317
18318 fts5MultiIterFree(pIter);
18319 if( p->rc==SQLITE_OK0 && bUseCksum && cksum!=cksum2 ){
18320 p->rc = FTS5_CORRUPT(11 | (1<<8));
18321 sqlite3Fts5ConfigErrmsg(p->pConfig,
18322 "fts5: checksum mismatch for table \"%s\"", p->pConfig->zName
18323 );
18324 }
18325#ifdef SQLITE_DEBUG
18326 /* In SQLITE_DEBUG builds, expensive extra checks were run as part of
18327 ** the integrity-check above. If no other errors were detected, but one
18328 ** of these tests failed, set the result to SQLITE_CORRUPT_VTAB here. */
18329 if( p->rc==SQLITE_OK0 && bTestFail ){
18330 p->rc = FTS5_CORRUPT(11 | (1<<8));
18331 }
18332 fts5BufferFree(&term)sqlite3Fts5BufferFree(&term);
18333#endif
18334
18335 fts5StructureRelease(pStruct);
18336 fts5BufferFree(&poslist)sqlite3Fts5BufferFree(&poslist);
18337 return fts5IndexReturn(p);
18338}
18339
18340/*************************************************************************
18341**************************************************************************
18342** Below this point is the implementation of the fts5_decode() scalar
18343** function only.
18344*/
18345
18346#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18347/*
18348** Decode a segment-data rowid from the %_data table. This function is
18349** the opposite of macro FTS5_SEGMENT_ROWID().
18350*/
18351static void fts5DecodeRowid(
18352 i64 iRowid, /* Rowid from %_data table */
18353 int *pbTombstone, /* OUT: Tombstone hash flag */
18354 int *piSegid, /* OUT: Segment id */
18355 int *pbDlidx, /* OUT: Dlidx flag */
18356 int *piHeight, /* OUT: Height */
18357 int *piPgno /* OUT: Page number */
18358){
18359 *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B31) - 1));
18360 iRowid >>= FTS5_DATA_PAGE_B31;
18361
18362 *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B5) - 1));
18363 iRowid >>= FTS5_DATA_HEIGHT_B5;
18364
18365 *pbDlidx = (int)(iRowid & 0x0001);
18366 iRowid >>= FTS5_DATA_DLI_B1;
18367
18368 *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B16) - 1));
18369 iRowid >>= FTS5_DATA_ID_B16;
18370
18371 *pbTombstone = (int)(iRowid & 0x0001);
18372}
18373#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18374
18375#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18376static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
18377 int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid components */
18378 fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno);
18379
18380 if( iSegid==0 ){
18381 if( iKey==FTS5_AVERAGES_ROWID1 ){
18382 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{averages} ");
18383 }else{
18384 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{structure}");
18385 }
18386 }
18387 else{
18388 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%s%ssegid=%d h=%d pgno=%d}",
18389 bDlidx ? "dlidx " : "",
18390 bTomb ? "tombstone " : "",
18391 iSegid, iHeight, iPgno
18392 );
18393 }
18394}
18395#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18396
18397#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18398static void fts5DebugStructure(
18399 int *pRc, /* IN/OUT: error code */
18400 Fts5Buffer *pBuf,
18401 Fts5Structure *p
18402){
18403 int iLvl, iSeg; /* Iterate through levels, segments */
18404
18405 for(iLvl=0; iLvl<p->nLevel; iLvl++){
18406 Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
18407 sqlite3Fts5BufferAppendPrintf(pRc, pBuf,
18408 " {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg
18409 );
18410 for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
18411 Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
18412 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d",
18413 pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast
18414 );
18415 if( pSeg->iOrigin1>0 ){
18416 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " origin=%lld..%lld",
18417 pSeg->iOrigin1, pSeg->iOrigin2
18418 );
18419 }
18420 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
18421 }
18422 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
18423 }
18424}
18425#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18426
18427#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18428/*
18429** This is part of the fts5_decode() debugging aid.
18430**
18431** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This
18432** function appends a human-readable representation of the same object
18433** to the buffer passed as the second argument.
18434*/
18435static void fts5DecodeStructure(
18436 int *pRc, /* IN/OUT: error code */
18437 Fts5Buffer *pBuf,
18438 const u8 *pBlob, int nBlob
18439){
18440 int rc; /* Return code */
18441 Fts5Structure *p = 0; /* Decoded structure object */
18442
18443 rc = fts5StructureDecode(pBlob, nBlob, 0, &p);
18444 if( rc!=SQLITE_OK0 ){
18445 *pRc = rc;
18446 return;
18447 }
18448
18449 fts5DebugStructure(pRc, pBuf, p);
18450 fts5StructureRelease(p);
18451}
18452#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18453
18454#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18455/*
18456** This is part of the fts5_decode() debugging aid.
18457**
18458** Arguments pBlob/nBlob contain an "averages" record. This function
18459** appends a human-readable representation of record to the buffer passed
18460** as the second argument.
18461*/
18462static void fts5DecodeAverages(
18463 int *pRc, /* IN/OUT: error code */
18464 Fts5Buffer *pBuf,
18465 const u8 *pBlob, int nBlob
18466){
18467 int i = 0;
18468 const char *zSpace = "";
18469
18470 while( i<nBlob ){
18471 u64 iVal;
18472 i += sqlite3Fts5GetVarint(&pBlob[i], &iVal);
18473 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "%s%d", zSpace, (int)iVal);
18474 zSpace = " ";
18475 }
18476}
18477#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18478
18479#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18480/*
18481** Buffer (a/n) is assumed to contain a list of serialized varints. Read
18482** each varint and append its string representation to buffer pBuf. Return
18483** after either the input buffer is exhausted or a 0 value is read.
18484**
18485** The return value is the number of bytes read from the input buffer.
18486*/
18487static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
18488 int iOff = 0;
18489 while( iOff<n ){
18490 int iVal;
18491 iOff += fts5GetVarint32(&a[iOff], iVal)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(iVal));
18492 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal);
18493 }
18494 return iOff;
18495}
18496#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18497
18498#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18499/*
18500** The start of buffer (a/n) contains the start of a doclist. The doclist
18501** may or may not finish within the buffer. This function appends a text
18502** representation of the part of the doclist that is present to buffer
18503** pBuf.
18504**
18505** The return value is the number of bytes read from the input buffer.
18506*/
18507static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
18508 i64 iDocid = 0;
18509 int iOff = 0;
18510
18511 if( n>0 ){
18512 iOff = sqlite3Fts5GetVarint(a, (u64*)&iDocid);
18513 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
18514 }
18515 while( iOff<n ){
18516 int nPos;
18517 int bDel;
18518 iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDel);
18519 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " nPos=%d%s", nPos, bDel?"*":"");
18520 iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos)(((n-iOff) < (nPos)) ? (n-iOff) : (nPos)));
18521 if( iOff<n ){
18522 i64 iDelta;
18523 iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDelta);
18524 iDocid += iDelta;
18525 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
18526 }
18527 }
18528
18529 return iOff;
18530}
18531#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18532
18533#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18534/*
18535** This function is part of the fts5_decode() debugging function. It is
18536** only ever used with detail=none tables.
18537**
18538** Buffer (pData/nData) contains a doclist in the format used by detail=none
18539** tables. This function appends a human-readable version of that list to
18540** buffer pBuf.
18541**
18542** If *pRc is other than SQLITE_OK when this function is called, it is a
18543** no-op. If an OOM or other error occurs within this function, *pRc is
18544** set to an SQLite error code before returning. The final state of buffer
18545** pBuf is undefined in this case.
18546*/
18547static void fts5DecodeRowidList(
18548 int *pRc, /* IN/OUT: Error code */
18549 Fts5Buffer *pBuf, /* Buffer to append text to */
18550 const u8 *pData, int nData /* Data to decode list-of-rowids from */
18551){
18552 int i = 0;
18553 i64 iRowid = 0;
18554
18555 while( i<nData ){
18556 const char *zApp = "";
18557 u64 iVal;
18558 i += sqlite3Fts5GetVarint(&pData[i], &iVal);
18559 iRowid += iVal;
18560
18561 if( i<nData && pData[i]==0x00 ){
18562 i++;
18563 if( i<nData && pData[i]==0x00 ){
18564 i++;
18565 zApp = "+";
18566 }else{
18567 zApp = "*";
18568 }
18569 }
18570
18571 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp);
18572 }
18573}
18574#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18575
18576#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18577static void fts5BufferAppendTerm(int *pRc, Fts5Buffer *pBuf, Fts5Buffer *pTerm){
18578 int ii;
18579 fts5BufferGrow(pRc, pBuf, pTerm->n*2 + 1)( (u32)((pBuf)->n) + (u32)(pTerm->n*2 + 1) <= (u32)(
(pBuf)->nSpace) ? 0 : sqlite3Fts5BufferSize((pRc),(pBuf),(
pTerm->n*2 + 1)+(pBuf)->n) );
18580 if( *pRc==SQLITE_OK0 ){
18581 for(ii=0; ii<pTerm->n; ii++){
18582 if( pTerm->p[ii]==0x00 ){
18583 pBuf->p[pBuf->n++] = '\\';
18584 pBuf->p[pBuf->n++] = '0';
18585 }else{
18586 pBuf->p[pBuf->n++] = pTerm->p[ii];
18587 }
18588 }
18589 pBuf->p[pBuf->n] = 0x00;
18590 }
18591}
18592#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18593
18594#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18595/*
18596** The implementation of user-defined scalar function fts5_decode().
18597*/
18598static void fts5DecodeFunction(
18599 sqlite3_context *pCtx, /* Function call context */
18600 int nArg, /* Number of args (always 2) */
18601 sqlite3_value **apVal /* Function arguments */
18602){
18603 i64 iRowid; /* Rowid for record being decoded */
18604 int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */
18605 int bTomb;
18606 const u8 *aBlob; int n; /* Record to decode */
18607 u8 *a = 0;
18608 Fts5Buffer s; /* Build up text to return here */
18609 int rc = SQLITE_OK0; /* Return code */
18610 sqlite3_int64 nSpace = 0;
18611 int eDetailNone = (sqlite3_user_datasqlite3_api->user_data(pCtx)!=0);
18612
18613 assert( nArg==2 )((void) (0));
18614 UNUSED_PARAM(nArg)(void)(nArg);
18615 memset(&s, 0, sizeof(Fts5Buffer));
18616 iRowid = sqlite3_value_int64sqlite3_api->value_int64(apVal[0]);
18617
18618 /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[]
18619 ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents
18620 ** buffer overreads even if the record is corrupt. */
18621 n = sqlite3_value_bytessqlite3_api->value_bytes(apVal[1]);
18622 aBlob = sqlite3_value_blobsqlite3_api->value_blob(apVal[1]);
18623 nSpace = ((i64)n) + FTS5_DATA_ZERO_PADDING8;
18624 a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace);
18625 if( a==0 ) goto decode_out;
18626 if( n>0 ) memcpy(a, aBlob, n);
18627
18628 fts5DecodeRowid(iRowid, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno);
18629
18630 fts5DebugRowid(&rc, &s, iRowid);
18631 if( bDlidx ){
18632 Fts5Data dlidx;
18633 Fts5DlidxLvl lvl;
18634
18635 dlidx.p = a;
18636 dlidx.nn = n;
18637
18638 memset(&lvl, 0, sizeof(Fts5DlidxLvl));
18639 lvl.pData = &dlidx;
18640 lvl.iLeafPgno = iPgno;
18641
18642 for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){
18643 sqlite3Fts5BufferAppendPrintf(&rc, &s,
18644 " %d(%lld)", lvl.iLeafPgno, lvl.iRowid
18645 );
18646 }
18647 }else if( bTomb ){
18648 u32 nElem = fts5GetU32(&a[4]);
18649 int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8;
18650 int nSlot = (n - 8) / szKey;
18651 int ii;
18652 sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem);
18653 if( aBlob[1] ){
18654 sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0");
18655 }
18656 for(ii=0; ii<nSlot; ii++){
18657 u64 iVal = 0;
18658 if( szKey==4 ){
18659 u32 *aSlot = (u32*)&aBlob[8];
18660 if( aSlot[ii] ) iVal = fts5GetU32((u8*)&aSlot[ii]);
18661 }else{
18662 u64 *aSlot = (u64*)&aBlob[8];
18663 if( aSlot[ii] ) iVal = fts5GetU64((u8*)&aSlot[ii]);
18664 }
18665 if( iVal!=0 ){
18666 sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", (i64)iVal);
18667 }
18668 }
18669 }else if( iSegid==0 ){
18670 if( iRowid==FTS5_AVERAGES_ROWID1 ){
18671 fts5DecodeAverages(&rc, &s, a, n);
18672 }else{
18673 fts5DecodeStructure(&rc, &s, a, n);
18674 }
18675 }else if( eDetailNone ){
18676 Fts5Buffer term; /* Current term read from page */
18677 int szLeaf;
18678 int iPgidxOff = szLeaf = fts5GetU16(&a[2]);
18679 int iTermOff;
18680 int nKeep = 0;
18681 int iOff;
18682
18683 memset(&term, 0, sizeof(Fts5Buffer));
18684
18685 /* Decode any entries that occur before the first term. */
18686 if( szLeaf<n ){
18687 iPgidxOff += fts5GetVarint32(&a[iPgidxOff], iTermOff)sqlite3Fts5GetVarint32(&a[iPgidxOff],(u32*)&(iTermOff
));
18688 }else{
18689 iTermOff = szLeaf;
18690 }
18691 fts5DecodeRowidList(&rc, &s, &a[4], iTermOff-4);
18692
18693 iOff = iTermOff;
18694 while( iOff<szLeaf && rc==SQLITE_OK0 ){
18695 int nAppend;
18696
18697 /* Read the term data for the next term*/
18698 iOff += fts5GetVarint32(&a[iOff], nAppend)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nAppend));
18699 term.n = nKeep;
18700 fts5BufferAppendBlob(&rc, &term, nAppend, &a[iOff])sqlite3Fts5BufferAppendBlob(&rc,&term,nAppend,&a[
iOff]);
18701 sqlite3Fts5BufferAppendPrintf(&rc, &s, " term=");
18702 fts5BufferAppendTerm(&rc, &s, &term);
18703 iOff += nAppend;
18704
18705 /* Figure out where the doclist for this term ends */
18706 if( iPgidxOff<n ){
18707 int nIncr;
18708 iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nIncr)sqlite3Fts5GetVarint32(&a[iPgidxOff],(u32*)&(nIncr));
18709 iTermOff += nIncr;
18710 }else{
18711 iTermOff = szLeaf;
18712 }
18713 if( iTermOff>szLeaf ){
18714 rc = FTS5_CORRUPT(11 | (1<<8));
18715 }else{
18716 fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff);
18717 }
18718 iOff = iTermOff;
18719 if( iOff<szLeaf ){
18720 iOff += fts5GetVarint32(&a[iOff], nKeep)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nKeep));
18721 }
18722 }
18723
18724 fts5BufferFree(&term)sqlite3Fts5BufferFree(&term);
18725 }else{
18726 Fts5Buffer term; /* Current term read from page */
18727 int szLeaf; /* Offset of pgidx in a[] */
18728 int iPgidxOff;
18729 int iPgidxPrev = 0; /* Previous value read from pgidx */
18730 int iTermOff = 0;
18731 int iRowidOff = 0;
18732 int iOff;
18733 int nDoclist;
18734
18735 memset(&term, 0, sizeof(Fts5Buffer));
18736
18737 if( n<4 ){
18738 sqlite3Fts5BufferSet(&rc, &s, 7, (const u8*)"corrupt");
18739 goto decode_out;
18740 }else{
18741 iRowidOff = fts5GetU16(&a[0]);
18742 iPgidxOff = szLeaf = fts5GetU16(&a[2]);
18743 if( iPgidxOff<n ){
18744 fts5GetVarint32(&a[iPgidxOff], iTermOff)sqlite3Fts5GetVarint32(&a[iPgidxOff],(u32*)&(iTermOff
));
18745 }else if( iPgidxOff>n ){
18746 rc = FTS5_CORRUPT(11 | (1<<8));
18747 goto decode_out;
18748 }
18749 }
18750
18751 /* Decode the position list tail at the start of the page */
18752 if( iRowidOff!=0 ){
18753 iOff = iRowidOff;
18754 }else if( iTermOff!=0 ){
18755 iOff = iTermOff;
18756 }else{
18757 iOff = szLeaf;
18758 }
18759 if( iOff>n ){
18760 rc = FTS5_CORRUPT(11 | (1<<8));
18761 goto decode_out;
18762 }
18763 fts5DecodePoslist(&rc, &s, &a[4], iOff-4);
18764
18765 /* Decode any more doclist data that appears on the page before the
18766 ** first term. */
18767 nDoclist = (iTermOff ? iTermOff : szLeaf) - iOff;
18768 if( nDoclist+iOff>n ){
18769 rc = FTS5_CORRUPT(11 | (1<<8));
18770 goto decode_out;
18771 }
18772 fts5DecodeDoclist(&rc, &s, &a[iOff], nDoclist);
18773
18774 while( iPgidxOff<n && rc==SQLITE_OK0 ){
18775 int bFirst = (iPgidxOff==szLeaf); /* True for first term on page */
18776 int nByte; /* Bytes of data */
18777 int iEnd;
18778
18779 iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nByte)sqlite3Fts5GetVarint32(&a[iPgidxOff],(u32*)&(nByte));
18780 iPgidxPrev += nByte;
18781 iOff = iPgidxPrev;
18782
18783 if( iPgidxOff<n ){
18784 fts5GetVarint32(&a[iPgidxOff], nByte)sqlite3Fts5GetVarint32(&a[iPgidxOff],(u32*)&(nByte));
18785 iEnd = iPgidxPrev + nByte;
18786 }else{
18787 iEnd = szLeaf;
18788 }
18789 if( iEnd>szLeaf ){
18790 rc = FTS5_CORRUPT(11 | (1<<8));
18791 break;
18792 }
18793
18794 if( bFirst==0 ){
18795 iOff += fts5GetVarint32(&a[iOff], nByte)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nByte));
18796 if( nByte>term.n ){
18797 rc = FTS5_CORRUPT(11 | (1<<8));
18798 break;
18799 }
18800 term.n = nByte;
18801 }
18802 iOff += fts5GetVarint32(&a[iOff], nByte)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nByte));
18803 if( iOff+nByte>n ){
18804 rc = FTS5_CORRUPT(11 | (1<<8));
18805 break;
18806 }
18807 fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff])sqlite3Fts5BufferAppendBlob(&rc,&term,nByte,&a[iOff
]);
18808 iOff += nByte;
18809
18810 sqlite3Fts5BufferAppendPrintf(&rc, &s, " term=");
18811 fts5BufferAppendTerm(&rc, &s, &term);
18812 iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], iEnd-iOff);
18813 }
18814
18815 fts5BufferFree(&term)sqlite3Fts5BufferFree(&term);
18816 }
18817
18818 decode_out:
18819 sqlite3_freesqlite3_api->free(a);
18820 if( rc==SQLITE_OK0 ){
18821 sqlite3_result_textsqlite3_api->result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
18822 }else{
18823 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
18824 }
18825 fts5BufferFree(&s)sqlite3Fts5BufferFree(&s);
18826}
18827#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18828
18829#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18830/*
18831** The implementation of user-defined scalar function fts5_rowid().
18832*/
18833static void fts5RowidFunction(
18834 sqlite3_context *pCtx, /* Function call context */
18835 int nArg, /* Number of args (always 2) */
18836 sqlite3_value **apVal /* Function arguments */
18837){
18838 const char *zArg;
18839 if( nArg==0 ){
18840 sqlite3_result_errorsqlite3_api->result_error(pCtx, "should be: fts5_rowid(subject, ....)", -1);
18841 }else{
18842 zArg = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[0]);
18843 if( 0==sqlite3_stricmpsqlite3_api->stricmp(zArg, "segment") ){
18844 i64 iRowid;
18845 int segid, pgno;
18846 if( nArg!=3 ){
18847 sqlite3_result_errorsqlite3_api->result_error(pCtx,
18848 "should be: fts5_rowid('segment', segid, pgno))", -1
18849 );
18850 }else{
18851 segid = sqlite3_value_intsqlite3_api->value_int(apVal[1]);
18852 pgno = sqlite3_value_intsqlite3_api->value_int(apVal[2]);
18853 iRowid = FTS5_SEGMENT_ROWID(segid, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pgno)) );
18854 sqlite3_result_int64sqlite3_api->result_int64(pCtx, iRowid);
18855 }
18856 }else{
18857 sqlite3_result_errorsqlite3_api->result_error(pCtx,
18858 "first arg to fts5_rowid() must be 'segment'" , -1
18859 );
18860 }
18861 }
18862}
18863#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18864
18865#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18866
18867typedef struct Fts5StructVtab Fts5StructVtab;
18868struct Fts5StructVtab {
18869 sqlite3_vtab base;
18870};
18871
18872typedef struct Fts5StructVcsr Fts5StructVcsr;
18873struct Fts5StructVcsr {
18874 sqlite3_vtab_cursor base;
18875 Fts5Structure *pStruct;
18876 int iLevel;
18877 int iSeg;
18878 int iRowid;
18879};
18880
18881/*
18882** Create a new fts5_structure() table-valued function.
18883*/
18884static int fts5structConnectMethod(
18885 sqlite3 *db,
18886 void *pAux,
18887 int argc, const char *const*argv,
18888 sqlite3_vtab **ppVtab,
18889 char **pzErr
18890){
18891 Fts5StructVtab *pNew = 0;
18892 int rc = SQLITE_OK0;
18893
18894 rc = sqlite3_declare_vtabsqlite3_api->declare_vtab(db,
18895 "CREATE TABLE xyz("
18896 "level, segment, merge, segid, leaf1, leaf2, loc1, loc2, "
18897 "npgtombstone, nentrytombstone, nentry, struct HIDDEN);"
18898 );
18899 if( rc==SQLITE_OK0 ){
18900 pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
18901 }
18902
18903 *ppVtab = (sqlite3_vtab*)pNew;
18904 return rc;
18905}
18906
18907/*
18908** We must have a single struct=? constraint that will be passed through
18909** into the xFilter method. If there is no valid struct=? constraint,
18910** then return an SQLITE_CONSTRAINT error.
18911*/
18912static int fts5structBestIndexMethod(
18913 sqlite3_vtab *tab,
18914 sqlite3_index_info *pIdxInfo
18915){
18916 int i;
18917 int rc = SQLITE_CONSTRAINT19;
18918 struct sqlite3_index_constraint *p;
18919 pIdxInfo->estimatedCost = (double)100;
18920 pIdxInfo->estimatedRows = 100;
18921 pIdxInfo->idxNum = 0;
18922 for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){
18923 if( p->usable==0 ) continue;
18924 if( p->op==SQLITE_INDEX_CONSTRAINT_EQ2 && p->iColumn==11 ){
18925 rc = SQLITE_OK0;
18926 pIdxInfo->aConstraintUsage[i].omit = 1;
18927 pIdxInfo->aConstraintUsage[i].argvIndex = 1;
18928 break;
18929 }
18930 }
18931 return rc;
18932}
18933
18934/*
18935** This method is the destructor for bytecodevtab objects.
18936*/
18937static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){
18938 Fts5StructVtab *p = (Fts5StructVtab*)pVtab;
18939 sqlite3_freesqlite3_api->free(p);
18940 return SQLITE_OK0;
18941}
18942
18943/*
18944** Constructor for a new bytecodevtab_cursor object.
18945*/
18946static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
18947 int rc = SQLITE_OK0;
18948 Fts5StructVcsr *pNew = 0;
18949
18950 pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
18951 *ppCsr = (sqlite3_vtab_cursor*)pNew;
18952
18953 return SQLITE_OK0;
18954}
18955
18956/*
18957** Destructor for a bytecodevtab_cursor.
18958*/
18959static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){
18960 Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
18961 fts5StructureRelease(pCsr->pStruct);
18962 sqlite3_freesqlite3_api->free(pCsr);
18963 return SQLITE_OK0;
18964}
18965
18966
18967/*
18968** Advance a bytecodevtab_cursor to its next row of output.
18969*/
18970static int fts5structNextMethod(sqlite3_vtab_cursor *cur){
18971 Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
18972 Fts5Structure *p = pCsr->pStruct;
18973
18974 assert( pCsr->pStruct )((void) (0));
18975 pCsr->iSeg++;
18976 pCsr->iRowid++;
18977 while( pCsr->iLevel<p->nLevel && pCsr->iSeg>=p->aLevel[pCsr->iLevel].nSeg ){
18978 pCsr->iLevel++;
18979 pCsr->iSeg = 0;
18980 }
18981 if( pCsr->iLevel>=p->nLevel ){
18982 fts5StructureRelease(pCsr->pStruct);
18983 pCsr->pStruct = 0;
18984 }
18985 return SQLITE_OK0;
18986}
18987
18988/*
18989** Return TRUE if the cursor has been moved off of the last
18990** row of output.
18991*/
18992static int fts5structEofMethod(sqlite3_vtab_cursor *cur){
18993 Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
18994 return pCsr->pStruct==0;
18995}
18996
18997static int fts5structRowidMethod(
18998 sqlite3_vtab_cursor *cur,
18999 sqlite_int64 *piRowid
19000){
19001 Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
19002 *piRowid = pCsr->iRowid;
19003 return SQLITE_OK0;
19004}
19005
19006/*
19007** Return values of columns for the row at which the bytecodevtab_cursor
19008** is currently pointing.
19009*/
19010static int fts5structColumnMethod(
19011 sqlite3_vtab_cursor *cur, /* The cursor */
19012 sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
19013 int i /* Which column to return */
19014){
19015 Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
19016 Fts5Structure *p = pCsr->pStruct;
19017 Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg];
19018
19019 switch( i ){
19020 case 0: /* level */
19021 sqlite3_result_intsqlite3_api->result_int(ctx, pCsr->iLevel);
19022 break;
19023 case 1: /* segment */
19024 sqlite3_result_intsqlite3_api->result_int(ctx, pCsr->iSeg);
19025 break;
19026 case 2: /* merge */
19027 sqlite3_result_intsqlite3_api->result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge);
19028 break;
19029 case 3: /* segid */
19030 sqlite3_result_intsqlite3_api->result_int(ctx, pSeg->iSegid);
19031 break;
19032 case 4: /* leaf1 */
19033 sqlite3_result_intsqlite3_api->result_int(ctx, pSeg->pgnoFirst);
19034 break;
19035 case 5: /* leaf2 */
19036 sqlite3_result_intsqlite3_api->result_int(ctx, pSeg->pgnoLast);
19037 break;
19038 case 6: /* origin1 */
19039 sqlite3_result_int64sqlite3_api->result_int64(ctx, pSeg->iOrigin1);
19040 break;
19041 case 7: /* origin2 */
19042 sqlite3_result_int64sqlite3_api->result_int64(ctx, pSeg->iOrigin2);
19043 break;
19044 case 8: /* npgtombstone */
19045 sqlite3_result_intsqlite3_api->result_int(ctx, pSeg->nPgTombstone);
19046 break;
19047 case 9: /* nentrytombstone */
19048 sqlite3_result_int64sqlite3_api->result_int64(ctx, pSeg->nEntryTombstone);
19049 break;
19050 case 10: /* nentry */
19051 sqlite3_result_int64sqlite3_api->result_int64(ctx, pSeg->nEntry);
19052 break;
19053 }
19054 return SQLITE_OK0;
19055}
19056
19057/*
19058** Initialize a cursor.
19059**
19060** idxNum==0 means show all subprograms
19061** idxNum==1 means show only the main bytecode and omit subprograms.
19062*/
19063static int fts5structFilterMethod(
19064 sqlite3_vtab_cursor *pVtabCursor,
19065 int idxNum, const char *idxStr,
19066 int argc, sqlite3_value **argv
19067){
19068 Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor;
19069 int rc = SQLITE_OK0;
19070
19071 const u8 *aBlob = 0;
19072 int nBlob = 0;
19073
19074 assert( argc==1 )((void) (0));
19075 fts5StructureRelease(pCsr->pStruct);
19076 pCsr->pStruct = 0;
19077
19078 nBlob = sqlite3_value_bytessqlite3_api->value_bytes(argv[0]);
19079 aBlob = (const u8*)sqlite3_value_blobsqlite3_api->value_blob(argv[0]);
19080 rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct);
19081 if( rc==SQLITE_OK0 ){
19082 pCsr->iLevel = 0;
19083 pCsr->iRowid = 0;
19084 pCsr->iSeg = -1;
19085 rc = fts5structNextMethod(pVtabCursor);
19086 }
19087
19088 return rc;
19089}
19090
19091#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
19092
19093/*
19094** This is called as part of registering the FTS5 module with database
19095** connection db. It registers several user-defined scalar functions useful
19096** with FTS5.
19097**
19098** If successful, SQLITE_OK is returned. If an error occurs, some other
19099** SQLite error code is returned instead.
19100*/
19101static int sqlite3Fts5IndexInit(sqlite3 *db){
19102#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
19103 int rc = sqlite3_create_functionsqlite3_api->create_function(
19104 db, "fts5_decode", 2, SQLITE_UTF81, 0, fts5DecodeFunction, 0, 0
19105 );
19106
19107 if( rc==SQLITE_OK0 ){
19108 rc = sqlite3_create_functionsqlite3_api->create_function(
19109 db, "fts5_decode_none", 2,
19110 SQLITE_UTF81, (void*)db, fts5DecodeFunction, 0, 0
19111 );
19112 }
19113
19114 if( rc==SQLITE_OK0 ){
19115 rc = sqlite3_create_functionsqlite3_api->create_function(
19116 db, "fts5_rowid", -1, SQLITE_UTF81, 0, fts5RowidFunction, 0, 0
19117 );
19118 }
19119
19120 if( rc==SQLITE_OK0 ){
19121 static const sqlite3_module fts5structure_module = {
19122 0, /* iVersion */
19123 0, /* xCreate */
19124 fts5structConnectMethod, /* xConnect */
19125 fts5structBestIndexMethod, /* xBestIndex */
19126 fts5structDisconnectMethod, /* xDisconnect */
19127 0, /* xDestroy */
19128 fts5structOpenMethod, /* xOpen */
19129 fts5structCloseMethod, /* xClose */
19130 fts5structFilterMethod, /* xFilter */
19131 fts5structNextMethod, /* xNext */
19132 fts5structEofMethod, /* xEof */
19133 fts5structColumnMethod, /* xColumn */
19134 fts5structRowidMethod, /* xRowid */
19135 0, /* xUpdate */
19136 0, /* xBegin */
19137 0, /* xSync */
19138 0, /* xCommit */
19139 0, /* xRollback */
19140 0, /* xFindFunction */
19141 0, /* xRename */
19142 0, /* xSavepoint */
19143 0, /* xRelease */
19144 0, /* xRollbackTo */
19145 0, /* xShadowName */
19146 0 /* xIntegrity */
19147 };
19148 rc = sqlite3_create_modulesqlite3_api->create_module(db, "fts5_structure", &fts5structure_module, 0);
19149 }
19150 return rc;
19151#else
19152 return SQLITE_OK0;
19153 UNUSED_PARAM(db)(void)(db);
19154#endif
19155}
19156
19157
19158static int sqlite3Fts5IndexReset(Fts5Index *p){
19159 assert( p->pStruct==0 || p->iStructVersion!=0 )((void) (0));
19160 if( fts5IndexDataVersion(p)!=p->iStructVersion ){
19161 fts5StructureInvalidate(p);
19162 }
19163 return fts5IndexReturn(p);
19164}
19165
19166#line 1 "fts5_main.c"
19167/*
19168** 2014 Jun 09
19169**
19170** The author disclaims copyright to this source code. In place of
19171** a legal notice, here is a blessing:
19172**
19173** May you do good and not evil.
19174** May you find forgiveness for yourself and forgive others.
19175** May you share freely, never taking more than you give.
19176**
19177******************************************************************************
19178**
19179** This is an SQLite module implementing full-text search.
19180*/
19181
19182
19183/* #include "fts5Int.h" */
19184
19185/*
19186** This variable is set to false when running tests for which the on disk
19187** structures should not be corrupt. Otherwise, true. If it is false, extra
19188** assert() conditions in the fts5 code are activated - conditions that are
19189** only true if it is guaranteed that the fts5 database is not corrupt.
19190*/
19191#ifdef SQLITE_DEBUG
19192int sqlite3_fts5_may_be_corrupt = 1;
19193#endif
19194
19195
19196typedef struct Fts5Auxdata Fts5Auxdata;
19197typedef struct Fts5Auxiliary Fts5Auxiliary;
19198typedef struct Fts5Cursor Fts5Cursor;
19199typedef struct Fts5FullTable Fts5FullTable;
19200typedef struct Fts5Sorter Fts5Sorter;
19201typedef struct Fts5TokenizerModule Fts5TokenizerModule;
19202
19203/*
19204** NOTES ON TRANSACTIONS:
19205**
19206** SQLite invokes the following virtual table methods as transactions are
19207** opened and closed by the user:
19208**
19209** xBegin(): Start of a new transaction.
19210** xSync(): Initial part of two-phase commit.
19211** xCommit(): Final part of two-phase commit.
19212** xRollback(): Rollback the transaction.
19213**
19214** Anything that is required as part of a commit that may fail is performed
19215** in the xSync() callback. Current versions of SQLite ignore any errors
19216** returned by xCommit().
19217**
19218** And as sub-transactions are opened/closed:
19219**
19220** xSavepoint(int S): Open savepoint S.
19221** xRelease(int S): Commit and close savepoint S.
19222** xRollbackTo(int S): Rollback to start of savepoint S.
19223**
19224** During a write-transaction the fts5_index.c module may cache some data
19225** in-memory. It is flushed to disk whenever xSync(), xRelease() or
19226** xSavepoint() is called. And discarded whenever xRollback() or xRollbackTo()
19227** is called.
19228**
19229** Additionally, if SQLITE_DEBUG is defined, an instance of the following
19230** structure is used to record the current transaction state. This information
19231** is not required, but it is used in the assert() statements executed by
19232** function fts5CheckTransactionState() (see below).
19233*/
19234struct Fts5TransactionState {
19235 int eState; /* 0==closed, 1==open, 2==synced */
19236 int iSavepoint; /* Number of open savepoints (0 -> none) */
19237};
19238
19239/*
19240** A single object of this type is allocated when the FTS5 module is
19241** registered with a database handle. It is used to store pointers to
19242** all registered FTS5 extensions - tokenizers and auxiliary functions.
19243*/
19244struct Fts5Global {
19245 fts5_api api; /* User visible part of object (see fts5.h) */
19246 sqlite3 *db; /* Associated database connection */
19247 i64 iNextId; /* Used to allocate unique cursor ids */
19248 Fts5Auxiliary *pAux; /* First in list of all aux. functions */
19249 Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */
19250 Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */
19251 Fts5Cursor *pCsr; /* First in list of all open cursors */
19252 u32 aLocaleHdr[4];
19253};
19254
19255/*
19256** Size of header on fts5_locale() values. And macro to access a buffer
19257** containing a copy of the header from an Fts5Config pointer.
19258*/
19259#define FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )) ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ))
19260#define FTS5_LOCALE_HDR(pConfig)((const u8*)(pConfig->pGlobal->aLocaleHdr)) ((const u8*)(pConfig->pGlobal->aLocaleHdr))
19261
19262#define FTS5_INSTTOKEN_SUBTYPE73 73
19263
19264/*
19265** Each auxiliary function registered with the FTS5 module is represented
19266** by an object of the following type. All such objects are stored as part
19267** of the Fts5Global.pAux list.
19268*/
19269struct Fts5Auxiliary {
19270 Fts5Global *pGlobal; /* Global context for this function */
19271 char *zFunc; /* Function name (nul-terminated) */
19272 void *pUserData; /* User-data pointer */
19273 fts5_extension_function xFunc; /* Callback function */
19274 void (*xDestroy)(void*); /* Destructor function */
19275 Fts5Auxiliary *pNext; /* Next registered auxiliary function */
19276};
19277
19278/*
19279** Each tokenizer module registered with the FTS5 module is represented
19280** by an object of the following type. All such objects are stored as part
19281** of the Fts5Global.pTok list.
19282**
19283** bV2Native:
19284** True if the tokenizer was registered using xCreateTokenizer_v2(), false
19285** for xCreateTokenizer(). If this variable is true, then x2 is populated
19286** with the routines as supplied by the caller and x1 contains synthesized
19287** wrapper routines. In this case the user-data pointer passed to
19288** x1.xCreate should be a pointer to the Fts5TokenizerModule structure,
19289** not a copy of pUserData.
19290**
19291** Of course, if bV2Native is false, then x1 contains the real routines and
19292** x2 the synthesized ones. In this case a pointer to the Fts5TokenizerModule
19293** object should be passed to x2.xCreate.
19294**
19295** The synthesized wrapper routines are necessary for xFindTokenizer(_v2)
19296** calls.
19297*/
19298struct Fts5TokenizerModule {
19299 char *zName; /* Name of tokenizer */
19300 void *pUserData; /* User pointer passed to xCreate() */
19301 int bV2Native; /* True if v2 native tokenizer */
19302 fts5_tokenizer x1; /* Tokenizer functions */
19303 fts5_tokenizer_v2 x2; /* V2 tokenizer functions */
19304 void (*xDestroy)(void*); /* Destructor function */
19305 Fts5TokenizerModule *pNext; /* Next registered tokenizer module */
19306};
19307
19308struct Fts5FullTable {
19309 Fts5Table p; /* Public class members from fts5Int.h */
19310 Fts5Storage *pStorage; /* Document store */
19311 Fts5Global *pGlobal; /* Global (connection wide) data */
19312 Fts5Cursor *pSortCsr; /* Sort data from this cursor */
19313 int iSavepoint; /* Successful xSavepoint()+1 */
19314
19315#ifdef SQLITE_DEBUG
19316 struct Fts5TransactionState ts;
19317#endif
19318};
19319
19320struct Fts5MatchPhrase {
19321 Fts5Buffer *pPoslist; /* Pointer to current poslist */
19322 int nTerm; /* Size of phrase in terms */
19323};
19324
19325/*
19326** pStmt:
19327** SELECT rowid, <fts> FROM <fts> ORDER BY +rank;
19328**
19329** aIdx[]:
19330** There is one entry in the aIdx[] array for each phrase in the query,
19331** the value of which is the offset within aPoslist[] following the last
19332** byte of the position list for the corresponding phrase.
19333*/
19334struct Fts5Sorter {
19335 sqlite3_stmt *pStmt;
19336 i64 iRowid; /* Current rowid */
19337 const u8 *aPoslist; /* Position lists for current row */
19338 int nIdx; /* Number of entries in aIdx[] */
19339 int aIdx[FLEXARRAY]; /* Offsets into aPoslist for current row */
19340};
19341
19342/* Size (int bytes) of an Fts5Sorter object with N indexes */
19343#define SZ_FTS5SORTER(N)(__builtin_offsetof(Fts5Sorter, nIdx)+((N+2)/2)*sizeof(i64)) (offsetof(Fts5Sorter,nIdx)__builtin_offsetof(Fts5Sorter, nIdx)+((N+2)/2)*sizeof(i64))
19344
19345/*
19346** Virtual-table cursor object.
19347**
19348** iSpecial:
19349** If this is a 'special' query (refer to function fts5SpecialMatch()),
19350** then this variable contains the result of the query.
19351**
19352** iFirstRowid, iLastRowid:
19353** These variables are only used for FTS5_PLAN_MATCH cursors. Assuming the
19354** cursor iterates in ascending order of rowids, iFirstRowid is the lower
19355** limit of rowids to return, and iLastRowid the upper. In other words, the
19356** WHERE clause in the user's query might have been:
19357**
19358** <tbl> MATCH <expr> AND rowid BETWEEN $iFirstRowid AND $iLastRowid
19359**
19360** If the cursor iterates in descending order of rowid, iFirstRowid
19361** is the upper limit (i.e. the "first" rowid visited) and iLastRowid
19362** the lower.
19363*/
19364struct Fts5Cursor {
19365 sqlite3_vtab_cursor base; /* Base class used by SQLite core */
19366 Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */
19367 int *aColumnSize; /* Values for xColumnSize() */
19368 i64 iCsrId; /* Cursor id */
19369
19370 /* Zero from this point onwards on cursor reset */
19371 int ePlan; /* FTS5_PLAN_XXX value */
19372 int bDesc; /* True for "ORDER BY rowid DESC" queries */
19373 i64 iFirstRowid; /* Return no rowids earlier than this */
19374 i64 iLastRowid; /* Return no rowids later than this */
19375 sqlite3_stmt *pStmt; /* Statement used to read %_content */
19376 Fts5Expr *pExpr; /* Expression for MATCH queries */
19377 Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */
19378 int csrflags; /* Mask of cursor flags (see below) */
19379 i64 iSpecial; /* Result of special query */
19380
19381 /* "rank" function. Populated on demand from vtab.xColumn(). */
19382 char *zRank; /* Custom rank function */
19383 char *zRankArgs; /* Custom rank function args */
19384 Fts5Auxiliary *pRank; /* Rank callback (or NULL) */
19385 int nRankArg; /* Number of trailing arguments for rank() */
19386 sqlite3_value **apRankArg; /* Array of trailing arguments */
19387 sqlite3_stmt *pRankArgStmt; /* Origin of objects in apRankArg[] */
19388
19389 /* Auxiliary data storage */
19390 Fts5Auxiliary *pAux; /* Currently executing extension function */
19391 Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */
19392
19393 /* Cache used by auxiliary API functions xInst() and xInstCount() */
19394 Fts5PoslistReader *aInstIter; /* One for each phrase */
19395 int nInstAlloc; /* Size of aInst[] array (entries / 3) */
19396 int nInstCount; /* Number of phrase instances */
19397 int *aInst; /* 3 integers per phrase instance */
19398};
19399
19400/*
19401** Bits that make up the "idxNum" parameter passed indirectly by
19402** xBestIndex() to xFilter().
19403*/
19404#define FTS5_BI_MATCH0x0001 0x0001 /* <tbl> MATCH ? */
19405#define FTS5_BI_RANK0x0002 0x0002 /* rank MATCH ? */
19406#define FTS5_BI_ROWID_EQ0x0004 0x0004 /* rowid == ? */
19407#define FTS5_BI_ROWID_LE0x0008 0x0008 /* rowid <= ? */
19408#define FTS5_BI_ROWID_GE0x0010 0x0010 /* rowid >= ? */
19409
19410#define FTS5_BI_ORDER_RANK0x0020 0x0020
19411#define FTS5_BI_ORDER_ROWID0x0040 0x0040
19412#define FTS5_BI_ORDER_DESC0x0080 0x0080
19413
19414/*
19415** Values for Fts5Cursor.csrflags
19416*/
19417#define FTS5CSR_EOF0x01 0x01
19418#define FTS5CSR_REQUIRE_CONTENT0x02 0x02
19419#define FTS5CSR_REQUIRE_DOCSIZE0x04 0x04
19420#define FTS5CSR_REQUIRE_INST0x08 0x08
19421#define FTS5CSR_FREE_ZRANK0x10 0x10
19422#define FTS5CSR_REQUIRE_RESEEK0x20 0x20
19423#define FTS5CSR_REQUIRE_POSLIST0x40 0x40
19424
19425#define BitFlagAllTest(x,y)(((x) & (y))==(y)) (((x) & (y))==(y))
19426#define BitFlagTest(x,y)(((x) & (y))!=0) (((x) & (y))!=0)
19427
19428
19429/*
19430** Macros to Set(), Clear() and Test() cursor flags.
19431*/
19432#define CsrFlagSet(pCsr, flag)((pCsr)->csrflags |= (flag)) ((pCsr)->csrflags |= (flag))
19433#define CsrFlagClear(pCsr, flag)((pCsr)->csrflags &= ~(flag)) ((pCsr)->csrflags &= ~(flag))
19434#define CsrFlagTest(pCsr, flag)((pCsr)->csrflags & (flag)) ((pCsr)->csrflags & (flag))
19435
19436struct Fts5Auxdata {
19437 Fts5Auxiliary *pAux; /* Extension to which this belongs */
19438 void *pPtr; /* Pointer value */
19439 void(*xDelete)(void*); /* Destructor */
19440 Fts5Auxdata *pNext; /* Next object in linked list */
19441};
19442
19443#ifdef SQLITE_DEBUG
19444#define FTS5_BEGIN 1
19445#define FTS5_SYNC 2
19446#define FTS5_COMMIT 3
19447#define FTS5_ROLLBACK 4
19448#define FTS5_SAVEPOINT 5
19449#define FTS5_RELEASE 6
19450#define FTS5_ROLLBACKTO 7
19451static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){
19452 switch( op ){
19453 case FTS5_BEGIN:
19454 assert( p->ts.eState==0 )((void) (0));
19455 p->ts.eState = 1;
19456 p->ts.iSavepoint = -1;
19457 break;
19458
19459 case FTS5_SYNC:
19460 assert( p->ts.eState==1 || p->ts.eState==2 )((void) (0));
19461 p->ts.eState = 2;
19462 break;
19463
19464 case FTS5_COMMIT:
19465 assert( p->ts.eState==2 )((void) (0));
19466 p->ts.eState = 0;
19467 break;
19468
19469 case FTS5_ROLLBACK:
19470 assert( p->ts.eState==1 || p->ts.eState==2 || p->ts.eState==0 )((void) (0));
19471 p->ts.eState = 0;
19472 break;
19473
19474 case FTS5_SAVEPOINT:
19475 assert( p->ts.eState>=1 )((void) (0));
19476 assert( iSavepoint>=0 )((void) (0));
19477 assert( iSavepoint>=p->ts.iSavepoint )((void) (0));
19478 p->ts.iSavepoint = iSavepoint;
19479 break;
19480
19481 case FTS5_RELEASE:
19482 assert( p->ts.eState>=1 )((void) (0));
19483 assert( iSavepoint>=0 )((void) (0));
19484 assert( iSavepoint<=p->ts.iSavepoint )((void) (0));
19485 p->ts.iSavepoint = iSavepoint-1;
19486 break;
19487
19488 case FTS5_ROLLBACKTO:
19489 assert( p->ts.eState>=1 )((void) (0));
19490 assert( iSavepoint>=-1 )((void) (0));
19491 /* The following assert() can fail if another vtab strikes an error
19492 ** within an xSavepoint() call then SQLite calls xRollbackTo() - without
19493 ** having called xSavepoint() on this vtab. */
19494 /* assert( iSavepoint<=p->ts.iSavepoint ); */
19495 p->ts.iSavepoint = iSavepoint;
19496 break;
19497 }
19498}
19499#else
19500# define fts5CheckTransactionState(x,y,z)
19501#endif
19502
19503/*
19504** Return true if pTab is a contentless table. If parameter bIncludeUnindexed
19505** is true, this includes contentless tables that store UNINDEXED columns
19506** only.
19507*/
19508static int fts5IsContentless(Fts5FullTable *pTab, int bIncludeUnindexed){
19509 int eContent = pTab->p.pConfig->eContent;
19510 return (
19511 eContent==FTS5_CONTENT_NONE1
19512 || (bIncludeUnindexed && eContent==FTS5_CONTENT_UNINDEXED3)
19513 );
19514}
19515
19516/*
19517** Delete a virtual table handle allocated by fts5InitVtab().
19518*/
19519static void fts5FreeVtab(Fts5FullTable *pTab){
19520 if( pTab ){
19521 sqlite3Fts5IndexClose(pTab->p.pIndex);
19522 sqlite3Fts5StorageClose(pTab->pStorage);
19523 sqlite3Fts5ConfigFree(pTab->p.pConfig);
19524 sqlite3_freesqlite3_api->free(pTab);
19525 }
19526}
19527
19528/*
19529** The xDisconnect() virtual table method.
19530*/
19531static int fts5DisconnectMethod(sqlite3_vtab *pVtab){
19532 fts5FreeVtab((Fts5FullTable*)pVtab);
19533 return SQLITE_OK0;
19534}
19535
19536/*
19537** The xDestroy() virtual table method.
19538*/
19539static int fts5DestroyMethod(sqlite3_vtab *pVtab){
19540 Fts5Table *pTab = (Fts5Table*)pVtab;
19541 int rc = sqlite3Fts5DropAll(pTab->pConfig);
19542 if( rc==SQLITE_OK0 ){
19543 fts5FreeVtab((Fts5FullTable*)pVtab);
19544 }
19545 return rc;
19546}
19547
19548/*
19549** This function is the implementation of both the xConnect and xCreate
19550** methods of the FTS3 virtual table.
19551**
19552** The argv[] array contains the following:
19553**
19554** argv[0] -> module name ("fts5")
19555** argv[1] -> database name
19556** argv[2] -> table name
19557** argv[...] -> "column name" and other module argument fields.
19558*/
19559static int fts5InitVtab(
19560 int bCreate, /* True for xCreate, false for xConnect */
19561 sqlite3 *db, /* The SQLite database connection */
19562 void *pAux, /* Hash table containing tokenizers */
19563 int argc, /* Number of elements in argv array */
19564 const char * const *argv, /* xCreate/xConnect argument array */
19565 sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
19566 char **pzErr /* Write any error message here */
19567){
19568 Fts5Global *pGlobal = (Fts5Global*)pAux;
19569 const char **azConfig = (const char**)argv;
19570 int rc = SQLITE_OK0; /* Return code */
19571 Fts5Config *pConfig = 0; /* Results of parsing argc/argv */
19572 Fts5FullTable *pTab = 0; /* New virtual table object */
19573
19574 /* Allocate the new vtab object and parse the configuration */
19575 pTab = (Fts5FullTable*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5FullTable));
19576 if( rc==SQLITE_OK0 ){
19577 rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
19578 assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 )((void) (0));
19579 }
19580 if( rc==SQLITE_OK0 ){
19581 pConfig->pzErrmsg = pzErr;
19582 pTab->p.pConfig = pConfig;
19583 pTab->pGlobal = pGlobal;
19584 if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){
19585 rc = sqlite3Fts5LoadTokenizer(pConfig);
19586 }
19587 }
19588
19589 /* Open the index sub-system */
19590 if( rc==SQLITE_OK0 ){
19591 rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->p.pIndex, pzErr);
19592 }
19593
19594 /* Open the storage sub-system */
19595 if( rc==SQLITE_OK0 ){
19596 rc = sqlite3Fts5StorageOpen(
19597 pConfig, pTab->p.pIndex, bCreate, &pTab->pStorage, pzErr
19598 );
19599 }
19600
19601 /* Call sqlite3_declare_vtab() */
19602 if( rc==SQLITE_OK0 ){
19603 rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
19604 }
19605
19606 /* Load the initial configuration */
19607 if( rc==SQLITE_OK0 ){
19608 rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie-1);
19609 }
19610
19611 if( rc==SQLITE_OK0 && pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
19612 rc = sqlite3_vtab_configsqlite3_api->vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT1, (int)1);
19613 }
19614 if( rc==SQLITE_OK0 ){
19615 rc = sqlite3_vtab_configsqlite3_api->vtab_config(db, SQLITE_VTAB_INNOCUOUS2);
19616 }
19617
19618 if( pConfig ) pConfig->pzErrmsg = 0;
19619 if( rc!=SQLITE_OK0 ){
19620 fts5FreeVtab(pTab);
19621 pTab = 0;
19622 }else if( bCreate ){
19623 fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
19624 }
19625 *ppVTab = (sqlite3_vtab*)pTab;
19626 return rc;
19627}
19628
19629/*
19630** The xConnect() and xCreate() methods for the virtual table. All the
19631** work is done in function fts5InitVtab().
19632*/
19633static int fts5ConnectMethod(
19634 sqlite3 *db, /* Database connection */
19635 void *pAux, /* Pointer to tokenizer hash table */
19636 int argc, /* Number of elements in argv array */
19637 const char * const *argv, /* xCreate/xConnect argument array */
19638 sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
19639 char **pzErr /* OUT: sqlite3_malloc'd error message */
19640){
19641 return fts5InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
19642}
19643static int fts5CreateMethod(
19644 sqlite3 *db, /* Database connection */
19645 void *pAux, /* Pointer to tokenizer hash table */
19646 int argc, /* Number of elements in argv array */
19647 const char * const *argv, /* xCreate/xConnect argument array */
19648 sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
19649 char **pzErr /* OUT: sqlite3_malloc'd error message */
19650){
19651 return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
19652}
19653
19654/*
19655** The different query plans.
19656*/
19657#define FTS5_PLAN_MATCH1 1 /* (<tbl> MATCH ?) */
19658#define FTS5_PLAN_SOURCE2 2 /* A source cursor for SORTED_MATCH */
19659#define FTS5_PLAN_SPECIAL3 3 /* An internal query */
19660#define FTS5_PLAN_SORTED_MATCH4 4 /* (<tbl> MATCH ? ORDER BY rank) */
19661#define FTS5_PLAN_SCAN5 5 /* No usable constraint */
19662#define FTS5_PLAN_ROWID6 6 /* (rowid = ?) */
19663
19664/*
19665** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this
19666** extension is currently being used by a version of SQLite too old to
19667** support index-info flags. In that case this function is a no-op.
19668*/
19669static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){
19670#if SQLITE_VERSION_NUMBER3053002>=3008012
19671#ifndef SQLITE_CORE
19672 if( sqlite3_libversion_numbersqlite3_api->libversion_number()>=3008012 )
19673#endif
19674 {
19675 pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE0x00000001;
19676 }
19677#endif
19678}
19679
19680static void fts5SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){
19681#if SQLITE_VERSION_NUMBER3053002>=3008002
19682#ifndef SQLITE_CORE
19683 if( sqlite3_libversion_numbersqlite3_api->libversion_number()>=3008002 )
19684#endif
19685 {
19686 pIdxInfo->estimatedRows = MAX(1, nRow)(((1) > (nRow)) ? (1) : (nRow));
19687 }
19688#endif
19689}
19690
19691static int fts5UsePatternMatch(
19692 Fts5Config *pConfig,
19693 struct sqlite3_index_constraint *p
19694){
19695 assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB )((void) (0));
19696 assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE )((void) (0));
19697 if( pConfig->t.ePattern==FTS5_PATTERN_GLOB66 && p->op==FTS5_PATTERN_GLOB66 ){
19698 return 1;
19699 }
19700 if( pConfig->t.ePattern==FTS5_PATTERN_LIKE65
19701 && (p->op==FTS5_PATTERN_LIKE65 || p->op==FTS5_PATTERN_GLOB66)
19702 ){
19703 return 1;
19704 }
19705 return 0;
19706}
19707
19708/*
19709** Implementation of the xBestIndex method for FTS5 tables. Within the
19710** WHERE constraint, it searches for the following:
19711**
19712** 1. A MATCH constraint against the table column.
19713** 2. A MATCH constraint against the "rank" column.
19714** 3. A MATCH constraint against some other column.
19715** 4. An == constraint against the rowid column.
19716** 5. A < or <= constraint against the rowid column.
19717** 6. A > or >= constraint against the rowid column.
19718**
19719** Within the ORDER BY, the following are supported:
19720**
19721** 5. ORDER BY rank [ASC|DESC]
19722** 6. ORDER BY rowid [ASC|DESC]
19723**
19724** Information for the xFilter call is passed via both the idxNum and
19725** idxStr variables. Specifically, idxNum is a bitmask of the following
19726** flags used to encode the ORDER BY clause:
19727**
19728** FTS5_BI_ORDER_RANK
19729** FTS5_BI_ORDER_ROWID
19730** FTS5_BI_ORDER_DESC
19731**
19732** idxStr is used to encode data from the WHERE clause. For each argument
19733** passed to the xFilter method, the following is appended to idxStr:
19734**
19735** Match against table column: "m"
19736** Match against rank column: "r"
19737** Match against other column: "M<column-number>"
19738** LIKE against other column: "L<column-number>"
19739** GLOB against other column: "G<column-number>"
19740** Equality constraint against the rowid: "="
19741** A < or <= against the rowid: "<"
19742** A > or >= against the rowid: ">"
19743**
19744** This function ensures that there is at most one "r" or "=". And that if
19745** there exists an "=" then there is no "<" or ">".
19746**
19747** If an unusable MATCH operator is present in the WHERE clause, then
19748** SQLITE_CONSTRAINT is returned.
19749**
19750** Costs are assigned as follows:
19751**
19752** a) If a MATCH operator is present, the cost depends on the other
19753** constraints also present. As follows:
19754**
19755** * No other constraints: cost=50000.0
19756** * One rowid range constraint: cost=37500.0
19757** * Both rowid range constraints: cost=30000.0
19758** * An == rowid constraint: cost=25000.0
19759**
19760** b) Otherwise, if there is no MATCH:
19761**
19762** * No other constraints: cost=3000000.0
19763** * One rowid range constraints: cost=2250000.0
19764** * Both rowid range constraint: cost=750000.0
19765** * An == rowid constraint: cost=25.0
19766**
19767** Costs are not modified by the ORDER BY clause.
19768**
19769** The ratios used in case (a) are based on informal results obtained from
19770** the tool/fts5cost.tcl script. The "MATCH and ==" combination has the
19771** cost set quite high because the query may be a prefix query. Unless
19772** there is a prefix index, prefix queries with rowid constraints are much
19773** more expensive than non-prefix queries with rowid constraints.
19774**
19775** The estimated rows returned is set to the cost/40. For simple queries,
19776** experimental results show that cost/4 might be about right. But for
19777** more complex queries that use multiple terms the number of rows might
19778** be far fewer than this. So we compromise and use cost/40.
19779*/
19780static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
19781 Fts5Table *pTab = (Fts5Table*)pVTab;
19782 Fts5Config *pConfig = pTab->pConfig;
19783 const int nCol = pConfig->nCol;
19784 int idxFlags = 0; /* Parameter passed through to xFilter() */
19785 int i;
19786
19787 char *idxStr;
19788 int iIdxStr = 0;
19789 int iCons = 0;
19790
19791 int bSeenEq = 0;
19792 int bSeenGt = 0;
19793 int bSeenLt = 0;
19794 int nSeenMatch = 0;
19795 int bSeenRank = 0;
19796
19797
19798 assert( SQLITE_INDEX_CONSTRAINT_EQ<SQLITE_INDEX_CONSTRAINT_MATCH )((void) (0));
19799 assert( SQLITE_INDEX_CONSTRAINT_GT<SQLITE_INDEX_CONSTRAINT_MATCH )((void) (0));
19800 assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH )((void) (0));
19801 assert( SQLITE_INDEX_CONSTRAINT_GE<SQLITE_INDEX_CONSTRAINT_MATCH )((void) (0));
19802 assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH )((void) (0));
19803
19804 if( pConfig->bLock ){
19805 pTab->base.zErrMsg = sqlite3_mprintfsqlite3_api->mprintf(
19806 "recursively defined fts5 content table"
19807 );
19808 return SQLITE_ERROR1;
19809 }
19810
19811 idxStr = (char*)sqlite3_malloc64sqlite3_api->malloc64((i64)pInfo->nConstraint * 8 + 1);
19812 if( idxStr==0 ) return SQLITE_NOMEM7;
19813 pInfo->idxStr = idxStr;
19814 pInfo->needToFreeIdxStr = 1;
19815
19816 for(i=0; i<pInfo->nConstraint; i++){
19817 struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
19818 int iCol = p->iColumn;
19819 if( p->op==SQLITE_INDEX_CONSTRAINT_MATCH64
19820 || (p->op==SQLITE_INDEX_CONSTRAINT_EQ2 && iCol>=nCol)
19821 ){
19822 /* A MATCH operator or equivalent */
19823 if( p->usable==0 || iCol<0 ){
19824 /* As there exists an unusable MATCH constraint this is an
19825 ** unusable plan. Return SQLITE_CONSTRAINT. */
19826 idxStr[iIdxStr] = 0;
19827 return SQLITE_CONSTRAINT19;
19828 }else{
19829 if( iCol==nCol+1 ){
19830 if( bSeenRank ) continue;
19831 idxStr[iIdxStr++] = 'r';
19832 bSeenRank = 1;
19833 }else{
19834 nSeenMatch++;
19835 idxStr[iIdxStr++] = 'M';
19836 sqlite3_snprintfsqlite3_api->xsnprintf(6, &idxStr[iIdxStr], "%d", iCol);
19837 iIdxStr += (int)strlen(&idxStr[iIdxStr]);
19838 assert( idxStr[iIdxStr]=='\0' )((void) (0));
19839 }
19840 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
19841 pInfo->aConstraintUsage[i].omit = 1;
19842 }
19843 }else if( p->usable ){
19844 if( iCol>=0 && iCol<nCol && fts5UsePatternMatch(pConfig, p) ){
19845 assert( p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB )((void) (0));
19846 idxStr[iIdxStr++] = p->op==FTS5_PATTERN_LIKE65 ? 'L' : 'G';
19847 sqlite3_snprintfsqlite3_api->xsnprintf(6, &idxStr[iIdxStr], "%d", iCol);
19848 idxStr += strlen(&idxStr[iIdxStr]);
19849 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
19850 assert( idxStr[iIdxStr]=='\0' )((void) (0));
19851 nSeenMatch++;
19852 }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ2 && iCol<0 ){
19853 idxStr[iIdxStr++] = '=';
19854 bSeenEq = 1;
19855 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
19856 pInfo->aConstraintUsage[i].omit = 1;
19857 }
19858 }
19859 }
19860
19861 if( bSeenEq==0 ){
19862 for(i=0; i<pInfo->nConstraint; i++){
19863 struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
19864 if( p->iColumn<0 && p->usable ){
19865 int op = p->op;
19866 if( op==SQLITE_INDEX_CONSTRAINT_LT16 || op==SQLITE_INDEX_CONSTRAINT_LE8 ){
19867 if( bSeenLt ) continue;
19868 idxStr[iIdxStr++] = '<';
19869 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
19870 bSeenLt = 1;
19871 }else
19872 if( op==SQLITE_INDEX_CONSTRAINT_GT4 || op==SQLITE_INDEX_CONSTRAINT_GE32 ){
19873 if( bSeenGt ) continue;
19874 idxStr[iIdxStr++] = '>';
19875 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
19876 bSeenGt = 1;
19877 }
19878 }
19879 }
19880 }
19881 idxStr[iIdxStr] = '\0';
19882
19883 /* Set idxFlags flags for the ORDER BY clause
19884 **
19885 ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC".
19886 */
19887 if( pInfo->nOrderBy==1 ){
19888 int iSort = pInfo->aOrderBy[0].iColumn;
19889 if( iSort==(pConfig->nCol+1) && nSeenMatch>0 ){
19890 idxFlags |= FTS5_BI_ORDER_RANK0x0020;
19891 }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){
19892 idxFlags |= FTS5_BI_ORDER_ROWID0x0040;
19893 }
19894 if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID)(((idxFlags) & (0x0020|0x0040))!=0) ){
19895 pInfo->orderByConsumed = 1;
19896 if( pInfo->aOrderBy[0].desc ){
19897 idxFlags |= FTS5_BI_ORDER_DESC0x0080;
19898 }
19899 }
19900 }
19901
19902 /* Calculate the estimated cost based on the flags set in idxFlags. */
19903 if( bSeenEq ){
19904 pInfo->estimatedCost = nSeenMatch ? 25000.0 : 25.0;
19905 fts5SetEstimatedRows(pInfo, 1);
19906 fts5SetUniqueFlag(pInfo);
19907 }else{
19908 i64 nEstRows;
19909 if( nSeenMatch ){
19910 if( bSeenLt && bSeenGt ){
19911 pInfo->estimatedCost = 50000.0;
19912 }else if( bSeenLt || bSeenGt ){
19913 pInfo->estimatedCost = 37500.0;
19914 }else{
19915 pInfo->estimatedCost = 50000.0;
19916 }
19917 nEstRows = (i64)(pInfo->estimatedCost / 40.0);
19918 for(i=1; i<nSeenMatch; i++){
19919 pInfo->estimatedCost *= 2.5;
19920 nEstRows = nEstRows / 2;
19921 }
19922 }else{
19923 if( bSeenLt && bSeenGt ){
19924 pInfo->estimatedCost = 750000.0;
19925 }else if( bSeenLt || bSeenGt ){
19926 pInfo->estimatedCost = 2250000.0;
19927 }else{
19928 pInfo->estimatedCost = 3000000.0;
19929 }
19930 nEstRows = (i64)(pInfo->estimatedCost / 4.0);
19931 }
19932 fts5SetEstimatedRows(pInfo, nEstRows);
19933 }
19934
19935 pInfo->idxNum = idxFlags;
19936 return SQLITE_OK0;
19937}
19938
19939static int fts5NewTransaction(Fts5FullTable *pTab){
19940 Fts5Cursor *pCsr;
19941 for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
19942 if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK0;
19943 }
19944 return sqlite3Fts5StorageReset(pTab->pStorage);
19945}
19946
19947/*
19948** Implementation of xOpen method.
19949*/
19950static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
19951 Fts5FullTable *pTab = (Fts5FullTable*)pVTab;
19952 Fts5Config *pConfig = pTab->p.pConfig;
19953 Fts5Cursor *pCsr = 0; /* New cursor object */
19954 sqlite3_int64 nByte; /* Bytes of space to allocate */
19955 int rc; /* Return code */
19956
19957 rc = fts5NewTransaction(pTab);
19958 if( rc==SQLITE_OK0 ){
19959 nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
19960 pCsr = (Fts5Cursor*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
19961 if( pCsr ){
19962 Fts5Global *pGlobal = pTab->pGlobal;
19963 memset(pCsr, 0, (size_t)nByte);
19964 pCsr->aColumnSize = (int*)&pCsr[1];
19965 pCsr->pNext = pGlobal->pCsr;
19966 pGlobal->pCsr = pCsr;
19967 pCsr->iCsrId = ++pGlobal->iNextId;
19968 }else{
19969 rc = SQLITE_NOMEM7;
19970 }
19971 }
19972 *ppCsr = (sqlite3_vtab_cursor*)pCsr;
19973 return rc;
19974}
19975
19976static int fts5StmtType(Fts5Cursor *pCsr){
19977 if( pCsr->ePlan==FTS5_PLAN_SCAN5 ){
19978 return (pCsr->bDesc) ? FTS5_STMT_SCAN_DESC1 : FTS5_STMT_SCAN_ASC0;
19979 }
19980 return FTS5_STMT_LOOKUP2;
19981}
19982
19983/*
19984** This function is called after the cursor passed as the only argument
19985** is moved to point at a different row. It clears all cached data
19986** specific to the previous row stored by the cursor object.
19987*/
19988static void fts5CsrNewrow(Fts5Cursor *pCsr){
19989 CsrFlagSet(pCsr,((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40))
19990 FTS5CSR_REQUIRE_CONTENT((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40))
19991 | FTS5CSR_REQUIRE_DOCSIZE((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40))
19992 | FTS5CSR_REQUIRE_INST((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40))
19993 | FTS5CSR_REQUIRE_POSLIST((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40))
19994 )((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40));
19995}
19996
19997static void fts5FreeCursorComponents(Fts5Cursor *pCsr){
19998 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
19999 Fts5Auxdata *pData;
20000 Fts5Auxdata *pNext;
20001
20002 sqlite3_freesqlite3_api->free(pCsr->aInstIter);
20003 sqlite3_freesqlite3_api->free(pCsr->aInst);
20004 if( pCsr->pStmt ){
20005 int eStmt = fts5StmtType(pCsr);
20006 sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
20007 }
20008 if( pCsr->pSorter ){
20009 Fts5Sorter *pSorter = pCsr->pSorter;
20010 sqlite3_finalizesqlite3_api->finalize(pSorter->pStmt);
20011 sqlite3_freesqlite3_api->free(pSorter);
20012 }
20013
20014 if( pCsr->ePlan!=FTS5_PLAN_SOURCE2 ){
20015 sqlite3Fts5ExprFree(pCsr->pExpr);
20016 }
20017
20018 for(pData=pCsr->pAuxdata; pData; pData=pNext){
20019 pNext = pData->pNext;
20020 if( pData->xDelete ) pData->xDelete(pData->pPtr);
20021 sqlite3_freesqlite3_api->free(pData);
20022 }
20023
20024 sqlite3_finalizesqlite3_api->finalize(pCsr->pRankArgStmt);
20025 sqlite3_freesqlite3_api->free(pCsr->apRankArg);
20026
20027 if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK)((pCsr)->csrflags & (0x10)) ){
20028 sqlite3_freesqlite3_api->free(pCsr->zRank);
20029 sqlite3_freesqlite3_api->free(pCsr->zRankArgs);
20030 }
20031
20032 sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
20033 memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr));
20034}
20035
20036
20037/*
20038** Close the cursor. For additional information see the documentation
20039** on the xClose method of the virtual table interface.
20040*/
20041static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
20042 if( pCursor ){
20043 Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab);
20044 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
20045 Fts5Cursor **pp;
20046
20047 fts5FreeCursorComponents(pCsr);
20048 /* Remove the cursor from the Fts5Global.pCsr list */
20049 for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
20050 *pp = pCsr->pNext;
20051
20052 sqlite3_freesqlite3_api->free(pCsr);
20053 }
20054 return SQLITE_OK0;
20055}
20056
20057static int fts5SorterNext(Fts5Cursor *pCsr){
20058 Fts5Sorter *pSorter = pCsr->pSorter;
20059 int rc;
20060
20061 rc = sqlite3_stepsqlite3_api->step(pSorter->pStmt);
20062 if( rc==SQLITE_DONE101 ){
20063 rc = SQLITE_OK0;
20064 CsrFlagSet(pCsr, FTS5CSR_EOF|FTS5CSR_REQUIRE_CONTENT)((pCsr)->csrflags |= (0x01|0x02));
20065 }else if( rc==SQLITE_ROW100 ){
20066 const u8 *a;
20067 const u8 *aBlob;
20068 int nBlob;
20069 int i;
20070 int iOff = 0;
20071 rc = SQLITE_OK0;
20072
20073 pSorter->iRowid = sqlite3_column_int64sqlite3_api->column_int64(pSorter->pStmt, 0);
20074 nBlob = sqlite3_column_bytessqlite3_api->column_bytes(pSorter->pStmt, 1);
20075 aBlob = a = sqlite3_column_blobsqlite3_api->column_blob(pSorter->pStmt, 1);
20076
20077 /* nBlob==0 in detail=none mode. */
20078 if( nBlob>0 ){
20079 for(i=0; i<(pSorter->nIdx-1); i++){
20080 int iVal;
20081 a += fts5GetVarint32(a, iVal)sqlite3Fts5GetVarint32(a,(u32*)&(iVal));
20082 iOff += iVal;
20083 pSorter->aIdx[i] = iOff;
20084 }
20085 pSorter->aIdx[i] = &aBlob[nBlob] - a;
20086 pSorter->aPoslist = a;
20087 }
20088
20089 fts5CsrNewrow(pCsr);
20090 }
20091
20092 return rc;
20093}
20094
20095
20096/*
20097** Set the FTS5CSR_REQUIRE_RESEEK flag on all FTS5_PLAN_MATCH cursors
20098** open on table pTab.
20099*/
20100static void fts5TripCursors(Fts5FullTable *pTab){
20101 Fts5Cursor *pCsr;
20102 for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
20103 if( pCsr->ePlan==FTS5_PLAN_MATCH1
20104 && pCsr->base.pVtab==(sqlite3_vtab*)pTab
20105 ){
20106 CsrFlagSet(pCsr, FTS5CSR_REQUIRE_RESEEK)((pCsr)->csrflags |= (0x20));
20107 }
20108 }
20109}
20110
20111/*
20112** If the REQUIRE_RESEEK flag is set on the cursor passed as the first
20113** argument, close and reopen all Fts5IndexIter iterators that the cursor
20114** is using. Then attempt to move the cursor to a rowid equal to or laster
20115** (in the cursors sort order - ASC or DESC) than the current rowid.
20116**
20117** If the new rowid is not equal to the old, set output parameter *pbSkip
20118** to 1 before returning. Otherwise, leave it unchanged.
20119**
20120** Return SQLITE_OK if successful or if no reseek was required, or an
20121** error code if an error occurred.
20122*/
20123static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
20124 int rc = SQLITE_OK0;
20125 assert( *pbSkip==0 )((void) (0));
20126 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_RESEEK)((pCsr)->csrflags & (0x20)) ){
20127 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
20128 int bDesc = pCsr->bDesc;
20129 i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
20130
20131 rc = sqlite3Fts5ExprFirst(
20132 pCsr->pExpr, pTab->p.pIndex, iRowid, pCsr->iLastRowid, bDesc
20133 );
20134 if( rc==SQLITE_OK0 && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){
20135 *pbSkip = 1;
20136 }
20137
20138 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK)((pCsr)->csrflags &= ~(0x20));
20139 fts5CsrNewrow(pCsr);
20140 if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
20141 CsrFlagSet(pCsr, FTS5CSR_EOF)((pCsr)->csrflags |= (0x01));
20142 *pbSkip = 1;
20143 }
20144 }
20145 return rc;
20146}
20147
20148
20149/*
20150** Advance the cursor to the next row in the table that matches the
20151** search criteria.
20152**
20153** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
20154** even if we reach end-of-file. The fts5EofMethod() will be called
20155** subsequently to determine whether or not an EOF was hit.
20156*/
20157static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
20158 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
20159 int rc;
20160
20161 assert( (pCsr->ePlan<3)==((void) (0))
20162 (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE)((void) (0))
20163 )((void) (0));
20164 assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) )((void) (0));
20165
20166 /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table,
20167 ** clear any token mappings accumulated at the fts5_index.c level. In
20168 ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH,
20169 ** we need to retain the mappings for the entire query. */
20170 if( pCsr->ePlan==FTS5_PLAN_MATCH1
20171 && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata
20172 ){
20173 sqlite3Fts5ExprClearTokens(pCsr->pExpr);
20174 }
20175
20176 if( pCsr->ePlan<3 ){
20177 int bSkip = 0;
20178 if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
20179 rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
20180 CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr))((pCsr)->csrflags |= (sqlite3Fts5ExprEof(pCsr->pExpr)));
20181 fts5CsrNewrow(pCsr);
20182 }else{
20183 switch( pCsr->ePlan ){
20184 case FTS5_PLAN_SPECIAL3: {
20185 CsrFlagSet(pCsr, FTS5CSR_EOF)((pCsr)->csrflags |= (0x01));
20186 rc = SQLITE_OK0;
20187 break;
20188 }
20189
20190 case FTS5_PLAN_SORTED_MATCH4: {
20191 rc = fts5SorterNext(pCsr);
20192 break;
20193 }
20194
20195 default: {
20196 Fts5Config *pConfig = ((Fts5Table*)pCursor->pVtab)->pConfig;
20197 pConfig->bLock++;
20198 rc = sqlite3_stepsqlite3_api->step(pCsr->pStmt);
20199 pConfig->bLock--;
20200 if( rc!=SQLITE_ROW100 ){
20201 CsrFlagSet(pCsr, FTS5CSR_EOF)((pCsr)->csrflags |= (0x01));
20202 rc = sqlite3_resetsqlite3_api->reset(pCsr->pStmt);
20203 if( rc!=SQLITE_OK0 ){
20204 pCursor->pVtab->zErrMsg = sqlite3_mprintfsqlite3_api->mprintf(
20205 "%s", sqlite3_errmsgsqlite3_api->errmsg(pConfig->db)
20206 );
20207 }
20208 }else{
20209 rc = SQLITE_OK0;
20210 CsrFlagSet(pCsr, FTS5CSR_REQUIRE_DOCSIZE)((pCsr)->csrflags |= (0x04));
20211 }
20212 break;
20213 }
20214 }
20215 }
20216
20217 return rc;
20218}
20219
20220
20221static int fts5PrepareStatement(
20222 sqlite3_stmt **ppStmt,
20223 Fts5Config *pConfig,
20224 const char *zFmt,
20225 ...
20226){
20227 sqlite3_stmt *pRet = 0;
20228 int rc;
20229 char *zSql;
20230 va_list ap;
20231
20232 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
20233 zSql = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
20234 if( zSql==0 ){
20235 rc = SQLITE_NOMEM7;
20236 }else{
20237 rc = sqlite3_prepare_v3sqlite3_api->prepare_v3(pConfig->db, zSql, -1,
20238 SQLITE_PREPARE_PERSISTENT0x01, &pRet, 0);
20239 if( rc!=SQLITE_OK0 ){
20240 sqlite3Fts5ConfigErrmsg(pConfig, "%s", sqlite3_errmsgsqlite3_api->errmsg(pConfig->db));
20241 }
20242 sqlite3_freesqlite3_api->free(zSql);
20243 }
20244
20245 va_end(ap)__builtin_va_end(ap);
20246 *ppStmt = pRet;
20247 return rc;
20248}
20249
20250static int fts5CursorFirstSorted(
20251 Fts5FullTable *pTab,
20252 Fts5Cursor *pCsr,
20253 int bDesc
20254){
20255 Fts5Config *pConfig = pTab->p.pConfig;
20256 Fts5Sorter *pSorter;
20257 int nPhrase;
20258 sqlite3_int64 nByte;
20259 int rc;
20260 const char *zRank = pCsr->zRank;
20261 const char *zRankArgs = pCsr->zRankArgs;
20262
20263 nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
20264 nByte = SZ_FTS5SORTER(nPhrase)(__builtin_offsetof(Fts5Sorter, nIdx)+((nPhrase+2)/2)*sizeof(
i64));
20265 pSorter = (Fts5Sorter*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
20266 if( pSorter==0 ) return SQLITE_NOMEM7;
20267 memset(pSorter, 0, (size_t)nByte);
20268 pSorter->nIdx = nPhrase;
20269
20270 /* TODO: It would be better to have some system for reusing statement
20271 ** handles here, rather than preparing a new one for each query. But that
20272 ** is not possible as SQLite reference counts the virtual table objects.
20273 ** And since the statement required here reads from this very virtual
20274 ** table, saving it creates a circular reference.
20275 **
20276 ** If SQLite a built-in statement cache, this wouldn't be a problem. */
20277 rc = fts5PrepareStatement(&pSorter->pStmt, pConfig,
20278 "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(\"%w\"%s%s) %s",
20279 pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
20280 (zRankArgs ? ", " : ""),
20281 (zRankArgs ? zRankArgs : ""),
20282 bDesc ? "DESC" : "ASC"
20283 );
20284
20285 pCsr->pSorter = pSorter;
20286 if( rc==SQLITE_OK0 ){
20287 assert( pTab->pSortCsr==0 )((void) (0));
20288 pTab->pSortCsr = pCsr;
20289 rc = fts5SorterNext(pCsr);
20290 pTab->pSortCsr = 0;
20291 }
20292
20293 if( rc!=SQLITE_OK0 ){
20294 sqlite3_finalizesqlite3_api->finalize(pSorter->pStmt);
20295 sqlite3_freesqlite3_api->free(pSorter);
20296 pCsr->pSorter = 0;
20297 }
20298
20299 return rc;
20300}
20301
20302static int fts5CursorFirst(Fts5FullTable *pTab, Fts5Cursor *pCsr, int bDesc){
20303 int rc;
20304 Fts5Expr *pExpr = pCsr->pExpr;
24
←
'pExpr' initialized to a null pointer value
→
20305 rc = sqlite3Fts5ExprFirst(
26
←
Calling 'sqlite3Fts5ExprFirst'
→
20306 pExpr, pTab->p.pIndex, pCsr->iFirstRowid, pCsr->iLastRowid, bDesc
25
←
Passing null pointer value via 1st parameter 'p'
→
20307 );
20308 if( sqlite3Fts5ExprEof(pExpr) ){
20309 CsrFlagSet(pCsr, FTS5CSR_EOF)((pCsr)->csrflags |= (0x01));
20310 }
20311 fts5CsrNewrow(pCsr);
20312 return rc;
20313}
20314
20315/*
20316** Process a "special" query. A special query is identified as one with a
20317** MATCH expression that begins with a '*' character. The remainder of
20318** the text passed to the MATCH operator are used as the special query
20319** parameters.
20320*/
20321static int fts5SpecialMatch(
20322 Fts5FullTable *pTab,
20323 Fts5Cursor *pCsr,
20324 const char *zQuery
20325){
20326 int rc = SQLITE_OK0; /* Return code */
20327 const char *z = zQuery; /* Special query text */
20328 int n; /* Number of bytes in text at z */
20329
20330 while( z[0]==' ' ) z++;
20331 for(n=0; z[n] && z[n]!=' '; n++);
20332
20333 assert( pTab->p.base.zErrMsg==0 )((void) (0));
20334 pCsr->ePlan = FTS5_PLAN_SPECIAL3;
20335
20336 if( n==5 && 0==sqlite3_strnicmpsqlite3_api->strnicmp("reads", z, n) ){
20337 pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->p.pIndex);
20338 }
20339 else if( n==2 && 0==sqlite3_strnicmpsqlite3_api->strnicmp("id", z, n) ){
20340 pCsr->iSpecial = pCsr->iCsrId;
20341 }
20342 else{
20343 /* An unrecognized directive. Return an error message. */
20344 pTab->p.base.zErrMsg = sqlite3_mprintfsqlite3_api->mprintf("unknown special query: %.*s", n, z);
20345 rc = SQLITE_ERROR1;
20346 }
20347
20348 return rc;
20349}
20350
20351/*
20352** Search for an auxiliary function named zName that can be used with table
20353** pTab. If one is found, return a pointer to the corresponding Fts5Auxiliary
20354** structure. Otherwise, if no such function exists, return NULL.
20355*/
20356static Fts5Auxiliary *fts5FindAuxiliary(Fts5FullTable *pTab, const char *zName){
20357 Fts5Auxiliary *pAux;
20358
20359 for(pAux=pTab->pGlobal->pAux; pAux; pAux=pAux->pNext){
20360 if( sqlite3_stricmpsqlite3_api->stricmp(zName, pAux->zFunc)==0 ) return pAux;
20361 }
20362
20363 /* No function of the specified name was found. Return 0. */
20364 return 0;
20365}
20366
20367
20368static int fts5FindRankFunction(Fts5Cursor *pCsr){
20369 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
20370 Fts5Config *pConfig = pTab->p.pConfig;
20371 int rc = SQLITE_OK0;
20372 Fts5Auxiliary *pAux = 0;
20373 const char *zRank = pCsr->zRank;
20374 const char *zRankArgs = pCsr->zRankArgs;
20375
20376 if( zRankArgs ){
20377 char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs);
20378 if( zSql ){
20379 sqlite3_stmt *pStmt = 0;
20380 rc = sqlite3_prepare_v3sqlite3_api->prepare_v3(pConfig->db, zSql, -1,
20381 SQLITE_PREPARE_PERSISTENT0x01, &pStmt, 0);
20382 sqlite3_freesqlite3_api->free(zSql);
20383 assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 )((void) (0));
20384 if( rc==SQLITE_OK0 ){
20385 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pStmt) ){
20386 sqlite3_int64 nByte;
20387 pCsr->nRankArg = sqlite3_column_countsqlite3_api->column_count(pStmt);
20388 nByte = sizeof(sqlite3_value*)*pCsr->nRankArg;
20389 pCsr->apRankArg = (sqlite3_value**)sqlite3Fts5MallocZero(&rc, nByte);
20390 if( rc==SQLITE_OK0 ){
20391 int i;
20392 for(i=0; i<pCsr->nRankArg; i++){
20393 pCsr->apRankArg[i] = sqlite3_column_valuesqlite3_api->column_value(pStmt, i);
20394 }
20395 }
20396 pCsr->pRankArgStmt = pStmt;
20397 }else{
20398 rc = sqlite3_finalizesqlite3_api->finalize(pStmt);
20399 assert( rc!=SQLITE_OK )((void) (0));
20400 }
20401 }
20402 }
20403 }
20404
20405 if( rc==SQLITE_OK0 ){
20406 pAux = fts5FindAuxiliary(pTab, zRank);
20407 if( pAux==0 ){
20408 assert( pTab->p.base.zErrMsg==0 )((void) (0));
20409 pTab->p.base.zErrMsg = sqlite3_mprintfsqlite3_api->mprintf("no such function: %s", zRank);
20410 rc = SQLITE_ERROR1;
20411 }
20412 }
20413
20414 pCsr->pRank = pAux;
20415 return rc;
20416}
20417
20418
20419static int fts5CursorParseRank(
20420 Fts5Config *pConfig,
20421 Fts5Cursor *pCsr,
20422 sqlite3_value *pRank
20423){
20424 int rc = SQLITE_OK0;
20425 if( pRank ){
20426 const char *z = (const char*)sqlite3_value_textsqlite3_api->value_text(pRank);
20427 char *zRank = 0;
20428 char *zRankArgs = 0;
20429
20430 if( z==0 ){
20431 if( sqlite3_value_typesqlite3_api->value_type(pRank)==SQLITE_NULL5 ) rc = SQLITE_ERROR1;
20432 }else{
20433 rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);
20434 }
20435 if( rc==SQLITE_OK0 ){
20436 pCsr->zRank = zRank;
20437 pCsr->zRankArgs = zRankArgs;
20438 CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK)((pCsr)->csrflags |= (0x10));
20439 }else if( rc==SQLITE_ERROR1 ){
20440 pCsr->base.pVtab->zErrMsg = sqlite3_mprintfsqlite3_api->mprintf(
20441 "parse error in rank function: %s", z
20442 );
20443 }
20444 }else{
20445 if( pConfig->zRank ){
20446 pCsr->zRank = (char*)pConfig->zRank;
20447 pCsr->zRankArgs = (char*)pConfig->zRankArgs;
20448 }else{
20449 pCsr->zRank = (char*)FTS5_DEFAULT_RANK"bm25";
20450 pCsr->zRankArgs = 0;
20451 }
20452 }
20453 return rc;
20454}
20455
20456static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){
20457 if( pVal ){
20458 int eType = sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal);
20459 if( eType==SQLITE_INTEGER1 ){
20460 return sqlite3_value_int64sqlite3_api->value_int64(pVal);
20461 }
20462 }
20463 return iDefault;
20464}
20465
20466/*
20467** Set the error message on the virtual table passed as the first argument.
20468*/
20469static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){
20470 va_list ap; /* ... printf arguments */
20471 va_start(ap, zFormat)__builtin_va_start(ap, zFormat);
20472 sqlite3_freesqlite3_api->free(p->p.base.zErrMsg);
20473 p->p.base.zErrMsg = sqlite3_vmprintfsqlite3_api->vmprintf(zFormat, ap);
20474 va_end(ap)__builtin_va_end(ap);
20475}
20476
20477/*
20478** Arrange for subsequent calls to sqlite3Fts5Tokenize() to use the locale
20479** specified by pLocale/nLocale. The buffer indicated by pLocale must remain
20480** valid until after the final call to sqlite3Fts5Tokenize() that will use
20481** the locale.
20482*/
20483static void sqlite3Fts5SetLocale(
20484 Fts5Config *pConfig,
20485 const char *zLocale,
20486 int nLocale
20487){
20488 Fts5TokenizerConfig *pT = &pConfig->t;
20489 pT->pLocale = zLocale;
20490 pT->nLocale = nLocale;
20491}
20492
20493/*
20494** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale().
20495*/
20496static void sqlite3Fts5ClearLocale(Fts5Config *pConfig){
20497 sqlite3Fts5SetLocale(pConfig, 0, 0);
20498}
20499
20500/*
20501** Return true if the value passed as the only argument is an
20502** fts5_locale() value.
20503*/
20504static int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){
20505 int ret = 0;
20506 if( sqlite3_value_typesqlite3_api->value_type(pVal)==SQLITE_BLOB4 ){
20507 /* Call sqlite3_value_bytes() after sqlite3_value_blob() in this case.
20508 ** If the blob was created using zeroblob(), then sqlite3_value_blob()
20509 ** may call malloc(). If this malloc() fails, then the values returned
20510 ** by both value_blob() and value_bytes() will be 0. If value_bytes() were
20511 ** called first, then the NULL pointer returned by value_blob() might
20512 ** be dereferenced. */
20513 const u8 *pBlob = sqlite3_value_blobsqlite3_api->value_blob(pVal);
20514 int nBlob = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
20515 if( nBlob>FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ))
20516 && 0==memcmp(pBlob, FTS5_LOCALE_HDR(pConfig)((const u8*)(pConfig->pGlobal->aLocaleHdr)), FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )))
20517 ){
20518 ret = 1;
20519 }
20520 }
20521 return ret;
20522}
20523
20524/*
20525** Value pVal is guaranteed to be an fts5_locale() value, according to
20526** sqlite3Fts5IsLocaleValue(). This function extracts the text and locale
20527** from the value and returns them separately.
20528**
20529** If successful, SQLITE_OK is returned and (*ppText) and (*ppLoc) set
20530** to point to buffers containing the text and locale, as utf-8,
20531** respectively. In this case output parameters (*pnText) and (*pnLoc) are
20532** set to the sizes in bytes of these two buffers.
20533**
20534** Or, if an error occurs, then an SQLite error code is returned. The final
20535** value of the four output parameters is undefined in this case.
20536*/
20537static int sqlite3Fts5DecodeLocaleValue(
20538 sqlite3_value *pVal,
20539 const char **ppText,
20540 int *pnText,
20541 const char **ppLoc,
20542 int *pnLoc
20543){
20544 const char *p = sqlite3_value_blobsqlite3_api->value_blob(pVal);
20545 int n = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
20546 int nLoc = 0;
20547
20548 assert( sqlite3_value_type(pVal)==SQLITE_BLOB )((void) (0));
20549 assert( n>FTS5_LOCALE_HDR_SIZE )((void) (0));
20550
20551 for(nLoc=FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )); p[nLoc]; nLoc++){
20552 if( nLoc==(n-1) ){
20553 return SQLITE_MISMATCH20;
20554 }
20555 }
20556 *ppLoc = &p[FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ))];
20557 *pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ));
20558
20559 *ppText = &p[nLoc+1];
20560 *pnText = n - nLoc - 1;
20561 return SQLITE_OK0;
20562}
20563
20564/*
20565** Argument pVal is the text of a full-text search expression. It may or
20566** may not have been wrapped by fts5_locale(). This function extracts
20567** the text of the expression, and sets output variable (*pzText) to
20568** point to a nul-terminated buffer containing the expression.
20569**
20570** If pVal was an fts5_locale() value, then sqlite3Fts5SetLocale() is called
20571** to set the tokenizer to use the specified locale.
20572**
20573** If output variable (*pbFreeAndReset) is set to true, then the caller
20574** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer
20575** locale, and (b) call sqlite3_free() to free (*pzText).
20576*/
20577static int fts5ExtractExprText(
20578 Fts5Config *pConfig, /* Fts5 configuration */
20579 sqlite3_value *pVal, /* Value to extract expression text from */
20580 char **pzText, /* OUT: nul-terminated buffer of text */
20581 int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */
20582){
20583 int rc = SQLITE_OK0;
20584
20585 if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
20586 const char *pText = 0;
20587 int nText = 0;
20588 const char *pLoc = 0;
20589 int nLoc = 0;
20590 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
20591 *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, pText);
20592 if( rc==SQLITE_OK0 ){
20593 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
20594 }
20595 *pbFreeAndReset = 1;
20596 }else{
20597 *pzText = (char*)sqlite3_value_textsqlite3_api->value_text(pVal);
20598 *pbFreeAndReset = 0;
20599 }
20600
20601 return rc;
20602}
20603
20604
20605/*
20606** This is the xFilter interface for the virtual table. See
20607** the virtual table xFilter method documentation for additional
20608** information.
20609**
20610** There are three possible query strategies:
20611**
20612** 1. Full-text search using a MATCH operator.
20613** 2. A by-rowid lookup.
20614** 3. A full-table scan.
20615*/
20616static int fts5FilterMethod(
20617 sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
20618 int idxNum, /* Strategy index */
20619 const char *idxStr, /* Unused */
20620 int nVal, /* Number of elements in apVal */
20621 sqlite3_value **apVal /* Arguments for the indexing scheme */
20622){
20623 Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab);
20624 Fts5Config *pConfig = pTab->p.pConfig;
20625 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
20626 int rc = SQLITE_OK0; /* Error code */
20627 int bDesc; /* True if ORDER BY [rank|rowid] DESC */
20628 int bOrderByRank; /* True if ORDER BY rank */
20629 sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */
20630 sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */
20631 sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */
20632 sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */
20633 int iCol; /* Column on LHS of MATCH operator */
20634 char **pzErrmsg = pConfig->pzErrmsg;
20635 int bPrefixInsttoken = pConfig->bPrefixInsttoken;
20636 int i;
20637 int iIdxStr = 0;
20638 Fts5Expr *pExpr = 0;
20639
20640 assert( pConfig->bLock==0 )((void) (0));
20641 if( pCsr->ePlan ){
20642 fts5FreeCursorComponents(pCsr);
20643 memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr));
20644 }
20645
20646 assert( pCsr->pStmt==0 )((void) (0));
20647 assert( pCsr->pExpr==0 )((void) (0));
20648 assert( pCsr->csrflags==0 )((void) (0));
20649 assert( pCsr->pRank==0 )((void) (0));
20650 assert( pCsr->zRank==0 )((void) (0));
20651 assert( pCsr->zRankArgs==0 )((void) (0));
20652 assert( pTab->pSortCsr==0 || nVal==0 )((void) (0));
20653
20654 assert( pzErrmsg==0 || pzErrmsg==&pTab->p.base.zErrMsg )((void) (0));
20655 pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
20656
20657 /* Decode the arguments passed through to this function. */
20658 for(i=0; i<nVal; i++){
20659 switch( idxStr[iIdxStr++] ){
20660 case 'r':
20661 pRank = apVal[i];
20662 break;
20663 case 'M': {
20664 char *zText = 0;
20665 int bFreeAndReset = 0;
20666 int bInternal = 0;
20667
20668 rc = fts5ExtractExprText(pConfig, apVal[i], &zText, &bFreeAndReset);
20669 if( rc!=SQLITE_OK0 ) goto filter_out;
20670 if( zText==0 ) zText = "";
20671 if( sqlite3_value_subtypesqlite3_api->value_subtype(apVal[i])==FTS5_INSTTOKEN_SUBTYPE73 ){
20672 pConfig->bPrefixInsttoken = 1;
20673 }
20674
20675 iCol = 0;
20676 do{
20677 iCol = iCol*10 + (idxStr[iIdxStr]-'0');
20678 iIdxStr++;
20679 }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' );
20680
20681 if( zText[0]=='*' ){
20682 /* The user has issued a query of the form "MATCH '*...'". This
20683 ** indicates that the MATCH expression is not a full text query,
20684 ** but a request for an internal parameter. */
20685 rc = fts5SpecialMatch(pTab, pCsr, &zText[1]);
20686 bInternal = 1;
20687 }else{
20688 char **pzErr = &pTab->p.base.zErrMsg;
20689 rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr);
20690 if( rc==SQLITE_OK0 ){
20691 rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr);
20692 pExpr = 0;
20693 }
20694 }
20695
20696 if( bFreeAndReset ){
20697 sqlite3_freesqlite3_api->free(zText);
20698 sqlite3Fts5ClearLocale(pConfig);
20699 }
20700
20701 if( bInternal || rc!=SQLITE_OK0 ) goto filter_out;
20702
20703 break;
20704 }
20705 case 'L':
20706 case 'G': {
20707 int bGlob = (idxStr[iIdxStr-1]=='G');
20708 const char *zText = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[i]);
20709 iCol = 0;
20710 do{
20711 iCol = iCol*10 + (idxStr[iIdxStr]-'0');
20712 iIdxStr++;
20713 }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' );
20714 if( zText ){
20715 rc = sqlite3Fts5ExprPattern(pConfig, bGlob, iCol, zText, &pExpr);
20716 }
20717 if( rc==SQLITE_OK0 ){
20718 rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr);
20719 pExpr = 0;
20720 }
20721 if( rc!=SQLITE_OK0 ) goto filter_out;
20722 break;
20723 }
20724 case '=':
20725 pRowidEq = apVal[i];
20726 break;
20727 case '<':
20728 pRowidLe = apVal[i];
20729 break;
20730 default: assert( idxStr[iIdxStr-1]=='>' )((void) (0));
20731 pRowidGe = apVal[i];
20732 break;
20733 }
20734 }
20735 bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK0x0020) ? 1 : 0);
20736 pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC0x0080) ? 1 : 0);
20737
20738 /* Set the cursor upper and lower rowid limits. Only some strategies
20739 ** actually use them. This is ok, as the xBestIndex() method leaves the
20740 ** sqlite3_index_constraint.omit flag clear for range constraints
20741 ** on the rowid field. */
20742 if( pRowidEq ){
20743 pRowidLe = pRowidGe = pRowidEq;
20744 }
20745 if( bDesc ){
20746 pCsr->iFirstRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)));
20747 pCsr->iLastRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32))));
20748 }else{
20749 pCsr->iLastRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)));
20750 pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32))));
20751 }
20752
20753 rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
20754 if( rc!=SQLITE_OK0 ) goto filter_out;
20755
20756 if( pTab->pSortCsr ){
20757 /* If pSortCsr is non-NULL, then this call is being made as part of
20758 ** processing for a "... MATCH <expr> ORDER BY rank" query (ePlan is
20759 ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will
20760 ** return results to the user for this query. The current cursor
20761 ** (pCursor) is used to execute the query issued by function
20762 ** fts5CursorFirstSorted() above. */
20763 assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 )((void) (0));
20764 assert( nVal==0 && bOrderByRank==0 && bDesc==0 )((void) (0));
20765 assert( pCsr->iLastRowid==LARGEST_INT64 )((void) (0));
20766 assert( pCsr->iFirstRowid==SMALLEST_INT64 )((void) (0));
20767 if( pTab->pSortCsr->bDesc ){
20768 pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid;
20769 pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid;
20770 }else{
20771 pCsr->iLastRowid = pTab->pSortCsr->iLastRowid;
20772 pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid;
20773 }
20774 pCsr->ePlan = FTS5_PLAN_SOURCE2;
20775 pCsr->pExpr = pTab->pSortCsr->pExpr;
20776 rc = fts5CursorFirst(pTab, pCsr, bDesc);
20777 }else if( pCsr->pExpr ){
20778 assert( rc==SQLITE_OK )((void) (0));
20779 rc = fts5CursorParseRank(pConfig, pCsr, pRank);
20780 if( rc==SQLITE_OK0 ){
20781 if( bOrderByRank ){
20782 pCsr->ePlan = FTS5_PLAN_SORTED_MATCH4;
20783 rc = fts5CursorFirstSorted(pTab, pCsr, bDesc);
20784 }else{
20785 pCsr->ePlan = FTS5_PLAN_MATCH1;
20786 rc = fts5CursorFirst(pTab, pCsr, bDesc);
20787 }
20788 }
20789 }else if( pConfig->zContent==0 ){
20790 fts5SetVtabError(pTab,"%s: table does not support scanning",pConfig->zName);
20791 rc = SQLITE_ERROR1;
20792 }else{
20793 /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup
20794 ** by rowid (ePlan==FTS5_PLAN_ROWID). */
20795 pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID6 : FTS5_PLAN_SCAN5);
20796 rc = sqlite3Fts5StorageStmt(
20797 pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->p.base.zErrMsg
20798 );
20799 if( rc==SQLITE_OK0 ){
20800 if( pRowidEq!=0 ){
20801 assert( pCsr->ePlan==FTS5_PLAN_ROWID )((void) (0));
20802 sqlite3_bind_valuesqlite3_api->bind_value(pCsr->pStmt, 1, pRowidEq);
20803 }else{
20804 sqlite3_bind_int64sqlite3_api->bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid);
20805 sqlite3_bind_int64sqlite3_api->bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid);
20806 }
20807 rc = fts5NextMethod(pCursor);
20808 }
20809 }
20810
20811 filter_out:
20812 sqlite3Fts5ExprFree(pExpr);
20813 pConfig->pzErrmsg = pzErrmsg;
20814 pConfig->bPrefixInsttoken = bPrefixInsttoken;
20815 return rc;
20816}
20817
20818/*
20819** This is the xEof method of the virtual table. SQLite calls this
20820** routine to find out if it has reached the end of a result set.
20821*/
20822static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
20823 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
20824 return (CsrFlagTest(pCsr, FTS5CSR_EOF)((pCsr)->csrflags & (0x01)) ? 1 : 0);
20825}
20826
20827/*
20828** Return the rowid that the cursor currently points to.
20829*/
20830static i64 fts5CursorRowid(Fts5Cursor *pCsr){
20831 assert( pCsr->ePlan==FTS5_PLAN_MATCH((void) (0))
20832 || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH((void) (0))
20833 || pCsr->ePlan==FTS5_PLAN_SOURCE((void) (0))
20834 || pCsr->ePlan==FTS5_PLAN_SCAN((void) (0))
20835 || pCsr->ePlan==FTS5_PLAN_ROWID((void) (0))
20836 )((void) (0));
20837 if( pCsr->pSorter ){
20838 return pCsr->pSorter->iRowid;
20839 }else if( pCsr->ePlan>=FTS5_PLAN_SCAN5 ){
20840 return sqlite3_column_int64sqlite3_api->column_int64(pCsr->pStmt, 0);
20841 }else{
20842 return sqlite3Fts5ExprRowid(pCsr->pExpr);
20843 }
20844}
20845
20846/*
20847** This is the xRowid method. The SQLite core calls this routine to
20848** retrieve the rowid for the current row of the result set. fts5
20849** exposes %_content.rowid as the rowid for the virtual table. The
20850** rowid should be written to *pRowid.
20851*/
20852static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
20853 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
20854 int ePlan = pCsr->ePlan;
20855
20856 assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 )((void) (0));
20857 if( ePlan==FTS5_PLAN_SPECIAL3 ){
20858 *pRowid = 0;
20859 }else{
20860 *pRowid = fts5CursorRowid(pCsr);
20861 }
20862
20863 return SQLITE_OK0;
20864}
20865
20866
20867/*
20868** If the cursor requires seeking (bSeekRequired flag is set), seek it.
20869** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise.
20870**
20871** If argument bErrormsg is true and an error occurs, an error message may
20872** be left in sqlite3_vtab.zErrMsg.
20873*/
20874static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){
20875 int rc = SQLITE_OK0;
20876
20877 /* If the cursor does not yet have a statement handle, obtain one now. */
20878 if( pCsr->pStmt==0 ){
20879 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
20880 int eStmt = fts5StmtType(pCsr);
20881 rc = sqlite3Fts5StorageStmt(
20882 pTab->pStorage, eStmt, &pCsr->pStmt, (bErrormsg?&pTab->p.base.zErrMsg:0)
20883 );
20884 assert( rc!=SQLITE_OK || pTab->p.base.zErrMsg==0 )((void) (0));
20885 assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) )((void) (0));
20886 }
20887
20888 if( rc==SQLITE_OK0 && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT)((pCsr)->csrflags & (0x02)) ){
20889 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
20890 assert( pCsr->pExpr )((void) (0));
20891 sqlite3_resetsqlite3_api->reset(pCsr->pStmt);
20892 sqlite3_bind_int64sqlite3_api->bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr));
20893 pTab->pConfig->bLock++;
20894 rc = sqlite3_stepsqlite3_api->step(pCsr->pStmt);
20895 pTab->pConfig->bLock--;
20896 if( rc==SQLITE_ROW100 ){
20897 rc = SQLITE_OK0;
20898 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT)((pCsr)->csrflags &= ~(0x02));
20899 }else{
20900 rc = sqlite3_resetsqlite3_api->reset(pCsr->pStmt);
20901 if( rc==SQLITE_OK0 ){
20902 rc = FTS5_CORRUPT(11 | (1<<8));
20903 fts5SetVtabError((Fts5FullTable*)pTab,
20904 "fts5: missing row %lld from content table %s",
20905 fts5CursorRowid(pCsr),
20906 pTab->pConfig->zContent
20907 );
20908 }else if( pTab->pConfig->pzErrmsg ){
20909 fts5SetVtabError((Fts5FullTable*)pTab,
20910 "%s", sqlite3_errmsgsqlite3_api->errmsg(pTab->pConfig->db)
20911 );
20912 }
20913 }
20914 }
20915 return rc;
20916}
20917
20918/*
20919** This function is called to handle an FTS INSERT command. In other words,
20920** an INSERT statement of the form:
20921**
20922** INSERT INTO fts(fts) VALUES($pCmd)
20923** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal)
20924**
20925** Argument pVal is the value assigned to column "fts" by the INSERT
20926** statement. This function returns SQLITE_OK if successful, or an SQLite
20927** error code if an error occurs.
20928**
20929** The commands implemented by this function are documented in the "Special
20930** INSERT Directives" section of the documentation. It should be updated if
20931** more commands are added to this function.
20932*/
20933static int fts5SpecialInsert(
20934 Fts5FullTable *pTab, /* Fts5 table object */
20935 const char *zCmd, /* Text inserted into table-name column */
20936 sqlite3_value *pVal /* Value inserted into rank column */
20937){
20938 Fts5Config *pConfig = pTab->p.pConfig;
20939 int rc = SQLITE_OK0;
20940 int bError = 0;
20941 int bLoadConfig = 0;
20942
20943 if( 0==sqlite3_stricmpsqlite3_api->stricmp("delete-all", zCmd) ){
20944 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
20945 fts5SetVtabError(pTab,
20946 "'delete-all' may only be used with a "
20947 "contentless or external content fts5 table"
20948 );
20949 rc = SQLITE_ERROR1;
20950 }else{
20951 rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
20952 }
20953 bLoadConfig = 1;
20954 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("rebuild", zCmd) ){
20955 if( fts5IsContentless(pTab, 1) ){
20956 fts5SetVtabError(pTab,
20957 "'rebuild' may not be used with a contentless fts5 table"
20958 );
20959 rc = SQLITE_ERROR1;
20960 }else{
20961 rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
20962 }
20963 bLoadConfig = 1;
20964 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("optimize", zCmd) ){
20965 rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
20966 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("merge", zCmd) ){
20967 int nMerge = sqlite3_value_intsqlite3_api->value_int(pVal);
20968 rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge);
20969 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("integrity-check", zCmd) ){
20970 int iArg = sqlite3_value_intsqlite3_api->value_int(pVal);
20971 rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, iArg);
20972#ifdef SQLITE_DEBUG
20973 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("prefix-index", zCmd) ){
20974 pConfig->bPrefixIndex = sqlite3_value_intsqlite3_api->value_int(pVal);
20975#endif
20976 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("flush", zCmd) ){
20977 rc = sqlite3Fts5FlushToDisk(&pTab->p);
20978 }else{
20979 rc = sqlite3Fts5FlushToDisk(&pTab->p);
20980 if( rc==SQLITE_OK0 ){
20981 rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
20982 }
20983 if( rc==SQLITE_OK0 ){
20984 rc = sqlite3Fts5ConfigSetValue(pTab->p.pConfig, zCmd, pVal, &bError);
20985 }
20986 if( rc==SQLITE_OK0 ){
20987 if( bError ){
20988 rc = SQLITE_ERROR1;
20989 }else{
20990 rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, zCmd, pVal, 0);
20991 }
20992 }
20993 }
20994
20995 if( rc==SQLITE_OK0 && bLoadConfig ){
20996 pTab->p.pConfig->iCookie--;
20997 rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
20998 }
20999
21000 return rc;
21001}
21002
21003static int fts5SpecialDelete(
21004 Fts5FullTable *pTab,
21005 sqlite3_value **apVal
21006){
21007 int rc = SQLITE_OK0;
21008 int eType1 = sqlite3_value_typesqlite3_api->value_type(apVal[1]);
21009 if( eType1==SQLITE_INTEGER1 ){
21010 sqlite3_int64 iDel = sqlite3_value_int64sqlite3_api->value_int64(apVal[1]);
21011 rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2], 0);
21012 }
21013 return rc;
21014}
21015
21016static void fts5StorageInsert(
21017 int *pRc,
21018 Fts5FullTable *pTab,
21019 sqlite3_value **apVal,
21020 i64 *piRowid
21021){
21022 int rc = *pRc;
21023 if( rc==SQLITE_OK0 ){
21024 rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, 0, apVal, piRowid);
21025 }
21026 if( rc==SQLITE_OK0 ){
21027 rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid);
21028 }
21029 *pRc = rc;
21030}
21031
21032/*
21033**
21034** This function is called when the user attempts an UPDATE on a contentless
21035** table. Parameter bRowidModified is true if the UPDATE statement modifies
21036** the rowid value. Parameter apVal[] contains the new values for each user
21037** defined column of the fts5 table. pConfig is the configuration object of the
21038** table being updated (guaranteed to be contentless). The contentless_delete=1
21039** and contentless_unindexed=1 options may or may not be set.
21040**
21041** This function returns SQLITE_OK if the UPDATE can go ahead, or an SQLite
21042** error code if it cannot. In this case an error message is also loaded into
21043** pConfig. Output parameter (*pbContent) is set to true if the caller should
21044** update the %_content table only - not the FTS index or any other shadow
21045** table. This occurs when an UPDATE modifies only UNINDEXED columns of the
21046** table.
21047**
21048** An UPDATE may proceed if:
21049**
21050** * The only columns modified are UNINDEXED columns, or
21051**
21052** * The contentless_delete=1 option was specified and all of the indexed
21053** columns (not a subset) have been modified.
21054*/
21055static int fts5ContentlessUpdate(
21056 Fts5Config *pConfig,
21057 sqlite3_value **apVal,
21058 int bRowidModified,
21059 int *pbContent
21060){
21061 int ii;
21062 int bSeenIndex = 0; /* Have seen modified indexed column */
21063 int bSeenIndexNC = 0; /* Have seen unmodified indexed column */
21064 int rc = SQLITE_OK0;
21065
21066 for(ii=0; ii<pConfig->nCol; ii++){
21067 if( pConfig->abUnindexed[ii]==0 ){
21068 if( sqlite3_value_nochangesqlite3_api->value_nochange(apVal[ii]) ){
21069 bSeenIndexNC++;
21070 }else{
21071 bSeenIndex++;
21072 }
21073 }
21074 }
21075
21076 if( bSeenIndex==0 && bRowidModified==0 ){
21077 *pbContent = 1;
21078 }else{
21079 if( bSeenIndexNC || pConfig->bContentlessDelete==0 ){
21080 rc = SQLITE_ERROR1;
21081 sqlite3Fts5ConfigErrmsg(pConfig,
21082 (pConfig->bContentlessDelete ?
21083 "%s a subset of columns on fts5 contentless-delete table: %s" :
21084 "%s contentless fts5 table: %s")
21085 , "cannot UPDATE", pConfig->zName
21086 );
21087 }
21088 }
21089
21090 return rc;
21091}
21092
21093/*
21094** This function is the implementation of the xUpdate callback used by
21095** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
21096** inserted, updated or deleted.
21097**
21098** A delete specifies a single argument - the rowid of the row to remove.
21099**
21100** Update and insert operations pass:
21101**
21102** 1. The "old" rowid, or NULL.
21103** 2. The "new" rowid.
21104** 3. Values for each of the nCol matchable columns.
21105** 4. Values for the two hidden columns (<tablename> and "rank").
21106*/
21107static int fts5UpdateMethod(
21108 sqlite3_vtab *pVtab, /* Virtual table handle */
21109 int nArg, /* Size of argument array */
21110 sqlite3_value **apVal, /* Array of arguments */
21111 sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
21112){
21113 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
21114 Fts5Config *pConfig = pTab->p.pConfig;
21115 int eType0; /* value_type() of apVal[0] */
21116 int rc = SQLITE_OK0; /* Return code */
21117
21118 /* A transaction must be open when this is called. */
21119 assert( pTab->ts.eState==1 || pTab->ts.eState==2 )((void) (0));
21120
21121 assert( pVtab->zErrMsg==0 )((void) (0));
21122 assert( nArg==1 || nArg==(2+pConfig->nCol+2) )((void) (0));
21123 assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER((void) (0))
21124 || sqlite3_value_type(apVal[0])==SQLITE_NULL((void) (0))
21125 )((void) (0));
21126 assert( pTab->p.pConfig->pzErrmsg==0 )((void) (0));
21127 if( pConfig->pgsz==0 ){
21128 rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie);
21129 if( rc!=SQLITE_OK0 ) return rc;
21130 }
21131
21132 pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
21133
21134 /* Put any active cursors into REQUIRE_SEEK state. */
21135 fts5TripCursors(pTab);
21136
21137 eType0 = sqlite3_value_typesqlite3_api->value_type(apVal[0]);
21138 if( eType0==SQLITE_NULL5
21139 && sqlite3_value_typesqlite3_api->value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL5
21140 ){
21141 /* A "special" INSERT op. These are handled separately. */
21142 const char *z = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[2+pConfig->nCol]);
21143 if( pConfig->eContent!=FTS5_CONTENT_NORMAL0
21144 && 0==sqlite3_stricmpsqlite3_api->stricmp("delete", z)
21145 ){
21146 if( pConfig->bContentlessDelete ){
21147 fts5SetVtabError(pTab,
21148 "'delete' may not be used with a contentless_delete=1 table"
21149 );
21150 rc = SQLITE_ERROR1;
21151 }else{
21152 rc = fts5SpecialDelete(pTab, apVal);
21153 }
21154 }else{
21155 rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
21156 }
21157 }else{
21158 /* A regular INSERT, UPDATE or DELETE statement. The trick here is that
21159 ** any conflict on the rowid value must be detected before any
21160 ** modifications are made to the database file. There are 4 cases:
21161 **
21162 ** 1) DELETE
21163 ** 2) UPDATE (rowid not modified)
21164 ** 3) UPDATE (rowid modified)
21165 ** 4) INSERT
21166 **
21167 ** Cases 3 and 4 may violate the rowid constraint.
21168 */
21169 int eConflict = SQLITE_ABORT4;
21170 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 || pConfig->bContentlessDelete ){
21171 eConflict = sqlite3_vtab_on_conflictsqlite3_api->vtab_on_conflict(pConfig->db);
21172 }
21173
21174 assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL )((void) (0));
21175 assert( nArg!=1 || eType0==SQLITE_INTEGER )((void) (0));
21176
21177 /* DELETE */
21178 if( nArg==1 ){
21179 /* It is only possible to DELETE from a contentless table if the
21180 ** contentless_delete=1 flag is set. */
21181 if( fts5IsContentless(pTab, 1) && pConfig->bContentlessDelete==0 ){
21182 fts5SetVtabError(pTab,
21183 "cannot DELETE from contentless fts5 table: %s", pConfig->zName
21184 );
21185 rc = SQLITE_ERROR1;
21186 }else{
21187 i64 iDel = sqlite3_value_int64sqlite3_api->value_int64(apVal[0]); /* Rowid to delete */
21188 rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0, 0);
21189 }
21190 }
21191
21192 /* INSERT or UPDATE */
21193 else{
21194 int eType1 = sqlite3_value_numeric_typesqlite3_api->value_numeric_type(apVal[1]);
21195
21196 /* It is an error to write an fts5_locale() value to a table without
21197 ** the locale=1 option. */
21198 if( pConfig->bLocale==0 ){
21199 int ii;
21200 for(ii=0; ii<pConfig->nCol; ii++){
21201 sqlite3_value *pVal = apVal[ii+2];
21202 if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
21203 fts5SetVtabError(pTab, "fts5_locale() requires locale=1");
21204 rc = SQLITE_MISMATCH20;
21205 goto update_out;
21206 }
21207 }
21208 }
21209
21210 if( eType0!=SQLITE_INTEGER1 ){
21211 /* An INSERT statement. If the conflict-mode is REPLACE, first remove
21212 ** the current entry (if any). */
21213 if( eConflict==SQLITE_REPLACE5 && eType1==SQLITE_INTEGER1 ){
21214 i64 iNew = sqlite3_value_int64sqlite3_api->value_int64(apVal[1]); /* Rowid to delete */
21215 rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0);
21216 }
21217 fts5StorageInsert(&rc, pTab, apVal, pRowid);
21218 }
21219
21220 /* UPDATE */
21221 else{
21222 Fts5Storage *pStorage = pTab->pStorage;
21223 i64 iOld = sqlite3_value_int64sqlite3_api->value_int64(apVal[0]); /* Old rowid */
21224 i64 iNew = sqlite3_value_int64sqlite3_api->value_int64(apVal[1]); /* New rowid */
21225 int bContent = 0; /* Content only update */
21226
21227 /* If this is a contentless table (including contentless_unindexed=1
21228 ** tables), check if the UPDATE may proceed. */
21229 if( fts5IsContentless(pTab, 1) ){
21230 rc = fts5ContentlessUpdate(pConfig, &apVal[2], iOld!=iNew, &bContent);
21231 if( rc!=SQLITE_OK0 ) goto update_out;
21232 }
21233
21234 if( eType1!=SQLITE_INTEGER1 ){
21235 rc = SQLITE_MISMATCH20;
21236 }else if( iOld!=iNew ){
21237 assert( bContent==0 )((void) (0));
21238 if( eConflict==SQLITE_REPLACE5 ){
21239 rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1);
21240 if( rc==SQLITE_OK0 ){
21241 rc = sqlite3Fts5StorageDelete(pStorage, iNew, 0, 0);
21242 }
21243 fts5StorageInsert(&rc, pTab, apVal, pRowid);
21244 }else{
21245 rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld);
21246 if( rc==SQLITE_OK0 ){
21247 rc = sqlite3Fts5StorageContentInsert(pStorage, 0, apVal, pRowid);
21248 }
21249 if( rc==SQLITE_OK0 ){
21250 rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 0);
21251 }
21252 if( rc==SQLITE_OK0 ){
21253 rc = sqlite3Fts5StorageIndexInsert(pStorage, apVal, *pRowid);
21254 }
21255 }
21256 }else if( bContent ){
21257 /* This occurs when an UPDATE on a contentless table affects *only*
21258 ** UNINDEXED columns. This is a no-op for contentless_unindexed=0
21259 ** tables, or a write to the %_content table only for =1 tables. */
21260 assert( fts5IsContentless(pTab, 1) )((void) (0));
21261 rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld);
21262 if( rc==SQLITE_OK0 ){
21263 rc = sqlite3Fts5StorageContentInsert(pStorage, 1, apVal, pRowid);
21264 }
21265 }else{
21266 rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1);
21267 fts5StorageInsert(&rc, pTab, apVal, pRowid);
21268 }
21269 sqlite3Fts5StorageReleaseDeleteRow(pStorage);
21270 }
21271 }
21272 }
21273
21274 update_out:
21275 sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
21276 pTab->p.pConfig->pzErrmsg = 0;
21277 return rc;
21278}
21279
21280/*
21281** Implementation of xSync() method.
21282*/
21283static int fts5SyncMethod(sqlite3_vtab *pVtab){
21284 int rc;
21285 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
21286 fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
21287 pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
21288 rc = sqlite3Fts5FlushToDisk(&pTab->p);
21289 pTab->p.pConfig->pzErrmsg = 0;
21290 return rc;
21291}
21292
21293/*
21294** Implementation of xBegin() method.
21295*/
21296static int fts5BeginMethod(sqlite3_vtab *pVtab){
21297 int rc = fts5NewTransaction((Fts5FullTable*)pVtab);
21298 if( rc==SQLITE_OK0 ){
21299 fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0);
21300 }
21301 return rc;
21302}
21303
21304/*
21305** Implementation of xCommit() method. This is a no-op. The contents of
21306** the pending-terms hash-table have already been flushed into the database
21307** by fts5SyncMethod().
21308*/
21309static int fts5CommitMethod(sqlite3_vtab *pVtab){
21310 UNUSED_PARAM(pVtab)(void)(pVtab); /* Call below is a no-op for NDEBUG builds */
21311 fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_COMMIT, 0);
21312 return SQLITE_OK0;
21313}
21314
21315/*
21316** Implementation of xRollback(). Discard the contents of the pending-terms
21317** hash-table. Any changes made to the database are reverted by SQLite.
21318*/
21319static int fts5RollbackMethod(sqlite3_vtab *pVtab){
21320 int rc;
21321 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
21322 fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0);
21323 rc = sqlite3Fts5StorageRollback(pTab->pStorage);
21324 pTab->p.pConfig->pgsz = 0;
21325 return rc;
21326}
21327
21328static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*);
21329
21330static void *fts5ApiUserData(Fts5Context *pCtx){
21331 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21332 return pCsr->pAux->pUserData;
21333}
21334
21335static int fts5ApiColumnCount(Fts5Context *pCtx){
21336 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21337 return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol;
21338}
21339
21340static int fts5ApiColumnTotalSize(
21341 Fts5Context *pCtx,
21342 int iCol,
21343 sqlite3_int64 *pnToken
21344){
21345 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21346 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
21347 return sqlite3Fts5StorageSize(pTab->pStorage, iCol, pnToken);
21348}
21349
21350static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){
21351 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21352 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
21353 return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow);
21354}
21355
21356/*
21357** Implementation of xTokenize_v2() API.
21358*/
21359static int fts5ApiTokenize_v2(
21360 Fts5Context *pCtx,
21361 const char *pText, int nText,
21362 const char *pLoc, int nLoc,
21363 void *pUserData,
21364 int (*xToken)(void*, int, const char*, int, int, int)
21365){
21366 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21367 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
21368 int rc = SQLITE_OK0;
21369
21370 sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc);
21371 rc = sqlite3Fts5Tokenize(pTab->pConfig,
21372 FTS5_TOKENIZE_AUX0x0008, pText, nText, pUserData, xToken
21373 );
21374 sqlite3Fts5SetLocale(pTab->pConfig, 0, 0);
21375
21376 return rc;
21377}
21378
21379/*
21380** Implementation of xTokenize() API. This is just xTokenize_v2() with NULL/0
21381** passed as the locale.
21382*/
21383static int fts5ApiTokenize(
21384 Fts5Context *pCtx,
21385 const char *pText, int nText,
21386 void *pUserData,
21387 int (*xToken)(void*, int, const char*, int, int, int)
21388){
21389 return fts5ApiTokenize_v2(pCtx, pText, nText, 0, 0, pUserData, xToken);
21390}
21391
21392static int fts5ApiPhraseCount(Fts5Context *pCtx){
21393 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21394 return sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
21395}
21396
21397static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
21398 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21399 return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
21400}
21401
21402/*
21403** Argument pStmt is an SQL statement of the type used by Fts5Cursor. This
21404** function extracts the text value of column iCol of the current row.
21405** Additionally, if there is an associated locale, it invokes
21406** sqlite3Fts5SetLocale() to configure the tokenizer. In all cases the caller
21407** should invoke sqlite3Fts5ClearLocale() to clear the locale at some point
21408** after this function returns.
21409**
21410** If successful, (*ppText) is set to point to a buffer containing the text
21411** value as utf-8 and SQLITE_OK returned. (*pnText) is set to the size of that
21412** buffer in bytes. It is not guaranteed to be nul-terminated. If an error
21413** occurs, an SQLite error code is returned. The final values of the two
21414** output parameters are undefined in this case.
21415*/
21416static int fts5TextFromStmt(
21417 Fts5Config *pConfig,
21418 sqlite3_stmt *pStmt,
21419 int iCol,
21420 const char **ppText,
21421 int *pnText
21422){
21423 sqlite3_value *pVal = sqlite3_column_valuesqlite3_api->column_value(pStmt, iCol+1);
21424 const char *pLoc = 0;
21425 int nLoc = 0;
21426 int rc = SQLITE_OK0;
21427
21428 if( pConfig->bLocale
21429 && pConfig->eContent==FTS5_CONTENT_EXTERNAL2
21430 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
21431 ){
21432 rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc);
21433 }else{
21434 *ppText = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
21435 *pnText = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
21436 if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
21437 pLoc = (const char*)sqlite3_column_textsqlite3_api->column_text(pStmt, iCol+1+pConfig->nCol);
21438 nLoc = sqlite3_column_bytessqlite3_api->column_bytes(pStmt, iCol+1+pConfig->nCol);
21439 }
21440 }
21441 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
21442 return rc;
21443}
21444
21445static int fts5ApiColumnText(
21446 Fts5Context *pCtx,
21447 int iCol,
21448 const char **pz,
21449 int *pn
21450){
21451 int rc = SQLITE_OK0;
21452 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21453 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
21454
21455 assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL )((void) (0));
21456 if( iCol<0 || iCol>=pTab->pConfig->nCol ){
21457 rc = SQLITE_RANGE25;
21458 }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab), 0) ){
21459 *pz = 0;
21460 *pn = 0;
21461 }else{
21462 rc = fts5SeekCursor(pCsr, 0);
21463 if( rc==SQLITE_OK0 ){
21464 rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn);
21465 sqlite3Fts5ClearLocale(pTab->pConfig);
21466 }
21467 }
21468 return rc;
21469}
21470
21471/*
21472** This is called by various API functions - xInst, xPhraseFirst,
21473** xPhraseFirstColumn etc. - to obtain the position list for phrase iPhrase
21474** of the current row. This function works for both detail=full tables (in
21475** which case the position-list was read from the fts index) or for other
21476** detail= modes if the row content is available.
21477*/
21478static int fts5CsrPoslist(
21479 Fts5Cursor *pCsr, /* Fts5 cursor object */
21480 int iPhrase, /* Phrase to find position list for */
21481 const u8 **pa, /* OUT: Pointer to position list buffer */
21482 int *pn /* OUT: Size of (*pa) in bytes */
21483){
21484 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
21485 int rc = SQLITE_OK0;
21486 int bLive = (pCsr->pSorter==0);
21487
21488 if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){
21489 rc = SQLITE_RANGE25;
21490 }else if( pConfig->eDetail!=FTS5_DETAIL_FULL0
21491 && fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1)
21492 ){
21493 *pa = 0;
21494 *pn = 0;
21495 return SQLITE_OK0;
21496 }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST)((pCsr)->csrflags & (0x40)) ){
21497 if( pConfig->eDetail!=FTS5_DETAIL_FULL0 ){
21498 Fts5PoslistPopulator *aPopulator;
21499 int i;
21500
21501 aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive);
21502 if( aPopulator==0 ) rc = SQLITE_NOMEM7;
21503 if( rc==SQLITE_OK0 ){
21504 rc = fts5SeekCursor(pCsr, 0);
21505 }
21506 for(i=0; i<pConfig->nCol && rc==SQLITE_OK0; i++){
21507 const char *z = 0;
21508 int n = 0;
21509 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
21510 if( rc==SQLITE_OK0 ){
21511 rc = sqlite3Fts5ExprPopulatePoslists(
21512 pConfig, pCsr->pExpr, aPopulator, i, z, n
21513 );
21514 }
21515 sqlite3Fts5ClearLocale(pConfig);
21516 }
21517 sqlite3_freesqlite3_api->free(aPopulator);
21518
21519 if( pCsr->pSorter ){
21520 sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid);
21521 }
21522 }
21523 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST)((pCsr)->csrflags &= ~(0x40));
21524 }
21525
21526 if( rc==SQLITE_OK0 ){
21527 if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL0 ){
21528 Fts5Sorter *pSorter = pCsr->pSorter;
21529 int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
21530 *pn = pSorter->aIdx[iPhrase] - i1;
21531 *pa = &pSorter->aPoslist[i1];
21532 }else{
21533 *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
21534 }
21535 }else{
21536 *pa = 0;
21537 *pn = 0;
21538 }
21539
21540 return rc;
21541}
21542
21543/*
21544** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated
21545** correctly for the current view. Return SQLITE_OK if successful, or an
21546** SQLite error code otherwise.
21547*/
21548static int fts5CacheInstArray(Fts5Cursor *pCsr){
21549 int rc = SQLITE_OK0;
21550 Fts5PoslistReader *aIter; /* One iterator for each phrase */
21551 int nIter; /* Number of iterators/phrases */
21552 int nCol = ((Fts5Table*)pCsr->base.pVtab)->pConfig->nCol;
21553
21554 nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
21555 if( pCsr->aInstIter==0 ){
21556 sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nIter;
21557 pCsr->aInstIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte);
21558 }
21559 aIter = pCsr->aInstIter;
21560
21561 if( aIter ){
21562 int nInst = 0; /* Number instances seen so far */
21563 int i;
21564
21565 /* Initialize all iterators */
21566 for(i=0; i<nIter && rc==SQLITE_OK0; i++){
21567 const u8 *a;
21568 int n;
21569 rc = fts5CsrPoslist(pCsr, i, &a, &n);
21570 if( rc==SQLITE_OK0 ){
21571 sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
21572 }
21573 }
21574
21575 if( rc==SQLITE_OK0 ){
21576 while( 1 ){
21577 int *aInst;
21578 int iBest = -1;
21579 for(i=0; i<nIter; i++){
21580 if( (aIter[i].bEof==0)
21581 && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos)
21582 ){
21583 iBest = i;
21584 }
21585 }
21586 if( iBest<0 ) break;
21587
21588 nInst++;
21589 if( nInst>=pCsr->nInstAlloc ){
21590 int nNewSize = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
21591 aInst = (int*)sqlite3_realloc64sqlite3_api->realloc64(
21592 pCsr->aInst, nNewSize*sizeof(int)*3
21593 );
21594 if( aInst ){
21595 pCsr->aInst = aInst;
21596 pCsr->nInstAlloc = nNewSize;
21597 }else{
21598 nInst--;
21599 rc = SQLITE_NOMEM7;
21600 break;
21601 }
21602 }
21603
21604 aInst = &pCsr->aInst[3 * (nInst-1)];
21605 aInst[0] = iBest;
21606 aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos)(int)((aIter[iBest].iPos >> 32) & 0x7FFFFFFF);
21607 aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos)(int)(aIter[iBest].iPos & 0x7FFFFFFF);
21608 assert( aInst[1]>=0 )((void) (0));
21609 if( aInst[1]>=nCol ){
21610 rc = FTS5_CORRUPT(11 | (1<<8));
21611 break;
21612 }
21613 sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
21614 }
21615 }
21616
21617 pCsr->nInstCount = nInst;
21618 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST)((pCsr)->csrflags &= ~(0x08));
21619 }
21620 return rc;
21621}
21622
21623static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){
21624 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21625 int rc = SQLITE_OK0;
21626 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)((pCsr)->csrflags & (0x08))==0
21627 || SQLITE_OK0==(rc = fts5CacheInstArray(pCsr)) ){
21628 *pnInst = pCsr->nInstCount;
21629 }
21630 return rc;
21631}
21632
21633static int fts5ApiInst(
21634 Fts5Context *pCtx,
21635 int iIdx,
21636 int *piPhrase,
21637 int *piCol,
21638 int *piOff
21639){
21640 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21641 int rc = SQLITE_OK0;
21642 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)((pCsr)->csrflags & (0x08))==0
21643 || SQLITE_OK0==(rc = fts5CacheInstArray(pCsr))
21644 ){
21645 if( iIdx<0 || iIdx>=pCsr->nInstCount ){
21646 rc = SQLITE_RANGE25;
21647 }else{
21648 *piPhrase = pCsr->aInst[iIdx*3];
21649 *piCol = pCsr->aInst[iIdx*3 + 1];
21650 *piOff = pCsr->aInst[iIdx*3 + 2];
21651 }
21652 }
21653 return rc;
21654}
21655
21656static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
21657 return fts5CursorRowid((Fts5Cursor*)pCtx);
21658}
21659
21660static int fts5ColumnSizeCb(
21661 void *pContext, /* Pointer to int */
21662 int tflags,
21663 const char *pUnused, /* Buffer containing token */
21664 int nUnused, /* Size of token in bytes */
21665 int iUnused1, /* Start offset of token */
21666 int iUnused2 /* End offset of token */
21667){
21668 int *pCnt = (int*)pContext;
21669 UNUSED_PARAM2(pUnused, nUnused)(void)(pUnused), (void)(nUnused);
21670 UNUSED_PARAM2(iUnused1, iUnused2)(void)(iUnused1), (void)(iUnused2);
21671 if( (tflags & FTS5_TOKEN_COLOCATED0x0001)==0 ){
21672 (*pCnt)++;
21673 }
21674 return SQLITE_OK0;
21675}
21676
21677static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
21678 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21679 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
21680 Fts5Config *pConfig = pTab->p.pConfig;
21681 int rc = SQLITE_OK0;
21682
21683 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE)((pCsr)->csrflags & (0x04)) ){
21684 if( pConfig->bColumnsize ){
21685 i64 iRowid = fts5CursorRowid(pCsr);
21686 rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
21687 }else if( !pConfig->zContent || pConfig->eContent==FTS5_CONTENT_UNINDEXED3 ){
21688 int i;
21689 for(i=0; i<pConfig->nCol; i++){
21690 if( pConfig->abUnindexed[i]==0 ){
21691 pCsr->aColumnSize[i] = -1;
21692 }
21693 }
21694 }else{
21695 int i;
21696 rc = fts5SeekCursor(pCsr, 0);
21697 for(i=0; rc==SQLITE_OK0 && i<pConfig->nCol; i++){
21698 if( pConfig->abUnindexed[i]==0 ){
21699 const char *z = 0;
21700 int n = 0;
21701 pCsr->aColumnSize[i] = 0;
21702 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
21703 if( rc==SQLITE_OK0 ){
21704 rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX0x0008,
21705 z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb
21706 );
21707 }
21708 sqlite3Fts5ClearLocale(pConfig);
21709 }
21710 }
21711 }
21712 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE)((pCsr)->csrflags &= ~(0x04));
21713 }
21714 if( iCol<0 ){
21715 int i;
21716 *pnToken = 0;
21717 for(i=0; i<pConfig->nCol; i++){
21718 *pnToken += pCsr->aColumnSize[i];
21719 }
21720 }else if( iCol<pConfig->nCol ){
21721 *pnToken = pCsr->aColumnSize[iCol];
21722 }else{
21723 *pnToken = 0;
21724 rc = SQLITE_RANGE25;
21725 }
21726 return rc;
21727}
21728
21729/*
21730** Implementation of the xSetAuxdata() method.
21731*/
21732static int fts5ApiSetAuxdata(
21733 Fts5Context *pCtx, /* Fts5 context */
21734 void *pPtr, /* Pointer to save as auxdata */
21735 void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */
21736){
21737 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21738 Fts5Auxdata *pData;
21739
21740 /* Search through the cursors list of Fts5Auxdata objects for one that
21741 ** corresponds to the currently executing auxiliary function. */
21742 for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){
21743 if( pData->pAux==pCsr->pAux ) break;
21744 }
21745
21746 if( pData ){
21747 if( pData->xDelete ){
21748 pData->xDelete(pData->pPtr);
21749 }
21750 }else{
21751 int rc = SQLITE_OK0;
21752 pData = (Fts5Auxdata*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Auxdata));
21753 if( pData==0 ){
21754 if( xDelete ) xDelete(pPtr);
21755 return rc;
21756 }
21757 pData->pAux = pCsr->pAux;
21758 pData->pNext = pCsr->pAuxdata;
21759 pCsr->pAuxdata = pData;
21760 }
21761
21762 pData->xDelete = xDelete;
21763 pData->pPtr = pPtr;
21764 return SQLITE_OK0;
21765}
21766
21767static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){
21768 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21769 Fts5Auxdata *pData;
21770 void *pRet = 0;
21771
21772 for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){
21773 if( pData->pAux==pCsr->pAux ) break;
21774 }
21775
21776 if( pData ){
21777 pRet = pData->pPtr;
21778 if( bClear ){
21779 pData->pPtr = 0;
21780 pData->xDelete = 0;
21781 }
21782 }
21783
21784 return pRet;
21785}
21786
21787static void fts5ApiPhraseNext(
21788 Fts5Context *pCtx,
21789 Fts5PhraseIter *pIter,
21790 int *piCol, int *piOff
21791){
21792 if( pIter->a>=pIter->b ){
21793 *piCol = -1;
21794 *piOff = -1;
21795 }else{
21796 int iVal;
21797 pIter->a += fts5GetVarint32(pIter->a, iVal)sqlite3Fts5GetVarint32(pIter->a,(u32*)&(iVal));
21798 if( iVal==1 ){
21799 /* Avoid returning a (*piCol) value that is too large for the table,
21800 ** even if the position-list is corrupt. The caller might not be
21801 ** expecting it. */
21802 int nCol = ((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab))->pConfig->nCol;
21803 pIter->a += fts5GetVarint32(pIter->a, iVal)sqlite3Fts5GetVarint32(pIter->a,(u32*)&(iVal));
21804 *piCol = (iVal>=nCol ? nCol-1 : iVal);
21805 *piOff = 0;
21806 pIter->a += fts5GetVarint32(pIter->a, iVal)sqlite3Fts5GetVarint32(pIter->a,(u32*)&(iVal));
21807 }
21808 *piOff += (iVal-2);
21809 }
21810}
21811
21812static int fts5ApiPhraseFirst(
21813 Fts5Context *pCtx,
21814 int iPhrase,
21815 Fts5PhraseIter *pIter,
21816 int *piCol, int *piOff
21817){
21818 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21819 int n;
21820 int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
21821 if( rc==SQLITE_OK0 ){
21822 assert( pIter->a || n==0 )((void) (0));
21823 pIter->b = (pIter->a ? &pIter->a[n] : 0);
21824 *piCol = 0;
21825 *piOff = 0;
21826 fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
21827 }
21828 return rc;
21829}
21830
21831static void fts5ApiPhraseNextColumn(
21832 Fts5Context *pCtx,
21833 Fts5PhraseIter *pIter,
21834 int *piCol
21835){
21836 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21837 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
21838
21839 if( pConfig->eDetail==FTS5_DETAIL_COLUMNS2 ){
21840 if( pIter->a>=pIter->b ){
21841 *piCol = -1;
21842 }else{
21843 int iIncr;
21844 pIter->a += fts5GetVarint32(&pIter->a[0], iIncr)sqlite3Fts5GetVarint32(&pIter->a[0],(u32*)&(iIncr)
);
21845 *piCol += (iIncr-2);
21846 }
21847 }else{
21848 while( 1 ){
21849 int dummy;
21850 if( pIter->a>=pIter->b ){
21851 *piCol = -1;
21852 return;
21853 }
21854 if( pIter->a[0]==0x01 ) break;
21855 pIter->a += fts5GetVarint32(pIter->a, dummy)sqlite3Fts5GetVarint32(pIter->a,(u32*)&(dummy));
21856 }
21857 pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol)sqlite3Fts5GetVarint32(&pIter->a[1],(u32*)&(*piCol
));
21858 }
21859}
21860
21861static int fts5ApiPhraseFirstColumn(
21862 Fts5Context *pCtx,
21863 int iPhrase,
21864 Fts5PhraseIter *pIter,
21865 int *piCol
21866){
21867 int rc = SQLITE_OK0;
21868 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21869 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
21870
21871 if( pConfig->eDetail==FTS5_DETAIL_COLUMNS2 ){
21872 Fts5Sorter *pSorter = pCsr->pSorter;
21873 int n;
21874 if( pSorter ){
21875 int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
21876 n = pSorter->aIdx[iPhrase] - i1;
21877 pIter->a = &pSorter->aPoslist[i1];
21878 }else{
21879 rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
21880 }
21881 if( rc==SQLITE_OK0 ){
21882 assert( pIter->a || n==0 )((void) (0));
21883 pIter->b = (pIter->a ? &pIter->a[n] : 0);
21884 *piCol = 0;
21885 fts5ApiPhraseNextColumn(pCtx, pIter, piCol);
21886 }
21887 }else{
21888 int n;
21889 rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
21890 if( rc==SQLITE_OK0 ){
21891 assert( pIter->a || n==0 )((void) (0));
21892 pIter->b = (pIter->a ? &pIter->a[n] : 0);
21893 if( n<=0 ){
21894 *piCol = -1;
21895 }else if( pIter->a[0]==0x01 ){
21896 pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol)sqlite3Fts5GetVarint32(&pIter->a[1],(u32*)&(*piCol
));
21897 }else{
21898 *piCol = 0;
21899 }
21900 }
21901 }
21902
21903 return rc;
21904}
21905
21906/*
21907** xQueryToken() API implemenetation.
21908*/
21909static int fts5ApiQueryToken(
21910 Fts5Context* pCtx,
21911 int iPhrase,
21912 int iToken,
21913 const char **ppOut,
21914 int *pnOut
21915){
21916 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21917 return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut);
21918}
21919
21920/*
21921** xInstToken() API implemenetation.
21922*/
21923static int fts5ApiInstToken(
21924 Fts5Context *pCtx,
21925 int iIdx,
21926 int iToken,
21927 const char **ppOut, int *pnOut
21928){
21929 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21930 int rc = SQLITE_OK0;
21931 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)((pCsr)->csrflags & (0x08))==0
21932 || SQLITE_OK0==(rc = fts5CacheInstArray(pCsr))
21933 ){
21934 if( iIdx<0 || iIdx>=pCsr->nInstCount ){
21935 rc = SQLITE_RANGE25;
21936 }else{
21937 int iPhrase = pCsr->aInst[iIdx*3];
21938 int iCol = pCsr->aInst[iIdx*3 + 1];
21939 int iOff = pCsr->aInst[iIdx*3 + 2];
21940 i64 iRowid = fts5CursorRowid(pCsr);
21941 rc = sqlite3Fts5ExprInstToken(
21942 pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut
21943 );
21944 }
21945 }
21946 return rc;
21947}
21948
21949
21950static int fts5ApiQueryPhrase(Fts5Context*, int, void*,
21951 int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
21952);
21953
21954/*
21955** The xColumnLocale() API.
21956*/
21957static int fts5ApiColumnLocale(
21958 Fts5Context *pCtx,
21959 int iCol,
21960 const char **pzLocale,
21961 int *pnLocale
21962){
21963 int rc = SQLITE_OK0;
21964 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21965 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
21966
21967 *pzLocale = 0;
21968 *pnLocale = 0;
21969
21970 assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL )((void) (0));
21971 if( iCol<0 || iCol>=pConfig->nCol ){
21972 rc = SQLITE_RANGE25;
21973 }else if(
21974 pConfig->abUnindexed[iCol]==0
21975 && 0==fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1)
21976 && pConfig->bLocale
21977 ){
21978 rc = fts5SeekCursor(pCsr, 0);
21979 if( rc==SQLITE_OK0 ){
21980 const char *zDummy = 0;
21981 int nDummy = 0;
21982 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &zDummy, &nDummy);
21983 if( rc==SQLITE_OK0 ){
21984 *pzLocale = pConfig->t.pLocale;
21985 *pnLocale = pConfig->t.nLocale;
21986 }
21987 sqlite3Fts5ClearLocale(pConfig);
21988 }
21989 }
21990
21991 return rc;
21992}
21993
21994static const Fts5ExtensionApi sFts5Api = {
21995 4, /* iVersion */
21996 fts5ApiUserData,
21997 fts5ApiColumnCount,
21998 fts5ApiRowCount,
21999 fts5ApiColumnTotalSize,
22000 fts5ApiTokenize,
22001 fts5ApiPhraseCount,
22002 fts5ApiPhraseSize,
22003 fts5ApiInstCount,
22004 fts5ApiInst,
22005 fts5ApiRowid,
22006 fts5ApiColumnText,
22007 fts5ApiColumnSize,
22008 fts5ApiQueryPhrase,
22009 fts5ApiSetAuxdata,
22010 fts5ApiGetAuxdata,
22011 fts5ApiPhraseFirst,
22012 fts5ApiPhraseNext,
22013 fts5ApiPhraseFirstColumn,
22014 fts5ApiPhraseNextColumn,
22015 fts5ApiQueryToken,
22016 fts5ApiInstToken,
22017 fts5ApiColumnLocale,
22018 fts5ApiTokenize_v2
22019};
22020
22021/*
22022** Implementation of API function xQueryPhrase().
22023*/
22024static int fts5ApiQueryPhrase(
22025 Fts5Context *pCtx,
22026 int iPhrase,
22027 void *pUserData,
22028 int(*xCallback)(const Fts5ExtensionApi*, Fts5Context*, void*)
22029){
22030 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
22031 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
22032 int rc;
22033 Fts5Cursor *pNew = 0;
22034
22035 rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew);
22036 if( rc
0.1
'rc' is equal to SQLITE_OK
==SQLITE_OK0 ){
1
Taking true branch
→
22037 pNew->ePlan = FTS5_PLAN_MATCH1;
22038 pNew->iFirstRowid = SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32)));
22039 pNew->iLastRowid = LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32));
22040 pNew->base.pVtab = (sqlite3_vtab*)pTab;
22041 rc = sqlite3Fts5ExprClonePhrase(pCsr->pExpr, iPhrase, &pNew->pExpr);
2
←
Calling 'sqlite3Fts5ExprClonePhrase'
→
21
←
Returning from 'sqlite3Fts5ExprClonePhrase'
→
22042 }
22043
22044 if( rc
21.1
'rc' is equal to SQLITE_OK
==SQLITE_OK0 ){
22
←
Taking true branch
→
22045 for(rc = fts5CursorFirst(pTab, pNew, 0);
23
←
Calling 'fts5CursorFirst'
→
22046 rc==SQLITE_OK0 && CsrFlagTest(pNew, FTS5CSR_EOF)((pNew)->csrflags & (0x01))==0;
22047 rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew)
22048 ){
22049 rc = xCallback(&sFts5Api, (Fts5Context*)pNew, pUserData);
22050 if( rc!=SQLITE_OK0 ){
22051 if( rc==SQLITE_DONE101 ) rc = SQLITE_OK0;
22052 break;
22053 }
22054 }
22055 }
22056
22057 fts5CloseMethod((sqlite3_vtab_cursor*)pNew);
22058 return rc;
22059}
22060
22061static void fts5ApiInvoke(
22062 Fts5Auxiliary *pAux,
22063 Fts5Cursor *pCsr,
22064 sqlite3_context *context,
22065 int argc,
22066 sqlite3_value **argv
22067){
22068 assert( pCsr->pAux==0 )((void) (0));
22069 assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL )((void) (0));
22070 pCsr->pAux = pAux;
22071 pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv);
22072 pCsr->pAux = 0;
22073}
22074
22075static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){
22076 Fts5Cursor *pCsr;
22077 for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
22078 if( pCsr->iCsrId==iCsrId ) break;
22079 }
22080 return pCsr;
22081}
22082
22083/*
22084** Parameter zFmt is a printf() style formatting string. This function
22085** formats it using the trailing arguments and returns the result as
22086** an error message to the context passed as the first argument.
22087*/
22088static void fts5ResultError(sqlite3_context *pCtx, const char *zFmt, ...){
22089 char *zErr = 0;
22090 va_list ap;
22091 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
22092 zErr = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
22093 sqlite3_result_errorsqlite3_api->result_error(pCtx, zErr, -1);
22094 sqlite3_freesqlite3_api->free(zErr);
22095 va_end(ap)__builtin_va_end(ap);
22096}
22097
22098static void fts5ApiCallback(
22099 sqlite3_context *context,
22100 int argc,
22101 sqlite3_value **argv
22102){
22103
22104 Fts5Auxiliary *pAux;
22105 Fts5Cursor *pCsr;
22106 i64 iCsrId;
22107
22108 assert( argc>=1 )((void) (0));
22109 pAux = (Fts5Auxiliary*)sqlite3_user_datasqlite3_api->user_data(context);
22110 iCsrId = sqlite3_value_int64sqlite3_api->value_int64(argv[0]);
22111
22112 pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId);
22113 if( pCsr==0 || (pCsr->ePlan==0 || pCsr->ePlan==FTS5_PLAN_SPECIAL3) ){
22114 fts5ResultError(context, "no such cursor: %lld", iCsrId);
22115 }else{
22116 sqlite3_vtab *pTab = pCsr->base.pVtab;
22117 fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]);
22118 sqlite3_freesqlite3_api->free(pTab->zErrMsg);
22119 pTab->zErrMsg = 0;
22120 }
22121}
22122
22123
22124/*
22125** Given cursor id iId, return a pointer to the corresponding Fts5Table
22126** object. Or NULL If the cursor id does not exist.
22127*/
22128static Fts5Table *sqlite3Fts5TableFromCsrid(
22129 Fts5Global *pGlobal, /* FTS5 global context for db handle */
22130 i64 iCsrId /* Id of cursor to find */
22131){
22132 Fts5Cursor *pCsr;
22133 pCsr = fts5CursorFromCsrid(pGlobal, iCsrId);
22134 if( pCsr ){
22135 return (Fts5Table*)pCsr->base.pVtab;
22136 }
22137 return 0;
22138}
22139
22140/*
22141** Return a "position-list blob" corresponding to the current position of
22142** cursor pCsr via sqlite3_result_blob(). A position-list blob contains
22143** the current position-list for each phrase in the query associated with
22144** cursor pCsr.
22145**
22146** A position-list blob begins with (nPhrase-1) varints, where nPhrase is
22147** the number of phrases in the query. Following the varints are the
22148** concatenated position lists for each phrase, in order.
22149**
22150** The first varint (if it exists) contains the size of the position list
22151** for phrase 0. The second (same disclaimer) contains the size of position
22152** list 1. And so on. There is no size field for the final position list,
22153** as it can be derived from the total size of the blob.
22154*/
22155static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
22156 int i;
22157 int rc = SQLITE_OK0;
22158 int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
22159 Fts5Buffer val;
22160
22161 memset(&val, 0, sizeof(Fts5Buffer));
22162 switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){
22163 case FTS5_DETAIL_FULL0:
22164
22165 /* Append the varints */
22166 for(i=0; i<(nPhrase-1); i++){
22167 const u8 *dummy;
22168 int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
22169 sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
22170 }
22171
22172 /* Append the position lists */
22173 for(i=0; i<nPhrase; i++){
22174 const u8 *pPoslist;
22175 int nPoslist;
22176 nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
22177 sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
22178 }
22179 break;
22180
22181 case FTS5_DETAIL_COLUMNS2:
22182
22183 /* Append the varints */
22184 for(i=0; rc==SQLITE_OK0 && i<(nPhrase-1); i++){
22185 const u8 *dummy;
22186 int nByte;
22187 rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte);
22188 sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
22189 }
22190
22191 /* Append the position lists */
22192 for(i=0; rc==SQLITE_OK0 && i<nPhrase; i++){
22193 const u8 *pPoslist;
22194 int nPoslist;
22195 rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &pPoslist, &nPoslist);
22196 sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
22197 }
22198 break;
22199
22200 default:
22201 break;
22202 }
22203
22204 sqlite3_result_blobsqlite3_api->result_blob(pCtx, val.p, val.n, sqlite3_freesqlite3_api->free);
22205 return rc;
22206}
22207
22208/*
22209** This is the xColumn method, called by SQLite to request a value from
22210** the row that the supplied cursor currently points to.
22211*/
22212static int fts5ColumnMethod(
22213 sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
22214 sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
22215 int iCol /* Index of column to read value from */
22216){
22217 Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab);
22218 Fts5Config *pConfig = pTab->p.pConfig;
22219 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
22220 int rc = SQLITE_OK0;
22221
22222 assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 )((void) (0));
22223
22224 if( pCsr->ePlan==FTS5_PLAN_SPECIAL3 ){
22225 if( iCol==pConfig->nCol ){
22226 sqlite3_result_int64sqlite3_api->result_int64(pCtx, pCsr->iSpecial);
22227 }
22228 }else
22229
22230 if( iCol==pConfig->nCol ){
22231 /* User is requesting the value of the special column with the same name
22232 ** as the table. Return the cursor integer id number. This value is only
22233 ** useful in that it may be passed as the first argument to an FTS5
22234 ** auxiliary function. */
22235 sqlite3_result_int64sqlite3_api->result_int64(pCtx, pCsr->iCsrId);
22236 }else if( iCol==pConfig->nCol+1 ){
22237 /* The value of the "rank" column. */
22238
22239 if( pCsr->ePlan==FTS5_PLAN_SOURCE2 ){
22240 fts5PoslistBlob(pCtx, pCsr);
22241 }else if(
22242 pCsr->ePlan==FTS5_PLAN_MATCH1
22243 || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH4
22244 ){
22245 if( pCsr->pRank || SQLITE_OK0==(rc = fts5FindRankFunction(pCsr)) ){
22246 fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg);
22247 }
22248 }
22249 }else{
22250 if( !sqlite3_vtab_nochangesqlite3_api->vtab_nochange(pCtx) && pConfig->eContent!=FTS5_CONTENT_NONE1 ){
22251 pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
22252 rc = fts5SeekCursor(pCsr, 1);
22253 if( rc==SQLITE_OK0 ){
22254 sqlite3_value *pVal = sqlite3_column_valuesqlite3_api->column_value(pCsr->pStmt, iCol+1);
22255 if( pConfig->bLocale
22256 && pConfig->eContent==FTS5_CONTENT_EXTERNAL2
22257 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
22258 ){
22259 const char *z = 0;
22260 int n = 0;
22261 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n);
22262 if( rc==SQLITE_OK0 ){
22263 sqlite3_result_textsqlite3_api->result_text(pCtx, z, n, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
22264 }
22265 sqlite3Fts5ClearLocale(pConfig);
22266 }else{
22267 sqlite3_result_valuesqlite3_api->result_value(pCtx, pVal);
22268 }
22269 }
22270
22271 pConfig->pzErrmsg = 0;
22272 }
22273 }
22274
22275 return rc;
22276}
22277
22278
22279/*
22280** This routine implements the xFindFunction method for the FTS3
22281** virtual table.
22282*/
22283static int fts5FindFunctionMethod(
22284 sqlite3_vtab *pVtab, /* Virtual table handle */
22285 int nUnused, /* Number of SQL function arguments */
22286 const char *zName, /* Name of SQL function */
22287 void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
22288 void **ppArg /* OUT: User data for *pxFunc */
22289){
22290 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22291 Fts5Auxiliary *pAux;
22292
22293 UNUSED_PARAM(nUnused)(void)(nUnused);
22294 pAux = fts5FindAuxiliary(pTab, zName);
22295 if( pAux ){
22296 *pxFunc = fts5ApiCallback;
22297 *ppArg = (void*)pAux;
22298 return 1;
22299 }
22300
22301 /* No function of the specified name was found. Return 0. */
22302 return 0;
22303}
22304
22305/*
22306** Implementation of FTS5 xRename method. Rename an fts5 table.
22307*/
22308static int fts5RenameMethod(
22309 sqlite3_vtab *pVtab, /* Virtual table handle */
22310 const char *zName /* New name of table */
22311){
22312 int rc;
22313 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22314 rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
22315 return rc;
22316}
22317
22318static int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
22319 fts5TripCursors((Fts5FullTable*)pTab);
22320 return sqlite3Fts5StorageSync(((Fts5FullTable*)pTab)->pStorage);
22321}
22322
22323/*
22324** The xSavepoint() method.
22325**
22326** Flush the contents of the pending-terms table to disk.
22327*/
22328static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
22329 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22330 int rc = SQLITE_OK0;
22331
22332 fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
22333 rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
22334 if( rc==SQLITE_OK0 ){
22335 pTab->iSavepoint = iSavepoint+1;
22336 }
22337 return rc;
22338}
22339
22340/*
22341** The xRelease() method.
22342**
22343** This is a no-op.
22344*/
22345static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
22346 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22347 int rc = SQLITE_OK0;
22348 fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
22349 if( (iSavepoint+1)<pTab->iSavepoint ){
22350 rc = sqlite3Fts5FlushToDisk(&pTab->p);
22351 if( rc==SQLITE_OK0 ){
22352 pTab->iSavepoint = iSavepoint;
22353 }
22354 }
22355 return rc;
22356}
22357
22358/*
22359** The xRollbackTo() method.
22360**
22361** Discard the contents of the pending terms table.
22362*/
22363static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
22364 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22365 int rc = SQLITE_OK0;
22366 fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
22367 fts5TripCursors(pTab);
22368 if( (iSavepoint+1)<=pTab->iSavepoint ){
22369 pTab->p.pConfig->pgsz = 0;
22370 rc = sqlite3Fts5StorageRollback(pTab->pStorage);
22371 }
22372 return rc;
22373}
22374
22375/*
22376** Register a new auxiliary function with global context pGlobal.
22377*/
22378static int fts5CreateAux(
22379 fts5_api *pApi, /* Global context (one per db handle) */
22380 const char *zName, /* Name of new function */
22381 void *pUserData, /* User data for aux. function */
22382 fts5_extension_function xFunc, /* Aux. function implementation */
22383 void(*xDestroy)(void*) /* Destructor for pUserData */
22384){
22385 Fts5Global *pGlobal = (Fts5Global*)pApi;
22386 int rc = sqlite3_overload_functionsqlite3_api->overload_function(pGlobal->db, zName, -1);
22387 if( rc==SQLITE_OK0 ){
22388 Fts5Auxiliary *pAux;
22389 sqlite3_int64 nName; /* Size of zName in bytes, including \0 */
22390 sqlite3_int64 nByte; /* Bytes of space to allocate */
22391
22392 nName = strlen(zName) + 1;
22393 nByte = sizeof(Fts5Auxiliary) + nName;
22394 pAux = (Fts5Auxiliary*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
22395 if( pAux ){
22396 memset(pAux, 0, (size_t)nByte);
22397 pAux->zFunc = (char*)&pAux[1];
22398 memcpy(pAux->zFunc, zName, nName);
22399 pAux->pGlobal = pGlobal;
22400 pAux->pUserData = pUserData;
22401 pAux->xFunc = xFunc;
22402 pAux->xDestroy = xDestroy;
22403 pAux->pNext = pGlobal->pAux;
22404 pGlobal->pAux = pAux;
22405 }else{
22406 rc = SQLITE_NOMEM7;
22407 }
22408 }
22409
22410 return rc;
22411}
22412
22413/*
22414** This function is used by xCreateTokenizer_v2() and xCreateTokenizer().
22415** It allocates and partially populates a new Fts5TokenizerModule object.
22416** The new object is already linked into the Fts5Global context before
22417** returning.
22418**
22419** If successful, SQLITE_OK is returned and a pointer to the new
22420** Fts5TokenizerModule object returned via output parameter (*ppNew). All
22421** that is required is for the caller to fill in the methods in
22422** Fts5TokenizerModule.x1 and x2, and to set Fts5TokenizerModule.bV2Native
22423** as appropriate.
22424**
22425** If an error occurs, an SQLite error code is returned and the final value
22426** of (*ppNew) undefined.
22427*/
22428static int fts5NewTokenizerModule(
22429 Fts5Global *pGlobal, /* Global context (one per db handle) */
22430 const char *zName, /* Name of new function */
22431 void *pUserData, /* User data for aux. function */
22432 void(*xDestroy)(void*), /* Destructor for pUserData */
22433 Fts5TokenizerModule **ppNew
22434){
22435 int rc = SQLITE_OK0;
22436 Fts5TokenizerModule *pNew;
22437 sqlite3_int64 nName; /* Size of zName and its \0 terminator */
22438 sqlite3_int64 nByte; /* Bytes of space to allocate */
22439
22440 nName = strlen(zName) + 1;
22441 nByte = sizeof(Fts5TokenizerModule) + nName;
22442 *ppNew = pNew = (Fts5TokenizerModule*)sqlite3Fts5MallocZero(&rc, nByte);
22443 if( pNew ){
22444 pNew->zName = (char*)&pNew[1];
22445 memcpy(pNew->zName, zName, nName);
22446 pNew->pUserData = pUserData;
22447 pNew->xDestroy = xDestroy;
22448 pNew->pNext = pGlobal->pTok;
22449 pGlobal->pTok = pNew;
22450 if( pNew->pNext==0 ){
22451 pGlobal->pDfltTok = pNew;
22452 }
22453 }
22454
22455 return rc;
22456}
22457
22458/*
22459** An instance of this type is used as the Fts5Tokenizer object for
22460** wrapper tokenizers - those that provide access to a v1 tokenizer via
22461** the fts5_tokenizer_v2 API, and those that provide access to a v2 tokenizer
22462** via the fts5_tokenizer API.
22463*/
22464typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer;
22465struct Fts5VtoVTokenizer {
22466 int bV2Native; /* True if v2 native tokenizer */
22467 fts5_tokenizer x1; /* Tokenizer functions */
22468 fts5_tokenizer_v2 x2; /* V2 tokenizer functions */
22469 Fts5Tokenizer *pReal;
22470};
22471
22472/*
22473** Create a wrapper tokenizer. The context argument pCtx points to the
22474** Fts5TokenizerModule object.
22475*/
22476static int fts5VtoVCreate(
22477 void *pCtx,
22478 const char **azArg,
22479 int nArg,
22480 Fts5Tokenizer **ppOut
22481){
22482 Fts5TokenizerModule *pMod = (Fts5TokenizerModule*)pCtx;
22483 Fts5VtoVTokenizer *pNew = 0;
22484 int rc = SQLITE_OK0;
22485
22486 pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
22487 if( rc==SQLITE_OK0 ){
22488 pNew->x1 = pMod->x1;
22489 pNew->x2 = pMod->x2;
22490 pNew->bV2Native = pMod->bV2Native;
22491 if( pMod->bV2Native ){
22492 rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal);
22493 }else{
22494 rc = pMod->x1.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal);
22495 }
22496 if( rc!=SQLITE_OK0 ){
22497 sqlite3_freesqlite3_api->free(pNew);
22498 pNew = 0;
22499 }
22500 }
22501
22502 *ppOut = (Fts5Tokenizer*)pNew;
22503 return rc;
22504}
22505
22506/*
22507** Delete an Fts5VtoVTokenizer wrapper tokenizer.
22508*/
22509static void fts5VtoVDelete(Fts5Tokenizer *pTok){
22510 Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
22511 if( p ){
22512 if( p->bV2Native ){
22513 p->x2.xDelete(p->pReal);
22514 }else{
22515 p->x1.xDelete(p->pReal);
22516 }
22517 sqlite3_freesqlite3_api->free(p);
22518 }
22519}
22520
22521
22522/*
22523** xTokenizer method for a wrapper tokenizer that offers the v1 interface
22524** (no support for locales).
22525*/
22526static int fts5V1toV2Tokenize(
22527 Fts5Tokenizer *pTok,
22528 void *pCtx, int flags,
22529 const char *pText, int nText,
22530 int (*xToken)(void*, int, const char*, int, int, int)
22531){
22532 Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
22533 assert( p->bV2Native )((void) (0));
22534 return p->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken);
22535}
22536
22537/*
22538** xTokenizer method for a wrapper tokenizer that offers the v2 interface
22539** (with locale support).
22540*/
22541static int fts5V2toV1Tokenize(
22542 Fts5Tokenizer *pTok,
22543 void *pCtx, int flags,
22544 const char *pText, int nText,
22545 const char *pLocale, int nLocale,
22546 int (*xToken)(void*, int, const char*, int, int, int)
22547){
22548 Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
22549 assert( p->bV2Native==0 )((void) (0));
22550 UNUSED_PARAM2(pLocale,nLocale)(void)(pLocale), (void)(nLocale);
22551 return p->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken);
22552}
22553
22554/*
22555** Register a new tokenizer. This is the implementation of the
22556** fts5_api.xCreateTokenizer_v2() method.
22557*/
22558static int fts5CreateTokenizer_v2(
22559 fts5_api *pApi, /* Global context (one per db handle) */
22560 const char *zName, /* Name of new function */
22561 void *pUserData, /* User data for aux. function */
22562 fts5_tokenizer_v2 *pTokenizer, /* Tokenizer implementation */
22563 void(*xDestroy)(void*) /* Destructor for pUserData */
22564){
22565 Fts5Global *pGlobal = (Fts5Global*)pApi;
22566 int rc = SQLITE_OK0;
22567
22568 if( pTokenizer->iVersion>2 ){
22569 rc = SQLITE_ERROR1;
22570 }else{
22571 Fts5TokenizerModule *pNew = 0;
22572 rc = fts5NewTokenizerModule(pGlobal, zName, pUserData, xDestroy, &pNew);
22573 if( pNew ){
22574 pNew->x2 = *pTokenizer;
22575 pNew->bV2Native = 1;
22576 pNew->x1.xCreate = fts5VtoVCreate;
22577 pNew->x1.xTokenize = fts5V1toV2Tokenize;
22578 pNew->x1.xDelete = fts5VtoVDelete;
22579 }
22580 }
22581
22582 return rc;
22583}
22584
22585/*
22586** The fts5_api.xCreateTokenizer() method.
22587*/
22588static int fts5CreateTokenizer(
22589 fts5_api *pApi, /* Global context (one per db handle) */
22590 const char *zName, /* Name of new function */
22591 void *pUserData, /* User data for aux. function */
22592 fts5_tokenizer *pTokenizer, /* Tokenizer implementation */
22593 void(*xDestroy)(void*) /* Destructor for pUserData */
22594){
22595 Fts5TokenizerModule *pNew = 0;
22596 int rc = SQLITE_OK0;
22597
22598 rc = fts5NewTokenizerModule(
22599 (Fts5Global*)pApi, zName, pUserData, xDestroy, &pNew
22600 );
22601 if( pNew ){
22602 pNew->x1 = *pTokenizer;
22603 pNew->x2.xCreate = fts5VtoVCreate;
22604 pNew->x2.xTokenize = fts5V2toV1Tokenize;
22605 pNew->x2.xDelete = fts5VtoVDelete;
22606 }
22607 return rc;
22608}
22609
22610/*
22611** Search the global context passed as the first argument for a tokenizer
22612** module named zName. If found, return a pointer to the Fts5TokenizerModule
22613** object. Otherwise, return NULL.
22614*/
22615static Fts5TokenizerModule *fts5LocateTokenizer(
22616 Fts5Global *pGlobal, /* Global (one per db handle) object */
22617 const char *zName /* Name of tokenizer module to find */
22618){
22619 Fts5TokenizerModule *pMod = 0;
22620
22621 if( zName==0 ){
22622 pMod = pGlobal->pDfltTok;
22623 }else{
22624 for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
22625 if( sqlite3_stricmpsqlite3_api->stricmp(zName, pMod->zName)==0 ) break;
22626 }
22627 }
22628
22629 return pMod;
22630}
22631
22632/*
22633** Find a tokenizer. This is the implementation of the
22634** fts5_api.xFindTokenizer_v2() method.
22635*/
22636static int fts5FindTokenizer_v2(
22637 fts5_api *pApi, /* Global context (one per db handle) */
22638 const char *zName, /* Name of tokenizer */
22639 void **ppUserData,
22640 fts5_tokenizer_v2 **ppTokenizer /* Populate this object */
22641){
22642 int rc = SQLITE_OK0;
22643 Fts5TokenizerModule *pMod;
22644
22645 pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName);
22646 if( pMod ){
22647 if( pMod->bV2Native ){
22648 *ppUserData = pMod->pUserData;
22649 }else{
22650 *ppUserData = (void*)pMod;
22651 }
22652 *ppTokenizer = &pMod->x2;
22653 }else{
22654 *ppTokenizer = 0;
22655 *ppUserData = 0;
22656 rc = SQLITE_ERROR1;
22657 }
22658
22659 return rc;
22660}
22661
22662/*
22663** Find a tokenizer. This is the implementation of the
22664** fts5_api.xFindTokenizer() method.
22665*/
22666static int fts5FindTokenizer(
22667 fts5_api *pApi, /* Global context (one per db handle) */
22668 const char *zName, /* Name of new function */
22669 void **ppUserData,
22670 fts5_tokenizer *pTokenizer /* Populate this object */
22671){
22672 int rc = SQLITE_OK0;
22673 Fts5TokenizerModule *pMod;
22674
22675 pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName);
22676 if( pMod ){
22677 if( pMod->bV2Native==0 ){
22678 *ppUserData = pMod->pUserData;
22679 }else{
22680 *ppUserData = (void*)pMod;
22681 }
22682 *pTokenizer = pMod->x1;
22683 }else{
22684 memset(pTokenizer, 0, sizeof(*pTokenizer));
22685 *ppUserData = 0;
22686 rc = SQLITE_ERROR1;
22687 }
22688
22689 return rc;
22690}
22691
22692/*
22693** Attempt to instantiate the tokenizer.
22694*/
22695static int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){
22696 const char **azArg = pConfig->t.azArg;
22697 const int nArg = pConfig->t.nArg;
22698 Fts5TokenizerModule *pMod = 0;
22699 int rc = SQLITE_OK0;
22700
22701 pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]);
22702 if( pMod==0 ){
22703 assert( nArg>0 )((void) (0));
22704 rc = SQLITE_ERROR1;
22705 sqlite3Fts5ConfigErrmsg(pConfig, "no such tokenizer: %s", azArg[0]);
22706 }else{
22707 int (*xCreate)(void*, const char**, int, Fts5Tokenizer**) = 0;
22708 if( pMod->bV2Native ){
22709 xCreate = pMod->x2.xCreate;
22710 pConfig->t.pApi2 = &pMod->x2;
22711 }else{
22712 pConfig->t.pApi1 = &pMod->x1;
22713 xCreate = pMod->x1.xCreate;
22714 }
22715
22716 rc = xCreate(pMod->pUserData,
22717 (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok
22718 );
22719
22720 if( rc!=SQLITE_OK0 ){
22721 if( rc!=SQLITE_NOMEM7 ){
22722 sqlite3Fts5ConfigErrmsg(pConfig, "error in tokenizer constructor");
22723 }
22724 }else if( pMod->bV2Native==0 ){
22725 pConfig->t.ePattern = sqlite3Fts5TokenizerPattern(
22726 pMod->x1.xCreate, pConfig->t.pTok
22727 );
22728 }
22729 }
22730
22731 if( rc!=SQLITE_OK0 ){
22732 pConfig->t.pApi1 = 0;
22733 pConfig->t.pApi2 = 0;
22734 pConfig->t.pTok = 0;
22735 }
22736
22737 return rc;
22738}
22739
22740
22741/*
22742** xDestroy callback passed to sqlite3_create_module(). This is invoked
22743** when the db handle is being closed. Free memory associated with
22744** tokenizers and aux functions registered with this db handle.
22745*/
22746static void fts5ModuleDestroy(void *pCtx){
22747 Fts5TokenizerModule *pTok, *pNextTok;
22748 Fts5Auxiliary *pAux, *pNextAux;
22749 Fts5Global *pGlobal = (Fts5Global*)pCtx;
22750
22751 for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){
22752 pNextAux = pAux->pNext;
22753 if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData);
22754 sqlite3_freesqlite3_api->free(pAux);
22755 }
22756
22757 for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){
22758 pNextTok = pTok->pNext;
22759 if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData);
22760 sqlite3_freesqlite3_api->free(pTok);
22761 }
22762
22763 sqlite3_freesqlite3_api->free(pGlobal);
22764}
22765
22766/*
22767** Implementation of the fts5() function used by clients to obtain the
22768** API pointer.
22769*/
22770static void fts5Fts5Func(
22771 sqlite3_context *pCtx, /* Function call context */
22772 int nArg, /* Number of args */
22773 sqlite3_value **apArg /* Function arguments */
22774){
22775 Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_datasqlite3_api->user_data(pCtx);
22776 fts5_api **ppApi;
22777 UNUSED_PARAM(nArg)(void)(nArg);
22778 assert( nArg==1 )((void) (0));
22779 ppApi = (fts5_api**)sqlite3_value_pointersqlite3_api->value_pointer(apArg[0], "fts5_api_ptr");
22780 if( ppApi ) *ppApi = &pGlobal->api;
22781}
22782
22783/*
22784** Implementation of fts5_source_id() function.
22785*/
22786static void fts5SourceIdFunc(
22787 sqlite3_context *pCtx, /* Function call context */
22788 int nArg, /* Number of args */
22789 sqlite3_value **apUnused /* Function arguments */
22790){
22791 assert( nArg==0 )((void) (0));
22792 UNUSED_PARAM2(nArg, apUnused)(void)(nArg), (void)(apUnused);
22793 sqlite3_result_textsqlite3_api->result_text(pCtx, "fts5: 2026-06-03 19:12:13 d6e03d8c777cfa2d35e3b60d8ec3e0187f3e9f99d8e2ee9cac695fd6fcdf1a24", -1, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
22794}
22795
22796/*
22797** Implementation of fts5_locale(LOCALE, TEXT) function.
22798**
22799** If parameter LOCALE is NULL, or a zero-length string, then a copy of
22800** TEXT is returned. Otherwise, both LOCALE and TEXT are interpreted as
22801** text, and the value returned is a blob consisting of:
22802**
22803** * The 4 bytes 0x00, 0xE0, 0xB2, 0xEb (FTS5_LOCALE_HEADER).
22804** * The LOCALE, as utf-8 text, followed by
22805** * 0x00, followed by
22806** * The TEXT, as utf-8 text.
22807**
22808** There is no final nul-terminator following the TEXT value.
22809*/
22810static void fts5LocaleFunc(
22811 sqlite3_context *pCtx, /* Function call context */
22812 int nArg, /* Number of args */
22813 sqlite3_value **apArg /* Function arguments */
22814){
22815 const char *zLocale = 0;
22816 i64 nLocale = 0;
22817 const char *zText = 0;
22818 i64 nText = 0;
22819
22820 assert( nArg==2 )((void) (0));
22821 UNUSED_PARAM(nArg)(void)(nArg);
22822
22823 zLocale = (const char*)sqlite3_value_textsqlite3_api->value_text(apArg[0]);
22824 nLocale = sqlite3_value_bytessqlite3_api->value_bytes(apArg[0]);
22825
22826 zText = (const char*)sqlite3_value_textsqlite3_api->value_text(apArg[1]);
22827 nText = sqlite3_value_bytessqlite3_api->value_bytes(apArg[1]);
22828
22829 if( zLocale==0 || zLocale[0]=='\0' ){
22830 sqlite3_result_textsqlite3_api->result_text(pCtx, zText, nText, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
22831 }else{
22832 Fts5Global *p = (Fts5Global*)sqlite3_user_datasqlite3_api->user_data(pCtx);
22833 u8 *pBlob = 0;
22834 u8 *pCsr = 0;
22835 i64 nBlob = 0;
22836
22837 nBlob = FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )) + nLocale + 1 + nText;
22838 pBlob = (u8*)sqlite3_malloc64sqlite3_api->malloc64(nBlob);
22839 if( pBlob==0 ){
22840 sqlite3_result_error_nomemsqlite3_api->result_error_nomem(pCtx);
22841 return;
22842 }
22843
22844 pCsr = pBlob;
22845 memcpy(pCsr, (const u8*)p->aLocaleHdr, FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )));
22846 pCsr += FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ));
22847 memcpy(pCsr, zLocale, nLocale);
22848 pCsr += nLocale;
22849 (*pCsr++) = 0x00;
22850 if( zText ) memcpy(pCsr, zText, nText);
22851 assert( &pCsr[nText]==&pBlob[nBlob] )((void) (0));
22852
22853 sqlite3_result_blobsqlite3_api->result_blob(pCtx, pBlob, nBlob, sqlite3_freesqlite3_api->free);
22854 }
22855}
22856
22857/*
22858** Implementation of fts5_insttoken() function.
22859*/
22860static void fts5InsttokenFunc(
22861 sqlite3_context *pCtx, /* Function call context */
22862 int nArg, /* Number of args */
22863 sqlite3_value **apArg /* Function arguments */
22864){
22865 assert( nArg==1 )((void) (0));
22866 (void)nArg;
22867 sqlite3_result_valuesqlite3_api->result_value(pCtx, apArg[0]);
22868 sqlite3_result_subtypesqlite3_api->result_subtype(pCtx, FTS5_INSTTOKEN_SUBTYPE73);
22869}
22870
22871/*
22872** Return true if zName is the extension on one of the shadow tables used
22873** by this module.
22874*/
22875static int fts5ShadowName(const char *zName){
22876 static const char *azName[] = {
22877 "config", "content", "data", "docsize", "idx"
22878 };
22879 unsigned int i;
22880 for(i=0; i<sizeof(azName)/sizeof(azName[0]); i++){
22881 if( sqlite3_stricmpsqlite3_api->stricmp(zName, azName[i])==0 ) return 1;
22882 }
22883 return 0;
22884}
22885
22886/*
22887** Run an integrity check on the FTS5 data structures. Return a string
22888** if anything is found amiss. Return a NULL pointer if everything is
22889** OK.
22890*/
22891static int fts5IntegrityMethod(
22892 sqlite3_vtab *pVtab, /* the FTS5 virtual table to check */
22893 const char *zSchema, /* Name of schema in which this table lives */
22894 const char *zTabname, /* Name of the table itself */
22895 int isQuick, /* True if this is a quick-check */
22896 char **pzErr /* Write error message here */
22897){
22898 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22899 int rc;
22900
22901 assert( pzErr!=0 && *pzErr==0 )((void) (0));
22902 UNUSED_PARAM(isQuick)(void)(isQuick);
22903 assert( pTab->p.pConfig->pzErrmsg==0 )((void) (0));
22904 pTab->p.pConfig->pzErrmsg = pzErr;
22905 rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0);
22906 if( *pzErr==0 && rc!=SQLITE_OK0 ){
22907 if( (rc&0xff)==SQLITE_CORRUPT11 ){
22908 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed inverted index for FTS5 table %s.%s",
22909 zSchema, zTabname);
22910 rc = (*pzErr) ? SQLITE_OK0 : SQLITE_NOMEM7;
22911 }else{
22912 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("unable to validate the inverted index for"
22913 " FTS5 table %s.%s: %s",
22914 zSchema, zTabname, sqlite3_errstrsqlite3_api->errstr(rc));
22915 }
22916 }else if( (rc&0xff)==SQLITE_CORRUPT11 ){
22917 rc = SQLITE_OK0;
22918 }
22919 sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
22920 pTab->p.pConfig->pzErrmsg = 0;
22921
22922 return rc;
22923}
22924
22925static int fts5Init(sqlite3 *db){
22926 static const sqlite3_module fts5Mod = {
22927 /* iVersion */ 4,
22928 /* xCreate */ fts5CreateMethod,
22929 /* xConnect */ fts5ConnectMethod,
22930 /* xBestIndex */ fts5BestIndexMethod,
22931 /* xDisconnect */ fts5DisconnectMethod,
22932 /* xDestroy */ fts5DestroyMethod,
22933 /* xOpen */ fts5OpenMethod,
22934 /* xClose */ fts5CloseMethod,
22935 /* xFilter */ fts5FilterMethod,
22936 /* xNext */ fts5NextMethod,
22937 /* xEof */ fts5EofMethod,
22938 /* xColumn */ fts5ColumnMethod,
22939 /* xRowid */ fts5RowidMethod,
22940 /* xUpdate */ fts5UpdateMethod,
22941 /* xBegin */ fts5BeginMethod,
22942 /* xSync */ fts5SyncMethod,
22943 /* xCommit */ fts5CommitMethod,
22944 /* xRollback */ fts5RollbackMethod,
22945 /* xFindFunction */ fts5FindFunctionMethod,
22946 /* xRename */ fts5RenameMethod,
22947 /* xSavepoint */ fts5SavepointMethod,
22948 /* xRelease */ fts5ReleaseMethod,
22949 /* xRollbackTo */ fts5RollbackToMethod,
22950 /* xShadowName */ fts5ShadowName,
22951 /* xIntegrity */ fts5IntegrityMethod
22952 };
22953
22954 int rc;
22955 Fts5Global *pGlobal = 0;
22956
22957 pGlobal = (Fts5Global*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5Global));
22958 if( pGlobal==0 ){
22959 rc = SQLITE_NOMEM7;
22960 }else{
22961 void *p = (void*)pGlobal;
22962 memset(pGlobal, 0, sizeof(Fts5Global));
22963 pGlobal->db = db;
22964 pGlobal->api.iVersion = 3;
22965 pGlobal->api.xCreateFunction = fts5CreateAux;
22966 pGlobal->api.xCreateTokenizer = fts5CreateTokenizer;
22967 pGlobal->api.xFindTokenizer = fts5FindTokenizer;
22968 pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2;
22969 pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2;
22970
22971 /* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector.
22972 ** The constants below were generated randomly. */
22973 sqlite3_randomnesssqlite3_api->randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr);
22974 pGlobal->aLocaleHdr[0] ^= 0xF924976D;
22975 pGlobal->aLocaleHdr[1] ^= 0x16596E13;
22976 pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA;
22977 pGlobal->aLocaleHdr[3] ^= 0x9B03A67F;
22978 assert( sizeof(pGlobal->aLocaleHdr)==16 )((void) (0));
22979
22980 rc = sqlite3_create_module_v2sqlite3_api->create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy);
22981 if( rc==SQLITE_OK0 ) rc = sqlite3Fts5IndexInit(db);
22982 if( rc==SQLITE_OK0 ) rc = sqlite3Fts5ExprInit(pGlobal, db);
22983 if( rc==SQLITE_OK0 ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
22984 if( rc==SQLITE_OK0 ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
22985 if( rc==SQLITE_OK0 ) rc = sqlite3Fts5VocabInit(pGlobal, db);
22986 if( rc==SQLITE_OK0 ){
22987 rc = sqlite3_create_functionsqlite3_api->create_function(
22988 db, "fts5", 1, SQLITE_UTF81, p, fts5Fts5Func, 0, 0
22989 );
22990 }
22991 if( rc==SQLITE_OK0 ){
22992 rc = sqlite3_create_functionsqlite3_api->create_function(
22993 db, "fts5_source_id", 0,
22994 SQLITE_UTF81|SQLITE_DETERMINISTIC0x000000800|SQLITE_INNOCUOUS0x000200000,
22995 p, fts5SourceIdFunc, 0, 0
22996 );
22997 }
22998 if( rc==SQLITE_OK0 ){
22999 rc = sqlite3_create_functionsqlite3_api->create_function(
23000 db, "fts5_locale", 2,
23001 SQLITE_UTF81|SQLITE_INNOCUOUS0x000200000|SQLITE_RESULT_SUBTYPE0x001000000|SQLITE_SUBTYPE0x000100000,
23002 p, fts5LocaleFunc, 0, 0
23003 );
23004 }
23005 if( rc==SQLITE_OK0 ){
23006 rc = sqlite3_create_functionsqlite3_api->create_function(
23007 db, "fts5_insttoken", 1,
23008 SQLITE_UTF81|SQLITE_INNOCUOUS0x000200000|SQLITE_RESULT_SUBTYPE0x001000000,
23009 p, fts5InsttokenFunc, 0, 0
23010 );
23011 }
23012 }
23013
23014 /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file
23015 ** fts5_test_mi.c is compiled and linked into the executable. And call
23016 ** its entry point to enable the matchinfo() demo. */
23017#ifdef SQLITE_FTS5_ENABLE_TEST_MI
23018 if( rc==SQLITE_OK0 ){
23019 extern int sqlite3Fts5TestRegisterMatchinfoAPI(fts5_api*);
23020 rc = sqlite3Fts5TestRegisterMatchinfoAPI(&pGlobal->api);
23021 }
23022#endif
23023
23024 return rc;
23025}
23026
23027/*
23028** The following functions are used to register the module with SQLite. If
23029** this module is being built as part of the SQLite core (SQLITE_CORE is
23030** defined), then sqlite3_open() will call sqlite3Fts5Init() directly.
23031**
23032** Or, if this module is being built as a loadable extension,
23033** sqlite3Fts5Init() is omitted and the two standard entry points
23034** sqlite3_fts_init() and sqlite3_fts5_init() defined instead.
23035*/
23036#ifndef SQLITE_CORE
23037#ifdef _WIN32
23038__declspec(dllexport)
23039#endif
23040int sqlite3_fts_init(
23041 sqlite3 *db,
23042 char **pzErrMsg,
23043 const sqlite3_api_routines *pApi
23044){
23045 SQLITE_EXTENSION_INIT2(pApi)sqlite3_api=pApi;;
23046 (void)pzErrMsg; /* Unused parameter */
23047 return fts5Init(db);
23048}
23049
23050#ifdef _WIN32
23051__declspec(dllexport)
23052#endif
23053int sqlite3_fts5_init(
23054 sqlite3 *db,
23055 char **pzErrMsg,
23056 const sqlite3_api_routines *pApi
23057){
23058 SQLITE_EXTENSION_INIT2(pApi)sqlite3_api=pApi;;
23059 (void)pzErrMsg; /* Unused parameter */
23060 return fts5Init(db);
23061}
23062#else
23063int sqlite3Fts5Init(sqlite3 *db){
23064 return fts5Init(db);
23065}
23066#endif
23067
23068#line 1 "fts5_storage.c"
23069/*
23070** 2014 May 31
23071**
23072** The author disclaims copyright to this source code. In place of
23073** a legal notice, here is a blessing:
23074**
23075** May you do good and not evil.
23076** May you find forgiveness for yourself and forgive others.
23077** May you share freely, never taking more than you give.
23078**
23079******************************************************************************
23080**
23081*/
23082
23083
23084
23085/* #include "fts5Int.h" */
23086
23087/*
23088** pSavedRow:
23089** SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it
23090** does a by-rowid lookup to retrieve a single row from the %_content
23091** table or equivalent external-content table/view.
23092**
23093** However, FTS5_STMT_LOOKUP2 is only used when retrieving the original
23094** values for a row being UPDATEd. In that case, the SQL statement is
23095** not reset and pSavedRow is set to point at it. This is so that the
23096** insert operation that follows the delete may access the original
23097** row values for any new values for which sqlite3_value_nochange() returns
23098** true. i.e. if the user executes:
23099**
23100** CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1);
23101** ...
23102** UPDATE fts SET a=?, b=? WHERE rowid=?;
23103**
23104** then the value passed to the xUpdate() method of this table as the
23105** new.c value is an sqlite3_value_nochange() value. So in this case it
23106** must be read from the saved row stored in Fts5Storage.pSavedRow.
23107**
23108** This is necessary - using sqlite3_value_nochange() instead of just having
23109** SQLite pass the original value back via xUpdate() - so as not to discard
23110** any locale information associated with such values.
23111**
23112*/
23113struct Fts5Storage {
23114 Fts5Config *pConfig;
23115 Fts5Index *pIndex;
23116 int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */
23117 i64 nTotalRow; /* Total number of rows in FTS table */
23118 i64 *aTotalSize; /* Total sizes of each column */
23119 sqlite3_stmt *pSavedRow;
23120 sqlite3_stmt *aStmt[12];
23121};
23122
23123
23124#if FTS5_STMT_SCAN_ASC0!=0
23125# error "FTS5_STMT_SCAN_ASC mismatch"
23126#endif
23127#if FTS5_STMT_SCAN_DESC1!=1
23128# error "FTS5_STMT_SCAN_DESC mismatch"
23129#endif
23130#if FTS5_STMT_LOOKUP2!=2
23131# error "FTS5_STMT_LOOKUP mismatch"
23132#endif
23133
23134#define FTS5_STMT_LOOKUP23 3
23135#define FTS5_STMT_INSERT_CONTENT4 4
23136#define FTS5_STMT_REPLACE_CONTENT5 5
23137#define FTS5_STMT_DELETE_CONTENT6 6
23138#define FTS5_STMT_REPLACE_DOCSIZE7 7
23139#define FTS5_STMT_DELETE_DOCSIZE8 8
23140#define FTS5_STMT_LOOKUP_DOCSIZE9 9
23141#define FTS5_STMT_REPLACE_CONFIG10 10
23142#define FTS5_STMT_SCAN11 11
23143
23144/*
23145** Prepare the two insert statements - Fts5Storage.pInsertContent and
23146** Fts5Storage.pInsertDocsize - if they have not already been prepared.
23147** Return SQLITE_OK if successful, or an SQLite error code if an error
23148** occurs.
23149*/
23150static int fts5StorageGetStmt(
23151 Fts5Storage *p, /* Storage handle */
23152 int eStmt, /* FTS5_STMT_XXX constant */
23153 sqlite3_stmt **ppStmt, /* OUT: Prepared statement handle */
23154 char **pzErrMsg /* OUT: Error message (if any) */
23155){
23156 int rc = SQLITE_OK0;
23157
23158 /* If there is no %_docsize table, there should be no requests for
23159 ** statements to operate on it. */
23160 assert( p->pConfig->bColumnsize || (((void) (0))
23161 eStmt!=FTS5_STMT_REPLACE_DOCSIZE((void) (0))
23162 && eStmt!=FTS5_STMT_DELETE_DOCSIZE((void) (0))
23163 && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE((void) (0))
23164 ))((void) (0));
23165
23166 assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) )((void) (0));
23167 if( p->aStmt[eStmt]==0 ){
23168 const char *azStmt[] = {
23169 "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
23170 "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
23171 "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */
23172 "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP2 */
23173
23174 "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */
23175 "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */
23176 "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */
23177 "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */
23178 "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */
23179
23180 "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */
23181
23182 "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */
23183 "SELECT %s FROM %s AS T", /* SCAN */
23184 };
23185 Fts5Config *pC = p->pConfig;
23186 char *zSql = 0;
23187
23188 assert( ArraySize(azStmt)==ArraySize(p->aStmt) )((void) (0));
23189
23190 switch( eStmt ){
23191 case FTS5_STMT_SCAN11:
23192 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt],
23193 pC->zContentExprlist, pC->zContent
23194 );
23195 break;
23196
23197 case FTS5_STMT_SCAN_ASC0:
23198 case FTS5_STMT_SCAN_DESC1:
23199 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt], pC->zContentExprlist,
23200 pC->zContent, pC->zContentRowid, pC->zContentRowid,
23201 pC->zContentRowid
23202 );
23203 break;
23204
23205 case FTS5_STMT_LOOKUP2:
23206 case FTS5_STMT_LOOKUP23:
23207 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt],
23208 pC->zContentExprlist, pC->zContent, pC->zContentRowid
23209 );
23210 break;
23211
23212 case FTS5_STMT_INSERT_CONTENT4:
23213 case FTS5_STMT_REPLACE_CONTENT5: {
23214 char *zBind = 0;
23215 int i;
23216
23217 assert( pC->eContent==FTS5_CONTENT_NORMAL((void) (0))
23218 || pC->eContent==FTS5_CONTENT_UNINDEXED((void) (0))
23219 )((void) (0));
23220
23221 /* Add bindings for the "c*" columns - those that store the actual
23222 ** table content. If eContent==NORMAL, then there is one binding
23223 ** for each column. Or, if eContent==UNINDEXED, then there are only
23224 ** bindings for the UNINDEXED columns. */
23225 for(i=0; rc==SQLITE_OK0 && i<(pC->nCol+1); i++){
23226 if( !i || pC->eContent==FTS5_CONTENT_NORMAL0 || pC->abUnindexed[i-1] ){
23227 zBind = sqlite3Fts5Mprintf(&rc, "%z%s?%d", zBind, zBind?",":"",i+1);
23228 }
23229 }
23230
23231 /* Add bindings for any "l*" columns. Only non-UNINDEXED columns
23232 ** require these. */
23233 if( pC->bLocale && pC->eContent==FTS5_CONTENT_NORMAL0 ){
23234 for(i=0; rc==SQLITE_OK0 && i<pC->nCol; i++){
23235 if( pC->abUnindexed[i]==0 ){
23236 zBind = sqlite3Fts5Mprintf(&rc, "%z,?%d", zBind, pC->nCol+i+2);
23237 }
23238 }
23239 }
23240
23241 zSql = sqlite3Fts5Mprintf(&rc, azStmt[eStmt], pC->zDb, pC->zName,zBind);
23242 sqlite3_freesqlite3_api->free(zBind);
23243 break;
23244 }
23245
23246 case FTS5_STMT_REPLACE_DOCSIZE7:
23247 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt], pC->zDb, pC->zName,
23248 (pC->bContentlessDelete ? ",?" : "")
23249 );
23250 break;
23251
23252 case FTS5_STMT_LOOKUP_DOCSIZE9:
23253 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt],
23254 (pC->bContentlessDelete ? ",origin" : ""),
23255 pC->zDb, pC->zName
23256 );
23257 break;
23258
23259 default:
23260 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt], pC->zDb, pC->zName);
23261 break;
23262 }
23263
23264 if( zSql==0 ){
23265 rc = SQLITE_NOMEM7;
23266 }else{
23267 int f = SQLITE_PREPARE_PERSISTENT0x01;
23268 if( eStmt>FTS5_STMT_LOOKUP23 ) f |= SQLITE_PREPARE_NO_VTAB0x04;
23269 p->pConfig->bLock++;
23270 rc = sqlite3_prepare_v3sqlite3_api->prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0);
23271 p->pConfig->bLock--;
23272 sqlite3_freesqlite3_api->free(zSql);
23273 if( rc!=SQLITE_OK0 && pzErrMsg ){
23274 *pzErrMsg = sqlite3_mprintfsqlite3_api->mprintf("%s", sqlite3_errmsgsqlite3_api->errmsg(pC->db));
23275 }
23276 if( rc==SQLITE_ERROR1 && eStmt>FTS5_STMT_LOOKUP23 && eStmt<FTS5_STMT_SCAN11 ){
23277 /* One of the internal tables - not the %_content table - is missing.
23278 ** This counts as a corrupted table. */
23279 rc = SQLITE_CORRUPT11;
23280 }
23281 }
23282 }
23283
23284 *ppStmt = p->aStmt[eStmt];
23285 sqlite3_resetsqlite3_api->reset(*ppStmt);
23286 return rc;
23287}
23288
23289
23290static int fts5ExecPrintf(
23291 sqlite3 *db,
23292 char **pzErr,
23293 const char *zFormat,
23294 ...
23295){
23296 int rc;
23297 va_list ap; /* ... printf arguments */
23298 char *zSql;
23299
23300 va_start(ap, zFormat)__builtin_va_start(ap, zFormat);
23301 zSql = sqlite3_vmprintfsqlite3_api->vmprintf(zFormat, ap);
23302
23303 if( zSql==0 ){
23304 rc = SQLITE_NOMEM7;
23305 }else{
23306 rc = sqlite3_execsqlite3_api->exec(db, zSql, 0, 0, pzErr);
23307 sqlite3_freesqlite3_api->free(zSql);
23308 }
23309
23310 va_end(ap)__builtin_va_end(ap);
23311 return rc;
23312}
23313
23314/*
23315** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error
23316** code otherwise.
23317*/
23318static int sqlite3Fts5DropAll(Fts5Config *pConfig){
23319 int rc = fts5ExecPrintf(pConfig->db, 0,
23320 "DROP TABLE IF EXISTS %Q.'%q_data';"
23321 "DROP TABLE IF EXISTS %Q.'%q_idx';"
23322 "DROP TABLE IF EXISTS %Q.'%q_config';",
23323 pConfig->zDb, pConfig->zName,
23324 pConfig->zDb, pConfig->zName,
23325 pConfig->zDb, pConfig->zName
23326 );
23327 if( rc==SQLITE_OK0 && pConfig->bColumnsize ){
23328 rc = fts5ExecPrintf(pConfig->db, 0,
23329 "DROP TABLE IF EXISTS %Q.'%q_docsize';",
23330 pConfig->zDb, pConfig->zName
23331 );
23332 }
23333 if( rc==SQLITE_OK0 && pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
23334 rc = fts5ExecPrintf(pConfig->db, 0,
23335 "DROP TABLE IF EXISTS %Q.'%q_content';",
23336 pConfig->zDb, pConfig->zName
23337 );
23338 }
23339 return rc;
23340}
23341
23342static void fts5StorageRenameOne(
23343 Fts5Config *pConfig, /* Current FTS5 configuration */
23344 int *pRc, /* IN/OUT: Error code */
23345 const char *zTail, /* Tail of table name e.g. "data", "config" */
23346 const char *zName /* New name of FTS5 table */
23347){
23348 if( *pRc==SQLITE_OK0 ){
23349 *pRc = fts5ExecPrintf(pConfig->db, 0,
23350 "ALTER TABLE %Q.'%q_%s' RENAME TO '%q_%s';",
23351 pConfig->zDb, pConfig->zName, zTail, zName, zTail
23352 );
23353 }
23354}
23355
23356static int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){
23357 Fts5Config *pConfig = pStorage->pConfig;
23358 int rc = sqlite3Fts5StorageSync(pStorage);
23359
23360 fts5StorageRenameOne(pConfig, &rc, "data", zName);
23361 fts5StorageRenameOne(pConfig, &rc, "idx", zName);
23362 fts5StorageRenameOne(pConfig, &rc, "config", zName);
23363 if( pConfig->bColumnsize ){
23364 fts5StorageRenameOne(pConfig, &rc, "docsize", zName);
23365 }
23366 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
23367 fts5StorageRenameOne(pConfig, &rc, "content", zName);
23368 }
23369 return rc;
23370}
23371
23372/*
23373** Create the shadow table named zPost, with definition zDefn. Return
23374** SQLITE_OK if successful, or an SQLite error code otherwise.
23375*/
23376static int sqlite3Fts5CreateTable(
23377 Fts5Config *pConfig, /* FTS5 configuration */
23378 const char *zPost, /* Shadow table to create (e.g. "content") */
23379 const char *zDefn, /* Columns etc. for shadow table */
23380 int bWithout, /* True for without rowid */
23381 char **pzErr /* OUT: Error message */
23382){
23383 int rc;
23384 char *zErr = 0;
23385
23386 rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s",
23387 pConfig->zDb, pConfig->zName, zPost, zDefn,
23388#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID
23389 bWithout?" WITHOUT ROWID":
23390#endif
23391 ""
23392 );
23393 if( zErr ){
23394 *pzErr = sqlite3_mprintfsqlite3_api->mprintf(
23395 "fts5: error creating shadow table %q_%s: %s",
23396 pConfig->zName, zPost, zErr
23397 );
23398 sqlite3_freesqlite3_api->free(zErr);
23399 }
23400
23401 return rc;
23402}
23403
23404/*
23405** Open a new Fts5Index handle. If the bCreate argument is true, create
23406** and initialize the underlying tables
23407**
23408** If successful, set *pp to point to the new object and return SQLITE_OK.
23409** Otherwise, set *pp to NULL and return an SQLite error code.
23410*/
23411static int sqlite3Fts5StorageOpen(
23412 Fts5Config *pConfig,
23413 Fts5Index *pIndex,
23414 int bCreate,
23415 Fts5Storage **pp,
23416 char **pzErr /* OUT: Error message */
23417){
23418 int rc = SQLITE_OK0;
23419 Fts5Storage *p; /* New object */
23420 sqlite3_int64 nByte; /* Bytes of space to allocate */
23421
23422 nByte = sizeof(Fts5Storage) /* Fts5Storage object */
23423 + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */
23424 *pp = p = (Fts5Storage*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
23425 if( !p ) return SQLITE_NOMEM7;
23426
23427 memset(p, 0, (size_t)nByte);
23428 p->aTotalSize = (i64*)&p[1];
23429 p->pConfig = pConfig;
23430 p->pIndex = pIndex;
23431
23432 if( bCreate ){
23433 if( pConfig->eContent==FTS5_CONTENT_NORMAL0
23434 || pConfig->eContent==FTS5_CONTENT_UNINDEXED3
23435 ){
23436 int nDefn = 32 + pConfig->nCol*10;
23437 char *zDefn = sqlite3_malloc64sqlite3_api->malloc64(32 + (sqlite3_int64)pConfig->nCol * 20);
23438 if( zDefn==0 ){
23439 rc = SQLITE_NOMEM7;
23440 }else{
23441 int i;
23442 int iOff;
23443 sqlite3_snprintfsqlite3_api->xsnprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY");
23444 iOff = (int)strlen(zDefn);
23445 for(i=0; i<pConfig->nCol; i++){
23446 if( pConfig->eContent==FTS5_CONTENT_NORMAL0
23447 || pConfig->abUnindexed[i]
23448 ){
23449 sqlite3_snprintfsqlite3_api->xsnprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
23450 iOff += (int)strlen(&zDefn[iOff]);
23451 }
23452 }
23453 if( pConfig->bLocale ){
23454 for(i=0; i<pConfig->nCol; i++){
23455 if( pConfig->abUnindexed[i]==0 ){
23456 sqlite3_snprintfsqlite3_api->xsnprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i);
23457 iOff += (int)strlen(&zDefn[iOff]);
23458 }
23459 }
23460 }
23461 rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
23462 }
23463 sqlite3_freesqlite3_api->free(zDefn);
23464 }
23465
23466 if( rc==SQLITE_OK0 && pConfig->bColumnsize ){
23467 const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
23468 if( pConfig->bContentlessDelete ){
23469 zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER";
23470 }
23471 rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
23472 }
23473 if( rc==SQLITE_OK0 ){
23474 rc = sqlite3Fts5CreateTable(
23475 pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
23476 );
23477 }
23478 if( rc==SQLITE_OK0 ){
23479 rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION4);
23480 }
23481 }
23482
23483 if( rc ){
23484 sqlite3Fts5StorageClose(p);
23485 *pp = 0;
23486 }
23487 return rc;
23488}
23489
23490/*
23491** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen().
23492*/
23493static int sqlite3Fts5StorageClose(Fts5Storage *p){
23494 int rc = SQLITE_OK0;
23495 if( p ){
23496 int i;
23497
23498 /* Finalize all SQL statements */
23499 for(i=0; i<ArraySize(p->aStmt)((int)(sizeof(p->aStmt) / sizeof(p->aStmt[0]))); i++){
23500 sqlite3_finalizesqlite3_api->finalize(p->aStmt[i]);
23501 }
23502
23503 sqlite3_freesqlite3_api->free(p);
23504 }
23505 return rc;
23506}
23507
23508typedef struct Fts5InsertCtx Fts5InsertCtx;
23509struct Fts5InsertCtx {
23510 Fts5Storage *pStorage;
23511 int iCol;
23512 int szCol; /* Size of column value in tokens */
23513};
23514
23515/*
23516** Tokenization callback used when inserting tokens into the FTS index.
23517*/
23518static int fts5StorageInsertCallback(
23519 void *pContext, /* Pointer to Fts5InsertCtx object */
23520 int tflags,
23521 const char *pToken, /* Buffer containing token */
23522 int nToken, /* Size of token in bytes */
23523 int iUnused1, /* Start offset of token */
23524 int iUnused2 /* End offset of token */
23525){
23526 Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
23527 Fts5Index *pIdx = pCtx->pStorage->pIndex;
23528 UNUSED_PARAM2(iUnused1, iUnused2)(void)(iUnused1), (void)(iUnused2);
23529 if( nToken>FTS5_MAX_TOKEN_SIZE32768 ) nToken = FTS5_MAX_TOKEN_SIZE32768;
23530 if( (tflags & FTS5_TOKEN_COLOCATED0x0001)==0 || pCtx->szCol==0 ){
23531 pCtx->szCol++;
23532 }
23533 return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
23534}
23535
23536/*
23537** This function is used as part of an UPDATE statement that modifies the
23538** rowid of a row. In that case, this function is called first to set
23539** Fts5Storage.pSavedRow to point to a statement that may be used to
23540** access the original values of the row being deleted - iDel.
23541**
23542** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
23543** It is not considered an error if row iDel does not exist. In this case
23544** pSavedRow is not set and SQLITE_OK returned.
23545*/
23546static int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){
23547 int rc = SQLITE_OK0;
23548 sqlite3_stmt *pSeek = 0;
23549
23550 assert( p->pSavedRow==0 )((void) (0));
23551 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP2+1, &pSeek, 0);
23552 if( rc==SQLITE_OK0 ){
23553 sqlite3_bind_int64sqlite3_api->bind_int64(pSeek, 1, iDel);
23554 if( sqlite3_stepsqlite3_api->step(pSeek)!=SQLITE_ROW100 ){
23555 rc = sqlite3_resetsqlite3_api->reset(pSeek);
23556 }else{
23557 p->pSavedRow = pSeek;
23558 }
23559 }
23560
23561 return rc;
23562}
23563
23564/*
23565** If a row with rowid iDel is present in the %_content table, add the
23566** delete-markers to the FTS index necessary to delete it. Do not actually
23567** remove the %_content row at this time though.
23568**
23569** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left
23570** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access
23571** the original values of the row being deleted. This is used by UPDATE
23572** statements.
23573*/
23574static int fts5StorageDeleteFromIndex(
23575 Fts5Storage *p,
23576 i64 iDel,
23577 sqlite3_value **apVal,
23578 int bSaveRow /* True to set pSavedRow */
23579){
23580 Fts5Config *pConfig = p->pConfig;
23581 sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */
23582 int rc = SQLITE_OK0; /* Return code */
23583 int rc2; /* sqlite3_reset() return code */
23584 int iCol;
23585 Fts5InsertCtx ctx;
23586
23587 assert( bSaveRow==0 || apVal==0 )((void) (0));
23588 assert( bSaveRow==0 || bSaveRow==1 )((void) (0));
23589 assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 )((void) (0));
23590
23591 if( apVal==0 ){
23592 if( p->pSavedRow && bSaveRow ){
23593 pSeek = p->pSavedRow;
23594 p->pSavedRow = 0;
23595 }else{
23596 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP2+bSaveRow, &pSeek, 0);
23597 if( rc!=SQLITE_OK0 ) return rc;
23598 sqlite3_bind_int64sqlite3_api->bind_int64(pSeek, 1, iDel);
23599 if( sqlite3_stepsqlite3_api->step(pSeek)!=SQLITE_ROW100 ){
23600 return sqlite3_resetsqlite3_api->reset(pSeek);
23601 }
23602 }
23603 }
23604
23605 ctx.pStorage = p;
23606 ctx.iCol = -1;
23607 for(iCol=1; rc==SQLITE_OK0 && iCol<=pConfig->nCol; iCol++){
23608 if( pConfig->abUnindexed[iCol-1]==0 ){
23609 sqlite3_value *pVal = 0;
23610 sqlite3_value *pFree = 0;
23611 const char *pText = 0;
23612 int nText = 0;
23613 const char *pLoc = 0;
23614 int nLoc = 0;
23615
23616 assert( pSeek==0 || apVal==0 )((void) (0));
23617 assert( pSeek!=0 || apVal!=0 )((void) (0));
23618 if( pSeek ){
23619 pVal = sqlite3_column_valuesqlite3_api->column_value(pSeek, iCol);
23620 }else{
23621 pVal = apVal[iCol-1];
23622 }
23623
23624 if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
23625 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
23626 }else{
23627 if( sqlite3_value_typesqlite3_api->value_type(pVal)!=SQLITE_TEXT3 ){
23628 /* Make a copy of the value to work with. This is because the call
23629 ** to sqlite3_value_text() below forces the type of the value to
23630 ** SQLITE_TEXT, and we may need to use it again later. */
23631 pFree = pVal = sqlite3_value_dupsqlite3_api->value_dup(pVal);
23632 if( pVal==0 ){
23633 rc = SQLITE_NOMEM7;
23634 }
23635 }
23636 if( rc==SQLITE_OK0 ){
23637 pText = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
23638 nText = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
23639 if( pConfig->bLocale && pSeek ){
23640 pLoc = (const char*)sqlite3_column_textsqlite3_api->column_text(pSeek, iCol+pConfig->nCol);
23641 nLoc = sqlite3_column_bytessqlite3_api->column_bytes(pSeek, iCol + pConfig->nCol);
23642 }
23643 }
23644 }
23645
23646 if( rc==SQLITE_OK0 ){
23647 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
23648 ctx.szCol = 0;
23649 rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT0x0004,
23650 pText, nText, (void*)&ctx, fts5StorageInsertCallback
23651 );
23652 p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
23653 if( rc==SQLITE_OK0 && p->aTotalSize[iCol-1]<0 ){
23654 rc = FTS5_CORRUPT(11 | (1<<8));
23655 }
23656 sqlite3Fts5ClearLocale(pConfig);
23657 }
23658 sqlite3_value_freesqlite3_api->value_free(pFree);
23659 }
23660 }
23661 if( rc==SQLITE_OK0 && p->nTotalRow<1 ){
23662 rc = FTS5_CORRUPT(11 | (1<<8));
23663 }else{
23664 p->nTotalRow--;
23665 }
23666
23667 if( rc==SQLITE_OK0 && bSaveRow ){
23668 assert( p->pSavedRow==0 )((void) (0));
23669 p->pSavedRow = pSeek;
23670 }else{
23671 rc2 = sqlite3_resetsqlite3_api->reset(pSeek);
23672 if( rc==SQLITE_OK0 ) rc = rc2;
23673 }
23674 return rc;
23675}
23676
23677/*
23678** Reset any saved statement pSavedRow. Zero pSavedRow as well. This
23679** should be called by the xUpdate() method of the fts5 table before
23680** returning from any operation that may have set Fts5Storage.pSavedRow.
23681*/
23682static void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){
23683 assert( pStorage->pSavedRow==0((void) (0))
23684 || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2]((void) (0))
23685 )((void) (0));
23686 sqlite3_resetsqlite3_api->reset(pStorage->pSavedRow);
23687 pStorage->pSavedRow = 0;
23688}
23689
23690/*
23691** This function is called to process a DELETE on a contentless_delete=1
23692** table. It adds the tombstone required to delete the entry with rowid
23693** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs,
23694** an SQLite error code.
23695*/
23696static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
23697 i64 iOrigin = 0;
23698 sqlite3_stmt *pLookup = 0;
23699 int rc = SQLITE_OK0;
23700
23701 assert( p->pConfig->bContentlessDelete )((void) (0));
23702 assert( p->pConfig->eContent==FTS5_CONTENT_NONE((void) (0))
23703 || p->pConfig->eContent==FTS5_CONTENT_UNINDEXED((void) (0))
23704 )((void) (0));
23705
23706 /* Look up the origin of the document in the %_docsize table. Store
23707 ** this in stack variable iOrigin. */
23708 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE9, &pLookup, 0);
23709 if( rc==SQLITE_OK0 ){
23710 sqlite3_bind_int64sqlite3_api->bind_int64(pLookup, 1, iDel);
23711 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pLookup) ){
23712 iOrigin = sqlite3_column_int64sqlite3_api->column_int64(pLookup, 1);
23713 }
23714 rc = sqlite3_resetsqlite3_api->reset(pLookup);
23715 }
23716
23717 if( rc==SQLITE_OK0 && iOrigin!=0 ){
23718 rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel);
23719 }
23720
23721 return rc;
23722}
23723
23724/*
23725** Insert a record into the %_docsize table. Specifically, do:
23726**
23727** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
23728**
23729** If there is no %_docsize table (as happens if the columnsize=0 option
23730** is specified when the FTS5 table is created), this function is a no-op.
23731*/
23732static int fts5StorageInsertDocsize(
23733 Fts5Storage *p, /* Storage module to write to */
23734 i64 iRowid, /* id value */
23735 Fts5Buffer *pBuf /* sz value */
23736){
23737 int rc = SQLITE_OK0;
23738 if( p->pConfig->bColumnsize ){
23739 sqlite3_stmt *pReplace = 0;
23740 rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE7, &pReplace, 0);
23741 if( rc==SQLITE_OK0 ){
23742 sqlite3_bind_int64sqlite3_api->bind_int64(pReplace, 1, iRowid);
23743 if( p->pConfig->bContentlessDelete ){
23744 i64 iOrigin = 0;
23745 rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
23746 sqlite3_bind_int64sqlite3_api->bind_int64(pReplace, 3, iOrigin);
23747 }
23748 }
23749 if( rc==SQLITE_OK0 ){
23750 sqlite3_bind_blobsqlite3_api->bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC((sqlite3_destructor_type)0));
23751 sqlite3_stepsqlite3_api->step(pReplace);
23752 rc = sqlite3_resetsqlite3_api->reset(pReplace);
23753 sqlite3_bind_nullsqlite3_api->bind_null(pReplace, 2);
23754 }
23755 }
23756 return rc;
23757}
23758
23759/*
23760** Load the contents of the "averages" record from disk into the
23761** p->nTotalRow and p->aTotalSize[] variables. If successful, and if
23762** argument bCache is true, set the p->bTotalsValid flag to indicate
23763** that the contents of aTotalSize[] and nTotalRow are valid until
23764** further notice.
23765**
23766** Return SQLITE_OK if successful, or an SQLite error code if an error
23767** occurs.
23768*/
23769static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){
23770 int rc = SQLITE_OK0;
23771 if( p->bTotalsValid==0 ){
23772 rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize);
23773 p->bTotalsValid = bCache;
23774 }
23775 return rc;
23776}
23777
23778/*
23779** Store the current contents of the p->nTotalRow and p->aTotalSize[]
23780** variables in the "averages" record on disk.
23781**
23782** Return SQLITE_OK if successful, or an SQLite error code if an error
23783** occurs.
23784*/
23785static int fts5StorageSaveTotals(Fts5Storage *p){
23786 int nCol = p->pConfig->nCol;
23787 int i;
23788 Fts5Buffer buf;
23789 int rc = SQLITE_OK0;
23790 memset(&buf, 0, sizeof(buf));
23791
23792 sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow);
23793 for(i=0; i<nCol; i++){
23794 sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]);
23795 }
23796 if( rc==SQLITE_OK0 ){
23797 rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n);
23798 }
23799 sqlite3_freesqlite3_api->free(buf.p);
23800
23801 return rc;
23802}
23803
23804/*
23805** Remove a row from the FTS table.
23806*/
23807static int sqlite3Fts5StorageDelete(
23808 Fts5Storage *p, /* Storage object */
23809 i64 iDel, /* Rowid to delete from table */
23810 sqlite3_value **apVal, /* Optional - values to remove from index */
23811 int bSaveRow /* If true, set pSavedRow for deleted row */
23812){
23813 Fts5Config *pConfig = p->pConfig;
23814 int rc;
23815 sqlite3_stmt *pDel = 0;
23816
23817 assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 )((void) (0));
23818 rc = fts5StorageLoadTotals(p, 1);
23819
23820 /* Delete the index records */
23821 if( rc==SQLITE_OK0 ){
23822 rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
23823 }
23824
23825 if( rc==SQLITE_OK0 ){
23826 if( p->pConfig->bContentlessDelete ){
23827 rc = fts5StorageContentlessDelete(p, iDel);
23828 if( rc==SQLITE_OK0
23829 && bSaveRow
23830 && p->pConfig->eContent==FTS5_CONTENT_UNINDEXED3
23831 ){
23832 rc = sqlite3Fts5StorageFindDeleteRow(p, iDel);
23833 }
23834 }else{
23835 rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow);
23836 }
23837 }
23838
23839 /* Delete the %_docsize record */
23840 if( rc==SQLITE_OK0 && pConfig->bColumnsize ){
23841 rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE8, &pDel, 0);
23842 if( rc==SQLITE_OK0 ){
23843 sqlite3_bind_int64sqlite3_api->bind_int64(pDel, 1, iDel);
23844 sqlite3_stepsqlite3_api->step(pDel);
23845 rc = sqlite3_resetsqlite3_api->reset(pDel);
23846 }
23847 }
23848
23849 /* Delete the %_content record */
23850 if( pConfig->eContent==FTS5_CONTENT_NORMAL0
23851 || pConfig->eContent==FTS5_CONTENT_UNINDEXED3
23852 ){
23853 if( rc==SQLITE_OK0 ){
23854 rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT6, &pDel, 0);
23855 }
23856 if( rc==SQLITE_OK0 ){
23857 sqlite3_bind_int64sqlite3_api->bind_int64(pDel, 1, iDel);
23858 sqlite3_stepsqlite3_api->step(pDel);
23859 rc = sqlite3_resetsqlite3_api->reset(pDel);
23860 }
23861 }
23862
23863 return rc;
23864}
23865
23866/*
23867** Delete all entries in the FTS5 index.
23868*/
23869static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
23870 Fts5Config *pConfig = p->pConfig;
23871 int rc;
23872
23873 p->bTotalsValid = 0;
23874
23875 /* Delete the contents of the %_data and %_docsize tables. */
23876 rc = fts5ExecPrintf(pConfig->db, 0,
23877 "DELETE FROM %Q.'%q_data';"
23878 "DELETE FROM %Q.'%q_idx';",
23879 pConfig->zDb, pConfig->zName,
23880 pConfig->zDb, pConfig->zName
23881 );
23882 if( rc==SQLITE_OK0 && pConfig->bColumnsize ){
23883 rc = fts5ExecPrintf(pConfig->db, 0,
23884 "DELETE FROM %Q.'%q_docsize';", pConfig->zDb, pConfig->zName
23885 );
23886 }
23887
23888 if( rc==SQLITE_OK0 && pConfig->eContent==FTS5_CONTENT_UNINDEXED3 ){
23889 rc = fts5ExecPrintf(pConfig->db, 0,
23890 "DELETE FROM %Q.'%q_content';", pConfig->zDb, pConfig->zName
23891 );
23892 }
23893
23894 /* Reinitialize the %_data table. This call creates the initial structure
23895 ** and averages records. */
23896 if( rc==SQLITE_OK0 ){
23897 rc = sqlite3Fts5IndexReinit(p->pIndex);
23898 }
23899 if( rc==SQLITE_OK0 ){
23900 rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION4);
23901 }
23902 return rc;
23903}
23904
23905static int sqlite3Fts5StorageRebuild(Fts5Storage *p){
23906 Fts5Buffer buf = {0,0,0};
23907 Fts5Config *pConfig = p->pConfig;
23908 sqlite3_stmt *pScan = 0;
23909 Fts5InsertCtx ctx;
23910 int rc, rc2;
23911
23912 memset(&ctx, 0, sizeof(Fts5InsertCtx));
23913 ctx.pStorage = p;
23914 rc = sqlite3Fts5StorageDeleteAll(p);
23915 if( rc==SQLITE_OK0 ){
23916 rc = fts5StorageLoadTotals(p, 1);
23917 }
23918
23919 if( rc==SQLITE_OK0 ){
23920 rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN11, &pScan, pConfig->pzErrmsg);
23921 }
23922
23923 while( rc==SQLITE_OK0 && SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pScan) ){
23924 i64 iRowid = sqlite3_column_int64sqlite3_api->column_int64(pScan, 0);
23925
23926 sqlite3Fts5BufferZero(&buf);
23927 rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
23928 for(ctx.iCol=0; rc==SQLITE_OK0 && ctx.iCol<pConfig->nCol; ctx.iCol++){
23929 ctx.szCol = 0;
23930 if( pConfig->abUnindexed[ctx.iCol]==0 ){
23931 int nText = 0; /* Size of pText in bytes */
23932 const char *pText = 0; /* Pointer to buffer containing text value */
23933 int nLoc = 0; /* Size of pLoc in bytes */
23934 const char *pLoc = 0; /* Pointer to buffer containing text value */
23935
23936 sqlite3_value *pVal = sqlite3_column_valuesqlite3_api->column_value(pScan, ctx.iCol+1);
23937 if( pConfig->eContent==FTS5_CONTENT_EXTERNAL2
23938 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
23939 ){
23940 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
23941 }else{
23942 pText = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
23943 nText = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
23944 if( pConfig->bLocale ){
23945 int iCol = ctx.iCol + 1 + pConfig->nCol;
23946 pLoc = (const char*)sqlite3_column_textsqlite3_api->column_text(pScan, iCol);
23947 nLoc = sqlite3_column_bytessqlite3_api->column_bytes(pScan, iCol);
23948 }
23949 }
23950
23951 if( rc==SQLITE_OK0 ){
23952 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
23953 rc = sqlite3Fts5Tokenize(pConfig,
23954 FTS5_TOKENIZE_DOCUMENT0x0004,
23955 pText, nText,
23956 (void*)&ctx,
23957 fts5StorageInsertCallback
23958 );
23959 sqlite3Fts5ClearLocale(pConfig);
23960 }
23961 }
23962 sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
23963 p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
23964 }
23965 p->nTotalRow++;
23966
23967 if( rc==SQLITE_OK0 ){
23968 rc = fts5StorageInsertDocsize(p, iRowid, &buf);
23969 }
23970 }
23971 sqlite3_freesqlite3_api->free(buf.p);
23972 rc2 = sqlite3_resetsqlite3_api->reset(pScan);
23973 if( rc==SQLITE_OK0 ) rc = rc2;
23974
23975 /* Write the averages record */
23976 if( rc==SQLITE_OK0 ){
23977 rc = fts5StorageSaveTotals(p);
23978 }
23979 return rc;
23980}
23981
23982static int sqlite3Fts5StorageOptimize(Fts5Storage *p){
23983 return sqlite3Fts5IndexOptimize(p->pIndex);
23984}
23985
23986static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
23987 return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
23988}
23989
23990static int sqlite3Fts5StorageReset(Fts5Storage *p){
23991 return sqlite3Fts5IndexReset(p->pIndex);
23992}
23993
23994/*
23995** Allocate a new rowid. This is used for "external content" tables when
23996** a NULL value is inserted into the rowid column. The new rowid is allocated
23997** by inserting a dummy row into the %_docsize table. The dummy will be
23998** overwritten later.
23999**
24000** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In
24001** this case the user is required to provide a rowid explicitly.
24002*/
24003static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
24004 int rc = SQLITE_MISMATCH20;
24005 if( p->pConfig->bColumnsize ){
24006 sqlite3_stmt *pReplace = 0;
24007 rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE7, &pReplace, 0);
24008 if( rc==SQLITE_OK0 ){
24009 sqlite3_bind_nullsqlite3_api->bind_null(pReplace, 1);
24010 sqlite3_bind_nullsqlite3_api->bind_null(pReplace, 2);
24011 sqlite3_stepsqlite3_api->step(pReplace);
24012 rc = sqlite3_resetsqlite3_api->reset(pReplace);
24013 }
24014 if( rc==SQLITE_OK0 ){
24015 *piRowid = sqlite3_last_insert_rowidsqlite3_api->last_insert_rowid(p->pConfig->db);
24016 }
24017 }
24018 return rc;
24019}
24020
24021/*
24022** Insert a new row into the FTS content table.
24023*/
24024static int sqlite3Fts5StorageContentInsert(
24025 Fts5Storage *p,
24026 int bReplace, /* True to use REPLACE instead of INSERT */
24027 sqlite3_value **apVal,
24028 i64 *piRowid
24029){
24030 Fts5Config *pConfig = p->pConfig;
24031 int rc = SQLITE_OK0;
24032
24033 /* Insert the new row into the %_content table. */
24034 if( pConfig->eContent!=FTS5_CONTENT_NORMAL0
24035 && pConfig->eContent!=FTS5_CONTENT_UNINDEXED3
24036 ){
24037 if( sqlite3_value_typesqlite3_api->value_type(apVal[1])==SQLITE_INTEGER1 ){
24038 *piRowid = sqlite3_value_int64sqlite3_api->value_int64(apVal[1]);
24039 }else{
24040 rc = fts5StorageNewRowid(p, piRowid);
24041 }
24042 }else{
24043 sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */
24044 int i; /* Counter variable */
24045
24046 assert( FTS5_STMT_INSERT_CONTENT+1==FTS5_STMT_REPLACE_CONTENT )((void) (0));
24047 assert( bReplace==0 || bReplace==1 )((void) (0));
24048 rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT4+bReplace, &pInsert, 0);
24049 if( pInsert ) sqlite3_clear_bindingssqlite3_api->clear_bindings(pInsert);
24050
24051 /* Bind the rowid value */
24052 sqlite3_bind_valuesqlite3_api->bind_value(pInsert, 1, apVal[1]);
24053
24054 /* Loop through values for user-defined columns. i=2 is the leftmost
24055 ** user-defined column. As is column 1 of pSavedRow. */
24056 for(i=2; rc==SQLITE_OK0 && i<=pConfig->nCol+1; i++){
24057 int bUnindexed = pConfig->abUnindexed[i-2];
24058 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 || bUnindexed ){
24059 sqlite3_value *pVal = apVal[i];
24060
24061 if( sqlite3_value_nochangesqlite3_api->value_nochange(pVal) && p->pSavedRow ){
24062 /* This is an UPDATE statement, and user-defined column (i-2) was not
24063 ** modified. Retrieve the value from Fts5Storage.pSavedRow. */
24064 pVal = sqlite3_column_valuesqlite3_api->column_value(p->pSavedRow, i-1);
24065 if( pConfig->bLocale && bUnindexed==0 ){
24066 sqlite3_bind_valuesqlite3_api->bind_value(pInsert, pConfig->nCol + i,
24067 sqlite3_column_valuesqlite3_api->column_value(p->pSavedRow, pConfig->nCol + i - 1)
24068 );
24069 }
24070 }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
24071 const char *pText = 0;
24072 const char *pLoc = 0;
24073 int nText = 0;
24074 int nLoc = 0;
24075 assert( pConfig->bLocale )((void) (0));
24076
24077 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
24078 if( rc==SQLITE_OK0 ){
24079 sqlite3_bind_textsqlite3_api->bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
24080 if( bUnindexed==0 ){
24081 int iLoc = pConfig->nCol + i;
24082 sqlite3_bind_textsqlite3_api->bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
24083 }
24084 }
24085
24086 continue;
24087 }
24088
24089 rc = sqlite3_bind_valuesqlite3_api->bind_value(pInsert, i, pVal);
24090 }
24091 }
24092 if( rc==SQLITE_OK0 ){
24093 sqlite3_stepsqlite3_api->step(pInsert);
24094 rc = sqlite3_resetsqlite3_api->reset(pInsert);
24095 }
24096 *piRowid = sqlite3_last_insert_rowidsqlite3_api->last_insert_rowid(pConfig->db);
24097 }
24098
24099 return rc;
24100}
24101
24102/*
24103** Insert new entries into the FTS index and %_docsize table.
24104*/
24105static int sqlite3Fts5StorageIndexInsert(
24106 Fts5Storage *p,
24107 sqlite3_value **apVal,
24108 i64 iRowid
24109){
24110 Fts5Config *pConfig = p->pConfig;
24111 int rc = SQLITE_OK0; /* Return code */
24112 Fts5InsertCtx ctx; /* Tokenization callback context object */
24113 Fts5Buffer buf; /* Buffer used to build up %_docsize blob */
24114
24115 memset(&buf, 0, sizeof(Fts5Buffer));
24116 ctx.pStorage = p;
24117 rc = fts5StorageLoadTotals(p, 1);
24118
24119 if( rc==SQLITE_OK0 ){
24120 rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
24121 }
24122 for(ctx.iCol=0; rc==SQLITE_OK0 && ctx.iCol<pConfig->nCol; ctx.iCol++){
24123 ctx.szCol = 0;
24124 if( pConfig->abUnindexed[ctx.iCol]==0 ){
24125 int nText = 0; /* Size of pText in bytes */
24126 const char *pText = 0; /* Pointer to buffer containing text value */
24127 int nLoc = 0; /* Size of pText in bytes */
24128 const char *pLoc = 0; /* Pointer to buffer containing text value */
24129
24130 sqlite3_value *pVal = apVal[ctx.iCol+2];
24131 if( p->pSavedRow && sqlite3_value_nochangesqlite3_api->value_nochange(pVal) ){
24132 pVal = sqlite3_column_valuesqlite3_api->column_value(p->pSavedRow, ctx.iCol+1);
24133 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 && pConfig->bLocale ){
24134 int iCol = ctx.iCol + 1 + pConfig->nCol;
24135 pLoc = (const char*)sqlite3_column_textsqlite3_api->column_text(p->pSavedRow, iCol);
24136 nLoc = sqlite3_column_bytessqlite3_api->column_bytes(p->pSavedRow, iCol);
24137 }
24138 }else{
24139 pVal = apVal[ctx.iCol+2];
24140 }
24141
24142 if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
24143 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
24144 }else{
24145 pText = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
24146 nText = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
24147 }
24148
24149 if( rc==SQLITE_OK0 ){
24150 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
24151 rc = sqlite3Fts5Tokenize(pConfig,
24152 FTS5_TOKENIZE_DOCUMENT0x0004, pText, nText, (void*)&ctx,
24153 fts5StorageInsertCallback
24154 );
24155 sqlite3Fts5ClearLocale(pConfig);
24156 }
24157 }
24158 sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
24159 p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
24160 }
24161 p->nTotalRow++;
24162
24163 /* Write the %_docsize record */
24164 if( rc==SQLITE_OK0 ){
24165 rc = fts5StorageInsertDocsize(p, iRowid, &buf);
24166 }
24167 sqlite3_freesqlite3_api->free(buf.p);
24168
24169 return rc;
24170}
24171
24172static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){
24173 Fts5Config *pConfig = p->pConfig;
24174 char *zSql;
24175 int rc;
24176
24177 zSql = sqlite3_mprintfsqlite3_api->mprintf("SELECT count(*) FROM %Q.'%q_%s'",
24178 pConfig->zDb, pConfig->zName, zSuffix
24179 );
24180 if( zSql==0 ){
24181 rc = SQLITE_NOMEM7;
24182 }else{
24183 sqlite3_stmt *pCnt = 0;
24184 rc = sqlite3_prepare_v2sqlite3_api->prepare_v2(pConfig->db, zSql, -1, &pCnt, 0);
24185 if( rc==SQLITE_OK0 ){
24186 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pCnt) ){
24187 *pnRow = sqlite3_column_int64sqlite3_api->column_int64(pCnt, 0);
24188 }
24189 rc = sqlite3_finalizesqlite3_api->finalize(pCnt);
24190 }
24191 }
24192
24193 sqlite3_freesqlite3_api->free(zSql);
24194 return rc;
24195}
24196
24197/*
24198** Context object used by sqlite3Fts5StorageIntegrity().
24199*/
24200typedef struct Fts5IntegrityCtx Fts5IntegrityCtx;
24201struct Fts5IntegrityCtx {
24202 i64 iRowid;
24203 int iCol;
24204 int szCol;
24205 u64 cksum;
24206 Fts5Termset *pTermset;
24207 Fts5Config *pConfig;
24208};
24209
24210
24211/*
24212** Tokenization callback used by integrity check.
24213*/
24214static int fts5StorageIntegrityCallback(
24215 void *pContext, /* Pointer to Fts5IntegrityCtx object */
24216 int tflags,
24217 const char *pToken, /* Buffer containing token */
24218 int nToken, /* Size of token in bytes */
24219 int iUnused1, /* Start offset of token */
24220 int iUnused2 /* End offset of token */
24221){
24222 Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
24223 Fts5Termset *pTermset = pCtx->pTermset;
24224 int bPresent;
24225 int ii;
24226 int rc = SQLITE_OK0;
24227 int iPos;
24228 int iCol;
24229
24230 UNUSED_PARAM2(iUnused1, iUnused2)(void)(iUnused1), (void)(iUnused2);
24231 if( nToken>FTS5_MAX_TOKEN_SIZE32768 ) nToken = FTS5_MAX_TOKEN_SIZE32768;
24232
24233 if( (tflags & FTS5_TOKEN_COLOCATED0x0001)==0 || pCtx->szCol==0 ){
24234 pCtx->szCol++;
24235 }
24236
24237 switch( pCtx->pConfig->eDetail ){
24238 case FTS5_DETAIL_FULL0:
24239 iPos = pCtx->szCol-1;
24240 iCol = pCtx->iCol;
24241 break;
24242
24243 case FTS5_DETAIL_COLUMNS2:
24244 iPos = pCtx->iCol;
24245 iCol = 0;
24246 break;
24247
24248 default:
24249 assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE )((void) (0));
24250 iPos = 0;
24251 iCol = 0;
24252 break;
24253 }
24254
24255 rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
24256 if( rc==SQLITE_OK0 && bPresent==0 ){
24257 pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
24258 pCtx->iRowid, iCol, iPos, 0, pToken, nToken
24259 );
24260 }
24261
24262 for(ii=0; rc==SQLITE_OK0 && ii<pCtx->pConfig->nPrefix; ii++){
24263 const int nChar = pCtx->pConfig->aPrefix[ii];
24264 int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
24265 if( nByte ){
24266 rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent);
24267 if( bPresent==0 ){
24268 pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
24269 pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte
24270 );
24271 }
24272 }
24273 }
24274
24275 return rc;
24276}
24277
24278/*
24279** Check that the contents of the FTS index match that of the %_content
24280** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return
24281** some other SQLite error code if an error occurs while attempting to
24282** determine this.
24283*/
24284static int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
24285 Fts5Config *pConfig = p->pConfig;
24286 int rc = SQLITE_OK0; /* Return code */
24287 int *aColSize; /* Array of size pConfig->nCol */
24288 i64 *aTotalSize; /* Array of size pConfig->nCol */
24289 Fts5IntegrityCtx ctx;
24290 sqlite3_stmt *pScan;
24291 int bUseCksum;
24292
24293 memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
24294 ctx.pConfig = p->pConfig;
24295 aTotalSize = (i64*)sqlite3_malloc64sqlite3_api->malloc64(pConfig->nCol*(sizeof(int)+sizeof(i64)));
24296 if( !aTotalSize ) return SQLITE_NOMEM7;
24297 aColSize = (int*)&aTotalSize[pConfig->nCol];
24298 memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
24299
24300 bUseCksum = (pConfig->eContent==FTS5_CONTENT_NORMAL0
24301 || (pConfig->eContent==FTS5_CONTENT_EXTERNAL2 && iArg)
24302 );
24303 if( bUseCksum ){
24304 /* Generate the expected index checksum based on the contents of the
24305 ** %_content table. This block stores the checksum in ctx.cksum. */
24306 rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN11, &pScan, 0);
24307 if( rc==SQLITE_OK0 ){
24308 int rc2;
24309 while( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pScan) ){
24310 int i;
24311 ctx.iRowid = sqlite3_column_int64sqlite3_api->column_int64(pScan, 0);
24312 ctx.szCol = 0;
24313 if( pConfig->bColumnsize ){
24314 rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
24315 }
24316 if( rc==SQLITE_OK0 && pConfig->eDetail==FTS5_DETAIL_NONE1 ){
24317 rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
24318 }
24319 for(i=0; rc==SQLITE_OK0 && i<pConfig->nCol; i++){
24320 if( pConfig->abUnindexed[i]==0 ){
24321 const char *pText = 0;
24322 int nText = 0;
24323 const char *pLoc = 0;
24324 int nLoc = 0;
24325 sqlite3_value *pVal = sqlite3_column_valuesqlite3_api->column_value(pScan, i+1);
24326
24327 if( pConfig->eContent==FTS5_CONTENT_EXTERNAL2
24328 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
24329 ){
24330 rc = sqlite3Fts5DecodeLocaleValue(
24331 pVal, &pText, &nText, &pLoc, &nLoc
24332 );
24333 }else{
24334 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 && pConfig->bLocale ){
24335 int iCol = i + 1 + pConfig->nCol;
24336 pLoc = (const char*)sqlite3_column_textsqlite3_api->column_text(pScan, iCol);
24337 nLoc = sqlite3_column_bytessqlite3_api->column_bytes(pScan, iCol);
24338 }
24339 pText = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
24340 nText = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
24341 }
24342
24343 ctx.iCol = i;
24344 ctx.szCol = 0;
24345
24346 if( rc==SQLITE_OK0 && pConfig->eDetail==FTS5_DETAIL_COLUMNS2 ){
24347 rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
24348 }
24349
24350 if( rc==SQLITE_OK0 ){
24351 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
24352 rc = sqlite3Fts5Tokenize(pConfig,
24353 FTS5_TOKENIZE_DOCUMENT0x0004,
24354 pText, nText,
24355 (void*)&ctx,
24356 fts5StorageIntegrityCallback
24357 );
24358 sqlite3Fts5ClearLocale(pConfig);
24359 }
24360
24361 /* If this is not a columnsize=0 database, check that the number
24362 ** of tokens in the value matches the aColSize[] value read from
24363 ** the %_docsize table. */
24364 if( rc==SQLITE_OK0
24365 && pConfig->bColumnsize
24366 && ctx.szCol!=aColSize[i]
24367 ){
24368 rc = FTS5_CORRUPT(11 | (1<<8));
24369 }
24370 aTotalSize[i] += ctx.szCol;
24371 if( pConfig->eDetail==FTS5_DETAIL_COLUMNS2 ){
24372 sqlite3Fts5TermsetFree(ctx.pTermset);
24373 ctx.pTermset = 0;
24374 }
24375 }
24376 }
24377 sqlite3Fts5TermsetFree(ctx.pTermset);
24378 ctx.pTermset = 0;
24379
24380 if( rc!=SQLITE_OK0 ) break;
24381 }
24382 rc2 = sqlite3_resetsqlite3_api->reset(pScan);
24383 if( rc==SQLITE_OK0 ) rc = rc2;
24384 }
24385
24386 /* Test that the "totals" (sometimes called "averages") record looks Ok */
24387 if( rc==SQLITE_OK0 ){
24388 int i;
24389 rc = fts5StorageLoadTotals(p, 0);
24390 for(i=0; rc==SQLITE_OK0 && i<pConfig->nCol; i++){
24391 if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT(11 | (1<<8));
24392 }
24393 }
24394
24395 /* Check that the %_docsize and %_content tables contain the expected
24396 ** number of rows. */
24397 if( rc==SQLITE_OK0 && pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
24398 i64 nRow = 0;
24399 rc = fts5StorageCount(p, "content", &nRow);
24400 if( rc==SQLITE_OK0 && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT(11 | (1<<8));
24401 }
24402 if( rc==SQLITE_OK0 && pConfig->bColumnsize ){
24403 i64 nRow = 0;
24404 rc = fts5StorageCount(p, "docsize", &nRow);
24405 if( rc==SQLITE_OK0 && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT(11 | (1<<8));
24406 }
24407 }
24408
24409 /* Pass the expected checksum down to the FTS index module. It will
24410 ** verify, amongst other things, that it matches the checksum generated by
24411 ** inspecting the index itself. */
24412 if( rc==SQLITE_OK0 ){
24413 rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum, bUseCksum);
24414 }
24415
24416 sqlite3_freesqlite3_api->free(aTotalSize);
24417 return rc;
24418}
24419
24420/*
24421** Obtain an SQLite statement handle that may be used to read data from the
24422** %_content table.
24423*/
24424static int sqlite3Fts5StorageStmt(
24425 Fts5Storage *p,
24426 int eStmt,
24427 sqlite3_stmt **pp,
24428 char **pzErrMsg
24429){
24430 int rc;
24431 assert( eStmt==FTS5_STMT_SCAN_ASC((void) (0))
24432 || eStmt==FTS5_STMT_SCAN_DESC((void) (0))
24433 || eStmt==FTS5_STMT_LOOKUP((void) (0))
24434 )((void) (0));
24435 rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg);
24436 if( rc==SQLITE_OK0 ){
24437 assert( p->aStmt[eStmt]==*pp )((void) (0));
24438 p->aStmt[eStmt] = 0;
24439 }
24440 return rc;
24441}
24442
24443/*
24444** Release an SQLite statement handle obtained via an earlier call to
24445** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function
24446** must match that passed to the sqlite3Fts5StorageStmt() call.
24447*/
24448static void sqlite3Fts5StorageStmtRelease(
24449 Fts5Storage *p,
24450 int eStmt,
24451 sqlite3_stmt *pStmt
24452){
24453 assert( eStmt==FTS5_STMT_SCAN_ASC((void) (0))
24454 || eStmt==FTS5_STMT_SCAN_DESC((void) (0))
24455 || eStmt==FTS5_STMT_LOOKUP((void) (0))
24456 )((void) (0));
24457 if( p->aStmt[eStmt]==0 ){
24458 sqlite3_resetsqlite3_api->reset(pStmt);
24459 p->aStmt[eStmt] = pStmt;
24460 }else{
24461 sqlite3_finalizesqlite3_api->finalize(pStmt);
24462 }
24463}
24464
24465static int fts5StorageDecodeSizeArray(
24466 int *aCol, int nCol, /* Array to populate */
24467 const u8 *aBlob, int nBlob /* Record to read varints from */
24468){
24469 int i;
24470 int iOff = 0;
24471 for(i=0; i<nCol; i++){
24472 if( iOff>=nBlob ) return 1;
24473 iOff += fts5GetVarint32(&aBlob[iOff], aCol[i])sqlite3Fts5GetVarint32(&aBlob[iOff],(u32*)&(aCol[i]));
24474 }
24475 return (iOff!=nBlob);
24476}
24477
24478/*
24479** Argument aCol points to an array of integers containing one entry for
24480** each table column. This function reads the %_docsize record for the
24481** specified rowid and populates aCol[] with the results.
24482**
24483** An SQLite error code is returned if an error occurs, or SQLITE_OK
24484** otherwise.
24485*/
24486static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
24487 int nCol = p->pConfig->nCol; /* Number of user columns in table */
24488 sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */
24489 int rc; /* Return Code */
24490
24491 assert( p->pConfig->bColumnsize )((void) (0));
24492 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE9, &pLookup, 0);
24493 if( pLookup ){
24494 int bCorrupt = 1;
24495 assert( rc==SQLITE_OK )((void) (0));
24496 sqlite3_bind_int64sqlite3_api->bind_int64(pLookup, 1, iRowid);
24497 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pLookup) ){
24498 const u8 *aBlob = sqlite3_column_blobsqlite3_api->column_blob(pLookup, 0);
24499 int nBlob = sqlite3_column_bytessqlite3_api->column_bytes(pLookup, 0);
24500 if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
24501 bCorrupt = 0;
24502 }
24503 }
24504 rc = sqlite3_resetsqlite3_api->reset(pLookup);
24505 if( bCorrupt && rc==SQLITE_OK0 ){
24506 rc = FTS5_CORRUPT(11 | (1<<8));
24507 }
24508 }else{
24509 assert( rc!=SQLITE_OK )((void) (0));
24510 }
24511
24512 return rc;
24513}
24514
24515static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
24516 int rc = fts5StorageLoadTotals(p, 0);
24517 if( rc==SQLITE_OK0 ){
24518 *pnToken = 0;
24519 if( iCol<0 ){
24520 int i;
24521 for(i=0; i<p->pConfig->nCol; i++){
24522 *pnToken += p->aTotalSize[i];
24523 }
24524 }else if( iCol<p->pConfig->nCol ){
24525 *pnToken = p->aTotalSize[iCol];
24526 }else{
24527 rc = SQLITE_RANGE25;
24528 }
24529 }
24530 return rc;
24531}
24532
24533static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
24534 int rc = fts5StorageLoadTotals(p, 0);
24535 if( rc==SQLITE_OK0 ){
24536 /* nTotalRow being zero does not necessarily indicate a corrupt
24537 ** database - it might be that the FTS5 table really does contain zero
24538 ** rows. However this function is only called from the xRowCount() API,
24539 ** and there is no way for that API to be invoked if the table contains
24540 ** no rows. Hence the FTS5_CORRUPT return. */
24541 *pnRow = p->nTotalRow;
24542 if( p->nTotalRow<=0 ) rc = FTS5_CORRUPT(11 | (1<<8));
24543 }
24544 return rc;
24545}
24546
24547/*
24548** Flush any data currently held in-memory to disk.
24549*/
24550static int sqlite3Fts5StorageSync(Fts5Storage *p){
24551 int rc = SQLITE_OK0;
24552 i64 iLastRowid = sqlite3_last_insert_rowidsqlite3_api->last_insert_rowid(p->pConfig->db);
24553 if( p->bTotalsValid ){
24554 rc = fts5StorageSaveTotals(p);
24555 if( rc==SQLITE_OK0 ){
24556 p->bTotalsValid = 0;
24557 }
24558 }
24559 if( rc==SQLITE_OK0 ){
24560 rc = sqlite3Fts5IndexSync(p->pIndex);
24561 }
24562 sqlite3_set_last_insert_rowidsqlite3_api->set_last_insert_rowid(p->pConfig->db, iLastRowid);
24563 return rc;
24564}
24565
24566static int sqlite3Fts5StorageRollback(Fts5Storage *p){
24567 p->bTotalsValid = 0;
24568 return sqlite3Fts5IndexRollback(p->pIndex);
24569}
24570
24571static int sqlite3Fts5StorageConfigValue(
24572 Fts5Storage *p,
24573 const char *z,
24574 sqlite3_value *pVal,
24575 int iVal
24576){
24577 sqlite3_stmt *pReplace = 0;
24578 int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG10, &pReplace, 0);
24579 if( rc==SQLITE_OK0 ){
24580 sqlite3_bind_textsqlite3_api->bind_text(pReplace, 1, z, -1, SQLITE_STATIC((sqlite3_destructor_type)0));
24581 if( pVal ){
24582 sqlite3_bind_valuesqlite3_api->bind_value(pReplace, 2, pVal);
24583 }else{
24584 sqlite3_bind_intsqlite3_api->bind_int(pReplace, 2, iVal);
24585 }
24586 sqlite3_stepsqlite3_api->step(pReplace);
24587 rc = sqlite3_resetsqlite3_api->reset(pReplace);
24588 sqlite3_bind_nullsqlite3_api->bind_null(pReplace, 1);
24589 }
24590 if( rc==SQLITE_OK0 && pVal ){
24591 int iNew = p->pConfig->iCookie + 1;
24592 rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
24593 if( rc==SQLITE_OK0 ){
24594 p->pConfig->iCookie = iNew;
24595 }
24596 }
24597 return rc;
24598}
24599
24600#line 1 "fts5_tokenize.c"
24601/*
24602** 2014 May 31
24603**
24604** The author disclaims copyright to this source code. In place of
24605** a legal notice, here is a blessing:
24606**
24607** May you do good and not evil.
24608** May you find forgiveness for yourself and forgive others.
24609** May you share freely, never taking more than you give.
24610**
24611******************************************************************************
24612*/
24613
24614
24615/* #include "fts5Int.h" */
24616
24617/**************************************************************************
24618** Start of ascii tokenizer implementation.
24619*/
24620
24621/*
24622** For tokenizers with no "unicode" modifier, the set of token characters
24623** is the same as the set of ASCII range alphanumeric characters.
24624*/
24625static unsigned char aAsciiTokenChar[128] = {
24626 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */
24627 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */
24628 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */
24629 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */
24630 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */
24631 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */
24632 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */
24633 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */
24634};
24635
24636typedef struct AsciiTokenizer AsciiTokenizer;
24637struct AsciiTokenizer {
24638 unsigned char aTokenChar[128];
24639};
24640
24641static void fts5AsciiAddExceptions(
24642 AsciiTokenizer *p,
24643 const char *zArg,
24644 int bTokenChars
24645){
24646 int i;
24647 for(i=0; zArg[i]; i++){
24648 if( (zArg[i] & 0x80)==0 ){
24649 p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars;
24650 }
24651 }
24652}
24653
24654/*
24655** Delete a "ascii" tokenizer.
24656*/
24657static void fts5AsciiDelete(Fts5Tokenizer *p){
24658 sqlite3_freesqlite3_api->free(p);
24659}
24660
24661/*
24662** Create an "ascii" tokenizer.
24663*/
24664static int fts5AsciiCreate(
24665 void *pUnused,
24666 const char **azArg, int nArg,
24667 Fts5Tokenizer **ppOut
24668){
24669 int rc = SQLITE_OK0;
24670 AsciiTokenizer *p = 0;
24671 UNUSED_PARAM(pUnused)(void)(pUnused);
24672 if( nArg%2 ){
24673 rc = SQLITE_ERROR1;
24674 }else{
24675 p = sqlite3_malloc64sqlite3_api->malloc64(sizeof(AsciiTokenizer));
24676 if( p==0 ){
24677 rc = SQLITE_NOMEM7;
24678 }else{
24679 int i;
24680 memset(p, 0, sizeof(AsciiTokenizer));
24681 memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
24682 for(i=0; rc==SQLITE_OK0 && i<nArg; i+=2){
24683 const char *zArg = azArg[i+1];
24684 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "tokenchars") ){
24685 fts5AsciiAddExceptions(p, zArg, 1);
24686 }else
24687 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "separators") ){
24688 fts5AsciiAddExceptions(p, zArg, 0);
24689 }else{
24690 rc = SQLITE_ERROR1;
24691 }
24692 }
24693 if( rc!=SQLITE_OK0 ){
24694 fts5AsciiDelete((Fts5Tokenizer*)p);
24695 p = 0;
24696 }
24697 }
24698 }
24699
24700 *ppOut = (Fts5Tokenizer*)p;
24701 return rc;
24702}
24703
24704
24705static void asciiFold(char *aOut, const char *aIn, int nByte){
24706 int i;
24707 for(i=0; i<nByte; i++){
24708 char c = aIn[i];
24709 if( c>='A' && c<='Z' ) c += 32;
24710 aOut[i] = c;
24711 }
24712}
24713
24714/*
24715** Tokenize some text using the ascii tokenizer.
24716*/
24717static int fts5AsciiTokenize(
24718 Fts5Tokenizer *pTokenizer,
24719 void *pCtx,
24720 int iUnused,
24721 const char *pText, int nText,
24722 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
24723){
24724 AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer;
24725 int rc = SQLITE_OK0;
24726 int ie;
24727 int is = 0;
24728
24729 char aFold[64];
24730 int nFold = sizeof(aFold);
24731 char *pFold = aFold;
24732 unsigned char *a = p->aTokenChar;
24733
24734 UNUSED_PARAM(iUnused)(void)(iUnused);
24735
24736 while( is<nText && rc==SQLITE_OK0 ){
24737 int nByte;
24738
24739 /* Skip any leading divider characters. */
24740 while( is<nText && ((pText[is]&0x80)==0 && a[(int)pText[is]]==0) ){
24741 is++;
24742 }
24743 if( is==nText ) break;
24744
24745 /* Count the token characters */
24746 ie = is+1;
24747 while( ie<nText && ((pText[ie]&0x80) || a[(int)pText[ie]] ) ){
24748 ie++;
24749 }
24750
24751 /* Fold to lower case */
24752 nByte = ie-is;
24753 if( nByte>nFold ){
24754 if( pFold!=aFold ) sqlite3_freesqlite3_api->free(pFold);
24755 pFold = sqlite3_malloc64sqlite3_api->malloc64((sqlite3_int64)nByte*2);
24756 if( pFold==0 ){
24757 rc = SQLITE_NOMEM7;
24758 break;
24759 }
24760 nFold = nByte*2;
24761 }
24762 asciiFold(pFold, &pText[is], nByte);
24763
24764 /* Invoke the token callback */
24765 rc = xToken(pCtx, 0, pFold, nByte, is, ie);
24766 is = ie+1;
24767 }
24768
24769 if( pFold!=aFold ) sqlite3_freesqlite3_api->free(pFold);
24770 if( rc==SQLITE_DONE101 ) rc = SQLITE_OK0;
24771 return rc;
24772}
24773
24774/**************************************************************************
24775** Start of unicode61 tokenizer implementation.
24776*/
24777
24778
24779/*
24780** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied
24781** from the sqlite3 source file utf.c. If this file is compiled as part
24782** of the amalgamation, they are not required.
24783*/
24784#ifndef SQLITE_AMALGAMATION
24785
24786static const unsigned char sqlite3Utf8Trans1[] = {
24787 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
24788 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
24789 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
24790 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
24791 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
24792 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
24793 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
24794 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
24795};
24796
24797#define READ_UTF8(zIn, zTerm, c)c = *(zIn++); if( c>=0xc0 ){ c = sqlite3Utf8Trans1[c-0xc0]
; while( zIn<zTerm && (*zIn & 0xc0)==0x80 ){ c
= (c<<6) + (0x3f & *(zIn++)); } if( c<0x80 || (
c&0xFFFFF800)==0xD800 || (c&0xFFFFFFFE)==0xFFFE ){ c =
0xFFFD; } } \
24798 c = *(zIn++); \
24799 if( c>=0xc0 ){ \
24800 c = sqlite3Utf8Trans1[c-0xc0]; \
24801 while( zIn<zTerm && (*zIn & 0xc0)==0x80 ){ \
24802 c = (c<<6) + (0x3f & *(zIn++)); \
24803 } \
24804 if( c<0x80 \
24805 || (c&0xFFFFF800)==0xD800 \
24806 || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
24807 }
24808
24809
24810#define WRITE_UTF8(zOut, c){ if( c<0x00080 ){ *zOut++ = (unsigned char)(c&0xFF); }
else if( c<0x00800 ){ *zOut++ = 0xC0 + (unsigned char)((c
>>6)&0x1F); *zOut++ = 0x80 + (unsigned char)(c &
0x3F); } else if( c<0x10000 ){ *zOut++ = 0xE0 + (unsigned
char)((c>>12)&0x0F); *zOut++ = 0x80 + (unsigned char
)((c>>6) & 0x3F); *zOut++ = 0x80 + (unsigned char)(
c & 0x3F); }else{ *zOut++ = 0xF0 + (unsigned char)((c>>
18) & 0x07); *zOut++ = 0x80 + (unsigned char)((c>>12
) & 0x3F); *zOut++ = 0x80 + (unsigned char)((c>>6) &
0x3F); *zOut++ = 0x80 + (unsigned char)(c & 0x3F); } } { \
24811 if( c<0x00080 ){ \
24812 *zOut++ = (unsigned char)(c&0xFF); \
24813 } \
24814 else if( c<0x00800 ){ \
24815 *zOut++ = 0xC0 + (unsigned char)((c>>6)&0x1F); \
24816 *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
24817 } \
24818 else if( c<0x10000 ){ \
24819 *zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \
24820 *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
24821 *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
24822 }else{ \
24823 *zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \
24824 *zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \
24825 *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
24826 *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
24827 } \
24828}
24829
24830#endif /* ifndef SQLITE_AMALGAMATION */
24831
24832#define FTS5_SKIP_UTF8(zIn){ if( ((unsigned char)(*(zIn++)))>=0xc0 ){ while( (((unsigned
char)*zIn) & 0xc0)==0x80 ){ zIn++; } } } { \
24833 if( ((unsigned char)(*(zIn++)))>=0xc0 ){ \
24834 while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; } \
24835 } \
24836}
24837
24838typedef struct Unicode61Tokenizer Unicode61Tokenizer;
24839struct Unicode61Tokenizer {
24840 unsigned char aTokenChar[128]; /* ASCII range token characters */
24841 char *aFold; /* Buffer to fold text into */
24842 int nFold; /* Size of aFold[] in bytes */
24843 int eRemoveDiacritic; /* True if remove_diacritics=1 is set */
24844 int nException;
24845 int *aiException;
24846
24847 unsigned char aCategory[32]; /* True for token char categories */
24848};
24849
24850/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
24851#define FTS5_REMOVE_DIACRITICS_NONE0 0
24852#define FTS5_REMOVE_DIACRITICS_SIMPLE1 1
24853#define FTS5_REMOVE_DIACRITICS_COMPLEX2 2
24854
24855static int fts5UnicodeAddExceptions(
24856 Unicode61Tokenizer *p, /* Tokenizer object */
24857 const char *z, /* Characters to treat as exceptions */
24858 int bTokenChars /* 1 for 'tokenchars', 0 for 'separators' */
24859){
24860 int rc = SQLITE_OK0;
24861 int n = (int)strlen(z);
24862 int *aNew;
24863
24864 if( n>0 ){
24865 aNew = (int*)sqlite3_realloc64sqlite3_api->realloc64(p->aiException,
24866 (n+p->nException)*sizeof(int));
24867 if( aNew ){
24868 int nNew = p->nException;
24869 const unsigned char *zCsr = (const unsigned char*)z;
24870 const unsigned char *zTerm = (const unsigned char*)&z[n];
24871 while( zCsr<zTerm ){
24872 u32 iCode;
24873 int bToken;
24874 READ_UTF8(zCsr, zTerm, iCode)iCode = *(zCsr++); if( iCode>=0xc0 ){ iCode = sqlite3Utf8Trans1
[iCode-0xc0]; while( zCsr<zTerm && (*zCsr & 0xc0
)==0x80 ){ iCode = (iCode<<6) + (0x3f & *(zCsr++));
} if( iCode<0x80 || (iCode&0xFFFFF800)==0xD800 || (iCode
&0xFFFFFFFE)==0xFFFE ){ iCode = 0xFFFD; } };
24875 if( iCode<128 ){
24876 p->aTokenChar[iCode] = (unsigned char)bTokenChars;
24877 }else{
24878 bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
24879 assert( (bToken==0 || bToken==1) )((void) (0));
24880 assert( (bTokenChars==0 || bTokenChars==1) )((void) (0));
24881 if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
24882 int i;
24883 for(i=0; i<nNew; i++){
24884 if( (u32)aNew[i]>iCode ) break;
24885 }
24886 memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int));
24887 aNew[i] = iCode;
24888 nNew++;
24889 }
24890 }
24891 }
24892 p->aiException = aNew;
24893 p->nException = nNew;
24894 }else{
24895 rc = SQLITE_NOMEM7;
24896 }
24897 }
24898
24899 return rc;
24900}
24901
24902/*
24903** Return true if the p->aiException[] array contains the value iCode.
24904*/
24905static int fts5UnicodeIsException(Unicode61Tokenizer *p, int iCode){
24906 if( p->nException>0 ){
24907 int *a = p->aiException;
24908 int iLo = 0;
24909 int iHi = p->nException-1;
24910
24911 while( iHi>=iLo ){
24912 int iTest = (iHi + iLo) / 2;
24913 if( iCode==a[iTest] ){
24914 return 1;
24915 }else if( iCode>a[iTest] ){
24916 iLo = iTest+1;
24917 }else{
24918 iHi = iTest-1;
24919 }
24920 }
24921 }
24922
24923 return 0;
24924}
24925
24926/*
24927** Delete a "unicode61" tokenizer.
24928*/
24929static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
24930 if( pTok ){
24931 Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok;
24932 sqlite3_freesqlite3_api->free(p->aiException);
24933 sqlite3_freesqlite3_api->free(p->aFold);
24934 sqlite3_freesqlite3_api->free(p);
24935 }
24936 return;
24937}
24938
24939static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){
24940 const char *z = zCat;
24941
24942 while( *z ){
24943 while( *z==' ' || *z=='\t' ) z++;
24944 if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){
24945 return SQLITE_ERROR1;
24946 }
24947 while( *z!=' ' && *z!='\t' && *z!='\0' ) z++;
24948 }
24949
24950 sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar);
24951 return SQLITE_OK0;
24952}
24953
24954/*
24955** Create a "unicode61" tokenizer.
24956*/
24957static int fts5UnicodeCreate(
24958 void *pUnused,
24959 const char **azArg, int nArg,
24960 Fts5Tokenizer **ppOut
24961){
24962 int rc = SQLITE_OK0; /* Return code */
24963 Unicode61Tokenizer *p = 0; /* New tokenizer object */
24964
24965 UNUSED_PARAM(pUnused)(void)(pUnused);
24966
24967 if( nArg%2 ){
24968 rc = SQLITE_ERROR1;
24969 }else{
24970 p = (Unicode61Tokenizer*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(Unicode61Tokenizer));
24971 if( p ){
24972 const char *zCat = "L* N* Co";
24973 int i;
24974 memset(p, 0, sizeof(Unicode61Tokenizer));
24975
24976 p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE1;
24977 p->nFold = 64;
24978 p->aFold = sqlite3_malloc64sqlite3_api->malloc64(p->nFold * sizeof(char));
24979 if( p->aFold==0 ){
24980 rc = SQLITE_NOMEM7;
24981 }
24982
24983 /* Search for a "categories" argument */
24984 for(i=0; rc==SQLITE_OK0 && i<nArg; i+=2){
24985 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "categories") ){
24986 zCat = azArg[i+1];
24987 }
24988 }
24989 if( rc==SQLITE_OK0 ){
24990 rc = unicodeSetCategories(p, zCat);
24991 }
24992
24993 for(i=0; rc==SQLITE_OK0 && i<nArg; i+=2){
24994 const char *zArg = azArg[i+1];
24995 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "remove_diacritics") ){
24996 if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
24997 rc = SQLITE_ERROR1;
24998 }else{
24999 p->eRemoveDiacritic = (zArg[0] - '0');
25000 assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE((void) (0))
25001 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE((void) (0))
25002 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX((void) (0))
25003 )((void) (0));
25004 }
25005 }else
25006 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "tokenchars") ){
25007 rc = fts5UnicodeAddExceptions(p, zArg, 1);
25008 }else
25009 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "separators") ){
25010 rc = fts5UnicodeAddExceptions(p, zArg, 0);
25011 }else
25012 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "categories") ){
25013 /* no-op */
25014 }else{
25015 rc = SQLITE_ERROR1;
25016 }
25017 }
25018 }else{
25019 rc = SQLITE_NOMEM7;
25020 }
25021 if( rc!=SQLITE_OK0 ){
25022 fts5UnicodeDelete((Fts5Tokenizer*)p);
25023 p = 0;
25024 }
25025 *ppOut = (Fts5Tokenizer*)p;
25026 }
25027 return rc;
25028}
25029
25030/*
25031** Return true if, for the purposes of tokenizing with the tokenizer
25032** passed as the first argument, codepoint iCode is considered a token
25033** character (not a separator).
25034*/
25035static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
25036 return (
25037 p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)]
25038 ^ fts5UnicodeIsException(p, iCode)
25039 );
25040}
25041
25042static int fts5UnicodeTokenize(
25043 Fts5Tokenizer *pTokenizer,
25044 void *pCtx,
25045 int iUnused,
25046 const char *pText, int nText,
25047 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
25048){
25049 Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer;
25050 int rc = SQLITE_OK0;
25051 unsigned char *a = p->aTokenChar;
25052
25053 unsigned char *zTerm = (unsigned char*)&pText[nText];
25054 unsigned char *zCsr = (unsigned char *)pText;
25055
25056 /* Output buffer */
25057 char *aFold = p->aFold;
25058 int nFold = p->nFold;
25059 const char *pEnd = &aFold[nFold-6];
25060
25061 UNUSED_PARAM(iUnused)(void)(iUnused);
25062
25063 /* Each iteration of this loop gobbles up a contiguous run of separators,
25064 ** then the next token. */
25065 while( rc==SQLITE_OK0 ){
25066 u32 iCode; /* non-ASCII codepoint read from input */
25067 char *zOut = aFold;
25068 int is;
25069 int ie;
25070
25071 /* Skip any separator characters. */
25072 while( 1 ){
25073 if( zCsr>=zTerm ) goto tokenize_done;
25074 if( *zCsr & 0x80 ) {
25075 /* A character outside of the ascii range. Skip past it if it is
25076 ** a separator character. Or break out of the loop if it is not. */
25077 is = zCsr - (unsigned char*)pText;
25078 READ_UTF8(zCsr, zTerm, iCode)iCode = *(zCsr++); if( iCode>=0xc0 ){ iCode = sqlite3Utf8Trans1
[iCode-0xc0]; while( zCsr<zTerm && (*zCsr & 0xc0
)==0x80 ){ iCode = (iCode<<6) + (0x3f & *(zCsr++));
} if( iCode<0x80 || (iCode&0xFFFFF800)==0xD800 || (iCode
&0xFFFFFFFE)==0xFFFE ){ iCode = 0xFFFD; } };
25079 if( fts5UnicodeIsAlnum(p, iCode) ){
25080 goto non_ascii_tokenchar;
25081 }
25082 }else{
25083 if( a[*zCsr] ){
25084 is = zCsr - (unsigned char*)pText;
25085 goto ascii_tokenchar;
25086 }
25087 zCsr++;
25088 }
25089 }
25090
25091 /* Run through the tokenchars. Fold them into the output buffer along
25092 ** the way. */
25093 while( zCsr<zTerm ){
25094
25095 /* Grow the output buffer so that there is sufficient space to fit the
25096 ** largest possible utf-8 character. */
25097 if( zOut>pEnd ){
25098 aFold = sqlite3_malloc64sqlite3_api->malloc64((sqlite3_int64)nFold*2);
25099 if( aFold==0 ){
25100 rc = SQLITE_NOMEM7;
25101 goto tokenize_done;
25102 }
25103 zOut = &aFold[zOut - p->aFold];
25104 memcpy(aFold, p->aFold, nFold);
25105 sqlite3_freesqlite3_api->free(p->aFold);
25106 p->aFold = aFold;
25107 p->nFold = nFold = nFold*2;
25108 pEnd = &aFold[nFold-6];
25109 }
25110
25111 if( *zCsr & 0x80 ){
25112 /* An non-ascii-range character. Fold it into the output buffer if
25113 ** it is a token character, or break out of the loop if it is not. */
25114 READ_UTF8(zCsr, zTerm, iCode)iCode = *(zCsr++); if( iCode>=0xc0 ){ iCode = sqlite3Utf8Trans1
[iCode-0xc0]; while( zCsr<zTerm && (*zCsr & 0xc0
)==0x80 ){ iCode = (iCode<<6) + (0x3f & *(zCsr++));
} if( iCode<0x80 || (iCode&0xFFFFF800)==0xD800 || (iCode
&0xFFFFFFFE)==0xFFFE ){ iCode = 0xFFFD; } };
25115 if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
25116 non_ascii_tokenchar:
25117 iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
25118 if( iCode ) WRITE_UTF8(zOut, iCode){ if( iCode<0x00080 ){ *zOut++ = (unsigned char)(iCode&
0xFF); } else if( iCode<0x00800 ){ *zOut++ = 0xC0 + (unsigned
char)((iCode>>6)&0x1F); *zOut++ = 0x80 + (unsigned
char)(iCode & 0x3F); } else if( iCode<0x10000 ){ *zOut
++ = 0xE0 + (unsigned char)((iCode>>12)&0x0F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); }else{ *zOut++
= 0xF0 + (unsigned char)((iCode>>18) & 0x07); *zOut
++ = 0x80 + (unsigned char)((iCode>>12) & 0x3F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); } };
25119 }else{
25120 break;
25121 }
25122 }else if( a[*zCsr]==0 ){
25123 /* An ascii-range separator character. End of token. */
25124 break;
25125 }else{
25126 ascii_tokenchar:
25127 if( *zCsr>='A' && *zCsr<='Z' ){
25128 *zOut++ = *zCsr + 32;
25129 }else{
25130 *zOut++ = *zCsr;
25131 }
25132 zCsr++;
25133 }
25134 ie = zCsr - (unsigned char*)pText;
25135 }
25136
25137 /* Invoke the token callback */
25138 rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie);
25139 }
25140
25141 tokenize_done:
25142 if( rc==SQLITE_DONE101 ) rc = SQLITE_OK0;
25143 return rc;
25144}
25145
25146/**************************************************************************
25147** Start of porter stemmer implementation.
25148*/
25149
25150/* Any tokens larger than this (in bytes) are passed through without
25151** stemming. */
25152#define FTS5_PORTER_MAX_TOKEN64 64
25153
25154typedef struct PorterTokenizer PorterTokenizer;
25155struct PorterTokenizer {
25156 fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */
25157 Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */
25158 char aBuf[FTS5_PORTER_MAX_TOKEN64 + 64];
25159};
25160
25161/*
25162** Delete a "porter" tokenizer.
25163*/
25164static void fts5PorterDelete(Fts5Tokenizer *pTok){
25165 if( pTok ){
25166 PorterTokenizer *p = (PorterTokenizer*)pTok;
25167 if( p->pTokenizer ){
25168 p->tokenizer_v2.xDelete(p->pTokenizer);
25169 }
25170 sqlite3_freesqlite3_api->free(p);
25171 }
25172}
25173
25174/*
25175** Create a "porter" tokenizer.
25176*/
25177static int fts5PorterCreate(
25178 void *pCtx,
25179 const char **azArg, int nArg,
25180 Fts5Tokenizer **ppOut
25181){
25182 fts5_api *pApi = (fts5_api*)pCtx;
25183 int rc = SQLITE_OK0;
25184 PorterTokenizer *pRet;
25185 void *pUserdata = 0;
25186 const char *zBase = "unicode61";
25187 fts5_tokenizer_v2 *pV2 = 0;
25188
25189 while( nArg>0 ){
25190 if( sqlite3_stricmpsqlite3_api->stricmp(azArg[0],"porter")==0 ){
25191 nArg--;
25192 azArg++;
25193 }else{
25194 zBase = azArg[0];
25195 break;
25196 }
25197 }
25198
25199 pRet = (PorterTokenizer*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(PorterTokenizer));
25200 if( pRet ){
25201 memset(pRet, 0, sizeof(PorterTokenizer));
25202 rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2);
25203 }else{
25204 rc = SQLITE_NOMEM7;
25205 }
25206 if( rc==SQLITE_OK0 ){
25207 int nArg2 = (nArg>0 ? nArg-1 : 0);
25208 const char **az2 = (nArg2 ? &azArg[1] : 0);
25209 memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2));
25210 rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer);
25211 }
25212
25213 if( rc!=SQLITE_OK0 ){
25214 fts5PorterDelete((Fts5Tokenizer*)pRet);
25215 pRet = 0;
25216 }
25217 *ppOut = (Fts5Tokenizer*)pRet;
25218 return rc;
25219}
25220
25221typedef struct PorterContext PorterContext;
25222struct PorterContext {
25223 void *pCtx;
25224 int (*xToken)(void*, int, const char*, int, int, int);
25225 char *aBuf;
25226};
25227
25228typedef struct PorterRule PorterRule;
25229struct PorterRule {
25230 const char *zSuffix;
25231 int nSuffix;
25232 int (*xCond)(char *zStem, int nStem);
25233 const char *zOutput;
25234 int nOutput;
25235};
25236
25237#if 0
25238static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){
25239 int ret = -1;
25240 int nBuf = *pnBuf;
25241 PorterRule *p;
25242
25243 for(p=aRule; p->zSuffix; p++){
25244 assert( strlen(p->zSuffix)==p->nSuffix )((void) (0));
25245 assert( strlen(p->zOutput)==p->nOutput )((void) (0));
25246 if( nBuf<p->nSuffix ) continue;
25247 if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break;
25248 }
25249
25250 if( p->zSuffix ){
25251 int nStem = nBuf - p->nSuffix;
25252 if( p->xCond==0 || p->xCond(aBuf, nStem) ){
25253 memcpy(&aBuf[nStem], p->zOutput, p->nOutput);
25254 *pnBuf = nStem + p->nOutput;
25255 ret = p - aRule;
25256 }
25257 }
25258
25259 return ret;
25260}
25261#endif
25262
25263static int fts5PorterIsVowel(char c, int bYIsVowel){
25264 return (
25265 c=='a' || c=='e' || c=='i' || c=='o' || c=='u' || (bYIsVowel && c=='y')
25266 );
25267}
25268
25269static int fts5PorterGobbleVC(char *zStem, int nStem, int bPrevCons){
25270 int i;
25271 int bCons = bPrevCons;
25272
25273 /* Scan for a vowel */
25274 for(i=0; i<nStem; i++){
25275 if( 0==(bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) break;
25276 }
25277
25278 /* Scan for a consonent */
25279 for(i++; i<nStem; i++){
25280 if( (bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) return i+1;
25281 }
25282 return 0;
25283}
25284
25285/* porter rule condition: (m > 0) */
25286static int fts5Porter_MGt0(char *zStem, int nStem){
25287 return !!fts5PorterGobbleVC(zStem, nStem, 0);
25288}
25289
25290/* porter rule condition: (m > 1) */
25291static int fts5Porter_MGt1(char *zStem, int nStem){
25292 int n;
25293 n = fts5PorterGobbleVC(zStem, nStem, 0);
25294 if( n && fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
25295 return 1;
25296 }
25297 return 0;
25298}
25299
25300/* porter rule condition: (m = 1) */
25301static int fts5Porter_MEq1(char *zStem, int nStem){
25302 int n;
25303 n = fts5PorterGobbleVC(zStem, nStem, 0);
25304 if( n && 0==fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
25305 return 1;
25306 }
25307 return 0;
25308}
25309
25310/* porter rule condition: (*o) */
25311static int fts5Porter_Ostar(char *zStem, int nStem){
25312 if( zStem[nStem-1]=='w' || zStem[nStem-1]=='x' || zStem[nStem-1]=='y' ){
25313 return 0;
25314 }else{
25315 int i;
25316 int mask = 0;
25317 int bCons = 0;
25318 for(i=0; i<nStem; i++){
25319 bCons = !fts5PorterIsVowel(zStem[i], bCons);
25320 assert( bCons==0 || bCons==1 )((void) (0));
25321 mask = (mask << 1) + bCons;
25322 }
25323 return ((mask & 0x0007)==0x0005);
25324 }
25325}
25326
25327/* porter rule condition: (m > 1 and (*S or *T)) */
25328static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){
25329 assert( nStem>0 )((void) (0));
25330 return (zStem[nStem-1]=='s' || zStem[nStem-1]=='t')
25331 && fts5Porter_MGt1(zStem, nStem);
25332}
25333
25334/* porter rule condition: (*v*) */
25335static int fts5Porter_Vowel(char *zStem, int nStem){
25336 int i;
25337 for(i=0; i<nStem; i++){
25338 if( fts5PorterIsVowel(zStem[i], i>0) ){
25339 return 1;
25340 }
25341 }
25342 return 0;
25343}
25344
25345
25346/**************************************************************************
25347***************************************************************************
25348** GENERATED CODE STARTS HERE (mkportersteps.tcl)
25349*/
25350
25351static int fts5PorterStep4(char *aBuf, int *pnBuf){
25352 int ret = 0;
25353 int nBuf = *pnBuf;
25354 switch( aBuf[nBuf-2] ){
25355
25356 case 'a':
25357 if( nBuf>2 && 0==memcmp("al", &aBuf[nBuf-2], 2) ){
25358 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
25359 *pnBuf = nBuf - 2;
25360 }
25361 }
25362 break;
25363
25364 case 'c':
25365 if( nBuf>4 && 0==memcmp("ance", &aBuf[nBuf-4], 4) ){
25366 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
25367 *pnBuf = nBuf - 4;
25368 }
25369 }else if( nBuf>4 && 0==memcmp("ence", &aBuf[nBuf-4], 4) ){
25370 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
25371 *pnBuf = nBuf - 4;
25372 }
25373 }
25374 break;
25375
25376 case 'e':
25377 if( nBuf>2 && 0==memcmp("er", &aBuf[nBuf-2], 2) ){
25378 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
25379 *pnBuf = nBuf - 2;
25380 }
25381 }
25382 break;
25383
25384 case 'i':
25385 if( nBuf>2 && 0==memcmp("ic", &aBuf[nBuf-2], 2) ){
25386 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
25387 *pnBuf = nBuf - 2;
25388 }
25389 }
25390 break;
25391
25392 case 'l':
25393 if( nBuf>4 && 0==memcmp("able", &aBuf[nBuf-4], 4) ){
25394 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
25395 *pnBuf = nBuf - 4;
25396 }
25397 }else if( nBuf>4 && 0==memcmp("ible", &aBuf[nBuf-4], 4) ){
25398 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
25399 *pnBuf = nBuf - 4;
25400 }
25401 }
25402 break;
25403
25404 case 'n':
25405 if( nBuf>3 && 0==memcmp("ant", &aBuf[nBuf-3], 3) ){
25406 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25407 *pnBuf = nBuf - 3;
25408 }
25409 }else if( nBuf>5 && 0==memcmp("ement", &aBuf[nBuf-5], 5) ){
25410 if( fts5Porter_MGt1(aBuf, nBuf-5) ){
25411 *pnBuf = nBuf - 5;
25412 }
25413 }else if( nBuf>4 && 0==memcmp("ment", &aBuf[nBuf-4], 4) ){
25414 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
25415 *pnBuf = nBuf - 4;
25416 }
25417 }else if( nBuf>3 && 0==memcmp("ent", &aBuf[nBuf-3], 3) ){
25418 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25419 *pnBuf = nBuf - 3;
25420 }
25421 }
25422 break;
25423
25424 case 'o':
25425 if( nBuf>3 && 0==memcmp("ion", &aBuf[nBuf-3], 3) ){
25426 if( fts5Porter_MGt1_and_S_or_T(aBuf, nBuf-3) ){
25427 *pnBuf = nBuf - 3;
25428 }
25429 }else if( nBuf>2 && 0==memcmp("ou", &aBuf[nBuf-2], 2) ){
25430 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
25431 *pnBuf = nBuf - 2;
25432 }
25433 }
25434 break;
25435
25436 case 's':
25437 if( nBuf>3 && 0==memcmp("ism", &aBuf[nBuf-3], 3) ){
25438 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25439 *pnBuf = nBuf - 3;
25440 }
25441 }
25442 break;
25443
25444 case 't':
25445 if( nBuf>3 && 0==memcmp("ate", &aBuf[nBuf-3], 3) ){
25446 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25447 *pnBuf = nBuf - 3;
25448 }
25449 }else if( nBuf>3 && 0==memcmp("iti", &aBuf[nBuf-3], 3) ){
25450 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25451 *pnBuf = nBuf - 3;
25452 }
25453 }
25454 break;
25455
25456 case 'u':
25457 if( nBuf>3 && 0==memcmp("ous", &aBuf[nBuf-3], 3) ){
25458 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25459 *pnBuf = nBuf - 3;
25460 }
25461 }
25462 break;
25463
25464 case 'v':
25465 if( nBuf>3 && 0==memcmp("ive", &aBuf[nBuf-3], 3) ){
25466 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25467 *pnBuf = nBuf - 3;
25468 }
25469 }
25470 break;
25471
25472 case 'z':
25473 if( nBuf>3 && 0==memcmp("ize", &aBuf[nBuf-3], 3) ){
25474 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25475 *pnBuf = nBuf - 3;
25476 }
25477 }
25478 break;
25479
25480 }
25481 return ret;
25482}
25483
25484
25485static int fts5PorterStep1B2(char *aBuf, int *pnBuf){
25486 int ret = 0;
25487 int nBuf = *pnBuf;
25488 switch( aBuf[nBuf-2] ){
25489
25490 case 'a':
25491 if( nBuf>2 && 0==memcmp("at", &aBuf[nBuf-2], 2) ){
25492 memcpy(&aBuf[nBuf-2], "ate", 3);
25493 *pnBuf = nBuf - 2 + 3;
25494 ret = 1;
25495 }
25496 break;
25497
25498 case 'b':
25499 if( nBuf>2 && 0==memcmp("bl", &aBuf[nBuf-2], 2) ){
25500 memcpy(&aBuf[nBuf-2], "ble", 3);
25501 *pnBuf = nBuf - 2 + 3;
25502 ret = 1;
25503 }
25504 break;
25505
25506 case 'i':
25507 if( nBuf>2 && 0==memcmp("iz", &aBuf[nBuf-2], 2) ){
25508 memcpy(&aBuf[nBuf-2], "ize", 3);
25509 *pnBuf = nBuf - 2 + 3;
25510 ret = 1;
25511 }
25512 break;
25513
25514 }
25515 return ret;
25516}
25517
25518
25519static int fts5PorterStep2(char *aBuf, int *pnBuf){
25520 int ret = 0;
25521 int nBuf = *pnBuf;
25522 switch( aBuf[nBuf-2] ){
25523
25524 case 'a':
25525 if( nBuf>7 && 0==memcmp("ational", &aBuf[nBuf-7], 7) ){
25526 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
25527 memcpy(&aBuf[nBuf-7], "ate", 3);
25528 *pnBuf = nBuf - 7 + 3;
25529 }
25530 }else if( nBuf>6 && 0==memcmp("tional", &aBuf[nBuf-6], 6) ){
25531 if( fts5Porter_MGt0(aBuf, nBuf-6) ){
25532 memcpy(&aBuf[nBuf-6], "tion", 4);
25533 *pnBuf = nBuf - 6 + 4;
25534 }
25535 }
25536 break;
25537
25538 case 'c':
25539 if( nBuf>4 && 0==memcmp("enci", &aBuf[nBuf-4], 4) ){
25540 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25541 memcpy(&aBuf[nBuf-4], "ence", 4);
25542 *pnBuf = nBuf - 4 + 4;
25543 }
25544 }else if( nBuf>4 && 0==memcmp("anci", &aBuf[nBuf-4], 4) ){
25545 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25546 memcpy(&aBuf[nBuf-4], "ance", 4);
25547 *pnBuf = nBuf - 4 + 4;
25548 }
25549 }
25550 break;
25551
25552 case 'e':
25553 if( nBuf>4 && 0==memcmp("izer", &aBuf[nBuf-4], 4) ){
25554 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25555 memcpy(&aBuf[nBuf-4], "ize", 3);
25556 *pnBuf = nBuf - 4 + 3;
25557 }
25558 }
25559 break;
25560
25561 case 'g':
25562 if( nBuf>4 && 0==memcmp("logi", &aBuf[nBuf-4], 4) ){
25563 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25564 memcpy(&aBuf[nBuf-4], "log", 3);
25565 *pnBuf = nBuf - 4 + 3;
25566 }
25567 }
25568 break;
25569
25570 case 'l':
25571 if( nBuf>3 && 0==memcmp("bli", &aBuf[nBuf-3], 3) ){
25572 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
25573 memcpy(&aBuf[nBuf-3], "ble", 3);
25574 *pnBuf = nBuf - 3 + 3;
25575 }
25576 }else if( nBuf>4 && 0==memcmp("alli", &aBuf[nBuf-4], 4) ){
25577 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25578 memcpy(&aBuf[nBuf-4], "al", 2);
25579 *pnBuf = nBuf - 4 + 2;
25580 }
25581 }else if( nBuf>5 && 0==memcmp("entli", &aBuf[nBuf-5], 5) ){
25582 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25583 memcpy(&aBuf[nBuf-5], "ent", 3);
25584 *pnBuf = nBuf - 5 + 3;
25585 }
25586 }else if( nBuf>3 && 0==memcmp("eli", &aBuf[nBuf-3], 3) ){
25587 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
25588 memcpy(&aBuf[nBuf-3], "e", 1);
25589 *pnBuf = nBuf - 3 + 1;
25590 }
25591 }else if( nBuf>5 && 0==memcmp("ousli", &aBuf[nBuf-5], 5) ){
25592 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25593 memcpy(&aBuf[nBuf-5], "ous", 3);
25594 *pnBuf = nBuf - 5 + 3;
25595 }
25596 }
25597 break;
25598
25599 case 'o':
25600 if( nBuf>7 && 0==memcmp("ization", &aBuf[nBuf-7], 7) ){
25601 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
25602 memcpy(&aBuf[nBuf-7], "ize", 3);
25603 *pnBuf = nBuf - 7 + 3;
25604 }
25605 }else if( nBuf>5 && 0==memcmp("ation", &aBuf[nBuf-5], 5) ){
25606 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25607 memcpy(&aBuf[nBuf-5], "ate", 3);
25608 *pnBuf = nBuf - 5 + 3;
25609 }
25610 }else if( nBuf>4 && 0==memcmp("ator", &aBuf[nBuf-4], 4) ){
25611 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25612 memcpy(&aBuf[nBuf-4], "ate", 3);
25613 *pnBuf = nBuf - 4 + 3;
25614 }
25615 }
25616 break;
25617
25618 case 's':
25619 if( nBuf>5 && 0==memcmp("alism", &aBuf[nBuf-5], 5) ){
25620 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25621 memcpy(&aBuf[nBuf-5], "al", 2);
25622 *pnBuf = nBuf - 5 + 2;
25623 }
25624 }else if( nBuf>7 && 0==memcmp("iveness", &aBuf[nBuf-7], 7) ){
25625 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
25626 memcpy(&aBuf[nBuf-7], "ive", 3);
25627 *pnBuf = nBuf - 7 + 3;
25628 }
25629 }else if( nBuf>7 && 0==memcmp("fulness", &aBuf[nBuf-7], 7) ){
25630 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
25631 memcpy(&aBuf[nBuf-7], "ful", 3);
25632 *pnBuf = nBuf - 7 + 3;
25633 }
25634 }else if( nBuf>7 && 0==memcmp("ousness", &aBuf[nBuf-7], 7) ){
25635 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
25636 memcpy(&aBuf[nBuf-7], "ous", 3);
25637 *pnBuf = nBuf - 7 + 3;
25638 }
25639 }
25640 break;
25641
25642 case 't':
25643 if( nBuf>5 && 0==memcmp("aliti", &aBuf[nBuf-5], 5) ){
25644 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25645 memcpy(&aBuf[nBuf-5], "al", 2);
25646 *pnBuf = nBuf - 5 + 2;
25647 }
25648 }else if( nBuf>5 && 0==memcmp("iviti", &aBuf[nBuf-5], 5) ){
25649 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25650 memcpy(&aBuf[nBuf-5], "ive", 3);
25651 *pnBuf = nBuf - 5 + 3;
25652 }
25653 }else if( nBuf>6 && 0==memcmp("biliti", &aBuf[nBuf-6], 6) ){
25654 if( fts5Porter_MGt0(aBuf, nBuf-6) ){
25655 memcpy(&aBuf[nBuf-6], "ble", 3);
25656 *pnBuf = nBuf - 6 + 3;
25657 }
25658 }
25659 break;
25660
25661 }
25662 return ret;
25663}
25664
25665
25666static int fts5PorterStep3(char *aBuf, int *pnBuf){
25667 int ret = 0;
25668 int nBuf = *pnBuf;
25669 switch( aBuf[nBuf-2] ){
25670
25671 case 'a':
25672 if( nBuf>4 && 0==memcmp("ical", &aBuf[nBuf-4], 4) ){
25673 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25674 memcpy(&aBuf[nBuf-4], "ic", 2);
25675 *pnBuf = nBuf - 4 + 2;
25676 }
25677 }
25678 break;
25679
25680 case 's':
25681 if( nBuf>4 && 0==memcmp("ness", &aBuf[nBuf-4], 4) ){
25682 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25683 *pnBuf = nBuf - 4;
25684 }
25685 }
25686 break;
25687
25688 case 't':
25689 if( nBuf>5 && 0==memcmp("icate", &aBuf[nBuf-5], 5) ){
25690 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25691 memcpy(&aBuf[nBuf-5], "ic", 2);
25692 *pnBuf = nBuf - 5 + 2;
25693 }
25694 }else if( nBuf>5 && 0==memcmp("iciti", &aBuf[nBuf-5], 5) ){
25695 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25696 memcpy(&aBuf[nBuf-5], "ic", 2);
25697 *pnBuf = nBuf - 5 + 2;
25698 }
25699 }
25700 break;
25701
25702 case 'u':
25703 if( nBuf>3 && 0==memcmp("ful", &aBuf[nBuf-3], 3) ){
25704 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
25705 *pnBuf = nBuf - 3;
25706 }
25707 }
25708 break;
25709
25710 case 'v':
25711 if( nBuf>5 && 0==memcmp("ative", &aBuf[nBuf-5], 5) ){
25712 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25713 *pnBuf = nBuf - 5;
25714 }
25715 }
25716 break;
25717
25718 case 'z':
25719 if( nBuf>5 && 0==memcmp("alize", &aBuf[nBuf-5], 5) ){
25720 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25721 memcpy(&aBuf[nBuf-5], "al", 2);
25722 *pnBuf = nBuf - 5 + 2;
25723 }
25724 }
25725 break;
25726
25727 }
25728 return ret;
25729}
25730
25731
25732static int fts5PorterStep1B(char *aBuf, int *pnBuf){
25733 int ret = 0;
25734 int nBuf = *pnBuf;
25735 switch( aBuf[nBuf-2] ){
25736
25737 case 'e':
25738 if( nBuf>3 && 0==memcmp("eed", &aBuf[nBuf-3], 3) ){
25739 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
25740 memcpy(&aBuf[nBuf-3], "ee", 2);
25741 *pnBuf = nBuf - 3 + 2;
25742 }
25743 }else if( nBuf>2 && 0==memcmp("ed", &aBuf[nBuf-2], 2) ){
25744 if( fts5Porter_Vowel(aBuf, nBuf-2) ){
25745 *pnBuf = nBuf - 2;
25746 ret = 1;
25747 }
25748 }
25749 break;
25750
25751 case 'n':
25752 if( nBuf>3 && 0==memcmp("ing", &aBuf[nBuf-3], 3) ){
25753 if( fts5Porter_Vowel(aBuf, nBuf-3) ){
25754 *pnBuf = nBuf - 3;
25755 ret = 1;
25756 }
25757 }
25758 break;
25759
25760 }
25761 return ret;
25762}
25763
25764/*
25765** GENERATED CODE ENDS HERE (mkportersteps.tcl)
25766***************************************************************************
25767**************************************************************************/
25768
25769static void fts5PorterStep1A(char *aBuf, int *pnBuf){
25770 int nBuf = *pnBuf;
25771 if( aBuf[nBuf-1]=='s' ){
25772 if( aBuf[nBuf-2]=='e' ){
25773 if( (nBuf>4 && aBuf[nBuf-4]=='s' && aBuf[nBuf-3]=='s')
25774 || (nBuf>3 && aBuf[nBuf-3]=='i' )
25775 ){
25776 *pnBuf = nBuf-2;
25777 }else{
25778 *pnBuf = nBuf-1;
25779 }
25780 }
25781 else if( aBuf[nBuf-2]!='s' ){
25782 *pnBuf = nBuf-1;
25783 }
25784 }
25785}
25786
25787static int fts5PorterCb(
25788 void *pCtx,
25789 int tflags,
25790 const char *pToken,
25791 int nToken,
25792 int iStart,
25793 int iEnd
25794){
25795 PorterContext *p = (PorterContext*)pCtx;
25796
25797 char *aBuf;
25798 int nBuf;
25799
25800 if( nToken>FTS5_PORTER_MAX_TOKEN64 || nToken<3 ) goto pass_through;
25801 aBuf = p->aBuf;
25802 nBuf = nToken;
25803 memcpy(aBuf, pToken, nBuf);
25804
25805 /* Step 1. */
25806 fts5PorterStep1A(aBuf, &nBuf);
25807 if( fts5PorterStep1B(aBuf, &nBuf) ){
25808 if( fts5PorterStep1B2(aBuf, &nBuf)==0 ){
25809 char c = aBuf[nBuf-1];
25810 if( fts5PorterIsVowel(c, 0)==0
25811 && c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2]
25812 ){
25813 nBuf--;
25814 }else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){
25815 aBuf[nBuf++] = 'e';
25816 }
25817 }
25818 }
25819
25820 /* Step 1C. */
25821 if( aBuf[nBuf-1]=='y' && fts5Porter_Vowel(aBuf, nBuf-1) ){
25822 aBuf[nBuf-1] = 'i';
25823 }
25824
25825 /* Steps 2 through 4. */
25826 fts5PorterStep2(aBuf, &nBuf);
25827 fts5PorterStep3(aBuf, &nBuf);
25828 fts5PorterStep4(aBuf, &nBuf);
25829
25830 /* Step 5a. */
25831 assert( nBuf>0 )((void) (0));
25832 if( aBuf[nBuf-1]=='e' ){
25833 if( fts5Porter_MGt1(aBuf, nBuf-1)
25834 || (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1))
25835 ){
25836 nBuf--;
25837 }
25838 }
25839
25840 /* Step 5b. */
25841 if( nBuf>1 && aBuf[nBuf-1]=='l'
25842 && aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1)
25843 ){
25844 nBuf--;
25845 }
25846
25847 return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd);
25848
25849 pass_through:
25850 return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
25851}
25852
25853/*
25854** Tokenize using the porter tokenizer.
25855*/
25856static int fts5PorterTokenize(
25857 Fts5Tokenizer *pTokenizer,
25858 void *pCtx,
25859 int flags,
25860 const char *pText, int nText,
25861 const char *pLoc, int nLoc,
25862 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
25863){
25864 PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
25865 PorterContext sCtx;
25866 sCtx.xToken = xToken;
25867 sCtx.pCtx = pCtx;
25868 sCtx.aBuf = p->aBuf;
25869 return p->tokenizer_v2.xTokenize(
25870 p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb
25871 );
25872}
25873
25874/**************************************************************************
25875** Start of trigram implementation.
25876*/
25877typedef struct TrigramTokenizer TrigramTokenizer;
25878struct TrigramTokenizer {
25879 int bFold; /* True to fold to lower-case */
25880 int iFoldParam; /* Parameter to pass to Fts5UnicodeFold() */
25881};
25882
25883/*
25884** Free a trigram tokenizer.
25885*/
25886static void fts5TriDelete(Fts5Tokenizer *p){
25887 sqlite3_freesqlite3_api->free(p);
25888}
25889
25890/*
25891** Allocate a trigram tokenizer.
25892*/
25893static int fts5TriCreate(
25894 void *pUnused,
25895 const char **azArg,
25896 int nArg,
25897 Fts5Tokenizer **ppOut
25898){
25899 int rc = SQLITE_OK0;
25900 TrigramTokenizer *pNew = 0;
25901 UNUSED_PARAM(pUnused)(void)(pUnused);
25902 if( nArg%2 ){
25903 rc = SQLITE_ERROR1;
25904 }else{
25905 int i;
25906 pNew = (TrigramTokenizer*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(*pNew));
25907 if( pNew==0 ){
25908 rc = SQLITE_NOMEM7;
25909 }else{
25910 pNew->bFold = 1;
25911 pNew->iFoldParam = 0;
25912
25913 for(i=0; rc==SQLITE_OK0 && i<nArg; i+=2){
25914 const char *zArg = azArg[i+1];
25915 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "case_sensitive") ){
25916 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
25917 rc = SQLITE_ERROR1;
25918 }else{
25919 pNew->bFold = (zArg[0]=='0');
25920 }
25921 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "remove_diacritics") ){
25922 if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
25923 rc = SQLITE_ERROR1;
25924 }else{
25925 pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
25926 }
25927 }else{
25928 rc = SQLITE_ERROR1;
25929 }
25930 }
25931
25932 if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
25933 rc = SQLITE_ERROR1;
25934 }
25935
25936 if( rc!=SQLITE_OK0 ){
25937 fts5TriDelete((Fts5Tokenizer*)pNew);
25938 pNew = 0;
25939 }
25940 }
25941 }
25942 *ppOut = (Fts5Tokenizer*)pNew;
25943 return rc;
25944}
25945
25946/*
25947** Trigram tokenizer tokenize routine.
25948*/
25949static int fts5TriTokenize(
25950 Fts5Tokenizer *pTok,
25951 void *pCtx,
25952 int unusedFlags,
25953 const char *pText, int nText,
25954 int (*xToken)(void*, int, const char*, int, int, int)
25955){
25956 TrigramTokenizer *p = (TrigramTokenizer*)pTok;
25957 int rc = SQLITE_OK0;
25958 char aBuf[32];
25959 char *zOut = aBuf;
25960 int ii;
25961 const unsigned char *zIn = (const unsigned char*)pText;
25962 const unsigned char *zEof = (zIn ? &zIn[nText] : 0);
25963 u32 iCode = 0;
25964 int aStart[3]; /* Input offset of each character in aBuf[] */
25965
25966 UNUSED_PARAM(unusedFlags)(void)(unusedFlags);
25967
25968 /* Populate aBuf[] with the characters for the first trigram. */
25969 for(ii=0; ii<3; ii++){
25970 do {
25971 aStart[ii] = zIn - (const unsigned char*)pText;
25972 if( zIn>=zEof ) return SQLITE_OK0;
25973 READ_UTF8(zIn, zEof, iCode)iCode = *(zIn++); if( iCode>=0xc0 ){ iCode = sqlite3Utf8Trans1
[iCode-0xc0]; while( zIn<zEof && (*zIn & 0xc0)
==0x80 ){ iCode = (iCode<<6) + (0x3f & *(zIn++)); }
if( iCode<0x80 || (iCode&0xFFFFF800)==0xD800 || (iCode
&0xFFFFFFFE)==0xFFFE ){ iCode = 0xFFFD; } };
25974 if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
25975 }while( iCode==0 );
25976 WRITE_UTF8(zOut, iCode){ if( iCode<0x00080 ){ *zOut++ = (unsigned char)(iCode&
0xFF); } else if( iCode<0x00800 ){ *zOut++ = 0xC0 + (unsigned
char)((iCode>>6)&0x1F); *zOut++ = 0x80 + (unsigned
char)(iCode & 0x3F); } else if( iCode<0x10000 ){ *zOut
++ = 0xE0 + (unsigned char)((iCode>>12)&0x0F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); }else{ *zOut++
= 0xF0 + (unsigned char)((iCode>>18) & 0x07); *zOut
++ = 0x80 + (unsigned char)((iCode>>12) & 0x3F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); } };
25977 }
25978
25979 /* At the start of each iteration of this loop:
25980 **
25981 ** aBuf: Contains 3 characters. The 3 characters of the next trigram.
25982 ** zOut: Points to the byte following the last character in aBuf.
25983 ** aStart[3]: Contains the byte offset in the input text corresponding
25984 ** to the start of each of the three characters in the buffer.
25985 */
25986 assert( zIn<=zEof )((void) (0));
25987 while( 1 ){
25988 int iNext; /* Start of character following current tri */
25989 const char *z1;
25990
25991 /* Read characters from the input up until the first non-diacritic */
25992 do {
25993 iNext = zIn - (const unsigned char*)pText;
25994 if( zIn>=zEof ){
25995 iCode = 0;
25996 break;
25997 }
25998 READ_UTF8(zIn, zEof, iCode)iCode = *(zIn++); if( iCode>=0xc0 ){ iCode = sqlite3Utf8Trans1
[iCode-0xc0]; while( zIn<zEof && (*zIn & 0xc0)
==0x80 ){ iCode = (iCode<<6) + (0x3f & *(zIn++)); }
if( iCode<0x80 || (iCode&0xFFFFF800)==0xD800 || (iCode
&0xFFFFFFFE)==0xFFFE ){ iCode = 0xFFFD; } };
25999 if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
26000 }while( iCode==0 );
26001
26002 /* Pass the current trigram back to fts5 */
26003 rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext);
26004 if( iCode==0 || rc!=SQLITE_OK0 ) break;
26005
26006 /* Remove the first character from buffer aBuf[]. Append the character
26007 ** with codepoint iCode. */
26008 z1 = aBuf;
26009 FTS5_SKIP_UTF8(z1){ if( ((unsigned char)(*(z1++)))>=0xc0 ){ while( (((unsigned
char)*z1) & 0xc0)==0x80 ){ z1++; } } };
26010 memmove(aBuf, z1, zOut - z1);
26011 zOut -= (z1 - aBuf);
26012 WRITE_UTF8(zOut, iCode){ if( iCode<0x00080 ){ *zOut++ = (unsigned char)(iCode&
0xFF); } else if( iCode<0x00800 ){ *zOut++ = 0xC0 + (unsigned
char)((iCode>>6)&0x1F); *zOut++ = 0x80 + (unsigned
char)(iCode & 0x3F); } else if( iCode<0x10000 ){ *zOut
++ = 0xE0 + (unsigned char)((iCode>>12)&0x0F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); }else{ *zOut++
= 0xF0 + (unsigned char)((iCode>>18) & 0x07); *zOut
++ = 0x80 + (unsigned char)((iCode>>12) & 0x3F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); } };
26013
26014 /* Update the aStart[] array */
26015 aStart[0] = aStart[1];
26016 aStart[1] = aStart[2];
26017 aStart[2] = iNext;
26018 }
26019
26020 return rc;
26021}
26022
26023/*
26024** Argument xCreate is a pointer to a constructor function for a tokenizer.
26025** pTok is a tokenizer previously created using the same method. This function
26026** returns one of FTS5_PATTERN_NONE, FTS5_PATTERN_LIKE or FTS5_PATTERN_GLOB
26027** indicating the style of pattern matching that the tokenizer can support.
26028** In practice, this is:
26029**
26030** "trigram" tokenizer, case_sensitive=1 - FTS5_PATTERN_GLOB
26031** "trigram" tokenizer, case_sensitive=0 (the default) - FTS5_PATTERN_LIKE
26032** all other tokenizers - FTS5_PATTERN_NONE
26033*/
26034static int sqlite3Fts5TokenizerPattern(
26035 int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
26036 Fts5Tokenizer *pTok
26037){
26038 if( xCreate==fts5TriCreate ){
26039 TrigramTokenizer *p = (TrigramTokenizer*)pTok;
26040 if( p->iFoldParam==0 ){
26041 return p->bFold ? FTS5_PATTERN_LIKE65 : FTS5_PATTERN_GLOB66;
26042 }
26043 }
26044 return FTS5_PATTERN_NONE0;
26045}
26046
26047/*
26048** Return true if the tokenizer described by p->azArg[] is the trigram
26049** tokenizer. This tokenizer needs to be loaded before xBestIndex is
26050** called for the first time in order to correctly handle LIKE/GLOB.
26051*/
26052static int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){
26053 return (p->nArg>=1 && 0==sqlite3_stricmpsqlite3_api->stricmp(p->azArg[0], "trigram"));
26054}
26055
26056
26057/*
26058** Register all built-in tokenizers with FTS5.
26059*/
26060static int sqlite3Fts5TokenizerInit(fts5_api *pApi){
26061 struct BuiltinTokenizer {
26062 const char *zName;
26063 fts5_tokenizer x;
26064 } aBuiltin[] = {
26065 { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
26066 { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
26067 { "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}},
26068 };
26069
26070 int rc = SQLITE_OK0; /* Return code */
26071 int i; /* To iterate through builtin functions */
26072
26073 for(i=0; rc==SQLITE_OK0 && i<ArraySize(aBuiltin)((int)(sizeof(aBuiltin) / sizeof(aBuiltin[0]))); i++){
26074 rc = pApi->xCreateTokenizer(pApi,
26075 aBuiltin[i].zName,
26076 (void*)pApi,
26077 &aBuiltin[i].x,
26078 0
26079 );
26080 }
26081 if( rc==SQLITE_OK0 ){
26082 fts5_tokenizer_v2 sPorter = {
26083 2,
26084 fts5PorterCreate,
26085 fts5PorterDelete,
26086 fts5PorterTokenize
26087 };
26088 rc = pApi->xCreateTokenizer_v2(pApi,
26089 "porter",
26090 (void*)pApi,
26091 &sPorter,
26092 0
26093 );
26094 }
26095 return rc;
26096}
26097
26098#line 1 "fts5_unicode2.c"
26099/*
26100** 2012-05-25
26101**
26102** The author disclaims copyright to this source code. In place of
26103** a legal notice, here is a blessing:
26104**
26105** May you do good and not evil.
26106** May you find forgiveness for yourself and forgive others.
26107** May you share freely, never taking more than you give.
26108**
26109******************************************************************************
26110*/
26111
26112/*
26113** DO NOT EDIT THIS MACHINE GENERATED FILE.
26114*/
26115
26116
26117#include <assert.h>
26118
26119
26120
26121/*
26122** If the argument is a codepoint corresponding to a lowercase letter
26123** in the ASCII range with a diacritic added, return the codepoint
26124** of the ASCII letter only. For example, if passed 235 - "LATIN
26125** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
26126** E"). The resuls of passing a codepoint that corresponds to an
26127** uppercase letter are undefined.
26128*/
26129static int fts5_remove_diacritic(int c, int bComplex){
26130 unsigned short aDia[] = {
26131 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995,
26132 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286,
26133 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732,
26134 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336,
26135 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896,
26136 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106,
26137 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344,
26138 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198,
26139 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468,
26140 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704,
26141 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914,
26142 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218,
26143 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554,
26144 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766,
26145 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118,
26146 63182, 63242, 63274, 63310, 63368, 63390,
26147 };
26148#define HIBIT((unsigned char)0x80) ((unsigned char)0x80)
26149 unsigned char aChar[] = {
26150 '\0', 'a', 'c', 'e', 'i', 'n',
26151 'o', 'u', 'y', 'y', 'a', 'c',
26152 'd', 'e', 'e', 'g', 'h', 'i',
26153 'j', 'k', 'l', 'n', 'o', 'r',
26154 's', 't', 'u', 'u', 'w', 'y',
26155 'z', 'o', 'u', 'a', 'i', 'o',
26156 'u', 'u'|HIBIT((unsigned char)0x80), 'a'|HIBIT((unsigned char)0x80), 'g', 'k', 'o',
26157 'o'|HIBIT((unsigned char)0x80), 'j', 'g', 'n', 'a'|HIBIT((unsigned char)0x80), 'a',
26158 'e', 'i', 'o', 'r', 'u', 's',
26159 't', 'h', 'a', 'e', 'o'|HIBIT((unsigned char)0x80), 'o',
26160 'o'|HIBIT((unsigned char)0x80), 'y', '\0', '\0', '\0', '\0',
26161 '\0', '\0', '\0', '\0', 'a', 'b',
26162 'c'|HIBIT((unsigned char)0x80), 'd', 'd', 'e'|HIBIT((unsigned char)0x80), 'e', 'e'|HIBIT((unsigned char)0x80),
26163 'f', 'g', 'h', 'h', 'i', 'i'|HIBIT((unsigned char)0x80),
26164 'k', 'l', 'l'|HIBIT((unsigned char)0x80), 'l', 'm', 'n',
26165 'o'|HIBIT((unsigned char)0x80), 'p', 'r', 'r'|HIBIT((unsigned char)0x80), 'r', 's',
26166 's'|HIBIT((unsigned char)0x80), 't', 'u', 'u'|HIBIT((unsigned char)0x80), 'v', 'w',
26167 'w', 'x', 'y', 'z', 'h', 't',
26168 'w', 'y', 'a', 'a'|HIBIT((unsigned char)0x80), 'a'|HIBIT((unsigned char)0x80), 'a'|HIBIT((unsigned char)0x80),
26169 'e', 'e'|HIBIT((unsigned char)0x80), 'e'|HIBIT((unsigned char)0x80), 'i', 'o', 'o'|HIBIT((unsigned char)0x80),
26170 'o'|HIBIT((unsigned char)0x80), 'o'|HIBIT((unsigned char)0x80), 'u', 'u'|HIBIT((unsigned char)0x80), 'u'|HIBIT((unsigned char)0x80), 'y',
26171 };
26172
26173 unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
26174 int iRes = 0;
26175 int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
26176 int iLo = 0;
26177 while( iHi>=iLo ){
26178 int iTest = (iHi + iLo) / 2;
26179 if( key >= aDia[iTest] ){
26180 iRes = iTest;
26181 iLo = iTest+1;
26182 }else{
26183 iHi = iTest-1;
26184 }
26185 }
26186 assert( key>=aDia[iRes] )((void) (0));
26187 if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
26188 return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
26189}
26190
26191
26192/*
26193** Return true if the argument interpreted as a unicode codepoint
26194** is a diacritical modifier character.
26195*/
26196static int sqlite3Fts5UnicodeIsdiacritic(int c){
26197 unsigned int mask0 = 0x08029FDF;
26198 unsigned int mask1 = 0x000361F8;
26199 if( c<768 || c>817 ) return 0;
26200 return (c < 768+32) ?
26201 (mask0 & ((unsigned int)1 << (c-768))) :
26202 (mask1 & ((unsigned int)1 << (c-768-32)));
26203}
26204
26205
26206/*
26207** Interpret the argument as a unicode codepoint. If the codepoint
26208** is an upper case character that has a lower case equivalent,
26209** return the codepoint corresponding to the lower case version.
26210** Otherwise, return a copy of the argument.
26211**
26212** The results are undefined if the value passed to this function
26213** is less than zero.
26214*/
26215static int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
26216 /* Each entry in the following array defines a rule for folding a range
26217 ** of codepoints to lower case. The rule applies to a range of nRange
26218 ** codepoints starting at codepoint iCode.
26219 **
26220 ** If the least significant bit in flags is clear, then the rule applies
26221 ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
26222 ** need to be folded). Or, if it is set, then the rule only applies to
26223 ** every second codepoint in the range, starting with codepoint C.
26224 **
26225 ** The 7 most significant bits in flags are an index into the aiOff[]
26226 ** array. If a specific codepoint C does require folding, then its lower
26227 ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF).
26228 **
26229 ** The contents of this array are generated by parsing the CaseFolding.txt
26230 ** file distributed as part of the "Unicode Character Database". See
26231 ** http://www.unicode.org for details.
26232 */
26233 static const struct TableEntry {
26234 unsigned short iCode;
26235 unsigned char flags;
26236 unsigned char nRange;
26237 } aEntry[] = {
26238 {65, 14, 26}, {181, 64, 1}, {192, 14, 23},
26239 {216, 14, 7}, {256, 1, 48}, {306, 1, 6},
26240 {313, 1, 16}, {330, 1, 46}, {376, 116, 1},
26241 {377, 1, 6}, {383, 104, 1}, {385, 50, 1},
26242 {386, 1, 4}, {390, 44, 1}, {391, 0, 1},
26243 {393, 42, 2}, {395, 0, 1}, {398, 32, 1},
26244 {399, 38, 1}, {400, 40, 1}, {401, 0, 1},
26245 {403, 42, 1}, {404, 46, 1}, {406, 52, 1},
26246 {407, 48, 1}, {408, 0, 1}, {412, 52, 1},
26247 {413, 54, 1}, {415, 56, 1}, {416, 1, 6},
26248 {422, 60, 1}, {423, 0, 1}, {425, 60, 1},
26249 {428, 0, 1}, {430, 60, 1}, {431, 0, 1},
26250 {433, 58, 2}, {435, 1, 4}, {439, 62, 1},
26251 {440, 0, 1}, {444, 0, 1}, {452, 2, 1},
26252 {453, 0, 1}, {455, 2, 1}, {456, 0, 1},
26253 {458, 2, 1}, {459, 1, 18}, {478, 1, 18},
26254 {497, 2, 1}, {498, 1, 4}, {502, 122, 1},
26255 {503, 134, 1}, {504, 1, 40}, {544, 110, 1},
26256 {546, 1, 18}, {570, 70, 1}, {571, 0, 1},
26257 {573, 108, 1}, {574, 68, 1}, {577, 0, 1},
26258 {579, 106, 1}, {580, 28, 1}, {581, 30, 1},
26259 {582, 1, 10}, {837, 36, 1}, {880, 1, 4},
26260 {886, 0, 1}, {902, 18, 1}, {904, 16, 3},
26261 {908, 26, 1}, {910, 24, 2}, {913, 14, 17},
26262 {931, 14, 9}, {962, 0, 1}, {975, 4, 1},
26263 {976, 140, 1}, {977, 142, 1}, {981, 146, 1},
26264 {982, 144, 1}, {984, 1, 24}, {1008, 136, 1},
26265 {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1},
26266 {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1},
26267 {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32},
26268 {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1},
26269 {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38},
26270 {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1},
26271 {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1},
26272 {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6},
26273 {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6},
26274 {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8},
26275 {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2},
26276 {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1},
26277 {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2},
26278 {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2},
26279 {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2},
26280 {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1},
26281 {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16},
26282 {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47},
26283 {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1},
26284 {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1},
26285 {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1},
26286 {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2},
26287 {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1},
26288 {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14},
26289 {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1},
26290 {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1},
26291 {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1},
26292 {65313, 14, 26},
26293 };
26294 static const unsigned short aiOff[] = {
26295 1, 2, 8, 15, 16, 26, 28, 32,
26296 37, 38, 40, 48, 63, 64, 69, 71,
26297 79, 80, 116, 202, 203, 205, 206, 207,
26298 209, 210, 211, 213, 214, 217, 218, 219,
26299 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721,
26300 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274,
26301 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406,
26302 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462,
26303 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511,
26304 65514, 65521, 65527, 65528, 65529,
26305 };
26306
26307 int ret = c;
26308
26309 assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 )((void) (0));
26310
26311 if( c<128 ){
26312 if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
26313 }else if( c<65536 ){
26314 const struct TableEntry *p;
26315 int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
26316 int iLo = 0;
26317 int iRes = -1;
26318
26319 assert( c>aEntry[0].iCode )((void) (0));
26320 while( iHi>=iLo ){
26321 int iTest = (iHi + iLo) / 2;
26322 int cmp = (c - aEntry[iTest].iCode);
26323 if( cmp>=0 ){
26324 iRes = iTest;
26325 iLo = iTest+1;
26326 }else{
26327 iHi = iTest-1;
26328 }
26329 }
26330
26331 assert( iRes>=0 && c>=aEntry[iRes].iCode )((void) (0));
26332 p = &aEntry[iRes];
26333 if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
26334 ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
26335 assert( ret>0 )((void) (0));
26336 }
26337
26338 if( eRemoveDiacritic ){
26339 ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
26340 }
26341 }
26342
26343 else if( c>=66560 && c<66600 ){
26344 ret = c + 40;
26345 }
26346
26347 return ret;
26348}
26349
26350
26351static int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){
26352 aArray[0] = 1;
26353 switch( zCat[0] ){
26354 case 'C':
26355 switch( zCat[1] ){
26356 case 'c': aArray[1] = 1; break;
26357 case 'f': aArray[2] = 1; break;
26358 case 'n': aArray[3] = 1; break;
26359 case 's': aArray[4] = 1; break;
26360 case 'o': aArray[31] = 1; break;
26361 case '*':
26362 aArray[1] = 1;
26363 aArray[2] = 1;
26364 aArray[3] = 1;
26365 aArray[4] = 1;
26366 aArray[31] = 1;
26367 break;
26368 default: return 1; }
26369 break;
26370
26371 case 'L':
26372 switch( zCat[1] ){
26373 case 'l': aArray[5] = 1; break;
26374 case 'm': aArray[6] = 1; break;
26375 case 'o': aArray[7] = 1; break;
26376 case 't': aArray[8] = 1; break;
26377 case 'u': aArray[9] = 1; break;
26378 case 'C': aArray[30] = 1; break;
26379 case '*':
26380 aArray[5] = 1;
26381 aArray[6] = 1;
26382 aArray[7] = 1;
26383 aArray[8] = 1;
26384 aArray[9] = 1;
26385 aArray[30] = 1;
26386 break;
26387 default: return 1; }
26388 break;
26389
26390 case 'M':
26391 switch( zCat[1] ){
26392 case 'c': aArray[10] = 1; break;
26393 case 'e': aArray[11] = 1; break;
26394 case 'n': aArray[12] = 1; break;
26395 case '*':
26396 aArray[10] = 1;
26397 aArray[11] = 1;
26398 aArray[12] = 1;
26399 break;
26400 default: return 1; }
26401 break;
26402
26403 case 'N':
26404 switch( zCat[1] ){
26405 case 'd': aArray[13] = 1; break;
26406 case 'l': aArray[14] = 1; break;
26407 case 'o': aArray[15] = 1; break;
26408 case '*':
26409 aArray[13] = 1;
26410 aArray[14] = 1;
26411 aArray[15] = 1;
26412 break;
26413 default: return 1; }
26414 break;
26415
26416 case 'P':
26417 switch( zCat[1] ){
26418 case 'c': aArray[16] = 1; break;
26419 case 'd': aArray[17] = 1; break;
26420 case 'e': aArray[18] = 1; break;
26421 case 'f': aArray[19] = 1; break;
26422 case 'i': aArray[20] = 1; break;
26423 case 'o': aArray[21] = 1; break;
26424 case 's': aArray[22] = 1; break;
26425 case '*':
26426 aArray[16] = 1;
26427 aArray[17] = 1;
26428 aArray[18] = 1;
26429 aArray[19] = 1;
26430 aArray[20] = 1;
26431 aArray[21] = 1;
26432 aArray[22] = 1;
26433 break;
26434 default: return 1; }
26435 break;
26436
26437 case 'S':
26438 switch( zCat[1] ){
26439 case 'c': aArray[23] = 1; break;
26440 case 'k': aArray[24] = 1; break;
26441 case 'm': aArray[25] = 1; break;
26442 case 'o': aArray[26] = 1; break;
26443 case '*':
26444 aArray[23] = 1;
26445 aArray[24] = 1;
26446 aArray[25] = 1;
26447 aArray[26] = 1;
26448 break;
26449 default: return 1; }
26450 break;
26451
26452 case 'Z':
26453 switch( zCat[1] ){
26454 case 'l': aArray[27] = 1; break;
26455 case 'p': aArray[28] = 1; break;
26456 case 's': aArray[29] = 1; break;
26457 case '*':
26458 aArray[27] = 1;
26459 aArray[28] = 1;
26460 aArray[29] = 1;
26461 break;
26462 default: return 1; }
26463 break;
26464
26465
26466 default:
26467 return 1;
26468 }
26469 return 0;
26470}
26471
26472static u16 aFts5UnicodeBlock[] = {
26473 0, 1471, 1753, 1760, 1760, 1760, 1760, 1760, 1760, 1760,
26474 1760, 1760, 1760, 1760, 1760, 1763, 1765,
26475 };
26476static u16 aFts5UnicodeMap[] = {
26477 0, 32, 33, 36, 37, 40, 41, 42, 43, 44,
26478 45, 46, 48, 58, 60, 63, 65, 91, 92, 93,
26479 94, 95, 96, 97, 123, 124, 125, 126, 127, 160,
26480 161, 162, 166, 167, 168, 169, 170, 171, 172, 173,
26481 174, 175, 176, 177, 178, 180, 181, 182, 184, 185,
26482 186, 187, 188, 191, 192, 215, 216, 223, 247, 248,
26483 256, 312, 313, 329, 330, 377, 383, 385, 387, 388,
26484 391, 394, 396, 398, 402, 403, 405, 406, 409, 412,
26485 414, 415, 417, 418, 423, 427, 428, 431, 434, 436,
26486 437, 440, 442, 443, 444, 446, 448, 452, 453, 454,
26487 455, 456, 457, 458, 459, 460, 461, 477, 478, 496,
26488 497, 498, 499, 500, 503, 505, 506, 564, 570, 572,
26489 573, 575, 577, 580, 583, 584, 592, 660, 661, 688,
26490 706, 710, 722, 736, 741, 748, 749, 750, 751, 768,
26491 880, 884, 885, 886, 890, 891, 894, 900, 902, 903,
26492 904, 908, 910, 912, 913, 931, 940, 975, 977, 978,
26493 981, 984, 1008, 1012, 1014, 1015, 1018, 1020, 1021, 1072,
26494 1120, 1154, 1155, 1160, 1162, 1217, 1231, 1232, 1329, 1369,
26495 1370, 1377, 1417, 1418, 1423, 1425, 1470, 1471, 1472, 1473,
26496 1475, 1476, 1478, 1479, 1488, 1520, 1523, 1536, 1542, 1545,
26497 1547, 1548, 1550, 1552, 1563, 1566, 1568, 1600, 1601, 1611,
26498 1632, 1642, 1646, 1648, 1649, 1748, 1749, 1750, 1757, 1758,
26499 1759, 1765, 1767, 1769, 1770, 1774, 1776, 1786, 1789, 1791,
26500 1792, 1807, 1808, 1809, 1810, 1840, 1869, 1958, 1969, 1984,
26501 1994, 2027, 2036, 2038, 2039, 2042, 2048, 2070, 2074, 2075,
26502 2084, 2085, 2088, 2089, 2096, 2112, 2137, 2142, 2208, 2210,
26503 2276, 2304, 2307, 2308, 2362, 2363, 2364, 2365, 2366, 2369,
26504 2377, 2381, 2382, 2384, 2385, 2392, 2402, 2404, 2406, 2416,
26505 2417, 2418, 2425, 2433, 2434, 2437, 2447, 2451, 2474, 2482,
26506 2486, 2492, 2493, 2494, 2497, 2503, 2507, 2509, 2510, 2519,
26507 2524, 2527, 2530, 2534, 2544, 2546, 2548, 2554, 2555, 2561,
26508 2563, 2565, 2575, 2579, 2602, 2610, 2613, 2616, 2620, 2622,
26509 2625, 2631, 2635, 2641, 2649, 2654, 2662, 2672, 2674, 2677,
26510 2689, 2691, 2693, 2703, 2707, 2730, 2738, 2741, 2748, 2749,
26511 2750, 2753, 2759, 2761, 2763, 2765, 2768, 2784, 2786, 2790,
26512 2800, 2801, 2817, 2818, 2821, 2831, 2835, 2858, 2866, 2869,
26513 2876, 2877, 2878, 2879, 2880, 2881, 2887, 2891, 2893, 2902,
26514 2903, 2908, 2911, 2914, 2918, 2928, 2929, 2930, 2946, 2947,
26515 2949, 2958, 2962, 2969, 2972, 2974, 2979, 2984, 2990, 3006,
26516 3008, 3009, 3014, 3018, 3021, 3024, 3031, 3046, 3056, 3059,
26517 3065, 3066, 3073, 3077, 3086, 3090, 3114, 3125, 3133, 3134,
26518 3137, 3142, 3146, 3157, 3160, 3168, 3170, 3174, 3192, 3199,
26519 3202, 3205, 3214, 3218, 3242, 3253, 3260, 3261, 3262, 3263,
26520 3264, 3270, 3271, 3274, 3276, 3285, 3294, 3296, 3298, 3302,
26521 3313, 3330, 3333, 3342, 3346, 3389, 3390, 3393, 3398, 3402,
26522 3405, 3406, 3415, 3424, 3426, 3430, 3440, 3449, 3450, 3458,
26523 3461, 3482, 3507, 3517, 3520, 3530, 3535, 3538, 3542, 3544,
26524 3570, 3572, 3585, 3633, 3634, 3636, 3647, 3648, 3654, 3655,
26525 3663, 3664, 3674, 3713, 3716, 3719, 3722, 3725, 3732, 3737,
26526 3745, 3749, 3751, 3754, 3757, 3761, 3762, 3764, 3771, 3773,
26527 3776, 3782, 3784, 3792, 3804, 3840, 3841, 3844, 3859, 3860,
26528 3861, 3864, 3866, 3872, 3882, 3892, 3893, 3894, 3895, 3896,
26529 3897, 3898, 3899, 3900, 3901, 3902, 3904, 3913, 3953, 3967,
26530 3968, 3973, 3974, 3976, 3981, 3993, 4030, 4038, 4039, 4046,
26531 4048, 4053, 4057, 4096, 4139, 4141, 4145, 4146, 4152, 4153,
26532 4155, 4157, 4159, 4160, 4170, 4176, 4182, 4184, 4186, 4190,
26533 4193, 4194, 4197, 4199, 4206, 4209, 4213, 4226, 4227, 4229,
26534 4231, 4237, 4238, 4239, 4240, 4250, 4253, 4254, 4256, 4295,
26535 4301, 4304, 4347, 4348, 4349, 4682, 4688, 4696, 4698, 4704,
26536 4746, 4752, 4786, 4792, 4800, 4802, 4808, 4824, 4882, 4888,
26537 4957, 4960, 4969, 4992, 5008, 5024, 5120, 5121, 5741, 5743,
26538 5760, 5761, 5787, 5788, 5792, 5867, 5870, 5888, 5902, 5906,
26539 5920, 5938, 5941, 5952, 5970, 5984, 5998, 6002, 6016, 6068,
26540 6070, 6071, 6078, 6086, 6087, 6089, 6100, 6103, 6104, 6107,
26541 6108, 6109, 6112, 6128, 6144, 6150, 6151, 6155, 6158, 6160,
26542 6176, 6211, 6212, 6272, 6313, 6314, 6320, 6400, 6432, 6435,
26543 6439, 6441, 6448, 6450, 6451, 6457, 6464, 6468, 6470, 6480,
26544 6512, 6528, 6576, 6593, 6600, 6608, 6618, 6622, 6656, 6679,
26545 6681, 6686, 6688, 6741, 6742, 6743, 6744, 6752, 6753, 6754,
26546 6755, 6757, 6765, 6771, 6783, 6784, 6800, 6816, 6823, 6824,
26547 6912, 6916, 6917, 6964, 6965, 6966, 6971, 6972, 6973, 6978,
26548 6979, 6981, 6992, 7002, 7009, 7019, 7028, 7040, 7042, 7043,
26549 7073, 7074, 7078, 7080, 7082, 7083, 7084, 7086, 7088, 7098,
26550 7142, 7143, 7144, 7146, 7149, 7150, 7151, 7154, 7164, 7168,
26551 7204, 7212, 7220, 7222, 7227, 7232, 7245, 7248, 7258, 7288,
26552 7294, 7360, 7376, 7379, 7380, 7393, 7394, 7401, 7405, 7406,
26553 7410, 7412, 7413, 7424, 7468, 7531, 7544, 7545, 7579, 7616,
26554 7676, 7680, 7830, 7838, 7936, 7944, 7952, 7960, 7968, 7976,
26555 7984, 7992, 8000, 8008, 8016, 8025, 8027, 8029, 8031, 8033,
26556 8040, 8048, 8064, 8072, 8080, 8088, 8096, 8104, 8112, 8118,
26557 8120, 8124, 8125, 8126, 8127, 8130, 8134, 8136, 8140, 8141,
26558 8144, 8150, 8152, 8157, 8160, 8168, 8173, 8178, 8182, 8184,
26559 8188, 8189, 8192, 8203, 8208, 8214, 8216, 8217, 8218, 8219,
26560 8221, 8222, 8223, 8224, 8232, 8233, 8234, 8239, 8240, 8249,
26561 8250, 8251, 8255, 8257, 8260, 8261, 8262, 8263, 8274, 8275,
26562 8276, 8277, 8287, 8288, 8298, 8304, 8305, 8308, 8314, 8317,
26563 8318, 8319, 8320, 8330, 8333, 8334, 8336, 8352, 8400, 8413,
26564 8417, 8418, 8421, 8448, 8450, 8451, 8455, 8456, 8458, 8459,
26565 8462, 8464, 8467, 8468, 8469, 8470, 8472, 8473, 8478, 8484,
26566 8485, 8486, 8487, 8488, 8489, 8490, 8494, 8495, 8496, 8500,
26567 8501, 8505, 8506, 8508, 8510, 8512, 8517, 8519, 8522, 8523,
26568 8524, 8526, 8527, 8528, 8544, 8579, 8581, 8585, 8592, 8597,
26569 8602, 8604, 8608, 8609, 8611, 8612, 8614, 8615, 8622, 8623,
26570 8654, 8656, 8658, 8659, 8660, 8661, 8692, 8960, 8968, 8972,
26571 8992, 8994, 9001, 9002, 9003, 9084, 9085, 9115, 9140, 9180,
26572 9186, 9216, 9280, 9312, 9372, 9450, 9472, 9655, 9656, 9665,
26573 9666, 9720, 9728, 9839, 9840, 9985, 10088, 10089, 10090, 10091,
26574 10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101,
26575 10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217,
26576 10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627,
26577 10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637,
26578 10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647,
26579 10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750,
26580 11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365,
26581 11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393,
26582 11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520,
26583 11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696,
26584 11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780,
26585 11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800,
26586 11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812,
26587 11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904,
26588 11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296,
26589 12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306,
26590 12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317,
26591 12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347,
26592 12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449,
26593 12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736,
26594 12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938,
26595 12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981,
26596 40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528,
26597 42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624,
26598 42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800,
26599 42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912,
26600 43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043,
26601 43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136,
26602 43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264,
26603 43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395,
26604 43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472,
26605 43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588,
26606 43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643,
26607 43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713,
26608 43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762,
26609 43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003,
26610 44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203,
26611 55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112,
26612 64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320,
26613 64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020,
26614 65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075,
26615 65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086,
26616 65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097,
26617 65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118,
26618 65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279,
26619 65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294,
26620 65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343,
26621 65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378,
26622 65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490,
26623 65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529,
26624 65532, 0, 13, 40, 60, 63, 80, 128, 256, 263,
26625 311, 320, 373, 377, 394, 400, 464, 509, 640, 672,
26626 768, 800, 816, 833, 834, 842, 896, 927, 928, 968,
26627 976, 977, 1024, 1064, 1104, 1184, 2048, 2056, 2058, 2103,
26628 2108, 2111, 2135, 2136, 2304, 2326, 2335, 2336, 2367, 2432,
26629 2494, 2560, 2561, 2565, 2572, 2576, 2581, 2585, 2616, 2623,
26630 2624, 2640, 2656, 2685, 2687, 2816, 2873, 2880, 2904, 2912,
26631 2936, 3072, 3680, 4096, 4097, 4098, 4099, 4152, 4167, 4178,
26632 4198, 4224, 4226, 4227, 4272, 4275, 4279, 4281, 4283, 4285,
26633 4286, 4304, 4336, 4352, 4355, 4391, 4396, 4397, 4406, 4416,
26634 4480, 4482, 4483, 4531, 4534, 4543, 4545, 4549, 4560, 5760,
26635 5803, 5804, 5805, 5806, 5808, 5814, 5815, 5824, 8192, 9216,
26636 9328, 12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248,
26637 53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637,
26638 53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298,
26639 54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441,
26640 54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541,
26641 54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662,
26642 54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922,
26643 54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062,
26644 55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178,
26645 55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961,
26646 60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003,
26647 61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028,
26648 61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099,
26649 61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744,
26650 61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368,
26651 62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971,
26652 63045, 63104, 63232, 0, 42710, 42752, 46900, 46912, 47133, 63488,
26653 1, 32, 256, 0, 65533,
26654 };
26655static u16 aFts5UnicodeData[] = {
26656 1025, 61, 117, 55, 117, 54, 50, 53, 57, 53,
26657 49, 85, 333, 85, 121, 85, 841, 54, 53, 50,
26658 56, 48, 56, 837, 54, 57, 50, 57, 1057, 61,
26659 53, 151, 58, 53, 56, 58, 39, 52, 57, 34,
26660 58, 56, 58, 57, 79, 56, 37, 85, 56, 47,
26661 39, 51, 111, 53, 745, 57, 233, 773, 57, 261,
26662 1822, 37, 542, 37, 1534, 222, 69, 73, 37, 126,
26663 126, 73, 69, 137, 37, 73, 37, 105, 101, 73,
26664 37, 73, 37, 190, 158, 37, 126, 126, 73, 37,
26665 126, 94, 37, 39, 94, 69, 135, 41, 40, 37,
26666 41, 40, 37, 41, 40, 37, 542, 37, 606, 37,
26667 41, 40, 37, 126, 73, 37, 1886, 197, 73, 37,
26668 73, 69, 126, 105, 37, 286, 2181, 39, 869, 582,
26669 152, 390, 472, 166, 248, 38, 56, 38, 568, 3596,
26670 158, 38, 56, 94, 38, 101, 53, 88, 41, 53,
26671 105, 41, 73, 37, 553, 297, 1125, 94, 37, 105,
26672 101, 798, 133, 94, 57, 126, 94, 37, 1641, 1541,
26673 1118, 58, 172, 75, 1790, 478, 37, 2846, 1225, 38,
26674 213, 1253, 53, 49, 55, 1452, 49, 44, 53, 76,
26675 53, 76, 53, 44, 871, 103, 85, 162, 121, 85,
26676 55, 85, 90, 364, 53, 85, 1031, 38, 327, 684,
26677 333, 149, 71, 44, 3175, 53, 39, 236, 34, 58,
26678 204, 70, 76, 58, 140, 71, 333, 103, 90, 39,
26679 469, 34, 39, 44, 967, 876, 2855, 364, 39, 333,
26680 1063, 300, 70, 58, 117, 38, 711, 140, 38, 300,
26681 38, 108, 38, 172, 501, 807, 108, 53, 39, 359,
26682 876, 108, 42, 1735, 44, 42, 44, 39, 106, 268,
26683 138, 44, 74, 39, 236, 327, 76, 85, 333, 53,
26684 38, 199, 231, 44, 74, 263, 71, 711, 231, 39,
26685 135, 44, 39, 106, 140, 74, 74, 44, 39, 42,
26686 71, 103, 76, 333, 71, 87, 207, 58, 55, 76,
26687 42, 199, 71, 711, 231, 71, 71, 71, 44, 106,
26688 76, 76, 108, 44, 135, 39, 333, 76, 103, 44,
26689 76, 42, 295, 103, 711, 231, 71, 167, 44, 39,
26690 106, 172, 76, 42, 74, 44, 39, 71, 76, 333,
26691 53, 55, 44, 74, 263, 71, 711, 231, 71, 167,
26692 44, 39, 42, 44, 42, 140, 74, 74, 44, 44,
26693 42, 71, 103, 76, 333, 58, 39, 207, 44, 39,
26694 199, 103, 135, 71, 39, 71, 71, 103, 391, 74,
26695 44, 74, 106, 106, 44, 39, 42, 333, 111, 218,
26696 55, 58, 106, 263, 103, 743, 327, 167, 39, 108,
26697 138, 108, 140, 76, 71, 71, 76, 333, 239, 58,
26698 74, 263, 103, 743, 327, 167, 44, 39, 42, 44,
26699 170, 44, 74, 74, 76, 74, 39, 71, 76, 333,
26700 71, 74, 263, 103, 1319, 39, 106, 140, 106, 106,
26701 44, 39, 42, 71, 76, 333, 207, 58, 199, 74,
26702 583, 775, 295, 39, 231, 44, 106, 108, 44, 266,
26703 74, 53, 1543, 44, 71, 236, 55, 199, 38, 268,
26704 53, 333, 85, 71, 39, 71, 39, 39, 135, 231,
26705 103, 39, 39, 71, 135, 44, 71, 204, 76, 39,
26706 167, 38, 204, 333, 135, 39, 122, 501, 58, 53,
26707 122, 76, 218, 333, 335, 58, 44, 58, 44, 58,
26708 44, 54, 50, 54, 50, 74, 263, 1159, 460, 42,
26709 172, 53, 76, 167, 364, 1164, 282, 44, 218, 90,
26710 181, 154, 85, 1383, 74, 140, 42, 204, 42, 76,
26711 74, 76, 39, 333, 213, 199, 74, 76, 135, 108,
26712 39, 106, 71, 234, 103, 140, 423, 44, 74, 76,
26713 202, 44, 39, 42, 333, 106, 44, 90, 1225, 41,
26714 41, 1383, 53, 38, 10631, 135, 231, 39, 135, 1319,
26715 135, 1063, 135, 231, 39, 135, 487, 1831, 135, 2151,
26716 108, 309, 655, 519, 346, 2727, 49, 19847, 85, 551,
26717 61, 839, 54, 50, 2407, 117, 110, 423, 135, 108,
26718 583, 108, 85, 583, 76, 423, 103, 76, 1671, 76,
26719 42, 236, 266, 44, 74, 364, 117, 38, 117, 55,
26720 39, 44, 333, 335, 213, 49, 149, 108, 61, 333,
26721 1127, 38, 1671, 1319, 44, 39, 2247, 935, 108, 138,
26722 76, 106, 74, 44, 202, 108, 58, 85, 333, 967,
26723 167, 1415, 554, 231, 74, 333, 47, 1114, 743, 76,
26724 106, 85, 1703, 42, 44, 42, 236, 44, 42, 44,
26725 74, 268, 202, 332, 44, 333, 333, 245, 38, 213,
26726 140, 42, 1511, 44, 42, 172, 42, 44, 170, 44,
26727 74, 231, 333, 245, 346, 300, 314, 76, 42, 967,
26728 42, 140, 74, 76, 42, 44, 74, 71, 333, 1415,
26729 44, 42, 76, 106, 44, 42, 108, 74, 149, 1159,
26730 266, 268, 74, 76, 181, 333, 103, 333, 967, 198,
26731 85, 277, 108, 53, 428, 42, 236, 135, 44, 135,
26732 74, 44, 71, 1413, 2022, 421, 38, 1093, 1190, 1260,
26733 140, 4830, 261, 3166, 261, 265, 197, 201, 261, 265,
26734 261, 265, 197, 201, 261, 41, 41, 41, 94, 229,
26735 265, 453, 261, 264, 261, 264, 261, 264, 165, 69,
26736 137, 40, 56, 37, 120, 101, 69, 137, 40, 120,
26737 133, 69, 137, 120, 261, 169, 120, 101, 69, 137,
26738 40, 88, 381, 162, 209, 85, 52, 51, 54, 84,
26739 51, 54, 52, 277, 59, 60, 162, 61, 309, 52,
26740 51, 149, 80, 117, 57, 54, 50, 373, 57, 53,
26741 48, 341, 61, 162, 194, 47, 38, 207, 121, 54,
26742 50, 38, 335, 121, 54, 50, 422, 855, 428, 139,
26743 44, 107, 396, 90, 41, 154, 41, 90, 37, 105,
26744 69, 105, 37, 58, 41, 90, 57, 169, 218, 41,
26745 58, 41, 58, 41, 58, 137, 58, 37, 137, 37,
26746 135, 37, 90, 69, 73, 185, 94, 101, 58, 57,
26747 90, 37, 58, 527, 1134, 94, 142, 47, 185, 186,
26748 89, 154, 57, 90, 57, 90, 57, 250, 57, 1018,
26749 89, 90, 57, 58, 57, 1018, 8601, 282, 153, 666,
26750 89, 250, 54, 50, 2618, 57, 986, 825, 1306, 217,
26751 602, 1274, 378, 1935, 2522, 719, 5882, 57, 314, 57,
26752 1754, 281, 3578, 57, 4634, 3322, 54, 50, 54, 50,
26753 54, 50, 54, 50, 54, 50, 54, 50, 54, 50,
26754 975, 1434, 185, 54, 50, 1017, 54, 50, 54, 50,
26755 54, 50, 54, 50, 54, 50, 537, 8218, 4217, 54,
26756 50, 54, 50, 54, 50, 54, 50, 54, 50, 54,
26757 50, 54, 50, 54, 50, 54, 50, 54, 50, 54,
26758 50, 2041, 54, 50, 54, 50, 1049, 54, 50, 8281,
26759 1562, 697, 90, 217, 346, 1513, 1509, 126, 73, 69,
26760 254, 105, 37, 94, 37, 94, 165, 70, 105, 37,
26761 3166, 37, 218, 158, 108, 94, 149, 47, 85, 1221,
26762 37, 37, 1799, 38, 53, 44, 743, 231, 231, 231,
26763 231, 231, 231, 231, 231, 1036, 85, 52, 51, 52,
26764 51, 117, 52, 51, 53, 52, 51, 309, 49, 85,
26765 49, 53, 52, 51, 85, 52, 51, 54, 50, 54,
26766 50, 54, 50, 54, 50, 181, 38, 341, 81, 858,
26767 2874, 6874, 410, 61, 117, 58, 38, 39, 46, 54,
26768 50, 54, 50, 54, 50, 54, 50, 54, 50, 90,
26769 54, 50, 54, 50, 54, 50, 54, 50, 49, 54,
26770 82, 58, 302, 140, 74, 49, 166, 90, 110, 38,
26771 39, 53, 90, 2759, 76, 88, 70, 39, 49, 2887,
26772 53, 102, 39, 1319, 3015, 90, 143, 346, 871, 1178,
26773 519, 1018, 335, 986, 271, 58, 495, 1050, 335, 1274,
26774 495, 2042, 8218, 39, 39, 2074, 39, 39, 679, 38,
26775 36583, 1786, 1287, 198, 85, 8583, 38, 117, 519, 333,
26776 71, 1502, 39, 44, 107, 53, 332, 53, 38, 798,
26777 44, 2247, 334, 76, 213, 760, 294, 88, 478, 69,
26778 2014, 38, 261, 190, 350, 38, 88, 158, 158, 382,
26779 70, 37, 231, 44, 103, 44, 135, 44, 743, 74,
26780 76, 42, 154, 207, 90, 55, 58, 1671, 149, 74,
26781 1607, 522, 44, 85, 333, 588, 199, 117, 39, 333,
26782 903, 268, 85, 743, 364, 74, 53, 935, 108, 42,
26783 1511, 44, 74, 140, 74, 44, 138, 437, 38, 333,
26784 85, 1319, 204, 74, 76, 74, 76, 103, 44, 263,
26785 44, 42, 333, 149, 519, 38, 199, 122, 39, 42,
26786 1543, 44, 39, 108, 71, 76, 167, 76, 39, 44,
26787 39, 71, 38, 85, 359, 42, 76, 74, 85, 39,
26788 70, 42, 44, 199, 199, 199, 231, 231, 1127, 74,
26789 44, 74, 44, 74, 53, 42, 44, 333, 39, 39,
26790 743, 1575, 36, 68, 68, 36, 63, 63, 11719, 3399,
26791 229, 165, 39, 44, 327, 57, 423, 167, 39, 71,
26792 71, 3463, 536, 11623, 54, 50, 2055, 1735, 391, 55,
26793 58, 524, 245, 54, 50, 53, 236, 53, 81, 80,
26794 54, 50, 54, 50, 54, 50, 54, 50, 54, 50,
26795 54, 50, 54, 50, 54, 50, 85, 54, 50, 149,
26796 112, 117, 149, 49, 54, 50, 54, 50, 54, 50,
26797 117, 57, 49, 121, 53, 55, 85, 167, 4327, 34,
26798 117, 55, 117, 54, 50, 53, 57, 53, 49, 85,
26799 333, 85, 121, 85, 841, 54, 53, 50, 56, 48,
26800 56, 837, 54, 57, 50, 57, 54, 50, 53, 54,
26801 50, 85, 327, 38, 1447, 70, 999, 199, 199, 199,
26802 103, 87, 57, 56, 58, 87, 58, 153, 90, 98,
26803 90, 391, 839, 615, 71, 487, 455, 3943, 117, 1455,
26804 314, 1710, 143, 570, 47, 410, 1466, 44, 935, 1575,
26805 999, 143, 551, 46, 263, 46, 967, 53, 1159, 263,
26806 53, 174, 1289, 1285, 2503, 333, 199, 39, 1415, 71,
26807 39, 743, 53, 271, 711, 207, 53, 839, 53, 1799,
26808 71, 39, 108, 76, 140, 135, 103, 871, 108, 44,
26809 271, 309, 935, 79, 53, 1735, 245, 711, 271, 615,
26810 271, 2343, 1007, 42, 44, 42, 1703, 492, 245, 655,
26811 333, 76, 42, 1447, 106, 140, 74, 76, 85, 34,
26812 149, 807, 333, 108, 1159, 172, 42, 268, 333, 149,
26813 76, 42, 1543, 106, 300, 74, 135, 149, 333, 1383,
26814 44, 42, 44, 74, 204, 42, 44, 333, 28135, 3182,
26815 149, 34279, 18215, 2215, 39, 1482, 140, 422, 71, 7898,
26816 1274, 1946, 74, 108, 122, 202, 258, 268, 90, 236,
26817 986, 140, 1562, 2138, 108, 58, 2810, 591, 841, 837,
26818 841, 229, 581, 841, 837, 41, 73, 41, 73, 137,
26819 265, 133, 37, 229, 357, 841, 837, 73, 137, 265,
26820 233, 837, 73, 137, 169, 41, 233, 837, 841, 837,
26821 841, 837, 841, 837, 841, 837, 841, 837, 841, 901,
26822 809, 57, 805, 57, 197, 809, 57, 805, 57, 197,
26823 809, 57, 805, 57, 197, 809, 57, 805, 57, 197,
26824 809, 57, 805, 57, 197, 94, 1613, 135, 871, 71,
26825 39, 39, 327, 135, 39, 39, 39, 39, 39, 39,
26826 103, 71, 39, 39, 39, 39, 39, 39, 71, 39,
26827 135, 231, 135, 135, 39, 327, 551, 103, 167, 551,
26828 89, 1434, 3226, 506, 474, 506, 506, 367, 1018, 1946,
26829 1402, 954, 1402, 314, 90, 1082, 218, 2266, 666, 1210,
26830 186, 570, 2042, 58, 5850, 154, 2010, 154, 794, 2266,
26831 378, 2266, 3738, 39, 39, 39, 39, 39, 39, 17351,
26832 34, 3074, 7692, 63, 63,
26833 };
26834
26835static int sqlite3Fts5UnicodeCategory(u32 iCode) {
26836 int iRes = -1;
26837 int iHi;
26838 int iLo;
26839 int ret;
26840 u16 iKey;
26841
26842 if( iCode>=(1<<20) ){
26843 return 0;
26844 }
26845 iLo = aFts5UnicodeBlock[(iCode>>16)];
26846 iHi = aFts5UnicodeBlock[1+(iCode>>16)];
26847 iKey = (iCode & 0xFFFF);
26848 while( iHi>iLo ){
26849 int iTest = (iHi + iLo) / 2;
26850 assert( iTest>=iLo && iTest<iHi )((void) (0));
26851 if( iKey>=aFts5UnicodeMap[iTest] ){
26852 iRes = iTest;
26853 iLo = iTest+1;
26854 }else{
26855 iHi = iTest;
26856 }
26857 }
26858
26859 if( iRes<0 ) return 0;
26860 if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
26861 ret = aFts5UnicodeData[iRes] & 0x1F;
26862 if( ret!=30 ) return ret;
26863 return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9;
26864}
26865
26866static void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
26867 int i = 0;
26868 int iTbl = 0;
26869 while( i<128 ){
26870 int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
26871 int n = (aFts5UnicodeData[iTbl] >> 5) + i;
26872 for(; i<128 && i<n; i++){
26873 aAscii[i] = (u8)bToken;
26874 }
26875 iTbl++;
26876 }
26877 aAscii[0] = 0; /* 0x00 is never a token character */
26878}
26879
26880#line 1 "fts5_varint.c"
26881/*
26882** 2015 May 30
26883**
26884** The author disclaims copyright to this source code. In place of
26885** a legal notice, here is a blessing:
26886**
26887** May you do good and not evil.
26888** May you find forgiveness for yourself and forgive others.
26889** May you share freely, never taking more than you give.
26890**
26891******************************************************************************
26892**
26893** Routines for varint serialization and deserialization.
26894*/
26895
26896
26897/* #include "fts5Int.h" */
26898
26899/*
26900** This is a copy of the sqlite3GetVarint32() routine from the SQLite core.
26901** Except, this version does handle the single byte case that the core
26902** version depends on being handled before its function is called.
26903*/
26904static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){
26905 u32 a,b;
26906
26907 /* The 1-byte case. Overwhelmingly the most common. */
26908 a = *p;
26909 /* a: p0 (unmasked) */
26910 if (!(a&0x80))
26911 {
26912 /* Values between 0 and 127 */
26913 *v = a;
26914 return 1;
26915 }
26916
26917 /* The 2-byte case */
26918 p++;
26919 b = *p;
26920 /* b: p1 (unmasked) */
26921 if (!(b&0x80))
26922 {
26923 /* Values between 128 and 16383 */
26924 a &= 0x7f;
26925 a = a<<7;
26926 *v = a | b;
26927 return 2;
26928 }
26929
26930 /* The 3-byte case */
26931 p++;
26932 a = a<<14;
26933 a |= *p;
26934 /* a: p0<<14 | p2 (unmasked) */
26935 if (!(a&0x80))
26936 {
26937 /* Values between 16384 and 2097151 */
26938 a &= (0x7f<<14)|(0x7f);
26939 b &= 0x7f;
26940 b = b<<7;
26941 *v = a | b;
26942 return 3;
26943 }
26944
26945 /* A 32-bit varint is used to store size information in btrees.
26946 ** Objects are rarely larger than 2MiB limit of a 3-byte varint.
26947 ** A 3-byte varint is sufficient, for example, to record the size
26948 ** of a 1048569-byte BLOB or string.
26949 **
26950 ** We only unroll the first 1-, 2-, and 3- byte cases. The very
26951 ** rare larger cases can be handled by the slower 64-bit varint
26952 ** routine.
26953 */
26954 {
26955 u64 v64;
26956 u8 n;
26957 p -= 2;
26958 n = sqlite3Fts5GetVarint(p, &v64);
26959 *v = ((u32)v64) & 0x7FFFFFFF;
26960 assert( n>3 && n<=9 )((void) (0));
26961 return n;
26962 }
26963}
26964
26965
26966/*
26967** Bitmasks used by sqlite3GetVarint(). These precomputed constants
26968** are defined here rather than simply putting the constant expressions
26969** inline in order to work around bugs in the RVT compiler.
26970**
26971** SLOT_2_0 A mask for (0x7f<<14) | 0x7f
26972**
26973** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0
26974*/
26975#define SLOT_2_00x001fc07f 0x001fc07f
26976#define SLOT_4_2_00xf01fc07f 0xf01fc07f
26977
26978/*
26979** Read a 64-bit variable-length integer from memory starting at p[0].
26980** Return the number of bytes read. The value is stored in *v.
26981*/
26982static u8 sqlite3Fts5GetVarint(const unsigned char *p, u64 *v){
26983 u32 a,b,s;
26984
26985 a = *p;
26986 /* a: p0 (unmasked) */
26987 if (!(a&0x80))
26988 {
26989 *v = a;
26990 return 1;
26991 }
26992
26993 p++;
26994 b = *p;
26995 /* b: p1 (unmasked) */
26996 if (!(b&0x80))
26997 {
26998 a &= 0x7f;
26999 a = a<<7;
27000 a |= b;
27001 *v = a;
27002 return 2;
27003 }
27004
27005 /* Verify that constants are precomputed correctly */
27006 assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) )((void) (0));
27007 assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) )((void) (0));
27008
27009 p++;
27010 a = a<<14;
27011 a |= *p;
27012 /* a: p0<<14 | p2 (unmasked) */
27013 if (!(a&0x80))
27014 {
27015 a &= SLOT_2_00x001fc07f;
27016 b &= 0x7f;
27017 b = b<<7;
27018 a |= b;
27019 *v = a;
27020 return 3;
27021 }
27022
27023 /* CSE1 from below */
27024 a &= SLOT_2_00x001fc07f;
27025 p++;
27026 b = b<<14;
27027 b |= *p;
27028 /* b: p1<<14 | p3 (unmasked) */
27029 if (!(b&0x80))
27030 {
27031 b &= SLOT_2_00x001fc07f;
27032 /* moved CSE1 up */
27033 /* a &= (0x7f<<14)|(0x7f); */
27034 a = a<<7;
27035 a |= b;
27036 *v = a;
27037 return 4;
27038 }
27039
27040 /* a: p0<<14 | p2 (masked) */
27041 /* b: p1<<14 | p3 (unmasked) */
27042 /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
27043 /* moved CSE1 up */
27044 /* a &= (0x7f<<14)|(0x7f); */
27045 b &= SLOT_2_00x001fc07f;
27046 s = a;
27047 /* s: p0<<14 | p2 (masked) */
27048
27049 p++;
27050 a = a<<14;
27051 a |= *p;
27052 /* a: p0<<28 | p2<<14 | p4 (unmasked) */
27053 if (!(a&0x80))
27054 {
27055 /* we can skip these cause they were (effectively) done above in calc'ing s */
27056 /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
27057 /* b &= (0x7f<<14)|(0x7f); */
27058 b = b<<7;
27059 a |= b;
27060 s = s>>18;
27061 *v = ((u64)s)<<32 | a;
27062 return 5;
27063 }
27064
27065 /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
27066 s = s<<7;
27067 s |= b;
27068 /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
27069
27070 p++;
27071 b = b<<14;
27072 b |= *p;
27073 /* b: p1<<28 | p3<<14 | p5 (unmasked) */
27074 if (!(b&0x80))
27075 {
27076 /* we can skip this cause it was (effectively) done above in calc'ing s */
27077 /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
27078 a &= SLOT_2_00x001fc07f;
27079 a = a<<7;
27080 a |= b;
27081 s = s>>18;
27082 *v = ((u64)s)<<32 | a;
27083 return 6;
27084 }
27085
27086 p++;
27087 a = a<<14;
27088 a |= *p;
27089 /* a: p2<<28 | p4<<14 | p6 (unmasked) */
27090 if (!(a&0x80))
27091 {
27092 a &= SLOT_4_2_00xf01fc07f;
27093 b &= SLOT_2_00x001fc07f;
27094 b = b<<7;
27095 a |= b;
27096 s = s>>11;
27097 *v = ((u64)s)<<32 | a;
27098 return 7;
27099 }
27100
27101 /* CSE2 from below */
27102 a &= SLOT_2_00x001fc07f;
27103 p++;
27104 b = b<<14;
27105 b |= *p;
27106 /* b: p3<<28 | p5<<14 | p7 (unmasked) */
27107 if (!(b&0x80))
27108 {
27109 b &= SLOT_4_2_00xf01fc07f;
27110 /* moved CSE2 up */
27111 /* a &= (0x7f<<14)|(0x7f); */
27112 a = a<<7;
27113 a |= b;
27114 s = s>>4;
27115 *v = ((u64)s)<<32 | a;
27116 return 8;
27117 }
27118
27119 p++;
27120 a = a<<15;
27121 a |= *p;
27122 /* a: p4<<29 | p6<<15 | p8 (unmasked) */
27123
27124 /* moved CSE2 up */
27125 /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */
27126 b &= SLOT_2_00x001fc07f;
27127 b = b<<8;
27128 a |= b;
27129
27130 s = s<<4;
27131 b = p[-4];
27132 b &= 0x7f;
27133 b = b>>3;
27134 s |= b;
27135
27136 *v = ((u64)s)<<32 | a;
27137
27138 return 9;
27139}
27140
27141/*
27142** The variable-length integer encoding is as follows:
27143**
27144** KEY:
27145** A = 0xxxxxxx 7 bits of data and one flag bit
27146** B = 1xxxxxxx 7 bits of data and one flag bit
27147** C = xxxxxxxx 8 bits of data
27148**
27149** 7 bits - A
27150** 14 bits - BA
27151** 21 bits - BBA
27152** 28 bits - BBBA
27153** 35 bits - BBBBA
27154** 42 bits - BBBBBA
27155** 49 bits - BBBBBBA
27156** 56 bits - BBBBBBBA
27157** 64 bits - BBBBBBBBC
27158*/
27159
27160#ifdef SQLITE_NOINLINE
27161# define FTS5_NOINLINE SQLITE_NOINLINE
27162#else
27163# define FTS5_NOINLINE
27164#endif
27165
27166/*
27167** Write a 64-bit variable-length integer to memory starting at p[0].
27168** The length of data write will be between 1 and 9 bytes. The number
27169** of bytes written is returned.
27170**
27171** A variable-length integer consists of the lower 7 bits of each byte
27172** for all bytes that have the 8th bit set and one byte with the 8th
27173** bit clear. Except, if we get to the 9th byte, it stores the full
27174** 8 bits and is the last byte.
27175*/
27176static int FTS5_NOINLINE fts5PutVarint64(unsigned char *p, u64 v){
27177 int i, j, n;
27178 u8 buf[10];
27179 if( v & (((u64)0xff000000)<<32) ){
27180 p[8] = (u8)v;
27181 v >>= 8;
27182 for(i=7; i>=0; i--){
27183 p[i] = (u8)((v & 0x7f) | 0x80);
27184 v >>= 7;
27185 }
27186 return 9;
27187 }
27188 n = 0;
27189 do{
27190 buf[n++] = (u8)((v & 0x7f) | 0x80);
27191 v >>= 7;
27192 }while( v!=0 );
27193 buf[0] &= 0x7f;
27194 assert( n<=9 )((void) (0));
27195 for(i=0, j=n-1; j>=0; j--, i++){
27196 p[i] = buf[j];
27197 }
27198 return n;
27199}
27200
27201static int sqlite3Fts5PutVarint(unsigned char *p, u64 v){
27202 if( v<=0x7f ){
27203 p[0] = v&0x7f;
27204 return 1;
27205 }
27206 if( v<=0x3fff ){
27207 p[0] = ((v>>7)&0x7f)|0x80;
27208 p[1] = v&0x7f;
27209 return 2;
27210 }
27211 return fts5PutVarint64(p,v);
27212}
27213
27214
27215static int sqlite3Fts5GetVarintLen(u32 iVal){
27216#if 0
27217 if( iVal<(1 << 7 ) ) return 1;
27218#endif
27219 assert( iVal>=(1 << 7) )((void) (0));
27220 if( iVal<(1 << 14) ) return 2;
27221 if( iVal<(1 << 21) ) return 3;
27222 if( iVal<(1 << 28) ) return 4;
27223 return 5;
27224}
27225
27226#line 1 "fts5_vocab.c"
27227/*
27228** 2015 May 08
27229**
27230** The author disclaims copyright to this source code. In place of
27231** a legal notice, here is a blessing:
27232**
27233** May you do good and not evil.
27234** May you find forgiveness for yourself and forgive others.
27235** May you share freely, never taking more than you give.
27236**
27237******************************************************************************
27238**
27239** This is an SQLite virtual table module implementing direct access to an
27240** existing FTS5 index. The module may create several different types of
27241** tables:
27242**
27243** col:
27244** CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col));
27245**
27246** One row for each term/column combination. The value of $doc is set to
27247** the number of fts5 rows that contain at least one instance of term
27248** $term within column $col. Field $cnt is set to the total number of
27249** instances of term $term in column $col (in any row of the fts5 table).
27250**
27251** row:
27252** CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term));
27253**
27254** One row for each term in the database. The value of $doc is set to
27255** the number of fts5 rows that contain at least one instance of term
27256** $term. Field $cnt is set to the total number of instances of term
27257** $term in the database.
27258**
27259** instance:
27260** CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY(<all-fields>));
27261**
27262** One row for each term instance in the database.
27263*/
27264
27265
27266/* #include "fts5Int.h" */
27267
27268
27269typedef struct Fts5VocabTable Fts5VocabTable;
27270typedef struct Fts5VocabCursor Fts5VocabCursor;
27271
27272struct Fts5VocabTable {
27273 sqlite3_vtab base;
27274 char *zFts5Tbl; /* Name of fts5 table */
27275 char *zFts5Db; /* Db containing fts5 table */
27276 sqlite3 *db; /* Database handle */
27277 Fts5Global *pGlobal; /* FTS5 global object for this database */
27278 int eType; /* FTS5_VOCAB_COL, ROW or INSTANCE */
27279 unsigned bBusy; /* True if busy */
27280};
27281
27282struct Fts5VocabCursor {
27283 sqlite3_vtab_cursor base;
27284 sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */
27285 Fts5Table *pFts5; /* Associated FTS5 table */
27286
27287 int bEof; /* True if this cursor is at EOF */
27288 Fts5IndexIter *pIter; /* Term/rowid iterator object */
27289 void *pStruct; /* From sqlite3Fts5StructureRef() */
27290
27291 int nLeTerm; /* Size of zLeTerm in bytes */
27292 char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */
27293 int colUsed; /* Copy of sqlite3_index_info.colUsed */
27294
27295 /* These are used by 'col' tables only */
27296 int iCol;
27297 i64 *aCnt;
27298 i64 *aDoc;
27299
27300 /* Output values used by all tables. */
27301 i64 rowid; /* This table's current rowid value */
27302 Fts5Buffer term; /* Current value of 'term' column */
27303
27304 /* Output values Used by 'instance' tables only */
27305 i64 iInstPos;
27306 int iInstOff;
27307};
27308
27309#define FTS5_VOCAB_COL0 0
27310#define FTS5_VOCAB_ROW1 1
27311#define FTS5_VOCAB_INSTANCE2 2
27312
27313#define FTS5_VOCAB_COL_SCHEMA"term, col, doc, cnt" "term, col, doc, cnt"
27314#define FTS5_VOCAB_ROW_SCHEMA"term, doc, cnt" "term, doc, cnt"
27315#define FTS5_VOCAB_INST_SCHEMA"term, doc, col, offset" "term, doc, col, offset"
27316
27317/*
27318** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
27319*/
27320#define FTS5_VOCAB_TERM_EQ0x0100 0x0100
27321#define FTS5_VOCAB_TERM_GE0x0200 0x0200
27322#define FTS5_VOCAB_TERM_LE0x0400 0x0400
27323
27324#define FTS5_VOCAB_COLUSED_MASK0xFF 0xFF
27325
27326
27327/*
27328** Translate a string containing an fts5vocab table type to an
27329** FTS5_VOCAB_XXX constant. If successful, set *peType to the output
27330** value and return SQLITE_OK. Otherwise, set *pzErr to an error message
27331** and return SQLITE_ERROR.
27332*/
27333static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){
27334 int rc = SQLITE_OK0;
27335 char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1);
27336 if( rc==SQLITE_OK0 ){
27337 sqlite3Fts5Dequote(zCopy);
27338 if( sqlite3_stricmpsqlite3_api->stricmp(zCopy, "col")==0 ){
27339 *peType = FTS5_VOCAB_COL0;
27340 }else
27341
27342 if( sqlite3_stricmpsqlite3_api->stricmp(zCopy, "row")==0 ){
27343 *peType = FTS5_VOCAB_ROW1;
27344 }else
27345 if( sqlite3_stricmpsqlite3_api->stricmp(zCopy, "instance")==0 ){
27346 *peType = FTS5_VOCAB_INSTANCE2;
27347 }else
27348 {
27349 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("fts5vocab: unknown table type: %Q", zCopy);
27350 rc = SQLITE_ERROR1;
27351 }
27352 sqlite3_freesqlite3_api->free(zCopy);
27353 }
27354
27355 return rc;
27356}
27357
27358
27359/*
27360** The xDisconnect() virtual table method.
27361*/
27362static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){
27363 Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
27364 sqlite3_freesqlite3_api->free(pTab);
27365 return SQLITE_OK0;
27366}
27367
27368/*
27369** The xDestroy() virtual table method.
27370*/
27371static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){
27372 Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
27373 sqlite3_freesqlite3_api->free(pTab);
27374 return SQLITE_OK0;
27375}
27376
27377/*
27378** This function is the implementation of both the xConnect and xCreate
27379** methods of the FTS3 virtual table.
27380**
27381** The argv[] array contains the following:
27382**
27383** argv[0] -> module name ("fts5vocab")
27384** argv[1] -> database name
27385** argv[2] -> table name
27386**
27387** then:
27388**
27389** argv[3] -> name of fts5 table
27390** argv[4] -> type of fts5vocab table
27391**
27392** or, for tables in the TEMP schema only.
27393**
27394** argv[3] -> name of fts5 tables database
27395** argv[4] -> name of fts5 table
27396** argv[5] -> type of fts5vocab table
27397*/
27398static int fts5VocabInitVtab(
27399 sqlite3 *db, /* The SQLite database connection */
27400 void *pAux, /* Pointer to Fts5Global object */
27401 int argc, /* Number of elements in argv array */
27402 const char * const *argv, /* xCreate/xConnect argument array */
27403 sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
27404 char **pzErr /* Write any error message here */
27405){
27406 const char *azSchema[] = {
27407 "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA"term, col, doc, cnt" ")",
27408 "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA"term, doc, cnt" ")",
27409 "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA"term, doc, col, offset" ")"
27410 };
27411
27412 Fts5VocabTable *pRet = 0;
27413 int rc = SQLITE_OK0; /* Return code */
27414 int bDb;
27415
27416 bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0);
27417
27418 if( argc!=5 && bDb==0 ){
27419 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("wrong number of vtable arguments");
27420 rc = SQLITE_ERROR1;
27421 }else{
27422 i64 nByte; /* Bytes of space to allocate */
27423 const char *zDb = bDb ? argv[3] : argv[1];
27424 const char *zTab = bDb ? argv[4] : argv[3];
27425 const char *zType = bDb ? argv[5] : argv[4];
27426 i64 nDb = strlen(zDb)+1;
27427 i64 nTab = strlen(zTab)+1;
27428 int eType = 0;
27429
27430 rc = fts5VocabTableType(zType, pzErr, &eType);
27431 if( rc==SQLITE_OK0 ){
27432 assert( eType>=0 && eType<ArraySize(azSchema) )((void) (0));
27433 rc = sqlite3_declare_vtabsqlite3_api->declare_vtab(db, azSchema[eType]);
27434 }
27435
27436 nByte = sizeof(Fts5VocabTable) + nDb + nTab;
27437 pRet = sqlite3Fts5MallocZero(&rc, nByte);
27438 if( pRet ){
27439 pRet->pGlobal = (Fts5Global*)pAux;
27440 pRet->eType = eType;
27441 pRet->db = db;
27442 pRet->zFts5Tbl = (char*)&pRet[1];
27443 pRet->zFts5Db = &pRet->zFts5Tbl[nTab];
27444 memcpy(pRet->zFts5Tbl, zTab, nTab);
27445 memcpy(pRet->zFts5Db, zDb, nDb);
27446 sqlite3Fts5Dequote(pRet->zFts5Tbl);
27447 sqlite3Fts5Dequote(pRet->zFts5Db);
27448 }
27449 }
27450
27451 *ppVTab = (sqlite3_vtab*)pRet;
27452 return rc;
27453}
27454
27455
27456/*
27457** The xConnect() and xCreate() methods for the virtual table. All the
27458** work is done in function fts5VocabInitVtab().
27459*/
27460static int fts5VocabConnectMethod(
27461 sqlite3 *db, /* Database connection */
27462 void *pAux, /* Pointer to tokenizer hash table */
27463 int argc, /* Number of elements in argv array */
27464 const char * const *argv, /* xCreate/xConnect argument array */
27465 sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
27466 char **pzErr /* OUT: sqlite3_malloc'd error message */
27467){
27468 return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
27469}
27470static int fts5VocabCreateMethod(
27471 sqlite3 *db, /* Database connection */
27472 void *pAux, /* Pointer to tokenizer hash table */
27473 int argc, /* Number of elements in argv array */
27474 const char * const *argv, /* xCreate/xConnect argument array */
27475 sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
27476 char **pzErr /* OUT: sqlite3_malloc'd error message */
27477){
27478 return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
27479}
27480
27481/*
27482** Implementation of the xBestIndex method.
27483**
27484** Only constraints of the form:
27485**
27486** term <= ?
27487** term == ?
27488** term >= ?
27489**
27490** are interpreted. Less-than and less-than-or-equal are treated
27491** identically, as are greater-than and greater-than-or-equal.
27492*/
27493static int fts5VocabBestIndexMethod(
27494 sqlite3_vtab *pUnused,
27495 sqlite3_index_info *pInfo
27496){
27497 int i;
27498 int iTermEq = -1;
27499 int iTermGe = -1;
27500 int iTermLe = -1;
27501 int idxNum = (int)pInfo->colUsed;
27502 int nArg = 0;
27503
27504 UNUSED_PARAM(pUnused)(void)(pUnused);
27505
27506 assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed )((void) (0));
27507
27508 for(i=0; i<pInfo->nConstraint; i++){
27509 struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
27510 if( p->usable==0 ) continue;
27511 if( p->iColumn==0 ){ /* term column */
27512 if( p->op==SQLITE_INDEX_CONSTRAINT_EQ2 ) iTermEq = i;
27513 if( p->op==SQLITE_INDEX_CONSTRAINT_LE8 ) iTermLe = i;
27514 if( p->op==SQLITE_INDEX_CONSTRAINT_LT16 ) iTermLe = i;
27515 if( p->op==SQLITE_INDEX_CONSTRAINT_GE32 ) iTermGe = i;
27516 if( p->op==SQLITE_INDEX_CONSTRAINT_GT4 ) iTermGe = i;
27517 }
27518 }
27519
27520 if( iTermEq>=0 ){
27521 idxNum |= FTS5_VOCAB_TERM_EQ0x0100;
27522 pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg;
27523 pInfo->estimatedCost = 100;
27524 }else{
27525 pInfo->estimatedCost = 1000000;
27526 if( iTermGe>=0 ){
27527 idxNum |= FTS5_VOCAB_TERM_GE0x0200;
27528 pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg;
27529 pInfo->estimatedCost = pInfo->estimatedCost / 2;
27530 }
27531 if( iTermLe>=0 ){
27532 idxNum |= FTS5_VOCAB_TERM_LE0x0400;
27533 pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg;
27534 pInfo->estimatedCost = pInfo->estimatedCost / 2;
27535 }
27536 }
27537
27538 /* This virtual table always delivers results in ascending order of
27539 ** the "term" column (column 0). So if the user has requested this
27540 ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the
27541 ** sqlite3_index_info.orderByConsumed flag to tell the core the results
27542 ** are already in sorted order. */
27543 if( pInfo->nOrderBy==1
27544 && pInfo->aOrderBy[0].iColumn==0
27545 && pInfo->aOrderBy[0].desc==0
27546 ){
27547 pInfo->orderByConsumed = 1;
27548 }
27549
27550 pInfo->idxNum = idxNum;
27551 return SQLITE_OK0;
27552}
27553
27554/*
27555** Implementation of xOpen method.
27556*/
27557static int fts5VocabOpenMethod(
27558 sqlite3_vtab *pVTab,
27559 sqlite3_vtab_cursor **ppCsr
27560){
27561 Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;
27562 Fts5Table *pFts5 = 0;
27563 Fts5VocabCursor *pCsr = 0;
27564 int rc = SQLITE_OK0;
27565 sqlite3_stmt *pStmt = 0;
27566 char *zSql = 0;
27567
27568 if( pTab->bBusy ){
27569 pVTab->zErrMsg = sqlite3_mprintfsqlite3_api->mprintf(
27570 "recursive definition for %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
27571 );
27572 return SQLITE_ERROR1;
27573 }
27574 zSql = sqlite3Fts5Mprintf(&rc,
27575 "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
27576 pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl
27577 );
27578 if( zSql ){
27579 rc = sqlite3_prepare_v2sqlite3_api->prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
27580 }
27581 sqlite3_freesqlite3_api->free(zSql);
27582 assert( rc==SQLITE_OK || pStmt==0 )((void) (0));
27583 if( rc==SQLITE_ERROR1 ) rc = SQLITE_OK0;
27584
27585 pTab->bBusy = 1;
27586 if( pStmt && sqlite3_stepsqlite3_api->step(pStmt)==SQLITE_ROW100 ){
27587 i64 iId = sqlite3_column_int64sqlite3_api->column_int64(pStmt, 0);
27588 pFts5 = sqlite3Fts5TableFromCsrid(pTab->pGlobal, iId);
27589 }
27590 pTab->bBusy = 0;
27591
27592 if( rc==SQLITE_OK0 ){
27593 if( pFts5==0 ){
27594 rc = sqlite3_finalizesqlite3_api->finalize(pStmt);
27595 pStmt = 0;
27596 if( rc==SQLITE_OK0 ){
27597 pVTab->zErrMsg = sqlite3_mprintfsqlite3_api->mprintf(
27598 "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
27599 );
27600 rc = SQLITE_ERROR1;
27601 }
27602 }else{
27603 rc = sqlite3Fts5FlushToDisk(pFts5);
27604 }
27605 }
27606
27607 if( rc==SQLITE_OK0 ){
27608 i64 nByte = pFts5->pConfig->nCol * sizeof(i64)*2 + sizeof(Fts5VocabCursor);
27609 pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
27610 }
27611
27612 if( pCsr ){
27613 pCsr->pFts5 = pFts5;
27614 pCsr->pStmt = pStmt;
27615 pCsr->aCnt = (i64*)&pCsr[1];
27616 pCsr->aDoc = &pCsr->aCnt[pFts5->pConfig->nCol];
27617 }else{
27618 sqlite3_finalizesqlite3_api->finalize(pStmt);
27619 }
27620
27621 *ppCsr = (sqlite3_vtab_cursor*)pCsr;
27622 return rc;
27623}
27624
27625/*
27626** Restore cursor pCsr to the state it was in immediately after being
27627** created by the xOpen() method.
27628*/
27629static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){
27630 int nCol = pCsr->pFts5->pConfig->nCol;
27631 pCsr->rowid = 0;
27632 sqlite3Fts5IterClose(pCsr->pIter);
27633 sqlite3Fts5StructureRelease(pCsr->pStruct);
27634 pCsr->pStruct = 0;
27635 pCsr->pIter = 0;
27636 sqlite3_freesqlite3_api->free(pCsr->zLeTerm);
27637 pCsr->nLeTerm = -1;
27638 pCsr->zLeTerm = 0;
27639 pCsr->bEof = 0;
27640 pCsr->iCol = 0;
27641 pCsr->iInstPos = 0;
27642 pCsr->iInstOff = 0;
27643 pCsr->colUsed = 0;
27644 memset(pCsr->aCnt, 0, sizeof(i64)*nCol);
27645 memset(pCsr->aDoc, 0, sizeof(i64)*nCol);
27646}
27647
27648/*
27649** Close the cursor. For additional information see the documentation
27650** on the xClose method of the virtual table interface.
27651*/
27652static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){
27653 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
27654 fts5VocabResetCursor(pCsr);
27655 sqlite3Fts5BufferFree(&pCsr->term);
27656 sqlite3_finalizesqlite3_api->finalize(pCsr->pStmt);
27657 sqlite3_freesqlite3_api->free(pCsr);
27658 return SQLITE_OK0;
27659}
27660
27661static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){
27662 int rc = SQLITE_OK0;
27663
27664 if( sqlite3Fts5IterEof(pCsr->pIter)((pCsr->pIter)->bEof) ){
27665 pCsr->bEof = 1;
27666 }else{
27667 const char *zTerm;
27668 int nTerm;
27669 zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
27670 if( pCsr->nLeTerm>=0 ){
27671 int nCmp = MIN(nTerm, pCsr->nLeTerm)(((nTerm) < (pCsr->nLeTerm)) ? (nTerm) : (pCsr->nLeTerm
));
27672 int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
27673 if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
27674 pCsr->bEof = 1;
27675 }
27676 }
27677
27678 sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
27679 }
27680 return rc;
27681}
27682
27683static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
27684 int eDetail = pCsr->pFts5->pConfig->eDetail;
27685 int rc = SQLITE_OK0;
27686 Fts5IndexIter *pIter = pCsr->pIter;
27687 i64 *pp = &pCsr->iInstPos;
27688 int *po = &pCsr->iInstOff;
27689
27690 assert( sqlite3Fts5IterEof(pIter)==0 )((void) (0));
27691 assert( pCsr->bEof==0 )((void) (0));
27692 while( eDetail==FTS5_DETAIL_NONE1
27693 || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp)
27694 ){
27695 pCsr->iInstPos = 0;
27696 pCsr->iInstOff = 0;
27697
27698 rc = sqlite3Fts5IterNextScan(pCsr->pIter);
27699 if( rc==SQLITE_OK0 ){
27700 rc = fts5VocabInstanceNewTerm(pCsr);
27701 if( pCsr->bEof || eDetail==FTS5_DETAIL_NONE1 ) break;
27702 }
27703 if( rc ){
27704 pCsr->bEof = 1;
27705 break;
27706 }
27707 }
27708
27709 return rc;
27710}
27711
27712/*
27713** Advance the cursor to the next row in the table.
27714*/
27715static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
27716 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
27717 Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
27718 int nCol = pCsr->pFts5->pConfig->nCol;
27719 int rc;
27720
27721 rc = sqlite3Fts5StructureTest(pCsr->pFts5->pIndex, pCsr->pStruct);
27722 if( rc!=SQLITE_OK0 ) return rc;
27723 pCsr->rowid++;
27724
27725 if( pTab->eType==FTS5_VOCAB_INSTANCE2 ){
27726 return fts5VocabInstanceNext(pCsr);
27727 }
27728
27729 if( pTab->eType==FTS5_VOCAB_COL0 ){
27730 for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
27731 if( pCsr->aDoc[pCsr->iCol] ) break;
27732 }
27733 }
27734
27735 if( pTab->eType!=FTS5_VOCAB_COL0 || pCsr->iCol>=nCol ){
27736 if( sqlite3Fts5IterEof(pCsr->pIter)((pCsr->pIter)->bEof) ){
27737 pCsr->bEof = 1;
27738 }else{
27739 const char *zTerm;
27740 int nTerm;
27741
27742 zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
27743 assert( nTerm>=0 )((void) (0));
27744 if( pCsr->nLeTerm>=0 ){
27745 int nCmp = MIN(nTerm, pCsr->nLeTerm)(((nTerm) < (pCsr->nLeTerm)) ? (nTerm) : (pCsr->nLeTerm
));
27746 int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
27747 if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
27748 pCsr->bEof = 1;
27749 return SQLITE_OK0;
27750 }
27751 }
27752
27753 sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
27754 memset(pCsr->aCnt, 0, nCol * sizeof(i64));
27755 memset(pCsr->aDoc, 0, nCol * sizeof(i64));
27756 pCsr->iCol = 0;
27757
27758 assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW )((void) (0));
27759 while( rc==SQLITE_OK0 ){
27760 int eDetail = pCsr->pFts5->pConfig->eDetail;
27761 const u8 *pPos; int nPos; /* Position list */
27762 i64 iPos = 0; /* 64-bit position read from poslist */
27763 int iOff = 0; /* Current offset within position list */
27764
27765 pPos = pCsr->pIter->pData;
27766 nPos = pCsr->pIter->nData;
27767
27768 switch( pTab->eType ){
27769 case FTS5_VOCAB_ROW1:
27770 /* Do not bother counting the number of instances if the "cnt"
27771 ** column is not being read (according to colUsed). */
27772 if( eDetail==FTS5_DETAIL_FULL0 && (pCsr->colUsed & 0x04) ){
27773 while( iPos<nPos ){
27774 u32 ii;
27775 fts5FastGetVarint32(pPos, iPos, ii){ ii = (pPos)[iPos++]; if( ii & 0x80 ){ iPos--; iPos += sqlite3Fts5GetVarint32
(&(pPos)[iPos],(u32*)&(ii)); } };
27776 if( ii==1 ){
27777 /* New column in the position list */
27778 fts5FastGetVarint32(pPos, iPos, ii){ ii = (pPos)[iPos++]; if( ii & 0x80 ){ iPos--; iPos += sqlite3Fts5GetVarint32
(&(pPos)[iPos],(u32*)&(ii)); } };
27779 }else{
27780 /* An instance - increment pCsr->aCnt[] */
27781 pCsr->aCnt[0]++;
27782 }
27783 }
27784 }
27785 pCsr->aDoc[0]++;
27786 break;
27787
27788 case FTS5_VOCAB_COL0:
27789 if( eDetail==FTS5_DETAIL_FULL0 ){
27790 int iCol = -1;
27791 while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
27792 int ii = FTS5_POS2COLUMN(iPos)(int)((iPos >> 32) & 0x7FFFFFFF);
27793 if( iCol!=ii ){
27794 if( ii>=nCol ){
27795 rc = FTS5_CORRUPT(11 | (1<<8));
27796 break;
27797 }
27798 pCsr->aDoc[ii]++;
27799 iCol = ii;
27800 }
27801 pCsr->aCnt[ii]++;
27802 }
27803 }else if( eDetail==FTS5_DETAIL_COLUMNS2 ){
27804 while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
27805 assert_nc( iPos>=0 && iPos<nCol )((void) (0));
27806 if( iPos>=nCol ){
27807 rc = FTS5_CORRUPT(11 | (1<<8));
27808 break;
27809 }
27810 pCsr->aDoc[iPos]++;
27811 }
27812 }else{
27813 assert( eDetail==FTS5_DETAIL_NONE )((void) (0));
27814 pCsr->aDoc[0]++;
27815 }
27816 break;
27817
27818 default:
27819 assert( pTab->eType==FTS5_VOCAB_INSTANCE )((void) (0));
27820 break;
27821 }
27822
27823 if( rc==SQLITE_OK0 ){
27824 rc = sqlite3Fts5IterNextScan(pCsr->pIter);
27825 }
27826 if( pTab->eType==FTS5_VOCAB_INSTANCE2 ) break;
27827
27828 if( rc==SQLITE_OK0 ){
27829 zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
27830 if( nTerm!=pCsr->term.n
27831 || (nTerm>0 && memcmp(zTerm, pCsr->term.p, nTerm))
27832 ){
27833 break;
27834 }
27835 if( sqlite3Fts5IterEof(pCsr->pIter)((pCsr->pIter)->bEof) ) break;
27836 }
27837 }
27838 }
27839 }
27840
27841 if( rc==SQLITE_OK0 && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL0 ){
27842 for(/* noop */; pCsr->iCol<nCol && pCsr->aDoc[pCsr->iCol]==0; pCsr->iCol++);
27843 if( pCsr->iCol==nCol ){
27844 rc = FTS5_CORRUPT(11 | (1<<8));
27845 }
27846 }
27847 return rc;
27848}
27849
27850/*
27851** This is the xFilter implementation for the virtual table.
27852*/
27853static int fts5VocabFilterMethod(
27854 sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
27855 int idxNum, /* Strategy index */
27856 const char *zUnused, /* Unused */
27857 int nUnused, /* Number of elements in apVal */
27858 sqlite3_value **apVal /* Arguments for the indexing scheme */
27859){
27860 Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
27861 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
27862 int eType = pTab->eType;
27863 int rc = SQLITE_OK0;
27864
27865 int iVal = 0;
27866 int f = FTS5INDEX_QUERY_SCAN0x0008;
27867 const char *zTerm = 0;
27868 int nTerm = 0;
27869
27870 sqlite3_value *pEq = 0;
27871 sqlite3_value *pGe = 0;
27872 sqlite3_value *pLe = 0;
27873
27874 UNUSED_PARAM2(zUnused, nUnused)(void)(zUnused), (void)(nUnused);
27875
27876 fts5VocabResetCursor(pCsr);
27877 if( idxNum & FTS5_VOCAB_TERM_EQ0x0100 ) pEq = apVal[iVal++];
27878 if( idxNum & FTS5_VOCAB_TERM_GE0x0200 ) pGe = apVal[iVal++];
27879 if( idxNum & FTS5_VOCAB_TERM_LE0x0400 ) pLe = apVal[iVal++];
27880 pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK0xFF);
27881
27882 if( pEq ){
27883 zTerm = (const char *)sqlite3_value_textsqlite3_api->value_text(pEq);
27884 nTerm = sqlite3_value_bytessqlite3_api->value_bytes(pEq);
27885 f = FTS5INDEX_QUERY_NOTOKENDATA0x0080;
27886 }else{
27887 if( pGe ){
27888 zTerm = (const char *)sqlite3_value_textsqlite3_api->value_text(pGe);
27889 nTerm = sqlite3_value_bytessqlite3_api->value_bytes(pGe);
27890 }
27891 if( pLe ){
27892 const char *zCopy = (const char *)sqlite3_value_textsqlite3_api->value_text(pLe);
27893 if( zCopy==0 ) zCopy = "";
27894 pCsr->nLeTerm = sqlite3_value_bytessqlite3_api->value_bytes(pLe);
27895 pCsr->zLeTerm = sqlite3_malloc64sqlite3_api->malloc64((i64)pCsr->nLeTerm+1);
27896 if( pCsr->zLeTerm==0 ){
27897 rc = SQLITE_NOMEM7;
27898 }else{
27899 memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1);
27900 }
27901 }
27902 }
27903
27904 if( rc==SQLITE_OK0 ){
27905 Fts5Index *pIndex = pCsr->pFts5->pIndex;
27906 rc = sqlite3Fts5IndexQuery(pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
27907 if( rc==SQLITE_OK0 ){
27908 pCsr->pStruct = sqlite3Fts5StructureRef(pIndex);
27909 }
27910 }
27911 if( rc==SQLITE_OK0 && eType==FTS5_VOCAB_INSTANCE2 ){
27912 rc = fts5VocabInstanceNewTerm(pCsr);
27913 }
27914 if( rc==SQLITE_OK0 && !pCsr->bEof
27915 && (eType!=FTS5_VOCAB_INSTANCE2
27916 || pCsr->pFts5->pConfig->eDetail!=FTS5_DETAIL_NONE1)
27917 ){
27918 rc = fts5VocabNextMethod(pCursor);
27919 }
27920
27921 return rc;
27922}
27923
27924/*
27925** This is the xEof method of the virtual table. SQLite calls this
27926** routine to find out if it has reached the end of a result set.
27927*/
27928static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){
27929 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
27930 return pCsr->bEof;
27931}
27932
27933static int fts5VocabColumnMethod(
27934 sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
27935 sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
27936 int iCol /* Index of column to read value from */
27937){
27938 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
27939 int eDetail = pCsr->pFts5->pConfig->eDetail;
27940 int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType;
27941 i64 iVal = 0;
27942
27943 if( iCol==0 ){
27944 sqlite3_result_textsqlite3_api->result_text(
27945 pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT((sqlite3_destructor_type)-1)
27946 );
27947 }else if( eType==FTS5_VOCAB_COL0 ){
27948 assert( iCol==1 || iCol==2 || iCol==3 )((void) (0));
27949 if( iCol==1 ){
27950 if( eDetail!=FTS5_DETAIL_NONE1 ){
27951 const char *z = pCsr->pFts5->pConfig->azCol[pCsr->iCol];
27952 sqlite3_result_textsqlite3_api->result_text(pCtx, z, -1, SQLITE_STATIC((sqlite3_destructor_type)0));
27953 }
27954 }else if( iCol==2 ){
27955 iVal = pCsr->aDoc[pCsr->iCol];
27956 }else{
27957 iVal = pCsr->aCnt[pCsr->iCol];
27958 }
27959 }else if( eType==FTS5_VOCAB_ROW1 ){
27960 assert( iCol==1 || iCol==2 )((void) (0));
27961 if( iCol==1 ){
27962 iVal = pCsr->aDoc[0];
27963 }else{
27964 iVal = pCsr->aCnt[0];
27965 }
27966 }else{
27967 assert( eType==FTS5_VOCAB_INSTANCE )((void) (0));
27968 switch( iCol ){
27969 case 1:
27970 sqlite3_result_int64sqlite3_api->result_int64(pCtx, pCsr->pIter->iRowid);
27971 break;
27972 case 2: {
27973 int ii = -1;
27974 if( eDetail==FTS5_DETAIL_FULL0 ){
27975 ii = FTS5_POS2COLUMN(pCsr->iInstPos)(int)((pCsr->iInstPos >> 32) & 0x7FFFFFFF);
27976 }else if( eDetail==FTS5_DETAIL_COLUMNS2 ){
27977 ii = (int)pCsr->iInstPos;
27978 }
27979 if( ii>=0 && ii<pCsr->pFts5->pConfig->nCol ){
27980 const char *z = pCsr->pFts5->pConfig->azCol[ii];
27981 sqlite3_result_textsqlite3_api->result_text(pCtx, z, -1, SQLITE_STATIC((sqlite3_destructor_type)0));
27982 }
27983 break;
27984 }
27985 default: {
27986 assert( iCol==3 )((void) (0));
27987 if( eDetail==FTS5_DETAIL_FULL0 ){
27988 int ii = FTS5_POS2OFFSET(pCsr->iInstPos)(int)(pCsr->iInstPos & 0x7FFFFFFF);
27989 sqlite3_result_intsqlite3_api->result_int(pCtx, ii);
27990 }
27991 break;
27992 }
27993 }
27994 }
27995
27996 if( iVal>0 ) sqlite3_result_int64sqlite3_api->result_int64(pCtx, iVal);
27997 return SQLITE_OK0;
27998}
27999
28000/*
28001** This is the xRowid method. The SQLite core calls this routine to
28002** retrieve the rowid for the current row of the result set. The
28003** rowid should be written to *pRowid.
28004*/
28005static int fts5VocabRowidMethod(
28006 sqlite3_vtab_cursor *pCursor,
28007 sqlite_int64 *pRowid
28008){
28009 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
28010 *pRowid = pCsr->rowid;
28011 return SQLITE_OK0;
28012}
28013
28014static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
28015 static const sqlite3_module fts5Vocab = {
28016 /* iVersion */ 2,
28017 /* xCreate */ fts5VocabCreateMethod,
28018 /* xConnect */ fts5VocabConnectMethod,
28019 /* xBestIndex */ fts5VocabBestIndexMethod,
28020 /* xDisconnect */ fts5VocabDisconnectMethod,
28021 /* xDestroy */ fts5VocabDestroyMethod,
28022 /* xOpen */ fts5VocabOpenMethod,
28023 /* xClose */ fts5VocabCloseMethod,
28024 /* xFilter */ fts5VocabFilterMethod,
28025 /* xNext */ fts5VocabNextMethod,
28026 /* xEof */ fts5VocabEofMethod,
28027 /* xColumn */ fts5VocabColumnMethod,
28028 /* xRowid */ fts5VocabRowidMethod,
28029 /* xUpdate */ 0,
28030 /* xBegin */ 0,
28031 /* xSync */ 0,
28032 /* xCommit */ 0,
28033 /* xRollback */ 0,
28034 /* xFindFunction */ 0,
28035 /* xRename */ 0,
28036 /* xSavepoint */ 0,
28037 /* xRelease */ 0,
28038 /* xRollbackTo */ 0,
28039 /* xShadowName */ 0,
28040 /* xIntegrity */ 0
28041 };
28042 void *p = (void*)pGlobal;
28043
28044 return sqlite3_create_module_v2sqlite3_api->create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
28045}
28046
28047
28048/* Here ends the fts5.c composite file. */
28049#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */