Bug Summary

File:root/firefox-clang/third_party/sqlite3/ext/fts5.c
Warning:line 27683, column 7
Access to field 'bEof' results in a dereference of a null pointer (loaded from field 'pIter')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O2 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name fts5.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/sqlite3/ext -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/sqlite3/ext -resource-dir /usr/lib/llvm-23/lib/clang/23 -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG=1 -D SQLITE_VEC_ENABLE_DISKANN=0 -I /root/firefox-clang/third_party/sqlite3/ext -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/sqlite3/ext -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/llvm-23/lib/clang/23/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=tautological-type-limit-compare -Wno-range-loop-analysis -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-unknown-warning-option -Wno-character-conversion -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -fdiagnostics-absolute-paths -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -mllvm -dwarf-linkage-names=Abstract -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-09-01-224014-2642839-1 -x c /root/firefox-clang/third_party/sqlite3/ext/fts5.c
1
2/*
3** This, the "fts5.c" source file, is a composite file that is itself
4** assembled from the following files:
5**
6** fts5.h
7** fts5Int.h
8** fts5parse.h <--- Generated from fts5parse.y by Lemon
9** fts5parse.c <--- Generated from fts5parse.y by Lemon
10** fts5_aux.c
11** fts5_buffer.c
12** fts5_config.c
13** fts5_expr.c
14** fts5_hash.c
15** fts5_index.c
16** fts5_main.c
17** fts5_storage.c
18** fts5_tokenize.c
19** fts5_unicode2.c
20** fts5_varint.c
21** fts5_vocab.c
22*/
23#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5)
24
25#if !defined(NDEBUG1) && !defined(SQLITE_DEBUG)
26# define NDEBUG1 1
27#endif
28#if defined(NDEBUG1) && defined(SQLITE_DEBUG)
29# undef NDEBUG1
30#endif
31
32#ifdef HAVE_STDINT_H1
33#include <stdint.h>
34#endif
35#ifdef HAVE_INTTYPES_H1
36#include <inttypes.h>
37#endif
38#line 1 "fts5.h"
39/*
40** 2014 May 31
41**
42** The author disclaims copyright to this source code. In place of
43** a legal notice, here is a blessing:
44**
45** May you do good and not evil.
46** May you find forgiveness for yourself and forgive others.
47** May you share freely, never taking more than you give.
48**
49******************************************************************************
50**
51** Interfaces to extend FTS5. Using the interfaces defined in this file,
52** FTS5 may be extended with:
53**
54** * custom tokenizers, and
55** * custom auxiliary functions.
56*/
57
58
59#ifndef _FTS5_H
60#define _FTS5_H
61
62#include "sqlite3.h"
63
64#ifdef __cplusplus
65extern "C" {
66#endif
67
68/*************************************************************************
69** CUSTOM AUXILIARY FUNCTIONS
70**
71** Virtual table implementations may overload SQL functions by implementing
72** the sqlite3_module.xFindFunction() method.
73*/
74
75typedef struct Fts5ExtensionApi Fts5ExtensionApi;
76typedef struct Fts5Context Fts5Context;
77typedef struct Fts5PhraseIter Fts5PhraseIter;
78
79typedef void (*fts5_extension_function)(
80 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
81 Fts5Context *pFts, /* First arg to pass to pApi functions */
82 sqlite3_context *pCtx, /* Context for returning result/error */
83 int nVal, /* Number of values in apVal[] array */
84 sqlite3_value **apVal /* Array of trailing arguments */
85);
86
87struct Fts5PhraseIter {
88 const unsigned char *a;
89 const unsigned char *b;
90};
91
92/*
93** EXTENSION API FUNCTIONS
94**
95** xUserData(pFts):
96** Return a copy of the pUserData pointer passed to the xCreateFunction()
97** API when the extension function was registered.
98**
99** xColumnTotalSize(pFts, iCol, pnToken):
100** If parameter iCol is less than zero, set output variable *pnToken
101** to the total number of tokens in the FTS5 table. Or, if iCol is
102** non-negative but less than the number of columns in the table, return
103** the total number of tokens in column iCol, considering all rows in
104** the FTS5 table.
105**
106** If parameter iCol is greater than or equal to the number of columns
107** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
108** an OOM condition or IO error), an appropriate SQLite error code is
109** returned.
110**
111** xColumnCount(pFts):
112** Return the number of columns in the table.
113**
114** xColumnSize(pFts, iCol, pnToken):
115** If parameter iCol is less than zero, set output variable *pnToken
116** to the total number of tokens in the current row. Or, if iCol is
117** non-negative but less than the number of columns in the table, set
118** *pnToken to the number of tokens in column iCol of the current row.
119**
120** If parameter iCol is greater than or equal to the number of columns
121** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
122** an OOM condition or IO error), an appropriate SQLite error code is
123** returned.
124**
125** This function may be quite inefficient if used with an FTS5 table
126** created with the "columnsize=0" option.
127**
128** xColumnText:
129** If parameter iCol is less than zero, or greater than or equal to the
130** number of columns in the table, SQLITE_RANGE is returned.
131**
132** Otherwise, this function attempts to retrieve the text of column iCol of
133** the current document. If successful, (*pz) is set to point to a buffer
134** containing the text in utf-8 encoding, (*pn) is set to the size in bytes
135** (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
136** if an error occurs, an SQLite error code is returned and the final values
137** of (*pz) and (*pn) are undefined.
138**
139** xPhraseCount:
140** Returns the number of phrases in the current query expression.
141**
142** xPhraseSize:
143** If parameter iCol is less than zero, or greater than or equal to the
144** number of phrases in the current query, as returned by xPhraseCount,
145** 0 is returned. Otherwise, this function returns the number of tokens in
146** phrase iPhrase of the query. Phrases are numbered starting from zero.
147**
148** xInstCount:
149** Set *pnInst to the total number of occurrences of all phrases within
150** the query within the current row. Return SQLITE_OK if successful, or
151** an error code (i.e. SQLITE_NOMEM) if an error occurs.
152**
153** This API can be quite slow if used with an FTS5 table created with the
154** "detail=none" or "detail=column" option. If the FTS5 table is created
155** with either "detail=none" or "detail=column" and "content=" option
156** (i.e. if it is a contentless table), then this API always returns 0.
157**
158** xInst:
159** Query for the details of phrase match iIdx within the current row.
160** Phrase matches are numbered starting from zero, so the iIdx argument
161** should be greater than or equal to zero and smaller than the value
162** output by xInstCount(). If iIdx is less than zero or greater than
163** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.
164**
165** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol
166** to the column in which it occurs and *piOff the token offset of the
167** first token of the phrase. SQLITE_OK is returned if successful, or an
168** error code (i.e. SQLITE_NOMEM) if an error occurs.
169**
170** This API can be quite slow if used with an FTS5 table created with the
171** "detail=none" or "detail=column" option.
172**
173** xRowid:
174** Returns the rowid of the current row.
175**
176** xTokenize:
177** Tokenize text using the tokenizer belonging to the FTS5 table.
178**
179** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
180** This API function is used to query the FTS table for phrase iPhrase
181** of the current query. Specifically, a query equivalent to:
182**
183** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid
184**
185** with $p set to a phrase equivalent to the phrase iPhrase of the
186** current query is executed. Any column filter that applies to
187** phrase iPhrase of the current query is included in $p. For each
188** row visited, the callback function passed as the fourth argument
189** is invoked. The context and API objects passed to the callback
190** function may be used to access the properties of each matched row.
191** Invoking Api.xUserData() returns a copy of the pointer passed as
192** the third argument to pUserData.
193**
194** If parameter iPhrase is less than zero, or greater than or equal to
195** the number of phrases in the query, as returned by xPhraseCount(),
196** this function returns SQLITE_RANGE.
197**
198** If the callback function returns any value other than SQLITE_OK, the
199** query is abandoned and the xQueryPhrase function returns immediately.
200** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
201** Otherwise, the error code is propagated upwards.
202**
203** If the query runs to completion without incident, SQLITE_OK is returned.
204** Or, if some error occurs before the query completes or is aborted by
205** the callback, an SQLite error code is returned.
206**
207**
208** xSetAuxdata(pFts5, pAux, xDelete)
209**
210** Save the pointer passed as the second argument as the extension function's
211** "auxiliary data". The pointer may then be retrieved by the current or any
212** future invocation of the same fts5 extension function made as part of
213** the same MATCH query using the xGetAuxdata() API.
214**
215** Each extension function is allocated a single auxiliary data slot for
216** each FTS query (MATCH expression). If the extension function is invoked
217** more than once for a single FTS query, then all invocations share a
218** single auxiliary data context.
219**
220** If there is already an auxiliary data pointer when this function is
221** invoked, then it is replaced by the new pointer. If an xDelete callback
222** was specified along with the original pointer, it is invoked at this
223** point.
224**
225** The xDelete callback, if one is specified, is also invoked on the
226** auxiliary data pointer after the FTS5 query has finished.
227**
228** If an error (e.g. an OOM condition) occurs within this function,
229** the auxiliary data is set to NULL and an error code returned. If the
230** xDelete parameter was not NULL, it is invoked on the auxiliary data
231** pointer before returning.
232**
233**
234** xGetAuxdata(pFts5, bClear)
235**
236** Returns the current auxiliary data pointer for the fts5 extension
237** function. See the xSetAuxdata() method for details.
238**
239** If the bClear argument is non-zero, then the auxiliary data is cleared
240** (set to NULL) before this function returns. In this case the xDelete,
241** if any, is not invoked.
242**
243**
244** xRowCount(pFts5, pnRow)
245**
246** This function is used to retrieve the total number of rows in the table.
247** In other words, the same value that would be returned by:
248**
249** SELECT count(*) FROM ftstable;
250**
251** xPhraseFirst()
252** This function is used, along with type Fts5PhraseIter and the xPhraseNext
253** method, to iterate through all instances of a single query phrase within
254** the current row. This is the same information as is accessible via the
255** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient
256** to use, this API may be faster under some circumstances. To iterate
257** through instances of phrase iPhrase, use the following code:
258**
259** Fts5PhraseIter iter;
260** int iCol, iOff;
261** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
262** iCol>=0;
263** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
264** ){
265** // An instance of phrase iPhrase at offset iOff of column iCol
266** }
267**
268** The Fts5PhraseIter structure is defined above. Applications should not
269** modify this structure directly - it should only be used as shown above
270** with the xPhraseFirst() and xPhraseNext() API methods (and by
271** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).
272**
273** This API can be quite slow if used with an FTS5 table created with the
274** "detail=none" or "detail=column" option. If the FTS5 table is created
275** with either "detail=none" or "detail=column" and "content=" option
276** (i.e. if it is a contentless table), then this API always iterates
277** through an empty set (all calls to xPhraseFirst() set iCol to -1).
278**
279** In all cases, matches are visited in (column ASC, offset ASC) order.
280** i.e. all those in column 0, sorted by offset, followed by those in
281** column 1, etc.
282**
283** xPhraseNext()
284** See xPhraseFirst above.
285**
286** xPhraseFirstColumn()
287** This function and xPhraseNextColumn() are similar to the xPhraseFirst()
288** and xPhraseNext() APIs described above. The difference is that instead
289** of iterating through all instances of a phrase in the current row, these
290** APIs are used to iterate through the set of columns in the current row
291** that contain one or more instances of a specified phrase. For example:
292**
293** Fts5PhraseIter iter;
294** int iCol;
295** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
296** iCol>=0;
297** pApi->xPhraseNextColumn(pFts, &iter, &iCol)
298** ){
299** // Column iCol contains at least one instance of phrase iPhrase
300** }
301**
302** This API can be quite slow if used with an FTS5 table created with the
303** "detail=none" option. If the FTS5 table is created with either
304** "detail=none" "content=" option (i.e. if it is a contentless table),
305** then this API always iterates through an empty set (all calls to
306** xPhraseFirstColumn() set iCol to -1).
307**
308** The information accessed using this API and its companion
309** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext
310** (or xInst/xInstCount). The chief advantage of this API is that it is
311** significantly more efficient than those alternatives when used with
312** "detail=column" tables.
313**
314** xPhraseNextColumn()
315** See xPhraseFirstColumn above.
316**
317** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
318** This is used to access token iToken of phrase iPhrase of the current
319** query. Before returning, output parameter *ppToken is set to point
320** to a buffer containing the requested token, and *pnToken to the
321** size of this buffer in bytes.
322**
323** If iPhrase or iToken are less than zero, or if iPhrase is greater than
324** or equal to the number of phrases in the query as reported by
325** xPhraseCount(), or if iToken is equal to or greater than the number of
326** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken
327 are both zeroed.
328**
329** The output text is not a copy of the query text that specified the
330** token. It is the output of the tokenizer module. For tokendata=1
331** tables, this includes any embedded 0x00 and trailing data.
332**
333** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
334** This is used to access token iToken of phrase hit iIdx within the
335** current row. If iIdx is less than zero or greater than or equal to the
336** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise,
337** output variable (*ppToken) is set to point to a buffer containing the
338** matching document token, and (*pnToken) to the size of that buffer in
339** bytes.
340**
341** The output text is not a copy of the document text that was tokenized.
342** It is the output of the tokenizer module. For tokendata=1 tables, this
343** includes any embedded 0x00 and trailing data.
344**
345** This API may be slow in some cases if the token identified by parameters
346** iIdx and iToken matched a prefix token in the query. In most cases, the
347** first call to this API for each prefix token in the query is forced
348** to scan the portion of the full-text index that matches the prefix
349** token to collect the extra data required by this API. If the prefix
350** token matches a large number of token instances in the document set,
351** this may be a performance problem.
352**
353** If the user knows in advance that a query may use this API for a
354** prefix token, FTS5 may be configured to collect all required data as part
355** of the initial querying of the full-text index, avoiding the second scan
356** entirely. This also causes prefix queries that do not use this API to
357** run more slowly and use more memory. FTS5 may be configured in this way
358** either on a per-table basis using the [FTS5 insttoken | 'insttoken']
359** option, or on a per-query basis using the
360** [fts5_insttoken | fts5_insttoken()] user function.
361**
362** This API can be quite slow if used with an FTS5 table created with the
363** "detail=none" or "detail=column" option.
364**
365** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale)
366** If parameter iCol is less than zero, or greater than or equal to the
367** number of columns in the table, SQLITE_RANGE is returned.
368**
369** Otherwise, this function attempts to retrieve the locale associated
370** with column iCol of the current row. Usually, there is no associated
371** locale, and output parameters (*pzLocale) and (*pnLocale) are set
372** to NULL and 0, respectively. However, if the fts5_locale() function
373** was used to associate a locale with the value when it was inserted
374** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated
375** buffer containing the name of the locale in utf-8 encoding. (*pnLocale)
376** is set to the size in bytes of the buffer, not including the
377** nul-terminator.
378**
379** If successful, SQLITE_OK is returned. Or, if an error occurs, an
380** SQLite error code is returned. The final value of the output parameters
381** is undefined in this case.
382**
383** xTokenize_v2:
384** Tokenize text using the tokenizer belonging to the FTS5 table. This
385** API is the same as the xTokenize() API, except that it allows a tokenizer
386** locale to be specified.
387*/
388struct Fts5ExtensionApi {
389 int iVersion; /* Currently always set to 4 */
390
391 void *(*xUserData)(Fts5Context*);
392
393 int (*xColumnCount)(Fts5Context*);
394 int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
395 int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);
396
397 int (*xTokenize)(Fts5Context*,
398 const char *pText, int nText, /* Text to tokenize */
399 void *pCtx, /* Context passed to xToken() */
400 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
401 );
402
403 int (*xPhraseCount)(Fts5Context*);
404 int (*xPhraseSize)(Fts5Context*, int iPhrase);
405
406 int (*xInstCount)(Fts5Context*, int *pnInst);
407 int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff);
408
409 sqlite3_int64 (*xRowid)(Fts5Context*);
410 int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
411 int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
412
413 int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
414 int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
415 );
416 int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
417 void *(*xGetAuxdata)(Fts5Context*, int bClear);
418
419 int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
420 void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);
421
422 int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
423 void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
424
425 /* Below this point are iVersion>=3 only */
426 int (*xQueryToken)(Fts5Context*,
427 int iPhrase, int iToken,
428 const char **ppToken, int *pnToken
429 );
430 int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
431
432 /* Below this point are iVersion>=4 only */
433 int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn);
434 int (*xTokenize_v2)(Fts5Context*,
435 const char *pText, int nText, /* Text to tokenize */
436 const char *pLocale, int nLocale, /* Locale to pass to tokenizer */
437 void *pCtx, /* Context passed to xToken() */
438 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
439 );
440};
441
442/*
443** CUSTOM AUXILIARY FUNCTIONS
444*************************************************************************/
445
446/*************************************************************************
447** CUSTOM TOKENIZERS
448**
449** Applications may also register custom tokenizer types. A tokenizer
450** is registered by providing fts5 with a populated instance of the
451** following structure. All structure methods must be defined, setting
452** any member of the fts5_tokenizer struct to NULL leads to undefined
453** behaviour. The structure methods are expected to function as follows:
454**
455** xCreate:
456** This function is used to allocate and initialize a tokenizer instance.
457** A tokenizer instance is required to actually tokenize text.
458**
459** The first argument passed to this function is a copy of the (void*)
460** pointer provided by the application when the fts5_tokenizer_v2 object
461** was registered with FTS5 (the third argument to xCreateTokenizer()).
462** The second and third arguments are an array of nul-terminated strings
463** containing the tokenizer arguments, if any, specified following the
464** tokenizer name as part of the CREATE VIRTUAL TABLE statement used
465** to create the FTS5 table.
466**
467** The final argument is an output variable. If successful, (*ppOut)
468** should be set to point to the new tokenizer handle and SQLITE_OK
469** returned. If an error occurs, some value other than SQLITE_OK should
470** be returned. In this case, fts5 assumes that the final value of *ppOut
471** is undefined.
472**
473** xDelete:
474** This function is invoked to delete a tokenizer handle previously
475** allocated using xCreate(). Fts5 guarantees that this function will
476** be invoked exactly once for each successful call to xCreate().
477**
478** xTokenize:
479** This function is expected to tokenize the nText byte string indicated
480** by argument pText. pText may or may not be nul-terminated. The first
481** argument passed to this function is a pointer to an Fts5Tokenizer object
482** returned by an earlier call to xCreate().
483**
484** The third argument indicates the reason that FTS5 is requesting
485** tokenization of the supplied text. This is always one of the following
486** four values:
487**
488** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into
489** or removed from the FTS table. The tokenizer is being invoked to
490** determine the set of tokens to add to (or delete from) the
491** FTS index.
492**
493** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed
494** against the FTS index. The tokenizer is being called to tokenize
495** a bareword or quoted string specified as part of the query.
496**
497** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as
498** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is
499** followed by a "*" character, indicating that the last token
500** returned by the tokenizer will be treated as a token prefix.
501**
502** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to
503** satisfy an fts5_api.xTokenize() request made by an auxiliary
504** function. Or an fts5_api.xColumnSize() request made by the same
505** on a columnsize=0 database.
506** </ul>
507**
508** The sixth and seventh arguments passed to xTokenize() - pLocale and
509** nLocale - are a pointer to a buffer containing the locale to use for
510** tokenization (e.g. "en_US") and its size in bytes, respectively. The
511** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in
512** which case nLocale is always 0) to indicate that the tokenizer should
513** use its default locale.
514**
515** For each token in the input string, the supplied callback xToken() must
516** be invoked. The first argument to it should be a copy of the pointer
517** passed as the second argument to xTokenize(). The third and fourth
518** arguments are a pointer to a buffer containing the token text, and the
519** size of the token in bytes. The 4th and 5th arguments are the byte offsets
520** of the first byte of and first byte immediately following the text from
521** which the token is derived within the input.
522**
523** The second argument passed to the xToken() callback ("tflags") should
524** normally be set to 0. The exception is if the tokenizer supports
525** synonyms. In this case see the discussion below for details.
526**
527** FTS5 assumes the xToken() callback is invoked for each token in the
528** order that they occur within the input text.
529**
530** If an xToken() callback returns any value other than SQLITE_OK, then
531** the tokenization should be abandoned and the xTokenize() method should
532** immediately return a copy of the xToken() return value. Or, if the
533** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally,
534** if an error occurs with the xTokenize() implementation itself, it
535** may abandon the tokenization and return any error code other than
536** SQLITE_OK or SQLITE_DONE.
537**
538** If the tokenizer is registered using an fts5_tokenizer_v2 object,
539** then the xTokenize() method has two additional arguments - pLocale
540** and nLocale. These specify the locale that the tokenizer should use
541** for the current request. If pLocale and nLocale are both 0, then the
542** tokenizer should use its default locale. Otherwise, pLocale points to
543** an nLocale byte buffer containing the name of the locale to use as utf-8
544** text. pLocale is not nul-terminated.
545**
546** FTS5_TOKENIZER
547**
548** There is also an fts5_tokenizer object. This is an older, deprecated,
549** version of fts5_tokenizer_v2. It is similar except that:
550**
551** <ul>
552** <li> There is no "iVersion" field, and
553** <li> The xTokenize() method does not take a locale argument.
554** </ul>
555**
556** Legacy fts5_tokenizer tokenizers must be registered using the
557** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2().
558**
559** Tokenizer implementations registered using either API may be retrieved
560** using both xFindTokenizer() and xFindTokenizer_v2().
561**
562** SYNONYM SUPPORT
563**
564** Custom tokenizers may also support synonyms. Consider a case in which a
565** user wishes to query for a phrase such as "first place". Using the
566** built-in tokenizers, the FTS5 query 'first + place' will match instances
567** of "first place" within the document set, but not alternative forms
568** such as "1st place". In some applications, it would be better to match
569** all instances of "first place" or "1st place" regardless of which form
570** the user specified in the MATCH query text.
571**
572** There are several ways to approach this in FTS5:
573**
574** <ol><li> By mapping all synonyms to a single token. In this case, using
575** the above example, this means that the tokenizer returns the
576** same token for inputs "first" and "1st". Say that token is in
577** fact "first", so that when the user inserts the document "I won
578** 1st place" entries are added to the index for tokens "i", "won",
579** "first" and "place". If the user then queries for '1st + place',
580** the tokenizer substitutes "first" for "1st" and the query works
581** as expected.
582**
583** <li> By querying the index for all synonyms of each query term
584** separately. In this case, when tokenizing query text, the
585** tokenizer may provide multiple synonyms for a single term
586** within the document. FTS5 then queries the index for each
587** synonym individually. For example, faced with the query:
588**
589** <codeblock>
590** ... MATCH 'first place'</codeblock>
591**
592** the tokenizer offers both "1st" and "first" as synonyms for the
593** first token in the MATCH query and FTS5 effectively runs a query
594** similar to:
595**
596** <codeblock>
597** ... MATCH '(first OR 1st) place'</codeblock>
598**
599** except that, for the purposes of auxiliary functions, the query
600** still appears to contain just two phrases - "(first OR 1st)"
601** being treated as a single phrase.
602**
603** <li> By adding multiple synonyms for a single term to the FTS index.
604** Using this method, when tokenizing document text, the tokenizer
605** provides multiple synonyms for each token. So that when a
606** document such as "I won first place" is tokenized, entries are
607** added to the FTS index for "i", "won", "first", "1st" and
608** "place".
609**
610** This way, even if the tokenizer does not provide synonyms
611** when tokenizing query text (it should not - to do so would be
612** inefficient), it doesn't matter if the user queries for
613** 'first + place' or '1st + place', as there are entries in the
614** FTS index corresponding to both forms of the first token.
615** </ol>
616**
617** Whether it is parsing document or query text, any call to xToken that
618** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit
619** is considered to supply a synonym for the previous token. For example,
620** when parsing the document "I won first place", a tokenizer that supports
621** synonyms would call xToken() 5 times, as follows:
622**
623** <codeblock>
624** xToken(pCtx, 0, "i", 1, 0, 1);
625** xToken(pCtx, 0, "won", 3, 2, 5);
626** xToken(pCtx, 0, "first", 5, 6, 11);
627** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11);
628** xToken(pCtx, 0, "place", 5, 12, 17);
629**</codeblock>
630**
631** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time
632** xToken() is called. Multiple synonyms may be specified for a single token
633** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence.
634** There is no limit to the number of synonyms that may be provided for a
635** single token.
636**
637** In many cases, method (1) above is the best approach. It does not add
638** extra data to the FTS index or require FTS5 to query for multiple terms,
639** so it is efficient in terms of disk space and query speed. However, it
640** does not support prefix queries very well. If, as suggested above, the
641** token "first" is substituted for "1st" by the tokenizer, then the query:
642**
643** <codeblock>
644** ... MATCH '1s*'</codeblock>
645**
646** will not match documents that contain the token "1st" (as the tokenizer
647** will probably not map "1s" to any prefix of "first").
648**
649** For full prefix support, method (3) may be preferred. In this case,
650** because the index contains entries for both "first" and "1st", prefix
651** queries such as 'fi*' or '1s*' will match correctly. However, because
652** extra entries are added to the FTS index, this method uses more space
653** within the database.
654**
655** Method (2) offers a midpoint between (1) and (3). Using this method,
656** a query such as '1s*' will match documents that contain the literal
657** token "1st", but not "first" (assuming the tokenizer is not able to
658** provide synonyms for prefixes). However, a non-prefix query like '1st'
659** will match against "1st" and "first". This method does not require
660** extra disk space, as no extra entries are added to the FTS index.
661** On the other hand, it may require more CPU cycles to run MATCH queries,
662** as separate queries of the FTS index are required for each synonym.
663**
664** When using methods (2) or (3), it is important that the tokenizer only
665** provide synonyms when tokenizing document text (method (3)) or query
666** text (method (2)), not both. Doing so will not cause any errors, but is
667** inefficient.
668*/
669typedef struct Fts5Tokenizer Fts5Tokenizer;
670typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2;
671struct fts5_tokenizer_v2 {
672 int iVersion; /* Currently always 2 */
673
674 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
675 void (*xDelete)(Fts5Tokenizer*);
676 int (*xTokenize)(Fts5Tokenizer*,
677 void *pCtx,
678 int flags, /* Mask of FTS5_TOKENIZE_* flags */
679 const char *pText, int nText,
680 const char *pLocale, int nLocale,
681 int (*xToken)(
682 void *pCtx, /* Copy of 2nd argument to xTokenize() */
683 int tflags, /* Mask of FTS5_TOKEN_* flags */
684 const char *pToken, /* Pointer to buffer containing token */
685 int nToken, /* Size of token in bytes */
686 int iStart, /* Byte offset of token within input text */
687 int iEnd /* Byte offset of end of token within input text */
688 )
689 );
690};
691
692/*
693** New code should use the fts5_tokenizer_v2 type to define tokenizer
694** implementations. The following type is included for legacy applications
695** that still use it.
696*/
697typedef struct fts5_tokenizer fts5_tokenizer;
698struct fts5_tokenizer {
699 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
700 void (*xDelete)(Fts5Tokenizer*);
701 int (*xTokenize)(Fts5Tokenizer*,
702 void *pCtx,
703 int flags, /* Mask of FTS5_TOKENIZE_* flags */
704 const char *pText, int nText,
705 int (*xToken)(
706 void *pCtx, /* Copy of 2nd argument to xTokenize() */
707 int tflags, /* Mask of FTS5_TOKEN_* flags */
708 const char *pToken, /* Pointer to buffer containing token */
709 int nToken, /* Size of token in bytes */
710 int iStart, /* Byte offset of token within input text */
711 int iEnd /* Byte offset of end of token within input text */
712 )
713 );
714};
715
716
717/* Flags that may be passed as the third argument to xTokenize() */
718#define FTS5_TOKENIZE_QUERY0x0001 0x0001
719#define FTS5_TOKENIZE_PREFIX0x0002 0x0002
720#define FTS5_TOKENIZE_DOCUMENT0x0004 0x0004
721#define FTS5_TOKENIZE_AUX0x0008 0x0008
722
723/* Flags that may be passed by the tokenizer implementation back to FTS5
724** as the third argument to the supplied xToken callback. */
725#define FTS5_TOKEN_COLOCATED0x0001 0x0001 /* Same position as prev. token */
726
727/*
728** END OF CUSTOM TOKENIZERS
729*************************************************************************/
730
731/*************************************************************************
732** FTS5 EXTENSION REGISTRATION API
733*/
734typedef struct fts5_api fts5_api;
735struct fts5_api {
736 int iVersion; /* Currently always set to 3 */
737
738 /* Create a new tokenizer */
739 int (*xCreateTokenizer)(
740 fts5_api *pApi,
741 const char *zName,
742 void *pUserData,
743 fts5_tokenizer *pTokenizer,
744 void (*xDestroy)(void*)
745 );
746
747 /* Find an existing tokenizer */
748 int (*xFindTokenizer)(
749 fts5_api *pApi,
750 const char *zName,
751 void **ppUserData,
752 fts5_tokenizer *pTokenizer
753 );
754
755 /* Create a new auxiliary function */
756 int (*xCreateFunction)(
757 fts5_api *pApi,
758 const char *zName,
759 void *pUserData,
760 fts5_extension_function xFunction,
761 void (*xDestroy)(void*)
762 );
763
764 /* APIs below this point are only available if iVersion>=3 */
765
766 /* Create a new tokenizer */
767 int (*xCreateTokenizer_v2)(
768 fts5_api *pApi,
769 const char *zName,
770 void *pUserData,
771 fts5_tokenizer_v2 *pTokenizer,
772 void (*xDestroy)(void*)
773 );
774
775 /* Find an existing tokenizer */
776 int (*xFindTokenizer_v2)(
777 fts5_api *pApi,
778 const char *zName,
779 void **ppUserData,
780 fts5_tokenizer_v2 **ppTokenizer
781 );
782};
783
784/*
785** END OF REGISTRATION API
786*************************************************************************/
787
788#ifdef __cplusplus
789} /* end of the 'extern "C"' block */
790#endif
791
792#endif /* _FTS5_H */
793
794#line 1 "fts5Int.h"
795/*
796** 2014 May 31
797**
798** The author disclaims copyright to this source code. In place of
799** a legal notice, here is a blessing:
800**
801** May you do good and not evil.
802** May you find forgiveness for yourself and forgive others.
803** May you share freely, never taking more than you give.
804**
805******************************************************************************
806**
807*/
808#ifndef _FTS5INT_H
809#define _FTS5INT_H
810
811/* #include "fts5.h" */
812#include "sqlite3ext.h"
813SQLITE_EXTENSION_INIT1const sqlite3_api_routines *sqlite3_api=0;
814
815#include <string.h>
816#include <assert.h>
817#include <stddef.h>
818
819#ifndef SQLITE_AMALGAMATION
820
821typedef unsigned char u8;
822typedef unsigned int u32;
823typedef unsigned short u16;
824typedef short i16;
825typedef sqlite3_int64 i64;
826typedef sqlite3_uint64 u64;
827
828#ifndef ArraySize
829# define ArraySize(x)((int)(sizeof(x) / sizeof(x[0]))) ((int)(sizeof(x) / sizeof(x[0])))
830#endif
831
832#define testcase(x)
833
834#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
835# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1
836#endif
837#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS)
838# define ALWAYS(X)(X) (1)
839# define NEVER(X)(X) (0)
840#elif !defined(NDEBUG1)
841# define ALWAYS(X)(X) ((X)?1:(assert(0)((void) (0)),0))
842# define NEVER(X)(X) ((X)?(assert(0)((void) (0)),1):0)
843#else
844# define ALWAYS(X)(X) (X)
845# define NEVER(X)(X) (X)
846#endif
847
848#define MIN(x,y)(((x) < (y)) ? (x) : (y)) (((x) < (y)) ? (x) : (y))
849#define MAX(x,y)(((x) > (y)) ? (x) : (y)) (((x) > (y)) ? (x) : (y))
850
851/*
852** Constants for the largest and smallest possible 64-bit signed integers.
853*/
854# define LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)) (0xffffffff|(((i64)0x7fffffff)<<32))
855# define SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32))) (((i64)-1) - LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)))
856
857/*
858** This macro is used in a single assert() within fts5 to check that an
859** allocation is aligned to an 8-byte boundary. But it is a complicated
860** macro to get right for multiple platforms without generating warnings.
861** So instead of reproducing the entire definition from sqliteInt.h, we
862** just do without this assert() for the rare non-amalgamation builds.
863*/
864#define EIGHT_BYTE_ALIGNMENT(x)1 1
865
866/*
867** Macros needed to provide flexible arrays in a portable way
868*/
869#ifndef offsetof
870# define offsetof(ST,M)__builtin_offsetof(ST, M) ((size_t)((char*)&((ST*)0)->M - (char*)0))
871#endif
872#if defined(__STDC_VERSION__201710L) && (__STDC_VERSION__201710L >= 199901L)
873# define FLEXARRAY
874#else
875# define FLEXARRAY 1
876#endif
877
878#endif /* SQLITE_AMALGAMATION */
879
880/*
881** Constants for the largest and smallest possible 32-bit signed integers.
882*/
883# define LARGEST_INT32((int)(0x7fffffff)) ((int)(0x7fffffff))
884# define SMALLEST_INT32((int)((-1) - ((int)(0x7fffffff)))) ((int)((-1) - LARGEST_INT32((int)(0x7fffffff))))
885
886/* Truncate very long tokens to this many bytes. Hard limit is
887** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
888** field that occurs at the start of each leaf page (see fts5_index.c). */
889#define FTS5_MAX_TOKEN_SIZE32768 32768
890
891/*
892** Maximum number of prefix indexes on single FTS5 table. This must be
893** less than 32. If it is set to anything large than that, an #error
894** directive in fts5_index.c will cause the build to fail.
895*/
896#define FTS5_MAX_PREFIX_INDEXES31 31
897
898/*
899** Maximum segments permitted in a single index
900*/
901#define FTS5_MAX_SEGMENT2000 2000
902
903#define FTS5_DEFAULT_NEARDIST10 10
904#define FTS5_DEFAULT_RANK"bm25" "bm25"
905
906/* Name of rank and rowid columns */
907#define FTS5_RANK_NAME"rank" "rank"
908#define FTS5_ROWID_NAME"rowid" "rowid"
909
910#ifdef SQLITE_DEBUG
911# define FTS5_CORRUPT(11 | (1<<8)) sqlite3Fts5Corrupt()
912static int sqlite3Fts5Corrupt(void);
913#else
914# define FTS5_CORRUPT(11 | (1<<8)) SQLITE_CORRUPT_VTAB(11 | (1<<8))
915#endif
916
917/*
918** The assert_nc() macro is similar to the assert() macro, except that it
919** is used for assert() conditions that are true only if it can be
920** guranteed that the database is not corrupt.
921*/
922#ifdef SQLITE_DEBUG
923extern int sqlite3_fts5_may_be_corrupt;
924# define assert_nc(x)((void) (0)) assert(sqlite3_fts5_may_be_corrupt || (x))((void) (0))
925#else
926# define assert_nc(x)((void) (0)) assert(x)((void) (0))
927#endif
928
929/*
930** A version of memcmp() that does not cause asan errors if one of the pointer
931** parameters is NULL and the number of bytes to compare is zero.
932*/
933#define fts5Memcmp(s1, s2, n)((n)<=0 ? 0 : memcmp((s1), (s2), (n))) ((n)<=0 ? 0 : memcmp((s1), (s2), (n)))
934
935/* Mark a function parameter as unused, to suppress nuisance compiler
936** warnings. */
937#ifndef UNUSED_PARAM
938# define UNUSED_PARAM(X)(void)(X) (void)(X)
939#endif
940
941#ifndef UNUSED_PARAM2
942# define UNUSED_PARAM2(X, Y)(void)(X), (void)(Y) (void)(X), (void)(Y)
943#endif
944
945typedef struct Fts5Global Fts5Global;
946typedef struct Fts5Colset Fts5Colset;
947
948/* If a NEAR() clump or phrase may only match a specific set of columns,
949** then an object of the following type is used to record the set of columns.
950** Each entry in the aiCol[] array is a column that may be matched.
951**
952** This object is used by fts5_expr.c and fts5_index.c.
953*/
954struct Fts5Colset {
955 int nCol;
956 int aiCol[FLEXARRAY];
957};
958
959/* Size (int bytes) of a complete Fts5Colset object with N columns. */
960#define SZ_FTS5COLSET(N)(sizeof(i64)*((N+2)/2)) (sizeof(i64)*((N+2)/2))
961
962/**************************************************************************
963** Interface to code in fts5_config.c. fts5_config.c contains contains code
964** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
965*/
966
967typedef struct Fts5Config Fts5Config;
968typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;
969
970struct Fts5TokenizerConfig {
971 Fts5Tokenizer *pTok;
972 fts5_tokenizer_v2 *pApi2;
973 fts5_tokenizer *pApi1;
974 const char **azArg;
975 int nArg;
976 int ePattern; /* FTS_PATTERN_XXX constant */
977 const char *pLocale; /* Current locale to use */
978 int nLocale; /* Size of pLocale in bytes */
979};
980
981/*
982** An instance of the following structure encodes all information that can
983** be gleaned from the CREATE VIRTUAL TABLE statement.
984**
985** And all information loaded from the %_config table.
986**
987** nAutomerge:
988** The minimum number of segments that an auto-merge operation should
989** attempt to merge together. A value of 1 sets the object to use the
990** compile time default. Zero disables auto-merge altogether.
991**
992** bContentlessDelete:
993** True if the contentless_delete option was present in the CREATE
994** VIRTUAL TABLE statement.
995**
996** zContent:
997**
998** zContentRowid:
999** The value of the content_rowid= option, if one was specified. Or
1000** the string "rowid" otherwise. This text is not quoted - if it is
1001** used as part of an SQL statement it needs to be quoted appropriately.
1002**
1003** zContentExprlist:
1004**
1005** pzErrmsg:
1006** This exists in order to allow the fts5_index.c module to return a
1007** decent error message if it encounters a file-format version it does
1008** not understand.
1009**
1010** bColumnsize:
1011** True if the %_docsize table is created.
1012**
1013** bPrefixIndex:
1014** This is only used for debugging. If set to false, any prefix indexes
1015** are ignored. This value is configured using:
1016**
1017** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
1018**
1019** bLocale:
1020** Set to true if locale=1 was specified when the table was created.
1021*/
1022struct Fts5Config {
1023 sqlite3 *db; /* Database handle */
1024 Fts5Global *pGlobal; /* Global fts5 object for handle db */
1025 char *zDb; /* Database holding FTS index (e.g. "main") */
1026 char *zName; /* Name of FTS index */
1027 int nCol; /* Number of columns */
1028 char **azCol; /* Column names */
1029 u8 *abUnindexed; /* True for unindexed columns */
1030 int nPrefix; /* Number of prefix indexes */
1031 int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */
1032 int eContent; /* An FTS5_CONTENT value */
1033 int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */
1034 int bContentlessUnindexed; /* "contentless_unindexed=" option (dflt=0) */
1035 char *zContent; /* content table */
1036 char *zContentRowid; /* "content_rowid=" option value */
1037 int bColumnsize; /* "columnsize=" option value (dflt==1) */
1038 int bTokendata; /* "tokendata=" option value (dflt==0) */
1039 int bLocale; /* "locale=" option value (dflt==0) */
1040 int eDetail; /* FTS5_DETAIL_XXX value */
1041 char *zContentExprlist;
1042 Fts5TokenizerConfig t;
1043 int bLock; /* True when table is preparing statement */
1044
1045
1046 /* Values loaded from the %_config table */
1047 int iVersion; /* fts5 file format 'version' */
1048 int iCookie; /* Incremented when %_config is modified */
1049 int pgsz; /* Approximate page size used in %_data */
1050 int nAutomerge; /* 'automerge' setting */
1051 int nCrisisMerge; /* Maximum allowed segments per level */
1052 int nUsermerge; /* 'usermerge' setting */
1053 int nHashSize; /* Bytes of memory for in-memory hash */
1054 char *zRank; /* Name of rank function */
1055 char *zRankArgs; /* Arguments to rank function */
1056 int bSecureDelete; /* 'secure-delete' */
1057 int nDeleteMerge; /* 'deletemerge' */
1058 int bPrefixInsttoken; /* 'prefix-insttoken' */
1059
1060 /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
1061 char **pzErrmsg;
1062
1063#ifdef SQLITE_DEBUG
1064 int bPrefixIndex; /* True to use prefix-indexes */
1065#endif
1066};
1067
1068/* Current expected value of %_config table 'version' field. And
1069** the expected version if the 'secure-delete' option has ever been
1070** set on the table. */
1071#define FTS5_CURRENT_VERSION4 4
1072#define FTS5_CURRENT_VERSION_SECUREDELETE5 5
1073
1074#define FTS5_CONTENT_NORMAL0 0
1075#define FTS5_CONTENT_NONE1 1
1076#define FTS5_CONTENT_EXTERNAL2 2
1077#define FTS5_CONTENT_UNINDEXED3 3
1078
1079#define FTS5_DETAIL_FULL0 0
1080#define FTS5_DETAIL_NONE1 1
1081#define FTS5_DETAIL_COLUMNS2 2
1082
1083#define FTS5_PATTERN_NONE0 0
1084#define FTS5_PATTERN_LIKE65 65 /* matches SQLITE_INDEX_CONSTRAINT_LIKE */
1085#define FTS5_PATTERN_GLOB66 66 /* matches SQLITE_INDEX_CONSTRAINT_GLOB */
1086
1087static int sqlite3Fts5ConfigParse(
1088 Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
1089);
1090static void sqlite3Fts5ConfigFree(Fts5Config*);
1091
1092static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
1093
1094static int sqlite3Fts5Tokenize(
1095 Fts5Config *pConfig, /* FTS5 Configuration object */
1096 int flags, /* FTS5_TOKENIZE_* flags */
1097 const char *pText, int nText, /* Text to tokenize */
1098 void *pCtx, /* Context passed to xToken() */
1099 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
1100);
1101
1102static void sqlite3Fts5Dequote(char *z);
1103
1104/* Load the contents of the %_config table */
1105static int sqlite3Fts5ConfigLoad(Fts5Config*, int);
1106
1107/* Set the value of a single config attribute */
1108static int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
1109
1110static int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
1111
1112static void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...);
1113
1114/*
1115** End of interface to code in fts5_config.c.
1116**************************************************************************/
1117
1118/**************************************************************************
1119** Interface to code in fts5_buffer.c.
1120*/
1121
1122/*
1123** Buffer object for the incremental building of string data.
1124*/
1125typedef struct Fts5Buffer Fts5Buffer;
1126struct Fts5Buffer {
1127 u8 *p;
1128 int n;
1129 int nSpace;
1130};
1131
1132static int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
1133static void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
1134static void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
1135static void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
1136static void sqlite3Fts5BufferFree(Fts5Buffer*);
1137static void sqlite3Fts5BufferZero(Fts5Buffer*);
1138static void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
1139static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
1140
1141static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
1142
1143#define fts5BufferZero(x)sqlite3Fts5BufferZero(x) sqlite3Fts5BufferZero(x)
1144#define fts5BufferAppendVarint(a,b,c)sqlite3Fts5BufferAppendVarint(a,b,(i64)c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c)
1145#define fts5BufferFree(a)sqlite3Fts5BufferFree(a) sqlite3Fts5BufferFree(a)
1146#define fts5BufferAppendBlob(a,b,c,d)sqlite3Fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
1147#define fts5BufferSet(a,b,c,d)sqlite3Fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d)
1148
1149#define fts5BufferGrow(pRc,pBuf,nn)( (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace
) ? 0 : sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n)
) ( \
1150 (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
1151 sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
1152)
1153
1154/* Write and decode big-endian 32-bit integer values */
1155static void sqlite3Fts5Put32(u8*, int);
1156static int sqlite3Fts5Get32(const u8*);
1157
1158#define FTS5_POS2COLUMN(iPos)(int)((iPos >> 32) & 0x7FFFFFFF) (int)((iPos >> 32) & 0x7FFFFFFF)
1159#define FTS5_POS2OFFSET(iPos)(int)(iPos & 0x7FFFFFFF) (int)(iPos & 0x7FFFFFFF)
1160
1161typedef struct Fts5PoslistReader Fts5PoslistReader;
1162struct Fts5PoslistReader {
1163 /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
1164 const u8 *a; /* Position list to iterate through */
1165 int n; /* Size of buffer at a[] in bytes */
1166 int i; /* Current offset in a[] */
1167
1168 u8 bFlag; /* For client use (any custom purpose) */
1169
1170 /* Output variables */
1171 u8 bEof; /* Set to true at EOF */
1172 i64 iPos; /* (iCol<<32) + iPos */
1173};
1174static int sqlite3Fts5PoslistReaderInit(
1175 const u8 *a, int n, /* Poslist buffer to iterate through */
1176 Fts5PoslistReader *pIter /* Iterator object to initialize */
1177);
1178static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
1179
1180typedef struct Fts5PoslistWriter Fts5PoslistWriter;
1181struct Fts5PoslistWriter {
1182 i64 iPrev;
1183};
1184static int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
1185static void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
1186
1187static int sqlite3Fts5PoslistNext64(
1188 const u8 *a, int n, /* Buffer containing poslist */
1189 int *pi, /* IN/OUT: Offset within a[] */
1190 i64 *piOff /* IN/OUT: Current offset */
1191);
1192
1193/* Malloc utility */
1194static void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte);
1195static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
1196
1197/* Character set tests (like isspace(), isalpha() etc.) */
1198static int sqlite3Fts5IsBareword(char t);
1199
1200
1201/* Bucket of terms object used by the integrity-check in offsets=0 mode. */
1202typedef struct Fts5Termset Fts5Termset;
1203static int sqlite3Fts5TermsetNew(Fts5Termset**);
1204static int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
1205static void sqlite3Fts5TermsetFree(Fts5Termset*);
1206
1207/*
1208** End of interface to code in fts5_buffer.c.
1209**************************************************************************/
1210
1211/**************************************************************************
1212** Interface to code in fts5_index.c. fts5_index.c contains contains code
1213** to access the data stored in the %_data table.
1214*/
1215
1216typedef struct Fts5Index Fts5Index;
1217typedef struct Fts5IndexIter Fts5IndexIter;
1218
1219struct Fts5IndexIter {
1220 i64 iRowid;
1221 const u8 *pData;
1222 int nData;
1223 u8 bEof;
1224};
1225
1226#define sqlite3Fts5IterEof(x)((x)->bEof) ((x)->bEof)
1227
1228/*
1229** Values used as part of the flags argument passed to IndexQuery().
1230*/
1231#define FTS5INDEX_QUERY_PREFIX0x0001 0x0001 /* Prefix query */
1232#define FTS5INDEX_QUERY_DESC0x0002 0x0002 /* Docs in descending rowid order */
1233#define FTS5INDEX_QUERY_TEST_NOIDX0x0004 0x0004 /* Do not use prefix index */
1234#define FTS5INDEX_QUERY_SCAN0x0008 0x0008 /* Scan query (fts5vocab) */
1235
1236/* The following are used internally by the fts5_index.c module. They are
1237** defined here only to make it easier to avoid clashes with the flags
1238** above. */
1239#define FTS5INDEX_QUERY_SKIPEMPTY0x0010 0x0010
1240#define FTS5INDEX_QUERY_NOOUTPUT0x0020 0x0020
1241#define FTS5INDEX_QUERY_SKIPHASH0x0040 0x0040
1242#define FTS5INDEX_QUERY_NOTOKENDATA0x0080 0x0080
1243#define FTS5INDEX_QUERY_SCANONETERM0x0100 0x0100
1244
1245/*
1246** Create/destroy an Fts5Index object.
1247*/
1248static int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
1249static int sqlite3Fts5IndexClose(Fts5Index *p);
1250
1251/*
1252** Return a simple checksum value based on the arguments.
1253*/
1254static u64 sqlite3Fts5IndexEntryCksum(
1255 i64 iRowid,
1256 int iCol,
1257 int iPos,
1258 int iIdx,
1259 const char *pTerm,
1260 int nTerm
1261);
1262
1263/*
1264** Argument p points to a buffer containing utf-8 text that is n bytes in
1265** size. Return the number of bytes in the nChar character prefix of the
1266** buffer, or 0 if there are less than nChar characters in total.
1267*/
1268static int sqlite3Fts5IndexCharlenToBytelen(
1269 const char *p,
1270 int nByte,
1271 int nChar
1272);
1273
1274/*
1275** Open a new iterator to iterate though all rowids that match the
1276** specified token or token prefix.
1277*/
1278static int sqlite3Fts5IndexQuery(
1279 Fts5Index *p, /* FTS index to query */
1280 const char *pToken, int nToken, /* Token (or prefix) to query for */
1281 int flags, /* Mask of FTS5INDEX_QUERY_X flags */
1282 Fts5Colset *pColset, /* Match these columns only */
1283 Fts5IndexIter **ppIter /* OUT: New iterator object */
1284);
1285
1286/*
1287** The various operations on open token or token prefix iterators opened
1288** using sqlite3Fts5IndexQuery().
1289*/
1290static int sqlite3Fts5IterNext(Fts5IndexIter*);
1291static int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
1292
1293/*
1294** Close an iterator opened by sqlite3Fts5IndexQuery().
1295*/
1296static void sqlite3Fts5IterClose(Fts5IndexIter*);
1297
1298/*
1299** Close the reader blob handle, if it is open.
1300*/
1301static void sqlite3Fts5IndexCloseReader(Fts5Index*);
1302
1303/*
1304** This interface is used by the fts5vocab module.
1305*/
1306static const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
1307static int sqlite3Fts5IterNextScan(Fts5IndexIter*);
1308static void *sqlite3Fts5StructureRef(Fts5Index*);
1309static void sqlite3Fts5StructureRelease(void*);
1310static int sqlite3Fts5StructureTest(Fts5Index*, void*);
1311
1312/*
1313** Used by xInstToken():
1314*/
1315static int sqlite3Fts5IterToken(
1316 Fts5IndexIter *pIndexIter,
1317 const char *pToken, int nToken,
1318 i64 iRowid,
1319 int iCol,
1320 int iOff,
1321 const char **ppOut, int *pnOut
1322);
1323
1324/*
1325** Insert or remove data to or from the index. Each time a document is
1326** added to or removed from the index, this function is called one or more
1327** times.
1328**
1329** For an insert, it must be called once for each token in the new document.
1330** If the operation is a delete, it must be called (at least) once for each
1331** unique token in the document with an iCol value less than zero. The iPos
1332** argument is ignored for a delete.
1333*/
1334static int sqlite3Fts5IndexWrite(
1335 Fts5Index *p, /* Index to write to */
1336 int iCol, /* Column token appears in (-ve -> delete) */
1337 int iPos, /* Position of token within column */
1338 const char *pToken, int nToken /* Token to add or remove to or from index */
1339);
1340
1341/*
1342** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
1343** document iDocid.
1344*/
1345static int sqlite3Fts5IndexBeginWrite(
1346 Fts5Index *p, /* Index to write to */
1347 int bDelete, /* True if current operation is a delete */
1348 i64 iDocid /* Docid to add or remove data from */
1349);
1350
1351/*
1352** Flush any data stored in the in-memory hash tables to the database.
1353** Also close any open blob handles.
1354*/
1355static int sqlite3Fts5IndexSync(Fts5Index *p);
1356
1357/*
1358** Discard any data stored in the in-memory hash tables. Do not write it
1359** to the database. Additionally, assume that the contents of the %_data
1360** table may have changed on disk. So any in-memory caches of %_data
1361** records must be invalidated.
1362*/
1363static int sqlite3Fts5IndexRollback(Fts5Index *p);
1364
1365/*
1366** Get or set the "averages" values.
1367*/
1368static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
1369static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
1370
1371/*
1372** Functions called by the storage module as part of integrity-check.
1373*/
1374static int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum, int bUseCksum);
1375
1376/*
1377** Called during virtual module initialization to register UDF
1378** fts5_decode() with SQLite
1379*/
1380static int sqlite3Fts5IndexInit(sqlite3*);
1381
1382static int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
1383
1384/*
1385** Return the total number of entries read from the %_data table by
1386** this connection since it was created.
1387*/
1388static int sqlite3Fts5IndexReads(Fts5Index *p);
1389
1390static int sqlite3Fts5IndexReinit(Fts5Index *p);
1391static int sqlite3Fts5IndexOptimize(Fts5Index *p);
1392static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
1393static int sqlite3Fts5IndexReset(Fts5Index *p);
1394
1395static int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
1396
1397static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
1398static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
1399
1400static void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
1401
1402/* Used to populate hash tables for xInstToken in detail=none/column mode. */
1403static int sqlite3Fts5IndexIterWriteTokendata(
1404 Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff
1405);
1406
1407/*
1408** End of interface to code in fts5_index.c.
1409**************************************************************************/
1410
1411/**************************************************************************
1412** Interface to code in fts5_varint.c.
1413*/
1414static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
1415static int sqlite3Fts5GetVarintLen(u32 iVal);
1416static u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
1417static int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
1418
1419#define fts5GetVarint32(a,b)sqlite3Fts5GetVarint32(a,(u32*)&(b)) sqlite3Fts5GetVarint32(a,(u32*)&(b))
1420#define fts5GetVarintsqlite3Fts5GetVarint sqlite3Fts5GetVarint
1421
1422#define fts5FastGetVarint32(a, iOff, nVal){ nVal = (a)[iOff++]; if( nVal & 0x80 ){ iOff--; iOff += sqlite3Fts5GetVarint32
(&(a)[iOff],(u32*)&(nVal)); } } { \
1423 nVal = (a)[iOff++]; \
1424 if( nVal & 0x80 ){ \
1425 iOff--; \
1426 iOff += fts5GetVarint32(&(a)[iOff], nVal)sqlite3Fts5GetVarint32(&(a)[iOff],(u32*)&(nVal)); \
1427 } \
1428}
1429
1430
1431/*
1432** End of interface to code in fts5_varint.c.
1433**************************************************************************/
1434
1435
1436/**************************************************************************
1437** Interface to code in fts5_main.c.
1438*/
1439
1440/*
1441** Virtual-table object.
1442*/
1443typedef struct Fts5Table Fts5Table;
1444struct Fts5Table {
1445 sqlite3_vtab base; /* Base class used by SQLite core */
1446 Fts5Config *pConfig; /* Virtual table configuration */
1447 Fts5Index *pIndex; /* Full-text index */
1448};
1449
1450static int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig);
1451
1452static Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
1453
1454static int sqlite3Fts5FlushToDisk(Fts5Table*);
1455
1456static void sqlite3Fts5ClearLocale(Fts5Config *pConfig);
1457static void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc);
1458
1459static int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal);
1460static int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal,
1461 const char **ppText, int *pnText, const char **ppLoc, int *pnLoc
1462);
1463
1464/*
1465** End of interface to code in fts5.c.
1466**************************************************************************/
1467
1468/**************************************************************************
1469** Interface to code in fts5_hash.c.
1470*/
1471typedef struct Fts5Hash Fts5Hash;
1472
1473/*
1474** Create a hash table, free a hash table.
1475*/
1476static int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
1477static void sqlite3Fts5HashFree(Fts5Hash*);
1478
1479static int sqlite3Fts5HashWrite(
1480 Fts5Hash*,
1481 i64 iRowid, /* Rowid for this entry */
1482 int iCol, /* Column token appears in (-ve -> delete) */
1483 int iPos, /* Position of token within column */
1484 char bByte,
1485 const char *pToken, int nToken /* Token to add or remove to or from index */
1486);
1487
1488/*
1489** Empty (but do not delete) a hash table.
1490*/
1491static void sqlite3Fts5HashClear(Fts5Hash*);
1492
1493/*
1494** Return true if the hash is empty, false otherwise.
1495*/
1496static int sqlite3Fts5HashIsEmpty(Fts5Hash*);
1497
1498static int sqlite3Fts5HashQuery(
1499 Fts5Hash*, /* Hash table to query */
1500 int nPre,
1501 const char *pTerm, int nTerm, /* Query term */
1502 void **ppObj, /* OUT: Pointer to doclist for pTerm */
1503 int *pnDoclist /* OUT: Size of doclist in bytes */
1504);
1505
1506static int sqlite3Fts5HashScanInit(
1507 Fts5Hash*, /* Hash table to query */
1508 const char *pTerm, int nTerm /* Query prefix */
1509);
1510static void sqlite3Fts5HashScanNext(Fts5Hash*);
1511static int sqlite3Fts5HashScanEof(Fts5Hash*);
1512static void sqlite3Fts5HashScanEntry(Fts5Hash *,
1513 const char **pzTerm, /* OUT: term (nul-terminated) */
1514 int *pnTerm, /* OUT: Size of term in bytes */
1515 const u8 **ppDoclist, /* OUT: pointer to doclist */
1516 int *pnDoclist /* OUT: size of doclist in bytes */
1517);
1518
1519
1520
1521/*
1522** End of interface to code in fts5_hash.c.
1523**************************************************************************/
1524
1525/**************************************************************************
1526** Interface to code in fts5_storage.c. fts5_storage.c contains contains
1527** code to access the data stored in the %_content and %_docsize tables.
1528*/
1529
1530#define FTS5_STMT_SCAN_ASC0 0 /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
1531#define FTS5_STMT_SCAN_DESC1 1 /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
1532#define FTS5_STMT_LOOKUP2 2 /* SELECT rowid, * FROM ... WHERE rowid=? */
1533
1534typedef struct Fts5Storage Fts5Storage;
1535
1536static int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
1537static int sqlite3Fts5StorageClose(Fts5Storage *p);
1538static int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
1539
1540static int sqlite3Fts5DropAll(Fts5Config*);
1541static int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
1542
1543static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int);
1544static int sqlite3Fts5StorageContentInsert(Fts5Storage *p, int, sqlite3_value**, i64*);
1545static int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
1546
1547static int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg);
1548
1549static int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
1550static void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
1551
1552static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
1553static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
1554static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
1555
1556static int sqlite3Fts5StorageSync(Fts5Storage *p);
1557static int sqlite3Fts5StorageRollback(Fts5Storage *p);
1558
1559static int sqlite3Fts5StorageConfigValue(
1560 Fts5Storage *p, const char*, sqlite3_value*, int
1561);
1562
1563static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
1564static int sqlite3Fts5StorageRebuild(Fts5Storage *p);
1565static int sqlite3Fts5StorageOptimize(Fts5Storage *p);
1566static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
1567static int sqlite3Fts5StorageReset(Fts5Storage *p);
1568
1569static void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*);
1570static int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel);
1571
1572/*
1573** End of interface to code in fts5_storage.c.
1574**************************************************************************/
1575
1576
1577/**************************************************************************
1578** Interface to code in fts5_expr.c.
1579*/
1580typedef struct Fts5Expr Fts5Expr;
1581typedef struct Fts5ExprNode Fts5ExprNode;
1582typedef struct Fts5Parse Fts5Parse;
1583typedef struct Fts5Token Fts5Token;
1584typedef struct Fts5ExprPhrase Fts5ExprPhrase;
1585typedef struct Fts5ExprNearset Fts5ExprNearset;
1586
1587struct Fts5Token {
1588 const char *p; /* Token text (not NULL terminated) */
1589 int n; /* Size of buffer p in bytes */
1590};
1591
1592/* Parse a MATCH expression. */
1593static int sqlite3Fts5ExprNew(
1594 Fts5Config *pConfig,
1595 int bPhraseToAnd,
1596 int iCol, /* Column on LHS of MATCH operator */
1597 const char *zExpr,
1598 Fts5Expr **ppNew,
1599 char **pzErr
1600);
1601static int sqlite3Fts5ExprPattern(
1602 Fts5Config *pConfig,
1603 int bGlob,
1604 int iCol,
1605 const char *zText,
1606 Fts5Expr **pp
1607);
1608
1609/*
1610** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
1611** rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
1612** rc = sqlite3Fts5ExprNext(pExpr)
1613** ){
1614** // The document with rowid iRowid matches the expression!
1615** i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
1616** }
1617*/
1618static int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, i64, int bDesc);
1619static int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
1620static int sqlite3Fts5ExprEof(Fts5Expr*);
1621static i64 sqlite3Fts5ExprRowid(Fts5Expr*);
1622
1623static void sqlite3Fts5ExprFree(Fts5Expr*);
1624static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2);
1625
1626/* Called during startup to register a UDF with SQLite */
1627static int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
1628
1629static int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
1630static int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
1631static int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
1632
1633typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
1634static Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
1635static int sqlite3Fts5ExprPopulatePoslists(
1636 Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
1637);
1638static void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
1639
1640static int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
1641
1642static int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
1643
1644static int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
1645static int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
1646static void sqlite3Fts5ExprClearTokens(Fts5Expr*);
1647
1648/*******************************************
1649** The fts5_expr.c API above this point is used by the other hand-written
1650** C code in this module. The interfaces below this point are called by
1651** the parser code in fts5parse.y. */
1652
1653static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
1654
1655static Fts5ExprNode *sqlite3Fts5ParseNode(
1656 Fts5Parse *pParse,
1657 int eType,
1658 Fts5ExprNode *pLeft,
1659 Fts5ExprNode *pRight,
1660 Fts5ExprNearset *pNear
1661);
1662
1663static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
1664 Fts5Parse *pParse,
1665 Fts5ExprNode *pLeft,
1666 Fts5ExprNode *pRight
1667);
1668
1669static Fts5ExprPhrase *sqlite3Fts5ParseTerm(
1670 Fts5Parse *pParse,
1671 Fts5ExprPhrase *pPhrase,
1672 Fts5Token *pToken,
1673 int bPrefix
1674);
1675
1676static void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
1677
1678static Fts5ExprNearset *sqlite3Fts5ParseNearset(
1679 Fts5Parse*,
1680 Fts5ExprNearset*,
1681 Fts5ExprPhrase*
1682);
1683
1684static Fts5Colset *sqlite3Fts5ParseColset(
1685 Fts5Parse*,
1686 Fts5Colset*,
1687 Fts5Token *
1688);
1689
1690static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
1691static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
1692static void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
1693
1694static void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
1695static void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
1696static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
1697static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
1698static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
1699
1700/*
1701** End of interface to code in fts5_expr.c.
1702**************************************************************************/
1703
1704
1705
1706/**************************************************************************
1707** Interface to code in fts5_aux.c.
1708*/
1709
1710static int sqlite3Fts5AuxInit(fts5_api*);
1711/*
1712** End of interface to code in fts5_aux.c.
1713**************************************************************************/
1714
1715/**************************************************************************
1716** Interface to code in fts5_tokenizer.c.
1717*/
1718
1719static int sqlite3Fts5TokenizerInit(fts5_api*);
1720static int sqlite3Fts5TokenizerPattern(
1721 int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
1722 Fts5Tokenizer *pTok
1723);
1724static int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*);
1725/*
1726** End of interface to code in fts5_tokenizer.c.
1727**************************************************************************/
1728
1729/**************************************************************************
1730** Interface to code in fts5_vocab.c.
1731*/
1732
1733static int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
1734
1735/*
1736** End of interface to code in fts5_vocab.c.
1737**************************************************************************/
1738
1739
1740/**************************************************************************
1741** Interface to automatically generated code in fts5_unicode2.c.
1742*/
1743static int sqlite3Fts5UnicodeIsdiacritic(int c);
1744static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
1745
1746static int sqlite3Fts5UnicodeCatParse(const char*, u8*);
1747static int sqlite3Fts5UnicodeCategory(u32 iCode);
1748static void sqlite3Fts5UnicodeAscii(u8*, u8*);
1749/*
1750** End of interface to code in fts5_unicode2.c.
1751**************************************************************************/
1752
1753#endif
1754
1755#line 1 "fts5parse.h"
1756#define FTS5_OR1 1
1757#define FTS5_AND2 2
1758#define FTS5_NOT3 3
1759#define FTS5_TERM4 4
1760#define FTS5_COLON5 5
1761#define FTS5_MINUS6 6
1762#define FTS5_LCP7 7
1763#define FTS5_RCP8 8
1764#define FTS5_STRING9 9
1765#define FTS5_LP10 10
1766#define FTS5_RP11 11
1767#define FTS5_CARET12 12
1768#define FTS5_COMMA13 13
1769#define FTS5_PLUS14 14
1770#define FTS5_STAR15 15
1771
1772#line 1 "fts5parse.c"
1773/* This file is automatically generated by Lemon from input grammar
1774** source file "fts5parse.y".
1775*/
1776/*
1777** 2000-05-29
1778**
1779** The author disclaims copyright to this source code. In place of
1780** a legal notice, here is a blessing:
1781**
1782** May you do good and not evil.
1783** May you find forgiveness for yourself and forgive others.
1784** May you share freely, never taking more than you give.
1785**
1786*************************************************************************
1787** Driver template for the LEMON parser generator.
1788**
1789** The "lemon" program processes an LALR(1) input grammar file, then uses
1790** this template to construct a parser. The "lemon" program inserts text
1791** at each "%%" line. Also, any "P-a-r-s-e" identifier prefix (without the
1792** interstitial "-" characters) contained in this template is changed into
1793** the value of the %name directive from the grammar. Otherwise, the content
1794** of this template is copied straight through into the generate parser
1795** source file.
1796**
1797** The following is the concatenation of all %include directives from the
1798** input grammar file:
1799*/
1800/************ Begin %include sections from the grammar ************************/
1801#line 47 "fts5parse.y"
1802
1803/* #include "fts5Int.h" */
1804/* #include "fts5parse.h" */
1805
1806/*
1807** Disable all error recovery processing in the parser push-down
1808** automaton.
1809*/
1810#define fts5YYNOERRORRECOVERY1 1
1811
1812/*
1813** Make fts5yytestcase() the same as testcase()
1814*/
1815#define fts5yytestcase(X) testcase(X)
1816
1817/*
1818** Indicate that sqlite3ParserFree() will never be called with a null
1819** pointer.
1820*/
1821#define fts5YYPARSEFREENOTNULL1 1
1822
1823/*
1824** Alternative datatype for the argument to the malloc() routine passed
1825** into sqlite3ParserAlloc(). The default is size_t.
1826*/
1827#define fts5YYMALLOCARGTYPEu64 u64
1828
1829#line 58 "fts5parse.c"
1830/**************** End of %include directives **********************************/
1831/* These constants specify the various numeric values for terminal symbols.
1832***************** Begin token definitions *************************************/
1833#ifndef FTS5_OR1
1834#define FTS5_OR1 1
1835#define FTS5_AND2 2
1836#define FTS5_NOT3 3
1837#define FTS5_TERM4 4
1838#define FTS5_COLON5 5
1839#define FTS5_MINUS6 6
1840#define FTS5_LCP7 7
1841#define FTS5_RCP8 8
1842#define FTS5_STRING9 9
1843#define FTS5_LP10 10
1844#define FTS5_RP11 11
1845#define FTS5_CARET12 12
1846#define FTS5_COMMA13 13
1847#define FTS5_PLUS14 14
1848#define FTS5_STAR15 15
1849#endif
1850/**************** End token definitions ***************************************/
1851
1852/* The next sections is a series of control #defines.
1853** various aspects of the generated parser.
1854** fts5YYCODETYPE is the data type used to store the integer codes
1855** that represent terminal and non-terminal symbols.
1856** "unsigned char" is used if there are fewer than
1857** 256 symbols. Larger types otherwise.
1858** fts5YYNOCODE is a number of type fts5YYCODETYPE that is not used for
1859** any terminal or nonterminal symbol.
1860** fts5YYFALLBACK If defined, this indicates that one or more tokens
1861** (also known as: "terminal symbols") have fall-back
1862** values which should be used if the original symbol
1863** would not parse. This permits keywords to sometimes
1864** be used as identifiers, for example.
1865** fts5YYACTIONTYPE is the data type used for "action codes" - numbers
1866** that indicate what to do in response to the next
1867** token.
1868** sqlite3Fts5ParserFTS5TOKENTYPE is the data type used for minor type for terminal
1869** symbols. Background: A "minor type" is a semantic
1870** value associated with a terminal or non-terminal
1871** symbols. For example, for an "ID" terminal symbol,
1872** the minor type might be the name of the identifier.
1873** Each non-terminal can have a different minor type.
1874** Terminal symbols all have the same minor type, though.
1875** This macros defines the minor type for terminal
1876** symbols.
1877** fts5YYMINORTYPE is the data type used for all minor types.
1878** This is typically a union of many types, one of
1879** which is sqlite3Fts5ParserFTS5TOKENTYPE. The entry in the union
1880** for terminal symbols is called "fts5yy0".
1881** fts5YYSTACKDEPTH is the maximum depth of the parser's stack. If
1882** zero the stack is dynamically sized using realloc()
1883** sqlite3Fts5ParserARG_SDECL A static variable declaration for the %extra_argument
1884** sqlite3Fts5ParserARG_PDECL A parameter declaration for the %extra_argument
1885** sqlite3Fts5ParserARG_PARAM Code to pass %extra_argument as a subroutine parameter
1886** sqlite3Fts5ParserARG_STORE Code to store %extra_argument into fts5yypParser
1887** sqlite3Fts5ParserARG_FETCH Code to extract %extra_argument from fts5yypParser
1888** sqlite3Fts5ParserCTX_* As sqlite3Fts5ParserARG_ except for %extra_context
1889** fts5YYREALLOC Name of the realloc() function to use
1890** fts5YYFREE Name of the free() function to use
1891** fts5YYDYNSTACK True if stack space should be extended on heap
1892** fts5YYERRORSYMBOL is the code number of the error symbol. If not
1893** defined, then do no error processing.
1894** fts5YYNSTATE the combined number of states.
1895** fts5YYNRULE the number of rules in the grammar
1896** fts5YYNFTS5TOKEN Number of terminal symbols
1897** fts5YY_MAX_SHIFT Maximum value for shift actions
1898** fts5YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions
1899** fts5YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions
1900** fts5YY_ERROR_ACTION The fts5yy_action[] code for syntax error
1901** fts5YY_ACCEPT_ACTION The fts5yy_action[] code for accept
1902** fts5YY_NO_ACTION The fts5yy_action[] code for no-op
1903** fts5YY_MIN_REDUCE Minimum value for reduce actions
1904** fts5YY_MAX_REDUCE Maximum value for reduce actions
1905** fts5YY_MIN_DSTRCTR Minimum symbol value that has a destructor
1906** fts5YY_MAX_DSTRCTR Maximum symbol value that has a destructor
1907*/
1908#ifndef INTERFACE1
1909# define INTERFACE1 1
1910#endif
1911/************* Begin control #defines *****************************************/
1912#define fts5YYCODETYPEunsigned char unsigned char
1913#define fts5YYNOCODE27 27
1914#define fts5YYACTIONTYPEunsigned char unsigned char
1915#define sqlite3Fts5ParserFTS5TOKENTYPEFts5Token Fts5Token
1916typedef union {
1917 int fts5yyinit;
1918 sqlite3Fts5ParserFTS5TOKENTYPEFts5Token fts5yy0;
1919 int fts5yy4;
1920 Fts5Colset* fts5yy11;
1921 Fts5ExprNode* fts5yy24;
1922 Fts5ExprNearset* fts5yy46;
1923 Fts5ExprPhrase* fts5yy53;
1924} fts5YYMINORTYPE;
1925#ifndef fts5YYSTACKDEPTH100
1926#define fts5YYSTACKDEPTH100 100
1927#endif
1928#define sqlite3Fts5ParserARG_SDECLFts5Parse *pParse; Fts5Parse *pParse;
1929#define sqlite3Fts5ParserARG_PDECL,Fts5Parse *pParse ,Fts5Parse *pParse
1930#define sqlite3Fts5ParserARG_PARAM,pParse ,pParse
1931#define sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse; Fts5Parse *pParse=fts5yypParser->pParse;
1932#define sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse; fts5yypParser->pParse=pParse;
1933#undef fts5YYREALLOCrealloc
1934#define fts5YYREALLOCrealloc realloc
1935#undef fts5YYFREEfree
1936#define fts5YYFREEfree free
1937#undef fts5YYDYNSTACK0
1938#define fts5YYDYNSTACK0 0
1939#undef fts5YYSIZELIMIT
1940#define sqlite3Fts5ParserCTX(P)0 0
1941#define sqlite3Fts5ParserCTX_SDECL
1942#define sqlite3Fts5ParserCTX_PDECL
1943#define sqlite3Fts5ParserCTX_PARAM
1944#define sqlite3Fts5ParserCTX_FETCH
1945#define sqlite3Fts5ParserCTX_STORE
1946#undef fts5YYERRORSYMBOL
1947#undef fts5YYERRSYMDT
1948#undef fts5YYFALLBACK
1949#define fts5YYNSTATE35 35
1950#define fts5YYNRULE28 28
1951#define fts5YYNRULE_WITH_ACTION28 28
1952#define fts5YYNFTS5TOKEN16 16
1953#define fts5YY_MAX_SHIFT34 34
1954#define fts5YY_MIN_SHIFTREDUCE52 52
1955#define fts5YY_MAX_SHIFTREDUCE79 79
1956#define fts5YY_ERROR_ACTION80 80
1957#define fts5YY_ACCEPT_ACTION81 81
1958#define fts5YY_NO_ACTION82 82
1959#define fts5YY_MIN_REDUCE83 83
1960#define fts5YY_MAX_REDUCE110 110
1961#define fts5YY_MIN_DSTRCTR16 16
1962#define fts5YY_MAX_DSTRCTR24 24
1963/************* End control #defines *******************************************/
1964#define fts5YY_NLOOKAHEAD((int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0]))) ((int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0])))
1965
1966/* Define the fts5yytestcase() macro to be a no-op if is not already defined
1967** otherwise.
1968**
1969** Applications can choose to define fts5yytestcase() in the %include section
1970** to a macro that can assist in verifying code coverage. For production
1971** code the fts5yytestcase() macro should be turned off. But it is useful
1972** for testing.
1973*/
1974#ifndef fts5yytestcase
1975# define fts5yytestcase(X)
1976#endif
1977
1978/* Macro to determine if stack space has the ability to grow using
1979** heap memory.
1980*/
1981#if fts5YYSTACKDEPTH100<=0 || fts5YYDYNSTACK0
1982# define fts5YYGROWABLESTACK0 1
1983#else
1984# define fts5YYGROWABLESTACK0 0
1985#endif
1986
1987/* Guarantee a minimum number of initial stack slots.
1988*/
1989#if fts5YYSTACKDEPTH100<=0
1990# undef fts5YYSTACKDEPTH100
1991# define fts5YYSTACKDEPTH100 2 /* Need a minimum stack size */
1992#endif
1993
1994
1995/* Next are the tables used to determine what action to take based on the
1996** current state and lookahead token. These tables are used to implement
1997** functions that take a state number and lookahead value and return an
1998** action integer.
1999**
2000** Suppose the action integer is N. Then the action is determined as
2001** follows
2002**
2003** 0 <= N <= fts5YY_MAX_SHIFT Shift N. That is, push the lookahead
2004** token onto the stack and goto state N.
2005**
2006** N between fts5YY_MIN_SHIFTREDUCE Shift to an arbitrary state then
2007** and fts5YY_MAX_SHIFTREDUCE reduce by rule N-fts5YY_MIN_SHIFTREDUCE.
2008**
2009** N == fts5YY_ERROR_ACTION A syntax error has occurred.
2010**
2011** N == fts5YY_ACCEPT_ACTION The parser accepts its input.
2012**
2013** N == fts5YY_NO_ACTION No such action. Denotes unused
2014** slots in the fts5yy_action[] table.
2015**
2016** N between fts5YY_MIN_REDUCE Reduce by rule N-fts5YY_MIN_REDUCE
2017** and fts5YY_MAX_REDUCE
2018**
2019** The action table is constructed as a single large table named fts5yy_action[].
2020** Given state S and lookahead X, the action is computed as either:
2021**
2022** (A) N = fts5yy_action[ fts5yy_shift_ofst[S] + X ]
2023** (B) N = fts5yy_default[S]
2024**
2025** The (A) formula is preferred. The B formula is used instead if
2026** fts5yy_lookahead[fts5yy_shift_ofst[S]+X] is not equal to X.
2027**
2028** The formulas above are for computing the action when the lookahead is
2029** a terminal symbol. If the lookahead is a non-terminal (as occurs after
2030** a reduce action) then the fts5yy_reduce_ofst[] array is used in place of
2031** the fts5yy_shift_ofst[] array.
2032**
2033** The following are the tables generated in this section:
2034**
2035** fts5yy_action[] A single table containing all actions.
2036** fts5yy_lookahead[] A table containing the lookahead for each entry in
2037** fts5yy_action. Used to detect hash collisions.
2038** fts5yy_shift_ofst[] For each state, the offset into fts5yy_action for
2039** shifting terminals.
2040** fts5yy_reduce_ofst[] For each state, the offset into fts5yy_action for
2041** shifting non-terminals after a reduce.
2042** fts5yy_default[] Default action for each state.
2043**
2044*********** Begin parsing tables **********************************************/
2045#define fts5YY_ACTTAB_COUNT(105) (105)
2046static const fts5YYACTIONTYPEunsigned char fts5yy_action[] = {
2047 /* 0 */ 81, 20, 96, 6, 28, 99, 98, 26, 26, 18,
2048 /* 10 */ 96, 6, 28, 17, 98, 56, 26, 19, 96, 6,
2049 /* 20 */ 28, 14, 98, 14, 26, 31, 92, 96, 6, 28,
2050 /* 30 */ 108, 98, 25, 26, 21, 96, 6, 28, 78, 98,
2051 /* 40 */ 58, 26, 29, 96, 6, 28, 107, 98, 22, 26,
2052 /* 50 */ 24, 16, 12, 11, 1, 13, 13, 24, 16, 23,
2053 /* 60 */ 11, 33, 34, 13, 97, 8, 27, 32, 98, 7,
2054 /* 70 */ 26, 3, 4, 5, 3, 4, 5, 3, 83, 4,
2055 /* 80 */ 5, 3, 63, 5, 3, 62, 12, 2, 86, 13,
2056 /* 90 */ 9, 30, 10, 10, 54, 57, 75, 78, 78, 53,
2057 /* 100 */ 57, 15, 82, 82, 71,
2058};
2059static const fts5YYCODETYPEunsigned char fts5yy_lookahead[] = {
2060 /* 0 */ 16, 17, 18, 19, 20, 22, 22, 24, 24, 17,
2061 /* 10 */ 18, 19, 20, 7, 22, 9, 24, 17, 18, 19,
2062 /* 20 */ 20, 9, 22, 9, 24, 13, 17, 18, 19, 20,
2063 /* 30 */ 26, 22, 24, 24, 17, 18, 19, 20, 15, 22,
2064 /* 40 */ 9, 24, 17, 18, 19, 20, 26, 22, 21, 24,
2065 /* 50 */ 6, 7, 9, 9, 10, 12, 12, 6, 7, 21,
2066 /* 60 */ 9, 24, 25, 12, 18, 5, 20, 14, 22, 5,
2067 /* 70 */ 24, 3, 1, 2, 3, 1, 2, 3, 0, 1,
2068 /* 80 */ 2, 3, 11, 2, 3, 11, 9, 10, 5, 12,
2069 /* 90 */ 23, 24, 10, 10, 8, 9, 9, 15, 15, 8,
2070 /* 100 */ 9, 9, 27, 27, 11, 27, 27, 27, 27, 27,
2071 /* 110 */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
2072 /* 120 */ 27,
2073};
2074#define fts5YY_SHIFT_COUNT(34) (34)
2075#define fts5YY_SHIFT_MIN(0) (0)
2076#define fts5YY_SHIFT_MAX(93) (93)
2077static const unsigned char fts5yy_shift_ofst[] = {
2078 /* 0 */ 44, 44, 44, 44, 44, 44, 51, 77, 43, 12,
2079 /* 10 */ 14, 83, 82, 14, 23, 23, 31, 31, 71, 74,
2080 /* 20 */ 78, 81, 86, 91, 6, 53, 53, 60, 64, 68,
2081 /* 30 */ 53, 87, 92, 53, 93,
2082};
2083#define fts5YY_REDUCE_COUNT(17) (17)
2084#define fts5YY_REDUCE_MIN(-17) (-17)
2085#define fts5YY_REDUCE_MAX(67) (67)
2086static const signed char fts5yy_reduce_ofst[] = {
2087 /* 0 */ -16, -8, 0, 9, 17, 25, 46, -17, -17, 37,
2088 /* 10 */ 67, 4, 4, 8, 4, 20, 27, 38,
2089};
2090static const fts5YYACTIONTYPEunsigned char fts5yy_default[] = {
2091 /* 0 */ 80, 80, 80, 80, 80, 80, 95, 80, 80, 105,
2092 /* 10 */ 80, 110, 110, 80, 110, 110, 80, 80, 80, 80,
2093 /* 20 */ 80, 91, 80, 80, 80, 101, 100, 80, 80, 90,
2094 /* 30 */ 103, 80, 80, 104, 80,
2095};
2096/********** End of lemon-generated parsing tables *****************************/
2097
2098/* The next table maps tokens (terminal symbols) into fallback tokens.
2099** If a construct like the following:
2100**
2101** %fallback ID X Y Z.
2102**
2103** appears in the grammar, then ID becomes a fallback token for X, Y,
2104** and Z. Whenever one of the tokens X, Y, or Z is input to the parser
2105** but it does not parse, the type of the token is changed to ID and
2106** the parse is retried before an error is thrown.
2107**
2108** This feature can be used, for example, to cause some keywords in a language
2109** to revert to identifiers if they keyword does not apply in the context where
2110** it appears.
2111*/
2112#ifdef fts5YYFALLBACK
2113static const fts5YYCODETYPEunsigned char fts5yyFallback[] = {
2114};
2115#endif /* fts5YYFALLBACK */
2116
2117/* The following structure represents a single element of the
2118** parser's stack. Information stored includes:
2119**
2120** + The state number for the parser at this level of the stack.
2121**
2122** + The value of the token stored at this level of the stack.
2123** (In other words, the "major" token.)
2124**
2125** + The semantic value stored at this level of the stack. This is
2126** the information used by the action routines in the grammar.
2127** It is sometimes called the "minor" token.
2128**
2129** After the "shift" half of a SHIFTREDUCE action, the stateno field
2130** actually contains the reduce action for the second half of the
2131** SHIFTREDUCE.
2132*/
2133struct fts5yyStackEntry {
2134 fts5YYACTIONTYPEunsigned char stateno; /* The state-number, or reduce action in SHIFTREDUCE */
2135 fts5YYCODETYPEunsigned char major; /* The major token value. This is the code
2136 ** number for the token at this stack level */
2137 fts5YYMINORTYPE minor; /* The user-supplied minor token value. This
2138 ** is the value of the token */
2139};
2140typedef struct fts5yyStackEntry fts5yyStackEntry;
2141
2142/* The state of the parser is completely contained in an instance of
2143** the following structure */
2144struct fts5yyParser {
2145 fts5yyStackEntry *fts5yytos; /* Pointer to top element of the stack */
2146#ifdef fts5YYTRACKMAXSTACKDEPTH
2147 int fts5yyhwm; /* High-water mark of the stack */
2148#endif
2149#ifndef fts5YYNOERRORRECOVERY1
2150 int fts5yyerrcnt; /* Shifts left before out of the error */
2151#endif
2152 sqlite3Fts5ParserARG_SDECLFts5Parse *pParse; /* A place to hold %extra_argument */
2153 sqlite3Fts5ParserCTX_SDECL /* A place to hold %extra_context */
2154 fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */
2155 fts5yyStackEntry *fts5yystack; /* The parser stack */
2156 fts5yyStackEntry fts5yystk0[fts5YYSTACKDEPTH100]; /* Initial stack space */
2157};
2158typedef struct fts5yyParser fts5yyParser;
2159
2160#include <assert.h>
2161#ifndef NDEBUG1
2162#include <stdio.h>
2163static FILE *fts5yyTraceFILE = 0;
2164static char *fts5yyTracePrompt = 0;
2165#endif /* NDEBUG */
2166
2167#ifndef NDEBUG1
2168/*
2169** Turn parser tracing on by giving a stream to which to write the trace
2170** and a prompt to preface each trace message. Tracing is turned off
2171** by making either argument NULL
2172**
2173** Inputs:
2174** <ul>
2175** <li> A FILE* to which trace output should be written.
2176** If NULL, then tracing is turned off.
2177** <li> A prefix string written at the beginning of every
2178** line of trace output. If NULL, then tracing is
2179** turned off.
2180** </ul>
2181**
2182** Outputs:
2183** None.
2184*/
2185static void sqlite3Fts5ParserTrace(FILE *TraceFILE, char *zTracePrompt){
2186 fts5yyTraceFILE = TraceFILE;
2187 fts5yyTracePrompt = zTracePrompt;
2188 if( fts5yyTraceFILE==0 ) fts5yyTracePrompt = 0;
2189 else if( fts5yyTracePrompt==0 ) fts5yyTraceFILE = 0;
2190}
2191#endif /* NDEBUG */
2192
2193#if defined(fts5YYCOVERAGE) || !defined(NDEBUG1)
2194/* For tracing shifts, the names of all terminals and nonterminals
2195** are required. The following table supplies these names */
2196static const char *const fts5yyTokenName[] = {
2197 /* 0 */ "$",
2198 /* 1 */ "OR",
2199 /* 2 */ "AND",
2200 /* 3 */ "NOT",
2201 /* 4 */ "TERM",
2202 /* 5 */ "COLON",
2203 /* 6 */ "MINUS",
2204 /* 7 */ "LCP",
2205 /* 8 */ "RCP",
2206 /* 9 */ "STRING",
2207 /* 10 */ "LP",
2208 /* 11 */ "RP",
2209 /* 12 */ "CARET",
2210 /* 13 */ "COMMA",
2211 /* 14 */ "PLUS",
2212 /* 15 */ "STAR",
2213 /* 16 */ "input",
2214 /* 17 */ "expr",
2215 /* 18 */ "cnearset",
2216 /* 19 */ "exprlist",
2217 /* 20 */ "colset",
2218 /* 21 */ "colsetlist",
2219 /* 22 */ "nearset",
2220 /* 23 */ "nearphrases",
2221 /* 24 */ "phrase",
2222 /* 25 */ "neardist_opt",
2223 /* 26 */ "star_opt",
2224};
2225#endif /* defined(fts5YYCOVERAGE) || !defined(NDEBUG) */
2226
2227#ifndef NDEBUG1
2228/* For tracing reduce actions, the names of all rules are required.
2229*/
2230static const char *const fts5yyRuleName[] = {
2231 /* 0 */ "input ::= expr",
2232 /* 1 */ "colset ::= MINUS LCP colsetlist RCP",
2233 /* 2 */ "colset ::= LCP colsetlist RCP",
2234 /* 3 */ "colset ::= STRING",
2235 /* 4 */ "colset ::= MINUS STRING",
2236 /* 5 */ "colsetlist ::= colsetlist STRING",
2237 /* 6 */ "colsetlist ::= STRING",
2238 /* 7 */ "expr ::= expr AND expr",
2239 /* 8 */ "expr ::= expr OR expr",
2240 /* 9 */ "expr ::= expr NOT expr",
2241 /* 10 */ "expr ::= colset COLON LP expr RP",
2242 /* 11 */ "expr ::= LP expr RP",
2243 /* 12 */ "expr ::= exprlist",
2244 /* 13 */ "exprlist ::= cnearset",
2245 /* 14 */ "exprlist ::= exprlist cnearset",
2246 /* 15 */ "cnearset ::= nearset",
2247 /* 16 */ "cnearset ::= colset COLON nearset",
2248 /* 17 */ "nearset ::= phrase",
2249 /* 18 */ "nearset ::= CARET phrase",
2250 /* 19 */ "nearset ::= STRING LP nearphrases neardist_opt RP",
2251 /* 20 */ "nearphrases ::= phrase",
2252 /* 21 */ "nearphrases ::= nearphrases phrase",
2253 /* 22 */ "neardist_opt ::=",
2254 /* 23 */ "neardist_opt ::= COMMA STRING",
2255 /* 24 */ "phrase ::= phrase PLUS STRING star_opt",
2256 /* 25 */ "phrase ::= STRING star_opt",
2257 /* 26 */ "star_opt ::= STAR",
2258 /* 27 */ "star_opt ::=",
2259};
2260#endif /* NDEBUG */
2261
2262
2263#if fts5YYGROWABLESTACK0
2264/*
2265** Try to increase the size of the parser stack. Return the number
2266** of errors. Return 0 on success.
2267*/
2268static int fts5yyGrowStack(fts5yyParser *p)1{
2269 int oldSize = 1 + (int)(p->fts5yystackEnd - p->fts5yystack);
2270 int newSize;
2271 int idx;
2272 fts5yyStackEntry *pNew;
2273#ifdef fts5YYSIZELIMIT
2274 int nLimit = fts5YYSIZELIMIT(sqlite3Fts5ParserCTX(p)0);
2275#endif
2276
2277 newSize = oldSize*2 + 100;
2278#ifdef fts5YYSIZELIMIT
2279 if( newSize>nLimit ){
2280 newSize = nLimit;
2281 if( newSize<=oldSize ) return 1;
2282 }
2283#endif
2284 idx = (int)(p->fts5yytos - p->fts5yystack);
2285 if( p->fts5yystack==p->fts5yystk0 ){
2286 pNew = fts5YYREALLOCrealloc(0, newSize*sizeof(pNew[0]), sqlite3Fts5ParserCTX(p)0);
2287 if( pNew==0 ) return 1;
2288 memcpy(pNew, p->fts5yystack, oldSize*sizeof(pNew[0]));
2289 }else{
2290 pNew = fts5YYREALLOCrealloc(p->fts5yystack, newSize*sizeof(pNew[0]), sqlite3Fts5ParserCTX(p)0);
2291 if( pNew==0 ) return 1;
2292 }
2293 p->fts5yystack = pNew;
2294 p->fts5yytos = &p->fts5yystack[idx];
2295#ifndef NDEBUG1
2296 if( fts5yyTraceFILE ){
2297 fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n",
2298 fts5yyTracePrompt, oldSize, newSize);
2299 }
2300#endif
2301 p->fts5yystackEnd = &p->fts5yystack[newSize-1];
2302 return 0;
2303}
2304#endif /* fts5YYGROWABLESTACK */
2305
2306#if !fts5YYGROWABLESTACK0
2307/* For builds that do no have a growable stack, fts5yyGrowStack always
2308** returns an error.
2309*/
2310# define fts5yyGrowStack(X)1 1
2311#endif
2312
2313/* Datatype of the argument to the memory allocated passed as the
2314** second argument to sqlite3Fts5ParserAlloc() below. This can be changed by
2315** putting an appropriate #define in the %include section of the input
2316** grammar.
2317*/
2318#ifndef fts5YYMALLOCARGTYPEu64
2319# define fts5YYMALLOCARGTYPEu64 size_t
2320#endif
2321
2322/* Initialize a new parser that has already been allocated.
2323*/
2324static void sqlite3Fts5ParserInit(void *fts5yypRawParser sqlite3Fts5ParserCTX_PDECL){
2325 fts5yyParser *fts5yypParser = (fts5yyParser*)fts5yypRawParser;
2326 sqlite3Fts5ParserCTX_STORE
2327#ifdef fts5YYTRACKMAXSTACKDEPTH
2328 fts5yypParser->fts5yyhwm = 0;
2329#endif
2330 fts5yypParser->fts5yystack = fts5yypParser->fts5yystk0;
2331 fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH100-1];
2332#ifndef fts5YYNOERRORRECOVERY1
2333 fts5yypParser->fts5yyerrcnt = -1;
2334#endif
2335 fts5yypParser->fts5yytos = fts5yypParser->fts5yystack;
2336 fts5yypParser->fts5yystack[0].stateno = 0;
2337 fts5yypParser->fts5yystack[0].major = 0;
2338}
2339
2340#ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK
2341/*
2342** This function allocates a new parser.
2343** The only argument is a pointer to a function which works like
2344** malloc.
2345**
2346** Inputs:
2347** A pointer to the function used to allocate memory.
2348**
2349** Outputs:
2350** A pointer to a parser. This pointer is used in subsequent calls
2351** to sqlite3Fts5Parser and sqlite3Fts5ParserFree.
2352*/
2353static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(fts5YYMALLOCARGTYPEu64) sqlite3Fts5ParserCTX_PDECL){
2354 fts5yyParser *fts5yypParser;
2355 fts5yypParser = (fts5yyParser*)(*mallocProc)( (fts5YYMALLOCARGTYPEu64)sizeof(fts5yyParser) );
2356 if( fts5yypParser ){
2357 sqlite3Fts5ParserCTX_STORE
2358 sqlite3Fts5ParserInit(fts5yypParser sqlite3Fts5ParserCTX_PARAM);
2359 }
2360 return (void*)fts5yypParser;
2361}
2362#endif /* sqlite3Fts5Parser_ENGINEALWAYSONSTACK */
2363
2364
2365/* The following function deletes the "minor type" or semantic value
2366** associated with a symbol. The symbol can be either a terminal
2367** or nonterminal. "fts5yymajor" is the symbol code, and "fts5yypminor" is
2368** a pointer to the value to be deleted. The code used to do the
2369** deletions is derived from the %destructor and/or %token_destructor
2370** directives of the input grammar.
2371*/
2372static void fts5yy_destructor(
2373 fts5yyParser *fts5yypParser, /* The parser */
2374 fts5YYCODETYPEunsigned char fts5yymajor, /* Type code for object to destroy */
2375 fts5YYMINORTYPE *fts5yypminor /* The object to be destroyed */
2376){
2377 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
2378 sqlite3Fts5ParserCTX_FETCH
2379 switch( fts5yymajor ){
2380 /* Here is inserted the actions which take place when a
2381 ** terminal or non-terminal is destroyed. This can happen
2382 ** when the symbol is popped from the stack during a
2383 ** reduce or during error processing or when a parser is
2384 ** being destroyed before it is finished parsing.
2385 **
2386 ** Note: during a reduce, the only symbols destroyed are those
2387 ** which appear on the RHS of the rule, but which are *not* used
2388 ** inside the C code.
2389 */
2390/********* Begin destructor definitions ***************************************/
2391 case 16: /* input */
2392{
2393#line 83 "fts5parse.y"
2394 (void)pParse;
2395#line 624 "fts5parse.c"
2396}
2397 break;
2398 case 17: /* expr */
2399 case 18: /* cnearset */
2400 case 19: /* exprlist */
2401{
2402#line 89 "fts5parse.y"
2403 sqlite3Fts5ParseNodeFree((fts5yypminor->fts5yy24));
2404#line 633 "fts5parse.c"
2405}
2406 break;
2407 case 20: /* colset */
2408 case 21: /* colsetlist */
2409{
2410#line 93 "fts5parse.y"
2411 sqlite3_freesqlite3_api->free((fts5yypminor->fts5yy11));
2412#line 641 "fts5parse.c"
2413}
2414 break;
2415 case 22: /* nearset */
2416 case 23: /* nearphrases */
2417{
2418#line 148 "fts5parse.y"
2419 sqlite3Fts5ParseNearsetFree((fts5yypminor->fts5yy46));
2420#line 649 "fts5parse.c"
2421}
2422 break;
2423 case 24: /* phrase */
2424{
2425#line 183 "fts5parse.y"
2426 sqlite3Fts5ParsePhraseFree((fts5yypminor->fts5yy53));
2427#line 656 "fts5parse.c"
2428}
2429 break;
2430/********* End destructor definitions *****************************************/
2431 default: break; /* If no destructor action specified: do nothing */
2432 }
2433}
2434
2435/*
2436** Pop the parser's stack once.
2437**
2438** If there is a destructor routine associated with the token which
2439** is popped from the stack, then call it.
2440*/
2441static void fts5yy_pop_parser_stack(fts5yyParser *pParser){
2442 fts5yyStackEntry *fts5yytos;
2443 assert( pParser->fts5yytos!=0 )((void) (0));
2444 assert( pParser->fts5yytos > pParser->fts5yystack )((void) (0));
2445 fts5yytos = pParser->fts5yytos--;
2446#ifndef NDEBUG1
2447 if( fts5yyTraceFILE ){
2448 fprintf(fts5yyTraceFILE,"%sPopping %s\n",
2449 fts5yyTracePrompt,
2450 fts5yyTokenName[fts5yytos->major]);
2451 }
2452#endif
2453 fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor);
2454}
2455
2456/*
2457** Clear all secondary memory allocations from the parser
2458*/
2459static void sqlite3Fts5ParserFinalize(void *p){
2460 fts5yyParser *pParser = (fts5yyParser*)p;
2461
2462 /* In-lined version of calling fts5yy_pop_parser_stack() for each
2463 ** element left in the stack */
2464 fts5yyStackEntry *fts5yytos = pParser->fts5yytos;
2465 while( fts5yytos>pParser->fts5yystack ){
2466#ifndef NDEBUG1
2467 if( fts5yyTraceFILE ){
2468 fprintf(fts5yyTraceFILE,"%sPopping %s\n",
2469 fts5yyTracePrompt,
2470 fts5yyTokenName[fts5yytos->major]);
2471 }
2472#endif
2473 if( fts5yytos->major>=fts5YY_MIN_DSTRCTR16 ){
2474 fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor);
2475 }
2476 fts5yytos--;
2477 }
2478
2479#if fts5YYGROWABLESTACK0
2480 if( pParser->fts5yystack!=pParser->fts5yystk0 ){
2481 fts5YYFREEfree(pParser->fts5yystack, sqlite3Fts5ParserCTX(pParser)0);
2482 }
2483#endif
2484}
2485
2486#ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK
2487/*
2488** Deallocate and destroy a parser. Destructors are called for
2489** all stack elements before shutting the parser down.
2490**
2491** If the fts5YYPARSEFREENEVERNULL macro exists (for example because it
2492** is defined in a %include section of the input grammar) then it is
2493** assumed that the input pointer is never NULL.
2494*/
2495static void sqlite3Fts5ParserFree(
2496 void *p, /* The parser to be deleted */
2497 void (*freeProc)(void*) /* Function used to reclaim memory */
2498){
2499#ifndef fts5YYPARSEFREENEVERNULL
2500 if( p==0 ) return;
2501#endif
2502 sqlite3Fts5ParserFinalize(p);
2503 (*freeProc)(p);
2504}
2505#endif /* sqlite3Fts5Parser_ENGINEALWAYSONSTACK */
2506
2507/*
2508** Return the peak depth of the stack for a parser.
2509*/
2510#ifdef fts5YYTRACKMAXSTACKDEPTH
2511static int sqlite3Fts5ParserStackPeak(void *p){
2512 fts5yyParser *pParser = (fts5yyParser*)p;
2513 return pParser->fts5yyhwm;
2514}
2515#endif
2516
2517/* This array of booleans keeps track of the parser statement
2518** coverage. The element fts5yycoverage[X][Y] is set when the parser
2519** is in state X and has a lookahead token Y. In a well-tested
2520** systems, every element of this matrix should end up being set.
2521*/
2522#if defined(fts5YYCOVERAGE)
2523static unsigned char fts5yycoverage[fts5YYNSTATE35][fts5YYNFTS5TOKEN16];
2524#endif
2525
2526/*
2527** Write into out a description of every state/lookahead combination that
2528**
2529** (1) has not been used by the parser, and
2530** (2) is not a syntax error.
2531**
2532** Return the number of missed state/lookahead combinations.
2533*/
2534#if defined(fts5YYCOVERAGE)
2535static int sqlite3Fts5ParserCoverage(FILE *out){
2536 int stateno, iLookAhead, i;
2537 int nMissed = 0;
2538 for(stateno=0; stateno<fts5YYNSTATE35; stateno++){
2539 i = fts5yy_shift_ofst[stateno];
2540 for(iLookAhead=0; iLookAhead<fts5YYNFTS5TOKEN16; iLookAhead++){
2541 if( fts5yy_lookahead[i+iLookAhead]!=iLookAhead ) continue;
2542 if( fts5yycoverage[stateno][iLookAhead]==0 ) nMissed++;
2543 if( out ){
2544 fprintf(out,"State %d lookahead %s %s\n", stateno,
2545 fts5yyTokenName[iLookAhead],
2546 fts5yycoverage[stateno][iLookAhead] ? "ok" : "missed");
2547 }
2548 }
2549 }
2550 return nMissed;
2551}
2552#endif
2553
2554/*
2555** Find the appropriate action for a parser given the terminal
2556** look-ahead token iLookAhead.
2557*/
2558static fts5YYACTIONTYPEunsigned char fts5yy_find_shift_action(
2559 fts5YYCODETYPEunsigned char iLookAhead, /* The look-ahead token */
2560 fts5YYACTIONTYPEunsigned char stateno /* Current state number */
2561){
2562 int i;
2563
2564 if( stateno>fts5YY_MAX_SHIFT34 ) return stateno;
2565 assert( stateno <= fts5YY_SHIFT_COUNT )((void) (0));
2566#if defined(fts5YYCOVERAGE)
2567 fts5yycoverage[stateno][iLookAhead] = 1;
2568#endif
2569 do{
2570 i = fts5yy_shift_ofst[stateno];
2571 assert( i>=0 )((void) (0));
2572 assert( i<=fts5YY_ACTTAB_COUNT )((void) (0));
2573 assert( i+fts5YYNFTS5TOKEN<=(int)fts5YY_NLOOKAHEAD )((void) (0));
2574 assert( iLookAhead!=fts5YYNOCODE )((void) (0));
2575 assert( iLookAhead < fts5YYNFTS5TOKEN )((void) (0));
2576 i += iLookAhead;
2577 assert( i<(int)fts5YY_NLOOKAHEAD )((void) (0));
2578 if( fts5yy_lookahead[i]!=iLookAhead ){
2579#ifdef fts5YYFALLBACK
2580 fts5YYCODETYPEunsigned char iFallback; /* Fallback token */
2581 assert( iLookAhead<sizeof(fts5yyFallback)/sizeof(fts5yyFallback[0]) )((void) (0));
2582 iFallback = fts5yyFallback[iLookAhead];
2583 if( iFallback!=0 ){
2584#ifndef NDEBUG1
2585 if( fts5yyTraceFILE ){
2586 fprintf(fts5yyTraceFILE, "%sFALLBACK %s => %s\n",
2587 fts5yyTracePrompt, fts5yyTokenName[iLookAhead], fts5yyTokenName[iFallback]);
2588 }
2589#endif
2590 assert( fts5yyFallback[iFallback]==0 )((void) (0)); /* Fallback loop must terminate */
2591 iLookAhead = iFallback;
2592 continue;
2593 }
2594#endif
2595#ifdef fts5YYWILDCARD
2596 {
2597 int j = i - iLookAhead + fts5YYWILDCARD;
2598 assert( j<(int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0])) )((void) (0));
2599 if( fts5yy_lookahead[j]==fts5YYWILDCARD && iLookAhead>0 ){
2600#ifndef NDEBUG1
2601 if( fts5yyTraceFILE ){
2602 fprintf(fts5yyTraceFILE, "%sWILDCARD %s => %s\n",
2603 fts5yyTracePrompt, fts5yyTokenName[iLookAhead],
2604 fts5yyTokenName[fts5YYWILDCARD]);
2605 }
2606#endif /* NDEBUG */
2607 return fts5yy_action[j];
2608 }
2609 }
2610#endif /* fts5YYWILDCARD */
2611 return fts5yy_default[stateno];
2612 }else{
2613 assert( i>=0 && i<(int)(sizeof(fts5yy_action)/sizeof(fts5yy_action[0])) )((void) (0));
2614 return fts5yy_action[i];
2615 }
2616 }while(1);
2617}
2618
2619/*
2620** Find the appropriate action for a parser given the non-terminal
2621** look-ahead token iLookAhead.
2622*/
2623static fts5YYACTIONTYPEunsigned char fts5yy_find_reduce_action(
2624 fts5YYACTIONTYPEunsigned char stateno, /* Current state number */
2625 fts5YYCODETYPEunsigned char iLookAhead /* The look-ahead token */
2626){
2627 int i;
2628#ifdef fts5YYERRORSYMBOL
2629 if( stateno>fts5YY_REDUCE_COUNT(17) ){
2630 return fts5yy_default[stateno];
2631 }
2632#else
2633 assert( stateno<=fts5YY_REDUCE_COUNT )((void) (0));
2634#endif
2635 i = fts5yy_reduce_ofst[stateno];
2636 assert( iLookAhead!=fts5YYNOCODE )((void) (0));
2637 i += iLookAhead;
2638#ifdef fts5YYERRORSYMBOL
2639 if( i<0 || i>=fts5YY_ACTTAB_COUNT(105) || fts5yy_lookahead[i]!=iLookAhead ){
2640 return fts5yy_default[stateno];
2641 }
2642#else
2643 assert( i>=0 && i<fts5YY_ACTTAB_COUNT )((void) (0));
2644 assert( fts5yy_lookahead[i]==iLookAhead )((void) (0));
2645#endif
2646 return fts5yy_action[i];
2647}
2648
2649/*
2650** The following routine is called if the stack overflows.
2651*/
2652static void fts5yyStackOverflow(fts5yyParser *fts5yypParser){
2653 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
2654 sqlite3Fts5ParserCTX_FETCH
2655#ifndef NDEBUG1
2656 if( fts5yyTraceFILE ){
2657 fprintf(fts5yyTraceFILE,"%sStack Overflow!\n",fts5yyTracePrompt);
2658 }
2659#endif
2660 while( fts5yypParser->fts5yytos>fts5yypParser->fts5yystack ) fts5yy_pop_parser_stack(fts5yypParser);
2661 /* Here code is inserted which will execute if the parser
2662 ** stack every overflows */
2663/******** Begin %stack_overflow code ******************************************/
2664#line 36 "fts5parse.y"
2665
2666 sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow");
2667#line 896 "fts5parse.c"
2668/******** End %stack_overflow code ********************************************/
2669 sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse; /* Suppress warning about unused %extra_argument var */
2670 sqlite3Fts5ParserCTX_STORE
2671}
2672
2673/*
2674** Print tracing information for a SHIFT action
2675*/
2676#ifndef NDEBUG1
2677static void fts5yyTraceShift(fts5yyParser *fts5yypParser, int fts5yyNewState, const char *zTag){
2678 if( fts5yyTraceFILE ){
2679 if( fts5yyNewState<fts5YYNSTATE35 ){
2680 fprintf(fts5yyTraceFILE,"%s%s '%s', go to state %d\n",
2681 fts5yyTracePrompt, zTag, fts5yyTokenName[fts5yypParser->fts5yytos->major],
2682 fts5yyNewState);
2683 }else{
2684 fprintf(fts5yyTraceFILE,"%s%s '%s', pending reduce %d\n",
2685 fts5yyTracePrompt, zTag, fts5yyTokenName[fts5yypParser->fts5yytos->major],
2686 fts5yyNewState - fts5YY_MIN_REDUCE83);
2687 }
2688 }
2689}
2690#else
2691# define fts5yyTraceShift(X,Y,Z)
2692#endif
2693
2694/*
2695** Perform a shift action.
2696*/
2697static void fts5yy_shift(
2698 fts5yyParser *fts5yypParser, /* The parser to be shifted */
2699 fts5YYACTIONTYPEunsigned char fts5yyNewState, /* The new state to shift in */
2700 fts5YYCODETYPEunsigned char fts5yyMajor, /* The major token to shift in */
2701 sqlite3Fts5ParserFTS5TOKENTYPEFts5Token fts5yyMinor /* The minor token to shift in */
2702){
2703 fts5yyStackEntry *fts5yytos;
2704 fts5yypParser->fts5yytos++;
2705#ifdef fts5YYTRACKMAXSTACKDEPTH
2706 if( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)>fts5yypParser->fts5yyhwm ){
2707 fts5yypParser->fts5yyhwm++;
2708 assert( fts5yypParser->fts5yyhwm == (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack) )((void) (0));
2709 }
2710#endif
2711 fts5yytos = fts5yypParser->fts5yytos;
2712 if( fts5yytos>fts5yypParser->fts5yystackEnd ){
2713 if( fts5yyGrowStack(fts5yypParser)1 ){
2714 fts5yypParser->fts5yytos--;
2715 fts5yyStackOverflow(fts5yypParser);
2716 return;
2717 }
2718 fts5yytos = fts5yypParser->fts5yytos;
2719 assert( fts5yytos <= fts5yypParser->fts5yystackEnd )((void) (0));
2720 }
2721 if( fts5yyNewState > fts5YY_MAX_SHIFT34 ){
2722 fts5yyNewState += fts5YY_MIN_REDUCE83 - fts5YY_MIN_SHIFTREDUCE52;
2723 }
2724 fts5yytos->stateno = fts5yyNewState;
2725 fts5yytos->major = fts5yyMajor;
2726 fts5yytos->minor.fts5yy0 = fts5yyMinor;
2727 fts5yyTraceShift(fts5yypParser, fts5yyNewState, "Shift");
2728}
2729
2730/* For rule J, fts5yyRuleInfoLhs[J] contains the symbol on the left-hand side
2731** of that rule */
2732static const fts5YYCODETYPEunsigned char fts5yyRuleInfoLhs[] = {
2733 16, /* (0) input ::= expr */
2734 20, /* (1) colset ::= MINUS LCP colsetlist RCP */
2735 20, /* (2) colset ::= LCP colsetlist RCP */
2736 20, /* (3) colset ::= STRING */
2737 20, /* (4) colset ::= MINUS STRING */
2738 21, /* (5) colsetlist ::= colsetlist STRING */
2739 21, /* (6) colsetlist ::= STRING */
2740 17, /* (7) expr ::= expr AND expr */
2741 17, /* (8) expr ::= expr OR expr */
2742 17, /* (9) expr ::= expr NOT expr */
2743 17, /* (10) expr ::= colset COLON LP expr RP */
2744 17, /* (11) expr ::= LP expr RP */
2745 17, /* (12) expr ::= exprlist */
2746 19, /* (13) exprlist ::= cnearset */
2747 19, /* (14) exprlist ::= exprlist cnearset */
2748 18, /* (15) cnearset ::= nearset */
2749 18, /* (16) cnearset ::= colset COLON nearset */
2750 22, /* (17) nearset ::= phrase */
2751 22, /* (18) nearset ::= CARET phrase */
2752 22, /* (19) nearset ::= STRING LP nearphrases neardist_opt RP */
2753 23, /* (20) nearphrases ::= phrase */
2754 23, /* (21) nearphrases ::= nearphrases phrase */
2755 25, /* (22) neardist_opt ::= */
2756 25, /* (23) neardist_opt ::= COMMA STRING */
2757 24, /* (24) phrase ::= phrase PLUS STRING star_opt */
2758 24, /* (25) phrase ::= STRING star_opt */
2759 26, /* (26) star_opt ::= STAR */
2760 26, /* (27) star_opt ::= */
2761};
2762
2763/* For rule J, fts5yyRuleInfoNRhs[J] contains the negative of the number
2764** of symbols on the right-hand side of that rule. */
2765static const signed char fts5yyRuleInfoNRhs[] = {
2766 -1, /* (0) input ::= expr */
2767 -4, /* (1) colset ::= MINUS LCP colsetlist RCP */
2768 -3, /* (2) colset ::= LCP colsetlist RCP */
2769 -1, /* (3) colset ::= STRING */
2770 -2, /* (4) colset ::= MINUS STRING */
2771 -2, /* (5) colsetlist ::= colsetlist STRING */
2772 -1, /* (6) colsetlist ::= STRING */
2773 -3, /* (7) expr ::= expr AND expr */
2774 -3, /* (8) expr ::= expr OR expr */
2775 -3, /* (9) expr ::= expr NOT expr */
2776 -5, /* (10) expr ::= colset COLON LP expr RP */
2777 -3, /* (11) expr ::= LP expr RP */
2778 -1, /* (12) expr ::= exprlist */
2779 -1, /* (13) exprlist ::= cnearset */
2780 -2, /* (14) exprlist ::= exprlist cnearset */
2781 -1, /* (15) cnearset ::= nearset */
2782 -3, /* (16) cnearset ::= colset COLON nearset */
2783 -1, /* (17) nearset ::= phrase */
2784 -2, /* (18) nearset ::= CARET phrase */
2785 -5, /* (19) nearset ::= STRING LP nearphrases neardist_opt RP */
2786 -1, /* (20) nearphrases ::= phrase */
2787 -2, /* (21) nearphrases ::= nearphrases phrase */
2788 0, /* (22) neardist_opt ::= */
2789 -2, /* (23) neardist_opt ::= COMMA STRING */
2790 -4, /* (24) phrase ::= phrase PLUS STRING star_opt */
2791 -2, /* (25) phrase ::= STRING star_opt */
2792 -1, /* (26) star_opt ::= STAR */
2793 0, /* (27) star_opt ::= */
2794};
2795
2796static void fts5yy_accept(fts5yyParser*); /* Forward Declaration */
2797
2798/*
2799** Perform a reduce action and the shift that must immediately
2800** follow the reduce.
2801**
2802** The fts5yyLookahead and fts5yyLookaheadToken parameters provide reduce actions
2803** access to the lookahead token (if any). The fts5yyLookahead will be fts5YYNOCODE
2804** if the lookahead token has already been consumed. As this procedure is
2805** only called from one place, optimizing compilers will in-line it, which
2806** means that the extra parameters have no performance impact.
2807*/
2808static fts5YYACTIONTYPEunsigned char fts5yy_reduce(
2809 fts5yyParser *fts5yypParser, /* The parser */
2810 unsigned int fts5yyruleno, /* Number of the rule by which to reduce */
2811 int fts5yyLookahead, /* Lookahead token, or fts5YYNOCODE if none */
2812 sqlite3Fts5ParserFTS5TOKENTYPEFts5Token fts5yyLookaheadToken /* Value of the lookahead token */
2813 sqlite3Fts5ParserCTX_PDECL /* %extra_context */
2814){
2815 int fts5yygoto; /* The next state */
2816 fts5YYACTIONTYPEunsigned char fts5yyact; /* The next action */
2817 fts5yyStackEntry *fts5yymsp; /* The top of the parser's stack */
2818 int fts5yysize; /* Amount to pop the stack */
2819 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
2820 (void)fts5yyLookahead;
2821 (void)fts5yyLookaheadToken;
2822 fts5yymsp = fts5yypParser->fts5yytos;
2823
2824 switch( fts5yyruleno ){
2825 /* Beginning here are the reduction cases. A typical example
2826 ** follows:
2827 ** case 0:
2828 ** #line <lineno> <grammarfile>
2829 ** { ... } // User supplied code
2830 ** #line <lineno> <thisfile>
2831 ** break;
2832 */
2833/********** Begin reduce actions **********************************************/
2834 fts5YYMINORTYPE fts5yylhsminor;
2835 case 0: /* input ::= expr */
2836#line 82 "fts5parse.y"
2837{ sqlite3Fts5ParseFinished(pParse, fts5yymsp[0].minor.fts5yy24); }
2838#line 1067 "fts5parse.c"
2839 break;
2840 case 1: /* colset ::= MINUS LCP colsetlist RCP */
2841#line 97 "fts5parse.y"
2842{
2843 fts5yymsp[-3].minor.fts5yy11 = sqlite3Fts5ParseColsetInvert(pParse, fts5yymsp[-1].minor.fts5yy11);
2844}
2845#line 1074 "fts5parse.c"
2846 break;
2847 case 2: /* colset ::= LCP colsetlist RCP */
2848#line 100 "fts5parse.y"
2849{ fts5yymsp[-2].minor.fts5yy11 = fts5yymsp[-1].minor.fts5yy11; }
2850#line 1079 "fts5parse.c"
2851 break;
2852 case 3: /* colset ::= STRING */
2853#line 101 "fts5parse.y"
2854{
2855 fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0);
2856}
2857#line 1086 "fts5parse.c"
2858 fts5yymsp[0].minor.fts5yy11 = fts5yylhsminor.fts5yy11;
2859 break;
2860 case 4: /* colset ::= MINUS STRING */
2861#line 104 "fts5parse.y"
2862{
2863 fts5yymsp[-1].minor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0);
2864 fts5yymsp[-1].minor.fts5yy11 = sqlite3Fts5ParseColsetInvert(pParse, fts5yymsp[-1].minor.fts5yy11);
2865}
2866#line 1095 "fts5parse.c"
2867 break;
2868 case 5: /* colsetlist ::= colsetlist STRING */
2869#line 109 "fts5parse.y"
2870{
2871 fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, fts5yymsp[-1].minor.fts5yy11, &fts5yymsp[0].minor.fts5yy0); }
2872#line 1101 "fts5parse.c"
2873 fts5yymsp[-1].minor.fts5yy11 = fts5yylhsminor.fts5yy11;
2874 break;
2875 case 6: /* colsetlist ::= STRING */
2876#line 111 "fts5parse.y"
2877{
2878 fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0);
2879}
2880#line 1109 "fts5parse.c"
2881 fts5yymsp[0].minor.fts5yy11 = fts5yylhsminor.fts5yy11;
2882 break;
2883 case 7: /* expr ::= expr AND expr */
2884#line 115 "fts5parse.y"
2885{
2886 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_AND2, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0);
2887}
2888#line 1117 "fts5parse.c"
2889 fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2890 break;
2891 case 8: /* expr ::= expr OR expr */
2892#line 118 "fts5parse.y"
2893{
2894 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_OR1, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0);
2895}
2896#line 1125 "fts5parse.c"
2897 fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2898 break;
2899 case 9: /* expr ::= expr NOT expr */
2900#line 121 "fts5parse.y"
2901{
2902 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_NOT3, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0);
2903}
2904#line 1133 "fts5parse.c"
2905 fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2906 break;
2907 case 10: /* expr ::= colset COLON LP expr RP */
2908#line 125 "fts5parse.y"
2909{
2910 sqlite3Fts5ParseSetColset(pParse, fts5yymsp[-1].minor.fts5yy24, fts5yymsp[-4].minor.fts5yy11);
2911 fts5yylhsminor.fts5yy24 = fts5yymsp[-1].minor.fts5yy24;
2912}
2913#line 1142 "fts5parse.c"
2914 fts5yymsp[-4].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2915 break;
2916 case 11: /* expr ::= LP expr RP */
2917#line 129 "fts5parse.y"
2918{fts5yymsp[-2].minor.fts5yy24 = fts5yymsp[-1].minor.fts5yy24;}
2919#line 1148 "fts5parse.c"
2920 break;
2921 case 12: /* expr ::= exprlist */
2922 case 13: /* exprlist ::= cnearset */ fts5yytestcase(fts5yyruleno==13);
2923#line 130 "fts5parse.y"
2924{fts5yylhsminor.fts5yy24 = fts5yymsp[0].minor.fts5yy24;}
2925#line 1154 "fts5parse.c"
2926 fts5yymsp[0].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2927 break;
2928 case 14: /* exprlist ::= exprlist cnearset */
2929#line 133 "fts5parse.y"
2930{
2931 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseImplicitAnd(pParse, fts5yymsp[-1].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24);
2932}
2933#line 1162 "fts5parse.c"
2934 fts5yymsp[-1].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2935 break;
2936 case 15: /* cnearset ::= nearset */
2937#line 137 "fts5parse.y"
2938{
2939 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_STRING9, 0, 0, fts5yymsp[0].minor.fts5yy46);
2940}
2941#line 1170 "fts5parse.c"
2942 fts5yymsp[0].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2943 break;
2944 case 16: /* cnearset ::= colset COLON nearset */
2945#line 140 "fts5parse.y"
2946{
2947 fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_STRING9, 0, 0, fts5yymsp[0].minor.fts5yy46);
2948 sqlite3Fts5ParseSetColset(pParse, fts5yylhsminor.fts5yy24, fts5yymsp[-2].minor.fts5yy11);
2949}
2950#line 1179 "fts5parse.c"
2951 fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24;
2952 break;
2953 case 17: /* nearset ::= phrase */
2954#line 151 "fts5parse.y"
2955{ fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53); }
2956#line 1185 "fts5parse.c"
2957 fts5yymsp[0].minor.fts5yy46 = fts5yylhsminor.fts5yy46;
2958 break;
2959 case 18: /* nearset ::= CARET phrase */
2960#line 152 "fts5parse.y"
2961{
2962 sqlite3Fts5ParseSetCaret(fts5yymsp[0].minor.fts5yy53);
2963 fts5yymsp[-1].minor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53);
2964}
2965#line 1194 "fts5parse.c"
2966 break;
2967 case 19: /* nearset ::= STRING LP nearphrases neardist_opt RP */
2968#line 156 "fts5parse.y"
2969{
2970 sqlite3Fts5ParseNear(pParse, &fts5yymsp[-4].minor.fts5yy0);
2971 sqlite3Fts5ParseSetDistance(pParse, fts5yymsp[-2].minor.fts5yy46, &fts5yymsp[-1].minor.fts5yy0);
2972 fts5yylhsminor.fts5yy46 = fts5yymsp[-2].minor.fts5yy46;
2973}
2974#line 1203 "fts5parse.c"
2975 fts5yymsp[-4].minor.fts5yy46 = fts5yylhsminor.fts5yy46;
2976 break;
2977 case 20: /* nearphrases ::= phrase */
2978#line 162 "fts5parse.y"
2979{
2980 fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53);
2981}
2982#line 1211 "fts5parse.c"
2983 fts5yymsp[0].minor.fts5yy46 = fts5yylhsminor.fts5yy46;
2984 break;
2985 case 21: /* nearphrases ::= nearphrases phrase */
2986#line 165 "fts5parse.y"
2987{
2988 fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, fts5yymsp[-1].minor.fts5yy46, fts5yymsp[0].minor.fts5yy53);
2989}
2990#line 1219 "fts5parse.c"
2991 fts5yymsp[-1].minor.fts5yy46 = fts5yylhsminor.fts5yy46;
2992 break;
2993 case 22: /* neardist_opt ::= */
2994#line 172 "fts5parse.y"
2995{ fts5yymsp[1].minor.fts5yy0.p = 0; fts5yymsp[1].minor.fts5yy0.n = 0; }
2996#line 1225 "fts5parse.c"
2997 break;
2998 case 23: /* neardist_opt ::= COMMA STRING */
2999#line 173 "fts5parse.y"
3000{ fts5yymsp[-1].minor.fts5yy0 = fts5yymsp[0].minor.fts5yy0; }
3001#line 1230 "fts5parse.c"
3002 break;
3003 case 24: /* phrase ::= phrase PLUS STRING star_opt */
3004#line 185 "fts5parse.y"
3005{
3006 fts5yylhsminor.fts5yy53 = sqlite3Fts5ParseTerm(pParse, fts5yymsp[-3].minor.fts5yy53, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy4);
3007}
3008#line 1237 "fts5parse.c"
3009 fts5yymsp[-3].minor.fts5yy53 = fts5yylhsminor.fts5yy53;
3010 break;
3011 case 25: /* phrase ::= STRING star_opt */
3012#line 188 "fts5parse.y"
3013{
3014 fts5yylhsminor.fts5yy53 = sqlite3Fts5ParseTerm(pParse, 0, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy4);
3015}
3016#line 1245 "fts5parse.c"
3017 fts5yymsp[-1].minor.fts5yy53 = fts5yylhsminor.fts5yy53;
3018 break;
3019 case 26: /* star_opt ::= STAR */
3020#line 196 "fts5parse.y"
3021{ fts5yymsp[0].minor.fts5yy4 = 1; }
3022#line 1251 "fts5parse.c"
3023 break;
3024 case 27: /* star_opt ::= */
3025#line 197 "fts5parse.y"
3026{ fts5yymsp[1].minor.fts5yy4 = 0; }
3027#line 1256 "fts5parse.c"
3028 break;
3029 default:
3030 break;
3031/********** End reduce actions ************************************************/
3032 };
3033 assert( fts5yyruleno<sizeof(fts5yyRuleInfoLhs)/sizeof(fts5yyRuleInfoLhs[0]) )((void) (0));
3034 fts5yygoto = fts5yyRuleInfoLhs[fts5yyruleno];
3035 fts5yysize = fts5yyRuleInfoNRhs[fts5yyruleno];
3036 fts5yyact = fts5yy_find_reduce_action(fts5yymsp[fts5yysize].stateno,(fts5YYCODETYPEunsigned char)fts5yygoto);
3037
3038 /* There are no SHIFTREDUCE actions on nonterminals because the table
3039 ** generator has simplified them to pure REDUCE actions. */
3040 assert( !(fts5yyact>fts5YY_MAX_SHIFT && fts5yyact<=fts5YY_MAX_SHIFTREDUCE) )((void) (0));
3041
3042 /* It is not possible for a REDUCE to be followed by an error */
3043 assert( fts5yyact!=fts5YY_ERROR_ACTION )((void) (0));
3044
3045 fts5yymsp += fts5yysize+1;
3046 fts5yypParser->fts5yytos = fts5yymsp;
3047 fts5yymsp->stateno = (fts5YYACTIONTYPEunsigned char)fts5yyact;
3048 fts5yymsp->major = (fts5YYCODETYPEunsigned char)fts5yygoto;
3049 fts5yyTraceShift(fts5yypParser, fts5yyact, "... then shift");
3050 return fts5yyact;
3051}
3052
3053/*
3054** The following code executes when the parse fails
3055*/
3056#ifndef fts5YYNOERRORRECOVERY1
3057static void fts5yy_parse_failed(
3058 fts5yyParser *fts5yypParser /* The parser */
3059){
3060 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
3061 sqlite3Fts5ParserCTX_FETCH
3062#ifndef NDEBUG1
3063 if( fts5yyTraceFILE ){
3064 fprintf(fts5yyTraceFILE,"%sFail!\n",fts5yyTracePrompt);
3065 }
3066#endif
3067 while( fts5yypParser->fts5yytos>fts5yypParser->fts5yystack ) fts5yy_pop_parser_stack(fts5yypParser);
3068 /* Here code is inserted which will be executed whenever the
3069 ** parser fails */
3070/************ Begin %parse_failure code ***************************************/
3071/************ End %parse_failure code *****************************************/
3072 sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse; /* Suppress warning about unused %extra_argument variable */
3073 sqlite3Fts5ParserCTX_STORE
3074}
3075#endif /* fts5YYNOERRORRECOVERY */
3076
3077/*
3078** The following code executes when a syntax error first occurs.
3079*/
3080static void fts5yy_syntax_error(
3081 fts5yyParser *fts5yypParser, /* The parser */
3082 int fts5yymajor, /* The major type of the error token */
3083 sqlite3Fts5ParserFTS5TOKENTYPEFts5Token fts5yyminor /* The minor type of the error token */
3084){
3085 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
3086 sqlite3Fts5ParserCTX_FETCH
3087#define FTS5TOKENfts5yyminor fts5yyminor
3088/************ Begin %syntax_error code ****************************************/
3089#line 30 "fts5parse.y"
3090
3091 UNUSED_PARAM(fts5yymajor)(void)(fts5yymajor); /* Silence a compiler warning */
3092 sqlite3Fts5ParseError(
3093 pParse, "fts5: syntax error near \"%.*s\"",FTS5TOKENfts5yyminor.n,FTS5TOKENfts5yyminor.p
3094 );
3095#line 1324 "fts5parse.c"
3096/************ End %syntax_error code ******************************************/
3097 sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse; /* Suppress warning about unused %extra_argument variable */
3098 sqlite3Fts5ParserCTX_STORE
3099}
3100
3101/*
3102** The following is executed when the parser accepts
3103*/
3104static void fts5yy_accept(
3105 fts5yyParser *fts5yypParser /* The parser */
3106){
3107 sqlite3Fts5ParserARG_FETCHFts5Parse *pParse=fts5yypParser->pParse;
3108 sqlite3Fts5ParserCTX_FETCH
3109#ifndef NDEBUG1
3110 if( fts5yyTraceFILE ){
3111 fprintf(fts5yyTraceFILE,"%sAccept!\n",fts5yyTracePrompt);
3112 }
3113#endif
3114#ifndef fts5YYNOERRORRECOVERY1
3115 fts5yypParser->fts5yyerrcnt = -1;
3116#endif
3117 assert( fts5yypParser->fts5yytos==fts5yypParser->fts5yystack )((void) (0));
3118 /* Here code is inserted which will be executed whenever the
3119 ** parser accepts */
3120/*********** Begin %parse_accept code *****************************************/
3121/*********** End %parse_accept code *******************************************/
3122 sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse; /* Suppress warning about unused %extra_argument variable */
3123 sqlite3Fts5ParserCTX_STORE
3124}
3125
3126/* The main parser program.
3127** The first argument is a pointer to a structure obtained from
3128** "sqlite3Fts5ParserAlloc" which describes the current state of the parser.
3129** The second argument is the major token number. The third is
3130** the minor token. The fourth optional argument is whatever the
3131** user wants (and specified in the grammar) and is available for
3132** use by the action routines.
3133**
3134** Inputs:
3135** <ul>
3136** <li> A pointer to the parser (an opaque structure.)
3137** <li> The major token number.
3138** <li> The minor token number.
3139** <li> An option argument of a grammar-specified type.
3140** </ul>
3141**
3142** Outputs:
3143** None.
3144*/
3145static void sqlite3Fts5Parser(
3146 void *fts5yyp, /* The parser */
3147 int fts5yymajor, /* The major token code number */
3148 sqlite3Fts5ParserFTS5TOKENTYPEFts5Token fts5yyminor /* The value for the token */
3149 sqlite3Fts5ParserARG_PDECL,Fts5Parse *pParse /* Optional %extra_argument parameter */
3150){
3151 fts5YYMINORTYPE fts5yyminorunion;
3152 fts5YYACTIONTYPEunsigned char fts5yyact; /* The parser action. */
3153#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY1)
3154 int fts5yyendofinput; /* True if we are at the end of input */
3155#endif
3156#ifdef fts5YYERRORSYMBOL
3157 int fts5yyerrorhit = 0; /* True if fts5yymajor has invoked an error */
3158#endif
3159 fts5yyParser *fts5yypParser = (fts5yyParser*)fts5yyp; /* The parser */
3160 sqlite3Fts5ParserCTX_FETCH
3161 sqlite3Fts5ParserARG_STOREfts5yypParser->pParse=pParse;
3162
3163 assert( fts5yypParser->fts5yytos!=0 )((void) (0));
3164#if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY1)
3165 fts5yyendofinput = (fts5yymajor==0);
3166#endif
3167
3168 fts5yyact = fts5yypParser->fts5yytos->stateno;
3169#ifndef NDEBUG1
3170 if( fts5yyTraceFILE ){
3171 if( fts5yyact < fts5YY_MIN_REDUCE83 ){
3172 fprintf(fts5yyTraceFILE,"%sInput '%s' in state %d\n",
3173 fts5yyTracePrompt,fts5yyTokenName[fts5yymajor],fts5yyact);
3174 }else{
3175 fprintf(fts5yyTraceFILE,"%sInput '%s' with pending reduce %d\n",
3176 fts5yyTracePrompt,fts5yyTokenName[fts5yymajor],fts5yyact-fts5YY_MIN_REDUCE83);
3177 }
3178 }
3179#endif
3180
3181 while(1){ /* Exit by "break" */
3182 assert( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystack )((void) (0));
3183 assert( fts5yyact==fts5yypParser->fts5yytos->stateno )((void) (0));
3184 fts5yyact = fts5yy_find_shift_action((fts5YYCODETYPEunsigned char)fts5yymajor,fts5yyact);
3185 if( fts5yyact >= fts5YY_MIN_REDUCE83 ){
3186 unsigned int fts5yyruleno = fts5yyact - fts5YY_MIN_REDUCE83; /* Reduce by this rule */
3187#ifndef NDEBUG1
3188 assert( fts5yyruleno<(int)(sizeof(fts5yyRuleName)/sizeof(fts5yyRuleName[0])) )((void) (0));
3189 if( fts5yyTraceFILE ){
3190 int fts5yysize = fts5yyRuleInfoNRhs[fts5yyruleno];
3191 if( fts5yysize ){
3192 fprintf(fts5yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n",
3193 fts5yyTracePrompt,
3194 fts5yyruleno, fts5yyRuleName[fts5yyruleno],
3195 fts5yyruleno<fts5YYNRULE_WITH_ACTION28 ? "" : " without external action",
3196 fts5yypParser->fts5yytos[fts5yysize].stateno);
3197 }else{
3198 fprintf(fts5yyTraceFILE, "%sReduce %d [%s]%s.\n",
3199 fts5yyTracePrompt, fts5yyruleno, fts5yyRuleName[fts5yyruleno],
3200 fts5yyruleno<fts5YYNRULE_WITH_ACTION28 ? "" : " without external action");
3201 }
3202 }
3203#endif /* NDEBUG */
3204
3205 /* Check that the stack is large enough to grow by a single entry
3206 ** if the RHS of the rule is empty. This ensures that there is room
3207 ** enough on the stack to push the LHS value */
3208 if( fts5yyRuleInfoNRhs[fts5yyruleno]==0 ){
3209#ifdef fts5YYTRACKMAXSTACKDEPTH
3210 if( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)>fts5yypParser->fts5yyhwm ){
3211 fts5yypParser->fts5yyhwm++;
3212 assert( fts5yypParser->fts5yyhwm ==((void) (0))
3213 (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack))((void) (0));
3214 }
3215#endif
3216 if( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystackEnd ){
3217 if( fts5yyGrowStack(fts5yypParser)1 ){
3218 fts5yyStackOverflow(fts5yypParser);
3219 break;
3220 }
3221 }
3222 }
3223 fts5yyact = fts5yy_reduce(fts5yypParser,fts5yyruleno,fts5yymajor,fts5yyminor sqlite3Fts5ParserCTX_PARAM);
3224 }else if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE79 ){
3225 fts5yy_shift(fts5yypParser,fts5yyact,(fts5YYCODETYPEunsigned char)fts5yymajor,fts5yyminor);
3226#ifndef fts5YYNOERRORRECOVERY1
3227 fts5yypParser->fts5yyerrcnt--;
3228#endif
3229 break;
3230 }else if( fts5yyact==fts5YY_ACCEPT_ACTION81 ){
3231 fts5yypParser->fts5yytos--;
3232 fts5yy_accept(fts5yypParser);
3233 return;
3234 }else{
3235 assert( fts5yyact == fts5YY_ERROR_ACTION )((void) (0));
3236 fts5yyminorunion.fts5yy0 = fts5yyminor;
3237#ifdef fts5YYERRORSYMBOL
3238 int fts5yymx;
3239#endif
3240#ifndef NDEBUG1
3241 if( fts5yyTraceFILE ){
3242 fprintf(fts5yyTraceFILE,"%sSyntax Error!\n",fts5yyTracePrompt);
3243 }
3244#endif
3245#ifdef fts5YYERRORSYMBOL
3246 /* A syntax error has occurred.
3247 ** The response to an error depends upon whether or not the
3248 ** grammar defines an error token "ERROR".
3249 **
3250 ** This is what we do if the grammar does define ERROR:
3251 **
3252 ** * Call the %syntax_error function.
3253 **
3254 ** * Begin popping the stack until we enter a state where
3255 ** it is legal to shift the error symbol, then shift
3256 ** the error symbol.
3257 **
3258 ** * Set the error count to three.
3259 **
3260 ** * Begin accepting and shifting new tokens. No new error
3261 ** processing will occur until three tokens have been
3262 ** shifted successfully.
3263 **
3264 */
3265 if( fts5yypParser->fts5yyerrcnt<0 ){
3266 fts5yy_syntax_error(fts5yypParser,fts5yymajor,fts5yyminor);
3267 }
3268 fts5yymx = fts5yypParser->fts5yytos->major;
3269 if( fts5yymx==fts5YYERRORSYMBOL || fts5yyerrorhit ){
3270#ifndef NDEBUG1
3271 if( fts5yyTraceFILE ){
3272 fprintf(fts5yyTraceFILE,"%sDiscard input token %s\n",
3273 fts5yyTracePrompt,fts5yyTokenName[fts5yymajor]);
3274 }
3275#endif
3276 fts5yy_destructor(fts5yypParser, (fts5YYCODETYPEunsigned char)fts5yymajor, &fts5yyminorunion);
3277 fts5yymajor = fts5YYNOCODE27;
3278 }else{
3279 while( fts5yypParser->fts5yytos > fts5yypParser->fts5yystack ){
3280 fts5yyact = fts5yy_find_reduce_action(fts5yypParser->fts5yytos->stateno,
3281 fts5YYERRORSYMBOL);
3282 if( fts5yyact<=fts5YY_MAX_SHIFTREDUCE79 ) break;
3283 fts5yy_pop_parser_stack(fts5yypParser);
3284 }
3285 if( fts5yypParser->fts5yytos <= fts5yypParser->fts5yystack || fts5yymajor==0 ){
3286 fts5yy_destructor(fts5yypParser,(fts5YYCODETYPEunsigned char)fts5yymajor,&fts5yyminorunion);
3287 fts5yy_parse_failed(fts5yypParser);
3288#ifndef fts5YYNOERRORRECOVERY1
3289 fts5yypParser->fts5yyerrcnt = -1;
3290#endif
3291 fts5yymajor = fts5YYNOCODE27;
3292 }else if( fts5yymx!=fts5YYERRORSYMBOL ){
3293 fts5yy_shift(fts5yypParser,fts5yyact,fts5YYERRORSYMBOL,fts5yyminor);
3294 }
3295 }
3296 fts5yypParser->fts5yyerrcnt = 3;
3297 fts5yyerrorhit = 1;
3298 if( fts5yymajor==fts5YYNOCODE27 ) break;
3299 fts5yyact = fts5yypParser->fts5yytos->stateno;
3300#elif defined(fts5YYNOERRORRECOVERY1)
3301 /* If the fts5YYNOERRORRECOVERY macro is defined, then do not attempt to
3302 ** do any kind of error recovery. Instead, simply invoke the syntax
3303 ** error routine and continue going as if nothing had happened.
3304 **
3305 ** Applications can set this macro (for example inside %include) if
3306 ** they intend to abandon the parse upon the first syntax error seen.
3307 */
3308 fts5yy_syntax_error(fts5yypParser,fts5yymajor, fts5yyminor);
3309 fts5yy_destructor(fts5yypParser,(fts5YYCODETYPEunsigned char)fts5yymajor,&fts5yyminorunion);
3310 break;
3311#else /* fts5YYERRORSYMBOL is not defined */
3312 /* This is what we do if the grammar does not define ERROR:
3313 **
3314 ** * Report an error message, and throw away the input token.
3315 **
3316 ** * If the input token is $, then fail the parse.
3317 **
3318 ** As before, subsequent error messages are suppressed until
3319 ** three input tokens have been successfully shifted.
3320 */
3321 if( fts5yypParser->fts5yyerrcnt<=0 ){
3322 fts5yy_syntax_error(fts5yypParser,fts5yymajor, fts5yyminor);
3323 }
3324 fts5yypParser->fts5yyerrcnt = 3;
3325 fts5yy_destructor(fts5yypParser,(fts5YYCODETYPEunsigned char)fts5yymajor,&fts5yyminorunion);
3326 if( fts5yyendofinput ){
3327 fts5yy_parse_failed(fts5yypParser);
3328#ifndef fts5YYNOERRORRECOVERY1
3329 fts5yypParser->fts5yyerrcnt = -1;
3330#endif
3331 }
3332 break;
3333#endif
3334 }
3335 }
3336#ifndef NDEBUG1
3337 if( fts5yyTraceFILE ){
3338 fts5yyStackEntry *i;
3339 char cDiv = '[';
3340 fprintf(fts5yyTraceFILE,"%sReturn. Stack=",fts5yyTracePrompt);
3341 for(i=&fts5yypParser->fts5yystack[1]; i<=fts5yypParser->fts5yytos; i++){
3342 fprintf(fts5yyTraceFILE,"%c%s", cDiv, fts5yyTokenName[i->major]);
3343 cDiv = ' ';
3344 }
3345 fprintf(fts5yyTraceFILE,"]\n");
3346 }
3347#endif
3348 return;
3349}
3350
3351/*
3352** Return the fallback token corresponding to canonical token iToken, or
3353** 0 if iToken has no fallback.
3354*/
3355static int sqlite3Fts5ParserFallback(int iToken){
3356#ifdef fts5YYFALLBACK
3357 assert( iToken<(int)(sizeof(fts5yyFallback)/sizeof(fts5yyFallback[0])) )((void) (0));
3358 return fts5yyFallback[iToken];
3359#else
3360 (void)iToken;
3361 return 0;
3362#endif
3363}
3364
3365#line 1 "fts5_aux.c"
3366/*
3367** 2014 May 31
3368**
3369** The author disclaims copyright to this source code. In place of
3370** a legal notice, here is a blessing:
3371**
3372** May you do good and not evil.
3373** May you find forgiveness for yourself and forgive others.
3374** May you share freely, never taking more than you give.
3375**
3376******************************************************************************
3377*/
3378
3379
3380/* #include "fts5Int.h" */
3381#include <math.h> /* amalgamator: keep */
3382
3383/*
3384** Object used to iterate through all "coalesced phrase instances" in
3385** a single column of the current row. If the phrase instances in the
3386** column being considered do not overlap, this object simply iterates
3387** through them. Or, if they do overlap (share one or more tokens in
3388** common), each set of overlapping instances is treated as a single
3389** match. See documentation for the highlight() auxiliary function for
3390** details.
3391**
3392** Usage is:
3393**
3394** for(rc = fts5CInstIterNext(pApi, pFts, iCol, &iter);
3395** (rc==SQLITE_OK && 0==fts5CInstIterEof(&iter);
3396** rc = fts5CInstIterNext(&iter)
3397** ){
3398** printf("instance starts at %d, ends at %d\n", iter.iStart, iter.iEnd);
3399** }
3400**
3401*/
3402typedef struct CInstIter CInstIter;
3403struct CInstIter {
3404 const Fts5ExtensionApi *pApi; /* API offered by current FTS version */
3405 Fts5Context *pFts; /* First arg to pass to pApi functions */
3406 int iCol; /* Column to search */
3407 int iInst; /* Next phrase instance index */
3408 int nInst; /* Total number of phrase instances */
3409
3410 /* Output variables */
3411 int iStart; /* First token in coalesced phrase instance */
3412 int iEnd; /* Last token in coalesced phrase instance */
3413};
3414
3415/*
3416** Advance the iterator to the next coalesced phrase instance. Return
3417** an SQLite error code if an error occurs, or SQLITE_OK otherwise.
3418*/
3419static int fts5CInstIterNext(CInstIter *pIter){
3420 int rc = SQLITE_OK0;
3421 pIter->iStart = -1;
3422 pIter->iEnd = -1;
3423
3424 while( rc==SQLITE_OK0 && pIter->iInst<pIter->nInst ){
3425 int ip; int ic; int io;
3426 rc = pIter->pApi->xInst(pIter->pFts, pIter->iInst, &ip, &ic, &io);
3427 if( rc==SQLITE_OK0 ){
3428 if( ic==pIter->iCol ){
3429 int iEnd = io - 1 + pIter->pApi->xPhraseSize(pIter->pFts, ip);
3430 if( pIter->iStart<0 ){
3431 pIter->iStart = io;
3432 pIter->iEnd = iEnd;
3433 }else if( io<=pIter->iEnd ){
3434 if( iEnd>pIter->iEnd ) pIter->iEnd = iEnd;
3435 }else{
3436 break;
3437 }
3438 }
3439 pIter->iInst++;
3440 }
3441 }
3442
3443 return rc;
3444}
3445
3446/*
3447** Initialize the iterator object indicated by the final parameter to
3448** iterate through coalesced phrase instances in column iCol.
3449*/
3450static int fts5CInstIterInit(
3451 const Fts5ExtensionApi *pApi,
3452 Fts5Context *pFts,
3453 int iCol,
3454 CInstIter *pIter
3455){
3456 int rc;
3457
3458 memset(pIter, 0, sizeof(CInstIter));
3459 pIter->pApi = pApi;
3460 pIter->pFts = pFts;
3461 pIter->iCol = iCol;
3462 rc = pApi->xInstCount(pFts, &pIter->nInst);
3463
3464 if( rc==SQLITE_OK0 ){
3465 rc = fts5CInstIterNext(pIter);
3466 }
3467
3468 return rc;
3469}
3470
3471
3472
3473/*************************************************************************
3474** Start of highlight() implementation.
3475*/
3476typedef struct HighlightContext HighlightContext;
3477struct HighlightContext {
3478 /* Constant parameters to fts5HighlightCb() */
3479 int iRangeStart; /* First token to include */
3480 int iRangeEnd; /* If non-zero, last token to include */
3481 const char *zOpen; /* Opening highlight */
3482 const char *zClose; /* Closing highlight */
3483 const char *zIn; /* Input text */
3484 int nIn; /* Size of input text in bytes */
3485
3486 /* Variables modified by fts5HighlightCb() */
3487 CInstIter iter; /* Coalesced Instance Iterator */
3488 int iPos; /* Current token offset in zIn[] */
3489 int iOff; /* Have copied up to this offset in zIn[] */
3490 int bOpen; /* True if highlight is open */
3491 char *zOut; /* Output value */
3492};
3493
3494/*
3495** Append text to the HighlightContext output string - p->zOut. Argument
3496** z points to a buffer containing n bytes of text to append. If n is
3497** negative, everything up until the first '\0' is appended to the output.
3498**
3499** If *pRc is set to any value other than SQLITE_OK when this function is
3500** called, it is a no-op. If an error (i.e. an OOM condition) is encountered,
3501** *pRc is set to an error code before returning.
3502*/
3503static void fts5HighlightAppend(
3504 int *pRc,
3505 HighlightContext *p,
3506 const char *z, int n
3507){
3508 if( *pRc==SQLITE_OK0 && z ){
3509 if( n<0 ) n = (int)strlen(z);
3510 p->zOut = sqlite3_mprintfsqlite3_api->mprintf("%z%.*s", p->zOut, n, z);
3511 if( p->zOut==0 ) *pRc = SQLITE_NOMEM7;
3512 }
3513}
3514
3515/*
3516** Tokenizer callback used by implementation of highlight() function.
3517*/
3518static int fts5HighlightCb(
3519 void *pContext, /* Pointer to HighlightContext object */
3520 int tflags, /* Mask of FTS5_TOKEN_* flags */
3521 const char *pToken, /* Buffer containing token */
3522 int nToken, /* Size of token in bytes */
3523 int iStartOff, /* Start byte offset of token */
3524 int iEndOff /* End byte offset of token */
3525){
3526 HighlightContext *p = (HighlightContext*)pContext;
3527 int rc = SQLITE_OK0;
3528 int iPos;
3529
3530 UNUSED_PARAM2(pToken, nToken)(void)(pToken), (void)(nToken);
3531
3532 if( tflags & FTS5_TOKEN_COLOCATED0x0001 ) return SQLITE_OK0;
3533 iPos = p->iPos++;
3534
3535 if( p->iRangeEnd>=0 ){
3536 if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK0;
3537 if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
3538 }
3539
3540 /* If the parenthesis is open, and this token is not part of the current
3541 ** phrase, and the starting byte offset of this token is past the point
3542 ** that has currently been copied into the output buffer, close the
3543 ** parenthesis. */
3544 if( p->bOpen
3545 && (iPos<=p->iter.iStart || p->iter.iStart<0)
3546 && iStartOff>p->iOff
3547 ){
3548 fts5HighlightAppend(&rc, p, p->zClose, -1);
3549 p->bOpen = 0;
3550 }
3551
3552 /* If this is the start of a new phrase, and the highlight is not open:
3553 **
3554 ** * copy text from the input up to the start of the phrase, and
3555 ** * open the highlight.
3556 */
3557 if( iPos==p->iter.iStart && p->bOpen==0 ){
3558 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
3559 fts5HighlightAppend(&rc, p, p->zOpen, -1);
3560 p->iOff = iStartOff;
3561 p->bOpen = 1;
3562 }
3563
3564 if( iPos==p->iter.iEnd ){
3565 if( p->bOpen==0 ){
3566 assert( p->iRangeEnd>=0 )((void) (0));
3567 fts5HighlightAppend(&rc, p, p->zOpen, -1);
3568 p->bOpen = 1;
3569 }
3570 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
3571 p->iOff = iEndOff;
3572
3573 if( rc==SQLITE_OK0 ){
3574 rc = fts5CInstIterNext(&p->iter);
3575 }
3576 }
3577
3578 if( iPos==p->iRangeEnd ){
3579 if( p->bOpen ){
3580 if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){
3581 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
3582 p->iOff = iEndOff;
3583 }
3584 fts5HighlightAppend(&rc, p, p->zClose, -1);
3585 p->bOpen = 0;
3586 }
3587 fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
3588 p->iOff = iEndOff;
3589 }
3590
3591 return rc;
3592}
3593
3594
3595/*
3596** Implementation of highlight() function.
3597*/
3598static void fts5HighlightFunction(
3599 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
3600 Fts5Context *pFts, /* First arg to pass to pApi functions */
3601 sqlite3_context *pCtx, /* Context for returning result/error */
3602 int nVal, /* Number of values in apVal[] array */
3603 sqlite3_value **apVal /* Array of trailing arguments */
3604){
3605 HighlightContext ctx;
3606 int rc;
3607 int iCol;
3608
3609 if( nVal!=3 ){
3610 const char *zErr = "wrong number of arguments to function highlight()";
3611 sqlite3_result_errorsqlite3_api->result_error(pCtx, zErr, -1);
3612 return;
3613 }
3614
3615 iCol = sqlite3_value_intsqlite3_api->value_int(apVal[0]);
3616 memset(&ctx, 0, sizeof(HighlightContext));
3617 ctx.zOpen = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[1]);
3618 ctx.zClose = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[2]);
3619 ctx.iRangeEnd = -1;
3620 rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);
3621 if( rc==SQLITE_RANGE25 ){
3622 sqlite3_result_textsqlite3_api->result_text(pCtx, "", -1, SQLITE_STATIC((sqlite3_destructor_type)0));
3623 rc = SQLITE_OK0;
3624 }else if( ctx.zIn ){
3625 const char *pLoc = 0; /* Locale of column iCol */
3626 int nLoc = 0; /* Size of pLoc in bytes */
3627 if( rc==SQLITE_OK0 ){
3628 rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
3629 }
3630
3631 if( rc==SQLITE_OK0 ){
3632 rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc);
3633 }
3634 if( rc==SQLITE_OK0 ){
3635 rc = pApi->xTokenize_v2(
3636 pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb
3637 );
3638 }
3639 if( ctx.bOpen ){
3640 fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
3641 }
3642 fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
3643
3644 if( rc==SQLITE_OK0 ){
3645 sqlite3_result_textsqlite3_api->result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
3646 }
3647 sqlite3_freesqlite3_api->free(ctx.zOut);
3648 }
3649 if( rc!=SQLITE_OK0 ){
3650 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
3651 }
3652}
3653/*
3654** End of highlight() implementation.
3655**************************************************************************/
3656
3657/*
3658** Context object passed to the fts5SentenceFinderCb() function.
3659*/
3660typedef struct Fts5SFinder Fts5SFinder;
3661struct Fts5SFinder {
3662 int iPos; /* Current token position */
3663 int nFirstAlloc; /* Allocated size of aFirst[] */
3664 int nFirst; /* Number of entries in aFirst[] */
3665 int *aFirst; /* Array of first token in each sentence */
3666 const char *zDoc; /* Document being tokenized */
3667};
3668
3669/*
3670** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if
3671** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an
3672** error occurs.
3673*/
3674static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
3675 if( p->nFirstAlloc==p->nFirst ){
3676 int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
3677 int *aNew;
3678
3679 aNew = (int*)sqlite3_realloc64sqlite3_api->realloc64(p->aFirst, nNew*sizeof(int));
3680 if( aNew==0 ) return SQLITE_NOMEM7;
3681 p->aFirst = aNew;
3682 p->nFirstAlloc = nNew;
3683 }
3684 p->aFirst[p->nFirst++] = iAdd;
3685 return SQLITE_OK0;
3686}
3687
3688/*
3689** This function is an xTokenize() callback used by the auxiliary snippet()
3690** function. Its job is to identify tokens that are the first in a sentence.
3691** For each such token, an entry is added to the SFinder.aFirst[] array.
3692*/
3693static int fts5SentenceFinderCb(
3694 void *pContext, /* Pointer to HighlightContext object */
3695 int tflags, /* Mask of FTS5_TOKEN_* flags */
3696 const char *pToken, /* Buffer containing token */
3697 int nToken, /* Size of token in bytes */
3698 int iStartOff, /* Start offset of token */
3699 int iEndOff /* End offset of token */
3700){
3701 int rc = SQLITE_OK0;
3702
3703 UNUSED_PARAM2(pToken, nToken)(void)(pToken), (void)(nToken);
3704 UNUSED_PARAM(iEndOff)(void)(iEndOff);
3705
3706 if( (tflags & FTS5_TOKEN_COLOCATED0x0001)==0 ){
3707 Fts5SFinder *p = (Fts5SFinder*)pContext;
3708 if( p->iPos>0 ){
3709 int i;
3710 char c = 0;
3711 for(i=iStartOff-1; i>=0; i--){
3712 c = p->zDoc[i];
3713 if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break;
3714 }
3715 if( i!=iStartOff-1 && (c=='.' || c==':') ){
3716 rc = fts5SentenceFinderAdd(p, p->iPos);
3717 }
3718 }else{
3719 rc = fts5SentenceFinderAdd(p, 0);
3720 }
3721 p->iPos++;
3722 }
3723 return rc;
3724}
3725
3726static int fts5SnippetScore(
3727 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
3728 Fts5Context *pFts, /* First arg to pass to pApi functions */
3729 int nDocsize, /* Size of column in tokens */
3730 unsigned char *aSeen, /* Array with one element per query phrase */
3731 int iCol, /* Column to score */
3732 int iPos, /* Starting offset to score */
3733 int nToken, /* Max tokens per snippet */
3734 int *pnScore, /* OUT: Score */
3735 int *piPos /* OUT: Adjusted offset */
3736){
3737 int rc;
3738 int i;
3739 int ip = 0;
3740 int ic = 0;
3741 int iOff = 0;
3742 int iFirst = -1;
3743 int nInst;
3744 int nScore = 0;
3745 int iLast = 0;
3746 sqlite3_int64 iEnd = (sqlite3_int64)iPos + nToken;
3747
3748 rc = pApi->xInstCount(pFts, &nInst);
3749 for(i=0; i<nInst && rc==SQLITE_OK0; i++){
3750 rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
3751 if( rc==SQLITE_OK0 && ic==iCol && iOff>=iPos && iOff<iEnd ){
3752 nScore += (aSeen[ip] ? 1 : 1000);
3753 aSeen[ip] = 1;
3754 if( iFirst<0 ) iFirst = iOff;
3755 iLast = iOff + pApi->xPhraseSize(pFts, ip);
3756 }
3757 }
3758
3759 *pnScore = nScore;
3760 if( piPos ){
3761 sqlite3_int64 iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
3762 if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
3763 if( iAdj<0 ) iAdj = 0;
3764 *piPos = (int)iAdj;
3765 }
3766
3767 return rc;
3768}
3769
3770/*
3771** Return the value in pVal interpreted as utf-8 text. Except, if pVal
3772** contains a NULL value, return a pointer to a static string zero
3773** bytes in length instead of a NULL pointer.
3774*/
3775static const char *fts5ValueToText(sqlite3_value *pVal){
3776 const char *zRet = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
3777 return zRet ? zRet : "";
3778}
3779
3780/*
3781** Implementation of snippet() function.
3782*/
3783static void fts5SnippetFunction(
3784 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
3785 Fts5Context *pFts, /* First arg to pass to pApi functions */
3786 sqlite3_context *pCtx, /* Context for returning result/error */
3787 int nVal, /* Number of values in apVal[] array */
3788 sqlite3_value **apVal /* Array of trailing arguments */
3789){
3790 HighlightContext ctx;
3791 int rc = SQLITE_OK0; /* Return code */
3792 int iCol; /* 1st argument to snippet() */
3793 const char *zEllips; /* 4th argument to snippet() */
3794 i64 nToken; /* 5th argument to snippet() */
3795 int nInst = 0; /* Number of instance matches this row */
3796 int i; /* Used to iterate through instances */
3797 int nPhrase; /* Number of phrases in query */
3798 unsigned char *aSeen; /* Array of "seen instance" flags */
3799 int iBestCol; /* Column containing best snippet */
3800 int iBestStart = 0; /* First token of best snippet */
3801 int nBestScore = 0; /* Score of best snippet */
3802 int nColSize = 0; /* Total size of iBestCol in tokens */
3803 Fts5SFinder sFinder; /* Used to find the beginnings of sentences */
3804 int nCol;
3805
3806 if( nVal!=5 ){
3807 const char *zErr = "wrong number of arguments to function snippet()";
3808 sqlite3_result_errorsqlite3_api->result_error(pCtx, zErr, -1);
3809 return;
3810 }
3811
3812 nCol = pApi->xColumnCount(pFts);
3813 memset(&ctx, 0, sizeof(HighlightContext));
3814 iCol = sqlite3_value_intsqlite3_api->value_int(apVal[0]);
3815 ctx.zOpen = fts5ValueToText(apVal[1]);
3816 ctx.zClose = fts5ValueToText(apVal[2]);
3817 ctx.iRangeEnd = -1;
3818 zEllips = fts5ValueToText(apVal[3]);
3819 nToken = (int)(MIN( MAX(sqlite3_value_int64(apVal[4]), 0), 64)((((((sqlite3_api->value_int64(apVal[4])) > (0)) ? (sqlite3_api
->value_int64(apVal[4])) : (0))) < (64)) ? ((((sqlite3_api
->value_int64(apVal[4])) > (0)) ? (sqlite3_api->value_int64
(apVal[4])) : (0))) : (64)));
3820
3821 iBestCol = (iCol>=0 ? iCol : 0);
3822 nPhrase = pApi->xPhraseCount(pFts);
3823 aSeen = sqlite3_malloc64sqlite3_api->malloc64(nPhrase);
3824 if( aSeen==0 ){
3825 rc = SQLITE_NOMEM7;
3826 }
3827 if( rc==SQLITE_OK0 ){
3828 rc = pApi->xInstCount(pFts, &nInst);
3829 }
3830
3831 memset(&sFinder, 0, sizeof(Fts5SFinder));
3832 for(i=0; i<nCol; i++){
3833 if( iCol<0 || iCol==i ){
3834 const char *pLoc = 0; /* Locale of column iCol */
3835 int nLoc = 0; /* Size of pLoc in bytes */
3836 int nDoc;
3837 int nDocsize;
3838 int ii;
3839 sFinder.iPos = 0;
3840 sFinder.nFirst = 0;
3841 rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
3842 if( rc!=SQLITE_OK0 ) break;
3843 rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc);
3844 if( rc!=SQLITE_OK0 ) break;
3845 rc = pApi->xTokenize_v2(pFts,
3846 sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb
3847 );
3848 if( rc!=SQLITE_OK0 ) break;
3849 rc = pApi->xColumnSize(pFts, i, &nDocsize);
3850 if( rc!=SQLITE_OK0 ) break;
3851
3852 for(ii=0; rc==SQLITE_OK0 && ii<nInst; ii++){
3853 int ip, ic, io;
3854 int iAdj;
3855 int nScore;
3856 int jj;
3857
3858 rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
3859 if( ic!=i ) continue;
3860 if( io>nDocsize ) rc = FTS5_CORRUPT(11 | (1<<8));
3861 if( rc!=SQLITE_OK0 ) continue;
3862 memset(aSeen, 0, nPhrase);
3863 rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
3864 io, nToken, &nScore, &iAdj
3865 );
3866 if( rc==SQLITE_OK0 && nScore>nBestScore ){
3867 nBestScore = nScore;
3868 iBestCol = i;
3869 iBestStart = iAdj;
3870 nColSize = nDocsize;
3871 }
3872
3873 if( rc==SQLITE_OK0 && sFinder.nFirst && nDocsize>nToken ){
3874 for(jj=0; jj<(sFinder.nFirst-1); jj++){
3875 if( sFinder.aFirst[jj+1]>io ) break;
3876 }
3877
3878 if( sFinder.aFirst[jj]<io ){
3879 memset(aSeen, 0, nPhrase);
3880 rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
3881 sFinder.aFirst[jj], nToken, &nScore, 0
3882 );
3883
3884 nScore += (sFinder.aFirst[jj]==0 ? 120 : 100);
3885 if( rc==SQLITE_OK0 && nScore>nBestScore ){
3886 nBestScore = nScore;
3887 iBestCol = i;
3888 iBestStart = sFinder.aFirst[jj];
3889 nColSize = nDocsize;
3890 }
3891 }
3892 }
3893 }
3894 }
3895 }
3896
3897 if( rc==SQLITE_OK0 ){
3898 rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
3899 }
3900 if( rc==SQLITE_OK0 && nColSize==0 ){
3901 rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
3902 }
3903 if( ctx.zIn ){
3904 const char *pLoc = 0; /* Locale of column iBestCol */
3905 int nLoc = 0; /* Bytes in pLoc */
3906
3907 if( rc==SQLITE_OK0 ){
3908 rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
3909 }
3910
3911 ctx.iRangeStart = iBestStart;
3912 ctx.iRangeEnd = iBestStart + nToken - 1;
3913
3914 if( iBestStart>0 ){
3915 fts5HighlightAppend(&rc, &ctx, zEllips, -1);
3916 }
3917
3918 /* Advance iterator ctx.iter so that it points to the first coalesced
3919 ** phrase instance at or following position iBestStart. */
3920 while( ctx.iter.iStart>=0 && ctx.iter.iStart<iBestStart && rc==SQLITE_OK0 ){
3921 rc = fts5CInstIterNext(&ctx.iter);
3922 }
3923
3924 if( rc==SQLITE_OK0 ){
3925 rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc);
3926 }
3927 if( rc==SQLITE_OK0 ){
3928 rc = pApi->xTokenize_v2(
3929 pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb
3930 );
3931 }
3932 if( ctx.bOpen ){
3933 fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
3934 }
3935 if( ctx.iRangeEnd>=(nColSize-1) ){
3936 fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
3937 }else{
3938 fts5HighlightAppend(&rc, &ctx, zEllips, -1);
3939 }
3940 }
3941 if( rc==SQLITE_OK0 ){
3942 sqlite3_result_textsqlite3_api->result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
3943 }else{
3944 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
3945 }
3946 sqlite3_freesqlite3_api->free(ctx.zOut);
3947 sqlite3_freesqlite3_api->free(aSeen);
3948 sqlite3_freesqlite3_api->free(sFinder.aFirst);
3949}
3950
3951/************************************************************************/
3952
3953/*
3954** The first time the bm25() function is called for a query, an instance
3955** of the following structure is allocated and populated.
3956*/
3957typedef struct Fts5Bm25Data Fts5Bm25Data;
3958struct Fts5Bm25Data {
3959 int nPhrase; /* Number of phrases in query */
3960 double avgdl; /* Average number of tokens in each row */
3961 double *aIDF; /* IDF for each phrase */
3962 double *aFreq; /* Array used to calculate phrase freq. */
3963};
3964
3965/*
3966** Callback used by fts5Bm25GetData() to count the number of rows in the
3967** table matched by each individual phrase within the query.
3968*/
3969static int fts5CountCb(
3970 const Fts5ExtensionApi *pApi,
3971 Fts5Context *pFts,
3972 void *pUserData /* Pointer to sqlite3_int64 variable */
3973){
3974 sqlite3_int64 *pn = (sqlite3_int64*)pUserData;
3975 UNUSED_PARAM2(pApi, pFts)(void)(pApi), (void)(pFts);
3976 (*pn)++;
3977 return SQLITE_OK0;
3978}
3979
3980/*
3981** Set *ppData to point to the Fts5Bm25Data object for the current query.
3982** If the object has not already been allocated, allocate and populate it
3983** now.
3984*/
3985static int fts5Bm25GetData(
3986 const Fts5ExtensionApi *pApi,
3987 Fts5Context *pFts,
3988 Fts5Bm25Data **ppData /* OUT: bm25-data object for this query */
3989){
3990 int rc = SQLITE_OK0; /* Return code */
3991 Fts5Bm25Data *p; /* Object to return */
3992
3993 p = (Fts5Bm25Data*)pApi->xGetAuxdata(pFts, 0);
3994 if( p==0 ){
3995 int nPhrase; /* Number of phrases in query */
3996 sqlite3_int64 nRow = 0; /* Number of rows in table */
3997 sqlite3_int64 nToken = 0; /* Number of tokens in table */
3998 sqlite3_int64 nByte; /* Bytes of space to allocate */
3999 int i;
4000
4001 /* Allocate the Fts5Bm25Data object */
4002 nPhrase = pApi->xPhraseCount(pFts);
4003 nByte = sizeof(Fts5Bm25Data) + nPhrase*2*sizeof(double);
4004 p = (Fts5Bm25Data*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
4005 if( p==0 ){
4006 rc = SQLITE_NOMEM7;
4007 }else{
4008 memset(p, 0, (size_t)nByte);
4009 p->nPhrase = nPhrase;
4010 p->aIDF = (double*)&p[1];
4011 p->aFreq = &p->aIDF[nPhrase];
4012 }
4013
4014 /* Calculate the average document length for this FTS5 table */
4015 if( rc==SQLITE_OK0 ) rc = pApi->xRowCount(pFts, &nRow);
4016 assert( rc!=SQLITE_OK || nRow>0 )((void) (0));
4017 if( rc==SQLITE_OK0 ) rc = pApi->xColumnTotalSize(pFts, -1, &nToken);
4018 if( rc==SQLITE_OK0 ) p->avgdl = (double)nToken / (double)nRow;
4019
4020 /* Calculate an IDF for each phrase in the query */
4021 for(i=0; rc==SQLITE_OK0 && i<nPhrase; i++){
4022 sqlite3_int64 nHit = 0;
4023 rc = pApi->xQueryPhrase(pFts, i, (void*)&nHit, fts5CountCb);
4024 if( rc==SQLITE_OK0 ){
4025 /* Calculate the IDF (Inverse Document Frequency) for phrase i.
4026 ** This is done using the standard BM25 formula as found on wikipedia:
4027 **
4028 ** IDF = log( (N - nHit + 0.5) / (nHit + 0.5) )
4029 **
4030 ** where "N" is the total number of documents in the set and nHit
4031 ** is the number that contain at least one instance of the phrase
4032 ** under consideration.
4033 **
4034 ** The problem with this is that if (N < 2*nHit), the IDF is
4035 ** negative. Which is undesirable. So the minimum allowable IDF is
4036 ** (1e-6) - roughly the same as a term that appears in just over
4037 ** half of set of 5,000,000 documents. */
4038 double idf = log( (nRow - nHit + 0.5) / (nHit + 0.5) );
4039 if( idf<=0.0 ) idf = 1e-6;
4040 p->aIDF[i] = idf;
4041 }
4042 }
4043
4044 if( rc!=SQLITE_OK0 ){
4045 sqlite3_freesqlite3_api->free(p);
4046 }else{
4047 rc = pApi->xSetAuxdata(pFts, p, sqlite3_freesqlite3_api->free);
4048 }
4049 if( rc!=SQLITE_OK0 ) p = 0;
4050 }
4051 *ppData = p;
4052 return rc;
4053}
4054
4055/*
4056** Implementation of bm25() function.
4057*/
4058static void fts5Bm25Function(
4059 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
4060 Fts5Context *pFts, /* First arg to pass to pApi functions */
4061 sqlite3_context *pCtx, /* Context for returning result/error */
4062 int nVal, /* Number of values in apVal[] array */
4063 sqlite3_value **apVal /* Array of trailing arguments */
4064){
4065 const double k1 = 1.2; /* Constant "k1" from BM25 formula */
4066 const double b = 0.75; /* Constant "b" from BM25 formula */
4067 int rc; /* Error code */
4068 double score = 0.0; /* SQL function return value */
4069 Fts5Bm25Data *pData; /* Values allocated/calculated once only */
4070 int i; /* Iterator variable */
4071 int nInst = 0; /* Value returned by xInstCount() */
4072 double D = 0.0; /* Total number of tokens in row */
4073 double *aFreq = 0; /* Array of phrase freq. for current row */
4074
4075 /* Calculate the phrase frequency (symbol "f(qi,D)" in the documentation)
4076 ** for each phrase in the query for the current row. */
4077 rc = fts5Bm25GetData(pApi, pFts, &pData);
4078 if( rc==SQLITE_OK0 ){
4079 aFreq = pData->aFreq;
4080 memset(aFreq, 0, sizeof(double) * pData->nPhrase);
4081 rc = pApi->xInstCount(pFts, &nInst);
4082 }
4083 for(i=0; rc==SQLITE_OK0 && i<nInst; i++){
4084 int ip; int ic; int io;
4085 rc = pApi->xInst(pFts, i, &ip, &ic, &io);
4086 if( rc==SQLITE_OK0 ){
4087 double w = (nVal > ic) ? sqlite3_value_doublesqlite3_api->value_double(apVal[ic]) : 1.0;
4088 aFreq[ip] += w;
4089 }
4090 }
4091
4092 /* Figure out the total size of the current row in tokens. */
4093 if( rc==SQLITE_OK0 ){
4094 int nTok;
4095 rc = pApi->xColumnSize(pFts, -1, &nTok);
4096 D = (double)nTok;
4097 }
4098
4099 /* Determine and return the BM25 score for the current row. Or, if an
4100 ** error has occurred, throw an exception. */
4101 if( rc==SQLITE_OK0 ){
4102 for(i=0; i<pData->nPhrase; i++){
4103 score += pData->aIDF[i] * (
4104 ( aFreq[i] * (k1 + 1.0) ) /
4105 ( aFreq[i] + k1 * (1 - b + b * D / pData->avgdl) )
4106 );
4107 }
4108 sqlite3_result_doublesqlite3_api->result_double(pCtx, -1.0 * score);
4109 }else{
4110 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
4111 }
4112}
4113
4114/*
4115** Implementation of fts5_get_locale() function.
4116*/
4117static void fts5GetLocaleFunction(
4118 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
4119 Fts5Context *pFts, /* First arg to pass to pApi functions */
4120 sqlite3_context *pCtx, /* Context for returning result/error */
4121 int nVal, /* Number of values in apVal[] array */
4122 sqlite3_value **apVal /* Array of trailing arguments */
4123){
4124 int iCol = 0;
4125 int eType = 0;
4126 int rc = SQLITE_OK0;
4127 const char *zLocale = 0;
4128 int nLocale = 0;
4129
4130 /* xColumnLocale() must be available */
4131 assert( pApi->iVersion>=4 )((void) (0));
4132
4133 if( nVal!=1 ){
4134 const char *z = "wrong number of arguments to function fts5_get_locale()";
4135 sqlite3_result_errorsqlite3_api->result_error(pCtx, z, -1);
4136 return;
4137 }
4138
4139 eType = sqlite3_value_numeric_typesqlite3_api->value_numeric_type(apVal[0]);
4140 if( eType!=SQLITE_INTEGER1 ){
4141 const char *z = "non-integer argument passed to function fts5_get_locale()";
4142 sqlite3_result_errorsqlite3_api->result_error(pCtx, z, -1);
4143 return;
4144 }
4145
4146 iCol = sqlite3_value_intsqlite3_api->value_int(apVal[0]);
4147 if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){
4148 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, SQLITE_RANGE25);
4149 return;
4150 }
4151
4152 rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale);
4153 if( rc!=SQLITE_OK0 ){
4154 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
4155 return;
4156 }
4157
4158 sqlite3_result_textsqlite3_api->result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
4159}
4160
4161static int sqlite3Fts5AuxInit(fts5_api *pApi){
4162 struct Builtin {
4163 const char *zFunc; /* Function name (nul-terminated) */
4164 void *pUserData; /* User-data pointer */
4165 fts5_extension_function xFunc;/* Callback function */
4166 void (*xDestroy)(void*); /* Destructor function */
4167 } aBuiltin [] = {
4168 { "snippet", 0, fts5SnippetFunction, 0 },
4169 { "highlight", 0, fts5HighlightFunction, 0 },
4170 { "bm25", 0, fts5Bm25Function, 0 },
4171 { "fts5_get_locale", 0, fts5GetLocaleFunction, 0 },
4172 };
4173 int rc = SQLITE_OK0; /* Return code */
4174 int i; /* To iterate through builtin functions */
4175
4176 for(i=0; rc==SQLITE_OK0 && i<ArraySize(aBuiltin)((int)(sizeof(aBuiltin) / sizeof(aBuiltin[0]))); i++){
4177 rc = pApi->xCreateFunction(pApi,
4178 aBuiltin[i].zFunc,
4179 aBuiltin[i].pUserData,
4180 aBuiltin[i].xFunc,
4181 aBuiltin[i].xDestroy
4182 );
4183 }
4184
4185 return rc;
4186}
4187
4188#line 1 "fts5_buffer.c"
4189/*
4190** 2014 May 31
4191**
4192** The author disclaims copyright to this source code. In place of
4193** a legal notice, here is a blessing:
4194**
4195** May you do good and not evil.
4196** May you find forgiveness for yourself and forgive others.
4197** May you share freely, never taking more than you give.
4198**
4199******************************************************************************
4200*/
4201
4202
4203
4204/* #include "fts5Int.h" */
4205
4206static int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
4207 if( (u32)pBuf->nSpace<nByte ){
4208 u64 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
4209 u8 *pNew;
4210 while( nNew<nByte ){
4211 nNew = nNew * 2;
4212 }
4213 pNew = sqlite3_realloc64sqlite3_api->realloc64(pBuf->p, nNew);
4214 if( pNew==0 ){
4215 *pRc = SQLITE_NOMEM7;
4216 return 1;
4217 }else{
4218 pBuf->nSpace = (int)nNew;
4219 pBuf->p = pNew;
4220 }
4221 }
4222 return 0;
4223}
4224
4225
4226/*
4227** Encode value iVal as an SQLite varint and append it to the buffer object
4228** pBuf. If an OOM error occurs, set the error code in p.
4229*/
4230static void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){
4231 if( fts5BufferGrow(pRc, pBuf, 9)( (u32)((pBuf)->n) + (u32)(9) <= (u32)((pBuf)->nSpace
) ? 0 : sqlite3Fts5BufferSize((pRc),(pBuf),(9)+(pBuf)->n) ) ) return;
4232 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iVal);
4233}
4234
4235static void sqlite3Fts5Put32(u8 *aBuf, int iVal){
4236 aBuf[0] = (iVal>>24) & 0x00FF;
4237 aBuf[1] = (iVal>>16) & 0x00FF;
4238 aBuf[2] = (iVal>> 8) & 0x00FF;
4239 aBuf[3] = (iVal>> 0) & 0x00FF;
4240}
4241
4242static int sqlite3Fts5Get32(const u8 *aBuf){
4243 return (int)((((u32)aBuf[0])<<24) + (aBuf[1]<<16) + (aBuf[2]<<8) + aBuf[3]);
4244}
4245
4246/*
4247** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set
4248** the error code in p. If an error has already occurred when this function
4249** is called, it is a no-op.
4250*/
4251static void sqlite3Fts5BufferAppendBlob(
4252 int *pRc,
4253 Fts5Buffer *pBuf,
4254 u32 nData,
4255 const u8 *pData
4256){
4257 if( nData ){
4258 if( fts5BufferGrow(pRc, pBuf, nData)( (u32)((pBuf)->n) + (u32)(nData) <= (u32)((pBuf)->nSpace
) ? 0 : sqlite3Fts5BufferSize((pRc),(pBuf),(nData)+(pBuf)->
n) ) ) return;
4259 assert( pBuf->p!=0 )((void) (0));
4260 memcpy(&pBuf->p[pBuf->n], pData, nData);
4261 pBuf->n += nData;
4262 }
4263}
4264
4265/*
4266** Append the nul-terminated string zStr to the buffer pBuf. This function
4267** ensures that the byte following the buffer data is set to 0x00, even
4268** though this byte is not included in the pBuf->n count.
4269*/
4270static void sqlite3Fts5BufferAppendString(
4271 int *pRc,
4272 Fts5Buffer *pBuf,
4273 const char *zStr
4274){
4275 int nStr = (int)strlen(zStr);
4276 sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr+1, (const u8*)zStr);
4277 pBuf->n--;
4278}
4279
4280/*
4281** Argument zFmt is a printf() style format string. This function performs
4282** the printf() style processing, then appends the results to buffer pBuf.
4283**
4284** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte
4285** following the buffer data is set to 0x00, even though this byte is not
4286** included in the pBuf->n count.
4287*/
4288static void sqlite3Fts5BufferAppendPrintf(
4289 int *pRc,
4290 Fts5Buffer *pBuf,
4291 char *zFmt, ...
4292){
4293 if( *pRc==SQLITE_OK0 ){
4294 char *zTmp;
4295 va_list ap;
4296 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
4297 zTmp = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
4298 va_end(ap)__builtin_va_end(ap);
4299
4300 if( zTmp==0 ){
4301 *pRc = SQLITE_NOMEM7;
4302 }else{
4303 sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp);
4304 sqlite3_freesqlite3_api->free(zTmp);
4305 }
4306 }
4307}
4308
4309static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...){
4310 char *zRet = 0;
4311 if( *pRc==SQLITE_OK0 ){
4312 va_list ap;
4313 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
4314 zRet = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
4315 va_end(ap)__builtin_va_end(ap);
4316 if( zRet==0 ){
4317 *pRc = SQLITE_NOMEM7;
4318 }
4319 }
4320 return zRet;
4321}
4322
4323
4324/*
4325** Free any buffer allocated by pBuf. Zero the structure before returning.
4326*/
4327static void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){
4328 sqlite3_freesqlite3_api->free(pBuf->p);
4329 memset(pBuf, 0, sizeof(Fts5Buffer));
4330}
4331
4332/*
4333** Zero the contents of the buffer object. But do not free the associated
4334** memory allocation.
4335*/
4336static void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){
4337 pBuf->n = 0;
4338}
4339
4340/*
4341** Set the buffer to contain nData/pData. If an OOM error occurs, leave an
4342** the error code in p. If an error has already occurred when this function
4343** is called, it is a no-op.
4344*/
4345static void sqlite3Fts5BufferSet(
4346 int *pRc,
4347 Fts5Buffer *pBuf,
4348 int nData,
4349 const u8 *pData
4350){
4351 pBuf->n = 0;
4352 sqlite3Fts5BufferAppendBlob(pRc, pBuf, nData, pData);
4353}
4354
4355static int sqlite3Fts5PoslistNext64(
4356 const u8 *a, int n, /* Buffer containing poslist */
4357 int *pi, /* IN/OUT: Offset within a[] */
4358 i64 *piOff /* IN/OUT: Current offset */
4359){
4360 int i = *pi;
4361 assert( a!=0 || i==0 )((void) (0));
4362 if( i>=n ){
4363 /* EOF */
4364 *piOff = -1;
4365 return 1;
4366 }else{
4367 i64 iOff = *piOff;
4368 u32 iVal;
4369 assert( a!=0 )((void) (0));
4370 fts5FastGetVarint32(a, i, iVal){ iVal = (a)[i++]; if( iVal & 0x80 ){ i--; i += sqlite3Fts5GetVarint32
(&(a)[i],(u32*)&(iVal)); } };
4371 if( iVal<=1 ){
4372 if( iVal==0 ){
4373 *pi = i;
4374 return 0;
4375 }
4376 fts5FastGetVarint32(a, i, iVal){ iVal = (a)[i++]; if( iVal & 0x80 ){ i--; i += sqlite3Fts5GetVarint32
(&(a)[i],(u32*)&(iVal)); } };
4377 iOff = ((i64)iVal) << 32;
4378 assert( iOff>=0 )((void) (0));
4379 fts5FastGetVarint32(a, i, iVal){ iVal = (a)[i++]; if( iVal & 0x80 ){ i--; i += sqlite3Fts5GetVarint32
(&(a)[i],(u32*)&(iVal)); } };
4380 if( iVal<2 ){
4381 /* This is a corrupt record. So stop parsing it here. */
4382 *piOff = -1;
4383 return 1;
4384 }
4385 *piOff = iOff + ((iVal-2) & 0x7FFFFFFF);
4386 }else{
4387 *piOff = (iOff & (i64)0x7FFFFFFF<<32)+((iOff + (iVal-2)) & 0x7FFFFFFF);
4388 }
4389 *pi = i;
4390 assert_nc( *piOff>=iOff )((void) (0));
4391 return 0;
4392 }
4393}
4394
4395
4396/*
4397** Advance the iterator object passed as the only argument. Return true
4398** if the iterator reaches EOF, or false otherwise.
4399*/
4400static int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){
4401 if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) ){
4402 pIter->bEof = 1;
4403 }
4404 return pIter->bEof;
4405}
4406
4407static int sqlite3Fts5PoslistReaderInit(
4408 const u8 *a, int n, /* Poslist buffer to iterate through */
4409 Fts5PoslistReader *pIter /* Iterator object to initialize */
4410){
4411 memset(pIter, 0, sizeof(*pIter));
4412 pIter->a = a;
4413 pIter->n = n;
4414 sqlite3Fts5PoslistReaderNext(pIter);
4415 return pIter->bEof;
4416}
4417
4418/*
4419** Append position iPos to the position list being accumulated in buffer
4420** pBuf, which must be already be large enough to hold the new data.
4421** The previous position written to this list is *piPrev. *piPrev is set
4422** to iPos before returning.
4423*/
4424static void sqlite3Fts5PoslistSafeAppend(
4425 Fts5Buffer *pBuf,
4426 i64 *piPrev,
4427 i64 iPos
4428){
4429 if( iPos>=*piPrev ){
4430 static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
4431 if( (iPos & colmask) != (*piPrev & colmask) ){
4432 pBuf->p[pBuf->n++] = 1;
4433 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
4434 *piPrev = (iPos & colmask);
4435 }
4436 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
4437 *piPrev = iPos;
4438 }
4439}
4440
4441static int sqlite3Fts5PoslistWriterAppend(
4442 Fts5Buffer *pBuf,
4443 Fts5PoslistWriter *pWriter,
4444 i64 iPos
4445){
4446 int rc = 0; /* Initialized only to suppress erroneous warning from Clang */
4447 if( fts5BufferGrow(&rc, pBuf, 5+5+5)( (u32)((pBuf)->n) + (u32)(5+5+5) <= (u32)((pBuf)->nSpace
) ? 0 : sqlite3Fts5BufferSize((&rc),(pBuf),(5+5+5)+(pBuf)
->n) ) ) return rc;
4448 sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
4449 return SQLITE_OK0;
4450}
4451
4452static void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte){
4453 void *pRet = 0;
4454 if( *pRc==SQLITE_OK0 ){
4455 pRet = sqlite3_malloc64sqlite3_api->malloc64(nByte);
4456 if( pRet==0 ){
4457 if( nByte>0 ) *pRc = SQLITE_NOMEM7;
4458 }else{
4459 memset(pRet, 0, (size_t)nByte);
4460 }
4461 }
4462 return pRet;
4463}
4464
4465/*
4466** Return a nul-terminated copy of the string indicated by pIn. If nIn
4467** is non-negative, then it is the length of the string in bytes. Otherwise,
4468** the length of the string is determined using strlen().
4469**
4470** It is the responsibility of the caller to eventually free the returned
4471** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned.
4472*/
4473static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){
4474 char *zRet = 0;
4475 if( *pRc==SQLITE_OK0 ){
4476 if( nIn<0 ){
4477 nIn = (int)strlen(pIn);
4478 }
4479 zRet = (char*)sqlite3_malloc64sqlite3_api->malloc64((i64)nIn+1);
4480 if( zRet ){
4481 memcpy(zRet, pIn, nIn);
4482 zRet[nIn] = '\0';
4483 }else{
4484 *pRc = SQLITE_NOMEM7;
4485 }
4486 }
4487 return zRet;
4488}
4489
4490
4491/*
4492** Return true if character 't' may be part of an FTS5 bareword, or false
4493** otherwise. Characters that may be part of barewords:
4494**
4495** * All non-ASCII characters,
4496** * The 52 upper and lower case ASCII characters, and
4497** * The 10 integer ASCII characters.
4498** * The underscore character "_" (0x5F).
4499** * The unicode "substitute" character (0x1A).
4500*/
4501static int sqlite3Fts5IsBareword(char t){
4502 u8 aBareword[128] = {
4503 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */
4504 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */
4505 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */
4506 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */
4507 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */
4508 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */
4509 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */
4510 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */
4511 };
4512
4513 return (t & 0x80) || aBareword[(int)t];
4514}
4515
4516
4517/*************************************************************************
4518*/
4519typedef struct Fts5TermsetEntry Fts5TermsetEntry;
4520struct Fts5TermsetEntry {
4521 char *pTerm;
4522 int nTerm;
4523 int iIdx; /* Index (main or aPrefix[] entry) */
4524 Fts5TermsetEntry *pNext;
4525};
4526
4527struct Fts5Termset {
4528 Fts5TermsetEntry *apHash[512];
4529};
4530
4531static int sqlite3Fts5TermsetNew(Fts5Termset **pp){
4532 int rc = SQLITE_OK0;
4533 *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset));
4534 return rc;
4535}
4536
4537static int sqlite3Fts5TermsetAdd(
4538 Fts5Termset *p,
4539 int iIdx,
4540 const char *pTerm, int nTerm,
4541 int *pbPresent
4542){
4543 int rc = SQLITE_OK0;
4544 *pbPresent = 0;
4545 if( p ){
4546 int i;
4547 u32 hash = 13;
4548 Fts5TermsetEntry *pEntry;
4549
4550 /* Calculate a hash value for this term. This is the same hash checksum
4551 ** used by the fts5_hash.c module. This is not important for correct
4552 ** operation of the module, but is necessary to ensure that some tests
4553 ** designed to produce hash table collisions really do work. */
4554 for(i=nTerm-1; i>=0; i--){
4555 hash = (hash << 3) ^ hash ^ pTerm[i];
4556 }
4557 hash = (hash << 3) ^ hash ^ iIdx;
4558 hash = hash % ArraySize(p->apHash)((int)(sizeof(p->apHash) / sizeof(p->apHash[0])));
4559
4560 for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
4561 if( pEntry->iIdx==iIdx
4562 && pEntry->nTerm==nTerm
4563 && memcmp(pEntry->pTerm, pTerm, nTerm)==0
4564 ){
4565 *pbPresent = 1;
4566 break;
4567 }
4568 }
4569
4570 if( pEntry==0 ){
4571 pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm);
4572 if( pEntry ){
4573 pEntry->pTerm = (char*)&pEntry[1];
4574 pEntry->nTerm = nTerm;
4575 pEntry->iIdx = iIdx;
4576 memcpy(pEntry->pTerm, pTerm, nTerm);
4577 pEntry->pNext = p->apHash[hash];
4578 p->apHash[hash] = pEntry;
4579 }
4580 }
4581 }
4582
4583 return rc;
4584}
4585
4586static void sqlite3Fts5TermsetFree(Fts5Termset *p){
4587 if( p ){
4588 u32 i;
4589 for(i=0; i<ArraySize(p->apHash)((int)(sizeof(p->apHash) / sizeof(p->apHash[0]))); i++){
4590 Fts5TermsetEntry *pEntry = p->apHash[i];
4591 while( pEntry ){
4592 Fts5TermsetEntry *pDel = pEntry;
4593 pEntry = pEntry->pNext;
4594 sqlite3_freesqlite3_api->free(pDel);
4595 }
4596 }
4597 sqlite3_freesqlite3_api->free(p);
4598 }
4599}
4600
4601#line 1 "fts5_config.c"
4602/*
4603** 2014 Jun 09
4604**
4605** The author disclaims copyright to this source code. In place of
4606** a legal notice, here is a blessing:
4607**
4608** May you do good and not evil.
4609** May you find forgiveness for yourself and forgive others.
4610** May you share freely, never taking more than you give.
4611**
4612******************************************************************************
4613**
4614** This is an SQLite module implementing full-text search.
4615*/
4616
4617
4618/* #include "fts5Int.h" */
4619
4620#define FTS5_DEFAULT_PAGE_SIZE4050 4050
4621#define FTS5_DEFAULT_AUTOMERGE4 4
4622#define FTS5_DEFAULT_USERMERGE4 4
4623#define FTS5_DEFAULT_CRISISMERGE16 16
4624#define FTS5_DEFAULT_HASHSIZE(1024*1024) (1024*1024)
4625
4626#define FTS5_DEFAULT_DELETE_AUTOMERGE10 10 /* default 10% */
4627
4628/* Maximum allowed page size */
4629#define FTS5_MAX_PAGE_SIZE(64*1024) (64*1024)
4630
4631static int fts5_iswhitespace(char x){
4632 return (x==' ');
4633}
4634
4635static int fts5_isopenquote(char x){
4636 return (x=='"' || x=='\'' || x=='[' || x=='`');
4637}
4638
4639/*
4640** Argument pIn points to a character that is part of a nul-terminated
4641** string. Return a pointer to the first character following *pIn in
4642** the string that is not a white-space character.
4643*/
4644static const char *fts5ConfigSkipWhitespace(const char *pIn){
4645 const char *p = pIn;
4646 if( p ){
4647 while( fts5_iswhitespace(*p) ){ p++; }
4648 }
4649 return p;
4650}
4651
4652/*
4653** Argument pIn points to a character that is part of a nul-terminated
4654** string. Return a pointer to the first character following *pIn in
4655** the string that is not a "bareword" character.
4656*/
4657static const char *fts5ConfigSkipBareword(const char *pIn){
4658 const char *p = pIn;
4659 while ( sqlite3Fts5IsBareword(*p) ) p++;
4660 if( p==pIn ) p = 0;
4661 return p;
4662}
4663
4664static int fts5_isdigit(char a){
4665 return (a>='0' && a<='9');
4666}
4667
4668
4669
4670static const char *fts5ConfigSkipLiteral(const char *pIn){
4671 const char *p = pIn;
4672 switch( *p ){
4673 case 'n': case 'N':
4674 if( sqlite3_strnicmpsqlite3_api->strnicmp("null", p, 4)==0 ){
4675 p = &p[4];
4676 }else{
4677 p = 0;
4678 }
4679 break;
4680
4681 case 'x': case 'X':
4682 p++;
4683 if( *p=='\'' ){
4684 p++;
4685 while( (*p>='a' && *p<='f')
4686 || (*p>='A' && *p<='F')
4687 || (*p>='0' && *p<='9')
4688 ){
4689 p++;
4690 }
4691 if( *p=='\'' && 0==((p-pIn)%2) ){
4692 p++;
4693 }else{
4694 p = 0;
4695 }
4696 }else{
4697 p = 0;
4698 }
4699 break;
4700
4701 case '\'':
4702 p++;
4703 while( p ){
4704 if( *p=='\'' ){
4705 p++;
4706 if( *p!='\'' ) break;
4707 }
4708 p++;
4709 if( *p==0 ) p = 0;
4710 }
4711 break;
4712
4713 default:
4714 /* maybe a number */
4715 if( *p=='+' || *p=='-' ) p++;
4716 while( fts5_isdigit(*p) ) p++;
4717
4718 /* At this point, if the literal was an integer, the parse is
4719 ** finished. Or, if it is a floating point value, it may continue
4720 ** with either a decimal point or an 'E' character. */
4721 if( *p=='.' && fts5_isdigit(p[1]) ){
4722 p += 2;
4723 while( fts5_isdigit(*p) ) p++;
4724 }
4725 if( p==pIn ) p = 0;
4726
4727 break;
4728 }
4729
4730 return p;
4731}
4732
4733/*
4734** The first character of the string pointed to by argument z is guaranteed
4735** to be an open-quote character (see function fts5_isopenquote()).
4736**
4737** This function searches for the corresponding close-quote character within
4738** the string and, if found, dequotes the string in place and adds a new
4739** nul-terminator byte.
4740**
4741** If the close-quote is found, the value returned is the byte offset of
4742** the character immediately following it. Or, if the close-quote is not
4743** found, -1 is returned. If -1 is returned, the buffer is left in an
4744** undefined state.
4745*/
4746static int fts5Dequote(char *z){
4747 char q;
4748 int iIn = 1;
4749 int iOut = 0;
4750 q = z[0];
4751
4752 /* Set stack variable q to the close-quote character */
4753 assert( q=='[' || q=='\'' || q=='"' || q=='`' )((void) (0));
4754 if( q=='[' ) q = ']';
4755
4756 while( z[iIn] ){
4757 if( z[iIn]==q ){
4758 if( z[iIn+1]!=q ){
4759 /* Character iIn was the close quote. */
4760 iIn++;
4761 break;
4762 }else{
4763 /* Character iIn and iIn+1 form an escaped quote character. Skip
4764 ** the input cursor past both and copy a single quote character
4765 ** to the output buffer. */
4766 iIn += 2;
4767 z[iOut++] = q;
4768 }
4769 }else{
4770 z[iOut++] = z[iIn++];
4771 }
4772 }
4773
4774 z[iOut] = '\0';
4775 return iIn;
4776}
4777
4778/*
4779** Convert an SQL-style quoted string into a normal string by removing
4780** the quote characters. The conversion is done in-place. If the
4781** input does not begin with a quote character, then this routine
4782** is a no-op.
4783**
4784** Examples:
4785**
4786** "abc" becomes abc
4787** 'xyz' becomes xyz
4788** [pqr] becomes pqr
4789** `mno` becomes mno
4790*/
4791static void sqlite3Fts5Dequote(char *z){
4792 char quote; /* Quote character (if any ) */
4793
4794 assert( 0==fts5_iswhitespace(z[0]) )((void) (0));
4795 quote = z[0];
4796 if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
4797 fts5Dequote(z);
4798 }
4799}
4800
4801
4802struct Fts5Enum {
4803 const char *zName;
4804 int eVal;
4805};
4806typedef struct Fts5Enum Fts5Enum;
4807
4808static int fts5ConfigSetEnum(
4809 const Fts5Enum *aEnum,
4810 const char *zEnum,
4811 int *peVal
4812){
4813 int nEnum = (int)strlen(zEnum);
4814 int i;
4815 int iVal = -1;
4816
4817 for(i=0; aEnum[i].zName; i++){
4818 if( sqlite3_strnicmpsqlite3_api->strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
4819 if( iVal>=0 ) return SQLITE_ERROR1;
4820 iVal = aEnum[i].eVal;
4821 }
4822 }
4823
4824 *peVal = iVal;
4825 return iVal<0 ? SQLITE_ERROR1 : SQLITE_OK0;
4826}
4827
4828/*
4829** Parse a "special" CREATE VIRTUAL TABLE directive and update
4830** configuration object pConfig as appropriate.
4831**
4832** If successful, object pConfig is updated and SQLITE_OK returned. If
4833** an error occurs, an SQLite error code is returned and an error message
4834** may be left in *pzErr. It is the responsibility of the caller to
4835** eventually free any such error message using sqlite3_free().
4836*/
4837static int fts5ConfigParseSpecial(
4838 Fts5Config *pConfig, /* Configuration object to update */
4839 const char *zCmd, /* Special command to parse */
4840 const char *zArg, /* Argument to parse */
4841 char **pzErr /* OUT: Error message */
4842){
4843 int rc = SQLITE_OK0;
4844 int nCmd = (int)strlen(zCmd);
4845
4846 if( sqlite3_strnicmpsqlite3_api->strnicmp("prefix", zCmd, nCmd)==0 ){
4847 const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES31;
4848 const char *p;
4849 int bFirst = 1;
4850 if( pConfig->aPrefix==0 ){
4851 pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
4852 if( rc ) return rc;
4853 }
4854
4855 p = zArg;
4856 while( 1 ){
4857 int nPre = 0;
4858
4859 while( p[0]==' ' ) p++;
4860 if( bFirst==0 && p[0]==',' ){
4861 p++;
4862 while( p[0]==' ' ) p++;
4863 }else if( p[0]=='\0' ){
4864 break;
4865 }
4866 if( p[0]<'0' || p[0]>'9' ){
4867 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed prefix=... directive");
4868 rc = SQLITE_ERROR1;
4869 break;
4870 }
4871
4872 if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES31 ){
4873 *pzErr = sqlite3_mprintfsqlite3_api->mprintf(
4874 "too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES31
4875 );
4876 rc = SQLITE_ERROR1;
4877 break;
4878 }
4879
4880 while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
4881 nPre = nPre*10 + (p[0] - '0');
4882 p++;
4883 }
4884
4885 if( nPre<=0 || nPre>=1000 ){
4886 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("prefix length out of range (max 999)");
4887 rc = SQLITE_ERROR1;
4888 break;
4889 }
4890
4891 pConfig->aPrefix[pConfig->nPrefix] = nPre;
4892 pConfig->nPrefix++;
4893 bFirst = 0;
4894 }
4895 assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES )((void) (0));
4896 return rc;
4897 }
4898
4899 if( sqlite3_strnicmpsqlite3_api->strnicmp("tokenize", zCmd, nCmd)==0 ){
4900 const char *p = (const char*)zArg;
4901 sqlite3_int64 nArg = strlen(zArg) + 1;
4902 char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg);
4903
4904 if( azArg ){
4905 char *pSpace = (char*)&azArg[nArg];
4906 if( pConfig->t.azArg ){
4907 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("multiple tokenize=... directives");
4908 rc = SQLITE_ERROR1;
4909 }else{
4910 for(nArg=0; p && *p; nArg++){
4911 const char *p2 = fts5ConfigSkipWhitespace(p);
4912 if( *p2=='\'' ){
4913 p = fts5ConfigSkipLiteral(p2);
4914 }else{
4915 p = fts5ConfigSkipBareword(p2);
4916 }
4917 if( p ){
4918 memcpy(pSpace, p2, p-p2);
4919 azArg[nArg] = pSpace;
4920 sqlite3Fts5Dequote(pSpace);
4921 pSpace += (p - p2) + 1;
4922 p = fts5ConfigSkipWhitespace(p);
4923 }
4924 }
4925 if( p==0 ){
4926 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("parse error in tokenize directive");
4927 rc = SQLITE_ERROR1;
4928 }else{
4929 pConfig->t.azArg = (const char**)azArg;
4930 pConfig->t.nArg = nArg;
4931 azArg = 0;
4932 }
4933 }
4934 }
4935 sqlite3_freesqlite3_api->free(azArg);
4936
4937 return rc;
4938 }
4939
4940 if( sqlite3_strnicmpsqlite3_api->strnicmp("content", zCmd, nCmd)==0 ){
4941 if( pConfig->eContent!=FTS5_CONTENT_NORMAL0 ){
4942 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("multiple content=... directives");
4943 rc = SQLITE_ERROR1;
4944 }else{
4945 if( zArg[0] ){
4946 pConfig->eContent = FTS5_CONTENT_EXTERNAL2;
4947 pConfig->zContent = sqlite3Fts5Mprintf(&rc, "%Q.%Q", pConfig->zDb,zArg);
4948 }else{
4949 pConfig->eContent = FTS5_CONTENT_NONE1;
4950 }
4951 }
4952 return rc;
4953 }
4954
4955 if( sqlite3_strnicmpsqlite3_api->strnicmp("contentless_delete", zCmd, nCmd)==0 ){
4956 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
4957 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed contentless_delete=... directive");
4958 rc = SQLITE_ERROR1;
4959 }else{
4960 pConfig->bContentlessDelete = (zArg[0]=='1');
4961 }
4962 return rc;
4963 }
4964
4965 if( sqlite3_strnicmpsqlite3_api->strnicmp("contentless_unindexed", zCmd, nCmd)==0 ){
4966 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
4967 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed contentless_delete=... directive");
4968 rc = SQLITE_ERROR1;
4969 }else{
4970 pConfig->bContentlessUnindexed = (zArg[0]=='1');
4971 }
4972 return rc;
4973 }
4974
4975 if( sqlite3_strnicmpsqlite3_api->strnicmp("content_rowid", zCmd, nCmd)==0 ){
4976 if( pConfig->zContentRowid ){
4977 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("multiple content_rowid=... directives");
4978 rc = SQLITE_ERROR1;
4979 }else{
4980 pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1);
4981 }
4982 return rc;
4983 }
4984
4985 if( sqlite3_strnicmpsqlite3_api->strnicmp("columnsize", zCmd, nCmd)==0 ){
4986 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
4987 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed columnsize=... directive");
4988 rc = SQLITE_ERROR1;
4989 }else{
4990 pConfig->bColumnsize = (zArg[0]=='1');
4991 }
4992 return rc;
4993 }
4994
4995 if( sqlite3_strnicmpsqlite3_api->strnicmp("locale", zCmd, nCmd)==0 ){
4996 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
4997 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed locale=... directive");
4998 rc = SQLITE_ERROR1;
4999 }else{
5000 pConfig->bLocale = (zArg[0]=='1');
5001 }
5002 return rc;
5003 }
5004
5005 if( sqlite3_strnicmpsqlite3_api->strnicmp("detail", zCmd, nCmd)==0 ){
5006 const Fts5Enum aDetail[] = {
5007 { "none", FTS5_DETAIL_NONE1 },
5008 { "full", FTS5_DETAIL_FULL0 },
5009 { "columns", FTS5_DETAIL_COLUMNS2 },
5010 { 0, 0 }
5011 };
5012
5013 if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){
5014 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed detail=... directive");
5015 }
5016 return rc;
5017 }
5018
5019 if( sqlite3_strnicmpsqlite3_api->strnicmp("tokendata", zCmd, nCmd)==0 ){
5020 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
5021 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed tokendata=... directive");
5022 rc = SQLITE_ERROR1;
5023 }else{
5024 pConfig->bTokendata = (zArg[0]=='1');
5025 }
5026 return rc;
5027 }
5028
5029 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
5030 return SQLITE_ERROR1;
5031}
5032
5033/*
5034** Gobble up the first bareword or quoted word from the input buffer zIn.
5035** Return a pointer to the character immediately following the last in
5036** the gobbled word if successful, or a NULL pointer otherwise (failed
5037** to find close-quote character).
5038**
5039** Before returning, set pzOut to point to a new buffer containing a
5040** nul-terminated, dequoted copy of the gobbled word. If the word was
5041** quoted, *pbQuoted is also set to 1 before returning.
5042**
5043** If *pRc is other than SQLITE_OK when this function is called, it is
5044** a no-op (NULL is returned). Otherwise, if an OOM occurs within this
5045** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not*
5046** set if a parse error (failed to find close quote) occurs.
5047*/
5048static const char *fts5ConfigGobbleWord(
5049 int *pRc, /* IN/OUT: Error code */
5050 const char *zIn, /* Buffer to gobble string/bareword from */
5051 char **pzOut, /* OUT: malloc'd buffer containing str/bw */
5052 int *pbQuoted /* OUT: Set to true if dequoting required */
5053){
5054 const char *zRet = 0;
5055
5056 sqlite3_int64 nIn = strlen(zIn);
5057 char *zOut = sqlite3_malloc64sqlite3_api->malloc64(nIn+1);
5058
5059 assert( *pRc==SQLITE_OK )((void) (0));
5060 *pbQuoted = 0;
5061 *pzOut = 0;
5062
5063 if( zOut==0 ){
5064 *pRc = SQLITE_NOMEM7;
5065 }else{
5066 memcpy(zOut, zIn, (size_t)(nIn+1));
5067 if( fts5_isopenquote(zOut[0]) ){
5068 int ii = fts5Dequote(zOut);
5069 zRet = &zIn[ii];
5070 *pbQuoted = 1;
5071 }else{
5072 zRet = fts5ConfigSkipBareword(zIn);
5073 if( zRet ){
5074 zOut[zRet-zIn] = '\0';
5075 }
5076 }
5077 }
5078
5079 if( zRet==0 ){
5080 sqlite3_freesqlite3_api->free(zOut);
5081 }else{
5082 *pzOut = zOut;
5083 }
5084
5085 return zRet;
5086}
5087
5088static int fts5ConfigParseColumn(
5089 Fts5Config *p,
5090 char *zCol,
5091 char *zArg,
5092 char **pzErr,
5093 int *pbUnindexed
5094){
5095 int rc = SQLITE_OK0;
5096 if( 0==sqlite3_stricmpsqlite3_api->stricmp(zCol, FTS5_RANK_NAME"rank")
5097 || 0==sqlite3_stricmpsqlite3_api->stricmp(zCol, FTS5_ROWID_NAME"rowid")
5098 ){
5099 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("reserved fts5 column name: %s", zCol);
5100 rc = SQLITE_ERROR1;
5101 }else if( zArg ){
5102 if( 0==sqlite3_stricmpsqlite3_api->stricmp(zArg, "unindexed") ){
5103 p->abUnindexed[p->nCol] = 1;
5104 *pbUnindexed = 1;
5105 }else{
5106 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("unrecognized column option: %s", zArg);
5107 rc = SQLITE_ERROR1;
5108 }
5109 }
5110
5111 p->azCol[p->nCol++] = zCol;
5112 return rc;
5113}
5114
5115/*
5116** Populate the Fts5Config.zContentExprlist string.
5117*/
5118static int fts5ConfigMakeExprlist(Fts5Config *p){
5119 int i;
5120 int rc = SQLITE_OK0;
5121 Fts5Buffer buf = {0, 0, 0};
5122
5123 sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid);
5124 if( p->eContent!=FTS5_CONTENT_NONE1 ){
5125 assert( p->eContent==FTS5_CONTENT_EXTERNAL((void) (0))
5126 || p->eContent==FTS5_CONTENT_NORMAL((void) (0))
5127 || p->eContent==FTS5_CONTENT_UNINDEXED((void) (0))
5128 )((void) (0));
5129 for(i=0; i<p->nCol; i++){
5130 if( p->eContent==FTS5_CONTENT_EXTERNAL2 ){
5131 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]);
5132 }else if( p->eContent==FTS5_CONTENT_NORMAL0 || p->abUnindexed[i] ){
5133 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i);
5134 }else{
5135 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
5136 }
5137 }
5138 }
5139 if( p->eContent==FTS5_CONTENT_NORMAL0 && p->bLocale ){
5140 for(i=0; i<p->nCol; i++){
5141 if( p->abUnindexed[i]==0 ){
5142 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i);
5143 }else{
5144 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
5145 }
5146 }
5147 }
5148
5149 assert( p->zContentExprlist==0 )((void) (0));
5150 p->zContentExprlist = (char*)buf.p;
5151 return rc;
5152}
5153
5154/*
5155** Arguments nArg/azArg contain the string arguments passed to the xCreate
5156** or xConnect method of the virtual table. This function attempts to
5157** allocate an instance of Fts5Config containing the results of parsing
5158** those arguments.
5159**
5160** If successful, SQLITE_OK is returned and *ppOut is set to point to the
5161** new Fts5Config object. If an error occurs, an SQLite error code is
5162** returned, *ppOut is set to NULL and an error message may be left in
5163** *pzErr. It is the responsibility of the caller to eventually free any
5164** such error message using sqlite3_free().
5165*/
5166static int sqlite3Fts5ConfigParse(
5167 Fts5Global *pGlobal,
5168 sqlite3 *db,
5169 int nArg, /* Number of arguments */
5170 const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */
5171 Fts5Config **ppOut, /* OUT: Results of parse */
5172 char **pzErr /* OUT: Error message */
5173){
5174 int rc = SQLITE_OK0; /* Return code */
5175 Fts5Config *pRet; /* New object to return */
5176 int i;
5177 sqlite3_int64 nByte;
5178 int bUnindexed = 0; /* True if there are one or more UNINDEXED */
5179
5180 *ppOut = pRet = (Fts5Config*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5Config));
5181 if( pRet==0 ) return SQLITE_NOMEM7;
5182 memset(pRet, 0, sizeof(Fts5Config));
5183 pRet->pGlobal = pGlobal;
5184 pRet->db = db;
5185 pRet->iCookie = -1;
5186
5187 nByte = nArg * (sizeof(char*) + sizeof(u8));
5188 pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
5189 pRet->abUnindexed = pRet->azCol ? (u8*)&pRet->azCol[nArg] : 0;
5190 pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
5191 pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
5192 pRet->bColumnsize = 1;
5193 pRet->eDetail = FTS5_DETAIL_FULL0;
5194#ifdef SQLITE_DEBUG
5195 pRet->bPrefixIndex = 1;
5196#endif
5197 if( rc==SQLITE_OK0 && sqlite3_stricmpsqlite3_api->stricmp(pRet->zName, FTS5_RANK_NAME"rank")==0 ){
5198 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("reserved fts5 table name: %s", pRet->zName);
5199 rc = SQLITE_ERROR1;
5200 }
5201
5202 assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK )((void) (0));
5203 for(i=3; rc==SQLITE_OK0 && i<nArg; i++){
5204 const char *zOrig = azArg[i];
5205 const char *z;
5206 char *zOne = 0;
5207 char *zTwo = 0;
5208 int bOption = 0;
5209 int bMustBeCol = 0;
5210
5211 z = fts5ConfigGobbleWord(&rc, zOrig, &zOne, &bMustBeCol);
5212 z = fts5ConfigSkipWhitespace(z);
5213 if( z && *z=='=' ){
5214 bOption = 1;
5215 assert( zOne!=0 )((void) (0));
5216 z++;
5217 if( bMustBeCol ) z = 0;
5218 }
5219 z = fts5ConfigSkipWhitespace(z);
5220 if( z && z[0] ){
5221 int bDummy;
5222 z = fts5ConfigGobbleWord(&rc, z, &zTwo, &bDummy);
5223 if( z && z[0] ) z = 0;
5224 }
5225
5226 if( rc==SQLITE_OK0 ){
5227 if( z==0 ){
5228 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("parse error in \"%s\"", zOrig);
5229 rc = SQLITE_ERROR1;
5230 }else{
5231 if( bOption ){
5232 rc = fts5ConfigParseSpecial(pRet,
5233 ALWAYS(zOne)(zOne)?zOne:"",
5234 zTwo?zTwo:"",
5235 pzErr
5236 );
5237 }else{
5238 rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr, &bUnindexed);
5239 zOne = 0;
5240 }
5241 }
5242 }
5243
5244 sqlite3_freesqlite3_api->free(zOne);
5245 sqlite3_freesqlite3_api->free(zTwo);
5246 }
5247
5248 /* We only allow contentless_delete=1 if the table is indeed contentless. */
5249 if( rc==SQLITE_OK0
5250 && pRet->bContentlessDelete
5251 && pRet->eContent!=FTS5_CONTENT_NONE1
5252 ){
5253 *pzErr = sqlite3_mprintfsqlite3_api->mprintf(
5254 "contentless_delete=1 requires a contentless table"
5255 );
5256 rc = SQLITE_ERROR1;
5257 }
5258
5259 /* We only allow contentless_delete=1 if columnsize=0 is not present.
5260 **
5261 ** This restriction may be removed at some point.
5262 */
5263 if( rc==SQLITE_OK0 && pRet->bContentlessDelete && pRet->bColumnsize==0 ){
5264 *pzErr = sqlite3_mprintfsqlite3_api->mprintf(
5265 "contentless_delete=1 is incompatible with columnsize=0"
5266 );
5267 rc = SQLITE_ERROR1;
5268 }
5269
5270 /* We only allow contentless_unindexed=1 if the table is actually a
5271 ** contentless one.
5272 */
5273 if( rc==SQLITE_OK0
5274 && pRet->bContentlessUnindexed
5275 && pRet->eContent!=FTS5_CONTENT_NONE1
5276 ){
5277 *pzErr = sqlite3_mprintfsqlite3_api->mprintf(
5278 "contentless_unindexed=1 requires a contentless table"
5279 );
5280 rc = SQLITE_ERROR1;
5281 }
5282
5283 /* If no zContent option was specified, fill in the default values. */
5284 if( rc==SQLITE_OK0 && pRet->zContent==0 ){
5285 const char *zTail = 0;
5286 assert( pRet->eContent==FTS5_CONTENT_NORMAL((void) (0))
5287 || pRet->eContent==FTS5_CONTENT_NONE((void) (0))
5288 )((void) (0));
5289 if( pRet->eContent==FTS5_CONTENT_NORMAL0 ){
5290 zTail = "content";
5291 }else if( bUnindexed && pRet->bContentlessUnindexed ){
5292 pRet->eContent = FTS5_CONTENT_UNINDEXED3;
5293 zTail = "content";
5294 }else if( pRet->bColumnsize ){
5295 zTail = "docsize";
5296 }
5297
5298 if( zTail ){
5299 pRet->zContent = sqlite3Fts5Mprintf(
5300 &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail
5301 );
5302 }
5303 }
5304
5305 if( rc==SQLITE_OK0 && pRet->zContentRowid==0 ){
5306 pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1);
5307 }
5308
5309 /* Formulate the zContentExprlist text */
5310 if( rc==SQLITE_OK0 ){
5311 rc = fts5ConfigMakeExprlist(pRet);
5312 }
5313
5314 if( rc!=SQLITE_OK0 ){
5315 sqlite3Fts5ConfigFree(pRet);
5316 *ppOut = 0;
5317 }
5318 return rc;
5319}
5320
5321/*
5322** Free the configuration object passed as the only argument.
5323*/
5324static void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
5325 if( pConfig ){
5326 int i;
5327 if( pConfig->t.pTok ){
5328 if( pConfig->t.pApi1 ){
5329 pConfig->t.pApi1->xDelete(pConfig->t.pTok);
5330 }else{
5331 pConfig->t.pApi2->xDelete(pConfig->t.pTok);
5332 }
5333 }
5334 sqlite3_freesqlite3_api->free((char*)pConfig->t.azArg);
5335 sqlite3_freesqlite3_api->free(pConfig->zDb);
5336 sqlite3_freesqlite3_api->free(pConfig->zName);
5337 for(i=0; i<pConfig->nCol; i++){
5338 sqlite3_freesqlite3_api->free(pConfig->azCol[i]);
5339 }
5340 sqlite3_freesqlite3_api->free(pConfig->azCol);
5341 sqlite3_freesqlite3_api->free(pConfig->aPrefix);
5342 sqlite3_freesqlite3_api->free(pConfig->zRank);
5343 sqlite3_freesqlite3_api->free(pConfig->zRankArgs);
5344 sqlite3_freesqlite3_api->free(pConfig->zContent);
5345 sqlite3_freesqlite3_api->free(pConfig->zContentRowid);
5346 sqlite3_freesqlite3_api->free(pConfig->zContentExprlist);
5347 sqlite3_freesqlite3_api->free(pConfig);
5348 }
5349}
5350
5351/*
5352** Call sqlite3_declare_vtab() based on the contents of the configuration
5353** object passed as the only argument. Return SQLITE_OK if successful, or
5354** an SQLite error code if an error occurs.
5355*/
5356static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){
5357 int i;
5358 int rc = SQLITE_OK0;
5359 char *zSql;
5360
5361 zSql = sqlite3Fts5Mprintf(&rc, "CREATE TABLE x(");
5362 for(i=0; zSql && i<pConfig->nCol; i++){
5363 const char *zSep = (i==0?"":", ");
5364 zSql = sqlite3Fts5Mprintf(&rc, "%z%s%Q", zSql, zSep, pConfig->azCol[i]);
5365 }
5366 zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)",
5367 zSql, pConfig->zName, FTS5_RANK_NAME"rank"
5368 );
5369
5370 assert( zSql || rc==SQLITE_NOMEM )((void) (0));
5371 if( zSql ){
5372 rc = sqlite3_declare_vtabsqlite3_api->declare_vtab(pConfig->db, zSql);
5373 sqlite3_freesqlite3_api->free(zSql);
5374 }
5375
5376 return rc;
5377}
5378
5379/*
5380** Tokenize the text passed via the second and third arguments.
5381**
5382** The callback is invoked once for each token in the input text. The
5383** arguments passed to it are, in order:
5384**
5385** void *pCtx // Copy of 4th argument to sqlite3Fts5Tokenize()
5386** const char *pToken // Pointer to buffer containing token
5387** int nToken // Size of token in bytes
5388** int iStart // Byte offset of start of token within input text
5389** int iEnd // Byte offset of end of token within input text
5390** int iPos // Position of token in input (first token is 0)
5391**
5392** If the callback returns a non-zero value the tokenization is abandoned
5393** and no further callbacks are issued.
5394**
5395** This function returns SQLITE_OK if successful or an SQLite error code
5396** if an error occurs. If the tokenization was abandoned early because
5397** the callback returned SQLITE_DONE, this is not an error and this function
5398** still returns SQLITE_OK. Or, if the tokenization was abandoned early
5399** because the callback returned another non-zero value, it is assumed
5400** to be an SQLite error code and returned to the caller.
5401*/
5402static int sqlite3Fts5Tokenize(
5403 Fts5Config *pConfig, /* FTS5 Configuration object */
5404 int flags, /* FTS5_TOKENIZE_* flags */
5405 const char *pText, int nText, /* Text to tokenize */
5406 void *pCtx, /* Context passed to xToken() */
5407 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
5408){
5409 int rc = SQLITE_OK0;
5410 if( pText ){
5411 if( pConfig->t.pTok==0 ){
5412 rc = sqlite3Fts5LoadTokenizer(pConfig);
5413 }
5414 if( rc==SQLITE_OK0 ){
5415 if( pConfig->t.pApi1 ){
5416 rc = pConfig->t.pApi1->xTokenize(
5417 pConfig->t.pTok, pCtx, flags, pText, nText, xToken
5418 );
5419 }else{
5420 rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags,
5421 pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken
5422 );
5423 }
5424 }
5425 }
5426 return rc;
5427}
5428
5429/*
5430** Argument pIn points to the first character in what is expected to be
5431** a comma-separated list of SQL literals followed by a ')' character.
5432** If it actually is this, return a pointer to the ')'. Otherwise, return
5433** NULL to indicate a parse error.
5434*/
5435static const char *fts5ConfigSkipArgs(const char *pIn){
5436 const char *p = pIn;
5437
5438 while( 1 ){
5439 p = fts5ConfigSkipWhitespace(p);
5440 p = fts5ConfigSkipLiteral(p);
5441 p = fts5ConfigSkipWhitespace(p);
5442 if( p==0 || *p==')' ) break;
5443 if( *p!=',' ){
5444 p = 0;
5445 break;
5446 }
5447 p++;
5448 }
5449
5450 return p;
5451}
5452
5453/*
5454** Parameter zIn contains a rank() function specification. The format of
5455** this is:
5456**
5457** + Bareword (function name)
5458** + Open parenthesis - "("
5459** + Zero or more SQL literals in a comma separated list
5460** + Close parenthesis - ")"
5461*/
5462static int sqlite3Fts5ConfigParseRank(
5463 const char *zIn, /* Input string */
5464 char **pzRank, /* OUT: Rank function name */
5465 char **pzRankArgs /* OUT: Rank function arguments */
5466){
5467 const char *p = zIn;
5468 const char *pRank;
5469 char *zRank = 0;
5470 char *zRankArgs = 0;
5471 int rc = SQLITE_OK0;
5472
5473 *pzRank = 0;
5474 *pzRankArgs = 0;
5475
5476 if( p==0 ){
5477 rc = SQLITE_ERROR1;
5478 }else{
5479 p = fts5ConfigSkipWhitespace(p);
5480 pRank = p;
5481 p = fts5ConfigSkipBareword(p);
5482
5483 if( p ){
5484 zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
5485 if( zRank ) memcpy(zRank, pRank, p-pRank);
5486 }else{
5487 rc = SQLITE_ERROR1;
5488 }
5489
5490 if( rc==SQLITE_OK0 ){
5491 p = fts5ConfigSkipWhitespace(p);
5492 if( *p!='(' ) rc = SQLITE_ERROR1;
5493 p++;
5494 }
5495 if( rc==SQLITE_OK0 ){
5496 const char *pArgs;
5497 p = fts5ConfigSkipWhitespace(p);
5498 pArgs = p;
5499 if( *p!=')' ){
5500 p = fts5ConfigSkipArgs(p);
5501 if( p==0 ){
5502 rc = SQLITE_ERROR1;
5503 }else{
5504 zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
5505 if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
5506 }
5507 }
5508 }
5509 }
5510
5511 if( rc!=SQLITE_OK0 ){
5512 sqlite3_freesqlite3_api->free(zRank);
5513 assert( zRankArgs==0 )((void) (0));
5514 }else{
5515 *pzRank = zRank;
5516 *pzRankArgs = zRankArgs;
5517 }
5518 return rc;
5519}
5520
5521static int sqlite3Fts5ConfigSetValue(
5522 Fts5Config *pConfig,
5523 const char *zKey,
5524 sqlite3_value *pVal,
5525 int *pbBadkey
5526){
5527 int rc = SQLITE_OK0;
5528
5529 if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "pgsz") ){
5530 int pgsz = 0;
5531 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5532 pgsz = sqlite3_value_intsqlite3_api->value_int(pVal);
5533 }
5534 if( pgsz<32 || pgsz>FTS5_MAX_PAGE_SIZE(64*1024) ){
5535 *pbBadkey = 1;
5536 }else{
5537 pConfig->pgsz = pgsz;
5538 }
5539 }
5540
5541 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "hashsize") ){
5542 int nHashSize = -1;
5543 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5544 nHashSize = sqlite3_value_intsqlite3_api->value_int(pVal);
5545 }
5546 if( nHashSize<=0 ){
5547 *pbBadkey = 1;
5548 }else{
5549 pConfig->nHashSize = nHashSize;
5550 }
5551 }
5552
5553 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "automerge") ){
5554 int nAutomerge = -1;
5555 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5556 nAutomerge = sqlite3_value_intsqlite3_api->value_int(pVal);
5557 }
5558 if( nAutomerge<0 || nAutomerge>64 ){
5559 *pbBadkey = 1;
5560 }else{
5561 if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE4;
5562 pConfig->nAutomerge = nAutomerge;
5563 }
5564 }
5565
5566 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "usermerge") ){
5567 int nUsermerge = -1;
5568 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5569 nUsermerge = sqlite3_value_intsqlite3_api->value_int(pVal);
5570 }
5571 if( nUsermerge<2 || nUsermerge>16 ){
5572 *pbBadkey = 1;
5573 }else{
5574 pConfig->nUsermerge = nUsermerge;
5575 }
5576 }
5577
5578 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "crisismerge") ){
5579 int nCrisisMerge = -1;
5580 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5581 nCrisisMerge = sqlite3_value_intsqlite3_api->value_int(pVal);
5582 }
5583 if( nCrisisMerge<0 ){
5584 *pbBadkey = 1;
5585 }else{
5586 if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE16;
5587 if( nCrisisMerge>=FTS5_MAX_SEGMENT2000 ) nCrisisMerge = FTS5_MAX_SEGMENT2000-1;
5588 pConfig->nCrisisMerge = nCrisisMerge;
5589 }
5590 }
5591
5592 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "deletemerge") ){
5593 int nVal = -1;
5594 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5595 nVal = sqlite3_value_intsqlite3_api->value_int(pVal);
5596 }else{
5597 *pbBadkey = 1;
5598 }
5599 if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE10;
5600 if( nVal>100 ) nVal = 0;
5601 pConfig->nDeleteMerge = nVal;
5602 }
5603
5604 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "rank") ){
5605 const char *zIn = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
5606 char *zRank;
5607 char *zRankArgs;
5608 rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs);
5609 if( rc==SQLITE_OK0 ){
5610 sqlite3_freesqlite3_api->free(pConfig->zRank);
5611 sqlite3_freesqlite3_api->free(pConfig->zRankArgs);
5612 pConfig->zRank = zRank;
5613 pConfig->zRankArgs = zRankArgs;
5614 }else if( rc==SQLITE_ERROR1 ){
5615 rc = SQLITE_OK0;
5616 *pbBadkey = 1;
5617 }
5618 }
5619
5620 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "secure-delete") ){
5621 int bVal = -1;
5622 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5623 bVal = sqlite3_value_intsqlite3_api->value_int(pVal);
5624 }
5625 if( bVal<0 ){
5626 *pbBadkey = 1;
5627 }else{
5628 pConfig->bSecureDelete = (bVal ? 1 : 0);
5629 }
5630 }
5631
5632 else if( 0==sqlite3_stricmpsqlite3_api->stricmp(zKey, "insttoken") ){
5633 int bVal = -1;
5634 if( SQLITE_INTEGER1==sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal) ){
5635 bVal = sqlite3_value_intsqlite3_api->value_int(pVal);
5636 }
5637 if( bVal<0 ){
5638 *pbBadkey = 1;
5639 }else{
5640 pConfig->bPrefixInsttoken = (bVal ? 1 : 0);
5641 }
5642
5643 }else{
5644 *pbBadkey = 1;
5645 }
5646 return rc;
5647}
5648
5649/*
5650** Load the contents of the %_config table into memory.
5651*/
5652static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
5653 const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
5654 char *zSql;
5655 sqlite3_stmt *p = 0;
5656 int rc = SQLITE_OK0;
5657 int iVersion = 0;
5658
5659 /* Set default values */
5660 pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE4050;
5661 pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE4;
5662 pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE4;
5663 pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE16;
5664 pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE(1024*1024);
5665 pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE10;
5666
5667 zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
5668 if( zSql ){
5669 rc = sqlite3_prepare_v2sqlite3_api->prepare_v2(pConfig->db, zSql, -1, &p, 0);
5670 sqlite3_freesqlite3_api->free(zSql);
5671 }
5672
5673 assert( rc==SQLITE_OK || p==0 )((void) (0));
5674 if( rc==SQLITE_OK0 ){
5675 while( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(p) ){
5676 const char *zK = (const char*)sqlite3_column_textsqlite3_api->column_text(p, 0);
5677 sqlite3_value *pVal = sqlite3_column_valuesqlite3_api->column_value(p, 1);
5678 if( 0==sqlite3_stricmpsqlite3_api->stricmp(zK, "version") ){
5679 iVersion = sqlite3_value_intsqlite3_api->value_int(pVal);
5680 }else{
5681 int bDummy = 0;
5682 sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, &bDummy);
5683 }
5684 }
5685 rc = sqlite3_finalizesqlite3_api->finalize(p);
5686 }
5687
5688 if( rc==SQLITE_OK0
5689 && iVersion!=FTS5_CURRENT_VERSION4
5690 && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE5
5691 ){
5692 rc = SQLITE_ERROR1;
5693 sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format "
5694 "(found %d, expected %d or %d) - run 'rebuild'",
5695 iVersion, FTS5_CURRENT_VERSION4, FTS5_CURRENT_VERSION_SECUREDELETE5
5696 );
5697 }else{
5698 pConfig->iVersion = iVersion;
5699 }
5700
5701 if( rc==SQLITE_OK0 ){
5702 pConfig->iCookie = iCookie;
5703 }
5704 return rc;
5705}
5706
5707/*
5708** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer
5709** containing the error message created using printf() style formatting
5710** string zFmt and its trailing arguments.
5711*/
5712static void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){
5713 va_list ap; /* ... printf arguments */
5714 char *zMsg = 0;
5715
5716 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
5717 zMsg = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
5718 if( pConfig->pzErrmsg ){
5719 assert( *pConfig->pzErrmsg==0 )((void) (0));
5720 *pConfig->pzErrmsg = zMsg;
5721 }else{
5722 sqlite3_freesqlite3_api->free(zMsg);
5723 }
5724
5725 va_end(ap)__builtin_va_end(ap);
5726}
5727
5728#line 1 "fts5_expr.c"
5729/*
5730** 2014 May 31
5731**
5732** The author disclaims copyright to this source code. In place of
5733** a legal notice, here is a blessing:
5734**
5735** May you do good and not evil.
5736** May you find forgiveness for yourself and forgive others.
5737** May you share freely, never taking more than you give.
5738**
5739******************************************************************************
5740**
5741*/
5742
5743
5744
5745/* #include "fts5Int.h" */
5746/* #include "fts5parse.h" */
5747
5748#ifndef SQLITE_FTS5_MAX_EXPR_DEPTH256
5749# define SQLITE_FTS5_MAX_EXPR_DEPTH256 256
5750#endif
5751
5752/*
5753** All token types in the generated fts5parse.h file are greater than 0.
5754*/
5755#define FTS5_EOF0 0
5756
5757#define FTS5_LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)) (0xffffffff|(((i64)0x7fffffff)<<32))
5758
5759typedef struct Fts5ExprTerm Fts5ExprTerm;
5760
5761/*
5762** Functions generated by lemon from fts5parse.y.
5763*/
5764static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(u64));
5765static void sqlite3Fts5ParserFree(void*, void (*freeProc)(void*));
5766static void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);
5767#ifndef NDEBUG1
5768#include <stdio.h>
5769static void sqlite3Fts5ParserTrace(FILE*, char*);
5770#endif
5771static int sqlite3Fts5ParserFallback(int);
5772
5773
5774struct Fts5Expr {
5775 Fts5Index *pIndex;
5776 Fts5Config *pConfig;
5777 Fts5ExprNode *pRoot;
5778 int bDesc; /* Iterate in descending rowid order */
5779 int nPhrase; /* Number of phrases in expression */
5780 Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */
5781};
5782
5783/*
5784** eType:
5785** Expression node type. Usually one of:
5786**
5787** FTS5_AND (nChild, apChild valid)
5788** FTS5_OR (nChild, apChild valid)
5789** FTS5_NOT (nChild, apChild valid)
5790** FTS5_STRING (pNear valid)
5791** FTS5_TERM (pNear valid)
5792**
5793** An expression node with eType==0 may also exist. It always matches zero
5794** rows. This is created when a phrase containing no tokens is parsed.
5795** e.g. "".
5796**
5797** iHeight:
5798** Distance from this node to furthest leaf. This is always 0 for nodes
5799** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one
5800** greater than the largest child value.
5801*/
5802struct Fts5ExprNode {
5803 int eType; /* Node type */
5804 int bEof; /* True at EOF */
5805 int bNomatch; /* True if entry is not a match */
5806 int iHeight; /* Distance to tree leaf nodes */
5807
5808 /* Next method for this node. */
5809 int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64);
5810
5811 i64 iRowid; /* Current rowid */
5812 Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */
5813
5814 /* Child nodes. For a NOT node, this array always contains 2 entries. For
5815 ** AND or OR nodes, it contains 2 or more entries. */
5816 int nChild; /* Number of child nodes */
5817 Fts5ExprNode *apChild[FLEXARRAY]; /* Array of child nodes */
5818};
5819
5820/* Size (in bytes) of an Fts5ExprNode object that holds up to N children */
5821#define SZ_FTS5EXPRNODE(N)(__builtin_offsetof(Fts5ExprNode, apChild) + (N)*sizeof(Fts5ExprNode
*)) \
5822 (offsetof(Fts5ExprNode,apChild)__builtin_offsetof(Fts5ExprNode, apChild) + (N)*sizeof(Fts5ExprNode*))
5823
5824#define Fts5NodeIsString(p)((p)->eType==4 || (p)->eType==9) ((p)->eType==FTS5_TERM4 || (p)->eType==FTS5_STRING9)
5825
5826/*
5827** Invoke the xNext method of an Fts5ExprNode object. This macro should be
5828** used as if it has the same signature as the xNext() methods themselves.
5829*/
5830#define fts5ExprNodeNext(a,b,c,d)(b)->xNext((a), (b), (c), (d)) (b)->xNext((a), (b), (c), (d))
5831
5832/*
5833** An instance of the following structure represents a single search term
5834** or term prefix.
5835*/
5836struct Fts5ExprTerm {
5837 u8 bPrefix; /* True for a prefix term */
5838 u8 bFirst; /* True if token must be first in column */
5839 char *pTerm; /* Term data */
5840 int nQueryTerm; /* Effective size of term in bytes */
5841 int nFullTerm; /* Size of term in bytes incl. tokendata */
5842 Fts5IndexIter *pIter; /* Iterator for this term */
5843 Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */
5844};
5845
5846/*
5847** A phrase. One or more terms that must appear in a contiguous sequence
5848** within a document for it to match.
5849*/
5850struct Fts5ExprPhrase {
5851 Fts5ExprNode *pNode; /* FTS5_STRING node this phrase is part of */
5852 Fts5Buffer poslist; /* Current position list */
5853 int nTerm; /* Number of entries in aTerm[] */
5854 Fts5ExprTerm aTerm[FLEXARRAY]; /* Terms that make up this phrase */
5855};
5856
5857/* Size (in bytes) of an Fts5ExprPhrase object that holds up to N terms */
5858#define SZ_FTS5EXPRPHRASE(N)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (N)*sizeof(Fts5ExprTerm
)) \
5859 (offsetof(Fts5ExprPhrase,aTerm)__builtin_offsetof(Fts5ExprPhrase, aTerm) + (N)*sizeof(Fts5ExprTerm))
5860
5861/*
5862** One or more phrases that must appear within a certain token distance of
5863** each other within each matching document.
5864*/
5865struct Fts5ExprNearset {
5866 int nNear; /* NEAR parameter */
5867 Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */
5868 int nPhrase; /* Number of entries in aPhrase[] array */
5869 Fts5ExprPhrase *apPhrase[FLEXARRAY]; /* Array of phrase pointers */
5870};
5871
5872/* Size (in bytes) of an Fts5ExprNearset object covering up to N phrases */
5873#define SZ_FTS5EXPRNEARSET(N)(__builtin_offsetof(Fts5ExprNearset, apPhrase)+(N)*sizeof(Fts5ExprPhrase
*)) \
5874 (offsetof(Fts5ExprNearset,apPhrase)__builtin_offsetof(Fts5ExprNearset, apPhrase)+(N)*sizeof(Fts5ExprPhrase*))
5875
5876/*
5877** Parse context.
5878*/
5879struct Fts5Parse {
5880 Fts5Config *pConfig;
5881 char *zErr;
5882 int rc;
5883 int nPhrase; /* Size of apPhrase array */
5884 Fts5ExprPhrase **apPhrase; /* Array of all phrases */
5885 Fts5ExprNode *pExpr; /* Result of a successful parse */
5886 int bPhraseToAnd; /* Convert "a+b" to "a AND b" */
5887};
5888
5889/*
5890** Check that the Fts5ExprNode.iHeight variables are set correctly in
5891** the expression tree passed as the only argument.
5892*/
5893#ifndef NDEBUG1
5894static void assert_expr_depth_ok(int rc, Fts5ExprNode *p){
5895 if( rc==SQLITE_OK0 ){
5896 if( p->eType==FTS5_TERM4 || p->eType==FTS5_STRING9 || p->eType==0 ){
5897 assert( p->iHeight==0 )((void) (0));
5898 }else{
5899 int ii;
5900 int iMaxChild = 0;
5901 for(ii=0; ii<p->nChild; ii++){
5902 Fts5ExprNode *pChild = p->apChild[ii];
5903 iMaxChild = MAX(iMaxChild, pChild->iHeight)(((iMaxChild) > (pChild->iHeight)) ? (iMaxChild) : (pChild
->iHeight));
5904 assert_expr_depth_ok(SQLITE_OK, pChild);
5905 }
5906 assert( p->iHeight==iMaxChild+1 )((void) (0));
5907 }
5908 }
5909}
5910#else
5911# define assert_expr_depth_ok(rc, p)
5912#endif
5913
5914static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
5915 va_list ap;
5916 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
5917 if( pParse->rc==SQLITE_OK0 ){
5918 assert( pParse->zErr==0 )((void) (0));
5919 pParse->zErr = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
5920 pParse->rc = SQLITE_ERROR1;
5921 }
5922 va_end(ap)__builtin_va_end(ap);
5923}
5924
5925static int fts5ExprIsspace(char t){
5926 return t==' ' || t=='\t' || t=='\n' || t=='\r';
5927}
5928
5929/*
5930** Read the first token from the nul-terminated string at *pz.
5931*/
5932static int fts5ExprGetToken(
5933 Fts5Parse *pParse,
5934 const char **pz, /* IN/OUT: Pointer into buffer */
5935 Fts5Token *pToken
5936){
5937 const char *z = *pz;
5938 int tok;
5939
5940 /* Skip past any whitespace */
5941 while( fts5ExprIsspace(*z) ) z++;
5942
5943 pToken->p = z;
5944 pToken->n = 1;
5945 switch( *z ){
5946 case '(': tok = FTS5_LP10; break;
5947 case ')': tok = FTS5_RP11; break;
5948 case '{': tok = FTS5_LCP7; break;
5949 case '}': tok = FTS5_RCP8; break;
5950 case ':': tok = FTS5_COLON5; break;
5951 case ',': tok = FTS5_COMMA13; break;
5952 case '+': tok = FTS5_PLUS14; break;
5953 case '*': tok = FTS5_STAR15; break;
5954 case '-': tok = FTS5_MINUS6; break;
5955 case '^': tok = FTS5_CARET12; break;
5956 case '\0': tok = FTS5_EOF0; break;
5957
5958 case '"': {
5959 const char *z2;
5960 tok = FTS5_STRING9;
5961
5962 for(z2=&z[1]; 1; z2++){
5963 if( z2[0]=='"' ){
5964 z2++;
5965 if( z2[0]!='"' ) break;
5966 }
5967 if( z2[0]=='\0' ){
5968 sqlite3Fts5ParseError(pParse, "unterminated string");
5969 return FTS5_EOF0;
5970 }
5971 }
5972 pToken->n = (z2 - z);
5973 break;
5974 }
5975
5976 default: {
5977 const char *z2;
5978 if( sqlite3Fts5IsBareword(z[0])==0 ){
5979 sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z);
5980 return FTS5_EOF0;
5981 }
5982 tok = FTS5_STRING9;
5983 for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++);
5984 pToken->n = (z2 - z);
5985 if( pToken->n==2 && memcmp(pToken->p, "OR", 2)==0 ) tok = FTS5_OR1;
5986 if( pToken->n==3 && memcmp(pToken->p, "NOT", 3)==0 ) tok = FTS5_NOT3;
5987 if( pToken->n==3 && memcmp(pToken->p, "AND", 3)==0 ) tok = FTS5_AND2;
5988 break;
5989 }
5990 }
5991
5992 *pz = &pToken->p[pToken->n];
5993 return tok;
5994}
5995
5996static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc64sqlite3_api->malloc64((sqlite3_int64)t);}
5997static void fts5ParseFree(void *p){ sqlite3_freesqlite3_api->free(p); }
5998
5999static int sqlite3Fts5ExprNew(
6000 Fts5Config *pConfig, /* FTS5 Configuration */
6001 int bPhraseToAnd,
6002 int iCol,
6003 const char *zExpr, /* Expression text */
6004 Fts5Expr **ppNew,
6005 char **pzErr
6006){
6007 Fts5Parse sParse;
6008 Fts5Token token;
6009 const char *z = zExpr;
6010 int t; /* Next token type */
6011 void *pEngine;
6012 Fts5Expr *pNew;
6013
6014 *ppNew = 0;
6015 *pzErr = 0;
6016 memset(&sParse, 0, sizeof(sParse));
6017 sParse.bPhraseToAnd = bPhraseToAnd;
6018 pEngine = sqlite3Fts5ParserAlloc(fts5ParseAlloc);
6019 if( pEngine==0 ){ return SQLITE_NOMEM7; }
6020 sParse.pConfig = pConfig;
6021
6022 do {
6023 t = fts5ExprGetToken(&sParse, &z, &token);
6024 sqlite3Fts5Parser(pEngine, t, token, &sParse);
6025 }while( sParse.rc==SQLITE_OK0 && t!=FTS5_EOF0 );
6026 sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
6027
6028 assert( sParse.pExpr || sParse.rc!=SQLITE_OK )((void) (0));
6029 assert_expr_depth_ok(sParse.rc, sParse.pExpr);
6030
6031 /* If the LHS of the MATCH expression was a user column, apply the
6032 ** implicit column-filter. */
6033 if( sParse.rc==SQLITE_OK0 && iCol<pConfig->nCol ){
6034 int n = SZ_FTS5COLSET(1)(sizeof(i64)*((1 +2)/2));
6035 Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
6036 if( pColset ){
6037 pColset->nCol = 1;
6038 pColset->aiCol[0] = iCol;
6039 sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset);
6040 }
6041 }
6042
6043 assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 )((void) (0));
6044 if( sParse.rc==SQLITE_OK0 ){
6045 *ppNew = pNew = sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5Expr));
6046 if( pNew==0 ){
6047 sParse.rc = SQLITE_NOMEM7;
6048 sqlite3Fts5ParseNodeFree(sParse.pExpr);
6049 }else{
6050 pNew->pRoot = sParse.pExpr;
6051 pNew->pIndex = 0;
6052 pNew->pConfig = pConfig;
6053 pNew->apExprPhrase = sParse.apPhrase;
6054 pNew->nPhrase = sParse.nPhrase;
6055 pNew->bDesc = 0;
6056 sParse.apPhrase = 0;
6057 }
6058 }else{
6059 sqlite3Fts5ParseNodeFree(sParse.pExpr);
6060 }
6061
6062 sqlite3_freesqlite3_api->free(sParse.apPhrase);
6063 if( 0==*pzErr ){
6064 *pzErr = sParse.zErr;
6065 }else{
6066 sqlite3_freesqlite3_api->free(sParse.zErr);
6067 }
6068 return sParse.rc;
6069}
6070
6071/*
6072** Assuming that buffer z is at least nByte bytes in size and contains a
6073** valid utf-8 string, return the number of characters in the string.
6074*/
6075static int fts5ExprCountChar(const char *z, int nByte){
6076 int nRet = 0;
6077 int ii;
6078 for(ii=0; ii<nByte; ii++){
6079 if( (z[ii] & 0xC0)!=0x80 ) nRet++;
6080 }
6081 return nRet;
6082}
6083
6084/*
6085** This function is only called when using the special 'trigram' tokenizer.
6086** Argument zText contains the text of a LIKE or GLOB pattern matched
6087** against column iCol. This function creates and compiles an FTS5 MATCH
6088** expression that will match a superset of the rows matched by the LIKE or
6089** GLOB. If successful, SQLITE_OK is returned. Otherwise, an SQLite error
6090** code.
6091*/
6092static int sqlite3Fts5ExprPattern(
6093 Fts5Config *pConfig, int bGlob, int iCol, const char *zText, Fts5Expr **pp
6094){
6095 i64 nText = strlen(zText);
6096 char *zExpr = (char*)sqlite3_malloc64sqlite3_api->malloc64(nText*4 + 1);
6097 int rc = SQLITE_OK0;
6098
6099 if( zExpr==0 ){
6100 rc = SQLITE_NOMEM7;
6101 }else{
6102 char aSpec[3];
6103 int iOut = 0;
6104 int i = 0;
6105 int iFirst = 0;
6106
6107 if( bGlob==0 ){
6108 aSpec[0] = '_';
6109 aSpec[1] = '%';
6110 aSpec[2] = 0;
6111 }else{
6112 aSpec[0] = '*';
6113 aSpec[1] = '?';
6114 aSpec[2] = '[';
6115 }
6116
6117 while( i<=nText ){
6118 if( i==nText
6119 || zText[i]==aSpec[0] || zText[i]==aSpec[1] || zText[i]==aSpec[2]
6120 ){
6121
6122 if( fts5ExprCountChar(&zText[iFirst], i-iFirst)>=3 ){
6123 int jj;
6124 zExpr[iOut++] = '"';
6125 for(jj=iFirst; jj<i; jj++){
6126 zExpr[iOut++] = zText[jj];
6127 if( zText[jj]=='"' ) zExpr[iOut++] = '"';
6128 }
6129 zExpr[iOut++] = '"';
6130 zExpr[iOut++] = ' ';
6131 }
6132 if( zText[i]==aSpec[2] ){
6133 i += 2;
6134 if( zText[i-1]=='^' ) i++;
6135 while( i<nText && zText[i]!=']' ) i++;
6136 }
6137 iFirst = i+1;
6138 }
6139 i++;
6140 }
6141 if( iOut>0 ){
6142 int bAnd = 0;
6143 if( pConfig->eDetail!=FTS5_DETAIL_FULL0 ){
6144 bAnd = 1;
6145 if( pConfig->eDetail==FTS5_DETAIL_NONE1 ){
6146 iCol = pConfig->nCol;
6147 }
6148 }
6149 zExpr[iOut] = '\0';
6150 rc = sqlite3Fts5ExprNew(pConfig, bAnd, iCol, zExpr, pp,pConfig->pzErrmsg);
6151 }else{
6152 *pp = 0;
6153 }
6154 sqlite3_freesqlite3_api->free(zExpr);
6155 }
6156
6157 return rc;
6158}
6159
6160/*
6161** Free the expression node object passed as the only argument.
6162*/
6163static void sqlite3Fts5ParseNodeFree(Fts5ExprNode *p){
6164 if( p ){
6165 int i;
6166 for(i=0; i<p->nChild; i++){
6167 sqlite3Fts5ParseNodeFree(p->apChild[i]);
6168 }
6169 sqlite3Fts5ParseNearsetFree(p->pNear);
6170 sqlite3_freesqlite3_api->free(p);
6171 }
6172}
6173
6174/*
6175** Free the expression object passed as the only argument.
6176*/
6177static void sqlite3Fts5ExprFree(Fts5Expr *p){
6178 if( p ){
6179 sqlite3Fts5ParseNodeFree(p->pRoot);
6180 sqlite3_freesqlite3_api->free(p->apExprPhrase);
6181 sqlite3_freesqlite3_api->free(p);
6182 }
6183}
6184
6185static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
6186 Fts5Parse sParse;
6187 memset(&sParse, 0, sizeof(sParse));
6188
6189 if( *pp1 && p2 ){
6190 Fts5Expr *p1 = *pp1;
6191 int nPhrase = p1->nPhrase + p2->nPhrase;
6192
6193 p1->pRoot = sqlite3Fts5ParseNode(&sParse, FTS5_AND2, p1->pRoot, p2->pRoot,0);
6194 p2->pRoot = 0;
6195
6196 if( sParse.rc==SQLITE_OK0 ){
6197 Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc64sqlite3_api->realloc64(
6198 p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*)
6199 );
6200 if( ap==0 ){
6201 sParse.rc = SQLITE_NOMEM7;
6202 }else{
6203 int i;
6204 memmove(&ap[p2->nPhrase], ap, p1->nPhrase*sizeof(Fts5ExprPhrase*));
6205 for(i=0; i<p2->nPhrase; i++){
6206 ap[i] = p2->apExprPhrase[i];
6207 }
6208 p1->nPhrase = nPhrase;
6209 p1->apExprPhrase = ap;
6210 }
6211 }
6212 sqlite3_freesqlite3_api->free(p2->apExprPhrase);
6213 sqlite3_freesqlite3_api->free(p2);
6214 }else if( p2 ){
6215 *pp1 = p2;
6216 }
6217
6218 return sParse.rc;
6219}
6220
6221/*
6222** Argument pTerm must be a synonym iterator. Return the current rowid
6223** that it points to.
6224*/
6225static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
6226 i64 iRet = 0;
6227 int bRetValid = 0;
6228 Fts5ExprTerm *p;
6229
6230 assert( pTerm )((void) (0));
6231 assert( pTerm->pSynonym )((void) (0));
6232 assert( bDesc==0 || bDesc==1 )((void) (0));
6233 for(p=pTerm; p; p=p->pSynonym){
6234 if( 0==sqlite3Fts5IterEof(p->pIter)((p->pIter)->bEof) ){
6235 i64 iRowid = p->pIter->iRowid;
6236 if( bRetValid==0 || (bDesc!=(iRowid<iRet)) ){
6237 iRet = iRowid;
6238 bRetValid = 1;
6239 }
6240 }
6241 }
6242
6243 if( pbEof && bRetValid==0 ) *pbEof = 1;
6244 return iRet;
6245}
6246
6247/*
6248** Argument pTerm must be a synonym iterator.
6249*/
6250static int fts5ExprSynonymList(
6251 Fts5ExprTerm *pTerm,
6252 i64 iRowid,
6253 Fts5Buffer *pBuf, /* Use this buffer for space if required */
6254 u8 **pa, int *pn
6255){
6256 Fts5PoslistReader aStatic[4];
6257 Fts5PoslistReader *aIter = aStatic;
6258 int nIter = 0;
6259 int nAlloc = 4;
6260 int rc = SQLITE_OK0;
6261 Fts5ExprTerm *p;
6262
6263 assert( pTerm->pSynonym )((void) (0));
6264 for(p=pTerm; p; p=p->pSynonym){
6265 Fts5IndexIter *pIter = p->pIter;
6266 if( sqlite3Fts5IterEof(pIter)((pIter)->bEof)==0 && pIter->iRowid==iRowid ){
6267 if( pIter->nData==0 ) continue;
6268 if( nIter==nAlloc ){
6269 sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
6270 Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
6271 if( aNew==0 ){
6272 rc = SQLITE_NOMEM7;
6273 goto synonym_poslist_out;
6274 }
6275 memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter);
6276 nAlloc = nAlloc*2;
6277 if( aIter!=aStatic ) sqlite3_freesqlite3_api->free(aIter);
6278 aIter = aNew;
6279 }
6280 sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]);
6281 assert( aIter[nIter].bEof==0 )((void) (0));
6282 nIter++;
6283 }
6284 }
6285
6286 if( nIter==1 ){
6287 *pa = (u8*)aIter[0].a;
6288 *pn = aIter[0].n;
6289 }else{
6290 Fts5PoslistWriter writer = {0};
6291 i64 iPrev = -1;
6292 fts5BufferZero(pBuf)sqlite3Fts5BufferZero(pBuf);
6293 while( 1 ){
6294 int i;
6295 i64 iMin = FTS5_LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32));
6296 for(i=0; i<nIter; i++){
6297 if( aIter[i].bEof==0 ){
6298 if( aIter[i].iPos==iPrev ){
6299 if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) continue;
6300 }
6301 if( aIter[i].iPos<iMin ){
6302 iMin = aIter[i].iPos;
6303 }
6304 }
6305 }
6306 if( iMin==FTS5_LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)) || rc!=SQLITE_OK0 ) break;
6307 rc = sqlite3Fts5PoslistWriterAppend(pBuf, &writer, iMin);
6308 iPrev = iMin;
6309 }
6310 if( rc==SQLITE_OK0 ){
6311 *pa = pBuf->p;
6312 *pn = pBuf->n;
6313 }
6314 }
6315
6316 synonym_poslist_out:
6317 if( aIter!=aStatic ) sqlite3_freesqlite3_api->free(aIter);
6318 return rc;
6319}
6320
6321
6322/*
6323** All individual term iterators in pPhrase are guaranteed to be valid and
6324** pointing to the same rowid when this function is called. This function
6325** checks if the current rowid really is a match, and if so populates
6326** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch
6327** is set to true if this is really a match, or false otherwise.
6328**
6329** SQLITE_OK is returned if an error occurs, or an SQLite error code
6330** otherwise. It is not considered an error code if the current rowid is
6331** not a match.
6332*/
6333static int fts5ExprPhraseIsMatch(
6334 Fts5ExprNode *pNode, /* Node pPhrase belongs to */
6335 Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */
6336 int *pbMatch /* OUT: Set to true if really a match */
6337){
6338 Fts5PoslistWriter writer = {0};
6339 Fts5PoslistReader aStatic[4];
6340 Fts5PoslistReader *aIter = aStatic;
6341 int i;
6342 int rc = SQLITE_OK0;
6343 int bFirst = pPhrase->aTerm[0].bFirst;
6344
6345 fts5BufferZero(&pPhrase->poslist)sqlite3Fts5BufferZero(&pPhrase->poslist);
6346
6347 /* If the aStatic[] array is not large enough, allocate a large array
6348 ** using sqlite3_malloc(). This approach could be improved upon. */
6349 if( pPhrase->nTerm>ArraySize(aStatic)((int)(sizeof(aStatic) / sizeof(aStatic[0]))) ){
6350 sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
6351 aIter = (Fts5PoslistReader*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
6352 if( !aIter ) return SQLITE_NOMEM7;
6353 }
6354 memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm);
6355
6356 /* Initialize a term iterator for each term in the phrase */
6357 for(i=0; i<pPhrase->nTerm; i++){
6358 Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
6359 int n = 0;
6360 int bFlag = 0;
6361 u8 *a = 0;
6362 if( pTerm->pSynonym ){
6363 Fts5Buffer buf = {0, 0, 0};
6364 rc = fts5ExprSynonymList(pTerm, pNode->iRowid, &buf, &a, &n);
6365 if( rc ){
6366 sqlite3_freesqlite3_api->free(a);
6367 goto ismatch_out;
6368 }
6369 if( a==buf.p ) bFlag = 1;
6370 }else{
6371 a = (u8*)pTerm->pIter->pData;
6372 n = pTerm->pIter->nData;
6373 }
6374 sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
6375 aIter[i].bFlag = (u8)bFlag;
6376 if( aIter[i].bEof ) goto ismatch_out;
6377 }
6378
6379 while( 1 ){
6380 int bMatch;
6381 i64 iPos = aIter[0].iPos;
6382 do {
6383 bMatch = 1;
6384 for(i=0; i<pPhrase->nTerm; i++){
6385 Fts5PoslistReader *pPos = &aIter[i];
6386 i64 iAdj = iPos + i;
6387 if( pPos->iPos!=iAdj ){
6388 bMatch = 0;
6389 while( pPos->iPos<iAdj ){
6390 if( sqlite3Fts5PoslistReaderNext(pPos) ) goto ismatch_out;
6391 }
6392 if( pPos->iPos>iAdj ) iPos = pPos->iPos-i;
6393 }
6394 }
6395 }while( bMatch==0 );
6396
6397 /* Append position iPos to the output */
6398 if( bFirst==0 || FTS5_POS2OFFSET(iPos)(int)(iPos & 0x7FFFFFFF)==0 ){
6399 rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
6400 if( rc!=SQLITE_OK0 ) goto ismatch_out;
6401 }
6402
6403 for(i=0; i<pPhrase->nTerm; i++){
6404 if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
6405 }
6406 }
6407
6408 ismatch_out:
6409 *pbMatch = (pPhrase->poslist.n>0);
6410 for(i=0; i<pPhrase->nTerm; i++){
6411 if( aIter[i].bFlag ) sqlite3_freesqlite3_api->free((u8*)aIter[i].a);
6412 }
6413 if( aIter!=aStatic ) sqlite3_freesqlite3_api->free(aIter);
6414 return rc;
6415}
6416
6417typedef struct Fts5LookaheadReader Fts5LookaheadReader;
6418struct Fts5LookaheadReader {
6419 const u8 *a; /* Buffer containing position list */
6420 int n; /* Size of buffer a[] in bytes */
6421 int i; /* Current offset in position list */
6422 i64 iPos; /* Current position */
6423 i64 iLookahead; /* Next position */
6424};
6425
6426#define FTS5_LOOKAHEAD_EOF(((i64)1) << 62) (((i64)1) << 62)
6427
6428static int fts5LookaheadReaderNext(Fts5LookaheadReader *p){
6429 p->iPos = p->iLookahead;
6430 if( sqlite3Fts5PoslistNext64(p->a, p->n, &p->i, &p->iLookahead) ){
6431 p->iLookahead = FTS5_LOOKAHEAD_EOF(((i64)1) << 62);
6432 }
6433 return (p->iPos==FTS5_LOOKAHEAD_EOF(((i64)1) << 62));
6434}
6435
6436static int fts5LookaheadReaderInit(
6437 const u8 *a, int n, /* Buffer to read position list from */
6438 Fts5LookaheadReader *p /* Iterator object to initialize */
6439){
6440 memset(p, 0, sizeof(Fts5LookaheadReader));
6441 p->a = a;
6442 p->n = n;
6443 fts5LookaheadReaderNext(p);
6444 return fts5LookaheadReaderNext(p);
6445}
6446
6447typedef struct Fts5NearTrimmer Fts5NearTrimmer;
6448struct Fts5NearTrimmer {
6449 Fts5LookaheadReader reader; /* Input iterator */
6450 Fts5PoslistWriter writer; /* Writer context */
6451 Fts5Buffer *pOut; /* Output poslist */
6452};
6453
6454/*
6455** The near-set object passed as the first argument contains more than
6456** one phrase. All phrases currently point to the same row. The
6457** Fts5ExprPhrase.poslist buffers are populated accordingly. This function
6458** tests if the current row contains instances of each phrase sufficiently
6459** close together to meet the NEAR constraint. Non-zero is returned if it
6460** does, or zero otherwise.
6461**
6462** If in/out parameter (*pRc) is set to other than SQLITE_OK when this
6463** function is called, it is a no-op. Or, if an error (e.g. SQLITE_NOMEM)
6464** occurs within this function (*pRc) is set accordingly before returning.
6465** The return value is undefined in both these cases.
6466**
6467** If no error occurs and non-zero (a match) is returned, the position-list
6468** of each phrase object is edited to contain only those entries that
6469** meet the constraint before returning.
6470*/
6471static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){
6472 Fts5NearTrimmer aStatic[4];
6473 Fts5NearTrimmer *a = aStatic;
6474 Fts5ExprPhrase **apPhrase = pNear->apPhrase;
6475
6476 int i;
6477 int rc = *pRc;
6478 int bMatch;
6479
6480 assert( pNear->nPhrase>1 )((void) (0));
6481
6482 /* If the aStatic[] array is not large enough, allocate a large array
6483 ** using sqlite3_malloc(). This approach could be improved upon. */
6484 if( pNear->nPhrase>ArraySize(aStatic)((int)(sizeof(aStatic) / sizeof(aStatic[0]))) ){
6485 sqlite3_int64 nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
6486 a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte);
6487 }else{
6488 memset(aStatic, 0, sizeof(aStatic));
6489 }
6490 if( rc!=SQLITE_OK0 ){
6491 *pRc = rc;
6492 return 0;
6493 }
6494
6495 /* Initialize a lookahead iterator for each phrase. After passing the
6496 ** buffer and buffer size to the lookaside-reader init function, zero
6497 ** the phrase poslist buffer. The new poslist for the phrase (containing
6498 ** the same entries as the original with some entries removed on account
6499 ** of the NEAR constraint) is written over the original even as it is
6500 ** being read. This is safe as the entries for the new poslist are a
6501 ** subset of the old, so it is not possible for data yet to be read to
6502 ** be overwritten. */
6503 for(i=0; i<pNear->nPhrase; i++){
6504 Fts5Buffer *pPoslist = &apPhrase[i]->poslist;
6505 fts5LookaheadReaderInit(pPoslist->p, pPoslist->n, &a[i].reader);
6506 pPoslist->n = 0;
6507 a[i].pOut = pPoslist;
6508 }
6509
6510 while( 1 ){
6511 int iAdv;
6512 i64 iMin;
6513 i64 iMax;
6514
6515 /* This block advances the phrase iterators until they point to a set of
6516 ** entries that together comprise a match. */
6517 iMax = a[0].reader.iPos;
6518 do {
6519 bMatch = 1;
6520 for(i=0; i<pNear->nPhrase; i++){
6521 Fts5LookaheadReader *pPos = &a[i].reader;
6522 iMin = iMax - pNear->apPhrase[i]->nTerm - pNear->nNear;
6523 if( pPos->iPos<iMin || pPos->iPos>iMax ){
6524 bMatch = 0;
6525 while( pPos->iPos<iMin ){
6526 if( fts5LookaheadReaderNext(pPos) ) goto ismatch_out;
6527 }
6528 if( pPos->iPos>iMax ) iMax = pPos->iPos;
6529 }
6530 }
6531 }while( bMatch==0 );
6532
6533 /* Add an entry to each output position list */
6534 for(i=0; i<pNear->nPhrase; i++){
6535 i64 iPos = a[i].reader.iPos;
6536 Fts5PoslistWriter *pWriter = &a[i].writer;
6537 if( a[i].pOut->n==0 || iPos!=pWriter->iPrev ){
6538 sqlite3Fts5PoslistSafeAppend(a[i].pOut, &pWriter->iPrev, iPos);
6539 }
6540 }
6541
6542 iAdv = 0;
6543 iMin = a[0].reader.iLookahead;
6544 for(i=0; i<pNear->nPhrase; i++){
6545 if( a[i].reader.iLookahead < iMin ){
6546 iMin = a[i].reader.iLookahead;
6547 iAdv = i;
6548 }
6549 }
6550 if( fts5LookaheadReaderNext(&a[iAdv].reader) ) goto ismatch_out;
6551 }
6552
6553 ismatch_out: {
6554 int bRet = a[0].pOut->n>0;
6555 *pRc = rc;
6556 if( a!=aStatic ) sqlite3_freesqlite3_api->free(a);
6557 return bRet;
6558 }
6559}
6560
6561/*
6562** Advance iterator pIter until it points to a value equal to or laster
6563** than the initial value of *piLast. If this means the iterator points
6564** to a value laster than *piLast, update *piLast to the new lastest value.
6565**
6566** If the iterator reaches EOF, set *pbEof to true before returning. If
6567** an error occurs, set *pRc to an error code. If either *pbEof or *pRc
6568** are set, return a non-zero value. Otherwise, return zero.
6569*/
6570static int fts5ExprAdvanceto(
6571 Fts5IndexIter *pIter, /* Iterator to advance */
6572 int bDesc, /* True if iterator is "rowid DESC" */
6573 i64 *piLast, /* IN/OUT: Lastest rowid seen so far */
6574 int *pRc, /* OUT: Error code */
6575 int *pbEof /* OUT: Set to true if EOF */
6576){
6577 i64 iLast = *piLast;
6578 i64 iRowid;
6579
6580 iRowid = pIter->iRowid;
6581 if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
6582 int rc = sqlite3Fts5IterNextFrom(pIter, iLast);
6583 if( rc || sqlite3Fts5IterEof(pIter)((pIter)->bEof) ){
6584 *pRc = rc;
6585 *pbEof = 1;
6586 return 1;
6587 }
6588 iRowid = pIter->iRowid;
6589 assert( (bDesc==0 && iRowid>=iLast) || (bDesc==1 && iRowid<=iLast) )((void) (0));
6590 }
6591 *piLast = iRowid;
6592
6593 return 0;
6594}
6595
6596static int fts5ExprSynonymAdvanceto(
6597 Fts5ExprTerm *pTerm, /* Term iterator to advance */
6598 int bDesc, /* True if iterator is "rowid DESC" */
6599 i64 *piLast, /* IN/OUT: Lastest rowid seen so far */
6600 int *pRc /* OUT: Error code */
6601){
6602 int rc = SQLITE_OK0;
6603 i64 iLast = *piLast;
6604 Fts5ExprTerm *p;
6605 int bEof = 0;
6606
6607 for(p=pTerm; rc==SQLITE_OK0 && p; p=p->pSynonym){
6608 if( sqlite3Fts5IterEof(p->pIter)((p->pIter)->bEof)==0 ){
6609 i64 iRowid = p->pIter->iRowid;
6610 if( (bDesc==0 && iLast>iRowid) || (bDesc && iLast<iRowid) ){
6611 rc = sqlite3Fts5IterNextFrom(p->pIter, iLast);
6612 }
6613 }
6614 }
6615
6616 if( rc!=SQLITE_OK0 ){
6617 *pRc = rc;
6618 bEof = 1;
6619 }else{
6620 *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof);
6621 }
6622 return bEof;
6623}
6624
6625
6626static int fts5ExprNearTest(
6627 int *pRc,
6628 Fts5Expr *pExpr, /* Expression that pNear is a part of */
6629 Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */
6630){
6631 Fts5ExprNearset *pNear = pNode->pNear;
6632 int rc = *pRc;
6633
6634 if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL0 ){
6635 Fts5ExprTerm *pTerm;
6636 Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
6637 pPhrase->poslist.n = 0;
6638 for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
6639 Fts5IndexIter *pIter = pTerm->pIter;
6640 if( sqlite3Fts5IterEof(pIter)((pIter)->bEof)==0 ){
6641 if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){
6642 pPhrase->poslist.n = 1;
6643 }
6644 }
6645 }
6646 return pPhrase->poslist.n;
6647 }else{
6648 int i;
6649
6650 /* Check that each phrase in the nearset matches the current row.
6651 ** Populate the pPhrase->poslist buffers at the same time. If any
6652 ** phrase is not a match, break out of the loop early. */
6653 for(i=0; rc==SQLITE_OK0 && i<pNear->nPhrase; i++){
6654 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
6655 if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym
6656 || pNear->pColset || pPhrase->aTerm[0].bFirst
6657 ){
6658 int bMatch = 0;
6659 rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
6660 if( bMatch==0 ) break;
6661 }else{
6662 Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
6663 fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData)sqlite3Fts5BufferSet(&rc,&pPhrase->poslist,pIter->
nData,pIter->pData);
6664 }
6665 }
6666
6667 *pRc = rc;
6668 if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
6669 return 1;
6670 }
6671 return 0;
6672 }
6673}
6674
6675
6676/*
6677** Initialize all term iterators in the pNear object. If any term is found
6678** to match no documents at all, return immediately without initializing any
6679** further iterators.
6680**
6681** If an error occurs, return an SQLite error code. Otherwise, return
6682** SQLITE_OK. It is not considered an error if some term matches zero
6683** documents.
6684*/
6685static int fts5ExprNearInitAll(
6686 Fts5Expr *pExpr,
6687 Fts5ExprNode *pNode
6688){
6689 Fts5ExprNearset *pNear = pNode->pNear;
6690 int i;
6691
6692 assert( pNode->bNomatch==0 )((void) (0));
6693 for(i=0; i<pNear->nPhrase; i++){
6694 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
6695 if( pPhrase->nTerm==0 ){
6696 pNode->bEof = 1;
6697 return SQLITE_OK0;
6698 }else{
6699 int j;
6700 for(j=0; j<pPhrase->nTerm; j++){
6701 Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
6702 Fts5ExprTerm *p;
6703 int bHit = 0;
6704
6705 for(p=pTerm; p; p=p->pSynonym){
6706 int rc;
6707 if( p->pIter ){
6708 sqlite3Fts5IterClose(p->pIter);
6709 p->pIter = 0;
6710 }
6711 rc = sqlite3Fts5IndexQuery(
6712 pExpr->pIndex, p->pTerm, p->nQueryTerm,
6713 (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX0x0001 : 0) |
6714 (pExpr->bDesc ? FTS5INDEX_QUERY_DESC0x0002 : 0),
6715 pNear->pColset,
6716 &p->pIter
6717 );
6718 assert( (rc==SQLITE_OK)==(p->pIter!=0) )((void) (0));
6719 if( rc!=SQLITE_OK0 ) return rc;
6720 if( 0==sqlite3Fts5IterEof(p->pIter)((p->pIter)->bEof) ){
6721 bHit = 1;
6722 }
6723 }
6724
6725 if( bHit==0 ){
6726 pNode->bEof = 1;
6727 return SQLITE_OK0;
6728 }
6729 }
6730 }
6731 }
6732
6733 pNode->bEof = 0;
6734 return SQLITE_OK0;
6735}
6736
6737/*
6738** If pExpr is an ASC iterator, this function returns a value with the
6739** same sign as:
6740**
6741** (iLhs - iRhs)
6742**
6743** Otherwise, if this is a DESC iterator, the opposite is returned:
6744**
6745** (iRhs - iLhs)
6746*/
6747static int fts5RowidCmp(
6748 Fts5Expr *pExpr,
6749 i64 iLhs,
6750 i64 iRhs
6751){
6752 assert( pExpr->bDesc==0 || pExpr->bDesc==1 )((void) (0));
6753 if( pExpr->bDesc==0 ){
6754 if( iLhs<iRhs ) return -1;
6755 return (iLhs > iRhs);
6756 }else{
6757 if( iLhs>iRhs ) return -1;
6758 return (iLhs < iRhs);
6759 }
6760}
6761
6762static void fts5ExprSetEof(Fts5ExprNode *pNode){
6763 int i;
6764 pNode->bEof = 1;
6765 pNode->bNomatch = 0;
6766 for(i=0; i<pNode->nChild; i++){
6767 fts5ExprSetEof(pNode->apChild[i]);
6768 }
6769}
6770
6771static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){
6772 if( pNode->eType==FTS5_STRING9 || pNode->eType==FTS5_TERM4 ){
6773 Fts5ExprNearset *pNear = pNode->pNear;
6774 int i;
6775 for(i=0; i<pNear->nPhrase; i++){
6776 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
6777 pPhrase->poslist.n = 0;
6778 }
6779 }else{
6780 int i;
6781 for(i=0; i<pNode->nChild; i++){
6782 fts5ExprNodeZeroPoslist(pNode->apChild[i]);
6783 }
6784 }
6785}
6786
6787
6788
6789/*
6790** Compare the values currently indicated by the two nodes as follows:
6791**
6792** res = (*p1) - (*p2)
6793**
6794** Nodes that point to values that come later in the iteration order are
6795** considered to be larger. Nodes at EOF are the largest of all.
6796**
6797** This means that if the iteration order is ASC, then numerically larger
6798** rowids are considered larger. Or if it is the default DESC, numerically
6799** smaller rowids are larger.
6800*/
6801static int fts5NodeCompare(
6802 Fts5Expr *pExpr,
6803 Fts5ExprNode *p1,
6804 Fts5ExprNode *p2
6805){
6806 if( p2->bEof ) return -1;
6807 if( p1->bEof ) return +1;
6808 return fts5RowidCmp(pExpr, p1->iRowid, p2->iRowid);
6809}
6810
6811/*
6812** All individual term iterators in pNear are guaranteed to be valid when
6813** this function is called. This function checks if all term iterators
6814** point to the same rowid, and if not, advances them until they do.
6815** If an EOF is reached before this happens, *pbEof is set to true before
6816** returning.
6817**
6818** SQLITE_OK is returned if an error occurs, or an SQLite error code
6819** otherwise. It is not considered an error code if an iterator reaches
6820** EOF.
6821*/
6822static int fts5ExprNodeTest_STRING(
6823 Fts5Expr *pExpr, /* Expression pPhrase belongs to */
6824 Fts5ExprNode *pNode
6825){
6826 Fts5ExprNearset *pNear = pNode->pNear;
6827 Fts5ExprPhrase *pLeft = pNear->apPhrase[0];
6828 int rc = SQLITE_OK0;
6829 i64 iLast; /* Lastest rowid any iterator points to */
6830 int i, j; /* Phrase and token index, respectively */
6831 int bMatch; /* True if all terms are at the same rowid */
6832 const int bDesc = pExpr->bDesc;
6833
6834 /* Check that this node should not be FTS5_TERM */
6835 assert( pNear->nPhrase>1((void) (0))
6836 || pNear->apPhrase[0]->nTerm>1((void) (0))
6837 || pNear->apPhrase[0]->aTerm[0].pSynonym((void) (0))
6838 || pNear->apPhrase[0]->aTerm[0].bFirst((void) (0))
6839 )((void) (0));
6840
6841 /* Initialize iLast, the "lastest" rowid any iterator points to. If the
6842 ** iterator skips through rowids in the default ascending order, this means
6843 ** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
6844 ** means the minimum rowid. */
6845 if( pLeft->aTerm[0].pSynonym ){
6846 iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0);
6847 }else{
6848 iLast = pLeft->aTerm[0].pIter->iRowid;
6849 }
6850
6851 do {
6852 bMatch = 1;
6853 for(i=0; i<pNear->nPhrase; i++){
6854 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
6855 for(j=0; j<pPhrase->nTerm; j++){
6856 Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
6857 if( pTerm->pSynonym ){
6858 i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
6859 if( iRowid==iLast ) continue;
6860 bMatch = 0;
6861 if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
6862 pNode->bNomatch = 0;
6863 pNode->bEof = 1;
6864 return rc;
6865 }
6866 }else{
6867 Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
6868 if( pIter->iRowid==iLast ) continue;
6869 bMatch = 0;
6870 if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
6871 return rc;
6872 }
6873 }
6874 }
6875 }
6876 }while( bMatch==0 );
6877
6878 pNode->iRowid = iLast;
6879 pNode->bNomatch = ((0==fts5ExprNearTest(&rc, pExpr, pNode)) && rc==SQLITE_OK0);
6880 assert( pNode->bEof==0 || pNode->bNomatch==0 )((void) (0));
6881
6882 return rc;
6883}
6884
6885/*
6886** Advance the first term iterator in the first phrase of pNear. Set output
6887** variable *pbEof to true if it reaches EOF or if an error occurs.
6888**
6889** Return SQLITE_OK if successful, or an SQLite error code if an error
6890** occurs.
6891*/
6892static int fts5ExprNodeNext_STRING(
6893 Fts5Expr *pExpr, /* Expression pPhrase belongs to */
6894 Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */
6895 int bFromValid,
6896 i64 iFrom
6897){
6898 Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0];
6899 int rc = SQLITE_OK0;
6900
6901 pNode->bNomatch = 0;
6902 if( pTerm->pSynonym ){
6903 int bEof = 1;
6904 Fts5ExprTerm *p;
6905
6906 /* Find the firstest rowid any synonym points to. */
6907 i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0);
6908
6909 /* Advance each iterator that currently points to iRowid. Or, if iFrom
6910 ** is valid - each iterator that points to a rowid before iFrom. */
6911 for(p=pTerm; p; p=p->pSynonym){
6912 if( sqlite3Fts5IterEof(p->pIter)((p->pIter)->bEof)==0 ){
6913 i64 ii = p->pIter->iRowid;
6914 if( ii==iRowid
6915 || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc)
6916 ){
6917 if( bFromValid ){
6918 rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom);
6919 }else{
6920 rc = sqlite3Fts5IterNext(p->pIter);
6921 }
6922 if( rc!=SQLITE_OK0 ) break;
6923 if( sqlite3Fts5IterEof(p->pIter)((p->pIter)->bEof)==0 ){
6924 bEof = 0;
6925 }
6926 }else{
6927 bEof = 0;
6928 }
6929 }
6930 }
6931
6932 /* Set the EOF flag if either all synonym iterators are at EOF or an
6933 ** error has occurred. */
6934 pNode->bEof = (rc || bEof);
6935 }else{
6936 Fts5IndexIter *pIter = pTerm->pIter;
6937
6938 assert( Fts5NodeIsString(pNode) )((void) (0));
6939 if( bFromValid ){
6940 rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
6941 }else{
6942 rc = sqlite3Fts5IterNext(pIter);
6943 }
6944
6945 pNode->bEof = (rc || sqlite3Fts5IterEof(pIter)((pIter)->bEof));
6946 }
6947
6948 if( pNode->bEof==0 ){
6949 assert( rc==SQLITE_OK )((void) (0));
6950 rc = fts5ExprNodeTest_STRING(pExpr, pNode);
6951 }
6952
6953 return rc;
6954}
6955
6956
6957static int fts5ExprNodeTest_TERM(
6958 Fts5Expr *pExpr, /* Expression that pNear is a part of */
6959 Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */
6960){
6961 /* As this "NEAR" object is actually a single phrase that consists
6962 ** of a single term only, grab pointers into the poslist managed by the
6963 ** fts5_index.c iterator object. This is much faster than synthesizing
6964 ** a new poslist the way we have to for more complicated phrase or NEAR
6965 ** expressions. */
6966 Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0];
6967 Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
6968
6969 assert( pNode->eType==FTS5_TERM )((void) (0));
6970 assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 )((void) (0));
6971 assert( pPhrase->aTerm[0].pSynonym==0 )((void) (0));
6972
6973 pPhrase->poslist.n = pIter->nData;
6974 if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL0 ){
6975 pPhrase->poslist.p = (u8*)pIter->pData;
6976 }
6977 pNode->iRowid = pIter->iRowid;
6978 pNode->bNomatch = (pPhrase->poslist.n==0);
6979 return SQLITE_OK0;
6980}
6981
6982/*
6983** xNext() method for a node of type FTS5_TERM.
6984*/
6985static int fts5ExprNodeNext_TERM(
6986 Fts5Expr *pExpr,
6987 Fts5ExprNode *pNode,
6988 int bFromValid,
6989 i64 iFrom
6990){
6991 int rc;
6992 Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter;
6993
6994 assert( pNode->bEof==0 )((void) (0));
6995 if( bFromValid ){
6996 rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
6997 }else{
6998 rc = sqlite3Fts5IterNext(pIter);
6999 }
7000 if( rc==SQLITE_OK0 && sqlite3Fts5IterEof(pIter)((pIter)->bEof)==0 ){
7001 rc = fts5ExprNodeTest_TERM(pExpr, pNode);
7002 }else{
7003 pNode->bEof = 1;
7004 pNode->bNomatch = 0;
7005 }
7006 return rc;
7007}
7008
7009static void fts5ExprNodeTest_OR(
7010 Fts5Expr *pExpr, /* Expression of which pNode is a part */
7011 Fts5ExprNode *pNode /* Expression node to test */
7012){
7013 Fts5ExprNode *pNext = pNode->apChild[0];
7014 int i;
7015
7016 for(i=1; i<pNode->nChild; i++){
7017 Fts5ExprNode *pChild = pNode->apChild[i];
7018 int cmp = fts5NodeCompare(pExpr, pNext, pChild);
7019 if( cmp>0 || (cmp==0 && pChild->bNomatch==0) ){
7020 pNext = pChild;
7021 }
7022 }
7023 pNode->iRowid = pNext->iRowid;
7024 pNode->bEof = pNext->bEof;
7025 pNode->bNomatch = pNext->bNomatch;
7026}
7027
7028static int fts5ExprNodeNext_OR(
7029 Fts5Expr *pExpr,
7030 Fts5ExprNode *pNode,
7031 int bFromValid,
7032 i64 iFrom
7033){
7034 int i;
7035 i64 iLast = pNode->iRowid;
7036
7037 for(i=0; i<pNode->nChild; i++){
7038 Fts5ExprNode *p1 = pNode->apChild[i];
7039 assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 )((void) (0));
7040 if( p1->bEof==0 ){
7041 if( (p1->iRowid==iLast)
7042 || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0)
7043 ){
7044 int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom)(p1)->xNext((pExpr), (p1), (bFromValid), (iFrom));
7045 if( rc!=SQLITE_OK0 ){
7046 pNode->bNomatch = 0;
7047 return rc;
7048 }
7049 }
7050 }
7051 }
7052
7053 fts5ExprNodeTest_OR(pExpr, pNode);
7054 return SQLITE_OK0;
7055}
7056
7057/*
7058** Argument pNode is an FTS5_AND node.
7059*/
7060static int fts5ExprNodeTest_AND(
7061 Fts5Expr *pExpr, /* Expression pPhrase belongs to */
7062 Fts5ExprNode *pAnd /* FTS5_AND node to advance */
7063){
7064 int iChild;
7065 i64 iLast = pAnd->iRowid;
7066 int rc = SQLITE_OK0;
7067 int bMatch;
7068
7069 assert( pAnd->bEof==0 )((void) (0));
7070 do {
7071 pAnd->bNomatch = 0;
7072 bMatch = 1;
7073 for(iChild=0; iChild<pAnd->nChild; iChild++){
7074 Fts5ExprNode *pChild = pAnd->apChild[iChild];
7075 int cmp = fts5RowidCmp(pExpr, iLast, pChild->iRowid);
7076 if( cmp>0 ){
7077 /* Advance pChild until it points to iLast or laster */
7078 rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast)(pChild)->xNext((pExpr), (pChild), (1), (iLast));
7079 if( rc!=SQLITE_OK0 ){
7080 pAnd->bNomatch = 0;
7081 return rc;
7082 }
7083 }
7084
7085 /* If the child node is now at EOF, so is the parent AND node. Otherwise,
7086 ** the child node is guaranteed to have advanced at least as far as
7087 ** rowid iLast. So if it is not at exactly iLast, pChild->iRowid is the
7088 ** new lastest rowid seen so far. */
7089 assert( pChild->bEof || fts5RowidCmp(pExpr, iLast, pChild->iRowid)<=0 )((void) (0));
7090 if( pChild->bEof ){
7091 fts5ExprSetEof(pAnd);
7092 bMatch = 1;
7093 break;
7094 }else if( iLast!=pChild->iRowid ){
7095 bMatch = 0;
7096 iLast = pChild->iRowid;
7097 }
7098
7099 if( pChild->bNomatch ){
7100 pAnd->bNomatch = 1;
7101 }
7102 }
7103 }while( bMatch==0 );
7104
7105 if( pAnd->bNomatch && pAnd!=pExpr->pRoot ){
7106 fts5ExprNodeZeroPoslist(pAnd);
7107 }
7108 pAnd->iRowid = iLast;
7109 return SQLITE_OK0;
7110}
7111
7112static int fts5ExprNodeNext_AND(
7113 Fts5Expr *pExpr,
7114 Fts5ExprNode *pNode,
7115 int bFromValid,
7116 i64 iFrom
7117){
7118 int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom)(pNode->apChild[0])->xNext((pExpr), (pNode->apChild[
0]), (bFromValid), (iFrom));
7119 if( rc==SQLITE_OK0 ){
7120 rc = fts5ExprNodeTest_AND(pExpr, pNode);
7121 }else{
7122 pNode->bNomatch = 0;
7123 }
7124 return rc;
7125}
7126
7127static int fts5ExprNodeTest_NOT(
7128 Fts5Expr *pExpr, /* Expression pPhrase belongs to */
7129 Fts5ExprNode *pNode /* FTS5_NOT node to advance */
7130){
7131 int rc = SQLITE_OK0;
7132 Fts5ExprNode *p1 = pNode->apChild[0];
7133 Fts5ExprNode *p2 = pNode->apChild[1];
7134 assert( pNode->nChild==2 )((void) (0));
7135
7136 while( rc==SQLITE_OK0 && p1->bEof==0 ){
7137 int cmp = fts5NodeCompare(pExpr, p1, p2);
7138 if( cmp>0 ){
7139 rc = fts5ExprNodeNext(pExpr, p2, 1, p1->iRowid)(p2)->xNext((pExpr), (p2), (1), (p1->iRowid));
7140 cmp = fts5NodeCompare(pExpr, p1, p2);
7141 }
7142 assert( rc!=SQLITE_OK || cmp<=0 )((void) (0));
7143 if( cmp || p2->bNomatch ) break;
7144 rc = fts5ExprNodeNext(pExpr, p1, 0, 0)(p1)->xNext((pExpr), (p1), (0), (0));
7145 }
7146 pNode->bEof = p1->bEof;
7147 pNode->bNomatch = p1->bNomatch;
7148 pNode->iRowid = p1->iRowid;
7149 if( p1->bEof ){
7150 fts5ExprNodeZeroPoslist(p2);
7151 }
7152 return rc;
7153}
7154
7155static int fts5ExprNodeNext_NOT(
7156 Fts5Expr *pExpr,
7157 Fts5ExprNode *pNode,
7158 int bFromValid,
7159 i64 iFrom
7160){
7161 int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom)(pNode->apChild[0])->xNext((pExpr), (pNode->apChild[
0]), (bFromValid), (iFrom));
7162 if( rc==SQLITE_OK0 ){
7163 rc = fts5ExprNodeTest_NOT(pExpr, pNode);
7164 }
7165 if( rc!=SQLITE_OK0 ){
7166 pNode->bNomatch = 0;
7167 }
7168 return rc;
7169}
7170
7171/*
7172** If pNode currently points to a match, this function returns SQLITE_OK
7173** without modifying it. Otherwise, pNode is advanced until it does point
7174** to a match or EOF is reached.
7175*/
7176static int fts5ExprNodeTest(
7177 Fts5Expr *pExpr, /* Expression of which pNode is a part */
7178 Fts5ExprNode *pNode /* Expression node to test */
7179){
7180 int rc = SQLITE_OK0;
7181 if( pNode->bEof==0 ){
7182 switch( pNode->eType ){
7183
7184 case FTS5_STRING9: {
7185 rc = fts5ExprNodeTest_STRING(pExpr, pNode);
7186 break;
7187 }
7188
7189 case FTS5_TERM4: {
7190 rc = fts5ExprNodeTest_TERM(pExpr, pNode);
7191 break;
7192 }
7193
7194 case FTS5_AND2: {
7195 rc = fts5ExprNodeTest_AND(pExpr, pNode);
7196 break;
7197 }
7198
7199 case FTS5_OR1: {
7200 fts5ExprNodeTest_OR(pExpr, pNode);
7201 break;
7202 }
7203
7204 default: assert( pNode->eType==FTS5_NOT )((void) (0)); {
7205 rc = fts5ExprNodeTest_NOT(pExpr, pNode);
7206 break;
7207 }
7208 }
7209 }
7210 return rc;
7211}
7212
7213
7214/*
7215** Set node pNode, which is part of expression pExpr, to point to the first
7216** match. If there are no matches, set the Node.bEof flag to indicate EOF.
7217**
7218** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise.
7219** It is not an error if there are no matches.
7220*/
7221static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){
7222 int rc = SQLITE_OK0;
7223 pNode->bEof = 0;
7224 pNode->bNomatch = 0;
7225
7226 if( Fts5NodeIsString(pNode)((pNode)->eType==4 || (pNode)->eType==9) ){
7227 /* Initialize all term iterators in the NEAR object. */
7228 rc = fts5ExprNearInitAll(pExpr, pNode);
7229 }else if( pNode->xNext==0 ){
7230 pNode->bEof = 1;
7231 }else{
7232 int i;
7233 int nEof = 0;
7234 for(i=0; i<pNode->nChild && rc==SQLITE_OK0; i++){
7235 Fts5ExprNode *pChild = pNode->apChild[i];
7236 rc = fts5ExprNodeFirst(pExpr, pNode->apChild[i]);
7237 assert( pChild->bEof==0 || pChild->bEof==1 )((void) (0));
7238 nEof += pChild->bEof;
7239 }
7240 pNode->iRowid = pNode->apChild[0]->iRowid;
7241
7242 switch( pNode->eType ){
7243 case FTS5_AND2:
7244 if( nEof>0 ) fts5ExprSetEof(pNode);
7245 break;
7246
7247 case FTS5_OR1:
7248 if( pNode->nChild==nEof ) fts5ExprSetEof(pNode);
7249 break;
7250
7251 default:
7252 assert( pNode->eType==FTS5_NOT )((void) (0));
7253 pNode->bEof = pNode->apChild[0]->bEof;
7254 break;
7255 }
7256 }
7257
7258 if( rc==SQLITE_OK0 ){
7259 rc = fts5ExprNodeTest(pExpr, pNode);
7260 }
7261 return rc;
7262}
7263
7264
7265/*
7266** Begin iterating through the set of documents in index pIdx matched by
7267** the MATCH expression passed as the first argument. If the "bDesc"
7268** parameter is passed a non-zero value, iteration is in descending rowid
7269** order. Or, if it is zero, in ascending order.
7270**
7271** If iterating in ascending rowid order (bDesc==0), the first document
7272** visited is that with the smallest rowid that is larger than or equal
7273** to parameter iFirst. Or, if iterating in ascending order (bDesc==1),
7274** then the first document visited must have a rowid smaller than or
7275** equal to iFirst.
7276**
7277** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
7278** is not considered an error if the query does not match any documents.
7279*/
7280static int sqlite3Fts5ExprFirst(
7281 Fts5Expr *p,
7282 Fts5Index *pIdx,
7283 i64 iFirst,
7284 i64 iLast,
7285 int bDesc
7286){
7287 Fts5ExprNode *pRoot = p->pRoot;
7288 int rc; /* Return code */
7289
7290 p->pIndex = pIdx;
7291 p->bDesc = bDesc;
7292 rc = fts5ExprNodeFirst(p, pRoot);
7293
7294 /* If not at EOF but the current rowid occurs earlier than iFirst in
7295 ** the iteration order, move to document iFirst or later. */
7296 if( rc==SQLITE_OK0
7297 && 0==pRoot->bEof
7298 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0
7299 ){
7300 rc = fts5ExprNodeNext(p, pRoot, 1, iFirst)(pRoot)->xNext((p), (pRoot), (1), (iFirst));
7301 }
7302
7303 /* If the iterator is not at a real match, skip forward until it is. */
7304 while( pRoot->bNomatch && rc==SQLITE_OK0 ){
7305 assert( pRoot->bEof==0 )((void) (0));
7306 rc = fts5ExprNodeNext(p, pRoot, 0, 0)(pRoot)->xNext((p), (pRoot), (0), (0));
7307 }
7308 if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){
7309 pRoot->bEof = 1;
7310 }
7311 return rc;
7312}
7313
7314/*
7315** Move to the next document
7316**
7317** Return SQLITE_OK if successful, or an SQLite error code otherwise. It
7318** is not considered an error if the query does not match any documents.
7319*/
7320static int sqlite3Fts5ExprNext(Fts5Expr *p, i64 iLast){
7321 int rc;
7322 Fts5ExprNode *pRoot = p->pRoot;
7323 assert( pRoot->bEof==0 && pRoot->bNomatch==0 )((void) (0));
7324 do {
7325 rc = fts5ExprNodeNext(p, pRoot, 0, 0)(pRoot)->xNext((p), (pRoot), (0), (0));
7326 assert( pRoot->bNomatch==0 || (rc==SQLITE_OK && pRoot->bEof==0) )((void) (0));
7327 }while( pRoot->bNomatch );
7328 if( fts5RowidCmp(p, pRoot->iRowid, iLast)>0 ){
7329 pRoot->bEof = 1;
7330 }
7331 return rc;
7332}
7333
7334static int sqlite3Fts5ExprEof(Fts5Expr *p){
7335 return p->pRoot->bEof;
7336}
7337
7338static i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
7339 return p->pRoot->iRowid;
7340}
7341
7342static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
7343 int rc = SQLITE_OK0;
7344 *pz = sqlite3Fts5Strndup(&rc, pToken->p, pToken->n);
7345 return rc;
7346}
7347
7348/*
7349** Free the phrase object passed as the only argument.
7350*/
7351static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
7352 if( pPhrase ){
7353 int i;
7354 for(i=0; i<pPhrase->nTerm; i++){
7355 Fts5ExprTerm *pSyn;
7356 Fts5ExprTerm *pNext;
7357 Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
7358 sqlite3_freesqlite3_api->free(pTerm->pTerm);
7359 sqlite3Fts5IterClose(pTerm->pIter);
7360 for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
7361 pNext = pSyn->pSynonym;
7362 sqlite3Fts5IterClose(pSyn->pIter);
7363 fts5BufferFree((Fts5Buffer*)&pSyn[1])sqlite3Fts5BufferFree((Fts5Buffer*)&pSyn[1]);
7364 sqlite3_freesqlite3_api->free(pSyn);
7365 }
7366 }
7367 if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist)sqlite3Fts5BufferFree(&pPhrase->poslist);
7368 sqlite3_freesqlite3_api->free(pPhrase);
7369 }
7370}
7371
7372/*
7373** Set the "bFirst" flag on the first token of the phrase passed as the
7374** only argument.
7375*/
7376static void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){
7377 if( pPhrase && pPhrase->nTerm ){
7378 pPhrase->aTerm[0].bFirst = 1;
7379 }
7380}
7381
7382/*
7383** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
7384** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
7385** appended to it and the results returned.
7386**
7387** If an OOM error occurs, both the pNear and pPhrase objects are freed and
7388** NULL returned.
7389*/
7390static Fts5ExprNearset *sqlite3Fts5ParseNearset(
7391 Fts5Parse *pParse, /* Parse context */
7392 Fts5ExprNearset *pNear, /* Existing nearset, or NULL */
7393 Fts5ExprPhrase *pPhrase /* Recently parsed phrase */
7394){
7395 const int SZALLOC = 8;
7396 Fts5ExprNearset *pRet = 0;
7397
7398 if( pParse->rc==SQLITE_OK0 ){
7399 if( pNear==0 ){
7400 sqlite3_int64 nByte;
7401 nByte = SZ_FTS5EXPRNEARSET(SZALLOC+1)(__builtin_offsetof(Fts5ExprNearset, apPhrase)+(SZALLOC+1)*sizeof
(Fts5ExprPhrase*));
7402 pRet = sqlite3_malloc64sqlite3_api->malloc64(nByte);
7403 if( pRet==0 ){
7404 pParse->rc = SQLITE_NOMEM7;
7405 }else{
7406 memset(pRet, 0, (size_t)nByte);
7407 }
7408 }else if( (pNear->nPhrase % SZALLOC)==0 ){
7409 int nNew = pNear->nPhrase + SZALLOC;
7410 sqlite3_int64 nByte;
7411
7412 nByte = SZ_FTS5EXPRNEARSET(nNew+1)(__builtin_offsetof(Fts5ExprNearset, apPhrase)+(nNew+1)*sizeof
(Fts5ExprPhrase*));
7413 pRet = (Fts5ExprNearset*)sqlite3_realloc64sqlite3_api->realloc64(pNear, nByte);
7414 if( pRet==0 ){
7415 pParse->rc = SQLITE_NOMEM7;
7416 }
7417 }else{
7418 pRet = pNear;
7419 }
7420 }
7421
7422 if( pRet==0 ){
7423 assert( pParse->rc!=SQLITE_OK )((void) (0));
7424 sqlite3Fts5ParseNearsetFree(pNear);
7425 sqlite3Fts5ParsePhraseFree(pPhrase);
7426 }else{
7427 if( pRet->nPhrase>0 ){
7428 Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1];
7429 assert( pParse!=0 )((void) (0));
7430 assert( pParse->apPhrase!=0 )((void) (0));
7431 assert( pParse->nPhrase>=2 )((void) (0));
7432 assert( pLast==pParse->apPhrase[pParse->nPhrase-2] )((void) (0));
7433 if( pPhrase->nTerm==0 ){
7434 fts5ExprPhraseFree(pPhrase);
7435 pRet->nPhrase--;
7436 pParse->nPhrase--;
7437 pPhrase = pLast;
7438 }else if( pLast->nTerm==0 ){
7439 fts5ExprPhraseFree(pLast);
7440 pParse->apPhrase[pParse->nPhrase-2] = pPhrase;
7441 pParse->nPhrase--;
7442 pRet->nPhrase--;
7443 }
7444 }
7445 pRet->apPhrase[pRet->nPhrase++] = pPhrase;
7446 }
7447 return pRet;
7448}
7449
7450typedef struct TokenCtx TokenCtx;
7451struct TokenCtx {
7452 Fts5ExprPhrase *pPhrase;
7453 Fts5Config *pConfig;
7454 int rc;
7455};
7456
7457/*
7458** Callback for tokenizing terms used by ParseTerm().
7459*/
7460static int fts5ParseTokenize(
7461 void *pContext, /* Pointer to Fts5InsertCtx object */
7462 int tflags, /* Mask of FTS5_TOKEN_* flags */
7463 const char *pToken, /* Buffer containing token */
7464 int nToken, /* Size of token in bytes */
7465 int iUnused1, /* Start offset of token */
7466 int iUnused2 /* End offset of token */
7467){
7468 int rc = SQLITE_OK0;
7469 const int SZALLOC = 8;
7470 TokenCtx *pCtx = (TokenCtx*)pContext;
7471 Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
7472
7473 UNUSED_PARAM2(iUnused1, iUnused2)(void)(iUnused1), (void)(iUnused2);
7474
7475 /* If an error has already occurred, this is a no-op */
7476 if( pCtx->rc!=SQLITE_OK0 ) return pCtx->rc;
7477 if( nToken>FTS5_MAX_TOKEN_SIZE32768 ) nToken = FTS5_MAX_TOKEN_SIZE32768;
7478
7479 if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED0x0001) ){
7480 Fts5ExprTerm *pSyn;
7481 sqlite3_int64 nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
7482 pSyn = (Fts5ExprTerm*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
7483 if( pSyn==0 ){
7484 rc = SQLITE_NOMEM7;
7485 }else{
7486 memset(pSyn, 0, (size_t)nByte);
7487 pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
7488 pSyn->nFullTerm = pSyn->nQueryTerm = nToken;
7489 memcpy(pSyn->pTerm, pToken, nToken);
7490 if( pCtx->pConfig->bTokendata ){
7491 pSyn->nQueryTerm = (int)strlen(pSyn->pTerm);
7492 }
7493 pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
7494 pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
7495 }
7496 }else{
7497 Fts5ExprTerm *pTerm;
7498 if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
7499 Fts5ExprPhrase *pNew;
7500 int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0);
7501
7502 pNew = (Fts5ExprPhrase*)sqlite3_realloc64sqlite3_api->realloc64(pPhrase,
7503 SZ_FTS5EXPRPHRASE(nNew+1)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (nNew+1)*sizeof(
Fts5ExprTerm))
7504 );
7505 if( pNew==0 ){
7506 rc = SQLITE_NOMEM7;
7507 }else{
7508 if( pPhrase==0 ) memset(pNew, 0, SZ_FTS5EXPRPHRASE(1)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (1)*sizeof(Fts5ExprTerm
)));
7509 pCtx->pPhrase = pPhrase = pNew;
7510 pNew->nTerm = nNew - SZALLOC;
7511 }
7512 }
7513
7514 if( rc==SQLITE_OK0 ){
7515 pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
7516 memset(pTerm, 0, sizeof(Fts5ExprTerm));
7517 pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
7518 pTerm->nFullTerm = pTerm->nQueryTerm = nToken;
7519 if( pCtx->pConfig->bTokendata && rc==SQLITE_OK0 ){
7520 pTerm->nQueryTerm = (int)strlen(pTerm->pTerm);
7521 }
7522 }
7523 }
7524
7525 pCtx->rc = rc;
7526 return rc;
7527}
7528
7529
7530/*
7531** Free the phrase object passed as the only argument.
7532*/
7533static void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase *pPhrase){
7534 fts5ExprPhraseFree(pPhrase);
7535}
7536
7537/*
7538** Free the phrase object passed as the second argument.
7539*/
7540static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset *pNear){
7541 if( pNear ){
7542 int i;
7543 for(i=0; i<pNear->nPhrase; i++){
7544 fts5ExprPhraseFree(pNear->apPhrase[i]);
7545 }
7546 sqlite3_freesqlite3_api->free(pNear->pColset);
7547 sqlite3_freesqlite3_api->free(pNear);
7548 }
7549}
7550
7551static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p){
7552 assert( pParse->pExpr==0 )((void) (0));
7553 pParse->pExpr = p;
7554}
7555
7556static int parseGrowPhraseArray(Fts5Parse *pParse){
7557 if( (pParse->nPhrase % 8)==0 ){
7558 sqlite3_int64 nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8);
7559 Fts5ExprPhrase **apNew;
7560 apNew = (Fts5ExprPhrase**)sqlite3_realloc64sqlite3_api->realloc64(pParse->apPhrase, nByte);
7561 if( apNew==0 ){
7562 pParse->rc = SQLITE_NOMEM7;
7563 return SQLITE_NOMEM7;
7564 }
7565 pParse->apPhrase = apNew;
7566 }
7567 return SQLITE_OK0;
7568}
7569
7570/*
7571** This function is called by the parser to process a string token. The
7572** string may or may not be quoted. In any case it is tokenized and a
7573** phrase object consisting of all tokens returned.
7574*/
7575static Fts5ExprPhrase *sqlite3Fts5ParseTerm(
7576 Fts5Parse *pParse, /* Parse context */
7577 Fts5ExprPhrase *pAppend, /* Phrase to append to */
7578 Fts5Token *pToken, /* String to tokenize */
7579 int bPrefix /* True if there is a trailing "*" */
7580){
7581 Fts5Config *pConfig = pParse->pConfig;
7582 TokenCtx sCtx; /* Context object passed to callback */
7583 int rc; /* Tokenize return code */
7584 char *z = 0;
7585
7586 memset(&sCtx, 0, sizeof(TokenCtx));
7587 sCtx.pPhrase = pAppend;
7588 sCtx.pConfig = pConfig;
7589
7590 rc = fts5ParseStringFromToken(pToken, &z);
7591 if( rc==SQLITE_OK0 ){
7592 int flags = FTS5_TOKENIZE_QUERY0x0001 | (bPrefix ? FTS5_TOKENIZE_PREFIX0x0002 : 0);
7593 int n;
7594 sqlite3Fts5Dequote(z);
7595 n = (int)strlen(z);
7596 rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
7597 }
7598 sqlite3_freesqlite3_api->free(z);
7599 if( rc || (rc = sCtx.rc) ){
7600 pParse->rc = rc;
7601 fts5ExprPhraseFree(sCtx.pPhrase);
7602 sCtx.pPhrase = 0;
7603 }else{
7604
7605 if( pAppend==0 ){
7606 if( parseGrowPhraseArray(pParse) ){
7607 fts5ExprPhraseFree(sCtx.pPhrase);
7608 return 0;
7609 }
7610 pParse->nPhrase++;
7611 }
7612
7613 if( sCtx.pPhrase==0 ){
7614 /* This happens when parsing a token or quoted phrase that contains
7615 ** no token characters at all. (e.g ... MATCH '""'). */
7616 sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, SZ_FTS5EXPRPHRASE(1)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (1)*sizeof(Fts5ExprTerm
)));
7617 }else if( sCtx.pPhrase->nTerm ){
7618 sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
7619 }
7620 assert( pParse->apPhrase!=0 )((void) (0));
7621 pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
7622 }
7623
7624 return sCtx.pPhrase;
7625}
7626
7627/*
7628** Create a new FTS5 expression by cloning phrase iPhrase of the
7629** expression passed as the second argument.
7630*/
7631static int sqlite3Fts5ExprClonePhrase(
7632 Fts5Expr *pExpr,
7633 int iPhrase,
7634 Fts5Expr **ppNew
7635){
7636 int rc = SQLITE_OK0; /* Return code */
7637 Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */
7638 Fts5Expr *pNew = 0; /* Expression to return via *ppNew */
7639 TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */
7640 if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){
7641 rc = SQLITE_RANGE25;
7642 }else{
7643 pOrig = pExpr->apExprPhrase[iPhrase];
7644 pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
7645 }
7646 if( rc==SQLITE_OK0 ){
7647 pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc,
7648 sizeof(Fts5ExprPhrase*));
7649 }
7650 if( rc==SQLITE_OK0 ){
7651 pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRNODE(1)(__builtin_offsetof(Fts5ExprNode, apChild) + (1)*sizeof(Fts5ExprNode
*)));
7652 }
7653 if( rc==SQLITE_OK0 ){
7654 pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc,
7655 SZ_FTS5EXPRNEARSET(2)(__builtin_offsetof(Fts5ExprNearset, apPhrase)+(2)*sizeof(Fts5ExprPhrase
*)));
7656 }
7657 if( rc==SQLITE_OK0 && ALWAYS(pOrig!=0)(pOrig!=0) ){
7658 Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset;
7659 if( pColsetOrig ){
7660 sqlite3_int64 nByte;
7661 Fts5Colset *pColset;
7662 nByte = SZ_FTS5COLSET(pColsetOrig->nCol)(sizeof(i64)*((pColsetOrig->nCol+2)/2));
7663 pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte);
7664 if( pColset ){
7665 memcpy(pColset, pColsetOrig, (size_t)nByte);
7666 }
7667 pNew->pRoot->pNear->pColset = pColset;
7668 }
7669 }
7670
7671 if( rc==SQLITE_OK0 ){
7672 if( pOrig->nTerm ){
7673 int i; /* Used to iterate through phrase terms */
7674 sCtx.pConfig = pExpr->pConfig;
7675 for(i=0; rc==SQLITE_OK0 && i<pOrig->nTerm; i++){
7676 int tflags = 0;
7677 Fts5ExprTerm *p;
7678 for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK0; p=p->pSynonym){
7679 rc = fts5ParseTokenize((void*)&sCtx,tflags,p->pTerm,p->nFullTerm,0,0);
7680 tflags = FTS5_TOKEN_COLOCATED0x0001;
7681 }
7682 if( rc==SQLITE_OK0 ){
7683 sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
7684 sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
7685 }
7686 }
7687 }else{
7688 /* This happens when parsing a token or quoted phrase that contains
7689 ** no token characters at all. (e.g ... MATCH '""'). */
7690 sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, SZ_FTS5EXPRPHRASE(1)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (1)*sizeof(Fts5ExprTerm
)));
7691 }
7692 }
7693
7694 if( rc==SQLITE_OK0 && ALWAYS(sCtx.pPhrase)(sCtx.pPhrase) ){
7695 /* All the allocations succeeded. Put the expression object together. */
7696 pNew->pIndex = pExpr->pIndex;
7697 pNew->pConfig = pExpr->pConfig;
7698 pNew->nPhrase = 1;
7699 pNew->apExprPhrase[0] = sCtx.pPhrase;
7700 pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
7701 pNew->pRoot->pNear->nPhrase = 1;
7702 sCtx.pPhrase->pNode = pNew->pRoot;
7703
7704 if( pOrig->nTerm==1
7705 && pOrig->aTerm[0].pSynonym==0
7706 && pOrig->aTerm[0].bFirst==0
7707 ){
7708 pNew->pRoot->eType = FTS5_TERM4;
7709 pNew->pRoot->xNext = fts5ExprNodeNext_TERM;
7710 }else{
7711 pNew->pRoot->eType = FTS5_STRING9;
7712 pNew->pRoot->xNext = fts5ExprNodeNext_STRING;
7713 }
7714 }else{
7715 sqlite3Fts5ExprFree(pNew);
7716 fts5ExprPhraseFree(sCtx.pPhrase);
7717 pNew = 0;
7718 }
7719
7720 *ppNew = pNew;
7721 return rc;
7722}
7723
7724
7725/*
7726** Token pTok has appeared in a MATCH expression where the NEAR operator
7727** is expected. If token pTok does not contain "NEAR", store an error
7728** in the pParse object.
7729*/
7730static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token *pTok){
7731 if( pTok->n!=4 || memcmp("NEAR", pTok->p, 4) ){
7732 sqlite3Fts5ParseError(
7733 pParse, "fts5: syntax error near \"%.*s\"", pTok->n, pTok->p
7734 );
7735 }
7736}
7737
7738static void sqlite3Fts5ParseSetDistance(
7739 Fts5Parse *pParse,
7740 Fts5ExprNearset *pNear,
7741 Fts5Token *p
7742){
7743 if( pNear ){
7744 int nNear = 0;
7745 int i;
7746 if( p->n ){
7747 for(i=0; i<p->n; i++){
7748 char c = (char)p->p[i];
7749 if( c<'0' || c>'9' ){
7750 sqlite3Fts5ParseError(
7751 pParse, "expected integer, got \"%.*s\"", p->n, p->p
7752 );
7753 return;
7754 }
7755 if( nNear<214748363 ) nNear = nNear * 10 + (p->p[i] - '0');
7756 /* ^^^^^^^^^^^^^^^--- Prevent integer overflow */
7757 }
7758 }else{
7759 nNear = FTS5_DEFAULT_NEARDIST10;
7760 }
7761 pNear->nNear = nNear;
7762 }
7763}
7764
7765/*
7766** The second argument passed to this function may be NULL, or it may be
7767** an existing Fts5Colset object. This function returns a pointer to
7768** a new colset object containing the contents of (p) with new value column
7769** number iCol appended.
7770**
7771** If an OOM error occurs, store an error code in pParse and return NULL.
7772** The old colset object (if any) is not freed in this case.
7773*/
7774static Fts5Colset *fts5ParseColset(
7775 Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */
7776 Fts5Colset *p, /* Existing colset object */
7777 int iCol /* New column to add to colset object */
7778){
7779 int nCol = p ? p->nCol : 0; /* Num. columns already in colset object */
7780 Fts5Colset *pNew; /* New colset object to return */
7781
7782 assert( pParse->rc==SQLITE_OK )((void) (0));
7783 assert( iCol>=0 && iCol<pParse->pConfig->nCol )((void) (0));
7784
7785 pNew = sqlite3_realloc64sqlite3_api->realloc64(p, SZ_FTS5COLSET(nCol+1)(sizeof(i64)*((nCol+1 +2)/2)));
7786 if( pNew==0 ){
7787 pParse->rc = SQLITE_NOMEM7;
7788 }else{
7789 int *aiCol = pNew->aiCol;
7790 int i, j;
7791 for(i=0; i<nCol; i++){
7792 if( aiCol[i]==iCol ) return pNew;
7793 if( aiCol[i]>iCol ) break;
7794 }
7795 for(j=nCol; j>i; j--){
7796 aiCol[j] = aiCol[j-1];
7797 }
7798 aiCol[i] = iCol;
7799 pNew->nCol = nCol+1;
7800
7801#ifndef NDEBUG1
7802 /* Check that the array is in order and contains no duplicate entries. */
7803 for(i=1; i<pNew->nCol; i++) assert( pNew->aiCol[i]>pNew->aiCol[i-1] )((void) (0));
7804#endif
7805 }
7806
7807 return pNew;
7808}
7809
7810/*
7811** Allocate and return an Fts5Colset object specifying the inverse of
7812** the colset passed as the second argument. Free the colset passed
7813** as the second argument before returning.
7814*/
7815static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){
7816 Fts5Colset *pRet;
7817 int nCol = pParse->pConfig->nCol;
7818
7819 pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc,
7820 SZ_FTS5COLSET(nCol+1)(sizeof(i64)*((nCol+1 +2)/2))
7821 );
7822 if( pRet ){
7823 int i;
7824 int iOld = 0;
7825 for(i=0; i<nCol; i++){
7826 if( iOld>=p->nCol || p->aiCol[iOld]!=i ){
7827 pRet->aiCol[pRet->nCol++] = i;
7828 }else{
7829 iOld++;
7830 }
7831 }
7832 }
7833
7834 sqlite3_freesqlite3_api->free(p);
7835 return pRet;
7836}
7837
7838static Fts5Colset *sqlite3Fts5ParseColset(
7839 Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */
7840 Fts5Colset *pColset, /* Existing colset object */
7841 Fts5Token *p
7842){
7843 Fts5Colset *pRet = 0;
7844 int iCol;
7845 char *z; /* Dequoted copy of token p */
7846
7847 z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n);
7848 if( pParse->rc==SQLITE_OK0 ){
7849 Fts5Config *pConfig = pParse->pConfig;
7850 sqlite3Fts5Dequote(z);
7851 for(iCol=0; iCol<pConfig->nCol; iCol++){
7852 if( 0==sqlite3_stricmpsqlite3_api->stricmp(pConfig->azCol[iCol], z) ) break;
7853 }
7854 if( iCol==pConfig->nCol ){
7855 sqlite3Fts5ParseError(pParse, "no such column: %s", z);
7856 }else{
7857 pRet = fts5ParseColset(pParse, pColset, iCol);
7858 }
7859 sqlite3_freesqlite3_api->free(z);
7860 }
7861
7862 if( pRet==0 ){
7863 assert( pParse->rc!=SQLITE_OK )((void) (0));
7864 sqlite3_freesqlite3_api->free(pColset);
7865 }
7866
7867 return pRet;
7868}
7869
7870/*
7871** If argument pOrig is NULL, or if (*pRc) is set to anything other than
7872** SQLITE_OK when this function is called, NULL is returned.
7873**
7874** Otherwise, a copy of (*pOrig) is made into memory obtained from
7875** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
7876** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
7877*/
7878static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
7879 Fts5Colset *pRet;
7880 if( pOrig ){
7881 sqlite3_int64 nByte = SZ_FTS5COLSET(pOrig->nCol)(sizeof(i64)*((pOrig->nCol+2)/2));
7882 pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
7883 if( pRet ){
7884 memcpy(pRet, pOrig, (size_t)nByte);
7885 }
7886 }else{
7887 pRet = 0;
7888 }
7889 return pRet;
7890}
7891
7892/*
7893** Remove from colset pColset any columns that are not also in colset pMerge.
7894*/
7895static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
7896 int iIn = 0; /* Next input in pColset */
7897 int iMerge = 0; /* Next input in pMerge */
7898 int iOut = 0; /* Next output slot in pColset */
7899
7900 while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
7901 int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
7902 if( iDiff==0 ){
7903 pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
7904 iMerge++;
7905 iIn++;
7906 }else if( iDiff>0 ){
7907 iMerge++;
7908 }else{
7909 iIn++;
7910 }
7911 }
7912 pColset->nCol = iOut;
7913}
7914
7915/*
7916** Recursively apply colset pColset to expression node pNode and all of
7917** its decendents. If (*ppFree) is not NULL, it contains a spare copy
7918** of pColset. This function may use the spare copy and set (*ppFree) to
7919** zero, or it may create copies of pColset using fts5CloneColset().
7920*/
7921static void fts5ParseSetColset(
7922 Fts5Parse *pParse,
7923 Fts5ExprNode *pNode,
7924 Fts5Colset *pColset,
7925 Fts5Colset **ppFree
7926){
7927 if( pParse->rc==SQLITE_OK0 ){
7928 assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING((void) (0))
7929 || pNode->eType==FTS5_AND || pNode->eType==FTS5_OR((void) (0))
7930 || pNode->eType==FTS5_NOT || pNode->eType==FTS5_EOF((void) (0))
7931 )((void) (0));
7932 if( pNode->eType==FTS5_STRING9 || pNode->eType==FTS5_TERM4 ){
7933 Fts5ExprNearset *pNear = pNode->pNear;
7934 if( pNear->pColset ){
7935 fts5MergeColset(pNear->pColset, pColset);
7936 if( pNear->pColset->nCol==0 ){
7937 pNode->eType = FTS5_EOF0;
7938 pNode->xNext = 0;
7939 }
7940 }else if( *ppFree ){
7941 pNear->pColset = pColset;
7942 *ppFree = 0;
7943 }else{
7944 pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
7945 }
7946 }else{
7947 int i;
7948 assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 )((void) (0));
7949 for(i=0; i<pNode->nChild; i++){
7950 fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
7951 }
7952 }
7953 }
7954}
7955
7956/*
7957** Apply colset pColset to expression node pExpr and all of its descendents.
7958*/
7959static void sqlite3Fts5ParseSetColset(
7960 Fts5Parse *pParse,
7961 Fts5ExprNode *pExpr,
7962 Fts5Colset *pColset
7963){
7964 Fts5Colset *pFree = pColset;
7965 if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE1 ){
7966 sqlite3Fts5ParseError(pParse,
7967 "fts5: column queries are not supported (detail=none)"
7968 );
7969 }else{
7970 fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
7971 }
7972 sqlite3_freesqlite3_api->free(pFree);
7973}
7974
7975static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
7976 switch( pNode->eType ){
7977 case FTS5_STRING9: {
7978 Fts5ExprNearset *pNear = pNode->pNear;
7979 if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1
7980 && pNear->apPhrase[0]->aTerm[0].pSynonym==0
7981 && pNear->apPhrase[0]->aTerm[0].bFirst==0
7982 ){
7983 pNode->eType = FTS5_TERM4;
7984 pNode->xNext = fts5ExprNodeNext_TERM;
7985 }else{
7986 pNode->xNext = fts5ExprNodeNext_STRING;
7987 }
7988 break;
7989 };
7990
7991 case FTS5_OR1: {
7992 pNode->xNext = fts5ExprNodeNext_OR;
7993 break;
7994 };
7995
7996 case FTS5_AND2: {
7997 pNode->xNext = fts5ExprNodeNext_AND;
7998 break;
7999 };
8000
8001 default: assert( pNode->eType==FTS5_NOT )((void) (0)); {
8002 pNode->xNext = fts5ExprNodeNext_NOT;
8003 break;
8004 };
8005 }
8006}
8007
8008/*
8009** Add pSub as a child of p.
8010*/
8011static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
8012 int ii = p->nChild;
8013 if( p->eType!=FTS5_NOT3 && pSub->eType==p->eType ){
8014 int nByte = sizeof(Fts5ExprNode*) * pSub->nChild;
8015 memcpy(&p->apChild[p->nChild], pSub->apChild, nByte);
8016 p->nChild += pSub->nChild;
8017 sqlite3_freesqlite3_api->free(pSub);
8018 }else{
8019 p->apChild[p->nChild++] = pSub;
8020 }
8021 for( ; ii<p->nChild; ii++){
8022 p->iHeight = MAX(p->iHeight, p->apChild[ii]->iHeight + 1)(((p->iHeight) > (p->apChild[ii]->iHeight + 1)) ?
(p->iHeight) : (p->apChild[ii]->iHeight + 1));
8023 }
8024}
8025
8026/*
8027** This function is used when parsing LIKE or GLOB patterns against
8028** trigram indexes that specify either detail=column or detail=none.
8029** It converts a phrase:
8030**
8031** abc + def + ghi
8032**
8033** into an AND tree:
8034**
8035** abc AND def AND ghi
8036*/
8037static Fts5ExprNode *fts5ParsePhraseToAnd(
8038 Fts5Parse *pParse,
8039 Fts5ExprNearset *pNear
8040){
8041 int nTerm = pNear->apPhrase[0]->nTerm;
8042 int ii;
8043 int nByte;
8044 Fts5ExprNode *pRet;
8045
8046 assert( pNear->nPhrase==1 )((void) (0));
8047 assert( pParse->bPhraseToAnd )((void) (0));
8048
8049 nByte = SZ_FTS5EXPRNODE(nTerm+1)(__builtin_offsetof(Fts5ExprNode, apChild) + (nTerm+1)*sizeof
(Fts5ExprNode*));
8050 pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte);
8051 if( pRet ){
8052 pRet->eType = FTS5_AND2;
8053 pRet->nChild = nTerm;
8054 pRet->iHeight = 1;
8055 fts5ExprAssignXNext(pRet);
8056 pParse->nPhrase--;
8057 for(ii=0; ii<nTerm; ii++){
8058 Fts5ExprPhrase *pPhrase = (Fts5ExprPhrase*)sqlite3Fts5MallocZero(
8059 &pParse->rc, SZ_FTS5EXPRPHRASE(1)(__builtin_offsetof(Fts5ExprPhrase, aTerm) + (1)*sizeof(Fts5ExprTerm
))
8060 );
8061 if( pPhrase ){
8062 if( parseGrowPhraseArray(pParse) ){
8063 fts5ExprPhraseFree(pPhrase);
8064 }else{
8065 Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii];
8066 Fts5ExprTerm *pTo = &pPhrase->aTerm[0];
8067 pParse->apPhrase[pParse->nPhrase++] = pPhrase;
8068 pPhrase->nTerm = 1;
8069 pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm);
8070 pTo->nQueryTerm = p->nQueryTerm;
8071 pTo->nFullTerm = p->nFullTerm;
8072 pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING9,
8073 0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase)
8074 );
8075 }
8076 }
8077 }
8078
8079 if( pParse->rc ){
8080 sqlite3Fts5ParseNodeFree(pRet);
8081 pRet = 0;
8082 }else{
8083 sqlite3Fts5ParseNearsetFree(pNear);
8084 }
8085 }
8086
8087 return pRet;
8088}
8089
8090/*
8091** Allocate and return a new expression object. If anything goes wrong (i.e.
8092** OOM error), leave an error code in pParse and return NULL.
8093*/
8094static Fts5ExprNode *sqlite3Fts5ParseNode(
8095 Fts5Parse *pParse, /* Parse context */
8096 int eType, /* FTS5_STRING, AND, OR or NOT */
8097 Fts5ExprNode *pLeft, /* Left hand child expression */
8098 Fts5ExprNode *pRight, /* Right hand child expression */
8099 Fts5ExprNearset *pNear /* For STRING expressions, the near cluster */
8100){
8101 Fts5ExprNode *pRet = 0;
8102
8103 if( pParse->rc==SQLITE_OK0 ){
8104 int nChild = 0; /* Number of children of returned node */
8105 sqlite3_int64 nByte; /* Bytes of space to allocate for this node */
8106
8107 assert( (eType!=FTS5_STRING && !pNear)((void) (0))
8108 || (eType==FTS5_STRING && !pLeft && !pRight)((void) (0))
8109 )((void) (0));
8110 if( eType==FTS5_STRING9 && pNear==0 ) return 0;
8111 if( eType!=FTS5_STRING9 && pLeft==0 ) return pRight;
8112 if( eType!=FTS5_STRING9 && pRight==0 ) return pLeft;
8113
8114 if( eType==FTS5_STRING9
8115 && pParse->bPhraseToAnd
8116 && pNear->apPhrase[0]->nTerm>1
8117 ){
8118 pRet = fts5ParsePhraseToAnd(pParse, pNear);
8119 }else{
8120 if( eType==FTS5_NOT3 ){
8121 nChild = 2;
8122 }else if( eType==FTS5_AND2 || eType==FTS5_OR1 ){
8123 nChild = 2;
8124 if( pLeft->eType==eType ) nChild += pLeft->nChild-1;
8125 if( pRight->eType==eType ) nChild += pRight->nChild-1;
8126 }
8127
8128 nByte = SZ_FTS5EXPRNODE(nChild)(__builtin_offsetof(Fts5ExprNode, apChild) + (nChild)*sizeof(
Fts5ExprNode*));
8129 pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte);
8130
8131 if( pRet ){
8132 pRet->eType = eType;
8133 pRet->pNear = pNear;
8134 fts5ExprAssignXNext(pRet);
8135 if( eType==FTS5_STRING9 ){
8136 int iPhrase;
8137 for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
8138 pNear->apPhrase[iPhrase]->pNode = pRet;
8139 if( pNear->apPhrase[iPhrase]->nTerm==0 ){
8140 pRet->xNext = 0;
8141 pRet->eType = FTS5_EOF0;
8142 }
8143 }
8144
8145 if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL0 ){
8146 Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
8147 if( pNear->nPhrase!=1
8148 || pPhrase->nTerm>1
8149 || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst)
8150 ){
8151 sqlite3Fts5ParseError(pParse,
8152 "fts5: %s queries are not supported (detail!=full)",
8153 pNear->nPhrase==1 ? "phrase": "NEAR"
8154 );
8155 sqlite3Fts5ParseNodeFree(pRet);
8156 pRet = 0;
8157 pNear = 0;
8158 assert( pLeft==0 && pRight==0 )((void) (0));
8159 }
8160 }
8161 }else{
8162 assert( pNear==0 )((void) (0));
8163 fts5ExprAddChildren(pRet, pLeft);
8164 fts5ExprAddChildren(pRet, pRight);
8165 pLeft = pRight = 0;
8166 if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH256 ){
8167 sqlite3Fts5ParseError(pParse,
8168 "fts5 expression tree is too large (maximum depth %d)",
8169 SQLITE_FTS5_MAX_EXPR_DEPTH256
8170 );
8171 sqlite3Fts5ParseNodeFree(pRet);
8172 pRet = 0;
8173 }
8174 }
8175 }
8176 }
8177 }
8178
8179 if( pRet==0 ){
8180 assert( pParse->rc!=SQLITE_OK )((void) (0));
8181 sqlite3Fts5ParseNodeFree(pLeft);
8182 sqlite3Fts5ParseNodeFree(pRight);
8183 sqlite3Fts5ParseNearsetFree(pNear);
8184 }
8185 return pRet;
8186}
8187
8188static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
8189 Fts5Parse *pParse, /* Parse context */
8190 Fts5ExprNode *pLeft, /* Left hand child expression */
8191 Fts5ExprNode *pRight /* Right hand child expression */
8192){
8193 Fts5ExprNode *pRet = 0;
8194 Fts5ExprNode *pPrev;
8195
8196 if( pParse->rc ){
8197 sqlite3Fts5ParseNodeFree(pLeft);
8198 sqlite3Fts5ParseNodeFree(pRight);
8199 }else{
8200
8201 assert( pLeft->eType==FTS5_STRING((void) (0))
8202 || pLeft->eType==FTS5_TERM((void) (0))
8203 || pLeft->eType==FTS5_EOF((void) (0))
8204 || pLeft->eType==FTS5_AND((void) (0))
8205 )((void) (0));
8206 assert( pRight->eType==FTS5_STRING((void) (0))
8207 || pRight->eType==FTS5_TERM((void) (0))
8208 || pRight->eType==FTS5_EOF((void) (0))
8209 || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd)((void) (0))
8210 )((void) (0));
8211
8212 if( pLeft->eType==FTS5_AND2 ){
8213 pPrev = pLeft->apChild[pLeft->nChild-1];
8214 }else{
8215 pPrev = pLeft;
8216 }
8217 assert( pPrev->eType==FTS5_STRING((void) (0))
8218 || pPrev->eType==FTS5_TERM((void) (0))
8219 || pPrev->eType==FTS5_EOF((void) (0))
8220 )((void) (0));
8221
8222 if( pRight->eType==FTS5_EOF0 ){
8223 assert( pParse->apPhrase!=0 )((void) (0));
8224 assert( pParse->nPhrase>0 )((void) (0));
8225 assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] )((void) (0));
8226 sqlite3Fts5ParseNodeFree(pRight);
8227 pRet = pLeft;
8228 pParse->nPhrase--;
8229 }
8230 else if( pPrev->eType==FTS5_EOF0 ){
8231 Fts5ExprPhrase **ap;
8232
8233 if( pPrev==pLeft ){
8234 pRet = pRight;
8235 }else{
8236 pLeft->apChild[pLeft->nChild-1] = pRight;
8237 pRet = pLeft;
8238 }
8239
8240 ap = &pParse->apPhrase[pParse->nPhrase-1-pRight->pNear->nPhrase];
8241 assert( ap[0]==pPrev->pNear->apPhrase[0] )((void) (0));
8242 memmove(ap, &ap[1], sizeof(Fts5ExprPhrase*)*pRight->pNear->nPhrase);
8243 pParse->nPhrase--;
8244
8245 sqlite3Fts5ParseNodeFree(pPrev);
8246 }
8247 else{
8248 pRet = sqlite3Fts5ParseNode(pParse, FTS5_AND2, pLeft, pRight, 0);
8249 }
8250 }
8251
8252 return pRet;
8253}
8254
8255#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
8256static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
8257 sqlite3_int64 nByte = 0;
8258 Fts5ExprTerm *p;
8259 char *zQuoted;
8260
8261 /* Determine the maximum amount of space required. */
8262 for(p=pTerm; p; p=p->pSynonym){
8263 nByte += pTerm->nQueryTerm * 2 + 3 + 2;
8264 }
8265 zQuoted = sqlite3_malloc64sqlite3_api->malloc64(nByte);
8266
8267 if( zQuoted ){
8268 int i = 0;
8269 for(p=pTerm; p; p=p->pSynonym){
8270 char *zIn = p->pTerm;
8271 char *zEnd = &zIn[p->nQueryTerm];
8272 zQuoted[i++] = '"';
8273 while( zIn<zEnd ){
8274 if( *zIn=='"' ) zQuoted[i++] = '"';
8275 zQuoted[i++] = *zIn++;
8276 }
8277 zQuoted[i++] = '"';
8278 if( p->pSynonym ) zQuoted[i++] = '|';
8279 }
8280 if( pTerm->bPrefix ){
8281 zQuoted[i++] = ' ';
8282 zQuoted[i++] = '*';
8283 }
8284 zQuoted[i++] = '\0';
8285 }
8286 return zQuoted;
8287}
8288
8289static char *fts5PrintfAppend(char *zApp, const char *zFmt, ...){
8290 char *zNew;
8291 va_list ap;
8292 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
8293 zNew = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
8294 va_end(ap)__builtin_va_end(ap);
8295 if( zApp && zNew ){
8296 char *zNew2 = sqlite3_mprintfsqlite3_api->mprintf("%s%s", zApp, zNew);
8297 sqlite3_freesqlite3_api->free(zNew);
8298 zNew = zNew2;
8299 }
8300 sqlite3_freesqlite3_api->free(zApp);
8301 return zNew;
8302}
8303
8304/*
8305** Compose a tcl-readable representation of expression pExpr. Return a
8306** pointer to a buffer containing that representation. It is the
8307** responsibility of the caller to at some point free the buffer using
8308** sqlite3_free().
8309*/
8310static char *fts5ExprPrintTcl(
8311 Fts5Config *pConfig,
8312 const char *zNearsetCmd,
8313 Fts5ExprNode *pExpr
8314){
8315 char *zRet = 0;
8316 if( pExpr->eType==FTS5_STRING9 || pExpr->eType==FTS5_TERM4 ){
8317 Fts5ExprNearset *pNear = pExpr->pNear;
8318 int i;
8319 int iTerm;
8320
8321 zRet = fts5PrintfAppend(zRet, "%s ", zNearsetCmd);
8322 if( zRet==0 ) return 0;
8323 if( pNear->pColset ){
8324 int *aiCol = pNear->pColset->aiCol;
8325 int nCol = pNear->pColset->nCol;
8326 if( nCol==1 ){
8327 zRet = fts5PrintfAppend(zRet, "-col %d ", aiCol[0]);
8328 }else{
8329 zRet = fts5PrintfAppend(zRet, "-col {%d", aiCol[0]);
8330 for(i=1; i<pNear->pColset->nCol; i++){
8331 zRet = fts5PrintfAppend(zRet, " %d", aiCol[i]);
8332 }
8333 zRet = fts5PrintfAppend(zRet, "} ");
8334 }
8335 if( zRet==0 ) return 0;
8336 }
8337
8338 if( pNear->nPhrase>1 ){
8339 zRet = fts5PrintfAppend(zRet, "-near %d ", pNear->nNear);
8340 if( zRet==0 ) return 0;
8341 }
8342
8343 zRet = fts5PrintfAppend(zRet, "--");
8344 if( zRet==0 ) return 0;
8345
8346 for(i=0; i<pNear->nPhrase; i++){
8347 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
8348
8349 zRet = fts5PrintfAppend(zRet, " {");
8350 for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
8351 Fts5ExprTerm *p = &pPhrase->aTerm[iTerm];
8352 zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ",
8353 p->nQueryTerm, p->pTerm
8354 );
8355 if( pPhrase->aTerm[iTerm].bPrefix ){
8356 zRet = fts5PrintfAppend(zRet, "*");
8357 }
8358 }
8359
8360 if( zRet ) zRet = fts5PrintfAppend(zRet, "}");
8361 if( zRet==0 ) return 0;
8362 }
8363
8364 }else if( pExpr->eType==0 ){
8365 zRet = sqlite3_mprintfsqlite3_api->mprintf("{}");
8366 }else{
8367 char const *zOp = 0;
8368 int i;
8369 switch( pExpr->eType ){
8370 case FTS5_AND2: zOp = "AND"; break;
8371 case FTS5_NOT3: zOp = "NOT"; break;
8372 default:
8373 assert( pExpr->eType==FTS5_OR )((void) (0));
8374 zOp = "OR";
8375 break;
8376 }
8377
8378 zRet = sqlite3_mprintfsqlite3_api->mprintf("%s", zOp);
8379 for(i=0; zRet && i<pExpr->nChild; i++){
8380 char *z = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->apChild[i]);
8381 if( !z ){
8382 sqlite3_freesqlite3_api->free(zRet);
8383 zRet = 0;
8384 }else{
8385 zRet = fts5PrintfAppend(zRet, " [%z]", z);
8386 }
8387 }
8388 }
8389
8390 return zRet;
8391}
8392
8393static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){
8394 char *zRet = 0;
8395 if( pExpr->eType==0 ){
8396 return sqlite3_mprintfsqlite3_api->mprintf("\"\"");
8397 }else
8398 if( pExpr->eType==FTS5_STRING9 || pExpr->eType==FTS5_TERM4 ){
8399 Fts5ExprNearset *pNear = pExpr->pNear;
8400 int i;
8401 int iTerm;
8402
8403 if( pNear->pColset ){
8404 int ii;
8405 Fts5Colset *pColset = pNear->pColset;
8406 if( pColset->nCol>1 ) zRet = fts5PrintfAppend(zRet, "{");
8407 for(ii=0; ii<pColset->nCol; ii++){
8408 zRet = fts5PrintfAppend(zRet, "%s%s",
8409 pConfig->azCol[pColset->aiCol[ii]], ii==pColset->nCol-1 ? "" : " "
8410 );
8411 }
8412 if( zRet ){
8413 zRet = fts5PrintfAppend(zRet, "%s : ", pColset->nCol>1 ? "}" : "");
8414 }
8415 if( zRet==0 ) return 0;
8416 }
8417
8418 if( pNear->nPhrase>1 ){
8419 zRet = fts5PrintfAppend(zRet, "NEAR(");
8420 if( zRet==0 ) return 0;
8421 }
8422
8423 for(i=0; i<pNear->nPhrase; i++){
8424 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
8425 if( i!=0 ){
8426 zRet = fts5PrintfAppend(zRet, " ");
8427 if( zRet==0 ) return 0;
8428 }
8429 for(iTerm=0; iTerm<pPhrase->nTerm; iTerm++){
8430 char *zTerm = fts5ExprTermPrint(&pPhrase->aTerm[iTerm]);
8431 if( zTerm ){
8432 zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" + ", zTerm);
8433 sqlite3_freesqlite3_api->free(zTerm);
8434 }
8435 if( zTerm==0 || zRet==0 ){
8436 sqlite3_freesqlite3_api->free(zRet);
8437 return 0;
8438 }
8439 }
8440 }
8441
8442 if( pNear->nPhrase>1 ){
8443 zRet = fts5PrintfAppend(zRet, ", %d)", pNear->nNear);
8444 if( zRet==0 ) return 0;
8445 }
8446
8447 }else{
8448 char const *zOp = 0;
8449 int i;
8450
8451 switch( pExpr->eType ){
8452 case FTS5_AND2: zOp = " AND "; break;
8453 case FTS5_NOT3: zOp = " NOT "; break;
8454 default:
8455 assert( pExpr->eType==FTS5_OR )((void) (0));
8456 zOp = " OR ";
8457 break;
8458 }
8459
8460 for(i=0; i<pExpr->nChild; i++){
8461 char *z = fts5ExprPrint(pConfig, pExpr->apChild[i]);
8462 if( z==0 ){
8463 sqlite3_freesqlite3_api->free(zRet);
8464 zRet = 0;
8465 }else{
8466 int e = pExpr->apChild[i]->eType;
8467 int b = (e!=FTS5_STRING9 && e!=FTS5_TERM4 && e!=FTS5_EOF0);
8468 zRet = fts5PrintfAppend(zRet, "%s%s%z%s",
8469 (i==0 ? "" : zOp),
8470 (b?"(":""), z, (b?")":"")
8471 );
8472 }
8473 if( zRet==0 ) break;
8474 }
8475 }
8476
8477 return zRet;
8478}
8479
8480/*
8481** The implementation of user-defined scalar functions fts5_expr() (bTcl==0)
8482** and fts5_expr_tcl() (bTcl!=0).
8483*/
8484static void fts5ExprFunction(
8485 sqlite3_context *pCtx, /* Function call context */
8486 int nArg, /* Number of args */
8487 sqlite3_value **apVal, /* Function arguments */
8488 int bTcl
8489){
8490 Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_datasqlite3_api->user_data(pCtx);
8491 sqlite3 *db = sqlite3_context_db_handlesqlite3_api->context_db_handle(pCtx);
8492 const char *zExpr = 0;
8493 char *zErr = 0;
8494 Fts5Expr *pExpr = 0;
8495 int rc;
8496 int i;
8497
8498 const char **azConfig; /* Array of arguments for Fts5Config */
8499 const char *zNearsetCmd = "nearset";
8500 int nConfig; /* Size of azConfig[] */
8501 Fts5Config *pConfig = 0;
8502 int iArg = 1;
8503
8504 if( nArg<1 ){
8505 zErr = sqlite3_mprintfsqlite3_api->mprintf("wrong number of arguments to function %s",
8506 bTcl ? "fts5_expr_tcl" : "fts5_expr"
8507 );
8508 sqlite3_result_errorsqlite3_api->result_error(pCtx, zErr, -1);
8509 sqlite3_freesqlite3_api->free(zErr);
8510 return;
8511 }
8512
8513 if( bTcl && nArg>1 ){
8514 zNearsetCmd = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[1]);
8515 iArg = 2;
8516 }
8517
8518 nConfig = 3 + (nArg-iArg);
8519 azConfig = (const char**)sqlite3_malloc64sqlite3_api->malloc64(sizeof(char*) * nConfig);
8520 if( azConfig==0 ){
8521 sqlite3_result_error_nomemsqlite3_api->result_error_nomem(pCtx);
8522 return;
8523 }
8524 azConfig[0] = 0;
8525 azConfig[1] = "main";
8526 azConfig[2] = "tbl";
8527 for(i=3; iArg<nArg; iArg++){
8528 const char *z = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[iArg]);
8529 azConfig[i++] = (z ? z : "");
8530 }
8531
8532 zExpr = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[0]);
8533 if( zExpr==0 ) zExpr = "";
8534
8535 rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
8536 if( rc==SQLITE_OK0 ){
8537 rc = sqlite3Fts5ExprNew(pConfig, 0, pConfig->nCol, zExpr, &pExpr, &zErr);
8538 }
8539 if( rc==SQLITE_OK0 ){
8540 char *zText;
8541 if( pExpr->pRoot->xNext==0 ){
8542 zText = sqlite3_mprintfsqlite3_api->mprintf("");
8543 }else if( bTcl ){
8544 zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
8545 }else{
8546 zText = fts5ExprPrint(pConfig, pExpr->pRoot);
8547 }
8548 if( zText==0 ){
8549 rc = SQLITE_NOMEM7;
8550 }else{
8551 sqlite3_result_textsqlite3_api->result_text(pCtx, zText, -1, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
8552 sqlite3_freesqlite3_api->free(zText);
8553 }
8554 }
8555
8556 if( rc!=SQLITE_OK0 ){
8557 if( zErr ){
8558 sqlite3_result_errorsqlite3_api->result_error(pCtx, zErr, -1);
8559 sqlite3_freesqlite3_api->free(zErr);
8560 }else{
8561 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
8562 }
8563 }
8564 sqlite3_freesqlite3_api->free((void *)azConfig);
8565 sqlite3Fts5ConfigFree(pConfig);
8566 sqlite3Fts5ExprFree(pExpr);
8567}
8568
8569static void fts5ExprFunctionHr(
8570 sqlite3_context *pCtx, /* Function call context */
8571 int nArg, /* Number of args */
8572 sqlite3_value **apVal /* Function arguments */
8573){
8574 fts5ExprFunction(pCtx, nArg, apVal, 0);
8575}
8576static void fts5ExprFunctionTcl(
8577 sqlite3_context *pCtx, /* Function call context */
8578 int nArg, /* Number of args */
8579 sqlite3_value **apVal /* Function arguments */
8580){
8581 fts5ExprFunction(pCtx, nArg, apVal, 1);
8582}
8583
8584/*
8585** The implementation of an SQLite user-defined-function that accepts a
8586** single integer as an argument. If the integer is an alpha-numeric
8587** unicode code point, 1 is returned. Otherwise 0.
8588*/
8589static void fts5ExprIsAlnum(
8590 sqlite3_context *pCtx, /* Function call context */
8591 int nArg, /* Number of args */
8592 sqlite3_value **apVal /* Function arguments */
8593){
8594 int iCode;
8595 u8 aArr[32];
8596 if( nArg!=1 ){
8597 sqlite3_result_errorsqlite3_api->result_error(pCtx,
8598 "wrong number of arguments to function fts5_isalnum", -1
8599 );
8600 return;
8601 }
8602 memset(aArr, 0, sizeof(aArr));
8603 sqlite3Fts5UnicodeCatParse("L*", aArr);
8604 sqlite3Fts5UnicodeCatParse("N*", aArr);
8605 sqlite3Fts5UnicodeCatParse("Co", aArr);
8606 iCode = sqlite3_value_intsqlite3_api->value_int(apVal[0]);
8607 sqlite3_result_intsqlite3_api->result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory((u32)iCode)]);
8608}
8609
8610static void fts5ExprFold(
8611 sqlite3_context *pCtx, /* Function call context */
8612 int nArg, /* Number of args */
8613 sqlite3_value **apVal /* Function arguments */
8614){
8615 if( nArg!=1 && nArg!=2 ){
8616 sqlite3_result_errorsqlite3_api->result_error(pCtx,
8617 "wrong number of arguments to function fts5_fold", -1
8618 );
8619 }else{
8620 int iCode;
8621 int bRemoveDiacritics = 0;
8622 iCode = sqlite3_value_intsqlite3_api->value_int(apVal[0]);
8623 if( nArg==2 ) bRemoveDiacritics = sqlite3_value_intsqlite3_api->value_int(apVal[1]);
8624 sqlite3_result_intsqlite3_api->result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics));
8625 }
8626}
8627#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */
8628
8629/*
8630** This is called during initialization to register the fts5_expr() scalar
8631** UDF with the SQLite handle passed as the only argument.
8632*/
8633static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
8634#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
8635 struct Fts5ExprFunc {
8636 const char *z;
8637 void (*x)(sqlite3_context*,int,sqlite3_value**);
8638 } aFunc[] = {
8639 { "fts5_expr", fts5ExprFunctionHr },
8640 { "fts5_expr_tcl", fts5ExprFunctionTcl },
8641 { "fts5_isalnum", fts5ExprIsAlnum },
8642 { "fts5_fold", fts5ExprFold },
8643 };
8644 int i;
8645 int rc = SQLITE_OK0;
8646 void *pCtx = (void*)pGlobal;
8647
8648 for(i=0; rc==SQLITE_OK0 && i<ArraySize(aFunc)((int)(sizeof(aFunc) / sizeof(aFunc[0]))); i++){
8649 struct Fts5ExprFunc *p = &aFunc[i];
8650 rc = sqlite3_create_functionsqlite3_api->create_function(db, p->z, -1, SQLITE_UTF81, pCtx, p->x, 0, 0);
8651 }
8652#else
8653 int rc = SQLITE_OK0;
8654 UNUSED_PARAM2(pGlobal,db)(void)(pGlobal), (void)(db);
8655#endif
8656
8657 /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and
8658 ** sqlite3Fts5ParserFallback() are unused */
8659#ifndef NDEBUG1
8660 (void)sqlite3Fts5ParserTrace;
8661#endif
8662 (void)sqlite3Fts5ParserFallback;
8663
8664 return rc;
8665}
8666
8667/*
8668** Return the number of phrases in expression pExpr.
8669*/
8670static int sqlite3Fts5ExprPhraseCount(Fts5Expr *pExpr){
8671 return (pExpr ? pExpr->nPhrase : 0);
8672}
8673
8674/*
8675** Return the number of terms in the iPhrase'th phrase in pExpr.
8676*/
8677static int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){
8678 if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0;
8679 return pExpr->apExprPhrase[iPhrase]->nTerm;
8680}
8681
8682/*
8683** This function is used to access the current position list for phrase
8684** iPhrase.
8685*/
8686static int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){
8687 int nRet;
8688 Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
8689 Fts5ExprNode *pNode = pPhrase->pNode;
8690 if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){
8691 *pa = pPhrase->poslist.p;
8692 nRet = pPhrase->poslist.n;
8693 }else{
8694 *pa = 0;
8695 nRet = 0;
8696 }
8697 return nRet;
8698}
8699
8700struct Fts5PoslistPopulator {
8701 Fts5PoslistWriter writer;
8702 int bOk; /* True if ok to populate */
8703 int bMiss;
8704};
8705
8706/*
8707** Clear the position lists associated with all phrases in the expression
8708** passed as the first argument. Argument bLive is true if the expression
8709** might be pointing to a real entry, otherwise it has just been reset.
8710**
8711** At present this function is only used for detail=col and detail=none
8712** fts5 tables. This implies that all phrases must be at most 1 token
8713** in size, as phrase matches are not supported without detail=full.
8714*/
8715static Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){
8716 Fts5PoslistPopulator *pRet;
8717 pRet = sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
8718 if( pRet ){
8719 int i;
8720 memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
8721 for(i=0; i<pExpr->nPhrase; i++){
8722 Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
8723 Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
8724 assert( pExpr->apExprPhrase[i]->nTerm<=1 )((void) (0));
8725 if( bLive &&
8726 (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof)
8727 ){
8728 pRet[i].bMiss = 1;
8729 }else{
8730 pBuf->n = 0;
8731 }
8732 }
8733 }
8734 return pRet;
8735}
8736
8737struct Fts5ExprCtx {
8738 Fts5Expr *pExpr;
8739 Fts5PoslistPopulator *aPopulator;
8740 i64 iOff;
8741};
8742typedef struct Fts5ExprCtx Fts5ExprCtx;
8743
8744/*
8745** TODO: Make this more efficient!
8746*/
8747static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
8748 int i;
8749 for(i=0; i<pColset->nCol; i++){
8750 if( pColset->aiCol[i]==iCol ) return 1;
8751 }
8752 return 0;
8753}
8754
8755/*
8756** pToken is a buffer nToken bytes in size that may or may not contain
8757** an embedded 0x00 byte. If it does, return the number of bytes in
8758** the buffer before the 0x00. If it does not, return nToken.
8759*/
8760static int fts5QueryTerm(const char *pToken, int nToken){
8761 int ii;
8762 for(ii=0; ii<nToken && pToken[ii]; ii++){}
8763 return ii;
8764}
8765
8766static int fts5ExprPopulatePoslistsCb(
8767 void *pCtx, /* Copy of 2nd argument to xTokenize() */
8768 int tflags, /* Mask of FTS5_TOKEN_* flags */
8769 const char *pToken, /* Pointer to buffer containing token */
8770 int nToken, /* Size of token in bytes */
8771 int iUnused1, /* Byte offset of token within input text */
8772 int iUnused2 /* Byte offset of end of token within input text */
8773){
8774 Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
8775 Fts5Expr *pExpr = p->pExpr;
8776 int i;
8777 int nQuery = nToken;
8778 i64 iRowid = pExpr->pRoot->iRowid;
8779
8780 UNUSED_PARAM2(iUnused1, iUnused2)(void)(iUnused1), (void)(iUnused2);
8781
8782 if( nQuery>FTS5_MAX_TOKEN_SIZE32768 ) nQuery = FTS5_MAX_TOKEN_SIZE32768;
8783 if( pExpr->pConfig->bTokendata ){
8784 nQuery = fts5QueryTerm(pToken, nQuery);
8785 }
8786 if( (tflags & FTS5_TOKEN_COLOCATED0x0001)==0 ) p->iOff++;
8787 for(i=0; i<pExpr->nPhrase; i++){
8788 Fts5ExprTerm *pT;
8789 if( p->aPopulator[i].bOk==0 ) continue;
8790 for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
8791 if( (pT->nQueryTerm==nQuery || (pT->nQueryTerm<nQuery && pT->bPrefix))
8792 && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0
8793 ){
8794 int rc = sqlite3Fts5PoslistWriterAppend(
8795 &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
8796 );
8797 if( rc==SQLITE_OK0 && (pExpr->pConfig->bTokendata || pT->bPrefix) ){
8798 int iCol = p->iOff>>32;
8799 int iTokOff = p->iOff & 0x7FFFFFFF;
8800 rc = sqlite3Fts5IndexIterWriteTokendata(
8801 pT->pIter, pToken, nToken, iRowid, iCol, iTokOff
8802 );
8803 }
8804 if( rc ) return rc;
8805 break;
8806 }
8807 }
8808 }
8809 return SQLITE_OK0;
8810}
8811
8812static int sqlite3Fts5ExprPopulatePoslists(
8813 Fts5Config *pConfig,
8814 Fts5Expr *pExpr,
8815 Fts5PoslistPopulator *aPopulator,
8816 int iCol,
8817 const char *z, int n
8818){
8819 int i;
8820 Fts5ExprCtx sCtx;
8821 sCtx.pExpr = pExpr;
8822 sCtx.aPopulator = aPopulator;
8823 sCtx.iOff = (((i64)iCol) << 32) - 1;
8824
8825 for(i=0; i<pExpr->nPhrase; i++){
8826 Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
8827 Fts5Colset *pColset = pNode->pNear->pColset;
8828 if( (pColset && 0==fts5ExprColsetTest(pColset, iCol))
8829 || aPopulator[i].bMiss
8830 ){
8831 aPopulator[i].bOk = 0;
8832 }else{
8833 aPopulator[i].bOk = 1;
8834 }
8835 }
8836
8837 return sqlite3Fts5Tokenize(pConfig,
8838 FTS5_TOKENIZE_DOCUMENT0x0004, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
8839 );
8840}
8841
8842static void fts5ExprClearPoslists(Fts5ExprNode *pNode){
8843 if( pNode->eType==FTS5_TERM4 || pNode->eType==FTS5_STRING9 ){
8844 pNode->pNear->apPhrase[0]->poslist.n = 0;
8845 }else{
8846 int i;
8847 for(i=0; i<pNode->nChild; i++){
8848 fts5ExprClearPoslists(pNode->apChild[i]);
8849 }
8850 }
8851}
8852
8853static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
8854 pNode->iRowid = iRowid;
8855 pNode->bEof = 0;
8856 switch( pNode->eType ){
8857 case 0:
8858 case FTS5_TERM4:
8859 case FTS5_STRING9:
8860 return (pNode->pNear->apPhrase[0]->poslist.n>0);
8861
8862 case FTS5_AND2: {
8863 int i;
8864 for(i=0; i<pNode->nChild; i++){
8865 if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
8866 fts5ExprClearPoslists(pNode);
8867 return 0;
8868 }
8869 }
8870 break;
8871 }
8872
8873 case FTS5_OR1: {
8874 int i;
8875 int bRet = 0;
8876 for(i=0; i<pNode->nChild; i++){
8877 if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
8878 bRet = 1;
8879 }
8880 }
8881 return bRet;
8882 }
8883
8884 default: {
8885 assert( pNode->eType==FTS5_NOT )((void) (0));
8886 if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
8887 || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
8888 ){
8889 fts5ExprClearPoslists(pNode);
8890 return 0;
8891 }
8892 break;
8893 }
8894 }
8895 return 1;
8896}
8897
8898static void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
8899 fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
8900}
8901
8902/*
8903** This function is only called for detail=columns tables.
8904*/
8905static int sqlite3Fts5ExprPhraseCollist(
8906 Fts5Expr *pExpr,
8907 int iPhrase,
8908 const u8 **ppCollist,
8909 int *pnCollist
8910){
8911 Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
8912 Fts5ExprNode *pNode = pPhrase->pNode;
8913 int rc = SQLITE_OK0;
8914
8915 assert( iPhrase>=0 && iPhrase<pExpr->nPhrase )((void) (0));
8916 assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS )((void) (0));
8917
8918 if( pNode->bEof==0
8919 && pNode->iRowid==pExpr->pRoot->iRowid
8920 && pPhrase->poslist.n>0
8921 ){
8922 Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
8923 if( pTerm->pSynonym ){
8924 Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1];
8925 rc = fts5ExprSynonymList(
8926 pTerm, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist
8927 );
8928 }else{
8929 *ppCollist = pPhrase->aTerm[0].pIter->pData;
8930 *pnCollist = pPhrase->aTerm[0].pIter->nData;
8931 }
8932 }else{
8933 *ppCollist = 0;
8934 *pnCollist = 0;
8935 }
8936
8937 return rc;
8938}
8939
8940/*
8941** Does the work of the fts5_api.xQueryToken() API method.
8942*/
8943static int sqlite3Fts5ExprQueryToken(
8944 Fts5Expr *pExpr,
8945 int iPhrase,
8946 int iToken,
8947 const char **ppOut,
8948 int *pnOut
8949){
8950 Fts5ExprPhrase *pPhrase = 0;
8951
8952 if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
8953 return SQLITE_RANGE25;
8954 }
8955 pPhrase = pExpr->apExprPhrase[iPhrase];
8956 if( iToken<0 || iToken>=pPhrase->nTerm ){
8957 return SQLITE_RANGE25;
8958 }
8959
8960 *ppOut = pPhrase->aTerm[iToken].pTerm;
8961 *pnOut = pPhrase->aTerm[iToken].nFullTerm;
8962 return SQLITE_OK0;
8963}
8964
8965/*
8966** Does the work of the fts5_api.xInstToken() API method.
8967*/
8968static int sqlite3Fts5ExprInstToken(
8969 Fts5Expr *pExpr,
8970 i64 iRowid,
8971 int iPhrase,
8972 int iCol,
8973 int iOff,
8974 int iToken,
8975 const char **ppOut,
8976 int *pnOut
8977){
8978 Fts5ExprPhrase *pPhrase = 0;
8979 Fts5ExprTerm *pTerm = 0;
8980 int rc = SQLITE_OK0;
8981
8982 if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
8983 return SQLITE_RANGE25;
8984 }
8985 pPhrase = pExpr->apExprPhrase[iPhrase];
8986 if( iToken<0 || iToken>=pPhrase->nTerm ){
8987 return SQLITE_RANGE25;
8988 }
8989 pTerm = &pPhrase->aTerm[iToken];
8990 if( pExpr->pConfig->bTokendata || pTerm->bPrefix ){
8991 rc = sqlite3Fts5IterToken(
8992 pTerm->pIter, pTerm->pTerm, pTerm->nQueryTerm,
8993 iRowid, iCol, iOff+iToken, ppOut, pnOut
8994 );
8995 }else{
8996 *ppOut = pTerm->pTerm;
8997 *pnOut = pTerm->nFullTerm;
8998 }
8999 return rc;
9000}
9001
9002/*
9003** Clear the token mappings for all Fts5IndexIter objects managed by
9004** the expression passed as the only argument.
9005*/
9006static void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){
9007 int ii;
9008 for(ii=0; ii<pExpr->nPhrase; ii++){
9009 Fts5ExprTerm *pT;
9010 for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){
9011 sqlite3Fts5IndexIterClearTokendata(pT->pIter);
9012 }
9013 }
9014}
9015
9016#line 1 "fts5_hash.c"
9017/*
9018** 2014 August 11
9019**
9020** The author disclaims copyright to this source code. In place of
9021** a legal notice, here is a blessing:
9022**
9023** May you do good and not evil.
9024** May you find forgiveness for yourself and forgive others.
9025** May you share freely, never taking more than you give.
9026**
9027******************************************************************************
9028**
9029*/
9030
9031
9032
9033/* #include "fts5Int.h" */
9034
9035typedef struct Fts5HashEntry Fts5HashEntry;
9036
9037/*
9038** This file contains the implementation of an in-memory hash table used
9039** to accumulate "term -> doclist" content before it is flushed to a level-0
9040** segment.
9041*/
9042
9043
9044struct Fts5Hash {
9045 int eDetail; /* Copy of Fts5Config.eDetail */
9046 int *pnByte; /* Pointer to bytes counter */
9047 int nEntry; /* Number of entries currently in hash */
9048 int nSlot; /* Size of aSlot[] array */
9049 Fts5HashEntry *pScan; /* Current ordered scan item */
9050 Fts5HashEntry **aSlot; /* Array of hash slots */
9051};
9052
9053/*
9054** Each entry in the hash table is represented by an object of the
9055** following type. Each object, its key, and its current data are stored
9056** in a single memory allocation. The key immediately follows the object
9057** in memory. The position list data immediately follows the key data
9058** in memory.
9059**
9060** The key is Fts5HashEntry.nKey bytes in size. It consists of a single
9061** byte identifying the index (either the main term index or a prefix-index),
9062** followed by the term data. For example: "0token". There is no
9063** nul-terminator - in this case nKey=6.
9064**
9065** The data that follows the key is in a similar, but not identical format
9066** to the doclist data stored in the database. It is:
9067**
9068** * Rowid, as a varint
9069** * Position list, without 0x00 terminator.
9070** * Size of previous position list and rowid, as a 4 byte
9071** big-endian integer.
9072**
9073** iRowidOff:
9074** Offset of last rowid written to data area. Relative to first byte of
9075** structure.
9076**
9077** nData:
9078** Bytes of data written since iRowidOff.
9079*/
9080struct Fts5HashEntry {
9081 Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */
9082 Fts5HashEntry *pScanNext; /* Next entry in sorted order */
9083
9084 int nAlloc; /* Total size of allocation */
9085 int iSzPoslist; /* Offset of space for 4-byte poslist size */
9086 int nData; /* Total bytes of data (incl. structure) */
9087 int nKey; /* Length of key in bytes */
9088 u8 bDel; /* Set delete-flag @ iSzPoslist */
9089 u8 bContent; /* Set content-flag (detail=none mode) */
9090 i16 iCol; /* Column of last value written */
9091 int iPos; /* Position of last value written */
9092 i64 iRowid; /* Rowid of last value written */
9093};
9094
9095/*
9096** Equivalent to:
9097**
9098** char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; }
9099*/
9100#define fts5EntryKey(p)( ((char *)(&(p)[1])) ) ( ((char *)(&(p)[1])) )
9101
9102
9103/*
9104** Allocate a new hash table.
9105*/
9106static int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){
9107 int rc = SQLITE_OK0;
9108 Fts5Hash *pNew;
9109
9110 *ppNew = pNew = (Fts5Hash*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5Hash));
9111 if( pNew==0 ){
9112 rc = SQLITE_NOMEM7;
9113 }else{
9114 sqlite3_int64 nByte;
9115 memset(pNew, 0, sizeof(Fts5Hash));
9116 pNew->pnByte = pnByte;
9117 pNew->eDetail = pConfig->eDetail;
9118
9119 pNew->nSlot = 1024;
9120 nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
9121 pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc64sqlite3_api->malloc64(nByte);
9122 if( pNew->aSlot==0 ){
9123 sqlite3_freesqlite3_api->free(pNew);
9124 *ppNew = 0;
9125 rc = SQLITE_NOMEM7;
9126 }else{
9127 memset(pNew->aSlot, 0, (size_t)nByte);
9128 }
9129 }
9130 return rc;
9131}
9132
9133/*
9134** Free a hash table object.
9135*/
9136static void sqlite3Fts5HashFree(Fts5Hash *pHash){
9137 if( pHash ){
9138 sqlite3Fts5HashClear(pHash);
9139 sqlite3_freesqlite3_api->free(pHash->aSlot);
9140 sqlite3_freesqlite3_api->free(pHash);
9141 }
9142}
9143
9144/*
9145** Empty (but do not delete) a hash table.
9146*/
9147static void sqlite3Fts5HashClear(Fts5Hash *pHash){
9148 int i;
9149 for(i=0; i<pHash->nSlot; i++){
9150 Fts5HashEntry *pNext;
9151 Fts5HashEntry *pSlot;
9152 for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){
9153 pNext = pSlot->pHashNext;
9154 sqlite3_freesqlite3_api->free(pSlot);
9155 }
9156 }
9157 memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*));
9158 pHash->nEntry = 0;
9159}
9160
9161static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){
9162 int i;
9163 unsigned int h = 13;
9164 for(i=n-1; i>=0; i--){
9165 h = (h << 3) ^ h ^ p[i];
9166 }
9167 return (h % nSlot);
9168}
9169
9170static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){
9171 int i;
9172 unsigned int h = 13;
9173 for(i=n-1; i>=0; i--){
9174 h = (h << 3) ^ h ^ p[i];
9175 }
9176 h = (h << 3) ^ h ^ b;
9177 return (h % nSlot);
9178}
9179
9180/*
9181** Resize the hash table by doubling the number of slots.
9182*/
9183static int fts5HashResize(Fts5Hash *pHash){
9184 int nNew = pHash->nSlot*2;
9185 int i;
9186 Fts5HashEntry **apNew;
9187 Fts5HashEntry **apOld = pHash->aSlot;
9188
9189 apNew = (Fts5HashEntry**)sqlite3_malloc64sqlite3_api->malloc64(nNew*sizeof(Fts5HashEntry*));
9190 if( !apNew ) return SQLITE_NOMEM7;
9191 memset(apNew, 0, nNew*sizeof(Fts5HashEntry*));
9192
9193 for(i=0; i<pHash->nSlot; i++){
9194 while( apOld[i] ){
9195 unsigned int iHash;
9196 Fts5HashEntry *p = apOld[i];
9197 apOld[i] = p->pHashNext;
9198 iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p)( ((char *)(&(p)[1])) ), p->nKey);
9199 p->pHashNext = apNew[iHash];
9200 apNew[iHash] = p;
9201 }
9202 }
9203
9204 sqlite3_freesqlite3_api->free(apOld);
9205 pHash->nSlot = nNew;
9206 pHash->aSlot = apNew;
9207 return SQLITE_OK0;
9208}
9209
9210static int fts5HashAddPoslistSize(
9211 Fts5Hash *pHash,
9212 Fts5HashEntry *p,
9213 Fts5HashEntry *p2
9214){
9215 int nRet = 0;
9216 if( p->iSzPoslist ){
9217 u8 *pPtr = p2 ? (u8*)p2 : (u8*)p;
9218 int nData = p->nData;
9219 if( pHash->eDetail==FTS5_DETAIL_NONE1 ){
9220 assert( nData==p->iSzPoslist )((void) (0));
9221 if( p->bDel ){
9222 pPtr[nData++] = 0x00;
9223 if( p->bContent ){
9224 pPtr[nData++] = 0x00;
9225 }
9226 }
9227 }else{
9228 int nSz = (nData - p->iSzPoslist - 1); /* Size in bytes */
9229 int nPos = nSz*2 + p->bDel; /* Value of nPos field */
9230
9231 assert( p->bDel==0 || p->bDel==1 )((void) (0));
9232 if( nPos<=127 ){
9233 pPtr[p->iSzPoslist] = (u8)nPos;
9234 }else{
9235 int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
9236 memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
9237 sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
9238 nData += (nByte-1);
9239 }
9240 }
9241
9242 nRet = nData - p->nData;
9243 if( p2==0 ){
9244 p->iSzPoslist = 0;
9245 p->bDel = 0;
9246 p->bContent = 0;
9247 p->nData = nData;
9248 }
9249 }
9250 return nRet;
9251}
9252
9253/*
9254** Add an entry to the in-memory hash table. The key is the concatenation
9255** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos).
9256**
9257** (bByte || pToken) -> (iRowid,iCol,iPos)
9258**
9259** Or, if iCol is negative, then the value is a delete marker.
9260*/
9261static int sqlite3Fts5HashWrite(
9262 Fts5Hash *pHash,
9263 i64 iRowid, /* Rowid for this entry */
9264 int iCol, /* Column token appears in (-ve -> delete) */
9265 int iPos, /* Position of token within column */
9266 char bByte, /* First byte of token */
9267 const char *pToken, int nToken /* Token to add or remove to or from index */
9268){
9269 unsigned int iHash;
9270 Fts5HashEntry *p;
9271 u8 *pPtr;
9272 int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */
9273 int bNew; /* If non-delete entry should be written */
9274
9275 bNew = (pHash->eDetail==FTS5_DETAIL_FULL0);
9276
9277 /* Attempt to locate an existing hash entry */
9278 iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
9279 for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
9280 char *zKey = fts5EntryKey(p)( ((char *)(&(p)[1])) );
9281 if( zKey[0]==bByte
9282 && p->nKey==nToken+1
9283 && memcmp(&zKey[1], pToken, nToken)==0
9284 ){
9285 break;
9286 }
9287 }
9288
9289 /* If an existing hash entry cannot be found, create a new one. */
9290 if( p==0 ){
9291 /* Figure out how much space to allocate */
9292 char *zKey;
9293 sqlite3_int64 nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64;
9294 if( nByte<128 ) nByte = 128;
9295
9296 /* Grow the Fts5Hash.aSlot[] array if necessary. */
9297 if( (pHash->nEntry*2)>=pHash->nSlot ){
9298 int rc = fts5HashResize(pHash);
9299 if( rc!=SQLITE_OK0 ) return rc;
9300 iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
9301 }
9302
9303 /* Allocate new Fts5HashEntry and add it to the hash table. */
9304 p = (Fts5HashEntry*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
9305 if( !p ) return SQLITE_NOMEM7;
9306 memset(p, 0, sizeof(Fts5HashEntry));
9307 p->nAlloc = (int)nByte;
9308 zKey = fts5EntryKey(p)( ((char *)(&(p)[1])) );
9309 zKey[0] = bByte;
9310 memcpy(&zKey[1], pToken, nToken);
9311 assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) )((void) (0));
9312 p->nKey = nToken+1;
9313 zKey[nToken+1] = '\0';
9314 p->nData = nToken+1 + sizeof(Fts5HashEntry);
9315 p->pHashNext = pHash->aSlot[iHash];
9316 pHash->aSlot[iHash] = p;
9317 pHash->nEntry++;
9318
9319 /* Add the first rowid field to the hash-entry */
9320 p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
9321 p->iRowid = iRowid;
9322
9323 p->iSzPoslist = p->nData;
9324 if( pHash->eDetail!=FTS5_DETAIL_NONE1 ){
9325 p->nData += 1;
9326 p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL0 ? 0 : -1);
9327 }
9328
9329 }else{
9330
9331 /* Appending to an existing hash-entry. Check that there is enough
9332 ** space to append the largest possible new entry. Worst case scenario
9333 ** is:
9334 **
9335 ** + 9 bytes for a new rowid,
9336 ** + 4 byte reserved for the "poslist size" varint.
9337 ** + 1 byte for a "new column" byte,
9338 ** + 3 bytes for a new column number (16-bit max) as a varint,
9339 ** + 5 bytes for the new position offset (32-bit max).
9340 */
9341 if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
9342 sqlite3_int64 nNew = p->nAlloc * 2;
9343 Fts5HashEntry *pNew;
9344 Fts5HashEntry **pp;
9345 pNew = (Fts5HashEntry*)sqlite3_realloc64sqlite3_api->realloc64(p, nNew);
9346 if( pNew==0 ) return SQLITE_NOMEM7;
9347 pNew->nAlloc = (int)nNew;
9348 for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
9349 *pp = pNew;
9350 p = pNew;
9351 }
9352 nIncr -= p->nData;
9353 }
9354 assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) )((void) (0));
9355
9356 pPtr = (u8*)p;
9357
9358 /* If this is a new rowid, append the 4-byte size field for the previous
9359 ** entry, and the new rowid for this entry. */
9360 if( iRowid!=p->iRowid ){
9361 u64 iDiff = (u64)iRowid - (u64)p->iRowid;
9362 fts5HashAddPoslistSize(pHash, p, 0);
9363 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iDiff);
9364 p->iRowid = iRowid;
9365 bNew = 1;
9366 p->iSzPoslist = p->nData;
9367 if( pHash->eDetail!=FTS5_DETAIL_NONE1 ){
9368 p->nData += 1;
9369 p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL0 ? 0 : -1);
9370 p->iPos = 0;
9371 }
9372 }
9373
9374 if( iCol>=0 ){
9375 if( pHash->eDetail==FTS5_DETAIL_NONE1 ){
9376 p->bContent = 1;
9377 }else{
9378 /* Append a new column value, if necessary */
9379 assert_nc( iCol>=p->iCol )((void) (0));
9380 if( iCol!=p->iCol ){
9381 if( pHash->eDetail==FTS5_DETAIL_FULL0 ){
9382 pPtr[p->nData++] = 0x01;
9383 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
9384 p->iCol = (i16)iCol;
9385 p->iPos = 0;
9386 }else{
9387 bNew = 1;
9388 p->iCol = (i16)(iPos = iCol);
9389 }
9390 }
9391
9392 /* Append the new position offset, if necessary */
9393 if( bNew ){
9394 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
9395 p->iPos = iPos;
9396 }
9397 }
9398 }else{
9399 /* This is a delete. Set the delete flag. */
9400 p->bDel = 1;
9401 }
9402
9403 nIncr += p->nData;
9404 *pHash->pnByte += nIncr;
9405 return SQLITE_OK0;
9406}
9407
9408
9409/*
9410** Arguments pLeft and pRight point to linked-lists of hash-entry objects,
9411** each sorted in key order. This function merges the two lists into a
9412** single list and returns a pointer to its first element.
9413*/
9414static Fts5HashEntry *fts5HashEntryMerge(
9415 Fts5HashEntry *pLeft,
9416 Fts5HashEntry *pRight
9417){
9418 Fts5HashEntry *p1 = pLeft;
9419 Fts5HashEntry *p2 = pRight;
9420 Fts5HashEntry *pRet = 0;
9421 Fts5HashEntry **ppOut = &pRet;
9422
9423 while( p1 || p2 ){
9424 if( p1==0 ){
9425 *ppOut = p2;
9426 p2 = 0;
9427 }else if( p2==0 ){
9428 *ppOut = p1;
9429 p1 = 0;
9430 }else{
9431 char *zKey1 = fts5EntryKey(p1)( ((char *)(&(p1)[1])) );
9432 char *zKey2 = fts5EntryKey(p2)( ((char *)(&(p2)[1])) );
9433 int nMin = MIN(p1->nKey, p2->nKey)(((p1->nKey) < (p2->nKey)) ? (p1->nKey) : (p2->
nKey));
9434
9435 int cmp = memcmp(zKey1, zKey2, nMin);
9436 if( cmp==0 ){
9437 cmp = p1->nKey - p2->nKey;
9438 }
9439 assert( cmp!=0 )((void) (0));
9440
9441 if( cmp>0 ){
9442 /* p2 is smaller */
9443 *ppOut = p2;
9444 ppOut = &p2->pScanNext;
9445 p2 = p2->pScanNext;
9446 }else{
9447 /* p1 is smaller */
9448 *ppOut = p1;
9449 ppOut = &p1->pScanNext;
9450 p1 = p1->pScanNext;
9451 }
9452 *ppOut = 0;
9453 }
9454 }
9455
9456 return pRet;
9457}
9458
9459/*
9460** Link all tokens from hash table iHash into a list in sorted order. The
9461** tokens are not removed from the hash table.
9462*/
9463static int fts5HashEntrySort(
9464 Fts5Hash *pHash,
9465 const char *pTerm, int nTerm, /* Query prefix, if any */
9466 Fts5HashEntry **ppSorted
9467){
9468 const int nMergeSlot = 32;
9469 Fts5HashEntry **ap;
9470 Fts5HashEntry *pList;
9471 int iSlot;
9472 int i;
9473
9474 *ppSorted = 0;
9475 ap = sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5HashEntry*) * nMergeSlot);
9476 if( !ap ) return SQLITE_NOMEM7;
9477 memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot);
9478
9479 for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
9480 Fts5HashEntry *pIter;
9481 for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
9482 if( pTerm==0
9483 || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter)( ((char *)(&(pIter)[1])) ), pTerm, nTerm))
9484 ){
9485 Fts5HashEntry *pEntry = pIter;
9486 pEntry->pScanNext = 0;
9487 for(i=0; ap[i]; i++){
9488 pEntry = fts5HashEntryMerge(pEntry, ap[i]);
9489 ap[i] = 0;
9490 }
9491 ap[i] = pEntry;
9492 }
9493 }
9494 }
9495
9496 pList = 0;
9497 for(i=0; i<nMergeSlot; i++){
9498 pList = fts5HashEntryMerge(pList, ap[i]);
9499 }
9500
9501 sqlite3_freesqlite3_api->free(ap);
9502 *ppSorted = pList;
9503 return SQLITE_OK0;
9504}
9505
9506/*
9507** Query the hash table for a doclist associated with term pTerm/nTerm.
9508*/
9509static int sqlite3Fts5HashQuery(
9510 Fts5Hash *pHash, /* Hash table to query */
9511 int nPre,
9512 const char *pTerm, int nTerm, /* Query term */
9513 void **ppOut, /* OUT: Pointer to new object */
9514 int *pnDoclist /* OUT: Size of doclist in bytes */
9515){
9516 unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm);
9517 char *zKey = 0;
9518 Fts5HashEntry *p;
9519
9520 for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
9521 zKey = fts5EntryKey(p)( ((char *)(&(p)[1])) );
9522 if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break;
9523 }
9524
9525 if( p ){
9526 int nHashPre = sizeof(Fts5HashEntry) + nTerm;
9527 int nList = p->nData - nHashPre;
9528 u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64sqlite3_api->malloc64(nPre + nList + 10));
9529 if( pRet ){
9530 Fts5HashEntry *pFaux = (Fts5HashEntry*)&pRet[nPre-nHashPre];
9531 memcpy(&pRet[nPre], &((u8*)p)[nHashPre], nList);
9532 nList += fts5HashAddPoslistSize(pHash, p, pFaux);
9533 *pnDoclist = nList;
9534 }else{
9535 *pnDoclist = 0;
9536 return SQLITE_NOMEM7;
9537 }
9538 }else{
9539 *ppOut = 0;
9540 *pnDoclist = 0;
9541 }
9542
9543 return SQLITE_OK0;
9544}
9545
9546static int sqlite3Fts5HashScanInit(
9547 Fts5Hash *p, /* Hash table to query */
9548 const char *pTerm, int nTerm /* Query prefix */
9549){
9550 return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
9551}
9552
9553#ifdef SQLITE_DEBUG
9554static int fts5HashCount(Fts5Hash *pHash){
9555 int nEntry = 0;
9556 int ii;
9557 for(ii=0; ii<pHash->nSlot; ii++){
9558 Fts5HashEntry *p = 0;
9559 for(p=pHash->aSlot[ii]; p; p=p->pHashNext){
9560 nEntry++;
9561 }
9562 }
9563 return nEntry;
9564}
9565#endif
9566
9567/*
9568** Return true if the hash table is empty, false otherwise.
9569*/
9570static int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){
9571 assert( pHash->nEntry==fts5HashCount(pHash) )((void) (0));
9572 return pHash->nEntry==0;
9573}
9574
9575static void sqlite3Fts5HashScanNext(Fts5Hash *p){
9576 assert( !sqlite3Fts5HashScanEof(p) )((void) (0));
9577 p->pScan = p->pScan->pScanNext;
9578}
9579
9580static int sqlite3Fts5HashScanEof(Fts5Hash *p){
9581 return (p->pScan==0);
9582}
9583
9584static void sqlite3Fts5HashScanEntry(
9585 Fts5Hash *pHash,
9586 const char **pzTerm, /* OUT: term (nul-terminated) */
9587 int *pnTerm, /* OUT: Size of term in bytes */
9588 const u8 **ppDoclist, /* OUT: pointer to doclist */
9589 int *pnDoclist /* OUT: size of doclist in bytes */
9590){
9591 Fts5HashEntry *p;
9592 if( (p = pHash->pScan) ){
9593 char *zKey = fts5EntryKey(p)( ((char *)(&(p)[1])) );
9594 int nTerm = p->nKey;
9595 fts5HashAddPoslistSize(pHash, p, 0);
9596 *pzTerm = zKey;
9597 *pnTerm = nTerm;
9598 *ppDoclist = (const u8*)&zKey[nTerm];
9599 *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm);
9600 }else{
9601 *pzTerm = 0;
9602 *pnTerm = 0;
9603 *ppDoclist = 0;
9604 *pnDoclist = 0;
9605 }
9606}
9607
9608#line 1 "fts5_index.c"
9609/*
9610** 2014 May 31
9611**
9612** The author disclaims copyright to this source code. In place of
9613** a legal notice, here is a blessing:
9614**
9615** May you do good and not evil.
9616** May you find forgiveness for yourself and forgive others.
9617** May you share freely, never taking more than you give.
9618**
9619******************************************************************************
9620**
9621** Low level access to the FTS index stored in the database file. The
9622** routines in this file file implement all read and write access to the
9623** %_data table. Other parts of the system access this functionality via
9624** the interface defined in fts5Int.h.
9625*/
9626
9627
9628/* #include "fts5Int.h" */
9629
9630/*
9631** Overview:
9632**
9633** The %_data table contains all the FTS indexes for an FTS5 virtual table.
9634** As well as the main term index, there may be up to 31 prefix indexes.
9635** The format is similar to FTS3/4, except that:
9636**
9637** * all segment b-tree leaf data is stored in fixed size page records
9638** (e.g. 1000 bytes). A single doclist may span multiple pages. Care is
9639** taken to ensure it is possible to iterate in either direction through
9640** the entries in a doclist, or to seek to a specific entry within a
9641** doclist, without loading it into memory.
9642**
9643** * large doclists that span many pages have associated "doclist index"
9644** records that contain a copy of the first rowid on each page spanned by
9645** the doclist. This is used to speed up seek operations, and merges of
9646** large doclists with very small doclists.
9647**
9648** * extra fields in the "structure record" record the state of ongoing
9649** incremental merge operations.
9650**
9651*/
9652
9653
9654#define FTS5_OPT_WORK_UNIT1000 1000 /* Number of leaf pages per optimize step */
9655#define FTS5_WORK_UNIT64 64 /* Number of leaf pages in unit of work */
9656
9657#define FTS5_MIN_DLIDX_SIZE4 4 /* Add dlidx if this many empty pages */
9658
9659#define FTS5_MAIN_PREFIX'0' '0'
9660
9661#if FTS5_MAX_PREFIX_INDEXES31 > 31
9662# error "FTS5_MAX_PREFIX_INDEXES is too large"
9663#endif
9664
9665#define FTS5_MAX_LEVEL64 64
9666
9667/*
9668** There are two versions of the format used for the structure record:
9669**
9670** 1. the legacy format, that may be read by all fts5 versions, and
9671**
9672** 2. the V2 format, which is used by contentless_delete=1 databases.
9673**
9674** Both begin with a 4-byte "configuration cookie" value. Then, a legacy
9675** format structure record contains a varint - the number of levels in
9676** the structure. Whereas a V2 structure record contains the constant
9677** 4 bytes [0xff 0x00 0x00 0x01]. This is unambiguous as the value of a
9678** varint has to be at least 16256 to begin with "0xFF". And the default
9679** maximum number of levels is 64.
9680**
9681** See below for more on structure record formats.
9682*/
9683#define FTS5_STRUCTURE_V2"\xFF\x00\x00\x01" "\xFF\x00\x00\x01"
9684
9685/*
9686** Details:
9687**
9688** The %_data table managed by this module,
9689**
9690** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB);
9691**
9692** , contains the following 6 types of records. See the comments surrounding
9693** the FTS5_*_ROWID macros below for a description of how %_data rowids are
9694** assigned to each fo them.
9695**
9696** 1. Structure Records:
9697**
9698** The set of segments that make up an index - the index structure - are
9699** recorded in a single record within the %_data table. The record consists
9700** of a single 32-bit configuration cookie value followed by a list of
9701** SQLite varints.
9702**
9703** If the structure record is a V2 record, the configuration cookie is
9704** followed by the following 4 bytes: [0xFF 0x00 0x00 0x01].
9705**
9706** Next, the record continues with three varints:
9707**
9708** + number of levels,
9709** + total number of segments on all levels,
9710** + value of write counter.
9711**
9712** Then, for each level from 0 to nMax:
9713**
9714** + number of input segments in ongoing merge.
9715** + total number of segments in level.
9716** + for each segment from oldest to newest:
9717** + segment id (always > 0)
9718** + first leaf page number (often 1, always greater than 0)
9719** + final leaf page number
9720**
9721** Then, for V2 structures only:
9722**
9723** + lower origin counter value,
9724** + upper origin counter value,
9725** + the number of tombstone hash pages.
9726**
9727** 2. The Averages Record:
9728**
9729** A single record within the %_data table. The data is a list of varints.
9730** The first value is the number of rows in the index. Then, for each column
9731** from left to right, the total number of tokens in the column for all
9732** rows of the table.
9733**
9734** 3. Segment leaves:
9735**
9736** TERM/DOCLIST FORMAT:
9737**
9738** Most of each segment leaf is taken up by term/doclist data. The
9739** general format of term/doclist, starting with the first term
9740** on the leaf page, is:
9741**
9742** varint : size of first term
9743** blob: first term data
9744** doclist: first doclist
9745** zero-or-more {
9746** varint: number of bytes in common with previous term
9747** varint: number of bytes of new term data (nNew)
9748** blob: nNew bytes of new term data
9749** doclist: next doclist
9750** }
9751**
9752** doclist format:
9753**
9754** varint: first rowid
9755** poslist: first poslist
9756** zero-or-more {
9757** varint: rowid delta (always > 0)
9758** poslist: next poslist
9759** }
9760**
9761** poslist format:
9762**
9763** varint: size of poslist in bytes multiplied by 2, not including
9764** this field. Plus 1 if this entry carries the "delete" flag.
9765** collist: collist for column 0
9766** zero-or-more {
9767** 0x01 byte
9768** varint: column number (I)
9769** collist: collist for column I
9770** }
9771**
9772** collist format:
9773**
9774** varint: first offset + 2
9775** zero-or-more {
9776** varint: offset delta + 2
9777** }
9778**
9779** PAGE FORMAT
9780**
9781** Each leaf page begins with a 4-byte header containing 2 16-bit
9782** unsigned integer fields in big-endian format. They are:
9783**
9784** * The byte offset of the first rowid on the page, if it exists
9785** and occurs before the first term (otherwise 0).
9786**
9787** * The byte offset of the start of the page footer. If the page
9788** footer is 0 bytes in size, then this field is the same as the
9789** size of the leaf page in bytes.
9790**
9791** The page footer consists of a single varint for each term located
9792** on the page. Each varint is the byte offset of the current term
9793** within the page, delta-compressed against the previous value. In
9794** other words, the first varint in the footer is the byte offset of
9795** the first term, the second is the byte offset of the second less that
9796** of the first, and so on.
9797**
9798** The term/doclist format described above is accurate if the entire
9799** term/doclist data fits on a single leaf page. If this is not the case,
9800** the format is changed in two ways:
9801**
9802** + if the first rowid on a page occurs before the first term, it
9803** is stored as a literal value:
9804**
9805** varint: first rowid
9806**
9807** + the first term on each page is stored in the same way as the
9808** very first term of the segment:
9809**
9810** varint : size of first term
9811** blob: first term data
9812**
9813** 5. Segment doclist indexes:
9814**
9815** Doclist indexes are themselves b-trees, however they usually consist of
9816** a single leaf record only. The format of each doclist index leaf page
9817** is:
9818**
9819** * Flags byte. Bits are:
9820** 0x01: Clear if leaf is also the root page, otherwise set.
9821**
9822** * Page number of fts index leaf page. As a varint.
9823**
9824** * First rowid on page indicated by previous field. As a varint.
9825**
9826** * A list of varints, one for each subsequent termless page. A
9827** positive delta if the termless page contains at least one rowid,
9828** or an 0x00 byte otherwise.
9829**
9830** Internal doclist index nodes are:
9831**
9832** * Flags byte. Bits are:
9833** 0x01: Clear for root page, otherwise set.
9834**
9835** * Page number of first child page. As a varint.
9836**
9837** * Copy of first rowid on page indicated by previous field. As a varint.
9838**
9839** * A list of delta-encoded varints - the first rowid on each subsequent
9840** child page.
9841**
9842** 6. Tombstone Hash Page
9843**
9844** These records are only ever present in contentless_delete=1 tables.
9845** There are zero or more of these associated with each segment. They
9846** are used to store the tombstone rowids for rows contained in the
9847** associated segments.
9848**
9849** The set of nHashPg tombstone hash pages associated with a single
9850** segment together form a single hash table containing tombstone rowids.
9851** To find the page of the hash on which a key might be stored:
9852**
9853** iPg = (rowid % nHashPg)
9854**
9855** Then, within page iPg, which has nSlot slots:
9856**
9857** iSlot = (rowid / nHashPg) % nSlot
9858**
9859** Each tombstone hash page begins with an 8 byte header:
9860**
9861** 1-byte: Key-size (the size in bytes of each slot). Either 4 or 8.
9862** 1-byte: rowid-0-tombstone flag. This flag is only valid on the
9863** first tombstone hash page for each segment (iPg=0). If set,
9864** the hash table contains rowid 0. If clear, it does not.
9865** Rowid 0 is handled specially.
9866** 2-bytes: unused.
9867** 4-bytes: Big-endian integer containing number of entries on page.
9868**
9869** Following this are nSlot 4 or 8 byte slots (depending on the key-size
9870** in the first byte of the page header). The number of slots may be
9871** determined based on the size of the page record and the key-size:
9872**
9873** nSlot = (nByte - 8) / key-size
9874*/
9875
9876/*
9877** Rowids for the averages and structure records in the %_data table.
9878*/
9879#define FTS5_AVERAGES_ROWID1 1 /* Rowid used for the averages record */
9880#define FTS5_STRUCTURE_ROWID10 10 /* The structure record */
9881
9882/*
9883** Macros determining the rowids used by segment leaves and dlidx leaves
9884** and nodes. All nodes and leaves are stored in the %_data table with large
9885** positive rowids.
9886**
9887** Each segment has a unique non-zero 16-bit id.
9888**
9889** The rowid for each segment leaf is found by passing the segment id and
9890** the leaf page number to the FTS5_SEGMENT_ROWID macro. Leaves are numbered
9891** sequentially starting from 1.
9892*/
9893#define FTS5_DATA_ID_B16 16 /* Max seg id number 65535 */
9894#define FTS5_DATA_DLI_B1 1 /* Doclist-index flag (1 bit) */
9895#define FTS5_DATA_HEIGHT_B5 5 /* Max dlidx tree height of 32 */
9896#define FTS5_DATA_PAGE_B31 31 /* Max page number of 2147483648 */
9897
9898#define fts5_dri(segid, dlidx, height, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(dlidx) <<
(31 + 5)) + ((i64)(height) << (31)) + ((i64)(pgno)) ) ( \
9899 ((i64)(segid) << (FTS5_DATA_PAGE_B31+FTS5_DATA_HEIGHT_B5+FTS5_DATA_DLI_B1)) + \
9900 ((i64)(dlidx) << (FTS5_DATA_PAGE_B31 + FTS5_DATA_HEIGHT_B5)) + \
9901 ((i64)(height) << (FTS5_DATA_PAGE_B31)) + \
9902 ((i64)(pgno)) \
9903)
9904
9905#define FTS5_SEGMENT_ROWID(segid, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pgno)) ) fts5_dri(segid, 0, 0, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pgno)) )
9906#define FTS5_DLIDX_ROWID(segid, height, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(1) << (31
+ 5)) + ((i64)(height) << (31)) + ((i64)(pgno)) ) fts5_dri(segid, 1, height, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(1) << (31
+ 5)) + ((i64)(height) << (31)) + ((i64)(pgno)) )
9907#define FTS5_TOMBSTONE_ROWID(segid,ipg)( ((i64)(segid+(1<<16)) << (31 +5 +1)) + ((i64)(0
) << (31 + 5)) + ((i64)(0) << (31)) + ((i64)(ipg)
) ) fts5_dri(segid+(1<<16), 0, 0, ipg)( ((i64)(segid+(1<<16)) << (31 +5 +1)) + ((i64)(0
) << (31 + 5)) + ((i64)(0) << (31)) + ((i64)(ipg)
) )
9908
9909#ifdef SQLITE_DEBUG
9910static int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB(11 | (1<<8)); }
9911#endif
9912
9913
9914/*
9915** Each time a blob is read from the %_data table, it is padded with this
9916** many zero bytes. This makes it easier to decode the various record formats
9917** without overreading if the records are corrupt.
9918*/
9919#define FTS5_DATA_ZERO_PADDING8 8
9920#define FTS5_DATA_PADDING20 20
9921
9922typedef struct Fts5Data Fts5Data;
9923typedef struct Fts5DlidxIter Fts5DlidxIter;
9924typedef struct Fts5DlidxLvl Fts5DlidxLvl;
9925typedef struct Fts5DlidxWriter Fts5DlidxWriter;
9926typedef struct Fts5Iter Fts5Iter;
9927typedef struct Fts5PageWriter Fts5PageWriter;
9928typedef struct Fts5SegIter Fts5SegIter;
9929typedef struct Fts5DoclistIter Fts5DoclistIter;
9930typedef struct Fts5SegWriter Fts5SegWriter;
9931typedef struct Fts5Structure Fts5Structure;
9932typedef struct Fts5StructureLevel Fts5StructureLevel;
9933typedef struct Fts5StructureSegment Fts5StructureSegment;
9934typedef struct Fts5TokenDataIter Fts5TokenDataIter;
9935typedef struct Fts5TokenDataMap Fts5TokenDataMap;
9936typedef struct Fts5TombstoneArray Fts5TombstoneArray;
9937
9938struct Fts5Data {
9939 u8 *p; /* Pointer to buffer containing record */
9940 int nn; /* Size of record in bytes */
9941 int szLeaf; /* Size of leaf without page-index */
9942};
9943
9944/*
9945** One object per %_data table.
9946**
9947** nContentlessDelete:
9948** The number of contentless delete operations since the most recent
9949** call to fts5IndexFlush() or fts5IndexDiscardData(). This is tracked
9950** so that extra auto-merge work can be done by fts5IndexFlush() to
9951** account for the delete operations.
9952*/
9953struct Fts5Index {
9954 Fts5Config *pConfig; /* Virtual table configuration */
9955 char *zDataTbl; /* Name of %_data table */
9956 int nWorkUnit; /* Leaf pages in a "unit" of work */
9957
9958 /*
9959 ** Variables related to the accumulation of tokens and doclists within the
9960 ** in-memory hash tables before they are flushed to disk.
9961 */
9962 Fts5Hash *pHash; /* Hash table for in-memory data */
9963 int nPendingData; /* Current bytes of pending data */
9964 i64 iWriteRowid; /* Rowid for current doc being written */
9965 int bDelete; /* Current write is a delete */
9966 int nContentlessDelete; /* Number of contentless delete ops */
9967 int nPendingRow; /* Number of INSERT in hash table */
9968
9969 /* Error state. */
9970 int rc; /* Current error code */
9971 int flushRc;
9972
9973 /* State used by the fts5DataXXX() functions. */
9974 sqlite3_blob *pReader; /* RO incr-blob open on %_data table */
9975 sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */
9976 sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */
9977 sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */
9978 sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=?" */
9979 sqlite3_stmt *pIdxSelect;
9980 sqlite3_stmt *pIdxNextSelect;
9981 int nRead; /* Total number of blocks read */
9982
9983 sqlite3_stmt *pDeleteFromIdx;
9984
9985 sqlite3_stmt *pDataVersion;
9986 i64 iStructVersion; /* data_version when pStruct read */
9987 Fts5Structure *pStruct; /* Current db structure (or NULL) */
9988};
9989
9990struct Fts5DoclistIter {
9991 u8 *aEof; /* Pointer to 1 byte past end of doclist */
9992
9993 /* Output variables. aPoslist==0 at EOF */
9994 i64 iRowid;
9995 u8 *aPoslist;
9996 int nPoslist;
9997 int nSize;
9998};
9999
10000/*
10001** The contents of the "structure" record for each index are represented
10002** using an Fts5Structure record in memory. Which uses instances of the
10003** other Fts5StructureXXX types as components.
10004**
10005** nOriginCntr:
10006** This value is set to non-zero for structure records created for
10007** contentlessdelete=1 tables only. In that case it represents the
10008** origin value to apply to the next top-level segment created.
10009*/
10010struct Fts5StructureSegment {
10011 int iSegid; /* Segment id */
10012 int pgnoFirst; /* First leaf page number in segment */
10013 int pgnoLast; /* Last leaf page number in segment */
10014
10015 /* contentlessdelete=1 tables only: */
10016 u64 iOrigin1;
10017 u64 iOrigin2;
10018 int nPgTombstone; /* Number of tombstone hash table pages */
10019 u64 nEntryTombstone; /* Number of tombstone entries that "count" */
10020 u64 nEntry; /* Number of rows in this segment */
10021};
10022struct Fts5StructureLevel {
10023 int nMerge; /* Number of segments in incr-merge */
10024 int nSeg; /* Total number of segments on level */
10025 Fts5StructureSegment *aSeg; /* Array of segments. aSeg[0] is oldest. */
10026};
10027struct Fts5Structure {
10028 int nRef; /* Object reference count */
10029 u64 nWriteCounter; /* Total leaves written to level 0 */
10030 u64 nOriginCntr; /* Origin value for next top-level segment */
10031 int nSegment; /* Total segments in this structure */
10032 int nLevel; /* Number of levels in this index */
10033 Fts5StructureLevel aLevel[FLEXARRAY]; /* Array of nLevel level objects */
10034};
10035
10036/* Size (in bytes) of an Fts5Structure object holding up to N levels */
10037#define SZ_FTS5STRUCTURE(N)(__builtin_offsetof(Fts5Structure, aLevel) + (N)*sizeof(Fts5StructureLevel
)) \
10038 (offsetof(Fts5Structure,aLevel)__builtin_offsetof(Fts5Structure, aLevel) + (N)*sizeof(Fts5StructureLevel))
10039
10040/*
10041** An object of type Fts5SegWriter is used to write to segments.
10042*/
10043struct Fts5PageWriter {
10044 int pgno; /* Page number for this page */
10045 int iPrevPgidx; /* Previous value written into pgidx */
10046 Fts5Buffer buf; /* Buffer containing leaf data */
10047 Fts5Buffer pgidx; /* Buffer containing page-index */
10048 Fts5Buffer term; /* Buffer containing previous term on page */
10049};
10050struct Fts5DlidxWriter {
10051 int pgno; /* Page number for this page */
10052 int bPrevValid; /* True if iPrev is valid */
10053 i64 iPrev; /* Previous rowid value written to page */
10054 Fts5Buffer buf; /* Buffer containing page data */
10055};
10056struct Fts5SegWriter {
10057 int iSegid; /* Segid to write to */
10058 Fts5PageWriter writer; /* PageWriter object */
10059 i64 iPrevRowid; /* Previous rowid written to current leaf */
10060 u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */
10061 u8 bFirstRowidInPage; /* True if next rowid is first in page */
10062 /* TODO1: Can use (writer.pgidx.n==0) instead of bFirstTermInPage */
10063 u8 bFirstTermInPage; /* True if next term will be first in leaf */
10064 int nLeafWritten; /* Number of leaf pages written */
10065 int nEmpty; /* Number of contiguous term-less nodes */
10066
10067 int nDlidx; /* Allocated size of aDlidx[] array */
10068 Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */
10069
10070 /* Values to insert into the %_idx table */
10071 Fts5Buffer btterm; /* Next term to insert into %_idx table */
10072 int iBtPage; /* Page number corresponding to btterm */
10073};
10074
10075typedef struct Fts5CResult Fts5CResult;
10076struct Fts5CResult {
10077 u16 iFirst; /* aSeg[] index of firstest iterator */
10078 u8 bTermEq; /* True if the terms are equal */
10079};
10080
10081/*
10082** Object for iterating through a single segment, visiting each term/rowid
10083** pair in the segment.
10084**
10085** pSeg:
10086** The segment to iterate through.
10087**
10088** iLeafPgno:
10089** Current leaf page number within segment.
10090**
10091** iLeafOffset:
10092** Byte offset within the current leaf that is the first byte of the
10093** position list data (one byte passed the position-list size field).
10094**
10095** pLeaf:
10096** Buffer containing current leaf page data. Set to NULL at EOF.
10097**
10098** iTermLeafPgno, iTermLeafOffset:
10099** Leaf page number containing the last term read from the segment. And
10100** the offset immediately following the term data.
10101**
10102** flags:
10103** Mask of FTS5_SEGITER_XXX values. Interpreted as follows:
10104**
10105** FTS5_SEGITER_ONETERM:
10106** If set, set the iterator to point to EOF after the current doclist
10107** has been exhausted. Do not proceed to the next term in the segment.
10108**
10109** FTS5_SEGITER_REVERSE:
10110** This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If
10111** it is set, iterate through rowid in descending order instead of the
10112** default ascending order.
10113**
10114** iRowidOffset/nRowidOffset/aRowidOffset:
10115** These are used if the FTS5_SEGITER_REVERSE flag is set.
10116**
10117** For each rowid on the page corresponding to the current term, the
10118** corresponding aRowidOffset[] entry is set to the byte offset of the
10119** start of the "position-list-size" field within the page.
10120**
10121** iTermIdx:
10122** Index of current term on iTermLeafPgno.
10123**
10124** apTombstone/nTombstone:
10125** These are used for contentless_delete=1 tables only. When the cursor
10126** is first allocated, the apTombstone[] array is allocated so that it
10127** is large enough for all tombstones hash pages associated with the
10128** segment. The pages themselves are loaded lazily from the database as
10129** they are required.
10130*/
10131struct Fts5SegIter {
10132 Fts5StructureSegment *pSeg; /* Segment to iterate through */
10133 int flags; /* Mask of configuration flags */
10134 int iLeafPgno; /* Current leaf page number */
10135 Fts5Data *pLeaf; /* Current leaf data */
10136 Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */
10137 i64 iLeafOffset; /* Byte offset within current leaf */
10138 Fts5TombstoneArray *pTombArray; /* Array of tombstone pages */
10139
10140 /* Next method */
10141 void (*xNext)(Fts5Index*, Fts5SegIter*, int*);
10142
10143 /* The page and offset from which the current term was read. The offset
10144 ** is the offset of the first rowid in the current doclist. */
10145 int iTermLeafPgno;
10146 int iTermLeafOffset;
10147
10148 int iPgidxOff; /* Next offset in pgidx */
10149 int iEndofDoclist;
10150
10151 /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */
10152 int iRowidOffset; /* Current entry in aRowidOffset[] */
10153 int nRowidOffset; /* Allocated size of aRowidOffset[] array */
10154 int *aRowidOffset; /* Array of offset to rowid fields */
10155
10156 Fts5DlidxIter *pDlidx; /* If there is a doclist-index */
10157
10158 /* Variables populated based on current entry. */
10159 Fts5Buffer term; /* Current term */
10160 i64 iRowid; /* Current rowid */
10161 int nPos; /* Number of bytes in current position list */
10162 u8 bDel; /* True if the delete flag is set */
10163};
10164
10165static int fts5IndexCorruptRowid(Fts5Index *pIdx, i64 iRowid){
10166 pIdx->rc = FTS5_CORRUPT(11 | (1<<8));
10167 sqlite3Fts5ConfigErrmsg(pIdx->pConfig,
10168 "fts5: corruption found reading blob %lld from table \"%s\"",
10169 iRowid, pIdx->pConfig->zName
10170 );
10171 return SQLITE_CORRUPT_VTAB(11 | (1<<8));
10172}
10173#define FTS5_CORRUPT_ROWID(pIdx, iRowid)fts5IndexCorruptRowid(pIdx, iRowid) fts5IndexCorruptRowid(pIdx, iRowid)
10174
10175static int fts5IndexCorruptIter(Fts5Index *pIdx, Fts5SegIter *pIter){
10176 pIdx->rc = FTS5_CORRUPT(11 | (1<<8));
10177 sqlite3Fts5ConfigErrmsg(pIdx->pConfig,
10178 "fts5: corruption on page %d, segment %d, table \"%s\"",
10179 pIter->iLeafPgno, pIter->pSeg->iSegid, pIdx->pConfig->zName
10180 );
10181 return SQLITE_CORRUPT_VTAB(11 | (1<<8));
10182}
10183#define FTS5_CORRUPT_ITER(pIdx, pIter)fts5IndexCorruptIter(pIdx, pIter) fts5IndexCorruptIter(pIdx, pIter)
10184
10185static int fts5IndexCorruptIdx(Fts5Index *pIdx){
10186 pIdx->rc = FTS5_CORRUPT(11 | (1<<8));
10187 sqlite3Fts5ConfigErrmsg(pIdx->pConfig,
10188 "fts5: corruption in table \"%s\"", pIdx->pConfig->zName
10189 );
10190 return SQLITE_CORRUPT_VTAB(11 | (1<<8));
10191}
10192#define FTS5_CORRUPT_IDX(pIdx)fts5IndexCorruptIdx(pIdx) fts5IndexCorruptIdx(pIdx)
10193
10194
10195/*
10196** Array of tombstone pages. Reference counted.
10197*/
10198struct Fts5TombstoneArray {
10199 int nRef; /* Number of pointers to this object */
10200 int nTombstone;
10201 Fts5Data *apTombstone[FLEXARRAY]; /* Array of tombstone pages */
10202};
10203
10204/* Size (in bytes) of an Fts5TombstoneArray holding up to N tombstones */
10205#define SZ_FTS5TOMBSTONEARRAY(N)(__builtin_offsetof(Fts5TombstoneArray, apTombstone)+(N)*sizeof
(Fts5Data*)) \
10206 (offsetof(Fts5TombstoneArray,apTombstone)__builtin_offsetof(Fts5TombstoneArray, apTombstone)+(N)*sizeof(Fts5Data*))
10207
10208/*
10209** Argument is a pointer to an Fts5Data structure that contains a
10210** leaf page.
10211*/
10212#define ASSERT_SZLEAF_OK(x)((void) (0)) assert( \((void) (0))
10213 (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \((void) (0))
10214)((void) (0))
10215
10216#define FTS5_SEGITER_ONETERM0x01 0x01
10217#define FTS5_SEGITER_REVERSE0x02 0x02
10218
10219/*
10220** Argument is a pointer to an Fts5Data structure that contains a leaf
10221** page. This macro evaluates to true if the leaf contains no terms, or
10222** false if it contains at least one term.
10223*/
10224#define fts5LeafIsTermless(x)((x)->szLeaf >= (x)->nn) ((x)->szLeaf >= (x)->nn)
10225
10226#define fts5LeafTermOff(x, i)(fts5GetU16(&(x)->p[(x)->szLeaf + (i)*2])) (fts5GetU16(&(x)->p[(x)->szLeaf + (i)*2]))
10227
10228#define fts5LeafFirstRowidOff(x)(fts5GetU16((x)->p)) (fts5GetU16((x)->p))
10229
10230/*
10231** Object for iterating through the merged results of one or more segments,
10232** visiting each term/rowid pair in the merged data.
10233**
10234** nSeg is always a power of two greater than or equal to the number of
10235** segments that this object is merging data from. Both the aSeg[] and
10236** aFirst[] arrays are sized at nSeg entries. The aSeg[] array is padded
10237** with zeroed objects - these are handled as if they were iterators opened
10238** on empty segments.
10239**
10240** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an
10241** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the
10242** comparison in this context is the index of the iterator that currently
10243** points to the smaller term/rowid combination. Iterators at EOF are
10244** considered to be greater than all other iterators.
10245**
10246** aFirst[1] contains the index in aSeg[] of the iterator that points to
10247** the smallest key overall. aFirst[0] is unused.
10248**
10249** poslist:
10250** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
10251** There is no way to tell if this is populated or not.
10252**
10253** pColset:
10254** If not NULL, points to an object containing a set of column indices.
10255** Only matches that occur in one of these columns will be returned.
10256** The Fts5Iter does not own the Fts5Colset object, and so it is not
10257** freed when the iterator is closed - it is owned by the upper layer.
10258*/
10259struct Fts5Iter {
10260 Fts5IndexIter base; /* Base class containing output vars */
10261 Fts5TokenDataIter *pTokenDataIter;
10262
10263 Fts5Index *pIndex; /* Index that owns this iterator */
10264 Fts5Buffer poslist; /* Buffer containing current poslist */
10265 Fts5Colset *pColset; /* Restrict matches to these columns */
10266
10267 /* Invoked to set output variables. */
10268 void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*);
10269
10270 int nSeg; /* Size of aSeg[] array */
10271 int bRev; /* True to iterate in reverse order */
10272 u8 bSkipEmpty; /* True to skip deleted entries */
10273
10274 i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */
10275 Fts5CResult *aFirst; /* Current merge state (see above) */
10276 Fts5SegIter aSeg[FLEXARRAY]; /* Array of segment iterators */
10277};
10278
10279/* Size (in bytes) of an Fts5Iter object holding up to N segment iterators */
10280#define SZ_FTS5ITER(N)(__builtin_offsetof(Fts5Iter, aSeg)+(N)*sizeof(Fts5SegIter)) (offsetof(Fts5Iter,aSeg)__builtin_offsetof(Fts5Iter, aSeg)+(N)*sizeof(Fts5SegIter))
10281
10282/*
10283** An instance of the following type is used to iterate through the contents
10284** of a doclist-index record.
10285**
10286** pData:
10287** Record containing the doclist-index data.
10288**
10289** bEof:
10290** Set to true once iterator has reached EOF.
10291**
10292** iOff:
10293** Set to the current offset within record pData.
10294*/
10295struct Fts5DlidxLvl {
10296 Fts5Data *pData; /* Data for current page of this level */
10297 int iOff; /* Current offset into pData */
10298 int bEof; /* At EOF already */
10299 int iFirstOff; /* Used by reverse iterators */
10300
10301 /* Output variables */
10302 int iLeafPgno; /* Page number of current leaf page */
10303 i64 iRowid; /* First rowid on leaf iLeafPgno */
10304};
10305struct Fts5DlidxIter {
10306 int nLvl;
10307 int iSegid;
10308 Fts5DlidxLvl aLvl[FLEXARRAY];
10309};
10310
10311/* Size (in bytes) of an Fts5DlidxIter object with up to N levels */
10312#define SZ_FTS5DLIDXITER(N)(__builtin_offsetof(Fts5DlidxIter, aLvl)+(N)*sizeof(Fts5DlidxLvl
)) \
10313 (offsetof(Fts5DlidxIter,aLvl)__builtin_offsetof(Fts5DlidxIter, aLvl)+(N)*sizeof(Fts5DlidxLvl))
10314
10315static void fts5PutU16(u8 *aOut, u16 iVal){
10316 aOut[0] = (iVal>>8);
10317 aOut[1] = (iVal&0xFF);
10318}
10319
10320static u16 fts5GetU16(const u8 *aIn){
10321 return ((u16)aIn[0] << 8) + aIn[1];
10322}
10323
10324/*
10325** The only argument points to a buffer at least 8 bytes in size. This
10326** function interprets the first 8 bytes of the buffer as a 64-bit big-endian
10327** unsigned integer and returns the result.
10328*/
10329static u64 fts5GetU64(u8 *a){
10330 return ((u64)a[0] << 56)
10331 + ((u64)a[1] << 48)
10332 + ((u64)a[2] << 40)
10333 + ((u64)a[3] << 32)
10334 + ((u64)a[4] << 24)
10335 + ((u64)a[5] << 16)
10336 + ((u64)a[6] << 8)
10337 + ((u64)a[7] << 0);
10338}
10339
10340/*
10341** The only argument points to a buffer at least 4 bytes in size. This
10342** function interprets the first 4 bytes of the buffer as a 32-bit big-endian
10343** unsigned integer and returns the result.
10344*/
10345static u32 fts5GetU32(const u8 *a){
10346 return ((u32)a[0] << 24)
10347 + ((u32)a[1] << 16)
10348 + ((u32)a[2] << 8)
10349 + ((u32)a[3] << 0);
10350}
10351
10352/*
10353** Write iVal, formated as a 64-bit big-endian unsigned integer, to the
10354** buffer indicated by the first argument.
10355*/
10356static void fts5PutU64(u8 *a, u64 iVal){
10357 a[0] = ((iVal >> 56) & 0xFF);
10358 a[1] = ((iVal >> 48) & 0xFF);
10359 a[2] = ((iVal >> 40) & 0xFF);
10360 a[3] = ((iVal >> 32) & 0xFF);
10361 a[4] = ((iVal >> 24) & 0xFF);
10362 a[5] = ((iVal >> 16) & 0xFF);
10363 a[6] = ((iVal >> 8) & 0xFF);
10364 a[7] = ((iVal >> 0) & 0xFF);
10365}
10366
10367/*
10368** Write iVal, formated as a 32-bit big-endian unsigned integer, to the
10369** buffer indicated by the first argument.
10370*/
10371static void fts5PutU32(u8 *a, u32 iVal){
10372 a[0] = ((iVal >> 24) & 0xFF);
10373 a[1] = ((iVal >> 16) & 0xFF);
10374 a[2] = ((iVal >> 8) & 0xFF);
10375 a[3] = ((iVal >> 0) & 0xFF);
10376}
10377
10378/*
10379** Allocate and return a buffer at least nByte bytes in size.
10380**
10381** If an OOM error is encountered, return NULL and set the error code in
10382** the Fts5Index handle passed as the first argument.
10383*/
10384static void *fts5IdxMalloc(Fts5Index *p, sqlite3_int64 nByte){
10385 return sqlite3Fts5MallocZero(&p->rc, nByte);
10386}
10387
10388/*
10389** Compare the contents of the pLeft buffer with the pRight/nRight blob.
10390**
10391** Return -ve if pLeft is smaller than pRight, 0 if they are equal or
10392** +ve if pRight is smaller than pLeft. In other words:
10393**
10394** res = *pLeft - *pRight
10395*/
10396#ifdef SQLITE_DEBUG
10397static int fts5BufferCompareBlob(
10398 Fts5Buffer *pLeft, /* Left hand side of comparison */
10399 const u8 *pRight, int nRight /* Right hand side of comparison */
10400){
10401 int nCmp = MIN(pLeft->n, nRight)(((pLeft->n) < (nRight)) ? (pLeft->n) : (nRight));
10402 int res = memcmp(pLeft->p, pRight, nCmp);
10403 return (res==0 ? (pLeft->n - nRight) : res);
10404}
10405#endif
10406
10407/*
10408** Compare the contents of the two buffers using memcmp(). If one buffer
10409** is a prefix of the other, it is considered the lesser.
10410**
10411** Return -ve if pLeft is smaller than pRight, 0 if they are equal or
10412** +ve if pRight is smaller than pLeft. In other words:
10413**
10414** res = *pLeft - *pRight
10415*/
10416static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){
10417 int nCmp, res;
10418 nCmp = MIN(pLeft->n, pRight->n)(((pLeft->n) < (pRight->n)) ? (pLeft->n) : (pRight
->n));
10419 assert( nCmp<=0 || pLeft->p!=0 )((void) (0));
10420 assert( nCmp<=0 || pRight->p!=0 )((void) (0));
10421 res = fts5Memcmp(pLeft->p, pRight->p, nCmp)((nCmp)<=0 ? 0 : memcmp((pLeft->p), (pRight->p), (nCmp
)));
10422 return (res==0 ? (pLeft->n - pRight->n) : res);
10423}
10424
10425static int fts5LeafFirstTermOff(Fts5Data *pLeaf){
10426 int ret;
10427 fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret)sqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(ret));
10428 return ret;
10429}
10430
10431/*
10432** Close the read-only blob handle, if it is open.
10433*/
10434static void fts5IndexCloseReader(Fts5Index *p){
10435 if( p->pReader ){
10436 int rc;
10437 sqlite3_blob *pReader = p->pReader;
10438 p->pReader = 0;
10439 rc = sqlite3_blob_closesqlite3_api->blob_close(pReader);
10440 if( p->rc==SQLITE_OK0 ) p->rc = rc;
10441 }
10442}
10443
10444/*
10445** Retrieve a record from the %_data table.
10446**
10447** If an error occurs, NULL is returned and an error left in the
10448** Fts5Index object.
10449*/
10450static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
10451 Fts5Data *pRet = 0;
10452 if( p->rc==SQLITE_OK0 ){
10453 int rc = SQLITE_OK0;
10454
10455 if( p->pReader ){
10456 /* This call may return SQLITE_ABORT if there has been a savepoint
10457 ** rollback since it was last used. In this case a new blob handle
10458 ** is required. */
10459 sqlite3_blob *pBlob = p->pReader;
10460 p->pReader = 0;
10461 rc = sqlite3_blob_reopensqlite3_api->blob_reopen(pBlob, iRowid);
10462 assert( p->pReader==0 )((void) (0));
10463 p->pReader = pBlob;
10464 if( rc!=SQLITE_OK0 ){
10465 fts5IndexCloseReader(p);
10466 }
10467 if( rc==SQLITE_ABORT4 ) rc = SQLITE_OK0;
10468 }
10469
10470 /* If the blob handle is not open at this point, open it and seek
10471 ** to the requested entry. */
10472 if( p->pReader==0 && rc==SQLITE_OK0 ){
10473 Fts5Config *pConfig = p->pConfig;
10474 rc = sqlite3_blob_opensqlite3_api->blob_open(pConfig->db,
10475 pConfig->zDb, p->zDataTbl, "block", iRowid, 0, &p->pReader
10476 );
10477 }
10478
10479 /* If either of the sqlite3_blob_open() or sqlite3_blob_reopen() calls
10480 ** above returned SQLITE_ERROR, return SQLITE_CORRUPT_VTAB instead.
10481 ** All the reasons those functions might return SQLITE_ERROR - missing
10482 ** table, missing row, non-blob/text in block column - indicate
10483 ** backing store corruption. */
10484 if( rc==SQLITE_ERROR1 ) rc = FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
10485
10486 if( rc==SQLITE_OK0 ){
10487 u8 *aOut = 0; /* Read blob data into this buffer */
10488 i64 nByte = sqlite3_blob_bytessqlite3_api->blob_bytes(p->pReader);
10489 i64 szData = (sizeof(Fts5Data) + 7) & ~7;
10490 i64 nAlloc = szData + nByte + FTS5_DATA_PADDING20;
10491 pRet = (Fts5Data*)sqlite3_malloc64sqlite3_api->malloc64(nAlloc);
10492 if( pRet ){
10493 pRet->nn = nByte;
10494 pRet->szLeaf = 0;
10495 aOut = pRet->p = (u8*)pRet + szData;
10496 }else{
10497 rc = SQLITE_NOMEM7;
10498 }
10499
10500 if( rc==SQLITE_OK0 ){
10501 rc = sqlite3_blob_readsqlite3_api->blob_read(p->pReader, aOut, nByte, 0);
10502 }
10503 if( rc!=SQLITE_OK0 ){
10504 sqlite3_freesqlite3_api->free(pRet);
10505 pRet = 0;
10506 }else{
10507 pRet->p[nByte] = 0x00;
10508 pRet->p[nByte+1] = 0x00;
10509 }
10510 }
10511 p->rc = rc;
10512 p->nRead++;
10513 }
10514
10515 assert( (pRet==0)==(p->rc!=SQLITE_OK) )((void) (0));
10516 assert( pRet==0 || EIGHT_BYTE_ALIGNMENT( pRet->p ) )((void) (0));
10517 return pRet;
10518}
10519
10520
10521/*
10522** Release a reference to data record returned by an earlier call to
10523** fts5DataRead().
10524*/
10525static void fts5DataRelease(Fts5Data *pData){
10526 sqlite3_freesqlite3_api->free(pData);
10527}
10528
10529/*
10530** Read a leaf-page record. This is similar to fts5DataRead(), except that
10531** it fills in the Fts5Data.szLeaf value before returning.
10532*/
10533static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){
10534 Fts5Data *pRet = fts5DataRead(p, iRowid);
10535 if( pRet ){
10536 assert( pRet->szLeaf==0 )((void) (0));
10537 if( pRet->nn>=4 ){
10538 pRet->szLeaf = fts5GetU16(&pRet->p[2]);
10539 }
10540 if( pRet->szLeaf<4 || pRet->szLeaf>pRet->nn ){
10541 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
10542 fts5DataRelease(pRet);
10543 pRet = 0;
10544 }
10545 }
10546 return pRet;
10547}
10548
10549static int fts5IndexPrepareStmt(
10550 Fts5Index *p,
10551 sqlite3_stmt **ppStmt,
10552 char *zSql
10553){
10554 if( p->rc==SQLITE_OK0 ){
10555 if( zSql ){
10556 int rc = sqlite3_prepare_v3sqlite3_api->prepare_v3(p->pConfig->db, zSql, -1,
10557 SQLITE_PREPARE_PERSISTENT0x01|SQLITE_PREPARE_NO_VTAB0x04,
10558 ppStmt, 0);
10559 /* If this prepare() call fails with SQLITE_ERROR, then one of the
10560 ** %_idx or %_data tables has been removed or modified. Call this
10561 ** corruption. */
10562 p->rc = (rc==SQLITE_ERROR1 ? SQLITE_CORRUPT11 : rc);
10563 }else{
10564 p->rc = SQLITE_NOMEM7;
10565 }
10566 }
10567 sqlite3_freesqlite3_api->free(zSql);
10568 return p->rc;
10569}
10570
10571
10572/*
10573** INSERT OR REPLACE a record into the %_data table.
10574*/
10575static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){
10576 if( p->rc!=SQLITE_OK0 ) return;
10577
10578 if( p->pWriter==0 ){
10579 Fts5Config *pConfig = p->pConfig;
10580 fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintfsqlite3_api->mprintf(
10581 "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)",
10582 pConfig->zDb, pConfig->zName
10583 ));
10584 if( p->rc ) return;
10585 }
10586
10587 sqlite3_bind_int64sqlite3_api->bind_int64(p->pWriter, 1, iRowid);
10588 sqlite3_bind_blobsqlite3_api->bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC((sqlite3_destructor_type)0));
10589 sqlite3_stepsqlite3_api->step(p->pWriter);
10590 p->rc = sqlite3_resetsqlite3_api->reset(p->pWriter);
10591 sqlite3_bind_nullsqlite3_api->bind_null(p->pWriter, 2);
10592}
10593
10594/*
10595** Execute the following SQL:
10596**
10597** DELETE FROM %_data WHERE id BETWEEN $iFirst AND $iLast
10598*/
10599static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){
10600 if( p->rc!=SQLITE_OK0 ) return;
10601
10602 if( p->pDeleter==0 ){
10603 Fts5Config *pConfig = p->pConfig;
10604 char *zSql = sqlite3_mprintfsqlite3_api->mprintf(
10605 "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?",
10606 pConfig->zDb, pConfig->zName
10607 );
10608 if( fts5IndexPrepareStmt(p, &p->pDeleter, zSql) ) return;
10609 }
10610
10611 sqlite3_bind_int64sqlite3_api->bind_int64(p->pDeleter, 1, iFirst);
10612 sqlite3_bind_int64sqlite3_api->bind_int64(p->pDeleter, 2, iLast);
10613 sqlite3_stepsqlite3_api->step(p->pDeleter);
10614 p->rc = sqlite3_resetsqlite3_api->reset(p->pDeleter);
10615}
10616
10617/*
10618** Remove all records associated with segment iSegid.
10619*/
10620static void fts5DataRemoveSegment(Fts5Index *p, Fts5StructureSegment *pSeg){
10621 int iSegid = pSeg->iSegid;
10622 i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(0)) );
10623 i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)( ((i64)(iSegid+1) << (31 +5 +1)) + ((i64)(0) << (
31 + 5)) + ((i64)(0) << (31)) + ((i64)(0)) )-1;
10624 fts5DataDelete(p, iFirst, iLast);
10625
10626 if( pSeg->nPgTombstone ){
10627 i64 iTomb1 = FTS5_TOMBSTONE_ROWID(iSegid, 0)( ((i64)(iSegid+(1<<16)) << (31 +5 +1)) + ((i64)(
0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64)(0))
);
10628 i64 iTomb2 = FTS5_TOMBSTONE_ROWID(iSegid, pSeg->nPgTombstone-1)( ((i64)(iSegid+(1<<16)) << (31 +5 +1)) + ((i64)(
0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64)(pSeg
->nPgTombstone-1)) );
10629 fts5DataDelete(p, iTomb1, iTomb2);
10630 }
10631 if( p->pIdxDeleter==0 ){
10632 Fts5Config *pConfig = p->pConfig;
10633 fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintfsqlite3_api->mprintf(
10634 "DELETE FROM '%q'.'%q_idx' WHERE segid=?",
10635 pConfig->zDb, pConfig->zName
10636 ));
10637 }
10638 if( p->rc==SQLITE_OK0 ){
10639 sqlite3_bind_intsqlite3_api->bind_int(p->pIdxDeleter, 1, iSegid);
10640 sqlite3_stepsqlite3_api->step(p->pIdxDeleter);
10641 p->rc = sqlite3_resetsqlite3_api->reset(p->pIdxDeleter);
10642 }
10643}
10644
10645/*
10646** Release a reference to an Fts5Structure object returned by an earlier
10647** call to fts5StructureRead() or fts5StructureDecode().
10648*/
10649static void fts5StructureRelease(Fts5Structure *pStruct){
10650 if( pStruct && 0>=(--pStruct->nRef) ){
10651 int i;
10652 assert( pStruct->nRef==0 )((void) (0));
10653 for(i=0; i<pStruct->nLevel; i++){
10654 sqlite3_freesqlite3_api->free(pStruct->aLevel[i].aSeg);
10655 }
10656 sqlite3_freesqlite3_api->free(pStruct);
10657 }
10658}
10659
10660static void fts5StructureRef(Fts5Structure *pStruct){
10661 pStruct->nRef++;
10662}
10663
10664static void *sqlite3Fts5StructureRef(Fts5Index *p){
10665 fts5StructureRef(p->pStruct);
10666 return (void*)p->pStruct;
10667}
10668static void sqlite3Fts5StructureRelease(void *p){
10669 if( p ){
10670 fts5StructureRelease((Fts5Structure*)p);
10671 }
10672}
10673static int sqlite3Fts5StructureTest(Fts5Index *p, void *pStruct){
10674 if( p->pStruct!=(Fts5Structure*)pStruct ){
10675 return SQLITE_ABORT4;
10676 }
10677 return SQLITE_OK0;
10678}
10679
10680/*
10681** Ensure that structure object (*pp) is writable.
10682**
10683** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. If
10684** an error occurs, (*pRc) is set to an SQLite error code before returning.
10685*/
10686static void fts5StructureMakeWritable(int *pRc, Fts5Structure **pp){
10687 Fts5Structure *p = *pp;
10688 if( *pRc==SQLITE_OK0 && p->nRef>1 ){
10689 i64 nByte = SZ_FTS5STRUCTURE(p->nLevel)(__builtin_offsetof(Fts5Structure, aLevel) + (p->nLevel)*sizeof
(Fts5StructureLevel));
10690 Fts5Structure *pNew;
10691 pNew = (Fts5Structure*)sqlite3Fts5MallocZero(pRc, nByte);
10692 if( pNew ){
10693 int i;
10694 memcpy(pNew, p, nByte);
10695 for(i=0; i<p->nLevel; i++) pNew->aLevel[i].aSeg = 0;
10696 for(i=0; i<p->nLevel; i++){
10697 Fts5StructureLevel *pLvl = &pNew->aLevel[i];
10698 nByte = sizeof(Fts5StructureSegment) * pNew->aLevel[i].nSeg;
10699 pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(pRc, nByte);
10700 if( pLvl->aSeg==0 ){
10701 for(i=0; i<p->nLevel; i++){
10702 sqlite3_freesqlite3_api->free(pNew->aLevel[i].aSeg);
10703 }
10704 sqlite3_freesqlite3_api->free(pNew);
10705 return;
10706 }
10707 memcpy(pLvl->aSeg, p->aLevel[i].aSeg, nByte);
10708 }
10709 p->nRef--;
10710 pNew->nRef = 1;
10711 }
10712 *pp = pNew;
10713 }
10714}
10715
10716/*
10717** Deserialize and return the structure record currently stored in serialized
10718** form within buffer pData/nData.
10719**
10720** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
10721** are over-allocated by one slot. This allows the structure contents
10722** to be more easily edited.
10723**
10724** If an error occurs, *ppOut is set to NULL and an SQLite error code
10725** returned. Otherwise, *ppOut is set to point to the new object and
10726** SQLITE_OK returned.
10727*/
10728static int fts5StructureDecode(
10729 const u8 *pData, /* Buffer containing serialized structure */
10730 int nData, /* Size of buffer pData in bytes */
10731 int *piCookie, /* Configuration cookie value */
10732 Fts5Structure **ppOut /* OUT: Deserialized object */
10733){
10734 int rc = SQLITE_OK0;
10735 int i = 0;
10736 int iLvl;
10737 int nLevel = 0;
10738 int nSegment = 0;
10739 sqlite3_int64 nByte; /* Bytes of space to allocate at pRet */
10740 Fts5Structure *pRet = 0; /* Structure object to return */
10741 int bStructureV2 = 0; /* True for FTS5_STRUCTURE_V2 */
10742 u64 nOriginCntr = 0; /* Largest origin value seen so far */
10743
10744 /* Grab the cookie value */
10745 if( piCookie ) *piCookie = sqlite3Fts5Get32(pData);
10746 i = 4;
10747
10748 /* Check if this is a V2 structure record. Set bStructureV2 if it is. */
10749 if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2"\xFF\x00\x00\x01", 4) ){
10750 i += 4;
10751 bStructureV2 = 1;
10752 }
10753
10754 /* Read the total number of levels and segments from the start of the
10755 ** structure record. */
10756 i += fts5GetVarint32(&pData[i], nLevel)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(nLevel));
10757 i += fts5GetVarint32(&pData[i], nSegment)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(nSegment));
10758 if( nLevel>FTS5_MAX_SEGMENT2000 || nLevel<0
10759 || nSegment>FTS5_MAX_SEGMENT2000 || nSegment<0
10760 ){
10761 return FTS5_CORRUPT(11 | (1<<8));
10762 }
10763 nByte = SZ_FTS5STRUCTURE(nLevel)(__builtin_offsetof(Fts5Structure, aLevel) + (nLevel)*sizeof(
Fts5StructureLevel));
10764 pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte);
10765
10766 if( pRet ){
10767 pRet->nRef = 1;
10768 pRet->nLevel = nLevel;
10769 pRet->nSegment = nSegment;
10770 i += sqlite3Fts5GetVarint(&pData[i], &pRet->nWriteCounter);
10771
10772 for(iLvl=0; rc==SQLITE_OK0 && iLvl<nLevel; iLvl++){
10773 Fts5StructureLevel *pLvl = &pRet->aLevel[iLvl];
10774 int nTotal = 0;
10775 int iSeg;
10776
10777 if( i>=nData ){
10778 rc = FTS5_CORRUPT(11 | (1<<8));
10779 }else{
10780 i += fts5GetVarint32(&pData[i], pLvl->nMerge)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(pLvl->nMerge
));
10781 i += fts5GetVarint32(&pData[i], nTotal)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(nTotal));
10782 if( nTotal<pLvl->nMerge ) rc = FTS5_CORRUPT(11 | (1<<8));
10783 pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc,
10784 (i64)nTotal * sizeof(Fts5StructureSegment)
10785 );
10786 nSegment -= nTotal;
10787 }
10788
10789 if( rc==SQLITE_OK0 ){
10790 pLvl->nSeg = nTotal;
10791 for(iSeg=0; iSeg<nTotal; iSeg++){
10792 Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
10793 if( i>=nData ){
10794 rc = FTS5_CORRUPT(11 | (1<<8));
10795 break;
10796 }
10797 assert( pSeg!=0 )((void) (0));
10798 i += fts5GetVarint32(&pData[i], pSeg->iSegid)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(pSeg->iSegid
));
10799 i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(pSeg->pgnoFirst
));
10800 i += fts5GetVarint32(&pData[i], pSeg->pgnoLast)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(pSeg->pgnoLast
));
10801 if( bStructureV2 ){
10802 i += fts5GetVarintsqlite3Fts5GetVarint(&pData[i], &pSeg->iOrigin1);
10803 i += fts5GetVarintsqlite3Fts5GetVarint(&pData[i], &pSeg->iOrigin2);
10804 i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone)sqlite3Fts5GetVarint32(&pData[i],(u32*)&(pSeg->nPgTombstone
));
10805 i += fts5GetVarintsqlite3Fts5GetVarint(&pData[i], &pSeg->nEntryTombstone);
10806 i += fts5GetVarintsqlite3Fts5GetVarint(&pData[i], &pSeg->nEntry);
10807 nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2)(((nOriginCntr) > (pSeg->iOrigin2)) ? (nOriginCntr) : (
pSeg->iOrigin2));
10808 }
10809 if( pSeg->pgnoLast<pSeg->pgnoFirst ){
10810 rc = FTS5_CORRUPT(11 | (1<<8));
10811 break;
10812 }
10813 }
10814 if( iLvl>0 && pLvl[-1].nMerge && nTotal==0 ) rc = FTS5_CORRUPT(11 | (1<<8));
10815 if( iLvl==nLevel-1 && pLvl->nMerge ) rc = FTS5_CORRUPT(11 | (1<<8));
10816 }
10817 }
10818 if( nSegment!=0 && rc==SQLITE_OK0 ) rc = FTS5_CORRUPT(11 | (1<<8));
10819 if( bStructureV2 ){
10820 pRet->nOriginCntr = nOriginCntr+1;
10821 }
10822
10823 if( rc!=SQLITE_OK0 ){
10824 fts5StructureRelease(pRet);
10825 pRet = 0;
10826 }
10827 }
10828
10829 *ppOut = pRet;
10830 return rc;
10831}
10832
10833/*
10834** Add a level to the Fts5Structure.aLevel[] array of structure object
10835** (*ppStruct).
10836*/
10837static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){
10838 fts5StructureMakeWritable(pRc, ppStruct);
10839 assert( (ppStruct!=0 && (*ppStruct)!=0) || (*pRc)!=SQLITE_OK )((void) (0));
10840 if( *pRc==SQLITE_OK0 ){
10841 Fts5Structure *pStruct = *ppStruct;
10842 int nLevel = pStruct->nLevel;
10843 sqlite3_int64 nByte = SZ_FTS5STRUCTURE(nLevel+2)(__builtin_offsetof(Fts5Structure, aLevel) + (nLevel+2)*sizeof
(Fts5StructureLevel));
10844
10845 pStruct = sqlite3_realloc64sqlite3_api->realloc64(pStruct, nByte);
10846 if( pStruct ){
10847 memset(&pStruct->aLevel[nLevel], 0, sizeof(Fts5StructureLevel));
10848 pStruct->nLevel++;
10849 *ppStruct = pStruct;
10850 }else{
10851 *pRc = SQLITE_NOMEM7;
10852 }
10853 }
10854}
10855
10856/*
10857** Extend level iLvl so that there is room for at least nExtra more
10858** segments.
10859*/
10860static void fts5StructureExtendLevel(
10861 int *pRc,
10862 Fts5Structure *pStruct,
10863 int iLvl,
10864 int nExtra,
10865 int bInsert
10866){
10867 if( *pRc==SQLITE_OK0 ){
10868 Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
10869 Fts5StructureSegment *aNew;
10870 sqlite3_int64 nByte;
10871
10872 nByte = (pLvl->nSeg + nExtra) * sizeof(Fts5StructureSegment);
10873 aNew = sqlite3_realloc64sqlite3_api->realloc64(pLvl->aSeg, nByte);
10874 if( aNew ){
10875 if( bInsert==0 ){
10876 memset(&aNew[pLvl->nSeg], 0, sizeof(Fts5StructureSegment) * nExtra);
10877 }else{
10878 int nMove = pLvl->nSeg * sizeof(Fts5StructureSegment);
10879 memmove(&aNew[nExtra], aNew, nMove);
10880 memset(aNew, 0, sizeof(Fts5StructureSegment) * nExtra);
10881 }
10882 pLvl->aSeg = aNew;
10883 }else{
10884 *pRc = SQLITE_NOMEM7;
10885 }
10886 }
10887}
10888
10889static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){
10890 Fts5Structure *pRet = 0;
10891 Fts5Config *pConfig = p->pConfig;
10892 int iCookie; /* Configuration cookie */
10893 Fts5Data *pData;
10894
10895 pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID10);
10896 if( p->rc==SQLITE_OK0 ){
10897 /* TODO: Do we need this if the leaf-index is appended? Probably... */
10898 memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING20);
10899 p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
10900 if( p->rc==SQLITE_OK0 ){
10901 if( (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){
10902 p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
10903 }
10904 }else if( p->rc==SQLITE_CORRUPT_VTAB(11 | (1<<8)) ){
10905 sqlite3Fts5ConfigErrmsg(p->pConfig,
10906 "fts5: corrupt structure record for table \"%s\"", p->pConfig->zName
10907 );
10908 }
10909 fts5DataRelease(pData);
10910 if( p->rc!=SQLITE_OK0 ){
10911 fts5StructureRelease(pRet);
10912 pRet = 0;
10913 }
10914 }
10915
10916 return pRet;
10917}
10918
10919static i64 fts5IndexDataVersion(Fts5Index *p){
10920 i64 iVersion = 0;
10921
10922 if( p->rc==SQLITE_OK0 ){
10923 if( p->pDataVersion==0 ){
10924 p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion,
10925 sqlite3_mprintfsqlite3_api->mprintf("PRAGMA %Q.data_version", p->pConfig->zDb)
10926 );
10927 if( p->rc ) return 0;
10928 }
10929
10930 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(p->pDataVersion) ){
10931 iVersion = sqlite3_column_int64sqlite3_api->column_int64(p->pDataVersion, 0);
10932 }
10933 p->rc = sqlite3_resetsqlite3_api->reset(p->pDataVersion);
10934 }
10935
10936 return iVersion;
10937}
10938
10939/*
10940** Read, deserialize and return the structure record.
10941**
10942** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
10943** are over-allocated as described for function fts5StructureDecode()
10944** above.
10945**
10946** If an error occurs, NULL is returned and an error code left in the
10947** Fts5Index handle. If an error has already occurred when this function
10948** is called, it is a no-op.
10949*/
10950static Fts5Structure *fts5StructureRead(Fts5Index *p){
10951
10952 if( p->pStruct==0 ){
10953 p->iStructVersion = fts5IndexDataVersion(p);
10954 if( p->rc==SQLITE_OK0 ){
10955 p->pStruct = fts5StructureReadUncached(p);
10956 }
10957 }
10958
10959#if 0
10960 else{
10961 Fts5Structure *pTest = fts5StructureReadUncached(p);
10962 if( pTest ){
10963 int i, j;
10964 assert_nc( p->pStruct->nSegment==pTest->nSegment )((void) (0));
10965 assert_nc( p->pStruct->nLevel==pTest->nLevel )((void) (0));
10966 for(i=0; i<pTest->nLevel; i++){
10967 assert_nc( p->pStruct->aLevel[i].nMerge==pTest->aLevel[i].nMerge )((void) (0));
10968 assert_nc( p->pStruct->aLevel[i].nSeg==pTest->aLevel[i].nSeg )((void) (0));
10969 for(j=0; j<pTest->aLevel[i].nSeg; j++){
10970 Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j];
10971 Fts5StructureSegment *p2 = &p->pStruct->aLevel[i].aSeg[j];
10972 assert_nc( p1->iSegid==p2->iSegid )((void) (0));
10973 assert_nc( p1->pgnoFirst==p2->pgnoFirst )((void) (0));
10974 assert_nc( p1->pgnoLast==p2->pgnoLast )((void) (0));
10975 }
10976 }
10977 fts5StructureRelease(pTest);
10978 }
10979 }
10980#endif
10981
10982 if( p->rc!=SQLITE_OK0 ) return 0;
10983 assert( p->iStructVersion!=0 )((void) (0));
10984 assert( p->pStruct!=0 )((void) (0));
10985 fts5StructureRef(p->pStruct);
10986 return p->pStruct;
10987}
10988
10989static void fts5StructureInvalidate(Fts5Index *p){
10990 if( p->pStruct ){
10991 fts5StructureRelease(p->pStruct);
10992 p->pStruct = 0;
10993 }
10994}
10995
10996/*
10997** Return the total number of segments in index structure pStruct. This
10998** function is only ever used as part of assert() conditions.
10999*/
11000#ifdef SQLITE_DEBUG
11001static int fts5StructureCountSegments(Fts5Structure *pStruct){
11002 int nSegment = 0; /* Total number of segments */
11003 if( pStruct ){
11004 int iLvl; /* Used to iterate through levels */
11005 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
11006 nSegment += pStruct->aLevel[iLvl].nSeg;
11007 }
11008 }
11009
11010 return nSegment;
11011}
11012#endif
11013
11014#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob){ ((void) (0)); memcpy(&(pBuf)->p[(pBuf)->n], pBlob
, nBlob); (pBuf)->n += nBlob; } { \
11015 assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) )((void) (0)); \
11016 memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \
11017 (pBuf)->n += nBlob; \
11018}
11019
11020#define fts5BufferSafeAppendVarint(pBuf, iVal){ (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf
)->n], (iVal)); ((void) (0)); } { \
11021 (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \
11022 assert( (pBuf)->nSpace>=(pBuf)->n )((void) (0)); \
11023}
11024
11025
11026/*
11027** Serialize and store the "structure" record.
11028**
11029** If an error occurs, leave an error code in the Fts5Index object. If an
11030** error has already occurred, this function is a no-op.
11031*/
11032static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
11033 if( p->rc==SQLITE_OK0 ){
11034 Fts5Buffer buf; /* Buffer to serialize record into */
11035 int iLvl; /* Used to iterate through levels */
11036 int iCookie; /* Cookie value to store */
11037 int nHdr = (pStruct->nOriginCntr>0 ? (4+4+9+9+9) : (4+9+9));
11038
11039 assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) )((void) (0));
11040 memset(&buf, 0, sizeof(Fts5Buffer));
11041
11042 /* Append the current configuration cookie */
11043 iCookie = p->pConfig->iCookie;
11044 if( iCookie<0 ) iCookie = 0;
11045
11046 if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){
11047 sqlite3Fts5Put32(buf.p, iCookie);
11048 buf.n = 4;
11049 if( pStruct->nOriginCntr>0 ){
11050 fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4){ ((void) (0)); memcpy(&(&buf)->p[(&buf)->n
], "\xFF\x00\x00\x01", 4); (&buf)->n += 4; };
11051 }
11052 fts5BufferSafeAppendVarint(&buf, pStruct->nLevel){ (&buf)->n += sqlite3Fts5PutVarint(&(&buf)->
p[(&buf)->n], (pStruct->nLevel)); ((void) (0)); };
11053 fts5BufferSafeAppendVarint(&buf, pStruct->nSegment){ (&buf)->n += sqlite3Fts5PutVarint(&(&buf)->
p[(&buf)->n], (pStruct->nSegment)); ((void) (0)); };
11054 fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter){ (&buf)->n += sqlite3Fts5PutVarint(&(&buf)->
p[(&buf)->n], ((i64)pStruct->nWriteCounter)); ((void
) (0)); };
11055 }
11056
11057 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
11058 int iSeg; /* Used to iterate through segments */
11059 Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
11060 fts5BufferAppendVarint(&p->rc, &buf, pLvl->nMerge)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pLvl
->nMerge);
11061 fts5BufferAppendVarint(&p->rc, &buf, pLvl->nSeg)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pLvl
->nSeg);
11062 assert( pLvl->nMerge<=pLvl->nSeg )((void) (0));
11063
11064 for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
11065 Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
11066 fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->iSegid);
11067 fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->pgnoFirst);
11068 fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->pgnoLast);
11069 if( pStruct->nOriginCntr>0 ){
11070 fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->iOrigin1);
11071 fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->iOrigin2);
11072 fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->nPgTombstone);
11073 fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->nEntryTombstone);
11074 fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->nEntry);
11075 }
11076 }
11077 }
11078
11079 fts5DataWrite(p, FTS5_STRUCTURE_ROWID10, buf.p, buf.n);
11080 fts5BufferFree(&buf)sqlite3Fts5BufferFree(&buf);
11081 }
11082}
11083
11084#if 0
11085static void fts5DebugStructure(int*,Fts5Buffer*,Fts5Structure*);
11086static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){
11087 int rc = SQLITE_OK0;
11088 Fts5Buffer buf;
11089 memset(&buf, 0, sizeof(buf));
11090 fts5DebugStructure(&rc, &buf, pStruct);
11091 fprintf(stdout, "%s: %s\n", zCaption, buf.p);
11092 fflush(stdout);
11093 fts5BufferFree(&buf)sqlite3Fts5BufferFree(&buf);
11094}
11095#else
11096# define fts5PrintStructure(x,y)
11097#endif
11098
11099static int fts5SegmentSize(Fts5StructureSegment *pSeg){
11100 return 1 + pSeg->pgnoLast - pSeg->pgnoFirst;
11101}
11102
11103/*
11104** Return a copy of index structure pStruct. Except, promote as many
11105** segments as possible to level iPromote. If an OOM occurs, NULL is
11106** returned.
11107*/
11108static void fts5StructurePromoteTo(
11109 Fts5Index *p,
11110 int iPromote,
11111 int szPromote,
11112 Fts5Structure *pStruct
11113){
11114 int il, is;
11115 Fts5StructureLevel *pOut = &pStruct->aLevel[iPromote];
11116
11117 if( pOut->nMerge==0 ){
11118 for(il=iPromote+1; il<pStruct->nLevel; il++){
11119 Fts5StructureLevel *pLvl = &pStruct->aLevel[il];
11120 if( pLvl->nMerge ) return;
11121 for(is=pLvl->nSeg-1; is>=0; is--){
11122 int sz = fts5SegmentSize(&pLvl->aSeg[is]);
11123 if( sz>szPromote ) return;
11124 fts5StructureExtendLevel(&p->rc, pStruct, iPromote, 1, 1);
11125 if( p->rc ) return;
11126 memcpy(pOut->aSeg, &pLvl->aSeg[is], sizeof(Fts5StructureSegment));
11127 pOut->nSeg++;
11128 pLvl->nSeg--;
11129 }
11130 }
11131 }
11132}
11133
11134/*
11135** A new segment has just been written to level iLvl of index structure
11136** pStruct. This function determines if any segments should be promoted
11137** as a result. Segments are promoted in two scenarios:
11138**
11139** a) If the segment just written is smaller than one or more segments
11140** within the previous populated level, it is promoted to the previous
11141** populated level.
11142**
11143** b) If the segment just written is larger than the newest segment on
11144** the next populated level, then that segment, and any other adjacent
11145** segments that are also smaller than the one just written, are
11146** promoted.
11147**
11148** If one or more segments are promoted, the structure object is updated
11149** to reflect this.
11150*/
11151static void fts5StructurePromote(
11152 Fts5Index *p, /* FTS5 backend object */
11153 int iLvl, /* Index level just updated */
11154 Fts5Structure *pStruct /* Index structure */
11155){
11156 if( p->rc==SQLITE_OK0 ){
11157 int iTst;
11158 int iPromote = -1;
11159 int szPromote = 0; /* Promote anything this size or smaller */
11160 Fts5StructureSegment *pSeg; /* Segment just written */
11161 int szSeg; /* Size of segment just written */
11162 int nSeg = pStruct->aLevel[iLvl].nSeg;
11163
11164 if( nSeg==0 ) return;
11165 pSeg = &pStruct->aLevel[iLvl].aSeg[pStruct->aLevel[iLvl].nSeg-1];
11166 szSeg = (1 + pSeg->pgnoLast - pSeg->pgnoFirst);
11167
11168 /* Check for condition (a) */
11169 for(iTst=iLvl-1; iTst>=0 && pStruct->aLevel[iTst].nSeg==0; iTst--);
11170 if( iTst>=0 ){
11171 int i;
11172 int szMax = 0;
11173 Fts5StructureLevel *pTst = &pStruct->aLevel[iTst];
11174 assert( pTst->nMerge==0 )((void) (0));
11175 for(i=0; i<pTst->nSeg; i++){
11176 int sz = pTst->aSeg[i].pgnoLast - pTst->aSeg[i].pgnoFirst + 1;
11177 if( sz>szMax ) szMax = sz;
11178 }
11179 if( szMax>=szSeg ){
11180 /* Condition (a) is true. Promote the newest segment on level
11181 ** iLvl to level iTst. */
11182 iPromote = iTst;
11183 szPromote = szMax;
11184 }
11185 }
11186
11187 /* If condition (a) is not met, assume (b) is true. StructurePromoteTo()
11188 ** is a no-op if it is not. */
11189 if( iPromote<0 ){
11190 iPromote = iLvl;
11191 szPromote = szSeg;
11192 }
11193 fts5StructurePromoteTo(p, iPromote, szPromote, pStruct);
11194 }
11195}
11196
11197
11198/*
11199** Advance the iterator passed as the only argument. If the end of the
11200** doclist-index page is reached, return non-zero.
11201*/
11202static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){
11203 Fts5Data *pData = pLvl->pData;
11204
11205 if( pLvl->iOff==0 ){
11206 assert( pLvl->bEof==0 )((void) (0));
11207 pLvl->iOff = 1;
11208 pLvl->iOff += fts5GetVarint32(&pData->p[1], pLvl->iLeafPgno)sqlite3Fts5GetVarint32(&pData->p[1],(u32*)&(pLvl->
iLeafPgno));
11209 pLvl->iOff += fts5GetVarintsqlite3Fts5GetVarint(&pData->p[pLvl->iOff], (u64*)&pLvl->iRowid);
11210 pLvl->iFirstOff = pLvl->iOff;
11211 }else{
11212 int iOff;
11213 for(iOff=pLvl->iOff; iOff<pData->nn; iOff++){
11214 if( pData->p[iOff] ) break;
11215 }
11216
11217 if( iOff<pData->nn ){
11218 u64 iVal;
11219 pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1;
11220 iOff += fts5GetVarintsqlite3Fts5GetVarint(&pData->p[iOff], &iVal);
11221 pLvl->iRowid += iVal;
11222 pLvl->iOff = iOff;
11223 }else{
11224 pLvl->bEof = 1;
11225 }
11226 }
11227
11228 return pLvl->bEof;
11229}
11230
11231/*
11232** Advance the iterator passed as the only argument.
11233*/
11234static int fts5DlidxIterNextR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){
11235 Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl];
11236
11237 assert( iLvl<pIter->nLvl )((void) (0));
11238 if( fts5DlidxLvlNext(pLvl) ){
11239 if( (iLvl+1) < pIter->nLvl ){
11240 fts5DlidxIterNextR(p, pIter, iLvl+1);
11241 if( pLvl[1].bEof==0 ){
11242 fts5DataRelease(pLvl->pData);
11243 memset(pLvl, 0, sizeof(Fts5DlidxLvl));
11244 pLvl->pData = fts5DataRead(p,
11245 FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno)( ((i64)(pIter->iSegid) << (31 +5 +1)) + ((i64)(1) <<
(31 + 5)) + ((i64)(iLvl) << (31)) + ((i64)(pLvl[1].iLeafPgno
)) )
11246 );
11247 if( pLvl->pData ) fts5DlidxLvlNext(pLvl);
11248 }
11249 }
11250 }
11251
11252 return pIter->aLvl[0].bEof;
11253}
11254static int fts5DlidxIterNext(Fts5Index *p, Fts5DlidxIter *pIter){
11255 return fts5DlidxIterNextR(p, pIter, 0);
11256}
11257
11258/*
11259** The iterator passed as the first argument has the following fields set
11260** as follows. This function sets up the rest of the iterator so that it
11261** points to the first rowid in the doclist-index.
11262**
11263** pData:
11264** pointer to doclist-index record,
11265**
11266** When this function is called pIter->iLeafPgno is the page number the
11267** doclist is associated with (the one featuring the term).
11268*/
11269static int fts5DlidxIterFirst(Fts5DlidxIter *pIter){
11270 int i;
11271 for(i=0; i<pIter->nLvl; i++){
11272 fts5DlidxLvlNext(&pIter->aLvl[i]);
11273 }
11274 return pIter->aLvl[0].bEof;
11275}
11276
11277
11278static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){
11279 return p->rc!=SQLITE_OK0 || pIter->aLvl[0].bEof;
11280}
11281
11282static void fts5DlidxIterLast(Fts5Index *p, Fts5DlidxIter *pIter){
11283 int i;
11284
11285 /* Advance each level to the last entry on the last page */
11286 for(i=pIter->nLvl-1; p->rc==SQLITE_OK0 && i>=0; i--){
11287 Fts5DlidxLvl *pLvl = &pIter->aLvl[i];
11288 while( fts5DlidxLvlNext(pLvl)==0 );
11289 pLvl->bEof = 0;
11290
11291 if( i>0 ){
11292 Fts5DlidxLvl *pChild = &pLvl[-1];
11293 fts5DataRelease(pChild->pData);
11294 memset(pChild, 0, sizeof(Fts5DlidxLvl));
11295 pChild->pData = fts5DataRead(p,
11296 FTS5_DLIDX_ROWID(pIter->iSegid, i-1, pLvl->iLeafPgno)( ((i64)(pIter->iSegid) << (31 +5 +1)) + ((i64)(1) <<
(31 + 5)) + ((i64)(i-1) << (31)) + ((i64)(pLvl->iLeafPgno
)) )
11297 );
11298 }
11299 }
11300}
11301
11302/*
11303** Move the iterator passed as the only argument to the previous entry.
11304*/
11305static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){
11306 int iOff = pLvl->iOff;
11307
11308 assert( pLvl->bEof==0 )((void) (0));
11309 if( iOff<=pLvl->iFirstOff ){
11310 pLvl->bEof = 1;
11311 }else{
11312 u8 *a = pLvl->pData->p;
11313
11314 pLvl->iOff = 0;
11315 fts5DlidxLvlNext(pLvl);
11316 while( 1 ){
11317 int nZero = 0;
11318 int ii = pLvl->iOff;
11319 u64 delta = 0;
11320
11321 while( a[ii]==0 ){
11322 nZero++;
11323 ii++;
11324 }
11325 ii += sqlite3Fts5GetVarint(&a[ii], &delta);
11326
11327 if( ii>=iOff ) break;
11328 pLvl->iLeafPgno += nZero+1;
11329 pLvl->iRowid += delta;
11330 pLvl->iOff = ii;
11331 }
11332 }
11333
11334 return pLvl->bEof;
11335}
11336
11337static int fts5DlidxIterPrevR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){
11338 Fts5DlidxLvl *pLvl = &pIter->aLvl[iLvl];
11339
11340 assert( iLvl<pIter->nLvl )((void) (0));
11341 if( fts5DlidxLvlPrev(pLvl) ){
11342 if( (iLvl+1) < pIter->nLvl ){
11343 fts5DlidxIterPrevR(p, pIter, iLvl+1);
11344 if( pLvl[1].bEof==0 ){
11345 fts5DataRelease(pLvl->pData);
11346 memset(pLvl, 0, sizeof(Fts5DlidxLvl));
11347 pLvl->pData = fts5DataRead(p,
11348 FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno)( ((i64)(pIter->iSegid) << (31 +5 +1)) + ((i64)(1) <<
(31 + 5)) + ((i64)(iLvl) << (31)) + ((i64)(pLvl[1].iLeafPgno
)) )
11349 );
11350 if( pLvl->pData ){
11351 while( fts5DlidxLvlNext(pLvl)==0 );
11352 pLvl->bEof = 0;
11353 }
11354 }
11355 }
11356 }
11357
11358 return pIter->aLvl[0].bEof;
11359}
11360static int fts5DlidxIterPrev(Fts5Index *p, Fts5DlidxIter *pIter){
11361 return fts5DlidxIterPrevR(p, pIter, 0);
11362}
11363
11364/*
11365** Free a doclist-index iterator object allocated by fts5DlidxIterInit().
11366*/
11367static void fts5DlidxIterFree(Fts5DlidxIter *pIter){
11368 if( pIter ){
11369 int i;
11370 for(i=0; i<pIter->nLvl; i++){
11371 fts5DataRelease(pIter->aLvl[i].pData);
11372 }
11373 sqlite3_freesqlite3_api->free(pIter);
11374 }
11375}
11376
11377static Fts5DlidxIter *fts5DlidxIterInit(
11378 Fts5Index *p, /* Fts5 Backend to iterate within */
11379 int bRev, /* True for ORDER BY ASC */
11380 int iSegid, /* Segment id */
11381 int iLeafPg /* Leaf page number to load dlidx for */
11382){
11383 Fts5DlidxIter *pIter = 0;
11384 int i;
11385 int bDone = 0;
11386
11387 for(i=0; p->rc==SQLITE_OK0 && bDone==0; i++){
11388 sqlite3_int64 nByte = SZ_FTS5DLIDXITER(i+1)(__builtin_offsetof(Fts5DlidxIter, aLvl)+(i+1)*sizeof(Fts5DlidxLvl
));
11389 Fts5DlidxIter *pNew;
11390
11391 pNew = (Fts5DlidxIter*)sqlite3_realloc64sqlite3_api->realloc64(pIter, nByte);
11392 if( pNew==0 ){
11393 p->rc = SQLITE_NOMEM7;
11394 }else{
11395 i64 iRowid = FTS5_DLIDX_ROWID(iSegid, i, iLeafPg)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(1) << (31
+ 5)) + ((i64)(i) << (31)) + ((i64)(iLeafPg)) );
11396 Fts5DlidxLvl *pLvl = &pNew->aLvl[i];
11397 pIter = pNew;
11398 memset(pLvl, 0, sizeof(Fts5DlidxLvl));
11399 pLvl->pData = fts5DataRead(p, iRowid);
11400 if( pLvl->pData && (pLvl->pData->p[0] & 0x0001)==0 ){
11401 bDone = 1;
11402 }
11403 pIter->nLvl = i+1;
11404 }
11405 }
11406
11407 if( p->rc==SQLITE_OK0 ){
11408 pIter->iSegid = iSegid;
11409 if( bRev==0 ){
11410 fts5DlidxIterFirst(pIter);
11411 }else{
11412 fts5DlidxIterLast(p, pIter);
11413 }
11414 }
11415
11416 if( p->rc!=SQLITE_OK0 ){
11417 fts5DlidxIterFree(pIter);
11418 pIter = 0;
11419 }
11420
11421 return pIter;
11422}
11423
11424static i64 fts5DlidxIterRowid(Fts5DlidxIter *pIter){
11425 return pIter->aLvl[0].iRowid;
11426}
11427static int fts5DlidxIterPgno(Fts5DlidxIter *pIter){
11428 return pIter->aLvl[0].iLeafPgno;
11429}
11430
11431/*
11432** Load the next leaf page into the segment iterator.
11433*/
11434static void fts5SegIterNextPage(
11435 Fts5Index *p, /* FTS5 backend object */
11436 Fts5SegIter *pIter /* Iterator to advance to next page */
11437){
11438 Fts5Data *pLeaf;
11439 Fts5StructureSegment *pSeg = pIter->pSeg;
11440 fts5DataRelease(pIter->pLeaf);
11441 pIter->iLeafPgno++;
11442 if( pIter->pNextLeaf ){
11443 pIter->pLeaf = pIter->pNextLeaf;
11444 pIter->pNextLeaf = 0;
11445 }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){
11446 pIter->pLeaf = fts5LeafRead(p,
11447 FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno)( ((i64)(pSeg->iSegid) << (31 +5 +1)) + ((i64)(0) <<
(31 + 5)) + ((i64)(0) << (31)) + ((i64)(pIter->iLeafPgno
)) )
11448 );
11449 }else{
11450 pIter->pLeaf = 0;
11451 }
11452 pLeaf = pIter->pLeaf;
11453
11454 if( pLeaf ){
11455 pIter->iPgidxOff = pLeaf->szLeaf;
11456 if( fts5LeafIsTermless(pLeaf)((pLeaf)->szLeaf >= (pLeaf)->nn) ){
11457 pIter->iEndofDoclist = pLeaf->nn+1;
11458 }else{
11459 pIter->iPgidxOff += fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff],sqlite3Fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff],
(u32*)&(pIter->iEndofDoclist))
11460 pIter->iEndofDoclistsqlite3Fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff],
(u32*)&(pIter->iEndofDoclist))
11461 )sqlite3Fts5GetVarint32(&pLeaf->p[pIter->iPgidxOff],
(u32*)&(pIter->iEndofDoclist));
11462 }
11463 }
11464}
11465
11466/*
11467** Argument p points to a buffer containing a varint to be interpreted as a
11468** position list size field. Read the varint and return the number of bytes
11469** read. Before returning, set *pnSz to the number of bytes in the position
11470** list, and *pbDel to true if the delete flag is set, or false otherwise.
11471*/
11472static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){
11473 int nSz;
11474 int n = 0;
11475 fts5FastGetVarint32(p, n, nSz){ nSz = (p)[n++]; if( nSz & 0x80 ){ n--; n += sqlite3Fts5GetVarint32
(&(p)[n],(u32*)&(nSz)); } };
11476 assert_nc( nSz>=0 )((void) (0));
11477 *pnSz = nSz/2;
11478 *pbDel = nSz & 0x0001;
11479 return n;
11480}
11481
11482/*
11483** Fts5SegIter.iLeafOffset currently points to the first byte of a
11484** position-list size field. Read the value of the field and store it
11485** in the following variables:
11486**
11487** Fts5SegIter.nPos
11488** Fts5SegIter.bDel
11489**
11490** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the
11491** position list content (if any).
11492*/
11493static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
11494 if( p->rc==SQLITE_OK0 ){
11495 int iOff = pIter->iLeafOffset; /* Offset to read at */
11496 ASSERT_SZLEAF_OK(pIter->pLeaf)((void) (0));
11497 if( p->pConfig->eDetail==FTS5_DETAIL_NONE1 ){
11498 int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf)(((pIter->iEndofDoclist) < (pIter->pLeaf->szLeaf)
) ? (pIter->iEndofDoclist) : (pIter->pLeaf->szLeaf));
11499 pIter->bDel = 0;
11500 pIter->nPos = 1;
11501 if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){
11502 pIter->bDel = 1;
11503 iOff++;
11504 if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){
11505 pIter->nPos = 1;
11506 iOff++;
11507 }else{
11508 pIter->nPos = 0;
11509 }
11510 }
11511 }else{
11512 int nSz;
11513 fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz){ nSz = (pIter->pLeaf->p)[iOff++]; if( nSz & 0x80 )
{ iOff--; iOff += sqlite3Fts5GetVarint32(&(pIter->pLeaf
->p)[iOff],(u32*)&(nSz)); } };
11514 pIter->bDel = (nSz & 0x0001);
11515 pIter->nPos = nSz>>1;
11516 assert_nc( pIter->nPos>=0 )((void) (0));
11517 }
11518 pIter->iLeafOffset = iOff;
11519 }
11520}
11521
11522static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
11523 u8 *a = pIter->pLeaf->p; /* Buffer to read data from */
11524 i64 iOff = pIter->iLeafOffset;
11525
11526 ASSERT_SZLEAF_OK(pIter->pLeaf)((void) (0));
11527 while( iOff>=pIter->pLeaf->szLeaf ){
11528 fts5SegIterNextPage(p, pIter);
11529 if( pIter->pLeaf==0 ){
11530 if( p->rc==SQLITE_OK0 ) FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
11531 return;
11532 }
11533 iOff = 4;
11534 a = pIter->pLeaf->p;
11535 }
11536 iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
11537 pIter->iLeafOffset = iOff;
11538}
11539
11540/*
11541** Fts5SegIter.iLeafOffset currently points to the first byte of the
11542** "nSuffix" field of a term. Function parameter nKeep contains the value
11543** of the "nPrefix" field (if there was one - it is passed 0 if this is
11544** the first term in the segment).
11545**
11546** This function populates:
11547**
11548** Fts5SegIter.term
11549** Fts5SegIter.rowid
11550**
11551** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the content of
11552** the first position list. The position list belonging to document
11553** (Fts5SegIter.iRowid).
11554*/
11555static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){
11556 u8 *a = pIter->pLeaf->p; /* Buffer to read data from */
11557 i64 iOff = pIter->iLeafOffset; /* Offset to read at */
11558 int nNew; /* Bytes of new data */
11559
11560 iOff += fts5GetVarint32(&a[iOff], nNew)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nNew));
11561 if( iOff+nNew>pIter->pLeaf->szLeaf || nKeep>pIter->term.n || nNew==0 ){
11562 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
11563 return;
11564 }
11565 pIter->term.n = nKeep;
11566 fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff])sqlite3Fts5BufferAppendBlob(&p->rc,&pIter->term
,nNew,&a[iOff]);
11567 assert( pIter->term.n<=pIter->term.nSpace )((void) (0));
11568 iOff += nNew;
11569 pIter->iTermLeafOffset = iOff;
11570 pIter->iTermLeafPgno = pIter->iLeafPgno;
11571 pIter->iLeafOffset = iOff;
11572
11573 if( pIter->iPgidxOff>=pIter->pLeaf->nn ){
11574 pIter->iEndofDoclist = pIter->pLeaf->nn+1;
11575 }else{
11576 int nExtra;
11577 pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra)sqlite3Fts5GetVarint32(&a[pIter->iPgidxOff],(u32*)&
(nExtra));
11578 pIter->iEndofDoclist += nExtra;
11579 }
11580
11581 fts5SegIterLoadRowid(p, pIter);
11582}
11583
11584static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*);
11585static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*);
11586static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*);
11587
11588static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){
11589 if( pIter->flags & FTS5_SEGITER_REVERSE0x02 ){
11590 pIter->xNext = fts5SegIterNext_Reverse;
11591 }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE1 ){
11592 pIter->xNext = fts5SegIterNext_None;
11593 }else{
11594 pIter->xNext = fts5SegIterNext;
11595 }
11596}
11597
11598/*
11599** Allocate a tombstone hash page array object (pIter->pTombArray) for
11600** the iterator passed as the second argument. If an OOM error occurs,
11601** leave an error in the Fts5Index object.
11602*/
11603static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){
11604 const i64 nTomb = (i64)pIter->pSeg->nPgTombstone;
11605 if( nTomb>0 ){
11606 i64 nByte = SZ_FTS5TOMBSTONEARRAY(nTomb+1)(__builtin_offsetof(Fts5TombstoneArray, apTombstone)+(nTomb+1
)*sizeof(Fts5Data*));
11607 Fts5TombstoneArray *pNew;
11608 pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte);
11609 if( pNew ){
11610 pNew->nTombstone = nTomb;
11611 pNew->nRef = 1;
11612 pIter->pTombArray = pNew;
11613 }
11614 }
11615}
11616
11617/*
11618** Initialize the iterator object pIter to iterate through the entries in
11619** segment pSeg. The iterator is left pointing to the first entry when
11620** this function returns.
11621**
11622** If an error occurs, Fts5Index.rc is set to an appropriate error code. If
11623** an error has already occurred when this function is called, it is a no-op.
11624*/
11625static void fts5SegIterInit(
11626 Fts5Index *p, /* FTS index object */
11627 Fts5StructureSegment *pSeg, /* Description of segment */
11628 Fts5SegIter *pIter /* Object to populate */
11629){
11630 if( pSeg->pgnoFirst==0 ){
11631 /* This happens if the segment is being used as an input to an incremental
11632 ** merge and all data has already been "trimmed". See function
11633 ** fts5TrimSegments() for details. In this case leave the iterator empty.
11634 ** The caller will see the (pIter->pLeaf==0) and assume the iterator is
11635 ** at EOF already. */
11636 assert( pIter->pLeaf==0 )((void) (0));
11637 return;
11638 }
11639
11640 if( p->rc==SQLITE_OK0 ){
11641 memset(pIter, 0, sizeof(*pIter));
11642 fts5SegIterSetNext(p, pIter);
11643 pIter->pSeg = pSeg;
11644 pIter->iLeafPgno = pSeg->pgnoFirst-1;
11645 do {
11646 fts5SegIterNextPage(p, pIter);
11647 }while( p->rc==SQLITE_OK0 && pIter->pLeaf && pIter->pLeaf->nn==4 );
11648 }
11649
11650 if( p->rc==SQLITE_OK0 && pIter->pLeaf ){
11651 pIter->iLeafOffset = 4;
11652 assert( pIter->pLeaf!=0 )((void) (0));
11653 assert_nc( pIter->pLeaf->nn>4 )((void) (0));
11654 assert_nc( fts5LeafFirstTermOff(pIter->pLeaf)==4 )((void) (0));
11655 pIter->iPgidxOff = pIter->pLeaf->szLeaf+1;
11656 fts5SegIterLoadTerm(p, pIter, 0);
11657 fts5SegIterLoadNPos(p, pIter);
11658 fts5SegIterAllocTombstone(p, pIter);
11659 }
11660}
11661
11662/*
11663** This function is only ever called on iterators created by calls to
11664** Fts5IndexQuery() with the FTS5INDEX_QUERY_DESC flag set.
11665**
11666** The iterator is in an unusual state when this function is called: the
11667** Fts5SegIter.iLeafOffset variable is set to the offset of the start of
11668** the position-list size field for the first relevant rowid on the page.
11669** Fts5SegIter.rowid is set, but nPos and bDel are not.
11670**
11671** This function advances the iterator so that it points to the last
11672** relevant rowid on the page and, if necessary, initializes the
11673** aRowidOffset[] and iRowidOffset variables. At this point the iterator
11674** is in its regular state - Fts5SegIter.iLeafOffset points to the first
11675** byte of the position list content associated with said rowid.
11676*/
11677static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){
11678 int eDetail = p->pConfig->eDetail;
11679 int n = pIter->pLeaf->szLeaf;
11680 int i = pIter->iLeafOffset;
11681 u8 *a = pIter->pLeaf->p;
11682 int iRowidOffset = 0;
11683
11684 if( n>pIter->iEndofDoclist ){
11685 n = pIter->iEndofDoclist;
11686 }
11687
11688 ASSERT_SZLEAF_OK(pIter->pLeaf)((void) (0));
11689 while( 1 ){
11690 u64 iDelta = 0;
11691
11692 if( i>=n ) break;
11693 if( eDetail==FTS5_DETAIL_NONE1 ){
11694 /* todo */
11695 if( i<n && a[i]==0 ){
11696 i++;
11697 if( i<n && a[i]==0 ) i++;
11698 }
11699 }else{
11700 int nPos;
11701 int bDummy;
11702 i += fts5GetPoslistSize(&a[i], &nPos, &bDummy);
11703 i += nPos;
11704 }
11705 if( i>=n ) break;
11706 i += fts5GetVarintsqlite3Fts5GetVarint(&a[i], &iDelta);
11707 pIter->iRowid += iDelta;
11708
11709 /* If necessary, grow the pIter->aRowidOffset[] array. */
11710 if( iRowidOffset>=pIter->nRowidOffset ){
11711 i64 nNew = pIter->nRowidOffset + 8;
11712 int *aNew = (int*)sqlite3_realloc64sqlite3_api->realloc64(pIter->aRowidOffset,nNew*sizeof(int));
11713 if( aNew==0 ){
11714 p->rc = SQLITE_NOMEM7;
11715 break;
11716 }
11717 pIter->aRowidOffset = aNew;
11718 pIter->nRowidOffset = nNew;
11719 }
11720
11721 pIter->aRowidOffset[iRowidOffset++] = pIter->iLeafOffset;
11722 pIter->iLeafOffset = i;
11723 }
11724 pIter->iRowidOffset = iRowidOffset;
11725 fts5SegIterLoadNPos(p, pIter);
11726}
11727
11728/*
11729**
11730*/
11731static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){
11732 assert( pIter->flags & FTS5_SEGITER_REVERSE )((void) (0));
11733 assert( pIter->flags & FTS5_SEGITER_ONETERM )((void) (0));
11734
11735 fts5DataRelease(pIter->pLeaf);
11736 pIter->pLeaf = 0;
11737 while( p->rc==SQLITE_OK0 && pIter->iLeafPgno>pIter->iTermLeafPgno ){
11738 Fts5Data *pNew;
11739 pIter->iLeafPgno--;
11740 pNew = fts5LeafRead(p, FTS5_SEGMENT_ROWID(( ((i64)(pIter->pSeg->iSegid) << (31 +5 +1)) + ((
i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64
)(pIter->iLeafPgno)) )
11741 pIter->pSeg->iSegid, pIter->iLeafPgno( ((i64)(pIter->pSeg->iSegid) << (31 +5 +1)) + ((
i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64
)(pIter->iLeafPgno)) )
11742 )( ((i64)(pIter->pSeg->iSegid) << (31 +5 +1)) + ((
i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64
)(pIter->iLeafPgno)) ));
11743 if( pNew ){
11744 /* iTermLeafOffset may be equal to szLeaf if the term is the last
11745 ** thing on the page - i.e. the first rowid is on the following page.
11746 ** In this case leave pIter->pLeaf==0, this iterator is at EOF. */
11747 if( pIter->iLeafPgno==pIter->iTermLeafPgno ){
11748 assert( pIter->pLeaf==0 )((void) (0));
11749 if( pIter->iTermLeafOffset<pNew->szLeaf ){
11750 pIter->pLeaf = pNew;
11751 pIter->iLeafOffset = pIter->iTermLeafOffset;
11752 }
11753 }else{
11754 int iRowidOff;
11755 iRowidOff = fts5LeafFirstRowidOff(pNew)(fts5GetU16((pNew)->p));
11756 if( iRowidOff ){
11757 if( iRowidOff>=pNew->szLeaf ){
11758 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
11759 }else{
11760 pIter->pLeaf = pNew;
11761 pIter->iLeafOffset = iRowidOff;
11762 }
11763 }
11764 }
11765
11766 if( pIter->pLeaf ){
11767 u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset];
11768 pIter->iLeafOffset += fts5GetVarintsqlite3Fts5GetVarint(a, (u64*)&pIter->iRowid);
11769 break;
11770 }else{
11771 fts5DataRelease(pNew);
11772 }
11773 }
11774 }
11775
11776 if( pIter->pLeaf ){
11777 pIter->iEndofDoclist = pIter->pLeaf->nn+1;
11778 fts5SegIterReverseInitPage(p, pIter);
11779 }
11780}
11781
11782/*
11783** Return true if the iterator passed as the second argument currently
11784** points to a delete marker. A delete marker is an entry with a 0 byte
11785** position-list.
11786*/
11787static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){
11788 Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
11789 return (p->rc==SQLITE_OK0 && pSeg->pLeaf && pSeg->nPos==0);
11790}
11791
11792/*
11793** Advance iterator pIter to the next entry.
11794**
11795** This version of fts5SegIterNext() is only used by reverse iterators.
11796*/
11797static void fts5SegIterNext_Reverse(
11798 Fts5Index *p, /* FTS5 backend object */
11799 Fts5SegIter *pIter, /* Iterator to advance */
11800 int *pbUnused /* Unused */
11801){
11802 assert( pIter->flags & FTS5_SEGITER_REVERSE )((void) (0));
11803 assert( pIter->pNextLeaf==0 )((void) (0));
11804 UNUSED_PARAM(pbUnused)(void)(pbUnused);
11805
11806 if( pIter->iRowidOffset>0 ){
11807 u8 *a = pIter->pLeaf->p;
11808 int iOff;
11809 u64 iDelta;
11810
11811 pIter->iRowidOffset--;
11812 pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset];
11813 fts5SegIterLoadNPos(p, pIter);
11814 iOff = pIter->iLeafOffset;
11815 if( p->pConfig->eDetail!=FTS5_DETAIL_NONE1 ){
11816 iOff += pIter->nPos;
11817 }
11818 fts5GetVarintsqlite3Fts5GetVarint(&a[iOff], &iDelta);
11819 pIter->iRowid -= iDelta;
11820 }else{
11821 fts5SegIterReverseNewPage(p, pIter);
11822 }
11823}
11824
11825/*
11826** Advance iterator pIter to the next entry.
11827**
11828** This version of fts5SegIterNext() is only used if detail=none and the
11829** iterator is not a reverse direction iterator.
11830*/
11831static void fts5SegIterNext_None(
11832 Fts5Index *p, /* FTS5 backend object */
11833 Fts5SegIter *pIter, /* Iterator to advance */
11834 int *pbNewTerm /* OUT: Set for new term */
11835){
11836 int iOff;
11837
11838 assert( p->rc==SQLITE_OK )((void) (0));
11839 assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 )((void) (0));
11840 assert( p->pConfig->eDetail==FTS5_DETAIL_NONE )((void) (0));
11841
11842 ASSERT_SZLEAF_OK(pIter->pLeaf)((void) (0));
11843 iOff = pIter->iLeafOffset;
11844
11845 /* Next entry is on the next page */
11846 while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
11847 fts5SegIterNextPage(p, pIter);
11848 if( p->rc || pIter->pLeaf==0 ) return;
11849 pIter->iRowid = 0;
11850 iOff = 4;
11851 }
11852
11853 if( iOff<pIter->iEndofDoclist ){
11854 /* Next entry is on the current page */
11855 u64 iDelta;
11856 iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta);
11857 pIter->iLeafOffset = iOff;
11858 pIter->iRowid += iDelta;
11859 }else if( (pIter->flags & FTS5_SEGITER_ONETERM0x01)==0 ){
11860 if( pIter->pSeg ){
11861 int nKeep = 0;
11862 if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){
11863 iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep)sqlite3Fts5GetVarint32(&pIter->pLeaf->p[iOff],(u32*
)&(nKeep));
11864 }
11865 pIter->iLeafOffset = iOff;
11866 fts5SegIterLoadTerm(p, pIter, nKeep);
11867 }else{
11868 const u8 *pList = 0;
11869 const char *zTerm = 0;
11870 int nTerm = 0;
11871 int nList;
11872 sqlite3Fts5HashScanNext(p->pHash);
11873 sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList);
11874 if( pList==0 ) goto next_none_eof;
11875 pIter->pLeaf->p = (u8*)pList;
11876 pIter->pLeaf->nn = nList;
11877 pIter->pLeaf->szLeaf = nList;
11878 pIter->iEndofDoclist = nList;
11879 sqlite3Fts5BufferSet(&p->rc,&pIter->term, nTerm, (u8*)zTerm);
11880 pIter->iLeafOffset = fts5GetVarintsqlite3Fts5GetVarint(pList, (u64*)&pIter->iRowid);
11881 }
11882
11883 if( pbNewTerm ) *pbNewTerm = 1;
11884 }else{
11885 goto next_none_eof;
11886 }
11887
11888 fts5SegIterLoadNPos(p, pIter);
11889
11890 return;
11891 next_none_eof:
11892 fts5DataRelease(pIter->pLeaf);
11893 pIter->pLeaf = 0;
11894}
11895
11896
11897/*
11898** Advance iterator pIter to the next entry.
11899**
11900** If an error occurs, Fts5Index.rc is set to an appropriate error code. It
11901** is not considered an error if the iterator reaches EOF. If an error has
11902** already occurred when this function is called, it is a no-op.
11903*/
11904static void fts5SegIterNext(
11905 Fts5Index *p, /* FTS5 backend object */
11906 Fts5SegIter *pIter, /* Iterator to advance */
11907 int *pbNewTerm /* OUT: Set for new term */
11908){
11909 Fts5Data *pLeaf = pIter->pLeaf;
11910 int iOff;
11911 int bNewTerm = 0;
11912 int nKeep = 0;
11913 u8 *a;
11914 int n;
11915
11916 assert( pbNewTerm==0 || *pbNewTerm==0 )((void) (0));
11917 assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE )((void) (0));
11918
11919 /* Search for the end of the position list within the current page. */
11920 a = pLeaf->p;
11921 n = pLeaf->szLeaf;
11922
11923 ASSERT_SZLEAF_OK(pLeaf)((void) (0));
11924 iOff = pIter->iLeafOffset + pIter->nPos;
11925
11926 if( iOff<n ){
11927 /* The next entry is on the current page. */
11928 assert_nc( iOff<=pIter->iEndofDoclist )((void) (0));
11929 if( iOff>=pIter->iEndofDoclist ){
11930 bNewTerm = 1;
11931 if( iOff!=fts5LeafFirstTermOff(pLeaf) ){
11932 iOff += fts5GetVarint32(&a[iOff], nKeep)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nKeep));
11933 }
11934 }else{
11935 u64 iDelta;
11936 iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta);
11937 pIter->iRowid += iDelta;
11938 assert_nc( iDelta>0 )((void) (0));
11939 }
11940 pIter->iLeafOffset = iOff;
11941
11942 }else if( pIter->pSeg==0 ){
11943 const u8 *pList = 0;
11944 const char *zTerm = 0;
11945 int nTerm = 0;
11946 int nList = 0;
11947 assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm )((void) (0));
11948 if( 0==(pIter->flags & FTS5_SEGITER_ONETERM0x01) ){
11949 sqlite3Fts5HashScanNext(p->pHash);
11950 sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList);
11951 }
11952 if( pList==0 ){
11953 fts5DataRelease(pIter->pLeaf);
11954 pIter->pLeaf = 0;
11955 }else{
11956 pIter->pLeaf->p = (u8*)pList;
11957 pIter->pLeaf->nn = nList;
11958 pIter->pLeaf->szLeaf = nList;
11959 pIter->iEndofDoclist = nList+1;
11960 sqlite3Fts5BufferSet(&p->rc, &pIter->term, nTerm, (u8*)zTerm);
11961 pIter->iLeafOffset = fts5GetVarintsqlite3Fts5GetVarint(pList, (u64*)&pIter->iRowid);
11962 *pbNewTerm = 1;
11963 }
11964 }else{
11965 iOff = 0;
11966 /* Next entry is not on the current page */
11967 while( iOff==0 ){
11968 fts5SegIterNextPage(p, pIter);
11969 pLeaf = pIter->pLeaf;
11970 if( pLeaf==0 ) break;
11971 ASSERT_SZLEAF_OK(pLeaf)((void) (0));
11972 if( (iOff = fts5LeafFirstRowidOff(pLeaf)(fts5GetU16((pLeaf)->p))) && iOff<pLeaf->szLeaf ){
11973 iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
11974 pIter->iLeafOffset = iOff;
11975
11976 if( pLeaf->nn>pLeaf->szLeaf ){
11977 pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(sqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(pIter->iEndofDoclist))
11978 &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclistsqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(pIter->iEndofDoclist))
11979 )sqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(pIter->iEndofDoclist));
11980 }
11981 }
11982 else if( pLeaf->nn>pLeaf->szLeaf ){
11983 pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(sqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(iOff))
11984 &pLeaf->p[pLeaf->szLeaf], iOffsqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(iOff))
11985 )sqlite3Fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf],(u32
*)&(iOff));
11986 pIter->iLeafOffset = iOff;
11987 pIter->iEndofDoclist = iOff;
11988 bNewTerm = 1;
11989 }
11990 assert_nc( iOff<pLeaf->szLeaf )((void) (0));
11991 if( iOff>pLeaf->szLeaf ){
11992 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
11993 return;
11994 }
11995 }
11996 }
11997
11998 /* Check if the iterator is now at EOF. If so, return early. */
11999 if( pIter->pLeaf ){
12000 if( bNewTerm ){
12001 if( pIter->flags & FTS5_SEGITER_ONETERM0x01 ){
12002 fts5DataRelease(pIter->pLeaf);
12003 pIter->pLeaf = 0;
12004 }else{
12005 fts5SegIterLoadTerm(p, pIter, nKeep);
12006 fts5SegIterLoadNPos(p, pIter);
12007 if( pbNewTerm ) *pbNewTerm = 1;
12008 }
12009 }else{
12010 /* The following could be done by calling fts5SegIterLoadNPos(). But
12011 ** this block is particularly performance critical, so equivalent
12012 ** code is inlined. */
12013 int nSz;
12014 assert_nc( pIter->iLeafOffset<=pIter->pLeaf->nn )((void) (0));
12015 fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz){ nSz = (pIter->pLeaf->p)[pIter->iLeafOffset++]; if(
nSz & 0x80 ){ pIter->iLeafOffset--; pIter->iLeafOffset
+= sqlite3Fts5GetVarint32(&(pIter->pLeaf->p)[pIter
->iLeafOffset],(u32*)&(nSz)); } };
12016 pIter->bDel = (nSz & 0x0001);
12017 pIter->nPos = nSz>>1;
12018 assert_nc( pIter->nPos>=0 )((void) (0));
12019 }
12020 }
12021}
12022
12023#define SWAPVAL(T, a, b){ T tmp; tmp=a; a=b; b=tmp; } { T tmp; tmp=a; a=b; b=tmp; }
12024
12025#define fts5IndexSkipVarint(a, iOff){ int iEnd = iOff+9; while( (a[iOff++] & 0x80) &&
iOff<iEnd ); } { \
12026 int iEnd = iOff+9; \
12027 while( (a[iOff++] & 0x80) && iOff<iEnd ); \
12028}
12029
12030/*
12031** Iterator pIter currently points to the first rowid in a doclist. This
12032** function sets the iterator up so that iterates in reverse order through
12033** the doclist.
12034*/
12035static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
12036 Fts5DlidxIter *pDlidx = pIter->pDlidx;
12037 Fts5Data *pLast = 0;
12038 int pgnoLast = 0;
12039
12040 if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION4 ){
12041 int iSegid = pIter->pSeg->iSegid;
12042 pgnoLast = fts5DlidxIterPgno(pDlidx);
12043 pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pgnoLast)) ));
12044 }else{
12045 Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */
12046
12047 /* Currently, Fts5SegIter.iLeafOffset points to the first byte of
12048 ** position-list content for the current rowid. Back it up so that it
12049 ** points to the start of the position-list size field. */
12050 int iPoslist;
12051 if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
12052 iPoslist = pIter->iTermLeafOffset;
12053 }else{
12054 iPoslist = 4;
12055 }
12056 fts5IndexSkipVarint(pLeaf->p, iPoslist){ int iEnd = iPoslist+9; while( (pLeaf->p[iPoslist++] &
0x80) && iPoslist<iEnd ); };
12057 pIter->iLeafOffset = iPoslist;
12058
12059 /* If this condition is true then the largest rowid for the current
12060 ** term may not be stored on the current page. So search forward to
12061 ** see where said rowid really is. */
12062 if( pIter->iEndofDoclist>=pLeaf->szLeaf ){
12063 int pgno;
12064 Fts5StructureSegment *pSeg = pIter->pSeg;
12065
12066 /* The last rowid in the doclist may not be on the current page. Search
12067 ** forward to find the page containing the last rowid. */
12068 for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){
12069 i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno)( ((i64)(pSeg->iSegid) << (31 +5 +1)) + ((i64)(0) <<
(31 + 5)) + ((i64)(0) << (31)) + ((i64)(pgno)) );
12070 Fts5Data *pNew = fts5LeafRead(p, iAbs);
12071 if( pNew ){
12072 int iRowid, bTermless;
12073 iRowid = fts5LeafFirstRowidOff(pNew)(fts5GetU16((pNew)->p));
12074 bTermless = fts5LeafIsTermless(pNew)((pNew)->szLeaf >= (pNew)->nn);
12075 if( iRowid ){
12076 SWAPVAL(Fts5Data*, pNew, pLast){ Fts5Data* tmp; tmp=pNew; pNew=pLast; pLast=tmp; };
12077 pgnoLast = pgno;
12078 }
12079 fts5DataRelease(pNew);
12080 if( bTermless==0 ) break;
12081 }
12082 }
12083 }
12084 }
12085
12086 /* If pLast is NULL at this point, then the last rowid for this doclist
12087 ** lies on the page currently indicated by the iterator. In this case
12088 ** pIter->iLeafOffset is already set to point to the position-list size
12089 ** field associated with the first relevant rowid on the page.
12090 **
12091 ** Or, if pLast is non-NULL, then it is the page that contains the last
12092 ** rowid. In this case configure the iterator so that it points to the
12093 ** first rowid on this page.
12094 */
12095 if( pLast ){
12096 int iOff;
12097 fts5DataRelease(pIter->pLeaf);
12098 pIter->pLeaf = pLast;
12099 pIter->iLeafPgno = pgnoLast;
12100 if( p->rc==SQLITE_OK0 ){
12101 iOff = fts5LeafFirstRowidOff(pLast)(fts5GetU16((pLast)->p));
12102 if( iOff>pLast->szLeaf ){
12103 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12104 return;
12105 }
12106 iOff += fts5GetVarintsqlite3Fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid);
12107 pIter->iLeafOffset = iOff;
12108
12109 if( fts5LeafIsTermless(pLast)((pLast)->szLeaf >= (pLast)->nn) ){
12110 pIter->iEndofDoclist = pLast->nn+1;
12111 }else{
12112 pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast);
12113 }
12114 }
12115 }
12116
12117 fts5SegIterReverseInitPage(p, pIter);
12118}
12119
12120/*
12121** Iterator pIter currently points to the first rowid of a doclist.
12122** There is a doclist-index associated with the final term on the current
12123** page. If the current term is the last term on the page, load the
12124** doclist-index from disk and initialize an iterator at (pIter->pDlidx).
12125*/
12126static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){
12127 int iSeg = pIter->pSeg->iSegid;
12128 int bRev = (pIter->flags & FTS5_SEGITER_REVERSE0x02);
12129 Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */
12130
12131 assert( pIter->flags & FTS5_SEGITER_ONETERM )((void) (0));
12132 assert( pIter->pDlidx==0 )((void) (0));
12133
12134 /* Check if the current doclist ends on this page. If it does, return
12135 ** early without loading the doclist-index (as it belongs to a different
12136 ** term. */
12137 if( pIter->iTermLeafPgno==pIter->iLeafPgno
12138 && pIter->iEndofDoclist<pLeaf->szLeaf
12139 ){
12140 return;
12141 }
12142
12143 pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
12144}
12145
12146/*
12147** The iterator object passed as the second argument currently contains
12148** no valid values except for the Fts5SegIter.pLeaf member variable. This
12149** function searches the leaf page for a term matching (pTerm/nTerm).
12150**
12151** If the specified term is found on the page, then the iterator is left
12152** pointing to it. If argument bGe is zero and the term is not found,
12153** the iterator is left pointing at EOF.
12154**
12155** If bGe is non-zero and the specified term is not found, then the
12156** iterator is left pointing to the smallest term in the segment that
12157** is larger than the specified term, even if this term is not on the
12158** current page.
12159*/
12160static void fts5LeafSeek(
12161 Fts5Index *p, /* Leave any error code here */
12162 int bGe, /* True for a >= search */
12163 Fts5SegIter *pIter, /* Iterator to seek */
12164 const u8 *pTerm, int nTerm /* Term to search for */
12165){
12166 u32 iOff;
12167 const u8 *a = pIter->pLeaf->p;
12168 u32 n = (u32)pIter->pLeaf->nn;
12169
12170 u32 nMatch = 0;
12171 u32 nKeep = 0;
12172 u32 nNew = 0;
12173 u32 iTermOff;
12174 u32 iPgidx; /* Current offset in pgidx */
12175 int bEndOfPage = 0;
12176
12177 assert( p->rc==SQLITE_OK )((void) (0));
12178
12179 iPgidx = (u32)pIter->pLeaf->szLeaf;
12180 iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff)sqlite3Fts5GetVarint32(&a[iPgidx],(u32*)&(iTermOff));
12181 iOff = iTermOff;
12182 if( iOff>n ){
12183 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12184 return;
12185 }
12186
12187 while( 1 ){
12188
12189 /* Figure out how many new bytes are in this term */
12190 fts5FastGetVarint32(a, iOff, nNew){ nNew = (a)[iOff++]; if( nNew & 0x80 ){ iOff--; iOff += sqlite3Fts5GetVarint32
(&(a)[iOff],(u32*)&(nNew)); } };
12191 if( nKeep<nMatch ){
12192 goto search_failed;
12193 }
12194 if( (iOff+nNew)>n ){
12195 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12196 return;
12197 }
12198
12199 assert( nKeep>=nMatch )((void) (0));
12200 if( nKeep==nMatch ){
12201 u32 nCmp;
12202 u32 i;
12203 nCmp = (u32)MIN(nNew, nTerm-nMatch)(((nNew) < (nTerm-nMatch)) ? (nNew) : (nTerm-nMatch));
12204 for(i=0; i<nCmp; i++){
12205 if( a[iOff+i]!=pTerm[nMatch+i] ) break;
12206 }
12207 nMatch += i;
12208
12209 if( (u32)nTerm==nMatch ){
12210 if( i==nNew ){
12211 goto search_success;
12212 }else{
12213 goto search_failed;
12214 }
12215 }else if( i<nNew && a[iOff+i]>pTerm[nMatch] ){
12216 goto search_failed;
12217 }
12218 }
12219
12220 if( iPgidx>=n ){
12221 bEndOfPage = 1;
12222 break;
12223 }
12224
12225 iPgidx += fts5GetVarint32(&a[iPgidx], nKeep)sqlite3Fts5GetVarint32(&a[iPgidx],(u32*)&(nKeep));
12226 iTermOff += nKeep;
12227 iOff = iTermOff;
12228
12229 if( iOff>=n ){
12230 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12231 return;
12232 }
12233
12234 /* Read the nKeep field of the next term. */
12235 fts5FastGetVarint32(a, iOff, nKeep){ nKeep = (a)[iOff++]; if( nKeep & 0x80 ){ iOff--; iOff +=
sqlite3Fts5GetVarint32(&(a)[iOff],(u32*)&(nKeep)); }
};
12236 }
12237
12238 search_failed:
12239 if( bGe==0 ){
12240 fts5DataRelease(pIter->pLeaf);
12241 pIter->pLeaf = 0;
12242 return;
12243 }else if( bEndOfPage ){
12244 do {
12245 fts5SegIterNextPage(p, pIter);
12246 if( pIter->pLeaf==0 ) return;
12247 a = pIter->pLeaf->p;
12248 if( fts5LeafIsTermless(pIter->pLeaf)((pIter->pLeaf)->szLeaf >= (pIter->pLeaf)->nn)==0 ){
12249 iPgidx = (u32)pIter->pLeaf->szLeaf;
12250 iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff)sqlite3Fts5GetVarint32(&pIter->pLeaf->p[iPgidx],(u32
*)&(iOff));
12251 if( iOff<4 || (i64)iOff>=pIter->pLeaf->szLeaf ){
12252 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12253 return;
12254 }else{
12255 nKeep = 0;
12256 iTermOff = iOff;
12257 n = (u32)pIter->pLeaf->nn;
12258 iOff += fts5GetVarint32(&a[iOff], nNew)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nNew));
12259 break;
12260 }
12261 }
12262 }while( 1 );
12263 }
12264
12265 search_success:
12266 if( (i64)iOff+nNew>n || nNew<1 ){
12267 FTS5_CORRUPT_ITER(p, pIter)fts5IndexCorruptIter(p, pIter);
12268 return;
12269 }
12270 pIter->iLeafOffset = iOff + nNew;
12271 pIter->iTermLeafOffset = pIter->iLeafOffset;
12272 pIter->iTermLeafPgno = pIter->iLeafPgno;
12273
12274 fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm)sqlite3Fts5BufferSet(&p->rc,&pIter->term,nKeep,
pTerm);
12275 fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff])sqlite3Fts5BufferAppendBlob(&p->rc,&pIter->term
,nNew,&a[iOff]);
12276
12277 if( iPgidx>=n ){
12278 pIter->iEndofDoclist = pIter->pLeaf->nn+1;
12279 }else{
12280 int nExtra;
12281 iPgidx += fts5GetVarint32(&a[iPgidx], nExtra)sqlite3Fts5GetVarint32(&a[iPgidx],(u32*)&(nExtra));
12282 pIter->iEndofDoclist = iTermOff + nExtra;
12283 }
12284 pIter->iPgidxOff = iPgidx;
12285
12286 fts5SegIterLoadRowid(p, pIter);
12287 fts5SegIterLoadNPos(p, pIter);
12288}
12289
12290static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){
12291 if( p->pIdxSelect==0 ){
12292 Fts5Config *pConfig = p->pConfig;
12293 fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintfsqlite3_api->mprintf(
12294 "SELECT pgno FROM '%q'.'%q_idx' WHERE "
12295 "segid=? AND term<=? ORDER BY term DESC LIMIT 1",
12296 pConfig->zDb, pConfig->zName
12297 ));
12298 }
12299 return p->pIdxSelect;
12300}
12301
12302/*
12303** Initialize the object pIter to point to term pTerm/nTerm within segment
12304** pSeg. If there is no such term in the index, the iterator is set to EOF.
12305**
12306** If an error occurs, Fts5Index.rc is set to an appropriate error code. If
12307** an error has already occurred when this function is called, it is a no-op.
12308*/
12309static void fts5SegIterSeekInit(
12310 Fts5Index *p, /* FTS5 backend */
12311 const u8 *pTerm, int nTerm, /* Term to seek to */
12312 int flags, /* Mask of FTS5INDEX_XXX flags */
12313 Fts5StructureSegment *pSeg, /* Description of segment */
12314 Fts5SegIter *pIter /* Object to populate */
12315){
12316 int iPg = 1;
12317 int bGe = (flags & FTS5INDEX_QUERY_SCAN0x0008);
12318 int bDlidx = 0; /* True if there is a doclist-index */
12319 sqlite3_stmt *pIdxSelect = 0;
12320
12321 assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 )((void) (0));
12322 assert( pTerm && nTerm )((void) (0));
12323 memset(pIter, 0, sizeof(*pIter));
12324 pIter->pSeg = pSeg;
12325
12326 /* This block sets stack variable iPg to the leaf page number that may
12327 ** contain term (pTerm/nTerm), if it is present in the segment. */
12328 pIdxSelect = fts5IdxSelectStmt(p);
12329 if( p->rc ) return;
12330 sqlite3_bind_intsqlite3_api->bind_int(pIdxSelect, 1, pSeg->iSegid);
12331 sqlite3_bind_blobsqlite3_api->bind_blob(pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC((sqlite3_destructor_type)0));
12332 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pIdxSelect) ){
12333 i64 val = sqlite3_column_intsqlite3_api->column_int(pIdxSelect, 0);
12334 iPg = (int)(val>>1);
12335 bDlidx = (val & 0x0001);
12336 }
12337 p->rc = sqlite3_resetsqlite3_api->reset(pIdxSelect);
12338 sqlite3_bind_nullsqlite3_api->bind_null(pIdxSelect, 2);
12339
12340 if( iPg<pSeg->pgnoFirst ){
12341 iPg = pSeg->pgnoFirst;
12342 bDlidx = 0;
12343 }
12344
12345 pIter->iLeafPgno = iPg - 1;
12346 fts5SegIterNextPage(p, pIter);
12347
12348 if( pIter->pLeaf ){
12349 fts5LeafSeek(p, bGe, pIter, pTerm, nTerm);
12350 }
12351
12352 if( p->rc==SQLITE_OK0 && (bGe==0 || (flags & FTS5INDEX_QUERY_SCANONETERM0x0100)) ){
12353 pIter->flags |= FTS5_SEGITER_ONETERM0x01;
12354 if( pIter->pLeaf ){
12355 if( flags & FTS5INDEX_QUERY_DESC0x0002 ){
12356 pIter->flags |= FTS5_SEGITER_REVERSE0x02;
12357 }
12358 if( bDlidx ){
12359 fts5SegIterLoadDlidx(p, pIter);
12360 }
12361 if( flags & FTS5INDEX_QUERY_DESC0x0002 ){
12362 fts5SegIterReverse(p, pIter);
12363 }
12364 }
12365 }
12366
12367 fts5SegIterSetNext(p, pIter);
12368 if( 0==(flags & FTS5INDEX_QUERY_SCANONETERM0x0100) ){
12369 fts5SegIterAllocTombstone(p, pIter);
12370 }
12371
12372 /* Either:
12373 **
12374 ** 1) an error has occurred, or
12375 ** 2) the iterator points to EOF, or
12376 ** 3) the iterator points to an entry with term (pTerm/nTerm), or
12377 ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points
12378 ** to an entry with a term greater than or equal to (pTerm/nTerm).
12379 */
12380 assert_nc( p->rc!=SQLITE_OK /* 1 */((void) (0))
12381 || pIter->pLeaf==0 /* 2 */((void) (0))
12382 || fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)==0 /* 3 */((void) (0))
12383 || (bGe && fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0) /* 4 */((void) (0))
12384 )((void) (0));
12385}
12386
12387
12388/*
12389** SQL used by fts5SegIterNextInit() to find the page to open.
12390*/
12391static sqlite3_stmt *fts5IdxNextStmt(Fts5Index *p){
12392 if( p->pIdxNextSelect==0 ){
12393 Fts5Config *pConfig = p->pConfig;
12394 fts5IndexPrepareStmt(p, &p->pIdxNextSelect, sqlite3_mprintfsqlite3_api->mprintf(
12395 "SELECT pgno FROM '%q'.'%q_idx' WHERE "
12396 "segid=? AND term>? ORDER BY term ASC LIMIT 1",
12397 pConfig->zDb, pConfig->zName
12398 ));
12399
12400 }
12401 return p->pIdxNextSelect;
12402}
12403
12404/*
12405** This is similar to fts5SegIterSeekInit(), except that it initializes
12406** the segment iterator to point to the first term following the page
12407** with pToken/nToken on it.
12408*/
12409static void fts5SegIterNextInit(
12410 Fts5Index *p,
12411 const char *pTerm, int nTerm,
12412 Fts5StructureSegment *pSeg, /* Description of segment */
12413 Fts5SegIter *pIter /* Object to populate */
12414){
12415 int iPg = -1; /* Page of segment to open */
12416 int bDlidx = 0;
12417 sqlite3_stmt *pSel = 0; /* SELECT to find iPg */
12418
12419 pSel = fts5IdxNextStmt(p);
12420 if( pSel ){
12421 assert( p->rc==SQLITE_OK )((void) (0));
12422 sqlite3_bind_intsqlite3_api->bind_int(pSel, 1, pSeg->iSegid);
12423 sqlite3_bind_blobsqlite3_api->bind_blob(pSel, 2, pTerm, nTerm, SQLITE_STATIC((sqlite3_destructor_type)0));
12424
12425 if( sqlite3_stepsqlite3_api->step(pSel)==SQLITE_ROW100 ){
12426 i64 val = sqlite3_column_int64sqlite3_api->column_int64(pSel, 0);
12427 iPg = (int)(val>>1);
12428 bDlidx = (val & 0x0001);
12429 }
12430 p->rc = sqlite3_resetsqlite3_api->reset(pSel);
12431 sqlite3_bind_nullsqlite3_api->bind_null(pSel, 2);
12432 if( p->rc ) return;
12433 }
12434
12435 memset(pIter, 0, sizeof(*pIter));
12436 pIter->pSeg = pSeg;
12437 pIter->flags |= FTS5_SEGITER_ONETERM0x01;
12438 if( iPg>=0 ){
12439 pIter->iLeafPgno = iPg - 1;
12440 fts5SegIterNextPage(p, pIter);
12441 fts5SegIterSetNext(p, pIter);
12442 }
12443 if( pIter->pLeaf ){
12444 const u8 *a = pIter->pLeaf->p;
12445 int iTermOff = 0;
12446
12447 pIter->iPgidxOff = pIter->pLeaf->szLeaf;
12448 pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], iTermOff)sqlite3Fts5GetVarint32(&a[pIter->iPgidxOff],(u32*)&
(iTermOff));
12449 if( iTermOff > pIter->pLeaf->szLeaf ){
12450 p->rc = FTS5_CORRUPT(11 | (1<<8));
12451 return;
12452 }
12453 pIter->iLeafOffset = iTermOff;
12454 fts5SegIterLoadTerm(p, pIter, 0);
12455 fts5SegIterLoadNPos(p, pIter);
12456 if( bDlidx ) fts5SegIterLoadDlidx(p, pIter);
12457
12458 assert( p->rc!=SQLITE_OK ||((void) (0))
12459 fts5BufferCompareBlob(&pIter->term, (const u8*)pTerm, nTerm)>0((void) (0))
12460 )((void) (0));
12461 }
12462}
12463
12464/*
12465** Initialize the object pIter to point to term pTerm/nTerm within the
12466** in-memory hash table. If there is no such term in the hash-table, the
12467** iterator is set to EOF.
12468**
12469** If an error occurs, Fts5Index.rc is set to an appropriate error code. If
12470** an error has already occurred when this function is called, it is a no-op.
12471*/
12472static void fts5SegIterHashInit(
12473 Fts5Index *p, /* FTS5 backend */
12474 const u8 *pTerm, int nTerm, /* Term to seek to */
12475 int flags, /* Mask of FTS5INDEX_XXX flags */
12476 Fts5SegIter *pIter /* Object to populate */
12477){
12478 int nList = 0;
12479 const u8 *z = 0;
12480 int n = 0;
12481 Fts5Data *pLeaf = 0;
12482
12483 assert( p->pHash )((void) (0));
12484 assert( p->rc==SQLITE_OK )((void) (0));
12485
12486 if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN0x0008) ){
12487 const u8 *pList = 0;
12488
12489 p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm);
12490 sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &n, &pList, &nList);
12491 if( pList ){
12492 pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data));
12493 if( pLeaf ){
12494 pLeaf->p = (u8*)pList;
12495 }
12496 }
12497
12498 /* The call to sqlite3Fts5HashScanInit() causes the hash table to
12499 ** fill the size field of all existing position lists. This means they
12500 ** can no longer be appended to. Since the only scenario in which they
12501 ** can be appended to is if the previous operation on this table was
12502 ** a DELETE, by clearing the Fts5Index.bDelete flag we can avoid this
12503 ** possibility altogether. */
12504 p->bDelete = 0;
12505 }else{
12506 p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data),
12507 (const char*)pTerm, nTerm, (void**)&pLeaf, &nList
12508 );
12509 if( pLeaf ){
12510 pLeaf->p = (u8*)&pLeaf[1];
12511 }
12512 z = pTerm;
12513 n = nTerm;
12514 pIter->flags |= FTS5_SEGITER_ONETERM0x01;
12515 }
12516
12517 if( pLeaf ){
12518 sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z);
12519 pLeaf->nn = pLeaf->szLeaf = nList;
12520 pIter->pLeaf = pLeaf;
12521 pIter->iLeafOffset = fts5GetVarintsqlite3Fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid);
12522 pIter->iEndofDoclist = pLeaf->nn;
12523
12524 if( flags & FTS5INDEX_QUERY_DESC0x0002 ){
12525 pIter->flags |= FTS5_SEGITER_REVERSE0x02;
12526 fts5SegIterReverseInitPage(p, pIter);
12527 }else{
12528 fts5SegIterLoadNPos(p, pIter);
12529 }
12530 }
12531
12532 fts5SegIterSetNext(p, pIter);
12533}
12534
12535/*
12536** Array ap[] contains n elements. Release each of these elements using
12537** fts5DataRelease(). Then free the array itself using sqlite3_free().
12538*/
12539static void fts5IndexFreeArray(Fts5Data **ap, int n){
12540 if( ap ){
12541 int ii;
12542 for(ii=0; ii<n; ii++){
12543 fts5DataRelease(ap[ii]);
12544 }
12545 sqlite3_freesqlite3_api->free(ap);
12546 }
12547}
12548
12549/*
12550** Decrement the ref-count of the object passed as the only argument. If it
12551** reaches 0, free it and its contents.
12552*/
12553static void fts5TombstoneArrayDelete(Fts5TombstoneArray *p){
12554 if( p ){
12555 p->nRef--;
12556 if( p->nRef<=0 ){
12557 int ii;
12558 for(ii=0; ii<p->nTombstone; ii++){
12559 fts5DataRelease(p->apTombstone[ii]);
12560 }
12561 sqlite3_freesqlite3_api->free(p);
12562 }
12563 }
12564}
12565
12566/*
12567** Zero the iterator passed as the only argument.
12568*/
12569static void fts5SegIterClear(Fts5SegIter *pIter){
12570 fts5BufferFree(&pIter->term)sqlite3Fts5BufferFree(&pIter->term);
12571 fts5DataRelease(pIter->pLeaf);
12572 fts5DataRelease(pIter->pNextLeaf);
12573 fts5TombstoneArrayDelete(pIter->pTombArray);
12574 fts5DlidxIterFree(pIter->pDlidx);
12575 sqlite3_freesqlite3_api->free(pIter->aRowidOffset);
12576 memset(pIter, 0, sizeof(Fts5SegIter));
12577}
12578
12579#ifdef SQLITE_DEBUG
12580
12581/*
12582** This function is used as part of the big assert() procedure implemented by
12583** fts5AssertMultiIterSetup(). It ensures that the result currently stored
12584** in *pRes is the correct result of comparing the current positions of the
12585** two iterators.
12586*/
12587static void fts5AssertComparisonResult(
12588 Fts5Iter *pIter,
12589 Fts5SegIter *p1,
12590 Fts5SegIter *p2,
12591 Fts5CResult *pRes
12592){
12593 int i1 = p1 - pIter->aSeg;
12594 int i2 = p2 - pIter->aSeg;
12595
12596 if( p1->pLeaf || p2->pLeaf ){
12597 if( p1->pLeaf==0 ){
12598 assert( pRes->iFirst==i2 )((void) (0));
12599 }else if( p2->pLeaf==0 ){
12600 assert( pRes->iFirst==i1 )((void) (0));
12601 }else{
12602 int nMin = MIN(p1->term.n, p2->term.n)(((p1->term.n) < (p2->term.n)) ? (p1->term.n) : (
p2->term.n));
12603 int res = fts5Memcmp(p1->term.p, p2->term.p, nMin)((nMin)<=0 ? 0 : memcmp((p1->term.p), (p2->term.p), (
nMin)));
12604 if( res==0 ) res = p1->term.n - p2->term.n;
12605
12606 if( res==0 ){
12607 assert( pRes->bTermEq==1 )((void) (0));
12608 assert( p1->iRowid!=p2->iRowid )((void) (0));
12609 res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : 1;
12610 }else{
12611 assert( pRes->bTermEq==0 )((void) (0));
12612 }
12613
12614 if( res<0 ){
12615 assert( pRes->iFirst==i1 )((void) (0));
12616 }else{
12617 assert( pRes->iFirst==i2 )((void) (0));
12618 }
12619 }
12620 }
12621}
12622
12623/*
12624** This function is a no-op unless SQLITE_DEBUG is defined when this module
12625** is compiled. In that case, this function is essentially an assert()
12626** statement used to verify that the contents of the pIter->aFirst[] array
12627** are correct.
12628*/
12629static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){
12630 if( p->rc==SQLITE_OK0 ){
12631 Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
12632 int i;
12633
12634 assert( (pFirst->pLeaf==0)==pIter->base.bEof )((void) (0));
12635
12636 /* Check that pIter->iSwitchRowid is set correctly. */
12637 for(i=0; i<pIter->nSeg; i++){
12638 Fts5SegIter *p1 = &pIter->aSeg[i];
12639 assert( p1==pFirst((void) (0))
12640 || p1->pLeaf==0((void) (0))
12641 || fts5BufferCompare(&pFirst->term, &p1->term)((void) (0))
12642 || p1->iRowid==pIter->iSwitchRowid((void) (0))
12643 || (p1->iRowid<pIter->iSwitchRowid)==pIter->bRev((void) (0))
12644 )((void) (0));
12645 }
12646
12647 for(i=0; i<pIter->nSeg; i+=2){
12648 Fts5SegIter *p1 = &pIter->aSeg[i];
12649 Fts5SegIter *p2 = &pIter->aSeg[i+1];
12650 Fts5CResult *pRes = &pIter->aFirst[(pIter->nSeg + i) / 2];
12651 fts5AssertComparisonResult(pIter, p1, p2, pRes);
12652 }
12653
12654 for(i=1; i<(pIter->nSeg / 2); i+=2){
12655 Fts5SegIter *p1 = &pIter->aSeg[ pIter->aFirst[i*2].iFirst ];
12656 Fts5SegIter *p2 = &pIter->aSeg[ pIter->aFirst[i*2+1].iFirst ];
12657 Fts5CResult *pRes = &pIter->aFirst[i];
12658 fts5AssertComparisonResult(pIter, p1, p2, pRes);
12659 }
12660 }
12661}
12662#else
12663# define fts5AssertMultiIterSetup(x,y)
12664#endif
12665
12666/*
12667** Do the comparison necessary to populate pIter->aFirst[iOut].
12668**
12669** If the returned value is non-zero, then it is the index of an entry
12670** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing
12671** to a key that is a duplicate of another, higher priority,
12672** segment-iterator in the pSeg->aSeg[] array.
12673*/
12674static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
12675 int i1; /* Index of left-hand Fts5SegIter */
12676 int i2; /* Index of right-hand Fts5SegIter */
12677 int iRes;
12678 Fts5SegIter *p1; /* Left-hand Fts5SegIter */
12679 Fts5SegIter *p2; /* Right-hand Fts5SegIter */
12680 Fts5CResult *pRes = &pIter->aFirst[iOut];
12681
12682 assert( iOut<pIter->nSeg && iOut>0 )((void) (0));
12683 assert( pIter->bRev==0 || pIter->bRev==1 )((void) (0));
12684
12685 if( iOut>=(pIter->nSeg/2) ){
12686 i1 = (iOut - pIter->nSeg/2) * 2;
12687 i2 = i1 + 1;
12688 }else{
12689 i1 = pIter->aFirst[iOut*2].iFirst;
12690 i2 = pIter->aFirst[iOut*2+1].iFirst;
12691 }
12692 p1 = &pIter->aSeg[i1];
12693 p2 = &pIter->aSeg[i2];
12694
12695 pRes->bTermEq = 0;
12696 if( p1->pLeaf==0 ){ /* If p1 is at EOF */
12697 iRes = i2;
12698 }else if( p2->pLeaf==0 ){ /* If p2 is at EOF */
12699 iRes = i1;
12700 }else{
12701 int res = fts5BufferCompare(&p1->term, &p2->term);
12702 if( res==0 ){
12703 assert_nc( i2>i1 )((void) (0));
12704 assert_nc( i2!=0 )((void) (0));
12705 pRes->bTermEq = 1;
12706 if( p1->iRowid==p2->iRowid ){
12707 return i2;
12708 }
12709 res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1;
12710 }
12711 assert( res!=0 )((void) (0));
12712 if( res<0 ){
12713 iRes = i1;
12714 }else{
12715 iRes = i2;
12716 }
12717 }
12718
12719 pRes->iFirst = (u16)iRes;
12720 return 0;
12721}
12722
12723/*
12724** Move the seg-iter so that it points to the first rowid on page iLeafPgno.
12725** It is an error if leaf iLeafPgno does not exist. Unless the db is
12726** a 'secure-delete' db, if it contains no rowids then this is also an error.
12727*/
12728static void fts5SegIterGotoPage(
12729 Fts5Index *p, /* FTS5 backend object */
12730 Fts5SegIter *pIter, /* Iterator to advance */
12731 int iLeafPgno
12732){
12733 assert( iLeafPgno>pIter->iLeafPgno )((void) (0));
12734
12735 if( iLeafPgno>pIter->pSeg->pgnoLast ){
12736 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
12737 }else{
12738 fts5DataRelease(pIter->pNextLeaf);
12739 pIter->pNextLeaf = 0;
12740 pIter->iLeafPgno = iLeafPgno-1;
12741
12742 while( p->rc==SQLITE_OK0 ){
12743 int iOff;
12744 fts5SegIterNextPage(p, pIter);
12745 if( pIter->pLeaf==0 ) break;
12746 iOff = fts5LeafFirstRowidOff(pIter->pLeaf)(fts5GetU16((pIter->pLeaf)->p));
12747 if( iOff>0 ){
12748 u8 *a = pIter->pLeaf->p;
12749 int n = pIter->pLeaf->szLeaf;
12750 if( iOff<4 || iOff>=n ){
12751 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
12752 }else{
12753 iOff += fts5GetVarintsqlite3Fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
12754 pIter->iLeafOffset = iOff;
12755 fts5SegIterLoadNPos(p, pIter);
12756 }
12757 break;
12758 }
12759 }
12760 }
12761}
12762
12763/*
12764** Advance the iterator passed as the second argument until it is at or
12765** past rowid iFrom. Regardless of the value of iFrom, the iterator is
12766** always advanced at least once.
12767*/
12768static void fts5SegIterNextFrom(
12769 Fts5Index *p, /* FTS5 backend object */
12770 Fts5SegIter *pIter, /* Iterator to advance */
12771 i64 iMatch /* Advance iterator at least this far */
12772){
12773 int bRev = (pIter->flags & FTS5_SEGITER_REVERSE0x02);
12774 Fts5DlidxIter *pDlidx = pIter->pDlidx;
12775 int iLeafPgno = pIter->iLeafPgno;
12776 int bMove = 1;
12777
12778 assert( pIter->flags & FTS5_SEGITER_ONETERM )((void) (0));
12779 assert( pIter->pDlidx )((void) (0));
12780 assert( pIter->pLeaf )((void) (0));
12781
12782 if( bRev==0 ){
12783 while( !fts5DlidxIterEof(p, pDlidx) && iMatch>fts5DlidxIterRowid(pDlidx) ){
12784 iLeafPgno = fts5DlidxIterPgno(pDlidx);
12785 fts5DlidxIterNext(p, pDlidx);
12786 }
12787 assert_nc( iLeafPgno>=pIter->iLeafPgno || p->rc )((void) (0));
12788 if( iLeafPgno>pIter->iLeafPgno ){
12789 fts5SegIterGotoPage(p, pIter, iLeafPgno);
12790 bMove = 0;
12791 }
12792 }else{
12793 assert( pIter->pNextLeaf==0 )((void) (0));
12794 assert( iMatch<pIter->iRowid )((void) (0));
12795 while( !fts5DlidxIterEof(p, pDlidx) && iMatch<fts5DlidxIterRowid(pDlidx) ){
12796 fts5DlidxIterPrev(p, pDlidx);
12797 }
12798 iLeafPgno = fts5DlidxIterPgno(pDlidx);
12799
12800 assert( fts5DlidxIterEof(p, pDlidx) || iLeafPgno<=pIter->iLeafPgno )((void) (0));
12801
12802 if( iLeafPgno<pIter->iLeafPgno ){
12803 pIter->iLeafPgno = iLeafPgno+1;
12804 fts5SegIterReverseNewPage(p, pIter);
12805 bMove = 0;
12806 }
12807 }
12808
12809 do{
12810 if( bMove && p->rc==SQLITE_OK0 ) pIter->xNext(p, pIter, 0);
12811 if( pIter->pLeaf==0 ) break;
12812 if( bRev==0 && pIter->iRowid>=iMatch ) break;
12813 if( bRev!=0 && pIter->iRowid<=iMatch ) break;
12814 bMove = 1;
12815 }while( p->rc==SQLITE_OK0 );
12816}
12817
12818/*
12819** Free the iterator object passed as the second argument.
12820*/
12821static void fts5MultiIterFree(Fts5Iter *pIter){
12822 if( pIter ){
12823 int i;
12824 for(i=0; i<pIter->nSeg; i++){
12825 fts5SegIterClear(&pIter->aSeg[i]);
12826 }
12827 fts5BufferFree(&pIter->poslist)sqlite3Fts5BufferFree(&pIter->poslist);
12828 sqlite3_freesqlite3_api->free(pIter);
12829 }
12830}
12831
12832static void fts5MultiIterAdvanced(
12833 Fts5Index *p, /* FTS5 backend to iterate within */
12834 Fts5Iter *pIter, /* Iterator to update aFirst[] array for */
12835 int iChanged, /* Index of sub-iterator just advanced */
12836 int iMinset /* Minimum entry in aFirst[] to set */
12837){
12838 int i;
12839 for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK0; i=i/2){
12840 int iEq;
12841 if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){
12842 Fts5SegIter *pSeg = &pIter->aSeg[iEq];
12843 assert( p->rc==SQLITE_OK )((void) (0));
12844 pSeg->xNext(p, pSeg, 0);
12845 i = pIter->nSeg + iEq;
12846 }
12847 }
12848}
12849
12850/*
12851** Sub-iterator iChanged of iterator pIter has just been advanced. It still
12852** points to the same term though - just a different rowid. This function
12853** attempts to update the contents of the pIter->aFirst[] accordingly.
12854** If it does so successfully, 0 is returned. Otherwise 1.
12855**
12856** If non-zero is returned, the caller should call fts5MultiIterAdvanced()
12857** on the iterator instead. That function does the same as this one, except
12858** that it deals with more complicated cases as well.
12859*/
12860static int fts5MultiIterAdvanceRowid(
12861 Fts5Iter *pIter, /* Iterator to update aFirst[] array for */
12862 int iChanged, /* Index of sub-iterator just advanced */
12863 Fts5SegIter **ppFirst
12864){
12865 Fts5SegIter *pNew = &pIter->aSeg[iChanged];
12866
12867 if( pNew->iRowid==pIter->iSwitchRowid
12868 || (pNew->iRowid<pIter->iSwitchRowid)==pIter->bRev
12869 ){
12870 int i;
12871 Fts5SegIter *pOther = &pIter->aSeg[iChanged ^ 0x0001];
12872 pIter->iSwitchRowid = pIter->bRev ? SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32))) : LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32));
12873 for(i=(pIter->nSeg+iChanged)/2; 1; i=i/2){
12874 Fts5CResult *pRes = &pIter->aFirst[i];
12875
12876 assert( pNew->pLeaf )((void) (0));
12877 assert( pRes->bTermEq==0 || pOther->pLeaf )((void) (0));
12878
12879 if( pRes->bTermEq ){
12880 if( pNew->iRowid==pOther->iRowid ){
12881 return 1;
12882 }else if( (pOther->iRowid>pNew->iRowid)==pIter->bRev ){
12883 pIter->iSwitchRowid = pOther->iRowid;
12884 pNew = pOther;
12885 }else if( (pOther->iRowid>pIter->iSwitchRowid)==pIter->bRev ){
12886 pIter->iSwitchRowid = pOther->iRowid;
12887 }
12888 }
12889 pRes->iFirst = (u16)(pNew - pIter->aSeg);
12890 if( i==1 ) break;
12891
12892 pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ];
12893 }
12894 }
12895
12896 *ppFirst = pNew;
12897 return 0;
12898}
12899
12900/*
12901** Set the pIter->bEof variable based on the state of the sub-iterators.
12902*/
12903static void fts5MultiIterSetEof(Fts5Iter *pIter){
12904 Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
12905 pIter->base.bEof = pSeg->pLeaf==0;
12906 pIter->iSwitchRowid = pSeg->iRowid;
12907}
12908
12909/*
12910** The argument to this macro must be an Fts5Data structure containing a
12911** tombstone hash page. This macro returns the key-size of the hash-page.
12912*/
12913#define TOMBSTONE_KEYSIZE(pPg)(pPg->p[0]==4 ? 4 : 8) (pPg->p[0]==4 ? 4 : 8)
12914
12915#define TOMBSTONE_NSLOT(pPg)((pPg->nn > 16) ? ((pPg->nn-8) / (pPg->p[0]==4 ? 4
: 8)) : 1) \
12916 ((pPg->nn > 16) ? ((pPg->nn-8) / TOMBSTONE_KEYSIZE(pPg)(pPg->p[0]==4 ? 4 : 8)) : 1)
12917
12918/*
12919** Query a single tombstone hash table for rowid iRowid. Return true if
12920** it is found or false otherwise. The tombstone hash table is one of
12921** nHashTable tables.
12922*/
12923static int fts5IndexTombstoneQuery(
12924 Fts5Data *pHash, /* Hash table page to query */
12925 int nHashTable, /* Number of pages attached to segment */
12926 u64 iRowid /* Rowid to query hash for */
12927){
12928 const int szKey = TOMBSTONE_KEYSIZE(pHash)(pHash->p[0]==4 ? 4 : 8);
12929 const int nSlot = TOMBSTONE_NSLOT(pHash)((pHash->nn > 16) ? ((pHash->nn-8) / (pHash->p[0]
==4 ? 4 : 8)) : 1);
12930 int iSlot = (iRowid / nHashTable) % nSlot;
12931 int nCollide = nSlot;
12932
12933 if( iRowid==0 ){
12934 return pHash->p[1];
12935 }else if( szKey==4 ){
12936 u32 *aSlot = (u32*)&pHash->p[8];
12937 while( aSlot[iSlot] ){
12938 if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1;
12939 if( nCollide--==0 ) break;
12940 iSlot = (iSlot+1)%nSlot;
12941 }
12942 }else{
12943 u64 *aSlot = (u64*)&pHash->p[8];
12944 while( aSlot[iSlot] ){
12945 if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1;
12946 if( nCollide--==0 ) break;
12947 iSlot = (iSlot+1)%nSlot;
12948 }
12949 }
12950
12951 return 0;
12952}
12953
12954/*
12955** Return true if the iterator passed as the only argument points
12956** to an segment entry for which there is a tombstone. Return false
12957** if there is no tombstone or if the iterator is already at EOF.
12958*/
12959static int fts5MultiIterIsDeleted(Fts5Iter *pIter){
12960 int iFirst = pIter->aFirst[1].iFirst;
12961 Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
12962 Fts5TombstoneArray *pArray = pSeg->pTombArray;
12963
12964 if( pSeg->pLeaf && pArray ){
12965 /* Figure out which page the rowid might be present on. */
12966 int iPg = ((u64)pSeg->iRowid) % pArray->nTombstone;
12967 assert( iPg>=0 )((void) (0));
12968
12969 /* If tombstone hash page iPg has not yet been loaded from the
12970 ** database, load it now. */
12971 if( pArray->apTombstone[iPg]==0 ){
12972 pArray->apTombstone[iPg] = fts5DataRead(pIter->pIndex,
12973 FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg)( ((i64)(pSeg->pSeg->iSegid+(1<<16)) << (31
+5 +1)) + ((i64)(0) << (31 + 5)) + ((i64)(0) << (
31)) + ((i64)(iPg)) )
12974 );
12975 if( pArray->apTombstone[iPg]==0 ) return 0;
12976 }
12977
12978 return fts5IndexTombstoneQuery(
12979 pArray->apTombstone[iPg],
12980 pArray->nTombstone,
12981 pSeg->iRowid
12982 );
12983 }
12984
12985 return 0;
12986}
12987
12988/*
12989** Move the iterator to the next entry.
12990**
12991** If an error occurs, an error code is left in Fts5Index.rc. It is not
12992** considered an error if the iterator reaches EOF, or if it is already at
12993** EOF when this function is called.
12994*/
12995static void fts5MultiIterNext(
12996 Fts5Index *p,
12997 Fts5Iter *pIter,
12998 int bFrom, /* True if argument iFrom is valid */
12999 i64 iFrom /* Advance at least as far as this */
13000){
13001 int bUseFrom = bFrom;
13002 assert( pIter->base.bEof==0 )((void) (0));
13003 while( p->rc==SQLITE_OK0 ){
13004 int iFirst = pIter->aFirst[1].iFirst;
13005 int bNewTerm = 0;
13006 Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
13007 assert( p->rc==SQLITE_OK )((void) (0));
13008 if( bUseFrom && pSeg->pDlidx ){
13009 fts5SegIterNextFrom(p, pSeg, iFrom);
13010 }else{
13011 pSeg->xNext(p, pSeg, &bNewTerm);
13012 }
13013
13014 if( pSeg->pLeaf==0 || bNewTerm
13015 || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg)
13016 ){
13017 fts5MultiIterAdvanced(p, pIter, iFirst, 1);
13018 fts5MultiIterSetEof(pIter);
13019 pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
13020 if( pSeg->pLeaf==0 ) return;
13021 }
13022
13023 fts5AssertMultiIterSetup(p, pIter);
13024 assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf )((void) (0));
13025 if( (pIter->bSkipEmpty==0 || pSeg->nPos)
13026 && 0==fts5MultiIterIsDeleted(pIter)
13027 ){
13028 pIter->xSetOutputs(pIter, pSeg);
13029 return;
13030 }
13031 bUseFrom = 0;
13032 }
13033}
13034
13035static void fts5MultiIterNext2(
13036 Fts5Index *p,
13037 Fts5Iter *pIter,
13038 int *pbNewTerm /* OUT: True if *might* be new term */
13039){
13040 assert( pIter->bSkipEmpty )((void) (0));
13041 if( p->rc==SQLITE_OK0 ){
13042 *pbNewTerm = 0;
13043 do{
13044 int iFirst = pIter->aFirst[1].iFirst;
13045 Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
13046 int bNewTerm = 0;
13047
13048 assert( p->rc==SQLITE_OK )((void) (0));
13049 pSeg->xNext(p, pSeg, &bNewTerm);
13050 if( pSeg->pLeaf==0 || bNewTerm
13051 || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg)
13052 ){
13053 fts5MultiIterAdvanced(p, pIter, iFirst, 1);
13054 fts5MultiIterSetEof(pIter);
13055 *pbNewTerm = 1;
13056 }
13057 fts5AssertMultiIterSetup(p, pIter);
13058
13059 }while( (fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter))
13060 && (p->rc==SQLITE_OK0)
13061 );
13062 }
13063}
13064
13065static void fts5IterSetOutputs_Noop(Fts5Iter *pUnused1, Fts5SegIter *pUnused2){
13066 UNUSED_PARAM2(pUnused1, pUnused2)(void)(pUnused1), (void)(pUnused2);
13067}
13068
13069static Fts5Iter *fts5MultiIterAlloc(
13070 Fts5Index *p, /* FTS5 backend to iterate within */
13071 int nSeg
13072){
13073 Fts5Iter *pNew;
13074 i64 nSlot; /* Power of two >= nSeg */
13075
13076 for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
13077 pNew = fts5IdxMalloc(p,
13078 SZ_FTS5ITER(nSlot)(__builtin_offsetof(Fts5Iter, aSeg)+(nSlot)*sizeof(Fts5SegIter
)) + /* pNew + pNew->aSeg[] */
13079 sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */
13080 );
13081 if( pNew ){
13082 pNew->nSeg = nSlot;
13083 pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot];
13084 pNew->pIndex = p;
13085 pNew->xSetOutputs = fts5IterSetOutputs_Noop;
13086 }
13087 return pNew;
13088}
13089
13090static void fts5PoslistCallback(
13091 Fts5Index *pUnused,
13092 void *pContext,
13093 const u8 *pChunk, int nChunk
13094){
13095 UNUSED_PARAM(pUnused)(void)(pUnused);
13096 assert_nc( nChunk>=0 )((void) (0));
13097 if( nChunk>0 ){
13098 fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk){ ((void) (0)); memcpy(&((Fts5Buffer*)pContext)->p[((Fts5Buffer
*)pContext)->n], pChunk, nChunk); ((Fts5Buffer*)pContext)->
n += nChunk; };
13099 }
13100}
13101
13102typedef struct PoslistCallbackCtx PoslistCallbackCtx;
13103struct PoslistCallbackCtx {
13104 Fts5Buffer *pBuf; /* Append to this buffer */
13105 Fts5Colset *pColset; /* Restrict matches to this column */
13106 int eState; /* See above */
13107};
13108
13109typedef struct PoslistOffsetsCtx PoslistOffsetsCtx;
13110struct PoslistOffsetsCtx {
13111 Fts5Buffer *pBuf; /* Append to this buffer */
13112 Fts5Colset *pColset; /* Restrict matches to this column */
13113 int iRead;
13114 int iWrite;
13115};
13116
13117/*
13118** TODO: Make this more efficient!
13119*/
13120static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){
13121 int i;
13122 for(i=0; i<pColset->nCol; i++){
13123 if( pColset->aiCol[i]==iCol ) return 1;
13124 }
13125 return 0;
13126}
13127
13128static void fts5PoslistOffsetsCallback(
13129 Fts5Index *pUnused,
13130 void *pContext,
13131 const u8 *pChunk, int nChunk
13132){
13133 PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext;
13134 UNUSED_PARAM(pUnused)(void)(pUnused);
13135 assert_nc( nChunk>=0 )((void) (0));
13136 if( nChunk>0 ){
13137 int i = 0;
13138 while( i<nChunk ){
13139 int iVal;
13140 i += fts5GetVarint32(&pChunk[i], iVal)sqlite3Fts5GetVarint32(&pChunk[i],(u32*)&(iVal));
13141 iVal += pCtx->iRead - 2;
13142 pCtx->iRead = iVal;
13143 if( fts5IndexColsetTest(pCtx->pColset, iVal) ){
13144 fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite){ (pCtx->pBuf)->n += sqlite3Fts5PutVarint(&(pCtx->
pBuf)->p[(pCtx->pBuf)->n], (iVal + 2 - pCtx->iWrite
)); ((void) (0)); };
13145 pCtx->iWrite = iVal;
13146 }
13147 }
13148 }
13149}
13150
13151static void fts5PoslistFilterCallback(
13152 Fts5Index *pUnused,
13153 void *pContext,
13154 const u8 *pChunk, int nChunk
13155){
13156 PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext;
13157 UNUSED_PARAM(pUnused)(void)(pUnused);
13158 assert_nc( nChunk>=0 )((void) (0));
13159 if( nChunk>0 ){
13160 /* Search through to find the first varint with value 1. This is the
13161 ** start of the next columns hits. */
13162 int i = 0;
13163 int iStart = 0;
13164
13165 if( pCtx->eState==2 ){
13166 int iCol;
13167 fts5FastGetVarint32(pChunk, i, iCol){ iCol = (pChunk)[i++]; if( iCol & 0x80 ){ i--; i += sqlite3Fts5GetVarint32
(&(pChunk)[i],(u32*)&(iCol)); } };
13168 if( fts5IndexColsetTest(pCtx->pColset, iCol) ){
13169 pCtx->eState = 1;
13170 fts5BufferSafeAppendVarint(pCtx->pBuf, 1){ (pCtx->pBuf)->n += sqlite3Fts5PutVarint(&(pCtx->
pBuf)->p[(pCtx->pBuf)->n], (1)); ((void) (0)); };
13171 }else{
13172 pCtx->eState = 0;
13173 }
13174 }
13175
13176 do {
13177 while( i<nChunk && pChunk[i]!=0x01 ){
13178 fts5IndexSkipVarint(pChunk, i){ int iEnd = i+9; while( (pChunk[i++] & 0x80) && i
<iEnd ); };
13179 }
13180 if( pCtx->eState ){
13181 fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart){ ((void) (0)); memcpy(&(pCtx->pBuf)->p[(pCtx->pBuf
)->n], &pChunk[iStart], i-iStart); (pCtx->pBuf)->
n += i-iStart; };
13182 }
13183 if( i<nChunk ){
13184 int iCol;
13185 iStart = i;
13186 i++;
13187 if( i>=nChunk ){
13188 pCtx->eState = 2;
13189 }else{
13190 fts5FastGetVarint32(pChunk, i, iCol){ iCol = (pChunk)[i++]; if( iCol & 0x80 ){ i--; i += sqlite3Fts5GetVarint32
(&(pChunk)[i],(u32*)&(iCol)); } };
13191 pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol);
13192 if( pCtx->eState ){
13193 fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart){ ((void) (0)); memcpy(&(pCtx->pBuf)->p[(pCtx->pBuf
)->n], &pChunk[iStart], i-iStart); (pCtx->pBuf)->
n += i-iStart; };
13194 iStart = i;
13195 }
13196 }
13197 }
13198 }while( i<nChunk );
13199 }
13200}
13201
13202static void fts5ChunkIterate(
13203 Fts5Index *p, /* Index object */
13204 Fts5SegIter *pSeg, /* Poslist of this iterator */
13205 void *pCtx, /* Context pointer for xChunk callback */
13206 void (*xChunk)(Fts5Index*, void*, const u8*, int)
13207){
13208 int nRem = pSeg->nPos; /* Number of bytes still to come */
13209 Fts5Data *pData = 0;
13210 u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset];
13211 int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset)(((nRem) < (pSeg->pLeaf->szLeaf - pSeg->iLeafOffset
)) ? (nRem) : (pSeg->pLeaf->szLeaf - pSeg->iLeafOffset
));
13212 int pgno = pSeg->iLeafPgno;
13213 int pgnoSave = 0;
13214
13215 /* This function does not work with detail=none databases. */
13216 assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE )((void) (0));
13217
13218 if( (pSeg->flags & FTS5_SEGITER_REVERSE0x02)==0 ){
13219 pgnoSave = pgno+1;
13220 }
13221
13222 while( 1 ){
13223 xChunk(p, pCtx, pChunk, nChunk);
13224 nRem -= nChunk;
13225 fts5DataRelease(pData);
13226 if( nRem<=0 ){
13227 break;
13228 }else if( pSeg->pSeg==0 ){
13229 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
13230 return;
13231 }else{
13232 pgno++;
13233 pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)( ((i64)(pSeg->pSeg->iSegid) << (31 +5 +1)) + ((i64
)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((i64)(pgno
)) ));
13234 if( pData==0 ) break;
13235 pChunk = &pData->p[4];
13236 nChunk = MIN(nRem, pData->szLeaf - 4)(((nRem) < (pData->szLeaf - 4)) ? (nRem) : (pData->szLeaf
- 4));
13237 if( pgno==pgnoSave ){
13238 assert( pSeg->pNextLeaf==0 )((void) (0));
13239 pSeg->pNextLeaf = pData;
13240 pData = 0;
13241 }
13242 }
13243 }
13244}
13245
13246/*
13247** Iterator pIter currently points to a valid entry (not EOF). This
13248** function appends the position list data for the current entry to
13249** buffer pBuf. It does not make a copy of the position-list size
13250** field.
13251*/
13252static void fts5SegiterPoslist(
13253 Fts5Index *p,
13254 Fts5SegIter *pSeg,
13255 Fts5Colset *pColset,
13256 Fts5Buffer *pBuf
13257){
13258 assert( pBuf!=0 )((void) (0));
13259 assert( pSeg!=0 )((void) (0));
13260 if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+FTS5_DATA_ZERO_PADDING)( (u32)((pBuf)->n) + (u32)(pSeg->nPos+8) <= (u32)((pBuf
)->nSpace) ? 0 : sqlite3Fts5BufferSize((&p->rc),(pBuf
),(pSeg->nPos+8)+(pBuf)->n) ) ){
13261 assert( pBuf->p!=0 )((void) (0));
13262 assert( pBuf->nSpace >= pBuf->n+pSeg->nPos+FTS5_DATA_ZERO_PADDING )((void) (0));
13263 memset(&pBuf->p[pBuf->n+pSeg->nPos], 0, FTS5_DATA_ZERO_PADDING8);
13264 if( pColset==0 ){
13265 fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
13266 }else{
13267 if( p->pConfig->eDetail==FTS5_DETAIL_FULL0 ){
13268 PoslistCallbackCtx sCtx;
13269 sCtx.pBuf = pBuf;
13270 sCtx.pColset = pColset;
13271 sCtx.eState = fts5IndexColsetTest(pColset, 0);
13272 assert( sCtx.eState==0 || sCtx.eState==1 )((void) (0));
13273 fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
13274 }else{
13275 PoslistOffsetsCtx sCtx;
13276 memset(&sCtx, 0, sizeof(sCtx));
13277 sCtx.pBuf = pBuf;
13278 sCtx.pColset = pColset;
13279 fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
13280 }
13281 }
13282 }
13283}
13284
13285/*
13286** Parameter pPos points to a buffer containing a position list, size nPos.
13287** This function filters it according to pColset (which must be non-NULL)
13288** and sets pIter->base.pData/nData to point to the new position list.
13289** If memory is required for the new position list, use buffer pIter->poslist.
13290** Or, if the new position list is a contiguous subset of the input, set
13291** pIter->base.pData/nData to point directly to it.
13292**
13293** This function is a no-op if *pRc is other than SQLITE_OK when it is
13294** called. If an OOM error is encountered, *pRc is set to SQLITE_NOMEM
13295** before returning.
13296*/
13297static void fts5IndexExtractColset(
13298 int *pRc,
13299 Fts5Colset *pColset, /* Colset to filter on */
13300 const u8 *pPos, int nPos, /* Position list */
13301 Fts5Iter *pIter
13302){
13303 if( *pRc==SQLITE_OK0 ){
13304 const u8 *p = pPos;
13305 const u8 *aCopy = p;
13306 const u8 *pEnd = &p[nPos]; /* One byte past end of position list */
13307 int i = 0;
13308 int iCurrent = 0;
13309
13310 if( pColset->nCol>1 && sqlite3Fts5BufferSize(pRc, &pIter->poslist, nPos) ){
13311 return;
13312 }
13313
13314 while( 1 ){
13315 while( pColset->aiCol[i]<iCurrent ){
13316 i++;
13317 if( i==pColset->nCol ){
13318 pIter->base.pData = pIter->poslist.p;
13319 pIter->base.nData = pIter->poslist.n;
13320 return;
13321 }
13322 }
13323
13324 /* Advance pointer p until it points to pEnd or an 0x01 byte that is
13325 ** not part of a varint */
13326 while( p<pEnd && *p!=0x01 ){
13327 while( p<pEnd && (*p++ & 0x80) );
13328 }
13329
13330 if( pColset->aiCol[i]==iCurrent ){
13331 if( pColset->nCol==1 ){
13332 pIter->base.pData = aCopy;
13333 pIter->base.nData = p-aCopy;
13334 return;
13335 }
13336 fts5BufferSafeAppendBlob(&pIter->poslist, aCopy, p-aCopy){ ((void) (0)); memcpy(&(&pIter->poslist)->p[(&
pIter->poslist)->n], aCopy, p-aCopy); (&pIter->poslist
)->n += p-aCopy; };
13337 }
13338 if( p>=pEnd ){
13339 pIter->base.pData = pIter->poslist.p;
13340 pIter->base.nData = pIter->poslist.n;
13341 return;
13342 }
13343 aCopy = p++;
13344 iCurrent = *p++;
13345 if( iCurrent & 0x80 ){
13346 p--;
13347 p += fts5GetVarint32(p, iCurrent)sqlite3Fts5GetVarint32(p,(u32*)&(iCurrent));
13348 }
13349 }
13350 }
13351
13352}
13353
13354/*
13355** xSetOutputs callback used by detail=none tables.
13356*/
13357static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
13358 assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE )((void) (0));
13359 pIter->base.iRowid = pSeg->iRowid;
13360 pIter->base.nData = pSeg->nPos;
13361}
13362
13363/*
13364** xSetOutputs callback used by detail=full and detail=col tables when no
13365** column filters are specified.
13366*/
13367static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
13368 pIter->base.iRowid = pSeg->iRowid;
13369 pIter->base.nData = pSeg->nPos;
13370
13371 assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE )((void) (0));
13372 assert( pIter->pColset==0 )((void) (0));
13373
13374 if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
13375 /* All data is stored on the current page. Populate the output
13376 ** variables to point into the body of the page object. */
13377 pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset];
13378 }else{
13379 /* The data is distributed over two or more pages. Copy it into the
13380 ** Fts5Iter.poslist buffer and then set the output pointer to point
13381 ** to this buffer. */
13382 fts5BufferZero(&pIter->poslist)sqlite3Fts5BufferZero(&pIter->poslist);
13383 fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
13384 pIter->base.pData = pIter->poslist.p;
13385 }
13386}
13387
13388/*
13389** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match
13390** against no columns at all).
13391*/
13392static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){
13393 UNUSED_PARAM(pSeg)(void)(pSeg);
13394 pIter->base.nData = 0;
13395}
13396
13397/*
13398** xSetOutputs callback used by detail=col when there is a column filter
13399** and there are 100 or more columns. Also called as a fallback from
13400** fts5IterSetOutputs_Col100 if the column-list spans more than one page.
13401*/
13402static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
13403 fts5BufferZero(&pIter->poslist)sqlite3Fts5BufferZero(&pIter->poslist);
13404 fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist);
13405 pIter->base.iRowid = pSeg->iRowid;
13406 pIter->base.pData = pIter->poslist.p;
13407 pIter->base.nData = pIter->poslist.n;
13408}
13409
13410/*
13411** xSetOutputs callback used when:
13412**
13413** * detail=col,
13414** * there is a column filter, and
13415** * the table contains 100 or fewer columns.
13416**
13417** The last point is to ensure all column numbers are stored as
13418** single-byte varints.
13419*/
13420static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){
13421
13422 assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS )((void) (0));
13423 assert( pIter->pColset )((void) (0));
13424 assert( pIter->poslist.nSpace>=pIter->pIndex->pConfig->nCol )((void) (0));
13425
13426 if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf
13427 || pSeg->nPos>pIter->pIndex->pConfig->nCol
13428 ){
13429 fts5IterSetOutputs_Col(pIter, pSeg);
13430 }else{
13431 u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
13432 u8 *pEnd = (u8*)&a[pSeg->nPos];
13433 int iPrev = 0;
13434 int *aiCol = pIter->pColset->aiCol;
13435 int *aiColEnd = &aiCol[pIter->pColset->nCol];
13436
13437 u8 *aOut = pIter->poslist.p;
13438 int iPrevOut = 0;
13439
13440 pIter->base.iRowid = pSeg->iRowid;
13441
13442 while( a<pEnd ){
13443 iPrev += (int)a++[0] - 2;
13444 while( *aiCol<iPrev ){
13445 aiCol++;
13446 if( aiCol==aiColEnd ) goto setoutputs_col_out;
13447 }
13448 if( *aiCol==iPrev ){
13449 *aOut++ = (u8)((iPrev - iPrevOut) + 2);
13450 iPrevOut = iPrev;
13451 }
13452 }
13453
13454setoutputs_col_out:
13455 pIter->base.pData = pIter->poslist.p;
13456 pIter->base.nData = aOut - pIter->poslist.p;
13457 }
13458}
13459
13460/*
13461** xSetOutputs callback used by detail=full when there is a column filter.
13462*/
13463static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
13464 Fts5Colset *pColset = pIter->pColset;
13465 pIter->base.iRowid = pSeg->iRowid;
13466
13467 assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL )((void) (0));
13468 assert( pColset )((void) (0));
13469
13470 if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
13471 /* All data is stored on the current page. Populate the output
13472 ** variables to point into the body of the page object. */
13473 const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
13474 int *pRc = &pIter->pIndex->rc;
13475 fts5BufferZero(&pIter->poslist)sqlite3Fts5BufferZero(&pIter->poslist);
13476 fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, pIter);
13477 }else{
13478 /* The data is distributed over two or more pages. Copy it into the
13479 ** Fts5Iter.poslist buffer and then set the output pointer to point
13480 ** to this buffer. */
13481 fts5BufferZero(&pIter->poslist)sqlite3Fts5BufferZero(&pIter->poslist);
13482 fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
13483 pIter->base.pData = pIter->poslist.p;
13484 pIter->base.nData = pIter->poslist.n;
13485 }
13486}
13487
13488static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){
13489 assert( pIter!=0 || (*pRc)!=SQLITE_OK )((void) (0));
13490 if( *pRc==SQLITE_OK0 ){
13491 Fts5Config *pConfig = pIter->pIndex->pConfig;
13492 if( pConfig->eDetail==FTS5_DETAIL_NONE1 ){
13493 pIter->xSetOutputs = fts5IterSetOutputs_None;
13494 }
13495
13496 else if( pIter->pColset==0 ){
13497 pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
13498 }
13499
13500 else if( pIter->pColset->nCol==0 ){
13501 pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset;
13502 }
13503
13504 else if( pConfig->eDetail==FTS5_DETAIL_FULL0 ){
13505 pIter->xSetOutputs = fts5IterSetOutputs_Full;
13506 }
13507
13508 else{
13509 assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS )((void) (0));
13510 if( pConfig->nCol<=100 ){
13511 pIter->xSetOutputs = fts5IterSetOutputs_Col100;
13512 sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol);
13513 }else{
13514 pIter->xSetOutputs = fts5IterSetOutputs_Col;
13515 }
13516 }
13517 }
13518}
13519
13520/*
13521** All the component segment-iterators of pIter have been set up. This
13522** functions finishes setup for iterator pIter itself.
13523*/
13524static void fts5MultiIterFinishSetup(Fts5Index *p, Fts5Iter *pIter){
13525 int iIter;
13526 for(iIter=pIter->nSeg-1; iIter>0; iIter--){
13527 int iEq;
13528 if( (iEq = fts5MultiIterDoCompare(pIter, iIter)) ){
13529 Fts5SegIter *pSeg = &pIter->aSeg[iEq];
13530 if( p->rc==SQLITE_OK0 ) pSeg->xNext(p, pSeg, 0);
13531 fts5MultiIterAdvanced(p, pIter, iEq, iIter);
13532 }
13533 }
13534 fts5MultiIterSetEof(pIter);
13535 fts5AssertMultiIterSetup(p, pIter);
13536
13537 if( (pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter))
13538 || fts5MultiIterIsDeleted(pIter)
13539 ){
13540 fts5MultiIterNext(p, pIter, 0, 0);
13541 }else if( pIter->base.bEof==0 ){
13542 Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
13543 pIter->xSetOutputs(pIter, pSeg);
13544 }
13545}
13546
13547/*
13548** Allocate a new Fts5Iter object.
13549**
13550** The new object will be used to iterate through data in structure pStruct.
13551** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel
13552** is zero or greater, data from the first nSegment segments on level iLevel
13553** is merged.
13554**
13555** The iterator initially points to the first term/rowid entry in the
13556** iterated data.
13557*/
13558static void fts5MultiIterNew(
13559 Fts5Index *p, /* FTS5 backend to iterate within */
13560 Fts5Structure *pStruct, /* Structure of specific index */
13561 int flags, /* FTS5INDEX_QUERY_XXX flags */
13562 Fts5Colset *pColset, /* Colset to filter on (or NULL) */
13563 const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */
13564 int iLevel, /* Level to iterate (-1 for all) */
13565 int nSegment, /* Number of segments to merge (iLevel>=0) */
13566 Fts5Iter **ppOut /* New object */
13567){
13568 int nSeg = 0; /* Number of segment-iters in use */
13569 int iIter = 0; /* */
13570 int iSeg; /* Used to iterate through segments */
13571 Fts5StructureLevel *pLvl;
13572 Fts5Iter *pNew;
13573
13574 assert( (pTerm==0 && nTerm==0) || iLevel<0 )((void) (0));
13575
13576 /* Allocate space for the new multi-seg-iterator. */
13577 if( p->rc==SQLITE_OK0 ){
13578 if( iLevel<0 ){
13579 assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) )((void) (0));
13580 nSeg = pStruct->nSegment;
13581 nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH0x0040));
13582 }else{
13583 nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment)(((pStruct->aLevel[iLevel].nSeg) < (nSegment)) ? (pStruct
->aLevel[iLevel].nSeg) : (nSegment));
13584 }
13585 }
13586 *ppOut = pNew = fts5MultiIterAlloc(p, nSeg);
13587 if( pNew==0 ){
13588 assert( p->rc!=SQLITE_OK )((void) (0));
13589 goto fts5MultiIterNew_post_check;
13590 }
13591 pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC0x0002));
13592 pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY0x0010));
13593 pNew->pColset = pColset;
13594 if( (flags & FTS5INDEX_QUERY_NOOUTPUT0x0020)==0 ){
13595 fts5IterSetOutputCb(&p->rc, pNew);
13596 }
13597
13598 /* Initialize each of the component segment iterators. */
13599 if( p->rc==SQLITE_OK0 ){
13600 if( iLevel<0 ){
13601 Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
13602 if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH0x0040) ){
13603 /* Add a segment iterator for the current contents of the hash table. */
13604 Fts5SegIter *pIter = &pNew->aSeg[iIter++];
13605 fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
13606 }
13607 for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){
13608 for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){
13609 Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
13610 Fts5SegIter *pIter = &pNew->aSeg[iIter++];
13611 if( pTerm==0 ){
13612 fts5SegIterInit(p, pSeg, pIter);
13613 }else{
13614 fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter);
13615 }
13616 }
13617 }
13618 }else{
13619 pLvl = &pStruct->aLevel[iLevel];
13620 for(iSeg=nSeg-1; iSeg>=0; iSeg--){
13621 fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]);
13622 }
13623 }
13624 assert( iIter==nSeg )((void) (0));
13625 }
13626
13627 /* If the above was successful, each component iterator now points
13628 ** to the first entry in its segment. In this case initialize the
13629 ** aFirst[] array. Or, if an error has occurred, free the iterator
13630 ** object and set the output variable to NULL. */
13631 if( p->rc==SQLITE_OK0 ){
13632 fts5MultiIterFinishSetup(p, pNew);
13633 }else{
13634 fts5MultiIterFree(pNew);
13635 *ppOut = 0;
13636 }
13637
13638fts5MultiIterNew_post_check:
13639 assert( (*ppOut)!=0 || p->rc!=SQLITE_OK )((void) (0));
13640 return;
13641}
13642
13643/*
13644** Create an Fts5Iter that iterates through the doclist provided
13645** as the second argument.
13646*/
13647static void fts5MultiIterNew2(
13648 Fts5Index *p, /* FTS5 backend to iterate within */
13649 Fts5Data *pData, /* Doclist to iterate through */
13650 int bDesc, /* True for descending rowid order */
13651 Fts5Iter **ppOut /* New object */
13652){
13653 Fts5Iter *pNew;
13654 pNew = fts5MultiIterAlloc(p, 2);
13655 if( pNew ){
13656 Fts5SegIter *pIter = &pNew->aSeg[1];
13657 pIter->flags = FTS5_SEGITER_ONETERM0x01;
13658 if( pData->szLeaf>0 ){
13659 pIter->pLeaf = pData;
13660 pIter->iLeafOffset = fts5GetVarintsqlite3Fts5GetVarint(pData->p, (u64*)&pIter->iRowid);
13661 pIter->iEndofDoclist = pData->nn;
13662 pNew->aFirst[1].iFirst = 1;
13663 if( bDesc ){
13664 pNew->bRev = 1;
13665 pIter->flags |= FTS5_SEGITER_REVERSE0x02;
13666 fts5SegIterReverseInitPage(p, pIter);
13667 }else{
13668 fts5SegIterLoadNPos(p, pIter);
13669 }
13670 pData = 0;
13671 }else{
13672 pNew->base.bEof = 1;
13673 }
13674 fts5SegIterSetNext(p, pIter);
13675
13676 *ppOut = pNew;
13677 }
13678
13679 fts5DataRelease(pData);
13680}
13681
13682/*
13683** Return true if the iterator is at EOF or if an error has occurred.
13684** False otherwise.
13685*/
13686static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){
13687 assert( pIter!=0 || p->rc!=SQLITE_OK )((void) (0));
13688 assert( p->rc!=SQLITE_OK((void) (0))
13689 || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof((void) (0))
13690 )((void) (0));
13691 return (p->rc || pIter->base.bEof);
13692}
13693
13694/*
13695** Return the rowid of the entry that the iterator currently points
13696** to. If the iterator points to EOF when this function is called the
13697** results are undefined.
13698*/
13699static i64 fts5MultiIterRowid(Fts5Iter *pIter){
13700 assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf )((void) (0));
13701 return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid;
13702}
13703
13704/*
13705** Move the iterator to the next entry at or following iMatch.
13706*/
13707static void fts5MultiIterNextFrom(
13708 Fts5Index *p,
13709 Fts5Iter *pIter,
13710 i64 iMatch
13711){
13712 while( 1 ){
13713 i64 iRowid;
13714 fts5MultiIterNext(p, pIter, 1, iMatch);
13715 if( fts5MultiIterEof(p, pIter) ) break;
13716 iRowid = fts5MultiIterRowid(pIter);
13717 if( pIter->bRev==0 && iRowid>=iMatch ) break;
13718 if( pIter->bRev!=0 && iRowid<=iMatch ) break;
13719 }
13720}
13721
13722/*
13723** Return a pointer to a buffer containing the term associated with the
13724** entry that the iterator currently points to.
13725*/
13726static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){
13727 Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
13728 *pn = p->term.n;
13729 return p->term.p;
13730}
13731
13732/*
13733** Allocate a new segment-id for the structure pStruct. The new segment
13734** id must be between 1 and 65335 inclusive, and must not be used by
13735** any currently existing segment. If a free segment id cannot be found,
13736** SQLITE_FULL is returned.
13737**
13738** If an error has already occurred, this function is a no-op. 0 is
13739** returned in this case.
13740*/
13741static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
13742 int iSegid = 0;
13743
13744 if( p->rc==SQLITE_OK0 ){
13745 if( pStruct->nSegment>=FTS5_MAX_SEGMENT2000 ){
13746 p->rc = SQLITE_FULL13;
13747 }else{
13748 /* FTS5_MAX_SEGMENT is currently defined as 2000. So the following
13749 ** array is 63 elements, or 252 bytes, in size. */
13750 u32 aUsed[(FTS5_MAX_SEGMENT2000+31) / 32];
13751 int iLvl, iSeg;
13752 int i;
13753 u32 mask;
13754 memset(aUsed, 0, sizeof(aUsed));
13755 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
13756 for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
13757 int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid;
13758 if( iId<=FTS5_MAX_SEGMENT2000 && iId>0 ){
13759 aUsed[(iId-1) / 32] |= (u32)1 << ((iId-1) % 32);
13760 }
13761 }
13762 }
13763
13764 for(i=0; aUsed[i]==0xFFFFFFFF; i++);
13765 mask = aUsed[i];
13766 for(iSegid=0; mask & ((u32)1 << iSegid); iSegid++);
13767 iSegid += 1 + i*32;
13768
13769#ifdef SQLITE_DEBUG
13770 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
13771 for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
13772 assert_nc( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid )((void) (0));
13773 }
13774 }
13775 assert_nc( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT )((void) (0));
13776
13777 {
13778 sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p);
13779 if( p->rc==SQLITE_OK0 ){
13780 u8 aBlob[2] = {0xff, 0xff};
13781 sqlite3_bind_intsqlite3_api->bind_int(pIdxSelect, 1, iSegid);
13782 sqlite3_bind_blobsqlite3_api->bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC((sqlite3_destructor_type)0));
13783 assert_nc( sqlite3_step(pIdxSelect)!=SQLITE_ROW )((void) (0));
13784 p->rc = sqlite3_resetsqlite3_api->reset(pIdxSelect);
13785 sqlite3_bind_nullsqlite3_api->bind_null(pIdxSelect, 2);
13786 }
13787 }
13788#endif
13789 }
13790 }
13791
13792 return iSegid;
13793}
13794
13795/*
13796** Discard all data currently cached in the hash-tables.
13797*/
13798static void fts5IndexDiscardData(Fts5Index *p){
13799 assert( p->pHash || p->nPendingData==0 )((void) (0));
13800 if( p->pHash ){
13801 sqlite3Fts5HashClear(p->pHash);
13802 p->nPendingData = 0;
13803 p->nPendingRow = 0;
13804 p->flushRc = SQLITE_OK0;
13805 }
13806 p->nContentlessDelete = 0;
13807}
13808
13809/*
13810** Return the size of the prefix, in bytes, that buffer
13811** (pNew/<length-unknown>) shares with buffer (pOld/nOld).
13812**
13813** Buffer (pNew/<length-unknown>) is guaranteed to be greater
13814** than buffer (pOld/nOld).
13815*/
13816static int fts5PrefixCompress(int nOld, const u8 *pOld, const u8 *pNew){
13817 int i;
13818 for(i=0; i<nOld; i++){
13819 if( pOld[i]!=pNew[i] ) break;
13820 }
13821 return i;
13822}
13823
13824static void fts5WriteDlidxClear(
13825 Fts5Index *p,
13826 Fts5SegWriter *pWriter,
13827 int bFlush /* If true, write dlidx to disk */
13828){
13829 int i;
13830 assert( bFlush==0 || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n>0) )((void) (0));
13831 for(i=0; i<pWriter->nDlidx; i++){
13832 Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i];
13833 if( pDlidx->buf.n==0 ) break;
13834 if( bFlush ){
13835 assert( pDlidx->pgno!=0 )((void) (0));
13836 fts5DataWrite(p,
13837 FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno)( ((i64)(pWriter->iSegid) << (31 +5 +1)) + ((i64)(1)
<< (31 + 5)) + ((i64)(i) << (31)) + ((i64)(pDlidx
->pgno)) ),
13838 pDlidx->buf.p, pDlidx->buf.n
13839 );
13840 }
13841 sqlite3Fts5BufferZero(&pDlidx->buf);
13842 pDlidx->bPrevValid = 0;
13843 }
13844}
13845
13846/*
13847** Grow the pWriter->aDlidx[] array to at least nLvl elements in size.
13848** Any new array elements are zeroed before returning.
13849*/
13850static int fts5WriteDlidxGrow(
13851 Fts5Index *p,
13852 Fts5SegWriter *pWriter,
13853 int nLvl
13854){
13855 if( p->rc==SQLITE_OK0 && nLvl>=pWriter->nDlidx ){
13856 Fts5DlidxWriter *aDlidx = (Fts5DlidxWriter*)sqlite3_realloc64sqlite3_api->realloc64(
13857 pWriter->aDlidx, sizeof(Fts5DlidxWriter) * nLvl
13858 );
13859 if( aDlidx==0 ){
13860 p->rc = SQLITE_NOMEM7;
13861 }else{
13862 size_t nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx);
13863 memset(&aDlidx[pWriter->nDlidx], 0, nByte);
13864 pWriter->aDlidx = aDlidx;
13865 pWriter->nDlidx = nLvl;
13866 }
13867 }
13868 return p->rc;
13869}
13870
13871/*
13872** If the current doclist-index accumulating in pWriter->aDlidx[] is large
13873** enough, flush it to disk and return 1. Otherwise discard it and return
13874** zero.
13875*/
13876static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){
13877 int bFlag = 0;
13878
13879 /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written
13880 ** to the database, also write the doclist-index to disk. */
13881 if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE4 ){
13882 bFlag = 1;
13883 }
13884 fts5WriteDlidxClear(p, pWriter, bFlag);
13885 pWriter->nEmpty = 0;
13886 return bFlag;
13887}
13888
13889/*
13890** This function is called whenever processing of the doclist for the
13891** last term on leaf page (pWriter->iBtPage) is completed.
13892**
13893** The doclist-index for that term is currently stored in-memory within the
13894** Fts5SegWriter.aDlidx[] array. If it is large enough, this function
13895** writes it out to disk. Or, if it is too small to bother with, discards
13896** it.
13897**
13898** Fts5SegWriter.btterm currently contains the first term on page iBtPage.
13899*/
13900static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){
13901 int bFlag;
13902
13903 assert( pWriter->iBtPage || pWriter->nEmpty==0 )((void) (0));
13904 if( pWriter->iBtPage==0 ) return;
13905 bFlag = fts5WriteFlushDlidx(p, pWriter);
13906
13907 if( p->rc==SQLITE_OK0 ){
13908 const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:"");
13909 /* The following was already done in fts5WriteInit(): */
13910 /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */
13911 sqlite3_bind_blobsqlite3_api->bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC((sqlite3_destructor_type)0));
13912 sqlite3_bind_int64sqlite3_api->bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1));
13913 sqlite3_stepsqlite3_api->step(p->pIdxWriter);
13914 p->rc = sqlite3_resetsqlite3_api->reset(p->pIdxWriter);
13915 sqlite3_bind_nullsqlite3_api->bind_null(p->pIdxWriter, 2);
13916 }
13917 pWriter->iBtPage = 0;
13918}
13919
13920/*
13921** This is called once for each leaf page except the first that contains
13922** at least one term. Argument (nTerm/pTerm) is the split-key - a term that
13923** is larger than all terms written to earlier leaves, and equal to or
13924** smaller than the first term on the new leaf.
13925**
13926** If an error occurs, an error code is left in Fts5Index.rc. If an error
13927** has already occurred when this function is called, it is a no-op.
13928*/
13929static void fts5WriteBtreeTerm(
13930 Fts5Index *p, /* FTS5 backend object */
13931 Fts5SegWriter *pWriter, /* Writer object */
13932 int nTerm, const u8 *pTerm /* First term on new page */
13933){
13934 fts5WriteFlushBtree(p, pWriter);
13935 if( p->rc==SQLITE_OK0 ){
13936 fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm)sqlite3Fts5BufferSet(&p->rc,&pWriter->btterm,nTerm
,pTerm);
13937 pWriter->iBtPage = pWriter->writer.pgno;
13938 }
13939}
13940
13941/*
13942** This function is called when flushing a leaf page that contains no
13943** terms at all to disk.
13944*/
13945static void fts5WriteBtreeNoTerm(
13946 Fts5Index *p, /* FTS5 backend object */
13947 Fts5SegWriter *pWriter /* Writer object */
13948){
13949 /* If there were no rowids on the leaf page either and the doclist-index
13950 ** has already been started, append an 0x00 byte to it. */
13951 if( pWriter->bFirstRowidInPage && pWriter->aDlidx[0].buf.n>0 ){
13952 Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[0];
13953 assert( pDlidx->bPrevValid )((void) (0));
13954 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, 0);
13955 }
13956
13957 /* Increment the "number of sequential leaves without a term" counter. */
13958 pWriter->nEmpty++;
13959}
13960
13961static i64 fts5DlidxExtractFirstRowid(Fts5Buffer *pBuf){
13962 i64 iRowid;
13963 int iOff;
13964
13965 iOff = 1 + fts5GetVarintsqlite3Fts5GetVarint(&pBuf->p[1], (u64*)&iRowid);
13966 fts5GetVarintsqlite3Fts5GetVarint(&pBuf->p[iOff], (u64*)&iRowid);
13967 return iRowid;
13968}
13969
13970/*
13971** Rowid iRowid has just been appended to the current leaf page. It is the
13972** first on the page. This function appends an appropriate entry to the current
13973** doclist-index.
13974*/
13975static void fts5WriteDlidxAppend(
13976 Fts5Index *p,
13977 Fts5SegWriter *pWriter,
13978 i64 iRowid
13979){
13980 int i;
13981 int bDone = 0;
13982
13983 for(i=0; p->rc==SQLITE_OK0 && bDone==0; i++){
13984 i64 iVal;
13985 Fts5DlidxWriter *pDlidx = &pWriter->aDlidx[i];
13986
13987 if( pDlidx->buf.n>=p->pConfig->pgsz ){
13988 /* The current doclist-index page is full. Write it to disk and push
13989 ** a copy of iRowid (which will become the first rowid on the next
13990 ** doclist-index leaf page) up into the next level of the b-tree
13991 ** hierarchy. If the node being flushed is currently the root node,
13992 ** also push its first rowid upwards. */
13993 pDlidx->buf.p[0] = 0x01; /* Not the root node */
13994 fts5DataWrite(p,
13995 FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno)( ((i64)(pWriter->iSegid) << (31 +5 +1)) + ((i64)(1)
<< (31 + 5)) + ((i64)(i) << (31)) + ((i64)(pDlidx
->pgno)) ),
13996 pDlidx->buf.p, pDlidx->buf.n
13997 );
13998 fts5WriteDlidxGrow(p, pWriter, i+2);
13999 pDlidx = &pWriter->aDlidx[i];
14000 if( p->rc==SQLITE_OK0 && pDlidx[1].buf.n==0 ){
14001 i64 iFirst = fts5DlidxExtractFirstRowid(&pDlidx->buf);
14002
14003 /* This was the root node. Push its first rowid up to the new root. */
14004 pDlidx[1].pgno = pDlidx->pgno;
14005 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, 0);
14006 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, pDlidx->pgno);
14007 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx[1].buf, iFirst);
14008 pDlidx[1].bPrevValid = 1;
14009 pDlidx[1].iPrev = iFirst;
14010 }
14011
14012 sqlite3Fts5BufferZero(&pDlidx->buf);
14013 pDlidx->bPrevValid = 0;
14014 pDlidx->pgno++;
14015 }else{
14016 bDone = 1;
14017 }
14018
14019 if( pDlidx->bPrevValid ){
14020 iVal = (u64)iRowid - (u64)pDlidx->iPrev;
14021 }else{
14022 i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno);
14023 assert( pDlidx->buf.n==0 )((void) (0));
14024 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone);
14025 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno);
14026 iVal = iRowid;
14027 }
14028
14029 sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iVal);
14030 pDlidx->bPrevValid = 1;
14031 pDlidx->iPrev = iRowid;
14032 }
14033}
14034
14035static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
14036 static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
14037 Fts5PageWriter *pPage = &pWriter->writer;
14038 i64 iRowid;
14039
14040 assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) )((void) (0));
14041
14042 /* Set the szLeaf header field. */
14043 assert( 0==fts5GetU16(&pPage->buf.p[2]) )((void) (0));
14044 fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n);
14045
14046 if( pWriter->bFirstTermInPage ){
14047 /* No term was written to this page. */
14048 assert( pPage->pgidx.n==0 )((void) (0));
14049 fts5WriteBtreeNoTerm(p, pWriter);
14050 }else{
14051 /* Append the pgidx to the page buffer. Set the szLeaf header field. */
14052 fts5BufferAppendBlob(&p->rc, &pPage->buf, pPage->pgidx.n, pPage->pgidx.p)sqlite3Fts5BufferAppendBlob(&p->rc,&pPage->buf,
pPage->pgidx.n,pPage->pgidx.p);
14053 }
14054
14055 /* Write the page out to disk */
14056 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, pPage->pgno)( ((i64)(pWriter->iSegid) << (31 +5 +1)) + ((i64)(0)
<< (31 + 5)) + ((i64)(0) << (31)) + ((i64)(pPage
->pgno)) );
14057 fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n);
14058
14059 /* Initialize the next page. */
14060 fts5BufferZero(&pPage->buf)sqlite3Fts5BufferZero(&pPage->buf);
14061 fts5BufferZero(&pPage->pgidx)sqlite3Fts5BufferZero(&pPage->pgidx);
14062 fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero)sqlite3Fts5BufferAppendBlob(&p->rc,&pPage->buf,
4,zero);
14063 pPage->iPrevPgidx = 0;
14064 pPage->pgno++;
14065
14066 /* Increase the leaves written counter */
14067 pWriter->nLeafWritten++;
14068
14069 /* The new leaf holds no terms or rowids */
14070 pWriter->bFirstTermInPage = 1;
14071 pWriter->bFirstRowidInPage = 1;
14072}
14073
14074/*
14075** Append term pTerm/nTerm to the segment being written by the writer passed
14076** as the second argument.
14077**
14078** If an error occurs, set the Fts5Index.rc error code. If an error has
14079** already occurred, this function is a no-op.
14080*/
14081static void fts5WriteAppendTerm(
14082 Fts5Index *p,
14083 Fts5SegWriter *pWriter,
14084 int nTerm, const u8 *pTerm
14085){
14086 int nPrefix; /* Bytes of prefix compression for term */
14087 Fts5PageWriter *pPage = &pWriter->writer;
14088 Fts5Buffer *pPgidx = &pWriter->writer.pgidx;
14089 int nMin = MIN(pPage->term.n, nTerm)(((pPage->term.n) < (nTerm)) ? (pPage->term.n) : (nTerm
));
14090
14091 assert( p->rc==SQLITE_OK )((void) (0));
14092 assert( pPage->buf.n>=4 )((void) (0));
14093 assert( pPage->buf.n>4 || pWriter->bFirstTermInPage )((void) (0));
14094
14095 /* If the current leaf page is full, flush it to disk. */
14096 if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){
14097 if( pPage->buf.n>4 ){
14098 fts5WriteFlushLeaf(p, pWriter);
14099 if( p->rc!=SQLITE_OK0 ) return;
14100 }
14101 fts5BufferGrow(&p->rc, &pPage->buf, nTerm+FTS5_DATA_PADDING)( (u32)((&pPage->buf)->n) + (u32)(nTerm+20) <= (
u32)((&pPage->buf)->nSpace) ? 0 : sqlite3Fts5BufferSize
((&p->rc),(&pPage->buf),(nTerm+20)+(&pPage->
buf)->n) );
14102 }
14103
14104 /* TODO1: Updating pgidx here. */
14105 pPgidx->n += sqlite3Fts5PutVarint(
14106 &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx
14107 );
14108 pPage->iPrevPgidx = pPage->buf.n;
14109#if 0
14110 fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n);
14111 pPgidx->n += 2;
14112#endif
14113
14114 if( pWriter->bFirstTermInPage ){
14115 nPrefix = 0;
14116 if( pPage->pgno!=1 ){
14117 /* This is the first term on a leaf that is not the leftmost leaf in
14118 ** the segment b-tree. In this case it is necessary to add a term to
14119 ** the b-tree hierarchy that is (a) larger than the largest term
14120 ** already written to the segment and (b) smaller than or equal to
14121 ** this term. In other words, a prefix of (pTerm/nTerm) that is one
14122 ** byte longer than the longest prefix (pTerm/nTerm) shares with the
14123 ** previous term.
14124 **
14125 ** Usually, the previous term is available in pPage->term. The exception
14126 ** is if this is the first term written in an incremental-merge step.
14127 ** In this case the previous term is not available, so just write a
14128 ** copy of (pTerm/nTerm) into the parent node. This is slightly
14129 ** inefficient, but still correct. */
14130 int n = nTerm;
14131 if( pPage->term.n ){
14132 n = 1 + fts5PrefixCompress(nMin, pPage->term.p, pTerm);
14133 }
14134 fts5WriteBtreeTerm(p, pWriter, n, pTerm);
14135 if( p->rc!=SQLITE_OK0 ) return;
14136 pPage = &pWriter->writer;
14137 }
14138 }else{
14139 nPrefix = fts5PrefixCompress(nMin, pPage->term.p, pTerm);
14140 fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix)sqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)nPrefix);
14141 }
14142
14143 /* Append the number of bytes of new data, then the term data itself
14144 ** to the page. */
14145 fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm - nPrefix)sqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)nTerm - nPrefix);
14146 fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm - nPrefix, &pTerm[nPrefix])sqlite3Fts5BufferAppendBlob(&p->rc,&pPage->buf,
nTerm - nPrefix,&pTerm[nPrefix]);
14147
14148 /* Update the Fts5PageWriter.term field. */
14149 fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm)sqlite3Fts5BufferSet(&p->rc,&pPage->term,nTerm,
pTerm);
14150 pWriter->bFirstTermInPage = 0;
14151
14152 pWriter->bFirstRowidInPage = 0;
14153 pWriter->bFirstRowidInDoclist = 1;
14154
14155 assert( p->rc || (pWriter->nDlidx>0 && pWriter->aDlidx[0].buf.n==0) )((void) (0));
14156 pWriter->aDlidx[0].pgno = pPage->pgno;
14157}
14158
14159/*
14160** Append a rowid and position-list size field to the writers output.
14161*/
14162static void fts5WriteAppendRowid(
14163 Fts5Index *p,
14164 Fts5SegWriter *pWriter,
14165 i64 iRowid
14166){
14167 if( p->rc==SQLITE_OK0 ){
14168 Fts5PageWriter *pPage = &pWriter->writer;
14169
14170 if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){
14171 fts5WriteFlushLeaf(p, pWriter);
14172 }
14173
14174 /* If this is to be the first rowid written to the page, set the
14175 ** rowid-pointer in the page-header. Also append a value to the dlidx
14176 ** buffer, in case a doclist-index is required. */
14177 if( pWriter->bFirstRowidInPage ){
14178 fts5PutU16(pPage->buf.p, (u16)pPage->buf.n);
14179 fts5WriteDlidxAppend(p, pWriter, iRowid);
14180 }
14181
14182 /* Write the rowid. */
14183 if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){
14184 fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid)sqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)iRowid);
14185 }else{
14186 assert_nc( p->rc || iRowid>pWriter->iPrevRowid )((void) (0));
14187 fts5BufferAppendVarint(&p->rc, &pPage->buf,sqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)(u64)iRowid - (u64)pWriter->iPrevRowid)
14188 (u64)iRowid - (u64)pWriter->iPrevRowidsqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)(u64)iRowid - (u64)pWriter->iPrevRowid)
14189 )sqlite3Fts5BufferAppendVarint(&p->rc,&pPage->buf
,(i64)(u64)iRowid - (u64)pWriter->iPrevRowid);
14190 }
14191 pWriter->iPrevRowid = iRowid;
14192 pWriter->bFirstRowidInDoclist = 0;
14193 pWriter->bFirstRowidInPage = 0;
14194 }
14195}
14196
14197static void fts5WriteAppendPoslistData(
14198 Fts5Index *p,
14199 Fts5SegWriter *pWriter,
14200 const u8 *aData,
14201 int nData
14202){
14203 Fts5PageWriter *pPage = &pWriter->writer;
14204 const u8 *a = aData;
14205 int n = nData;
14206
14207 assert( p->pConfig->pgsz>0 || p->rc!=SQLITE_OK )((void) (0));
14208 while( p->rc==SQLITE_OK0
14209 && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz
14210 ){
14211 int nReq = p->pConfig->pgsz - pPage->buf.n - pPage->pgidx.n;
14212 int nCopy = 0;
14213 while( nCopy<nReq ){
14214 i64 dummy;
14215 nCopy += fts5GetVarintsqlite3Fts5GetVarint(&a[nCopy], (u64*)&dummy);
14216 }
14217 fts5BufferAppendBlob(&p->rc, &pPage->buf, nCopy, a)sqlite3Fts5BufferAppendBlob(&p->rc,&pPage->buf,
nCopy,a);
14218 a += nCopy;
14219 n -= nCopy;
14220 fts5WriteFlushLeaf(p, pWriter);
14221 }
14222 if( n>0 ){
14223 fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a)sqlite3Fts5BufferAppendBlob(&p->rc,&pPage->buf,
n,a);
14224 }
14225}
14226
14227/*
14228** Flush any data cached by the writer object to the database. Free any
14229** allocations associated with the writer.
14230*/
14231static void fts5WriteFinish(
14232 Fts5Index *p,
14233 Fts5SegWriter *pWriter, /* Writer object */
14234 int *pnLeaf /* OUT: Number of leaf pages in b-tree */
14235){
14236 int i;
14237 Fts5PageWriter *pLeaf = &pWriter->writer;
14238 if( p->rc==SQLITE_OK0 ){
14239 assert( pLeaf->pgno>=1 )((void) (0));
14240 if( pLeaf->buf.n>4 ){
14241 fts5WriteFlushLeaf(p, pWriter);
14242 }
14243 *pnLeaf = pLeaf->pgno-1;
14244 if( pLeaf->pgno>1 ){
14245 fts5WriteFlushBtree(p, pWriter);
14246 }
14247 }
14248 fts5BufferFree(&pLeaf->term)sqlite3Fts5BufferFree(&pLeaf->term);
14249 fts5BufferFree(&pLeaf->buf)sqlite3Fts5BufferFree(&pLeaf->buf);
14250 fts5BufferFree(&pLeaf->pgidx)sqlite3Fts5BufferFree(&pLeaf->pgidx);
14251 fts5BufferFree(&pWriter->btterm)sqlite3Fts5BufferFree(&pWriter->btterm);
14252
14253 for(i=0; i<pWriter->nDlidx; i++){
14254 sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf);
14255 }
14256 sqlite3_freesqlite3_api->free(pWriter->aDlidx);
14257}
14258
14259static void fts5WriteInit(
14260 Fts5Index *p,
14261 Fts5SegWriter *pWriter,
14262 int iSegid
14263){
14264 const int nBuffer = p->pConfig->pgsz + FTS5_DATA_PADDING20;
14265
14266 memset(pWriter, 0, sizeof(Fts5SegWriter));
14267 pWriter->iSegid = iSegid;
14268
14269 fts5WriteDlidxGrow(p, pWriter, 1);
14270 pWriter->writer.pgno = 1;
14271 pWriter->bFirstTermInPage = 1;
14272 pWriter->iBtPage = 1;
14273
14274 assert( pWriter->writer.buf.n==0 )((void) (0));
14275 assert( pWriter->writer.pgidx.n==0 )((void) (0));
14276
14277 /* Grow the two buffers to pgsz + padding bytes in size. */
14278 sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.pgidx, nBuffer);
14279 sqlite3Fts5BufferSize(&p->rc, &pWriter->writer.buf, nBuffer);
14280
14281 if( p->pIdxWriter==0 ){
14282 Fts5Config *pConfig = p->pConfig;
14283 fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintfsqlite3_api->mprintf(
14284 "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)",
14285 pConfig->zDb, pConfig->zName
14286 ));
14287 }
14288
14289 if( p->rc==SQLITE_OK0 ){
14290 /* Initialize the 4-byte leaf-page header to 0x00. */
14291 memset(pWriter->writer.buf.p, 0, 4);
14292 pWriter->writer.buf.n = 4;
14293
14294 /* Bind the current output segment id to the index-writer. This is an
14295 ** optimization over binding the same value over and over as rows are
14296 ** inserted into %_idx by the current writer. */
14297 sqlite3_bind_intsqlite3_api->bind_int(p->pIdxWriter, 1, pWriter->iSegid);
14298 }
14299}
14300
14301/*
14302** Iterator pIter was used to iterate through the input segments of on an
14303** incremental merge operation. This function is called if the incremental
14304** merge step has finished but the input has not been completely exhausted.
14305*/
14306static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){
14307 int i;
14308 Fts5Buffer buf;
14309 memset(&buf, 0, sizeof(Fts5Buffer));
14310 for(i=0; i<pIter->nSeg && p->rc==SQLITE_OK0; i++){
14311 Fts5SegIter *pSeg = &pIter->aSeg[i];
14312 if( pSeg->pSeg==0 ){
14313 /* no-op */
14314 }else if( pSeg->pLeaf==0 ){
14315 /* All keys from this input segment have been transfered to the output.
14316 ** Set both the first and last page-numbers to 0 to indicate that the
14317 ** segment is now empty. */
14318 pSeg->pSeg->pgnoLast = 0;
14319 pSeg->pSeg->pgnoFirst = 0;
14320 }else{
14321 int iOff = pSeg->iTermLeafOffset; /* Offset on new first leaf page */
14322 i64 iLeafRowid;
14323 Fts5Data *pData;
14324 int iId = pSeg->pSeg->iSegid;
14325 u8 aHdr[4] = {0x00, 0x00, 0x00, 0x00};
14326
14327 iLeafRowid = FTS5_SEGMENT_ROWID(iId, pSeg->iTermLeafPgno)( ((i64)(iId) << (31 +5 +1)) + ((i64)(0) << (31 +
5)) + ((i64)(0) << (31)) + ((i64)(pSeg->iTermLeafPgno
)) );
14328 pData = fts5LeafRead(p, iLeafRowid);
14329 if( pData ){
14330 if( iOff>pData->szLeaf ){
14331 /* This can occur if the pages that the segments occupy overlap - if
14332 ** a single page has been assigned to more than one segment. In
14333 ** this case a prior iteration of this loop may have corrupted the
14334 ** segment currently being trimmed. */
14335 FTS5_CORRUPT_ROWID(p, iLeafRowid)fts5IndexCorruptRowid(p, iLeafRowid);
14336 }else{
14337 fts5BufferZero(&buf)sqlite3Fts5BufferZero(&buf);
14338 fts5BufferGrow(&p->rc, &buf, pData->nn)( (u32)((&buf)->n) + (u32)(pData->nn) <= (u32)((
&buf)->nSpace) ? 0 : sqlite3Fts5BufferSize((&p->
rc),(&buf),(pData->nn)+(&buf)->n) );
14339 fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr)sqlite3Fts5BufferAppendBlob(&p->rc,&buf,sizeof(aHdr
),aHdr);
14340 fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)pSeg
->term.n);
14341 fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p)sqlite3Fts5BufferAppendBlob(&p->rc,&buf,pSeg->term
.n,pSeg->term.p);
14342 fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff])sqlite3Fts5BufferAppendBlob(&p->rc,&buf,pData->
szLeaf-iOff,&pData->p[iOff]);
14343 if( p->rc==SQLITE_OK0 ){
14344 /* Set the szLeaf field */
14345 fts5PutU16(&buf.p[2], (u16)buf.n);
14346 }
14347
14348 /* Set up the new page-index array */
14349 fts5BufferAppendVarint(&p->rc, &buf, 4)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)4);
14350 if( pSeg->iLeafPgno==pSeg->iTermLeafPgno
14351 && pSeg->iEndofDoclist<pData->szLeaf
14352 && pSeg->iPgidxOff<=pData->nn
14353 ){
14354 int nDiff = pData->szLeaf - pSeg->iEndofDoclist;
14355 fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4)sqlite3Fts5BufferAppendVarint(&p->rc,&buf,(i64)buf
.n - 1 - nDiff - 4);
14356 fts5BufferAppendBlob(&p->rc, &buf,sqlite3Fts5BufferAppendBlob(&p->rc,&buf,pData->
nn - pSeg->iPgidxOff,&pData->p[pSeg->iPgidxOff])
14357 pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff]sqlite3Fts5BufferAppendBlob(&p->rc,&buf,pData->
nn - pSeg->iPgidxOff,&pData->p[pSeg->iPgidxOff])
14358 )sqlite3Fts5BufferAppendBlob(&p->rc,&buf,pData->
nn - pSeg->iPgidxOff,&pData->p[pSeg->iPgidxOff]);
14359 }
14360
14361 pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno;
14362 fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 1)( ((i64)(iId) << (31 +5 +1)) + ((i64)(0) << (31 +
5)) + ((i64)(0) << (31)) + ((i64)(1)) ), iLeafRowid);
14363 fts5DataWrite(p, iLeafRowid, buf.p, buf.n);
14364 }
14365 fts5DataRelease(pData);
14366 }
14367 }
14368 }
14369 fts5BufferFree(&buf)sqlite3Fts5BufferFree(&buf);
14370}
14371
14372static void fts5MergeChunkCallback(
14373 Fts5Index *p,
14374 void *pCtx,
14375 const u8 *pChunk, int nChunk
14376){
14377 Fts5SegWriter *pWriter = (Fts5SegWriter*)pCtx;
14378 fts5WriteAppendPoslistData(p, pWriter, pChunk, nChunk);
14379}
14380
14381/*
14382**
14383*/
14384static void fts5IndexMergeLevel(
14385 Fts5Index *p, /* FTS5 backend object */
14386 Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */
14387 int iLvl, /* Level to read input from */
14388 int *pnRem /* Write up to this many output leaves */
14389){
14390 Fts5Structure *pStruct = *ppStruct;
14391 Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
14392 Fts5StructureLevel *pLvlOut;
14393 Fts5Iter *pIter = 0; /* Iterator to read input data */
14394 int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */
14395 int nInput; /* Number of input segments */
14396 Fts5SegWriter writer; /* Writer object */
14397 Fts5StructureSegment *pSeg; /* Output segment */
14398 Fts5Buffer term;
14399 int bOldest; /* True if the output segment is the oldest */
14400 int eDetail = p->pConfig->eDetail;
14401 const int flags = FTS5INDEX_QUERY_NOOUTPUT0x0020;
14402 int bTermWritten = 0; /* True if current term already output */
14403
14404 assert( iLvl<pStruct->nLevel )((void) (0));
14405 assert( pLvl->nMerge<=pLvl->nSeg )((void) (0));
14406
14407 memset(&writer, 0, sizeof(Fts5SegWriter));
14408 memset(&term, 0, sizeof(Fts5Buffer));
14409 if( pLvl->nMerge ){
14410 pLvlOut = &pStruct->aLevel[iLvl+1];
14411 assert( pLvlOut->nSeg>0 )((void) (0));
14412 nInput = pLvl->nMerge;
14413 pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1];
14414
14415 fts5WriteInit(p, &writer, pSeg->iSegid);
14416 writer.writer.pgno = pSeg->pgnoLast+1;
14417 writer.iBtPage = 0;
14418 }else{
14419 int iSegid = fts5AllocateSegid(p, pStruct);
14420
14421 /* Extend the Fts5Structure object as required to ensure the output
14422 ** segment exists. */
14423 if( iLvl==pStruct->nLevel-1 ){
14424 fts5StructureAddLevel(&p->rc, ppStruct);
14425 pStruct = *ppStruct;
14426 }
14427 fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0);
14428 if( p->rc ) return;
14429 pLvl = &pStruct->aLevel[iLvl];
14430 pLvlOut = &pStruct->aLevel[iLvl+1];
14431
14432 fts5WriteInit(p, &writer, iSegid);
14433
14434 /* Add the new segment to the output level */
14435 pSeg = &pLvlOut->aSeg[pLvlOut->nSeg];
14436 pLvlOut->nSeg++;
14437 pSeg->pgnoFirst = 1;
14438 pSeg->iSegid = iSegid;
14439 pStruct->nSegment++;
14440
14441 /* Read input from all segments in the input level */
14442 nInput = pLvl->nSeg;
14443
14444 /* Set the range of origins that will go into the output segment. */
14445 if( pStruct->nOriginCntr>0 ){
14446 pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1;
14447 pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2;
14448 }
14449 }
14450 bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2);
14451
14452 assert( iLvl>=0 )((void) (0));
14453 for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter);
14454 fts5MultiIterEof(p, pIter)==0;
14455 fts5MultiIterNext(p, pIter, 0, 0)
14456 ){
14457 Fts5SegIter *pSegIter = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
14458 int nPos; /* position-list size field value */
14459 int nTerm;
14460 const u8 *pTerm;
14461
14462 pTerm = fts5MultiIterTerm(pIter, &nTerm);
14463 if( nTerm!=term.n || fts5Memcmp(pTerm, term.p, nTerm)((nTerm)<=0 ? 0 : memcmp((pTerm), (term.p), (nTerm))) ){
14464 if( pnRem && writer.nLeafWritten>nRem ){
14465 break;
14466 }
14467 fts5BufferSet(&p->rc, &term, nTerm, pTerm)sqlite3Fts5BufferSet(&p->rc,&term,nTerm,pTerm);
14468 bTermWritten =0;
14469 }
14470
14471 /* Check for key annihilation. */
14472 if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue;
14473
14474 if( p->rc==SQLITE_OK0 && bTermWritten==0 ){
14475 /* This is a new term. Append a term to the output segment. */
14476 fts5WriteAppendTerm(p, &writer, nTerm, pTerm);
14477 bTermWritten = 1;
14478 }
14479
14480 /* Append the rowid to the output */
14481 /* WRITEPOSLISTSIZE */
14482 fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter));
14483
14484 if( eDetail==FTS5_DETAIL_NONE1 ){
14485 if( pSegIter->bDel ){
14486 fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0)sqlite3Fts5BufferAppendVarint(&p->rc,&writer.writer
.buf,(i64)0);
14487 if( pSegIter->nPos>0 ){
14488 fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0)sqlite3Fts5BufferAppendVarint(&p->rc,&writer.writer
.buf,(i64)0);
14489 }
14490 }
14491 }else{
14492 /* Append the position-list data to the output */
14493 nPos = pSegIter->nPos*2 + pSegIter->bDel;
14494 fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos)sqlite3Fts5BufferAppendVarint(&p->rc,&writer.writer
.buf,(i64)nPos);
14495 fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback);
14496 }
14497 }
14498
14499 /* Flush the last leaf page to disk. Set the output segment b-tree height
14500 ** and last leaf page number at the same time. */
14501 fts5WriteFinish(p, &writer, &pSeg->pgnoLast);
14502
14503 assert( pIter!=0 || p->rc!=SQLITE_OK )((void) (0));
14504 if( fts5MultiIterEof(p, pIter) ){
14505 int i;
14506
14507 /* Remove the redundant segments from the %_data table */
14508 assert( pSeg->nEntry==0 )((void) (0));
14509 for(i=0; i<nInput; i++){
14510 Fts5StructureSegment *pOld = &pLvl->aSeg[i];
14511 pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone);
14512 fts5DataRemoveSegment(p, pOld);
14513 }
14514
14515 /* Remove the redundant segments from the input level */
14516 if( pLvl->nSeg!=nInput ){
14517 int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment);
14518 memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove);
14519 }
14520 pStruct->nSegment -= nInput;
14521 pLvl->nSeg -= nInput;
14522 pLvl->nMerge = 0;
14523 if( pSeg->pgnoLast==0 ){
14524 pLvlOut->nSeg--;
14525 pStruct->nSegment--;
14526 }
14527 }else{
14528 assert( pSeg->pgnoLast>0 )((void) (0));
14529 fts5TrimSegments(p, pIter);
14530 pLvl->nMerge = nInput;
14531 }
14532
14533 fts5MultiIterFree(pIter);
14534 fts5BufferFree(&term)sqlite3Fts5BufferFree(&term);
14535 if( pnRem ) *pnRem -= writer.nLeafWritten;
14536}
14537
14538/*
14539** If this is not a contentless_delete=1 table, or if the 'deletemerge'
14540** configuration option is set to 0, then this function always returns -1.
14541** Otherwise, it searches the structure object passed as the second argument
14542** for a level suitable for merging due to having a large number of
14543** tombstones in the tombstone hash. If one is found, its index is returned.
14544** Otherwise, if there is no suitable level, -1.
14545*/
14546static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){
14547 Fts5Config *pConfig = p->pConfig;
14548 int iRet = -1;
14549 if( pConfig->bContentlessDelete && pConfig->nDeleteMerge>0 ){
14550 int ii;
14551 int nBest = 0;
14552
14553 for(ii=0; ii<pStruct->nLevel; ii++){
14554 Fts5StructureLevel *pLvl = &pStruct->aLevel[ii];
14555 i64 nEntry = 0;
14556 i64 nTomb = 0;
14557 int iSeg;
14558 for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
14559 nEntry += pLvl->aSeg[iSeg].nEntry;
14560 nTomb += pLvl->aSeg[iSeg].nEntryTombstone;
14561 }
14562 assert_nc( nEntry>0 || pLvl->nSeg==0 )((void) (0));
14563 if( nEntry>0 ){
14564 int nPercent = (nTomb * 100) / nEntry;
14565 if( nPercent>=pConfig->nDeleteMerge && nPercent>nBest ){
14566 iRet = ii;
14567 nBest = nPercent;
14568 }
14569 }
14570
14571 /* If pLvl is already the input level to an ongoing merge, look no
14572 ** further for a merge candidate. The caller should be allowed to
14573 ** continue merging from pLvl first. */
14574 if( pLvl->nMerge ) break;
14575 }
14576 }
14577 return iRet;
14578}
14579
14580/*
14581** Do up to nPg pages of automerge work on the index.
14582**
14583** Return true if any changes were actually made, or false otherwise.
14584*/
14585static int fts5IndexMerge(
14586 Fts5Index *p, /* FTS5 backend object */
14587 Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */
14588 int nPg, /* Pages of work to do */
14589 int nMin /* Minimum number of segments to merge */
14590){
14591 int nRem = nPg;
14592 int bRet = 0;
14593 Fts5Structure *pStruct = *ppStruct;
14594 while( nRem>0 && p->rc==SQLITE_OK0 ){
14595 int iLvl; /* To iterate through levels */
14596 int iBestLvl = 0; /* Level offering the most input segments */
14597 int nBest = 0; /* Number of input segments on best level */
14598
14599 /* Set iBestLvl to the level to read input segments from. Or to -1 if
14600 ** there is no level suitable to merge segments from. */
14601 assert( pStruct->nLevel>0 )((void) (0));
14602 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
14603 Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
14604 if( pLvl->nMerge ){
14605 if( pLvl->nMerge>nBest ){
14606 iBestLvl = iLvl;
14607 nBest = nMin;
14608 }
14609 break;
14610 }
14611 if( pLvl->nSeg>nBest ){
14612 nBest = pLvl->nSeg;
14613 iBestLvl = iLvl;
14614 }
14615 }
14616 if( nBest<nMin ){
14617 iBestLvl = fts5IndexFindDeleteMerge(p, pStruct);
14618 }
14619
14620 if( iBestLvl<0 ) break;
14621 bRet = 1;
14622 fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
14623 if( p->rc==SQLITE_OK0 && pStruct->aLevel[iBestLvl].nMerge==0 ){
14624 fts5StructurePromote(p, iBestLvl+1, pStruct);
14625 }
14626
14627 if( nMin==1 ) nMin = 2;
14628 }
14629 *ppStruct = pStruct;
14630 return bRet;
14631}
14632
14633/*
14634** A total of nLeaf leaf pages of data has just been flushed to a level-0
14635** segment. This function updates the write-counter accordingly and, if
14636** necessary, performs incremental merge work.
14637**
14638** If an error occurs, set the Fts5Index.rc error code. If an error has
14639** already occurred, this function is a no-op.
14640*/
14641static void fts5IndexAutomerge(
14642 Fts5Index *p, /* FTS5 backend object */
14643 Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */
14644 int nLeaf /* Number of output leaves just written */
14645){
14646 if( p->rc==SQLITE_OK0 && p->pConfig->nAutomerge>0 && ALWAYS((*ppStruct)!=0)((*ppStruct)!=0) ){
14647 Fts5Structure *pStruct = *ppStruct;
14648 u64 nWrite; /* Initial value of write-counter */
14649 int nWork; /* Number of work-quanta to perform */
14650 int nRem; /* Number of leaf pages left to write */
14651
14652 /* Update the write-counter. While doing so, set nWork. */
14653 nWrite = pStruct->nWriteCounter;
14654 nWork = (int)(((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit));
14655 pStruct->nWriteCounter += nLeaf;
14656 nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel);
14657
14658 fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge);
14659 }
14660}
14661
14662static void fts5IndexCrisismerge(
14663 Fts5Index *p, /* FTS5 backend object */
14664 Fts5Structure **ppStruct /* IN/OUT: Current structure of index */
14665){
14666 const int nCrisis = p->pConfig->nCrisisMerge;
14667 Fts5Structure *pStruct = *ppStruct;
14668 if( pStruct && pStruct->nLevel>0 ){
14669 int iLvl = 0;
14670 while( p->rc==SQLITE_OK0 && pStruct->aLevel[iLvl].nSeg>=nCrisis ){
14671 fts5IndexMergeLevel(p, &pStruct, iLvl, 0);
14672 assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) )((void) (0));
14673 fts5StructurePromote(p, iLvl+1, pStruct);
14674 iLvl++;
14675 }
14676 *ppStruct = pStruct;
14677 }
14678}
14679
14680static int fts5IndexReturn(Fts5Index *p){
14681 int rc = p->rc;
14682 p->rc = SQLITE_OK0;
14683 return rc;
14684}
14685
14686/*
14687** Close the read-only blob handle, if it is open.
14688*/
14689static void sqlite3Fts5IndexCloseReader(Fts5Index *p){
14690 fts5IndexCloseReader(p);
14691 fts5IndexReturn(p);
14692}
14693
14694typedef struct Fts5FlushCtx Fts5FlushCtx;
14695struct Fts5FlushCtx {
14696 Fts5Index *pIdx;
14697 Fts5SegWriter writer;
14698};
14699
14700/*
14701** Buffer aBuf[] contains a list of varints, all small enough to fit
14702** in a 32-bit integer. Return the size of the largest prefix of this
14703** list nMax bytes or less in size.
14704*/
14705static int fts5PoslistPrefix(const u8 *aBuf, int nMax){
14706 int ret;
14707 u32 dummy;
14708 ret = fts5GetVarint32(aBuf, dummy)sqlite3Fts5GetVarint32(aBuf,(u32*)&(dummy));
14709 if( ret<nMax ){
14710 while( 1 ){
14711 int i = fts5GetVarint32(&aBuf[ret], dummy)sqlite3Fts5GetVarint32(&aBuf[ret],(u32*)&(dummy));
14712 if( (ret + i) > nMax ) break;
14713 ret += i;
14714 }
14715 }
14716 return ret;
14717}
14718
14719/*
14720** Execute the SQL statement:
14721**
14722** DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno);
14723**
14724** This is used when a secure-delete operation removes the last term
14725** from a segment leaf page. In that case the %_idx entry is removed
14726** too. This is done to ensure that if all instances of a token are
14727** removed from an fts5 database in secure-delete mode, no trace of
14728** the token itself remains in the database.
14729*/
14730static void fts5SecureDeleteIdxEntry(
14731 Fts5Index *p, /* FTS5 backend object */
14732 int iSegid, /* Id of segment to delete entry for */
14733 int iPgno /* Page number within segment */
14734){
14735 if( iPgno!=1 ){
14736 assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE )((void) (0));
14737 if( p->pDeleteFromIdx==0 ){
14738 fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintfsqlite3_api->mprintf(
14739 "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)",
14740 p->pConfig->zDb, p->pConfig->zName
14741 ));
14742 }
14743 if( p->rc==SQLITE_OK0 ){
14744 sqlite3_bind_intsqlite3_api->bind_int(p->pDeleteFromIdx, 1, iSegid);
14745 sqlite3_bind_intsqlite3_api->bind_int(p->pDeleteFromIdx, 2, iPgno);
14746 sqlite3_stepsqlite3_api->step(p->pDeleteFromIdx);
14747 p->rc = sqlite3_resetsqlite3_api->reset(p->pDeleteFromIdx);
14748 }
14749 }
14750}
14751
14752/*
14753** This is called when a secure-delete operation removes a position-list
14754** that overflows onto segment page iPgno of segment pSeg. This function
14755** rewrites node iPgno, and possibly one or more of its right-hand peers,
14756** to remove this portion of the position list.
14757**
14758** Output variable (*pbLastInDoclist) is set to true if the position-list
14759** removed is followed by a new term or the end-of-segment, or false if
14760** it is followed by another rowid/position list.
14761*/
14762static void fts5SecureDeleteOverflow(
14763 Fts5Index *p,
14764 Fts5StructureSegment *pSeg,
14765 int iPgno,
14766 int *pbLastInDoclist
14767){
14768 const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE1);
14769 int pgno;
14770 Fts5Data *pLeaf = 0;
14771 assert( iPgno!=1 )((void) (0));
14772
14773 *pbLastInDoclist = 1;
14774 for(pgno=iPgno; p->rc==SQLITE_OK0 && pgno<=pSeg->pgnoLast; pgno++){
14775 i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno)( ((i64)(pSeg->iSegid) << (31 +5 +1)) + ((i64)(0) <<
(31 + 5)) + ((i64)(0) << (31)) + ((i64)(pgno)) );
14776 int iNext = 0;
14777 u8 *aPg = 0;
14778
14779 pLeaf = fts5LeafRead(p, iRowid);
14780 if( pLeaf==0 ) break;
14781 aPg = pLeaf->p;
14782
14783 iNext = fts5GetU16(&aPg[0]);
14784 if( iNext!=0 ){
14785 *pbLastInDoclist = 0;
14786 }
14787 if( iNext==0 && pLeaf->szLeaf<pLeaf->nn ){
14788 fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext)sqlite3Fts5GetVarint32(&aPg[pLeaf->szLeaf],(u32*)&
(iNext));
14789 }
14790
14791 if( iNext==0 ){
14792 /* The page contains no terms or rowids. Replace it with an empty
14793 ** page and move on to the right-hand peer. */
14794 const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04};
14795 assert_nc( bDetailNone==0 || pLeaf->nn==4 )((void) (0));
14796 if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
14797 fts5DataRelease(pLeaf);
14798 pLeaf = 0;
14799 }else if( bDetailNone ){
14800 break;
14801 }else if( iNext>=pLeaf->szLeaf || pLeaf->nn<pLeaf->szLeaf || iNext<4 ){
14802 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
14803 break;
14804 }else{
14805 int nShift = iNext - 4;
14806 int nPg;
14807
14808 int nIdx = 0;
14809 u8 *aIdx = 0;
14810
14811 /* Unless the current page footer is 0 bytes in size (in which case
14812 ** the new page footer will be as well), allocate and populate a
14813 ** buffer containing the new page footer. Set stack variables aIdx
14814 ** and nIdx accordingly. */
14815 if( pLeaf->nn>pLeaf->szLeaf ){
14816 int iFirst = 0;
14817 int i1 = pLeaf->szLeaf;
14818 int i2 = 0;
14819
14820 i1 += fts5GetVarint32(&aPg[i1], iFirst)sqlite3Fts5GetVarint32(&aPg[i1],(u32*)&(iFirst));
14821 if( iFirst<iNext ){
14822 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
14823 break;
14824 }
14825 aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2);
14826 if( aIdx==0 ) break;
14827 i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift);
14828 if( i1<pLeaf->nn ){
14829 memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1);
14830 i2 += (pLeaf->nn-i1);
14831 }
14832 nIdx = i2;
14833 }
14834
14835 /* Modify the contents of buffer aPg[]. Set nPg to the new size
14836 ** in bytes. The new page is always smaller than the old. */
14837 nPg = pLeaf->szLeaf - nShift;
14838 memmove(&aPg[4], &aPg[4+nShift], nPg-4);
14839 fts5PutU16(&aPg[2], nPg);
14840 if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4);
14841 if( nIdx>0 ){
14842 memcpy(&aPg[nPg], aIdx, nIdx);
14843 nPg += nIdx;
14844 }
14845 sqlite3_freesqlite3_api->free(aIdx);
14846
14847 /* Write the new page to disk and exit the loop */
14848 assert( nPg>4 || fts5GetU16(aPg)==0 )((void) (0));
14849 fts5DataWrite(p, iRowid, aPg, nPg);
14850 break;
14851 }
14852 }
14853 fts5DataRelease(pLeaf);
14854}
14855
14856/*
14857** Completely remove the entry that pSeg currently points to from
14858** the database.
14859*/
14860static void fts5DoSecureDelete(
14861 Fts5Index *p,
14862 Fts5SegIter *pSeg
14863){
14864 const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE1);
14865 int iSegid = pSeg->pSeg->iSegid;
14866 u8 *aPg = pSeg->pLeaf->p;
14867 int nPg = pSeg->pLeaf->nn;
14868 int iPgIdx = pSeg->pLeaf->szLeaf; /* Offset of page footer */
14869
14870 u64 iDelta = 0;
14871 int iNextOff = 0;
14872 int iOff = 0;
14873 int nIdx = 0;
14874 u8 *aIdx = 0;
14875 int bLastInDoclist = 0;
14876 int iIdx = 0;
14877 int iStart = 0;
14878 int iDelKeyOff = 0; /* Offset of deleted key, if any */
14879
14880 nIdx = nPg-iPgIdx;
14881 aIdx = sqlite3Fts5MallocZero(&p->rc, ((i64)nIdx)+16);
14882 if( p->rc ) return;
14883 memcpy(aIdx, &aPg[iPgIdx], nIdx);
14884
14885 /* At this point segment iterator pSeg points to the entry
14886 ** this function should remove from the b-tree segment.
14887 **
14888 ** In detail=full or detail=column mode, pSeg->iLeafOffset is the
14889 ** offset of the first byte in the position-list for the entry to
14890 ** remove. Immediately before this comes two varints that will also
14891 ** need to be removed:
14892 **
14893 ** + the rowid or delta rowid value for the entry, and
14894 ** + the size of the position list in bytes.
14895 **
14896 ** Or, in detail=none mode, there is a single varint prior to
14897 ** pSeg->iLeafOffset - the rowid or delta rowid value.
14898 **
14899 ** This block sets the following variables:
14900 **
14901 ** iStart:
14902 ** The offset of the first byte of the rowid or delta-rowid
14903 ** value for the doclist entry being removed.
14904 **
14905 ** iDelta:
14906 ** The value of the rowid or delta-rowid value for the doclist
14907 ** entry being removed.
14908 **
14909 ** iNextOff:
14910 ** The offset of the next entry following the position list
14911 ** for the one being removed. If the position list for this
14912 ** entry overflows onto the next leaf page, this value will be
14913 ** greater than pLeaf->szLeaf.
14914 */
14915 {
14916 int iSOP; /* Start-Of-Position-list */
14917 if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
14918 iStart = pSeg->iTermLeafOffset;
14919 }else{
14920 iStart = fts5GetU16(&aPg[0]);
14921 }
14922 if( iStart>nPg ){
14923 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
14924 sqlite3_freesqlite3_api->free(aIdx);
14925 return;
14926 }
14927
14928 iSOP = iStart + fts5GetVarintsqlite3Fts5GetVarint(&aPg[iStart], &iDelta);
14929 assert_nc( iSOP<=pSeg->iLeafOffset )((void) (0));
14930
14931 if( bDetailNone ){
14932 while( iSOP<pSeg->iLeafOffset ){
14933 if( aPg[iSOP]==0x00 ) iSOP++;
14934 if( aPg[iSOP]==0x00 ) iSOP++;
14935 iStart = iSOP;
14936 iSOP = iStart + fts5GetVarintsqlite3Fts5GetVarint(&aPg[iStart], &iDelta);
14937 }
14938
14939 iNextOff = iSOP;
14940 if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
14941 if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
14942
14943 }else{
14944 int nPos = 0;
14945 iSOP += fts5GetVarint32(&aPg[iSOP], nPos)sqlite3Fts5GetVarint32(&aPg[iSOP],(u32*)&(nPos));
14946 while( iSOP<pSeg->iLeafOffset ){
14947 iStart = iSOP + (nPos/2);
14948 iSOP = iStart + fts5GetVarintsqlite3Fts5GetVarint(&aPg[iStart], &iDelta);
14949 iSOP += fts5GetVarint32(&aPg[iSOP], nPos)sqlite3Fts5GetVarint32(&aPg[iSOP],(u32*)&(nPos));
14950 }
14951 assert_nc( iSOP==pSeg->iLeafOffset )((void) (0));
14952 iNextOff = iSOP + pSeg->nPos;
14953 }
14954 }
14955
14956 iOff = iStart;
14957
14958 /* If the position-list for the entry being removed flows over past
14959 ** the end of this page, delete the portion of the position-list on the
14960 ** next page and beyond.
14961 **
14962 ** Set variable bLastInDoclist to true if this entry happens
14963 ** to be the last rowid in the doclist for its term. */
14964 if( iNextOff>=iPgIdx ){
14965 int pgno = pSeg->iLeafPgno+1;
14966 fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
14967 iNextOff = iPgIdx;
14968 }
14969
14970 if( pSeg->bDel==0 ){
14971 if( iNextOff!=iPgIdx ){
14972 /* Loop through the page-footer. If iNextOff (offset of the
14973 ** entry following the one we are removing) is equal to the
14974 ** offset of a key on this page, then the entry is the last
14975 ** in its doclist. */
14976 int iKeyOff = 0;
14977 for(iIdx=0; iIdx<nIdx; /* no-op */){
14978 u32 iVal = 0;
14979 iIdx += fts5GetVarint32(&aIdx[iIdx], iVal)sqlite3Fts5GetVarint32(&aIdx[iIdx],(u32*)&(iVal));
14980 iKeyOff += iVal;
14981 if( iKeyOff==iNextOff ){
14982 bLastInDoclist = 1;
14983 }
14984 }
14985 }
14986
14987 /* If this is (a) the first rowid on a page and (b) is not followed by
14988 ** another position list on the same page, set the "first-rowid" field
14989 ** of the header to 0. */
14990 if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist || iNextOff==iPgIdx) ){
14991 fts5PutU16(&aPg[0], 0);
14992 }
14993 }
14994
14995 if( pSeg->bDel ){
14996 iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta);
14997 aPg[iOff++] = 0x01;
14998 }else if( bLastInDoclist==0 ){
14999 if( iNextOff!=iPgIdx ){
15000 u64 iNextDelta = 0;
15001 iNextOff += fts5GetVarintsqlite3Fts5GetVarint(&aPg[iNextOff], &iNextDelta);
15002 iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
15003 }
15004 }else if(
15005 pSeg->iLeafPgno==pSeg->iTermLeafPgno
15006 && iStart==pSeg->iTermLeafOffset
15007 ){
15008 /* The entry being removed was the only position list in its
15009 ** doclist. Therefore the term needs to be removed as well. */
15010 int iKey = 0;
15011 int iKeyOff = 0;
15012
15013 /* Set iKeyOff to the offset of the term that will be removed - the
15014 ** last offset in the footer that is not greater than iStart. */
15015 for(iIdx=0; iIdx<nIdx; iKey++){
15016 u32 iVal = 0;
15017 iIdx += fts5GetVarint32(&aIdx[iIdx], iVal)sqlite3Fts5GetVarint32(&aIdx[iIdx],(u32*)&(iVal));
15018 if( (iKeyOff+iVal)>(u32)iStart ) break;
15019 iKeyOff += iVal;
15020 }
15021 assert_nc( iKey>=1 )((void) (0));
15022
15023 /* Set iDelKeyOff to the value of the footer entry to remove from
15024 ** the page. */
15025 iDelKeyOff = iOff = iKeyOff;
15026
15027 if( iNextOff!=iPgIdx ){
15028 /* This is the only position-list associated with the term, and there
15029 ** is another term following it on this page. So the subsequent term
15030 ** needs to be moved to replace the term associated with the entry
15031 ** being removed. */
15032 u64 nPrefix = 0;
15033 u64 nSuffix = 0;
15034 u64 nPrefix2 = 0;
15035 u64 nSuffix2 = 0;
15036
15037 iDelKeyOff = iNextOff;
15038 iNextOff += fts5GetVarintsqlite3Fts5GetVarint(&aPg[iNextOff], &nPrefix2);
15039 iNextOff += fts5GetVarintsqlite3Fts5GetVarint(&aPg[iNextOff], &nSuffix2);
15040
15041 if( iKey!=1 ){
15042 iKeyOff += fts5GetVarintsqlite3Fts5GetVarint(&aPg[iKeyOff], &nPrefix);
15043 }
15044 iKeyOff += fts5GetVarintsqlite3Fts5GetVarint(&aPg[iKeyOff], &nSuffix);
15045
15046 nPrefix = MIN(nPrefix, nPrefix2)(((nPrefix) < (nPrefix2)) ? (nPrefix) : (nPrefix2));
15047 nSuffix = (nPrefix2 + nSuffix2) - nPrefix;
15048
15049 if( (iKeyOff+nSuffix)>(u64)iPgIdx || (iNextOff+nSuffix2)>(u64)iPgIdx ){
15050 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
15051 }else{
15052 if( iKey!=1 ){
15053 iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix);
15054 }
15055 iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix);
15056 if( nPrefix2>(u64)pSeg->term.n ){
15057 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
15058 }else if( nPrefix2>nPrefix ){
15059 memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix);
15060 iOff += (nPrefix2-nPrefix);
15061 }
15062 memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2);
15063 iOff += nSuffix2;
15064 iNextOff += nSuffix2;
15065 }
15066 }
15067 }else if( iStart==4 ){
15068 int iPgno;
15069
15070 assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno )((void) (0));
15071 /* The entry being removed may be the only position list in
15072 ** its doclist. */
15073 for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
15074 Fts5Data *pPg = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(iPgno)) ));
15075 int bEmpty = (pPg && pPg->nn==4);
15076 fts5DataRelease(pPg);
15077 if( bEmpty==0 ) break;
15078 }
15079
15080 if( iPgno==pSeg->iTermLeafPgno ){
15081 i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pSeg->iTermLeafPgno
)) );
15082 Fts5Data *pTerm = fts5LeafRead(p, iId);
15083 if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
15084 u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
15085 int nTermIdx = pTerm->nn - pTerm->szLeaf;
15086 int iTermIdx = 0;
15087 i64 iTermOff = 0;
15088
15089 while( 1 ){
15090 u32 iVal = 0;
15091 int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal)sqlite3Fts5GetVarint32(&aTermIdx[iTermIdx],(u32*)&(iVal
));
15092 iTermOff += iVal;
15093 if( (iTermIdx+nByte)>=nTermIdx ) break;
15094 iTermIdx += nByte;
15095 }
15096 nTermIdx = iTermIdx;
15097
15098 if( iTermOff>pTerm->szLeaf ){
15099 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
15100 }else{
15101 memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
15102 fts5PutU16(&pTerm->p[2], iTermOff);
15103 fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
15104 if( nTermIdx==0 ){
15105 fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
15106 }
15107 }
15108 }
15109 fts5DataRelease(pTerm);
15110 }
15111 }
15112
15113 /* Assuming no error has occurred, this block does final edits to the
15114 ** leaf page before writing it back to disk. Input variables are:
15115 **
15116 ** nPg: Total initial size of leaf page.
15117 ** iPgIdx: Initial offset of page footer.
15118 **
15119 ** iOff: Offset to move data to
15120 ** iNextOff: Offset to move data from
15121 */
15122 if( p->rc==SQLITE_OK0 ){
15123 const int nMove = nPg - iNextOff; /* Number of bytes to move */
15124 int nShift = iNextOff - iOff; /* Distance to move them */
15125
15126 int iPrevKeyOut = 0;
15127 int iKeyIn = 0;
15128
15129 if( nMove>0 ){
15130 memmove(&aPg[iOff], &aPg[iNextOff], nMove);
15131 }
15132 iPgIdx -= nShift;
15133 nPg = iPgIdx;
15134 fts5PutU16(&aPg[2], iPgIdx);
15135
15136 for(iIdx=0; iIdx<nIdx; /* no-op */){
15137 u32 iVal = 0;
15138 iIdx += fts5GetVarint32(&aIdx[iIdx], iVal)sqlite3Fts5GetVarint32(&aIdx[iIdx],(u32*)&(iVal));
15139 iKeyIn += iVal;
15140 if( iKeyIn!=iDelKeyOff ){
15141 int iKeyOut = (iKeyIn - (iKeyIn>iOff ? nShift : 0));
15142 nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOut - iPrevKeyOut);
15143 iPrevKeyOut = iKeyOut;
15144 }
15145 }
15146
15147 if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
15148 fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
15149 }
15150
15151 assert_nc( nPg>4 || fts5GetU16(aPg)==0 )((void) (0));
15152 fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pSeg->iLeafPgno
)) ), aPg, nPg);
15153 }
15154 sqlite3_freesqlite3_api->free(aIdx);
15155}
15156
15157/*
15158** This is called as part of flushing a delete to disk in 'secure-delete'
15159** mode. It edits the segments within the database described by argument
15160** pStruct to remove the entries for term zTerm, rowid iRowid.
15161**
15162** Return SQLITE_OK if successful, or an SQLite error code if an error
15163** has occurred. Any error code is also stored in the Fts5Index handle.
15164*/
15165static int fts5FlushSecureDelete(
15166 Fts5Index *p,
15167 Fts5Structure *pStruct,
15168 const char *zTerm,
15169 int nTerm,
15170 i64 iRowid
15171){
15172 const int f = FTS5INDEX_QUERY_SKIPHASH0x0040;
15173 Fts5Iter *pIter = 0; /* Used to find term instance */
15174
15175 /* If the version number has not been set to SECUREDELETE, do so now. */
15176 if( p->pConfig->iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE5 ){
15177 Fts5Config *pConfig = p->pConfig;
15178 sqlite3_stmt *pStmt = 0;
15179 fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintfsqlite3_api->mprintf(
15180 "REPLACE INTO %Q.'%q_config' VALUES ('version', %d)",
15181 pConfig->zDb, pConfig->zName, FTS5_CURRENT_VERSION_SECUREDELETE5
15182 ));
15183 if( p->rc==SQLITE_OK0 ){
15184 int rc;
15185 sqlite3_stepsqlite3_api->step(pStmt);
15186 rc = sqlite3_finalizesqlite3_api->finalize(pStmt);
15187 if( p->rc==SQLITE_OK0 ) p->rc = rc;
15188 pConfig->iCookie++;
15189 pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE5;
15190 }
15191 }
15192
15193 fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter);
15194 if( fts5MultiIterEof(p, pIter)==0 ){
15195 i64 iThis = fts5MultiIterRowid(pIter);
15196 if( iThis<iRowid ){
15197 fts5MultiIterNextFrom(p, pIter, iRowid);
15198 }
15199
15200 if( p->rc==SQLITE_OK0
15201 && fts5MultiIterEof(p, pIter)==0
15202 && iRowid==fts5MultiIterRowid(pIter)
15203 ){
15204 Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
15205 fts5DoSecureDelete(p, pSeg);
15206 }
15207 }
15208
15209 fts5MultiIterFree(pIter);
15210 return p->rc;
15211}
15212
15213
15214/*
15215** Flush the contents of in-memory hash table iHash to a new level-0
15216** segment on disk. Also update the corresponding structure record.
15217**
15218** If an error occurs, set the Fts5Index.rc error code. If an error has
15219** already occurred, this function is a no-op.
15220*/
15221static void fts5FlushOneHash(Fts5Index *p){
15222 Fts5Hash *pHash = p->pHash;
15223 Fts5Structure *pStruct;
15224 int iSegid;
15225 int pgnoLast = 0; /* Last leaf page number in segment */
15226
15227 /* Obtain a reference to the index structure and allocate a new segment-id
15228 ** for the new level-0 segment. */
15229 pStruct = fts5StructureRead(p);
15230 fts5StructureInvalidate(p);
15231
15232 if( sqlite3Fts5HashIsEmpty(pHash)==0 ){
15233 iSegid = fts5AllocateSegid(p, pStruct);
15234 if( iSegid ){
15235 const int pgsz = p->pConfig->pgsz;
15236 int eDetail = p->pConfig->eDetail;
15237 int bSecureDelete = p->pConfig->bSecureDelete;
15238 Fts5StructureSegment *pSeg; /* New segment within pStruct */
15239 Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */
15240 Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */
15241
15242 Fts5SegWriter writer;
15243 fts5WriteInit(p, &writer, iSegid);
15244
15245 pBuf = &writer.writer.buf;
15246 pPgidx = &writer.writer.pgidx;
15247
15248 /* fts5WriteInit() should have initialized the buffers to (most likely)
15249 ** the maximum space required. */
15250 assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) )((void) (0));
15251 assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) )((void) (0));
15252
15253 /* Begin scanning through hash table entries. This loop runs once for each
15254 ** term/doclist currently stored within the hash table. */
15255 if( p->rc==SQLITE_OK0 ){
15256 p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0);
15257 }
15258 while( p->rc==SQLITE_OK0 && 0==sqlite3Fts5HashScanEof(pHash) ){
15259 const char *zTerm; /* Buffer containing term */
15260 int nTerm; /* Size of zTerm in bytes */
15261 const u8 *pDoclist; /* Pointer to doclist for this term */
15262 int nDoclist; /* Size of doclist in bytes */
15263
15264 /* Get the term and doclist for this entry. */
15265 sqlite3Fts5HashScanEntry(pHash, &zTerm, &nTerm, &pDoclist, &nDoclist);
15266 if( bSecureDelete==0 ){
15267 fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
15268 if( p->rc!=SQLITE_OK0 ) break;
15269 assert( writer.bFirstRowidInPage==0 )((void) (0));
15270 }
15271
15272 if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
15273 /* The entire doclist will fit on the current leaf. */
15274 fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist){ ((void) (0)); memcpy(&(pBuf)->p[(pBuf)->n], pDoclist
, nDoclist); (pBuf)->n += nDoclist; };
15275 }else{
15276 int bTermWritten = !bSecureDelete;
15277 i64 iRowid = 0;
15278 i64 iPrev = 0;
15279 int iOff = 0;
15280
15281 /* The entire doclist will not fit on this leaf. The following
15282 ** loop iterates through the poslists that make up the current
15283 ** doclist. */
15284 while( p->rc==SQLITE_OK0 && iOff<nDoclist ){
15285 u64 iDelta = 0;
15286 iOff += fts5GetVarintsqlite3Fts5GetVarint(&pDoclist[iOff], &iDelta);
15287 iRowid += iDelta;
15288
15289 /* If in secure delete mode, and if this entry in the poslist is
15290 ** in fact a delete, then edit the existing segments directly
15291 ** using fts5FlushSecureDelete(). */
15292 if( bSecureDelete ){
15293 if( eDetail==FTS5_DETAIL_NONE1 ){
15294 if( iOff<nDoclist && pDoclist[iOff]==0x00
15295 && !fts5FlushSecureDelete(p, pStruct, zTerm, nTerm, iRowid)
15296 ){
15297 iOff++;
15298 if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
15299 iOff++;
15300 nDoclist = 0;
15301 }else{
15302 continue;
15303 }
15304 }
15305 }else if( (pDoclist[iOff] & 0x01)
15306 && !fts5FlushSecureDelete(p, pStruct, zTerm, nTerm, iRowid)
15307 ){
15308 if( p->rc!=SQLITE_OK0 || pDoclist[iOff]==0x01 ){
15309 iOff++;
15310 continue;
15311 }
15312 }
15313 }
15314
15315 if( p->rc==SQLITE_OK0 && bTermWritten==0 ){
15316 fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
15317 bTermWritten = 1;
15318 assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 )((void) (0));
15319 }
15320
15321 if( writer.bFirstRowidInPage ){
15322 fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */
15323 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
15324 writer.bFirstRowidInPage = 0;
15325 fts5WriteDlidxAppend(p, &writer, iRowid);
15326 }else{
15327 u64 iRowidDelta = (u64)iRowid - (u64)iPrev;
15328 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowidDelta);
15329 }
15330 if( p->rc!=SQLITE_OK0 ) break;
15331 assert( pBuf->n<=pBuf->nSpace )((void) (0));
15332 iPrev = iRowid;
15333
15334 if( eDetail==FTS5_DETAIL_NONE1 ){
15335 if( iOff<nDoclist && pDoclist[iOff]==0 ){
15336 pBuf->p[pBuf->n++] = 0;
15337 iOff++;
15338 if( iOff<nDoclist && pDoclist[iOff]==0 ){
15339 pBuf->p[pBuf->n++] = 0;
15340 iOff++;
15341 }
15342 }
15343 if( (pBuf->n + pPgidx->n)>=pgsz ){
15344 fts5WriteFlushLeaf(p, &writer);
15345 }
15346 }else{
15347 int bDel = 0;
15348 int nPos = 0;
15349 int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDel);
15350 if( bDel && bSecureDelete ){
15351 fts5BufferAppendVarint(&p->rc, pBuf, nPos*2)sqlite3Fts5BufferAppendVarint(&p->rc,pBuf,(i64)nPos*2);
15352 iOff += nCopy;
15353 nCopy = nPos;
15354 }else{
15355 nCopy += nPos;
15356 }
15357 if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
15358 /* The entire poslist will fit on the current leaf. So copy
15359 ** it in one go. */
15360 fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy){ ((void) (0)); memcpy(&(pBuf)->p[(pBuf)->n], &
pDoclist[iOff], nCopy); (pBuf)->n += nCopy; };
15361 }else{
15362 /* The entire poslist will not fit on this leaf. So it needs
15363 ** to be broken into sections. The only qualification being
15364 ** that each varint must be stored contiguously. */
15365 const u8 *pPoslist = &pDoclist[iOff];
15366 int iPos = 0;
15367 while( p->rc==SQLITE_OK0 ){
15368 int nSpace = pgsz - pBuf->n - pPgidx->n;
15369 int n = 0;
15370 if( (nCopy - iPos)<=nSpace ){
15371 n = nCopy - iPos;
15372 }else{
15373 n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
15374 }
15375 assert( n>0 )((void) (0));
15376 fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n){ ((void) (0)); memcpy(&(pBuf)->p[(pBuf)->n], &
pPoslist[iPos], n); (pBuf)->n += n; };
15377 iPos += n;
15378 if( (pBuf->n + pPgidx->n)>=pgsz ){
15379 fts5WriteFlushLeaf(p, &writer);
15380 }
15381 if( iPos>=nCopy ) break;
15382 }
15383 }
15384 iOff += nCopy;
15385 }
15386 }
15387 }
15388
15389 /* TODO2: Doclist terminator written here. */
15390 /* pBuf->p[pBuf->n++] = '\0'; */
15391 assert( pBuf->n<=pBuf->nSpace )((void) (0));
15392 if( p->rc==SQLITE_OK0 ) sqlite3Fts5HashScanNext(pHash);
15393 }
15394 fts5WriteFinish(p, &writer, &pgnoLast);
15395
15396 assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 )((void) (0));
15397 if( pgnoLast>0 ){
15398 /* Update the Fts5Structure. It is written back to the database by the
15399 ** fts5StructureRelease() call below. */
15400 if( pStruct->nLevel==0 ){
15401 fts5StructureAddLevel(&p->rc, &pStruct);
15402 }
15403 fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
15404 if( p->rc==SQLITE_OK0 ){
15405 pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
15406 pSeg->iSegid = iSegid;
15407 pSeg->pgnoFirst = 1;
15408 pSeg->pgnoLast = pgnoLast;
15409 if( pStruct->nOriginCntr>0 ){
15410 pSeg->iOrigin1 = pStruct->nOriginCntr;
15411 pSeg->iOrigin2 = pStruct->nOriginCntr;
15412 pSeg->nEntry = p->nPendingRow;
15413 pStruct->nOriginCntr++;
15414 }
15415 pStruct->nSegment++;
15416 }
15417 fts5StructurePromote(p, 0, pStruct);
15418 }
15419 }
15420 }
15421
15422 fts5IndexAutomerge(p, &pStruct, pgnoLast + p->nContentlessDelete);
15423 fts5IndexCrisismerge(p, &pStruct);
15424 fts5StructureWrite(p, pStruct);
15425 fts5StructureRelease(pStruct);
15426}
15427
15428/*
15429** Flush any data stored in the in-memory hash tables to the database.
15430*/
15431static void fts5IndexFlush(Fts5Index *p){
15432 /* Unless it is empty, flush the hash table to disk */
15433 if( p->flushRc ){
15434 p->rc = p->flushRc;
15435 return;
15436 }
15437 if( p->nPendingData || p->nContentlessDelete ){
15438 assert( p->pHash )((void) (0));
15439 fts5FlushOneHash(p);
15440 if( p->rc==SQLITE_OK0 ){
15441 sqlite3Fts5HashClear(p->pHash);
15442 p->nPendingData = 0;
15443 p->nPendingRow = 0;
15444 p->nContentlessDelete = 0;
15445 }else if( p->nPendingData || p->nContentlessDelete ){
15446 p->flushRc = p->rc;
15447 }
15448 }
15449}
15450
15451static Fts5Structure *fts5IndexOptimizeStruct(
15452 Fts5Index *p,
15453 Fts5Structure *pStruct
15454){
15455 Fts5Structure *pNew = 0;
15456 sqlite3_int64 nByte = SZ_FTS5STRUCTURE(1)(__builtin_offsetof(Fts5Structure, aLevel) + (1)*sizeof(Fts5StructureLevel
));
15457 int nSeg = pStruct->nSegment;
15458 int i;
15459
15460 /* Figure out if this structure requires optimization. A structure does
15461 ** not require optimization if either:
15462 **
15463 ** 1. it consists of fewer than two segments, or
15464 ** 2. all segments are on the same level, or
15465 ** 3. all segments except one are currently inputs to a merge operation.
15466 **
15467 ** In the first case, if there are no tombstone hash pages, return NULL. In
15468 ** the second, increment the ref-count on *pStruct and return a copy of the
15469 ** pointer to it.
15470 */
15471 if( nSeg==0 ) return 0;
15472 for(i=0; i<pStruct->nLevel; i++){
15473 int nThis = pStruct->aLevel[i].nSeg;
15474 int nMerge = pStruct->aLevel[i].nMerge;
15475 if( nThis>0 && (nThis==nSeg || (nThis==nSeg-1 && nMerge==nThis)) ){
15476 if( nSeg==1 && nThis==1 && pStruct->aLevel[i].aSeg[0].nPgTombstone==0 ){
15477 return 0;
15478 }
15479 fts5StructureRef(pStruct);
15480 return pStruct;
15481 }
15482 assert( pStruct->aLevel[i].nMerge<=nThis )((void) (0));
15483 }
15484
15485 nByte += (((i64)pStruct->nLevel)+1) * sizeof(Fts5StructureLevel);
15486 assert( nByte==(i64)SZ_FTS5STRUCTURE(pStruct->nLevel+2) )((void) (0));
15487 pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
15488
15489 if( pNew ){
15490 Fts5StructureLevel *pLvl;
15491 nByte = nSeg * sizeof(Fts5StructureSegment);
15492 pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL)(((pStruct->nLevel+1) < (64)) ? (pStruct->nLevel+1) :
(64));
15493 pNew->nRef = 1;
15494 pNew->nWriteCounter = pStruct->nWriteCounter;
15495 pNew->nOriginCntr = pStruct->nOriginCntr;
15496 pLvl = &pNew->aLevel[pNew->nLevel-1];
15497 pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
15498 if( pLvl->aSeg ){
15499 int iLvl, iSeg;
15500 int iSegOut = 0;
15501 /* Iterate through all segments, from oldest to newest. Add them to
15502 ** the new Fts5Level object so that pLvl->aSeg[0] is the oldest
15503 ** segment in the data structure. */
15504 for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){
15505 for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
15506 pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg];
15507 iSegOut++;
15508 }
15509 }
15510 pNew->nSegment = pLvl->nSeg = nSeg;
15511 }else{
15512 sqlite3_freesqlite3_api->free(pNew);
15513 pNew = 0;
15514 }
15515 }
15516
15517 return pNew;
15518}
15519
15520static int sqlite3Fts5IndexOptimize(Fts5Index *p){
15521 Fts5Structure *pStruct;
15522 Fts5Structure *pNew = 0;
15523
15524 assert( p->rc==SQLITE_OK )((void) (0));
15525 fts5IndexFlush(p);
15526 assert( p->rc!=SQLITE_OK || p->nContentlessDelete==0 )((void) (0));
15527 pStruct = fts5StructureRead(p);
15528 assert( p->rc!=SQLITE_OK || pStruct!=0 )((void) (0));
15529 fts5StructureInvalidate(p);
15530
15531 if( pStruct ){
15532 pNew = fts5IndexOptimizeStruct(p, pStruct);
15533 }
15534 fts5StructureRelease(pStruct);
15535
15536 assert( pNew==0 || pNew->nSegment>0 )((void) (0));
15537 if( pNew ){
15538 int iLvl;
15539 for(iLvl=0; pNew->aLevel[iLvl].nSeg==0; iLvl++){}
15540 while( p->rc==SQLITE_OK0 && pNew->aLevel[iLvl].nSeg>0 ){
15541 int nRem = FTS5_OPT_WORK_UNIT1000;
15542 fts5IndexMergeLevel(p, &pNew, iLvl, &nRem);
15543 }
15544
15545 fts5StructureWrite(p, pNew);
15546 fts5StructureRelease(pNew);
15547 }
15548
15549 return fts5IndexReturn(p);
15550}
15551
15552/*
15553** This is called to implement the special "VALUES('merge', $nMerge)"
15554** INSERT command.
15555*/
15556static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
15557 Fts5Structure *pStruct = 0;
15558
15559 fts5IndexFlush(p);
15560 pStruct = fts5StructureRead(p);
15561 if( pStruct ){
15562 int nMin = p->pConfig->nUsermerge;
15563 fts5StructureInvalidate(p);
15564 if( nMerge<0 ){
15565 Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
15566 fts5StructureRelease(pStruct);
15567 pStruct = pNew;
15568 nMin = 1;
15569 nMerge = (nMerge==SMALLEST_INT32((int)((-1) - ((int)(0x7fffffff)))) ? LARGEST_INT32((int)(0x7fffffff)) : (nMerge*-1));
15570 }
15571 if( pStruct && pStruct->nLevel ){
15572 if( fts5IndexMerge(p, &pStruct, nMerge, nMin) ){
15573 fts5StructureWrite(p, pStruct);
15574 }
15575 }
15576 fts5StructureRelease(pStruct);
15577 }
15578 return fts5IndexReturn(p);
15579}
15580
15581static void fts5AppendRowid(
15582 Fts5Index *p,
15583 u64 iDelta,
15584 Fts5Iter *pUnused,
15585 Fts5Buffer *pBuf
15586){
15587 UNUSED_PARAM(pUnused)(void)(pUnused);
15588 fts5BufferAppendVarint(&p->rc, pBuf, iDelta)sqlite3Fts5BufferAppendVarint(&p->rc,pBuf,(i64)iDelta);
15589}
15590
15591static void fts5AppendPoslist(
15592 Fts5Index *p,
15593 u64 iDelta,
15594 Fts5Iter *pMulti,
15595 Fts5Buffer *pBuf
15596){
15597 int nData = pMulti->base.nData;
15598 int nByte = nData + 9 + 9 + FTS5_DATA_ZERO_PADDING8;
15599 assert( nData>0 )((void) (0));
15600 if( p->rc==SQLITE_OK0 && 0==fts5BufferGrow(&p->rc, pBuf, nByte)( (u32)((pBuf)->n) + (u32)(nByte) <= (u32)((pBuf)->nSpace
) ? 0 : sqlite3Fts5BufferSize((&p->rc),(pBuf),(nByte)+
(pBuf)->n) ) ){
15601 fts5BufferSafeAppendVarint(pBuf, iDelta){ (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf
)->n], (iDelta)); ((void) (0)); };
15602 fts5BufferSafeAppendVarint(pBuf, nData*2){ (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf
)->n], (nData*2)); ((void) (0)); };
15603 fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData){ ((void) (0)); memcpy(&(pBuf)->p[(pBuf)->n], pMulti
->base.pData, nData); (pBuf)->n += nData; };
15604 memset(&pBuf->p[pBuf->n], 0, FTS5_DATA_ZERO_PADDING8);
15605 }
15606}
15607
15608
15609static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
15610 u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;
15611
15612 assert( pIter->aPoslist || (p==0 && pIter->aPoslist==0) )((void) (0));
15613 if( p>=pIter->aEof ){
15614 pIter->aPoslist = 0;
15615 }else{
15616 i64 iDelta;
15617
15618 p += fts5GetVarintsqlite3Fts5GetVarint(p, (u64*)&iDelta);
15619 pIter->iRowid += iDelta;
15620
15621 /* Read position list size */
15622 if( p[0] & 0x80 ){
15623 int nPos;
15624 pIter->nSize = fts5GetVarint32(p, nPos)sqlite3Fts5GetVarint32(p,(u32*)&(nPos));
15625 pIter->nPoslist = (nPos>>1);
15626 }else{
15627 pIter->nPoslist = ((int)(p[0])) >> 1;
15628 pIter->nSize = 1;
15629 }
15630
15631 pIter->aPoslist = p;
15632 if( &pIter->aPoslist[pIter->nPoslist]>pIter->aEof ){
15633 pIter->aPoslist = 0;
15634 }
15635 }
15636}
15637
15638static void fts5DoclistIterInit(
15639 Fts5Buffer *pBuf,
15640 Fts5DoclistIter *pIter
15641){
15642 memset(pIter, 0, sizeof(*pIter));
15643 if( pBuf->n>0 ){
15644 pIter->aPoslist = pBuf->p;
15645 pIter->aEof = &pBuf->p[pBuf->n];
15646 fts5DoclistIterNext(pIter);
15647 }
15648}
15649
15650#if 0
15651/*
15652** Append a doclist to buffer pBuf.
15653**
15654** This function assumes that space within the buffer has already been
15655** allocated.
15656*/
15657static void fts5MergeAppendDocid({ ((void) (0)); { ((Fts5Buffer *pBuf))->n += sqlite3Fts5PutVarint
(&((Fts5Buffer *pBuf))->p[((Fts5Buffer *pBuf))->n],
((u64)(i64 iRowid) - (u64)(i64 *piLastRowid))); ((void) (0))
; }; (i64 *piLastRowid) = (i64 iRowid); }
15658 Fts5Buffer *pBuf, /* Buffer to write to */{ ((void) (0)); { ((Fts5Buffer *pBuf))->n += sqlite3Fts5PutVarint
(&((Fts5Buffer *pBuf))->p[((Fts5Buffer *pBuf))->n],
((u64)(i64 iRowid) - (u64)(i64 *piLastRowid))); ((void) (0))
; }; (i64 *piLastRowid) = (i64 iRowid); }
15659 i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */{ ((void) (0)); { ((Fts5Buffer *pBuf))->n += sqlite3Fts5PutVarint
(&((Fts5Buffer *pBuf))->p[((Fts5Buffer *pBuf))->n],
((u64)(i64 iRowid) - (u64)(i64 *piLastRowid))); ((void) (0))
; }; (i64 *piLastRowid) = (i64 iRowid); }
15660 i64 iRowid /* Rowid to append */{ ((void) (0)); { ((Fts5Buffer *pBuf))->n += sqlite3Fts5PutVarint
(&((Fts5Buffer *pBuf))->p[((Fts5Buffer *pBuf))->n],
((u64)(i64 iRowid) - (u64)(i64 *piLastRowid))); ((void) (0))
; }; (i64 *piLastRowid) = (i64 iRowid); }
15661){ ((void) (0)); { ((Fts5Buffer *pBuf))->n += sqlite3Fts5PutVarint
(&((Fts5Buffer *pBuf))->p[((Fts5Buffer *pBuf))->n],
((u64)(i64 iRowid) - (u64)(i64 *piLastRowid))); ((void) (0))
; }; (i64 *piLastRowid) = (i64 iRowid); }{
15662 assert( pBuf->n!=0 || (*piLastRowid)==0 )((void) (0));
15663 fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid){ (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf
)->n], (iRowid - *piLastRowid)); ((void) (0)); };
15664 *piLastRowid = iRowid;
15665}
15666#endif
15667
15668#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid){ ((void) (0)); { ((pBuf))->n += sqlite3Fts5PutVarint(&
((pBuf))->p[((pBuf))->n], ((u64)(iRowid) - (u64)(iLastRowid
))); ((void) (0)); }; (iLastRowid) = (iRowid); } { \
15669 assert( (pBuf)->n!=0 || (iLastRowid)==0 )((void) (0)); \
15670 fts5BufferSafeAppendVarint((pBuf), (u64)(iRowid) - (u64)(iLastRowid)){ ((pBuf))->n += sqlite3Fts5PutVarint(&((pBuf))->p[
((pBuf))->n], ((u64)(iRowid) - (u64)(iLastRowid))); ((void
) (0)); }; \
15671 (iLastRowid) = (iRowid); \
15672}
15673
15674/*
15675** Swap the contents of buffer *p1 with that of *p2.
15676*/
15677static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
15678 Fts5Buffer tmp = *p1;
15679 *p1 = *p2;
15680 *p2 = tmp;
15681}
15682
15683static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){
15684 int i = *piOff;
15685 if( i>=pBuf->n ){
15686 *piOff = -1;
15687 }else{
15688 u64 iVal;
15689 *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal);
15690 *piRowid += iVal;
15691 }
15692}
15693
15694/*
15695** This is the equivalent of fts5MergePrefixLists() for detail=none mode.
15696** In this case the buffers consist of a delta-encoded list of rowids only.
15697*/
15698static void fts5MergeRowidLists(
15699 Fts5Index *p, /* FTS5 backend object */
15700 Fts5Buffer *p1, /* First list to merge */
15701 int nBuf, /* Number of entries in apBuf[] */
15702 Fts5Buffer *aBuf /* Array of other lists to merge into p1 */
15703){
15704 int i1 = 0;
15705 int i2 = 0;
15706 i64 iRowid1 = 0;
15707 i64 iRowid2 = 0;
15708 i64 iOut = 0;
15709 Fts5Buffer *p2 = &aBuf[0];
15710 Fts5Buffer out;
15711
15712 (void)nBuf;
15713 memset(&out, 0, sizeof(out));
15714 assert( nBuf==1 )((void) (0));
15715 sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n);
15716 if( p->rc ) return;
15717
15718 fts5NextRowid(p1, &i1, &iRowid1);
15719 fts5NextRowid(p2, &i2, &iRowid2);
15720 while( i1>=0 || i2>=0 ){
15721 if( i1>=0 && (i2<0 || iRowid1<iRowid2) ){
15722 assert( iOut==0 || iRowid1>iOut )((void) (0));
15723 fts5BufferSafeAppendVarint(&out, iRowid1 - iOut){ (&out)->n += sqlite3Fts5PutVarint(&(&out)->
p[(&out)->n], (iRowid1 - iOut)); ((void) (0)); };
15724 iOut = iRowid1;
15725 fts5NextRowid(p1, &i1, &iRowid1);
15726 }else{
15727 assert( iOut==0 || iRowid2>iOut )((void) (0));
15728 fts5BufferSafeAppendVarint(&out, iRowid2 - iOut){ (&out)->n += sqlite3Fts5PutVarint(&(&out)->
p[(&out)->n], (iRowid2 - iOut)); ((void) (0)); };
15729 iOut = iRowid2;
15730 if( i1>=0 && iRowid1==iRowid2 ){
15731 fts5NextRowid(p1, &i1, &iRowid1);
15732 }
15733 fts5NextRowid(p2, &i2, &iRowid2);
15734 }
15735 }
15736
15737 fts5BufferSwap(&out, p1);
15738 fts5BufferFree(&out)sqlite3Fts5BufferFree(&out);
15739}
15740
15741typedef struct PrefixMerger PrefixMerger;
15742struct PrefixMerger {
15743 Fts5DoclistIter iter; /* Doclist iterator */
15744 i64 iPos; /* For iterating through a position list */
15745 int iOff;
15746 u8 *aPos;
15747 PrefixMerger *pNext; /* Next in docid/poslist order */
15748};
15749
15750static void fts5PrefixMergerInsertByRowid(
15751 PrefixMerger **ppHead,
15752 PrefixMerger *p
15753){
15754 if( p->iter.aPoslist ){
15755 PrefixMerger **pp = ppHead;
15756 while( *pp && p->iter.iRowid>(*pp)->iter.iRowid ){
15757 pp = &(*pp)->pNext;
15758 }
15759 p->pNext = *pp;
15760 *pp = p;
15761 }
15762}
15763
15764static void fts5PrefixMergerInsertByPosition(
15765 PrefixMerger **ppHead,
15766 PrefixMerger *p
15767){
15768 if( p->iPos>=0 ){
15769 PrefixMerger **pp = ppHead;
15770 while( *pp && p->iPos>(*pp)->iPos ){
15771 pp = &(*pp)->pNext;
15772 }
15773 p->pNext = *pp;
15774 *pp = p;
15775 }
15776}
15777
15778
15779/*
15780** Array aBuf[] contains nBuf doclists. These are all merged in with the
15781** doclist in buffer p1.
15782*/
15783static void fts5MergePrefixLists(
15784 Fts5Index *p, /* FTS5 backend object */
15785 Fts5Buffer *p1, /* First list to merge */
15786 int nBuf, /* Number of buffers in array aBuf[] */
15787 Fts5Buffer *aBuf /* Other lists to merge in */
15788){
15789#define fts5PrefixMergerNextPosition(p)sqlite3Fts5PoslistNext64((p)->aPos,(p)->iter.nPoslist,&
(p)->iOff,&(p)->iPos) \
15790 sqlite3Fts5PoslistNext64((p)->aPos,(p)->iter.nPoslist,&(p)->iOff,&(p)->iPos)
15791#define FTS5_MERGE_NLIST16 16
15792 PrefixMerger aMerger[FTS5_MERGE_NLIST16];
15793 PrefixMerger *pHead = 0;
15794 int i;
15795 int nOut = 0;
15796 Fts5Buffer out = {0, 0, 0};
15797 Fts5Buffer tmp = {0, 0, 0};
15798 i64 iLastRowid = 0;
15799
15800 /* Initialize a doclist-iterator for each input buffer. Arrange them in
15801 ** a linked-list starting at pHead in ascending order of rowid. Avoid
15802 ** linking any iterators already at EOF into the linked list at all. */
15803 assert( nBuf+1<=(int)(sizeof(aMerger)/sizeof(aMerger[0])) )((void) (0));
15804 memset(aMerger, 0, sizeof(PrefixMerger)*(nBuf+1));
15805 pHead = &aMerger[nBuf];
15806 fts5DoclistIterInit(p1, &pHead->iter);
15807 for(i=0; i<nBuf; i++){
15808 fts5DoclistIterInit(&aBuf[i], &aMerger[i].iter);
15809 fts5PrefixMergerInsertByRowid(&pHead, &aMerger[i]);
15810 nOut += aBuf[i].n;
15811 }
15812 if( nOut==0 ) return;
15813 nOut += p1->n + 9 + 10*nBuf;
15814
15815 /* The maximum size of the output is equal to the sum of the
15816 ** input sizes + 1 varint (9 bytes). The extra varint is because if the
15817 ** first rowid in one input is a large negative number, and the first in
15818 ** the other a non-negative number, the delta for the non-negative
15819 ** number will be larger on disk than the literal integer value
15820 ** was.
15821 **
15822 ** Or, if the input position-lists are corrupt, then the output might
15823 ** include up to (nBuf+1) extra 10-byte positions created by interpreting -1
15824 ** (the value PoslistNext64() uses for EOF) as a position and appending
15825 ** it to the output. This can happen at most once for each input
15826 ** position-list, hence (nBuf+1) 10 byte paddings. */
15827 if( sqlite3Fts5BufferSize(&p->rc, &out, nOut) ) return;
15828
15829 while( pHead ){
15830 fts5MergeAppendDocid(&out, iLastRowid, pHead->iter.iRowid){ ((void) (0)); { ((&out))->n += sqlite3Fts5PutVarint(
&((&out))->p[((&out))->n], ((u64)(pHead->
iter.iRowid) - (u64)(iLastRowid))); ((void) (0)); }; (iLastRowid
) = (pHead->iter.iRowid); };
15831
15832 if( pHead->pNext && iLastRowid==pHead->pNext->iter.iRowid ){
15833 /* Merge data from two or more poslists */
15834 i64 iPrev = 0;
15835 int nTmp = FTS5_DATA_ZERO_PADDING8;
15836 int nMerge = 0;
15837 PrefixMerger *pSave = pHead;
15838 PrefixMerger *pThis = 0;
15839 int nTail = 0;
15840
15841 pHead = 0;
15842 while( pSave && pSave->iter.iRowid==iLastRowid ){
15843 PrefixMerger *pNext = pSave->pNext;
15844 pSave->iOff = 0;
15845 pSave->iPos = 0;
15846 pSave->aPos = &pSave->iter.aPoslist[pSave->iter.nSize];
15847 fts5PrefixMergerNextPosition(pSave)sqlite3Fts5PoslistNext64((pSave)->aPos,(pSave)->iter.nPoslist
,&(pSave)->iOff,&(pSave)->iPos);
15848 nTmp += pSave->iter.nPoslist + 10;
15849 nMerge++;
15850 fts5PrefixMergerInsertByPosition(&pHead, pSave);
15851 pSave = pNext;
15852 }
15853
15854 if( pHead==0 || pHead->pNext==0 ){
15855 FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
15856 break;
15857 }
15858
15859 /* See the earlier comment in this function for an explanation of why
15860 ** corrupt input position lists might cause the output to consume
15861 ** at most nMerge*10 bytes of unexpected space. */
15862 if( sqlite3Fts5BufferSize(&p->rc, &tmp, nTmp+nMerge*10) ){
15863 break;
15864 }
15865 fts5BufferZero(&tmp)sqlite3Fts5BufferZero(&tmp);
15866
15867 pThis = pHead;
15868 pHead = pThis->pNext;
15869 sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pThis->iPos);
15870 fts5PrefixMergerNextPosition(pThis)sqlite3Fts5PoslistNext64((pThis)->aPos,(pThis)->iter.nPoslist
,&(pThis)->iOff,&(pThis)->iPos);
15871 fts5PrefixMergerInsertByPosition(&pHead, pThis);
15872
15873 while( pHead->pNext ){
15874 pThis = pHead;
15875 if( pThis->iPos!=iPrev ){
15876 sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pThis->iPos);
15877 }
15878 fts5PrefixMergerNextPosition(pThis)sqlite3Fts5PoslistNext64((pThis)->aPos,(pThis)->iter.nPoslist
,&(pThis)->iOff,&(pThis)->iPos);
15879 pHead = pThis->pNext;
15880 fts5PrefixMergerInsertByPosition(&pHead, pThis);
15881 }
15882
15883 if( pHead->iPos!=iPrev ){
15884 sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pHead->iPos);
15885 }
15886 nTail = pHead->iter.nPoslist - pHead->iOff;
15887
15888 /* WRITEPOSLISTSIZE */
15889 assert_nc( tmp.n+nTail<=nTmp )((void) (0));
15890 assert( tmp.n+nTail<=nTmp+nMerge*10 )((void) (0));
15891 if( tmp.n+nTail>nTmp-FTS5_DATA_ZERO_PADDING8 ){
15892 if( p->rc==SQLITE_OK0 ) FTS5_CORRUPT_IDX(p)fts5IndexCorruptIdx(p);
15893 break;
15894 }
15895 fts5BufferSafeAppendVarint(&out, (tmp.n+nTail) * 2){ (&out)->n += sqlite3Fts5PutVarint(&(&out)->
p[(&out)->n], ((tmp.n+nTail) * 2)); ((void) (0)); };
15896 fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n){ ((void) (0)); memcpy(&(&out)->p[(&out)->n
], tmp.p, tmp.n); (&out)->n += tmp.n; };
15897 if( nTail>0 ){
15898 fts5BufferSafeAppendBlob(&out, &pHead->aPos[pHead->iOff], nTail){ ((void) (0)); memcpy(&(&out)->p[(&out)->n
], &pHead->aPos[pHead->iOff], nTail); (&out)->
n += nTail; };
15899 }
15900
15901 pHead = pSave;
15902 for(i=0; i<nBuf+1; i++){
15903 PrefixMerger *pX = &aMerger[i];
15904 if( pX->iter.aPoslist && pX->iter.iRowid==iLastRowid ){
15905 fts5DoclistIterNext(&pX->iter);
15906 fts5PrefixMergerInsertByRowid(&pHead, pX);
15907 }
15908 }
15909
15910 }else{
15911 /* Copy poslist from pHead to output */
15912 PrefixMerger *pThis = pHead;
15913 Fts5DoclistIter *pI = &pThis->iter;
15914 fts5BufferSafeAppendBlob(&out, pI->aPoslist, pI->nPoslist+pI->nSize){ ((void) (0)); memcpy(&(&out)->p[(&out)->n
], pI->aPoslist, pI->nPoslist+pI->nSize); (&out)
->n += pI->nPoslist+pI->nSize; };
15915 fts5DoclistIterNext(pI);
15916 pHead = pThis->pNext;
15917 fts5PrefixMergerInsertByRowid(&pHead, pThis);
15918 }
15919 }
15920
15921 fts5BufferFree(p1)sqlite3Fts5BufferFree(p1);
15922 fts5BufferFree(&tmp)sqlite3Fts5BufferFree(&tmp);
15923 memset(&out.p[out.n], 0, FTS5_DATA_ZERO_PADDING8);
15924 *p1 = out;
15925}
15926
15927
15928/*
15929** Iterate through a range of entries in the FTS index, invoking the xVisit
15930** callback for each of them.
15931**
15932** Parameter pToken points to an nToken buffer containing an FTS index term
15933** (i.e. a document term with the preceding 1 byte index identifier -
15934** FTS5_MAIN_PREFIX or similar). If bPrefix is true, then the call visits
15935** all entries for terms that have pToken/nToken as a prefix. If bPrefix
15936** is false, then only entries with pToken/nToken as the entire key are
15937** visited.
15938**
15939** If the current table is a tokendata=1 table, then if bPrefix is true then
15940** each index term is treated separately. However, if bPrefix is false, then
15941** all index terms corresponding to pToken/nToken are collapsed into a single
15942** term before the callback is invoked.
15943**
15944** The callback invoked for each entry visited is specified by paramter xVisit.
15945** Each time it is invoked, it is passed a pointer to the Fts5Index object,
15946** a copy of the 7th paramter to this function (pCtx) and a pointer to the
15947** iterator that indicates the current entry. If the current entry is the
15948** first with a new term (i.e. different from that of the previous entry,
15949** including the very first term), then the final two parameters are passed
15950** a pointer to the term and its size in bytes, respectively. If the current
15951** entry is not the first associated with its term, these two parameters
15952** are passed 0.
15953**
15954** If parameter pColset is not NULL, then it is used to filter entries before
15955** the callback is invoked.
15956*/
15957static int fts5VisitEntries(
15958 Fts5Index *p, /* Fts5 index object */
15959 Fts5Colset *pColset, /* Columns filter to apply, or NULL */
15960 u8 *pToken, /* Buffer containing token */
15961 int nToken, /* Size of buffer pToken in bytes */
15962 int bPrefix, /* True for a prefix scan */
15963 void (*xVisit)(Fts5Index*, void *pCtx, Fts5Iter *pIter, const u8*, int),
15964 void *pCtx /* Passed as second argument to xVisit() */
15965){
15966 const int flags = (bPrefix ? FTS5INDEX_QUERY_SCAN0x0008 : 0)
15967 | FTS5INDEX_QUERY_SKIPEMPTY0x0010
15968 | FTS5INDEX_QUERY_NOOUTPUT0x0020;
15969 Fts5Iter *p1 = 0; /* Iterator used to gather data from index */
15970 int bNewTerm = 1;
15971 Fts5Structure *pStruct = fts5StructureRead(p);
15972
15973 fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1);
15974 fts5IterSetOutputCb(&p->rc, p1);
15975 for( /* no-op */ ;
15976 fts5MultiIterEof(p, p1)==0;
15977 fts5MultiIterNext2(p, p1, &bNewTerm)
15978 ){
15979 Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ];
15980 int nNew = 0;
15981 const u8 *pNew = 0;
15982
15983 p1->xSetOutputs(p1, pSeg);
15984 if( p->rc ) break;
15985
15986 if( bNewTerm ){
15987 nNew = pSeg->term.n;
15988 pNew = pSeg->term.p;
15989 if( nNew<nToken || memcmp(pToken, pNew, nToken) ) break;
15990 }
15991
15992 xVisit(p, pCtx, p1, pNew, nNew);
15993 }
15994 fts5MultiIterFree(p1);
15995
15996 fts5StructureRelease(pStruct);
15997 return p->rc;
15998}
15999
16000
16001/*
16002** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an
16003** array of these for each row it visits (so all iRowid fields are the same).
16004** Or, for an iterator used by an "ORDER BY rank" query, it accumulates an
16005** array of these for the entire query (in which case iRowid fields may take
16006** a variety of values).
16007**
16008** Each instance in the array indicates the iterator (and therefore term)
16009** associated with position iPos of rowid iRowid. This is used by the
16010** xInstToken() API.
16011**
16012** iRowid:
16013** Rowid for the current entry.
16014**
16015** iPos:
16016** Position of current entry within row. In the usual ((iCol<<32)+iOff)
16017** format (e.g. see macros FTS5_POS2COLUMN() and FTS5_POS2OFFSET()).
16018**
16019** iIter:
16020** If the Fts5TokenDataIter iterator that the entry is part of is
16021** actually an iterator (i.e. with nIter>0, not just a container for
16022** Fts5TokenDataMap structures), then this variable is an index into
16023** the apIter[] array. The corresponding term is that which the iterator
16024** at apIter[iIter] currently points to.
16025**
16026** Or, if the Fts5TokenDataIter iterator is just a container object
16027** (nIter==0), then iIter is an index into the term.p[] buffer where
16028** the term is stored.
16029**
16030** nByte:
16031** In the case where iIter is an index into term.p[], this variable
16032** is the size of the term in bytes. If iIter is an index into apIter[],
16033** this variable is unused.
16034*/
16035struct Fts5TokenDataMap {
16036 i64 iRowid; /* Row this token is located in */
16037 i64 iPos; /* Position of token */
16038 int iIter; /* Iterator token was read from */
16039 int nByte; /* Length of token in bytes (or 0) */
16040};
16041
16042/*
16043** An object used to supplement Fts5Iter for tokendata=1 iterators.
16044**
16045** This object serves two purposes. The first is as a container for an array
16046** of Fts5TokenDataMap structures, which are used to find the token required
16047** when the xInstToken() API is used. This is done by the nMapAlloc, nMap and
16048** aMap[] variables.
16049*/
16050struct Fts5TokenDataIter {
16051 i64 nMapAlloc; /* Allocated size of aMap[] in entries */
16052 i64 nMap; /* Number of valid entries in aMap[] */
16053 Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */
16054
16055 /* The following are used for prefix-queries only. */
16056 Fts5Buffer terms;
16057
16058 /* The following are used for other full-token tokendata queries only. */
16059 i64 nIter;
16060 i64 nIterAlloc;
16061 Fts5PoslistReader *aPoslistReader;
16062 int *aPoslistToIter;
16063 Fts5Iter *apIter[FLEXARRAY];
16064};
16065
16066/* Size in bytes of an Fts5TokenDataIter object holding up to N iterators */
16067#define SZ_FTS5TOKENDATAITER(N)(__builtin_offsetof(Fts5TokenDataIter, apIter) + (N)*sizeof(Fts5Iter
)) \
16068 (offsetof(Fts5TokenDataIter,apIter)__builtin_offsetof(Fts5TokenDataIter, apIter) + (N)*sizeof(Fts5Iter))
16069
16070/*
16071** The two input arrays - a1[] and a2[] - are in sorted order. This function
16072** merges the two arrays together and writes the result to output array
16073** aOut[]. aOut[] is guaranteed to be large enough to hold the result.
16074**
16075** Duplicate entries are copied into the output. So the size of the output
16076** array is always (n1+n2) entries.
16077*/
16078static void fts5TokendataMerge(
16079 Fts5TokenDataMap *a1, int n1, /* Input array 1 */
16080 Fts5TokenDataMap *a2, int n2, /* Input array 2 */
16081 Fts5TokenDataMap *aOut /* Output array */
16082){
16083 int i1 = 0;
16084 int i2 = 0;
16085
16086 assert( n1>=0 && n2>=0 )((void) (0));
16087 while( i1<n1 || i2<n2 ){
16088 Fts5TokenDataMap *pOut = &aOut[i1+i2];
16089 if( i2>=n2 || (i1<n1 && (
16090 a1[i1].iRowid<a2[i2].iRowid
16091 || (a1[i1].iRowid==a2[i2].iRowid && a1[i1].iPos<=a2[i2].iPos)
16092 ))){
16093 memcpy(pOut, &a1[i1], sizeof(Fts5TokenDataMap));
16094 i1++;
16095 }else{
16096 memcpy(pOut, &a2[i2], sizeof(Fts5TokenDataMap));
16097 i2++;
16098 }
16099 }
16100}
16101
16102
16103/*
16104** Append a mapping to the token-map belonging to object pT.
16105*/
16106static void fts5TokendataIterAppendMap(
16107 Fts5Index *p,
16108 Fts5TokenDataIter *pT,
16109 int iIter,
16110 int nByte,
16111 i64 iRowid,
16112 i64 iPos
16113){
16114 if( p->rc==SQLITE_OK0 ){
16115 if( pT->nMap==pT->nMapAlloc ){
16116 i64 nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64;
16117 i64 nAlloc = nNew * sizeof(Fts5TokenDataMap);
16118 Fts5TokenDataMap *aNew;
16119
16120 aNew = (Fts5TokenDataMap*)sqlite3_realloc64sqlite3_api->realloc64(pT->aMap, nAlloc);
16121 if( aNew==0 ){
16122 p->rc = SQLITE_NOMEM7;
16123 return;
16124 }
16125
16126 pT->aMap = aNew;
16127 pT->nMapAlloc = nNew;
16128 }
16129
16130 pT->aMap[pT->nMap].iRowid = iRowid;
16131 pT->aMap[pT->nMap].iPos = iPos;
16132 pT->aMap[pT->nMap].iIter = iIter;
16133 pT->aMap[pT->nMap].nByte = nByte;
16134 pT->nMap++;
16135 }
16136}
16137
16138/*
16139** Sort the contents of the pT->aMap[] array.
16140**
16141** The sorting algorithm requires a malloc(). If this fails, an error code
16142** is left in Fts5Index.rc before returning.
16143*/
16144static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){
16145 Fts5TokenDataMap *aTmp = 0;
16146 i64 nByte = pT->nMap * sizeof(Fts5TokenDataMap);
16147
16148 aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte);
16149 if( aTmp ){
16150 Fts5TokenDataMap *a1 = pT->aMap;
16151 Fts5TokenDataMap *a2 = aTmp;
16152 i64 nHalf;
16153
16154 for(nHalf=1; nHalf<pT->nMap; nHalf=nHalf*2){
16155 int i1;
16156 for(i1=0; i1<pT->nMap; i1+=(nHalf*2)){
16157 int n1 = MIN(nHalf, pT->nMap-i1)(((nHalf) < (pT->nMap-i1)) ? (nHalf) : (pT->nMap-i1)
);
16158 int n2 = MIN(nHalf, pT->nMap-i1-n1)(((nHalf) < (pT->nMap-i1-n1)) ? (nHalf) : (pT->nMap-
i1-n1));
16159 fts5TokendataMerge(&a1[i1], n1, &a1[i1+n1], n2, &a2[i1]);
16160 }
16161 SWAPVAL(Fts5TokenDataMap*, a1, a2){ Fts5TokenDataMap* tmp; tmp=a1; a1=a2; a2=tmp; };
16162 }
16163
16164 if( a1!=pT->aMap ){
16165 memcpy(pT->aMap, a1, pT->nMap*sizeof(Fts5TokenDataMap));
16166 }
16167 sqlite3_freesqlite3_api->free(aTmp);
16168
16169#ifdef SQLITE_DEBUG
16170 {
16171 int ii;
16172 for(ii=1; ii<pT->nMap; ii++){
16173 Fts5TokenDataMap *p1 = &pT->aMap[ii-1];
16174 Fts5TokenDataMap *p2 = &pT->aMap[ii];
16175 assert( p1->iRowid<p2->iRowid((void) (0))
16176 || (p1->iRowid==p2->iRowid && p1->iPos<=p2->iPos)((void) (0))
16177 )((void) (0));
16178 }
16179 }
16180#endif
16181 }
16182}
16183
16184/*
16185** Delete an Fts5TokenDataIter structure and its contents.
16186*/
16187static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){
16188 if( pSet ){
16189 int ii;
16190 for(ii=0; ii<pSet->nIter; ii++){
16191 fts5MultiIterFree(pSet->apIter[ii]);
16192 }
16193 fts5BufferFree(&pSet->terms)sqlite3Fts5BufferFree(&pSet->terms);
16194 sqlite3_freesqlite3_api->free(pSet->aPoslistReader);
16195 sqlite3_freesqlite3_api->free(pSet->aMap);
16196 sqlite3_freesqlite3_api->free(pSet);
16197 }
16198}
16199
16200
16201/*
16202** fts5VisitEntries() context object used by fts5SetupPrefixIterTokendata()
16203** to pass data to prefixIterSetupTokendataCb().
16204*/
16205typedef struct TokendataSetupCtx TokendataSetupCtx;
16206struct TokendataSetupCtx {
16207 Fts5TokenDataIter *pT; /* Object being populated with mappings */
16208 int iTermOff; /* Offset of current term in terms.p[] */
16209 int nTermByte; /* Size of current term in bytes */
16210};
16211
16212/*
16213** fts5VisitEntries() callback used by fts5SetupPrefixIterTokendata(). This
16214** callback adds an entry to the Fts5TokenDataIter.aMap[] array for each
16215** position in the current position-list. It doesn't matter that some of
16216** these may be out of order - they will be sorted later.
16217*/
16218static void prefixIterSetupTokendataCb(
16219 Fts5Index *p,
16220 void *pCtx,
16221 Fts5Iter *p1,
16222 const u8 *pNew,
16223 int nNew
16224){
16225 TokendataSetupCtx *pSetup = (TokendataSetupCtx*)pCtx;
16226 int iPosOff = 0;
16227 i64 iPos = 0;
16228
16229 if( pNew ){
16230 pSetup->nTermByte = nNew-1;
16231 pSetup->iTermOff = pSetup->pT->terms.n;
16232 fts5BufferAppendBlob(&p->rc, &pSetup->pT->terms, nNew-1, pNew+1)sqlite3Fts5BufferAppendBlob(&p->rc,&pSetup->pT->
terms,nNew-1,pNew+1);
16233 }
16234
16235 while( 0==sqlite3Fts5PoslistNext64(
16236 p1->base.pData, p1->base.nData, &iPosOff, &iPos
16237 ) ){
16238 fts5TokendataIterAppendMap(p,
16239 pSetup->pT, pSetup->iTermOff, pSetup->nTermByte, p1->base.iRowid, iPos
16240 );
16241 }
16242}
16243
16244
16245/*
16246** Context object passed by fts5SetupPrefixIter() to fts5VisitEntries().
16247*/
16248typedef struct PrefixSetupCtx PrefixSetupCtx;
16249struct PrefixSetupCtx {
16250 void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*);
16251 void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*);
16252 i64 iLastRowid;
16253 int nMerge;
16254 Fts5Buffer *aBuf;
16255 int nBuf;
16256 Fts5Buffer doclist;
16257 TokendataSetupCtx *pTokendata;
16258};
16259
16260/*
16261** fts5VisitEntries() callback used by fts5SetupPrefixIter()
16262*/
16263static void prefixIterSetupCb(
16264 Fts5Index *p,
16265 void *pCtx,
16266 Fts5Iter *p1,
16267 const u8 *pNew,
16268 int nNew
16269){
16270 PrefixSetupCtx *pSetup = (PrefixSetupCtx*)pCtx;
16271 const int nMerge = pSetup->nMerge;
16272
16273 if( p1->base.nData>0 ){
16274 if( p1->base.iRowid<=pSetup->iLastRowid && pSetup->doclist.n>0 ){
16275 int i;
16276 for(i=0; p->rc==SQLITE_OK0 && pSetup->doclist.n; i++){
16277 int i1 = i*nMerge;
16278 int iStore;
16279 assert( i1+nMerge<=pSetup->nBuf )((void) (0));
16280 for(iStore=i1; iStore<i1+nMerge; iStore++){
16281 if( pSetup->aBuf[iStore].n==0 ){
16282 fts5BufferSwap(&pSetup->doclist, &pSetup->aBuf[iStore]);
16283 fts5BufferZero(&pSetup->doclist)sqlite3Fts5BufferZero(&pSetup->doclist);
16284 break;
16285 }
16286 }
16287 if( iStore==i1+nMerge ){
16288 pSetup->xMerge(p, &pSetup->doclist, nMerge, &pSetup->aBuf[i1]);
16289 for(iStore=i1; iStore<i1+nMerge; iStore++){
16290 fts5BufferZero(&pSetup->aBuf[iStore])sqlite3Fts5BufferZero(&pSetup->aBuf[iStore]);
16291 }
16292 }
16293 }
16294 pSetup->iLastRowid = 0;
16295 }
16296
16297 pSetup->xAppend(
16298 p, (u64)p1->base.iRowid-(u64)pSetup->iLastRowid, p1, &pSetup->doclist
16299 );
16300 pSetup->iLastRowid = p1->base.iRowid;
16301 }
16302
16303 if( pSetup->pTokendata ){
16304 prefixIterSetupTokendataCb(p, (void*)pSetup->pTokendata, p1, pNew, nNew);
16305 }
16306}
16307
16308static void fts5SetupPrefixIter(
16309 Fts5Index *p, /* Index to read from */
16310 int bDesc, /* True for "ORDER BY rowid DESC" */
16311 int iIdx, /* Index to scan for data */
16312 u8 *pToken, /* Buffer containing prefix to match */
16313 int nToken, /* Size of buffer pToken in bytes */
16314 Fts5Colset *pColset, /* Restrict matches to these columns */
16315 Fts5Iter **ppIter /* OUT: New iterator */
16316){
16317 Fts5Structure *pStruct;
16318 PrefixSetupCtx s;
16319 TokendataSetupCtx s2;
16320
16321 memset(&s, 0, sizeof(s));
16322 memset(&s2, 0, sizeof(s2));
16323
16324 s.nMerge = 1;
16325 s.iLastRowid = 0;
16326 s.nBuf = 32;
16327 if( iIdx==0
16328 && p->pConfig->eDetail==FTS5_DETAIL_FULL0
16329 && p->pConfig->bPrefixInsttoken
16330 ){
16331 s.pTokendata = &s2;
16332 s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, SZ_FTS5TOKENDATAITER(1)(__builtin_offsetof(Fts5TokenDataIter, apIter) + (1)*sizeof(Fts5Iter
)));
16333 }
16334
16335 if( p->pConfig->eDetail==FTS5_DETAIL_NONE1 ){
16336 s.xMerge = fts5MergeRowidLists;
16337 s.xAppend = fts5AppendRowid;
16338 }else{
16339 s.nMerge = FTS5_MERGE_NLIST16-1;
16340 s.nBuf = s.nMerge*8; /* Sufficient to merge (16^8)==(2^32) lists */
16341 s.xMerge = fts5MergePrefixLists;
16342 s.xAppend = fts5AppendPoslist;
16343 }
16344
16345 s.aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*s.nBuf);
16346 pStruct = fts5StructureRead(p);
16347 assert( p->rc!=SQLITE_OK || (s.aBuf && pStruct) )((void) (0));
16348
16349 if( p->rc==SQLITE_OK0 ){
16350 void *pCtx = (void*)&s;
16351 int i;
16352 Fts5Data *pData;
16353
16354 /* If iIdx is non-zero, then it is the number of a prefix-index for
16355 ** prefixes 1 character longer than the prefix being queried for. That
16356 ** index contains all the doclists required, except for the one
16357 ** corresponding to the prefix itself. That one is extracted from the
16358 ** main term index here. */
16359 if( iIdx!=0 ){
16360 pToken[0] = FTS5_MAIN_PREFIX'0';
16361 fts5VisitEntries(p, pColset, pToken, nToken, 0, prefixIterSetupCb, pCtx);
16362 }
16363
16364 pToken[0] = FTS5_MAIN_PREFIX'0' + iIdx;
16365 fts5VisitEntries(p, pColset, pToken, nToken, 1, prefixIterSetupCb, pCtx);
16366
16367 assert( (s.nBuf%s.nMerge)==0 )((void) (0));
16368 for(i=0; i<s.nBuf; i+=s.nMerge){
16369 int iFree;
16370 if( p->rc==SQLITE_OK0 ){
16371 s.xMerge(p, &s.doclist, s.nMerge, &s.aBuf[i]);
16372 }
16373 for(iFree=i; iFree<i+s.nMerge; iFree++){
16374 fts5BufferFree(&s.aBuf[iFree])sqlite3Fts5BufferFree(&s.aBuf[iFree]);
16375 }
16376 }
16377
16378 pData = fts5IdxMalloc(p, sizeof(*pData)
16379 + ((i64)s.doclist.n)+FTS5_DATA_ZERO_PADDING8);
16380 assert( pData!=0 || p->rc!=SQLITE_OK )((void) (0));
16381 if( pData ){
16382 pData->p = (u8*)&pData[1];
16383 pData->nn = pData->szLeaf = s.doclist.n;
16384 if( s.doclist.n ) memcpy(pData->p, s.doclist.p, s.doclist.n);
16385 fts5MultiIterNew2(p, pData, bDesc, ppIter);
16386 }
16387
16388 assert( (*ppIter)!=0 || p->rc!=SQLITE_OK )((void) (0));
16389 if( p->rc==SQLITE_OK0 && s.pTokendata ){
16390 fts5TokendataIterSortMap(p, s2.pT);
16391 (*ppIter)->pTokenDataIter = s2.pT;
16392 s2.pT = 0;
16393 }
16394 }
16395
16396 fts5TokendataIterDelete(s2.pT);
16397 fts5BufferFree(&s.doclist)sqlite3Fts5BufferFree(&s.doclist);
16398 fts5StructureRelease(pStruct);
16399 sqlite3_freesqlite3_api->free(s.aBuf);
16400}
16401
16402
16403/*
16404** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
16405** to the document with rowid iRowid.
16406*/
16407static int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
16408 assert( p->rc==SQLITE_OK )((void) (0));
16409
16410 /* Allocate the hash table if it has not already been allocated */
16411 if( p->pHash==0 ){
16412 p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData);
16413 }
16414
16415 /* Flush the hash table to disk if required */
16416 if( iRowid<p->iWriteRowid
16417 || (iRowid==p->iWriteRowid && p->bDelete==0)
16418 || (p->nPendingData > p->pConfig->nHashSize)
16419 ){
16420 fts5IndexFlush(p);
16421 }
16422
16423 p->iWriteRowid = iRowid;
16424 p->bDelete = bDelete;
16425 if( bDelete==0 ){
16426 p->nPendingRow++;
16427 }
16428 return fts5IndexReturn(p);
16429}
16430
16431/*
16432** Commit data to disk.
16433*/
16434static int sqlite3Fts5IndexSync(Fts5Index *p){
16435 assert( p->rc==SQLITE_OK )((void) (0));
16436 fts5IndexFlush(p);
16437 fts5IndexCloseReader(p);
16438 return fts5IndexReturn(p);
16439}
16440
16441/*
16442** Discard any data stored in the in-memory hash tables. Do not write it
16443** to the database. Additionally, assume that the contents of the %_data
16444** table may have changed on disk. So any in-memory caches of %_data
16445** records must be invalidated.
16446*/
16447static int sqlite3Fts5IndexRollback(Fts5Index *p){
16448 fts5IndexCloseReader(p);
16449 fts5IndexDiscardData(p);
16450 fts5StructureInvalidate(p);
16451 return fts5IndexReturn(p);
16452}
16453
16454/*
16455** The %_data table is completely empty when this function is called. This
16456** function populates it with the initial structure objects for each index,
16457** and the initial version of the "averages" record (a zero-byte blob).
16458*/
16459static int sqlite3Fts5IndexReinit(Fts5Index *p){
16460 Fts5Structure *pTmp;
16461 union {
16462 Fts5Structure sFts;
16463 u8 tmpSpace[SZ_FTS5STRUCTURE(1)(__builtin_offsetof(Fts5Structure, aLevel) + (1)*sizeof(Fts5StructureLevel
))];
16464 } uFts;
16465 fts5StructureInvalidate(p);
16466 fts5IndexDiscardData(p);
16467 pTmp = &uFts.sFts;
16468 memset(uFts.tmpSpace, 0, sizeof(uFts.tmpSpace));
16469 if( p->pConfig->bContentlessDelete ){
16470 pTmp->nOriginCntr = 1;
16471 }
16472 fts5DataWrite(p, FTS5_AVERAGES_ROWID1, (const u8*)"", 0);
16473 fts5StructureWrite(p, pTmp);
16474 return fts5IndexReturn(p);
16475}
16476
16477/*
16478** Open a new Fts5Index handle. If the bCreate argument is true, create
16479** and initialize the underlying %_data table.
16480**
16481** If successful, set *pp to point to the new object and return SQLITE_OK.
16482** Otherwise, set *pp to NULL and return an SQLite error code.
16483*/
16484static int sqlite3Fts5IndexOpen(
16485 Fts5Config *pConfig,
16486 int bCreate,
16487 Fts5Index **pp,
16488 char **pzErr
16489){
16490 int rc = SQLITE_OK0;
16491 Fts5Index *p; /* New object */
16492
16493 *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index));
16494 if( rc==SQLITE_OK0 ){
16495 p->pConfig = pConfig;
16496 p->nWorkUnit = FTS5_WORK_UNIT64;
16497 p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName);
16498 if( p->zDataTbl && bCreate ){
16499 rc = sqlite3Fts5CreateTable(
16500 pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
16501 );
16502 if( rc==SQLITE_OK0 ){
16503 rc = sqlite3Fts5CreateTable(pConfig, "idx",
16504 "segid, term, pgno, PRIMARY KEY(segid, term)",
16505 1, pzErr
16506 );
16507 }
16508 if( rc==SQLITE_OK0 ){
16509 rc = sqlite3Fts5IndexReinit(p);
16510 }
16511 }
16512 }
16513
16514 assert( rc!=SQLITE_OK || p->rc==SQLITE_OK )((void) (0));
16515 if( rc ){
16516 sqlite3Fts5IndexClose(p);
16517 *pp = 0;
16518 }
16519 return rc;
16520}
16521
16522/*
16523** Close a handle opened by an earlier call to sqlite3Fts5IndexOpen().
16524*/
16525static int sqlite3Fts5IndexClose(Fts5Index *p){
16526 int rc = SQLITE_OK0;
16527 if( p ){
16528 assert( p->pReader==0 )((void) (0));
16529 fts5StructureInvalidate(p);
16530 sqlite3_finalizesqlite3_api->finalize(p->pWriter);
16531 sqlite3_finalizesqlite3_api->finalize(p->pDeleter);
16532 sqlite3_finalizesqlite3_api->finalize(p->pIdxWriter);
16533 sqlite3_finalizesqlite3_api->finalize(p->pIdxDeleter);
16534 sqlite3_finalizesqlite3_api->finalize(p->pIdxSelect);
16535 sqlite3_finalizesqlite3_api->finalize(p->pIdxNextSelect);
16536 sqlite3_finalizesqlite3_api->finalize(p->pDataVersion);
16537 sqlite3_finalizesqlite3_api->finalize(p->pDeleteFromIdx);
16538 sqlite3Fts5HashFree(p->pHash);
16539 sqlite3_freesqlite3_api->free(p->zDataTbl);
16540 sqlite3_freesqlite3_api->free(p);
16541 }
16542 return rc;
16543}
16544
16545/*
16546** Argument p points to a buffer containing utf-8 text that is n bytes in
16547** size. Return the number of bytes in the nChar character prefix of the
16548** buffer, or 0 if there are less than nChar characters in total.
16549*/
16550static int sqlite3Fts5IndexCharlenToBytelen(
16551 const char *p,
16552 int nByte,
16553 int nChar
16554){
16555 int n = 0;
16556 int i;
16557 for(i=0; i<nChar; i++){
16558 if( n>=nByte ) return 0; /* Input contains fewer than nChar chars */
16559 if( (unsigned char)p[n++]>=0xc0 ){
16560 if( n>=nByte ) return 0;
16561 while( (p[n] & 0xc0)==0x80 ){
16562 n++;
16563 if( n>=nByte ){
16564 if( i+1==nChar ) break;
16565 return 0;
16566 }
16567 }
16568 }
16569 }
16570 return n;
16571}
16572
16573/*
16574** pIn is a UTF-8 encoded string, nIn bytes in size. Return the number of
16575** unicode characters in the string.
16576*/
16577static int fts5IndexCharlen(const char *pIn, int nIn){
16578 int nChar = 0;
16579 int i = 0;
16580 while( i<nIn ){
16581 if( (unsigned char)pIn[i++]>=0xc0 ){
16582 while( i<nIn && (pIn[i] & 0xc0)==0x80 ) i++;
16583 }
16584 nChar++;
16585 }
16586 return nChar;
16587}
16588
16589/*
16590** Insert or remove data to or from the index. Each time a document is
16591** added to or removed from the index, this function is called one or more
16592** times.
16593**
16594** For an insert, it must be called once for each token in the new document.
16595** If the operation is a delete, it must be called (at least) once for each
16596** unique token in the document with an iCol value less than zero. The iPos
16597** argument is ignored for a delete.
16598*/
16599static int sqlite3Fts5IndexWrite(
16600 Fts5Index *p, /* Index to write to */
16601 int iCol, /* Column token appears in (-ve -> delete) */
16602 int iPos, /* Position of token within column */
16603 const char *pToken, int nToken /* Token to add or remove to or from index */
16604){
16605 int i; /* Used to iterate through indexes */
16606 int rc = SQLITE_OK0; /* Return code */
16607 Fts5Config *pConfig = p->pConfig;
16608
16609 assert( p->rc==SQLITE_OK )((void) (0));
16610 assert( (iCol<0)==p->bDelete )((void) (0));
16611
16612 /* Add the entry to the main terms index. */
16613 rc = sqlite3Fts5HashWrite(
16614 p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX'0', pToken, nToken
16615 );
16616
16617 for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK0; i++){
16618 const int nChar = pConfig->aPrefix[i];
16619 int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
16620 if( nByte ){
16621 rc = sqlite3Fts5HashWrite(p->pHash,
16622 p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX'0'+i+1), pToken,
16623 nByte
16624 );
16625 }
16626 }
16627
16628 return rc;
16629}
16630
16631/*
16632** pToken points to a buffer of size nToken bytes containing a search
16633** term, including the index number at the start, used on a tokendata=1
16634** table. This function returns true if the term in buffer pBuf matches
16635** token pToken/nToken.
16636*/
16637static int fts5IsTokendataPrefix(
16638 Fts5Buffer *pBuf,
16639 const u8 *pToken,
16640 int nToken
16641){
16642 return (
16643 pBuf->n>=nToken
16644 && 0==memcmp(pBuf->p, pToken, nToken)
16645 && (pBuf->n==nToken || pBuf->p[nToken]==0x00)
16646 );
16647}
16648
16649/*
16650** Ensure the segment-iterator passed as the only argument points to EOF.
16651*/
16652static void fts5SegIterSetEOF(Fts5SegIter *pSeg){
16653 fts5DataRelease(pSeg->pLeaf);
16654 pSeg->pLeaf = 0;
16655}
16656
16657static void fts5IterClose(Fts5IndexIter *pIndexIter){
16658 if( pIndexIter ){
16659 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
16660 Fts5Index *pIndex = pIter->pIndex;
16661 fts5TokendataIterDelete(pIter->pTokenDataIter);
16662 fts5MultiIterFree(pIter);
16663 fts5IndexCloseReader(pIndex);
16664 }
16665}
16666
16667/*
16668** This function appends iterator pAppend to Fts5TokenDataIter pIn and
16669** returns the result.
16670*/
16671static Fts5TokenDataIter *fts5AppendTokendataIter(
16672 Fts5Index *p, /* Index object (for error code) */
16673 Fts5TokenDataIter *pIn, /* Current Fts5TokenDataIter struct */
16674 Fts5Iter *pAppend /* Append this iterator */
16675){
16676 Fts5TokenDataIter *pRet = pIn;
16677
16678 if( p->rc==SQLITE_OK0 ){
16679 if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){
16680 i64 nAlloc = pIn ? pIn->nIterAlloc*2 : 16;
16681 i64 nByte = SZ_FTS5TOKENDATAITER(nAlloc+1)(__builtin_offsetof(Fts5TokenDataIter, apIter) + (nAlloc+1)*sizeof
(Fts5Iter));
16682 Fts5TokenDataIter *pNew;
16683 pNew = (Fts5TokenDataIter*)sqlite3_realloc64sqlite3_api->realloc64(pIn, nByte);
16684
16685 if( pNew==0 ){
16686 p->rc = SQLITE_NOMEM7;
16687 }else{
16688 if( pIn==0 ) memset(pNew, 0, nByte);
16689 pRet = pNew;
16690 pNew->nIterAlloc = nAlloc;
16691 }
16692 }
16693 }
16694 if( p->rc ){
16695 fts5IterClose((Fts5IndexIter*)pAppend);
16696 }else{
16697 pRet->apIter[pRet->nIter++] = pAppend;
16698 }
16699 assert( pRet==0 || pRet->nIter<=pRet->nIterAlloc )((void) (0));
16700
16701 return pRet;
16702}
16703
16704/*
16705** The iterator passed as the only argument must be a tokendata=1 iterator
16706** (pIter->pTokenDataIter!=0). This function sets the iterator output
16707** variables (pIter->base.*) according to the contents of the current
16708** row.
16709*/
16710static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){
16711 int ii;
16712 int nHit = 0;
16713 i64 iRowid = SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32)));
16714 int iMin = 0;
16715
16716 Fts5TokenDataIter *pT = pIter->pTokenDataIter;
16717
16718 pIter->base.nData = 0;
16719 pIter->base.pData = 0;
16720
16721 for(ii=0; ii<pT->nIter; ii++){
16722 Fts5Iter *p = pT->apIter[ii];
16723 if( p->base.bEof==0 ){
16724 if( nHit==0 || p->base.iRowid<iRowid ){
16725 iRowid = p->base.iRowid;
16726 nHit = 1;
16727 pIter->base.pData = p->base.pData;
16728 pIter->base.nData = p->base.nData;
16729 iMin = ii;
16730 }else if( p->base.iRowid==iRowid ){
16731 nHit++;
16732 }
16733 }
16734 }
16735
16736 if( nHit==0 ){
16737 pIter->base.bEof = 1;
16738 }else{
16739 int eDetail = pIter->pIndex->pConfig->eDetail;
16740 pIter->base.bEof = 0;
16741 pIter->base.iRowid = iRowid;
16742
16743 if( nHit==1 && eDetail==FTS5_DETAIL_FULL0 ){
16744 fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, 0, iRowid, -1);
16745 }else
16746 if( nHit>1 && eDetail!=FTS5_DETAIL_NONE1 ){
16747 int nReader = 0;
16748 int nByte = 0;
16749 i64 iPrev = 0;
16750
16751 /* Allocate array of iterators if they are not already allocated. */
16752 if( pT->aPoslistReader==0 ){
16753 pT->aPoslistReader = (Fts5PoslistReader*)sqlite3Fts5MallocZero(
16754 &pIter->pIndex->rc,
16755 pT->nIter * (sizeof(Fts5PoslistReader) + sizeof(int))
16756 );
16757 if( pT->aPoslistReader==0 ) return;
16758 pT->aPoslistToIter = (int*)&pT->aPoslistReader[pT->nIter];
16759 }
16760
16761 /* Populate an iterator for each poslist that will be merged */
16762 for(ii=0; ii<pT->nIter; ii++){
16763 Fts5Iter *p = pT->apIter[ii];
16764 if( iRowid==p->base.iRowid ){
16765 pT->aPoslistToIter[nReader] = ii;
16766 sqlite3Fts5PoslistReaderInit(
16767 p->base.pData, p->base.nData, &pT->aPoslistReader[nReader++]
16768 );
16769 nByte += p->base.nData;
16770 }
16771 }
16772
16773 /* Ensure the output buffer is large enough */
16774 if( fts5BufferGrow(&pIter->pIndex->rc, &pIter->poslist, nByte+nHit*10)( (u32)((&pIter->poslist)->n) + (u32)(nByte+nHit*10
) <= (u32)((&pIter->poslist)->nSpace) ? 0 : sqlite3Fts5BufferSize
((&pIter->pIndex->rc),(&pIter->poslist),(nByte
+nHit*10)+(&pIter->poslist)->n) ) ){
16775 return;
16776 }
16777
16778 /* Ensure the token-mapping is large enough */
16779 if( eDetail==FTS5_DETAIL_FULL0 && pT->nMapAlloc<(pT->nMap + nByte) ){
16780 i64 nNew = (pT->nMapAlloc + nByte) * 2;
16781 Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc64sqlite3_api->realloc64(
16782 pT->aMap, nNew*sizeof(Fts5TokenDataMap)
16783 );
16784 if( aNew==0 ){
16785 pIter->pIndex->rc = SQLITE_NOMEM7;
16786 return;
16787 }
16788 pT->aMap = aNew;
16789 pT->nMapAlloc = nNew;
16790 }
16791
16792 pIter->poslist.n = 0;
16793
16794 while( 1 ){
16795 i64 iMinPos = LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32));
16796
16797 /* Find smallest position */
16798 iMin = 0;
16799 for(ii=0; ii<nReader; ii++){
16800 Fts5PoslistReader *pReader = &pT->aPoslistReader[ii];
16801 if( pReader->bEof==0 ){
16802 if( pReader->iPos<iMinPos ){
16803 iMinPos = pReader->iPos;
16804 iMin = ii;
16805 }
16806 }
16807 }
16808
16809 /* If all readers were at EOF, break out of the loop. */
16810 if( iMinPos==LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)) ) break;
16811
16812 sqlite3Fts5PoslistSafeAppend(&pIter->poslist, &iPrev, iMinPos);
16813 sqlite3Fts5PoslistReaderNext(&pT->aPoslistReader[iMin]);
16814
16815 if( eDetail==FTS5_DETAIL_FULL0 ){
16816 pT->aMap[pT->nMap].iPos = iMinPos;
16817 pT->aMap[pT->nMap].iIter = pT->aPoslistToIter[iMin];
16818 pT->aMap[pT->nMap].iRowid = iRowid;
16819 pT->nMap++;
16820 }
16821 }
16822
16823 pIter->base.pData = pIter->poslist.p;
16824 pIter->base.nData = pIter->poslist.n;
16825 }
16826 }
16827}
16828
16829/*
16830** The iterator passed as the only argument must be a tokendata=1 iterator
16831** (pIter->pTokenDataIter!=0). This function advances the iterator. If
16832** argument bFrom is false, then the iterator is advanced to the next
16833** entry. Or, if bFrom is true, it is advanced to the first entry with
16834** a rowid of iFrom or greater.
16835*/
16836static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){
16837 int ii;
16838 Fts5TokenDataIter *pT = pIter->pTokenDataIter;
16839 Fts5Index *pIndex = pIter->pIndex;
16840
16841 for(ii=0; ii<pT->nIter; ii++){
16842 Fts5Iter *p = pT->apIter[ii];
16843 if( p->base.bEof==0
16844 && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowid<iFrom))
16845 ){
16846 fts5MultiIterNext(pIndex, p, bFrom, iFrom);
16847 while( bFrom && p->base.bEof==0
16848 && p->base.iRowid<iFrom
16849 && pIndex->rc==SQLITE_OK0
16850 ){
16851 fts5MultiIterNext(pIndex, p, 0, 0);
16852 }
16853 }
16854 }
16855
16856 if( pIndex->rc==SQLITE_OK0 ){
16857 fts5IterSetOutputsTokendata(pIter);
16858 }
16859}
16860
16861/*
16862** If the segment-iterator passed as the first argument is at EOF, then
16863** set pIter->term to a copy of buffer pTerm.
16864*/
16865static void fts5TokendataSetTermIfEof(Fts5Iter *pIter, Fts5Buffer *pTerm){
16866 if( pIter && pIter->aSeg[0].pLeaf==0 ){
16867 fts5BufferSet(&pIter->pIndex->rc, &pIter->aSeg[0].term, pTerm->n, pTerm->p)sqlite3Fts5BufferSet(&pIter->pIndex->rc,&pIter->
aSeg[0].term,pTerm->n,pTerm->p);
16868 }
16869}
16870
16871/*
16872** This function sets up an iterator to use for a non-prefix query on a
16873** tokendata=1 table.
16874*/
16875static Fts5Iter *fts5SetupTokendataIter(
16876 Fts5Index *p, /* FTS index to query */
16877 const u8 *pToken, /* Buffer containing query term */
16878 int nToken, /* Size of buffer pToken in bytes */
16879 Fts5Colset *pColset /* Colset to filter on */
16880){
16881 Fts5Iter *pRet = 0;
16882 Fts5TokenDataIter *pSet = 0;
16883 Fts5Structure *pStruct = 0;
16884 const int flags = FTS5INDEX_QUERY_SCANONETERM0x0100 | FTS5INDEX_QUERY_SCAN0x0008;
16885
16886 Fts5Buffer bSeek = {0, 0, 0};
16887 Fts5Buffer *pSmall = 0;
16888
16889 fts5IndexFlush(p);
16890 pStruct = fts5StructureRead(p);
16891
16892 while( p->rc==SQLITE_OK0 ){
16893 Fts5Iter *pPrev = pSet ? pSet->apIter[pSet->nIter-1] : 0;
16894 Fts5Iter *pNew = 0;
16895 Fts5SegIter *pNewIter = 0;
16896 Fts5SegIter *pPrevIter = 0;
16897
16898 int iLvl, iSeg, ii;
16899
16900 pNew = fts5MultiIterAlloc(p, pStruct->nSegment);
16901 if( pSmall ){
16902 fts5BufferSet(&p->rc, &bSeek, pSmall->n, pSmall->p)sqlite3Fts5BufferSet(&p->rc,&bSeek,pSmall->n,pSmall
->p);
16903 fts5BufferAppendBlob(&p->rc, &bSeek, 1, (const u8*)"\0")sqlite3Fts5BufferAppendBlob(&p->rc,&bSeek,1,(const
u8*)"\0");
16904 }else{
16905 fts5BufferSet(&p->rc, &bSeek, nToken, pToken)sqlite3Fts5BufferSet(&p->rc,&bSeek,nToken,pToken);
16906 }
16907 if( p->rc ){
16908 fts5IterClose((Fts5IndexIter*)pNew);
16909 break;
16910 }
16911
16912 pNewIter = &pNew->aSeg[0];
16913 pPrevIter = (pPrev ? &pPrev->aSeg[0] : 0);
16914 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
16915 for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){
16916 Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
16917 int bDone = 0;
16918
16919 if( pPrevIter ){
16920 if( fts5BufferCompare(pSmall, &pPrevIter->term) ){
16921 memcpy(pNewIter, pPrevIter, sizeof(Fts5SegIter));
16922 memset(pPrevIter, 0, sizeof(Fts5SegIter));
16923 bDone = 1;
16924 }else if( pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf ){
16925 fts5SegIterNextInit(p,(const char*)bSeek.p,bSeek.n-1,pSeg,pNewIter);
16926 bDone = 1;
16927 }
16928 }
16929
16930 if( bDone==0 ){
16931 fts5SegIterSeekInit(p, bSeek.p, bSeek.n, flags, pSeg, pNewIter);
16932 }
16933
16934 if( pPrevIter ){
16935 if( pPrevIter->pTombArray ){
16936 pNewIter->pTombArray = pPrevIter->pTombArray;
16937 pNewIter->pTombArray->nRef++;
16938 }
16939 }else{
16940 fts5SegIterAllocTombstone(p, pNewIter);
16941 }
16942
16943 pNewIter++;
16944 if( pPrevIter ) pPrevIter++;
16945 if( p->rc ) break;
16946 }
16947 }
16948 fts5TokendataSetTermIfEof(pPrev, pSmall);
16949
16950 pNew->bSkipEmpty = 1;
16951 pNew->pColset = pColset;
16952 fts5IterSetOutputCb(&p->rc, pNew);
16953
16954 /* Loop through all segments in the new iterator. Find the smallest
16955 ** term that any segment-iterator points to. Iterator pNew will be
16956 ** used for this term. Also, set any iterator that points to a term that
16957 ** does not match pToken/nToken to point to EOF */
16958 pSmall = 0;
16959 for(ii=0; ii<pNew->nSeg; ii++){
16960 Fts5SegIter *pII = &pNew->aSeg[ii];
16961 if( 0==fts5IsTokendataPrefix(&pII->term, pToken, nToken) ){
16962 fts5SegIterSetEOF(pII);
16963 }
16964 if( pII->pLeaf && (!pSmall || fts5BufferCompare(pSmall, &pII->term)>0) ){
16965 pSmall = &pII->term;
16966 }
16967 }
16968
16969 /* If pSmall is still NULL at this point, then the new iterator does
16970 ** not point to any terms that match the query. So delete it and break
16971 ** out of the loop - all required iterators have been collected. */
16972 if( pSmall==0 ){
16973 fts5IterClose((Fts5IndexIter*)pNew);
16974 break;
16975 }
16976
16977 /* Append this iterator to the set and continue. */
16978 pSet = fts5AppendTokendataIter(p, pSet, pNew);
16979 }
16980
16981 if( p->rc==SQLITE_OK0 && pSet ){
16982 int ii;
16983 for(ii=0; ii<pSet->nIter; ii++){
16984 Fts5Iter *pIter = pSet->apIter[ii];
16985 int iSeg;
16986 for(iSeg=0; iSeg<pIter->nSeg; iSeg++){
16987 pIter->aSeg[iSeg].flags |= FTS5_SEGITER_ONETERM0x01;
16988 }
16989 fts5MultiIterFinishSetup(p, pIter);
16990 }
16991 }
16992
16993 if( p->rc==SQLITE_OK0 ){
16994 pRet = fts5MultiIterAlloc(p, 0);
16995 }
16996 if( pRet ){
16997 pRet->nSeg = 0;
16998 pRet->pTokenDataIter = pSet;
16999 if( pSet ){
17000 fts5IterSetOutputsTokendata(pRet);
17001 }else{
17002 pRet->base.bEof = 1;
17003 }
17004 }else{
17005 fts5TokendataIterDelete(pSet);
17006 }
17007
17008 fts5StructureRelease(pStruct);
17009 fts5BufferFree(&bSeek)sqlite3Fts5BufferFree(&bSeek);
17010 return pRet;
17011}
17012
17013/*
17014** Open a new iterator to iterate though all rowid that match the
17015** specified token or token prefix.
17016*/
17017static int sqlite3Fts5IndexQuery(
17018 Fts5Index *p, /* FTS index to query */
17019 const char *pToken, int nToken, /* Token (or prefix) to query for */
17020 int flags, /* Mask of FTS5INDEX_QUERY_X flags */
17021 Fts5Colset *pColset, /* Match these columns only */
17022 Fts5IndexIter **ppIter /* OUT: New iterator object */
17023){
17024 Fts5Config *pConfig = p->pConfig;
17025 Fts5Iter *pRet = 0;
17026 Fts5Buffer buf = {0, 0, 0};
17027
17028 /* If the QUERY_SCAN flag is set, all other flags must be clear. */
17029 assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN )((void) (0));
17030
17031 if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
17
←
Assuming the condition is false
→
18
←
Taking false branch
→
17032 int iIdx = 0; /* Index to search */
17033 int iPrefixIdx = 0; /* +1 prefix index */
17034 int bTokendata = pConfig->bTokendata;
17035 assert( buf.p!=0 )((void) (0));
17036 if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken);
17037
17038 /* The NOTOKENDATA flag is set when each token in a tokendata=1 table
17039 ** should be treated individually, instead of merging all those with
17040 ** a common prefix into a single entry. This is used, for example, by
17041 ** queries performed as part of an integrity-check, or by the fts5vocab
17042 ** module. */
17043 if( flags & (FTS5INDEX_QUERY_NOTOKENDATA0x0080|FTS5INDEX_QUERY_SCAN0x0008) ){
17044 bTokendata = 0;
17045 }
17046
17047 /* Figure out which index to search and set iIdx accordingly. If this
17048 ** is a prefix query for which there is no prefix index, set iIdx to
17049 ** greater than pConfig->nPrefix to indicate that the query will be
17050 ** satisfied by scanning multiple terms in the main index.
17051 **
17052 ** If the QUERY_TEST_NOIDX flag was specified, then this must be a
17053 ** prefix-query. Instead of using a prefix-index (if one exists),
17054 ** evaluate the prefix query using the main FTS index. This is used
17055 ** for internal sanity checking by the integrity-check in debug
17056 ** mode only. */
17057#ifdef SQLITE_DEBUG
17058 if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX0x0004) ){
17059 assert( flags & FTS5INDEX_QUERY_PREFIX )((void) (0));
17060 iIdx = 1+pConfig->nPrefix;
17061 }else
17062#endif
17063 if( flags & FTS5INDEX_QUERY_PREFIX0x0001 ){
17064 int nChar = fts5IndexCharlen(pToken, nToken);
17065 for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
17066 int nIdxChar = pConfig->aPrefix[iIdx-1];
17067 if( nIdxChar==nChar ) break;
17068 if( nIdxChar==nChar+1 ) iPrefixIdx = iIdx;
17069 }
17070 }
17071
17072 if( bTokendata && iIdx==0 ){
17073 buf.p[0] = FTS5_MAIN_PREFIX'0';
17074 pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset);
17075 }else if( iIdx<=pConfig->nPrefix ){
17076 /* Straight index lookup */
17077 Fts5Structure *pStruct = fts5StructureRead(p);
17078 buf.p[0] = (u8)(FTS5_MAIN_PREFIX'0' + iIdx);
17079 if( pStruct ){
17080 fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY0x0010,
17081 pColset, buf.p, nToken+1, -1, 0, &pRet
17082 );
17083 fts5StructureRelease(pStruct);
17084 }
17085 }else{
17086 /* Scan multiple terms in the main index for a prefix query. */
17087 int bDesc = (flags & FTS5INDEX_QUERY_DESC0x0002)!=0;
17088 fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet);
17089 if( pRet==0 ){
17090 assert( p->rc!=SQLITE_OK )((void) (0));
17091 }else{
17092 assert( pRet->pColset==0 )((void) (0));
17093 fts5IterSetOutputCb(&p->rc, pRet);
17094 if( p->rc==SQLITE_OK0 ){
17095 Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst];
17096 if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
17097 }
17098 }
17099 }
17100
17101 if( p->rc ){
17102 fts5IterClose((Fts5IndexIter*)pRet);
17103 pRet = 0;
17104 fts5IndexCloseReader(p);
17105 }
17106
17107 *ppIter = (Fts5IndexIter*)pRet;
17108 sqlite3Fts5BufferFree(&buf);
17109 }
17110 return fts5IndexReturn(p);
19
←
Returning without writing to '*ppIter'
→
17111}
17112
17113/*
17114** Return true if the iterator passed as the only argument is at EOF.
17115*/
17116/*
17117** Move to the next matching rowid.
17118*/
17119static int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){
17120 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17121 assert( pIter->pIndex->rc==SQLITE_OK )((void) (0));
17122 if( pIter->nSeg==0 ){
17123 assert( pIter->pTokenDataIter )((void) (0));
17124 fts5TokendataIterNext(pIter, 0, 0);
17125 }else{
17126 fts5MultiIterNext(pIter->pIndex, pIter, 0, 0);
17127 }
17128 return fts5IndexReturn(pIter->pIndex);
17129}
17130
17131/*
17132** Move to the next matching term/rowid. Used by the fts5vocab module.
17133*/
17134static int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){
17135 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17136 Fts5Index *p = pIter->pIndex;
17137
17138 assert( pIter->pIndex->rc==SQLITE_OK )((void) (0));
17139
17140 fts5MultiIterNext(p, pIter, 0, 0);
17141 if( p->rc==SQLITE_OK0 ){
17142 Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
17143 if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX'0' ){
17144 fts5DataRelease(pSeg->pLeaf);
17145 pSeg->pLeaf = 0;
17146 pIter->base.bEof = 1;
17147 }
17148 }
17149
17150 return fts5IndexReturn(pIter->pIndex);
17151}
17152
17153/*
17154** Move to the next matching rowid that occurs at or after iMatch. The
17155** definition of "at or after" depends on whether this iterator iterates
17156** in ascending or descending rowid order.
17157*/
17158static int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){
17159 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17160 if( pIter->nSeg==0 ){
17161 assert( pIter->pTokenDataIter )((void) (0));
17162 fts5TokendataIterNext(pIter, 1, iMatch);
17163 }else{
17164 fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch);
17165 }
17166 return fts5IndexReturn(pIter->pIndex);
17167}
17168
17169/*
17170** Return the current term.
17171*/
17172static const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){
17173 int n;
17174 const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n);
17175 assert_nc( z || n<=1 )((void) (0));
17176 *pn = n-1;
17177 return (z ? &z[1] : 0);
17178}
17179
17180/*
17181** pIter is a prefix query. This function populates pIter->pTokenDataIter
17182** with an Fts5TokenDataIter object containing mappings for all rows
17183** matched by the query.
17184*/
17185static int fts5SetupPrefixIterTokendata(
17186 Fts5Iter *pIter,
17187 const char *pToken, /* Token prefix to search for */
17188 int nToken /* Size of pToken in bytes */
17189){
17190 Fts5Index *p = pIter->pIndex;
17191 Fts5Buffer token = {0, 0, 0};
17192 TokendataSetupCtx ctx;
17193
17194 memset(&ctx, 0, sizeof(ctx));
17195
17196 fts5BufferGrow(&p->rc, &token, nToken+1)( (u32)((&token)->n) + (u32)(nToken+1) <= (u32)((&
token)->nSpace) ? 0 : sqlite3Fts5BufferSize((&p->rc
),(&token),(nToken+1)+(&token)->n) );
17197 assert( token.p!=0 || p->rc!=SQLITE_OK )((void) (0));
17198 ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc,
17199 SZ_FTS5TOKENDATAITER(1)(__builtin_offsetof(Fts5TokenDataIter, apIter) + (1)*sizeof(Fts5Iter
)));
17200
17201 if( p->rc==SQLITE_OK0 ){
17202
17203 /* Fill in the token prefix to search for */
17204 token.p[0] = FTS5_MAIN_PREFIX'0';
17205 memcpy(&token.p[1], pToken, nToken);
17206 token.n = nToken+1;
17207
17208 fts5VisitEntries(
17209 p, 0, token.p, token.n, 1, prefixIterSetupTokendataCb, (void*)&ctx
17210 );
17211
17212 fts5TokendataIterSortMap(p, ctx.pT);
17213 }
17214
17215 if( p->rc==SQLITE_OK0 ){
17216 pIter->pTokenDataIter = ctx.pT;
17217 }else{
17218 fts5TokendataIterDelete(ctx.pT);
17219 }
17220 fts5BufferFree(&token)sqlite3Fts5BufferFree(&token);
17221
17222 return fts5IndexReturn(p);
17223}
17224
17225/*
17226** This is used by xInstToken() to access the token at offset iOff, column
17227** iCol of row iRowid. The token is returned via output variables *ppOut
17228** and *pnOut. The iterator passed as the first argument must be a tokendata=1
17229** iterator (pIter->pTokenDataIter!=0).
17230**
17231** pToken/nToken:
17232*/
17233static int sqlite3Fts5IterToken(
17234 Fts5IndexIter *pIndexIter,
17235 const char *pToken, int nToken,
17236 i64 iRowid,
17237 int iCol,
17238 int iOff,
17239 const char **ppOut, int *pnOut
17240){
17241 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17242 Fts5TokenDataIter *pT = pIter->pTokenDataIter;
17243 i64 iPos = (((i64)iCol)<<32) + iOff;
17244 Fts5TokenDataMap *aMap = 0;
17245 int i1 = 0;
17246 int i2 = 0;
17247 int iTest = 0;
17248
17249 assert( pT || (pToken && pIter->nSeg>0) )((void) (0));
17250 if( pT==0 ){
17251 int rc = fts5SetupPrefixIterTokendata(pIter, pToken, nToken);
17252 if( rc!=SQLITE_OK0 ) return rc;
17253 pT = pIter->pTokenDataIter;
17254 }
17255
17256 i2 = pT->nMap;
17257 aMap = pT->aMap;
17258
17259 while( i2>i1 ){
17260 iTest = (i1 + i2) / 2;
17261
17262 if( aMap[iTest].iRowid<iRowid ){
17263 i1 = iTest+1;
17264 }else if( aMap[iTest].iRowid>iRowid ){
17265 i2 = iTest;
17266 }else{
17267 if( aMap[iTest].iPos<iPos ){
17268 if( aMap[iTest].iPos<0 ){
17269 break;
17270 }
17271 i1 = iTest+1;
17272 }else if( aMap[iTest].iPos>iPos ){
17273 i2 = iTest;
17274 }else{
17275 break;
17276 }
17277 }
17278 }
17279
17280 if( i2>i1 ){
17281 if( pIter->nSeg==0 ){
17282 Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter];
17283 *ppOut = (const char*)pMap->aSeg[0].term.p+1;
17284 *pnOut = pMap->aSeg[0].term.n-1;
17285 }else{
17286 Fts5TokenDataMap *p = &aMap[iTest];
17287 *ppOut = (const char*)&pT->terms.p[p->iIter];
17288 *pnOut = aMap[iTest].nByte;
17289 }
17290 }
17291
17292 return SQLITE_OK0;
17293}
17294
17295/*
17296** Clear any existing entries from the token-map associated with the
17297** iterator passed as the only argument.
17298*/
17299static void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){
17300 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17301 if( pIter && pIter->pTokenDataIter
17302 && (pIter->nSeg==0 || pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL0)
17303 ){
17304 pIter->pTokenDataIter->nMap = 0;
17305 }
17306}
17307
17308/*
17309** Set a token-mapping for the iterator passed as the first argument. This
17310** is used in detail=column or detail=none mode when a token is requested
17311** using the xInstToken() API. In this case the caller tokenizers the
17312** current row and configures the token-mapping via multiple calls to this
17313** function.
17314*/
17315static int sqlite3Fts5IndexIterWriteTokendata(
17316 Fts5IndexIter *pIndexIter,
17317 const char *pToken, int nToken,
17318 i64 iRowid, int iCol, int iOff
17319){
17320 Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
17321 Fts5TokenDataIter *pT = pIter->pTokenDataIter;
17322 Fts5Index *p = pIter->pIndex;
17323 i64 iPos = (((i64)iCol)<<32) + iOff;
17324
17325 assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL )((void) (0));
17326 assert( pIter->pTokenDataIter || pIter->nSeg>0 )((void) (0));
17327 if( pIter->nSeg>0 ){
17328 /* This is a prefix term iterator. */
17329 if( pT==0 ){
17330 pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc,
17331 SZ_FTS5TOKENDATAITER(1)(__builtin_offsetof(Fts5TokenDataIter, apIter) + (1)*sizeof(Fts5Iter
)));
17332 pIter->pTokenDataIter = pT;
17333 }
17334 if( pT ){
17335 fts5TokendataIterAppendMap(p, pT, pT->terms.n, nToken, iRowid, iPos);
17336 fts5BufferAppendBlob(&p->rc, &pT->terms, nToken, (const u8*)pToken)sqlite3Fts5BufferAppendBlob(&p->rc,&pT->terms,nToken
,(const u8*)pToken);
17337 }
17338 }else{
17339 int ii;
17340 for(ii=0; ii<pT->nIter; ii++){
17341 Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term;
17342 if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break;
17343 }
17344 if( ii<pT->nIter ){
17345 fts5TokendataIterAppendMap(p, pT, ii, 0, iRowid, iPos);
17346 }
17347 }
17348 return fts5IndexReturn(p);
17349}
17350
17351/*
17352** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
17353*/
17354static void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){
17355 if( pIndexIter ){
17356 Fts5Index *pIndex = ((Fts5Iter*)pIndexIter)->pIndex;
17357 fts5IterClose(pIndexIter);
17358 fts5IndexReturn(pIndex);
17359 }
17360}
17361
17362/*
17363** Read and decode the "averages" record from the database.
17364**
17365** Parameter anSize must point to an array of size nCol, where nCol is
17366** the number of user defined columns in the FTS table.
17367*/
17368static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){
17369 int nCol = p->pConfig->nCol;
17370 Fts5Data *pData;
17371
17372 *pnRow = 0;
17373 memset(anSize, 0, sizeof(i64) * nCol);
17374 pData = fts5DataRead(p, FTS5_AVERAGES_ROWID1);
17375 if( p->rc==SQLITE_OK0 && pData->nn ){
17376 int i = 0;
17377 int iCol;
17378 i += fts5GetVarintsqlite3Fts5GetVarint(&pData->p[i], (u64*)pnRow);
17379 for(iCol=0; i<pData->nn && iCol<nCol; iCol++){
17380 i += fts5GetVarintsqlite3Fts5GetVarint(&pData->p[i], (u64*)&anSize[iCol]);
17381 }
17382 }
17383
17384 fts5DataRelease(pData);
17385 return fts5IndexReturn(p);
17386}
17387
17388/*
17389** Replace the current "averages" record with the contents of the buffer
17390** supplied as the second argument.
17391*/
17392static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){
17393 assert( p->rc==SQLITE_OK )((void) (0));
17394 fts5DataWrite(p, FTS5_AVERAGES_ROWID1, pData, nData);
17395 return fts5IndexReturn(p);
17396}
17397
17398/*
17399** Return the total number of blocks this module has read from the %_data
17400** table since it was created.
17401*/
17402static int sqlite3Fts5IndexReads(Fts5Index *p){
17403 return p->nRead;
17404}
17405
17406/*
17407** Set the 32-bit cookie value stored at the start of all structure
17408** records to the value passed as the second argument.
17409**
17410** Return SQLITE_OK if successful, or an SQLite error code if an error
17411** occurs.
17412*/
17413static int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){
17414 int rc; /* Return code */
17415 Fts5Config *pConfig = p->pConfig; /* Configuration object */
17416 u8 aCookie[4]; /* Binary representation of iNew */
17417 sqlite3_blob *pBlob = 0;
17418
17419 assert( p->rc==SQLITE_OK )((void) (0));
17420 sqlite3Fts5Put32(aCookie, iNew);
17421
17422 rc = sqlite3_blob_opensqlite3_api->blob_open(pConfig->db, pConfig->zDb, p->zDataTbl,
17423 "block", FTS5_STRUCTURE_ROWID10, 1, &pBlob
17424 );
17425 if( rc==SQLITE_OK0 ){
17426 sqlite3_blob_writesqlite3_api->blob_write(pBlob, aCookie, 4, 0);
17427 rc = sqlite3_blob_closesqlite3_api->blob_close(pBlob);
17428 }
17429
17430 return rc;
17431}
17432
17433static int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
17434 Fts5Structure *pStruct;
17435 pStruct = fts5StructureRead(p);
17436 fts5StructureRelease(pStruct);
17437 return fts5IndexReturn(p);
17438}
17439
17440/*
17441** Retrieve the origin value that will be used for the segment currently
17442** being accumulated in the in-memory hash table when it is flushed to
17443** disk. If successful, SQLITE_OK is returned and (*piOrigin) set to
17444** the queried value. Or, if an error occurs, an error code is returned
17445** and the final value of (*piOrigin) is undefined.
17446*/
17447static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){
17448 Fts5Structure *pStruct;
17449 pStruct = fts5StructureRead(p);
17450 if( pStruct ){
17451 *piOrigin = pStruct->nOriginCntr;
17452 fts5StructureRelease(pStruct);
17453 }
17454 return fts5IndexReturn(p);
17455}
17456
17457/*
17458** Buffer pPg contains a page of a tombstone hash table - one of nPg pages
17459** associated with the same segment. This function adds rowid iRowid to
17460** the hash table. The caller is required to guarantee that there is at
17461** least one free slot on the page.
17462**
17463** If parameter bForce is false and the hash table is deemed to be full
17464** (more than half of the slots are occupied), then non-zero is returned
17465** and iRowid not inserted. Or, if bForce is true or if the hash table page
17466** is not full, iRowid is inserted and zero returned.
17467*/
17468static int fts5IndexTombstoneAddToPage(
17469 Fts5Data *pPg,
17470 int bForce,
17471 int nPg,
17472 u64 iRowid
17473){
17474 const int szKey = TOMBSTONE_KEYSIZE(pPg)(pPg->p[0]==4 ? 4 : 8);
17475 const int nSlot = TOMBSTONE_NSLOT(pPg)((pPg->nn > 16) ? ((pPg->nn-8) / (pPg->p[0]==4 ? 4
: 8)) : 1);
17476 const int nElem = fts5GetU32(&pPg->p[4]);
17477 int iSlot = (iRowid / nPg) % nSlot;
17478 int nCollide = nSlot;
17479
17480 if( szKey==4 && iRowid>0xFFFFFFFF ) return 2;
17481 if( iRowid==0 ){
17482 pPg->p[1] = 0x01;
17483 return 0;
17484 }
17485
17486 if( bForce==0 && nElem>=(nSlot/2) ){
17487 return 1;
17488 }
17489
17490 fts5PutU32(&pPg->p[4], nElem+1);
17491 if( szKey==4 ){
17492 u32 *aSlot = (u32*)&pPg->p[8];
17493 while( aSlot[iSlot] ){
17494 iSlot = (iSlot + 1) % nSlot;
17495 if( nCollide--==0 ) return 0;
17496 }
17497 fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid);
17498 }else{
17499 u64 *aSlot = (u64*)&pPg->p[8];
17500 while( aSlot[iSlot] ){
17501 iSlot = (iSlot + 1) % nSlot;
17502 if( nCollide--==0 ) return 0;
17503 }
17504 fts5PutU64((u8*)&aSlot[iSlot], iRowid);
17505 }
17506
17507 return 0;
17508}
17509
17510/*
17511** This function attempts to build a new hash containing all the keys
17512** currently in the tombstone hash table for segment pSeg. The new
17513** hash will be stored in the nOut buffers passed in array apOut[].
17514** All pages of the new hash use key-size szKey (4 or 8).
17515**
17516** Return 0 if the hash is successfully rebuilt into the nOut pages.
17517** Or non-zero if it is not (because one page became overfull). In this
17518** case the caller should retry with a larger nOut parameter.
17519**
17520** Parameter pData1 is page iPg1 of the hash table being rebuilt.
17521*/
17522static int fts5IndexTombstoneRehash(
17523 Fts5Index *p,
17524 Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */
17525 Fts5Data *pData1, /* One page of current hash - or NULL */
17526 int iPg1, /* Which page of the current hash is pData1 */
17527 int szKey, /* 4 or 8, the keysize */
17528 int nOut, /* Number of output pages */
17529 Fts5Data **apOut /* Array of output hash pages */
17530){
17531 int ii;
17532 int res = 0;
17533
17534 /* Initialize the headers of all the output pages */
17535 for(ii=0; ii<nOut; ii++){
17536 apOut[ii]->p[0] = szKey;
17537 fts5PutU32(&apOut[ii]->p[4], 0);
17538 }
17539
17540 /* Loop through the current pages of the hash table. */
17541 for(ii=0; res==0 && ii<pSeg->nPgTombstone; ii++){
17542 Fts5Data *pData = 0; /* Page ii of the current hash table */
17543 Fts5Data *pFree = 0; /* Free this at the end of the loop */
17544
17545 if( iPg1==ii ){
17546 pData = pData1;
17547 }else{
17548 pFree = pData = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii)( ((i64)(pSeg->iSegid+(1<<16)) << (31 +5 +1)) +
((i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((
i64)(ii)) ));
17549 }
17550
17551 if( pData ){
17552 int szKeyIn = TOMBSTONE_KEYSIZE(pData)(pData->p[0]==4 ? 4 : 8);
17553 int nSlotIn = (pData->nn - 8) / szKeyIn;
17554 int iIn;
17555 for(iIn=0; iIn<nSlotIn; iIn++){
17556 u64 iVal = 0;
17557
17558 /* Read the value from slot iIn of the input page into iVal. */
17559 if( szKeyIn==4 ){
17560 u32 *aSlot = (u32*)&pData->p[8];
17561 if( aSlot[iIn] ) iVal = fts5GetU32((u8*)&aSlot[iIn]);
17562 }else{
17563 u64 *aSlot = (u64*)&pData->p[8];
17564 if( aSlot[iIn] ) iVal = fts5GetU64((u8*)&aSlot[iIn]);
17565 }
17566
17567 /* If iVal is not 0 at this point, insert it into the new hash table */
17568 if( iVal ){
17569 Fts5Data *pPg = apOut[(iVal % nOut)];
17570 res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal);
17571 if( res ) break;
17572 }
17573 }
17574
17575 /* If this is page 0 of the old hash, copy the rowid-0-flag from the
17576 ** old hash to the new. */
17577 if( ii==0 ){
17578 apOut[0]->p[1] = pData->p[1];
17579 }
17580 }
17581 fts5DataRelease(pFree);
17582 }
17583
17584 return res;
17585}
17586
17587/*
17588** This is called to rebuild the hash table belonging to segment pSeg.
17589** If parameter pData1 is not NULL, then one page of the existing hash table
17590** has already been loaded - pData1, which is page iPg1. The key-size for
17591** the new hash table is szKey (4 or 8).
17592**
17593** If successful, the new hash table is not written to disk. Instead,
17594** output parameter (*pnOut) is set to the number of pages in the new
17595** hash table, and (*papOut) to point to an array of buffers containing
17596** the new page data.
17597**
17598** If an error occurs, an error code is left in the Fts5Index object and
17599** both output parameters set to 0 before returning.
17600*/
17601static void fts5IndexTombstoneRebuild(
17602 Fts5Index *p,
17603 Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */
17604 Fts5Data *pData1, /* One page of current hash - or NULL */
17605 int iPg1, /* Which page of the current hash is pData1 */
17606 int szKey, /* 4 or 8, the keysize */
17607 int *pnOut, /* OUT: Number of output pages */
17608 Fts5Data ***papOut /* OUT: Output hash pages */
17609){
17610 const int MINSLOT = 32;
17611 int nSlotPerPage = MAX(MINSLOT, (p->pConfig->pgsz - 8) / szKey)(((MINSLOT) > ((p->pConfig->pgsz - 8) / szKey)) ? (MINSLOT
) : ((p->pConfig->pgsz - 8) / szKey));
17612 i64 nSlot = 0; /* Number of slots in each output page */
17613 i64 nOut = 0;
17614
17615 /* Figure out how many output pages (nOut) and how many slots per
17616 ** page (nSlot). There are three possibilities:
17617 **
17618 ** 1. The hash table does not yet exist. In this case the new hash
17619 ** table will consist of a single page with MINSLOT slots.
17620 **
17621 ** 2. The hash table exists but is currently a single page. In this
17622 ** case an attempt is made to grow the page to accommodate the new
17623 ** entry. The page is allowed to grow up to nSlotPerPage (see above)
17624 ** slots.
17625 **
17626 ** 3. The hash table already consists of more than one page, or of
17627 ** a single page already so large that it cannot be grown. In this
17628 ** case the new hash consists of (nPg*2+1) pages of nSlotPerPage
17629 ** slots each, where nPg is the current number of pages in the
17630 ** hash table.
17631 */
17632 if( pSeg->nPgTombstone==0 ){
17633 /* Case 1. */
17634 nOut = 1;
17635 nSlot = MINSLOT;
17636 }else if( pSeg->nPgTombstone==1 ){
17637 /* Case 2. */
17638 u32 nElem = fts5GetU32(&pData1->p[4]);
17639 assert( pData1 && iPg1==0 )((void) (0));
17640 if( nElem>((u32)nSlotPerPage/4) ){
17641 nOut = 0;
17642 }else{
17643 nOut = 1;
17644 nSlot = MAX((i64)nElem*4, MINSLOT)((((i64)nElem*4) > (MINSLOT)) ? ((i64)nElem*4) : (MINSLOT)
);
17645 }
17646 }
17647 if( nOut==0 ){
17648 /* Case 3. */
17649 nOut = ((i64)pSeg->nPgTombstone * 2 + 1);
17650 nSlot = nSlotPerPage;
17651 }
17652
17653 /* Allocate the required array and output pages */
17654 while( 1 ){
17655 int res = 0;
17656 i64 ii = 0;
17657 i64 szPage = 0;
17658 Fts5Data **apOut = 0;
17659
17660 /* Allocate space for the new hash table */
17661 assert( nSlot>=MINSLOT )((void) (0));
17662 apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut);
17663 szPage = 8 + nSlot*szKey;
17664 for(ii=0; ii<nOut; ii++){
17665 Fts5Data *pNew = (Fts5Data*)sqlite3Fts5MallocZero(&p->rc,
17666 sizeof(Fts5Data)+szPage
17667 );
17668 if( pNew ){
17669 pNew->nn = szPage;
17670 pNew->p = (u8*)&pNew[1];
17671 apOut[ii] = pNew;
17672 }
17673 }
17674
17675 /* Rebuild the hash table. */
17676 if( p->rc==SQLITE_OK0 ){
17677 res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut);
17678 }
17679 if( res==0 ){
17680 if( p->rc ){
17681 fts5IndexFreeArray(apOut, nOut);
17682 apOut = 0;
17683 nOut = 0;
17684 }
17685 *pnOut = nOut;
17686 *papOut = apOut;
17687 break;
17688 }
17689
17690 /* If control flows to here, it was not possible to rebuild the hash
17691 ** table. Free all buffers and then try again with more pages. */
17692 assert( p->rc==SQLITE_OK )((void) (0));
17693 fts5IndexFreeArray(apOut, nOut);
17694 nSlot = nSlotPerPage;
17695 nOut = nOut*2 + 1;
17696 }
17697}
17698
17699
17700/*
17701** Add a tombstone for rowid iRowid to segment pSeg.
17702*/
17703static void fts5IndexTombstoneAdd(
17704 Fts5Index *p,
17705 Fts5StructureSegment *pSeg,
17706 u64 iRowid
17707){
17708 Fts5Data *pPg = 0;
17709 int iPg = -1;
17710 int szKey = 0;
17711 int nHash = 0;
17712 Fts5Data **apHash = 0;
17713
17714 p->nContentlessDelete++;
17715
17716 if( pSeg->nPgTombstone>0 ){
17717 iPg = iRowid % pSeg->nPgTombstone;
17718 pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg)( ((i64)(pSeg->iSegid+(1<<16)) << (31 +5 +1)) +
((i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((
i64)(iPg)) ));
17719 if( pPg==0 ){
17720 assert( p->rc!=SQLITE_OK )((void) (0));
17721 return;
17722 }
17723
17724 if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){
17725 fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg)( ((i64)(pSeg->iSegid+(1<<16)) << (31 +5 +1)) +
((i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((
i64)(iPg)) ), pPg->p, pPg->nn);
17726 fts5DataRelease(pPg);
17727 return;
17728 }
17729 }
17730
17731 /* Have to rebuild the hash table. First figure out the key-size (4 or 8). */
17732 szKey = pPg ? TOMBSTONE_KEYSIZE(pPg)(pPg->p[0]==4 ? 4 : 8) : 4;
17733 if( iRowid>0xFFFFFFFF ) szKey = 8;
17734
17735 /* Rebuild the hash table */
17736 fts5IndexTombstoneRebuild(p, pSeg, pPg, iPg, szKey, &nHash, &apHash);
17737 assert( p->rc==SQLITE_OK || (nHash==0 && apHash==0) )((void) (0));
17738
17739 /* If all has succeeded, write the new rowid into one of the new hash
17740 ** table pages, then write them all out to disk. */
17741 if( nHash ){
17742 int ii = 0;
17743 fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid);
17744 for(ii=0; ii<nHash; ii++){
17745 i64 iTombstoneRowid = FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii)( ((i64)(pSeg->iSegid+(1<<16)) << (31 +5 +1)) +
((i64)(0) << (31 + 5)) + ((i64)(0) << (31)) + ((
i64)(ii)) );
17746 fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn);
17747 }
17748 pSeg->nPgTombstone = nHash;
17749 fts5StructureWrite(p, p->pStruct);
17750 }
17751
17752 fts5DataRelease(pPg);
17753 fts5IndexFreeArray(apHash, nHash);
17754}
17755
17756/*
17757** Add iRowid to the tombstone list of the segment or segments that contain
17758** rows from origin iOrigin. Return SQLITE_OK if successful, or an SQLite
17759** error code otherwise.
17760*/
17761static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){
17762 Fts5Structure *pStruct;
17763 pStruct = fts5StructureRead(p);
17764 if( pStruct ){
17765 int bFound = 0; /* True after pSeg->nEntryTombstone incr. */
17766 int iLvl;
17767 for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){
17768 int iSeg;
17769 for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){
17770 Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
17771 if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){
17772 if( bFound==0 ){
17773 pSeg->nEntryTombstone++;
17774 bFound = 1;
17775 }
17776 fts5IndexTombstoneAdd(p, pSeg, iRowid);
17777 }
17778 }
17779 }
17780 fts5StructureRelease(pStruct);
17781 }
17782 return fts5IndexReturn(p);
17783}
17784
17785/*************************************************************************
17786**************************************************************************
17787** Below this point is the implementation of the integrity-check
17788** functionality.
17789*/
17790
17791/*
17792** Return a simple checksum value based on the arguments.
17793*/
17794static u64 sqlite3Fts5IndexEntryCksum(
17795 i64 iRowid,
17796 int iCol,
17797 int iPos,
17798 int iIdx,
17799 const char *pTerm,
17800 int nTerm
17801){
17802 int i;
17803 u64 ret = iRowid;
17804 ret += (ret<<3) + iCol;
17805 ret += (ret<<3) + iPos;
17806 if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX'0' + iIdx);
17807 for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i];
17808 return ret;
17809}
17810
17811#ifdef SQLITE_DEBUG
17812/*
17813** This function is purely an internal test. It does not contribute to
17814** FTS functionality, or even the integrity-check, in any way.
17815**
17816** Instead, it tests that the same set of pgno/rowid combinations are
17817** visited regardless of whether the doclist-index identified by parameters
17818** iSegid/iLeaf is iterated in forwards or reverse order.
17819*/
17820static void fts5TestDlidxReverse(
17821 Fts5Index *p,
17822 int iSegid, /* Segment id to load from */
17823 int iLeaf /* Load doclist-index for this leaf */
17824){
17825 Fts5DlidxIter *pDlidx = 0;
17826 u64 cksum1 = 13;
17827 u64 cksum2 = 13;
17828
17829 for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iLeaf);
17830 fts5DlidxIterEof(p, pDlidx)==0;
17831 fts5DlidxIterNext(p, pDlidx)
17832 ){
17833 i64 iRowid = fts5DlidxIterRowid(pDlidx);
17834 int pgno = fts5DlidxIterPgno(pDlidx);
17835 assert( pgno>iLeaf )((void) (0));
17836 cksum1 += iRowid + ((i64)pgno<<32);
17837 }
17838 fts5DlidxIterFree(pDlidx);
17839 pDlidx = 0;
17840
17841 for(pDlidx=fts5DlidxIterInit(p, 1, iSegid, iLeaf);
17842 fts5DlidxIterEof(p, pDlidx)==0;
17843 fts5DlidxIterPrev(p, pDlidx)
17844 ){
17845 i64 iRowid = fts5DlidxIterRowid(pDlidx);
17846 int pgno = fts5DlidxIterPgno(pDlidx);
17847 assert( fts5DlidxIterPgno(pDlidx)>iLeaf )((void) (0));
17848 cksum2 += iRowid + ((i64)pgno<<32);
17849 }
17850 fts5DlidxIterFree(pDlidx);
17851 pDlidx = 0;
17852
17853 if( p->rc==SQLITE_OK0 && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT(11 | (1<<8));
17854}
17855
17856static int fts5QueryCksum(
17857 Fts5Index *p, /* Fts5 index object */
17858 int iIdx,
17859 const char *z, /* Index key to query for */
17860 int n, /* Size of index key in bytes */
17861 int flags, /* Flags for Fts5IndexQuery */
17862 u64 *pCksum /* IN/OUT: Checksum value */
17863){
17864 int eDetail = p->pConfig->eDetail;
17865 u64 cksum = *pCksum;
17866 Fts5IndexIter *pIter = 0;
17867 int rc = sqlite3Fts5IndexQuery(
17868 p, z, n, (flags | FTS5INDEX_QUERY_NOTOKENDATA0x0080), 0, &pIter
17869 );
17870
17871 while( rc==SQLITE_OK0 && ALWAYS(pIter!=0)(pIter!=0) && 0==sqlite3Fts5IterEof(pIter)((pIter)->bEof) ){
17872 i64 rowid = pIter->iRowid;
17873
17874 if( eDetail==FTS5_DETAIL_NONE1 ){
17875 cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n);
17876 }else{
17877 Fts5PoslistReader sReader;
17878 for(sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &sReader);
17879 sReader.bEof==0;
17880 sqlite3Fts5PoslistReaderNext(&sReader)
17881 ){
17882 int iCol = FTS5_POS2COLUMN(sReader.iPos)(int)((sReader.iPos >> 32) & 0x7FFFFFFF);
17883 int iOff = FTS5_POS2OFFSET(sReader.iPos)(int)(sReader.iPos & 0x7FFFFFFF);
17884 cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
17885 }
17886 }
17887 if( rc==SQLITE_OK0 ){
17888 rc = sqlite3Fts5IterNext(pIter);
17889 }
17890 }
17891 fts5IterClose(pIter);
17892
17893 *pCksum = cksum;
17894 return rc;
17895}
17896
17897/*
17898** Check if buffer z[], size n bytes, contains as series of valid utf-8
17899** encoded codepoints. If so, return 0. Otherwise, if the buffer does not
17900** contain valid utf-8, return non-zero.
17901*/
17902static int fts5TestUtf8(const char *z, int n){
17903 int i = 0;
17904 assert_nc( n>0 )((void) (0));
17905 while( i<n ){
17906 if( (z[i] & 0x80)==0x00 ){
17907 i++;
17908 }else
17909 if( (z[i] & 0xE0)==0xC0 ){
17910 if( i+1>=n || (z[i+1] & 0xC0)!=0x80 ) return 1;
17911 i += 2;
17912 }else
17913 if( (z[i] & 0xF0)==0xE0 ){
17914 if( i+2>=n || (z[i+1] & 0xC0)!=0x80 || (z[i+2] & 0xC0)!=0x80 ) return 1;
17915 i += 3;
17916 }else
17917 if( (z[i] & 0xF8)==0xF0 ){
17918 if( i+3>=n || (z[i+1] & 0xC0)!=0x80 || (z[i+2] & 0xC0)!=0x80 ) return 1;
17919 if( (z[i+2] & 0xC0)!=0x80 ) return 1;
17920 i += 3;
17921 }else{
17922 return 1;
17923 }
17924 }
17925
17926 return 0;
17927}
17928
17929/*
17930** This function is also purely an internal test. It does not contribute to
17931** FTS functionality, or even the integrity-check, in any way.
17932**
17933** This function sets output variable (*pbFail) to true if the test fails. Or
17934** leaves it unchanged if the test succeeds.
17935*/
17936static void fts5TestTerm(
17937 Fts5Index *p,
17938 Fts5Buffer *pPrev, /* Previous term */
17939 const char *z, int n, /* Possibly new term to test */
17940 u64 expected,
17941 u64 *pCksum,
17942 int *pbFail
17943){
17944 int rc = p->rc;
17945 if( pPrev->n==0 ){
17946 fts5BufferSet(&rc, pPrev, n, (const u8*)z)sqlite3Fts5BufferSet(&rc,pPrev,n,(const u8*)z);
17947 }else
17948 if( *pbFail==0
17949 && rc==SQLITE_OK0
17950 && (pPrev->n!=n || memcmp(pPrev->p, z, n))
17951 && (p->pHash==0 || p->pHash->nEntry==0)
17952 ){
17953 u64 cksum3 = *pCksum;
17954 const char *zTerm = (const char*)&pPrev->p[1]; /* term sans prefix-byte */
17955 int nTerm = pPrev->n-1; /* Size of zTerm in bytes */
17956 int iIdx = (pPrev->p[0] - FTS5_MAIN_PREFIX'0');
17957 int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX0x0001);
17958 u64 ck1 = 0;
17959 u64 ck2 = 0;
17960
17961 /* Check that the results returned for ASC and DESC queries are
17962 ** the same. If not, call this corruption. */
17963 rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1);
17964 if( rc==SQLITE_OK0 ){
17965 int f = flags|FTS5INDEX_QUERY_DESC0x0002;
17966 rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
17967 }
17968 if( rc==SQLITE_OK0 && ck1!=ck2 ) rc = FTS5_CORRUPT(11 | (1<<8));
17969
17970 /* If this is a prefix query, check that the results returned if the
17971 ** the index is disabled are the same. In both ASC and DESC order.
17972 **
17973 ** This check may only be performed if the hash table is empty. This
17974 ** is because the hash table only supports a single scan query at
17975 ** a time, and the multi-iter loop from which this function is called
17976 ** is already performing such a scan.
17977 **
17978 ** Also only do this if buffer zTerm contains nTerm bytes of valid
17979 ** utf-8. Otherwise, the last part of the buffer contents might contain
17980 ** a non-utf-8 sequence that happens to be a prefix of a valid utf-8
17981 ** character stored in the main fts index, which will cause the
17982 ** test to fail. */
17983 if( p->nPendingData==0 && 0==fts5TestUtf8(zTerm, nTerm) ){
17984 if( iIdx>0 && rc==SQLITE_OK0 ){
17985 int f = flags|FTS5INDEX_QUERY_TEST_NOIDX0x0004;
17986 ck2 = 0;
17987 rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
17988 if( rc==SQLITE_OK0 && ck1!=ck2 ) rc = FTS5_CORRUPT(11 | (1<<8));
17989 }
17990 if( iIdx>0 && rc==SQLITE_OK0 ){
17991 int f = flags|FTS5INDEX_QUERY_TEST_NOIDX0x0004|FTS5INDEX_QUERY_DESC0x0002;
17992 ck2 = 0;
17993 rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
17994 if( rc==SQLITE_OK0 && ck1!=ck2 ) rc = FTS5_CORRUPT(11 | (1<<8));
17995 }
17996 }
17997
17998 cksum3 ^= ck1;
17999 fts5BufferSet(&rc, pPrev, n, (const u8*)z)sqlite3Fts5BufferSet(&rc,pPrev,n,(const u8*)z);
18000
18001 if( rc==SQLITE_OK0 && cksum3!=expected ){
18002 *pbFail = 1;
18003 }
18004 *pCksum = cksum3;
18005 }
18006 p->rc = rc;
18007}
18008
18009#else
18010# define fts5TestDlidxReverse(x,y,z)
18011# define fts5TestTerm(t,u,v,w,x,y,z)
18012#endif
18013
18014/*
18015** Check that:
18016**
18017** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and
18018** contain zero terms.
18019** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and
18020** contain zero rowids.
18021*/
18022static void fts5IndexIntegrityCheckEmpty(
18023 Fts5Index *p,
18024 Fts5StructureSegment *pSeg, /* Segment to check internal consistency */
18025 int iFirst,
18026 int iNoRowid,
18027 int iLast
18028){
18029 int i;
18030
18031 /* Now check that the iter.nEmpty leaves following the current leaf
18032 ** (a) exist and (b) contain no terms. */
18033 for(i=iFirst; p->rc==SQLITE_OK0 && i<=iLast; i++){
18034 Fts5Data *pLeaf = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i)( ((i64)(pSeg->iSegid) << (31 +5 +1)) + ((i64)(0) <<
(31 + 5)) + ((i64)(0) << (31)) + ((i64)(i)) ));
18035 if( pLeaf ){
18036 if( !fts5LeafIsTermless(pLeaf)((pLeaf)->szLeaf >= (pLeaf)->nn)
18037 || (i>=iNoRowid && 0!=fts5LeafFirstRowidOff(pLeaf)(fts5GetU16((pLeaf)->p)))
18038 ){
18039 FTS5_CORRUPT_ROWID(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, i))fts5IndexCorruptRowid(p, ( ((i64)(pSeg->iSegid) << (
31 +5 +1)) + ((i64)(0) << (31 + 5)) + ((i64)(0) <<
(31)) + ((i64)(i)) ));
18040 }
18041 }
18042 fts5DataRelease(pLeaf);
18043 }
18044}
18045
18046static void fts5IntegrityCheckPgidx(Fts5Index *p, i64 iRowid, Fts5Data *pLeaf){
18047 i64 iTermOff = 0;
18048 int ii;
18049
18050 Fts5Buffer buf1 = {0,0,0};
18051 Fts5Buffer buf2 = {0,0,0};
18052
18053 ii = pLeaf->szLeaf;
18054 while( ii<pLeaf->nn && p->rc==SQLITE_OK0 ){
18055 int res;
18056 i64 iOff;
18057 int nIncr;
18058
18059 ii += fts5GetVarint32(&pLeaf->p[ii], nIncr)sqlite3Fts5GetVarint32(&pLeaf->p[ii],(u32*)&(nIncr
));
18060 iTermOff += nIncr;
18061 iOff = iTermOff;
18062
18063 if( iOff>=pLeaf->szLeaf ){
18064 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
18065 }else if( iTermOff==nIncr ){
18066 int nByte;
18067 iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte)sqlite3Fts5GetVarint32(&pLeaf->p[iOff],(u32*)&(nByte
));
18068 if( (iOff+nByte)>pLeaf->szLeaf ){
18069 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
18070 }else{
18071 fts5BufferSet(&p->rc, &buf1, nByte, &pLeaf->p[iOff])sqlite3Fts5BufferSet(&p->rc,&buf1,nByte,&pLeaf
->p[iOff]);
18072 }
18073 }else{
18074 int nKeep, nByte;
18075 iOff += fts5GetVarint32(&pLeaf->p[iOff], nKeep)sqlite3Fts5GetVarint32(&pLeaf->p[iOff],(u32*)&(nKeep
));
18076 iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte)sqlite3Fts5GetVarint32(&pLeaf->p[iOff],(u32*)&(nByte
));
18077 if( nKeep>buf1.n || (iOff+nByte)>pLeaf->szLeaf ){
18078 FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
18079 }else{
18080 buf1.n = nKeep;
18081 fts5BufferAppendBlob(&p->rc, &buf1, nByte, &pLeaf->p[iOff])sqlite3Fts5BufferAppendBlob(&p->rc,&buf1,nByte,&
pLeaf->p[iOff]);
18082 }
18083
18084 if( p->rc==SQLITE_OK0 ){
18085 res = fts5BufferCompare(&buf1, &buf2);
18086 if( res<=0 ) FTS5_CORRUPT_ROWID(p, iRowid)fts5IndexCorruptRowid(p, iRowid);
18087 }
18088 }
18089 fts5BufferSet(&p->rc, &buf2, buf1.n, buf1.p)sqlite3Fts5BufferSet(&p->rc,&buf2,buf1.n,buf1.p);
18090 }
18091
18092 fts5BufferFree(&buf1)sqlite3Fts5BufferFree(&buf1);
18093 fts5BufferFree(&buf2)sqlite3Fts5BufferFree(&buf2);
18094}
18095
18096static void fts5IndexIntegrityCheckSegment(
18097 Fts5Index *p, /* FTS5 backend object */
18098 Fts5StructureSegment *pSeg /* Segment to check internal consistency */
18099){
18100 Fts5Config *pConfig = p->pConfig;
18101 int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE5);
18102 sqlite3_stmt *pStmt = 0;
18103 int rc2;
18104 int iIdxPrevLeaf = pSeg->pgnoFirst-1;
18105 int iDlidxPrevLeaf = pSeg->pgnoLast;
18106
18107 if( pSeg->pgnoFirst==0 ) return;
18108
18109 fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintfsqlite3_api->mprintf(
18110 "SELECT segid, term, (pgno>>1), (pgno&1) FROM %Q.'%q_idx' WHERE segid=%d "
18111 "ORDER BY 1, 2",
18112 pConfig->zDb, pConfig->zName, pSeg->iSegid
18113 ));
18114
18115 /* Iterate through the b-tree hierarchy. */
18116 while( p->rc==SQLITE_OK0 && SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pStmt) ){
18117 i64 iRow; /* Rowid for this leaf */
18118 Fts5Data *pLeaf; /* Data for this leaf */
18119
18120 const char *zIdxTerm = (const char*)sqlite3_column_blobsqlite3_api->column_blob(pStmt, 1);
18121 int nIdxTerm = sqlite3_column_bytessqlite3_api->column_bytes(pStmt, 1);
18122 int iIdxLeaf = sqlite3_column_intsqlite3_api->column_int(pStmt, 2);
18123 int bIdxDlidx = sqlite3_column_intsqlite3_api->column_int(pStmt, 3);
18124
18125 /* If the leaf in question has already been trimmed from the segment,
18126 ** ignore this b-tree entry. Otherwise, load it into memory. */
18127 if( iIdxLeaf<pSeg->pgnoFirst ) continue;
18128 iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf)( ((i64)(pSeg->iSegid) << (31 +5 +1)) + ((i64)(0) <<
(31 + 5)) + ((i64)(0) << (31)) + ((i64)(iIdxLeaf)) );
18129 pLeaf = fts5LeafRead(p, iRow);
18130 if( pLeaf==0 ) break;
18131
18132 /* Check that the leaf contains at least one term, and that it is equal
18133 ** to or larger than the split-key in zIdxTerm. Also check that if there
18134 ** is also a rowid pointer within the leaf page header, it points to a
18135 ** location before the term. */
18136 if( pLeaf->nn<=pLeaf->szLeaf ){
18137
18138 if( nIdxTerm==0
18139 && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE5
18140 && pLeaf->nn==pLeaf->szLeaf
18141 && pLeaf->nn==4
18142 ){
18143 /* special case - the very first page in a segment keeps its %_idx
18144 ** entry even if all the terms are removed from it by secure-delete
18145 ** operations. */
18146 }else{
18147 FTS5_CORRUPT_ROWID(p, iRow)fts5IndexCorruptRowid(p, iRow);
18148 }
18149
18150 }else{
18151 int iOff; /* Offset of first term on leaf */
18152 int iRowidOff; /* Offset of first rowid on leaf */
18153 int nTerm; /* Size of term on leaf in bytes */
18154 int res; /* Comparison of term and split-key */
18155
18156 iOff = fts5LeafFirstTermOff(pLeaf);
18157 iRowidOff = fts5LeafFirstRowidOff(pLeaf)(fts5GetU16((pLeaf)->p));
18158 if( iRowidOff>=iOff || iOff>=pLeaf->szLeaf ){
18159 FTS5_CORRUPT_ROWID(p, iRow)fts5IndexCorruptRowid(p, iRow);
18160 }else{
18161 iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm)sqlite3Fts5GetVarint32(&pLeaf->p[iOff],(u32*)&(nTerm
));
18162 if( (i64)iOff+(i64)nTerm>(i64)pLeaf->szLeaf ){
18163 FTS5_CORRUPT_ROWID(p, iRow)fts5IndexCorruptRowid(p, iRow);
18164 }else{
18165 res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm))(((((nTerm) < (nIdxTerm)) ? (nTerm) : (nIdxTerm)))<=0 ?
0 : memcmp((&pLeaf->p[iOff]), (zIdxTerm), ((((nTerm) <
(nIdxTerm)) ? (nTerm) : (nIdxTerm)))));
18166 if( res==0 ) res = nTerm - nIdxTerm;
18167 if( res<0 ) FTS5_CORRUPT_ROWID(p, iRow)fts5IndexCorruptRowid(p, iRow);
18168 }
18169 }
18170
18171 fts5IntegrityCheckPgidx(p, iRow, pLeaf);
18172 }
18173 fts5DataRelease(pLeaf);
18174 if( p->rc ) break;
18175
18176 /* Now check that the iter.nEmpty leaves following the current leaf
18177 ** (a) exist and (b) contain no terms. */
18178 fts5IndexIntegrityCheckEmpty(
18179 p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1
18180 );
18181 if( p->rc ) break;
18182
18183 /* If there is a doclist-index, check that it looks right. */
18184 if( bIdxDlidx ){
18185 Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */
18186 int iPrevLeaf = iIdxLeaf;
18187 int iSegid = pSeg->iSegid;
18188 int iPg = 0;
18189 i64 iKey;
18190
18191 for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf);
18192 fts5DlidxIterEof(p, pDlidx)==0;
18193 fts5DlidxIterNext(p, pDlidx)
18194 ){
18195
18196 /* Check any rowid-less pages that occur before the current leaf. */
18197 for(iPg=iPrevLeaf+1; iPg<fts5DlidxIterPgno(pDlidx); iPg++){
18198 iKey = FTS5_SEGMENT_ROWID(iSegid, iPg)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(iPg)) );
18199 pLeaf = fts5LeafRead(p, iKey);
18200 if( pLeaf ){
18201 if( fts5LeafFirstRowidOff(pLeaf)(fts5GetU16((pLeaf)->p))!=0 ) FTS5_CORRUPT_ROWID(p, iKey)fts5IndexCorruptRowid(p, iKey);
18202 fts5DataRelease(pLeaf);
18203 }
18204 }
18205 iPrevLeaf = fts5DlidxIterPgno(pDlidx);
18206
18207 /* Check that the leaf page indicated by the iterator really does
18208 ** contain the rowid suggested by the same. */
18209 iKey = FTS5_SEGMENT_ROWID(iSegid, iPrevLeaf)( ((i64)(iSegid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(iPrevLeaf)) );
18210 pLeaf = fts5LeafRead(p, iKey);
18211 if( pLeaf ){
18212 i64 iRowid;
18213 int iRowidOff = fts5LeafFirstRowidOff(pLeaf)(fts5GetU16((pLeaf)->p));
18214 ASSERT_SZLEAF_OK(pLeaf)((void) (0));
18215 if( iRowidOff>=pLeaf->szLeaf ){
18216 FTS5_CORRUPT_ROWID(p, iKey)fts5IndexCorruptRowid(p, iKey);
18217 }else if( bSecureDelete==0 || iRowidOff>0 ){
18218 i64 iDlRowid = fts5DlidxIterRowid(pDlidx);
18219 fts5GetVarintsqlite3Fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
18220 if( iRowid<iDlRowid || (bSecureDelete==0 && iRowid!=iDlRowid) ){
18221 FTS5_CORRUPT_ROWID(p, iKey)fts5IndexCorruptRowid(p, iKey);
18222 }
18223 }
18224 fts5DataRelease(pLeaf);
18225 }
18226 }
18227
18228 iDlidxPrevLeaf = iPg;
18229 fts5DlidxIterFree(pDlidx);
18230 fts5TestDlidxReverse(p, iSegid, iIdxLeaf);
18231 }else{
18232 iDlidxPrevLeaf = pSeg->pgnoLast;
18233 /* TODO: Check there is no doclist index */
18234 }
18235
18236 iIdxPrevLeaf = iIdxLeaf;
18237 }
18238
18239 rc2 = sqlite3_finalizesqlite3_api->finalize(pStmt);
18240 if( p->rc==SQLITE_OK0 ) p->rc = rc2;
18241
18242 /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */
18243#if 0
18244 if( p->rc==SQLITE_OK0 && iter.iLeaf!=pSeg->pgnoLast ){
18245 p->rc = FTS5_CORRUPT(11 | (1<<8));
18246 }
18247#endif
18248}
18249
18250
18251/*
18252** Run internal checks to ensure that the FTS index (a) is internally
18253** consistent and (b) contains entries for which the XOR of the checksums
18254** as calculated by sqlite3Fts5IndexEntryCksum() is cksum.
18255**
18256** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
18257** checksum does not match. Return SQLITE_OK if all checks pass without
18258** error, or some other SQLite error code if another error (e.g. OOM)
18259** occurs.
18260*/
18261static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){
18262 int eDetail = p->pConfig->eDetail;
18263 u64 cksum2 = 0; /* Checksum based on contents of indexes */
18264 Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */
18265 Fts5Iter *pIter; /* Used to iterate through entire index */
18266 Fts5Structure *pStruct; /* Index structure */
18267 int iLvl, iSeg;
18268
18269#ifdef SQLITE_DEBUG
18270 /* Used by extra internal tests only run if NDEBUG is not defined */
18271 u64 cksum3 = 0; /* Checksum based on contents of indexes */
18272 Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */
18273 int bTestFail = 0;
18274#endif
18275 const int flags = FTS5INDEX_QUERY_NOOUTPUT0x0020;
18276
18277 /* Load the FTS index structure */
18278 pStruct = fts5StructureRead(p);
18279 if( pStruct==0 ){
18280 assert( p->rc!=SQLITE_OK )((void) (0));
18281 return fts5IndexReturn(p);
18282 }
18283
18284 /* Check that the internal nodes of each segment match the leaves */
18285 for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
18286 for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
18287 Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
18288 fts5IndexIntegrityCheckSegment(p, pSeg);
18289 }
18290 }
18291
18292 /* The cksum argument passed to this function is a checksum calculated
18293 ** based on all expected entries in the FTS index (including prefix index
18294 ** entries). This block checks that a checksum calculated based on the
18295 ** actual contents of FTS index is identical.
18296 **
18297 ** Two versions of the same checksum are calculated. The first (stack
18298 ** variable cksum2) based on entries extracted from the full-text index
18299 ** while doing a linear scan of each individual index in turn.
18300 **
18301 ** As each term visited by the linear scans, a separate query for the
18302 ** same term is performed. cksum3 is calculated based on the entries
18303 ** extracted by these queries.
18304 */
18305 for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, -1, 0, &pIter);
18306 fts5MultiIterEof(p, pIter)==0;
18307 fts5MultiIterNext(p, pIter, 0, 0)
18308 ){
18309 int n; /* Size of term in bytes */
18310 i64 iPos = 0; /* Position read from poslist */
18311 int iOff = 0; /* Offset within poslist */
18312 i64 iRowid = fts5MultiIterRowid(pIter);
18313 char *z = (char*)fts5MultiIterTerm(pIter, &n);
18314
18315 /* If this is a new term, query for it. Update cksum3 with the results. */
18316 fts5TestTerm(p, &term, z, n, cksum2, &cksum3, &bTestFail);
18317 if( p->rc ) break;
18318
18319 if( eDetail==FTS5_DETAIL_NONE1 ){
18320 if( 0==fts5MultiIterIsEmpty(p, pIter) ){
18321 cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n);
18322 }
18323 }else{
18324 poslist.n = 0;
18325 fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist);
18326 fts5BufferAppendBlob(&p->rc, &poslist, 4, (const u8*)"\0\0\0\0")sqlite3Fts5BufferAppendBlob(&p->rc,&poslist,4,(const
u8*)"\0\0\0\0");
18327 while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
18328 int iCol = FTS5_POS2COLUMN(iPos)(int)((iPos >> 32) & 0x7FFFFFFF);
18329 int iTokOff = FTS5_POS2OFFSET(iPos)(int)(iPos & 0x7FFFFFFF);
18330 cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
18331 }
18332 }
18333 }
18334 fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3, &bTestFail);
18335
18336 fts5MultiIterFree(pIter);
18337 if( p->rc==SQLITE_OK0 && bUseCksum && cksum!=cksum2 ){
18338 p->rc = FTS5_CORRUPT(11 | (1<<8));
18339 sqlite3Fts5ConfigErrmsg(p->pConfig,
18340 "fts5: checksum mismatch for table \"%s\"", p->pConfig->zName
18341 );
18342 }
18343#ifdef SQLITE_DEBUG
18344 /* In SQLITE_DEBUG builds, expensive extra checks were run as part of
18345 ** the integrity-check above. If no other errors were detected, but one
18346 ** of these tests failed, set the result to SQLITE_CORRUPT_VTAB here. */
18347 if( p->rc==SQLITE_OK0 && bTestFail ){
18348 p->rc = FTS5_CORRUPT(11 | (1<<8));
18349 }
18350 fts5BufferFree(&term)sqlite3Fts5BufferFree(&term);
18351#endif
18352
18353 fts5StructureRelease(pStruct);
18354 fts5BufferFree(&poslist)sqlite3Fts5BufferFree(&poslist);
18355 return fts5IndexReturn(p);
18356}
18357
18358/*************************************************************************
18359**************************************************************************
18360** Below this point is the implementation of the fts5_decode() scalar
18361** function only.
18362*/
18363
18364#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18365/*
18366** Decode a segment-data rowid from the %_data table. This function is
18367** the opposite of macro FTS5_SEGMENT_ROWID().
18368*/
18369static void fts5DecodeRowid(
18370 i64 iRowid, /* Rowid from %_data table */
18371 int *pbTombstone, /* OUT: Tombstone hash flag */
18372 int *piSegid, /* OUT: Segment id */
18373 int *pbDlidx, /* OUT: Dlidx flag */
18374 int *piHeight, /* OUT: Height */
18375 int *piPgno /* OUT: Page number */
18376){
18377 *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B31) - 1));
18378 iRowid >>= FTS5_DATA_PAGE_B31;
18379
18380 *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B5) - 1));
18381 iRowid >>= FTS5_DATA_HEIGHT_B5;
18382
18383 *pbDlidx = (int)(iRowid & 0x0001);
18384 iRowid >>= FTS5_DATA_DLI_B1;
18385
18386 *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B16) - 1));
18387 iRowid >>= FTS5_DATA_ID_B16;
18388
18389 *pbTombstone = (int)(iRowid & 0x0001);
18390}
18391#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18392
18393#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18394static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
18395 int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid components */
18396 fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno);
18397
18398 if( iSegid==0 ){
18399 if( iKey==FTS5_AVERAGES_ROWID1 ){
18400 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{averages} ");
18401 }else{
18402 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{structure}");
18403 }
18404 }
18405 else{
18406 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%s%ssegid=%d h=%d pgno=%d}",
18407 bDlidx ? "dlidx " : "",
18408 bTomb ? "tombstone " : "",
18409 iSegid, iHeight, iPgno
18410 );
18411 }
18412}
18413#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18414
18415#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18416static void fts5DebugStructure(
18417 int *pRc, /* IN/OUT: error code */
18418 Fts5Buffer *pBuf,
18419 Fts5Structure *p
18420){
18421 int iLvl, iSeg; /* Iterate through levels, segments */
18422
18423 for(iLvl=0; iLvl<p->nLevel; iLvl++){
18424 Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
18425 sqlite3Fts5BufferAppendPrintf(pRc, pBuf,
18426 " {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg
18427 );
18428 for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
18429 Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
18430 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d",
18431 pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast
18432 );
18433 if( pSeg->iOrigin1>0 ){
18434 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " origin=%lld..%lld",
18435 pSeg->iOrigin1, pSeg->iOrigin2
18436 );
18437 }
18438 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
18439 }
18440 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}");
18441 }
18442}
18443#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18444
18445#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18446/*
18447** This is part of the fts5_decode() debugging aid.
18448**
18449** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This
18450** function appends a human-readable representation of the same object
18451** to the buffer passed as the second argument.
18452*/
18453static void fts5DecodeStructure(
18454 int *pRc, /* IN/OUT: error code */
18455 Fts5Buffer *pBuf,
18456 const u8 *pBlob, int nBlob
18457){
18458 int rc; /* Return code */
18459 Fts5Structure *p = 0; /* Decoded structure object */
18460
18461 rc = fts5StructureDecode(pBlob, nBlob, 0, &p);
18462 if( rc!=SQLITE_OK0 ){
18463 *pRc = rc;
18464 return;
18465 }
18466
18467 fts5DebugStructure(pRc, pBuf, p);
18468 fts5StructureRelease(p);
18469}
18470#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18471
18472#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18473/*
18474** This is part of the fts5_decode() debugging aid.
18475**
18476** Arguments pBlob/nBlob contain an "averages" record. This function
18477** appends a human-readable representation of record to the buffer passed
18478** as the second argument.
18479*/
18480static void fts5DecodeAverages(
18481 int *pRc, /* IN/OUT: error code */
18482 Fts5Buffer *pBuf,
18483 const u8 *pBlob, int nBlob
18484){
18485 int i = 0;
18486 const char *zSpace = "";
18487
18488 while( i<nBlob ){
18489 u64 iVal;
18490 i += sqlite3Fts5GetVarint(&pBlob[i], &iVal);
18491 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "%s%d", zSpace, (int)iVal);
18492 zSpace = " ";
18493 }
18494}
18495#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18496
18497#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18498/*
18499** Buffer (a/n) is assumed to contain a list of serialized varints. Read
18500** each varint and append its string representation to buffer pBuf. Return
18501** after either the input buffer is exhausted or a 0 value is read.
18502**
18503** The return value is the number of bytes read from the input buffer.
18504*/
18505static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
18506 int iOff = 0;
18507 while( iOff<n ){
18508 int iVal;
18509 iOff += fts5GetVarint32(&a[iOff], iVal)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(iVal));
18510 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %d", iVal);
18511 }
18512 return iOff;
18513}
18514#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18515
18516#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18517/*
18518** The start of buffer (a/n) contains the start of a doclist. The doclist
18519** may or may not finish within the buffer. This function appends a text
18520** representation of the part of the doclist that is present to buffer
18521** pBuf.
18522**
18523** The return value is the number of bytes read from the input buffer.
18524*/
18525static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){
18526 i64 iDocid = 0;
18527 int iOff = 0;
18528
18529 if( n>0 ){
18530 iOff = sqlite3Fts5GetVarint(a, (u64*)&iDocid);
18531 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
18532 }
18533 while( iOff<n ){
18534 int nPos;
18535 int bDel;
18536 iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDel);
18537 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " nPos=%d%s", nPos, bDel?"*":"");
18538 iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos)(((n-iOff) < (nPos)) ? (n-iOff) : (nPos)));
18539 if( iOff<n ){
18540 i64 iDelta;
18541 iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDelta);
18542 iDocid += iDelta;
18543 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid);
18544 }
18545 }
18546
18547 return iOff;
18548}
18549#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18550
18551#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18552/*
18553** This function is part of the fts5_decode() debugging function. It is
18554** only ever used with detail=none tables.
18555**
18556** Buffer (pData/nData) contains a doclist in the format used by detail=none
18557** tables. This function appends a human-readable version of that list to
18558** buffer pBuf.
18559**
18560** If *pRc is other than SQLITE_OK when this function is called, it is a
18561** no-op. If an OOM or other error occurs within this function, *pRc is
18562** set to an SQLite error code before returning. The final state of buffer
18563** pBuf is undefined in this case.
18564*/
18565static void fts5DecodeRowidList(
18566 int *pRc, /* IN/OUT: Error code */
18567 Fts5Buffer *pBuf, /* Buffer to append text to */
18568 const u8 *pData, int nData /* Data to decode list-of-rowids from */
18569){
18570 int i = 0;
18571 i64 iRowid = 0;
18572
18573 while( i<nData ){
18574 const char *zApp = "";
18575 u64 iVal;
18576 i += sqlite3Fts5GetVarint(&pData[i], &iVal);
18577 iRowid += iVal;
18578
18579 if( i<nData && pData[i]==0x00 ){
18580 i++;
18581 if( i<nData && pData[i]==0x00 ){
18582 i++;
18583 zApp = "+";
18584 }else{
18585 zApp = "*";
18586 }
18587 }
18588
18589 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp);
18590 }
18591}
18592#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18593
18594#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18595static void fts5BufferAppendTerm(int *pRc, Fts5Buffer *pBuf, Fts5Buffer *pTerm){
18596 int ii;
18597 fts5BufferGrow(pRc, pBuf, pTerm->n*2 + 1)( (u32)((pBuf)->n) + (u32)(pTerm->n*2 + 1) <= (u32)(
(pBuf)->nSpace) ? 0 : sqlite3Fts5BufferSize((pRc),(pBuf),(
pTerm->n*2 + 1)+(pBuf)->n) );
18598 if( *pRc==SQLITE_OK0 ){
18599 for(ii=0; ii<pTerm->n; ii++){
18600 if( pTerm->p[ii]==0x00 ){
18601 pBuf->p[pBuf->n++] = '\\';
18602 pBuf->p[pBuf->n++] = '0';
18603 }else{
18604 pBuf->p[pBuf->n++] = pTerm->p[ii];
18605 }
18606 }
18607 pBuf->p[pBuf->n] = 0x00;
18608 }
18609}
18610#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18611
18612#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18613/*
18614** The implementation of user-defined scalar function fts5_decode().
18615*/
18616static void fts5DecodeFunction(
18617 sqlite3_context *pCtx, /* Function call context */
18618 int nArg, /* Number of args (always 2) */
18619 sqlite3_value **apVal /* Function arguments */
18620){
18621 i64 iRowid; /* Rowid for record being decoded */
18622 int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */
18623 int bTomb;
18624 const u8 *aBlob; int n; /* Record to decode */
18625 u8 *a = 0;
18626 Fts5Buffer s; /* Build up text to return here */
18627 int rc = SQLITE_OK0; /* Return code */
18628 sqlite3_int64 nSpace = 0;
18629 int eDetailNone = (sqlite3_user_datasqlite3_api->user_data(pCtx)!=0);
18630
18631 assert( nArg==2 )((void) (0));
18632 UNUSED_PARAM(nArg)(void)(nArg);
18633 memset(&s, 0, sizeof(Fts5Buffer));
18634 iRowid = sqlite3_value_int64sqlite3_api->value_int64(apVal[0]);
18635
18636 /* Make a copy of the second argument (a blob) in aBlob[]. The aBlob[]
18637 ** copy is followed by FTS5_DATA_ZERO_PADDING 0x00 bytes, which prevents
18638 ** buffer overreads even if the record is corrupt. */
18639 n = sqlite3_value_bytessqlite3_api->value_bytes(apVal[1]);
18640 aBlob = sqlite3_value_blobsqlite3_api->value_blob(apVal[1]);
18641 nSpace = ((i64)n) + FTS5_DATA_ZERO_PADDING8;
18642 a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace);
18643 if( a==0 ) goto decode_out;
18644 if( n>0 ) memcpy(a, aBlob, n);
18645
18646 fts5DecodeRowid(iRowid, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno);
18647
18648 fts5DebugRowid(&rc, &s, iRowid);
18649 if( bDlidx ){
18650 Fts5Data dlidx;
18651 Fts5DlidxLvl lvl;
18652
18653 dlidx.p = a;
18654 dlidx.nn = n;
18655
18656 memset(&lvl, 0, sizeof(Fts5DlidxLvl));
18657 lvl.pData = &dlidx;
18658 lvl.iLeafPgno = iPgno;
18659
18660 for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){
18661 sqlite3Fts5BufferAppendPrintf(&rc, &s,
18662 " %d(%lld)", lvl.iLeafPgno, lvl.iRowid
18663 );
18664 }
18665 }else if( bTomb ){
18666 u32 nElem = fts5GetU32(&a[4]);
18667 int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8;
18668 int nSlot = (n - 8) / szKey;
18669 int ii;
18670 sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem);
18671 if( aBlob[1] ){
18672 sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0");
18673 }
18674 for(ii=0; ii<nSlot; ii++){
18675 u64 iVal = 0;
18676 if( szKey==4 ){
18677 u32 *aSlot = (u32*)&aBlob[8];
18678 if( aSlot[ii] ) iVal = fts5GetU32((u8*)&aSlot[ii]);
18679 }else{
18680 u64 *aSlot = (u64*)&aBlob[8];
18681 if( aSlot[ii] ) iVal = fts5GetU64((u8*)&aSlot[ii]);
18682 }
18683 if( iVal!=0 ){
18684 sqlite3Fts5BufferAppendPrintf(&rc, &s, " %lld", (i64)iVal);
18685 }
18686 }
18687 }else if( iSegid==0 ){
18688 if( iRowid==FTS5_AVERAGES_ROWID1 ){
18689 fts5DecodeAverages(&rc, &s, a, n);
18690 }else{
18691 fts5DecodeStructure(&rc, &s, a, n);
18692 }
18693 }else if( eDetailNone ){
18694 Fts5Buffer term; /* Current term read from page */
18695 int szLeaf;
18696 int iPgidxOff = szLeaf = fts5GetU16(&a[2]);
18697 int iTermOff;
18698 int nKeep = 0;
18699 int iOff;
18700
18701 memset(&term, 0, sizeof(Fts5Buffer));
18702
18703 /* Decode any entries that occur before the first term. */
18704 if( szLeaf<n ){
18705 iPgidxOff += fts5GetVarint32(&a[iPgidxOff], iTermOff)sqlite3Fts5GetVarint32(&a[iPgidxOff],(u32*)&(iTermOff
));
18706 }else{
18707 iTermOff = szLeaf;
18708 }
18709 fts5DecodeRowidList(&rc, &s, &a[4], iTermOff-4);
18710
18711 iOff = iTermOff;
18712 while( iOff<szLeaf && rc==SQLITE_OK0 ){
18713 int nAppend;
18714
18715 /* Read the term data for the next term*/
18716 iOff += fts5GetVarint32(&a[iOff], nAppend)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nAppend));
18717 term.n = nKeep;
18718 fts5BufferAppendBlob(&rc, &term, nAppend, &a[iOff])sqlite3Fts5BufferAppendBlob(&rc,&term,nAppend,&a[
iOff]);
18719 sqlite3Fts5BufferAppendPrintf(&rc, &s, " term=");
18720 fts5BufferAppendTerm(&rc, &s, &term);
18721 iOff += nAppend;
18722
18723 /* Figure out where the doclist for this term ends */
18724 if( iPgidxOff<n ){
18725 int nIncr;
18726 iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nIncr)sqlite3Fts5GetVarint32(&a[iPgidxOff],(u32*)&(nIncr));
18727 iTermOff += nIncr;
18728 }else{
18729 iTermOff = szLeaf;
18730 }
18731 if( iTermOff>szLeaf ){
18732 rc = FTS5_CORRUPT(11 | (1<<8));
18733 }else{
18734 fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff);
18735 }
18736 iOff = iTermOff;
18737 if( iOff<szLeaf ){
18738 iOff += fts5GetVarint32(&a[iOff], nKeep)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nKeep));
18739 }
18740 }
18741
18742 fts5BufferFree(&term)sqlite3Fts5BufferFree(&term);
18743 }else{
18744 Fts5Buffer term; /* Current term read from page */
18745 int szLeaf; /* Offset of pgidx in a[] */
18746 int iPgidxOff;
18747 int iPgidxPrev = 0; /* Previous value read from pgidx */
18748 int iTermOff = 0;
18749 int iRowidOff = 0;
18750 int iOff;
18751 int nDoclist;
18752
18753 memset(&term, 0, sizeof(Fts5Buffer));
18754
18755 if( n<4 ){
18756 sqlite3Fts5BufferSet(&rc, &s, 7, (const u8*)"corrupt");
18757 goto decode_out;
18758 }else{
18759 iRowidOff = fts5GetU16(&a[0]);
18760 iPgidxOff = szLeaf = fts5GetU16(&a[2]);
18761 if( iPgidxOff<n ){
18762 fts5GetVarint32(&a[iPgidxOff], iTermOff)sqlite3Fts5GetVarint32(&a[iPgidxOff],(u32*)&(iTermOff
));
18763 }else if( iPgidxOff>n ){
18764 rc = FTS5_CORRUPT(11 | (1<<8));
18765 goto decode_out;
18766 }
18767 }
18768
18769 /* Decode the position list tail at the start of the page */
18770 if( iRowidOff!=0 ){
18771 iOff = iRowidOff;
18772 }else if( iTermOff!=0 ){
18773 iOff = iTermOff;
18774 }else{
18775 iOff = szLeaf;
18776 }
18777 if( iOff>n ){
18778 rc = FTS5_CORRUPT(11 | (1<<8));
18779 goto decode_out;
18780 }
18781 fts5DecodePoslist(&rc, &s, &a[4], iOff-4);
18782
18783 /* Decode any more doclist data that appears on the page before the
18784 ** first term. */
18785 nDoclist = (iTermOff ? iTermOff : szLeaf) - iOff;
18786 if( nDoclist+iOff>n ){
18787 rc = FTS5_CORRUPT(11 | (1<<8));
18788 goto decode_out;
18789 }
18790 fts5DecodeDoclist(&rc, &s, &a[iOff], nDoclist);
18791
18792 while( iPgidxOff<n && rc==SQLITE_OK0 ){
18793 int bFirst = (iPgidxOff==szLeaf); /* True for first term on page */
18794 int nByte; /* Bytes of data */
18795 int iEnd;
18796
18797 iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nByte)sqlite3Fts5GetVarint32(&a[iPgidxOff],(u32*)&(nByte));
18798 iPgidxPrev += nByte;
18799 iOff = iPgidxPrev;
18800
18801 if( iPgidxOff<n ){
18802 fts5GetVarint32(&a[iPgidxOff], nByte)sqlite3Fts5GetVarint32(&a[iPgidxOff],(u32*)&(nByte));
18803 iEnd = iPgidxPrev + nByte;
18804 }else{
18805 iEnd = szLeaf;
18806 }
18807 if( iEnd>szLeaf ){
18808 rc = FTS5_CORRUPT(11 | (1<<8));
18809 break;
18810 }
18811
18812 if( bFirst==0 ){
18813 iOff += fts5GetVarint32(&a[iOff], nByte)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nByte));
18814 if( nByte>term.n ){
18815 rc = FTS5_CORRUPT(11 | (1<<8));
18816 break;
18817 }
18818 term.n = nByte;
18819 }
18820 iOff += fts5GetVarint32(&a[iOff], nByte)sqlite3Fts5GetVarint32(&a[iOff],(u32*)&(nByte));
18821 if( iOff+nByte>n ){
18822 rc = FTS5_CORRUPT(11 | (1<<8));
18823 break;
18824 }
18825 fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff])sqlite3Fts5BufferAppendBlob(&rc,&term,nByte,&a[iOff
]);
18826 iOff += nByte;
18827
18828 sqlite3Fts5BufferAppendPrintf(&rc, &s, " term=");
18829 fts5BufferAppendTerm(&rc, &s, &term);
18830 iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], iEnd-iOff);
18831 }
18832
18833 fts5BufferFree(&term)sqlite3Fts5BufferFree(&term);
18834 }
18835
18836 decode_out:
18837 sqlite3_freesqlite3_api->free(a);
18838 if( rc==SQLITE_OK0 ){
18839 sqlite3_result_textsqlite3_api->result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
18840 }else{
18841 sqlite3_result_error_codesqlite3_api->result_error_code(pCtx, rc);
18842 }
18843 fts5BufferFree(&s)sqlite3Fts5BufferFree(&s);
18844}
18845#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18846
18847#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18848/*
18849** The implementation of user-defined scalar function fts5_rowid().
18850*/
18851static void fts5RowidFunction(
18852 sqlite3_context *pCtx, /* Function call context */
18853 int nArg, /* Number of args (always 2) */
18854 sqlite3_value **apVal /* Function arguments */
18855){
18856 const char *zArg;
18857 if( nArg==0 ){
18858 sqlite3_result_errorsqlite3_api->result_error(pCtx, "should be: fts5_rowid(subject, ....)", -1);
18859 }else{
18860 zArg = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[0]);
18861 if( 0==sqlite3_stricmpsqlite3_api->stricmp(zArg, "segment") ){
18862 i64 iRowid;
18863 int segid, pgno;
18864 if( nArg!=3 ){
18865 sqlite3_result_errorsqlite3_api->result_error(pCtx,
18866 "should be: fts5_rowid('segment', segid, pgno))", -1
18867 );
18868 }else{
18869 segid = sqlite3_value_intsqlite3_api->value_int(apVal[1]);
18870 pgno = sqlite3_value_intsqlite3_api->value_int(apVal[2]);
18871 iRowid = FTS5_SEGMENT_ROWID(segid, pgno)( ((i64)(segid) << (31 +5 +1)) + ((i64)(0) << (31
+ 5)) + ((i64)(0) << (31)) + ((i64)(pgno)) );
18872 sqlite3_result_int64sqlite3_api->result_int64(pCtx, iRowid);
18873 }
18874 }else{
18875 sqlite3_result_errorsqlite3_api->result_error(pCtx,
18876 "first arg to fts5_rowid() must be 'segment'" , -1
18877 );
18878 }
18879 }
18880}
18881#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
18882
18883#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
18884
18885typedef struct Fts5StructVtab Fts5StructVtab;
18886struct Fts5StructVtab {
18887 sqlite3_vtab base;
18888};
18889
18890typedef struct Fts5StructVcsr Fts5StructVcsr;
18891struct Fts5StructVcsr {
18892 sqlite3_vtab_cursor base;
18893 Fts5Structure *pStruct;
18894 int iLevel;
18895 int iSeg;
18896 int iRowid;
18897};
18898
18899/*
18900** Create a new fts5_structure() table-valued function.
18901*/
18902static int fts5structConnectMethod(
18903 sqlite3 *db,
18904 void *pAux,
18905 int argc, const char *const*argv,
18906 sqlite3_vtab **ppVtab,
18907 char **pzErr
18908){
18909 Fts5StructVtab *pNew = 0;
18910 int rc = SQLITE_OK0;
18911
18912 rc = sqlite3_declare_vtabsqlite3_api->declare_vtab(db,
18913 "CREATE TABLE xyz("
18914 "level, segment, merge, segid, leaf1, leaf2, loc1, loc2, "
18915 "npgtombstone, nentrytombstone, nentry, struct HIDDEN);"
18916 );
18917 if( rc==SQLITE_OK0 ){
18918 pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
18919 }
18920
18921 *ppVtab = (sqlite3_vtab*)pNew;
18922 return rc;
18923}
18924
18925/*
18926** We must have a single struct=? constraint that will be passed through
18927** into the xFilter method. If there is no valid struct=? constraint,
18928** then return an SQLITE_CONSTRAINT error.
18929*/
18930static int fts5structBestIndexMethod(
18931 sqlite3_vtab *tab,
18932 sqlite3_index_info *pIdxInfo
18933){
18934 int i;
18935 int rc = SQLITE_CONSTRAINT19;
18936 struct sqlite3_index_constraint *p;
18937 pIdxInfo->estimatedCost = (double)100;
18938 pIdxInfo->estimatedRows = 100;
18939 pIdxInfo->idxNum = 0;
18940 for(i=0, p=pIdxInfo->aConstraint; i<pIdxInfo->nConstraint; i++, p++){
18941 if( p->usable==0 ) continue;
18942 if( p->op==SQLITE_INDEX_CONSTRAINT_EQ2 && p->iColumn==11 ){
18943 rc = SQLITE_OK0;
18944 pIdxInfo->aConstraintUsage[i].omit = 1;
18945 pIdxInfo->aConstraintUsage[i].argvIndex = 1;
18946 break;
18947 }
18948 }
18949 return rc;
18950}
18951
18952/*
18953** This method is the destructor for bytecodevtab objects.
18954*/
18955static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){
18956 Fts5StructVtab *p = (Fts5StructVtab*)pVtab;
18957 sqlite3_freesqlite3_api->free(p);
18958 return SQLITE_OK0;
18959}
18960
18961/*
18962** Constructor for a new bytecodevtab_cursor object.
18963*/
18964static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
18965 int rc = SQLITE_OK0;
18966 Fts5StructVcsr *pNew = 0;
18967
18968 pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
18969 *ppCsr = (sqlite3_vtab_cursor*)pNew;
18970
18971 return SQLITE_OK0;
18972}
18973
18974/*
18975** Destructor for a bytecodevtab_cursor.
18976*/
18977static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){
18978 Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
18979 fts5StructureRelease(pCsr->pStruct);
18980 sqlite3_freesqlite3_api->free(pCsr);
18981 return SQLITE_OK0;
18982}
18983
18984
18985/*
18986** Advance a bytecodevtab_cursor to its next row of output.
18987*/
18988static int fts5structNextMethod(sqlite3_vtab_cursor *cur){
18989 Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
18990 Fts5Structure *p = pCsr->pStruct;
18991
18992 assert( pCsr->pStruct )((void) (0));
18993 pCsr->iSeg++;
18994 pCsr->iRowid++;
18995 while( pCsr->iLevel<p->nLevel && pCsr->iSeg>=p->aLevel[pCsr->iLevel].nSeg ){
18996 pCsr->iLevel++;
18997 pCsr->iSeg = 0;
18998 }
18999 if( pCsr->iLevel>=p->nLevel ){
19000 fts5StructureRelease(pCsr->pStruct);
19001 pCsr->pStruct = 0;
19002 }
19003 return SQLITE_OK0;
19004}
19005
19006/*
19007** Return TRUE if the cursor has been moved off of the last
19008** row of output.
19009*/
19010static int fts5structEofMethod(sqlite3_vtab_cursor *cur){
19011 Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
19012 return pCsr->pStruct==0;
19013}
19014
19015static int fts5structRowidMethod(
19016 sqlite3_vtab_cursor *cur,
19017 sqlite_int64 *piRowid
19018){
19019 Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
19020 *piRowid = pCsr->iRowid;
19021 return SQLITE_OK0;
19022}
19023
19024/*
19025** Return values of columns for the row at which the bytecodevtab_cursor
19026** is currently pointing.
19027*/
19028static int fts5structColumnMethod(
19029 sqlite3_vtab_cursor *cur, /* The cursor */
19030 sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
19031 int i /* Which column to return */
19032){
19033 Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur;
19034 Fts5Structure *p = pCsr->pStruct;
19035 Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg];
19036
19037 switch( i ){
19038 case 0: /* level */
19039 sqlite3_result_intsqlite3_api->result_int(ctx, pCsr->iLevel);
19040 break;
19041 case 1: /* segment */
19042 sqlite3_result_intsqlite3_api->result_int(ctx, pCsr->iSeg);
19043 break;
19044 case 2: /* merge */
19045 sqlite3_result_intsqlite3_api->result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge);
19046 break;
19047 case 3: /* segid */
19048 sqlite3_result_intsqlite3_api->result_int(ctx, pSeg->iSegid);
19049 break;
19050 case 4: /* leaf1 */
19051 sqlite3_result_intsqlite3_api->result_int(ctx, pSeg->pgnoFirst);
19052 break;
19053 case 5: /* leaf2 */
19054 sqlite3_result_intsqlite3_api->result_int(ctx, pSeg->pgnoLast);
19055 break;
19056 case 6: /* origin1 */
19057 sqlite3_result_int64sqlite3_api->result_int64(ctx, pSeg->iOrigin1);
19058 break;
19059 case 7: /* origin2 */
19060 sqlite3_result_int64sqlite3_api->result_int64(ctx, pSeg->iOrigin2);
19061 break;
19062 case 8: /* npgtombstone */
19063 sqlite3_result_intsqlite3_api->result_int(ctx, pSeg->nPgTombstone);
19064 break;
19065 case 9: /* nentrytombstone */
19066 sqlite3_result_int64sqlite3_api->result_int64(ctx, pSeg->nEntryTombstone);
19067 break;
19068 case 10: /* nentry */
19069 sqlite3_result_int64sqlite3_api->result_int64(ctx, pSeg->nEntry);
19070 break;
19071 }
19072 return SQLITE_OK0;
19073}
19074
19075/*
19076** Initialize a cursor.
19077**
19078** idxNum==0 means show all subprograms
19079** idxNum==1 means show only the main bytecode and omit subprograms.
19080*/
19081static int fts5structFilterMethod(
19082 sqlite3_vtab_cursor *pVtabCursor,
19083 int idxNum, const char *idxStr,
19084 int argc, sqlite3_value **argv
19085){
19086 Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor;
19087 int rc = SQLITE_OK0;
19088
19089 const u8 *aBlob = 0;
19090 int nBlob = 0;
19091
19092 assert( argc==1 )((void) (0));
19093 fts5StructureRelease(pCsr->pStruct);
19094 pCsr->pStruct = 0;
19095
19096 nBlob = sqlite3_value_bytessqlite3_api->value_bytes(argv[0]);
19097 aBlob = (const u8*)sqlite3_value_blobsqlite3_api->value_blob(argv[0]);
19098 rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct);
19099 if( rc==SQLITE_OK0 ){
19100 pCsr->iLevel = 0;
19101 pCsr->iRowid = 0;
19102 pCsr->iSeg = -1;
19103 rc = fts5structNextMethod(pVtabCursor);
19104 }
19105
19106 return rc;
19107}
19108
19109#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */
19110
19111/*
19112** This is called as part of registering the FTS5 module with database
19113** connection db. It registers several user-defined scalar functions useful
19114** with FTS5.
19115**
19116** If successful, SQLITE_OK is returned. If an error occurs, some other
19117** SQLite error code is returned instead.
19118*/
19119static int sqlite3Fts5IndexInit(sqlite3 *db){
19120#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
19121 int rc = sqlite3_create_functionsqlite3_api->create_function(
19122 db, "fts5_decode", 2, SQLITE_UTF81, 0, fts5DecodeFunction, 0, 0
19123 );
19124
19125 if( rc==SQLITE_OK0 ){
19126 rc = sqlite3_create_functionsqlite3_api->create_function(
19127 db, "fts5_decode_none", 2,
19128 SQLITE_UTF81, (void*)db, fts5DecodeFunction, 0, 0
19129 );
19130 }
19131
19132 if( rc==SQLITE_OK0 ){
19133 rc = sqlite3_create_functionsqlite3_api->create_function(
19134 db, "fts5_rowid", -1, SQLITE_UTF81, 0, fts5RowidFunction, 0, 0
19135 );
19136 }
19137
19138 if( rc==SQLITE_OK0 ){
19139 static const sqlite3_module fts5structure_module = {
19140 0, /* iVersion */
19141 0, /* xCreate */
19142 fts5structConnectMethod, /* xConnect */
19143 fts5structBestIndexMethod, /* xBestIndex */
19144 fts5structDisconnectMethod, /* xDisconnect */
19145 0, /* xDestroy */
19146 fts5structOpenMethod, /* xOpen */
19147 fts5structCloseMethod, /* xClose */
19148 fts5structFilterMethod, /* xFilter */
19149 fts5structNextMethod, /* xNext */
19150 fts5structEofMethod, /* xEof */
19151 fts5structColumnMethod, /* xColumn */
19152 fts5structRowidMethod, /* xRowid */
19153 0, /* xUpdate */
19154 0, /* xBegin */
19155 0, /* xSync */
19156 0, /* xCommit */
19157 0, /* xRollback */
19158 0, /* xFindFunction */
19159 0, /* xRename */
19160 0, /* xSavepoint */
19161 0, /* xRelease */
19162 0, /* xRollbackTo */
19163 0, /* xShadowName */
19164 0 /* xIntegrity */
19165 };
19166 rc = sqlite3_create_modulesqlite3_api->create_module(db, "fts5_structure", &fts5structure_module, 0);
19167 }
19168 return rc;
19169#else
19170 return SQLITE_OK0;
19171 UNUSED_PARAM(db)(void)(db);
19172#endif
19173}
19174
19175
19176static int sqlite3Fts5IndexReset(Fts5Index *p){
19177 assert( p->pStruct==0 || p->iStructVersion!=0 )((void) (0));
19178 if( fts5IndexDataVersion(p)!=p->iStructVersion ){
19179 fts5StructureInvalidate(p);
19180 }
19181 return fts5IndexReturn(p);
19182}
19183
19184#line 1 "fts5_main.c"
19185/*
19186** 2014 Jun 09
19187**
19188** The author disclaims copyright to this source code. In place of
19189** a legal notice, here is a blessing:
19190**
19191** May you do good and not evil.
19192** May you find forgiveness for yourself and forgive others.
19193** May you share freely, never taking more than you give.
19194**
19195******************************************************************************
19196**
19197** This is an SQLite module implementing full-text search.
19198*/
19199
19200
19201/* #include "fts5Int.h" */
19202
19203/*
19204** This variable is set to false when running tests for which the on disk
19205** structures should not be corrupt. Otherwise, true. If it is false, extra
19206** assert() conditions in the fts5 code are activated - conditions that are
19207** only true if it is guaranteed that the fts5 database is not corrupt.
19208*/
19209#ifdef SQLITE_DEBUG
19210int sqlite3_fts5_may_be_corrupt = 1;
19211#endif
19212
19213
19214typedef struct Fts5Auxdata Fts5Auxdata;
19215typedef struct Fts5Auxiliary Fts5Auxiliary;
19216typedef struct Fts5Cursor Fts5Cursor;
19217typedef struct Fts5FullTable Fts5FullTable;
19218typedef struct Fts5Sorter Fts5Sorter;
19219typedef struct Fts5TokenizerModule Fts5TokenizerModule;
19220
19221/*
19222** NOTES ON TRANSACTIONS:
19223**
19224** SQLite invokes the following virtual table methods as transactions are
19225** opened and closed by the user:
19226**
19227** xBegin(): Start of a new transaction.
19228** xSync(): Initial part of two-phase commit.
19229** xCommit(): Final part of two-phase commit.
19230** xRollback(): Rollback the transaction.
19231**
19232** Anything that is required as part of a commit that may fail is performed
19233** in the xSync() callback. Current versions of SQLite ignore any errors
19234** returned by xCommit().
19235**
19236** And as sub-transactions are opened/closed:
19237**
19238** xSavepoint(int S): Open savepoint S.
19239** xRelease(int S): Commit and close savepoint S.
19240** xRollbackTo(int S): Rollback to start of savepoint S.
19241**
19242** During a write-transaction the fts5_index.c module may cache some data
19243** in-memory. It is flushed to disk whenever xSync(), xRelease() or
19244** xSavepoint() is called. And discarded whenever xRollback() or xRollbackTo()
19245** is called.
19246**
19247** Additionally, if SQLITE_DEBUG is defined, an instance of the following
19248** structure is used to record the current transaction state. This information
19249** is not required, but it is used in the assert() statements executed by
19250** function fts5CheckTransactionState() (see below).
19251*/
19252struct Fts5TransactionState {
19253 int eState; /* 0==closed, 1==open, 2==synced */
19254 int iSavepoint; /* Number of open savepoints (0 -> none) */
19255};
19256
19257/*
19258** A single object of this type is allocated when the FTS5 module is
19259** registered with a database handle. It is used to store pointers to
19260** all registered FTS5 extensions - tokenizers and auxiliary functions.
19261*/
19262struct Fts5Global {
19263 fts5_api api; /* User visible part of object (see fts5.h) */
19264 sqlite3 *db; /* Associated database connection */
19265 i64 iNextId; /* Used to allocate unique cursor ids */
19266 Fts5Auxiliary *pAux; /* First in list of all aux. functions */
19267 Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */
19268 Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */
19269 Fts5Cursor *pCsr; /* First in list of all open cursors */
19270 u32 aLocaleHdr[4];
19271};
19272
19273/*
19274** Size of header on fts5_locale() values. And macro to access a buffer
19275** containing a copy of the header from an Fts5Config pointer.
19276*/
19277#define FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )) ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ))
19278#define FTS5_LOCALE_HDR(pConfig)((const u8*)(pConfig->pGlobal->aLocaleHdr)) ((const u8*)(pConfig->pGlobal->aLocaleHdr))
19279
19280#define FTS5_INSTTOKEN_SUBTYPE73 73
19281
19282/*
19283** Each auxiliary function registered with the FTS5 module is represented
19284** by an object of the following type. All such objects are stored as part
19285** of the Fts5Global.pAux list.
19286*/
19287struct Fts5Auxiliary {
19288 Fts5Global *pGlobal; /* Global context for this function */
19289 char *zFunc; /* Function name (nul-terminated) */
19290 void *pUserData; /* User-data pointer */
19291 fts5_extension_function xFunc; /* Callback function */
19292 void (*xDestroy)(void*); /* Destructor function */
19293 Fts5Auxiliary *pNext; /* Next registered auxiliary function */
19294};
19295
19296/*
19297** Each tokenizer module registered with the FTS5 module is represented
19298** by an object of the following type. All such objects are stored as part
19299** of the Fts5Global.pTok list.
19300**
19301** bV2Native:
19302** True if the tokenizer was registered using xCreateTokenizer_v2(), false
19303** for xCreateTokenizer(). If this variable is true, then x2 is populated
19304** with the routines as supplied by the caller and x1 contains synthesized
19305** wrapper routines. In this case the user-data pointer passed to
19306** x1.xCreate should be a pointer to the Fts5TokenizerModule structure,
19307** not a copy of pUserData.
19308**
19309** Of course, if bV2Native is false, then x1 contains the real routines and
19310** x2 the synthesized ones. In this case a pointer to the Fts5TokenizerModule
19311** object should be passed to x2.xCreate.
19312**
19313** The synthesized wrapper routines are necessary for xFindTokenizer(_v2)
19314** calls.
19315*/
19316struct Fts5TokenizerModule {
19317 char *zName; /* Name of tokenizer */
19318 void *pUserData; /* User pointer passed to xCreate() */
19319 int bV2Native; /* True if v2 native tokenizer */
19320 fts5_tokenizer x1; /* Tokenizer functions */
19321 fts5_tokenizer_v2 x2; /* V2 tokenizer functions */
19322 void (*xDestroy)(void*); /* Destructor function */
19323 Fts5TokenizerModule *pNext; /* Next registered tokenizer module */
19324};
19325
19326struct Fts5FullTable {
19327 Fts5Table p; /* Public class members from fts5Int.h */
19328 Fts5Storage *pStorage; /* Document store */
19329 Fts5Global *pGlobal; /* Global (connection wide) data */
19330 Fts5Cursor *pSortCsr; /* Sort data from this cursor */
19331 int iSavepoint; /* Successful xSavepoint()+1 */
19332
19333#ifdef SQLITE_DEBUG
19334 struct Fts5TransactionState ts;
19335#endif
19336};
19337
19338struct Fts5MatchPhrase {
19339 Fts5Buffer *pPoslist; /* Pointer to current poslist */
19340 int nTerm; /* Size of phrase in terms */
19341};
19342
19343/*
19344** pStmt:
19345** SELECT rowid, <fts> FROM <fts> ORDER BY +rank;
19346**
19347** aIdx[]:
19348** There is one entry in the aIdx[] array for each phrase in the query,
19349** the value of which is the offset within aPoslist[] following the last
19350** byte of the position list for the corresponding phrase.
19351*/
19352struct Fts5Sorter {
19353 sqlite3_stmt *pStmt;
19354 i64 iRowid; /* Current rowid */
19355 const u8 *aPoslist; /* Position lists for current row */
19356 int nIdx; /* Number of entries in aIdx[] */
19357 int aIdx[FLEXARRAY]; /* Offsets into aPoslist for current row */
19358};
19359
19360/* Size (int bytes) of an Fts5Sorter object with N indexes */
19361#define SZ_FTS5SORTER(N)(__builtin_offsetof(Fts5Sorter, nIdx)+((N+2)/2)*sizeof(i64)) (offsetof(Fts5Sorter,nIdx)__builtin_offsetof(Fts5Sorter, nIdx)+((N+2)/2)*sizeof(i64))
19362
19363/*
19364** Virtual-table cursor object.
19365**
19366** iSpecial:
19367** If this is a 'special' query (refer to function fts5SpecialMatch()),
19368** then this variable contains the result of the query.
19369**
19370** iFirstRowid, iLastRowid:
19371** These variables are only used for FTS5_PLAN_MATCH cursors. Assuming the
19372** cursor iterates in ascending order of rowids, iFirstRowid is the lower
19373** limit of rowids to return, and iLastRowid the upper. In other words, the
19374** WHERE clause in the user's query might have been:
19375**
19376** <tbl> MATCH <expr> AND rowid BETWEEN $iFirstRowid AND $iLastRowid
19377**
19378** If the cursor iterates in descending order of rowid, iFirstRowid
19379** is the upper limit (i.e. the "first" rowid visited) and iLastRowid
19380** the lower.
19381*/
19382struct Fts5Cursor {
19383 sqlite3_vtab_cursor base; /* Base class used by SQLite core */
19384 Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */
19385 int *aColumnSize; /* Values for xColumnSize() */
19386 i64 iCsrId; /* Cursor id */
19387
19388 /* Zero from this point onwards on cursor reset */
19389 int ePlan; /* FTS5_PLAN_XXX value */
19390 int bDesc; /* True for "ORDER BY rowid DESC" queries */
19391 i64 iFirstRowid; /* Return no rowids earlier than this */
19392 i64 iLastRowid; /* Return no rowids later than this */
19393 sqlite3_stmt *pStmt; /* Statement used to read %_content */
19394 Fts5Expr *pExpr; /* Expression for MATCH queries */
19395 Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */
19396 int csrflags; /* Mask of cursor flags (see below) */
19397 i64 iSpecial; /* Result of special query */
19398
19399 /* "rank" function. Populated on demand from vtab.xColumn(). */
19400 char *zRank; /* Custom rank function */
19401 char *zRankArgs; /* Custom rank function args */
19402 Fts5Auxiliary *pRank; /* Rank callback (or NULL) */
19403 int nRankArg; /* Number of trailing arguments for rank() */
19404 sqlite3_value **apRankArg; /* Array of trailing arguments */
19405 sqlite3_stmt *pRankArgStmt; /* Origin of objects in apRankArg[] */
19406
19407 /* Auxiliary data storage */
19408 Fts5Auxiliary *pAux; /* Currently executing extension function */
19409 Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */
19410
19411 /* Cache used by auxiliary API functions xInst() and xInstCount() */
19412 Fts5PoslistReader *aInstIter; /* One for each phrase */
19413 int nInstAlloc; /* Size of aInst[] array (entries / 3) */
19414 int nInstCount; /* Number of phrase instances */
19415 int *aInst; /* 3 integers per phrase instance */
19416};
19417
19418/*
19419** Bits that make up the "idxNum" parameter passed indirectly by
19420** xBestIndex() to xFilter().
19421*/
19422#define FTS5_BI_MATCH0x0001 0x0001 /* <tbl> MATCH ? */
19423#define FTS5_BI_RANK0x0002 0x0002 /* rank MATCH ? */
19424#define FTS5_BI_ROWID_EQ0x0004 0x0004 /* rowid == ? */
19425#define FTS5_BI_ROWID_LE0x0008 0x0008 /* rowid <= ? */
19426#define FTS5_BI_ROWID_GE0x0010 0x0010 /* rowid >= ? */
19427
19428#define FTS5_BI_ORDER_RANK0x0020 0x0020
19429#define FTS5_BI_ORDER_ROWID0x0040 0x0040
19430#define FTS5_BI_ORDER_DESC0x0080 0x0080
19431
19432/*
19433** Values for Fts5Cursor.csrflags
19434*/
19435#define FTS5CSR_EOF0x01 0x01
19436#define FTS5CSR_REQUIRE_CONTENT0x02 0x02
19437#define FTS5CSR_REQUIRE_DOCSIZE0x04 0x04
19438#define FTS5CSR_REQUIRE_INST0x08 0x08
19439#define FTS5CSR_FREE_ZRANK0x10 0x10
19440#define FTS5CSR_REQUIRE_RESEEK0x20 0x20
19441#define FTS5CSR_REQUIRE_POSLIST0x40 0x40
19442
19443#define BitFlagAllTest(x,y)(((x) & (y))==(y)) (((x) & (y))==(y))
19444#define BitFlagTest(x,y)(((x) & (y))!=0) (((x) & (y))!=0)
19445
19446
19447/*
19448** Macros to Set(), Clear() and Test() cursor flags.
19449*/
19450#define CsrFlagSet(pCsr, flag)((pCsr)->csrflags |= (flag)) ((pCsr)->csrflags |= (flag))
19451#define CsrFlagClear(pCsr, flag)((pCsr)->csrflags &= ~(flag)) ((pCsr)->csrflags &= ~(flag))
19452#define CsrFlagTest(pCsr, flag)((pCsr)->csrflags & (flag)) ((pCsr)->csrflags & (flag))
19453
19454struct Fts5Auxdata {
19455 Fts5Auxiliary *pAux; /* Extension to which this belongs */
19456 void *pPtr; /* Pointer value */
19457 void(*xDelete)(void*); /* Destructor */
19458 Fts5Auxdata *pNext; /* Next object in linked list */
19459};
19460
19461#ifdef SQLITE_DEBUG
19462#define FTS5_BEGIN 1
19463#define FTS5_SYNC 2
19464#define FTS5_COMMIT 3
19465#define FTS5_ROLLBACK 4
19466#define FTS5_SAVEPOINT 5
19467#define FTS5_RELEASE 6
19468#define FTS5_ROLLBACKTO 7
19469static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){
19470 switch( op ){
19471 case FTS5_BEGIN:
19472 assert( p->ts.eState==0 )((void) (0));
19473 p->ts.eState = 1;
19474 p->ts.iSavepoint = -1;
19475 break;
19476
19477 case FTS5_SYNC:
19478 assert( p->ts.eState==1 || p->ts.eState==2 )((void) (0));
19479 p->ts.eState = 2;
19480 break;
19481
19482 case FTS5_COMMIT:
19483 assert( p->ts.eState==2 )((void) (0));
19484 p->ts.eState = 0;
19485 break;
19486
19487 case FTS5_ROLLBACK:
19488 assert( p->ts.eState==1 || p->ts.eState==2 || p->ts.eState==0 )((void) (0));
19489 p->ts.eState = 0;
19490 break;
19491
19492 case FTS5_SAVEPOINT:
19493 assert( p->ts.eState>=1 )((void) (0));
19494 assert( iSavepoint>=0 )((void) (0));
19495 assert( iSavepoint>=p->ts.iSavepoint )((void) (0));
19496 p->ts.iSavepoint = iSavepoint;
19497 break;
19498
19499 case FTS5_RELEASE:
19500 assert( p->ts.eState>=1 )((void) (0));
19501 assert( iSavepoint>=0 )((void) (0));
19502 assert( iSavepoint<=p->ts.iSavepoint )((void) (0));
19503 p->ts.iSavepoint = iSavepoint-1;
19504 break;
19505
19506 case FTS5_ROLLBACKTO:
19507 assert( p->ts.eState>=1 )((void) (0));
19508 assert( iSavepoint>=-1 )((void) (0));
19509 /* The following assert() can fail if another vtab strikes an error
19510 ** within an xSavepoint() call then SQLite calls xRollbackTo() - without
19511 ** having called xSavepoint() on this vtab. */
19512 /* assert( iSavepoint<=p->ts.iSavepoint ); */
19513 p->ts.iSavepoint = iSavepoint;
19514 break;
19515 }
19516}
19517#else
19518# define fts5CheckTransactionState(x,y,z)
19519#endif
19520
19521/*
19522** Return true if pTab is a contentless table. If parameter bIncludeUnindexed
19523** is true, this includes contentless tables that store UNINDEXED columns
19524** only.
19525*/
19526static int fts5IsContentless(Fts5FullTable *pTab, int bIncludeUnindexed){
19527 int eContent = pTab->p.pConfig->eContent;
19528 return (
19529 eContent==FTS5_CONTENT_NONE1
19530 || (bIncludeUnindexed && eContent==FTS5_CONTENT_UNINDEXED3)
19531 );
19532}
19533
19534/*
19535** Delete a virtual table handle allocated by fts5InitVtab().
19536*/
19537static void fts5FreeVtab(Fts5FullTable *pTab){
19538 if( pTab ){
19539 sqlite3Fts5IndexClose(pTab->p.pIndex);
19540 sqlite3Fts5StorageClose(pTab->pStorage);
19541 sqlite3Fts5ConfigFree(pTab->p.pConfig);
19542 sqlite3_freesqlite3_api->free(pTab);
19543 }
19544}
19545
19546/*
19547** The xDisconnect() virtual table method.
19548*/
19549static int fts5DisconnectMethod(sqlite3_vtab *pVtab){
19550 fts5FreeVtab((Fts5FullTable*)pVtab);
19551 return SQLITE_OK0;
19552}
19553
19554/*
19555** The xDestroy() virtual table method.
19556*/
19557static int fts5DestroyMethod(sqlite3_vtab *pVtab){
19558 Fts5Table *pTab = (Fts5Table*)pVtab;
19559 int rc = sqlite3Fts5DropAll(pTab->pConfig);
19560 if( rc==SQLITE_OK0 ){
19561 fts5FreeVtab((Fts5FullTable*)pVtab);
19562 }
19563 return rc;
19564}
19565
19566/*
19567** This function is the implementation of both the xConnect and xCreate
19568** methods of the FTS3 virtual table.
19569**
19570** The argv[] array contains the following:
19571**
19572** argv[0] -> module name ("fts5")
19573** argv[1] -> database name
19574** argv[2] -> table name
19575** argv[...] -> "column name" and other module argument fields.
19576*/
19577static int fts5InitVtab(
19578 int bCreate, /* True for xCreate, false for xConnect */
19579 sqlite3 *db, /* The SQLite database connection */
19580 void *pAux, /* Hash table containing tokenizers */
19581 int argc, /* Number of elements in argv array */
19582 const char * const *argv, /* xCreate/xConnect argument array */
19583 sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
19584 char **pzErr /* Write any error message here */
19585){
19586 Fts5Global *pGlobal = (Fts5Global*)pAux;
19587 const char **azConfig = (const char**)argv;
19588 int rc = SQLITE_OK0; /* Return code */
19589 Fts5Config *pConfig = 0; /* Results of parsing argc/argv */
19590 Fts5FullTable *pTab = 0; /* New virtual table object */
19591
19592 /* Allocate the new vtab object and parse the configuration */
19593 pTab = (Fts5FullTable*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5FullTable));
19594 if( rc==SQLITE_OK0 ){
19595 rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
19596 assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 )((void) (0));
19597 }
19598 if( rc==SQLITE_OK0 ){
19599 pConfig->pzErrmsg = pzErr;
19600 pTab->p.pConfig = pConfig;
19601 pTab->pGlobal = pGlobal;
19602 if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){
19603 rc = sqlite3Fts5LoadTokenizer(pConfig);
19604 }
19605 }
19606
19607 /* Open the index sub-system */
19608 if( rc==SQLITE_OK0 ){
19609 rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->p.pIndex, pzErr);
19610 }
19611
19612 /* Open the storage sub-system */
19613 if( rc==SQLITE_OK0 ){
19614 rc = sqlite3Fts5StorageOpen(
19615 pConfig, pTab->p.pIndex, bCreate, &pTab->pStorage, pzErr
19616 );
19617 }
19618
19619 /* Call sqlite3_declare_vtab() */
19620 if( rc==SQLITE_OK0 ){
19621 rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
19622 }
19623
19624 /* Load the initial configuration */
19625 if( rc==SQLITE_OK0 ){
19626 rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie-1);
19627 }
19628
19629 if( rc==SQLITE_OK0 && pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
19630 rc = sqlite3_vtab_configsqlite3_api->vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT1, (int)1);
19631 }
19632 if( rc==SQLITE_OK0 ){
19633 rc = sqlite3_vtab_configsqlite3_api->vtab_config(db, SQLITE_VTAB_INNOCUOUS2);
19634 }
19635
19636 if( pConfig ) pConfig->pzErrmsg = 0;
19637 if( rc!=SQLITE_OK0 ){
19638 fts5FreeVtab(pTab);
19639 pTab = 0;
19640 }else if( bCreate ){
19641 fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
19642 }
19643 *ppVTab = (sqlite3_vtab*)pTab;
19644 return rc;
19645}
19646
19647/*
19648** The xConnect() and xCreate() methods for the virtual table. All the
19649** work is done in function fts5InitVtab().
19650*/
19651static int fts5ConnectMethod(
19652 sqlite3 *db, /* Database connection */
19653 void *pAux, /* Pointer to tokenizer hash table */
19654 int argc, /* Number of elements in argv array */
19655 const char * const *argv, /* xCreate/xConnect argument array */
19656 sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
19657 char **pzErr /* OUT: sqlite3_malloc'd error message */
19658){
19659 return fts5InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
19660}
19661static int fts5CreateMethod(
19662 sqlite3 *db, /* Database connection */
19663 void *pAux, /* Pointer to tokenizer hash table */
19664 int argc, /* Number of elements in argv array */
19665 const char * const *argv, /* xCreate/xConnect argument array */
19666 sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
19667 char **pzErr /* OUT: sqlite3_malloc'd error message */
19668){
19669 return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
19670}
19671
19672/*
19673** The different query plans.
19674*/
19675#define FTS5_PLAN_MATCH1 1 /* (<tbl> MATCH ?) */
19676#define FTS5_PLAN_SOURCE2 2 /* A source cursor for SORTED_MATCH */
19677#define FTS5_PLAN_SPECIAL3 3 /* An internal query */
19678#define FTS5_PLAN_SORTED_MATCH4 4 /* (<tbl> MATCH ? ORDER BY rank) */
19679#define FTS5_PLAN_SCAN5 5 /* No usable constraint */
19680#define FTS5_PLAN_ROWID6 6 /* (rowid = ?) */
19681
19682/*
19683** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this
19684** extension is currently being used by a version of SQLite too old to
19685** support index-info flags. In that case this function is a no-op.
19686*/
19687static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){
19688#if SQLITE_VERSION_NUMBER3053004>=3008012
19689#ifndef SQLITE_CORE
19690 if( sqlite3_libversion_numbersqlite3_api->libversion_number()>=3008012 )
19691#endif
19692 {
19693 pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE0x00000001;
19694 }
19695#endif
19696}
19697
19698static void fts5SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){
19699#if SQLITE_VERSION_NUMBER3053004>=3008002
19700#ifndef SQLITE_CORE
19701 if( sqlite3_libversion_numbersqlite3_api->libversion_number()>=3008002 )
19702#endif
19703 {
19704 pIdxInfo->estimatedRows = MAX(1, nRow)(((1) > (nRow)) ? (1) : (nRow));
19705 }
19706#endif
19707}
19708
19709static int fts5UsePatternMatch(
19710 Fts5Config *pConfig,
19711 struct sqlite3_index_constraint *p
19712){
19713 assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB )((void) (0));
19714 assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE )((void) (0));
19715 if( pConfig->t.ePattern==FTS5_PATTERN_GLOB66 && p->op==FTS5_PATTERN_GLOB66 ){
19716 return 1;
19717 }
19718 if( pConfig->t.ePattern==FTS5_PATTERN_LIKE65
19719 && (p->op==FTS5_PATTERN_LIKE65 || p->op==FTS5_PATTERN_GLOB66)
19720 ){
19721 return 1;
19722 }
19723 return 0;
19724}
19725
19726/*
19727** Implementation of the xBestIndex method for FTS5 tables. Within the
19728** WHERE constraint, it searches for the following:
19729**
19730** 1. A MATCH constraint against the table column.
19731** 2. A MATCH constraint against the "rank" column.
19732** 3. A MATCH constraint against some other column.
19733** 4. An == constraint against the rowid column.
19734** 5. A < or <= constraint against the rowid column.
19735** 6. A > or >= constraint against the rowid column.
19736**
19737** Within the ORDER BY, the following are supported:
19738**
19739** 5. ORDER BY rank [ASC|DESC]
19740** 6. ORDER BY rowid [ASC|DESC]
19741**
19742** Information for the xFilter call is passed via both the idxNum and
19743** idxStr variables. Specifically, idxNum is a bitmask of the following
19744** flags used to encode the ORDER BY clause:
19745**
19746** FTS5_BI_ORDER_RANK
19747** FTS5_BI_ORDER_ROWID
19748** FTS5_BI_ORDER_DESC
19749**
19750** idxStr is used to encode data from the WHERE clause. For each argument
19751** passed to the xFilter method, the following is appended to idxStr:
19752**
19753** Match against table column: "m"
19754** Match against rank column: "r"
19755** Match against other column: "M<column-number>"
19756** LIKE against other column: "L<column-number>"
19757** GLOB against other column: "G<column-number>"
19758** Equality constraint against the rowid: "="
19759** A < or <= against the rowid: "<"
19760** A > or >= against the rowid: ">"
19761**
19762** This function ensures that there is at most one "r" or "=". And that if
19763** there exists an "=" then there is no "<" or ">".
19764**
19765** If an unusable MATCH operator is present in the WHERE clause, then
19766** SQLITE_CONSTRAINT is returned.
19767**
19768** Costs are assigned as follows:
19769**
19770** a) If a MATCH operator is present, the cost depends on the other
19771** constraints also present. As follows:
19772**
19773** * No other constraints: cost=50000.0
19774** * One rowid range constraint: cost=37500.0
19775** * Both rowid range constraints: cost=30000.0
19776** * An == rowid constraint: cost=25000.0
19777**
19778** b) Otherwise, if there is no MATCH:
19779**
19780** * No other constraints: cost=3000000.0
19781** * One rowid range constraints: cost=2250000.0
19782** * Both rowid range constraint: cost=750000.0
19783** * An == rowid constraint: cost=25.0
19784**
19785** Costs are not modified by the ORDER BY clause.
19786**
19787** The ratios used in case (a) are based on informal results obtained from
19788** the tool/fts5cost.tcl script. The "MATCH and ==" combination has the
19789** cost set quite high because the query may be a prefix query. Unless
19790** there is a prefix index, prefix queries with rowid constraints are much
19791** more expensive than non-prefix queries with rowid constraints.
19792**
19793** The estimated rows returned is set to the cost/40. For simple queries,
19794** experimental results show that cost/4 might be about right. But for
19795** more complex queries that use multiple terms the number of rows might
19796** be far fewer than this. So we compromise and use cost/40.
19797*/
19798static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
19799 Fts5Table *pTab = (Fts5Table*)pVTab;
19800 Fts5Config *pConfig = pTab->pConfig;
19801 const int nCol = pConfig->nCol;
19802 int idxFlags = 0; /* Parameter passed through to xFilter() */
19803 int i;
19804
19805 char *idxStr;
19806 int iIdxStr = 0;
19807 int iCons = 0;
19808
19809 int bSeenEq = 0;
19810 int bSeenGt = 0;
19811 int bSeenLt = 0;
19812 int nSeenMatch = 0;
19813 int bSeenRank = 0;
19814
19815
19816 assert( SQLITE_INDEX_CONSTRAINT_EQ<SQLITE_INDEX_CONSTRAINT_MATCH )((void) (0));
19817 assert( SQLITE_INDEX_CONSTRAINT_GT<SQLITE_INDEX_CONSTRAINT_MATCH )((void) (0));
19818 assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH )((void) (0));
19819 assert( SQLITE_INDEX_CONSTRAINT_GE<SQLITE_INDEX_CONSTRAINT_MATCH )((void) (0));
19820 assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH )((void) (0));
19821
19822 if( pConfig->bLock ){
19823 pTab->base.zErrMsg = sqlite3_mprintfsqlite3_api->mprintf(
19824 "recursively defined fts5 content table"
19825 );
19826 return SQLITE_ERROR1;
19827 }
19828
19829 idxStr = (char*)sqlite3_malloc64sqlite3_api->malloc64((i64)pInfo->nConstraint * 8 + 1);
19830 if( idxStr==0 ) return SQLITE_NOMEM7;
19831 pInfo->idxStr = idxStr;
19832 pInfo->needToFreeIdxStr = 1;
19833
19834 for(i=0; i<pInfo->nConstraint; i++){
19835 struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
19836 int iCol = p->iColumn;
19837 if( p->op==SQLITE_INDEX_CONSTRAINT_MATCH64
19838 || (p->op==SQLITE_INDEX_CONSTRAINT_EQ2 && iCol>=nCol)
19839 ){
19840 /* A MATCH operator or equivalent */
19841 if( p->usable==0 || iCol<0 ){
19842 /* As there exists an unusable MATCH constraint this is an
19843 ** unusable plan. Return SQLITE_CONSTRAINT. */
19844 idxStr[iIdxStr] = 0;
19845 return SQLITE_CONSTRAINT19;
19846 }else{
19847 if( iCol==nCol+1 ){
19848 if( bSeenRank ) continue;
19849 idxStr[iIdxStr++] = 'r';
19850 bSeenRank = 1;
19851 }else{
19852 nSeenMatch++;
19853 idxStr[iIdxStr++] = 'M';
19854 sqlite3_snprintfsqlite3_api->xsnprintf(6, &idxStr[iIdxStr], "%d", iCol);
19855 iIdxStr += (int)strlen(&idxStr[iIdxStr]);
19856 assert( idxStr[iIdxStr]=='\0' )((void) (0));
19857 }
19858 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
19859 pInfo->aConstraintUsage[i].omit = 1;
19860 }
19861 }else if( p->usable ){
19862 if( iCol>=0 && iCol<nCol && fts5UsePatternMatch(pConfig, p) ){
19863 assert( p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB )((void) (0));
19864 idxStr[iIdxStr++] = p->op==FTS5_PATTERN_LIKE65 ? 'L' : 'G';
19865 sqlite3_snprintfsqlite3_api->xsnprintf(6, &idxStr[iIdxStr], "%d", iCol);
19866 idxStr += strlen(&idxStr[iIdxStr]);
19867 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
19868 assert( idxStr[iIdxStr]=='\0' )((void) (0));
19869 nSeenMatch++;
19870 }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ2 && iCol<0 ){
19871 idxStr[iIdxStr++] = '=';
19872 bSeenEq = 1;
19873 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
19874 pInfo->aConstraintUsage[i].omit = 1;
19875 }
19876 }
19877 }
19878
19879 if( bSeenEq==0 ){
19880 for(i=0; i<pInfo->nConstraint; i++){
19881 struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
19882 if( p->iColumn<0 && p->usable ){
19883 int op = p->op;
19884 if( op==SQLITE_INDEX_CONSTRAINT_LT16 || op==SQLITE_INDEX_CONSTRAINT_LE8 ){
19885 if( bSeenLt ) continue;
19886 idxStr[iIdxStr++] = '<';
19887 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
19888 bSeenLt = 1;
19889 }else
19890 if( op==SQLITE_INDEX_CONSTRAINT_GT4 || op==SQLITE_INDEX_CONSTRAINT_GE32 ){
19891 if( bSeenGt ) continue;
19892 idxStr[iIdxStr++] = '>';
19893 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
19894 bSeenGt = 1;
19895 }
19896 }
19897 }
19898 }
19899 idxStr[iIdxStr] = '\0';
19900
19901 /* Set idxFlags flags for the ORDER BY clause
19902 **
19903 ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC".
19904 */
19905 if( pInfo->nOrderBy==1 ){
19906 int iSort = pInfo->aOrderBy[0].iColumn;
19907 if( iSort==(pConfig->nCol+1) && nSeenMatch>0 ){
19908 idxFlags |= FTS5_BI_ORDER_RANK0x0020;
19909 }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){
19910 idxFlags |= FTS5_BI_ORDER_ROWID0x0040;
19911 }
19912 if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID)(((idxFlags) & (0x0020|0x0040))!=0) ){
19913 pInfo->orderByConsumed = 1;
19914 if( pInfo->aOrderBy[0].desc ){
19915 idxFlags |= FTS5_BI_ORDER_DESC0x0080;
19916 }
19917 }
19918 }
19919
19920 /* Calculate the estimated cost based on the flags set in idxFlags. */
19921 if( bSeenEq ){
19922 pInfo->estimatedCost = nSeenMatch ? 25000.0 : 25.0;
19923 fts5SetEstimatedRows(pInfo, 1);
19924 fts5SetUniqueFlag(pInfo);
19925 }else{
19926 i64 nEstRows;
19927 if( nSeenMatch ){
19928 if( bSeenLt && bSeenGt ){
19929 pInfo->estimatedCost = 50000.0;
19930 }else if( bSeenLt || bSeenGt ){
19931 pInfo->estimatedCost = 37500.0;
19932 }else{
19933 pInfo->estimatedCost = 50000.0;
19934 }
19935 nEstRows = (i64)(pInfo->estimatedCost / 40.0);
19936 for(i=1; i<nSeenMatch; i++){
19937 pInfo->estimatedCost *= 2.5;
19938 nEstRows = nEstRows / 2;
19939 }
19940 }else{
19941 if( bSeenLt && bSeenGt ){
19942 pInfo->estimatedCost = 750000.0;
19943 }else if( bSeenLt || bSeenGt ){
19944 pInfo->estimatedCost = 2250000.0;
19945 }else{
19946 pInfo->estimatedCost = 3000000.0;
19947 }
19948 nEstRows = (i64)(pInfo->estimatedCost / 4.0);
19949 }
19950 fts5SetEstimatedRows(pInfo, nEstRows);
19951 }
19952
19953 pInfo->idxNum = idxFlags;
19954 return SQLITE_OK0;
19955}
19956
19957static int fts5NewTransaction(Fts5FullTable *pTab){
19958 Fts5Cursor *pCsr;
19959 for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
19960 if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK0;
19961 }
19962 return sqlite3Fts5StorageReset(pTab->pStorage);
19963}
19964
19965/*
19966** Implementation of xOpen method.
19967*/
19968static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
19969 Fts5FullTable *pTab = (Fts5FullTable*)pVTab;
19970 Fts5Config *pConfig = pTab->p.pConfig;
19971 Fts5Cursor *pCsr = 0; /* New cursor object */
19972 sqlite3_int64 nByte; /* Bytes of space to allocate */
19973 int rc; /* Return code */
19974
19975 rc = fts5NewTransaction(pTab);
19976 if( rc==SQLITE_OK0 ){
19977 nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
19978 pCsr = (Fts5Cursor*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
19979 if( pCsr ){
19980 Fts5Global *pGlobal = pTab->pGlobal;
19981 memset(pCsr, 0, (size_t)nByte);
19982 pCsr->aColumnSize = (int*)&pCsr[1];
19983 pCsr->pNext = pGlobal->pCsr;
19984 pGlobal->pCsr = pCsr;
19985 pCsr->iCsrId = ++pGlobal->iNextId;
19986 }else{
19987 rc = SQLITE_NOMEM7;
19988 }
19989 }
19990 *ppCsr = (sqlite3_vtab_cursor*)pCsr;
19991 return rc;
19992}
19993
19994static int fts5StmtType(Fts5Cursor *pCsr){
19995 if( pCsr->ePlan==FTS5_PLAN_SCAN5 ){
19996 return (pCsr->bDesc) ? FTS5_STMT_SCAN_DESC1 : FTS5_STMT_SCAN_ASC0;
19997 }
19998 return FTS5_STMT_LOOKUP2;
19999}
20000
20001/*
20002** This function is called after the cursor passed as the only argument
20003** is moved to point at a different row. It clears all cached data
20004** specific to the previous row stored by the cursor object.
20005*/
20006static void fts5CsrNewrow(Fts5Cursor *pCsr){
20007 CsrFlagSet(pCsr,((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40))
20008 FTS5CSR_REQUIRE_CONTENT((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40))
20009 | FTS5CSR_REQUIRE_DOCSIZE((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40))
20010 | FTS5CSR_REQUIRE_INST((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40))
20011 | FTS5CSR_REQUIRE_POSLIST((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40))
20012 )((pCsr)->csrflags |= (0x02 | 0x04 | 0x08 | 0x40));
20013}
20014
20015static void fts5FreeCursorComponents(Fts5Cursor *pCsr){
20016 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
20017 Fts5Auxdata *pData;
20018 Fts5Auxdata *pNext;
20019
20020 sqlite3_freesqlite3_api->free(pCsr->aInstIter);
20021 sqlite3_freesqlite3_api->free(pCsr->aInst);
20022 if( pCsr->pStmt ){
20023 int eStmt = fts5StmtType(pCsr);
20024 sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
20025 }
20026 if( pCsr->pSorter ){
20027 Fts5Sorter *pSorter = pCsr->pSorter;
20028 sqlite3_finalizesqlite3_api->finalize(pSorter->pStmt);
20029 sqlite3_freesqlite3_api->free(pSorter);
20030 }
20031
20032 if( pCsr->ePlan!=FTS5_PLAN_SOURCE2 ){
20033 sqlite3Fts5ExprFree(pCsr->pExpr);
20034 }
20035
20036 for(pData=pCsr->pAuxdata; pData; pData=pNext){
20037 pNext = pData->pNext;
20038 if( pData->xDelete ) pData->xDelete(pData->pPtr);
20039 sqlite3_freesqlite3_api->free(pData);
20040 }
20041
20042 sqlite3_finalizesqlite3_api->finalize(pCsr->pRankArgStmt);
20043 sqlite3_freesqlite3_api->free(pCsr->apRankArg);
20044
20045 if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK)((pCsr)->csrflags & (0x10)) ){
20046 sqlite3_freesqlite3_api->free(pCsr->zRank);
20047 sqlite3_freesqlite3_api->free(pCsr->zRankArgs);
20048 }
20049
20050 sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
20051 memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr));
20052}
20053
20054
20055/*
20056** Close the cursor. For additional information see the documentation
20057** on the xClose method of the virtual table interface.
20058*/
20059static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
20060 if( pCursor ){
20061 Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab);
20062 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
20063 Fts5Cursor **pp;
20064
20065 fts5FreeCursorComponents(pCsr);
20066 /* Remove the cursor from the Fts5Global.pCsr list */
20067 for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
20068 *pp = pCsr->pNext;
20069
20070 sqlite3_freesqlite3_api->free(pCsr);
20071 }
20072 return SQLITE_OK0;
20073}
20074
20075static int fts5SorterNext(Fts5Cursor *pCsr){
20076 Fts5Sorter *pSorter = pCsr->pSorter;
20077 int rc;
20078
20079 rc = sqlite3_stepsqlite3_api->step(pSorter->pStmt);
20080 if( rc==SQLITE_DONE101 ){
20081 rc = SQLITE_OK0;
20082 CsrFlagSet(pCsr, FTS5CSR_EOF|FTS5CSR_REQUIRE_CONTENT)((pCsr)->csrflags |= (0x01|0x02));
20083 }else if( rc==SQLITE_ROW100 ){
20084 const u8 *a;
20085 const u8 *aBlob;
20086 int nBlob;
20087 int i;
20088 int iOff = 0;
20089 rc = SQLITE_OK0;
20090
20091 pSorter->iRowid = sqlite3_column_int64sqlite3_api->column_int64(pSorter->pStmt, 0);
20092 nBlob = sqlite3_column_bytessqlite3_api->column_bytes(pSorter->pStmt, 1);
20093 aBlob = a = sqlite3_column_blobsqlite3_api->column_blob(pSorter->pStmt, 1);
20094
20095 /* nBlob==0 in detail=none mode. */
20096 if( nBlob>0 ){
20097 for(i=0; i<(pSorter->nIdx-1); i++){
20098 int iVal;
20099 a += fts5GetVarint32(a, iVal)sqlite3Fts5GetVarint32(a,(u32*)&(iVal));
20100 iOff += iVal;
20101 pSorter->aIdx[i] = iOff;
20102 }
20103 pSorter->aIdx[i] = &aBlob[nBlob] - a;
20104 pSorter->aPoslist = a;
20105 }
20106
20107 fts5CsrNewrow(pCsr);
20108 }
20109
20110 return rc;
20111}
20112
20113
20114/*
20115** Set the FTS5CSR_REQUIRE_RESEEK flag on all FTS5_PLAN_MATCH cursors
20116** open on table pTab.
20117*/
20118static void fts5TripCursors(Fts5FullTable *pTab){
20119 Fts5Cursor *pCsr;
20120 for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
20121 if( pCsr->ePlan==FTS5_PLAN_MATCH1
20122 && pCsr->base.pVtab==(sqlite3_vtab*)pTab
20123 ){
20124 CsrFlagSet(pCsr, FTS5CSR_REQUIRE_RESEEK)((pCsr)->csrflags |= (0x20));
20125 }
20126 }
20127}
20128
20129/*
20130** If the REQUIRE_RESEEK flag is set on the cursor passed as the first
20131** argument, close and reopen all Fts5IndexIter iterators that the cursor
20132** is using. Then attempt to move the cursor to a rowid equal to or laster
20133** (in the cursors sort order - ASC or DESC) than the current rowid.
20134**
20135** If the new rowid is not equal to the old, set output parameter *pbSkip
20136** to 1 before returning. Otherwise, leave it unchanged.
20137**
20138** Return SQLITE_OK if successful or if no reseek was required, or an
20139** error code if an error occurred.
20140*/
20141static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
20142 int rc = SQLITE_OK0;
20143 assert( *pbSkip==0 )((void) (0));
20144 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_RESEEK)((pCsr)->csrflags & (0x20)) ){
20145 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
20146 int bDesc = pCsr->bDesc;
20147 i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
20148
20149 rc = sqlite3Fts5ExprFirst(
20150 pCsr->pExpr, pTab->p.pIndex, iRowid, pCsr->iLastRowid, bDesc
20151 );
20152 if( rc==SQLITE_OK0 && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){
20153 *pbSkip = 1;
20154 }
20155
20156 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK)((pCsr)->csrflags &= ~(0x20));
20157 fts5CsrNewrow(pCsr);
20158 if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
20159 CsrFlagSet(pCsr, FTS5CSR_EOF)((pCsr)->csrflags |= (0x01));
20160 *pbSkip = 1;
20161 }
20162 }
20163 return rc;
20164}
20165
20166
20167/*
20168** Advance the cursor to the next row in the table that matches the
20169** search criteria.
20170**
20171** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
20172** even if we reach end-of-file. The fts5EofMethod() will be called
20173** subsequently to determine whether or not an EOF was hit.
20174*/
20175static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
20176 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
20177 int rc;
20178
20179 assert( (pCsr->ePlan<3)==((void) (0))
20180 (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE)((void) (0))
20181 )((void) (0));
20182 assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) )((void) (0));
20183
20184 /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table,
20185 ** clear any token mappings accumulated at the fts5_index.c level. In
20186 ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH,
20187 ** we need to retain the mappings for the entire query. */
20188 if( pCsr->ePlan==FTS5_PLAN_MATCH1
20189 && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata
20190 ){
20191 sqlite3Fts5ExprClearTokens(pCsr->pExpr);
20192 }
20193
20194 if( pCsr->ePlan<3 ){
20195 int bSkip = 0;
20196 if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
20197 rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
20198 CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr))((pCsr)->csrflags |= (sqlite3Fts5ExprEof(pCsr->pExpr)));
20199 fts5CsrNewrow(pCsr);
20200 }else{
20201 switch( pCsr->ePlan ){
20202 case FTS5_PLAN_SPECIAL3: {
20203 CsrFlagSet(pCsr, FTS5CSR_EOF)((pCsr)->csrflags |= (0x01));
20204 rc = SQLITE_OK0;
20205 break;
20206 }
20207
20208 case FTS5_PLAN_SORTED_MATCH4: {
20209 rc = fts5SorterNext(pCsr);
20210 break;
20211 }
20212
20213 default: {
20214 Fts5Config *pConfig = ((Fts5Table*)pCursor->pVtab)->pConfig;
20215 pConfig->bLock++;
20216 rc = sqlite3_stepsqlite3_api->step(pCsr->pStmt);
20217 pConfig->bLock--;
20218 if( rc!=SQLITE_ROW100 ){
20219 CsrFlagSet(pCsr, FTS5CSR_EOF)((pCsr)->csrflags |= (0x01));
20220 rc = sqlite3_resetsqlite3_api->reset(pCsr->pStmt);
20221 if( rc!=SQLITE_OK0 ){
20222 pCursor->pVtab->zErrMsg = sqlite3_mprintfsqlite3_api->mprintf(
20223 "%s", sqlite3_errmsgsqlite3_api->errmsg(pConfig->db)
20224 );
20225 }
20226 }else{
20227 rc = SQLITE_OK0;
20228 CsrFlagSet(pCsr, FTS5CSR_REQUIRE_DOCSIZE)((pCsr)->csrflags |= (0x04));
20229 }
20230 break;
20231 }
20232 }
20233 }
20234
20235 return rc;
20236}
20237
20238
20239static int fts5PrepareStatement(
20240 sqlite3_stmt **ppStmt,
20241 Fts5Config *pConfig,
20242 const char *zFmt,
20243 ...
20244){
20245 sqlite3_stmt *pRet = 0;
20246 int rc;
20247 char *zSql;
20248 va_list ap;
20249
20250 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
20251 zSql = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
20252 if( zSql==0 ){
20253 rc = SQLITE_NOMEM7;
20254 }else{
20255 rc = sqlite3_prepare_v3sqlite3_api->prepare_v3(pConfig->db, zSql, -1,
20256 SQLITE_PREPARE_PERSISTENT0x01, &pRet, 0);
20257 if( rc!=SQLITE_OK0 ){
20258 sqlite3Fts5ConfigErrmsg(pConfig, "%s", sqlite3_errmsgsqlite3_api->errmsg(pConfig->db));
20259 }
20260 sqlite3_freesqlite3_api->free(zSql);
20261 }
20262
20263 va_end(ap)__builtin_va_end(ap);
20264 *ppStmt = pRet;
20265 return rc;
20266}
20267
20268static int fts5CursorFirstSorted(
20269 Fts5FullTable *pTab,
20270 Fts5Cursor *pCsr,
20271 int bDesc
20272){
20273 Fts5Config *pConfig = pTab->p.pConfig;
20274 Fts5Sorter *pSorter;
20275 int nPhrase;
20276 sqlite3_int64 nByte;
20277 int rc;
20278 const char *zRank = pCsr->zRank;
20279 const char *zRankArgs = pCsr->zRankArgs;
20280
20281 nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
20282 nByte = SZ_FTS5SORTER(nPhrase)(__builtin_offsetof(Fts5Sorter, nIdx)+((nPhrase+2)/2)*sizeof(
i64));
20283 pSorter = (Fts5Sorter*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
20284 if( pSorter==0 ) return SQLITE_NOMEM7;
20285 memset(pSorter, 0, (size_t)nByte);
20286 pSorter->nIdx = nPhrase;
20287
20288 /* TODO: It would be better to have some system for reusing statement
20289 ** handles here, rather than preparing a new one for each query. But that
20290 ** is not possible as SQLite reference counts the virtual table objects.
20291 ** And since the statement required here reads from this very virtual
20292 ** table, saving it creates a circular reference.
20293 **
20294 ** If SQLite a built-in statement cache, this wouldn't be a problem. */
20295 rc = fts5PrepareStatement(&pSorter->pStmt, pConfig,
20296 "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(\"%w\"%s%s) %s",
20297 pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
20298 (zRankArgs ? ", " : ""),
20299 (zRankArgs ? zRankArgs : ""),
20300 bDesc ? "DESC" : "ASC"
20301 );
20302
20303 pCsr->pSorter = pSorter;
20304 if( rc==SQLITE_OK0 ){
20305 assert( pTab->pSortCsr==0 )((void) (0));
20306 pTab->pSortCsr = pCsr;
20307 rc = fts5SorterNext(pCsr);
20308 pTab->pSortCsr = 0;
20309 }
20310
20311 if( rc!=SQLITE_OK0 ){
20312 sqlite3_finalizesqlite3_api->finalize(pSorter->pStmt);
20313 sqlite3_freesqlite3_api->free(pSorter);
20314 pCsr->pSorter = 0;
20315 }
20316
20317 return rc;
20318}
20319
20320static int fts5CursorFirst(Fts5FullTable *pTab, Fts5Cursor *pCsr, int bDesc){
20321 int rc;
20322 Fts5Expr *pExpr = pCsr->pExpr;
20323 rc = sqlite3Fts5ExprFirst(
20324 pExpr, pTab->p.pIndex, pCsr->iFirstRowid, pCsr->iLastRowid, bDesc
20325 );
20326 if( sqlite3Fts5ExprEof(pExpr) ){
20327 CsrFlagSet(pCsr, FTS5CSR_EOF)((pCsr)->csrflags |= (0x01));
20328 }
20329 fts5CsrNewrow(pCsr);
20330 return rc;
20331}
20332
20333/*
20334** Process a "special" query. A special query is identified as one with a
20335** MATCH expression that begins with a '*' character. The remainder of
20336** the text passed to the MATCH operator are used as the special query
20337** parameters.
20338*/
20339static int fts5SpecialMatch(
20340 Fts5FullTable *pTab,
20341 Fts5Cursor *pCsr,
20342 const char *zQuery
20343){
20344 int rc = SQLITE_OK0; /* Return code */
20345 const char *z = zQuery; /* Special query text */
20346 int n; /* Number of bytes in text at z */
20347
20348 while( z[0]==' ' ) z++;
20349 for(n=0; z[n] && z[n]!=' '; n++);
20350
20351 assert( pTab->p.base.zErrMsg==0 )((void) (0));
20352 pCsr->ePlan = FTS5_PLAN_SPECIAL3;
20353
20354 if( n==5 && 0==sqlite3_strnicmpsqlite3_api->strnicmp("reads", z, n) ){
20355 pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->p.pIndex);
20356 }
20357 else if( n==2 && 0==sqlite3_strnicmpsqlite3_api->strnicmp("id", z, n) ){
20358 pCsr->iSpecial = pCsr->iCsrId;
20359 }
20360 else{
20361 /* An unrecognized directive. Return an error message. */
20362 pTab->p.base.zErrMsg = sqlite3_mprintfsqlite3_api->mprintf("unknown special query: %.*s", n, z);
20363 rc = SQLITE_ERROR1;
20364 }
20365
20366 return rc;
20367}
20368
20369/*
20370** Search for an auxiliary function named zName that can be used with table
20371** pTab. If one is found, return a pointer to the corresponding Fts5Auxiliary
20372** structure. Otherwise, if no such function exists, return NULL.
20373*/
20374static Fts5Auxiliary *fts5FindAuxiliary(Fts5FullTable *pTab, const char *zName){
20375 Fts5Auxiliary *pAux;
20376
20377 for(pAux=pTab->pGlobal->pAux; pAux; pAux=pAux->pNext){
20378 if( sqlite3_stricmpsqlite3_api->stricmp(zName, pAux->zFunc)==0 ) return pAux;
20379 }
20380
20381 /* No function of the specified name was found. Return 0. */
20382 return 0;
20383}
20384
20385
20386static int fts5FindRankFunction(Fts5Cursor *pCsr){
20387 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
20388 Fts5Config *pConfig = pTab->p.pConfig;
20389 int rc = SQLITE_OK0;
20390 Fts5Auxiliary *pAux = 0;
20391 const char *zRank = pCsr->zRank;
20392 const char *zRankArgs = pCsr->zRankArgs;
20393
20394 if( zRankArgs ){
20395 char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs);
20396 if( zSql ){
20397 sqlite3_stmt *pStmt = 0;
20398 rc = sqlite3_prepare_v3sqlite3_api->prepare_v3(pConfig->db, zSql, -1,
20399 SQLITE_PREPARE_PERSISTENT0x01, &pStmt, 0);
20400 sqlite3_freesqlite3_api->free(zSql);
20401 assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 )((void) (0));
20402 if( rc==SQLITE_OK0 ){
20403 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pStmt) ){
20404 sqlite3_int64 nByte;
20405 pCsr->nRankArg = sqlite3_column_countsqlite3_api->column_count(pStmt);
20406 nByte = sizeof(sqlite3_value*)*pCsr->nRankArg;
20407 pCsr->apRankArg = (sqlite3_value**)sqlite3Fts5MallocZero(&rc, nByte);
20408 if( rc==SQLITE_OK0 ){
20409 int i;
20410 for(i=0; i<pCsr->nRankArg; i++){
20411 pCsr->apRankArg[i] = sqlite3_column_valuesqlite3_api->column_value(pStmt, i);
20412 }
20413 }
20414 pCsr->pRankArgStmt = pStmt;
20415 }else{
20416 rc = sqlite3_finalizesqlite3_api->finalize(pStmt);
20417 assert( rc!=SQLITE_OK )((void) (0));
20418 }
20419 }
20420 }
20421 }
20422
20423 if( rc==SQLITE_OK0 ){
20424 pAux = fts5FindAuxiliary(pTab, zRank);
20425 if( pAux==0 ){
20426 assert( pTab->p.base.zErrMsg==0 )((void) (0));
20427 pTab->p.base.zErrMsg = sqlite3_mprintfsqlite3_api->mprintf("no such function: %s", zRank);
20428 rc = SQLITE_ERROR1;
20429 }
20430 }
20431
20432 pCsr->pRank = pAux;
20433 return rc;
20434}
20435
20436
20437static int fts5CursorParseRank(
20438 Fts5Config *pConfig,
20439 Fts5Cursor *pCsr,
20440 sqlite3_value *pRank
20441){
20442 int rc = SQLITE_OK0;
20443 if( pRank ){
20444 const char *z = (const char*)sqlite3_value_textsqlite3_api->value_text(pRank);
20445 char *zRank = 0;
20446 char *zRankArgs = 0;
20447
20448 if( z==0 ){
20449 if( sqlite3_value_typesqlite3_api->value_type(pRank)==SQLITE_NULL5 ) rc = SQLITE_ERROR1;
20450 }else{
20451 rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);
20452 }
20453 if( rc==SQLITE_OK0 ){
20454 pCsr->zRank = zRank;
20455 pCsr->zRankArgs = zRankArgs;
20456 CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK)((pCsr)->csrflags |= (0x10));
20457 }else if( rc==SQLITE_ERROR1 ){
20458 pCsr->base.pVtab->zErrMsg = sqlite3_mprintfsqlite3_api->mprintf(
20459 "parse error in rank function: %s", z
20460 );
20461 }
20462 }else{
20463 if( pConfig->zRank ){
20464 pCsr->zRank = (char*)pConfig->zRank;
20465 pCsr->zRankArgs = (char*)pConfig->zRankArgs;
20466 }else{
20467 pCsr->zRank = (char*)FTS5_DEFAULT_RANK"bm25";
20468 pCsr->zRankArgs = 0;
20469 }
20470 }
20471 return rc;
20472}
20473
20474static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){
20475 if( pVal ){
20476 int eType = sqlite3_value_numeric_typesqlite3_api->value_numeric_type(pVal);
20477 if( eType==SQLITE_INTEGER1 ){
20478 return sqlite3_value_int64sqlite3_api->value_int64(pVal);
20479 }
20480 }
20481 return iDefault;
20482}
20483
20484/*
20485** Set the error message on the virtual table passed as the first argument.
20486*/
20487static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){
20488 va_list ap; /* ... printf arguments */
20489 va_start(ap, zFormat)__builtin_va_start(ap, zFormat);
20490 sqlite3_freesqlite3_api->free(p->p.base.zErrMsg);
20491 p->p.base.zErrMsg = sqlite3_vmprintfsqlite3_api->vmprintf(zFormat, ap);
20492 va_end(ap)__builtin_va_end(ap);
20493}
20494
20495/*
20496** Arrange for subsequent calls to sqlite3Fts5Tokenize() to use the locale
20497** specified by pLocale/nLocale. The buffer indicated by pLocale must remain
20498** valid until after the final call to sqlite3Fts5Tokenize() that will use
20499** the locale.
20500*/
20501static void sqlite3Fts5SetLocale(
20502 Fts5Config *pConfig,
20503 const char *zLocale,
20504 int nLocale
20505){
20506 Fts5TokenizerConfig *pT = &pConfig->t;
20507 pT->pLocale = zLocale;
20508 pT->nLocale = nLocale;
20509}
20510
20511/*
20512** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale().
20513*/
20514static void sqlite3Fts5ClearLocale(Fts5Config *pConfig){
20515 sqlite3Fts5SetLocale(pConfig, 0, 0);
20516}
20517
20518/*
20519** Return true if the value passed as the only argument is an
20520** fts5_locale() value.
20521*/
20522static int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){
20523 int ret = 0;
20524 if( sqlite3_value_typesqlite3_api->value_type(pVal)==SQLITE_BLOB4 ){
20525 /* Call sqlite3_value_bytes() after sqlite3_value_blob() in this case.
20526 ** If the blob was created using zeroblob(), then sqlite3_value_blob()
20527 ** may call malloc(). If this malloc() fails, then the values returned
20528 ** by both value_blob() and value_bytes() will be 0. If value_bytes() were
20529 ** called first, then the NULL pointer returned by value_blob() might
20530 ** be dereferenced. */
20531 const u8 *pBlob = sqlite3_value_blobsqlite3_api->value_blob(pVal);
20532 int nBlob = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
20533 if( nBlob>FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ))
20534 && 0==memcmp(pBlob, FTS5_LOCALE_HDR(pConfig)((const u8*)(pConfig->pGlobal->aLocaleHdr)), FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )))
20535 ){
20536 ret = 1;
20537 }
20538 }
20539 return ret;
20540}
20541
20542/*
20543** Value pVal is guaranteed to be an fts5_locale() value, according to
20544** sqlite3Fts5IsLocaleValue(). This function extracts the text and locale
20545** from the value and returns them separately.
20546**
20547** If successful, SQLITE_OK is returned and (*ppText) and (*ppLoc) set
20548** to point to buffers containing the text and locale, as utf-8,
20549** respectively. In this case output parameters (*pnText) and (*pnLoc) are
20550** set to the sizes in bytes of these two buffers.
20551**
20552** Or, if an error occurs, then an SQLite error code is returned. The final
20553** value of the four output parameters is undefined in this case.
20554*/
20555static int sqlite3Fts5DecodeLocaleValue(
20556 sqlite3_value *pVal,
20557 const char **ppText,
20558 int *pnText,
20559 const char **ppLoc,
20560 int *pnLoc
20561){
20562 const char *p = sqlite3_value_blobsqlite3_api->value_blob(pVal);
20563 int n = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
20564 int nLoc = 0;
20565
20566 assert( sqlite3_value_type(pVal)==SQLITE_BLOB )((void) (0));
20567 assert( n>FTS5_LOCALE_HDR_SIZE )((void) (0));
20568
20569 for(nLoc=FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )); p[nLoc]; nLoc++){
20570 if( nLoc==(n-1) ){
20571 return SQLITE_MISMATCH20;
20572 }
20573 }
20574 *ppLoc = &p[FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ))];
20575 *pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ));
20576
20577 *ppText = &p[nLoc+1];
20578 *pnText = n - nLoc - 1;
20579 return SQLITE_OK0;
20580}
20581
20582/*
20583** Argument pVal is the text of a full-text search expression. It may or
20584** may not have been wrapped by fts5_locale(). This function extracts
20585** the text of the expression, and sets output variable (*pzText) to
20586** point to a nul-terminated buffer containing the expression.
20587**
20588** If pVal was an fts5_locale() value, then sqlite3Fts5SetLocale() is called
20589** to set the tokenizer to use the specified locale.
20590**
20591** If output variable (*pbFreeAndReset) is set to true, then the caller
20592** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer
20593** locale, and (b) call sqlite3_free() to free (*pzText).
20594*/
20595static int fts5ExtractExprText(
20596 Fts5Config *pConfig, /* Fts5 configuration */
20597 sqlite3_value *pVal, /* Value to extract expression text from */
20598 char **pzText, /* OUT: nul-terminated buffer of text */
20599 int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */
20600){
20601 int rc = SQLITE_OK0;
20602
20603 if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
20604 const char *pText = 0;
20605 int nText = 0;
20606 const char *pLoc = 0;
20607 int nLoc = 0;
20608 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
20609 *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, pText);
20610 if( rc==SQLITE_OK0 ){
20611 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
20612 }
20613 *pbFreeAndReset = 1;
20614 }else{
20615 *pzText = (char*)sqlite3_value_textsqlite3_api->value_text(pVal);
20616 *pbFreeAndReset = 0;
20617 }
20618
20619 return rc;
20620}
20621
20622
20623/*
20624** This is the xFilter interface for the virtual table. See
20625** the virtual table xFilter method documentation for additional
20626** information.
20627**
20628** There are three possible query strategies:
20629**
20630** 1. Full-text search using a MATCH operator.
20631** 2. A by-rowid lookup.
20632** 3. A full-table scan.
20633*/
20634static int fts5FilterMethod(
20635 sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
20636 int idxNum, /* Strategy index */
20637 const char *idxStr, /* Unused */
20638 int nVal, /* Number of elements in apVal */
20639 sqlite3_value **apVal /* Arguments for the indexing scheme */
20640){
20641 Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab);
20642 Fts5Config *pConfig = pTab->p.pConfig;
20643 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
20644 int rc = SQLITE_OK0; /* Error code */
20645 int bDesc; /* True if ORDER BY [rank|rowid] DESC */
20646 int bOrderByRank; /* True if ORDER BY rank */
20647 sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */
20648 sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */
20649 sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */
20650 sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */
20651 int iCol; /* Column on LHS of MATCH operator */
20652 char **pzErrmsg = pConfig->pzErrmsg;
20653 int bPrefixInsttoken = pConfig->bPrefixInsttoken;
20654 int i;
20655 int iIdxStr = 0;
20656 Fts5Expr *pExpr = 0;
20657
20658 assert( pConfig->bLock==0 )((void) (0));
20659 if( pCsr->ePlan ){
20660 fts5FreeCursorComponents(pCsr);
20661 memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr));
20662 }
20663
20664 assert( pCsr->pStmt==0 )((void) (0));
20665 assert( pCsr->pExpr==0 )((void) (0));
20666 assert( pCsr->csrflags==0 )((void) (0));
20667 assert( pCsr->pRank==0 )((void) (0));
20668 assert( pCsr->zRank==0 )((void) (0));
20669 assert( pCsr->zRankArgs==0 )((void) (0));
20670 assert( pTab->pSortCsr==0 || nVal==0 )((void) (0));
20671
20672 assert( pzErrmsg==0 || pzErrmsg==&pTab->p.base.zErrMsg )((void) (0));
20673 pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
20674
20675 /* Decode the arguments passed through to this function. */
20676 for(i=0; i<nVal; i++){
20677 switch( idxStr[iIdxStr++] ){
20678 case 'r':
20679 pRank = apVal[i];
20680 break;
20681 case 'M': {
20682 char *zText = 0;
20683 int bFreeAndReset = 0;
20684 int bInternal = 0;
20685
20686 rc = fts5ExtractExprText(pConfig, apVal[i], &zText, &bFreeAndReset);
20687 if( rc!=SQLITE_OK0 ) goto filter_out;
20688 if( zText==0 ) zText = "";
20689 if( sqlite3_value_subtypesqlite3_api->value_subtype(apVal[i])==FTS5_INSTTOKEN_SUBTYPE73 ){
20690 pConfig->bPrefixInsttoken = 1;
20691 }
20692
20693 iCol = 0;
20694 do{
20695 iCol = iCol*10 + (idxStr[iIdxStr]-'0');
20696 iIdxStr++;
20697 }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' );
20698
20699 if( zText[0]=='*' ){
20700 /* The user has issued a query of the form "MATCH '*...'". This
20701 ** indicates that the MATCH expression is not a full text query,
20702 ** but a request for an internal parameter. */
20703 rc = fts5SpecialMatch(pTab, pCsr, &zText[1]);
20704 bInternal = 1;
20705 }else{
20706 char **pzErr = &pTab->p.base.zErrMsg;
20707 rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr);
20708 if( rc==SQLITE_OK0 ){
20709 rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr);
20710 pExpr = 0;
20711 }
20712 }
20713
20714 if( bFreeAndReset ){
20715 sqlite3_freesqlite3_api->free(zText);
20716 sqlite3Fts5ClearLocale(pConfig);
20717 }
20718
20719 if( bInternal || rc!=SQLITE_OK0 ) goto filter_out;
20720
20721 break;
20722 }
20723 case 'L':
20724 case 'G': {
20725 int bGlob = (idxStr[iIdxStr-1]=='G');
20726 const char *zText = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[i]);
20727 iCol = 0;
20728 do{
20729 iCol = iCol*10 + (idxStr[iIdxStr]-'0');
20730 iIdxStr++;
20731 }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' );
20732 if( zText ){
20733 rc = sqlite3Fts5ExprPattern(pConfig, bGlob, iCol, zText, &pExpr);
20734 }
20735 if( rc==SQLITE_OK0 ){
20736 rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr);
20737 pExpr = 0;
20738 }
20739 if( rc!=SQLITE_OK0 ) goto filter_out;
20740 break;
20741 }
20742 case '=':
20743 pRowidEq = apVal[i];
20744 break;
20745 case '<':
20746 pRowidLe = apVal[i];
20747 break;
20748 default: assert( idxStr[iIdxStr-1]=='>' )((void) (0));
20749 pRowidGe = apVal[i];
20750 break;
20751 }
20752 }
20753 bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK0x0020) ? 1 : 0);
20754 pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC0x0080) ? 1 : 0);
20755
20756 /* Set the cursor upper and lower rowid limits. Only some strategies
20757 ** actually use them. This is ok, as the xBestIndex() method leaves the
20758 ** sqlite3_index_constraint.omit flag clear for range constraints
20759 ** on the rowid field. */
20760 if( pRowidEq ){
20761 pRowidLe = pRowidGe = pRowidEq;
20762 }
20763 if( bDesc ){
20764 pCsr->iFirstRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)));
20765 pCsr->iLastRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32))));
20766 }else{
20767 pCsr->iLastRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32)));
20768 pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32))));
20769 }
20770
20771 rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
20772 if( rc!=SQLITE_OK0 ) goto filter_out;
20773
20774 if( pTab->pSortCsr ){
20775 /* If pSortCsr is non-NULL, then this call is being made as part of
20776 ** processing for a "... MATCH <expr> ORDER BY rank" query (ePlan is
20777 ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will
20778 ** return results to the user for this query. The current cursor
20779 ** (pCursor) is used to execute the query issued by function
20780 ** fts5CursorFirstSorted() above. */
20781 assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 )((void) (0));
20782 assert( nVal==0 && bOrderByRank==0 && bDesc==0 )((void) (0));
20783 assert( pCsr->iLastRowid==LARGEST_INT64 )((void) (0));
20784 assert( pCsr->iFirstRowid==SMALLEST_INT64 )((void) (0));
20785 if( pTab->pSortCsr->bDesc ){
20786 pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid;
20787 pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid;
20788 }else{
20789 pCsr->iLastRowid = pTab->pSortCsr->iLastRowid;
20790 pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid;
20791 }
20792 pCsr->ePlan = FTS5_PLAN_SOURCE2;
20793 pCsr->pExpr = pTab->pSortCsr->pExpr;
20794 rc = fts5CursorFirst(pTab, pCsr, bDesc);
20795 }else if( pCsr->pExpr ){
20796 assert( rc==SQLITE_OK )((void) (0));
20797 rc = fts5CursorParseRank(pConfig, pCsr, pRank);
20798 if( rc==SQLITE_OK0 ){
20799 if( bOrderByRank ){
20800 pCsr->ePlan = FTS5_PLAN_SORTED_MATCH4;
20801 rc = fts5CursorFirstSorted(pTab, pCsr, bDesc);
20802 }else{
20803 pCsr->ePlan = FTS5_PLAN_MATCH1;
20804 rc = fts5CursorFirst(pTab, pCsr, bDesc);
20805 }
20806 }
20807 }else if( pConfig->zContent==0 ){
20808 fts5SetVtabError(pTab,"%s: table does not support scanning",pConfig->zName);
20809 rc = SQLITE_ERROR1;
20810 }else{
20811 /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup
20812 ** by rowid (ePlan==FTS5_PLAN_ROWID). */
20813 pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID6 : FTS5_PLAN_SCAN5);
20814 rc = sqlite3Fts5StorageStmt(
20815 pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->p.base.zErrMsg
20816 );
20817 if( rc==SQLITE_OK0 ){
20818 if( pRowidEq!=0 ){
20819 assert( pCsr->ePlan==FTS5_PLAN_ROWID )((void) (0));
20820 sqlite3_bind_valuesqlite3_api->bind_value(pCsr->pStmt, 1, pRowidEq);
20821 }else{
20822 sqlite3_bind_int64sqlite3_api->bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid);
20823 sqlite3_bind_int64sqlite3_api->bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid);
20824 }
20825 rc = fts5NextMethod(pCursor);
20826 }
20827 }
20828
20829 filter_out:
20830 sqlite3Fts5ExprFree(pExpr);
20831 pConfig->pzErrmsg = pzErrmsg;
20832 pConfig->bPrefixInsttoken = bPrefixInsttoken;
20833 return rc;
20834}
20835
20836/*
20837** This is the xEof method of the virtual table. SQLite calls this
20838** routine to find out if it has reached the end of a result set.
20839*/
20840static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
20841 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
20842 return (CsrFlagTest(pCsr, FTS5CSR_EOF)((pCsr)->csrflags & (0x01)) ? 1 : 0);
20843}
20844
20845/*
20846** Return the rowid that the cursor currently points to.
20847*/
20848static i64 fts5CursorRowid(Fts5Cursor *pCsr){
20849 assert( pCsr->ePlan==FTS5_PLAN_MATCH((void) (0))
20850 || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH((void) (0))
20851 || pCsr->ePlan==FTS5_PLAN_SOURCE((void) (0))
20852 || pCsr->ePlan==FTS5_PLAN_SCAN((void) (0))
20853 || pCsr->ePlan==FTS5_PLAN_ROWID((void) (0))
20854 )((void) (0));
20855 if( pCsr->pSorter ){
20856 return pCsr->pSorter->iRowid;
20857 }else if( pCsr->ePlan>=FTS5_PLAN_SCAN5 ){
20858 return sqlite3_column_int64sqlite3_api->column_int64(pCsr->pStmt, 0);
20859 }else{
20860 return sqlite3Fts5ExprRowid(pCsr->pExpr);
20861 }
20862}
20863
20864/*
20865** This is the xRowid method. The SQLite core calls this routine to
20866** retrieve the rowid for the current row of the result set. fts5
20867** exposes %_content.rowid as the rowid for the virtual table. The
20868** rowid should be written to *pRowid.
20869*/
20870static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
20871 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
20872 int ePlan = pCsr->ePlan;
20873
20874 assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 )((void) (0));
20875 if( ePlan==FTS5_PLAN_SPECIAL3 ){
20876 *pRowid = 0;
20877 }else{
20878 *pRowid = fts5CursorRowid(pCsr);
20879 }
20880
20881 return SQLITE_OK0;
20882}
20883
20884
20885/*
20886** If the cursor requires seeking (bSeekRequired flag is set), seek it.
20887** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise.
20888**
20889** If argument bErrormsg is true and an error occurs, an error message may
20890** be left in sqlite3_vtab.zErrMsg.
20891*/
20892static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){
20893 int rc = SQLITE_OK0;
20894
20895 /* If the cursor does not yet have a statement handle, obtain one now. */
20896 if( pCsr->pStmt==0 ){
20897 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
20898 int eStmt = fts5StmtType(pCsr);
20899 rc = sqlite3Fts5StorageStmt(
20900 pTab->pStorage, eStmt, &pCsr->pStmt, (bErrormsg?&pTab->p.base.zErrMsg:0)
20901 );
20902 assert( rc!=SQLITE_OK || pTab->p.base.zErrMsg==0 )((void) (0));
20903 assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) )((void) (0));
20904 }
20905
20906 if( rc==SQLITE_OK0 && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT)((pCsr)->csrflags & (0x02)) ){
20907 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
20908 assert( pCsr->pExpr )((void) (0));
20909 sqlite3_resetsqlite3_api->reset(pCsr->pStmt);
20910 sqlite3_bind_int64sqlite3_api->bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr));
20911 pTab->pConfig->bLock++;
20912 rc = sqlite3_stepsqlite3_api->step(pCsr->pStmt);
20913 pTab->pConfig->bLock--;
20914 if( rc==SQLITE_ROW100 ){
20915 rc = SQLITE_OK0;
20916 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT)((pCsr)->csrflags &= ~(0x02));
20917 }else{
20918 rc = sqlite3_resetsqlite3_api->reset(pCsr->pStmt);
20919 if( rc==SQLITE_OK0 ){
20920 rc = FTS5_CORRUPT(11 | (1<<8));
20921 fts5SetVtabError((Fts5FullTable*)pTab,
20922 "fts5: missing row %lld from content table %s",
20923 fts5CursorRowid(pCsr),
20924 pTab->pConfig->zContent
20925 );
20926 }else if( pTab->pConfig->pzErrmsg ){
20927 fts5SetVtabError((Fts5FullTable*)pTab,
20928 "%s", sqlite3_errmsgsqlite3_api->errmsg(pTab->pConfig->db)
20929 );
20930 }
20931 }
20932 }
20933 return rc;
20934}
20935
20936/*
20937** This function is called to handle an FTS INSERT command. In other words,
20938** an INSERT statement of the form:
20939**
20940** INSERT INTO fts(fts) VALUES($pCmd)
20941** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal)
20942**
20943** Argument pVal is the value assigned to column "fts" by the INSERT
20944** statement. This function returns SQLITE_OK if successful, or an SQLite
20945** error code if an error occurs.
20946**
20947** The commands implemented by this function are documented in the "Special
20948** INSERT Directives" section of the documentation. It should be updated if
20949** more commands are added to this function.
20950*/
20951static int fts5SpecialInsert(
20952 Fts5FullTable *pTab, /* Fts5 table object */
20953 const char *zCmd, /* Text inserted into table-name column */
20954 sqlite3_value *pVal /* Value inserted into rank column */
20955){
20956 Fts5Config *pConfig = pTab->p.pConfig;
20957 int rc = SQLITE_OK0;
20958 int bError = 0;
20959 int bLoadConfig = 0;
20960
20961 if( 0==sqlite3_stricmpsqlite3_api->stricmp("delete-all", zCmd) ){
20962 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
20963 fts5SetVtabError(pTab,
20964 "'delete-all' may only be used with a "
20965 "contentless or external content fts5 table"
20966 );
20967 rc = SQLITE_ERROR1;
20968 }else{
20969 rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
20970 }
20971 bLoadConfig = 1;
20972 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("rebuild", zCmd) ){
20973 if( fts5IsContentless(pTab, 1) ){
20974 fts5SetVtabError(pTab,
20975 "'rebuild' may not be used with a contentless fts5 table"
20976 );
20977 rc = SQLITE_ERROR1;
20978 }else{
20979 rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
20980 }
20981 bLoadConfig = 1;
20982 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("optimize", zCmd) ){
20983 rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
20984 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("merge", zCmd) ){
20985 int nMerge = sqlite3_value_intsqlite3_api->value_int(pVal);
20986 rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge);
20987 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("integrity-check", zCmd) ){
20988 int iArg = sqlite3_value_intsqlite3_api->value_int(pVal);
20989 rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, iArg);
20990#ifdef SQLITE_DEBUG
20991 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("prefix-index", zCmd) ){
20992 pConfig->bPrefixIndex = sqlite3_value_intsqlite3_api->value_int(pVal);
20993#endif
20994 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp("flush", zCmd) ){
20995 rc = sqlite3Fts5FlushToDisk(&pTab->p);
20996 }else{
20997 rc = sqlite3Fts5FlushToDisk(&pTab->p);
20998 if( rc==SQLITE_OK0 ){
20999 rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
21000 }
21001 if( rc==SQLITE_OK0 ){
21002 rc = sqlite3Fts5ConfigSetValue(pTab->p.pConfig, zCmd, pVal, &bError);
21003 }
21004 if( rc==SQLITE_OK0 ){
21005 if( bError ){
21006 rc = SQLITE_ERROR1;
21007 }else{
21008 rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, zCmd, pVal, 0);
21009 }
21010 }
21011 }
21012
21013 if( rc==SQLITE_OK0 && bLoadConfig ){
21014 pTab->p.pConfig->iCookie--;
21015 rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
21016 }
21017
21018 return rc;
21019}
21020
21021static int fts5SpecialDelete(
21022 Fts5FullTable *pTab,
21023 sqlite3_value **apVal
21024){
21025 int rc = SQLITE_OK0;
21026 int eType1 = sqlite3_value_typesqlite3_api->value_type(apVal[1]);
21027 if( eType1==SQLITE_INTEGER1 ){
21028 sqlite3_int64 iDel = sqlite3_value_int64sqlite3_api->value_int64(apVal[1]);
21029 rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2], 0);
21030 }
21031 return rc;
21032}
21033
21034static void fts5StorageInsert(
21035 int *pRc,
21036 Fts5FullTable *pTab,
21037 sqlite3_value **apVal,
21038 i64 *piRowid
21039){
21040 int rc = *pRc;
21041 if( rc==SQLITE_OK0 ){
21042 rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, 0, apVal, piRowid);
21043 }
21044 if( rc==SQLITE_OK0 ){
21045 rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid);
21046 }
21047 *pRc = rc;
21048}
21049
21050/*
21051**
21052** This function is called when the user attempts an UPDATE on a contentless
21053** table. Parameter bRowidModified is true if the UPDATE statement modifies
21054** the rowid value. Parameter apVal[] contains the new values for each user
21055** defined column of the fts5 table. pConfig is the configuration object of the
21056** table being updated (guaranteed to be contentless). The contentless_delete=1
21057** and contentless_unindexed=1 options may or may not be set.
21058**
21059** This function returns SQLITE_OK if the UPDATE can go ahead, or an SQLite
21060** error code if it cannot. In this case an error message is also loaded into
21061** pConfig. Output parameter (*pbContent) is set to true if the caller should
21062** update the %_content table only - not the FTS index or any other shadow
21063** table. This occurs when an UPDATE modifies only UNINDEXED columns of the
21064** table.
21065**
21066** An UPDATE may proceed if:
21067**
21068** * The only columns modified are UNINDEXED columns, or
21069**
21070** * The contentless_delete=1 option was specified and all of the indexed
21071** columns (not a subset) have been modified.
21072*/
21073static int fts5ContentlessUpdate(
21074 Fts5Config *pConfig,
21075 sqlite3_value **apVal,
21076 int bRowidModified,
21077 int *pbContent
21078){
21079 int ii;
21080 int bSeenIndex = 0; /* Have seen modified indexed column */
21081 int bSeenIndexNC = 0; /* Have seen unmodified indexed column */
21082 int rc = SQLITE_OK0;
21083
21084 for(ii=0; ii<pConfig->nCol; ii++){
21085 if( pConfig->abUnindexed[ii]==0 ){
21086 if( sqlite3_value_nochangesqlite3_api->value_nochange(apVal[ii]) ){
21087 bSeenIndexNC++;
21088 }else{
21089 bSeenIndex++;
21090 }
21091 }
21092 }
21093
21094 if( bSeenIndex==0 && bRowidModified==0 ){
21095 *pbContent = 1;
21096 }else{
21097 if( bSeenIndexNC || pConfig->bContentlessDelete==0 ){
21098 rc = SQLITE_ERROR1;
21099 sqlite3Fts5ConfigErrmsg(pConfig,
21100 (pConfig->bContentlessDelete ?
21101 "%s a subset of columns on fts5 contentless-delete table: %s" :
21102 "%s contentless fts5 table: %s")
21103 , "cannot UPDATE", pConfig->zName
21104 );
21105 }
21106 }
21107
21108 return rc;
21109}
21110
21111/*
21112** This function is the implementation of the xUpdate callback used by
21113** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
21114** inserted, updated or deleted.
21115**
21116** A delete specifies a single argument - the rowid of the row to remove.
21117**
21118** Update and insert operations pass:
21119**
21120** 1. The "old" rowid, or NULL.
21121** 2. The "new" rowid.
21122** 3. Values for each of the nCol matchable columns.
21123** 4. Values for the two hidden columns (<tablename> and "rank").
21124*/
21125static int fts5UpdateMethod(
21126 sqlite3_vtab *pVtab, /* Virtual table handle */
21127 int nArg, /* Size of argument array */
21128 sqlite3_value **apVal, /* Array of arguments */
21129 sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
21130){
21131 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
21132 Fts5Config *pConfig = pTab->p.pConfig;
21133 int eType0; /* value_type() of apVal[0] */
21134 int rc = SQLITE_OK0; /* Return code */
21135
21136 /* A transaction must be open when this is called. */
21137 assert( pTab->ts.eState==1 || pTab->ts.eState==2 )((void) (0));
21138
21139 assert( pVtab->zErrMsg==0 )((void) (0));
21140 assert( nArg==1 || nArg==(2+pConfig->nCol+2) )((void) (0));
21141 assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER((void) (0))
21142 || sqlite3_value_type(apVal[0])==SQLITE_NULL((void) (0))
21143 )((void) (0));
21144 assert( pTab->p.pConfig->pzErrmsg==0 )((void) (0));
21145 if( pConfig->pgsz==0 ){
21146 rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie);
21147 if( rc!=SQLITE_OK0 ) return rc;
21148 }
21149
21150 pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
21151
21152 /* Put any active cursors into REQUIRE_SEEK state. */
21153 fts5TripCursors(pTab);
21154
21155 eType0 = sqlite3_value_typesqlite3_api->value_type(apVal[0]);
21156 if( eType0==SQLITE_NULL5
21157 && sqlite3_value_typesqlite3_api->value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL5
21158 ){
21159 /* A "special" INSERT op. These are handled separately. */
21160 const char *z = (const char*)sqlite3_value_textsqlite3_api->value_text(apVal[2+pConfig->nCol]);
21161 if( pConfig->eContent!=FTS5_CONTENT_NORMAL0
21162 && 0==sqlite3_stricmpsqlite3_api->stricmp("delete", z)
21163 ){
21164 if( pConfig->bContentlessDelete ){
21165 fts5SetVtabError(pTab,
21166 "'delete' may not be used with a contentless_delete=1 table"
21167 );
21168 rc = SQLITE_ERROR1;
21169 }else{
21170 rc = fts5SpecialDelete(pTab, apVal);
21171 }
21172 }else{
21173 rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
21174 }
21175 }else{
21176 /* A regular INSERT, UPDATE or DELETE statement. The trick here is that
21177 ** any conflict on the rowid value must be detected before any
21178 ** modifications are made to the database file. There are 4 cases:
21179 **
21180 ** 1) DELETE
21181 ** 2) UPDATE (rowid not modified)
21182 ** 3) UPDATE (rowid modified)
21183 ** 4) INSERT
21184 **
21185 ** Cases 3 and 4 may violate the rowid constraint.
21186 */
21187 int eConflict = SQLITE_ABORT4;
21188 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 || pConfig->bContentlessDelete ){
21189 eConflict = sqlite3_vtab_on_conflictsqlite3_api->vtab_on_conflict(pConfig->db);
21190 }
21191
21192 assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL )((void) (0));
21193 assert( nArg!=1 || eType0==SQLITE_INTEGER )((void) (0));
21194
21195 /* DELETE */
21196 if( nArg==1 ){
21197 /* It is only possible to DELETE from a contentless table if the
21198 ** contentless_delete=1 flag is set. */
21199 if( fts5IsContentless(pTab, 1) && pConfig->bContentlessDelete==0 ){
21200 fts5SetVtabError(pTab,
21201 "cannot DELETE from contentless fts5 table: %s", pConfig->zName
21202 );
21203 rc = SQLITE_ERROR1;
21204 }else{
21205 i64 iDel = sqlite3_value_int64sqlite3_api->value_int64(apVal[0]); /* Rowid to delete */
21206 rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0, 0);
21207 }
21208 }
21209
21210 /* INSERT or UPDATE */
21211 else{
21212 int eType1 = sqlite3_value_numeric_typesqlite3_api->value_numeric_type(apVal[1]);
21213
21214 /* It is an error to write an fts5_locale() value to a table without
21215 ** the locale=1 option. */
21216 if( pConfig->bLocale==0 ){
21217 int ii;
21218 for(ii=0; ii<pConfig->nCol; ii++){
21219 sqlite3_value *pVal = apVal[ii+2];
21220 if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
21221 fts5SetVtabError(pTab, "fts5_locale() requires locale=1");
21222 rc = SQLITE_MISMATCH20;
21223 goto update_out;
21224 }
21225 }
21226 }
21227
21228 if( eType0!=SQLITE_INTEGER1 ){
21229 /* An INSERT statement. If the conflict-mode is REPLACE, first remove
21230 ** the current entry (if any). */
21231 if( eConflict==SQLITE_REPLACE5 && eType1==SQLITE_INTEGER1 ){
21232 i64 iNew = sqlite3_value_int64sqlite3_api->value_int64(apVal[1]); /* Rowid to delete */
21233 rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0);
21234 }
21235 fts5StorageInsert(&rc, pTab, apVal, pRowid);
21236 }
21237
21238 /* UPDATE */
21239 else{
21240 Fts5Storage *pStorage = pTab->pStorage;
21241 i64 iOld = sqlite3_value_int64sqlite3_api->value_int64(apVal[0]); /* Old rowid */
21242 i64 iNew = sqlite3_value_int64sqlite3_api->value_int64(apVal[1]); /* New rowid */
21243 int bContent = 0; /* Content only update */
21244
21245 /* If this is a contentless table (including contentless_unindexed=1
21246 ** tables), check if the UPDATE may proceed. */
21247 if( fts5IsContentless(pTab, 1) ){
21248 rc = fts5ContentlessUpdate(pConfig, &apVal[2], iOld!=iNew, &bContent);
21249 if( rc!=SQLITE_OK0 ) goto update_out;
21250 }
21251
21252 if( eType1!=SQLITE_INTEGER1 ){
21253 rc = SQLITE_MISMATCH20;
21254 }else if( iOld!=iNew ){
21255 assert( bContent==0 )((void) (0));
21256 if( eConflict==SQLITE_REPLACE5 ){
21257 rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1);
21258 if( rc==SQLITE_OK0 ){
21259 rc = sqlite3Fts5StorageDelete(pStorage, iNew, 0, 0);
21260 }
21261 fts5StorageInsert(&rc, pTab, apVal, pRowid);
21262 }else{
21263 rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld);
21264 if( rc==SQLITE_OK0 ){
21265 rc = sqlite3Fts5StorageContentInsert(pStorage, 0, apVal, pRowid);
21266 }
21267 if( rc==SQLITE_OK0 ){
21268 rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 0);
21269 }
21270 if( rc==SQLITE_OK0 ){
21271 rc = sqlite3Fts5StorageIndexInsert(pStorage, apVal, *pRowid);
21272 }
21273 }
21274 }else if( bContent ){
21275 /* This occurs when an UPDATE on a contentless table affects *only*
21276 ** UNINDEXED columns. This is a no-op for contentless_unindexed=0
21277 ** tables, or a write to the %_content table only for =1 tables. */
21278 assert( fts5IsContentless(pTab, 1) )((void) (0));
21279 rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld);
21280 if( rc==SQLITE_OK0 ){
21281 rc = sqlite3Fts5StorageContentInsert(pStorage, 1, apVal, pRowid);
21282 }
21283 }else{
21284 rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1);
21285 fts5StorageInsert(&rc, pTab, apVal, pRowid);
21286 }
21287 sqlite3Fts5StorageReleaseDeleteRow(pStorage);
21288 }
21289 }
21290 }
21291
21292 update_out:
21293 sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
21294 pTab->p.pConfig->pzErrmsg = 0;
21295 return rc;
21296}
21297
21298/*
21299** Implementation of xSync() method.
21300*/
21301static int fts5SyncMethod(sqlite3_vtab *pVtab){
21302 int rc;
21303 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
21304 fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
21305 pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
21306 rc = sqlite3Fts5FlushToDisk(&pTab->p);
21307 pTab->p.pConfig->pzErrmsg = 0;
21308 return rc;
21309}
21310
21311/*
21312** Implementation of xBegin() method.
21313*/
21314static int fts5BeginMethod(sqlite3_vtab *pVtab){
21315 int rc = fts5NewTransaction((Fts5FullTable*)pVtab);
21316 if( rc==SQLITE_OK0 ){
21317 fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0);
21318 }
21319 return rc;
21320}
21321
21322/*
21323** Implementation of xCommit() method. This is a no-op. The contents of
21324** the pending-terms hash-table have already been flushed into the database
21325** by fts5SyncMethod().
21326*/
21327static int fts5CommitMethod(sqlite3_vtab *pVtab){
21328 UNUSED_PARAM(pVtab)(void)(pVtab); /* Call below is a no-op for NDEBUG builds */
21329 fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_COMMIT, 0);
21330 return SQLITE_OK0;
21331}
21332
21333/*
21334** Implementation of xRollback(). Discard the contents of the pending-terms
21335** hash-table. Any changes made to the database are reverted by SQLite.
21336*/
21337static int fts5RollbackMethod(sqlite3_vtab *pVtab){
21338 int rc;
21339 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
21340 fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0);
21341 rc = sqlite3Fts5StorageRollback(pTab->pStorage);
21342 pTab->p.pConfig->pgsz = 0;
21343 return rc;
21344}
21345
21346static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*);
21347
21348static void *fts5ApiUserData(Fts5Context *pCtx){
21349 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21350 return pCsr->pAux->pUserData;
21351}
21352
21353static int fts5ApiColumnCount(Fts5Context *pCtx){
21354 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21355 return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol;
21356}
21357
21358static int fts5ApiColumnTotalSize(
21359 Fts5Context *pCtx,
21360 int iCol,
21361 sqlite3_int64 *pnToken
21362){
21363 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21364 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
21365 return sqlite3Fts5StorageSize(pTab->pStorage, iCol, pnToken);
21366}
21367
21368static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){
21369 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21370 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
21371 return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow);
21372}
21373
21374/*
21375** Implementation of xTokenize_v2() API.
21376*/
21377static int fts5ApiTokenize_v2(
21378 Fts5Context *pCtx,
21379 const char *pText, int nText,
21380 const char *pLoc, int nLoc,
21381 void *pUserData,
21382 int (*xToken)(void*, int, const char*, int, int, int)
21383){
21384 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21385 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
21386 int rc = SQLITE_OK0;
21387
21388 sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc);
21389 rc = sqlite3Fts5Tokenize(pTab->pConfig,
21390 FTS5_TOKENIZE_AUX0x0008, pText, nText, pUserData, xToken
21391 );
21392 sqlite3Fts5SetLocale(pTab->pConfig, 0, 0);
21393
21394 return rc;
21395}
21396
21397/*
21398** Implementation of xTokenize() API. This is just xTokenize_v2() with NULL/0
21399** passed as the locale.
21400*/
21401static int fts5ApiTokenize(
21402 Fts5Context *pCtx,
21403 const char *pText, int nText,
21404 void *pUserData,
21405 int (*xToken)(void*, int, const char*, int, int, int)
21406){
21407 return fts5ApiTokenize_v2(pCtx, pText, nText, 0, 0, pUserData, xToken);
21408}
21409
21410static int fts5ApiPhraseCount(Fts5Context *pCtx){
21411 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21412 return sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
21413}
21414
21415static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
21416 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21417 return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
21418}
21419
21420/*
21421** Argument pStmt is an SQL statement of the type used by Fts5Cursor. This
21422** function extracts the text value of column iCol of the current row.
21423** Additionally, if there is an associated locale, it invokes
21424** sqlite3Fts5SetLocale() to configure the tokenizer. In all cases the caller
21425** should invoke sqlite3Fts5ClearLocale() to clear the locale at some point
21426** after this function returns.
21427**
21428** If successful, (*ppText) is set to point to a buffer containing the text
21429** value as utf-8 and SQLITE_OK returned. (*pnText) is set to the size of that
21430** buffer in bytes. It is not guaranteed to be nul-terminated. If an error
21431** occurs, an SQLite error code is returned. The final values of the two
21432** output parameters are undefined in this case.
21433*/
21434static int fts5TextFromStmt(
21435 Fts5Config *pConfig,
21436 sqlite3_stmt *pStmt,
21437 int iCol,
21438 const char **ppText,
21439 int *pnText
21440){
21441 sqlite3_value *pVal = sqlite3_column_valuesqlite3_api->column_value(pStmt, iCol+1);
21442 const char *pLoc = 0;
21443 int nLoc = 0;
21444 int rc = SQLITE_OK0;
21445
21446 if( pConfig->bLocale
21447 && pConfig->eContent==FTS5_CONTENT_EXTERNAL2
21448 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
21449 ){
21450 rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc);
21451 }else{
21452 *ppText = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
21453 *pnText = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
21454 if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
21455 pLoc = (const char*)sqlite3_column_textsqlite3_api->column_text(pStmt, iCol+1+pConfig->nCol);
21456 nLoc = sqlite3_column_bytessqlite3_api->column_bytes(pStmt, iCol+1+pConfig->nCol);
21457 }
21458 }
21459 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
21460 return rc;
21461}
21462
21463static int fts5ApiColumnText(
21464 Fts5Context *pCtx,
21465 int iCol,
21466 const char **pz,
21467 int *pn
21468){
21469 int rc = SQLITE_OK0;
21470 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21471 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
21472
21473 assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL )((void) (0));
21474 if( iCol<0 || iCol>=pTab->pConfig->nCol ){
21475 rc = SQLITE_RANGE25;
21476 }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab), 0) ){
21477 *pz = 0;
21478 *pn = 0;
21479 }else{
21480 rc = fts5SeekCursor(pCsr, 0);
21481 if( rc==SQLITE_OK0 ){
21482 rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn);
21483 sqlite3Fts5ClearLocale(pTab->pConfig);
21484 }
21485 }
21486 return rc;
21487}
21488
21489/*
21490** This is called by various API functions - xInst, xPhraseFirst,
21491** xPhraseFirstColumn etc. - to obtain the position list for phrase iPhrase
21492** of the current row. This function works for both detail=full tables (in
21493** which case the position-list was read from the fts index) or for other
21494** detail= modes if the row content is available.
21495*/
21496static int fts5CsrPoslist(
21497 Fts5Cursor *pCsr, /* Fts5 cursor object */
21498 int iPhrase, /* Phrase to find position list for */
21499 const u8 **pa, /* OUT: Pointer to position list buffer */
21500 int *pn /* OUT: Size of (*pa) in bytes */
21501){
21502 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
21503 int rc = SQLITE_OK0;
21504 int bLive = (pCsr->pSorter==0);
21505
21506 if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){
21507 rc = SQLITE_RANGE25;
21508 }else if( pConfig->eDetail!=FTS5_DETAIL_FULL0
21509 && fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1)
21510 ){
21511 *pa = 0;
21512 *pn = 0;
21513 return SQLITE_OK0;
21514 }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST)((pCsr)->csrflags & (0x40)) ){
21515 if( pConfig->eDetail!=FTS5_DETAIL_FULL0 ){
21516 Fts5PoslistPopulator *aPopulator;
21517 int i;
21518
21519 aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive);
21520 if( aPopulator==0 ) rc = SQLITE_NOMEM7;
21521 if( rc==SQLITE_OK0 ){
21522 rc = fts5SeekCursor(pCsr, 0);
21523 }
21524 for(i=0; i<pConfig->nCol && rc==SQLITE_OK0; i++){
21525 const char *z = 0;
21526 int n = 0;
21527 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
21528 if( rc==SQLITE_OK0 ){
21529 rc = sqlite3Fts5ExprPopulatePoslists(
21530 pConfig, pCsr->pExpr, aPopulator, i, z, n
21531 );
21532 }
21533 sqlite3Fts5ClearLocale(pConfig);
21534 }
21535 sqlite3_freesqlite3_api->free(aPopulator);
21536
21537 if( pCsr->pSorter ){
21538 sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid);
21539 }
21540 }
21541 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST)((pCsr)->csrflags &= ~(0x40));
21542 }
21543
21544 if( rc==SQLITE_OK0 ){
21545 if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL0 ){
21546 Fts5Sorter *pSorter = pCsr->pSorter;
21547 int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
21548 *pn = pSorter->aIdx[iPhrase] - i1;
21549 *pa = &pSorter->aPoslist[i1];
21550 }else{
21551 *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
21552 }
21553 }else{
21554 *pa = 0;
21555 *pn = 0;
21556 }
21557
21558 return rc;
21559}
21560
21561/*
21562** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated
21563** correctly for the current view. Return SQLITE_OK if successful, or an
21564** SQLite error code otherwise.
21565*/
21566static int fts5CacheInstArray(Fts5Cursor *pCsr){
21567 int rc = SQLITE_OK0;
21568 Fts5PoslistReader *aIter; /* One iterator for each phrase */
21569 int nIter; /* Number of iterators/phrases */
21570 int nCol = ((Fts5Table*)pCsr->base.pVtab)->pConfig->nCol;
21571
21572 nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
21573 if( pCsr->aInstIter==0 ){
21574 sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nIter;
21575 pCsr->aInstIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte);
21576 }
21577 aIter = pCsr->aInstIter;
21578
21579 if( aIter ){
21580 int nInst = 0; /* Number instances seen so far */
21581 int i;
21582
21583 /* Initialize all iterators */
21584 for(i=0; i<nIter && rc==SQLITE_OK0; i++){
21585 const u8 *a;
21586 int n;
21587 rc = fts5CsrPoslist(pCsr, i, &a, &n);
21588 if( rc==SQLITE_OK0 ){
21589 sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
21590 }
21591 }
21592
21593 if( rc==SQLITE_OK0 ){
21594 while( 1 ){
21595 int *aInst;
21596 int iBest = -1;
21597 for(i=0; i<nIter; i++){
21598 if( (aIter[i].bEof==0)
21599 && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos)
21600 ){
21601 iBest = i;
21602 }
21603 }
21604 if( iBest<0 ) break;
21605
21606 nInst++;
21607 if( nInst>=pCsr->nInstAlloc ){
21608 int nNewSize = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
21609 aInst = (int*)sqlite3_realloc64sqlite3_api->realloc64(
21610 pCsr->aInst, nNewSize*sizeof(int)*3
21611 );
21612 if( aInst ){
21613 pCsr->aInst = aInst;
21614 pCsr->nInstAlloc = nNewSize;
21615 }else{
21616 nInst--;
21617 rc = SQLITE_NOMEM7;
21618 break;
21619 }
21620 }
21621
21622 aInst = &pCsr->aInst[3 * (nInst-1)];
21623 aInst[0] = iBest;
21624 aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos)(int)((aIter[iBest].iPos >> 32) & 0x7FFFFFFF);
21625 aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos)(int)(aIter[iBest].iPos & 0x7FFFFFFF);
21626 assert( aInst[1]>=0 )((void) (0));
21627 if( aInst[1]>=nCol ){
21628 rc = FTS5_CORRUPT(11 | (1<<8));
21629 break;
21630 }
21631 sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
21632 }
21633 }
21634
21635 pCsr->nInstCount = nInst;
21636 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST)((pCsr)->csrflags &= ~(0x08));
21637 }
21638 return rc;
21639}
21640
21641static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){
21642 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21643 int rc = SQLITE_OK0;
21644 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)((pCsr)->csrflags & (0x08))==0
21645 || SQLITE_OK0==(rc = fts5CacheInstArray(pCsr)) ){
21646 *pnInst = pCsr->nInstCount;
21647 }
21648 return rc;
21649}
21650
21651static int fts5ApiInst(
21652 Fts5Context *pCtx,
21653 int iIdx,
21654 int *piPhrase,
21655 int *piCol,
21656 int *piOff
21657){
21658 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21659 int rc = SQLITE_OK0;
21660 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)((pCsr)->csrflags & (0x08))==0
21661 || SQLITE_OK0==(rc = fts5CacheInstArray(pCsr))
21662 ){
21663 if( iIdx<0 || iIdx>=pCsr->nInstCount ){
21664 rc = SQLITE_RANGE25;
21665 }else{
21666 *piPhrase = pCsr->aInst[iIdx*3];
21667 *piCol = pCsr->aInst[iIdx*3 + 1];
21668 *piOff = pCsr->aInst[iIdx*3 + 2];
21669 }
21670 }
21671 return rc;
21672}
21673
21674static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
21675 return fts5CursorRowid((Fts5Cursor*)pCtx);
21676}
21677
21678static int fts5ColumnSizeCb(
21679 void *pContext, /* Pointer to int */
21680 int tflags,
21681 const char *pUnused, /* Buffer containing token */
21682 int nUnused, /* Size of token in bytes */
21683 int iUnused1, /* Start offset of token */
21684 int iUnused2 /* End offset of token */
21685){
21686 int *pCnt = (int*)pContext;
21687 UNUSED_PARAM2(pUnused, nUnused)(void)(pUnused), (void)(nUnused);
21688 UNUSED_PARAM2(iUnused1, iUnused2)(void)(iUnused1), (void)(iUnused2);
21689 if( (tflags & FTS5_TOKEN_COLOCATED0x0001)==0 ){
21690 (*pCnt)++;
21691 }
21692 return SQLITE_OK0;
21693}
21694
21695static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
21696 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21697 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
21698 Fts5Config *pConfig = pTab->p.pConfig;
21699 int rc = SQLITE_OK0;
21700
21701 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE)((pCsr)->csrflags & (0x04)) ){
21702 if( pConfig->bColumnsize ){
21703 i64 iRowid = fts5CursorRowid(pCsr);
21704 rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
21705 }else if( !pConfig->zContent || pConfig->eContent==FTS5_CONTENT_UNINDEXED3 ){
21706 int i;
21707 for(i=0; i<pConfig->nCol; i++){
21708 if( pConfig->abUnindexed[i]==0 ){
21709 pCsr->aColumnSize[i] = -1;
21710 }
21711 }
21712 }else{
21713 int i;
21714 rc = fts5SeekCursor(pCsr, 0);
21715 for(i=0; rc==SQLITE_OK0 && i<pConfig->nCol; i++){
21716 if( pConfig->abUnindexed[i]==0 ){
21717 const char *z = 0;
21718 int n = 0;
21719 pCsr->aColumnSize[i] = 0;
21720 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
21721 if( rc==SQLITE_OK0 ){
21722 rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX0x0008,
21723 z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb
21724 );
21725 }
21726 sqlite3Fts5ClearLocale(pConfig);
21727 }
21728 }
21729 }
21730 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE)((pCsr)->csrflags &= ~(0x04));
21731 }
21732 if( iCol<0 ){
21733 int i;
21734 *pnToken = 0;
21735 for(i=0; i<pConfig->nCol; i++){
21736 *pnToken += pCsr->aColumnSize[i];
21737 }
21738 }else if( iCol<pConfig->nCol ){
21739 *pnToken = pCsr->aColumnSize[iCol];
21740 }else{
21741 *pnToken = 0;
21742 rc = SQLITE_RANGE25;
21743 }
21744 return rc;
21745}
21746
21747/*
21748** Implementation of the xSetAuxdata() method.
21749*/
21750static int fts5ApiSetAuxdata(
21751 Fts5Context *pCtx, /* Fts5 context */
21752 void *pPtr, /* Pointer to save as auxdata */
21753 void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */
21754){
21755 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21756 Fts5Auxdata *pData;
21757
21758 /* Search through the cursors list of Fts5Auxdata objects for one that
21759 ** corresponds to the currently executing auxiliary function. */
21760 for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){
21761 if( pData->pAux==pCsr->pAux ) break;
21762 }
21763
21764 if( pData ){
21765 if( pData->xDelete ){
21766 pData->xDelete(pData->pPtr);
21767 }
21768 }else{
21769 int rc = SQLITE_OK0;
21770 pData = (Fts5Auxdata*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Auxdata));
21771 if( pData==0 ){
21772 if( xDelete ) xDelete(pPtr);
21773 return rc;
21774 }
21775 pData->pAux = pCsr->pAux;
21776 pData->pNext = pCsr->pAuxdata;
21777 pCsr->pAuxdata = pData;
21778 }
21779
21780 pData->xDelete = xDelete;
21781 pData->pPtr = pPtr;
21782 return SQLITE_OK0;
21783}
21784
21785static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){
21786 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21787 Fts5Auxdata *pData;
21788 void *pRet = 0;
21789
21790 for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){
21791 if( pData->pAux==pCsr->pAux ) break;
21792 }
21793
21794 if( pData ){
21795 pRet = pData->pPtr;
21796 if( bClear ){
21797 pData->pPtr = 0;
21798 pData->xDelete = 0;
21799 }
21800 }
21801
21802 return pRet;
21803}
21804
21805static void fts5ApiPhraseNext(
21806 Fts5Context *pCtx,
21807 Fts5PhraseIter *pIter,
21808 int *piCol, int *piOff
21809){
21810 if( pIter->a>=pIter->b ){
21811 *piCol = -1;
21812 *piOff = -1;
21813 }else{
21814 int iVal;
21815 pIter->a += fts5GetVarint32(pIter->a, iVal)sqlite3Fts5GetVarint32(pIter->a,(u32*)&(iVal));
21816 if( iVal==1 ){
21817 /* Avoid returning a (*piCol) value that is too large for the table,
21818 ** even if the position-list is corrupt. The caller might not be
21819 ** expecting it. */
21820 int nCol = ((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab))->pConfig->nCol;
21821 pIter->a += fts5GetVarint32(pIter->a, iVal)sqlite3Fts5GetVarint32(pIter->a,(u32*)&(iVal));
21822 *piCol = (iVal>=nCol ? nCol-1 : iVal);
21823 *piOff = 0;
21824 pIter->a += fts5GetVarint32(pIter->a, iVal)sqlite3Fts5GetVarint32(pIter->a,(u32*)&(iVal));
21825 }
21826 *piOff += (iVal-2);
21827 }
21828}
21829
21830static int fts5ApiPhraseFirst(
21831 Fts5Context *pCtx,
21832 int iPhrase,
21833 Fts5PhraseIter *pIter,
21834 int *piCol, int *piOff
21835){
21836 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21837 int n;
21838 int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
21839 if( rc==SQLITE_OK0 ){
21840 assert( pIter->a || n==0 )((void) (0));
21841 pIter->b = (pIter->a ? &pIter->a[n] : 0);
21842 *piCol = 0;
21843 *piOff = 0;
21844 fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
21845 }
21846 return rc;
21847}
21848
21849static void fts5ApiPhraseNextColumn(
21850 Fts5Context *pCtx,
21851 Fts5PhraseIter *pIter,
21852 int *piCol
21853){
21854 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21855 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
21856
21857 if( pConfig->eDetail==FTS5_DETAIL_COLUMNS2 ){
21858 if( pIter->a>=pIter->b ){
21859 *piCol = -1;
21860 }else{
21861 int iIncr;
21862 pIter->a += fts5GetVarint32(&pIter->a[0], iIncr)sqlite3Fts5GetVarint32(&pIter->a[0],(u32*)&(iIncr)
);
21863 *piCol += (iIncr-2);
21864 }
21865 }else{
21866 while( 1 ){
21867 int dummy;
21868 if( pIter->a>=pIter->b ){
21869 *piCol = -1;
21870 return;
21871 }
21872 if( pIter->a[0]==0x01 ) break;
21873 pIter->a += fts5GetVarint32(pIter->a, dummy)sqlite3Fts5GetVarint32(pIter->a,(u32*)&(dummy));
21874 }
21875 pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol)sqlite3Fts5GetVarint32(&pIter->a[1],(u32*)&(*piCol
));
21876 }
21877}
21878
21879static int fts5ApiPhraseFirstColumn(
21880 Fts5Context *pCtx,
21881 int iPhrase,
21882 Fts5PhraseIter *pIter,
21883 int *piCol
21884){
21885 int rc = SQLITE_OK0;
21886 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21887 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
21888
21889 if( pConfig->eDetail==FTS5_DETAIL_COLUMNS2 ){
21890 Fts5Sorter *pSorter = pCsr->pSorter;
21891 if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){
21892 rc = SQLITE_RANGE25;
21893 }else{
21894 int n;
21895 if( pSorter ){
21896 int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
21897 n = pSorter->aIdx[iPhrase] - i1;
21898 pIter->a = &pSorter->aPoslist[i1];
21899 }else{
21900 rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
21901 }
21902 if( rc==SQLITE_OK0 ){
21903 assert( pIter->a || n==0 )((void) (0));
21904 pIter->b = (pIter->a ? &pIter->a[n] : 0);
21905 *piCol = 0;
21906 fts5ApiPhraseNextColumn(pCtx, pIter, piCol);
21907 }
21908 }
21909 }else{
21910 int n;
21911 rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
21912 if( rc==SQLITE_OK0 ){
21913 assert( pIter->a || n==0 )((void) (0));
21914 pIter->b = (pIter->a ? &pIter->a[n] : 0);
21915 if( n<=0 ){
21916 *piCol = -1;
21917 }else if( pIter->a[0]==0x01 ){
21918 pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol)sqlite3Fts5GetVarint32(&pIter->a[1],(u32*)&(*piCol
));
21919 }else{
21920 *piCol = 0;
21921 }
21922 }
21923 }
21924
21925 return rc;
21926}
21927
21928/*
21929** xQueryToken() API implemenetation.
21930*/
21931static int fts5ApiQueryToken(
21932 Fts5Context* pCtx,
21933 int iPhrase,
21934 int iToken,
21935 const char **ppOut,
21936 int *pnOut
21937){
21938 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21939 return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut);
21940}
21941
21942/*
21943** xInstToken() API implemenetation.
21944*/
21945static int fts5ApiInstToken(
21946 Fts5Context *pCtx,
21947 int iIdx,
21948 int iToken,
21949 const char **ppOut, int *pnOut
21950){
21951 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21952 int rc = SQLITE_OK0;
21953 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)((pCsr)->csrflags & (0x08))==0
21954 || SQLITE_OK0==(rc = fts5CacheInstArray(pCsr))
21955 ){
21956 if( iIdx<0 || iIdx>=pCsr->nInstCount ){
21957 rc = SQLITE_RANGE25;
21958 }else{
21959 int iPhrase = pCsr->aInst[iIdx*3];
21960 int iCol = pCsr->aInst[iIdx*3 + 1];
21961 int iOff = pCsr->aInst[iIdx*3 + 2];
21962 i64 iRowid = fts5CursorRowid(pCsr);
21963 rc = sqlite3Fts5ExprInstToken(
21964 pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut
21965 );
21966 }
21967 }
21968 return rc;
21969}
21970
21971
21972static int fts5ApiQueryPhrase(Fts5Context*, int, void*,
21973 int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
21974);
21975
21976/*
21977** The xColumnLocale() API.
21978*/
21979static int fts5ApiColumnLocale(
21980 Fts5Context *pCtx,
21981 int iCol,
21982 const char **pzLocale,
21983 int *pnLocale
21984){
21985 int rc = SQLITE_OK0;
21986 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
21987 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
21988
21989 *pzLocale = 0;
21990 *pnLocale = 0;
21991
21992 assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL )((void) (0));
21993 if( iCol<0 || iCol>=pConfig->nCol ){
21994 rc = SQLITE_RANGE25;
21995 }else if(
21996 pConfig->abUnindexed[iCol]==0
21997 && 0==fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1)
21998 && pConfig->bLocale
21999 ){
22000 rc = fts5SeekCursor(pCsr, 0);
22001 if( rc==SQLITE_OK0 ){
22002 const char *zDummy = 0;
22003 int nDummy = 0;
22004 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &zDummy, &nDummy);
22005 if( rc==SQLITE_OK0 ){
22006 *pzLocale = pConfig->t.pLocale;
22007 *pnLocale = pConfig->t.nLocale;
22008 }
22009 sqlite3Fts5ClearLocale(pConfig);
22010 }
22011 }
22012
22013 return rc;
22014}
22015
22016static const Fts5ExtensionApi sFts5Api = {
22017 4, /* iVersion */
22018 fts5ApiUserData,
22019 fts5ApiColumnCount,
22020 fts5ApiRowCount,
22021 fts5ApiColumnTotalSize,
22022 fts5ApiTokenize,
22023 fts5ApiPhraseCount,
22024 fts5ApiPhraseSize,
22025 fts5ApiInstCount,
22026 fts5ApiInst,
22027 fts5ApiRowid,
22028 fts5ApiColumnText,
22029 fts5ApiColumnSize,
22030 fts5ApiQueryPhrase,
22031 fts5ApiSetAuxdata,
22032 fts5ApiGetAuxdata,
22033 fts5ApiPhraseFirst,
22034 fts5ApiPhraseNext,
22035 fts5ApiPhraseFirstColumn,
22036 fts5ApiPhraseNextColumn,
22037 fts5ApiQueryToken,
22038 fts5ApiInstToken,
22039 fts5ApiColumnLocale,
22040 fts5ApiTokenize_v2
22041};
22042
22043/*
22044** Implementation of API function xQueryPhrase().
22045*/
22046static int fts5ApiQueryPhrase(
22047 Fts5Context *pCtx,
22048 int iPhrase,
22049 void *pUserData,
22050 int(*xCallback)(const Fts5ExtensionApi*, Fts5Context*, void*)
22051){
22052 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
22053 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
22054 int rc;
22055 Fts5Cursor *pNew = 0;
22056
22057 rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew);
22058 if( rc==SQLITE_OK0 ){
22059 pNew->ePlan = FTS5_PLAN_MATCH1;
22060 pNew->iFirstRowid = SMALLEST_INT64(((i64)-1) - (0xffffffff|(((i64)0x7fffffff)<<32)));
22061 pNew->iLastRowid = LARGEST_INT64(0xffffffff|(((i64)0x7fffffff)<<32));
22062 pNew->base.pVtab = (sqlite3_vtab*)pTab;
22063 rc = sqlite3Fts5ExprClonePhrase(pCsr->pExpr, iPhrase, &pNew->pExpr);
22064 }
22065
22066 if( rc==SQLITE_OK0 ){
22067 for(rc = fts5CursorFirst(pTab, pNew, 0);
22068 rc==SQLITE_OK0 && CsrFlagTest(pNew, FTS5CSR_EOF)((pNew)->csrflags & (0x01))==0;
22069 rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew)
22070 ){
22071 rc = xCallback(&sFts5Api, (Fts5Context*)pNew, pUserData);
22072 if( rc!=SQLITE_OK0 ){
22073 if( rc==SQLITE_DONE101 ) rc = SQLITE_OK0;
22074 break;
22075 }
22076 }
22077 }
22078
22079 fts5CloseMethod((sqlite3_vtab_cursor*)pNew);
22080 return rc;
22081}
22082
22083static void fts5ApiInvoke(
22084 Fts5Auxiliary *pAux,
22085 Fts5Cursor *pCsr,
22086 sqlite3_context *context,
22087 int argc,
22088 sqlite3_value **argv
22089){
22090 assert( pCsr->pAux==0 )((void) (0));
22091 assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL )((void) (0));
22092 pCsr->pAux = pAux;
22093 pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv);
22094 pCsr->pAux = 0;
22095}
22096
22097static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){
22098 Fts5Cursor *pCsr;
22099 for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
22100 if( pCsr->iCsrId==iCsrId ) break;
22101 }
22102 return pCsr;
22103}
22104
22105/*
22106** Parameter zFmt is a printf() style formatting string. This function
22107** formats it using the trailing arguments and returns the result as
22108** an error message to the context passed as the first argument.
22109*/
22110static void fts5ResultError(sqlite3_context *pCtx, const char *zFmt, ...){
22111 char *zErr = 0;
22112 va_list ap;
22113 va_start(ap, zFmt)__builtin_va_start(ap, zFmt);
22114 zErr = sqlite3_vmprintfsqlite3_api->vmprintf(zFmt, ap);
22115 sqlite3_result_errorsqlite3_api->result_error(pCtx, zErr, -1);
22116 sqlite3_freesqlite3_api->free(zErr);
22117 va_end(ap)__builtin_va_end(ap);
22118}
22119
22120static void fts5ApiCallback(
22121 sqlite3_context *context,
22122 int argc,
22123 sqlite3_value **argv
22124){
22125
22126 Fts5Auxiliary *pAux;
22127 Fts5Cursor *pCsr;
22128 i64 iCsrId;
22129
22130 assert( argc>=1 )((void) (0));
22131 pAux = (Fts5Auxiliary*)sqlite3_user_datasqlite3_api->user_data(context);
22132 iCsrId = sqlite3_value_int64sqlite3_api->value_int64(argv[0]);
22133
22134 pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId);
22135 if( pCsr==0 || (pCsr->ePlan==0 || pCsr->ePlan==FTS5_PLAN_SPECIAL3) ){
22136 fts5ResultError(context, "no such cursor: %lld", iCsrId);
22137 }else{
22138 sqlite3_vtab *pTab = pCsr->base.pVtab;
22139 fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]);
22140 sqlite3_freesqlite3_api->free(pTab->zErrMsg);
22141 pTab->zErrMsg = 0;
22142 }
22143}
22144
22145
22146/*
22147** Given cursor id iId, return a pointer to the corresponding Fts5Table
22148** object. Or NULL If the cursor id does not exist.
22149*/
22150static Fts5Table *sqlite3Fts5TableFromCsrid(
22151 Fts5Global *pGlobal, /* FTS5 global context for db handle */
22152 i64 iCsrId /* Id of cursor to find */
22153){
22154 Fts5Cursor *pCsr;
22155 pCsr = fts5CursorFromCsrid(pGlobal, iCsrId);
22156 if( pCsr ){
22157 return (Fts5Table*)pCsr->base.pVtab;
22158 }
22159 return 0;
22160}
22161
22162/*
22163** Return a "position-list blob" corresponding to the current position of
22164** cursor pCsr via sqlite3_result_blob(). A position-list blob contains
22165** the current position-list for each phrase in the query associated with
22166** cursor pCsr.
22167**
22168** A position-list blob begins with (nPhrase-1) varints, where nPhrase is
22169** the number of phrases in the query. Following the varints are the
22170** concatenated position lists for each phrase, in order.
22171**
22172** The first varint (if it exists) contains the size of the position list
22173** for phrase 0. The second (same disclaimer) contains the size of position
22174** list 1. And so on. There is no size field for the final position list,
22175** as it can be derived from the total size of the blob.
22176*/
22177static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
22178 int i;
22179 int rc = SQLITE_OK0;
22180 int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
22181 Fts5Buffer val;
22182
22183 memset(&val, 0, sizeof(Fts5Buffer));
22184 switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){
22185 case FTS5_DETAIL_FULL0:
22186
22187 /* Append the varints */
22188 for(i=0; i<(nPhrase-1); i++){
22189 const u8 *dummy;
22190 int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
22191 sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
22192 }
22193
22194 /* Append the position lists */
22195 for(i=0; i<nPhrase; i++){
22196 const u8 *pPoslist;
22197 int nPoslist;
22198 nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
22199 sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
22200 }
22201 break;
22202
22203 case FTS5_DETAIL_COLUMNS2:
22204
22205 /* Append the varints */
22206 for(i=0; rc==SQLITE_OK0 && i<(nPhrase-1); i++){
22207 const u8 *dummy;
22208 int nByte;
22209 rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte);
22210 sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
22211 }
22212
22213 /* Append the position lists */
22214 for(i=0; rc==SQLITE_OK0 && i<nPhrase; i++){
22215 const u8 *pPoslist;
22216 int nPoslist;
22217 rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &pPoslist, &nPoslist);
22218 sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
22219 }
22220 break;
22221
22222 default:
22223 break;
22224 }
22225
22226 sqlite3_result_blobsqlite3_api->result_blob(pCtx, val.p, val.n, sqlite3_freesqlite3_api->free);
22227 return rc;
22228}
22229
22230/*
22231** This is the xColumn method, called by SQLite to request a value from
22232** the row that the supplied cursor currently points to.
22233*/
22234static int fts5ColumnMethod(
22235 sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
22236 sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
22237 int iCol /* Index of column to read value from */
22238){
22239 Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab);
22240 Fts5Config *pConfig = pTab->p.pConfig;
22241 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
22242 int rc = SQLITE_OK0;
22243
22244 assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 )((void) (0));
22245
22246 if( pCsr->ePlan==FTS5_PLAN_SPECIAL3 ){
22247 if( iCol==pConfig->nCol ){
22248 sqlite3_result_int64sqlite3_api->result_int64(pCtx, pCsr->iSpecial);
22249 }
22250 }else
22251
22252 if( iCol==pConfig->nCol ){
22253 /* User is requesting the value of the special column with the same name
22254 ** as the table. Return the cursor integer id number. This value is only
22255 ** useful in that it may be passed as the first argument to an FTS5
22256 ** auxiliary function. */
22257 sqlite3_result_int64sqlite3_api->result_int64(pCtx, pCsr->iCsrId);
22258 }else if( iCol==pConfig->nCol+1 ){
22259 /* The value of the "rank" column. */
22260
22261 if( pCsr->ePlan==FTS5_PLAN_SOURCE2 ){
22262 fts5PoslistBlob(pCtx, pCsr);
22263 }else if(
22264 pCsr->ePlan==FTS5_PLAN_MATCH1
22265 || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH4
22266 ){
22267 if( pCsr->pRank || SQLITE_OK0==(rc = fts5FindRankFunction(pCsr)) ){
22268 fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg);
22269 }
22270 }
22271 }else{
22272 if( !sqlite3_vtab_nochangesqlite3_api->vtab_nochange(pCtx) && pConfig->eContent!=FTS5_CONTENT_NONE1 ){
22273 pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
22274 rc = fts5SeekCursor(pCsr, 1);
22275 if( rc==SQLITE_OK0 ){
22276 sqlite3_value *pVal = sqlite3_column_valuesqlite3_api->column_value(pCsr->pStmt, iCol+1);
22277 if( pConfig->bLocale
22278 && pConfig->eContent==FTS5_CONTENT_EXTERNAL2
22279 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
22280 ){
22281 const char *z = 0;
22282 int n = 0;
22283 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n);
22284 if( rc==SQLITE_OK0 ){
22285 sqlite3_result_textsqlite3_api->result_text(pCtx, z, n, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
22286 }
22287 sqlite3Fts5ClearLocale(pConfig);
22288 }else{
22289 sqlite3_result_valuesqlite3_api->result_value(pCtx, pVal);
22290 }
22291 }
22292
22293 pConfig->pzErrmsg = 0;
22294 }
22295 }
22296
22297 return rc;
22298}
22299
22300
22301/*
22302** This routine implements the xFindFunction method for the FTS3
22303** virtual table.
22304*/
22305static int fts5FindFunctionMethod(
22306 sqlite3_vtab *pVtab, /* Virtual table handle */
22307 int nUnused, /* Number of SQL function arguments */
22308 const char *zName, /* Name of SQL function */
22309 void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
22310 void **ppArg /* OUT: User data for *pxFunc */
22311){
22312 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22313 Fts5Auxiliary *pAux;
22314
22315 UNUSED_PARAM(nUnused)(void)(nUnused);
22316 pAux = fts5FindAuxiliary(pTab, zName);
22317 if( pAux ){
22318 *pxFunc = fts5ApiCallback;
22319 *ppArg = (void*)pAux;
22320 return 1;
22321 }
22322
22323 /* No function of the specified name was found. Return 0. */
22324 return 0;
22325}
22326
22327/*
22328** Implementation of FTS5 xRename method. Rename an fts5 table.
22329*/
22330static int fts5RenameMethod(
22331 sqlite3_vtab *pVtab, /* Virtual table handle */
22332 const char *zName /* New name of table */
22333){
22334 int rc;
22335 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22336 rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
22337 return rc;
22338}
22339
22340static int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
22341 fts5TripCursors((Fts5FullTable*)pTab);
22342 return sqlite3Fts5StorageSync(((Fts5FullTable*)pTab)->pStorage);
22343}
22344
22345/*
22346** The xSavepoint() method.
22347**
22348** Flush the contents of the pending-terms table to disk.
22349*/
22350static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
22351 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22352 int rc = SQLITE_OK0;
22353
22354 fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
22355 rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
22356 if( rc==SQLITE_OK0 ){
22357 pTab->iSavepoint = iSavepoint+1;
22358 }
22359 return rc;
22360}
22361
22362/*
22363** The xRelease() method.
22364**
22365** This is a no-op.
22366*/
22367static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
22368 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22369 int rc = SQLITE_OK0;
22370 fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
22371 if( (iSavepoint+1)<pTab->iSavepoint ){
22372 rc = sqlite3Fts5FlushToDisk(&pTab->p);
22373 if( rc==SQLITE_OK0 ){
22374 pTab->iSavepoint = iSavepoint;
22375 }
22376 }
22377 return rc;
22378}
22379
22380/*
22381** The xRollbackTo() method.
22382**
22383** Discard the contents of the pending terms table.
22384*/
22385static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
22386 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22387 int rc = SQLITE_OK0;
22388 fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
22389 fts5TripCursors(pTab);
22390 if( (iSavepoint+1)<=pTab->iSavepoint ){
22391 pTab->p.pConfig->pgsz = 0;
22392 rc = sqlite3Fts5StorageRollback(pTab->pStorage);
22393 }
22394 return rc;
22395}
22396
22397/*
22398** Register a new auxiliary function with global context pGlobal.
22399*/
22400static int fts5CreateAux(
22401 fts5_api *pApi, /* Global context (one per db handle) */
22402 const char *zName, /* Name of new function */
22403 void *pUserData, /* User data for aux. function */
22404 fts5_extension_function xFunc, /* Aux. function implementation */
22405 void(*xDestroy)(void*) /* Destructor for pUserData */
22406){
22407 Fts5Global *pGlobal = (Fts5Global*)pApi;
22408 int rc = sqlite3_overload_functionsqlite3_api->overload_function(pGlobal->db, zName, -1);
22409 if( rc==SQLITE_OK0 ){
22410 Fts5Auxiliary *pAux;
22411 sqlite3_int64 nName; /* Size of zName in bytes, including \0 */
22412 sqlite3_int64 nByte; /* Bytes of space to allocate */
22413
22414 nName = strlen(zName) + 1;
22415 nByte = sizeof(Fts5Auxiliary) + nName;
22416 pAux = (Fts5Auxiliary*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
22417 if( pAux ){
22418 memset(pAux, 0, (size_t)nByte);
22419 pAux->zFunc = (char*)&pAux[1];
22420 memcpy(pAux->zFunc, zName, nName);
22421 pAux->pGlobal = pGlobal;
22422 pAux->pUserData = pUserData;
22423 pAux->xFunc = xFunc;
22424 pAux->xDestroy = xDestroy;
22425 pAux->pNext = pGlobal->pAux;
22426 pGlobal->pAux = pAux;
22427 }else{
22428 rc = SQLITE_NOMEM7;
22429 }
22430 }
22431
22432 return rc;
22433}
22434
22435/*
22436** This function is used by xCreateTokenizer_v2() and xCreateTokenizer().
22437** It allocates and partially populates a new Fts5TokenizerModule object.
22438** The new object is already linked into the Fts5Global context before
22439** returning.
22440**
22441** If successful, SQLITE_OK is returned and a pointer to the new
22442** Fts5TokenizerModule object returned via output parameter (*ppNew). All
22443** that is required is for the caller to fill in the methods in
22444** Fts5TokenizerModule.x1 and x2, and to set Fts5TokenizerModule.bV2Native
22445** as appropriate.
22446**
22447** If an error occurs, an SQLite error code is returned and the final value
22448** of (*ppNew) undefined.
22449*/
22450static int fts5NewTokenizerModule(
22451 Fts5Global *pGlobal, /* Global context (one per db handle) */
22452 const char *zName, /* Name of new function */
22453 void *pUserData, /* User data for aux. function */
22454 void(*xDestroy)(void*), /* Destructor for pUserData */
22455 Fts5TokenizerModule **ppNew
22456){
22457 int rc = SQLITE_OK0;
22458 Fts5TokenizerModule *pNew;
22459 sqlite3_int64 nName; /* Size of zName and its \0 terminator */
22460 sqlite3_int64 nByte; /* Bytes of space to allocate */
22461
22462 nName = strlen(zName) + 1;
22463 nByte = sizeof(Fts5TokenizerModule) + nName;
22464 *ppNew = pNew = (Fts5TokenizerModule*)sqlite3Fts5MallocZero(&rc, nByte);
22465 if( pNew ){
22466 pNew->zName = (char*)&pNew[1];
22467 memcpy(pNew->zName, zName, nName);
22468 pNew->pUserData = pUserData;
22469 pNew->xDestroy = xDestroy;
22470 pNew->pNext = pGlobal->pTok;
22471 pGlobal->pTok = pNew;
22472 if( pNew->pNext==0 ){
22473 pGlobal->pDfltTok = pNew;
22474 }
22475 }
22476
22477 return rc;
22478}
22479
22480/*
22481** An instance of this type is used as the Fts5Tokenizer object for
22482** wrapper tokenizers - those that provide access to a v1 tokenizer via
22483** the fts5_tokenizer_v2 API, and those that provide access to a v2 tokenizer
22484** via the fts5_tokenizer API.
22485*/
22486typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer;
22487struct Fts5VtoVTokenizer {
22488 int bV2Native; /* True if v2 native tokenizer */
22489 fts5_tokenizer x1; /* Tokenizer functions */
22490 fts5_tokenizer_v2 x2; /* V2 tokenizer functions */
22491 Fts5Tokenizer *pReal;
22492};
22493
22494/*
22495** Create a wrapper tokenizer. The context argument pCtx points to the
22496** Fts5TokenizerModule object.
22497*/
22498static int fts5VtoVCreate(
22499 void *pCtx,
22500 const char **azArg,
22501 int nArg,
22502 Fts5Tokenizer **ppOut
22503){
22504 Fts5TokenizerModule *pMod = (Fts5TokenizerModule*)pCtx;
22505 Fts5VtoVTokenizer *pNew = 0;
22506 int rc = SQLITE_OK0;
22507
22508 pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
22509 if( rc==SQLITE_OK0 ){
22510 pNew->x1 = pMod->x1;
22511 pNew->x2 = pMod->x2;
22512 pNew->bV2Native = pMod->bV2Native;
22513 if( pMod->bV2Native ){
22514 rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal);
22515 }else{
22516 rc = pMod->x1.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal);
22517 }
22518 if( rc!=SQLITE_OK0 ){
22519 sqlite3_freesqlite3_api->free(pNew);
22520 pNew = 0;
22521 }
22522 }
22523
22524 *ppOut = (Fts5Tokenizer*)pNew;
22525 return rc;
22526}
22527
22528/*
22529** Delete an Fts5VtoVTokenizer wrapper tokenizer.
22530*/
22531static void fts5VtoVDelete(Fts5Tokenizer *pTok){
22532 Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
22533 if( p ){
22534 if( p->bV2Native ){
22535 p->x2.xDelete(p->pReal);
22536 }else{
22537 p->x1.xDelete(p->pReal);
22538 }
22539 sqlite3_freesqlite3_api->free(p);
22540 }
22541}
22542
22543
22544/*
22545** xTokenizer method for a wrapper tokenizer that offers the v1 interface
22546** (no support for locales).
22547*/
22548static int fts5V1toV2Tokenize(
22549 Fts5Tokenizer *pTok,
22550 void *pCtx, int flags,
22551 const char *pText, int nText,
22552 int (*xToken)(void*, int, const char*, int, int, int)
22553){
22554 Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
22555 assert( p->bV2Native )((void) (0));
22556 return p->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken);
22557}
22558
22559/*
22560** xTokenizer method for a wrapper tokenizer that offers the v2 interface
22561** (with locale support).
22562*/
22563static int fts5V2toV1Tokenize(
22564 Fts5Tokenizer *pTok,
22565 void *pCtx, int flags,
22566 const char *pText, int nText,
22567 const char *pLocale, int nLocale,
22568 int (*xToken)(void*, int, const char*, int, int, int)
22569){
22570 Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
22571 assert( p->bV2Native==0 )((void) (0));
22572 UNUSED_PARAM2(pLocale,nLocale)(void)(pLocale), (void)(nLocale);
22573 return p->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken);
22574}
22575
22576/*
22577** Register a new tokenizer. This is the implementation of the
22578** fts5_api.xCreateTokenizer_v2() method.
22579*/
22580static int fts5CreateTokenizer_v2(
22581 fts5_api *pApi, /* Global context (one per db handle) */
22582 const char *zName, /* Name of new function */
22583 void *pUserData, /* User data for aux. function */
22584 fts5_tokenizer_v2 *pTokenizer, /* Tokenizer implementation */
22585 void(*xDestroy)(void*) /* Destructor for pUserData */
22586){
22587 Fts5Global *pGlobal = (Fts5Global*)pApi;
22588 int rc = SQLITE_OK0;
22589
22590 if( pTokenizer->iVersion>2 ){
22591 rc = SQLITE_ERROR1;
22592 }else{
22593 Fts5TokenizerModule *pNew = 0;
22594 rc = fts5NewTokenizerModule(pGlobal, zName, pUserData, xDestroy, &pNew);
22595 if( pNew ){
22596 pNew->x2 = *pTokenizer;
22597 pNew->bV2Native = 1;
22598 pNew->x1.xCreate = fts5VtoVCreate;
22599 pNew->x1.xTokenize = fts5V1toV2Tokenize;
22600 pNew->x1.xDelete = fts5VtoVDelete;
22601 }
22602 }
22603
22604 return rc;
22605}
22606
22607/*
22608** The fts5_api.xCreateTokenizer() method.
22609*/
22610static int fts5CreateTokenizer(
22611 fts5_api *pApi, /* Global context (one per db handle) */
22612 const char *zName, /* Name of new function */
22613 void *pUserData, /* User data for aux. function */
22614 fts5_tokenizer *pTokenizer, /* Tokenizer implementation */
22615 void(*xDestroy)(void*) /* Destructor for pUserData */
22616){
22617 Fts5TokenizerModule *pNew = 0;
22618 int rc = SQLITE_OK0;
22619
22620 rc = fts5NewTokenizerModule(
22621 (Fts5Global*)pApi, zName, pUserData, xDestroy, &pNew
22622 );
22623 if( pNew ){
22624 pNew->x1 = *pTokenizer;
22625 pNew->x2.xCreate = fts5VtoVCreate;
22626 pNew->x2.xTokenize = fts5V2toV1Tokenize;
22627 pNew->x2.xDelete = fts5VtoVDelete;
22628 }
22629 return rc;
22630}
22631
22632/*
22633** Search the global context passed as the first argument for a tokenizer
22634** module named zName. If found, return a pointer to the Fts5TokenizerModule
22635** object. Otherwise, return NULL.
22636*/
22637static Fts5TokenizerModule *fts5LocateTokenizer(
22638 Fts5Global *pGlobal, /* Global (one per db handle) object */
22639 const char *zName /* Name of tokenizer module to find */
22640){
22641 Fts5TokenizerModule *pMod = 0;
22642
22643 if( zName==0 ){
22644 pMod = pGlobal->pDfltTok;
22645 }else{
22646 for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
22647 if( sqlite3_stricmpsqlite3_api->stricmp(zName, pMod->zName)==0 ) break;
22648 }
22649 }
22650
22651 return pMod;
22652}
22653
22654/*
22655** Find a tokenizer. This is the implementation of the
22656** fts5_api.xFindTokenizer_v2() method.
22657*/
22658static int fts5FindTokenizer_v2(
22659 fts5_api *pApi, /* Global context (one per db handle) */
22660 const char *zName, /* Name of tokenizer */
22661 void **ppUserData,
22662 fts5_tokenizer_v2 **ppTokenizer /* Populate this object */
22663){
22664 int rc = SQLITE_OK0;
22665 Fts5TokenizerModule *pMod;
22666
22667 pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName);
22668 if( pMod ){
22669 if( pMod->bV2Native ){
22670 *ppUserData = pMod->pUserData;
22671 }else{
22672 *ppUserData = (void*)pMod;
22673 }
22674 *ppTokenizer = &pMod->x2;
22675 }else{
22676 *ppTokenizer = 0;
22677 *ppUserData = 0;
22678 rc = SQLITE_ERROR1;
22679 }
22680
22681 return rc;
22682}
22683
22684/*
22685** Find a tokenizer. This is the implementation of the
22686** fts5_api.xFindTokenizer() method.
22687*/
22688static int fts5FindTokenizer(
22689 fts5_api *pApi, /* Global context (one per db handle) */
22690 const char *zName, /* Name of new function */
22691 void **ppUserData,
22692 fts5_tokenizer *pTokenizer /* Populate this object */
22693){
22694 int rc = SQLITE_OK0;
22695 Fts5TokenizerModule *pMod;
22696
22697 pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName);
22698 if( pMod ){
22699 if( pMod->bV2Native==0 ){
22700 *ppUserData = pMod->pUserData;
22701 }else{
22702 *ppUserData = (void*)pMod;
22703 }
22704 *pTokenizer = pMod->x1;
22705 }else{
22706 memset(pTokenizer, 0, sizeof(*pTokenizer));
22707 *ppUserData = 0;
22708 rc = SQLITE_ERROR1;
22709 }
22710
22711 return rc;
22712}
22713
22714/*
22715** Attempt to instantiate the tokenizer.
22716*/
22717static int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){
22718 const char **azArg = pConfig->t.azArg;
22719 const int nArg = pConfig->t.nArg;
22720 Fts5TokenizerModule *pMod = 0;
22721 int rc = SQLITE_OK0;
22722
22723 pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]);
22724 if( pMod==0 ){
22725 assert( nArg>0 )((void) (0));
22726 rc = SQLITE_ERROR1;
22727 sqlite3Fts5ConfigErrmsg(pConfig, "no such tokenizer: %s", azArg[0]);
22728 }else{
22729 int (*xCreate)(void*, const char**, int, Fts5Tokenizer**) = 0;
22730 if( pMod->bV2Native ){
22731 xCreate = pMod->x2.xCreate;
22732 pConfig->t.pApi2 = &pMod->x2;
22733 }else{
22734 pConfig->t.pApi1 = &pMod->x1;
22735 xCreate = pMod->x1.xCreate;
22736 }
22737
22738 rc = xCreate(pMod->pUserData,
22739 (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok
22740 );
22741
22742 if( rc!=SQLITE_OK0 ){
22743 if( rc!=SQLITE_NOMEM7 ){
22744 sqlite3Fts5ConfigErrmsg(pConfig, "error in tokenizer constructor");
22745 }
22746 }else if( pMod->bV2Native==0 ){
22747 pConfig->t.ePattern = sqlite3Fts5TokenizerPattern(
22748 pMod->x1.xCreate, pConfig->t.pTok
22749 );
22750 }
22751 }
22752
22753 if( rc!=SQLITE_OK0 ){
22754 pConfig->t.pApi1 = 0;
22755 pConfig->t.pApi2 = 0;
22756 pConfig->t.pTok = 0;
22757 }
22758
22759 return rc;
22760}
22761
22762
22763/*
22764** xDestroy callback passed to sqlite3_create_module(). This is invoked
22765** when the db handle is being closed. Free memory associated with
22766** tokenizers and aux functions registered with this db handle.
22767*/
22768static void fts5ModuleDestroy(void *pCtx){
22769 Fts5TokenizerModule *pTok, *pNextTok;
22770 Fts5Auxiliary *pAux, *pNextAux;
22771 Fts5Global *pGlobal = (Fts5Global*)pCtx;
22772
22773 for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){
22774 pNextAux = pAux->pNext;
22775 if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData);
22776 sqlite3_freesqlite3_api->free(pAux);
22777 }
22778
22779 for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){
22780 pNextTok = pTok->pNext;
22781 if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData);
22782 sqlite3_freesqlite3_api->free(pTok);
22783 }
22784
22785 sqlite3_freesqlite3_api->free(pGlobal);
22786}
22787
22788/*
22789** Implementation of the fts5() function used by clients to obtain the
22790** API pointer.
22791*/
22792static void fts5Fts5Func(
22793 sqlite3_context *pCtx, /* Function call context */
22794 int nArg, /* Number of args */
22795 sqlite3_value **apArg /* Function arguments */
22796){
22797 Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_datasqlite3_api->user_data(pCtx);
22798 fts5_api **ppApi;
22799 UNUSED_PARAM(nArg)(void)(nArg);
22800 assert( nArg==1 )((void) (0));
22801 ppApi = (fts5_api**)sqlite3_value_pointersqlite3_api->value_pointer(apArg[0], "fts5_api_ptr");
22802 if( ppApi ) *ppApi = &pGlobal->api;
22803}
22804
22805/*
22806** Implementation of fts5_source_id() function.
22807*/
22808static void fts5SourceIdFunc(
22809 sqlite3_context *pCtx, /* Function call context */
22810 int nArg, /* Number of args */
22811 sqlite3_value **apUnused /* Function arguments */
22812){
22813 assert( nArg==0 )((void) (0));
22814 UNUSED_PARAM2(nArg, apUnused)(void)(nArg), (void)(apUnused);
22815 sqlite3_result_textsqlite3_api->result_text(pCtx, "fts5: 2026-07-24 19:02:57 bf7c7f30031888f4e796e429ab3978879485813aaca6f641c7b33e4e09459bcc", -1, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
22816}
22817
22818/*
22819** Implementation of fts5_locale(LOCALE, TEXT) function.
22820**
22821** If parameter LOCALE is NULL, or a zero-length string, then a copy of
22822** TEXT is returned. Otherwise, both LOCALE and TEXT are interpreted as
22823** text, and the value returned is a blob consisting of:
22824**
22825** * The 4 bytes 0x00, 0xE0, 0xB2, 0xEb (FTS5_LOCALE_HEADER).
22826** * The LOCALE, as utf-8 text, followed by
22827** * 0x00, followed by
22828** * The TEXT, as utf-8 text.
22829**
22830** There is no final nul-terminator following the TEXT value.
22831*/
22832static void fts5LocaleFunc(
22833 sqlite3_context *pCtx, /* Function call context */
22834 int nArg, /* Number of args */
22835 sqlite3_value **apArg /* Function arguments */
22836){
22837 const char *zLocale = 0;
22838 i64 nLocale = 0;
22839 const char *zText = 0;
22840 i64 nText = 0;
22841
22842 assert( nArg==2 )((void) (0));
22843 UNUSED_PARAM(nArg)(void)(nArg);
22844
22845 zLocale = (const char*)sqlite3_value_textsqlite3_api->value_text(apArg[0]);
22846 nLocale = sqlite3_value_bytessqlite3_api->value_bytes(apArg[0]);
22847
22848 zText = (const char*)sqlite3_value_textsqlite3_api->value_text(apArg[1]);
22849 nText = sqlite3_value_bytessqlite3_api->value_bytes(apArg[1]);
22850
22851 if( zLocale==0 || zLocale[0]=='\0' ){
22852 sqlite3_result_textsqlite3_api->result_text(pCtx, zText, nText, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
22853 }else{
22854 Fts5Global *p = (Fts5Global*)sqlite3_user_datasqlite3_api->user_data(pCtx);
22855 u8 *pBlob = 0;
22856 u8 *pCsr = 0;
22857 i64 nBlob = 0;
22858
22859 nBlob = FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )) + nLocale + 1 + nText;
22860 pBlob = (u8*)sqlite3_malloc64sqlite3_api->malloc64(nBlob);
22861 if( pBlob==0 ){
22862 sqlite3_result_error_nomemsqlite3_api->result_error_nomem(pCtx);
22863 return;
22864 }
22865
22866 pCsr = pBlob;
22867 memcpy(pCsr, (const u8*)p->aLocaleHdr, FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )));
22868 pCsr += FTS5_LOCALE_HDR_SIZE((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ));
22869 memcpy(pCsr, zLocale, nLocale);
22870 pCsr += nLocale;
22871 (*pCsr++) = 0x00;
22872 if( zText ) memcpy(pCsr, zText, nText);
22873 assert( &pCsr[nText]==&pBlob[nBlob] )((void) (0));
22874
22875 sqlite3_result_blobsqlite3_api->result_blob(pCtx, pBlob, nBlob, sqlite3_freesqlite3_api->free);
22876 }
22877}
22878
22879/*
22880** Implementation of fts5_insttoken() function.
22881*/
22882static void fts5InsttokenFunc(
22883 sqlite3_context *pCtx, /* Function call context */
22884 int nArg, /* Number of args */
22885 sqlite3_value **apArg /* Function arguments */
22886){
22887 assert( nArg==1 )((void) (0));
22888 (void)nArg;
22889 sqlite3_result_valuesqlite3_api->result_value(pCtx, apArg[0]);
22890 sqlite3_result_subtypesqlite3_api->result_subtype(pCtx, FTS5_INSTTOKEN_SUBTYPE73);
22891}
22892
22893/*
22894** Return true if zName is the extension on one of the shadow tables used
22895** by this module.
22896*/
22897static int fts5ShadowName(const char *zName){
22898 static const char *azName[] = {
22899 "config", "content", "data", "docsize", "idx"
22900 };
22901 unsigned int i;
22902 for(i=0; i<sizeof(azName)/sizeof(azName[0]); i++){
22903 if( sqlite3_stricmpsqlite3_api->stricmp(zName, azName[i])==0 ) return 1;
22904 }
22905 return 0;
22906}
22907
22908/*
22909** Run an integrity check on the FTS5 data structures. Return a string
22910** if anything is found amiss. Return a NULL pointer if everything is
22911** OK.
22912*/
22913static int fts5IntegrityMethod(
22914 sqlite3_vtab *pVtab, /* the FTS5 virtual table to check */
22915 const char *zSchema, /* Name of schema in which this table lives */
22916 const char *zTabname, /* Name of the table itself */
22917 int isQuick, /* True if this is a quick-check */
22918 char **pzErr /* Write error message here */
22919){
22920 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
22921 int rc;
22922
22923 assert( pzErr!=0 && *pzErr==0 )((void) (0));
22924 UNUSED_PARAM(isQuick)(void)(isQuick);
22925 assert( pTab->p.pConfig->pzErrmsg==0 )((void) (0));
22926 pTab->p.pConfig->pzErrmsg = pzErr;
22927 rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0);
22928 if( *pzErr==0 && rc!=SQLITE_OK0 ){
22929 if( (rc&0xff)==SQLITE_CORRUPT11 ){
22930 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("malformed inverted index for FTS5 table %s.%s",
22931 zSchema, zTabname);
22932 rc = (*pzErr) ? SQLITE_OK0 : SQLITE_NOMEM7;
22933 }else{
22934 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("unable to validate the inverted index for"
22935 " FTS5 table %s.%s: %s",
22936 zSchema, zTabname, sqlite3_errstrsqlite3_api->errstr(rc));
22937 }
22938 }else if( (rc&0xff)==SQLITE_CORRUPT11 ){
22939 rc = SQLITE_OK0;
22940 }
22941 sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
22942 pTab->p.pConfig->pzErrmsg = 0;
22943
22944 return rc;
22945}
22946
22947static int fts5Init(sqlite3 *db){
22948 static const sqlite3_module fts5Mod = {
22949 /* iVersion */ 4,
22950 /* xCreate */ fts5CreateMethod,
22951 /* xConnect */ fts5ConnectMethod,
22952 /* xBestIndex */ fts5BestIndexMethod,
22953 /* xDisconnect */ fts5DisconnectMethod,
22954 /* xDestroy */ fts5DestroyMethod,
22955 /* xOpen */ fts5OpenMethod,
22956 /* xClose */ fts5CloseMethod,
22957 /* xFilter */ fts5FilterMethod,
22958 /* xNext */ fts5NextMethod,
22959 /* xEof */ fts5EofMethod,
22960 /* xColumn */ fts5ColumnMethod,
22961 /* xRowid */ fts5RowidMethod,
22962 /* xUpdate */ fts5UpdateMethod,
22963 /* xBegin */ fts5BeginMethod,
22964 /* xSync */ fts5SyncMethod,
22965 /* xCommit */ fts5CommitMethod,
22966 /* xRollback */ fts5RollbackMethod,
22967 /* xFindFunction */ fts5FindFunctionMethod,
22968 /* xRename */ fts5RenameMethod,
22969 /* xSavepoint */ fts5SavepointMethod,
22970 /* xRelease */ fts5ReleaseMethod,
22971 /* xRollbackTo */ fts5RollbackToMethod,
22972 /* xShadowName */ fts5ShadowName,
22973 /* xIntegrity */ fts5IntegrityMethod
22974 };
22975
22976 int rc;
22977 Fts5Global *pGlobal = 0;
22978
22979 pGlobal = (Fts5Global*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(Fts5Global));
22980 if( pGlobal==0 ){
22981 rc = SQLITE_NOMEM7;
22982 }else{
22983 void *p = (void*)pGlobal;
22984 memset(pGlobal, 0, sizeof(Fts5Global));
22985 pGlobal->db = db;
22986 pGlobal->api.iVersion = 3;
22987 pGlobal->api.xCreateFunction = fts5CreateAux;
22988 pGlobal->api.xCreateTokenizer = fts5CreateTokenizer;
22989 pGlobal->api.xFindTokenizer = fts5FindTokenizer;
22990 pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2;
22991 pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2;
22992
22993 /* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector.
22994 ** The constants below were generated randomly. */
22995 sqlite3_randomnesssqlite3_api->randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr);
22996 pGlobal->aLocaleHdr[0] ^= 0xF924976D;
22997 pGlobal->aLocaleHdr[1] ^= 0x16596E13;
22998 pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA;
22999 pGlobal->aLocaleHdr[3] ^= 0x9B03A67F;
23000 assert( sizeof(pGlobal->aLocaleHdr)==16 )((void) (0));
23001
23002 rc = sqlite3_create_module_v2sqlite3_api->create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy);
23003 if( rc==SQLITE_OK0 ) rc = sqlite3Fts5IndexInit(db);
23004 if( rc==SQLITE_OK0 ) rc = sqlite3Fts5ExprInit(pGlobal, db);
23005 if( rc==SQLITE_OK0 ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
23006 if( rc==SQLITE_OK0 ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
23007 if( rc==SQLITE_OK0 ) rc = sqlite3Fts5VocabInit(pGlobal, db);
23008 if( rc==SQLITE_OK0 ){
23009 rc = sqlite3_create_functionsqlite3_api->create_function(
23010 db, "fts5", 1, SQLITE_UTF81, p, fts5Fts5Func, 0, 0
23011 );
23012 }
23013 if( rc==SQLITE_OK0 ){
23014 rc = sqlite3_create_functionsqlite3_api->create_function(
23015 db, "fts5_source_id", 0,
23016 SQLITE_UTF81|SQLITE_DETERMINISTIC0x000000800|SQLITE_INNOCUOUS0x000200000,
23017 p, fts5SourceIdFunc, 0, 0
23018 );
23019 }
23020 if( rc==SQLITE_OK0 ){
23021 rc = sqlite3_create_functionsqlite3_api->create_function(
23022 db, "fts5_locale", 2,
23023 SQLITE_UTF81|SQLITE_INNOCUOUS0x000200000|SQLITE_RESULT_SUBTYPE0x001000000|SQLITE_SUBTYPE0x000100000,
23024 p, fts5LocaleFunc, 0, 0
23025 );
23026 }
23027 if( rc==SQLITE_OK0 ){
23028 rc = sqlite3_create_functionsqlite3_api->create_function(
23029 db, "fts5_insttoken", 1,
23030 SQLITE_UTF81|SQLITE_INNOCUOUS0x000200000|SQLITE_RESULT_SUBTYPE0x001000000,
23031 p, fts5InsttokenFunc, 0, 0
23032 );
23033 }
23034 }
23035
23036 /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file
23037 ** fts5_test_mi.c is compiled and linked into the executable. And call
23038 ** its entry point to enable the matchinfo() demo. */
23039#ifdef SQLITE_FTS5_ENABLE_TEST_MI
23040 if( rc==SQLITE_OK0 ){
23041 extern int sqlite3Fts5TestRegisterMatchinfoAPI(fts5_api*);
23042 rc = sqlite3Fts5TestRegisterMatchinfoAPI(&pGlobal->api);
23043 }
23044#endif
23045
23046 return rc;
23047}
23048
23049/*
23050** The following functions are used to register the module with SQLite. If
23051** this module is being built as part of the SQLite core (SQLITE_CORE is
23052** defined), then sqlite3_open() will call sqlite3Fts5Init() directly.
23053**
23054** Or, if this module is being built as a loadable extension,
23055** sqlite3Fts5Init() is omitted and the two standard entry points
23056** sqlite3_fts_init() and sqlite3_fts5_init() defined instead.
23057*/
23058#ifndef SQLITE_CORE
23059#ifdef _WIN32
23060__declspec(dllexport)
23061#endif
23062int sqlite3_fts_init(
23063 sqlite3 *db,
23064 char **pzErrMsg,
23065 const sqlite3_api_routines *pApi
23066){
23067 SQLITE_EXTENSION_INIT2(pApi)sqlite3_api=pApi;;
23068 (void)pzErrMsg; /* Unused parameter */
23069 return fts5Init(db);
23070}
23071
23072#ifdef _WIN32
23073__declspec(dllexport)
23074#endif
23075int sqlite3_fts5_init(
23076 sqlite3 *db,
23077 char **pzErrMsg,
23078 const sqlite3_api_routines *pApi
23079){
23080 SQLITE_EXTENSION_INIT2(pApi)sqlite3_api=pApi;;
23081 (void)pzErrMsg; /* Unused parameter */
23082 return fts5Init(db);
23083}
23084#else
23085int sqlite3Fts5Init(sqlite3 *db){
23086 return fts5Init(db);
23087}
23088#endif
23089
23090#line 1 "fts5_storage.c"
23091/*
23092** 2014 May 31
23093**
23094** The author disclaims copyright to this source code. In place of
23095** a legal notice, here is a blessing:
23096**
23097** May you do good and not evil.
23098** May you find forgiveness for yourself and forgive others.
23099** May you share freely, never taking more than you give.
23100**
23101******************************************************************************
23102**
23103*/
23104
23105
23106
23107/* #include "fts5Int.h" */
23108
23109/*
23110** pSavedRow:
23111** SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it
23112** does a by-rowid lookup to retrieve a single row from the %_content
23113** table or equivalent external-content table/view.
23114**
23115** However, FTS5_STMT_LOOKUP2 is only used when retrieving the original
23116** values for a row being UPDATEd. In that case, the SQL statement is
23117** not reset and pSavedRow is set to point at it. This is so that the
23118** insert operation that follows the delete may access the original
23119** row values for any new values for which sqlite3_value_nochange() returns
23120** true. i.e. if the user executes:
23121**
23122** CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1);
23123** ...
23124** UPDATE fts SET a=?, b=? WHERE rowid=?;
23125**
23126** then the value passed to the xUpdate() method of this table as the
23127** new.c value is an sqlite3_value_nochange() value. So in this case it
23128** must be read from the saved row stored in Fts5Storage.pSavedRow.
23129**
23130** This is necessary - using sqlite3_value_nochange() instead of just having
23131** SQLite pass the original value back via xUpdate() - so as not to discard
23132** any locale information associated with such values.
23133**
23134*/
23135struct Fts5Storage {
23136 Fts5Config *pConfig;
23137 Fts5Index *pIndex;
23138 int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */
23139 i64 nTotalRow; /* Total number of rows in FTS table */
23140 i64 *aTotalSize; /* Total sizes of each column */
23141 sqlite3_stmt *pSavedRow;
23142 sqlite3_stmt *aStmt[12];
23143};
23144
23145
23146#if FTS5_STMT_SCAN_ASC0!=0
23147# error "FTS5_STMT_SCAN_ASC mismatch"
23148#endif
23149#if FTS5_STMT_SCAN_DESC1!=1
23150# error "FTS5_STMT_SCAN_DESC mismatch"
23151#endif
23152#if FTS5_STMT_LOOKUP2!=2
23153# error "FTS5_STMT_LOOKUP mismatch"
23154#endif
23155
23156#define FTS5_STMT_LOOKUP23 3
23157#define FTS5_STMT_INSERT_CONTENT4 4
23158#define FTS5_STMT_REPLACE_CONTENT5 5
23159#define FTS5_STMT_DELETE_CONTENT6 6
23160#define FTS5_STMT_REPLACE_DOCSIZE7 7
23161#define FTS5_STMT_DELETE_DOCSIZE8 8
23162#define FTS5_STMT_LOOKUP_DOCSIZE9 9
23163#define FTS5_STMT_REPLACE_CONFIG10 10
23164#define FTS5_STMT_SCAN11 11
23165
23166/*
23167** Prepare the two insert statements - Fts5Storage.pInsertContent and
23168** Fts5Storage.pInsertDocsize - if they have not already been prepared.
23169** Return SQLITE_OK if successful, or an SQLite error code if an error
23170** occurs.
23171*/
23172static int fts5StorageGetStmt(
23173 Fts5Storage *p, /* Storage handle */
23174 int eStmt, /* FTS5_STMT_XXX constant */
23175 sqlite3_stmt **ppStmt, /* OUT: Prepared statement handle */
23176 char **pzErrMsg /* OUT: Error message (if any) */
23177){
23178 int rc = SQLITE_OK0;
23179
23180 /* If there is no %_docsize table, there should be no requests for
23181 ** statements to operate on it. */
23182 assert( p->pConfig->bColumnsize || (((void) (0))
23183 eStmt!=FTS5_STMT_REPLACE_DOCSIZE((void) (0))
23184 && eStmt!=FTS5_STMT_DELETE_DOCSIZE((void) (0))
23185 && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE((void) (0))
23186 ))((void) (0));
23187
23188 assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) )((void) (0));
23189 if( p->aStmt[eStmt]==0 ){
23190 const char *azStmt[] = {
23191 "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
23192 "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
23193 "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */
23194 "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP2 */
23195
23196 "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */
23197 "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */
23198 "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */
23199 "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */
23200 "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */
23201
23202 "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */
23203
23204 "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */
23205 "SELECT %s FROM %s AS T", /* SCAN */
23206 };
23207 Fts5Config *pC = p->pConfig;
23208 char *zSql = 0;
23209
23210 assert( ArraySize(azStmt)==ArraySize(p->aStmt) )((void) (0));
23211
23212 switch( eStmt ){
23213 case FTS5_STMT_SCAN11:
23214 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt],
23215 pC->zContentExprlist, pC->zContent
23216 );
23217 break;
23218
23219 case FTS5_STMT_SCAN_ASC0:
23220 case FTS5_STMT_SCAN_DESC1:
23221 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt], pC->zContentExprlist,
23222 pC->zContent, pC->zContentRowid, pC->zContentRowid,
23223 pC->zContentRowid
23224 );
23225 break;
23226
23227 case FTS5_STMT_LOOKUP2:
23228 case FTS5_STMT_LOOKUP23:
23229 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt],
23230 pC->zContentExprlist, pC->zContent, pC->zContentRowid
23231 );
23232 break;
23233
23234 case FTS5_STMT_INSERT_CONTENT4:
23235 case FTS5_STMT_REPLACE_CONTENT5: {
23236 char *zBind = 0;
23237 int i;
23238
23239 assert( pC->eContent==FTS5_CONTENT_NORMAL((void) (0))
23240 || pC->eContent==FTS5_CONTENT_UNINDEXED((void) (0))
23241 )((void) (0));
23242
23243 /* Add bindings for the "c*" columns - those that store the actual
23244 ** table content. If eContent==NORMAL, then there is one binding
23245 ** for each column. Or, if eContent==UNINDEXED, then there are only
23246 ** bindings for the UNINDEXED columns. */
23247 for(i=0; rc==SQLITE_OK0 && i<(pC->nCol+1); i++){
23248 if( !i || pC->eContent==FTS5_CONTENT_NORMAL0 || pC->abUnindexed[i-1] ){
23249 zBind = sqlite3Fts5Mprintf(&rc, "%z%s?%d", zBind, zBind?",":"",i+1);
23250 }
23251 }
23252
23253 /* Add bindings for any "l*" columns. Only non-UNINDEXED columns
23254 ** require these. */
23255 if( pC->bLocale && pC->eContent==FTS5_CONTENT_NORMAL0 ){
23256 for(i=0; rc==SQLITE_OK0 && i<pC->nCol; i++){
23257 if( pC->abUnindexed[i]==0 ){
23258 zBind = sqlite3Fts5Mprintf(&rc, "%z,?%d", zBind, pC->nCol+i+2);
23259 }
23260 }
23261 }
23262
23263 zSql = sqlite3Fts5Mprintf(&rc, azStmt[eStmt], pC->zDb, pC->zName,zBind);
23264 sqlite3_freesqlite3_api->free(zBind);
23265 break;
23266 }
23267
23268 case FTS5_STMT_REPLACE_DOCSIZE7:
23269 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt], pC->zDb, pC->zName,
23270 (pC->bContentlessDelete ? ",?" : "")
23271 );
23272 break;
23273
23274 case FTS5_STMT_LOOKUP_DOCSIZE9:
23275 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt],
23276 (pC->bContentlessDelete ? ",origin" : ""),
23277 pC->zDb, pC->zName
23278 );
23279 break;
23280
23281 default:
23282 zSql = sqlite3_mprintfsqlite3_api->mprintf(azStmt[eStmt], pC->zDb, pC->zName);
23283 break;
23284 }
23285
23286 if( zSql==0 ){
23287 rc = SQLITE_NOMEM7;
23288 }else{
23289 int f = SQLITE_PREPARE_PERSISTENT0x01;
23290 if( eStmt>FTS5_STMT_LOOKUP23 ) f |= SQLITE_PREPARE_NO_VTAB0x04;
23291 p->pConfig->bLock++;
23292 rc = sqlite3_prepare_v3sqlite3_api->prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0);
23293 p->pConfig->bLock--;
23294 sqlite3_freesqlite3_api->free(zSql);
23295 if( rc!=SQLITE_OK0 && pzErrMsg ){
23296 *pzErrMsg = sqlite3_mprintfsqlite3_api->mprintf("%s", sqlite3_errmsgsqlite3_api->errmsg(pC->db));
23297 }
23298 if( rc==SQLITE_ERROR1 && eStmt>FTS5_STMT_LOOKUP23 && eStmt<FTS5_STMT_SCAN11 ){
23299 /* One of the internal tables - not the %_content table - is missing.
23300 ** This counts as a corrupted table. */
23301 rc = SQLITE_CORRUPT11;
23302 }
23303 }
23304 }
23305
23306 *ppStmt = p->aStmt[eStmt];
23307 sqlite3_resetsqlite3_api->reset(*ppStmt);
23308 return rc;
23309}
23310
23311
23312static int fts5ExecPrintf(
23313 sqlite3 *db,
23314 char **pzErr,
23315 const char *zFormat,
23316 ...
23317){
23318 int rc;
23319 va_list ap; /* ... printf arguments */
23320 char *zSql;
23321
23322 va_start(ap, zFormat)__builtin_va_start(ap, zFormat);
23323 zSql = sqlite3_vmprintfsqlite3_api->vmprintf(zFormat, ap);
23324
23325 if( zSql==0 ){
23326 rc = SQLITE_NOMEM7;
23327 }else{
23328 rc = sqlite3_execsqlite3_api->exec(db, zSql, 0, 0, pzErr);
23329 sqlite3_freesqlite3_api->free(zSql);
23330 }
23331
23332 va_end(ap)__builtin_va_end(ap);
23333 return rc;
23334}
23335
23336/*
23337** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error
23338** code otherwise.
23339*/
23340static int sqlite3Fts5DropAll(Fts5Config *pConfig){
23341 int rc = fts5ExecPrintf(pConfig->db, 0,
23342 "DROP TABLE IF EXISTS %Q.'%q_data';"
23343 "DROP TABLE IF EXISTS %Q.'%q_idx';"
23344 "DROP TABLE IF EXISTS %Q.'%q_config';",
23345 pConfig->zDb, pConfig->zName,
23346 pConfig->zDb, pConfig->zName,
23347 pConfig->zDb, pConfig->zName
23348 );
23349 if( rc==SQLITE_OK0 && pConfig->bColumnsize ){
23350 rc = fts5ExecPrintf(pConfig->db, 0,
23351 "DROP TABLE IF EXISTS %Q.'%q_docsize';",
23352 pConfig->zDb, pConfig->zName
23353 );
23354 }
23355 if( rc==SQLITE_OK0 && pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
23356 rc = fts5ExecPrintf(pConfig->db, 0,
23357 "DROP TABLE IF EXISTS %Q.'%q_content';",
23358 pConfig->zDb, pConfig->zName
23359 );
23360 }
23361 return rc;
23362}
23363
23364static void fts5StorageRenameOne(
23365 Fts5Config *pConfig, /* Current FTS5 configuration */
23366 int *pRc, /* IN/OUT: Error code */
23367 const char *zTail, /* Tail of table name e.g. "data", "config" */
23368 const char *zName /* New name of FTS5 table */
23369){
23370 if( *pRc==SQLITE_OK0 ){
23371 *pRc = fts5ExecPrintf(pConfig->db, 0,
23372 "ALTER TABLE %Q.'%q_%s' RENAME TO '%q_%s';",
23373 pConfig->zDb, pConfig->zName, zTail, zName, zTail
23374 );
23375 }
23376}
23377
23378static int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){
23379 Fts5Config *pConfig = pStorage->pConfig;
23380 int rc = sqlite3Fts5StorageSync(pStorage);
23381
23382 fts5StorageRenameOne(pConfig, &rc, "data", zName);
23383 fts5StorageRenameOne(pConfig, &rc, "idx", zName);
23384 fts5StorageRenameOne(pConfig, &rc, "config", zName);
23385 if( pConfig->bColumnsize ){
23386 fts5StorageRenameOne(pConfig, &rc, "docsize", zName);
23387 }
23388 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
23389 fts5StorageRenameOne(pConfig, &rc, "content", zName);
23390 }
23391 return rc;
23392}
23393
23394/*
23395** Create the shadow table named zPost, with definition zDefn. Return
23396** SQLITE_OK if successful, or an SQLite error code otherwise.
23397*/
23398static int sqlite3Fts5CreateTable(
23399 Fts5Config *pConfig, /* FTS5 configuration */
23400 const char *zPost, /* Shadow table to create (e.g. "content") */
23401 const char *zDefn, /* Columns etc. for shadow table */
23402 int bWithout, /* True for without rowid */
23403 char **pzErr /* OUT: Error message */
23404){
23405 int rc;
23406 char *zErr = 0;
23407
23408 rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s",
23409 pConfig->zDb, pConfig->zName, zPost, zDefn,
23410#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID
23411 bWithout?" WITHOUT ROWID":
23412#endif
23413 ""
23414 );
23415 if( zErr ){
23416 *pzErr = sqlite3_mprintfsqlite3_api->mprintf(
23417 "fts5: error creating shadow table %q_%s: %s",
23418 pConfig->zName, zPost, zErr
23419 );
23420 sqlite3_freesqlite3_api->free(zErr);
23421 }
23422
23423 return rc;
23424}
23425
23426/*
23427** Open a new Fts5Index handle. If the bCreate argument is true, create
23428** and initialize the underlying tables
23429**
23430** If successful, set *pp to point to the new object and return SQLITE_OK.
23431** Otherwise, set *pp to NULL and return an SQLite error code.
23432*/
23433static int sqlite3Fts5StorageOpen(
23434 Fts5Config *pConfig,
23435 Fts5Index *pIndex,
23436 int bCreate,
23437 Fts5Storage **pp,
23438 char **pzErr /* OUT: Error message */
23439){
23440 int rc = SQLITE_OK0;
23441 Fts5Storage *p; /* New object */
23442 sqlite3_int64 nByte; /* Bytes of space to allocate */
23443
23444 nByte = sizeof(Fts5Storage) /* Fts5Storage object */
23445 + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */
23446 *pp = p = (Fts5Storage*)sqlite3_malloc64sqlite3_api->malloc64(nByte);
23447 if( !p ) return SQLITE_NOMEM7;
23448
23449 memset(p, 0, (size_t)nByte);
23450 p->aTotalSize = (i64*)&p[1];
23451 p->pConfig = pConfig;
23452 p->pIndex = pIndex;
23453
23454 if( bCreate ){
23455 if( pConfig->eContent==FTS5_CONTENT_NORMAL0
23456 || pConfig->eContent==FTS5_CONTENT_UNINDEXED3
23457 ){
23458 int i = 0;
23459 char *zDefn = 0;
23460 sqlite3_str *pDefn = sqlite3_str_newsqlite3_api->str_new(pConfig->db);
23461
23462 sqlite3_str_appendfsqlite3_api->str_appendf(pDefn, "id INTEGER PRIMARY KEY");
23463 for(i=0; i<pConfig->nCol; i++){
23464 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 || pConfig->abUnindexed[i] ){
23465 sqlite3_str_appendfsqlite3_api->str_appendf(pDefn, ", c%d", i);
23466 }
23467 }
23468 if( pConfig->bLocale ){
23469 for(i=0; i<pConfig->nCol; i++){
23470 if( pConfig->abUnindexed[i]==0 ){
23471 sqlite3_str_appendfsqlite3_api->str_appendf(pDefn, ", l%d", i);
23472 }
23473 }
23474 }
23475 zDefn = sqlite3_str_finishsqlite3_api->str_finish(pDefn);
23476
23477 if( zDefn ){
23478 rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
23479 sqlite3_freesqlite3_api->free(zDefn);
23480 }else{
23481 rc = SQLITE_NOMEM7;
23482 }
23483 }
23484
23485 if( rc==SQLITE_OK0 && pConfig->bColumnsize ){
23486 const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
23487 if( pConfig->bContentlessDelete ){
23488 zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER";
23489 }
23490 rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
23491 }
23492 if( rc==SQLITE_OK0 ){
23493 rc = sqlite3Fts5CreateTable(
23494 pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
23495 );
23496 }
23497 if( rc==SQLITE_OK0 ){
23498 rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION4);
23499 }
23500 }
23501
23502 if( rc ){
23503 sqlite3Fts5StorageClose(p);
23504 *pp = 0;
23505 }
23506 return rc;
23507}
23508
23509/*
23510** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen().
23511*/
23512static int sqlite3Fts5StorageClose(Fts5Storage *p){
23513 int rc = SQLITE_OK0;
23514 if( p ){
23515 int i;
23516
23517 /* Finalize all SQL statements */
23518 for(i=0; i<ArraySize(p->aStmt)((int)(sizeof(p->aStmt) / sizeof(p->aStmt[0]))); i++){
23519 sqlite3_finalizesqlite3_api->finalize(p->aStmt[i]);
23520 }
23521
23522 sqlite3_freesqlite3_api->free(p);
23523 }
23524 return rc;
23525}
23526
23527typedef struct Fts5InsertCtx Fts5InsertCtx;
23528struct Fts5InsertCtx {
23529 Fts5Storage *pStorage;
23530 int iCol;
23531 int szCol; /* Size of column value in tokens */
23532};
23533
23534/*
23535** Tokenization callback used when inserting tokens into the FTS index.
23536*/
23537static int fts5StorageInsertCallback(
23538 void *pContext, /* Pointer to Fts5InsertCtx object */
23539 int tflags,
23540 const char *pToken, /* Buffer containing token */
23541 int nToken, /* Size of token in bytes */
23542 int iUnused1, /* Start offset of token */
23543 int iUnused2 /* End offset of token */
23544){
23545 Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
23546 Fts5Index *pIdx = pCtx->pStorage->pIndex;
23547 UNUSED_PARAM2(iUnused1, iUnused2)(void)(iUnused1), (void)(iUnused2);
23548 if( nToken>FTS5_MAX_TOKEN_SIZE32768 ) nToken = FTS5_MAX_TOKEN_SIZE32768;
23549 if( (tflags & FTS5_TOKEN_COLOCATED0x0001)==0 || pCtx->szCol==0 ){
23550 pCtx->szCol++;
23551 }
23552 return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
23553}
23554
23555/*
23556** This function is used as part of an UPDATE statement that modifies the
23557** rowid of a row. In that case, this function is called first to set
23558** Fts5Storage.pSavedRow to point to a statement that may be used to
23559** access the original values of the row being deleted - iDel.
23560**
23561** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
23562** It is not considered an error if row iDel does not exist. In this case
23563** pSavedRow is not set and SQLITE_OK returned.
23564*/
23565static int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){
23566 int rc = SQLITE_OK0;
23567 sqlite3_stmt *pSeek = 0;
23568
23569 assert( p->pSavedRow==0 )((void) (0));
23570 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP2+1, &pSeek, 0);
23571 if( rc==SQLITE_OK0 ){
23572 sqlite3_bind_int64sqlite3_api->bind_int64(pSeek, 1, iDel);
23573 if( sqlite3_stepsqlite3_api->step(pSeek)!=SQLITE_ROW100 ){
23574 rc = sqlite3_resetsqlite3_api->reset(pSeek);
23575 }else{
23576 p->pSavedRow = pSeek;
23577 }
23578 }
23579
23580 return rc;
23581}
23582
23583/*
23584** If a row with rowid iDel is present in the %_content table, add the
23585** delete-markers to the FTS index necessary to delete it. Do not actually
23586** remove the %_content row at this time though.
23587**
23588** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left
23589** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access
23590** the original values of the row being deleted. This is used by UPDATE
23591** statements.
23592*/
23593static int fts5StorageDeleteFromIndex(
23594 Fts5Storage *p,
23595 i64 iDel,
23596 sqlite3_value **apVal,
23597 int bSaveRow /* True to set pSavedRow */
23598){
23599 Fts5Config *pConfig = p->pConfig;
23600 sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */
23601 int rc = SQLITE_OK0; /* Return code */
23602 int rc2; /* sqlite3_reset() return code */
23603 int iCol;
23604 Fts5InsertCtx ctx;
23605
23606 assert( bSaveRow==0 || apVal==0 )((void) (0));
23607 assert( bSaveRow==0 || bSaveRow==1 )((void) (0));
23608 assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 )((void) (0));
23609
23610 if( apVal==0 ){
23611 if( p->pSavedRow && bSaveRow ){
23612 pSeek = p->pSavedRow;
23613 p->pSavedRow = 0;
23614 }else{
23615 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP2+bSaveRow, &pSeek, 0);
23616 if( rc!=SQLITE_OK0 ) return rc;
23617 sqlite3_bind_int64sqlite3_api->bind_int64(pSeek, 1, iDel);
23618 if( sqlite3_stepsqlite3_api->step(pSeek)!=SQLITE_ROW100 ){
23619 return sqlite3_resetsqlite3_api->reset(pSeek);
23620 }
23621 }
23622 }
23623
23624 ctx.pStorage = p;
23625 ctx.iCol = -1;
23626 for(iCol=1; rc==SQLITE_OK0 && iCol<=pConfig->nCol; iCol++){
23627 if( pConfig->abUnindexed[iCol-1]==0 ){
23628 sqlite3_value *pVal = 0;
23629 sqlite3_value *pFree = 0;
23630 const char *pText = 0;
23631 int nText = 0;
23632 const char *pLoc = 0;
23633 int nLoc = 0;
23634
23635 assert( pSeek==0 || apVal==0 )((void) (0));
23636 assert( pSeek!=0 || apVal!=0 )((void) (0));
23637 if( pSeek ){
23638 pVal = sqlite3_column_valuesqlite3_api->column_value(pSeek, iCol);
23639 }else{
23640 pVal = apVal[iCol-1];
23641 }
23642
23643 if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
23644 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
23645 }else{
23646 if( sqlite3_value_typesqlite3_api->value_type(pVal)!=SQLITE_TEXT3 ){
23647 /* Make a copy of the value to work with. This is because the call
23648 ** to sqlite3_value_text() below forces the type of the value to
23649 ** SQLITE_TEXT, and we may need to use it again later. */
23650 pFree = pVal = sqlite3_value_dupsqlite3_api->value_dup(pVal);
23651 if( pVal==0 ){
23652 rc = SQLITE_NOMEM7;
23653 }
23654 }
23655 if( rc==SQLITE_OK0 ){
23656 pText = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
23657 nText = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
23658 if( pConfig->bLocale && pSeek ){
23659 pLoc = (const char*)sqlite3_column_textsqlite3_api->column_text(pSeek, iCol+pConfig->nCol);
23660 nLoc = sqlite3_column_bytessqlite3_api->column_bytes(pSeek, iCol + pConfig->nCol);
23661 }
23662 }
23663 }
23664
23665 if( rc==SQLITE_OK0 ){
23666 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
23667 ctx.szCol = 0;
23668 rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT0x0004,
23669 pText, nText, (void*)&ctx, fts5StorageInsertCallback
23670 );
23671 p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
23672 if( rc==SQLITE_OK0 && p->aTotalSize[iCol-1]<0 ){
23673 rc = FTS5_CORRUPT(11 | (1<<8));
23674 }
23675 sqlite3Fts5ClearLocale(pConfig);
23676 }
23677 sqlite3_value_freesqlite3_api->value_free(pFree);
23678 }
23679 }
23680 if( rc==SQLITE_OK0 && p->nTotalRow<1 ){
23681 rc = FTS5_CORRUPT(11 | (1<<8));
23682 }else{
23683 p->nTotalRow--;
23684 }
23685
23686 if( rc==SQLITE_OK0 && bSaveRow ){
23687 assert( p->pSavedRow==0 )((void) (0));
23688 p->pSavedRow = pSeek;
23689 }else{
23690 rc2 = sqlite3_resetsqlite3_api->reset(pSeek);
23691 if( rc==SQLITE_OK0 ) rc = rc2;
23692 }
23693 return rc;
23694}
23695
23696/*
23697** Reset any saved statement pSavedRow. Zero pSavedRow as well. This
23698** should be called by the xUpdate() method of the fts5 table before
23699** returning from any operation that may have set Fts5Storage.pSavedRow.
23700*/
23701static void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){
23702 assert( pStorage->pSavedRow==0((void) (0))
23703 || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2]((void) (0))
23704 )((void) (0));
23705 sqlite3_resetsqlite3_api->reset(pStorage->pSavedRow);
23706 pStorage->pSavedRow = 0;
23707}
23708
23709/*
23710** This function is called to process a DELETE on a contentless_delete=1
23711** table. It adds the tombstone required to delete the entry with rowid
23712** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs,
23713** an SQLite error code.
23714*/
23715static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
23716 i64 iOrigin = 0;
23717 sqlite3_stmt *pLookup = 0;
23718 int rc = SQLITE_OK0;
23719
23720 assert( p->pConfig->bContentlessDelete )((void) (0));
23721 assert( p->pConfig->eContent==FTS5_CONTENT_NONE((void) (0))
23722 || p->pConfig->eContent==FTS5_CONTENT_UNINDEXED((void) (0))
23723 )((void) (0));
23724
23725 /* Look up the origin of the document in the %_docsize table. Store
23726 ** this in stack variable iOrigin. */
23727 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE9, &pLookup, 0);
23728 if( rc==SQLITE_OK0 ){
23729 sqlite3_bind_int64sqlite3_api->bind_int64(pLookup, 1, iDel);
23730 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pLookup) ){
23731 iOrigin = sqlite3_column_int64sqlite3_api->column_int64(pLookup, 1);
23732 }
23733 rc = sqlite3_resetsqlite3_api->reset(pLookup);
23734 }
23735
23736 if( rc==SQLITE_OK0 && iOrigin!=0 ){
23737 rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel);
23738 }
23739
23740 return rc;
23741}
23742
23743/*
23744** Insert a record into the %_docsize table. Specifically, do:
23745**
23746** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
23747**
23748** If there is no %_docsize table (as happens if the columnsize=0 option
23749** is specified when the FTS5 table is created), this function is a no-op.
23750*/
23751static int fts5StorageInsertDocsize(
23752 Fts5Storage *p, /* Storage module to write to */
23753 i64 iRowid, /* id value */
23754 Fts5Buffer *pBuf /* sz value */
23755){
23756 int rc = SQLITE_OK0;
23757 if( p->pConfig->bColumnsize ){
23758 sqlite3_stmt *pReplace = 0;
23759 rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE7, &pReplace, 0);
23760 if( rc==SQLITE_OK0 ){
23761 sqlite3_bind_int64sqlite3_api->bind_int64(pReplace, 1, iRowid);
23762 if( p->pConfig->bContentlessDelete ){
23763 i64 iOrigin = 0;
23764 rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
23765 sqlite3_bind_int64sqlite3_api->bind_int64(pReplace, 3, iOrigin);
23766 }
23767 }
23768 if( rc==SQLITE_OK0 ){
23769 sqlite3_bind_blobsqlite3_api->bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC((sqlite3_destructor_type)0));
23770 sqlite3_stepsqlite3_api->step(pReplace);
23771 rc = sqlite3_resetsqlite3_api->reset(pReplace);
23772 sqlite3_bind_nullsqlite3_api->bind_null(pReplace, 2);
23773 }
23774 }
23775 return rc;
23776}
23777
23778/*
23779** Load the contents of the "averages" record from disk into the
23780** p->nTotalRow and p->aTotalSize[] variables. If successful, and if
23781** argument bCache is true, set the p->bTotalsValid flag to indicate
23782** that the contents of aTotalSize[] and nTotalRow are valid until
23783** further notice.
23784**
23785** Return SQLITE_OK if successful, or an SQLite error code if an error
23786** occurs.
23787*/
23788static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){
23789 int rc = SQLITE_OK0;
23790 if( p->bTotalsValid==0 ){
23791 rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize);
23792 p->bTotalsValid = bCache;
23793 }
23794 return rc;
23795}
23796
23797/*
23798** Store the current contents of the p->nTotalRow and p->aTotalSize[]
23799** variables in the "averages" record on disk.
23800**
23801** Return SQLITE_OK if successful, or an SQLite error code if an error
23802** occurs.
23803*/
23804static int fts5StorageSaveTotals(Fts5Storage *p){
23805 int nCol = p->pConfig->nCol;
23806 int i;
23807 Fts5Buffer buf;
23808 int rc = SQLITE_OK0;
23809 memset(&buf, 0, sizeof(buf));
23810
23811 sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow);
23812 for(i=0; i<nCol; i++){
23813 sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]);
23814 }
23815 if( rc==SQLITE_OK0 ){
23816 rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n);
23817 }
23818 sqlite3_freesqlite3_api->free(buf.p);
23819
23820 return rc;
23821}
23822
23823/*
23824** Remove a row from the FTS table.
23825*/
23826static int sqlite3Fts5StorageDelete(
23827 Fts5Storage *p, /* Storage object */
23828 i64 iDel, /* Rowid to delete from table */
23829 sqlite3_value **apVal, /* Optional - values to remove from index */
23830 int bSaveRow /* If true, set pSavedRow for deleted row */
23831){
23832 Fts5Config *pConfig = p->pConfig;
23833 int rc;
23834 sqlite3_stmt *pDel = 0;
23835
23836 assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 )((void) (0));
23837 rc = fts5StorageLoadTotals(p, 1);
23838
23839 /* Delete the index records */
23840 if( rc==SQLITE_OK0 ){
23841 rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
23842 }
23843
23844 if( rc==SQLITE_OK0 ){
23845 if( p->pConfig->bContentlessDelete ){
23846 rc = fts5StorageContentlessDelete(p, iDel);
23847 if( rc==SQLITE_OK0
23848 && bSaveRow
23849 && p->pConfig->eContent==FTS5_CONTENT_UNINDEXED3
23850 ){
23851 rc = sqlite3Fts5StorageFindDeleteRow(p, iDel);
23852 }
23853 }else{
23854 rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow);
23855 }
23856 }
23857
23858 /* Delete the %_docsize record */
23859 if( rc==SQLITE_OK0 && pConfig->bColumnsize ){
23860 rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE8, &pDel, 0);
23861 if( rc==SQLITE_OK0 ){
23862 sqlite3_bind_int64sqlite3_api->bind_int64(pDel, 1, iDel);
23863 sqlite3_stepsqlite3_api->step(pDel);
23864 rc = sqlite3_resetsqlite3_api->reset(pDel);
23865 }
23866 }
23867
23868 /* Delete the %_content record */
23869 if( pConfig->eContent==FTS5_CONTENT_NORMAL0
23870 || pConfig->eContent==FTS5_CONTENT_UNINDEXED3
23871 ){
23872 if( rc==SQLITE_OK0 ){
23873 rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT6, &pDel, 0);
23874 }
23875 if( rc==SQLITE_OK0 ){
23876 sqlite3_bind_int64sqlite3_api->bind_int64(pDel, 1, iDel);
23877 sqlite3_stepsqlite3_api->step(pDel);
23878 rc = sqlite3_resetsqlite3_api->reset(pDel);
23879 }
23880 }
23881
23882 return rc;
23883}
23884
23885/*
23886** Delete all entries in the FTS5 index.
23887*/
23888static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
23889 Fts5Config *pConfig = p->pConfig;
23890 int rc;
23891
23892 p->bTotalsValid = 0;
23893
23894 /* Delete the contents of the %_data and %_docsize tables. */
23895 rc = fts5ExecPrintf(pConfig->db, 0,
23896 "DELETE FROM %Q.'%q_data';"
23897 "DELETE FROM %Q.'%q_idx';",
23898 pConfig->zDb, pConfig->zName,
23899 pConfig->zDb, pConfig->zName
23900 );
23901 if( rc==SQLITE_OK0 && pConfig->bColumnsize ){
23902 rc = fts5ExecPrintf(pConfig->db, 0,
23903 "DELETE FROM %Q.'%q_docsize';", pConfig->zDb, pConfig->zName
23904 );
23905 }
23906
23907 if( rc==SQLITE_OK0 && pConfig->eContent==FTS5_CONTENT_UNINDEXED3 ){
23908 rc = fts5ExecPrintf(pConfig->db, 0,
23909 "DELETE FROM %Q.'%q_content';", pConfig->zDb, pConfig->zName
23910 );
23911 }
23912
23913 /* Reinitialize the %_data table. This call creates the initial structure
23914 ** and averages records. */
23915 if( rc==SQLITE_OK0 ){
23916 rc = sqlite3Fts5IndexReinit(p->pIndex);
23917 }
23918 if( rc==SQLITE_OK0 ){
23919 rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION4);
23920 }
23921 return rc;
23922}
23923
23924static int sqlite3Fts5StorageRebuild(Fts5Storage *p){
23925 Fts5Buffer buf = {0,0,0};
23926 Fts5Config *pConfig = p->pConfig;
23927 sqlite3_stmt *pScan = 0;
23928 Fts5InsertCtx ctx;
23929 int rc, rc2;
23930
23931 memset(&ctx, 0, sizeof(Fts5InsertCtx));
23932 ctx.pStorage = p;
23933 rc = sqlite3Fts5StorageDeleteAll(p);
23934 if( rc==SQLITE_OK0 ){
23935 rc = fts5StorageLoadTotals(p, 1);
23936 }
23937
23938 if( rc==SQLITE_OK0 ){
23939 rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN11, &pScan, pConfig->pzErrmsg);
23940 }
23941
23942 while( rc==SQLITE_OK0 && SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pScan) ){
23943 i64 iRowid = sqlite3_column_int64sqlite3_api->column_int64(pScan, 0);
23944
23945 sqlite3Fts5BufferZero(&buf);
23946 rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
23947 for(ctx.iCol=0; rc==SQLITE_OK0 && ctx.iCol<pConfig->nCol; ctx.iCol++){
23948 ctx.szCol = 0;
23949 if( pConfig->abUnindexed[ctx.iCol]==0 ){
23950 int nText = 0; /* Size of pText in bytes */
23951 const char *pText = 0; /* Pointer to buffer containing text value */
23952 int nLoc = 0; /* Size of pLoc in bytes */
23953 const char *pLoc = 0; /* Pointer to buffer containing text value */
23954
23955 sqlite3_value *pVal = sqlite3_column_valuesqlite3_api->column_value(pScan, ctx.iCol+1);
23956 if( pConfig->eContent==FTS5_CONTENT_EXTERNAL2
23957 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
23958 ){
23959 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
23960 }else{
23961 pText = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
23962 nText = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
23963 if( pConfig->bLocale ){
23964 int iCol = ctx.iCol + 1 + pConfig->nCol;
23965 pLoc = (const char*)sqlite3_column_textsqlite3_api->column_text(pScan, iCol);
23966 nLoc = sqlite3_column_bytessqlite3_api->column_bytes(pScan, iCol);
23967 }
23968 }
23969
23970 if( rc==SQLITE_OK0 ){
23971 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
23972 rc = sqlite3Fts5Tokenize(pConfig,
23973 FTS5_TOKENIZE_DOCUMENT0x0004,
23974 pText, nText,
23975 (void*)&ctx,
23976 fts5StorageInsertCallback
23977 );
23978 sqlite3Fts5ClearLocale(pConfig);
23979 }
23980 }
23981 sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
23982 p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
23983 }
23984 p->nTotalRow++;
23985
23986 if( rc==SQLITE_OK0 ){
23987 rc = fts5StorageInsertDocsize(p, iRowid, &buf);
23988 }
23989 }
23990 sqlite3_freesqlite3_api->free(buf.p);
23991 rc2 = sqlite3_resetsqlite3_api->reset(pScan);
23992 if( rc==SQLITE_OK0 ) rc = rc2;
23993
23994 /* Write the averages record */
23995 if( rc==SQLITE_OK0 ){
23996 rc = fts5StorageSaveTotals(p);
23997 }
23998 return rc;
23999}
24000
24001static int sqlite3Fts5StorageOptimize(Fts5Storage *p){
24002 return sqlite3Fts5IndexOptimize(p->pIndex);
24003}
24004
24005static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
24006 return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
24007}
24008
24009static int sqlite3Fts5StorageReset(Fts5Storage *p){
24010 return sqlite3Fts5IndexReset(p->pIndex);
24011}
24012
24013/*
24014** Allocate a new rowid. This is used for "external content" tables when
24015** a NULL value is inserted into the rowid column. The new rowid is allocated
24016** by inserting a dummy row into the %_docsize table. The dummy will be
24017** overwritten later.
24018**
24019** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In
24020** this case the user is required to provide a rowid explicitly.
24021*/
24022static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
24023 int rc = SQLITE_MISMATCH20;
24024 if( p->pConfig->bColumnsize ){
24025 sqlite3_stmt *pReplace = 0;
24026 rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE7, &pReplace, 0);
24027 if( rc==SQLITE_OK0 ){
24028 sqlite3_bind_nullsqlite3_api->bind_null(pReplace, 1);
24029 sqlite3_bind_nullsqlite3_api->bind_null(pReplace, 2);
24030 sqlite3_stepsqlite3_api->step(pReplace);
24031 rc = sqlite3_resetsqlite3_api->reset(pReplace);
24032 }
24033 if( rc==SQLITE_OK0 ){
24034 *piRowid = sqlite3_last_insert_rowidsqlite3_api->last_insert_rowid(p->pConfig->db);
24035 }
24036 }
24037 return rc;
24038}
24039
24040/*
24041** Insert a new row into the FTS content table.
24042*/
24043static int sqlite3Fts5StorageContentInsert(
24044 Fts5Storage *p,
24045 int bReplace, /* True to use REPLACE instead of INSERT */
24046 sqlite3_value **apVal,
24047 i64 *piRowid
24048){
24049 Fts5Config *pConfig = p->pConfig;
24050 int rc = SQLITE_OK0;
24051
24052 /* Insert the new row into the %_content table. */
24053 if( pConfig->eContent!=FTS5_CONTENT_NORMAL0
24054 && pConfig->eContent!=FTS5_CONTENT_UNINDEXED3
24055 ){
24056 if( sqlite3_value_typesqlite3_api->value_type(apVal[1])==SQLITE_INTEGER1 ){
24057 *piRowid = sqlite3_value_int64sqlite3_api->value_int64(apVal[1]);
24058 }else{
24059 rc = fts5StorageNewRowid(p, piRowid);
24060 }
24061 }else{
24062 sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */
24063 int i; /* Counter variable */
24064
24065 assert( FTS5_STMT_INSERT_CONTENT+1==FTS5_STMT_REPLACE_CONTENT )((void) (0));
24066 assert( bReplace==0 || bReplace==1 )((void) (0));
24067 rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT4+bReplace, &pInsert, 0);
24068 if( pInsert ) sqlite3_clear_bindingssqlite3_api->clear_bindings(pInsert);
24069
24070 /* Bind the rowid value */
24071 sqlite3_bind_valuesqlite3_api->bind_value(pInsert, 1, apVal[1]);
24072
24073 /* Loop through values for user-defined columns. i=2 is the leftmost
24074 ** user-defined column. As is column 1 of pSavedRow. */
24075 for(i=2; rc==SQLITE_OK0 && i<=pConfig->nCol+1; i++){
24076 int bUnindexed = pConfig->abUnindexed[i-2];
24077 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 || bUnindexed ){
24078 sqlite3_value *pVal = apVal[i];
24079
24080 if( sqlite3_value_nochangesqlite3_api->value_nochange(pVal) && p->pSavedRow ){
24081 /* This is an UPDATE statement, and user-defined column (i-2) was not
24082 ** modified. Retrieve the value from Fts5Storage.pSavedRow. */
24083 pVal = sqlite3_column_valuesqlite3_api->column_value(p->pSavedRow, i-1);
24084 if( pConfig->bLocale && bUnindexed==0 ){
24085 sqlite3_bind_valuesqlite3_api->bind_value(pInsert, pConfig->nCol + i,
24086 sqlite3_column_valuesqlite3_api->column_value(p->pSavedRow, pConfig->nCol + i - 1)
24087 );
24088 }
24089 }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
24090 const char *pText = 0;
24091 const char *pLoc = 0;
24092 int nText = 0;
24093 int nLoc = 0;
24094 assert( pConfig->bLocale )((void) (0));
24095
24096 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
24097 if( rc==SQLITE_OK0 ){
24098 sqlite3_bind_textsqlite3_api->bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
24099 if( bUnindexed==0 ){
24100 int iLoc = pConfig->nCol + i;
24101 sqlite3_bind_textsqlite3_api->bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT((sqlite3_destructor_type)-1));
24102 }
24103 }
24104
24105 continue;
24106 }
24107
24108 rc = sqlite3_bind_valuesqlite3_api->bind_value(pInsert, i, pVal);
24109 }
24110 }
24111 if( rc==SQLITE_OK0 ){
24112 sqlite3_stepsqlite3_api->step(pInsert);
24113 rc = sqlite3_resetsqlite3_api->reset(pInsert);
24114 }
24115 *piRowid = sqlite3_last_insert_rowidsqlite3_api->last_insert_rowid(pConfig->db);
24116 }
24117
24118 return rc;
24119}
24120
24121/*
24122** Insert new entries into the FTS index and %_docsize table.
24123*/
24124static int sqlite3Fts5StorageIndexInsert(
24125 Fts5Storage *p,
24126 sqlite3_value **apVal,
24127 i64 iRowid
24128){
24129 Fts5Config *pConfig = p->pConfig;
24130 int rc = SQLITE_OK0; /* Return code */
24131 Fts5InsertCtx ctx; /* Tokenization callback context object */
24132 Fts5Buffer buf; /* Buffer used to build up %_docsize blob */
24133
24134 memset(&buf, 0, sizeof(Fts5Buffer));
24135 ctx.pStorage = p;
24136 rc = fts5StorageLoadTotals(p, 1);
24137
24138 if( rc==SQLITE_OK0 ){
24139 rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
24140 }
24141 for(ctx.iCol=0; rc==SQLITE_OK0 && ctx.iCol<pConfig->nCol; ctx.iCol++){
24142 ctx.szCol = 0;
24143 if( pConfig->abUnindexed[ctx.iCol]==0 ){
24144 int nText = 0; /* Size of pText in bytes */
24145 const char *pText = 0; /* Pointer to buffer containing text value */
24146 int nLoc = 0; /* Size of pText in bytes */
24147 const char *pLoc = 0; /* Pointer to buffer containing text value */
24148
24149 sqlite3_value *pVal = apVal[ctx.iCol+2];
24150 if( p->pSavedRow && sqlite3_value_nochangesqlite3_api->value_nochange(pVal) ){
24151 pVal = sqlite3_column_valuesqlite3_api->column_value(p->pSavedRow, ctx.iCol+1);
24152 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 && pConfig->bLocale ){
24153 int iCol = ctx.iCol + 1 + pConfig->nCol;
24154 pLoc = (const char*)sqlite3_column_textsqlite3_api->column_text(p->pSavedRow, iCol);
24155 nLoc = sqlite3_column_bytessqlite3_api->column_bytes(p->pSavedRow, iCol);
24156 }
24157 }else{
24158 pVal = apVal[ctx.iCol+2];
24159 }
24160
24161 if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
24162 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
24163 }else{
24164 pText = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
24165 nText = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
24166 }
24167
24168 if( rc==SQLITE_OK0 ){
24169 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
24170 rc = sqlite3Fts5Tokenize(pConfig,
24171 FTS5_TOKENIZE_DOCUMENT0x0004, pText, nText, (void*)&ctx,
24172 fts5StorageInsertCallback
24173 );
24174 sqlite3Fts5ClearLocale(pConfig);
24175 }
24176 }
24177 sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
24178 p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
24179 }
24180 p->nTotalRow++;
24181
24182 /* Write the %_docsize record */
24183 if( rc==SQLITE_OK0 ){
24184 rc = fts5StorageInsertDocsize(p, iRowid, &buf);
24185 }
24186 sqlite3_freesqlite3_api->free(buf.p);
24187
24188 return rc;
24189}
24190
24191static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){
24192 Fts5Config *pConfig = p->pConfig;
24193 char *zSql;
24194 int rc;
24195
24196 zSql = sqlite3_mprintfsqlite3_api->mprintf("SELECT count(*) FROM %Q.'%q_%s'",
24197 pConfig->zDb, pConfig->zName, zSuffix
24198 );
24199 if( zSql==0 ){
24200 rc = SQLITE_NOMEM7;
24201 }else{
24202 sqlite3_stmt *pCnt = 0;
24203 rc = sqlite3_prepare_v2sqlite3_api->prepare_v2(pConfig->db, zSql, -1, &pCnt, 0);
24204 if( rc==SQLITE_OK0 ){
24205 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pCnt) ){
24206 *pnRow = sqlite3_column_int64sqlite3_api->column_int64(pCnt, 0);
24207 }
24208 rc = sqlite3_finalizesqlite3_api->finalize(pCnt);
24209 }
24210 }
24211
24212 sqlite3_freesqlite3_api->free(zSql);
24213 return rc;
24214}
24215
24216/*
24217** Context object used by sqlite3Fts5StorageIntegrity().
24218*/
24219typedef struct Fts5IntegrityCtx Fts5IntegrityCtx;
24220struct Fts5IntegrityCtx {
24221 i64 iRowid;
24222 int iCol;
24223 int szCol;
24224 u64 cksum;
24225 Fts5Termset *pTermset;
24226 Fts5Config *pConfig;
24227};
24228
24229
24230/*
24231** Tokenization callback used by integrity check.
24232*/
24233static int fts5StorageIntegrityCallback(
24234 void *pContext, /* Pointer to Fts5IntegrityCtx object */
24235 int tflags,
24236 const char *pToken, /* Buffer containing token */
24237 int nToken, /* Size of token in bytes */
24238 int iUnused1, /* Start offset of token */
24239 int iUnused2 /* End offset of token */
24240){
24241 Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
24242 Fts5Termset *pTermset = pCtx->pTermset;
24243 int bPresent;
24244 int ii;
24245 int rc = SQLITE_OK0;
24246 int iPos;
24247 int iCol;
24248
24249 UNUSED_PARAM2(iUnused1, iUnused2)(void)(iUnused1), (void)(iUnused2);
24250 if( nToken>FTS5_MAX_TOKEN_SIZE32768 ) nToken = FTS5_MAX_TOKEN_SIZE32768;
24251
24252 if( (tflags & FTS5_TOKEN_COLOCATED0x0001)==0 || pCtx->szCol==0 ){
24253 pCtx->szCol++;
24254 }
24255
24256 switch( pCtx->pConfig->eDetail ){
24257 case FTS5_DETAIL_FULL0:
24258 iPos = pCtx->szCol-1;
24259 iCol = pCtx->iCol;
24260 break;
24261
24262 case FTS5_DETAIL_COLUMNS2:
24263 iPos = pCtx->iCol;
24264 iCol = 0;
24265 break;
24266
24267 default:
24268 assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE )((void) (0));
24269 iPos = 0;
24270 iCol = 0;
24271 break;
24272 }
24273
24274 rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
24275 if( rc==SQLITE_OK0 && bPresent==0 ){
24276 pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
24277 pCtx->iRowid, iCol, iPos, 0, pToken, nToken
24278 );
24279 }
24280
24281 for(ii=0; rc==SQLITE_OK0 && ii<pCtx->pConfig->nPrefix; ii++){
24282 const int nChar = pCtx->pConfig->aPrefix[ii];
24283 int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
24284 if( nByte ){
24285 rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent);
24286 if( bPresent==0 ){
24287 pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
24288 pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte
24289 );
24290 }
24291 }
24292 }
24293
24294 return rc;
24295}
24296
24297/*
24298** Check that the contents of the FTS index match that of the %_content
24299** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return
24300** some other SQLite error code if an error occurs while attempting to
24301** determine this.
24302*/
24303static int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
24304 Fts5Config *pConfig = p->pConfig;
24305 int rc = SQLITE_OK0; /* Return code */
24306 int *aColSize; /* Array of size pConfig->nCol */
24307 i64 *aTotalSize; /* Array of size pConfig->nCol */
24308 Fts5IntegrityCtx ctx;
24309 sqlite3_stmt *pScan;
24310 int bUseCksum;
24311
24312 memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
24313 ctx.pConfig = p->pConfig;
24314 aTotalSize = (i64*)sqlite3_malloc64sqlite3_api->malloc64(pConfig->nCol*(sizeof(int)+sizeof(i64)));
24315 if( !aTotalSize ) return SQLITE_NOMEM7;
24316 aColSize = (int*)&aTotalSize[pConfig->nCol];
24317 memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
24318
24319 bUseCksum = (pConfig->eContent==FTS5_CONTENT_NORMAL0
24320 || (pConfig->eContent==FTS5_CONTENT_EXTERNAL2 && iArg)
24321 );
24322 if( bUseCksum ){
24323 /* Generate the expected index checksum based on the contents of the
24324 ** %_content table. This block stores the checksum in ctx.cksum. */
24325 rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN11, &pScan, 0);
24326 if( rc==SQLITE_OK0 ){
24327 int rc2;
24328 while( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pScan) ){
24329 int i;
24330 ctx.iRowid = sqlite3_column_int64sqlite3_api->column_int64(pScan, 0);
24331 ctx.szCol = 0;
24332 if( pConfig->bColumnsize ){
24333 rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
24334 }
24335 if( rc==SQLITE_OK0 && pConfig->eDetail==FTS5_DETAIL_NONE1 ){
24336 rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
24337 }
24338 for(i=0; rc==SQLITE_OK0 && i<pConfig->nCol; i++){
24339 if( pConfig->abUnindexed[i]==0 ){
24340 const char *pText = 0;
24341 int nText = 0;
24342 const char *pLoc = 0;
24343 int nLoc = 0;
24344 sqlite3_value *pVal = sqlite3_column_valuesqlite3_api->column_value(pScan, i+1);
24345
24346 if( pConfig->eContent==FTS5_CONTENT_EXTERNAL2
24347 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
24348 ){
24349 rc = sqlite3Fts5DecodeLocaleValue(
24350 pVal, &pText, &nText, &pLoc, &nLoc
24351 );
24352 }else{
24353 if( pConfig->eContent==FTS5_CONTENT_NORMAL0 && pConfig->bLocale ){
24354 int iCol = i + 1 + pConfig->nCol;
24355 pLoc = (const char*)sqlite3_column_textsqlite3_api->column_text(pScan, iCol);
24356 nLoc = sqlite3_column_bytessqlite3_api->column_bytes(pScan, iCol);
24357 }
24358 pText = (const char*)sqlite3_value_textsqlite3_api->value_text(pVal);
24359 nText = sqlite3_value_bytessqlite3_api->value_bytes(pVal);
24360 }
24361
24362 ctx.iCol = i;
24363 ctx.szCol = 0;
24364
24365 if( rc==SQLITE_OK0 && pConfig->eDetail==FTS5_DETAIL_COLUMNS2 ){
24366 rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
24367 }
24368
24369 if( rc==SQLITE_OK0 ){
24370 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
24371 rc = sqlite3Fts5Tokenize(pConfig,
24372 FTS5_TOKENIZE_DOCUMENT0x0004,
24373 pText, nText,
24374 (void*)&ctx,
24375 fts5StorageIntegrityCallback
24376 );
24377 sqlite3Fts5ClearLocale(pConfig);
24378 }
24379
24380 /* If this is not a columnsize=0 database, check that the number
24381 ** of tokens in the value matches the aColSize[] value read from
24382 ** the %_docsize table. */
24383 if( rc==SQLITE_OK0
24384 && pConfig->bColumnsize
24385 && ctx.szCol!=aColSize[i]
24386 ){
24387 rc = FTS5_CORRUPT(11 | (1<<8));
24388 }
24389 aTotalSize[i] += ctx.szCol;
24390 if( pConfig->eDetail==FTS5_DETAIL_COLUMNS2 ){
24391 sqlite3Fts5TermsetFree(ctx.pTermset);
24392 ctx.pTermset = 0;
24393 }
24394 }
24395 }
24396 sqlite3Fts5TermsetFree(ctx.pTermset);
24397 ctx.pTermset = 0;
24398
24399 if( rc!=SQLITE_OK0 ) break;
24400 }
24401 rc2 = sqlite3_resetsqlite3_api->reset(pScan);
24402 if( rc==SQLITE_OK0 ) rc = rc2;
24403 }
24404
24405 /* Test that the "totals" (sometimes called "averages") record looks Ok */
24406 if( rc==SQLITE_OK0 ){
24407 int i;
24408 rc = fts5StorageLoadTotals(p, 0);
24409 for(i=0; rc==SQLITE_OK0 && i<pConfig->nCol; i++){
24410 if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT(11 | (1<<8));
24411 }
24412 }
24413
24414 /* Check that the %_docsize and %_content tables contain the expected
24415 ** number of rows. */
24416 if( rc==SQLITE_OK0 && pConfig->eContent==FTS5_CONTENT_NORMAL0 ){
24417 i64 nRow = 0;
24418 rc = fts5StorageCount(p, "content", &nRow);
24419 if( rc==SQLITE_OK0 && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT(11 | (1<<8));
24420 }
24421 if( rc==SQLITE_OK0 && pConfig->bColumnsize ){
24422 i64 nRow = 0;
24423 rc = fts5StorageCount(p, "docsize", &nRow);
24424 if( rc==SQLITE_OK0 && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT(11 | (1<<8));
24425 }
24426 }
24427
24428 /* Pass the expected checksum down to the FTS index module. It will
24429 ** verify, amongst other things, that it matches the checksum generated by
24430 ** inspecting the index itself. */
24431 if( rc==SQLITE_OK0 ){
24432 rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum, bUseCksum);
24433 }
24434
24435 sqlite3_freesqlite3_api->free(aTotalSize);
24436 return rc;
24437}
24438
24439/*
24440** Obtain an SQLite statement handle that may be used to read data from the
24441** %_content table.
24442*/
24443static int sqlite3Fts5StorageStmt(
24444 Fts5Storage *p,
24445 int eStmt,
24446 sqlite3_stmt **pp,
24447 char **pzErrMsg
24448){
24449 int rc;
24450 assert( eStmt==FTS5_STMT_SCAN_ASC((void) (0))
24451 || eStmt==FTS5_STMT_SCAN_DESC((void) (0))
24452 || eStmt==FTS5_STMT_LOOKUP((void) (0))
24453 )((void) (0));
24454 rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg);
24455 if( rc==SQLITE_OK0 ){
24456 assert( p->aStmt[eStmt]==*pp )((void) (0));
24457 p->aStmt[eStmt] = 0;
24458 }
24459 return rc;
24460}
24461
24462/*
24463** Release an SQLite statement handle obtained via an earlier call to
24464** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function
24465** must match that passed to the sqlite3Fts5StorageStmt() call.
24466*/
24467static void sqlite3Fts5StorageStmtRelease(
24468 Fts5Storage *p,
24469 int eStmt,
24470 sqlite3_stmt *pStmt
24471){
24472 assert( eStmt==FTS5_STMT_SCAN_ASC((void) (0))
24473 || eStmt==FTS5_STMT_SCAN_DESC((void) (0))
24474 || eStmt==FTS5_STMT_LOOKUP((void) (0))
24475 )((void) (0));
24476 if( p->aStmt[eStmt]==0 ){
24477 sqlite3_resetsqlite3_api->reset(pStmt);
24478 p->aStmt[eStmt] = pStmt;
24479 }else{
24480 sqlite3_finalizesqlite3_api->finalize(pStmt);
24481 }
24482}
24483
24484static int fts5StorageDecodeSizeArray(
24485 int *aCol, int nCol, /* Array to populate */
24486 const u8 *aBlob, int nBlob /* Record to read varints from */
24487){
24488 int i;
24489 int iOff = 0;
24490 for(i=0; i<nCol; i++){
24491 if( iOff>=nBlob ) return 1;
24492 iOff += fts5GetVarint32(&aBlob[iOff], aCol[i])sqlite3Fts5GetVarint32(&aBlob[iOff],(u32*)&(aCol[i]));
24493 }
24494 return (iOff!=nBlob);
24495}
24496
24497/*
24498** Argument aCol points to an array of integers containing one entry for
24499** each table column. This function reads the %_docsize record for the
24500** specified rowid and populates aCol[] with the results.
24501**
24502** An SQLite error code is returned if an error occurs, or SQLITE_OK
24503** otherwise.
24504*/
24505static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
24506 int nCol = p->pConfig->nCol; /* Number of user columns in table */
24507 sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */
24508 int rc; /* Return Code */
24509
24510 assert( p->pConfig->bColumnsize )((void) (0));
24511 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE9, &pLookup, 0);
24512 if( pLookup ){
24513 int bCorrupt = 1;
24514 assert( rc==SQLITE_OK )((void) (0));
24515 sqlite3_bind_int64sqlite3_api->bind_int64(pLookup, 1, iRowid);
24516 if( SQLITE_ROW100==sqlite3_stepsqlite3_api->step(pLookup) ){
24517 const u8 *aBlob = sqlite3_column_blobsqlite3_api->column_blob(pLookup, 0);
24518 int nBlob = sqlite3_column_bytessqlite3_api->column_bytes(pLookup, 0);
24519 if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
24520 bCorrupt = 0;
24521 }
24522 }
24523 rc = sqlite3_resetsqlite3_api->reset(pLookup);
24524 if( bCorrupt && rc==SQLITE_OK0 ){
24525 rc = FTS5_CORRUPT(11 | (1<<8));
24526 }
24527 }else{
24528 assert( rc!=SQLITE_OK )((void) (0));
24529 }
24530
24531 return rc;
24532}
24533
24534static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
24535 int rc = fts5StorageLoadTotals(p, 0);
24536 if( rc==SQLITE_OK0 ){
24537 *pnToken = 0;
24538 if( iCol<0 ){
24539 int i;
24540 for(i=0; i<p->pConfig->nCol; i++){
24541 *pnToken += p->aTotalSize[i];
24542 }
24543 }else if( iCol<p->pConfig->nCol ){
24544 *pnToken = p->aTotalSize[iCol];
24545 }else{
24546 rc = SQLITE_RANGE25;
24547 }
24548 }
24549 return rc;
24550}
24551
24552static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
24553 int rc = fts5StorageLoadTotals(p, 0);
24554 if( rc==SQLITE_OK0 ){
24555 /* nTotalRow being zero does not necessarily indicate a corrupt
24556 ** database - it might be that the FTS5 table really does contain zero
24557 ** rows. However this function is only called from the xRowCount() API,
24558 ** and there is no way for that API to be invoked if the table contains
24559 ** no rows. Hence the FTS5_CORRUPT return. */
24560 *pnRow = p->nTotalRow;
24561 if( p->nTotalRow<=0 ) rc = FTS5_CORRUPT(11 | (1<<8));
24562 }
24563 return rc;
24564}
24565
24566/*
24567** Flush any data currently held in-memory to disk.
24568*/
24569static int sqlite3Fts5StorageSync(Fts5Storage *p){
24570 int rc = SQLITE_OK0;
24571 i64 iLastRowid = sqlite3_last_insert_rowidsqlite3_api->last_insert_rowid(p->pConfig->db);
24572 if( p->bTotalsValid ){
24573 rc = fts5StorageSaveTotals(p);
24574 if( rc==SQLITE_OK0 ){
24575 p->bTotalsValid = 0;
24576 }
24577 }
24578 if( rc==SQLITE_OK0 ){
24579 rc = sqlite3Fts5IndexSync(p->pIndex);
24580 }
24581 sqlite3_set_last_insert_rowidsqlite3_api->set_last_insert_rowid(p->pConfig->db, iLastRowid);
24582 return rc;
24583}
24584
24585static int sqlite3Fts5StorageRollback(Fts5Storage *p){
24586 p->bTotalsValid = 0;
24587 return sqlite3Fts5IndexRollback(p->pIndex);
24588}
24589
24590static int sqlite3Fts5StorageConfigValue(
24591 Fts5Storage *p,
24592 const char *z,
24593 sqlite3_value *pVal,
24594 int iVal
24595){
24596 sqlite3_stmt *pReplace = 0;
24597 int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG10, &pReplace, 0);
24598 if( rc==SQLITE_OK0 ){
24599 sqlite3_bind_textsqlite3_api->bind_text(pReplace, 1, z, -1, SQLITE_STATIC((sqlite3_destructor_type)0));
24600 if( pVal ){
24601 sqlite3_bind_valuesqlite3_api->bind_value(pReplace, 2, pVal);
24602 }else{
24603 sqlite3_bind_intsqlite3_api->bind_int(pReplace, 2, iVal);
24604 }
24605 sqlite3_stepsqlite3_api->step(pReplace);
24606 rc = sqlite3_resetsqlite3_api->reset(pReplace);
24607 sqlite3_bind_nullsqlite3_api->bind_null(pReplace, 1);
24608 }
24609 if( rc==SQLITE_OK0 && pVal ){
24610 int iNew = p->pConfig->iCookie + 1;
24611 rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
24612 if( rc==SQLITE_OK0 ){
24613 p->pConfig->iCookie = iNew;
24614 }
24615 }
24616 return rc;
24617}
24618
24619#line 1 "fts5_tokenize.c"
24620/*
24621** 2014 May 31
24622**
24623** The author disclaims copyright to this source code. In place of
24624** a legal notice, here is a blessing:
24625**
24626** May you do good and not evil.
24627** May you find forgiveness for yourself and forgive others.
24628** May you share freely, never taking more than you give.
24629**
24630******************************************************************************
24631*/
24632
24633
24634/* #include "fts5Int.h" */
24635
24636/**************************************************************************
24637** Start of ascii tokenizer implementation.
24638*/
24639
24640/*
24641** For tokenizers with no "unicode" modifier, the set of token characters
24642** is the same as the set of ASCII range alphanumeric characters.
24643*/
24644static unsigned char aAsciiTokenChar[128] = {
24645 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */
24646 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */
24647 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */
24648 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */
24649 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */
24650 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */
24651 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */
24652 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */
24653};
24654
24655typedef struct AsciiTokenizer AsciiTokenizer;
24656struct AsciiTokenizer {
24657 unsigned char aTokenChar[128];
24658};
24659
24660static void fts5AsciiAddExceptions(
24661 AsciiTokenizer *p,
24662 const char *zArg,
24663 int bTokenChars
24664){
24665 int i;
24666 for(i=0; zArg[i]; i++){
24667 if( (zArg[i] & 0x80)==0 ){
24668 p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars;
24669 }
24670 }
24671}
24672
24673/*
24674** Delete a "ascii" tokenizer.
24675*/
24676static void fts5AsciiDelete(Fts5Tokenizer *p){
24677 sqlite3_freesqlite3_api->free(p);
24678}
24679
24680/*
24681** Create an "ascii" tokenizer.
24682*/
24683static int fts5AsciiCreate(
24684 void *pUnused,
24685 const char **azArg, int nArg,
24686 Fts5Tokenizer **ppOut
24687){
24688 int rc = SQLITE_OK0;
24689 AsciiTokenizer *p = 0;
24690 UNUSED_PARAM(pUnused)(void)(pUnused);
24691 if( nArg%2 ){
24692 rc = SQLITE_ERROR1;
24693 }else{
24694 p = sqlite3_malloc64sqlite3_api->malloc64(sizeof(AsciiTokenizer));
24695 if( p==0 ){
24696 rc = SQLITE_NOMEM7;
24697 }else{
24698 int i;
24699 memset(p, 0, sizeof(AsciiTokenizer));
24700 memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
24701 for(i=0; rc==SQLITE_OK0 && i<nArg; i+=2){
24702 const char *zArg = azArg[i+1];
24703 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "tokenchars") ){
24704 fts5AsciiAddExceptions(p, zArg, 1);
24705 }else
24706 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "separators") ){
24707 fts5AsciiAddExceptions(p, zArg, 0);
24708 }else{
24709 rc = SQLITE_ERROR1;
24710 }
24711 }
24712 if( rc!=SQLITE_OK0 ){
24713 fts5AsciiDelete((Fts5Tokenizer*)p);
24714 p = 0;
24715 }
24716 }
24717 }
24718
24719 *ppOut = (Fts5Tokenizer*)p;
24720 return rc;
24721}
24722
24723
24724static void asciiFold(char *aOut, const char *aIn, int nByte){
24725 int i;
24726 for(i=0; i<nByte; i++){
24727 char c = aIn[i];
24728 if( c>='A' && c<='Z' ) c += 32;
24729 aOut[i] = c;
24730 }
24731}
24732
24733/*
24734** Tokenize some text using the ascii tokenizer.
24735*/
24736static int fts5AsciiTokenize(
24737 Fts5Tokenizer *pTokenizer,
24738 void *pCtx,
24739 int iUnused,
24740 const char *pText, int nText,
24741 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
24742){
24743 AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer;
24744 int rc = SQLITE_OK0;
24745 int ie;
24746 int is = 0;
24747
24748 char aFold[64];
24749 int nFold = sizeof(aFold);
24750 char *pFold = aFold;
24751 unsigned char *a = p->aTokenChar;
24752
24753 UNUSED_PARAM(iUnused)(void)(iUnused);
24754
24755 while( is<nText && rc==SQLITE_OK0 ){
24756 int nByte;
24757
24758 /* Skip any leading divider characters. */
24759 while( is<nText && ((pText[is]&0x80)==0 && a[(int)pText[is]]==0) ){
24760 is++;
24761 }
24762 if( is==nText ) break;
24763
24764 /* Count the token characters */
24765 ie = is+1;
24766 while( ie<nText && ((pText[ie]&0x80) || a[(int)pText[ie]] ) ){
24767 ie++;
24768 }
24769
24770 /* Fold to lower case */
24771 nByte = ie-is;
24772 if( nByte>nFold ){
24773 if( pFold!=aFold ) sqlite3_freesqlite3_api->free(pFold);
24774 pFold = sqlite3_malloc64sqlite3_api->malloc64((sqlite3_int64)nByte*2);
24775 if( pFold==0 ){
24776 rc = SQLITE_NOMEM7;
24777 break;
24778 }
24779 nFold = nByte*2;
24780 }
24781 asciiFold(pFold, &pText[is], nByte);
24782
24783 /* Invoke the token callback */
24784 rc = xToken(pCtx, 0, pFold, nByte, is, ie);
24785 is = ie+1;
24786 }
24787
24788 if( pFold!=aFold ) sqlite3_freesqlite3_api->free(pFold);
24789 if( rc==SQLITE_DONE101 ) rc = SQLITE_OK0;
24790 return rc;
24791}
24792
24793/**************************************************************************
24794** Start of unicode61 tokenizer implementation.
24795*/
24796
24797
24798/*
24799** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied
24800** from the sqlite3 source file utf.c. If this file is compiled as part
24801** of the amalgamation, they are not required.
24802*/
24803#ifndef SQLITE_AMALGAMATION
24804
24805static const unsigned char sqlite3Utf8Trans1[] = {
24806 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
24807 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
24808 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
24809 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
24810 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
24811 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
24812 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
24813 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
24814};
24815
24816#define READ_UTF8(zIn, zTerm, c)c = *(zIn++); if( c>=0xc0 ){ c = sqlite3Utf8Trans1[c-0xc0]
; while( zIn<zTerm && (*zIn & 0xc0)==0x80 ){ c
= (c<<6) + (0x3f & *(zIn++)); } if( c<0x80 || (
c&0xFFFFF800)==0xD800 || (c&0xFFFFFFFE)==0xFFFE ){ c =
0xFFFD; } } \
24817 c = *(zIn++); \
24818 if( c>=0xc0 ){ \
24819 c = sqlite3Utf8Trans1[c-0xc0]; \
24820 while( zIn<zTerm && (*zIn & 0xc0)==0x80 ){ \
24821 c = (c<<6) + (0x3f & *(zIn++)); \
24822 } \
24823 if( c<0x80 \
24824 || (c&0xFFFFF800)==0xD800 \
24825 || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
24826 }
24827
24828
24829#define WRITE_UTF8(zOut, c){ if( c<0x00080 ){ *zOut++ = (unsigned char)(c&0xFF); }
else if( c<0x00800 ){ *zOut++ = 0xC0 + (unsigned char)((c
>>6)&0x1F); *zOut++ = 0x80 + (unsigned char)(c &
0x3F); } else if( c<0x10000 ){ *zOut++ = 0xE0 + (unsigned
char)((c>>12)&0x0F); *zOut++ = 0x80 + (unsigned char
)((c>>6) & 0x3F); *zOut++ = 0x80 + (unsigned char)(
c & 0x3F); }else{ *zOut++ = 0xF0 + (unsigned char)((c>>
18) & 0x07); *zOut++ = 0x80 + (unsigned char)((c>>12
) & 0x3F); *zOut++ = 0x80 + (unsigned char)((c>>6) &
0x3F); *zOut++ = 0x80 + (unsigned char)(c & 0x3F); } } { \
24830 if( c<0x00080 ){ \
24831 *zOut++ = (unsigned char)(c&0xFF); \
24832 } \
24833 else if( c<0x00800 ){ \
24834 *zOut++ = 0xC0 + (unsigned char)((c>>6)&0x1F); \
24835 *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
24836 } \
24837 else if( c<0x10000 ){ \
24838 *zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \
24839 *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
24840 *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
24841 }else{ \
24842 *zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \
24843 *zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \
24844 *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
24845 *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
24846 } \
24847}
24848
24849#endif /* ifndef SQLITE_AMALGAMATION */
24850
24851#define FTS5_SKIP_UTF8(zIn){ if( ((unsigned char)(*(zIn++)))>=0xc0 ){ while( (((unsigned
char)*zIn) & 0xc0)==0x80 ){ zIn++; } } } { \
24852 if( ((unsigned char)(*(zIn++)))>=0xc0 ){ \
24853 while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; } \
24854 } \
24855}
24856
24857typedef struct Unicode61Tokenizer Unicode61Tokenizer;
24858struct Unicode61Tokenizer {
24859 unsigned char aTokenChar[128]; /* ASCII range token characters */
24860 char *aFold; /* Buffer to fold text into */
24861 int nFold; /* Size of aFold[] in bytes */
24862 int eRemoveDiacritic; /* True if remove_diacritics=1 is set */
24863 int nException;
24864 int *aiException;
24865
24866 unsigned char aCategory[32]; /* True for token char categories */
24867};
24868
24869/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
24870#define FTS5_REMOVE_DIACRITICS_NONE0 0
24871#define FTS5_REMOVE_DIACRITICS_SIMPLE1 1
24872#define FTS5_REMOVE_DIACRITICS_COMPLEX2 2
24873
24874static int fts5UnicodeAddExceptions(
24875 Unicode61Tokenizer *p, /* Tokenizer object */
24876 const char *z, /* Characters to treat as exceptions */
24877 int bTokenChars /* 1 for 'tokenchars', 0 for 'separators' */
24878){
24879 int rc = SQLITE_OK0;
24880 int n = (int)strlen(z);
24881 int *aNew;
24882
24883 if( n>0 ){
24884 aNew = (int*)sqlite3_realloc64sqlite3_api->realloc64(p->aiException,
24885 (n+p->nException)*sizeof(int));
24886 if( aNew ){
24887 int nNew = p->nException;
24888 const unsigned char *zCsr = (const unsigned char*)z;
24889 const unsigned char *zTerm = (const unsigned char*)&z[n];
24890 while( zCsr<zTerm ){
24891 u32 iCode;
24892 int bToken;
24893 READ_UTF8(zCsr, zTerm, iCode)iCode = *(zCsr++); if( iCode>=0xc0 ){ iCode = sqlite3Utf8Trans1
[iCode-0xc0]; while( zCsr<zTerm && (*zCsr & 0xc0
)==0x80 ){ iCode = (iCode<<6) + (0x3f & *(zCsr++));
} if( iCode<0x80 || (iCode&0xFFFFF800)==0xD800 || (iCode
&0xFFFFFFFE)==0xFFFE ){ iCode = 0xFFFD; } };
24894 if( iCode<128 ){
24895 p->aTokenChar[iCode] = (unsigned char)bTokenChars;
24896 }else{
24897 bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
24898 assert( (bToken==0 || bToken==1) )((void) (0));
24899 assert( (bTokenChars==0 || bTokenChars==1) )((void) (0));
24900 if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
24901 int i;
24902 for(i=0; i<nNew; i++){
24903 if( (u32)aNew[i]>iCode ) break;
24904 }
24905 memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int));
24906 aNew[i] = iCode;
24907 nNew++;
24908 }
24909 }
24910 }
24911 p->aiException = aNew;
24912 p->nException = nNew;
24913 }else{
24914 rc = SQLITE_NOMEM7;
24915 }
24916 }
24917
24918 return rc;
24919}
24920
24921/*
24922** Return true if the p->aiException[] array contains the value iCode.
24923*/
24924static int fts5UnicodeIsException(Unicode61Tokenizer *p, int iCode){
24925 if( p->nException>0 ){
24926 int *a = p->aiException;
24927 int iLo = 0;
24928 int iHi = p->nException-1;
24929
24930 while( iHi>=iLo ){
24931 int iTest = (iHi + iLo) / 2;
24932 if( iCode==a[iTest] ){
24933 return 1;
24934 }else if( iCode>a[iTest] ){
24935 iLo = iTest+1;
24936 }else{
24937 iHi = iTest-1;
24938 }
24939 }
24940 }
24941
24942 return 0;
24943}
24944
24945/*
24946** Delete a "unicode61" tokenizer.
24947*/
24948static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
24949 if( pTok ){
24950 Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok;
24951 sqlite3_freesqlite3_api->free(p->aiException);
24952 sqlite3_freesqlite3_api->free(p->aFold);
24953 sqlite3_freesqlite3_api->free(p);
24954 }
24955 return;
24956}
24957
24958static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){
24959 const char *z = zCat;
24960
24961 while( *z ){
24962 while( *z==' ' || *z=='\t' ) z++;
24963 if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){
24964 return SQLITE_ERROR1;
24965 }
24966 while( *z!=' ' && *z!='\t' && *z!='\0' ) z++;
24967 }
24968
24969 sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar);
24970 return SQLITE_OK0;
24971}
24972
24973/*
24974** Create a "unicode61" tokenizer.
24975*/
24976static int fts5UnicodeCreate(
24977 void *pUnused,
24978 const char **azArg, int nArg,
24979 Fts5Tokenizer **ppOut
24980){
24981 int rc = SQLITE_OK0; /* Return code */
24982 Unicode61Tokenizer *p = 0; /* New tokenizer object */
24983
24984 UNUSED_PARAM(pUnused)(void)(pUnused);
24985
24986 if( nArg%2 ){
24987 rc = SQLITE_ERROR1;
24988 }else{
24989 p = (Unicode61Tokenizer*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(Unicode61Tokenizer));
24990 if( p ){
24991 const char *zCat = "L* N* Co";
24992 int i;
24993 memset(p, 0, sizeof(Unicode61Tokenizer));
24994
24995 p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE1;
24996 p->nFold = 64;
24997 p->aFold = sqlite3_malloc64sqlite3_api->malloc64(p->nFold * sizeof(char));
24998 if( p->aFold==0 ){
24999 rc = SQLITE_NOMEM7;
25000 }
25001
25002 /* Search for a "categories" argument */
25003 for(i=0; rc==SQLITE_OK0 && i<nArg; i+=2){
25004 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "categories") ){
25005 zCat = azArg[i+1];
25006 }
25007 }
25008 if( rc==SQLITE_OK0 ){
25009 rc = unicodeSetCategories(p, zCat);
25010 }
25011
25012 for(i=0; rc==SQLITE_OK0 && i<nArg; i+=2){
25013 const char *zArg = azArg[i+1];
25014 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "remove_diacritics") ){
25015 if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
25016 rc = SQLITE_ERROR1;
25017 }else{
25018 p->eRemoveDiacritic = (zArg[0] - '0');
25019 assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE((void) (0))
25020 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE((void) (0))
25021 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX((void) (0))
25022 )((void) (0));
25023 }
25024 }else
25025 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "tokenchars") ){
25026 rc = fts5UnicodeAddExceptions(p, zArg, 1);
25027 }else
25028 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "separators") ){
25029 rc = fts5UnicodeAddExceptions(p, zArg, 0);
25030 }else
25031 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "categories") ){
25032 /* no-op */
25033 }else{
25034 rc = SQLITE_ERROR1;
25035 }
25036 }
25037 }else{
25038 rc = SQLITE_NOMEM7;
25039 }
25040 if( rc!=SQLITE_OK0 ){
25041 fts5UnicodeDelete((Fts5Tokenizer*)p);
25042 p = 0;
25043 }
25044 *ppOut = (Fts5Tokenizer*)p;
25045 }
25046 return rc;
25047}
25048
25049/*
25050** Return true if, for the purposes of tokenizing with the tokenizer
25051** passed as the first argument, codepoint iCode is considered a token
25052** character (not a separator).
25053*/
25054static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
25055 return (
25056 p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)]
25057 ^ fts5UnicodeIsException(p, iCode)
25058 );
25059}
25060
25061static int fts5UnicodeTokenize(
25062 Fts5Tokenizer *pTokenizer,
25063 void *pCtx,
25064 int iUnused,
25065 const char *pText, int nText,
25066 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
25067){
25068 Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer;
25069 int rc = SQLITE_OK0;
25070 unsigned char *a = p->aTokenChar;
25071
25072 unsigned char *zTerm = (unsigned char*)&pText[nText];
25073 unsigned char *zCsr = (unsigned char *)pText;
25074
25075 /* Output buffer */
25076 char *aFold = p->aFold;
25077 int nFold = p->nFold;
25078 const char *pEnd = &aFold[nFold-6];
25079
25080 UNUSED_PARAM(iUnused)(void)(iUnused);
25081
25082 /* Each iteration of this loop gobbles up a contiguous run of separators,
25083 ** then the next token. */
25084 while( rc==SQLITE_OK0 ){
25085 u32 iCode; /* non-ASCII codepoint read from input */
25086 char *zOut = aFold;
25087 int is;
25088 int ie;
25089
25090 /* Skip any separator characters. */
25091 while( 1 ){
25092 if( zCsr>=zTerm ) goto tokenize_done;
25093 if( *zCsr & 0x80 ) {
25094 /* A character outside of the ascii range. Skip past it if it is
25095 ** a separator character. Or break out of the loop if it is not. */
25096 is = zCsr - (unsigned char*)pText;
25097 READ_UTF8(zCsr, zTerm, iCode)iCode = *(zCsr++); if( iCode>=0xc0 ){ iCode = sqlite3Utf8Trans1
[iCode-0xc0]; while( zCsr<zTerm && (*zCsr & 0xc0
)==0x80 ){ iCode = (iCode<<6) + (0x3f & *(zCsr++));
} if( iCode<0x80 || (iCode&0xFFFFF800)==0xD800 || (iCode
&0xFFFFFFFE)==0xFFFE ){ iCode = 0xFFFD; } };
25098 if( fts5UnicodeIsAlnum(p, iCode) ){
25099 goto non_ascii_tokenchar;
25100 }
25101 }else{
25102 if( a[*zCsr] ){
25103 is = zCsr - (unsigned char*)pText;
25104 goto ascii_tokenchar;
25105 }
25106 zCsr++;
25107 }
25108 }
25109
25110 /* Run through the tokenchars. Fold them into the output buffer along
25111 ** the way. */
25112 while( zCsr<zTerm ){
25113
25114 /* Grow the output buffer so that there is sufficient space to fit the
25115 ** largest possible utf-8 character. */
25116 if( zOut>pEnd ){
25117 aFold = sqlite3_malloc64sqlite3_api->malloc64((sqlite3_int64)nFold*2);
25118 if( aFold==0 ){
25119 rc = SQLITE_NOMEM7;
25120 goto tokenize_done;
25121 }
25122 zOut = &aFold[zOut - p->aFold];
25123 memcpy(aFold, p->aFold, nFold);
25124 sqlite3_freesqlite3_api->free(p->aFold);
25125 p->aFold = aFold;
25126 p->nFold = nFold = nFold*2;
25127 pEnd = &aFold[nFold-6];
25128 }
25129
25130 if( *zCsr & 0x80 ){
25131 /* An non-ascii-range character. Fold it into the output buffer if
25132 ** it is a token character, or break out of the loop if it is not. */
25133 READ_UTF8(zCsr, zTerm, iCode)iCode = *(zCsr++); if( iCode>=0xc0 ){ iCode = sqlite3Utf8Trans1
[iCode-0xc0]; while( zCsr<zTerm && (*zCsr & 0xc0
)==0x80 ){ iCode = (iCode<<6) + (0x3f & *(zCsr++));
} if( iCode<0x80 || (iCode&0xFFFFF800)==0xD800 || (iCode
&0xFFFFFFFE)==0xFFFE ){ iCode = 0xFFFD; } };
25134 if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
25135 non_ascii_tokenchar:
25136 iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
25137 if( iCode ) WRITE_UTF8(zOut, iCode){ if( iCode<0x00080 ){ *zOut++ = (unsigned char)(iCode&
0xFF); } else if( iCode<0x00800 ){ *zOut++ = 0xC0 + (unsigned
char)((iCode>>6)&0x1F); *zOut++ = 0x80 + (unsigned
char)(iCode & 0x3F); } else if( iCode<0x10000 ){ *zOut
++ = 0xE0 + (unsigned char)((iCode>>12)&0x0F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); }else{ *zOut++
= 0xF0 + (unsigned char)((iCode>>18) & 0x07); *zOut
++ = 0x80 + (unsigned char)((iCode>>12) & 0x3F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); } };
25138 }else{
25139 break;
25140 }
25141 }else if( a[*zCsr]==0 ){
25142 /* An ascii-range separator character. End of token. */
25143 break;
25144 }else{
25145 ascii_tokenchar:
25146 if( *zCsr>='A' && *zCsr<='Z' ){
25147 *zOut++ = *zCsr + 32;
25148 }else{
25149 *zOut++ = *zCsr;
25150 }
25151 zCsr++;
25152 }
25153 ie = zCsr - (unsigned char*)pText;
25154 }
25155
25156 /* Invoke the token callback */
25157 rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie);
25158 }
25159
25160 tokenize_done:
25161 if( rc==SQLITE_DONE101 ) rc = SQLITE_OK0;
25162 return rc;
25163}
25164
25165/**************************************************************************
25166** Start of porter stemmer implementation.
25167*/
25168
25169/* Any tokens larger than this (in bytes) are passed through without
25170** stemming. */
25171#define FTS5_PORTER_MAX_TOKEN64 64
25172
25173typedef struct PorterTokenizer PorterTokenizer;
25174struct PorterTokenizer {
25175 fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */
25176 Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */
25177 char aBuf[FTS5_PORTER_MAX_TOKEN64 + 64];
25178};
25179
25180/*
25181** Delete a "porter" tokenizer.
25182*/
25183static void fts5PorterDelete(Fts5Tokenizer *pTok){
25184 if( pTok ){
25185 PorterTokenizer *p = (PorterTokenizer*)pTok;
25186 if( p->pTokenizer ){
25187 p->tokenizer_v2.xDelete(p->pTokenizer);
25188 }
25189 sqlite3_freesqlite3_api->free(p);
25190 }
25191}
25192
25193/*
25194** Create a "porter" tokenizer.
25195*/
25196static int fts5PorterCreate(
25197 void *pCtx,
25198 const char **azArg, int nArg,
25199 Fts5Tokenizer **ppOut
25200){
25201 fts5_api *pApi = (fts5_api*)pCtx;
25202 int rc = SQLITE_OK0;
25203 PorterTokenizer *pRet;
25204 void *pUserdata = 0;
25205 const char *zBase = "unicode61";
25206 fts5_tokenizer_v2 *pV2 = 0;
25207
25208 while( nArg>0 ){
25209 if( sqlite3_stricmpsqlite3_api->stricmp(azArg[0],"porter")==0 ){
25210 nArg--;
25211 azArg++;
25212 }else{
25213 zBase = azArg[0];
25214 break;
25215 }
25216 }
25217
25218 pRet = (PorterTokenizer*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(PorterTokenizer));
25219 if( pRet ){
25220 memset(pRet, 0, sizeof(PorterTokenizer));
25221 rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2);
25222 }else{
25223 rc = SQLITE_NOMEM7;
25224 }
25225 if( rc==SQLITE_OK0 ){
25226 int nArg2 = (nArg>0 ? nArg-1 : 0);
25227 const char **az2 = (nArg2 ? &azArg[1] : 0);
25228 memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2));
25229 rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer);
25230 }
25231
25232 if( rc!=SQLITE_OK0 ){
25233 fts5PorterDelete((Fts5Tokenizer*)pRet);
25234 pRet = 0;
25235 }
25236 *ppOut = (Fts5Tokenizer*)pRet;
25237 return rc;
25238}
25239
25240typedef struct PorterContext PorterContext;
25241struct PorterContext {
25242 void *pCtx;
25243 int (*xToken)(void*, int, const char*, int, int, int);
25244 char *aBuf;
25245};
25246
25247typedef struct PorterRule PorterRule;
25248struct PorterRule {
25249 const char *zSuffix;
25250 int nSuffix;
25251 int (*xCond)(char *zStem, int nStem);
25252 const char *zOutput;
25253 int nOutput;
25254};
25255
25256#if 0
25257static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){
25258 int ret = -1;
25259 int nBuf = *pnBuf;
25260 PorterRule *p;
25261
25262 for(p=aRule; p->zSuffix; p++){
25263 assert( strlen(p->zSuffix)==p->nSuffix )((void) (0));
25264 assert( strlen(p->zOutput)==p->nOutput )((void) (0));
25265 if( nBuf<p->nSuffix ) continue;
25266 if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break;
25267 }
25268
25269 if( p->zSuffix ){
25270 int nStem = nBuf - p->nSuffix;
25271 if( p->xCond==0 || p->xCond(aBuf, nStem) ){
25272 memcpy(&aBuf[nStem], p->zOutput, p->nOutput);
25273 *pnBuf = nStem + p->nOutput;
25274 ret = p - aRule;
25275 }
25276 }
25277
25278 return ret;
25279}
25280#endif
25281
25282static int fts5PorterIsVowel(char c, int bYIsVowel){
25283 return (
25284 c=='a' || c=='e' || c=='i' || c=='o' || c=='u' || (bYIsVowel && c=='y')
25285 );
25286}
25287
25288static int fts5PorterGobbleVC(char *zStem, int nStem, int bPrevCons){
25289 int i;
25290 int bCons = bPrevCons;
25291
25292 /* Scan for a vowel */
25293 for(i=0; i<nStem; i++){
25294 if( 0==(bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) break;
25295 }
25296
25297 /* Scan for a consonent */
25298 for(i++; i<nStem; i++){
25299 if( (bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) return i+1;
25300 }
25301 return 0;
25302}
25303
25304/* porter rule condition: (m > 0) */
25305static int fts5Porter_MGt0(char *zStem, int nStem){
25306 return !!fts5PorterGobbleVC(zStem, nStem, 0);
25307}
25308
25309/* porter rule condition: (m > 1) */
25310static int fts5Porter_MGt1(char *zStem, int nStem){
25311 int n;
25312 n = fts5PorterGobbleVC(zStem, nStem, 0);
25313 if( n && fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
25314 return 1;
25315 }
25316 return 0;
25317}
25318
25319/* porter rule condition: (m = 1) */
25320static int fts5Porter_MEq1(char *zStem, int nStem){
25321 int n;
25322 n = fts5PorterGobbleVC(zStem, nStem, 0);
25323 if( n && 0==fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
25324 return 1;
25325 }
25326 return 0;
25327}
25328
25329/* porter rule condition: (*o) */
25330static int fts5Porter_Ostar(char *zStem, int nStem){
25331 if( zStem[nStem-1]=='w' || zStem[nStem-1]=='x' || zStem[nStem-1]=='y' ){
25332 return 0;
25333 }else{
25334 int i;
25335 int mask = 0;
25336 int bCons = 0;
25337 for(i=0; i<nStem; i++){
25338 bCons = !fts5PorterIsVowel(zStem[i], bCons);
25339 assert( bCons==0 || bCons==1 )((void) (0));
25340 mask = (mask << 1) + bCons;
25341 }
25342 return ((mask & 0x0007)==0x0005);
25343 }
25344}
25345
25346/* porter rule condition: (m > 1 and (*S or *T)) */
25347static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){
25348 assert( nStem>0 )((void) (0));
25349 return (zStem[nStem-1]=='s' || zStem[nStem-1]=='t')
25350 && fts5Porter_MGt1(zStem, nStem);
25351}
25352
25353/* porter rule condition: (*v*) */
25354static int fts5Porter_Vowel(char *zStem, int nStem){
25355 int i;
25356 for(i=0; i<nStem; i++){
25357 if( fts5PorterIsVowel(zStem[i], i>0) ){
25358 return 1;
25359 }
25360 }
25361 return 0;
25362}
25363
25364
25365/**************************************************************************
25366***************************************************************************
25367** GENERATED CODE STARTS HERE (mkportersteps.tcl)
25368*/
25369
25370static int fts5PorterStep4(char *aBuf, int *pnBuf){
25371 int ret = 0;
25372 int nBuf = *pnBuf;
25373 switch( aBuf[nBuf-2] ){
25374
25375 case 'a':
25376 if( nBuf>2 && 0==memcmp("al", &aBuf[nBuf-2], 2) ){
25377 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
25378 *pnBuf = nBuf - 2;
25379 }
25380 }
25381 break;
25382
25383 case 'c':
25384 if( nBuf>4 && 0==memcmp("ance", &aBuf[nBuf-4], 4) ){
25385 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
25386 *pnBuf = nBuf - 4;
25387 }
25388 }else if( nBuf>4 && 0==memcmp("ence", &aBuf[nBuf-4], 4) ){
25389 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
25390 *pnBuf = nBuf - 4;
25391 }
25392 }
25393 break;
25394
25395 case 'e':
25396 if( nBuf>2 && 0==memcmp("er", &aBuf[nBuf-2], 2) ){
25397 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
25398 *pnBuf = nBuf - 2;
25399 }
25400 }
25401 break;
25402
25403 case 'i':
25404 if( nBuf>2 && 0==memcmp("ic", &aBuf[nBuf-2], 2) ){
25405 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
25406 *pnBuf = nBuf - 2;
25407 }
25408 }
25409 break;
25410
25411 case 'l':
25412 if( nBuf>4 && 0==memcmp("able", &aBuf[nBuf-4], 4) ){
25413 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
25414 *pnBuf = nBuf - 4;
25415 }
25416 }else if( nBuf>4 && 0==memcmp("ible", &aBuf[nBuf-4], 4) ){
25417 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
25418 *pnBuf = nBuf - 4;
25419 }
25420 }
25421 break;
25422
25423 case 'n':
25424 if( nBuf>3 && 0==memcmp("ant", &aBuf[nBuf-3], 3) ){
25425 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25426 *pnBuf = nBuf - 3;
25427 }
25428 }else if( nBuf>5 && 0==memcmp("ement", &aBuf[nBuf-5], 5) ){
25429 if( fts5Porter_MGt1(aBuf, nBuf-5) ){
25430 *pnBuf = nBuf - 5;
25431 }
25432 }else if( nBuf>4 && 0==memcmp("ment", &aBuf[nBuf-4], 4) ){
25433 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
25434 *pnBuf = nBuf - 4;
25435 }
25436 }else if( nBuf>3 && 0==memcmp("ent", &aBuf[nBuf-3], 3) ){
25437 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25438 *pnBuf = nBuf - 3;
25439 }
25440 }
25441 break;
25442
25443 case 'o':
25444 if( nBuf>3 && 0==memcmp("ion", &aBuf[nBuf-3], 3) ){
25445 if( fts5Porter_MGt1_and_S_or_T(aBuf, nBuf-3) ){
25446 *pnBuf = nBuf - 3;
25447 }
25448 }else if( nBuf>2 && 0==memcmp("ou", &aBuf[nBuf-2], 2) ){
25449 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
25450 *pnBuf = nBuf - 2;
25451 }
25452 }
25453 break;
25454
25455 case 's':
25456 if( nBuf>3 && 0==memcmp("ism", &aBuf[nBuf-3], 3) ){
25457 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25458 *pnBuf = nBuf - 3;
25459 }
25460 }
25461 break;
25462
25463 case 't':
25464 if( nBuf>3 && 0==memcmp("ate", &aBuf[nBuf-3], 3) ){
25465 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25466 *pnBuf = nBuf - 3;
25467 }
25468 }else if( nBuf>3 && 0==memcmp("iti", &aBuf[nBuf-3], 3) ){
25469 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25470 *pnBuf = nBuf - 3;
25471 }
25472 }
25473 break;
25474
25475 case 'u':
25476 if( nBuf>3 && 0==memcmp("ous", &aBuf[nBuf-3], 3) ){
25477 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25478 *pnBuf = nBuf - 3;
25479 }
25480 }
25481 break;
25482
25483 case 'v':
25484 if( nBuf>3 && 0==memcmp("ive", &aBuf[nBuf-3], 3) ){
25485 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25486 *pnBuf = nBuf - 3;
25487 }
25488 }
25489 break;
25490
25491 case 'z':
25492 if( nBuf>3 && 0==memcmp("ize", &aBuf[nBuf-3], 3) ){
25493 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
25494 *pnBuf = nBuf - 3;
25495 }
25496 }
25497 break;
25498
25499 }
25500 return ret;
25501}
25502
25503
25504static int fts5PorterStep1B2(char *aBuf, int *pnBuf){
25505 int ret = 0;
25506 int nBuf = *pnBuf;
25507 switch( aBuf[nBuf-2] ){
25508
25509 case 'a':
25510 if( nBuf>2 && 0==memcmp("at", &aBuf[nBuf-2], 2) ){
25511 memcpy(&aBuf[nBuf-2], "ate", 3);
25512 *pnBuf = nBuf - 2 + 3;
25513 ret = 1;
25514 }
25515 break;
25516
25517 case 'b':
25518 if( nBuf>2 && 0==memcmp("bl", &aBuf[nBuf-2], 2) ){
25519 memcpy(&aBuf[nBuf-2], "ble", 3);
25520 *pnBuf = nBuf - 2 + 3;
25521 ret = 1;
25522 }
25523 break;
25524
25525 case 'i':
25526 if( nBuf>2 && 0==memcmp("iz", &aBuf[nBuf-2], 2) ){
25527 memcpy(&aBuf[nBuf-2], "ize", 3);
25528 *pnBuf = nBuf - 2 + 3;
25529 ret = 1;
25530 }
25531 break;
25532
25533 }
25534 return ret;
25535}
25536
25537
25538static int fts5PorterStep2(char *aBuf, int *pnBuf){
25539 int ret = 0;
25540 int nBuf = *pnBuf;
25541 switch( aBuf[nBuf-2] ){
25542
25543 case 'a':
25544 if( nBuf>7 && 0==memcmp("ational", &aBuf[nBuf-7], 7) ){
25545 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
25546 memcpy(&aBuf[nBuf-7], "ate", 3);
25547 *pnBuf = nBuf - 7 + 3;
25548 }
25549 }else if( nBuf>6 && 0==memcmp("tional", &aBuf[nBuf-6], 6) ){
25550 if( fts5Porter_MGt0(aBuf, nBuf-6) ){
25551 memcpy(&aBuf[nBuf-6], "tion", 4);
25552 *pnBuf = nBuf - 6 + 4;
25553 }
25554 }
25555 break;
25556
25557 case 'c':
25558 if( nBuf>4 && 0==memcmp("enci", &aBuf[nBuf-4], 4) ){
25559 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25560 memcpy(&aBuf[nBuf-4], "ence", 4);
25561 *pnBuf = nBuf - 4 + 4;
25562 }
25563 }else if( nBuf>4 && 0==memcmp("anci", &aBuf[nBuf-4], 4) ){
25564 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25565 memcpy(&aBuf[nBuf-4], "ance", 4);
25566 *pnBuf = nBuf - 4 + 4;
25567 }
25568 }
25569 break;
25570
25571 case 'e':
25572 if( nBuf>4 && 0==memcmp("izer", &aBuf[nBuf-4], 4) ){
25573 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25574 memcpy(&aBuf[nBuf-4], "ize", 3);
25575 *pnBuf = nBuf - 4 + 3;
25576 }
25577 }
25578 break;
25579
25580 case 'g':
25581 if( nBuf>4 && 0==memcmp("logi", &aBuf[nBuf-4], 4) ){
25582 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25583 memcpy(&aBuf[nBuf-4], "log", 3);
25584 *pnBuf = nBuf - 4 + 3;
25585 }
25586 }
25587 break;
25588
25589 case 'l':
25590 if( nBuf>3 && 0==memcmp("bli", &aBuf[nBuf-3], 3) ){
25591 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
25592 memcpy(&aBuf[nBuf-3], "ble", 3);
25593 *pnBuf = nBuf - 3 + 3;
25594 }
25595 }else if( nBuf>4 && 0==memcmp("alli", &aBuf[nBuf-4], 4) ){
25596 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25597 memcpy(&aBuf[nBuf-4], "al", 2);
25598 *pnBuf = nBuf - 4 + 2;
25599 }
25600 }else if( nBuf>5 && 0==memcmp("entli", &aBuf[nBuf-5], 5) ){
25601 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25602 memcpy(&aBuf[nBuf-5], "ent", 3);
25603 *pnBuf = nBuf - 5 + 3;
25604 }
25605 }else if( nBuf>3 && 0==memcmp("eli", &aBuf[nBuf-3], 3) ){
25606 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
25607 memcpy(&aBuf[nBuf-3], "e", 1);
25608 *pnBuf = nBuf - 3 + 1;
25609 }
25610 }else if( nBuf>5 && 0==memcmp("ousli", &aBuf[nBuf-5], 5) ){
25611 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25612 memcpy(&aBuf[nBuf-5], "ous", 3);
25613 *pnBuf = nBuf - 5 + 3;
25614 }
25615 }
25616 break;
25617
25618 case 'o':
25619 if( nBuf>7 && 0==memcmp("ization", &aBuf[nBuf-7], 7) ){
25620 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
25621 memcpy(&aBuf[nBuf-7], "ize", 3);
25622 *pnBuf = nBuf - 7 + 3;
25623 }
25624 }else if( nBuf>5 && 0==memcmp("ation", &aBuf[nBuf-5], 5) ){
25625 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25626 memcpy(&aBuf[nBuf-5], "ate", 3);
25627 *pnBuf = nBuf - 5 + 3;
25628 }
25629 }else if( nBuf>4 && 0==memcmp("ator", &aBuf[nBuf-4], 4) ){
25630 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25631 memcpy(&aBuf[nBuf-4], "ate", 3);
25632 *pnBuf = nBuf - 4 + 3;
25633 }
25634 }
25635 break;
25636
25637 case 's':
25638 if( nBuf>5 && 0==memcmp("alism", &aBuf[nBuf-5], 5) ){
25639 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25640 memcpy(&aBuf[nBuf-5], "al", 2);
25641 *pnBuf = nBuf - 5 + 2;
25642 }
25643 }else if( nBuf>7 && 0==memcmp("iveness", &aBuf[nBuf-7], 7) ){
25644 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
25645 memcpy(&aBuf[nBuf-7], "ive", 3);
25646 *pnBuf = nBuf - 7 + 3;
25647 }
25648 }else if( nBuf>7 && 0==memcmp("fulness", &aBuf[nBuf-7], 7) ){
25649 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
25650 memcpy(&aBuf[nBuf-7], "ful", 3);
25651 *pnBuf = nBuf - 7 + 3;
25652 }
25653 }else if( nBuf>7 && 0==memcmp("ousness", &aBuf[nBuf-7], 7) ){
25654 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
25655 memcpy(&aBuf[nBuf-7], "ous", 3);
25656 *pnBuf = nBuf - 7 + 3;
25657 }
25658 }
25659 break;
25660
25661 case 't':
25662 if( nBuf>5 && 0==memcmp("aliti", &aBuf[nBuf-5], 5) ){
25663 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25664 memcpy(&aBuf[nBuf-5], "al", 2);
25665 *pnBuf = nBuf - 5 + 2;
25666 }
25667 }else if( nBuf>5 && 0==memcmp("iviti", &aBuf[nBuf-5], 5) ){
25668 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25669 memcpy(&aBuf[nBuf-5], "ive", 3);
25670 *pnBuf = nBuf - 5 + 3;
25671 }
25672 }else if( nBuf>6 && 0==memcmp("biliti", &aBuf[nBuf-6], 6) ){
25673 if( fts5Porter_MGt0(aBuf, nBuf-6) ){
25674 memcpy(&aBuf[nBuf-6], "ble", 3);
25675 *pnBuf = nBuf - 6 + 3;
25676 }
25677 }
25678 break;
25679
25680 }
25681 return ret;
25682}
25683
25684
25685static int fts5PorterStep3(char *aBuf, int *pnBuf){
25686 int ret = 0;
25687 int nBuf = *pnBuf;
25688 switch( aBuf[nBuf-2] ){
25689
25690 case 'a':
25691 if( nBuf>4 && 0==memcmp("ical", &aBuf[nBuf-4], 4) ){
25692 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25693 memcpy(&aBuf[nBuf-4], "ic", 2);
25694 *pnBuf = nBuf - 4 + 2;
25695 }
25696 }
25697 break;
25698
25699 case 's':
25700 if( nBuf>4 && 0==memcmp("ness", &aBuf[nBuf-4], 4) ){
25701 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
25702 *pnBuf = nBuf - 4;
25703 }
25704 }
25705 break;
25706
25707 case 't':
25708 if( nBuf>5 && 0==memcmp("icate", &aBuf[nBuf-5], 5) ){
25709 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25710 memcpy(&aBuf[nBuf-5], "ic", 2);
25711 *pnBuf = nBuf - 5 + 2;
25712 }
25713 }else if( nBuf>5 && 0==memcmp("iciti", &aBuf[nBuf-5], 5) ){
25714 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25715 memcpy(&aBuf[nBuf-5], "ic", 2);
25716 *pnBuf = nBuf - 5 + 2;
25717 }
25718 }
25719 break;
25720
25721 case 'u':
25722 if( nBuf>3 && 0==memcmp("ful", &aBuf[nBuf-3], 3) ){
25723 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
25724 *pnBuf = nBuf - 3;
25725 }
25726 }
25727 break;
25728
25729 case 'v':
25730 if( nBuf>5 && 0==memcmp("ative", &aBuf[nBuf-5], 5) ){
25731 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25732 *pnBuf = nBuf - 5;
25733 }
25734 }
25735 break;
25736
25737 case 'z':
25738 if( nBuf>5 && 0==memcmp("alize", &aBuf[nBuf-5], 5) ){
25739 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
25740 memcpy(&aBuf[nBuf-5], "al", 2);
25741 *pnBuf = nBuf - 5 + 2;
25742 }
25743 }
25744 break;
25745
25746 }
25747 return ret;
25748}
25749
25750
25751static int fts5PorterStep1B(char *aBuf, int *pnBuf){
25752 int ret = 0;
25753 int nBuf = *pnBuf;
25754 switch( aBuf[nBuf-2] ){
25755
25756 case 'e':
25757 if( nBuf>3 && 0==memcmp("eed", &aBuf[nBuf-3], 3) ){
25758 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
25759 memcpy(&aBuf[nBuf-3], "ee", 2);
25760 *pnBuf = nBuf - 3 + 2;
25761 }
25762 }else if( nBuf>2 && 0==memcmp("ed", &aBuf[nBuf-2], 2) ){
25763 if( fts5Porter_Vowel(aBuf, nBuf-2) ){
25764 *pnBuf = nBuf - 2;
25765 ret = 1;
25766 }
25767 }
25768 break;
25769
25770 case 'n':
25771 if( nBuf>3 && 0==memcmp("ing", &aBuf[nBuf-3], 3) ){
25772 if( fts5Porter_Vowel(aBuf, nBuf-3) ){
25773 *pnBuf = nBuf - 3;
25774 ret = 1;
25775 }
25776 }
25777 break;
25778
25779 }
25780 return ret;
25781}
25782
25783/*
25784** GENERATED CODE ENDS HERE (mkportersteps.tcl)
25785***************************************************************************
25786**************************************************************************/
25787
25788static void fts5PorterStep1A(char *aBuf, int *pnBuf){
25789 int nBuf = *pnBuf;
25790 if( aBuf[nBuf-1]=='s' ){
25791 if( aBuf[nBuf-2]=='e' ){
25792 if( (nBuf>4 && aBuf[nBuf-4]=='s' && aBuf[nBuf-3]=='s')
25793 || (nBuf>3 && aBuf[nBuf-3]=='i' )
25794 ){
25795 *pnBuf = nBuf-2;
25796 }else{
25797 *pnBuf = nBuf-1;
25798 }
25799 }
25800 else if( aBuf[nBuf-2]!='s' ){
25801 *pnBuf = nBuf-1;
25802 }
25803 }
25804}
25805
25806static int fts5PorterCb(
25807 void *pCtx,
25808 int tflags,
25809 const char *pToken,
25810 int nToken,
25811 int iStart,
25812 int iEnd
25813){
25814 PorterContext *p = (PorterContext*)pCtx;
25815
25816 char *aBuf;
25817 int nBuf;
25818
25819 if( nToken>FTS5_PORTER_MAX_TOKEN64 || nToken<3 ) goto pass_through;
25820 aBuf = p->aBuf;
25821 nBuf = nToken;
25822 memcpy(aBuf, pToken, nBuf);
25823
25824 /* Step 1. */
25825 fts5PorterStep1A(aBuf, &nBuf);
25826 if( fts5PorterStep1B(aBuf, &nBuf) ){
25827 if( fts5PorterStep1B2(aBuf, &nBuf)==0 ){
25828 char c = aBuf[nBuf-1];
25829 if( fts5PorterIsVowel(c, 0)==0
25830 && c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2]
25831 ){
25832 nBuf--;
25833 }else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){
25834 aBuf[nBuf++] = 'e';
25835 }
25836 }
25837 }
25838
25839 /* Step 1C. */
25840 if( aBuf[nBuf-1]=='y' && fts5Porter_Vowel(aBuf, nBuf-1) ){
25841 aBuf[nBuf-1] = 'i';
25842 }
25843
25844 /* Steps 2 through 4. */
25845 fts5PorterStep2(aBuf, &nBuf);
25846 fts5PorterStep3(aBuf, &nBuf);
25847 fts5PorterStep4(aBuf, &nBuf);
25848
25849 /* Step 5a. */
25850 assert( nBuf>0 )((void) (0));
25851 if( aBuf[nBuf-1]=='e' ){
25852 if( fts5Porter_MGt1(aBuf, nBuf-1)
25853 || (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1))
25854 ){
25855 nBuf--;
25856 }
25857 }
25858
25859 /* Step 5b. */
25860 if( nBuf>1 && aBuf[nBuf-1]=='l'
25861 && aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1)
25862 ){
25863 nBuf--;
25864 }
25865
25866 return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd);
25867
25868 pass_through:
25869 return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
25870}
25871
25872/*
25873** Tokenize using the porter tokenizer.
25874*/
25875static int fts5PorterTokenize(
25876 Fts5Tokenizer *pTokenizer,
25877 void *pCtx,
25878 int flags,
25879 const char *pText, int nText,
25880 const char *pLoc, int nLoc,
25881 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
25882){
25883 PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
25884 PorterContext sCtx;
25885 sCtx.xToken = xToken;
25886 sCtx.pCtx = pCtx;
25887 sCtx.aBuf = p->aBuf;
25888 return p->tokenizer_v2.xTokenize(
25889 p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb
25890 );
25891}
25892
25893/**************************************************************************
25894** Start of trigram implementation.
25895*/
25896typedef struct TrigramTokenizer TrigramTokenizer;
25897struct TrigramTokenizer {
25898 int bFold; /* True to fold to lower-case */
25899 int iFoldParam; /* Parameter to pass to Fts5UnicodeFold() */
25900};
25901
25902/*
25903** Free a trigram tokenizer.
25904*/
25905static void fts5TriDelete(Fts5Tokenizer *p){
25906 sqlite3_freesqlite3_api->free(p);
25907}
25908
25909/*
25910** Allocate a trigram tokenizer.
25911*/
25912static int fts5TriCreate(
25913 void *pUnused,
25914 const char **azArg,
25915 int nArg,
25916 Fts5Tokenizer **ppOut
25917){
25918 int rc = SQLITE_OK0;
25919 TrigramTokenizer *pNew = 0;
25920 UNUSED_PARAM(pUnused)(void)(pUnused);
25921 if( nArg%2 ){
25922 rc = SQLITE_ERROR1;
25923 }else{
25924 int i;
25925 pNew = (TrigramTokenizer*)sqlite3_malloc64sqlite3_api->malloc64(sizeof(*pNew));
25926 if( pNew==0 ){
25927 rc = SQLITE_NOMEM7;
25928 }else{
25929 pNew->bFold = 1;
25930 pNew->iFoldParam = 0;
25931
25932 for(i=0; rc==SQLITE_OK0 && i<nArg; i+=2){
25933 const char *zArg = azArg[i+1];
25934 if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "case_sensitive") ){
25935 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
25936 rc = SQLITE_ERROR1;
25937 }else{
25938 pNew->bFold = (zArg[0]=='0');
25939 }
25940 }else if( 0==sqlite3_stricmpsqlite3_api->stricmp(azArg[i], "remove_diacritics") ){
25941 if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
25942 rc = SQLITE_ERROR1;
25943 }else{
25944 pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
25945 }
25946 }else{
25947 rc = SQLITE_ERROR1;
25948 }
25949 }
25950
25951 if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
25952 rc = SQLITE_ERROR1;
25953 }
25954
25955 if( rc!=SQLITE_OK0 ){
25956 fts5TriDelete((Fts5Tokenizer*)pNew);
25957 pNew = 0;
25958 }
25959 }
25960 }
25961 *ppOut = (Fts5Tokenizer*)pNew;
25962 return rc;
25963}
25964
25965/*
25966** Trigram tokenizer tokenize routine.
25967*/
25968static int fts5TriTokenize(
25969 Fts5Tokenizer *pTok,
25970 void *pCtx,
25971 int unusedFlags,
25972 const char *pText, int nText,
25973 int (*xToken)(void*, int, const char*, int, int, int)
25974){
25975 TrigramTokenizer *p = (TrigramTokenizer*)pTok;
25976 int rc = SQLITE_OK0;
25977 char aBuf[32];
25978 char *zOut = aBuf;
25979 int ii;
25980 const unsigned char *zIn = (const unsigned char*)pText;
25981 const unsigned char *zEof = (zIn ? &zIn[nText] : 0);
25982 u32 iCode = 0;
25983 int aStart[3]; /* Input offset of each character in aBuf[] */
25984
25985 UNUSED_PARAM(unusedFlags)(void)(unusedFlags);
25986
25987 /* Populate aBuf[] with the characters for the first trigram. */
25988 for(ii=0; ii<3; ii++){
25989 do {
25990 aStart[ii] = zIn - (const unsigned char*)pText;
25991 if( zIn>=zEof ) return SQLITE_OK0;
25992 READ_UTF8(zIn, zEof, iCode)iCode = *(zIn++); if( iCode>=0xc0 ){ iCode = sqlite3Utf8Trans1
[iCode-0xc0]; while( zIn<zEof && (*zIn & 0xc0)
==0x80 ){ iCode = (iCode<<6) + (0x3f & *(zIn++)); }
if( iCode<0x80 || (iCode&0xFFFFF800)==0xD800 || (iCode
&0xFFFFFFFE)==0xFFFE ){ iCode = 0xFFFD; } };
25993 if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
25994 }while( iCode==0 );
25995 WRITE_UTF8(zOut, iCode){ if( iCode<0x00080 ){ *zOut++ = (unsigned char)(iCode&
0xFF); } else if( iCode<0x00800 ){ *zOut++ = 0xC0 + (unsigned
char)((iCode>>6)&0x1F); *zOut++ = 0x80 + (unsigned
char)(iCode & 0x3F); } else if( iCode<0x10000 ){ *zOut
++ = 0xE0 + (unsigned char)((iCode>>12)&0x0F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); }else{ *zOut++
= 0xF0 + (unsigned char)((iCode>>18) & 0x07); *zOut
++ = 0x80 + (unsigned char)((iCode>>12) & 0x3F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); } };
25996 }
25997
25998 /* At the start of each iteration of this loop:
25999 **
26000 ** aBuf: Contains 3 characters. The 3 characters of the next trigram.
26001 ** zOut: Points to the byte following the last character in aBuf.
26002 ** aStart[3]: Contains the byte offset in the input text corresponding
26003 ** to the start of each of the three characters in the buffer.
26004 */
26005 assert( zIn<=zEof )((void) (0));
26006 while( 1 ){
26007 int iNext; /* Start of character following current tri */
26008 const char *z1;
26009
26010 /* Read characters from the input up until the first non-diacritic */
26011 do {
26012 iNext = zIn - (const unsigned char*)pText;
26013 if( zIn>=zEof ){
26014 iCode = 0;
26015 break;
26016 }
26017 READ_UTF8(zIn, zEof, iCode)iCode = *(zIn++); if( iCode>=0xc0 ){ iCode = sqlite3Utf8Trans1
[iCode-0xc0]; while( zIn<zEof && (*zIn & 0xc0)
==0x80 ){ iCode = (iCode<<6) + (0x3f & *(zIn++)); }
if( iCode<0x80 || (iCode&0xFFFFF800)==0xD800 || (iCode
&0xFFFFFFFE)==0xFFFE ){ iCode = 0xFFFD; } };
26018 if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
26019 }while( iCode==0 );
26020
26021 /* Pass the current trigram back to fts5 */
26022 rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext);
26023 if( iCode==0 || rc!=SQLITE_OK0 ) break;
26024
26025 /* Remove the first character from buffer aBuf[]. Append the character
26026 ** with codepoint iCode. */
26027 z1 = aBuf;
26028 FTS5_SKIP_UTF8(z1){ if( ((unsigned char)(*(z1++)))>=0xc0 ){ while( (((unsigned
char)*z1) & 0xc0)==0x80 ){ z1++; } } };
26029 memmove(aBuf, z1, zOut - z1);
26030 zOut -= (z1 - aBuf);
26031 WRITE_UTF8(zOut, iCode){ if( iCode<0x00080 ){ *zOut++ = (unsigned char)(iCode&
0xFF); } else if( iCode<0x00800 ){ *zOut++ = 0xC0 + (unsigned
char)((iCode>>6)&0x1F); *zOut++ = 0x80 + (unsigned
char)(iCode & 0x3F); } else if( iCode<0x10000 ){ *zOut
++ = 0xE0 + (unsigned char)((iCode>>12)&0x0F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); }else{ *zOut++
= 0xF0 + (unsigned char)((iCode>>18) & 0x07); *zOut
++ = 0x80 + (unsigned char)((iCode>>12) & 0x3F); *zOut
++ = 0x80 + (unsigned char)((iCode>>6) & 0x3F); *zOut
++ = 0x80 + (unsigned char)(iCode & 0x3F); } };
26032
26033 /* Update the aStart[] array */
26034 aStart[0] = aStart[1];
26035 aStart[1] = aStart[2];
26036 aStart[2] = iNext;
26037 }
26038
26039 return rc;
26040}
26041
26042/*
26043** Argument xCreate is a pointer to a constructor function for a tokenizer.
26044** pTok is a tokenizer previously created using the same method. This function
26045** returns one of FTS5_PATTERN_NONE, FTS5_PATTERN_LIKE or FTS5_PATTERN_GLOB
26046** indicating the style of pattern matching that the tokenizer can support.
26047** In practice, this is:
26048**
26049** "trigram" tokenizer, case_sensitive=1 - FTS5_PATTERN_GLOB
26050** "trigram" tokenizer, case_sensitive=0 (the default) - FTS5_PATTERN_LIKE
26051** all other tokenizers - FTS5_PATTERN_NONE
26052*/
26053static int sqlite3Fts5TokenizerPattern(
26054 int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
26055 Fts5Tokenizer *pTok
26056){
26057 if( xCreate==fts5TriCreate ){
26058 TrigramTokenizer *p = (TrigramTokenizer*)pTok;
26059 if( p->iFoldParam==0 ){
26060 return p->bFold ? FTS5_PATTERN_LIKE65 : FTS5_PATTERN_GLOB66;
26061 }
26062 }
26063 return FTS5_PATTERN_NONE0;
26064}
26065
26066/*
26067** Return true if the tokenizer described by p->azArg[] is the trigram
26068** tokenizer. This tokenizer needs to be loaded before xBestIndex is
26069** called for the first time in order to correctly handle LIKE/GLOB.
26070*/
26071static int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){
26072 return (p->nArg>=1 && 0==sqlite3_stricmpsqlite3_api->stricmp(p->azArg[0], "trigram"));
26073}
26074
26075
26076/*
26077** Register all built-in tokenizers with FTS5.
26078*/
26079static int sqlite3Fts5TokenizerInit(fts5_api *pApi){
26080 struct BuiltinTokenizer {
26081 const char *zName;
26082 fts5_tokenizer x;
26083 } aBuiltin[] = {
26084 { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
26085 { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
26086 { "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}},
26087 };
26088
26089 int rc = SQLITE_OK0; /* Return code */
26090 int i; /* To iterate through builtin functions */
26091
26092 for(i=0; rc==SQLITE_OK0 && i<ArraySize(aBuiltin)((int)(sizeof(aBuiltin) / sizeof(aBuiltin[0]))); i++){
26093 rc = pApi->xCreateTokenizer(pApi,
26094 aBuiltin[i].zName,
26095 (void*)pApi,
26096 &aBuiltin[i].x,
26097 0
26098 );
26099 }
26100 if( rc==SQLITE_OK0 ){
26101 fts5_tokenizer_v2 sPorter = {
26102 2,
26103 fts5PorterCreate,
26104 fts5PorterDelete,
26105 fts5PorterTokenize
26106 };
26107 rc = pApi->xCreateTokenizer_v2(pApi,
26108 "porter",
26109 (void*)pApi,
26110 &sPorter,
26111 0
26112 );
26113 }
26114 return rc;
26115}
26116
26117#line 1 "fts5_unicode2.c"
26118/*
26119** 2012-05-25
26120**
26121** The author disclaims copyright to this source code. In place of
26122** a legal notice, here is a blessing:
26123**
26124** May you do good and not evil.
26125** May you find forgiveness for yourself and forgive others.
26126** May you share freely, never taking more than you give.
26127**
26128******************************************************************************
26129*/
26130
26131/*
26132** DO NOT EDIT THIS MACHINE GENERATED FILE.
26133*/
26134
26135
26136#include <assert.h>
26137
26138
26139
26140/*
26141** If the argument is a codepoint corresponding to a lowercase letter
26142** in the ASCII range with a diacritic added, return the codepoint
26143** of the ASCII letter only. For example, if passed 235 - "LATIN
26144** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER
26145** E"). The resuls of passing a codepoint that corresponds to an
26146** uppercase letter are undefined.
26147*/
26148static int fts5_remove_diacritic(int c, int bComplex){
26149 unsigned short aDia[] = {
26150 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995,
26151 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286,
26152 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732,
26153 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336,
26154 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896,
26155 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106,
26156 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344,
26157 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198,
26158 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468,
26159 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704,
26160 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914,
26161 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218,
26162 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554,
26163 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766,
26164 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118,
26165 63182, 63242, 63274, 63310, 63368, 63390,
26166 };
26167#define HIBIT((unsigned char)0x80) ((unsigned char)0x80)
26168 unsigned char aChar[] = {
26169 '\0', 'a', 'c', 'e', 'i', 'n',
26170 'o', 'u', 'y', 'y', 'a', 'c',
26171 'd', 'e', 'e', 'g', 'h', 'i',
26172 'j', 'k', 'l', 'n', 'o', 'r',
26173 's', 't', 'u', 'u', 'w', 'y',
26174 'z', 'o', 'u', 'a', 'i', 'o',
26175 'u', 'u'|HIBIT((unsigned char)0x80), 'a'|HIBIT((unsigned char)0x80), 'g', 'k', 'o',
26176 'o'|HIBIT((unsigned char)0x80), 'j', 'g', 'n', 'a'|HIBIT((unsigned char)0x80), 'a',
26177 'e', 'i', 'o', 'r', 'u', 's',
26178 't', 'h', 'a', 'e', 'o'|HIBIT((unsigned char)0x80), 'o',
26179 'o'|HIBIT((unsigned char)0x80), 'y', '\0', '\0', '\0', '\0',
26180 '\0', '\0', '\0', '\0', 'a', 'b',
26181 'c'|HIBIT((unsigned char)0x80), 'd', 'd', 'e'|HIBIT((unsigned char)0x80), 'e', 'e'|HIBIT((unsigned char)0x80),
26182 'f', 'g', 'h', 'h', 'i', 'i'|HIBIT((unsigned char)0x80),
26183 'k', 'l', 'l'|HIBIT((unsigned char)0x80), 'l', 'm', 'n',
26184 'o'|HIBIT((unsigned char)0x80), 'p', 'r', 'r'|HIBIT((unsigned char)0x80), 'r', 's',
26185 's'|HIBIT((unsigned char)0x80), 't', 'u', 'u'|HIBIT((unsigned char)0x80), 'v', 'w',
26186 'w', 'x', 'y', 'z', 'h', 't',
26187 'w', 'y', 'a', 'a'|HIBIT((unsigned char)0x80), 'a'|HIBIT((unsigned char)0x80), 'a'|HIBIT((unsigned char)0x80),
26188 'e', 'e'|HIBIT((unsigned char)0x80), 'e'|HIBIT((unsigned char)0x80), 'i', 'o', 'o'|HIBIT((unsigned char)0x80),
26189 'o'|HIBIT((unsigned char)0x80), 'o'|HIBIT((unsigned char)0x80), 'u', 'u'|HIBIT((unsigned char)0x80), 'u'|HIBIT((unsigned char)0x80), 'y',
26190 };
26191
26192 unsigned int key = (((unsigned int)c)<<3) | 0x00000007;
26193 int iRes = 0;
26194 int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1;
26195 int iLo = 0;
26196 while( iHi>=iLo ){
26197 int iTest = (iHi + iLo) / 2;
26198 if( key >= aDia[iTest] ){
26199 iRes = iTest;
26200 iLo = iTest+1;
26201 }else{
26202 iHi = iTest-1;
26203 }
26204 }
26205 assert( key>=aDia[iRes] )((void) (0));
26206 if( bComplex==0 && (aChar[iRes] & 0x80) ) return c;
26207 return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F);
26208}
26209
26210
26211/*
26212** Return true if the argument interpreted as a unicode codepoint
26213** is a diacritical modifier character.
26214*/
26215static int sqlite3Fts5UnicodeIsdiacritic(int c){
26216 unsigned int mask0 = 0x08029FDF;
26217 unsigned int mask1 = 0x000361F8;
26218 if( c<768 || c>817 ) return 0;
26219 return (c < 768+32) ?
26220 (mask0 & ((unsigned int)1 << (c-768))) :
26221 (mask1 & ((unsigned int)1 << (c-768-32)));
26222}
26223
26224
26225/*
26226** Interpret the argument as a unicode codepoint. If the codepoint
26227** is an upper case character that has a lower case equivalent,
26228** return the codepoint corresponding to the lower case version.
26229** Otherwise, return a copy of the argument.
26230**
26231** The results are undefined if the value passed to this function
26232** is less than zero.
26233*/
26234static int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){
26235 /* Each entry in the following array defines a rule for folding a range
26236 ** of codepoints to lower case. The rule applies to a range of nRange
26237 ** codepoints starting at codepoint iCode.
26238 **
26239 ** If the least significant bit in flags is clear, then the rule applies
26240 ** to all nRange codepoints (i.e. all nRange codepoints are upper case and
26241 ** need to be folded). Or, if it is set, then the rule only applies to
26242 ** every second codepoint in the range, starting with codepoint C.
26243 **
26244 ** The 7 most significant bits in flags are an index into the aiOff[]
26245 ** array. If a specific codepoint C does require folding, then its lower
26246 ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF).
26247 **
26248 ** The contents of this array are generated by parsing the CaseFolding.txt
26249 ** file distributed as part of the "Unicode Character Database". See
26250 ** http://www.unicode.org for details.
26251 */
26252 static const struct TableEntry {
26253 unsigned short iCode;
26254 unsigned char flags;
26255 unsigned char nRange;
26256 } aEntry[] = {
26257 {65, 14, 26}, {181, 64, 1}, {192, 14, 23},
26258 {216, 14, 7}, {256, 1, 48}, {306, 1, 6},
26259 {313, 1, 16}, {330, 1, 46}, {376, 116, 1},
26260 {377, 1, 6}, {383, 104, 1}, {385, 50, 1},
26261 {386, 1, 4}, {390, 44, 1}, {391, 0, 1},
26262 {393, 42, 2}, {395, 0, 1}, {398, 32, 1},
26263 {399, 38, 1}, {400, 40, 1}, {401, 0, 1},
26264 {403, 42, 1}, {404, 46, 1}, {406, 52, 1},
26265 {407, 48, 1}, {408, 0, 1}, {412, 52, 1},
26266 {413, 54, 1}, {415, 56, 1}, {416, 1, 6},
26267 {422, 60, 1}, {423, 0, 1}, {425, 60, 1},
26268 {428, 0, 1}, {430, 60, 1}, {431, 0, 1},
26269 {433, 58, 2}, {435, 1, 4}, {439, 62, 1},
26270 {440, 0, 1}, {444, 0, 1}, {452, 2, 1},
26271 {453, 0, 1}, {455, 2, 1}, {456, 0, 1},
26272 {458, 2, 1}, {459, 1, 18}, {478, 1, 18},
26273 {497, 2, 1}, {498, 1, 4}, {502, 122, 1},
26274 {503, 134, 1}, {504, 1, 40}, {544, 110, 1},
26275 {546, 1, 18}, {570, 70, 1}, {571, 0, 1},
26276 {573, 108, 1}, {574, 68, 1}, {577, 0, 1},
26277 {579, 106, 1}, {580, 28, 1}, {581, 30, 1},
26278 {582, 1, 10}, {837, 36, 1}, {880, 1, 4},
26279 {886, 0, 1}, {902, 18, 1}, {904, 16, 3},
26280 {908, 26, 1}, {910, 24, 2}, {913, 14, 17},
26281 {931, 14, 9}, {962, 0, 1}, {975, 4, 1},
26282 {976, 140, 1}, {977, 142, 1}, {981, 146, 1},
26283 {982, 144, 1}, {984, 1, 24}, {1008, 136, 1},
26284 {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1},
26285 {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1},
26286 {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32},
26287 {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1},
26288 {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38},
26289 {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1},
26290 {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1},
26291 {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6},
26292 {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6},
26293 {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8},
26294 {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2},
26295 {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1},
26296 {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2},
26297 {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2},
26298 {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2},
26299 {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1},
26300 {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16},
26301 {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47},
26302 {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1},
26303 {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1},
26304 {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1},
26305 {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2},
26306 {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1},
26307 {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14},
26308 {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1},
26309 {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1},
26310 {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1},
26311 {65313, 14, 26},
26312 };
26313 static const unsigned short aiOff[] = {
26314 1, 2, 8, 15, 16, 26, 28, 32,
26315 37, 38, 40, 48, 63, 64, 69, 71,
26316 79, 80, 116, 202, 203, 205, 206, 207,
26317 209, 210, 211, 213, 214, 217, 218, 219,
26318 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721,
26319 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274,
26320 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406,
26321 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462,
26322 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511,
26323 65514, 65521, 65527, 65528, 65529,
26324 };
26325
26326 int ret = c;
26327
26328 assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 )((void) (0));
26329
26330 if( c<128 ){
26331 if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
26332 }else if( c<65536 ){
26333 const struct TableEntry *p;
26334 int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
26335 int iLo = 0;
26336 int iRes = -1;
26337
26338 assert( c>aEntry[0].iCode )((void) (0));
26339 while( iHi>=iLo ){
26340 int iTest = (iHi + iLo) / 2;
26341 int cmp = (c - aEntry[iTest].iCode);
26342 if( cmp>=0 ){
26343 iRes = iTest;
26344 iLo = iTest+1;
26345 }else{
26346 iHi = iTest-1;
26347 }
26348 }
26349
26350 assert( iRes>=0 && c>=aEntry[iRes].iCode )((void) (0));
26351 p = &aEntry[iRes];
26352 if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
26353 ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
26354 assert( ret>0 )((void) (0));
26355 }
26356
26357 if( eRemoveDiacritic ){
26358 ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2);
26359 }
26360 }
26361
26362 else if( c>=66560 && c<66600 ){
26363 ret = c + 40;
26364 }
26365
26366 return ret;
26367}
26368
26369
26370static int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){
26371 aArray[0] = 1;
26372 switch( zCat[0] ){
26373 case 'C':
26374 switch( zCat[1] ){
26375 case 'c': aArray[1] = 1; break;
26376 case 'f': aArray[2] = 1; break;
26377 case 'n': aArray[3] = 1; break;
26378 case 's': aArray[4] = 1; break;
26379 case 'o': aArray[31] = 1; break;
26380 case '*':
26381 aArray[1] = 1;
26382 aArray[2] = 1;
26383 aArray[3] = 1;
26384 aArray[4] = 1;
26385 aArray[31] = 1;
26386 break;
26387 default: return 1; }
26388 break;
26389
26390 case 'L':
26391 switch( zCat[1] ){
26392 case 'l': aArray[5] = 1; break;
26393 case 'm': aArray[6] = 1; break;
26394 case 'o': aArray[7] = 1; break;
26395 case 't': aArray[8] = 1; break;
26396 case 'u': aArray[9] = 1; break;
26397 case 'C': aArray[30] = 1; break;
26398 case '*':
26399 aArray[5] = 1;
26400 aArray[6] = 1;
26401 aArray[7] = 1;
26402 aArray[8] = 1;
26403 aArray[9] = 1;
26404 aArray[30] = 1;
26405 break;
26406 default: return 1; }
26407 break;
26408
26409 case 'M':
26410 switch( zCat[1] ){
26411 case 'c': aArray[10] = 1; break;
26412 case 'e': aArray[11] = 1; break;
26413 case 'n': aArray[12] = 1; break;
26414 case '*':
26415 aArray[10] = 1;
26416 aArray[11] = 1;
26417 aArray[12] = 1;
26418 break;
26419 default: return 1; }
26420 break;
26421
26422 case 'N':
26423 switch( zCat[1] ){
26424 case 'd': aArray[13] = 1; break;
26425 case 'l': aArray[14] = 1; break;
26426 case 'o': aArray[15] = 1; break;
26427 case '*':
26428 aArray[13] = 1;
26429 aArray[14] = 1;
26430 aArray[15] = 1;
26431 break;
26432 default: return 1; }
26433 break;
26434
26435 case 'P':
26436 switch( zCat[1] ){
26437 case 'c': aArray[16] = 1; break;
26438 case 'd': aArray[17] = 1; break;
26439 case 'e': aArray[18] = 1; break;
26440 case 'f': aArray[19] = 1; break;
26441 case 'i': aArray[20] = 1; break;
26442 case 'o': aArray[21] = 1; break;
26443 case 's': aArray[22] = 1; break;
26444 case '*':
26445 aArray[16] = 1;
26446 aArray[17] = 1;
26447 aArray[18] = 1;
26448 aArray[19] = 1;
26449 aArray[20] = 1;
26450 aArray[21] = 1;
26451 aArray[22] = 1;
26452 break;
26453 default: return 1; }
26454 break;
26455
26456 case 'S':
26457 switch( zCat[1] ){
26458 case 'c': aArray[23] = 1; break;
26459 case 'k': aArray[24] = 1; break;
26460 case 'm': aArray[25] = 1; break;
26461 case 'o': aArray[26] = 1; break;
26462 case '*':
26463 aArray[23] = 1;
26464 aArray[24] = 1;
26465 aArray[25] = 1;
26466 aArray[26] = 1;
26467 break;
26468 default: return 1; }
26469 break;
26470
26471 case 'Z':
26472 switch( zCat[1] ){
26473 case 'l': aArray[27] = 1; break;
26474 case 'p': aArray[28] = 1; break;
26475 case 's': aArray[29] = 1; break;
26476 case '*':
26477 aArray[27] = 1;
26478 aArray[28] = 1;
26479 aArray[29] = 1;
26480 break;
26481 default: return 1; }
26482 break;
26483
26484
26485 default:
26486 return 1;
26487 }
26488 return 0;
26489}
26490
26491static u16 aFts5UnicodeBlock[] = {
26492 0, 1471, 1753, 1760, 1760, 1760, 1760, 1760, 1760, 1760,
26493 1760, 1760, 1760, 1760, 1760, 1763, 1765,
26494 };
26495static u16 aFts5UnicodeMap[] = {
26496 0, 32, 33, 36, 37, 40, 41, 42, 43, 44,
26497 45, 46, 48, 58, 60, 63, 65, 91, 92, 93,
26498 94, 95, 96, 97, 123, 124, 125, 126, 127, 160,
26499 161, 162, 166, 167, 168, 169, 170, 171, 172, 173,
26500 174, 175, 176, 177, 178, 180, 181, 182, 184, 185,
26501 186, 187, 188, 191, 192, 215, 216, 223, 247, 248,
26502 256, 312, 313, 329, 330, 377, 383, 385, 387, 388,
26503 391, 394, 396, 398, 402, 403, 405, 406, 409, 412,
26504 414, 415, 417, 418, 423, 427, 428, 431, 434, 436,
26505 437, 440, 442, 443, 444, 446, 448, 452, 453, 454,
26506 455, 456, 457, 458, 459, 460, 461, 477, 478, 496,
26507 497, 498, 499, 500, 503, 505, 506, 564, 570, 572,
26508 573, 575, 577, 580, 583, 584, 592, 660, 661, 688,
26509 706, 710, 722, 736, 741, 748, 749, 750, 751, 768,
26510 880, 884, 885, 886, 890, 891, 894, 900, 902, 903,
26511 904, 908, 910, 912, 913, 931, 940, 975, 977, 978,
26512 981, 984, 1008, 1012, 1014, 1015, 1018, 1020, 1021, 1072,
26513 1120, 1154, 1155, 1160, 1162, 1217, 1231, 1232, 1329, 1369,
26514 1370, 1377, 1417, 1418, 1423, 1425, 1470, 1471, 1472, 1473,
26515 1475, 1476, 1478, 1479, 1488, 1520, 1523, 1536, 1542, 1545,
26516 1547, 1548, 1550, 1552, 1563, 1566, 1568, 1600, 1601, 1611,
26517 1632, 1642, 1646, 1648, 1649, 1748, 1749, 1750, 1757, 1758,
26518 1759, 1765, 1767, 1769, 1770, 1774, 1776, 1786, 1789, 1791,
26519 1792, 1807, 1808, 1809, 1810, 1840, 1869, 1958, 1969, 1984,
26520 1994, 2027, 2036, 2038, 2039, 2042, 2048, 2070, 2074, 2075,
26521 2084, 2085, 2088, 2089, 2096, 2112, 2137, 2142, 2208, 2210,
26522 2276, 2304, 2307, 2308, 2362, 2363, 2364, 2365, 2366, 2369,
26523 2377, 2381, 2382, 2384, 2385, 2392, 2402, 2404, 2406, 2416,
26524 2417, 2418, 2425, 2433, 2434, 2437, 2447, 2451, 2474, 2482,
26525 2486, 2492, 2493, 2494, 2497, 2503, 2507, 2509, 2510, 2519,
26526 2524, 2527, 2530, 2534, 2544, 2546, 2548, 2554, 2555, 2561,
26527 2563, 2565, 2575, 2579, 2602, 2610, 2613, 2616, 2620, 2622,
26528 2625, 2631, 2635, 2641, 2649, 2654, 2662, 2672, 2674, 2677,
26529 2689, 2691, 2693, 2703, 2707, 2730, 2738, 2741, 2748, 2749,
26530 2750, 2753, 2759, 2761, 2763, 2765, 2768, 2784, 2786, 2790,
26531 2800, 2801, 2817, 2818, 2821, 2831, 2835, 2858, 2866, 2869,
26532 2876, 2877, 2878, 2879, 2880, 2881, 2887, 2891, 2893, 2902,
26533 2903, 2908, 2911, 2914, 2918, 2928, 2929, 2930, 2946, 2947,
26534 2949, 2958, 2962, 2969, 2972, 2974, 2979, 2984, 2990, 3006,
26535 3008, 3009, 3014, 3018, 3021, 3024, 3031, 3046, 3056, 3059,
26536 3065, 3066, 3073, 3077, 3086, 3090, 3114, 3125, 3133, 3134,
26537 3137, 3142, 3146, 3157, 3160, 3168, 3170, 3174, 3192, 3199,
26538 3202, 3205, 3214, 3218, 3242, 3253, 3260, 3261, 3262, 3263,
26539 3264, 3270, 3271, 3274, 3276, 3285, 3294, 3296, 3298, 3302,
26540 3313, 3330, 3333, 3342, 3346, 3389, 3390, 3393, 3398, 3402,
26541 3405, 3406, 3415, 3424, 3426, 3430, 3440, 3449, 3450, 3458,
26542 3461, 3482, 3507, 3517, 3520, 3530, 3535, 3538, 3542, 3544,
26543 3570, 3572, 3585, 3633, 3634, 3636, 3647, 3648, 3654, 3655,
26544 3663, 3664, 3674, 3713, 3716, 3719, 3722, 3725, 3732, 3737,
26545 3745, 3749, 3751, 3754, 3757, 3761, 3762, 3764, 3771, 3773,
26546 3776, 3782, 3784, 3792, 3804, 3840, 3841, 3844, 3859, 3860,
26547 3861, 3864, 3866, 3872, 3882, 3892, 3893, 3894, 3895, 3896,
26548 3897, 3898, 3899, 3900, 3901, 3902, 3904, 3913, 3953, 3967,
26549 3968, 3973, 3974, 3976, 3981, 3993, 4030, 4038, 4039, 4046,
26550 4048, 4053, 4057, 4096, 4139, 4141, 4145, 4146, 4152, 4153,
26551 4155, 4157, 4159, 4160, 4170, 4176, 4182, 4184, 4186, 4190,
26552 4193, 4194, 4197, 4199, 4206, 4209, 4213, 4226, 4227, 4229,
26553 4231, 4237, 4238, 4239, 4240, 4250, 4253, 4254, 4256, 4295,
26554 4301, 4304, 4347, 4348, 4349, 4682, 4688, 4696, 4698, 4704,
26555 4746, 4752, 4786, 4792, 4800, 4802, 4808, 4824, 4882, 4888,
26556 4957, 4960, 4969, 4992, 5008, 5024, 5120, 5121, 5741, 5743,
26557 5760, 5761, 5787, 5788, 5792, 5867, 5870, 5888, 5902, 5906,
26558 5920, 5938, 5941, 5952, 5970, 5984, 5998, 6002, 6016, 6068,
26559 6070, 6071, 6078, 6086, 6087, 6089, 6100, 6103, 6104, 6107,
26560 6108, 6109, 6112, 6128, 6144, 6150, 6151, 6155, 6158, 6160,
26561 6176, 6211, 6212, 6272, 6313, 6314, 6320, 6400, 6432, 6435,
26562 6439, 6441, 6448, 6450, 6451, 6457, 6464, 6468, 6470, 6480,
26563 6512, 6528, 6576, 6593, 6600, 6608, 6618, 6622, 6656, 6679,
26564 6681, 6686, 6688, 6741, 6742, 6743, 6744, 6752, 6753, 6754,
26565 6755, 6757, 6765, 6771, 6783, 6784, 6800, 6816, 6823, 6824,
26566 6912, 6916, 6917, 6964, 6965, 6966, 6971, 6972, 6973, 6978,
26567 6979, 6981, 6992, 7002, 7009, 7019, 7028, 7040, 7042, 7043,
26568 7073, 7074, 7078, 7080, 7082, 7083, 7084, 7086, 7088, 7098,
26569 7142, 7143, 7144, 7146, 7149, 7150, 7151, 7154, 7164, 7168,
26570 7204, 7212, 7220, 7222, 7227, 7232, 7245, 7248, 7258, 7288,
26571 7294, 7360, 7376, 7379, 7380, 7393, 7394, 7401, 7405, 7406,
26572 7410, 7412, 7413, 7424, 7468, 7531, 7544, 7545, 7579, 7616,
26573 7676, 7680, 7830, 7838, 7936, 7944, 7952, 7960, 7968, 7976,
26574 7984, 7992, 8000, 8008, 8016, 8025, 8027, 8029, 8031, 8033,
26575 8040, 8048, 8064, 8072, 8080, 8088, 8096, 8104, 8112, 8118,
26576 8120, 8124, 8125, 8126, 8127, 8130, 8134, 8136, 8140, 8141,
26577 8144, 8150, 8152, 8157, 8160, 8168, 8173, 8178, 8182, 8184,
26578 8188, 8189, 8192, 8203, 8208, 8214, 8216, 8217, 8218, 8219,
26579 8221, 8222, 8223, 8224, 8232, 8233, 8234, 8239, 8240, 8249,
26580 8250, 8251, 8255, 8257, 8260, 8261, 8262, 8263, 8274, 8275,
26581 8276, 8277, 8287, 8288, 8298, 8304, 8305, 8308, 8314, 8317,
26582 8318, 8319, 8320, 8330, 8333, 8334, 8336, 8352, 8400, 8413,
26583 8417, 8418, 8421, 8448, 8450, 8451, 8455, 8456, 8458, 8459,
26584 8462, 8464, 8467, 8468, 8469, 8470, 8472, 8473, 8478, 8484,
26585 8485, 8486, 8487, 8488, 8489, 8490, 8494, 8495, 8496, 8500,
26586 8501, 8505, 8506, 8508, 8510, 8512, 8517, 8519, 8522, 8523,
26587 8524, 8526, 8527, 8528, 8544, 8579, 8581, 8585, 8592, 8597,
26588 8602, 8604, 8608, 8609, 8611, 8612, 8614, 8615, 8622, 8623,
26589 8654, 8656, 8658, 8659, 8660, 8661, 8692, 8960, 8968, 8972,
26590 8992, 8994, 9001, 9002, 9003, 9084, 9085, 9115, 9140, 9180,
26591 9186, 9216, 9280, 9312, 9372, 9450, 9472, 9655, 9656, 9665,
26592 9666, 9720, 9728, 9839, 9840, 9985, 10088, 10089, 10090, 10091,
26593 10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101,
26594 10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217,
26595 10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627,
26596 10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637,
26597 10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647,
26598 10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750,
26599 11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365,
26600 11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393,
26601 11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520,
26602 11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696,
26603 11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780,
26604 11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800,
26605 11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812,
26606 11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904,
26607 11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296,
26608 12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306,
26609 12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317,
26610 12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347,
26611 12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449,
26612 12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736,
26613 12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938,
26614 12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981,
26615 40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528,
26616 42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624,
26617 42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800,
26618 42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912,
26619 43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043,
26620 43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136,
26621 43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264,
26622 43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395,
26623 43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472,
26624 43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588,
26625 43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643,
26626 43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713,
26627 43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762,
26628 43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003,
26629 44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203,
26630 55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112,
26631 64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320,
26632 64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020,
26633 65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075,
26634 65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086,
26635 65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097,
26636 65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118,
26637 65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279,
26638 65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294,
26639 65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343,
26640 65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378,
26641 65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490,
26642 65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529,
26643 65532, 0, 13, 40, 60, 63, 80, 128, 256, 263,
26644 311, 320, 373, 377, 394, 400, 464, 509, 640, 672,
26645 768, 800, 816, 833, 834, 842, 896, 927, 928, 968,
26646 976, 977, 1024, 1064, 1104, 1184, 2048, 2056, 2058, 2103,
26647 2108, 2111, 2135, 2136, 2304, 2326, 2335, 2336, 2367, 2432,
26648 2494, 2560, 2561, 2565, 2572, 2576, 2581, 2585, 2616, 2623,
26649 2624, 2640, 2656, 2685, 2687, 2816, 2873, 2880, 2904, 2912,
26650 2936, 3072, 3680, 4096, 4097, 4098, 4099, 4152, 4167, 4178,
26651 4198, 4224, 4226, 4227, 4272, 4275, 4279, 4281, 4283, 4285,
26652 4286, 4304, 4336, 4352, 4355, 4391, 4396, 4397, 4406, 4416,
26653 4480, 4482, 4483, 4531, 4534, 4543, 4545, 4549, 4560, 5760,
26654 5803, 5804, 5805, 5806, 5808, 5814, 5815, 5824, 8192, 9216,
26655 9328, 12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248,
26656 53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637,
26657 53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298,
26658 54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441,
26659 54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541,
26660 54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662,
26661 54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922,
26662 54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062,
26663 55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178,
26664 55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961,
26665 60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003,
26666 61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028,
26667 61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099,
26668 61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744,
26669 61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368,
26670 62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971,
26671 63045, 63104, 63232, 0, 42710, 42752, 46900, 46912, 47133, 63488,
26672 1, 32, 256, 0, 65533,
26673 };
26674static u16 aFts5UnicodeData[] = {
26675 1025, 61, 117, 55, 117, 54, 50, 53, 57, 53,
26676 49, 85, 333, 85, 121, 85, 841, 54, 53, 50,
26677 56, 48, 56, 837, 54, 57, 50, 57, 1057, 61,
26678 53, 151, 58, 53, 56, 58, 39, 52, 57, 34,
26679 58, 56, 58, 57, 79, 56, 37, 85, 56, 47,
26680 39, 51, 111, 53, 745, 57, 233, 773, 57, 261,
26681 1822, 37, 542, 37, 1534, 222, 69, 73, 37, 126,
26682 126, 73, 69, 137, 37, 73, 37, 105, 101, 73,
26683 37, 73, 37, 190, 158, 37, 126, 126, 73, 37,
26684 126, 94, 37, 39, 94, 69, 135, 41, 40, 37,
26685 41, 40, 37, 41, 40, 37, 542, 37, 606, 37,
26686 41, 40, 37, 126, 73, 37, 1886, 197, 73, 37,
26687 73, 69, 126, 105, 37, 286, 2181, 39, 869, 582,
26688 152, 390, 472, 166, 248, 38, 56, 38, 568, 3596,
26689 158, 38, 56, 94, 38, 101, 53, 88, 41, 53,
26690 105, 41, 73, 37, 553, 297, 1125, 94, 37, 105,
26691 101, 798, 133, 94, 57, 126, 94, 37, 1641, 1541,
26692 1118, 58, 172, 75, 1790, 478, 37, 2846, 1225, 38,
26693 213, 1253, 53, 49, 55, 1452, 49, 44, 53, 76,
26694 53, 76, 53, 44, 871, 103, 85, 162, 121, 85,
26695 55, 85, 90, 364, 53, 85, 1031, 38, 327, 684,
26696 333, 149, 71, 44, 3175, 53, 39, 236, 34, 58,
26697 204, 70, 76, 58, 140, 71, 333, 103, 90, 39,
26698 469, 34, 39, 44, 967, 876, 2855, 364, 39, 333,
26699 1063, 300, 70, 58, 117, 38, 711, 140, 38, 300,
26700 38, 108, 38, 172, 501, 807, 108, 53, 39, 359,
26701 876, 108, 42, 1735, 44, 42, 44, 39, 106, 268,
26702 138, 44, 74, 39, 236, 327, 76, 85, 333, 53,
26703 38, 199, 231, 44, 74, 263, 71, 711, 231, 39,
26704 135, 44, 39, 106, 140, 74, 74, 44, 39, 42,
26705 71, 103, 76, 333, 71, 87, 207, 58, 55, 76,
26706 42, 199, 71, 711, 231, 71, 71, 71, 44, 106,
26707 76, 76, 108, 44, 135, 39, 333, 76, 103, 44,
26708 76, 42, 295, 103, 711, 231, 71, 167, 44, 39,
26709 106, 172, 76, 42, 74, 44, 39, 71, 76, 333,
26710 53, 55, 44, 74, 263, 71, 711, 231, 71, 167,
26711 44, 39, 42, 44, 42, 140, 74, 74, 44, 44,
26712 42, 71, 103, 76, 333, 58, 39, 207, 44, 39,
26713 199, 103, 135, 71, 39, 71, 71, 103, 391, 74,
26714 44, 74, 106, 106, 44, 39, 42, 333, 111, 218,
26715 55, 58, 106, 263, 103, 743, 327, 167, 39, 108,
26716 138, 108, 140, 76, 71, 71, 76, 333, 239, 58,
26717 74, 263, 103, 743, 327, 167, 44, 39, 42, 44,
26718 170, 44, 74, 74, 76, 74, 39, 71, 76, 333,
26719 71, 74, 263, 103, 1319, 39, 106, 140, 106, 106,
26720 44, 39, 42, 71, 76, 333, 207, 58, 199, 74,
26721 583, 775, 295, 39, 231, 44, 106, 108, 44, 266,
26722 74, 53, 1543, 44, 71, 236, 55, 199, 38, 268,
26723 53, 333, 85, 71, 39, 71, 39, 39, 135, 231,
26724 103, 39, 39, 71, 135, 44, 71, 204, 76, 39,
26725 167, 38, 204, 333, 135, 39, 122, 501, 58, 53,
26726 122, 76, 218, 333, 335, 58, 44, 58, 44, 58,
26727 44, 54, 50, 54, 50, 74, 263, 1159, 460, 42,
26728 172, 53, 76, 167, 364, 1164, 282, 44, 218, 90,
26729 181, 154, 85, 1383, 74, 140, 42, 204, 42, 76,
26730 74, 76, 39, 333, 213, 199, 74, 76, 135, 108,
26731 39, 106, 71, 234, 103, 140, 423, 44, 74, 76,
26732 202, 44, 39, 42, 333, 106, 44, 90, 1225, 41,
26733 41, 1383, 53, 38, 10631, 135, 231, 39, 135, 1319,
26734 135, 1063, 135, 231, 39, 135, 487, 1831, 135, 2151,
26735 108, 309, 655, 519, 346, 2727, 49, 19847, 85, 551,
26736 61, 839, 54, 50, 2407, 117, 110, 423, 135, 108,
26737 583, 108, 85, 583, 76, 423, 103, 76, 1671, 76,
26738 42, 236, 266, 44, 74, 364, 117, 38, 117, 55,
26739 39, 44, 333, 335, 213, 49, 149, 108, 61, 333,
26740 1127, 38, 1671, 1319, 44, 39, 2247, 935, 108, 138,
26741 76, 106, 74, 44, 202, 108, 58, 85, 333, 967,
26742 167, 1415, 554, 231, 74, 333, 47, 1114, 743, 76,
26743 106, 85, 1703, 42, 44, 42, 236, 44, 42, 44,
26744 74, 268, 202, 332, 44, 333, 333, 245, 38, 213,
26745 140, 42, 1511, 44, 42, 172, 42, 44, 170, 44,
26746 74, 231, 333, 245, 346, 300, 314, 76, 42, 967,
26747 42, 140, 74, 76, 42, 44, 74, 71, 333, 1415,
26748 44, 42, 76, 106, 44, 42, 108, 74, 149, 1159,
26749 266, 268, 74, 76, 181, 333, 103, 333, 967, 198,
26750 85, 277, 108, 53, 428, 42, 236, 135, 44, 135,
26751 74, 44, 71, 1413, 2022, 421, 38, 1093, 1190, 1260,
26752 140, 4830, 261, 3166, 261, 265, 197, 201, 261, 265,
26753 261, 265, 197, 201, 261, 41, 41, 41, 94, 229,
26754 265, 453, 261, 264, 261, 264, 261, 264, 165, 69,
26755 137, 40, 56, 37, 120, 101, 69, 137, 40, 120,
26756 133, 69, 137, 120, 261, 169, 120, 101, 69, 137,
26757 40, 88, 381, 162, 209, 85, 52, 51, 54, 84,
26758 51, 54, 52, 277, 59, 60, 162, 61, 309, 52,
26759 51, 149, 80, 117, 57, 54, 50, 373, 57, 53,
26760 48, 341, 61, 162, 194, 47, 38, 207, 121, 54,
26761 50, 38, 335, 121, 54, 50, 422, 855, 428, 139,
26762 44, 107, 396, 90, 41, 154, 41, 90, 37, 105,
26763 69, 105, 37, 58, 41, 90, 57, 169, 218, 41,
26764 58, 41, 58, 41, 58, 137, 58, 37, 137, 37,
26765 135, 37, 90, 69, 73, 185, 94, 101, 58, 57,
26766 90, 37, 58, 527, 1134, 94, 142, 47, 185, 186,
26767 89, 154, 57, 90, 57, 90, 57, 250, 57, 1018,
26768 89, 90, 57, 58, 57, 1018, 8601, 282, 153, 666,
26769 89, 250, 54, 50, 2618, 57, 986, 825, 1306, 217,
26770 602, 1274, 378, 1935, 2522, 719, 5882, 57, 314, 57,
26771 1754, 281, 3578, 57, 4634, 3322, 54, 50, 54, 50,
26772 54, 50, 54, 50, 54, 50, 54, 50, 54, 50,
26773 975, 1434, 185, 54, 50, 1017, 54, 50, 54, 50,
26774 54, 50, 54, 50, 54, 50, 537, 8218, 4217, 54,
26775 50, 54, 50, 54, 50, 54, 50, 54, 50, 54,
26776 50, 54, 50, 54, 50, 54, 50, 54, 50, 54,
26777 50, 2041, 54, 50, 54, 50, 1049, 54, 50, 8281,
26778 1562, 697, 90, 217, 346, 1513, 1509, 126, 73, 69,
26779 254, 105, 37, 94, 37, 94, 165, 70, 105, 37,
26780 3166, 37, 218, 158, 108, 94, 149, 47, 85, 1221,
26781 37, 37, 1799, 38, 53, 44, 743, 231, 231, 231,
26782 231, 231, 231, 231, 231, 1036, 85, 52, 51, 52,
26783 51, 117, 52, 51, 53, 52, 51, 309, 49, 85,
26784 49, 53, 52, 51, 85, 52, 51, 54, 50, 54,
26785 50, 54, 50, 54, 50, 181, 38, 341, 81, 858,
26786 2874, 6874, 410, 61, 117, 58, 38, 39, 46, 54,
26787 50, 54, 50, 54, 50, 54, 50, 54, 50, 90,
26788 54, 50, 54, 50, 54, 50, 54, 50, 49, 54,
26789 82, 58, 302, 140, 74, 49, 166, 90, 110, 38,
26790 39, 53, 90, 2759, 76, 88, 70, 39, 49, 2887,
26791 53, 102, 39, 1319, 3015, 90, 143, 346, 871, 1178,
26792 519, 1018, 335, 986, 271, 58, 495, 1050, 335, 1274,
26793 495, 2042, 8218, 39, 39, 2074, 39, 39, 679, 38,
26794 36583, 1786, 1287, 198, 85, 8583, 38, 117, 519, 333,
26795 71, 1502, 39, 44, 107, 53, 332, 53, 38, 798,
26796 44, 2247, 334, 76, 213, 760, 294, 88, 478, 69,
26797 2014, 38, 261, 190, 350, 38, 88, 158, 158, 382,
26798 70, 37, 231, 44, 103, 44, 135, 44, 743, 74,
26799 76, 42, 154, 207, 90, 55, 58, 1671, 149, 74,
26800 1607, 522, 44, 85, 333, 588, 199, 117, 39, 333,
26801 903, 268, 85, 743, 364, 74, 53, 935, 108, 42,
26802 1511, 44, 74, 140, 74, 44, 138, 437, 38, 333,
26803 85, 1319, 204, 74, 76, 74, 76, 103, 44, 263,
26804 44, 42, 333, 149, 519, 38, 199, 122, 39, 42,
26805 1543, 44, 39, 108, 71, 76, 167, 76, 39, 44,
26806 39, 71, 38, 85, 359, 42, 76, 74, 85, 39,
26807 70, 42, 44, 199, 199, 199, 231, 231, 1127, 74,
26808 44, 74, 44, 74, 53, 42, 44, 333, 39, 39,
26809 743, 1575, 36, 68, 68, 36, 63, 63, 11719, 3399,
26810 229, 165, 39, 44, 327, 57, 423, 167, 39, 71,
26811 71, 3463, 536, 11623, 54, 50, 2055, 1735, 391, 55,
26812 58, 524, 245, 54, 50, 53, 236, 53, 81, 80,
26813 54, 50, 54, 50, 54, 50, 54, 50, 54, 50,
26814 54, 50, 54, 50, 54, 50, 85, 54, 50, 149,
26815 112, 117, 149, 49, 54, 50, 54, 50, 54, 50,
26816 117, 57, 49, 121, 53, 55, 85, 167, 4327, 34,
26817 117, 55, 117, 54, 50, 53, 57, 53, 49, 85,
26818 333, 85, 121, 85, 841, 54, 53, 50, 56, 48,
26819 56, 837, 54, 57, 50, 57, 54, 50, 53, 54,
26820 50, 85, 327, 38, 1447, 70, 999, 199, 199, 199,
26821 103, 87, 57, 56, 58, 87, 58, 153, 90, 98,
26822 90, 391, 839, 615, 71, 487, 455, 3943, 117, 1455,
26823 314, 1710, 143, 570, 47, 410, 1466, 44, 935, 1575,
26824 999, 143, 551, 46, 263, 46, 967, 53, 1159, 263,
26825 53, 174, 1289, 1285, 2503, 333, 199, 39, 1415, 71,
26826 39, 743, 53, 271, 711, 207, 53, 839, 53, 1799,
26827 71, 39, 108, 76, 140, 135, 103, 871, 108, 44,
26828 271, 309, 935, 79, 53, 1735, 245, 711, 271, 615,
26829 271, 2343, 1007, 42, 44, 42, 1703, 492, 245, 655,
26830 333, 76, 42, 1447, 106, 140, 74, 76, 85, 34,
26831 149, 807, 333, 108, 1159, 172, 42, 268, 333, 149,
26832 76, 42, 1543, 106, 300, 74, 135, 149, 333, 1383,
26833 44, 42, 44, 74, 204, 42, 44, 333, 28135, 3182,
26834 149, 34279, 18215, 2215, 39, 1482, 140, 422, 71, 7898,
26835 1274, 1946, 74, 108, 122, 202, 258, 268, 90, 236,
26836 986, 140, 1562, 2138, 108, 58, 2810, 591, 841, 837,
26837 841, 229, 581, 841, 837, 41, 73, 41, 73, 137,
26838 265, 133, 37, 229, 357, 841, 837, 73, 137, 265,
26839 233, 837, 73, 137, 169, 41, 233, 837, 841, 837,
26840 841, 837, 841, 837, 841, 837, 841, 837, 841, 901,
26841 809, 57, 805, 57, 197, 809, 57, 805, 57, 197,
26842 809, 57, 805, 57, 197, 809, 57, 805, 57, 197,
26843 809, 57, 805, 57, 197, 94, 1613, 135, 871, 71,
26844 39, 39, 327, 135, 39, 39, 39, 39, 39, 39,
26845 103, 71, 39, 39, 39, 39, 39, 39, 71, 39,
26846 135, 231, 135, 135, 39, 327, 551, 103, 167, 551,
26847 89, 1434, 3226, 506, 474, 506, 506, 367, 1018, 1946,
26848 1402, 954, 1402, 314, 90, 1082, 218, 2266, 666, 1210,
26849 186, 570, 2042, 58, 5850, 154, 2010, 154, 794, 2266,
26850 378, 2266, 3738, 39, 39, 39, 39, 39, 39, 17351,
26851 34, 3074, 7692, 63, 63,
26852 };
26853
26854static int sqlite3Fts5UnicodeCategory(u32 iCode) {
26855 int iRes = -1;
26856 int iHi;
26857 int iLo;
26858 int ret;
26859 u16 iKey;
26860
26861 if( iCode>=(1<<20) ){
26862 return 0;
26863 }
26864 iLo = aFts5UnicodeBlock[(iCode>>16)];
26865 iHi = aFts5UnicodeBlock[1+(iCode>>16)];
26866 iKey = (iCode & 0xFFFF);
26867 while( iHi>iLo ){
26868 int iTest = (iHi + iLo) / 2;
26869 assert( iTest>=iLo && iTest<iHi )((void) (0));
26870 if( iKey>=aFts5UnicodeMap[iTest] ){
26871 iRes = iTest;
26872 iLo = iTest+1;
26873 }else{
26874 iHi = iTest;
26875 }
26876 }
26877
26878 if( iRes<0 ) return 0;
26879 if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
26880 ret = aFts5UnicodeData[iRes] & 0x1F;
26881 if( ret!=30 ) return ret;
26882 return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9;
26883}
26884
26885static void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
26886 int i = 0;
26887 int iTbl = 0;
26888 while( i<128 ){
26889 int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
26890 int n = (aFts5UnicodeData[iTbl] >> 5) + i;
26891 for(; i<128 && i<n; i++){
26892 aAscii[i] = (u8)bToken;
26893 }
26894 iTbl++;
26895 }
26896 aAscii[0] = 0; /* 0x00 is never a token character */
26897}
26898
26899#line 1 "fts5_varint.c"
26900/*
26901** 2015 May 30
26902**
26903** The author disclaims copyright to this source code. In place of
26904** a legal notice, here is a blessing:
26905**
26906** May you do good and not evil.
26907** May you find forgiveness for yourself and forgive others.
26908** May you share freely, never taking more than you give.
26909**
26910******************************************************************************
26911**
26912** Routines for varint serialization and deserialization.
26913*/
26914
26915
26916/* #include "fts5Int.h" */
26917
26918/*
26919** This is a copy of the sqlite3GetVarint32() routine from the SQLite core.
26920** Except, this version does handle the single byte case that the core
26921** version depends on being handled before its function is called.
26922*/
26923static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v){
26924 u32 a,b;
26925
26926 /* The 1-byte case. Overwhelmingly the most common. */
26927 a = *p;
26928 /* a: p0 (unmasked) */
26929 if (!(a&0x80))
26930 {
26931 /* Values between 0 and 127 */
26932 *v = a;
26933 return 1;
26934 }
26935
26936 /* The 2-byte case */
26937 p++;
26938 b = *p;
26939 /* b: p1 (unmasked) */
26940 if (!(b&0x80))
26941 {
26942 /* Values between 128 and 16383 */
26943 a &= 0x7f;
26944 a = a<<7;
26945 *v = a | b;
26946 return 2;
26947 }
26948
26949 /* The 3-byte case */
26950 p++;
26951 a = a<<14;
26952 a |= *p;
26953 /* a: p0<<14 | p2 (unmasked) */
26954 if (!(a&0x80))
26955 {
26956 /* Values between 16384 and 2097151 */
26957 a &= (0x7f<<14)|(0x7f);
26958 b &= 0x7f;
26959 b = b<<7;
26960 *v = a | b;
26961 return 3;
26962 }
26963
26964 /* A 32-bit varint is used to store size information in btrees.
26965 ** Objects are rarely larger than 2MiB limit of a 3-byte varint.
26966 ** A 3-byte varint is sufficient, for example, to record the size
26967 ** of a 1048569-byte BLOB or string.
26968 **
26969 ** We only unroll the first 1-, 2-, and 3- byte cases. The very
26970 ** rare larger cases can be handled by the slower 64-bit varint
26971 ** routine.
26972 */
26973 {
26974 u64 v64;
26975 u8 n;
26976 p -= 2;
26977 n = sqlite3Fts5GetVarint(p, &v64);
26978 *v = ((u32)v64) & 0x7FFFFFFF;
26979 assert( n>3 && n<=9 )((void) (0));
26980 return n;
26981 }
26982}
26983
26984
26985/*
26986** Bitmasks used by sqlite3GetVarint(). These precomputed constants
26987** are defined here rather than simply putting the constant expressions
26988** inline in order to work around bugs in the RVT compiler.
26989**
26990** SLOT_2_0 A mask for (0x7f<<14) | 0x7f
26991**
26992** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0
26993*/
26994#define SLOT_2_00x001fc07f 0x001fc07f
26995#define SLOT_4_2_00xf01fc07f 0xf01fc07f
26996
26997/*
26998** Read a 64-bit variable-length integer from memory starting at p[0].
26999** Return the number of bytes read. The value is stored in *v.
27000*/
27001static u8 sqlite3Fts5GetVarint(const unsigned char *p, u64 *v){
27002 u32 a,b,s;
27003
27004 a = *p;
27005 /* a: p0 (unmasked) */
27006 if (!(a&0x80))
27007 {
27008 *v = a;
27009 return 1;
27010 }
27011
27012 p++;
27013 b = *p;
27014 /* b: p1 (unmasked) */
27015 if (!(b&0x80))
27016 {
27017 a &= 0x7f;
27018 a = a<<7;
27019 a |= b;
27020 *v = a;
27021 return 2;
27022 }
27023
27024 /* Verify that constants are precomputed correctly */
27025 assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) )((void) (0));
27026 assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) )((void) (0));
27027
27028 p++;
27029 a = a<<14;
27030 a |= *p;
27031 /* a: p0<<14 | p2 (unmasked) */
27032 if (!(a&0x80))
27033 {
27034 a &= SLOT_2_00x001fc07f;
27035 b &= 0x7f;
27036 b = b<<7;
27037 a |= b;
27038 *v = a;
27039 return 3;
27040 }
27041
27042 /* CSE1 from below */
27043 a &= SLOT_2_00x001fc07f;
27044 p++;
27045 b = b<<14;
27046 b |= *p;
27047 /* b: p1<<14 | p3 (unmasked) */
27048 if (!(b&0x80))
27049 {
27050 b &= SLOT_2_00x001fc07f;
27051 /* moved CSE1 up */
27052 /* a &= (0x7f<<14)|(0x7f); */
27053 a = a<<7;
27054 a |= b;
27055 *v = a;
27056 return 4;
27057 }
27058
27059 /* a: p0<<14 | p2 (masked) */
27060 /* b: p1<<14 | p3 (unmasked) */
27061 /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
27062 /* moved CSE1 up */
27063 /* a &= (0x7f<<14)|(0x7f); */
27064 b &= SLOT_2_00x001fc07f;
27065 s = a;
27066 /* s: p0<<14 | p2 (masked) */
27067
27068 p++;
27069 a = a<<14;
27070 a |= *p;
27071 /* a: p0<<28 | p2<<14 | p4 (unmasked) */
27072 if (!(a&0x80))
27073 {
27074 /* we can skip these cause they were (effectively) done above in calc'ing s */
27075 /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
27076 /* b &= (0x7f<<14)|(0x7f); */
27077 b = b<<7;
27078 a |= b;
27079 s = s>>18;
27080 *v = ((u64)s)<<32 | a;
27081 return 5;
27082 }
27083
27084 /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
27085 s = s<<7;
27086 s |= b;
27087 /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */
27088
27089 p++;
27090 b = b<<14;
27091 b |= *p;
27092 /* b: p1<<28 | p3<<14 | p5 (unmasked) */
27093 if (!(b&0x80))
27094 {
27095 /* we can skip this cause it was (effectively) done above in calc'ing s */
27096 /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */
27097 a &= SLOT_2_00x001fc07f;
27098 a = a<<7;
27099 a |= b;
27100 s = s>>18;
27101 *v = ((u64)s)<<32 | a;
27102 return 6;
27103 }
27104
27105 p++;
27106 a = a<<14;
27107 a |= *p;
27108 /* a: p2<<28 | p4<<14 | p6 (unmasked) */
27109 if (!(a&0x80))
27110 {
27111 a &= SLOT_4_2_00xf01fc07f;
27112 b &= SLOT_2_00x001fc07f;
27113 b = b<<7;
27114 a |= b;
27115 s = s>>11;
27116 *v = ((u64)s)<<32 | a;
27117 return 7;
27118 }
27119
27120 /* CSE2 from below */
27121 a &= SLOT_2_00x001fc07f;
27122 p++;
27123 b = b<<14;
27124 b |= *p;
27125 /* b: p3<<28 | p5<<14 | p7 (unmasked) */
27126 if (!(b&0x80))
27127 {
27128 b &= SLOT_4_2_00xf01fc07f;
27129 /* moved CSE2 up */
27130 /* a &= (0x7f<<14)|(0x7f); */
27131 a = a<<7;
27132 a |= b;
27133 s = s>>4;
27134 *v = ((u64)s)<<32 | a;
27135 return 8;
27136 }
27137
27138 p++;
27139 a = a<<15;
27140 a |= *p;
27141 /* a: p4<<29 | p6<<15 | p8 (unmasked) */
27142
27143 /* moved CSE2 up */
27144 /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */
27145 b &= SLOT_2_00x001fc07f;
27146 b = b<<8;
27147 a |= b;
27148
27149 s = s<<4;
27150 b = p[-4];
27151 b &= 0x7f;
27152 b = b>>3;
27153 s |= b;
27154
27155 *v = ((u64)s)<<32 | a;
27156
27157 return 9;
27158}
27159
27160/*
27161** The variable-length integer encoding is as follows:
27162**
27163** KEY:
27164** A = 0xxxxxxx 7 bits of data and one flag bit
27165** B = 1xxxxxxx 7 bits of data and one flag bit
27166** C = xxxxxxxx 8 bits of data
27167**
27168** 7 bits - A
27169** 14 bits - BA
27170** 21 bits - BBA
27171** 28 bits - BBBA
27172** 35 bits - BBBBA
27173** 42 bits - BBBBBA
27174** 49 bits - BBBBBBA
27175** 56 bits - BBBBBBBA
27176** 64 bits - BBBBBBBBC
27177*/
27178
27179#ifdef SQLITE_NOINLINE
27180# define FTS5_NOINLINE SQLITE_NOINLINE
27181#else
27182# define FTS5_NOINLINE
27183#endif
27184
27185/*
27186** Write a 64-bit variable-length integer to memory starting at p[0].
27187** The length of data write will be between 1 and 9 bytes. The number
27188** of bytes written is returned.
27189**
27190** A variable-length integer consists of the lower 7 bits of each byte
27191** for all bytes that have the 8th bit set and one byte with the 8th
27192** bit clear. Except, if we get to the 9th byte, it stores the full
27193** 8 bits and is the last byte.
27194*/
27195static int FTS5_NOINLINE fts5PutVarint64(unsigned char *p, u64 v){
27196 int i, j, n;
27197 u8 buf[10];
27198 if( v & (((u64)0xff000000)<<32) ){
27199 p[8] = (u8)v;
27200 v >>= 8;
27201 for(i=7; i>=0; i--){
27202 p[i] = (u8)((v & 0x7f) | 0x80);
27203 v >>= 7;
27204 }
27205 return 9;
27206 }
27207 n = 0;
27208 do{
27209 buf[n++] = (u8)((v & 0x7f) | 0x80);
27210 v >>= 7;
27211 }while( v!=0 );
27212 buf[0] &= 0x7f;
27213 assert( n<=9 )((void) (0));
27214 for(i=0, j=n-1; j>=0; j--, i++){
27215 p[i] = buf[j];
27216 }
27217 return n;
27218}
27219
27220static int sqlite3Fts5PutVarint(unsigned char *p, u64 v){
27221 if( v<=0x7f ){
27222 p[0] = v&0x7f;
27223 return 1;
27224 }
27225 if( v<=0x3fff ){
27226 p[0] = ((v>>7)&0x7f)|0x80;
27227 p[1] = v&0x7f;
27228 return 2;
27229 }
27230 return fts5PutVarint64(p,v);
27231}
27232
27233
27234static int sqlite3Fts5GetVarintLen(u32 iVal){
27235#if 0
27236 if( iVal<(1 << 7 ) ) return 1;
27237#endif
27238 assert( iVal>=(1 << 7) )((void) (0));
27239 if( iVal<(1 << 14) ) return 2;
27240 if( iVal<(1 << 21) ) return 3;
27241 if( iVal<(1 << 28) ) return 4;
27242 return 5;
27243}
27244
27245#line 1 "fts5_vocab.c"
27246/*
27247** 2015 May 08
27248**
27249** The author disclaims copyright to this source code. In place of
27250** a legal notice, here is a blessing:
27251**
27252** May you do good and not evil.
27253** May you find forgiveness for yourself and forgive others.
27254** May you share freely, never taking more than you give.
27255**
27256******************************************************************************
27257**
27258** This is an SQLite virtual table module implementing direct access to an
27259** existing FTS5 index. The module may create several different types of
27260** tables:
27261**
27262** col:
27263** CREATE TABLE vocab(term, col, doc, cnt, PRIMARY KEY(term, col));
27264**
27265** One row for each term/column combination. The value of $doc is set to
27266** the number of fts5 rows that contain at least one instance of term
27267** $term within column $col. Field $cnt is set to the total number of
27268** instances of term $term in column $col (in any row of the fts5 table).
27269**
27270** row:
27271** CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term));
27272**
27273** One row for each term in the database. The value of $doc is set to
27274** the number of fts5 rows that contain at least one instance of term
27275** $term. Field $cnt is set to the total number of instances of term
27276** $term in the database.
27277**
27278** instance:
27279** CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY(<all-fields>));
27280**
27281** One row for each term instance in the database.
27282*/
27283
27284
27285/* #include "fts5Int.h" */
27286
27287
27288typedef struct Fts5VocabTable Fts5VocabTable;
27289typedef struct Fts5VocabCursor Fts5VocabCursor;
27290
27291struct Fts5VocabTable {
27292 sqlite3_vtab base;
27293 char *zFts5Tbl; /* Name of fts5 table */
27294 char *zFts5Db; /* Db containing fts5 table */
27295 sqlite3 *db; /* Database handle */
27296 Fts5Global *pGlobal; /* FTS5 global object for this database */
27297 int eType; /* FTS5_VOCAB_COL, ROW or INSTANCE */
27298 unsigned bBusy; /* True if busy */
27299};
27300
27301struct Fts5VocabCursor {
27302 sqlite3_vtab_cursor base;
27303 sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */
27304 Fts5Table *pFts5; /* Associated FTS5 table */
27305
27306 int bEof; /* True if this cursor is at EOF */
27307 Fts5IndexIter *pIter; /* Term/rowid iterator object */
27308 void *pStruct; /* From sqlite3Fts5StructureRef() */
27309
27310 int nLeTerm; /* Size of zLeTerm in bytes */
27311 char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */
27312 int colUsed; /* Copy of sqlite3_index_info.colUsed */
27313
27314 /* These are used by 'col' tables only */
27315 int iCol;
27316 i64 *aCnt;
27317 i64 *aDoc;
27318
27319 /* Output values used by all tables. */
27320 i64 rowid; /* This table's current rowid value */
27321 Fts5Buffer term; /* Current value of 'term' column */
27322
27323 /* Output values Used by 'instance' tables only */
27324 i64 iInstPos;
27325 int iInstOff;
27326};
27327
27328#define FTS5_VOCAB_COL0 0
27329#define FTS5_VOCAB_ROW1 1
27330#define FTS5_VOCAB_INSTANCE2 2
27331
27332#define FTS5_VOCAB_COL_SCHEMA"term, col, doc, cnt" "term, col, doc, cnt"
27333#define FTS5_VOCAB_ROW_SCHEMA"term, doc, cnt" "term, doc, cnt"
27334#define FTS5_VOCAB_INST_SCHEMA"term, doc, col, offset" "term, doc, col, offset"
27335
27336/*
27337** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
27338*/
27339#define FTS5_VOCAB_TERM_EQ0x0100 0x0100
27340#define FTS5_VOCAB_TERM_GE0x0200 0x0200
27341#define FTS5_VOCAB_TERM_LE0x0400 0x0400
27342
27343#define FTS5_VOCAB_COLUSED_MASK0xFF 0xFF
27344
27345
27346/*
27347** Translate a string containing an fts5vocab table type to an
27348** FTS5_VOCAB_XXX constant. If successful, set *peType to the output
27349** value and return SQLITE_OK. Otherwise, set *pzErr to an error message
27350** and return SQLITE_ERROR.
27351*/
27352static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){
27353 int rc = SQLITE_OK0;
27354 char *zCopy = sqlite3Fts5Strndup(&rc, zType, -1);
27355 if( rc==SQLITE_OK0 ){
27356 sqlite3Fts5Dequote(zCopy);
27357 if( sqlite3_stricmpsqlite3_api->stricmp(zCopy, "col")==0 ){
27358 *peType = FTS5_VOCAB_COL0;
27359 }else
27360
27361 if( sqlite3_stricmpsqlite3_api->stricmp(zCopy, "row")==0 ){
27362 *peType = FTS5_VOCAB_ROW1;
27363 }else
27364 if( sqlite3_stricmpsqlite3_api->stricmp(zCopy, "instance")==0 ){
27365 *peType = FTS5_VOCAB_INSTANCE2;
27366 }else
27367 {
27368 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("fts5vocab: unknown table type: %Q", zCopy);
27369 rc = SQLITE_ERROR1;
27370 }
27371 sqlite3_freesqlite3_api->free(zCopy);
27372 }
27373
27374 return rc;
27375}
27376
27377
27378/*
27379** The xDisconnect() virtual table method.
27380*/
27381static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){
27382 Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
27383 sqlite3_freesqlite3_api->free(pTab);
27384 return SQLITE_OK0;
27385}
27386
27387/*
27388** The xDestroy() virtual table method.
27389*/
27390static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){
27391 Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab;
27392 sqlite3_freesqlite3_api->free(pTab);
27393 return SQLITE_OK0;
27394}
27395
27396/*
27397** This function is the implementation of both the xConnect and xCreate
27398** methods of the FTS3 virtual table.
27399**
27400** The argv[] array contains the following:
27401**
27402** argv[0] -> module name ("fts5vocab")
27403** argv[1] -> database name
27404** argv[2] -> table name
27405**
27406** then:
27407**
27408** argv[3] -> name of fts5 table
27409** argv[4] -> type of fts5vocab table
27410**
27411** or, for tables in the TEMP schema only.
27412**
27413** argv[3] -> name of fts5 tables database
27414** argv[4] -> name of fts5 table
27415** argv[5] -> type of fts5vocab table
27416*/
27417static int fts5VocabInitVtab(
27418 sqlite3 *db, /* The SQLite database connection */
27419 void *pAux, /* Pointer to Fts5Global object */
27420 int argc, /* Number of elements in argv array */
27421 const char * const *argv, /* xCreate/xConnect argument array */
27422 sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
27423 char **pzErr /* Write any error message here */
27424){
27425 const char *azSchema[] = {
27426 "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA"term, col, doc, cnt" ")",
27427 "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA"term, doc, cnt" ")",
27428 "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA"term, doc, col, offset" ")"
27429 };
27430
27431 Fts5VocabTable *pRet = 0;
27432 int rc = SQLITE_OK0; /* Return code */
27433 int bDb;
27434
27435 bDb = (argc==6 && strlen(argv[1])==4 && memcmp("temp", argv[1], 4)==0);
27436
27437 if( argc!=5 && bDb==0 ){
27438 *pzErr = sqlite3_mprintfsqlite3_api->mprintf("wrong number of vtable arguments");
27439 rc = SQLITE_ERROR1;
27440 }else{
27441 i64 nByte; /* Bytes of space to allocate */
27442 const char *zDb = bDb ? argv[3] : argv[1];
27443 const char *zTab = bDb ? argv[4] : argv[3];
27444 const char *zType = bDb ? argv[5] : argv[4];
27445 i64 nDb = strlen(zDb)+1;
27446 i64 nTab = strlen(zTab)+1;
27447 int eType = 0;
27448
27449 rc = fts5VocabTableType(zType, pzErr, &eType);
27450 if( rc==SQLITE_OK0 ){
27451 assert( eType>=0 && eType<ArraySize(azSchema) )((void) (0));
27452 rc = sqlite3_declare_vtabsqlite3_api->declare_vtab(db, azSchema[eType]);
27453 }
27454
27455 nByte = sizeof(Fts5VocabTable) + nDb + nTab;
27456 pRet = sqlite3Fts5MallocZero(&rc, nByte);
27457 if( pRet ){
27458 pRet->pGlobal = (Fts5Global*)pAux;
27459 pRet->eType = eType;
27460 pRet->db = db;
27461 pRet->zFts5Tbl = (char*)&pRet[1];
27462 pRet->zFts5Db = &pRet->zFts5Tbl[nTab];
27463 memcpy(pRet->zFts5Tbl, zTab, nTab);
27464 memcpy(pRet->zFts5Db, zDb, nDb);
27465 sqlite3Fts5Dequote(pRet->zFts5Tbl);
27466 sqlite3Fts5Dequote(pRet->zFts5Db);
27467 }
27468 }
27469
27470 *ppVTab = (sqlite3_vtab*)pRet;
27471 return rc;
27472}
27473
27474
27475/*
27476** The xConnect() and xCreate() methods for the virtual table. All the
27477** work is done in function fts5VocabInitVtab().
27478*/
27479static int fts5VocabConnectMethod(
27480 sqlite3 *db, /* Database connection */
27481 void *pAux, /* Pointer to tokenizer hash table */
27482 int argc, /* Number of elements in argv array */
27483 const char * const *argv, /* xCreate/xConnect argument array */
27484 sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
27485 char **pzErr /* OUT: sqlite3_malloc'd error message */
27486){
27487 return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
27488}
27489static int fts5VocabCreateMethod(
27490 sqlite3 *db, /* Database connection */
27491 void *pAux, /* Pointer to tokenizer hash table */
27492 int argc, /* Number of elements in argv array */
27493 const char * const *argv, /* xCreate/xConnect argument array */
27494 sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
27495 char **pzErr /* OUT: sqlite3_malloc'd error message */
27496){
27497 return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr);
27498}
27499
27500/*
27501** Implementation of the xBestIndex method.
27502**
27503** Only constraints of the form:
27504**
27505** term <= ?
27506** term == ?
27507** term >= ?
27508**
27509** are interpreted. Less-than and less-than-or-equal are treated
27510** identically, as are greater-than and greater-than-or-equal.
27511*/
27512static int fts5VocabBestIndexMethod(
27513 sqlite3_vtab *pUnused,
27514 sqlite3_index_info *pInfo
27515){
27516 int i;
27517 int iTermEq = -1;
27518 int iTermGe = -1;
27519 int iTermLe = -1;
27520 int idxNum = (int)pInfo->colUsed;
27521 int nArg = 0;
27522
27523 UNUSED_PARAM(pUnused)(void)(pUnused);
27524
27525 assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed )((void) (0));
27526
27527 for(i=0; i<pInfo->nConstraint; i++){
27528 struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
27529 if( p->usable==0 ) continue;
27530 if( p->iColumn==0 ){ /* term column */
27531 if( p->op==SQLITE_INDEX_CONSTRAINT_EQ2 ) iTermEq = i;
27532 if( p->op==SQLITE_INDEX_CONSTRAINT_LE8 ) iTermLe = i;
27533 if( p->op==SQLITE_INDEX_CONSTRAINT_LT16 ) iTermLe = i;
27534 if( p->op==SQLITE_INDEX_CONSTRAINT_GE32 ) iTermGe = i;
27535 if( p->op==SQLITE_INDEX_CONSTRAINT_GT4 ) iTermGe = i;
27536 }
27537 }
27538
27539 if( iTermEq>=0 ){
27540 idxNum |= FTS5_VOCAB_TERM_EQ0x0100;
27541 pInfo->aConstraintUsage[iTermEq].argvIndex = ++nArg;
27542 pInfo->estimatedCost = 100;
27543 }else{
27544 pInfo->estimatedCost = 1000000;
27545 if( iTermGe>=0 ){
27546 idxNum |= FTS5_VOCAB_TERM_GE0x0200;
27547 pInfo->aConstraintUsage[iTermGe].argvIndex = ++nArg;
27548 pInfo->estimatedCost = pInfo->estimatedCost / 2;
27549 }
27550 if( iTermLe>=0 ){
27551 idxNum |= FTS5_VOCAB_TERM_LE0x0400;
27552 pInfo->aConstraintUsage[iTermLe].argvIndex = ++nArg;
27553 pInfo->estimatedCost = pInfo->estimatedCost / 2;
27554 }
27555 }
27556
27557 /* This virtual table always delivers results in ascending order of
27558 ** the "term" column (column 0). So if the user has requested this
27559 ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the
27560 ** sqlite3_index_info.orderByConsumed flag to tell the core the results
27561 ** are already in sorted order. */
27562 if( pInfo->nOrderBy==1
27563 && pInfo->aOrderBy[0].iColumn==0
27564 && pInfo->aOrderBy[0].desc==0
27565 ){
27566 pInfo->orderByConsumed = 1;
27567 }
27568
27569 pInfo->idxNum = idxNum;
27570 return SQLITE_OK0;
27571}
27572
27573/*
27574** Implementation of xOpen method.
27575*/
27576static int fts5VocabOpenMethod(
27577 sqlite3_vtab *pVTab,
27578 sqlite3_vtab_cursor **ppCsr
27579){
27580 Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab;
27581 Fts5Table *pFts5 = 0;
27582 Fts5VocabCursor *pCsr = 0;
27583 int rc = SQLITE_OK0;
27584 sqlite3_stmt *pStmt = 0;
27585 char *zSql = 0;
27586
27587 if( pTab->bBusy ){
27588 pVTab->zErrMsg = sqlite3_mprintfsqlite3_api->mprintf(
27589 "recursive definition for %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
27590 );
27591 return SQLITE_ERROR1;
27592 }
27593 zSql = sqlite3Fts5Mprintf(&rc,
27594 "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'",
27595 pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl
27596 );
27597 if( zSql ){
27598 rc = sqlite3_prepare_v2sqlite3_api->prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
27599 }
27600 sqlite3_freesqlite3_api->free(zSql);
27601 assert( rc==SQLITE_OK || pStmt==0 )((void) (0));
27602 if( rc==SQLITE_ERROR1 ) rc = SQLITE_OK0;
27603
27604 pTab->bBusy = 1;
27605 if( pStmt && sqlite3_stepsqlite3_api->step(pStmt)==SQLITE_ROW100 ){
27606 i64 iId = sqlite3_column_int64sqlite3_api->column_int64(pStmt, 0);
27607 pFts5 = sqlite3Fts5TableFromCsrid(pTab->pGlobal, iId);
27608 }
27609 pTab->bBusy = 0;
27610
27611 if( rc==SQLITE_OK0 ){
27612 if( pFts5==0 ){
27613 rc = sqlite3_finalizesqlite3_api->finalize(pStmt);
27614 pStmt = 0;
27615 if( rc==SQLITE_OK0 ){
27616 pVTab->zErrMsg = sqlite3_mprintfsqlite3_api->mprintf(
27617 "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
27618 );
27619 rc = SQLITE_ERROR1;
27620 }
27621 }else{
27622 rc = sqlite3Fts5FlushToDisk(pFts5);
27623 }
27624 }
27625
27626 if( rc==SQLITE_OK0 ){
27627 i64 nByte = pFts5->pConfig->nCol * sizeof(i64)*2 + sizeof(Fts5VocabCursor);
27628 pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte);
27629 }
27630
27631 if( pCsr ){
27632 pCsr->pFts5 = pFts5;
27633 pCsr->pStmt = pStmt;
27634 pCsr->aCnt = (i64*)&pCsr[1];
27635 pCsr->aDoc = &pCsr->aCnt[pFts5->pConfig->nCol];
27636 }else{
27637 sqlite3_finalizesqlite3_api->finalize(pStmt);
27638 }
27639
27640 *ppCsr = (sqlite3_vtab_cursor*)pCsr;
27641 return rc;
27642}
27643
27644/*
27645** Restore cursor pCsr to the state it was in immediately after being
27646** created by the xOpen() method.
27647*/
27648static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){
27649 int nCol = pCsr->pFts5->pConfig->nCol;
27650 pCsr->rowid = 0;
27651 sqlite3Fts5IterClose(pCsr->pIter);
27652 sqlite3Fts5StructureRelease(pCsr->pStruct);
27653 pCsr->pStruct = 0;
27654 pCsr->pIter = 0;
2
←
Null pointer value stored to field 'pIter'
→
27655 sqlite3_freesqlite3_api->free(pCsr->zLeTerm);
27656 pCsr->nLeTerm = -1;
27657 pCsr->zLeTerm = 0;
27658 pCsr->bEof = 0;
27659 pCsr->iCol = 0;
27660 pCsr->iInstPos = 0;
27661 pCsr->iInstOff = 0;
27662 pCsr->colUsed = 0;
27663 memset(pCsr->aCnt, 0, sizeof(i64)*nCol);
27664 memset(pCsr->aDoc, 0, sizeof(i64)*nCol);
27665}
27666
27667/*
27668** Close the cursor. For additional information see the documentation
27669** on the xClose method of the virtual table interface.
27670*/
27671static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){
27672 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
27673 fts5VocabResetCursor(pCsr);
27674 sqlite3Fts5BufferFree(&pCsr->term);
27675 sqlite3_finalizesqlite3_api->finalize(pCsr->pStmt);
27676 sqlite3_freesqlite3_api->free(pCsr);
27677 return SQLITE_OK0;
27678}
27679
27680static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){
27681 int rc = SQLITE_OK0;
27682
27683 if( sqlite3Fts5IterEof(pCsr->pIter)((pCsr->pIter)->bEof) ){
26
←
Access to field 'bEof' results in a dereference of a null pointer (loaded from field 'pIter')
27684 pCsr->bEof = 1;
27685 }else{
27686 const char *zTerm;
27687 int nTerm;
27688 zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
27689 if( pCsr->nLeTerm>=0 ){
27690 int nCmp = MIN(nTerm, pCsr->nLeTerm)(((nTerm) < (pCsr->nLeTerm)) ? (nTerm) : (pCsr->nLeTerm
));
27691 int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
27692 if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
27693 pCsr->bEof = 1;
27694 }
27695 }
27696
27697 sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
27698 }
27699 return rc;
27700}
27701
27702static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){
27703 int eDetail = pCsr->pFts5->pConfig->eDetail;
27704 int rc = SQLITE_OK0;
27705 Fts5IndexIter *pIter = pCsr->pIter;
27706 i64 *pp = &pCsr->iInstPos;
27707 int *po = &pCsr->iInstOff;
27708
27709 assert( sqlite3Fts5IterEof(pIter)==0 )((void) (0));
27710 assert( pCsr->bEof==0 )((void) (0));
27711 while( eDetail==FTS5_DETAIL_NONE1
27712 || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp)
27713 ){
27714 pCsr->iInstPos = 0;
27715 pCsr->iInstOff = 0;
27716
27717 rc = sqlite3Fts5IterNextScan(pCsr->pIter);
27718 if( rc==SQLITE_OK0 ){
27719 rc = fts5VocabInstanceNewTerm(pCsr);
27720 if( pCsr->bEof || eDetail==FTS5_DETAIL_NONE1 ) break;
27721 }
27722 if( rc ){
27723 pCsr->bEof = 1;
27724 break;
27725 }
27726 }
27727
27728 return rc;
27729}
27730
27731/*
27732** Advance the cursor to the next row in the table.
27733*/
27734static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
27735 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
27736 Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
27737 int nCol = pCsr->pFts5->pConfig->nCol;
27738 int rc;
27739
27740 rc = sqlite3Fts5StructureTest(pCsr->pFts5->pIndex, pCsr->pStruct);
27741 if( rc!=SQLITE_OK0 ) return rc;
27742 pCsr->rowid++;
27743
27744 if( pTab->eType==FTS5_VOCAB_INSTANCE2 ){
27745 return fts5VocabInstanceNext(pCsr);
27746 }
27747
27748 if( pTab->eType==FTS5_VOCAB_COL0 ){
27749 for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
27750 if( pCsr->aDoc[pCsr->iCol] ) break;
27751 }
27752 }
27753
27754 if( pTab->eType!=FTS5_VOCAB_COL0 || pCsr->iCol>=nCol ){
27755 if( sqlite3Fts5IterEof(pCsr->pIter)((pCsr->pIter)->bEof) ){
27756 pCsr->bEof = 1;
27757 }else{
27758 const char *zTerm;
27759 int nTerm;
27760
27761 zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
27762 assert( nTerm>=0 )((void) (0));
27763 if( pCsr->nLeTerm>=0 ){
27764 int nCmp = MIN(nTerm, pCsr->nLeTerm)(((nTerm) < (pCsr->nLeTerm)) ? (nTerm) : (pCsr->nLeTerm
));
27765 int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp);
27766 if( bCmp<0 || (bCmp==0 && pCsr->nLeTerm<nTerm) ){
27767 pCsr->bEof = 1;
27768 return SQLITE_OK0;
27769 }
27770 }
27771
27772 sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
27773 memset(pCsr->aCnt, 0, nCol * sizeof(i64));
27774 memset(pCsr->aDoc, 0, nCol * sizeof(i64));
27775 pCsr->iCol = 0;
27776
27777 assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW )((void) (0));
27778 while( rc==SQLITE_OK0 ){
27779 int eDetail = pCsr->pFts5->pConfig->eDetail;
27780 const u8 *pPos; int nPos; /* Position list */
27781 i64 iPos = 0; /* 64-bit position read from poslist */
27782 int iOff = 0; /* Current offset within position list */
27783
27784 pPos = pCsr->pIter->pData;
27785 nPos = pCsr->pIter->nData;
27786
27787 switch( pTab->eType ){
27788 case FTS5_VOCAB_ROW1:
27789 /* Do not bother counting the number of instances if the "cnt"
27790 ** column is not being read (according to colUsed). */
27791 if( eDetail==FTS5_DETAIL_FULL0 && (pCsr->colUsed & 0x04) ){
27792 while( iPos<nPos ){
27793 u32 ii;
27794 fts5FastGetVarint32(pPos, iPos, ii){ ii = (pPos)[iPos++]; if( ii & 0x80 ){ iPos--; iPos += sqlite3Fts5GetVarint32
(&(pPos)[iPos],(u32*)&(ii)); } };
27795 if( ii==1 ){
27796 /* New column in the position list */
27797 fts5FastGetVarint32(pPos, iPos, ii){ ii = (pPos)[iPos++]; if( ii & 0x80 ){ iPos--; iPos += sqlite3Fts5GetVarint32
(&(pPos)[iPos],(u32*)&(ii)); } };
27798 }else{
27799 /* An instance - increment pCsr->aCnt[] */
27800 pCsr->aCnt[0]++;
27801 }
27802 }
27803 }
27804 pCsr->aDoc[0]++;
27805 break;
27806
27807 case FTS5_VOCAB_COL0:
27808 if( eDetail==FTS5_DETAIL_FULL0 ){
27809 int iCol = -1;
27810 while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
27811 int ii = FTS5_POS2COLUMN(iPos)(int)((iPos >> 32) & 0x7FFFFFFF);
27812 if( iCol!=ii ){
27813 if( ii>=nCol ){
27814 rc = FTS5_CORRUPT(11 | (1<<8));
27815 break;
27816 }
27817 pCsr->aDoc[ii]++;
27818 iCol = ii;
27819 }
27820 pCsr->aCnt[ii]++;
27821 }
27822 }else if( eDetail==FTS5_DETAIL_COLUMNS2 ){
27823 while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
27824 assert_nc( iPos>=0 && iPos<nCol )((void) (0));
27825 if( iPos>=nCol ){
27826 rc = FTS5_CORRUPT(11 | (1<<8));
27827 break;
27828 }
27829 pCsr->aDoc[iPos]++;
27830 }
27831 }else{
27832 assert( eDetail==FTS5_DETAIL_NONE )((void) (0));
27833 pCsr->aDoc[0]++;
27834 }
27835 break;
27836
27837 default:
27838 assert( pTab->eType==FTS5_VOCAB_INSTANCE )((void) (0));
27839 break;
27840 }
27841
27842 if( rc==SQLITE_OK0 ){
27843 rc = sqlite3Fts5IterNextScan(pCsr->pIter);
27844 }
27845 if( pTab->eType==FTS5_VOCAB_INSTANCE2 ) break;
27846
27847 if( rc==SQLITE_OK0 ){
27848 zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm);
27849 if( nTerm!=pCsr->term.n
27850 || (nTerm>0 && memcmp(zTerm, pCsr->term.p, nTerm))
27851 ){
27852 break;
27853 }
27854 if( sqlite3Fts5IterEof(pCsr->pIter)((pCsr->pIter)->bEof) ) break;
27855 }
27856 }
27857 }
27858 }
27859
27860 if( rc==SQLITE_OK0 && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL0 ){
27861 for(/* noop */; pCsr->iCol<nCol && pCsr->aDoc[pCsr->iCol]==0; pCsr->iCol++);
27862 if( pCsr->iCol==nCol ){
27863 rc = FTS5_CORRUPT(11 | (1<<8));
27864 }
27865 }
27866 return rc;
27867}
27868
27869/*
27870** This is the xFilter implementation for the virtual table.
27871*/
27872static int fts5VocabFilterMethod(
27873 sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
27874 int idxNum, /* Strategy index */
27875 const char *zUnused, /* Unused */
27876 int nUnused, /* Number of elements in apVal */
27877 sqlite3_value **apVal /* Arguments for the indexing scheme */
27878){
27879 Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab;
27880 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
27881 int eType = pTab->eType;
27882 int rc = SQLITE_OK0;
27883
27884 int iVal = 0;
27885 int f = FTS5INDEX_QUERY_SCAN0x0008;
27886 const char *zTerm = 0;
27887 int nTerm = 0;
27888
27889 sqlite3_value *pEq = 0;
27890 sqlite3_value *pGe = 0;
27891 sqlite3_value *pLe = 0;
27892
27893 UNUSED_PARAM2(zUnused, nUnused)(void)(zUnused), (void)(nUnused);
27894
27895 fts5VocabResetCursor(pCsr);
1
Calling 'fts5VocabResetCursor'
→
3
←
Returning from 'fts5VocabResetCursor'
→
27896 if( idxNum & FTS5_VOCAB_TERM_EQ0x0100 ) pEq = apVal[iVal++];
4
←
Assuming the condition is false
→
5
←
Taking false branch
→
27897 if( idxNum & FTS5_VOCAB_TERM_GE0x0200 ) pGe = apVal[iVal++];
6
←
Assuming the condition is true
→
7
←
Taking true branch
→
27898 if( idxNum & FTS5_VOCAB_TERM_LE0x0400 ) pLe = apVal[iVal++];
8
←
Assuming the condition is true
→
9
←
Taking true branch
→
27899 pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK0xFF);
27900
27901 if( pEq
9.1
'pEq' is null
 ){
10
←
Taking false branch
→
27902 zTerm = (const char *)sqlite3_value_textsqlite3_api->value_text(pEq);
27903 nTerm = sqlite3_value_bytessqlite3_api->value_bytes(pEq);
27904 f = FTS5INDEX_QUERY_NOTOKENDATA0x0080;
27905 }else{
27906 if( pGe ){
11
←
Assuming 'pGe' is null
→
12
←
Taking false branch
→
27907 zTerm = (const char *)sqlite3_value_textsqlite3_api->value_text(pGe);
27908 nTerm = sqlite3_value_bytessqlite3_api->value_bytes(pGe);
27909 }
27910 if( pLe ){
13
←
Assuming 'pLe' is null
→
14
←
Taking false branch
→
27911 const char *zCopy = (const char *)sqlite3_value_textsqlite3_api->value_text(pLe);
27912 if( zCopy==0 ) zCopy = "";
27913 pCsr->nLeTerm = sqlite3_value_bytessqlite3_api->value_bytes(pLe);
27914 pCsr->zLeTerm = sqlite3_malloc64sqlite3_api->malloc64((i64)pCsr->nLeTerm+1);
27915 if( pCsr->zLeTerm==0 ){
27916 rc = SQLITE_NOMEM7;
27917 }else{
27918 memcpy(pCsr->zLeTerm, zCopy, pCsr->nLeTerm+1);
27919 }
27920 }
27921 }
27922
27923 if( rc
14.1
'rc' is equal to SQLITE_OK
==SQLITE_OK0 ){
15
←
Taking true branch
→
27924 Fts5Index *pIndex = pCsr->pFts5->pIndex;
27925 rc = sqlite3Fts5IndexQuery(pIndex, zTerm, nTerm, f, 0, &pCsr->pIter);
16
←
Calling 'sqlite3Fts5IndexQuery'
→
20
←
Returning from 'sqlite3Fts5IndexQuery'
→
27926 if( rc==SQLITE_OK0 ){
21
←
Assuming 'rc' is equal to SQLITE_OK
→
22
←
Taking true branch
→
27927 pCsr->pStruct = sqlite3Fts5StructureRef(pIndex);
27928 }
27929 }
27930 if( rc
22.1
'rc' is equal to SQLITE_OK
==SQLITE_OK0 && eType==FTS5_VOCAB_INSTANCE2 ){
23
←
Assuming 'eType' is equal to FTS5_VOCAB_INSTANCE
→
24
←
Taking true branch
→
27931 rc = fts5VocabInstanceNewTerm(pCsr);
25
←
Calling 'fts5VocabInstanceNewTerm'
→
27932 }
27933 if( rc==SQLITE_OK0 && !pCsr->bEof
27934 && (eType!=FTS5_VOCAB_INSTANCE2
27935 || pCsr->pFts5->pConfig->eDetail!=FTS5_DETAIL_NONE1)
27936 ){
27937 rc = fts5VocabNextMethod(pCursor);
27938 }
27939
27940 return rc;
27941}
27942
27943/*
27944** This is the xEof method of the virtual table. SQLite calls this
27945** routine to find out if it has reached the end of a result set.
27946*/
27947static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){
27948 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
27949 return pCsr->bEof;
27950}
27951
27952static int fts5VocabColumnMethod(
27953 sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
27954 sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
27955 int iCol /* Index of column to read value from */
27956){
27957 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
27958 int eDetail = pCsr->pFts5->pConfig->eDetail;
27959 int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType;
27960 i64 iVal = 0;
27961
27962 if( iCol==0 ){
27963 sqlite3_result_textsqlite3_api->result_text(
27964 pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT((sqlite3_destructor_type)-1)
27965 );
27966 }else if( eType==FTS5_VOCAB_COL0 ){
27967 assert( iCol==1 || iCol==2 || iCol==3 )((void) (0));
27968 if( iCol==1 ){
27969 if( eDetail!=FTS5_DETAIL_NONE1 ){
27970 const char *z = pCsr->pFts5->pConfig->azCol[pCsr->iCol];
27971 sqlite3_result_textsqlite3_api->result_text(pCtx, z, -1, SQLITE_STATIC((sqlite3_destructor_type)0));
27972 }
27973 }else if( iCol==2 ){
27974 iVal = pCsr->aDoc[pCsr->iCol];
27975 }else{
27976 iVal = pCsr->aCnt[pCsr->iCol];
27977 }
27978 }else if( eType==FTS5_VOCAB_ROW1 ){
27979 assert( iCol==1 || iCol==2 )((void) (0));
27980 if( iCol==1 ){
27981 iVal = pCsr->aDoc[0];
27982 }else{
27983 iVal = pCsr->aCnt[0];
27984 }
27985 }else{
27986 assert( eType==FTS5_VOCAB_INSTANCE )((void) (0));
27987 switch( iCol ){
27988 case 1:
27989 sqlite3_result_int64sqlite3_api->result_int64(pCtx, pCsr->pIter->iRowid);
27990 break;
27991 case 2: {
27992 int ii = -1;
27993 if( eDetail==FTS5_DETAIL_FULL0 ){
27994 ii = FTS5_POS2COLUMN(pCsr->iInstPos)(int)((pCsr->iInstPos >> 32) & 0x7FFFFFFF);
27995 }else if( eDetail==FTS5_DETAIL_COLUMNS2 ){
27996 ii = (int)pCsr->iInstPos;
27997 }
27998 if( ii>=0 && ii<pCsr->pFts5->pConfig->nCol ){
27999 const char *z = pCsr->pFts5->pConfig->azCol[ii];
28000 sqlite3_result_textsqlite3_api->result_text(pCtx, z, -1, SQLITE_STATIC((sqlite3_destructor_type)0));
28001 }
28002 break;
28003 }
28004 default: {
28005 assert( iCol==3 )((void) (0));
28006 if( eDetail==FTS5_DETAIL_FULL0 ){
28007 int ii = FTS5_POS2OFFSET(pCsr->iInstPos)(int)(pCsr->iInstPos & 0x7FFFFFFF);
28008 sqlite3_result_intsqlite3_api->result_int(pCtx, ii);
28009 }
28010 break;
28011 }
28012 }
28013 }
28014
28015 if( iVal>0 ) sqlite3_result_int64sqlite3_api->result_int64(pCtx, iVal);
28016 return SQLITE_OK0;
28017}
28018
28019/*
28020** This is the xRowid method. The SQLite core calls this routine to
28021** retrieve the rowid for the current row of the result set. The
28022** rowid should be written to *pRowid.
28023*/
28024static int fts5VocabRowidMethod(
28025 sqlite3_vtab_cursor *pCursor,
28026 sqlite_int64 *pRowid
28027){
28028 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
28029 *pRowid = pCsr->rowid;
28030 return SQLITE_OK0;
28031}
28032
28033static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
28034 static const sqlite3_module fts5Vocab = {
28035 /* iVersion */ 2,
28036 /* xCreate */ fts5VocabCreateMethod,
28037 /* xConnect */ fts5VocabConnectMethod,
28038 /* xBestIndex */ fts5VocabBestIndexMethod,
28039 /* xDisconnect */ fts5VocabDisconnectMethod,
28040 /* xDestroy */ fts5VocabDestroyMethod,
28041 /* xOpen */ fts5VocabOpenMethod,
28042 /* xClose */ fts5VocabCloseMethod,
28043 /* xFilter */ fts5VocabFilterMethod,
28044 /* xNext */ fts5VocabNextMethod,
28045 /* xEof */ fts5VocabEofMethod,
28046 /* xColumn */ fts5VocabColumnMethod,
28047 /* xRowid */ fts5VocabRowidMethod,
28048 /* xUpdate */ 0,
28049 /* xBegin */ 0,
28050 /* xSync */ 0,
28051 /* xCommit */ 0,
28052 /* xRollback */ 0,
28053 /* xFindFunction */ 0,
28054 /* xRename */ 0,
28055 /* xSavepoint */ 0,
28056 /* xRelease */ 0,
28057 /* xRollbackTo */ 0,
28058 /* xShadowName */ 0,
28059 /* xIntegrity */ 0
28060 };
28061 void *p = (void*)pGlobal;
28062
28063 return sqlite3_create_module_v2sqlite3_api->create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
28064}
28065
28066
28067/* Here ends the fts5.c composite file. */
28068#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */