Overview
| SHA1 Hash: | f3d4a2db535a12c11b4ab28eba94421ce1880d7c |
|---|---|
| Date: | 2009-11-23 13:21:26 |
| User: | drh |
| Comment: | Update the SQLite used internally to the first 3.6.21 release candidate. |
| Timelines: | ancestors | descendants | both | trunk |
| Other Links: | files | ZIP archive | manifest |
Tags And Properties
- branch=trunk inherited from [a28c83647d]
- sym-trunk inherited from [a28c83647d]
Changes
[hide diffs]Modified src/sqlite3.c from [56de1fb10a] to [bb900ede45].
@@ -1,8 +1,8 @@
/******************************************************************************
** This file is an amalgamation of many separate C source files from SQLite
-** version 3.6.20. By combining all the individual C code files into this
+** version 3.6.21. By combining all the individual C code files into this
** single large file, the entire code can be compiled as a one translation
** unit. This allows many compilers to do optimizations that would not be
** possible if the files were compiled separately. Performance improvements
** of 5% are more are commonly seen when SQLite is compiled as a single
** translation unit.
@@ -91,12 +91,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
**
** This file defines various limits of what SQLite can process.
-**
-** @(#) $Id: sqliteLimit.h,v 1.10 2009/01/10 16:15:09 danielk1977 Exp $
*/
/*
** The maximum length of a TEXT or BLOB in bytes. This also
** limits the size of a row in a table or index.
@@ -277,15 +275,11 @@
** A value of 1 means that a trigger program will not be able to itself
** fire any triggers. A value of 0 means that no trigger programs at all
** may be executed.
*/
#ifndef SQLITE_MAX_TRIGGER_DEPTH
-#if defined(SQLITE_SMALL_STACK)
-# define SQLITE_MAX_TRIGGER_DEPTH 10
-#else
-# define SQLITE_MAX_TRIGGER_DEPTH 1000
-#endif
+# define SQLITE_MAX_TRIGGER_DEPTH 1000
#endif
/************** End of sqliteLimit.h *****************************************/
/************** Continuing where we left off in sqliteInt.h ******************/
@@ -647,13 +641,13 @@
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
**
** Requirements: [H10011] [H10014]
*/
-#define SQLITE_VERSION "3.6.20"
-#define SQLITE_VERSION_NUMBER 3006020
-#define SQLITE_SOURCE_ID "2009-11-04 23:02:52 f1c09acaca3e205acf5b077c9b2d0fe35f035c1e"
+#define SQLITE_VERSION "3.6.21"
+#define SQLITE_VERSION_NUMBER 3006021
+#define SQLITE_SOURCE_ID "2009-11-23 13:17:27 39214aee6553db76309851e7aa74fcc02d4f59b7"
/*
** CAPI3REF: Run-Time Library Version Numbers {H10020} <S60100>
** KEYWORDS: sqlite3_version
**
@@ -6316,12 +6310,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the header file for the generic hash-table implemenation
** used in SQLite.
-**
-** $Id: hash.h,v 1.15 2009/05/02 13:29:38 drh Exp $
*/
#ifndef _SQLITE_HASH_H_
#define _SQLITE_HASH_H_
/* Forward declarations of structures. */
@@ -6920,12 +6912,10 @@
**
*************************************************************************
** This header file defines the interface that the sqlite B-Tree file
** subsystem. See comments in the source code for a detailed description
** of what each interface routine does.
-**
-** @(#) $Id: btree.h,v 1.120 2009/07/22 00:35:24 drh Exp $
*/
#ifndef _BTREE_H_
#define _BTREE_H_
/* TODO: This definition is just included so other modules compile. It
@@ -7058,10 +7048,11 @@
int wrFlag, /* 1 for writing. 0 for read-only */
struct KeyInfo*, /* First argument to compare function */
BtCursor *pCursor /* Space to write cursor structure */
);
SQLITE_PRIVATE int sqlite3BtreeCursorSize(void);
+SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*);
SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*);
SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked(
BtCursor*,
UnpackedRecord *pUnKey,
@@ -7169,12 +7160,10 @@
** Header file for the Virtual DataBase Engine (VDBE)
**
** This header defines the interface to the virtual database engine
** or VDBE. The VDBE implements an abstract machine that runs a
** simple program to access and modify the underlying database.
-**
-** $Id: vdbe.h,v 1.142 2009/07/24 17:58:53 danielk1977 Exp $
*/
#ifndef _SQLITE_VDBE_H_
#define _SQLITE_VDBE_H_
/*
@@ -7198,11 +7187,11 @@
** as an instance of the following structure:
*/
struct VdbeOp {
u8 opcode; /* What operation to perform */
signed char p4type; /* One of the P4_xxx constants for p4 */
- u8 opflags; /* Not currently used */
+ u8 opflags; /* Mask of the OPFLG_* flags in opcodes.h */
u8 p5; /* Fifth parameter is an unsigned character */
int p1; /* First operand */
int p2; /* Second parameter (often the jump destination) */
int p3; /* The third parameter */
union { /* fourth parameter */
@@ -7476,26 +7465,27 @@
#define OPFLG_JUMP 0x0001 /* jump: P2 holds jmp target */
#define OPFLG_OUT2_PRERELEASE 0x0002 /* out2-prerelease: */
#define OPFLG_IN1 0x0004 /* in1: P1 is an input */
#define OPFLG_IN2 0x0008 /* in2: P2 is an input */
#define OPFLG_IN3 0x0010 /* in3: P3 is an input */
-#define OPFLG_OUT3 0x0020 /* out3: P3 is an output */
+#define OPFLG_OUT2 0x0020 /* out2: P2 is an output */
+#define OPFLG_OUT3 0x0040 /* out3: P3 is an output */
#define OPFLG_INITIALIZER {\
-/* 0 */ 0x00, 0x01, 0x01, 0x04, 0x04, 0x10, 0x00, 0x02,\
-/* 8 */ 0x02, 0x02, 0x02, 0x02, 0x00, 0x00, 0x04, 0x04,\
-/* 16 */ 0x00, 0x00, 0x00, 0x04, 0x04, 0x05, 0x04, 0x00,\
+/* 0 */ 0x00, 0x01, 0x05, 0x04, 0x04, 0x10, 0x00, 0x02,\
+/* 8 */ 0x02, 0x02, 0x02, 0x02, 0x00, 0x00, 0x24, 0x24,\
+/* 16 */ 0x00, 0x00, 0x00, 0x24, 0x04, 0x05, 0x04, 0x00,\
/* 24 */ 0x00, 0x01, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02,\
/* 32 */ 0x00, 0x00, 0x00, 0x02, 0x10, 0x00, 0x00, 0x00,\
/* 40 */ 0x00, 0x00, 0x00, 0x11, 0x11, 0x11, 0x11, 0x08,\
/* 48 */ 0x11, 0x11, 0x11, 0x11, 0x02, 0x02, 0x00, 0x00,\
/* 56 */ 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0x01,\
-/* 64 */ 0x01, 0x01, 0x01, 0x08, 0x2c, 0x2c, 0x00, 0x02,\
-/* 72 */ 0x11, 0x05, 0x05, 0x15, 0x15, 0x15, 0x15, 0x15,\
-/* 80 */ 0x15, 0x11, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c,\
-/* 88 */ 0x2c, 0x2c, 0x2c, 0x2c, 0x02, 0x04, 0x02, 0x00,\
+/* 64 */ 0x01, 0x01, 0x01, 0x08, 0x4c, 0x4c, 0x00, 0x02,\
+/* 72 */ 0x01, 0x05, 0x05, 0x15, 0x15, 0x15, 0x15, 0x15,\
+/* 80 */ 0x15, 0x01, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c, 0x4c,\
+/* 88 */ 0x4c, 0x4c, 0x4c, 0x4c, 0x02, 0x24, 0x02, 0x00,\
/* 96 */ 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\
-/* 104 */ 0x08, 0x21, 0x15, 0x01, 0x02, 0x00, 0x01, 0x08,\
+/* 104 */ 0x0c, 0x45, 0x15, 0x01, 0x02, 0x00, 0x01, 0x08,\
/* 112 */ 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x01, 0x00,\
/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01,\
/* 128 */ 0x00, 0x00, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00,\
/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04,\
/* 144 */ 0x04, 0x04,}
@@ -7511,10 +7501,11 @@
SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int);
SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int);
SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int);
SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int);
SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int);
+SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int);
SQLITE_PRIVATE int sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp);
SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1);
SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2);
SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3);
SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
@@ -7579,12 +7570,10 @@
**
*************************************************************************
** This header file defines the interface that the sqlite page cache
** subsystem. The page cache subsystem reads and writes a file a page
** at a time and provides a journal for rollback.
-**
-** @(#) $Id: pager.h,v 1.104 2009/07/24 19:01:19 drh Exp $
*/
#ifndef _PAGER_H_
#define _PAGER_H_
@@ -7750,12 +7739,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This header file defines the interface that the sqlite page cache
** subsystem.
-**
-** @(#) $Id: pcache.h,v 1.20 2009/07/25 11:46:49 danielk1977 Exp $
*/
#ifndef _PCACHE_H_
typedef struct PgHdr PgHdr;
@@ -7918,12 +7905,10 @@
** "os.c") attempt to abstract the underlying operating system so that
** the SQLite library will work on both POSIX and windows systems.
**
** This header file is #include-ed by sqliteInt.h and thus ends up
** being included by every source file.
-**
-** $Id: os.h,v 1.108 2009/02/05 16:31:46 drh Exp $
*/
#ifndef _SQLITE_OS_H_
#define _SQLITE_OS_H_
/*
@@ -8198,12 +8183,10 @@
** better organized.
**
** NOTE: source files should *not* #include this header file directly.
** Source files should #include the sqliteInt.h file and let that file
** include this one indirectly.
-**
-** $Id: mutex.h,v 1.9 2008/10/07 15:25:48 drh Exp $
*/
/*
** Figure out what version of the code to use. The choices are
@@ -8576,10 +8559,11 @@
#define SQLITE_FUNC_CASE 0x02 /* Case-sensitive LIKE-type function */
#define SQLITE_FUNC_EPHEM 0x04 /* Ephemeral. Delete with VDBE */
#define SQLITE_FUNC_NEEDCOLL 0x08 /* sqlite3GetFuncCollSeq() might be called */
#define SQLITE_FUNC_PRIVATE 0x10 /* Allowed for internal use only */
#define SQLITE_FUNC_COUNT 0x20 /* Built-in count(*) aggregate */
+#define SQLITE_FUNC_COALESCE 0x40 /* Built-in coalesce() or ifnull() function */
/*
** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are
** used to create the initializers for the FuncDef structures.
**
@@ -10290,10 +10274,13 @@
SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *);
SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*);
SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*);
SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*);
SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr*, int*);
+SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*);
+SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump(Vdbe*, const Expr*, int, int);
+SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
SQLITE_PRIVATE int sqlite3IsRowid(const char*);
SQLITE_PRIVATE void sqlite3GenerateRowDelete(Parse*, Table*, int, int, int, Trigger *, int);
SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int*);
SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int);
SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int,int,
@@ -10461,10 +10448,11 @@
SQLITE_PRIVATE char *sqlite3Utf8to16(sqlite3 *, u8, char *, int, int *);
#endif
SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value **);
SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8);
#ifndef SQLITE_AMALGAMATION
+SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[];
SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[];
SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[];
SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config;
SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions;
SQLITE_PRIVATE int sqlite3PendingByte;
@@ -10702,11 +10690,10 @@
*************************************************************************
**
** This file contains definitions of global variables and contants.
*/
-
/* An array to map all upper-case characters into their corresponding
** lower-case character.
**
** SQLite only considers US-ASCII (or EBCDIC) characters. We do not
** handle case conversions for the UTF character set since the tables
@@ -10758,10 +10745,11 @@
** isalpha() 0x02
** isdigit() 0x04
** isalnum() 0x06
** isxdigit() 0x08
** toupper() 0x20
+** SQLite identifier character 0x40
**
** Bit 0x20 is set if the mapped character requires translation to upper
** case. i.e. if the character is a lower-case ASCII character.
** If x is a lower-case ASCII character, then its upper-case equivalent
** is (x - 0x20). Therefore toupper() can be implemented as:
@@ -10769,50 +10757,55 @@
** (x & ~(map[x]&0x20))
**
** Standard function tolower() is implemented using the sqlite3UpperToLower[]
** array. tolower() is used more often than toupper() by SQLite.
**
+** Bit 0x40 is set if the character non-alphanumeric and can be used in an
+** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any
+** non-ASCII UTF character. Hence the test for whether or not a character is
+** part of an identifier is 0x46.
+**
** SQLite's versions are identical to the standard versions assuming a
** locale of "C". They are implemented as macros in sqliteInt.h.
*/
#ifdef SQLITE_ASCII
SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */
0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */
- 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */
+ 0x01, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, /* 20..27 !"#$%&' */
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */
0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */
0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */
0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */
0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */
0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */
- 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, /* 58..5f XYZ[\]^_ */
+ 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */
0x00, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */
0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */
0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */
0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 80..87 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 88..8f ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 90..97 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 98..9f ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* a0..a7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* a8..af ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* b0..b7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* b8..bf ........ */
-
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* c0..c7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* c8..cf ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* d0..d7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* d8..df ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* e0..e7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* e8..ef ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* f0..f7 ........ */
- 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /* f8..ff ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */
+
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */
};
#endif
@@ -10877,10 +10870,18 @@
** Changing the pending byte during operating results in undefined
** and dileterious behavior.
*/
SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000;
+/*
+** Properties of opcodes. The OPFLG_INITIALIZER macro is
+** created by mkopcodeh.awk during compilation. Data is obtained
+** from the comments following the "case OP_xxxx:" statements in
+** the vdbe.c file.
+*/
+SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER;
+
/************** End of global.c **********************************************/
/************** Begin file status.c ******************************************/
/*
** 2008 June 18
**
@@ -10893,12 +10894,10 @@
**
*************************************************************************
**
** This module implements the sqlite3_status() interface and related
** functionality.
-**
-** $Id: status.c,v 1.9 2008/09/02 00:52:52 drh Exp $
*/
/*
** Variables in which to record status information.
*/
@@ -11020,12 +11019,10 @@
** functions for SQLite.
**
** There is only one exported symbol in this file - the function
** sqlite3RegisterDateTimeFunctions() found at the bottom of the file.
** All other code has file scope.
-**
-** $Id: date.c,v 1.107 2009/05/03 20:23:53 drh Exp $
**
** SQLite processes all times and dates as Julian Day numbers. The
** dates and times are stored as the number of days since noon
** in Greenwich on November 24, 4714 B.C. according to the Gregorian
** calendar system.
@@ -12125,12 +12122,10 @@
**
******************************************************************************
**
** This file contains OS interface code that is common to all
** architectures.
-**
-** $Id: os.c,v 1.127 2009/07/27 11:41:21 danielk1977 Exp $
*/
#define _SQLITE_OS_C_ 1
#undef _SQLITE_OS_C_
/*
@@ -12252,10 +12247,11 @@
sqlite3_vfs *pVfs,
const char *zPath,
int nPathOut,
char *zPathOut
){
+ zPathOut[0] = 0;
return pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut);
}
#ifndef SQLITE_OMIT_LOAD_EXTENSION
SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *pVfs, const char *zPath){
return pVfs->xDlOpen(pVfs, zPath);
@@ -12423,14 +12419,10 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
**
-** $Id: fault.c,v 1.11 2008/09/02 00:52:52 drh Exp $
-*/
-
-/*
** This file contains code to support the concept of "benign"
** malloc failures (when the xMalloc() or xRealloc() method of the
** sqlite3_mem_methods structure fails to allocate a block of memory
** and returns 0).
**
@@ -12521,12 +12513,10 @@
** This file contains a no-op memory allocation drivers for use when
** SQLITE_ZERO_MALLOC is defined. The allocation drivers implemented
** here always fail. SQLite will not operate with these drivers. These
** are merely placeholders. Real drivers must be substituted using
** sqlite3_config() before SQLite will operate.
-**
-** $Id: mem0.c,v 1.1 2008/10/28 18:58:20 drh Exp $
*/
/*
** This version of the memory allocator is the default. It is
** used when no other memory allocator is specified using compile-time
@@ -12585,12 +12575,10 @@
** SQLite will use the standard C-library malloc/realloc/free interface
** to obtain the memory it needs.
**
** This file contains implementations of the low-level memory allocation
** routines specified in the sqlite3_mem_methods object.
-**
-** $Id: mem1.c,v 1.30 2009/03/23 04:33:33 danielk1977 Exp $
*/
/*
** This version of the memory allocator is the default. It is
** used when no other memory allocator is specified using compile-time
@@ -12734,12 +12722,10 @@
** information to each allocation in order to help detect and fix memory
** leaks and memory usage errors.
**
** This file contains implementations of the low-level memory allocation
** routines specified in the sqlite3_mem_methods object.
-**
-** $Id: mem2.c,v 1.45 2009/03/23 04:33:33 danielk1977 Exp $
*/
/*
** This version of the memory allocator is used only if the
** SQLITE_MEMDEBUG macro is defined
@@ -13183,12 +13169,10 @@
** the amount of memory available to SQLite is fixed and cannot
** be changed.
**
** This version of the memory allocation subsystem is included
** in the build only if SQLITE_ENABLE_MEMSYS3 is defined.
-**
-** $Id: mem3.c,v 1.25 2008/11/19 16:52:44 danielk1977 Exp $
*/
/*
** This version of the memory allocator is only built into the library
** SQLITE_ENABLE_MEMSYS3 is defined. Defining this symbol does not
@@ -14442,13 +14426,10 @@
**
*************************************************************************
** This file contains the C functions that implement mutexes.
**
** This file contains code that is common across all mutex implementations.
-
-**
-** $Id: mutex.c,v 1.31 2009/07/16 18:21:18 drh Exp $
*/
#if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT)
/*
** For debugging purposes, record when the mutex subsystem is initialized
@@ -14609,12 +14590,10 @@
** interface.
**
** If compiled with SQLITE_DEBUG, then additional logic is inserted
** that does error checking on mutexes to make sure they are being
** called correctly.
-**
-** $Id: mutex_noop.c,v 1.3 2008/12/05 17:17:08 drh Exp $
*/
#if defined(SQLITE_MUTEX_NOOP) && !defined(SQLITE_DEBUG)
/*
@@ -14783,12 +14762,10 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains the C functions that implement mutexes for OS/2
-**
-** $Id: mutex_os2.c,v 1.11 2008/11/22 19:50:54 pweilbacher Exp $
*/
/*
** The code in this file is only used if SQLITE_MUTEX_OS2 is defined.
** See the mutex.h file for details.
@@ -15058,12 +15035,10 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains the C functions that implement mutexes for pthreads
-**
-** $Id: mutex_unix.c,v 1.16 2008/12/08 18:19:18 drh Exp $
*/
/*
** The code in this file is only used if we are compiling threadsafe
** under unix with pthreads.
@@ -15389,12 +15364,10 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains the C functions that implement mutexes for win32
-**
-** $Id: mutex_w32.c,v 1.18 2009/08/10 03:23:21 shane Exp $
*/
/*
** The code in this file is only used if we are compiling multithreaded
** on a win32 system.
@@ -15676,12 +15649,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
**
** Memory allocation functions used throughout sqlite.
-**
-** $Id: malloc.c,v 1.66 2009/07/17 11:44:07 drh Exp $
*/
/*
** This routine runs when the memory allocator sees that the
** total memory allocation is about to exceed the soft heap
@@ -16382,12 +16353,10 @@
** The "printf" code that follows dates from the 1980's. It is in
** the public domain. The original comments are included here for
** completeness. They are very out-of-date but might be useful as
** an historical reference. Most of the "enhancements" have been backed
** out so that the functionality is now the same as standard printf().
-**
-** $Id: printf.c,v 1.104 2009/06/03 01:24:54 drh Exp $
**
**************************************************************************
**
** The following modules is an enhanced replacement for the "printf" subroutines
** found in the standard C library. The following enhancements are
@@ -17352,12 +17321,10 @@
** This file contains code to implement a pseudo-random number
** generator (PRNG) for SQLite.
**
** Random numbers are used by some of the database backends in order
** to generate random integer keys for tables or random filenames.
-**
-** $Id: random.c,v 1.29 2008/12/10 19:26:24 drh Exp $
*/
/* All threads share a single random number generator.
** This structure is the current state of the generator.
@@ -17499,12 +17466,10 @@
**
*************************************************************************
** This file contains routines used to translate between UTF-8,
** UTF-16, UTF-16BE, and UTF-16LE.
**
-** $Id: utf.c,v 1.73 2009/04/01 18:40:32 drh Exp $
-**
** Notes on UTF-8:
**
** Byte-0 Byte-1 Byte-2 Byte-3 Value
** 0xxxxxxx 00000000 00000000 0xxxxxxx
** 110yyyyy 10xxxxxx 00000000 00000yyy yyxxxxxx
@@ -17540,12 +17505,10 @@
** This is the header file for information that is private to the
** VDBE. This information used to all be at the top of the single
** source code file "vdbe.c". When that file became too big (over
** 6000 lines long) it was split up into several smaller files and
** this header information was factored out.
-**
-** $Id: vdbeInt.h,v 1.174 2009/06/23 14:15:04 drh Exp $
*/
#ifndef _VDBEINT_H_
#define _VDBEINT_H_
/*
@@ -17907,11 +17870,10 @@
SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,int,int,int,Mem*);
SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p);
SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p);
SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*);
SQLITE_PRIVATE const char *sqlite3OpcodeName(int);
-SQLITE_PRIVATE int sqlite3VdbeOpcodeHasProperty(int, int);
SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve);
SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int);
SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*);
SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *);
SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem);
@@ -19574,12 +19536,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This is the implementation of generic hash-tables
** used in SQLite.
-**
-** $Id: hash.c,v 1.38 2009/05/09 23:29:12 drh Exp $
*/
/* Turn bulk memory into a hash table object by initializing the
** fields of the Hash structure.
**
@@ -20011,12 +19971,10 @@
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file contains code that is specific to OS/2.
-**
-** $Id: os_os2.c,v 1.63 2008/12/10 19:26:24 drh Exp $
*/
#if SQLITE_OS_OS2
@@ -20074,12 +20032,10 @@
** all of the platform-specific files (os_*.c) and is #included into those
** files.
**
** This file should be #included by the os_*.c files only. It is not a
** general purpose header file.
-**
-** $Id: os_common.h,v 1.38 2009/02/24 18:40:50 danielk1977 Exp $
*/
#ifndef _OS_COMMON_H_
#define _OS_COMMON_H_
/*
@@ -20136,12 +20092,10 @@
**
******************************************************************************
**
** This file contains inline asm code for retrieving "high-performance"
** counters for x86 class CPUs.
-**
-** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
*/
#ifndef _HWTIME_H_
#define _HWTIME_H_
/*
@@ -21603,12 +21557,10 @@
** all of the platform-specific files (os_*.c) and is #included into those
** files.
**
** This file should be #included by the os_*.c files only. It is not a
** general purpose header file.
-**
-** $Id: os_common.h,v 1.38 2009/02/24 18:40:50 danielk1977 Exp $
*/
#ifndef _OS_COMMON_H_
#define _OS_COMMON_H_
/*
@@ -21665,12 +21617,10 @@
**
******************************************************************************
**
** This file contains inline asm code for retrieving "high-performance"
** counters for x86 class CPUs.
-**
-** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
*/
#ifndef _HWTIME_H_
#define _HWTIME_H_
/*
@@ -24685,10 +24635,23 @@
rc = ftruncate(((unixFile*)id)->h, (off_t)nByte);
if( rc ){
((unixFile*)id)->lastErrno = errno;
return SQLITE_IOERR_TRUNCATE;
}else{
+#ifndef NDEBUG
+ /* If we are doing a normal write to a database file (as opposed to
+ ** doing a hot-journal rollback or a write to some file other than a
+ ** normal database file) and we truncate the file to zero length,
+ ** that effectively updates the change counter. This might happen
+ ** when restoring a database using the backup API from a zero-length
+ ** source.
+ */
+ if( ((unixFile*)id)->inNormalWrite && nByte==0 ){
+ ((unixFile*)id)->transCntrChng = 1;
+ }
+#endif
+
return SQLITE_OK;
}
}
/*
@@ -27042,12 +27005,10 @@
** all of the platform-specific files (os_*.c) and is #included into those
** files.
**
** This file should be #included by the os_*.c files only. It is not a
** general purpose header file.
-**
-** $Id: os_common.h,v 1.38 2009/02/24 18:40:50 danielk1977 Exp $
*/
#ifndef _OS_COMMON_H_
#define _OS_COMMON_H_
/*
@@ -27104,12 +27065,10 @@
**
******************************************************************************
**
** This file contains inline asm code for retrieving "high-performance"
** counters for x86 class CPUs.
-**
-** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
*/
#ifndef _HWTIME_H_
#define _HWTIME_H_
/*
@@ -29147,12 +29106,10 @@
** 5 and 500 set operations per Bitvec object, though the number of sets can
** sometimes grow into tens of thousands or larger. The size of the
** Bitvec object is the number of pages in the database file at the
** start of a transaction, and is thus usually less than a few thousand,
** but can be as large as 2 billion for a really big database.
-**
-** @(#) $Id: bitvec.c,v 1.17 2009/07/25 17:33:26 drh Exp $
*/
/* Size of the Bitvec structure in bytes. */
#define BITVEC_SZ (sizeof(void*)*128) /* 512 on 32bit. 1024 on 64bit */
@@ -29536,12 +29493,10 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file implements that page cache.
-**
-** @(#) $Id: pcache.c,v 1.47 2009/07/25 11:46:49 danielk1977 Exp $
*/
/*
** A complete page cache is an instance of this structure.
*/
@@ -30119,12 +30074,10 @@
** This file implements the default page cache implementation (the
** sqlite3_pcache interface). It also contains part of the implementation
** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features.
** If the default page cache implementation is overriden, then neither of
** these two features are available.
-**
-** @(#) $Id: pcache1.c,v 1.19 2009/07/17 11:44:07 drh Exp $
*/
typedef struct PCache1 PCache1;
typedef struct PgHdr1 PgHdr1;
@@ -30770,19 +30723,11 @@
h = iNew%pCache->nHash;
pPage->iKey = iNew;
pPage->pNext = pCache->apHash[h];
pCache->apHash[h] = pPage;
-
- /* The xRekey() interface is only used to move pages earlier in the
- ** database file (in order to move all free pages to the end of the
- ** file where they can be truncated off.) Hence, it is not possible
- ** for the new page number to be greater than the largest previously
- ** fetched page. But we retain the following test in case xRekey()
- ** begins to be used in different ways in the future.
- */
- if( NEVER(iNew>pCache->iMaxKey) ){
+ if( iNew>pCache->iMaxKey ){
pCache->iMaxKey = iNew;
}
pcache1LeaveMutex();
}
@@ -30954,12 +30899,10 @@
** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST
** primitives are constant time. The cost of DESTROY is O(N).
**
** There is an added cost of O(N) when switching between TEST and
** SMALLEST primitives.
-**
-** $Id: rowset.c,v 1.7 2009/05/22 01:00:13 drh Exp $
*/
/*
** Target size for allocation chunks.
@@ -31338,12 +31281,10 @@
** atomic commit and rollback through the use of a journal file that
** is separate from the database file. The pager also implements file
** locking to prevent two processes from writing the same database
** file simultaneously, or one process from reading the database while
** another is writing.
-**
-** @(#) $Id: pager.c,v 1.629 2009/08/10 17:48:57 drh Exp $
*/
#ifndef SQLITE_OMIT_DISKIO
/*
** Macros for troubleshooting. Normally turned off
@@ -32233,14 +32174,14 @@
return rc;
}
/* Check that the values read from the page-size and sector-size fields
** are within range. To be 'in range', both values need to be a power
- ** of two greater than or equal to 512, and not greater than their
+ ** of two greater than or equal to 512 or 32, and not greater than their
** respective compile time maximum limits.
*/
- if( iPageSize<512 || iSectorSize<512
+ if( iPageSize<512 || iSectorSize<32
|| iPageSize>SQLITE_MAX_PAGE_SIZE || iSectorSize>MAX_SECTOR_SIZE
|| ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0
){
/* If the either the page-size or sector-size in the journal-header is
** invalid, then the process that wrote the journal-header must have
@@ -32743,28 +32684,28 @@
){
int rc;
PgHdr *pPg; /* An existing page in the cache */
Pgno pgno; /* The page number of a page in journal */
u32 cksum; /* Checksum used for sanity checking */
- u8 *aData; /* Temporary storage for the page */
+ char *aData; /* Temporary storage for the page */
sqlite3_file *jfd; /* The file descriptor for the journal file */
assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */
assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */
assert( isMainJrnl || pDone ); /* pDone always used on sub-journals */
assert( isSavepnt || pDone==0 ); /* pDone never used on non-savepoint */
- aData = (u8*)pPager->pTmpSpace;
+ aData = pPager->pTmpSpace;
assert( aData ); /* Temp storage must have already been allocated */
/* Read the page number and page data from the journal or sub-journal
** file. Return an error code to the caller if an IO error occurs.
*/
jfd = isMainJrnl ? pPager->jfd : pPager->sjfd;
rc = read32bits(jfd, *pOffset, &pgno);
if( rc!=SQLITE_OK ) return rc;
- rc = sqlite3OsRead(jfd, aData, pPager->pageSize, (*pOffset)+4);
+ rc = sqlite3OsRead(jfd, (u8*)aData, pPager->pageSize, (*pOffset)+4);
if( rc!=SQLITE_OK ) return rc;
*pOffset += pPager->pageSize + 4 + isMainJrnl*4;
/* Sanity checking on the page. This is more important that I originally
** thought. If a power failure occurs while the journal is being written,
@@ -32779,11 +32720,11 @@
return SQLITE_OK;
}
if( isMainJrnl ){
rc = read32bits(jfd, (*pOffset)-4, &cksum);
if( rc ) return rc;
- if( !isSavepnt && pager_cksum(pPager, aData)!=cksum ){
+ if( !isSavepnt && pager_cksum(pPager, (u8*)aData)!=cksum ){
return SQLITE_DONE;
}
}
if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){
@@ -32825,27 +32766,27 @@
** Do not attempt to write if database file has never been opened.
*/
pPg = pager_lookup(pPager, pgno);
assert( pPg || !MEMDB );
PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n",
- PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, aData),
- (isMainJrnl?"main-journal":"sub-journal")
+ PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData),
+ (isMainJrnl?"main-journal":"sub-journal")
));
if( (pPager->state>=PAGER_EXCLUSIVE)
&& (pPg==0 || 0==(pPg->flags&PGHDR_NEED_SYNC))
&& isOpen(pPager->fd)
&& !isUnsync
){
i64 ofst = (pgno-1)*(i64)pPager->pageSize;
- rc = sqlite3OsWrite(pPager->fd, aData, pPager->pageSize, ofst);
+ rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst);
if( pgno>pPager->dbFileSize ){
pPager->dbFileSize = pgno;
}
if( pPager->pBackup ){
CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM);
- sqlite3BackupUpdate(pPager->pBackup, pgno, aData);
- CODEC1(pPager, aData, pgno, 0, rc=SQLITE_NOMEM);
+ sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData);
+ CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM, aData);
}
}else if( !isMainJrnl && pPg==0 ){
/* If this is a rollback of a savepoint and data was not written to
** the database and the page is not in-memory, there is a potential
** problem. When the page is next fetched by the b-tree layer, it
@@ -32876,11 +32817,11 @@
** of an internal error resulting in an automatic call to
** sqlite3PagerRollback().
*/
void *pData;
pData = pPg->pData;
- memcpy(pData, aData, pPager->pageSize);
+ memcpy(pData, (u8*)aData, pPager->pageSize);
pPager->xReiniter(pPg);
if( isMainJrnl && (!isSavepnt || *pOffset<=pPager->journalHdr) ){
/* If the contents of this page were just restored from the main
** journal file, then its content must be as they were when the
** transaction was first opened. In this case we can mark the page
@@ -33101,12 +33042,12 @@
** master journal pointers within created journal files.
**
** For temporary files the effective sector size is always 512 bytes.
**
** Otherwise, for non-temporary files, the effective sector size is
-** the value returned by the xSectorSize() method rounded up to 512 if
-** it is less than 512, or rounded down to MAX_SECTOR_SIZE if it
+** the value returned by the xSectorSize() method rounded up to 32 if
+** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it
** is greater than MAX_SECTOR_SIZE.
*/
static void setSectorSize(Pager *pPager){
assert( isOpen(pPager->fd) || pPager->tempFile );
@@ -33115,11 +33056,11 @@
** may not have been opened yet, in which case the OsSectorSize()
** call will segfault.
*/
pPager->sectorSize = sqlite3OsSectorSize(pPager->fd);
}
- if( pPager->sectorSize<512 ){
+ if( pPager->sectorSize<32 ){
pPager->sectorSize = 512;
}
if( pPager->sectorSize>MAX_SECTOR_SIZE ){
assert( MAX_SECTOR_SIZE>=512 );
pPager->sectorSize = MAX_SECTOR_SIZE;
@@ -34193,11 +34134,13 @@
** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to
** make the file smaller (presumably by auto-vacuum code). Do not write
** any such pages to the file.
**
** Also, do not write out any page that has the PGHDR_DONT_WRITE flag
- ** set (set by sqlite3PagerDontWrite()).
+ ** set (set by sqlite3PagerDontWrite()). Note that if compiled with
+ ** SQLITE_SECURE_DELETE the PGHDR_DONT_WRITE bit is never set and so
+ ** the second test is always true.
*/
if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){
i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */
char *pData; /* Data to write */
@@ -35702,10 +35645,11 @@
SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){
return pPg->flags&PGHDR_DIRTY;
}
#endif
+#ifndef SQLITE_SECURE_DELETE
/*
** A call to this routine tells the pager that it is not necessary to
** write the information on page pPg back to the disk, even though
** that page might be marked as dirty. This happens, for example, when
** the page has been added as a leaf of the freelist and so its
@@ -35727,10 +35671,11 @@
#ifdef SQLITE_CHECK_PAGES
pPg->pageHash = pager_pagehash(pPg);
#endif
}
}
+#endif /* !defined(SQLITE_SECURE_DELETE) */
/*
** This routine is called to increment the value of the database file
** change-counter, stored as a 4-byte big-endian integer starting at
** byte offset 24 of the pager file.
@@ -35766,11 +35711,11 @@
#else
# define DIRECT_MODE isDirectMode
#endif
assert( pPager->state>=PAGER_RESERVED );
- if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){
+ if( !pPager->changeCountDone && pPager->dbSize>0 ){
PgHdr *pPgHdr; /* Reference to page 1 */
u32 change_counter; /* Initial value of change-counter field */
assert( !pPager->tempFile && isOpen(pPager->fd) );
@@ -36419,10 +36364,18 @@
int rc; /* Return code */
Pgno origPgno; /* The original page number */
assert( pPg->nRef>0 );
+ /* In order to be able to rollback, an in-memory database must journal
+ ** the page we are moving from.
+ */
+ if( MEMDB ){
+ rc = sqlite3PagerWrite(pPg);
+ if( rc ) return rc;
+ }
+
/* If the page being moved is dirty and has not been saved by the latest
** savepoint, then save the current contents of the page into the
** sub-journal now. This is required to handle the following scenario:
**
** BEGIN;
@@ -36472,11 +36425,18 @@
pPg->flags &= ~PGHDR_NEED_SYNC;
pPgOld = pager_lookup(pPager, pgno);
assert( !pPgOld || pPgOld->nRef==1 );
if( pPgOld ){
pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC);
- sqlite3PcacheDrop(pPgOld);
+ if( MEMDB ){
+ /* Do not discard pages from an in-memory database since we might
+ ** need to rollback later. Just move the page out of the way. */
+ assert( pPager->dbSizeValid );
+ sqlite3PcacheMove(pPgOld, pPager->dbSize+1);
+ }else{
+ sqlite3PcacheDrop(pPgOld);
+ }
}
origPgno = pPg->pgno;
sqlite3PcacheMove(pPg, pgno);
sqlite3PcacheMakeDirty(pPg);
@@ -36517,22 +36477,16 @@
sqlite3PagerUnref(pPgHdr);
}
/*
** For an in-memory database, make sure the original page continues
- ** to exist, in case the transaction needs to roll back. We allocate
- ** the page now, instead of at rollback, because we can better deal
- ** with an out-of-memory error now. Ticket #3761.
+ ** to exist, in case the transaction needs to roll back. Use pPgOld
+ ** as the original page since it has already been allocated.
*/
if( MEMDB ){
- DbPage *pNew;
- rc = sqlite3PagerAcquire(pPager, origPgno, &pNew, 1);
- if( rc!=SQLITE_OK ){
- sqlite3PcacheMove(pPg, origPgno);
- return rc;
- }
- sqlite3PagerUnref(pNew);
+ sqlite3PcacheMove(pPgOld, origPgno);
+ sqlite3PagerUnref(pPgOld);
}
return SQLITE_OK;
}
#endif
@@ -36655,12 +36609,10 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
**
-** $Id: btmutex.c,v 1.17 2009/07/20 12:33:33 drh Exp $
-**
** This file contains code used to implement mutexes on Btree objects.
** This code really belongs in btree.c. But btree.c is getting too
** big and we want to break it down some. This packaged seemed like
** a good breakout.
*/
@@ -36675,12 +36627,10 @@
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
-** $Id: btreeInt.h,v 1.52 2009/07/15 17:25:46 drh Exp $
-**
** This file implements a external (disk-based) database using BTrees.
** For a detailed discussion of BTrees, refer to
**
** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3:
** "Sorting And Searching", pages 473-480. Addison-Wesley
@@ -37660,12 +37610,10 @@
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
-** $Id: btree.c,v 1.705 2009/08/10 03:57:58 shane Exp $
-**
** This file implements a external (disk-based) database using BTrees.
** See the header comment on "btreeInt.h" for additional information.
** Including a description of file format and an overview of operation.
*/
@@ -40930,12 +40878,12 @@
**
** No checking is done to make sure that page iTable really is the
** root page of a b-tree. If it is not, then the cursor acquired
** will not work correctly.
**
-** It is assumed that the sqlite3BtreeCursorSize() bytes of memory
-** pointed to by pCur have been zeroed by the caller.
+** It is assumed that the sqlite3BtreeCursorZero() has been called
+** on pCur to initialize the memory space prior to invoking this routine.
*/
static int btreeCursor(
Btree *p, /* The btree */
int iTable, /* Root page of table to open */
int wrFlag, /* 1 to write. 0 read-only */
@@ -41004,11 +40952,23 @@
** sufficient storage to hold a cursor. The BtCursor object is opaque
** to users so they cannot do the sizeof() themselves - they must call
** this routine.
*/
SQLITE_PRIVATE int sqlite3BtreeCursorSize(void){
- return sizeof(BtCursor);
+ return ROUND8(sizeof(BtCursor));
+}
+
+/*
+** Initialize memory that will be converted into a BtCursor object.
+**
+** The simple approach here would be to memset() the entire object
+** to zero. But it turns out that the apPage[] and aiIdx[] arrays
+** do not need to be zeroed and they are large, so we can save a lot
+** of run-time by skipping the initialization of those elements.
+*/
+SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor *p){
+ memset(p, 0, offsetof(BtCursor, iPage));
}
/*
** Set the cached rowid value of every cursor in the same database file
** as pCur and having the same root page number as pCur. The value is
@@ -45489,12 +45449,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains the implementation of the sqlite3_backup_XXX()
** API functions and the related features.
-**
-** $Id: backup.c,v 1.19 2009/07/06 19:03:13 drh Exp $
*/
/* Macro to find the minimum of two numeric values.
*/
#ifndef MIN
@@ -46123,12 +46081,10 @@
**
** This file contains code use to manipulate "Mem" structure. A "Mem"
** stores a single value in the VDBE. Mem is an opaque structure visible
** only within the VDBE. Interface routines refer to a Mem using the
** name sqlite_value
-**
-** $Id: vdbemem.c,v 1.152 2009/07/22 18:07:41 drh Exp $
*/
/*
** Call sqlite3VdbeMemExpandBlob() on the supplied value (type Mem*)
** P if required.
@@ -46863,13 +46819,10 @@
SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){
int rc;
int f1, f2;
int combined_flags;
- /* Interchange pMem1 and pMem2 if the collating sequence specifies
- ** DESC order.
- */
f1 = pMem1->flags;
f2 = pMem2->flags;
combined_flags = f1|f2;
assert( (combined_flags & MEM_RowSet)==0 );
@@ -47116,11 +47069,11 @@
*ppVal = 0;
return SQLITE_OK;
}
op = pExpr->op;
if( op==TK_REGISTER ){
- op = pExpr->op2;
+ op = pExpr->op2; /* This only happens with SQLITE_ENABLE_STAT2 */
}
if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){
pVal = sqlite3ValueNew(db);
if( pVal==0 ) goto no_mem;
@@ -47229,12 +47182,10 @@
*************************************************************************
** This file contains code used for creating, destroying, and populating
** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior
** to version 2.8.7, all this code was combined into the vdbe.c source file.
** But that file was getting too big so this subroutines were split out.
-**
-** $Id: vdbeaux.c,v 1.480 2009/08/08 18:01:08 drh Exp $
*/
/*
@@ -47412,10 +47363,26 @@
sqlite3VdbeChangeP4(p, addr, zP4, p4type);
return addr;
}
/*
+** Add an opcode that includes the p4 value as an integer.
+*/
+SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(
+ Vdbe *p, /* Add the opcode to this VM */
+ int op, /* The new opcode */
+ int p1, /* The P1 operand */
+ int p2, /* The P2 operand */
+ int p3, /* The P3 operand */
+ int p4 /* The P4 operand as an integer */
+){
+ int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3);
+ sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32);
+ return addr;
+}
+
+/*
** Create a new symbolic label for an instruction that has yet to be
** coded. The symbolic label is really just a negative number. The
** label can be used as the P2 value of an operation. Later, when
** the label is resolved to a specific address, the VDBE will scan
** through its operation list and change all values of P2 which match
@@ -47586,10 +47553,12 @@
** This routine is called once after all opcodes have been inserted.
**
** Variable *pMaxFuncArgs is set to the maximum value of any P2 argument
** to an OP_Function, OP_AggStep or OP_VFilter opcode. This is used by
** sqlite3VdbeMakeReady() to size the Vdbe.apArg[] array.
+**
+** The Op.opflags field is set on all opcodes.
*/
static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
int i;
int nMaxArgs = *pMaxFuncArgs;
Op *pOp;
@@ -47596,29 +47565,28 @@
int *aLabel = p->aLabel;
p->readOnly = 1;
for(pOp=p->aOp, i=p->nOp-1; i>=0; i--, pOp++){
u8 opcode = pOp->opcode;
+ pOp->opflags = sqlite3OpcodeProperty[opcode];
if( opcode==OP_Function || opcode==OP_AggStep ){
if( pOp->p5>nMaxArgs ) nMaxArgs = pOp->p5;
+ }else if( opcode==OP_Transaction && pOp->p2!=0 ){
+ p->readOnly = 0;
#ifndef SQLITE_OMIT_VIRTUALTABLE
}else if( opcode==OP_VUpdate ){
if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2;
-#endif
- }else if( opcode==OP_Transaction && pOp->p2!=0 ){
- p->readOnly = 0;
-#ifndef SQLITE_OMIT_VIRTUALTABLE
}else if( opcode==OP_VFilter ){
int n;
assert( p->nOp - i >= 3 );
assert( pOp[-1].opcode==OP_Integer );
n = pOp[-1].p1;
if( n>nMaxArgs ) nMaxArgs = n;
#endif
}
- if( sqlite3VdbeOpcodeHasProperty(opcode, OPFLG_JUMP) && pOp->p2<0 ){
+ if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){
assert( -1-pOp->p2<p->nLabel );
pOp->p2 = aLabel[-1-pOp->p2];
}
}
sqlite3DbFree(p->db, p->aLabel);
@@ -47676,11 +47644,11 @@
for(i=0; i<nOp; i++, pIn++){
int p2 = pIn->p2;
VdbeOp *pOut = &p->aOp[i+addr];
pOut->opcode = pIn->opcode;
pOut->p1 = pIn->p1;
- if( p2<0 && sqlite3VdbeOpcodeHasProperty(pOut->opcode, OPFLG_JUMP) ){
+ if( p2<0 && (sqlite3OpcodeProperty[pOut->opcode] & OPFLG_JUMP)!=0 ){
pOut->p2 = addr + ADDR(p2);
}else{
pOut->p2 = p2;
}
pOut->p3 = pIn->p3;
@@ -49986,13 +49954,21 @@
Mem mem1;
pKeyInfo = pPKey2->pKeyInfo;
mem1.enc = pKeyInfo->enc;
mem1.db = pKeyInfo->db;
- mem1.flags = 0;
- mem1.u.i = 0; /* not needed, here to silence compiler warning */
- mem1.zMalloc = 0;
+ /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */
+ VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */
+
+ /* Compilers may complain that mem1.u.i is potentially uninitialized.
+ ** We could initialize it, as shown here, to silence those complaints.
+ ** But in fact, mem1.u.i will never actually be used initialized, and doing
+ ** the unnecessary initialization has a measurable negative performance
+ ** impact, since this routine is a very high runner. And so, we choose
+ ** to ignore the compiler warnings and leave this variable uninitialized.
+ */
+ /* mem1.u.i = 0; // not needed, here to silence compiler warning */
idx1 = getVarint32(aKey1, szHdr1);
d1 = szHdr1;
if( pPKey2->flags & UNPACKED_IGNORE_ROWID ){
szHdr1--;
@@ -50012,51 +49988,56 @@
/* Do the comparison
*/
rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i],
i<nField ? pKeyInfo->aColl[i] : 0);
if( rc!=0 ){
- break;
+ assert( mem1.zMalloc==0 ); /* See comment below */
+
+ /* Invert the result if we are using DESC sort order. */
+ if( pKeyInfo->aSortOrder && i<nField && pKeyInfo->aSortOrder[i] ){
+ rc = -rc;
+ }
+
+ /* If the PREFIX_SEARCH flag is set and all fields except the final
+ ** rowid field were equal, then clear the PREFIX_SEARCH flag and set
+ ** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1).
+ ** This is used by the OP_IsUnique opcode.
+ */
+ if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){
+ assert( idx1==szHdr1 && rc );
+ assert( mem1.flags & MEM_Int );
+ pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH;
+ pPKey2->rowid = mem1.u.i;
+ }
+
+ return rc;
}
i++;
}
- /* No memory allocation is ever used on mem1. */
- if( NEVER(mem1.zMalloc) ) sqlite3VdbeMemRelease(&mem1);
-
- /* If the PREFIX_SEARCH flag is set and all fields except the final
- ** rowid field were equal, then clear the PREFIX_SEARCH flag and set
- ** pPKey2->rowid to the value of the rowid field in (pKey1, nKey1).
- ** This is used by the OP_IsUnique opcode.
- */
- if( (pPKey2->flags & UNPACKED_PREFIX_SEARCH) && i==(pPKey2->nField-1) ){
- assert( idx1==szHdr1 && rc );
- assert( mem1.flags & MEM_Int );
- pPKey2->flags &= ~UNPACKED_PREFIX_SEARCH;
- pPKey2->rowid = mem1.u.i;
- }
-
- if( rc==0 ){
- /* rc==0 here means that one of the keys ran out of fields and
- ** all the fields up to that point were equal. If the UNPACKED_INCRKEY
- ** flag is set, then break the tie by treating key2 as larger.
- ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes
- ** are considered to be equal. Otherwise, the longer key is the
- ** larger. As it happens, the pPKey2 will always be the longer
- ** if there is a difference.
- */
- if( pPKey2->flags & UNPACKED_INCRKEY ){
- rc = -1;
- }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){
- /* Leave rc==0 */
- }else if( idx1<szHdr1 ){
- rc = 1;
- }
- }else if( pKeyInfo->aSortOrder && i<pKeyInfo->nField
- && pKeyInfo->aSortOrder[i] ){
- rc = -rc;
- }
-
+ /* No memory allocation is ever used on mem1. Prove this using
+ ** the following assert(). If the assert() fails, it indicates a
+ ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1).
+ */
+ assert( mem1.zMalloc==0 );
+
+ /* rc==0 here means that one of the keys ran out of fields and
+ ** all the fields up to that point were equal. If the UNPACKED_INCRKEY
+ ** flag is set, then break the tie by treating key2 as larger.
+ ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes
+ ** are considered to be equal. Otherwise, the longer key is the
+ ** larger. As it happens, the pPKey2 will always be the longer
+ ** if there is a difference.
+ */
+ assert( rc==0 );
+ if( pPKey2->flags & UNPACKED_INCRKEY ){
+ rc = -1;
+ }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){
+ /* Leave rc==0 */
+ }else if( idx1<szHdr1 ){
+ rc = 1;
+ }
return rc;
}
/*
@@ -50272,12 +50253,10 @@
**
*************************************************************************
**
** This file contains code use to implement APIs that are part of the
** VDBE.
-**
-** $Id: vdbeapi.c,v 1.167 2009/06/25 01:47:12 drh Exp $
*/
#ifndef SQLITE_OMIT_DEPRECATED
/*
** Return TRUE (non-zero) of the statement supplied as an argument needs
@@ -51528,12 +51507,10 @@
** Various scripts scan this source file in order to generate HTML
** documentation, headers files, or other derived files. The formatting
** of the code in this file is, therefore, important. See other comments
** in this file for details. If in doubt, do not deviate from existing
** commenting and indentation practices when changing or adding code.
-**
-** $Id: vdbe.c,v 1.874 2009/07/24 17:58:53 danielk1977 Exp $
*/
/*
** The following global variable is incremented every time a cursor
** moves, either by the OP_SeekXX, OP_Next, or OP_Prev opcodes. The test
@@ -51657,27 +51634,10 @@
pMem->type = SQLITE_BLOB;
}
}
/*
-** Properties of opcodes. The OPFLG_INITIALIZER macro is
-** created by mkopcodeh.awk during compilation. Data is obtained
-** from the comments following the "case OP_xxxx:" statements in
-** this file.
-*/
-static const unsigned char opcodeProperty[] = OPFLG_INITIALIZER;
-
-/*
-** Return true if an opcode has any of the OPFLG_xxx properties
-** specified by mask.
-*/
-SQLITE_PRIVATE int sqlite3VdbeOpcodeHasProperty(int opcode, int mask){
- assert( opcode>0 && opcode<(int)sizeof(opcodeProperty) );
- return (opcodeProperty[opcode]&mask)!=0;
-}
-
-/*
** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL
** if we run out of memory.
*/
static VdbeCursor *allocateCursor(
Vdbe *p, /* The virtual machine */
@@ -51707,11 +51667,11 @@
Mem *pMem = &p->aMem[p->nMem-iCur];
int nByte;
VdbeCursor *pCx = 0;
nByte =
- sizeof(VdbeCursor) +
+ ROUND8(sizeof(VdbeCursor)) +
(isBtreeCursor?sqlite3BtreeCursorSize():0) +
2*nField*sizeof(u32);
assert( iCur<p->nCursor );
if( p->apCsr[iCur] ){
@@ -51718,19 +51678,20 @@
sqlite3VdbeFreeCursor(p, p->apCsr[iCur]);
p->apCsr[iCur] = 0;
}
if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){
p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z;
- memset(pMem->z, 0, nByte);
+ memset(pCx, 0, sizeof(VdbeCursor));
pCx->iDb = iDb;
pCx->nField = nField;
if( nField ){
- pCx->aType = (u32 *)&pMem->z[sizeof(VdbeCursor)];
+ pCx->aType = (u32 *)&pMem->z[ROUND8(sizeof(VdbeCursor))];
}
if( isBtreeCursor ){
pCx->pCursor = (BtCursor*)
- &pMem->z[sizeof(VdbeCursor)+2*nField*sizeof(u32)];
+ &pMem->z[ROUND8(sizeof(VdbeCursor))+2*nField*sizeof(u32)];
+ sqlite3BtreeCursorZero(pCx->pCursor);
}
}
return pCx;
}
@@ -51968,12 +51929,10 @@
**
******************************************************************************
**
** This file contains inline asm code for retrieving "high-performance"
** counters for x86 class CPUs.
-**
-** $Id: hwtime.h,v 1.3 2008/08/01 14:33:15 shane Exp $
*/
#ifndef _HWTIME_H_
#define _HWTIME_H_
/*
@@ -52132,27 +52091,30 @@
*/
SQLITE_PRIVATE int sqlite3VdbeExec(
Vdbe *p /* The VDBE */
){
int pc; /* The program counter */
+ Op *aOp = p->aOp; /* Copy of p->aOp */
Op *pOp; /* Current operation */
int rc = SQLITE_OK; /* Value to return */
sqlite3 *db = p->db; /* The database */
+ u8 resetSchemaOnFault = 0; /* Reset schema after an error if true */
u8 encoding = ENC(db); /* The database encoding */
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+ u8 checkProgress; /* True if progress callbacks are enabled */
+ int nProgressOps = 0; /* Opcodes executed since progress callback. */
+#endif
+ Mem *aMem = p->aMem; /* Copy of p->aMem */
Mem *pIn1 = 0; /* 1st input operand */
Mem *pIn2 = 0; /* 2nd input operand */
Mem *pIn3 = 0; /* 3rd input operand */
Mem *pOut = 0; /* Output operand */
- u8 opProperty;
int iCompare = 0; /* Result of last OP_Compare operation */
int *aPermute = 0; /* Permutation of columns for OP_Compare */
#ifdef VDBE_PROFILE
u64 start; /* CPU clock count at start of opcode */
int origPc; /* Program counter at start of opcode */
-#endif
-#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
- int nProgressOps = 0; /* Opcodes executed since progress callback. */
#endif
/********************************************************************
** Automatically generated code
**
** The following union is automatically generated by the
@@ -52245,14 +52207,12 @@
int szHdr; /* Size of the header size field at start of record */
int avail; /* Number of bytes of available data */
Mem *pReg; /* PseudoTable input register */
} am;
struct OP_Affinity_stack_vars {
- char *zAffinity; /* The affinity to be applied */
- Mem *pData0; /* First register to which to apply affinity */
- Mem *pLast; /* Last register to which to apply affinity */
- Mem *pRec; /* Current register */
+ const char *zAffinity; /* The affinity to be applied */
+ char cAff; /* A single character of affinity */
} an;
struct OP_MakeRecord_stack_vars {
u8 *zNewRecord; /* A buffer to hold the data for the new record */
Mem *pRec; /* The new record */
u64 nData; /* Number of bytes of data space */
@@ -52333,18 +52293,19 @@
struct OP_Found_stack_vars {
int alreadyExists;
VdbeCursor *pC;
int res;
UnpackedRecord *pIdxKey;
+ UnpackedRecord r;
char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7];
} bb;
struct OP_IsUnique_stack_vars {
u16 ii;
VdbeCursor *pCx;
BtCursor *pCrsr;
u16 nField;
- Mem *aMem;
+ Mem *aMx;
UnpackedRecord r; /* B-Tree index search key */
i64 R; /* Rowid stored in register P3 */
} bc;
struct OP_NotExists_stack_vars {
VdbeCursor *pC;
@@ -52453,63 +52414,58 @@
int j; /* Loop counter */
int nErr; /* Number of errors reported */
char *z; /* Text of the error report */
Mem *pnErr; /* Register keeping track of errors remaining */
} bv;
- struct OP_RowSetAdd_stack_vars {
- Mem *pIdx;
- Mem *pVal;
- } bw;
struct OP_RowSetRead_stack_vars {
- Mem *pIdx;
i64 val;
- } bx;
+ } bw;
struct OP_RowSetTest_stack_vars {
int iSet;
int exists;
- } by;
+ } bx;
struct OP_Program_stack_vars {
int nMem; /* Number of memory registers for sub-program */
int nByte; /* Bytes of runtime space required for sub-program */
Mem *pRt; /* Register to allocate runtime space */
Mem *pMem; /* Used to iterate through memory cells */
Mem *pEnd; /* Last memory cell in new array */
VdbeFrame *pFrame; /* New vdbe frame to execute in */
SubProgram *pProgram; /* Sub-program to execute */
void *t; /* Token identifying trigger */
- } bz;
+ } by;
struct OP_Param_stack_vars {
VdbeFrame *pFrame;
Mem *pIn;
- } ca;
+ } bz;
struct OP_MemMax_stack_vars {
Mem *pIn1;
VdbeFrame *pFrame;
- } cb;
+ } ca;
struct OP_AggStep_stack_vars {
int n;
int i;
Mem *pMem;
Mem *pRec;
sqlite3_context ctx;
sqlite3_value **apVal;
- } cc;
+ } cb;
struct OP_AggFinal_stack_vars {
Mem *pMem;
- } cd;
+ } cc;
struct OP_IncrVacuum_stack_vars {
Btree *pBt;
- } ce;
+ } cd;
struct OP_VBegin_stack_vars {
VTable *pVTab;
- } cf;
+ } ce;
struct OP_VOpen_stack_vars {
VdbeCursor *pCur;
sqlite3_vtab_cursor *pVtabCursor;
sqlite3_vtab *pVtab;
sqlite3_module *pModule;
- } cg;
+ } cf;
struct OP_VFilter_stack_vars {
int nArg;
int iQuery;
const sqlite3_module *pModule;
Mem *pQuery;
@@ -52518,44 +52474,44 @@
sqlite3_vtab *pVtab;
VdbeCursor *pCur;
int res;
int i;
Mem **apArg;
- } ch;
+ } cg;
struct OP_VColumn_stack_vars {
sqlite3_vtab *pVtab;
const sqlite3_module *pModule;
Mem *pDest;
sqlite3_context sContext;
- } ci;
+ } ch;
struct OP_VNext_stack_vars {
sqlite3_vtab *pVtab;
const sqlite3_module *pModule;
int res;
VdbeCursor *pCur;
- } cj;
+ } ci;
struct OP_VRename_stack_vars {
sqlite3_vtab *pVtab;
Mem *pName;
- } ck;
+ } cj;
struct OP_VUpdate_stack_vars {
sqlite3_vtab *pVtab;
sqlite3_module *pModule;
int nArg;
int i;
sqlite_int64 rowid;
Mem **apArg;
Mem *pX;
- } cl;
+ } ck;
struct OP_Pagecount_stack_vars {
int p1;
int nPage;
Pager *pPager;
- } cm;
+ } cl;
struct OP_Trace_stack_vars {
char *zTrace;
- } cn;
+ } cm;
} u;
/* End automatically generated code
********************************************************************/
assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */
@@ -52571,20 +52527,23 @@
assert( p->explain==0 );
p->pResultSet = 0;
db->busyHandler.nBusy = 0;
CHECK_FOR_INTERRUPT;
sqlite3VdbeIOTraceSql(p);
+#ifndef SQLITE_OMIT_PROGRESS_CALLBACK
+ checkProgress = db->xProgress!=0;
+#endif
#ifdef SQLITE_DEBUG
sqlite3BeginBenignMalloc();
if( p->pc==0
&& ((p->db->flags & SQLITE_VdbeListing) || fileExists(db, "vdbe_explain"))
){
int i;
printf("VDBE Program Listing:\n");
sqlite3VdbePrintSql(p);
for(i=0; i<p->nOp; i++){
- sqlite3VdbePrintOp(stdout, i, &p->aOp[i]);
+ sqlite3VdbePrintOp(stdout, i, &aOp[i]);
}
}
if( fileExists(db, "vdbe_trace") ){
p->trace = stdout;
}
@@ -52595,11 +52554,11 @@
if( db->mallocFailed ) goto no_mem;
#ifdef VDBE_PROFILE
origPc = pc;
start = sqlite3Hwtime();
#endif
- pOp = &p->aOp[pc];
+ pOp = &aOp[pc];
/* Only allow tracing if SQLITE_DEBUG is defined.
*/
#ifdef SQLITE_DEBUG
if( p->trace ){
@@ -52636,11 +52595,11 @@
** of VDBE ops have been executed (either since this invocation of
** sqlite3VdbeExec() or since last time the progress callback was called).
** If the progress callback returns non-zero, exit the virtual machine with
** a return code SQLITE_ABORT.
*/
- if( db->xProgress ){
+ if( checkProgress ){
if( db->nProgressOps==nProgressOps ){
int prc;
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
prc =db->xProgress(db->pProgressArg);
if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
@@ -52652,69 +52611,50 @@
}
nProgressOps++;
}
#endif
- /* Do common setup processing for any opcode that is marked
- ** with the "out2-prerelease" tag. Such opcodes have a single
- ** output which is specified by the P2 parameter. The P2 register
- ** is initialized to a NULL.
- */
- opProperty = opcodeProperty[pOp->opcode];
- if( (opProperty & OPFLG_OUT2_PRERELEASE)!=0 ){
+ /* On any opcode with the "out2-prerelase" tag, free any
+ ** external allocations out of mem[p2] and set mem[p2] to be
+ ** an undefined integer. Opcodes will either fill in the integer
+ ** value or convert mem[p2] to a different type.
+ */
+ assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] );
+ if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){
assert( pOp->p2>0 );
assert( pOp->p2<=p->nMem );
- pOut = &p->aMem[pOp->p2];
+ pOut = &aMem[pOp->p2];
sqlite3VdbeMemReleaseExternal(pOut);
- pOut->flags = MEM_Null;
- pOut->n = 0;
- }else
-
- /* Do common setup for opcodes marked with one of the following
- ** combinations of properties.
- **
- ** in1
- ** in1 in2
- ** in1 in2 out3
- ** in1 in3
- **
- ** Variables pIn1, pIn2, and pIn3 are made to point to appropriate
- ** registers for inputs. Variable pOut points to the output register.
- */
- if( (opProperty & OPFLG_IN1)!=0 ){
+ pOut->flags = MEM_Int;
+ }
+
+ /* Sanity checking on other operands */
+#ifdef SQLITE_DEBUG
+ if( (pOp->opflags & OPFLG_IN1)!=0 ){
assert( pOp->p1>0 );
assert( pOp->p1<=p->nMem );
- pIn1 = &p->aMem[pOp->p1];
- REGISTER_TRACE(pOp->p1, pIn1);
- if( (opProperty & OPFLG_IN2)!=0 ){
- assert( pOp->p2>0 );
- assert( pOp->p2<=p->nMem );
- pIn2 = &p->aMem[pOp->p2];
- REGISTER_TRACE(pOp->p2, pIn2);
- /* As currently implemented, in2 implies out3. There is no reason
- ** why this has to be, it just worked out that way. */
- assert( (opProperty & OPFLG_OUT3)!=0 );
- assert( pOp->p3>0 );
- assert( pOp->p3<=p->nMem );
- pOut = &p->aMem[pOp->p3];
- }else if( (opProperty & OPFLG_IN3)!=0 ){
- assert( pOp->p3>0 );
- assert( pOp->p3<=p->nMem );
- pIn3 = &p->aMem[pOp->p3];
- REGISTER_TRACE(pOp->p3, pIn3);
- }
- }else if( (opProperty & OPFLG_IN2)!=0 ){
+ REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]);
+ }
+ if( (pOp->opflags & OPFLG_IN2)!=0 ){
+ assert( pOp->p2>0 );
+ assert( pOp->p2<=p->nMem );
+ REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]);
+ }
+ if( (pOp->opflags & OPFLG_IN3)!=0 ){
+ assert( pOp->p3>0 );
+ assert( pOp->p3<=p->nMem );
+ REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]);
+ }
+ if( (pOp->opflags & OPFLG_OUT2)!=0 ){
assert( pOp->p2>0 );
assert( pOp->p2<=p->nMem );
- pIn2 = &p->aMem[pOp->p2];
- REGISTER_TRACE(pOp->p2, pIn2);
- }else if( (opProperty & OPFLG_IN3)!=0 ){
+ }
+ if( (pOp->opflags & OPFLG_OUT3)!=0 ){
assert( pOp->p3>0 );
assert( pOp->p3<=p->nMem );
- pIn3 = &p->aMem[pOp->p3];
- REGISTER_TRACE(pOp->p3, pIn3);
- }
+ }
+#endif
switch( pOp->opcode ){
/*****************************************************************************
** What follows is a massive switch statement where each case implements a
@@ -52767,14 +52707,12 @@
/* Opcode: Gosub P1 P2 * * *
**
** Write the current address onto register P1
** and then jump to address P2.
*/
-case OP_Gosub: { /* jump */
- assert( pOp->p1>0 );
- assert( pOp->p1<=p->nMem );
- pIn1 = &p->aMem[pOp->p1];
+case OP_Gosub: { /* jump, in1 */
+ pIn1 = &aMem[pOp->p1];
assert( (pIn1->flags & MEM_Dyn)==0 );
pIn1->flags = MEM_Int;
pIn1->u.i = pc;
REGISTER_TRACE(pOp->p1, pIn1);
pc = pOp->p2 - 1;
@@ -52784,10 +52722,11 @@
/* Opcode: Return P1 * * * *
**
** Jump to the next instruction after the address in register P1.
*/
case OP_Return: { /* in1 */
+ pIn1 = &aMem[pOp->p1];
assert( pIn1->flags & MEM_Int );
pc = (int)pIn1->u.i;
break;
}
@@ -52797,10 +52736,11 @@
*/
case OP_Yield: { /* in1 */
#if 0 /* local variables moved into u.aa */
int pcDest;
#endif /* local variables moved into u.aa */
+ pIn1 = &aMem[pOp->p1];
assert( (pIn1->flags & MEM_Dyn)==0 );
pIn1->flags = MEM_Int;
u.aa.pcDest = (int)pIn1->u.i;
pIn1->u.i = pc;
REGISTER_TRACE(pOp->p1, pIn1);
@@ -52813,10 +52753,11 @@
** Check the value in register P3. If is is NULL then Halt using
** parameter P1, P2, and P4 as if this were a Halt instruction. If the
** value in register P3 is not NULL, then this routine is a no-op.
*/
case OP_HaltIfNull: { /* in3 */
+ pIn3 = &aMem[pOp->p3];
if( (pIn3->flags & MEM_Null)==0 ) break;
/* Fall through into OP_Halt */
}
/* Opcode: Halt P1 P2 * P4 *
@@ -52852,10 +52793,12 @@
** instruction is set to OE_Ignore, then the sub-program is throwing
** an IGNORE exception. In this case jump to the address specified
** as the p2 of the calling OP_Program. */
pc = p->aOp[pc].p2-1;
}
+ aOp = p->aOp;
+ aMem = p->aMem;
break;
}
p->rc = pOp->p1;
p->errorAction = (u8)pOp->p2;
@@ -52878,11 +52821,10 @@
/* Opcode: Integer P1 P2 * * *
**
** The 32-bit integer value P1 is written into register P2.
*/
case OP_Integer: { /* out2-prerelease */
- pOut->flags = MEM_Int;
pOut->u.i = pOp->p1;
break;
}
/* Opcode: Int64 * P2 * P4 *
@@ -52890,11 +52832,10 @@
** P4 is a pointer to a 64-bit integer value.
** Write that value into register P2.
*/
case OP_Int64: { /* out2-prerelease */
assert( pOp->p4.pI64!=0 );
- pOut->flags = MEM_Int;
pOut->u.i = *pOp->p4.pI64;
break;
}
/* Opcode: Real * P2 * P4 *
@@ -52960,10 +52901,11 @@
/* Opcode: Null * P2 * * *
**
** Write a NULL into register P2.
*/
case OP_Null: { /* out2-prerelease */
+ pOut->flags = MEM_Null;
break;
}
/* Opcode: Blob P1 P2 * P4
@@ -53009,11 +52951,11 @@
while( u.ab.n-- > 0 ){
u.ab.pVar = &p->aVar[u.ab.p1++];
if( sqlite3VdbeMemTooBig(u.ab.pVar) ){
goto too_big;
}
- pOut = &p->aMem[u.ab.p2++];
+ pOut = &aMem[u.ab.p2++];
sqlite3VdbeMemReleaseExternal(pOut);
pOut->flags = MEM_Null;
sqlite3VdbeMemShallowCopy(pOut, u.ab.pVar, MEM_Static);
UPDATE_MAX_BLOBSIZE(pOut);
}
@@ -53039,15 +52981,15 @@
u.ac.p1 = pOp->p1;
u.ac.p2 = pOp->p2;
assert( u.ac.n>0 && u.ac.p1>0 && u.ac.p2>0 );
assert( u.ac.p1+u.ac.n<=u.ac.p2 || u.ac.p2+u.ac.n<=u.ac.p1 );
- pIn1 = &p->aMem[u.ac.p1];
- pOut = &p->aMem[u.ac.p2];
+ pIn1 = &aMem[u.ac.p1];
+ pOut = &aMem[u.ac.p2];
while( u.ac.n-- ){
- assert( pOut<=&p->aMem[p->nMem] );
- assert( pIn1<=&p->aMem[p->nMem] );
+ assert( pOut<=&aMem[p->nMem] );
+ assert( pIn1<=&aMem[p->nMem] );
u.ac.zMalloc = pOut->zMalloc;
pOut->zMalloc = 0;
sqlite3VdbeMemMove(pOut, pIn1);
pIn1->zMalloc = u.ac.zMalloc;
REGISTER_TRACE(u.ac.p2++, pOut);
@@ -53062,14 +53004,13 @@
** Make a copy of register P1 into register P2.
**
** This instruction makes a deep copy of the value. A duplicate
** is made of any string or blob constant. See also OP_SCopy.
*/
-case OP_Copy: { /* in1 */
- assert( pOp->p2>0 );
- assert( pOp->p2<=p->nMem );
- pOut = &p->aMem[pOp->p2];
+case OP_Copy: { /* in1, out2 */
+ pIn1 = &aMem[pOp->p1];
+ pOut = &aMem[pOp->p2];
assert( pOut!=pIn1 );
sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
Deephemeralize(pOut);
REGISTER_TRACE(pOp->p2, pOut);
break;
@@ -53085,15 +53026,13 @@
** Worse, if the original is deallocated, the copy becomes invalid.
** Thus the program must guarantee that the original will not change
** during the lifetime of the copy. Use OP_Copy to make a complete
** copy.
*/
-case OP_SCopy: { /* in1 */
- REGISTER_TRACE(pOp->p1, pIn1);
- assert( pOp->p2>0 );
- assert( pOp->p2<=p->nMem );
- pOut = &p->aMem[pOp->p2];
+case OP_SCopy: { /* in1, out2 */
+ pIn1 = &aMem[pOp->p1];
+ pOut = &aMem[pOp->p2];
assert( pOut!=pIn1 );
sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem);
REGISTER_TRACE(pOp->p2, pOut);
break;
}
@@ -53150,11 +53089,11 @@
/* Make sure the results of the current row are \000 terminated
** and have an assigned type. The results are de-ephemeralized as
** as side effect.
*/
- u.ad.pMem = p->pResultSet = &p->aMem[pOp->p1];
+ u.ad.pMem = p->pResultSet = &aMem[pOp->p1];
for(u.ad.i=0; u.ad.i<pOp->p2; u.ad.i++){
sqlite3VdbeMemNulTerminate(&u.ad.pMem[u.ad.i]);
sqlite3VdbeMemStoreType(&u.ad.pMem[u.ad.i]);
REGISTER_TRACE(pOp->p1+u.ad.i, &u.ad.pMem[u.ad.i]);
}
@@ -53182,10 +53121,13 @@
case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */
#if 0 /* local variables moved into u.ae */
i64 nByte;
#endif /* local variables moved into u.ae */
+ pIn1 = &aMem[pOp->p1];
+ pIn2 = &aMem[pOp->p2];
+ pOut = &aMem[pOp->p3];
assert( pIn1!=pOut );
if( (pIn1->flags | pIn2->flags) & MEM_Null ){
sqlite3VdbeMemSetNull(pOut);
break;
}
@@ -53257,12 +53199,15 @@
i64 iB; /* Integer value of right operand */
double rA; /* Real value of left operand */
double rB; /* Real value of right operand */
#endif /* local variables moved into u.af */
+ pIn1 = &aMem[pOp->p1];
applyNumericAffinity(pIn1);
+ pIn2 = &aMem[pOp->p2];
applyNumericAffinity(pIn2);
+ pOut = &aMem[pOp->p3];
u.af.flags = pIn1->flags | pIn2->flags;
if( (u.af.flags & MEM_Null)!=0 ) goto arithmetic_result_is_null;
if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){
u.af.iA = pIn1->u.i;
u.af.iB = pIn2->u.i;
@@ -53375,11 +53320,11 @@
u.ag.apVal = p->apArg;
assert( u.ag.apVal || u.ag.n==0 );
assert( u.ag.n==0 || (pOp->p2>0 && pOp->p2+u.ag.n<=p->nMem+1) );
assert( pOp->p3<pOp->p2 || pOp->p3>=pOp->p2+u.ag.n );
- u.ag.pArg = &p->aMem[pOp->p2];
+ u.ag.pArg = &aMem[pOp->p2];
for(u.ag.i=0; u.ag.i<u.ag.n; u.ag.i++, u.ag.pArg++){
u.ag.apVal[u.ag.i] = u.ag.pArg;
sqlite3VdbeMemStoreType(u.ag.pArg);
REGISTER_TRACE(pOp->p2, u.ag.pArg);
}
@@ -53392,11 +53337,11 @@
u.ag.ctx.pVdbeFunc = (VdbeFunc*)pOp->p4.pVdbeFunc;
u.ag.ctx.pFunc = u.ag.ctx.pVdbeFunc->pFunc;
}
assert( pOp->p3>0 && pOp->p3<=p->nMem );
- pOut = &p->aMem[pOp->p3];
+ pOut = &aMem[pOp->p3];
u.ag.ctx.s.flags = MEM_Null;
u.ag.ctx.s.db = db;
u.ag.ctx.s.xDel = 0;
u.ag.ctx.s.zMalloc = 0;
@@ -53407,11 +53352,11 @@
sqlite3VdbeMemMove(&u.ag.ctx.s, pOut);
MemSetTypeFlag(&u.ag.ctx.s, MEM_Null);
u.ag.ctx.isError = 0;
if( u.ag.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
- assert( pOp>p->aOp );
+ assert( pOp>aOp );
assert( pOp[-1].p4type==P4_COLLSEQ );
assert( pOp[-1].opcode==OP_CollSeq );
u.ag.ctx.pColl = pOp[-1].p4.pColl;
}
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
@@ -53493,10 +53438,13 @@
#if 0 /* local variables moved into u.ah */
i64 a;
i64 b;
#endif /* local variables moved into u.ah */
+ pIn1 = &aMem[pOp->p1];
+ pIn2 = &aMem[pOp->p2];
+ pOut = &aMem[pOp->p3];
if( (pIn1->flags | pIn2->flags) & MEM_Null ){
sqlite3VdbeMemSetNull(pOut);
break;
}
u.ah.a = sqlite3VdbeIntValue(pIn2);
@@ -53519,10 +53467,11 @@
** The result is always an integer.
**
** To force any register to be an integer, just add 0.
*/
case OP_AddImm: { /* in1 */
+ pIn1 = &aMem[pOp->p1];
sqlite3VdbeMemIntegerify(pIn1);
pIn1->u.i += pOp->p2;
break;
}
@@ -53532,10 +53481,11 @@
** in P1 is not an integer and cannot be converted into an integer
** without data loss, then jump immediately to P2, or if P2==0
** raise an SQLITE_MISMATCH exception.
*/
case OP_MustBeInt: { /* jump, in1 */
+ pIn1 = &aMem[pOp->p1];
applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding);
if( (pIn1->flags & MEM_Int)==0 ){
if( pOp->p2==0 ){
rc = SQLITE_MISMATCH;
goto abort_due_to_error;
@@ -53556,10 +53506,11 @@
** has REAL affinity. Such column values may still be stored as
** integers, for space efficiency, but after extraction we want them
** to have only a real value.
*/
case OP_RealAffinity: { /* in1 */
+ pIn1 = &aMem[pOp->p1];
if( pIn1->flags & MEM_Int ){
sqlite3VdbeMemRealify(pIn1);
}
break;
}
@@ -53573,10 +53524,11 @@
** are afterwards simply interpreted as text.
**
** A NULL value is not changed by this routine. It remains NULL.
*/
case OP_ToText: { /* same as TK_TO_TEXT, in1 */
+ pIn1 = &aMem[pOp->p1];
if( pIn1->flags & MEM_Null ) break;
assert( MEM_Str==(MEM_Blob>>3) );
pIn1->flags |= (pIn1->flags&MEM_Blob)>>3;
applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
rc = ExpandBlob(pIn1);
@@ -53594,10 +53546,11 @@
** to the underlying data.
**
** A NULL value is not changed by this routine. It remains NULL.
*/
case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */
+ pIn1 = &aMem[pOp->p1];
if( pIn1->flags & MEM_Null ) break;
if( (pIn1->flags & MEM_Blob)==0 ){
applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding);
assert( pIn1->flags & MEM_Str || db->mallocFailed );
MemSetTypeFlag(pIn1, MEM_Blob);
@@ -53617,10 +53570,11 @@
** is possible.
**
** A NULL value is not changed by this routine. It remains NULL.
*/
case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */
+ pIn1 = &aMem[pOp->p1];
if( (pIn1->flags & (MEM_Null|MEM_Int|MEM_Real))==0 ){
sqlite3VdbeMemNumerify(pIn1);
}
break;
}
@@ -53634,10 +53588,11 @@
** equivalent of atoi() and store 0 if no such conversion is possible.
**
** A NULL value is not changed by this routine. It remains NULL.
*/
case OP_ToInt: { /* same as TK_TO_INT, in1 */
+ pIn1 = &aMem[pOp->p1];
if( (pIn1->flags & MEM_Null)==0 ){
sqlite3VdbeMemIntegerify(pIn1);
}
break;
}
@@ -53651,10 +53606,11 @@
** equivalent of atoi() and store 0.0 if no such conversion is possible.
**
** A NULL value is not changed by this routine. It remains NULL.
*/
case OP_ToReal: { /* same as TK_TO_REAL, in1 */
+ pIn1 = &aMem[pOp->p1];
if( (pIn1->flags & MEM_Null)==0 ){
sqlite3VdbeMemRealify(pIn1);
}
break;
}
@@ -53741,10 +53697,12 @@
#if 0 /* local variables moved into u.ai */
int res; /* Result of the comparison of pIn1 against pIn3 */
char affinity; /* Affinity to use for comparison */
#endif /* local variables moved into u.ai */
+ pIn1 = &aMem[pOp->p1];
+ pIn3 = &aMem[pOp->p3];
if( (pIn1->flags | pIn3->flags)&MEM_Null ){
/* One or both operands are NULL */
if( pOp->p5 & SQLITE_NULLEQ ){
/* If SQLITE_NULLEQ is set (which will only happen if the operator is
** OP_Eq or OP_Ne) then take the jump or not depending on whether
@@ -53756,11 +53714,11 @@
/* SQLITE_NULLEQ is clear and at least one operand is NULL,
** then the result is always NULL.
** The jump is taken if the SQLITE_JUMPIFNULL bit is set.
*/
if( pOp->p5 & SQLITE_STOREP2 ){
- pOut = &p->aMem[pOp->p2];
+ pOut = &aMem[pOp->p2];
MemSetTypeFlag(pOut, MEM_Null);
REGISTER_TRACE(pOp->p2, pOut);
}else if( pOp->p5 & SQLITE_JUMPIFNULL ){
pc = pOp->p2-1;
}
@@ -53788,11 +53746,11 @@
case OP_Gt: u.ai.res = u.ai.res>0; break;
default: u.ai.res = u.ai.res>=0; break;
}
if( pOp->p5 & SQLITE_STOREP2 ){
- pOut = &p->aMem[pOp->p2];
+ pOut = &aMem[pOp->p2];
MemSetTypeFlag(pOut, MEM_Int);
pOut->u.i = u.ai.res;
REGISTER_TRACE(pOp->p2, pOut);
}else if( u.ai.res ){
pc = pOp->p2-1;
@@ -53859,16 +53817,16 @@
assert( u.aj.p2>0 && u.aj.p2+u.aj.n<=p->nMem+1 );
}
#endif /* SQLITE_DEBUG */
for(u.aj.i=0; u.aj.i<u.aj.n; u.aj.i++){
u.aj.idx = aPermute ? aPermute[u.aj.i] : u.aj.i;
- REGISTER_TRACE(u.aj.p1+u.aj.idx, &p->aMem[u.aj.p1+u.aj.idx]);
- REGISTER_TRACE(u.aj.p2+u.aj.idx, &p->aMem[u.aj.p2+u.aj.idx]);
+ REGISTER_TRACE(u.aj.p1+u.aj.idx, &aMem[u.aj.p1+u.aj.idx]);
+ REGISTER_TRACE(u.aj.p2+u.aj.idx, &aMem[u.aj.p2+u.aj.idx]);
assert( u.aj.i<u.aj.pKeyInfo->nField );
u.aj.pColl = u.aj.pKeyInfo->aColl[u.aj.i];
u.aj.bRev = u.aj.pKeyInfo->aSortOrder[u.aj.i];
- iCompare = sqlite3MemCompare(&p->aMem[u.aj.p1+u.aj.idx], &p->aMem[u.aj.p2+u.aj.idx], u.aj.pColl);
+ iCompare = sqlite3MemCompare(&aMem[u.aj.p1+u.aj.idx], &aMem[u.aj.p2+u.aj.idx], u.aj.pColl);
if( iCompare ){
if( u.aj.bRev ) iCompare = -iCompare;
break;
}
}
@@ -53916,15 +53874,17 @@
#if 0 /* local variables moved into u.ak */
int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */
int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */
#endif /* local variables moved into u.ak */
+ pIn1 = &aMem[pOp->p1];
if( pIn1->flags & MEM_Null ){
u.ak.v1 = 2;
}else{
u.ak.v1 = sqlite3VdbeIntValue(pIn1)!=0;
}
+ pIn2 = &aMem[pOp->p2];
if( pIn2->flags & MEM_Null ){
u.ak.v2 = 2;
}else{
u.ak.v2 = sqlite3VdbeIntValue(pIn2)!=0;
}
@@ -53933,10 +53893,11 @@
u.ak.v1 = and_logic[u.ak.v1*3+u.ak.v2];
}else{
static const unsigned char or_logic[] = { 0, 1, 2, 1, 1, 1, 2, 1, 2 };
u.ak.v1 = or_logic[u.ak.v1*3+u.ak.v2];
}
+ pOut = &aMem[pOp->p3];
if( u.ak.v1==2 ){
MemSetTypeFlag(pOut, MEM_Null);
}else{
pOut->u.i = u.ak.v1;
MemSetTypeFlag(pOut, MEM_Int);
@@ -53948,12 +53909,13 @@
**
** Interpret the value in register P1 as a boolean value. Store the
** boolean complement in register P2. If the value in register P1 is
** NULL, then a NULL is stored in P2.
*/
-case OP_Not: { /* same as TK_NOT, in1 */
- pOut = &p->aMem[pOp->p2];
+case OP_Not: { /* same as TK_NOT, in1, out2 */
+ pIn1 = &aMem[pOp->p1];
+ pOut = &aMem[pOp->p2];
if( pIn1->flags & MEM_Null ){
sqlite3VdbeMemSetNull(pOut);
}else{
sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1));
}
@@ -53964,12 +53926,13 @@
**
** Interpret the content of register P1 as an integer. Store the
** ones-complement of the P1 value into register P2. If P1 holds
** a NULL then store a NULL in P2.
*/
-case OP_BitNot: { /* same as TK_BITNOT, in1 */
- pOut = &p->aMem[pOp->p2];
+case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */
+ pIn1 = &aMem[pOp->p1];
+ pOut = &aMem[pOp->p2];
if( pIn1->flags & MEM_Null ){
sqlite3VdbeMemSetNull(pOut);
}else{
sqlite3VdbeMemSetInt64(pOut, ~sqlite3VdbeIntValue(pIn1));
}
@@ -53991,10 +53954,11 @@
case OP_If: /* jump, in1 */
case OP_IfNot: { /* jump, in1 */
#if 0 /* local variables moved into u.al */
int c;
#endif /* local variables moved into u.al */
+ pIn1 = &aMem[pOp->p1];
if( pIn1->flags & MEM_Null ){
u.al.c = pOp->p3;
}else{
#ifdef SQLITE_OMIT_FLOATING_POINT
u.al.c = sqlite3VdbeIntValue(pIn1)!=0;
@@ -54012,10 +53976,11 @@
/* Opcode: IsNull P1 P2 * * *
**
** Jump to P2 if the value in register P1 is NULL.
*/
case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */
+ pIn1 = &aMem[pOp->p1];
if( (pIn1->flags & MEM_Null)!=0 ){
pc = pOp->p2 - 1;
}
break;
}
@@ -54023,10 +53988,11 @@
/* Opcode: NotNull P1 P2 * * *
**
** Jump to P2 if the value in register P1 is not NULL.
*/
case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */
+ pIn1 = &aMem[pOp->p1];
if( (pIn1->flags & MEM_Null)==0 ){
pc = pOp->p2 - 1;
}
break;
}
@@ -54081,11 +54047,11 @@
u.am.p2 = pOp->p2;
u.am.pC = 0;
memset(&u.am.sMem, 0, sizeof(u.am.sMem));
assert( u.am.p1<p->nCursor );
assert( pOp->p3>0 && pOp->p3<=p->nMem );
- u.am.pDest = &p->aMem[pOp->p3];
+ u.am.pDest = &aMem[pOp->p3];
MemSetTypeFlag(u.am.pDest, MEM_Null);
u.am.zRec = 0;
/* This block sets the variable u.am.payloadSize to be the total number of
** bytes in the record.
@@ -54127,11 +54093,11 @@
assert( sqlite3BtreeCursorIsValid(u.am.pCrsr) );
rc = sqlite3BtreeDataSize(u.am.pCrsr, &u.am.payloadSize);
assert( rc==SQLITE_OK ); /* DataSize() cannot fail */
}
}else if( u.am.pC->pseudoTableReg>0 ){
- u.am.pReg = &p->aMem[u.am.pC->pseudoTableReg];
+ u.am.pReg = &aMem[u.am.pC->pseudoTableReg];
assert( u.am.pReg->flags & MEM_Blob );
u.am.payloadSize = u.am.pReg->n;
u.am.zRec = u.am.pReg->z;
u.am.pC->cacheStatus = (pOp->p5&OPFLAG_CLEARCACHE) ? CACHE_STALE : p->cacheCtr;
assert( u.am.payloadSize==0 || u.am.zRec!=0 );
@@ -54339,22 +54305,23 @@
** string indicates the column affinity that should be used for the nth
** memory cell in the range.
*/
case OP_Affinity: {
#if 0 /* local variables moved into u.an */
- char *zAffinity; /* The affinity to be applied */
- Mem *pData0; /* First register to which to apply affinity */
- Mem *pLast; /* Last register to which to apply affinity */
- Mem *pRec; /* Current register */
+ const char *zAffinity; /* The affinity to be applied */
+ char cAff; /* A single character of affinity */
#endif /* local variables moved into u.an */
u.an.zAffinity = pOp->p4.z;
- u.an.pData0 = &p->aMem[pOp->p1];
- u.an.pLast = &u.an.pData0[pOp->p2-1];
- for(u.an.pRec=u.an.pData0; u.an.pRec<=u.an.pLast; u.an.pRec++){
- ExpandBlob(u.an.pRec);
- applyAffinity(u.an.pRec, u.an.zAffinity[u.an.pRec-u.an.pData0], encoding);
+ assert( u.an.zAffinity!=0 );
+ assert( u.an.zAffinity[pOp->p2]==0 );
+ pIn1 = &aMem[pOp->p1];
+ while( (u.an.cAff = *(u.an.zAffinity++))!=0 ){
+ assert( pIn1 <= &p->aMem[p->nMem] );
+ ExpandBlob(pIn1);
+ applyAffinity(pIn1, u.an.cAff, encoding);
+ pIn1++;
}
break;
}
/* Opcode: MakeRecord P1 P2 P3 P4 *
@@ -54414,11 +54381,11 @@
u.ao.nByte = 0; /* Data space required for this record */
u.ao.nZero = 0; /* Number of zero bytes at the end of the record */
u.ao.nField = pOp->p1;
u.ao.zAffinity = pOp->p4.z;
assert( u.ao.nField>0 && pOp->p2>0 && pOp->p2+u.ao.nField<=p->nMem+1 );
- u.ao.pData0 = &p->aMem[u.ao.nField];
+ u.ao.pData0 = &aMem[u.ao.nField];
u.ao.nField = pOp->p2;
u.ao.pLast = &u.ao.pData0[u.ao.nField-1];
u.ao.file_format = p->minWriteFileFormat;
/* Loop through the elements that will make up the record to figure
@@ -54458,11 +54425,11 @@
** the new record. The output register (pOp->p3) is not allowed to
** be one of the input registers (because the following call to
** sqlite3VdbeMemGrow() could clobber the value before it is used).
*/
assert( pOp->p3<pOp->p1 || pOp->p3>=pOp->p1+pOp->p2 );
- pOut = &p->aMem[pOp->p3];
+ pOut = &aMem[pOp->p3];
if( sqlite3VdbeMemGrow(pOut, (int)u.ao.nByte, 0) ){
goto no_mem;
}
u.ao.zNewRecord = (u8 *)pOut->z;
@@ -54507,11 +54474,10 @@
if( u.ap.pCrsr ){
rc = sqlite3BtreeCount(u.ap.pCrsr, &u.ap.nEntry);
}else{
u.ap.nEntry = 0;
}
- pOut->flags = MEM_Int;
pOut->u.i = u.ap.nEntry;
break;
}
#endif
@@ -54836,11 +54802,10 @@
assert( db->aDb[u.at.iDb].pBt!=0 );
assert( (p->btreeMask & (1<<u.at.iDb))!=0 );
sqlite3BtreeGetMeta(db->aDb[u.at.iDb].pBt, u.at.iCookie, (u32 *)&u.at.iMeta);
pOut->u.i = u.at.iMeta;
- MemSetTypeFlag(pOut, MEM_Int);
break;
}
/* Opcode: SetCookie P1 P2 P3 * *
**
@@ -54859,10 +54824,11 @@
assert( pOp->p2<SQLITE_N_BTREE_META );
assert( pOp->p1>=0 && pOp->p1<db->nDb );
assert( (p->btreeMask & (1<<pOp->p1))!=0 );
u.au.pDb = &db->aDb[pOp->p1];
assert( u.au.pDb->pBt!=0 );
+ pIn3 = &aMem[pOp->p3];
sqlite3VdbeMemIntegerify(pIn3);
/* See note about index shifting on OP_ReadCookie */
rc = sqlite3BtreeUpdateMeta(u.au.pDb->pBt, pOp->p2, (int)pIn3->u.i);
if( pOp->p2==BTREE_SCHEMA_VERSION ){
/* When the schema cookie changes, record the new cookie internally */
@@ -55021,11 +54987,11 @@
u.aw.wrFlag = 0;
}
if( pOp->p5 ){
assert( u.aw.p2>0 );
assert( u.aw.p2<=p->nMem );
- pIn2 = &p->aMem[u.aw.p2];
+ pIn2 = &aMem[u.aw.p2];
sqlite3VdbeMemIntegerify(pIn2);
u.aw.p2 = (int)pIn2->u.i;
/* The u.aw.p2 value always comes from a prior OP_CreateTable opcode and
** that opcode will always set the u.aw.p2 value to 2 or more or else fail.
** If there were a failure, the prepared statement would have halted
@@ -55243,17 +55209,21 @@
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
assert( pOp->p2!=0 );
u.az.pC = p->apCsr[pOp->p1];
assert( u.az.pC!=0 );
assert( u.az.pC->pseudoTableReg==0 );
+ assert( OP_SeekLe == OP_SeekLt+1 );
+ assert( OP_SeekGe == OP_SeekLt+2 );
+ assert( OP_SeekGt == OP_SeekLt+3 );
if( u.az.pC->pCursor!=0 ){
u.az.oc = pOp->opcode;
u.az.pC->nullRow = 0;
if( u.az.pC->isTable ){
/* The input value in P3 might be of any type: integer, real, string,
** blob, or NULL. But it needs to be an integer before we can do
** the seek, so covert it. */
+ pIn3 = &aMem[pOp->p3];
applyNumericAffinity(pIn3);
u.az.iKey = sqlite3VdbeIntValue(pIn3);
u.az.pC->rowidIsValid = 0;
/* If the P3 value could not be converted into an integer without
@@ -55272,16 +55242,16 @@
if( u.az.iKey==SMALLEST_INT64 && (pIn3->r<(double)u.az.iKey || pIn3->r>0) ){
/* The P3 value is too large in magnitude to be expressed as an
** integer. */
u.az.res = 1;
if( pIn3->r<0 ){
- if( u.az.oc==OP_SeekGt || u.az.oc==OP_SeekGe ){
+ if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe || u.az.oc==OP_SeekGt );
rc = sqlite3BtreeFirst(u.az.pC->pCursor, &u.az.res);
if( rc!=SQLITE_OK ) goto abort_due_to_error;
}
}else{
- if( u.az.oc==OP_SeekLt || u.az.oc==OP_SeekLe ){
+ if( u.az.oc<=OP_SeekLe ){ assert( u.az.oc==OP_SeekLt || u.az.oc==OP_SeekLe );
rc = sqlite3BtreeLast(u.az.pC->pCursor, &u.az.res);
if( rc!=SQLITE_OK ) goto abort_due_to_error;
}
}
if( u.az.res ){
@@ -55309,16 +55279,26 @@
u.az.nField = pOp->p4.i;
assert( pOp->p4type==P4_INT32 );
assert( u.az.nField>0 );
u.az.r.pKeyInfo = u.az.pC->pKeyInfo;
u.az.r.nField = (u16)u.az.nField;
- if( u.az.oc==OP_SeekGt || u.az.oc==OP_SeekLe ){
- u.az.r.flags = UNPACKED_INCRKEY;
- }else{
- u.az.r.flags = 0;
- }
- u.az.r.aMem = &p->aMem[pOp->p3];
+
+ /* The next line of code computes as follows, only faster:
+ ** if( u.az.oc==OP_SeekGt || u.az.oc==OP_SeekLe ){
+ ** u.az.r.flags = UNPACKED_INCRKEY;
+ ** }else{
+ ** u.az.r.flags = 0;
+ ** }
+ */
+ u.az.r.flags = UNPACKED_INCRKEY * (1 & (u.az.oc - OP_SeekLt));
+ assert( u.az.oc!=OP_SeekGt || u.az.r.flags==UNPACKED_INCRKEY );
+ assert( u.az.oc!=OP_SeekLe || u.az.r.flags==UNPACKED_INCRKEY );
+ assert( u.az.oc!=OP_SeekGe || u.az.r.flags==0 );
+ assert( u.az.oc!=OP_SeekLt || u.az.r.flags==0 );
+
+ u.az.r.aMem = &aMem[pOp->p3];
+ ExpandBlob(u.az.r.aMem);
rc = sqlite3BtreeMovetoUnpacked(u.az.pC->pCursor, &u.az.r, 0, 0, &u.az.res);
if( rc!=SQLITE_OK ){
goto abort_due_to_error;
}
u.az.pC->rowidIsValid = 0;
@@ -55326,11 +55306,11 @@
u.az.pC->deferredMoveto = 0;
u.az.pC->cacheStatus = CACHE_STALE;
#ifdef SQLITE_TEST
sqlite3_search_count++;
#endif
- if( u.az.oc==OP_SeekGe || u.az.oc==OP_SeekGt ){
+ if( u.az.oc>=OP_SeekGe ){ assert( u.az.oc==OP_SeekGe || u.az.oc==OP_SeekGt );
if( u.az.res<0 || (u.az.res==0 && u.az.oc==OP_SeekGt) ){
rc = sqlite3BtreeNext(u.az.pC->pCursor, &u.az.res);
if( rc!=SQLITE_OK ) goto abort_due_to_error;
u.az.pC->rowidIsValid = 0;
}else{
@@ -55381,45 +55361,40 @@
u.ba.pC = p->apCsr[pOp->p1];
assert( u.ba.pC!=0 );
if( ALWAYS(u.ba.pC->pCursor!=0) ){
assert( u.ba.pC->isTable );
u.ba.pC->nullRow = 0;
+ pIn2 = &aMem[pOp->p2];
u.ba.pC->movetoTarget = sqlite3VdbeIntValue(pIn2);
u.ba.pC->rowidIsValid = 0;
u.ba.pC->deferredMoveto = 1;
}
break;
}
-/* Opcode: Found P1 P2 P3 * *
-**
-** Register P3 holds a blob constructed by MakeRecord. P1 is an index.
-** If an entry that matches the value in register p3 exists in P1 then
-** jump to P2. If the P3 value does not match any entry in P1
-** then fall thru. The P1 cursor is left pointing at the matching entry
-** if it exists.
-**
-** This instruction is used to implement the IN operator where the
-** left-hand side is a SELECT statement. P1 may be a true index, or it
-** may be a temporary index that holds the results of the SELECT
-** statement. This instruction is also used to implement the
-** DISTINCT keyword in SELECT statements.
-**
-** This instruction checks if index P1 contains a record for which
-** the first N serialized values exactly match the N serialized values
-** in the record in register P3, where N is the total number of values in
-** the P3 record (the P3 record is a prefix of the P1 record).
-**
-** See also: NotFound, IsUnique, NotExists
-*/
-/* Opcode: NotFound P1 P2 P3 * *
-**
-** Register P3 holds a blob constructed by MakeRecord. P1 is
-** an index. If no entry exists in P1 that matches the blob then jump
-** to P2. If an entry does existing, fall through. The cursor is left
-** pointing to the entry that matches.
+/* Opcode: Found P1 P2 P3 P4 *
+**
+** If P4==0 then register P3 holds a blob constructed by MakeRecord. If
+** P4>0 then register P3 is the first of P4 registers that form an unpacked
+** record.
+**
+** Cursor P1 is on an index btree. If the record identified by P3 and P4
+** is a prefix of any entry in P1 then a jump is made to P2 and
+** P1 is left pointing at the matching entry.
+*/
+/* Opcode: NotFound P1 P2 P3 P4 *
+**
+** If P4==0 then register P3 holds a blob constructed by MakeRecord. If
+** P4>0 then register P3 is the first of P4 registers that form an unpacked
+** record.
+**
+** Cursor P1 is on an index btree. If the record identified by P3 and P4
+** is not the prefix of any entry in P1 then a jump is made to P2. If P1
+** does contain an entry whose prefix matches the P3/P4 record then control
+** falls through to the next instruction and P1 is left pointing at the
+** matching entry.
**
** See also: Found, NotExists, IsUnique
*/
case OP_NotFound: /* jump, in3 */
case OP_Found: { /* jump, in3 */
@@ -55426,36 +55401,47 @@
#if 0 /* local variables moved into u.bb */
int alreadyExists;
VdbeCursor *pC;
int res;
UnpackedRecord *pIdxKey;
+ UnpackedRecord r;
char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7];
#endif /* local variables moved into u.bb */
#ifdef SQLITE_TEST
sqlite3_found_count++;
#endif
u.bb.alreadyExists = 0;
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
+ assert( pOp->p4type==P4_INT32 );
u.bb.pC = p->apCsr[pOp->p1];
assert( u.bb.pC!=0 );
+ pIn3 = &aMem[pOp->p3];
if( ALWAYS(u.bb.pC->pCursor!=0) ){
assert( u.bb.pC->isTable==0 );
- assert( pIn3->flags & MEM_Blob );
- ExpandBlob(pIn3);
- u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z,
- u.bb.aTempRec, sizeof(u.bb.aTempRec));
- if( u.bb.pIdxKey==0 ){
- goto no_mem;
- }
- if( pOp->opcode==OP_Found ){
+ if( pOp->p4.i>0 ){
+ u.bb.r.pKeyInfo = u.bb.pC->pKeyInfo;
+ u.bb.r.nField = pOp->p4.i;
+ u.bb.r.aMem = pIn3;
+ u.bb.r.flags = UNPACKED_PREFIX_MATCH;
+ u.bb.pIdxKey = &u.bb.r;
+ }else{
+ assert( pIn3->flags & MEM_Blob );
+ ExpandBlob(pIn3);
+ u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z,
+ u.bb.aTempRec, sizeof(u.bb.aTempRec));
+ if( u.bb.pIdxKey==0 ){
+ goto no_mem;
+ }
u.bb.pIdxKey->flags |= UNPACKED_PREFIX_MATCH;
}
rc = sqlite3BtreeMovetoUnpacked(u.bb.pC->pCursor, u.bb.pIdxKey, 0, 0, &u.bb.res);
- sqlite3VdbeDeleteUnpackedRecord(u.bb.pIdxKey);
+ if( pOp->p4.i==0 ){
+ sqlite3VdbeDeleteUnpackedRecord(u.bb.pIdxKey);
+ }
if( rc!=SQLITE_OK ){
break;
}
u.bb.alreadyExists = (u.bb.res==0);
u.bb.pC->deferredMoveto = 0;
@@ -55469,13 +55455,14 @@
break;
}
/* Opcode: IsUnique P1 P2 P3 P4 *
**
-** Cursor P1 is open on an index. So it has no data and its key consists
-** of a record generated by OP_MakeRecord where the last field is the
-** rowid of the entry that the index refers to.
+** Cursor P1 is open on an index b-tree - that is to say, a btree which
+** no data and where the key are records generated by OP_MakeRecord with
+** the list field being the integer ROWID of the entry that the index
+** entry refers to.
**
** The P3 register contains an integer record number. Call this record
** number R. Register P4 is the first in a set of N contiguous registers
** that make up an unpacked index key that can be used with cursor P1.
** The value of N can be inferred from the cursor. N includes the rowid
@@ -55498,16 +55485,17 @@
#if 0 /* local variables moved into u.bc */
u16 ii;
VdbeCursor *pCx;
BtCursor *pCrsr;
u16 nField;
- Mem *aMem;
+ Mem *aMx;
UnpackedRecord r; /* B-Tree index search key */
i64 R; /* Rowid stored in register P3 */
#endif /* local variables moved into u.bc */
- u.bc.aMem = &p->aMem[pOp->p4.i];
+ pIn3 = &aMem[pOp->p3];
+ u.bc.aMx = &aMem[pOp->p4.i];
/* Assert that the values of parameters P1 and P4 are in range. */
assert( pOp->p4type==P4_INT32 );
assert( pOp->p4.i>0 && pOp->p4.i<=p->nMem );
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
@@ -55519,24 +55507,24 @@
u.bc.pCrsr = u.bc.pCx->pCursor;
/* If any of the values are NULL, take the jump. */
u.bc.nField = u.bc.pCx->pKeyInfo->nField;
for(u.bc.ii=0; u.bc.ii<u.bc.nField; u.bc.ii++){
- if( u.bc.aMem[u.bc.ii].flags & MEM_Null ){
+ if( u.bc.aMx[u.bc.ii].flags & MEM_Null ){
pc = pOp->p2 - 1;
u.bc.pCrsr = 0;
break;
}
}
- assert( (u.bc.aMem[u.bc.nField].flags & MEM_Null)==0 );
+ assert( (u.bc.aMx[u.bc.nField].flags & MEM_Null)==0 );
if( u.bc.pCrsr!=0 ){
/* Populate the index search key. */
u.bc.r.pKeyInfo = u.bc.pCx->pKeyInfo;
u.bc.r.nField = u.bc.nField + 1;
u.bc.r.flags = UNPACKED_PREFIX_SEARCH;
- u.bc.r.aMem = u.bc.aMem;
+ u.bc.r.aMem = u.bc.aMx;
/* Extract the value of u.bc.R from register P3. */
sqlite3VdbeMemIntegerify(pIn3);
u.bc.R = pIn3->u.i;
@@ -55573,10 +55561,11 @@
BtCursor *pCrsr;
int res;
u64 iKey;
#endif /* local variables moved into u.bd */
+ pIn3 = &aMem[pOp->p3];
assert( pIn3->flags & MEM_Int );
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
u.bd.pC = p->apCsr[pOp->p1];
assert( u.bd.pC!=0 );
assert( u.bd.pC->isTable );
@@ -55616,11 +55605,10 @@
*/
case OP_Sequence: { /* out2-prerelease */
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
assert( p->apCsr[pOp->p1]!=0 );
pOut->u.i = p->apCsr[pOp->p1]->seqCount++;
- MemSetTypeFlag(pOut, MEM_Int);
break;
}
/* Opcode: NewRowid P1 P2 P3 * *
@@ -55712,11 +55700,11 @@
assert( pOp->p3<=u.be.pFrame->nMem );
u.be.pMem = &u.be.pFrame->aMem[pOp->p3];
}else{
/* Assert that P3 is a valid memory cell. */
assert( pOp->p3<=p->nMem );
- u.be.pMem = &p->aMem[pOp->p3];
+ u.be.pMem = &aMem[pOp->p3];
}
REGISTER_TRACE(pOp->p3, u.be.pMem);
sqlite3VdbeMemIntegerify(u.be.pMem);
assert( (u.be.pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */
@@ -55755,11 +55743,10 @@
}
u.be.pC->rowidIsValid = 0;
u.be.pC->deferredMoveto = 0;
u.be.pC->cacheStatus = CACHE_STALE;
}
- MemSetTypeFlag(pOut, MEM_Int);
pOut->u.i = u.be.v;
break;
}
/* Opcode: Insert P1 P2 P3 P4 P5
@@ -55818,21 +55805,21 @@
const char *zDb; /* database name - used by the update hook */
const char *zTbl; /* Table name - used by the opdate hook */
int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */
#endif /* local variables moved into u.bf */
- u.bf.pData = &p->aMem[pOp->p2];
+ u.bf.pData = &aMem[pOp->p2];
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
u.bf.pC = p->apCsr[pOp->p1];
assert( u.bf.pC!=0 );
assert( u.bf.pC->pCursor!=0 );
assert( u.bf.pC->pseudoTableReg==0 );
assert( u.bf.pC->isTable );
REGISTER_TRACE(pOp->p2, u.bf.pData);
if( pOp->opcode==OP_Insert ){
- u.bf.pKey = &p->aMem[pOp->p3];
+ u.bf.pKey = &aMem[pOp->p3];
assert( u.bf.pKey->flags & MEM_Int );
REGISTER_TRACE(pOp->p3, u.bf.pKey);
u.bf.iKey = u.bf.pKey->u.i;
}else{
assert( pOp->opcode==OP_InsertInt );
@@ -55980,11 +55967,11 @@
BtCursor *pCrsr;
u32 n;
i64 n64;
#endif /* local variables moved into u.bh */
- pOut = &p->aMem[pOp->p2];
+ pOut = &aMem[pOp->p2];
/* Note that RowKey and RowData are really exactly the same instruction */
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
u.bh.pC = p->apCsr[pOp->p1];
assert( u.bh.pC->isTable || pOp->opcode==OP_RowKey );
@@ -56055,11 +56042,11 @@
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
u.bi.pC = p->apCsr[pOp->p1];
assert( u.bi.pC!=0 );
assert( u.bi.pC->pseudoTableReg==0 );
if( u.bi.pC->nullRow ){
- /* Do nothing so that reg[P2] remains NULL */
+ pOut->flags = MEM_Null;
break;
}else if( u.bi.pC->deferredMoveto ){
u.bi.v = u.bi.pC->movetoTarget;
#ifndef SQLITE_OMIT_VIRTUALTABLE
}else if( u.bi.pC->pVtabCursor ){
@@ -56083,11 +56070,10 @@
rc = sqlite3BtreeKeySize(u.bi.pC->pCursor, &u.bi.v);
assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */
}
}
pOut->u.i = u.bi.v;
- MemSetTypeFlag(pOut, MEM_Int);
break;
}
/* Opcode: NullRow P1 * * * *
**
@@ -56278,10 +56264,11 @@
#endif /* local variables moved into u.bn */
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
u.bn.pC = p->apCsr[pOp->p1];
assert( u.bn.pC!=0 );
+ pIn2 = &aMem[pOp->p2];
assert( pIn2->flags & MEM_Blob );
u.bn.pCrsr = u.bn.pC->pCursor;
if( ALWAYS(u.bn.pCrsr!=0) ){
assert( u.bn.pC->isTable==0 );
rc = ExpandBlob(pIn2);
@@ -56320,11 +56307,11 @@
u.bo.pCrsr = u.bo.pC->pCursor;
if( ALWAYS(u.bo.pCrsr!=0) ){
u.bo.r.pKeyInfo = u.bo.pC->pKeyInfo;
u.bo.r.nField = (u16)pOp->p3;
u.bo.r.flags = 0;
- u.bo.r.aMem = &p->aMem[pOp->p2];
+ u.bo.r.aMem = &aMem[pOp->p2];
rc = sqlite3BtreeMovetoUnpacked(u.bo.pCrsr, &u.bo.r, 0, 0, &u.bo.res);
if( rc==SQLITE_OK && u.bo.res==0 ){
rc = sqlite3BtreeDelete(u.bo.pCrsr);
}
assert( u.bo.pC->deferredMoveto==0 );
@@ -56350,10 +56337,11 @@
assert( pOp->p1>=0 && pOp->p1<p->nCursor );
u.bp.pC = p->apCsr[pOp->p1];
assert( u.bp.pC!=0 );
u.bp.pCrsr = u.bp.pC->pCursor;
+ pOut->flags = MEM_Null;
if( ALWAYS(u.bp.pCrsr!=0) ){
rc = sqlite3VdbeCursorMoveto(u.bp.pC);
if( NEVER(rc) ) goto abort_due_to_error;
assert( u.bp.pC->deferredMoveto==0 );
assert( u.bp.pC->isTable==0 );
@@ -56360,12 +56348,12 @@
if( !u.bp.pC->nullRow ){
rc = sqlite3VdbeIdxRowid(db, u.bp.pCrsr, &u.bp.rowid);
if( rc!=SQLITE_OK ){
goto abort_due_to_error;
}
- MemSetTypeFlag(pOut, MEM_Int);
pOut->u.i = u.bp.rowid;
+ pOut->flags = MEM_Int;
}
}
break;
}
@@ -56393,12 +56381,12 @@
** Otherwise fall through to the next instruction.
**
** If P5 is non-zero then the key value is increased by an epsilon prior
** to the comparison. This makes the opcode work like IdxLE.
*/
-case OP_IdxLT: /* jump, in3 */
-case OP_IdxGE: { /* jump, in3 */
+case OP_IdxLT: /* jump */
+case OP_IdxGE: { /* jump */
#if 0 /* local variables moved into u.bq */
VdbeCursor *pC;
int res;
UnpackedRecord r;
#endif /* local variables moved into u.bq */
@@ -56415,11 +56403,11 @@
if( pOp->p5 ){
u.bq.r.flags = UNPACKED_INCRKEY | UNPACKED_IGNORE_ROWID;
}else{
u.bq.r.flags = UNPACKED_IGNORE_ROWID;
}
- u.bq.r.aMem = &p->aMem[pOp->p3];
+ u.bq.r.aMem = &aMem[pOp->p3];
rc = sqlite3VdbeIdxKeyCompare(u.bq.pC, &u.bq.r, &u.bq.res);
if( pOp->opcode==OP_IdxLT ){
u.bq.res = -u.bq.res;
}else{
assert( pOp->opcode==OP_IdxGE );
@@ -56467,23 +56455,25 @@
}
}
#else
u.br.iCnt = db->activeVdbeCnt;
#endif
+ pOut->flags = MEM_Null;
if( u.br.iCnt>1 ){
rc = SQLITE_LOCKED;
p->errorAction = OE_Abort;
}else{
u.br.iDb = pOp->p3;
assert( u.br.iCnt==1 );
assert( (p->btreeMask & (1<<u.br.iDb))!=0 );
rc = sqlite3BtreeDropTable(db->aDb[u.br.iDb].pBt, pOp->p1, &u.br.iMoved);
- MemSetTypeFlag(pOut, MEM_Int);
+ pOut->flags = MEM_Int;
pOut->u.i = u.br.iMoved;
#ifndef SQLITE_OMIT_AUTOVACUUM
if( rc==SQLITE_OK && u.br.iMoved!=0 ){
sqlite3RootPageMoved(&db->aDb[u.br.iDb], u.br.iMoved, pOp->p1);
+ resetSchemaOnFault = 1;
}
#endif
}
break;
}
@@ -56517,11 +56507,11 @@
db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &u.bs.nChange : 0)
);
if( pOp->p3 ){
p->nChange += u.bs.nChange;
if( pOp->p3>0 ){
- p->aMem[pOp->p3].u.i += u.bs.nChange;
+ aMem[pOp->p3].u.i += u.bs.nChange;
}
}
break;
}
@@ -56566,11 +56556,10 @@
}else{
u.bt.flags = BTREE_ZERODATA;
}
rc = sqlite3BtreeCreateTable(u.bt.pDb->pBt, &u.bt.pgno, u.bt.flags);
pOut->u.i = u.bt.pgno;
- MemSetTypeFlag(pOut, MEM_Int);
break;
}
/* Opcode: ParseSchema P1 P2 * P4 *
**
@@ -56733,14 +56722,14 @@
u.bv.nRoot = pOp->p2;
assert( u.bv.nRoot>0 );
u.bv.aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(u.bv.nRoot+1) );
if( u.bv.aRoot==0 ) goto no_mem;
assert( pOp->p3>0 && pOp->p3<=p->nMem );
- u.bv.pnErr = &p->aMem[pOp->p3];
+ u.bv.pnErr = &aMem[pOp->p3];
assert( (u.bv.pnErr->flags & MEM_Int)!=0 );
assert( (u.bv.pnErr->flags & (MEM_Str|MEM_Blob))==0 );
- pIn1 = &p->aMem[pOp->p1];
+ pIn1 = &aMem[pOp->p1];
for(u.bv.j=0; u.bv.j<u.bv.nRoot; u.bv.j++){
u.bv.aRoot[u.bv.j] = (int)sqlite3VdbeIntValue(&pIn1[u.bv.j]);
}
u.bv.aRoot[u.bv.j] = 0;
assert( pOp->p5<db->nDb );
@@ -56768,53 +56757,43 @@
** Insert the integer value held by register P2 into a boolean index
** held in register P1.
**
** An assertion fails if P2 is not an integer.
*/
-case OP_RowSetAdd: { /* in2 */
-#if 0 /* local variables moved into u.bw */
- Mem *pIdx;
- Mem *pVal;
-#endif /* local variables moved into u.bw */
- assert( pOp->p1>0 && pOp->p1<=p->nMem );
- u.bw.pIdx = &p->aMem[pOp->p1];
- assert( pOp->p2>0 && pOp->p2<=p->nMem );
- u.bw.pVal = &p->aMem[pOp->p2];
- assert( (u.bw.pVal->flags & MEM_Int)!=0 );
- if( (u.bw.pIdx->flags & MEM_RowSet)==0 ){
- sqlite3VdbeMemSetRowSet(u.bw.pIdx);
- if( (u.bw.pIdx->flags & MEM_RowSet)==0 ) goto no_mem;
- }
- sqlite3RowSetInsert(u.bw.pIdx->u.pRowSet, u.bw.pVal->u.i);
+case OP_RowSetAdd: { /* in1, in2 */
+ pIn1 = &aMem[pOp->p1];
+ pIn2 = &aMem[pOp->p2];
+ assert( (pIn2->flags & MEM_Int)!=0 );
+ if( (pIn1->flags & MEM_RowSet)==0 ){
+ sqlite3VdbeMemSetRowSet(pIn1);
+ if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem;
+ }
+ sqlite3RowSetInsert(pIn1->u.pRowSet, pIn2->u.i);
break;
}
/* Opcode: RowSetRead P1 P2 P3 * *
**
** Extract the smallest value from boolean index P1 and put that value into
** register P3. Or, if boolean index P1 is initially empty, leave P3
** unchanged and jump to instruction P2.
*/
-case OP_RowSetRead: { /* jump, out3 */
-#if 0 /* local variables moved into u.bx */
- Mem *pIdx;
+case OP_RowSetRead: { /* jump, in1, out3 */
+#if 0 /* local variables moved into u.bw */
i64 val;
-#endif /* local variables moved into u.bx */
- assert( pOp->p1>0 && pOp->p1<=p->nMem );
+#endif /* local variables moved into u.bw */
CHECK_FOR_INTERRUPT;
- u.bx.pIdx = &p->aMem[pOp->p1];
- pOut = &p->aMem[pOp->p3];
- if( (u.bx.pIdx->flags & MEM_RowSet)==0
- || sqlite3RowSetNext(u.bx.pIdx->u.pRowSet, &u.bx.val)==0
+ pIn1 = &aMem[pOp->p1];
+ if( (pIn1->flags & MEM_RowSet)==0
+ || sqlite3RowSetNext(pIn1->u.pRowSet, &u.bw.val)==0
){
/* The boolean index is empty */
- sqlite3VdbeMemSetNull(u.bx.pIdx);
+ sqlite3VdbeMemSetNull(pIn1);
pc = pOp->p2 - 1;
}else{
/* A value was pulled from the index */
- assert( pOp->p3>0 && pOp->p3<=p->nMem );
- sqlite3VdbeMemSetInt64(pOut, u.bx.val);
+ sqlite3VdbeMemSetInt64(&aMem[pOp->p3], u.bw.val);
}
break;
}
/* Opcode: RowSetTest P1 P2 P3 P4
@@ -56839,16 +56818,18 @@
** inserted, there is no need to search to see if the same value was
** previously inserted as part of set X (only if it was previously
** inserted as part of some other set).
*/
case OP_RowSetTest: { /* jump, in1, in3 */
-#if 0 /* local variables moved into u.by */
+#if 0 /* local variables moved into u.bx */
int iSet;
int exists;
-#endif /* local variables moved into u.by */
-
- u.by.iSet = pOp->p4.i;
+#endif /* local variables moved into u.bx */
+
+ pIn1 = &aMem[pOp->p1];
+ pIn3 = &aMem[pOp->p3];
+ u.bx.iSet = pOp->p4.i;
assert( pIn3->flags&MEM_Int );
/* If there is anything other than a rowset object in memory cell P1,
** delete it now and initialize P1 with an empty rowset
*/
@@ -56856,21 +56837,21 @@
sqlite3VdbeMemSetRowSet(pIn1);
if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem;
}
assert( pOp->p4type==P4_INT32 );
- assert( u.by.iSet==-1 || u.by.iSet>=0 );
- if( u.by.iSet ){
- u.by.exists = sqlite3RowSetTest(pIn1->u.pRowSet,
- (u8)(u.by.iSet>=0 ? u.by.iSet & 0xf : 0xff),
+ assert( u.bx.iSet==-1 || u.bx.iSet>=0 );
+ if( u.bx.iSet ){
+ u.bx.exists = sqlite3RowSetTest(pIn1->u.pRowSet,
+ (u8)(u.bx.iSet>=0 ? u.bx.iSet & 0xf : 0xff),
pIn3->u.i);
- if( u.by.exists ){
+ if( u.bx.exists ){
pc = pOp->p2 - 1;
break;
}
}
- if( u.by.iSet>=0 ){
+ if( u.bx.iSet>=0 ){
sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i);
}
break;
}
@@ -56889,24 +56870,24 @@
** memory required by the sub-vdbe at runtime.
**
** P4 is a pointer to the VM containing the trigger program.
*/
case OP_Program: { /* jump */
-#if 0 /* local variables moved into u.bz */
+#if 0 /* local variables moved into u.by */
int nMem; /* Number of memory registers for sub-program */
int nByte; /* Bytes of runtime space required for sub-program */
Mem *pRt; /* Register to allocate runtime space */
Mem *pMem; /* Used to iterate through memory cells */
Mem *pEnd; /* Last memory cell in new array */
VdbeFrame *pFrame; /* New vdbe frame to execute in */
SubProgram *pProgram; /* Sub-program to execute */
void *t; /* Token identifying trigger */
-#endif /* local variables moved into u.bz */
-
- u.bz.pProgram = pOp->p4.pProgram;
- u.bz.pRt = &p->aMem[pOp->p3];
- assert( u.bz.pProgram->nOp>0 );
+#endif /* local variables moved into u.by */
+
+ u.by.pProgram = pOp->p4.pProgram;
+ u.by.pRt = &aMem[pOp->p3];
+ assert( u.by.pProgram->nOp>0 );
/* If the p5 flag is clear, then recursive invocation of triggers is
** disabled for backwards compatibility (p5 is set if this sub-program
** is really a trigger, not a foreign key action, and the flag set
** and cleared by the "PRAGMA recursive_triggers" command is clear).
@@ -56916,79 +56897,79 @@
** SubProgram (if the trigger may be executed with more than one different
** ON CONFLICT algorithm). SubProgram structures associated with a
** single trigger all have the same value for the SubProgram.token
** variable. */
if( pOp->p5 ){
- u.bz.t = u.bz.pProgram->token;
- for(u.bz.pFrame=p->pFrame; u.bz.pFrame && u.bz.pFrame->token!=u.bz.t; u.bz.pFrame=u.bz.pFrame->pParent);
- if( u.bz.pFrame ) break;
+ u.by.t = u.by.pProgram->token;
+ for(u.by.pFrame=p->pFrame; u.by.pFrame && u.by.pFrame->token!=u.by.t; u.by.pFrame=u.by.pFrame->pParent);
+ if( u.by.pFrame ) break;
}
if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){
rc = SQLITE_ERROR;
sqlite3SetString(&p->zErrMsg, db, "too many levels of trigger recursion");
break;
}
- /* Register u.bz.pRt is used to store the memory required to save the state
+ /* Register u.by.pRt is used to store the memory required to save the state
** of the current program, and the memory required at runtime to execute
- ** the trigger program. If this trigger has been fired before, then u.bz.pRt
+ ** the trigger program. If this trigger has been fired before, then u.by.pRt
** is already allocated. Otherwise, it must be initialized. */
- if( (u.bz.pRt->flags&MEM_Frame)==0 ){
+ if( (u.by.pRt->flags&MEM_Frame)==0 ){
/* SubProgram.nMem is set to the number of memory cells used by the
** program stored in SubProgram.aOp. As well as these, one memory
** cell is required for each cursor used by the program. Set local
- ** variable u.bz.nMem (and later, VdbeFrame.nChildMem) to this value.
- */
- u.bz.nMem = u.bz.pProgram->nMem + u.bz.pProgram->nCsr;
- u.bz.nByte = ROUND8(sizeof(VdbeFrame))
- + u.bz.nMem * sizeof(Mem)
- + u.bz.pProgram->nCsr * sizeof(VdbeCursor *);
- u.bz.pFrame = sqlite3DbMallocZero(db, u.bz.nByte);
- if( !u.bz.pFrame ){
+ ** variable u.by.nMem (and later, VdbeFrame.nChildMem) to this value.
+ */
+ u.by.nMem = u.by.pProgram->nMem + u.by.pProgram->nCsr;
+ u.by.nByte = ROUND8(sizeof(VdbeFrame))
+ + u.by.nMem * sizeof(Mem)
+ + u.by.pProgram->nCsr * sizeof(VdbeCursor *);
+ u.by.pFrame = sqlite3DbMallocZero(db, u.by.nByte);
+ if( !u.by.pFrame ){
goto no_mem;
}
- sqlite3VdbeMemRelease(u.bz.pRt);
- u.bz.pRt->flags = MEM_Frame;
- u.bz.pRt->u.pFrame = u.bz.pFrame;
-
- u.bz.pFrame->v = p;
- u.bz.pFrame->nChildMem = u.bz.nMem;
- u.bz.pFrame->nChildCsr = u.bz.pProgram->nCsr;
- u.bz.pFrame->pc = pc;
- u.bz.pFrame->aMem = p->aMem;
- u.bz.pFrame->nMem = p->nMem;
- u.bz.pFrame->apCsr = p->apCsr;
- u.bz.pFrame->nCursor = p->nCursor;
- u.bz.pFrame->aOp = p->aOp;
- u.bz.pFrame->nOp = p->nOp;
- u.bz.pFrame->token = u.bz.pProgram->token;
-
- u.bz.pEnd = &VdbeFrameMem(u.bz.pFrame)[u.bz.pFrame->nChildMem];
- for(u.bz.pMem=VdbeFrameMem(u.bz.pFrame); u.bz.pMem!=u.bz.pEnd; u.bz.pMem++){
- u.bz.pMem->flags = MEM_Null;
- u.bz.pMem->db = db;
- }
- }else{
- u.bz.pFrame = u.bz.pRt->u.pFrame;
- assert( u.bz.pProgram->nMem+u.bz.pProgram->nCsr==u.bz.pFrame->nChildMem );
- assert( u.bz.pProgram->nCsr==u.bz.pFrame->nChildCsr );
- assert( pc==u.bz.pFrame->pc );
+ sqlite3VdbeMemRelease(u.by.pRt);
+ u.by.pRt->flags = MEM_Frame;
+ u.by.pRt->u.pFrame = u.by.pFrame;
+
+ u.by.pFrame->v = p;
+ u.by.pFrame->nChildMem = u.by.nMem;
+ u.by.pFrame->nChildCsr = u.by.pProgram->nCsr;
+ u.by.pFrame->pc = pc;
+ u.by.pFrame->aMem = p->aMem;
+ u.by.pFrame->nMem = p->nMem;
+ u.by.pFrame->apCsr = p->apCsr;
+ u.by.pFrame->nCursor = p->nCursor;
+ u.by.pFrame->aOp = p->aOp;
+ u.by.pFrame->nOp = p->nOp;
+ u.by.pFrame->token = u.by.pProgram->token;
+
+ u.by.pEnd = &VdbeFrameMem(u.by.pFrame)[u.by.pFrame->nChildMem];
+ for(u.by.pMem=VdbeFrameMem(u.by.pFrame); u.by.pMem!=u.by.pEnd; u.by.pMem++){
+ u.by.pMem->flags = MEM_Null;
+ u.by.pMem->db = db;
+ }
+ }else{
+ u.by.pFrame = u.by.pRt->u.pFrame;
+ assert( u.by.pProgram->nMem+u.by.pProgram->nCsr==u.by.pFrame->nChildMem );
+ assert( u.by.pProgram->nCsr==u.by.pFrame->nChildCsr );
+ assert( pc==u.by.pFrame->pc );
}
p->nFrame++;
- u.bz.pFrame->pParent = p->pFrame;
- u.bz.pFrame->lastRowid = db->lastRowid;
- u.bz.pFrame->nChange = p->nChange;
+ u.by.pFrame->pParent = p->pFrame;
+ u.by.pFrame->lastRowid = db->lastRowid;
+ u.by.pFrame->nChange = p->nChange;
p->nChange = 0;
- p->pFrame = u.bz.pFrame;
- p->aMem = &VdbeFrameMem(u.bz.pFrame)[-1];
- p->nMem = u.bz.pFrame->nChildMem;
- p->nCursor = (u16)u.bz.pFrame->nChildCsr;
- p->apCsr = (VdbeCursor **)&p->aMem[p->nMem+1];
- p->aOp = u.bz.pProgram->aOp;
- p->nOp = u.bz.pProgram->nOp;
+ p->pFrame = u.by.pFrame;
+ p->aMem = aMem = &VdbeFrameMem(u.by.pFrame)[-1];
+ p->nMem = u.by.pFrame->nChildMem;
+ p->nCursor = (u16)u.by.pFrame->nChildCsr;
+ p->apCsr = (VdbeCursor **)&aMem[p->nMem+1];
+ p->aOp = aOp = u.by.pProgram->aOp;
+ p->nOp = u.by.pProgram->nOp;
pc = -1;
break;
}
@@ -57003,17 +56984,17 @@
** The address of the cell in the parent frame is determined by adding
** the value of the P1 argument to the value of the P1 argument to the
** calling OP_Program instruction.
*/
case OP_Param: { /* out2-prerelease */
-#if 0 /* local variables moved into u.ca */
+#if 0 /* local variables moved into u.bz */
VdbeFrame *pFrame;
Mem *pIn;
-#endif /* local variables moved into u.ca */
- u.ca.pFrame = p->pFrame;
- u.ca.pIn = &u.ca.pFrame->aMem[pOp->p1 + u.ca.pFrame->aOp[u.ca.pFrame->pc].p1];
- sqlite3VdbeMemShallowCopy(pOut, u.ca.pIn, MEM_Ephem);
+#endif /* local variables moved into u.bz */
+ u.bz.pFrame = p->pFrame;
+ u.bz.pIn = &u.bz.pFrame->aMem[pOp->p1 + u.bz.pFrame->aOp[u.bz.pFrame->pc].p1];
+ sqlite3VdbeMemShallowCopy(pOut, u.bz.pIn, MEM_Ephem);
break;
}
#endif /* #ifndef SQLITE_OMIT_TRIGGER */
@@ -57065,24 +57046,25 @@
**
** This instruction throws an error if the memory cell is not initially
** an integer.
*/
case OP_MemMax: { /* in2 */
-#if 0 /* local variables moved into u.cb */
+#if 0 /* local variables moved into u.ca */
Mem *pIn1;
VdbeFrame *pFrame;
-#endif /* local variables moved into u.cb */
+#endif /* local variables moved into u.ca */
if( p->pFrame ){
- for(u.cb.pFrame=p->pFrame; u.cb.pFrame->pParent; u.cb.pFrame=u.cb.pFrame->pParent);
- u.cb.pIn1 = &u.cb.pFrame->aMem[pOp->p1];
- }else{
- u.cb.pIn1 = &p->aMem[pOp->p1];
- }
- sqlite3VdbeMemIntegerify(u.cb.pIn1);
+ for(u.ca.pFrame=p->pFrame; u.ca.pFrame->pParent; u.ca.pFrame=u.ca.pFrame->pParent);
+ u.ca.pIn1 = &u.ca.pFrame->aMem[pOp->p1];
+ }else{
+ u.ca.pIn1 = &aMem[pOp->p1];
+ }
+ sqlite3VdbeMemIntegerify(u.ca.pIn1);
+ pIn2 = &aMem[pOp->p2];
sqlite3VdbeMemIntegerify(pIn2);
- if( u.cb.pIn1->u.i<pIn2->u.i){
- u.cb.pIn1->u.i = pIn2->u.i;
+ if( u.ca.pIn1->u.i<pIn2->u.i){
+ u.ca.pIn1->u.i = pIn2->u.i;
}
break;
}
#endif /* SQLITE_OMIT_AUTOINCREMENT */
@@ -57092,10 +57074,11 @@
**
** It is illegal to use this instruction on a register that does
** not contain an integer. An assertion fault will result if you try.
*/
case OP_IfPos: { /* jump, in1 */
+ pIn1 = &aMem[pOp->p1];
assert( pIn1->flags&MEM_Int );
if( pIn1->u.i>0 ){
pc = pOp->p2 - 1;
}
break;
@@ -57107,26 +57090,30 @@
**
** It is illegal to use this instruction on a register that does
** not contain an integer. An assertion fault will result if you try.
*/
case OP_IfNeg: { /* jump, in1 */
+ pIn1 = &aMem[pOp->p1];
assert( pIn1->flags&MEM_Int );
if( pIn1->u.i<0 ){
pc = pOp->p2 - 1;
}
break;
}
-/* Opcode: IfZero P1 P2 * * *
-**
-** If the value of register P1 is exactly 0, jump to P2.
+/* Opcode: IfZero P1 P2 P3 * *
+**
+** The register P1 must contain an integer. Add literal P3 to the
+** value in register P1. If the result is exactly 0, jump to P2.
**
** It is illegal to use this instruction on a register that does
** not contain an integer. An assertion fault will result if you try.
*/
case OP_IfZero: { /* jump, in1 */
+ pIn1 = &aMem[pOp->p1];
assert( pIn1->flags&MEM_Int );
+ pIn1->u.i += pOp->p3;
if( pIn1->u.i==0 ){
pc = pOp->p2 - 1;
}
break;
}
@@ -57140,51 +57127,51 @@
**
** The P5 arguments are taken from register P2 and its
** successors.
*/
case OP_AggStep: {
-#if 0 /* local variables moved into u.cc */
+#if 0 /* local variables moved into u.cb */
int n;
int i;
Mem *pMem;
Mem *pRec;
sqlite3_context ctx;
sqlite3_value **apVal;
-#endif /* local variables moved into u.cc */
-
- u.cc.n = pOp->p5;
- assert( u.cc.n>=0 );
- u.cc.pRec = &p->aMem[pOp->p2];
- u.cc.apVal = p->apArg;
- assert( u.cc.apVal || u.cc.n==0 );
- for(u.cc.i=0; u.cc.i<u.cc.n; u.cc.i++, u.cc.pRec++){
- u.cc.apVal[u.cc.i] = u.cc.pRec;
- sqlite3VdbeMemStoreType(u.cc.pRec);
- }
- u.cc.ctx.pFunc = pOp->p4.pFunc;
+#endif /* local variables moved into u.cb */
+
+ u.cb.n = pOp->p5;
+ assert( u.cb.n>=0 );
+ u.cb.pRec = &aMem[pOp->p2];
+ u.cb.apVal = p->apArg;
+ assert( u.cb.apVal || u.cb.n==0 );
+ for(u.cb.i=0; u.cb.i<u.cb.n; u.cb.i++, u.cb.pRec++){
+ u.cb.apVal[u.cb.i] = u.cb.pRec;
+ sqlite3VdbeMemStoreType(u.cb.pRec);
+ }
+ u.cb.ctx.pFunc = pOp->p4.pFunc;
assert( pOp->p3>0 && pOp->p3<=p->nMem );
- u.cc.ctx.pMem = u.cc.pMem = &p->aMem[pOp->p3];
- u.cc.pMem->n++;
- u.cc.ctx.s.flags = MEM_Null;
- u.cc.ctx.s.z = 0;
- u.cc.ctx.s.zMalloc = 0;
- u.cc.ctx.s.xDel = 0;
- u.cc.ctx.s.db = db;
- u.cc.ctx.isError = 0;
- u.cc.ctx.pColl = 0;
- if( u.cc.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
+ u.cb.ctx.pMem = u.cb.pMem = &aMem[pOp->p3];
+ u.cb.pMem->n++;
+ u.cb.ctx.s.flags = MEM_Null;
+ u.cb.ctx.s.z = 0;
+ u.cb.ctx.s.zMalloc = 0;
+ u.cb.ctx.s.xDel = 0;
+ u.cb.ctx.s.db = db;
+ u.cb.ctx.isError = 0;
+ u.cb.ctx.pColl = 0;
+ if( u.cb.ctx.pFunc->flags & SQLITE_FUNC_NEEDCOLL ){
assert( pOp>p->aOp );
assert( pOp[-1].p4type==P4_COLLSEQ );
assert( pOp[-1].opcode==OP_CollSeq );
- u.cc.ctx.pColl = pOp[-1].p4.pColl;
- }
- (u.cc.ctx.pFunc->xStep)(&u.cc.ctx, u.cc.n, u.cc.apVal);
- if( u.cc.ctx.isError ){
- sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cc.ctx.s));
- rc = u.cc.ctx.isError;
- }
- sqlite3VdbeMemRelease(&u.cc.ctx.s);
+ u.cb.ctx.pColl = pOp[-1].p4.pColl;
+ }
+ (u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal);
+ if( u.cb.ctx.isError ){
+ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cb.ctx.s));
+ rc = u.cb.ctx.isError;
+ }
+ sqlite3VdbeMemRelease(&u.cb.ctx.s);
break;
}
/* Opcode: AggFinal P1 P2 * P4 *
**
@@ -57197,23 +57184,23 @@
** functions that can take varying numbers of arguments. The
** P4 argument is only needed for the degenerate case where
** the step function was not previously called.
*/
case OP_AggFinal: {
-#if 0 /* local variables moved into u.cd */
+#if 0 /* local variables moved into u.cc */
Mem *pMem;
-#endif /* local variables moved into u.cd */
+#endif /* local variables moved into u.cc */
assert( pOp->p1>0 && pOp->p1<=p->nMem );
- u.cd.pMem = &p->aMem[pOp->p1];
- assert( (u.cd.pMem->flags & ~(MEM_Null|MEM_Agg))==0 );
- rc = sqlite3VdbeMemFinalize(u.cd.pMem, pOp->p4.pFunc);
+ u.cc.pMem = &aMem[pOp->p1];
+ assert( (u.cc.pMem->flags & ~(MEM_Null|MEM_Agg))==0 );
+ rc = sqlite3VdbeMemFinalize(u.cc.pMem, pOp->p4.pFunc);
if( rc ){
- sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(u.cd.pMem));
- }
- sqlite3VdbeChangeEncoding(u.cd.pMem, encoding);
- UPDATE_MAX_BLOBSIZE(u.cd.pMem);
- if( sqlite3VdbeMemTooBig(u.cd.pMem) ){
+ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(u.cc.pMem));
+ }
+ sqlite3VdbeChangeEncoding(u.cc.pMem, encoding);
+ UPDATE_MAX_BLOBSIZE(u.cc.pMem);
+ if( sqlite3VdbeMemTooBig(u.cc.pMem) ){
goto too_big;
}
break;
}
@@ -57239,18 +57226,18 @@
** Perform a single step of the incremental vacuum procedure on
** the P1 database. If the vacuum has finished, jump to instruction
** P2. Otherwise, fall through to the next instruction.
*/
case OP_IncrVacuum: { /* jump */
-#if 0 /* local variables moved into u.ce */
+#if 0 /* local variables moved into u.cd */
Btree *pBt;
-#endif /* local variables moved into u.ce */
+#endif /* local variables moved into u.cd */
assert( pOp->p1>=0 && pOp->p1<db->nDb );
assert( (p->btreeMask & (1<<pOp->p1))!=0 );
- u.ce.pBt = db->aDb[pOp->p1].pBt;
- rc = sqlite3BtreeIncrVacuum(u.ce.pBt);
+ u.cd.pBt = db->aDb[pOp->p1].pBt;
+ rc = sqlite3BtreeIncrVacuum(u.cd.pBt);
if( rc==SQLITE_DONE ){
pc = pOp->p2 - 1;
rc = SQLITE_OK;
}
break;
@@ -57316,19 +57303,19 @@
** Also, whether or not P4 is set, check that this is not being called from
** within a callback to a virtual table xSync() method. If it is, the error
** code will be set to SQLITE_LOCKED.
*/
case OP_VBegin: {
-#if 0 /* local variables moved into u.cf */
+#if 0 /* local variables moved into u.ce */
VTable *pVTab;
-#endif /* local variables moved into u.cf */
- u.cf.pVTab = pOp->p4.pVtab;
- rc = sqlite3VtabBegin(db, u.cf.pVTab);
- if( u.cf.pVTab ){
+#endif /* local variables moved into u.ce */
+ u.ce.pVTab = pOp->p4.pVtab;
+ rc = sqlite3VtabBegin(db, u.ce.pVTab);
+ if( u.ce.pVTab ){
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.cf.pVTab->pVtab->zErrMsg;
- u.cf.pVTab->pVtab->zErrMsg = 0;
+ p->zErrMsg = u.ce.pVTab->pVtab->zErrMsg;
+ u.ce.pVTab->pVtab->zErrMsg = 0;
}
break;
}
#endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -57364,40 +57351,40 @@
** P4 is a pointer to a virtual table object, an sqlite3_vtab structure.
** P1 is a cursor number. This opcode opens a cursor to the virtual
** table and stores that cursor in P1.
*/
case OP_VOpen: {
-#if 0 /* local variables moved into u.cg */
+#if 0 /* local variables moved into u.cf */
VdbeCursor *pCur;
sqlite3_vtab_cursor *pVtabCursor;
sqlite3_vtab *pVtab;
sqlite3_module *pModule;
-#endif /* local variables moved into u.cg */
-
- u.cg.pCur = 0;
- u.cg.pVtabCursor = 0;
- u.cg.pVtab = pOp->p4.pVtab->pVtab;
- u.cg.pModule = (sqlite3_module *)u.cg.pVtab->pModule;
- assert(u.cg.pVtab && u.cg.pModule);
+#endif /* local variables moved into u.cf */
+
+ u.cf.pCur = 0;
+ u.cf.pVtabCursor = 0;
+ u.cf.pVtab = pOp->p4.pVtab->pVtab;
+ u.cf.pModule = (sqlite3_module *)u.cf.pVtab->pModule;
+ assert(u.cf.pVtab && u.cf.pModule);
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
- rc = u.cg.pModule->xOpen(u.cg.pVtab, &u.cg.pVtabCursor);
+ rc = u.cf.pModule->xOpen(u.cf.pVtab, &u.cf.pVtabCursor);
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.cg.pVtab->zErrMsg;
- u.cg.pVtab->zErrMsg = 0;
+ p->zErrMsg = u.cf.pVtab->zErrMsg;
+ u.cf.pVtab->zErrMsg = 0;
if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
if( SQLITE_OK==rc ){
/* Initialize sqlite3_vtab_cursor base class */
- u.cg.pVtabCursor->pVtab = u.cg.pVtab;
+ u.cf.pVtabCursor->pVtab = u.cf.pVtab;
/* Initialise vdbe cursor object */
- u.cg.pCur = allocateCursor(p, pOp->p1, 0, -1, 0);
- if( u.cg.pCur ){
- u.cg.pCur->pVtabCursor = u.cg.pVtabCursor;
- u.cg.pCur->pModule = u.cg.pVtabCursor->pVtab->pModule;
+ u.cf.pCur = allocateCursor(p, pOp->p1, 0, -1, 0);
+ if( u.cf.pCur ){
+ u.cf.pCur->pVtabCursor = u.cf.pVtabCursor;
+ u.cf.pCur->pModule = u.cf.pVtabCursor->pVtab->pModule;
}else{
db->mallocFailed = 1;
- u.cg.pModule->xClose(u.cg.pVtabCursor);
+ u.cf.pModule->xClose(u.cf.pVtabCursor);
}
}
break;
}
#endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -57420,11 +57407,11 @@
** xFilter as argv. Register P3+2 becomes argv[0] when passed to xFilter.
**
** A jump is made to P2 if the result set after filtering would be empty.
*/
case OP_VFilter: { /* jump */
-#if 0 /* local variables moved into u.ch */
+#if 0 /* local variables moved into u.cg */
int nArg;
int iQuery;
const sqlite3_module *pModule;
Mem *pQuery;
Mem *pArgc;
@@ -57432,52 +57419,52 @@
sqlite3_vtab *pVtab;
VdbeCursor *pCur;
int res;
int i;
Mem **apArg;
-#endif /* local variables moved into u.ch */
-
- u.ch.pQuery = &p->aMem[pOp->p3];
- u.ch.pArgc = &u.ch.pQuery[1];
- u.ch.pCur = p->apCsr[pOp->p1];
- REGISTER_TRACE(pOp->p3, u.ch.pQuery);
- assert( u.ch.pCur->pVtabCursor );
- u.ch.pVtabCursor = u.ch.pCur->pVtabCursor;
- u.ch.pVtab = u.ch.pVtabCursor->pVtab;
- u.ch.pModule = u.ch.pVtab->pModule;
+#endif /* local variables moved into u.cg */
+
+ u.cg.pQuery = &aMem[pOp->p3];
+ u.cg.pArgc = &u.cg.pQuery[1];
+ u.cg.pCur = p->apCsr[pOp->p1];
+ REGISTER_TRACE(pOp->p3, u.cg.pQuery);
+ assert( u.cg.pCur->pVtabCursor );
+ u.cg.pVtabCursor = u.cg.pCur->pVtabCursor;
+ u.cg.pVtab = u.cg.pVtabCursor->pVtab;
+ u.cg.pModule = u.cg.pVtab->pModule;
/* Grab the index number and argc parameters */
- assert( (u.ch.pQuery->flags&MEM_Int)!=0 && u.ch.pArgc->flags==MEM_Int );
- u.ch.nArg = (int)u.ch.pArgc->u.i;
- u.ch.iQuery = (int)u.ch.pQuery->u.i;
+ assert( (u.cg.pQuery->flags&MEM_Int)!=0 && u.cg.pArgc->flags==MEM_Int );
+ u.cg.nArg = (int)u.cg.pArgc->u.i;
+ u.cg.iQuery = (int)u.cg.pQuery->u.i;
/* Invoke the xFilter method */
{
- u.ch.res = 0;
- u.ch.apArg = p->apArg;
- for(u.ch.i = 0; u.ch.i<u.ch.nArg; u.ch.i++){
- u.ch.apArg[u.ch.i] = &u.ch.pArgc[u.ch.i+1];
- sqlite3VdbeMemStoreType(u.ch.apArg[u.ch.i]);
+ u.cg.res = 0;
+ u.cg.apArg = p->apArg;
+ for(u.cg.i = 0; u.cg.i<u.cg.nArg; u.cg.i++){
+ u.cg.apArg[u.cg.i] = &u.cg.pArgc[u.cg.i+1];
+ sqlite3VdbeMemStoreType(u.cg.apArg[u.cg.i]);
}
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
p->inVtabMethod = 1;
- rc = u.ch.pModule->xFilter(u.ch.pVtabCursor, u.ch.iQuery, pOp->p4.z, u.ch.nArg, u.ch.apArg);
+ rc = u.cg.pModule->xFilter(u.cg.pVtabCursor, u.cg.iQuery, pOp->p4.z, u.cg.nArg, u.cg.apArg);
p->inVtabMethod = 0;
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.ch.pVtab->zErrMsg;
- u.ch.pVtab->zErrMsg = 0;
- if( rc==SQLITE_OK ){
- u.ch.res = u.ch.pModule->xEof(u.ch.pVtabCursor);
+ p->zErrMsg = u.cg.pVtab->zErrMsg;
+ u.cg.pVtab->zErrMsg = 0;
+ if( rc==SQLITE_OK ){
+ u.cg.res = u.cg.pModule->xEof(u.cg.pVtabCursor);
}
if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
- if( u.ch.res ){
+ if( u.cg.res ){
pc = pOp->p2 - 1;
}
}
- u.ch.pCur->nullRow = 0;
+ u.cg.pCur->nullRow = 0;
break;
}
#endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -57487,60 +57474,60 @@
** Store the value of the P2-th column of
** the row of the virtual-table that the
** P1 cursor is pointing to into register P3.
*/
case OP_VColumn: {
-#if 0 /* local variables moved into u.ci */
+#if 0 /* local variables moved into u.ch */
sqlite3_vtab *pVtab;
const sqlite3_module *pModule;
Mem *pDest;
sqlite3_context sContext;
-#endif /* local variables moved into u.ci */
+#endif /* local variables moved into u.ch */
VdbeCursor *pCur = p->apCsr[pOp->p1];
assert( pCur->pVtabCursor );
assert( pOp->p3>0 && pOp->p3<=p->nMem );
- u.ci.pDest = &p->aMem[pOp->p3];
+ u.ch.pDest = &aMem[pOp->p3];
if( pCur->nullRow ){
- sqlite3VdbeMemSetNull(u.ci.pDest);
+ sqlite3VdbeMemSetNull(u.ch.pDest);
break;
}
- u.ci.pVtab = pCur->pVtabCursor->pVtab;
- u.ci.pModule = u.ci.pVtab->pModule;
- assert( u.ci.pModule->xColumn );
- memset(&u.ci.sContext, 0, sizeof(u.ci.sContext));
+ u.ch.pVtab = pCur->pVtabCursor->pVtab;
+ u.ch.pModule = u.ch.pVtab->pModule;
+ assert( u.ch.pModule->xColumn );
+ memset(&u.ch.sContext, 0, sizeof(u.ch.sContext));
/* The output cell may already have a buffer allocated. Move
- ** the current contents to u.ci.sContext.s so in case the user-function
+ ** the current contents to u.ch.sContext.s so in case the user-function
** can use the already allocated buffer instead of allocating a
** new one.
*/
- sqlite3VdbeMemMove(&u.ci.sContext.s, u.ci.pDest);
- MemSetTypeFlag(&u.ci.sContext.s, MEM_Null);
+ sqlite3VdbeMemMove(&u.ch.sContext.s, u.ch.pDest);
+ MemSetTypeFlag(&u.ch.sContext.s, MEM_Null);
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
- rc = u.ci.pModule->xColumn(pCur->pVtabCursor, &u.ci.sContext, pOp->p2);
+ rc = u.ch.pModule->xColumn(pCur->pVtabCursor, &u.ch.sContext, pOp->p2);
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.ci.pVtab->zErrMsg;
- u.ci.pVtab->zErrMsg = 0;
- if( u.ci.sContext.isError ){
- rc = u.ci.sContext.isError;
+ p->zErrMsg = u.ch.pVtab->zErrMsg;
+ u.ch.pVtab->zErrMsg = 0;
+ if( u.ch.sContext.isError ){
+ rc = u.ch.sContext.isError;
}
/* Copy the result of the function to the P3 register. We
** do this regardless of whether or not an error occurred to ensure any
- ** dynamic allocation in u.ci.sContext.s (a Mem struct) is released.
- */
- sqlite3VdbeChangeEncoding(&u.ci.sContext.s, encoding);
- REGISTER_TRACE(pOp->p3, u.ci.pDest);
- sqlite3VdbeMemMove(u.ci.pDest, &u.ci.sContext.s);
- UPDATE_MAX_BLOBSIZE(u.ci.pDest);
+ ** dynamic allocation in u.ch.sContext.s (a Mem struct) is released.
+ */
+ sqlite3VdbeChangeEncoding(&u.ch.sContext.s, encoding);
+ REGISTER_TRACE(pOp->p3, u.ch.pDest);
+ sqlite3VdbeMemMove(u.ch.pDest, &u.ch.sContext.s);
+ UPDATE_MAX_BLOBSIZE(u.ch.pDest);
if( sqlite3SafetyOn(db) ){
goto abort_due_to_misuse;
}
- if( sqlite3VdbeMemTooBig(u.ci.pDest) ){
+ if( sqlite3VdbeMemTooBig(u.ch.pDest) ){
goto too_big;
}
break;
}
#endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -57551,46 +57538,46 @@
** Advance virtual table P1 to the next row in its result set and
** jump to instruction P2. Or, if the virtual table has reached
** the end of its result set, then fall through to the next instruction.
*/
case OP_VNext: { /* jump */
-#if 0 /* local variables moved into u.cj */
+#if 0 /* local variables moved into u.ci */
sqlite3_vtab *pVtab;
const sqlite3_module *pModule;
int res;
VdbeCursor *pCur;
-#endif /* local variables moved into u.cj */
-
- u.cj.res = 0;
- u.cj.pCur = p->apCsr[pOp->p1];
- assert( u.cj.pCur->pVtabCursor );
- if( u.cj.pCur->nullRow ){
+#endif /* local variables moved into u.ci */
+
+ u.ci.res = 0;
+ u.ci.pCur = p->apCsr[pOp->p1];
+ assert( u.ci.pCur->pVtabCursor );
+ if( u.ci.pCur->nullRow ){
break;
}
- u.cj.pVtab = u.cj.pCur->pVtabCursor->pVtab;
- u.cj.pModule = u.cj.pVtab->pModule;
- assert( u.cj.pModule->xNext );
+ u.ci.pVtab = u.ci.pCur->pVtabCursor->pVtab;
+ u.ci.pModule = u.ci.pVtab->pModule;
+ assert( u.ci.pModule->xNext );
/* Invoke the xNext() method of the module. There is no way for the
** underlying implementation to return an error if one occurs during
** xNext(). Instead, if an error occurs, true is returned (indicating that
** data is available) and the error code returned when xColumn or
** some other method is next invoked on the save virtual table cursor.
*/
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
p->inVtabMethod = 1;
- rc = u.cj.pModule->xNext(u.cj.pCur->pVtabCursor);
+ rc = u.ci.pModule->xNext(u.ci.pCur->pVtabCursor);
p->inVtabMethod = 0;
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.cj.pVtab->zErrMsg;
- u.cj.pVtab->zErrMsg = 0;
- if( rc==SQLITE_OK ){
- u.cj.res = u.cj.pModule->xEof(u.cj.pCur->pVtabCursor);
+ p->zErrMsg = u.ci.pVtab->zErrMsg;
+ u.ci.pVtab->zErrMsg = 0;
+ if( rc==SQLITE_OK ){
+ u.ci.res = u.ci.pModule->xEof(u.ci.pCur->pVtabCursor);
}
if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
- if( !u.cj.res ){
+ if( !u.ci.res ){
/* If there is data, jump to P2 */
pc = pOp->p2 - 1;
}
break;
}
@@ -57602,25 +57589,25 @@
** P4 is a pointer to a virtual table object, an sqlite3_vtab structure.
** This opcode invokes the corresponding xRename method. The value
** in register P1 is passed as the zName argument to the xRename method.
*/
case OP_VRename: {
-#if 0 /* local variables moved into u.ck */
+#if 0 /* local variables moved into u.cj */
sqlite3_vtab *pVtab;
Mem *pName;
-#endif /* local variables moved into u.ck */
-
- u.ck.pVtab = pOp->p4.pVtab->pVtab;
- u.ck.pName = &p->aMem[pOp->p1];
- assert( u.ck.pVtab->pModule->xRename );
- REGISTER_TRACE(pOp->p1, u.ck.pName);
- assert( u.ck.pName->flags & MEM_Str );
+#endif /* local variables moved into u.cj */
+
+ u.cj.pVtab = pOp->p4.pVtab->pVtab;
+ u.cj.pName = &aMem[pOp->p1];
+ assert( u.cj.pVtab->pModule->xRename );
+ REGISTER_TRACE(pOp->p1, u.cj.pName);
+ assert( u.cj.pName->flags & MEM_Str );
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
- rc = u.ck.pVtab->pModule->xRename(u.ck.pVtab, u.ck.pName->z);
+ rc = u.cj.pVtab->pModule->xRename(u.cj.pVtab, u.cj.pName->z);
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.ck.pVtab->zErrMsg;
- u.ck.pVtab->zErrMsg = 0;
+ p->zErrMsg = u.cj.pVtab->zErrMsg;
+ u.cj.pVtab->zErrMsg = 0;
if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
break;
}
#endif
@@ -57648,41 +57635,41 @@
** P1 is a boolean flag. If it is set to true and the xUpdate call
** is successful, then the value returned by sqlite3_last_insert_rowid()
** is set to the value of the rowid for the row just inserted.
*/
case OP_VUpdate: {
-#if 0 /* local variables moved into u.cl */
+#if 0 /* local variables moved into u.ck */
sqlite3_vtab *pVtab;
sqlite3_module *pModule;
int nArg;
int i;
sqlite_int64 rowid;
Mem **apArg;
Mem *pX;
-#endif /* local variables moved into u.cl */
-
- u.cl.pVtab = pOp->p4.pVtab->pVtab;
- u.cl.pModule = (sqlite3_module *)u.cl.pVtab->pModule;
- u.cl.nArg = pOp->p2;
+#endif /* local variables moved into u.ck */
+
+ u.ck.pVtab = pOp->p4.pVtab->pVtab;
+ u.ck.pModule = (sqlite3_module *)u.ck.pVtab->pModule;
+ u.ck.nArg = pOp->p2;
assert( pOp->p4type==P4_VTAB );
- if( ALWAYS(u.cl.pModule->xUpdate) ){
- u.cl.apArg = p->apArg;
- u.cl.pX = &p->aMem[pOp->p3];
- for(u.cl.i=0; u.cl.i<u.cl.nArg; u.cl.i++){
- sqlite3VdbeMemStoreType(u.cl.pX);
- u.cl.apArg[u.cl.i] = u.cl.pX;
- u.cl.pX++;
+ if( ALWAYS(u.ck.pModule->xUpdate) ){
+ u.ck.apArg = p->apArg;
+ u.ck.pX = &aMem[pOp->p3];
+ for(u.ck.i=0; u.ck.i<u.ck.nArg; u.ck.i++){
+ sqlite3VdbeMemStoreType(u.ck.pX);
+ u.ck.apArg[u.ck.i] = u.ck.pX;
+ u.ck.pX++;
}
if( sqlite3SafetyOff(db) ) goto abort_due_to_misuse;
- rc = u.cl.pModule->xUpdate(u.cl.pVtab, u.cl.nArg, u.cl.apArg, &u.cl.rowid);
+ rc = u.ck.pModule->xUpdate(u.ck.pVtab, u.ck.nArg, u.ck.apArg, &u.ck.rowid);
sqlite3DbFree(db, p->zErrMsg);
- p->zErrMsg = u.cl.pVtab->zErrMsg;
- u.cl.pVtab->zErrMsg = 0;
+ p->zErrMsg = u.ck.pVtab->zErrMsg;
+ u.ck.pVtab->zErrMsg = 0;
if( sqlite3SafetyOn(db) ) goto abort_due_to_misuse;
if( rc==SQLITE_OK && pOp->p1 ){
- assert( u.cl.nArg>1 && u.cl.apArg[0] && (u.cl.apArg[0]->flags&MEM_Null) );
- db->lastRowid = u.cl.rowid;
+ assert( u.ck.nArg>1 && u.ck.apArg[0] && (u.ck.apArg[0]->flags&MEM_Null) );
+ db->lastRowid = u.ck.rowid;
}
p->nChange++;
}
break;
}
@@ -57692,25 +57679,24 @@
/* Opcode: Pagecount P1 P2 * * *
**
** Write the current number of pages in database P1 to memory cell P2.
*/
case OP_Pagecount: { /* out2-prerelease */
-#if 0 /* local variables moved into u.cm */
+#if 0 /* local variables moved into u.cl */
int p1;
int nPage;
Pager *pPager;
-#endif /* local variables moved into u.cm */
-
- u.cm.p1 = pOp->p1;
- u.cm.pPager = sqlite3BtreePager(db->aDb[u.cm.p1].pBt);
- rc = sqlite3PagerPagecount(u.cm.pPager, &u.cm.nPage);
+#endif /* local variables moved into u.cl */
+
+ u.cl.p1 = pOp->p1;
+ u.cl.pPager = sqlite3BtreePager(db->aDb[u.cl.p1].pBt);
+ rc = sqlite3PagerPagecount(u.cl.pPager, &u.cl.nPage);
/* OP_Pagecount is always called from within a read transaction. The
** page count has already been successfully read and cached. So the
** sqlite3PagerPagecount() call above cannot fail. */
if( ALWAYS(rc==SQLITE_OK) ){
- pOut->flags = MEM_Int;
- pOut->u.i = u.cm.nPage;
+ pOut->u.i = u.cl.nPage;
}
break;
}
#endif
@@ -57719,22 +57705,22 @@
**
** If tracing is enabled (by the sqlite3_trace()) interface, then
** the UTF-8 string contained in P4 is emitted on the trace callback.
*/
case OP_Trace: {
-#if 0 /* local variables moved into u.cn */
+#if 0 /* local variables moved into u.cm */
char *zTrace;
-#endif /* local variables moved into u.cn */
-
- u.cn.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql);
- if( u.cn.zTrace ){
+#endif /* local variables moved into u.cm */
+
+ u.cm.zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql);
+ if( u.cm.zTrace ){
if( db->xTrace ){
- db->xTrace(db->pTraceArg, u.cn.zTrace);
+ db->xTrace(db->pTraceArg, u.cm.zTrace);
}
#ifdef SQLITE_DEBUG
if( (db->flags & SQLITE_SqlTrace)!=0 ){
- sqlite3DebugPrintf("SQL-trace: %s\n", u.cn.zTrace);
+ sqlite3DebugPrintf("SQL-trace: %s\n", u.cm.zTrace);
}
#endif /* SQLITE_DEBUG */
}
break;
}
@@ -57769,11 +57755,11 @@
u64 elapsed = sqlite3Hwtime() - start;
pOp->cycles += elapsed;
pOp->cnt++;
#if 0
fprintf(stdout, "%10llu ", elapsed);
- sqlite3VdbePrintOp(stdout, origPc, &p->aOp[origPc]);
+ sqlite3VdbePrintOp(stdout, origPc, &aOp[origPc]);
#endif
}
#endif
/* The following code adds nothing to the actual functionality
@@ -57785,15 +57771,15 @@
assert( pc>=-1 && pc<p->nOp );
#ifdef SQLITE_DEBUG
if( p->trace ){
if( rc!=0 ) fprintf(p->trace,"rc=%d\n",rc);
- if( opProperty & OPFLG_OUT2_PRERELEASE ){
- registerTrace(p->trace, pOp->p2, pOut);
- }
- if( opProperty & OPFLG_OUT3 ){
- registerTrace(p->trace, pOp->p3, pOut);
+ if( pOp->opflags & (OPFLG_OUT2_PRERELEASE|OPFLG_OUT2) ){
+ registerTrace(p->trace, pOp->p2, &aMem[pOp->p2]);
+ }
+ if( pOp->opflags & OPFLG_OUT3 ){
+ registerTrace(p->trace, pOp->p3, &aMem[pOp->p3]);
}
}
#endif /* SQLITE_DEBUG */
#endif /* NDEBUG */
} /* The end of the for(;;) loop the loops through opcodes */
@@ -57805,10 +57791,11 @@
assert( rc );
p->rc = rc;
sqlite3VdbeHalt(p);
if( rc==SQLITE_IOERR_NOMEM ) db->mallocFailed = 1;
rc = SQLITE_ERROR;
+ if( resetSchemaOnFault ) sqlite3ResetInternalSchema(db, 0);
/* This is the only way out of this procedure. We have to
** release the mutexes on btrees that were acquired at the
** top. */
vdbe_return:
@@ -57872,12 +57859,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
**
** This file contains code used to implement incremental BLOB I/O.
-**
-** $Id: vdbeblob.c,v 1.35 2009/07/02 07:47:33 danielk1977 Exp $
*/
#ifndef SQLITE_OMIT_INCRBLOB
@@ -58266,16 +58251,10 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
**
-** @(#) $Id: journal.c,v 1.9 2009/01/20 17:06:27 danielk1977 Exp $
-*/
-
-#ifdef SQLITE_ENABLE_ATOMIC_WRITE
-
-/*
** This file implements a special kind of sqlite3_file object used
** by SQLite to create journal files if the atomic-write optimization
** is enabled.
**
** The distinctive characteristic of this sqlite3_file is that the
@@ -58286,11 +58265,11 @@
**
** 1) The in-memory representation grows too large for the allocated
** buffer, or
** 2) The sqlite3JournalCreate() function is called.
*/
-
+#ifdef SQLITE_ENABLE_ATOMIC_WRITE
/*
** A JournalFile object is a subclass of sqlite3_file used by
** as an open file handle for journal files.
@@ -58511,12 +58490,10 @@
*************************************************************************
**
** This file contains code use to implement an in-memory rollback journal.
** The in-memory rollback journal is used to journal transactions for
** ":memory:" databases and when the journal_mode=MEMORY pragma is used.
-**
-** @(#) $Id: memjournal.c,v 1.12 2009/05/04 11:42:30 danielk1977 Exp $
*/
/* Forward references to internal structures */
typedef struct MemJournal MemJournal;
typedef struct FilePoint FilePoint;
@@ -58770,12 +58747,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains routines used for walking the parser tree for
** an SQL statement.
-**
-** $Id: walker.c,v 1.7 2009/06/15 23:15:59 drh Exp $
*/
/*
** Walk an expression tree. Invoke the callback once for each node
@@ -58910,12 +58885,10 @@
*************************************************************************
**
** This file contains routines used for walking the parser tree and
** resolve all identifiers by associating them with a particular
** table and column.
-**
-** $Id: resolve.c,v 1.30 2009/06/15 23:15:59 drh Exp $
*/
/*
** Turn the pExpr expression into an alias for the iCol-th column of the
** result set in pEList.
@@ -61344,10 +61317,98 @@
}
return rc;
}
/*
+** Return FALSE if there is no chance that the expression can be NULL.
+**
+** If the expression might be NULL or if the expression is too complex
+** to tell return TRUE.
+**
+** This routine is used as an optimization, to skip OP_IsNull opcodes
+** when we know that a value cannot be NULL. Hence, a false positive
+** (returning TRUE when in fact the expression can never be NULL) might
+** be a small performance hit but is otherwise harmless. On the other
+** hand, a false negative (returning FALSE when the result could be NULL)
+** will likely result in an incorrect answer. So when in doubt, return
+** TRUE.
+*/
+SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){
+ u8 op;
+ while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; }
+ op = p->op;
+ if( op==TK_REGISTER ) op = p->op2;
+ switch( op ){
+ case TK_INTEGER:
+ case TK_STRING:
+ case TK_FLOAT:
+ case TK_BLOB:
+ return 0;
+ default:
+ return 1;
+ }
+}
+
+/*
+** Generate an OP_IsNull instruction that tests register iReg and jumps
+** to location iDest if the value in iReg is NULL. The value in iReg
+** was computed by pExpr. If we can look at pExpr at compile-time and
+** determine that it can never generate a NULL, then the OP_IsNull operation
+** can be omitted.
+*/
+SQLITE_PRIVATE void sqlite3ExprCodeIsNullJump(
+ Vdbe *v, /* The VDBE under construction */
+ const Expr *pExpr, /* Only generate OP_IsNull if this expr can be NULL */
+ int iReg, /* Test the value in this register for NULL */
+ int iDest /* Jump here if the value is null */
+){
+ if( sqlite3ExprCanBeNull(pExpr) ){
+ sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iDest);
+ }
+}
+
+/*
+** Return TRUE if the given expression is a constant which would be
+** unchanged by OP_Affinity with the affinity given in the second
+** argument.
+**
+** This routine is used to determine if the OP_Affinity operation
+** can be omitted. When in doubt return FALSE. A false negative
+** is harmless. A false positive, however, can result in the wrong
+** answer.
+*/
+SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){
+ u8 op;
+ if( aff==SQLITE_AFF_NONE ) return 1;
+ while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; }
+ op = p->op;
+ if( op==TK_REGISTER ) op = p->op2;
+ switch( op ){
+ case TK_INTEGER: {
+ return aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC;
+ }
+ case TK_FLOAT: {
+ return aff==SQLITE_AFF_REAL || aff==SQLITE_AFF_NUMERIC;
+ }
+ case TK_STRING: {
+ return aff==SQLITE_AFF_TEXT;
+ }
+ case TK_BLOB: {
+ return 1;
+ }
+ case TK_COLUMN: {
+ assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */
+ return p->iColumn<0
+ && (aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC);
+ }
+ default: {
+ return 0;
+ }
+ }
+}
+
+/*
** Return TRUE if the given string is a row-id column name.
*/
SQLITE_PRIVATE int sqlite3IsRowid(const char *z){
if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1;
if( sqlite3StrICmp(z, "ROWID")==0 ) return 1;
@@ -61430,20 +61491,20 @@
** be found with <column> as its left-most column.
**
** When the b-tree is being used for membership tests, the calling function
** needs to know whether or not the structure contains an SQL NULL
** value in order to correctly evaluate expressions like "X IN (Y, Z)".
-** If there is a chance that the b-tree might contain a NULL value at
+** If there is any chance that the (...) might contain a NULL value at
** runtime, then a register is allocated and the register number written
-** to *prNotFound. If there is no chance that the b-tree contains a
+** to *prNotFound. If there is no chance that the (...) contains a
** NULL value, then *prNotFound is left unchanged.
**
** If a register is allocated and its location stored in *prNotFound, then
-** its initial value is NULL. If the b-tree does not remain constant
-** for the duration of the query (i.e. the SELECT that generates the b-tree
+** its initial value is NULL. If the (...) does not remain constant
+** for the duration of the query (i.e. the SELECT within the (...)
** is a correlated subquery) then the value of the allocated register is
-** reset to NULL each time the b-tree is repopulated. This allows the
+** reset to NULL each time the subquery is rerun. This allows the
** caller to use vdbe code equivalent to the following:
**
** if( register==NULL ){
** has_null = <test if data structure contains null>
** register = 1
@@ -61631,11 +61692,11 @@
}
affinity = sqlite3ExprAffinity(pLeft);
/* Whether this is an 'x IN(SELECT...)' or an 'x IN(<exprlist>)'
- ** expression it is handled the same way. A virtual table is
+ ** expression it is handled the same way. An ephemeral table is
** filled with single-field index keys representing the results
** from the SELECT or the <exprlist>.
**
** If the 'x' expression is a column value, or the SELECT...
** statement returns a column value, then the affinity of that
@@ -61776,10 +61837,132 @@
sqlite3VdbeJumpHere(v, testAddr-1);
}
sqlite3ExprCachePop(pParse, 1);
return rReg;
+}
+#endif /* SQLITE_OMIT_SUBQUERY */
+
+#ifndef SQLITE_OMIT_SUBQUERY
+/*
+** Generate code for an IN expression.
+**
+** x IN (SELECT ...)
+** x IN (value, value, ...)
+**
+** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS)
+** is an array of zero or more values. The expression is true if the LHS is
+** contained within the RHS. The value of the expression is unknown (NULL)
+** if the LHS is NULL or if the LHS is not contained within the RHS and the
+** RHS contains one or more NULL values.
+**
+** This routine generates code will jump to destIfFalse if the LHS is not
+** contained within the RHS. If due to NULLs we cannot determine if the LHS
+** is contained in the RHS then jump to destIfNull. If the LHS is contained
+** within the RHS then fall through.
+*/
+static void sqlite3ExprCodeIN(
+ Parse *pParse, /* Parsing and code generating context */
+ Expr *pExpr, /* The IN expression */
+ int destIfFalse, /* Jump here if LHS is not contained in the RHS */
+ int destIfNull /* Jump here if the results are unknown due to NULLs */
+){
+ int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */
+ char affinity; /* Comparison affinity to use */
+ int eType; /* Type of the RHS */
+ int r1; /* Temporary use register */
+ Vdbe *v; /* Statement under construction */
+
+ /* Compute the RHS. After this step, the table with cursor
+ ** pExpr->iTable will contains the values that make up the RHS.
+ */
+ v = pParse->pVdbe;
+ assert( v!=0 ); /* OOM detected prior to this routine */
+ VdbeNoopComment((v, "begin IN expr"));
+ eType = sqlite3FindInIndex(pParse, pExpr, &rRhsHasNull);
+
+ /* Figure out the affinity to use to create a key from the results
+ ** of the expression. affinityStr stores a static string suitable for
+ ** P4 of OP_MakeRecord.
+ */
+ affinity = comparisonAffinity(pExpr);
+
+ /* Code the LHS, the <expr> from "<expr> IN (...)".
+ */
+ sqlite3ExprCachePush(pParse);
+ r1 = sqlite3GetTempReg(pParse);
+ sqlite3ExprCode(pParse, pExpr->pLeft, r1);
+ sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull);
+
+
+ if( eType==IN_INDEX_ROWID ){
+ /* In this case, the RHS is the ROWID of table b-tree
+ */
+ sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse);
+ sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1);
+ }else{
+ /* In this case, the RHS is an index b-tree.
+ */
+ sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1);
+
+ /* If the set membership test fails, then the result of the
+ ** "x IN (...)" expression must be either 0 or NULL. If the set
+ ** contains no NULL values, then the result is 0. If the set
+ ** contains one or more NULL values, then the result of the
+ ** expression is also NULL.
+ */
+ if( rRhsHasNull==0 || destIfFalse==destIfNull ){
+ /* This branch runs if it is known at compile time that the RHS
+ ** cannot contain NULL values. This happens as the result
+ ** of a "NOT NULL" constraint in the database schema.
+ **
+ ** Also run this branch if NULL is equivalent to FALSE
+ ** for this particular IN operator.
+ */
+ sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1);
+
+ }else{
+ /* In this branch, the RHS of the IN might contain a NULL and
+ ** the presence of a NULL on the RHS makes a difference in the
+ ** outcome.
+ */
+ int j1, j2, j3;
+
+ /* First check to see if the LHS is contained in the RHS. If so,
+ ** then the presence of NULLs in the RHS does not matter, so jump
+ ** over all of the code that follows.
+ */
+ j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1);
+
+ /* Here we begin generating code that runs if the LHS is not
+ ** contained within the RHS. Generate additional code that
+ ** tests the RHS for NULLs. If the RHS contains a NULL then
+ ** jump to destIfNull. If there are no NULLs in the RHS then
+ ** jump to destIfFalse.
+ */
+ j2 = sqlite3VdbeAddOp1(v, OP_NotNull, rRhsHasNull);
+ j3 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1);
+ sqlite3VdbeAddOp2(v, OP_Integer, -1, rRhsHasNull);
+ sqlite3VdbeJumpHere(v, j3);
+ sqlite3VdbeAddOp2(v, OP_AddImm, rRhsHasNull, 1);
+ sqlite3VdbeJumpHere(v, j2);
+
+ /* Jump to the appropriate target depending on whether or not
+ ** the RHS contains a NULL
+ */
+ sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull);
+ sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse);
+
+ /* The OP_Found at the top of this branch jumps here when true,
+ ** causing the overall IN expression evaluation to fall through.
+ */
+ sqlite3VdbeJumpHere(v, j1);
+ }
+ }
+ sqlite3ReleaseTempReg(pParse, r1);
+ sqlite3ExprCachePop(pParse, 1);
+ VdbeComment((v, "end IN expr"));
}
#endif /* SQLITE_OMIT_SUBQUERY */
/*
** Duplicate an 8-byte value
@@ -62479,10 +62662,31 @@
pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0);
if( pDef==0 ){
sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId);
break;
}
+
+ /* Attempt a direct implementation of the built-in COALESCE() and
+ ** IFNULL() functions. This avoids unnecessary evalation of
+ ** arguments past the first non-NULL argument.
+ */
+ if( pDef->flags & SQLITE_FUNC_COALESCE ){
+ int endCoalesce = sqlite3VdbeMakeLabel(v);
+ assert( nFarg>=2 );
+ sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target);
+ for(i=1; i<nFarg; i++){
+ sqlite3VdbeAddOp2(v, OP_NotNull, target, endCoalesce);
+ sqlite3ExprCacheRemove(pParse, target);
+ sqlite3ExprCachePush(pParse);
+ sqlite3ExprCode(pParse, pFarg->a[i].pExpr, target);
+ sqlite3ExprCachePop(pParse, 1);
+ }
+ sqlite3VdbeResolveLabel(v, endCoalesce);
+ break;
+ }
+
+
if( pFarg ){
r1 = sqlite3GetTempRange(pParse, nFarg);
sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */
sqlite3ExprCodeExprList(pParse, pFarg, r1, 1);
sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */
@@ -62536,99 +62740,23 @@
testcase( op==TK_SELECT );
inReg = sqlite3CodeSubselect(pParse, pExpr, 0, 0);
break;
}
case TK_IN: {
- int rNotFound = 0;
- int rMayHaveNull = 0;
- int j2, j3, j4, j5;
- char affinity;
- int eType;
-
- VdbeNoopComment((v, "begin IN expr r%d", target));
- eType = sqlite3FindInIndex(pParse, pExpr, &rMayHaveNull);
- if( rMayHaveNull ){
- rNotFound = ++pParse->nMem;
- }
-
- /* Figure out the affinity to use to create a key from the results
- ** of the expression. affinityStr stores a static string suitable for
- ** P4 of OP_MakeRecord.
- */
- affinity = comparisonAffinity(pExpr);
-
-
- /* Code the <expr> from "<expr> IN (...)". The temporary table
- ** pExpr->iTable contains the values that make up the (...) set.
- */
- sqlite3ExprCachePush(pParse);
- sqlite3ExprCode(pParse, pExpr->pLeft, target);
- j2 = sqlite3VdbeAddOp1(v, OP_IsNull, target);
- if( eType==IN_INDEX_ROWID ){
- j3 = sqlite3VdbeAddOp1(v, OP_MustBeInt, target);
- j4 = sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, 0, target);
- sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
- j5 = sqlite3VdbeAddOp0(v, OP_Goto);
- sqlite3VdbeJumpHere(v, j3);
- sqlite3VdbeJumpHere(v, j4);
- sqlite3VdbeAddOp2(v, OP_Integer, 0, target);
- }else{
- r2 = regFree2 = sqlite3GetTempReg(pParse);
-
- /* Create a record and test for set membership. If the set contains
- ** the value, then jump to the end of the test code. The target
- ** register still contains the true (1) value written to it earlier.
- */
- sqlite3VdbeAddOp4(v, OP_MakeRecord, target, 1, r2, &affinity, 1);
- sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
- j5 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, r2);
-
- /* If the set membership test fails, then the result of the
- ** "x IN (...)" expression must be either 0 or NULL. If the set
- ** contains no NULL values, then the result is 0. If the set
- ** contains one or more NULL values, then the result of the
- ** expression is also NULL.
- */
- if( rNotFound==0 ){
- /* This branch runs if it is known at compile time (now) that
- ** the set contains no NULL values. This happens as the result
- ** of a "NOT NULL" constraint in the database schema. No need
- ** to test the data structure at runtime in this case.
- */
- sqlite3VdbeAddOp2(v, OP_Integer, 0, target);
- }else{
- /* This block populates the rNotFound register with either NULL
- ** or 0 (an integer value). If the data structure contains one
- ** or more NULLs, then set rNotFound to NULL. Otherwise, set it
- ** to 0. If register rMayHaveNull is already set to some value
- ** other than NULL, then the test has already been run and
- ** rNotFound is already populated.
- */
- static const char nullRecord[] = { 0x02, 0x00 };
- j3 = sqlite3VdbeAddOp1(v, OP_NotNull, rMayHaveNull);
- sqlite3VdbeAddOp2(v, OP_Null, 0, rNotFound);
- sqlite3VdbeAddOp4(v, OP_Blob, 2, rMayHaveNull, 0,
- nullRecord, P4_STATIC);
- j4 = sqlite3VdbeAddOp3(v, OP_Found, pExpr->iTable, 0, rMayHaveNull);
- sqlite3VdbeAddOp2(v, OP_Integer, 0, rNotFound);
- sqlite3VdbeJumpHere(v, j4);
- sqlite3VdbeJumpHere(v, j3);
-
- /* Copy the value of register rNotFound (which is either NULL or 0)
- ** into the target register. This will be the result of the
- ** expression.
- */
- sqlite3VdbeAddOp2(v, OP_Copy, rNotFound, target);
- }
- }
- sqlite3VdbeJumpHere(v, j2);
- sqlite3VdbeJumpHere(v, j5);
- sqlite3ExprCachePop(pParse, 1);
- VdbeComment((v, "end IN expr r%d", target));
- break;
- }
-#endif
+ int destIfFalse = sqlite3VdbeMakeLabel(v);
+ int destIfNull = sqlite3VdbeMakeLabel(v);
+ sqlite3VdbeAddOp2(v, OP_Null, 0, target);
+ sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull);
+ sqlite3VdbeAddOp2(v, OP_Integer, 1, target);
+ sqlite3VdbeResolveLabel(v, destIfFalse);
+ sqlite3VdbeAddOp2(v, OP_AddImm, target, 0);
+ sqlite3VdbeResolveLabel(v, destIfNull);
+ break;
+ }
+#endif /* SQLITE_OMIT_SUBQUERY */
+
+
/*
** x BETWEEN y AND z
**
** This is equivalent to
**
@@ -63056,10 +63184,66 @@
}
return n;
}
/*
+** Generate code for a BETWEEN operator.
+**
+** x BETWEEN y AND z
+**
+** The above is equivalent to
+**
+** x>=y AND x<=z
+**
+** Code it as such, taking care to do the common subexpression
+** elementation of x.
+*/
+static void exprCodeBetween(
+ Parse *pParse, /* Parsing and code generating context */
+ Expr *pExpr, /* The BETWEEN expression */
+ int dest, /* Jump here if the jump is taken */
+ int jumpIfTrue, /* Take the jump if the BETWEEN is true */
+ int jumpIfNull /* Take the jump if the BETWEEN is NULL */
+){
+ Expr exprAnd; /* The AND operator in x>=y AND x<=z */
+ Expr compLeft; /* The x>=y term */
+ Expr compRight; /* The x<=z term */
+ Expr exprX; /* The x subexpression */
+ int regFree1 = 0; /* Temporary use register */
+
+ assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
+ exprX = *pExpr->pLeft;
+ exprAnd.op = TK_AND;
+ exprAnd.pLeft = &compLeft;
+ exprAnd.pRight = &compRight;
+ compLeft.op = TK_GE;
+ compLeft.pLeft = &exprX;
+ compLeft.pRight = pExpr->x.pList->a[0].pExpr;
+ compRight.op = TK_LE;
+ compRight.pLeft = &exprX;
+ compRight.pRight = pExpr->x.pList->a[1].pExpr;
+ exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1);
+ exprX.op = TK_REGISTER;
+ if( jumpIfTrue ){
+ sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull);
+ }else{
+ sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull);
+ }
+ sqlite3ReleaseTempReg(pParse, regFree1);
+
+ /* Ensure adequate test coverage */
+ testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 );
+ testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 );
+ testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 );
+ testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 );
+ testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 );
+ testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 );
+ testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 );
+ testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 );
+}
+
+/*
** Generate code for a boolean expression such that a jump is made
** to the label "dest" if the expression is true but execution
** continues straight thru if the expression is false.
**
** If the expression evaluates to NULL (neither true nor false), then
@@ -63154,40 +63338,20 @@
sqlite3VdbeAddOp2(v, op, r1, dest);
testcase( regFree1==0 );
break;
}
case TK_BETWEEN: {
- /* x BETWEEN y AND z
- **
- ** Is equivalent to
- **
- ** x>=y AND x<=z
- **
- ** Code it as such, taking care to do the common subexpression
- ** elementation of x.
- */
- Expr exprAnd;
- Expr compLeft;
- Expr compRight;
- Expr exprX;
-
- assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
- exprX = *pExpr->pLeft;
- exprAnd.op = TK_AND;
- exprAnd.pLeft = &compLeft;
- exprAnd.pRight = &compRight;
- compLeft.op = TK_GE;
- compLeft.pLeft = &exprX;
- compLeft.pRight = pExpr->x.pList->a[0].pExpr;
- compRight.op = TK_LE;
- compRight.pLeft = &exprX;
- compRight.pRight = pExpr->x.pList->a[1].pExpr;
- exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1);
- testcase( regFree1==0 );
- exprX.op = TK_REGISTER;
testcase( jumpIfNull==0 );
- sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull);
+ exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull);
+ break;
+ }
+ case TK_IN: {
+ int destIfFalse = sqlite3VdbeMakeLabel(v);
+ int destIfNull = jumpIfNull ? dest : destIfFalse;
+ sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull);
+ sqlite3VdbeAddOp2(v, OP_Goto, 0, dest);
+ sqlite3VdbeResolveLabel(v, destIfFalse);
break;
}
default: {
r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1);
sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0);
@@ -63267,10 +63431,11 @@
sqlite3VdbeResolveLabel(v, d2);
sqlite3ExprCachePop(pParse, 1);
break;
}
case TK_NOT: {
+ testcase( jumpIfNull==0 );
sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull);
break;
}
case TK_LT:
case TK_LE:
@@ -63314,40 +63479,22 @@
sqlite3VdbeAddOp2(v, op, r1, dest);
testcase( regFree1==0 );
break;
}
case TK_BETWEEN: {
- /* x BETWEEN y AND z
- **
- ** Is equivalent to
- **
- ** x>=y AND x<=z
- **
- ** Code it as such, taking care to do the common subexpression
- ** elementation of x.
- */
- Expr exprAnd;
- Expr compLeft;
- Expr compRight;
- Expr exprX;
-
- assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
- exprX = *pExpr->pLeft;
- exprAnd.op = TK_AND;
- exprAnd.pLeft = &compLeft;
- exprAnd.pRight = &compRight;
- compLeft.op = TK_GE;
- compLeft.pLeft = &exprX;
- compLeft.pRight = pExpr->x.pList->a[0].pExpr;
- compRight.op = TK_LE;
- compRight.pLeft = &exprX;
- compRight.pRight = pExpr->x.pList->a[1].pExpr;
- exprX.iTable = sqlite3ExprCodeTemp(pParse, &exprX, ®Free1);
- testcase( regFree1==0 );
- exprX.op = TK_REGISTER;
testcase( jumpIfNull==0 );
- sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull);
+ exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull);
+ break;
+ }
+ case TK_IN: {
+ if( jumpIfNull ){
+ sqlite3ExprCodeIN(pParse, pExpr, dest, dest);
+ }else{
+ int destIfNull = sqlite3VdbeMakeLabel(v);
+ sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull);
+ sqlite3VdbeResolveLabel(v, destIfNull);
+ }
break;
}
default: {
r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1);
sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0);
@@ -63692,12 +63839,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains C code routines that used to generate VDBE code
** that implements the ALTER TABLE command.
-**
-** $Id: alter.c,v 1.62 2009/07/24 17:58:53 danielk1977 Exp $
*/
/*
** The code in this file only exists if we are not omitting the
** ALTER TABLE logic from the build.
@@ -64488,12 +64633,10 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains code associated with the ANALYZE command.
-**
-** @(#) $Id: analyze.c,v 1.52 2009/04/16 17:45:48 drh Exp $
*/
#ifndef SQLITE_OMIT_ANALYZE
/*
** This routine generates code that opens the sqlite_stat1 table for
@@ -65143,12 +65286,10 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains code used to implement the ATTACH and DETACH commands.
-**
-** $Id: attach.c,v 1.93 2009/05/31 21:21:41 drh Exp $
*/
#ifndef SQLITE_OMIT_ATTACH
/*
** Resolve an expression that was part of an ATTACH or DETACH statement. This
@@ -65685,12 +65826,10 @@
*************************************************************************
** This file contains code used to implement the sqlite3_set_authorizer()
** API. This facility is an optional feature of the library. Embedded
** systems that do not need this facility may omit it by recompiling
** the library with -DSQLITE_OMIT_AUTHORIZATION=1
-**
-** $Id: auth.c,v 1.32 2009/07/02 18:40:35 danielk1977 Exp $
*/
/*
** All of the code in this file may be omitted by defining a single
** macro.
@@ -65946,12 +66085,10 @@
** DROP INDEX
** creating ID lists
** BEGIN TRANSACTION
** COMMIT
** ROLLBACK
-**
-** $Id: build.c,v 1.557 2009/07/24 17:58:53 danielk1977 Exp $
*/
/*
** This routine is called when a new SQL statement is beginning to
** be parsed. Initialize the pParse structure as needed.
@@ -69645,12 +69782,10 @@
**
*************************************************************************
**
** This file contains functions used to access the internal hash tables
** of user defined functions and collation sequences.
-**
-** $Id: callback.c,v 1.42 2009/06/17 00:35:31 drh Exp $
*/
/*
** Invoke the 'collation needed' callback to request a collation sequence
@@ -70098,12 +70233,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains C code routines that are called by the parser
** in order to generate code for DELETE FROM statements.
-**
-** $Id: delete.c,v 1.207 2009/08/08 18:01:08 drh Exp $
*/
/*
** Look up every table that is named in pSrc. If any table is not found,
** add an error message to pParse->zErrMsg and return NULL. If all tables
@@ -70722,11 +70855,10 @@
}
sqlite3ReleaseTempRange(pParse, regBase, nCol+1);
return regBase;
}
-
/************** End of delete.c **********************************************/
/************** Begin file func.c ********************************************/
/*
** 2002 February 23
**
@@ -70880,10 +71012,12 @@
** p1 is 1-indexed. So substr(x,1,1) returns the first character
** of x. If x is text, then we actually count UTF-8 characters.
** If x is a blob, then we count bytes.
**
** If p1 is negative, then we begin abs(p1) from the end of x[].
+**
+** If p2 is negative, return the p2 characters preceeding p1.
*/
static void substrFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
@@ -70900,24 +71034,26 @@
|| (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL)
){
return;
}
p0type = sqlite3_value_type(argv[0]);
+ p1 = sqlite3_value_int(argv[1]);
if( p0type==SQLITE_BLOB ){
len = sqlite3_value_bytes(argv[0]);
z = sqlite3_value_blob(argv[0]);
if( z==0 ) return;
assert( len==sqlite3_value_bytes(argv[0]) );
}else{
z = sqlite3_value_text(argv[0]);
if( z==0 ) return;
len = 0;
- for(z2=z; *z2; len++){
- SQLITE_SKIP_UTF8(z2);
- }
- }
- p1 = sqlite3_value_int(argv[1]);
+ if( p1<0 ){
+ for(z2=z; *z2; len++){
+ SQLITE_SKIP_UTF8(z2);
+ }
+ }
+ }
if( argc==3 ){
p2 = sqlite3_value_int(argv[2]);
if( p2<0 ){
p2 = -p2;
negP2 = 1;
@@ -70943,14 +71079,10 @@
p2 += p1;
p1 = 0;
}
}
assert( p1>=0 && p2>=0 );
- if( p1+p2>len ){
- p2 = len-p1;
- if( p2<0 ) p2 = 0;
- }
if( p0type!=SQLITE_BLOB ){
while( *z && p1 ){
SQLITE_SKIP_UTF8(z);
p1--;
}
@@ -70957,10 +71089,14 @@
for(z2=z; *z2 && p2; p2--){
SQLITE_SKIP_UTF8(z2);
}
sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT);
}else{
+ if( p1+p2>len ){
+ p2 = len-p1;
+ if( p2<0 ) p2 = 0;
+ }
sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT);
}
}
/*
@@ -71058,10 +71194,18 @@
sqlite3_result_text(context, (char *)z1, -1, sqlite3_free);
}
}
}
+
+#if 0 /* This function is never used. */
+/*
+** The COALESCE() and IFNULL() functions used to be implemented as shown
+** here. But now they are implemented as VDBE code so that unused arguments
+** do not have to be computed. This legacy implementation is retained as
+** comment.
+*/
/*
** Implementation of the IFNULL(), NVL(), and COALESCE() functions.
** All three do the same thing. They return the first non-NULL
** argument.
*/
@@ -71076,10 +71220,12 @@
sqlite3_result_value(context, argv[i]);
break;
}
}
}
+#endif /* NOT USED */
+#define ifnullFunc versionFunc /* Substitute function - never called */
/*
** Implementation of random(). Return a random integer.
*/
static void randomFunc(
@@ -72160,14 +72306,16 @@
FUNCTION(round, 2, 0, 0, roundFunc ),
#endif
FUNCTION(upper, 1, 0, 0, upperFunc ),
FUNCTION(lower, 1, 0, 0, lowerFunc ),
FUNCTION(coalesce, 1, 0, 0, 0 ),
- FUNCTION(coalesce, -1, 0, 0, ifnullFunc ),
FUNCTION(coalesce, 0, 0, 0, 0 ),
+/* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */
+ {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0},
FUNCTION(hex, 1, 0, 0, hexFunc ),
- FUNCTION(ifnull, 2, 0, 1, ifnullFunc ),
+/* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */
+ {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0},
FUNCTION(random, 0, 0, 0, randomFunc ),
FUNCTION(randomblob, 1, 0, 0, randomBlob ),
FUNCTION(nullif, 2, 0, 1, nullifFunc ),
FUNCTION(sqlite_version, 0, 0, 0, versionFunc ),
FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ),
@@ -72613,11 +72761,11 @@
sqlite3VdbeAddOp2(v, OP_Goto, 0, iOk);
}
sqlite3VdbeAddOp3(v, OP_MakeRecord, regTemp, nCol, regRec);
sqlite3VdbeChangeP4(v, -1, sqlite3IndexAffinityStr(v, pIdx), 0);
- sqlite3VdbeAddOp3(v, OP_Found, iCur, iOk, regRec);
+ sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0);
sqlite3ReleaseTempReg(pParse, regRec);
sqlite3ReleaseTempRange(pParse, regTemp, nCol);
}
}
@@ -73419,12 +73567,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains C code routines that are called by the parser
** to handle INSERT statements in SQLite.
-**
-** $Id: insert.c,v 1.270 2009/07/24 17:58:53 danielk1977 Exp $
*/
/*
** Generate code that will open a table for reading.
*/
@@ -75240,12 +75386,10 @@
*************************************************************************
** Main file for the SQLite library. The routines in this file
** implement the programmer interface to the library. Routines in
** other files are for internal use by SQLite and should not be
** accessed by users of the library.
-**
-** $Id: legacy.c,v 1.35 2009/08/07 16:56:00 danielk1977 Exp $
*/
/*
** Execute SQL code. Return one of the SQLITE_ success/failure
@@ -75386,12 +75530,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains code used to dynamically load extensions into
** the SQLite library.
-**
-** $Id: loadext.c,v 1.60 2009/06/03 01:24:54 drh Exp $
*/
#ifndef SQLITE_CORE
#define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */
#endif
@@ -75411,12 +75553,10 @@
** This header file defines the SQLite interface for use by
** shared libraries that want to be imported as extensions into
** an SQLite instance. Shared libraries that intend to be loaded
** as extensions by SQLite should #include this file instead of
** sqlite3.h.
-**
-** @(#) $Id: sqlite3ext.h,v 1.25 2008/10/12 00:27:54 shane Exp $
*/
#ifndef _SQLITE3EXT_H_
#define _SQLITE3EXT_H_
typedef struct sqlite3_api_routines sqlite3_api_routines;
@@ -76380,12 +76520,10 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains code used to implement the PRAGMA command.
-**
-** $Id: pragma.c,v 1.214 2009/07/02 07:47:33 danielk1977 Exp $
*/
/* Ignore this whole file if pragmas are disabled
*/
#if !defined(SQLITE_OMIT_PRAGMA)
@@ -77499,10 +77637,11 @@
sqlite3VdbeAddOp2(v, OP_Integer, 0, 2); /* reg(2) will count entries */
loopTop = sqlite3VdbeAddOp2(v, OP_Rewind, 1, 0);
sqlite3VdbeAddOp2(v, OP_AddImm, 2, 1); /* increment entry count */
for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){
int jmp2;
+ int r1;
static const VdbeOpList idxErr[] = {
{ OP_AddImm, 1, -1, 0},
{ OP_String8, 0, 3, 0}, /* 1 */
{ OP_Rowid, 1, 4, 0},
{ OP_String8, 0, 5, 0}, /* 3 */
@@ -77512,12 +77651,12 @@
{ OP_Concat, 6, 3, 3},
{ OP_ResultRow, 3, 1, 0},
{ OP_IfPos, 1, 0, 0}, /* 9 */
{ OP_Halt, 0, 0, 0},
};
- sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 1);
- jmp2 = sqlite3VdbeAddOp3(v, OP_Found, j+2, 0, 3);
+ r1 = sqlite3GenerateIndexKey(pParse, pIdx, 1, 3, 0);
+ jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, j+2, 0, r1, pIdx->nColumn+1);
addr = sqlite3VdbeAddOpList(v, ArraySize(idxErr), idxErr);
sqlite3VdbeChangeP4(v, addr+1, "rowid ", P4_STATIC);
sqlite3VdbeChangeP4(v, addr+3, " missing from index ", P4_STATIC);
sqlite3VdbeChangeP4(v, addr+4, pIdx->zName, P4_STATIC);
sqlite3VdbeJumpHere(v, addr+9);
@@ -77820,12 +77959,10 @@
**
*************************************************************************
** This file contains the implementation of the sqlite3_prepare()
** interface, and routines that contribute to loading the database schema
** from disk.
-**
-** $Id: prepare.c,v 1.131 2009/08/06 17:43:31 drh Exp $
*/
/*
** Fill the InitData structure with an error message that indicates
** that the database is corrupt.
@@ -78694,12 +78831,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains C code routines that are called by the parser
** to handle SELECT statements in SQLite.
-**
-** $Id: select.c,v 1.526 2009/08/01 15:09:58 drh Exp $
*/
/*
** Delete all the content of a Select structure but do not deallocate
@@ -79123,12 +79258,12 @@
Vdbe *v;
int r1;
v = pParse->pVdbe;
r1 = sqlite3GetTempReg(pParse);
- sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1);
- sqlite3VdbeAddOp3(v, OP_Found, iTab, addrRepeat, r1);
+ sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N);
+ sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1);
sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1);
sqlite3ReleaseTempReg(pParse, r1);
}
/*
@@ -79365,12 +79500,11 @@
/* Jump to the end of the loop if the LIMIT is reached.
*/
if( p->iLimit ){
assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to
** pushOntoSorter() would have cleared p->iLimit */
- sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1);
- sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak);
+ sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1);
}
}
/*
** Given an expression list, generate a KeyInfo structure that records
@@ -79992,11 +80126,11 @@
*/
static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){
Vdbe *v = 0;
int iLimit = 0;
int iOffset;
- int addr1;
+ int addr1, n;
if( p->iLimit ) return;
/*
** "LIMIT -1" always shows all rows. There is some
** contraversy about what the correct behavior should be.
@@ -80007,14 +80141,22 @@
assert( p->pOffset==0 || p->pLimit!=0 );
if( p->pLimit ){
p->iLimit = iLimit = ++pParse->nMem;
v = sqlite3GetVdbe(pParse);
if( NEVER(v==0) ) return; /* VDBE should have already been allocated */
- sqlite3ExprCode(pParse, p->pLimit, iLimit);
- sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit);
- VdbeComment((v, "LIMIT counter"));
- sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak);
+ if( sqlite3ExprIsInteger(p->pLimit, &n) ){
+ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit);
+ VdbeComment((v, "LIMIT counter"));
+ if( n==0 ){
+ sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak);
+ }
+ }else{
+ sqlite3ExprCode(pParse, p->pLimit, iLimit);
+ sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit);
+ VdbeComment((v, "LIMIT counter"));
+ sqlite3VdbeAddOp2(v, OP_IfZero, iLimit, iBreak);
+ }
if( p->pOffset ){
p->iOffset = iOffset = ++pParse->nMem;
pParse->nMem++; /* Allocate an extra register for limit+offset */
sqlite3ExprCode(pParse, p->pOffset, iOffset);
sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset);
@@ -80346,11 +80488,11 @@
iCont = sqlite3VdbeMakeLabel(v);
computeLimitRegisters(pParse, p, iBreak);
sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak);
r1 = sqlite3GetTempReg(pParse);
iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1);
- sqlite3VdbeAddOp3(v, OP_NotFound, tab2, iCont, r1);
+ sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0);
sqlite3ReleaseTempReg(pParse, r1);
selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr,
0, -1, &dest, iCont, iBreak);
sqlite3VdbeResolveLabel(v, iCont);
sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart);
@@ -80565,12 +80707,11 @@
}
/* Jump to the end of the loop if the LIMIT is reached.
*/
if( p->iLimit ){
- sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1);
- sqlite3VdbeAddOp2(v, OP_IfZero, p->iLimit, iBreak);
+ sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1);
}
/* Generate the subroutine return
*/
sqlite3VdbeResolveLabel(v, iContinue);
@@ -82934,12 +83075,10 @@
** interface routines. These are just wrappers around the main
** interface routine of sqlite3_exec().
**
** These routines are in a separate files so that they will not be linked
** if they are not used.
-**
-** $Id: table.c,v 1.40 2009/04/10 14:28:00 drh Exp $
*/
#ifndef SQLITE_OMIT_GET_TABLE
/*
@@ -83126,13 +83265,11 @@
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
-**
-**
-** $Id: trigger.c,v 1.143 2009/08/10 03:57:58 shane Exp $
+** This file contains the implementation for TRIGGERs
*/
#ifndef SQLITE_OMIT_TRIGGER
/*
** Delete a linked list of TriggerStep structures.
@@ -84210,12 +84347,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains C code routines that are called by the parser
** to handle UPDATE statements.
-**
-** $Id: update.c,v 1.207 2009/08/08 18:01:08 drh Exp $
*/
#ifndef SQLITE_OMIT_VIRTUALTABLE
/* Forward declaration */
static void updateVirtualTable(
@@ -84843,12 +84978,10 @@
*************************************************************************
** This file contains code used to implement the VACUUM command.
**
** Most of the code in this file may be omitted by defining the
** SQLITE_OMIT_VACUUM macro.
-**
-** $Id: vacuum.c,v 1.91 2009/07/02 07:47:33 danielk1977 Exp $
*/
#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH)
/*
** Execute zSql on database db. Return an error code.
@@ -85149,12 +85282,10 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains code used to help implement virtual tables.
-**
-** $Id: vtab.c,v 1.94 2009/08/08 18:01:08 drh Exp $
*/
#ifndef SQLITE_OMIT_VIRTUALTABLE
/*
** The actual function that does the work of creating a new module.
@@ -86118,12 +86249,10 @@
** the WHERE clause of SQL statements. This module is responsible for
** generating the code that loops through a table looking for applicable
** rows. Indices are selected and used to speed the search when doing
** so is applicable. Because this module is responsible for selecting
** indices, you might also think of this module as the "query optimizer".
-**
-** $Id: where.c,v 1.411 2009/07/31 06:14:52 danielk1977 Exp $
*/
/*
** Trace output macros
*/
@@ -88698,20 +88827,43 @@
/*
** Code an OP_Affinity opcode to apply the column affinity string zAff
** to the n registers starting at base.
**
-** Buffer zAff was allocated using sqlite3DbMalloc(). It is the
-** responsibility of this function to arrange for it to be eventually
-** freed using sqlite3DbFree().
+** As an optimization, SQLITE_AFF_NONE entries (which are no-ops) at the
+** beginning and end of zAff are ignored. If all entries in zAff are
+** SQLITE_AFF_NONE, then no code gets generated.
+**
+** This routine makes its own copy of zAff so that the caller is free
+** to modify zAff after this routine returns.
*/
static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){
Vdbe *v = pParse->pVdbe;
+ if( zAff==0 ){
+ assert( pParse->db->mallocFailed );
+ return;
+ }
assert( v!=0 );
- sqlite3VdbeAddOp2(v, OP_Affinity, base, n);
- sqlite3VdbeChangeP4(v, -1, zAff, P4_DYNAMIC);
- sqlite3ExprCacheAffinityChange(pParse, base, n);
+
+ /* Adjust base and n to skip over SQLITE_AFF_NONE entries at the beginning
+ ** and end of the affinity string.
+ */
+ while( n>0 && zAff[0]==SQLITE_AFF_NONE ){
+ n--;
+ base++;
+ zAff++;
+ }
+ while( n>1 && zAff[n-1]==SQLITE_AFF_NONE ){
+ n--;
+ }
+
+ /* Code the OP_Affinity opcode if there is anything left to do. */
+ if( n>0 ){
+ sqlite3VdbeAddOp2(v, OP_Affinity, base, n);
+ sqlite3VdbeChangeP4(v, -1, zAff, n);
+ sqlite3ExprCacheAffinityChange(pParse, base, n);
+ }
}
/*
** Generate code for a single equality term of the WHERE clause. An equality
@@ -88778,11 +88930,11 @@
return iReg;
}
/*
** Generate code that will evaluate all == and IN constraints for an
-** index. The values for all constraints are left on the stack.
+** index.
**
** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c).
** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10
** The index has as many as three equality constraints, but in this
** example, the third "c" value is an inequality. So only two
@@ -88790,11 +88942,12 @@
** a==5 and b IN (1,2,3). The current values for a and b will be stored
** in consecutive registers and the index of the first register is returned.
**
** In the example above nEq==2. But this subroutine works for any value
** of nEq including 0. If nEq==0, this routine is nearly a no-op.
-** The only thing it does is allocate the pLevel->iMem memory cell.
+** The only thing it does is allocate the pLevel->iMem memory cell and
+** compute the affinity string.
**
** This routine always allocates at least one memory cell and returns
** the index of that memory cell. The code that
** calls this routine will use that memory cell to store the termination
** key value of the loop. If one or more IN operators appear, then
@@ -88868,15 +89021,19 @@
}
}
testcase( pTerm->eOperator & WO_ISNULL );
testcase( pTerm->eOperator & WO_IN );
if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){
- sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk);
- if( zAff
- && sqlite3CompareAffinity(pTerm->pExpr->pRight, zAff[j])==SQLITE_AFF_NONE
- ){
- zAff[j] = SQLITE_AFF_NONE;
+ Expr *pRight = pTerm->pExpr->pRight;
+ sqlite3ExprCodeIsNullJump(v, pRight, regBase+j, pLevel->addrBrk);
+ if( zAff ){
+ if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_NONE ){
+ zAff[j] = SQLITE_AFF_NONE;
+ }
+ if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){
+ zAff[j] = SQLITE_AFF_NONE;
+ }
}
}
}
*pzAff = zAff;
return regBase;
@@ -89198,18 +89355,21 @@
/* Seek the index cursor to the start of the range. */
nConstraint = nEq;
if( pRangeStart ){
Expr *pRight = pRangeStart->pExpr->pRight;
sqlite3ExprCode(pParse, pRight, regBase+nEq);
- sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt);
- if( zAff
- && sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE
- ){
- /* Since the comparison is to be performed with no conversions applied
- ** to the operands, set the affinity to apply to pRight to
- ** SQLITE_AFF_NONE. */
- zAff[nConstraint] = SQLITE_AFF_NONE;
+ sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+ if( zAff ){
+ if( sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE){
+ /* Since the comparison is to be performed with no conversions
+ ** applied to the operands, set the affinity to apply to pRight to
+ ** SQLITE_AFF_NONE. */
+ zAff[nConstraint] = SQLITE_AFF_NONE;
+ }
+ if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[nConstraint]) ){
+ zAff[nConstraint] = SQLITE_AFF_NONE;
+ }
}
nConstraint++;
}else if( isMinQuery ){
sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq);
nConstraint++;
@@ -89223,34 +89383,36 @@
testcase( op==OP_Last );
testcase( op==OP_SeekGt );
testcase( op==OP_SeekGe );
testcase( op==OP_SeekLe );
testcase( op==OP_SeekLt );
- sqlite3VdbeAddOp4(v, op, iIdxCur, addrNxt, regBase,
- SQLITE_INT_TO_PTR(nConstraint), P4_INT32);
+ sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);
/* Load the value for the inequality constraint at the end of the
** range (if any).
*/
nConstraint = nEq;
if( pRangeEnd ){
Expr *pRight = pRangeEnd->pExpr->pRight;
sqlite3ExprCacheRemove(pParse, regBase+nEq);
sqlite3ExprCode(pParse, pRight, regBase+nEq);
- sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt);
- zAff = sqlite3DbStrDup(pParse->db, zAff);
- if( zAff
- && sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE
- ){
- /* Since the comparison is to be performed with no conversions applied
- ** to the operands, set the affinity to apply to pRight to
- ** SQLITE_AFF_NONE. */
- zAff[nConstraint] = SQLITE_AFF_NONE;
+ sqlite3ExprCodeIsNullJump(v, pRight, regBase+nEq, addrNxt);
+ if( zAff ){
+ if( sqlite3CompareAffinity(pRight, zAff[nConstraint])==SQLITE_AFF_NONE){
+ /* Since the comparison is to be performed with no conversions
+ ** applied to the operands, set the affinity to apply to pRight to
+ ** SQLITE_AFF_NONE. */
+ zAff[nConstraint] = SQLITE_AFF_NONE;
+ }
+ if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[nConstraint]) ){
+ zAff[nConstraint] = SQLITE_AFF_NONE;
+ }
}
codeApplyAffinity(pParse, regBase, nEq+1, zAff);
nConstraint++;
}
+ sqlite3DbFree(pParse->db, zAff);
/* Top of the loop body */
pLevel->p2 = sqlite3VdbeCurrentAddr(v);
/* Check if the index cursor is past the end of the range. */
@@ -89257,12 +89419,11 @@
op = aEndOp[(pRangeEnd || nEq) * (1 + bRev)];
testcase( op==OP_Noop );
testcase( op==OP_IdxGE );
testcase( op==OP_IdxLT );
if( op!=OP_Noop ){
- sqlite3VdbeAddOp4(v, op, iIdxCur, addrNxt, regBase,
- SQLITE_INT_TO_PTR(nConstraint), P4_INT32);
+ sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint);
sqlite3VdbeChangeP5(v, endEq!=bRev ?1:0);
}
/* If there are inequality constraints, check that the value
** of the table column that the inequality contrains is not NULL.
@@ -89387,13 +89548,12 @@
if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){
int iSet = ((ii==pOrWc->nTerm-1)?-1:ii);
int r;
r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur,
regRowid, 0);
- sqlite3VdbeAddOp4(v, OP_RowSetTest, regRowset,
- sqlite3VdbeCurrentAddr(v)+2,
- r, SQLITE_INT_TO_PTR(iSet), P4_INT32);
+ sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset,
+ sqlite3VdbeCurrentAddr(v)+2, r, iSet);
}
sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody);
/* Finish the loop through table entries that match term pOrTerm. */
sqlite3WhereEnd(pSubWInfo);
@@ -89920,11 +90080,12 @@
sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op);
if( !pWInfo->okOnePass && pTab->nCol<BMS ){
Bitmask b = pTabItem->colUsed;
int n = 0;
for(; b; b=b>>1, n++){}
- sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, SQLITE_INT_TO_PTR(n), P4_INT32);
+ sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1,
+ SQLITE_INT_TO_PTR(n), P4_INT32);
assert( n<=pTab->nCol );
}
}else{
sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
}
@@ -90048,11 +90209,15 @@
}
sqlite3VdbeResolveLabel(v, pLevel->addrBrk);
if( pLevel->iLeftJoin ){
int addr;
addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin);
- sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);
+ assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0
+ || (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 );
+ if( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0 ){
+ sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor);
+ }
if( pLevel->iIdxCur>=0 ){
sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur);
}
if( pLevel->op==OP_Return ){
sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst);
@@ -90099,11 +90264,10 @@
*/
if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 && !db->mallocFailed){
int k, j, last;
VdbeOp *pOp;
Index *pIdx = pLevel->plan.u.pIdx;
- int useIndexOnly = pLevel->plan.wsFlags & WHERE_IDX_ONLY;
assert( pIdx!=0 );
pOp = sqlite3VdbeGetOp(v, pWInfo->iTop);
last = sqlite3VdbeCurrentAddr(v);
for(k=pWInfo->iTop; k<last; k++, pOp++){
@@ -90114,16 +90278,15 @@
pOp->p2 = j;
pOp->p1 = pLevel->iIdxCur;
break;
}
}
- assert(!useIndexOnly || j<pIdx->nColumn);
+ assert( (pLevel->plan.wsFlags & WHERE_IDX_ONLY)==0
+ || j<pIdx->nColumn );
}else if( pOp->opcode==OP_Rowid ){
pOp->p1 = pLevel->iIdxCur;
pOp->opcode = OP_IdxRowid;
- }else if( pOp->opcode==OP_NullRow && useIndexOnly ){
- pOp->opcode = OP_Noop;
}
}
}
}
@@ -90238,10 +90401,21 @@
Token *pPostOp /* The operand token for setting the span */
){
pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0);
pOut->zStart = pOperand->zStart;
pOut->zEnd = &pPostOp->z[pPostOp->n];
+ }
+
+ /* A routine to convert a binary TK_IS or TK_ISNOT expression into a
+ ** unary TK_ISNULL or TK_NOTNULL expression. */
+ static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){
+ sqlite3 *db = pParse->db;
+ if( db->mallocFailed==0 && pY->op==TK_NULL ){
+ pA->op = op;
+ sqlite3ExprDelete(db, pA->pRight);
+ pA->pRight = 0;
+ }
}
/* Construct an expression node for a unary prefix operator
*/
static void spanUnaryPrefix(
@@ -92857,21 +93031,17 @@
{spanUnaryPostfix(&yygotominor.yy346,pParse,TK_NOTNULL,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy0);}
break;
case 216: /* expr ::= expr IS expr */
{
spanBinaryExpr(&yygotominor.yy346,pParse,TK_IS,&yymsp[-2].minor.yy346,&yymsp[0].minor.yy346);
- if( pParse->db->mallocFailed==0 && yymsp[0].minor.yy346.pExpr->op==TK_NULL ){
- yygotominor.yy346.pExpr->op = TK_ISNULL;
- }
+ binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_ISNULL);
}
break;
case 217: /* expr ::= expr IS NOT expr */
{
spanBinaryExpr(&yygotominor.yy346,pParse,TK_ISNOT,&yymsp[-3].minor.yy346,&yymsp[0].minor.yy346);
- if( pParse->db->mallocFailed==0 && yymsp[0].minor.yy346.pExpr->op==TK_NULL ){
- yygotominor.yy346.pExpr->op = TK_NOTNULL;
- }
+ binaryToUnaryIfNull(pParse, yymsp[0].minor.yy346.pExpr, yygotominor.yy346.pExpr, TK_NOTNULL);
}
break;
case 218: /* expr ::= NOT expr */
case 219: /* expr ::= BITNOT expr */ yytestcase(yyruleno==219);
{spanUnaryPrefix(&yygotominor.yy346,pParse,yymsp[-1].major,&yymsp[0].minor.yy346,&yymsp[-1].minor.yy0);}
@@ -93551,12 +93721,10 @@
** An tokenizer for SQL
**
** This file contains C code that splits an SQL input string up into
** individual tokens and sends those tokens one-by-one over to the
** parser for analysis.
-**
-** $Id: tokenize.c,v 1.163 2009/07/03 22:54:37 drh Exp $
*/
/*
** The charMap() macro maps alphabetic characters into their
** lower-case ASCII equivalent. On ASCII machines, this is just
@@ -93892,20 +94060,11 @@
** middle of identfiers. But many SQL implementations do.
** SQLite will allow '$' in identifiers for compatibility.
** But the feature is undocumented.
*/
#ifdef SQLITE_ASCII
-SQLITE_PRIVATE const char sqlite3IsAsciiIdChar[] = {
-/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
- 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
-};
-#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsAsciiIdChar[c-0x20]))
+#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
#endif
#ifdef SQLITE_EBCDIC
SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */
@@ -94361,22 +94520,19 @@
**
** This file contains C code that implements the sqlite3_complete() API.
** This code used to be part of the tokenizer.c source file. But by
** separating it out, the code will be automatically omitted from
** static links that do not use it.
-**
-** $Id: complete.c,v 1.8 2009/04/28 04:46:42 drh Exp $
*/
#ifndef SQLITE_OMIT_COMPLETE
/*
** This is defined in tokenize.c. We just have to import the definition.
*/
#ifndef SQLITE_AMALGAMATION
#ifdef SQLITE_ASCII
-SQLITE_PRIVATE const char sqlite3IsAsciiIdChar[];
-#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsAsciiIdChar[c-0x20]))
+#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
#endif
#ifdef SQLITE_EBCDIC
SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[];
#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40]))
#endif
@@ -94529,11 +94685,13 @@
if( *zSql==0 ) return 0;
token = tkOTHER;
break;
}
default: {
- int c;
+#ifdef SQLITE_EBCDIC
+ unsigned char c;
+#endif
if( IdChar((u8)*zSql) ){
/* Keywords and unquoted identifiers */
int nId;
for(nId=1; IdChar(zSql[nId]); nId++){}
#ifdef SQLITE_OMIT_TRIGGER
@@ -97015,12 +97173,10 @@
**
*************************************************************************
**
** This file contains the implementation of the sqlite3_unlock_notify()
** API method and its associated functionality.
-**
-** $Id: notify.c,v 1.4 2009/04/07 22:06:57 drh Exp $
*/
/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */
#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
@@ -97615,380 +97771,17 @@
#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE)
# define SQLITE_CORE 1
#endif
-
-/************** Include fts3_expr.h in the middle of fts3.c ******************/
-/************** Begin file fts3_expr.h ***************************************/
-/*
-** 2008 Nov 28
-**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
-**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
-**
-******************************************************************************
-**
-*/
-
-/************** Include fts3_tokenizer.h in the middle of fts3_expr.h ********/
-/************** Begin file fts3_tokenizer.h **********************************/
-/*
-** 2006 July 10
-**
-** The author disclaims copyright to this source code.
-**
-*************************************************************************
-** Defines the interface to tokenizers used by fulltext-search. There
-** are three basic components:
-**
-** sqlite3_tokenizer_module is a singleton defining the tokenizer
-** interface functions. This is essentially the class structure for
-** tokenizers.
-**
-** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
-** including customization information defined at creation time.
-**
-** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
-** tokens from a particular input.
-*/
-#ifndef _FTS3_TOKENIZER_H_
-#define _FTS3_TOKENIZER_H_
-
-/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
-** If tokenizers are to be allowed to call sqlite3_*() functions, then
-** we will need a way to register the API consistently.
-*/
-
-/*
-** Structures used by the tokenizer interface. When a new tokenizer
-** implementation is registered, the caller provides a pointer to
-** an sqlite3_tokenizer_module containing pointers to the callback
-** functions that make up an implementation.
-**
-** When an fts3 table is created, it passes any arguments passed to
-** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
-** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
-** implementation. The xCreate() function in turn returns an
-** sqlite3_tokenizer structure representing the specific tokenizer to
-** be used for the fts3 table (customized by the tokenizer clause arguments).
-**
-** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
-** method is called. It returns an sqlite3_tokenizer_cursor object
-** that may be used to tokenize a specific input buffer based on
-** the tokenization rules supplied by a specific sqlite3_tokenizer
-** object.
-*/
-typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
-typedef struct sqlite3_tokenizer sqlite3_tokenizer;
-typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
-
-struct sqlite3_tokenizer_module {
-
- /*
- ** Structure version. Should always be set to 0.
- */
- int iVersion;
-
- /*
- ** Create a new tokenizer. The values in the argv[] array are the
- ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
- ** TABLE statement that created the fts3 table. For example, if
- ** the following SQL is executed:
- **
- ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
- **
- ** then argc is set to 2, and the argv[] array contains pointers
- ** to the strings "arg1" and "arg2".
- **
- ** This method should return either SQLITE_OK (0), or an SQLite error
- ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
- ** to point at the newly created tokenizer structure. The generic
- ** sqlite3_tokenizer.pModule variable should not be initialised by
- ** this callback. The caller will do so.
- */
- int (*xCreate)(
- int argc, /* Size of argv array */
- const char *const*argv, /* Tokenizer argument strings */
- sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
- );
-
- /*
- ** Destroy an existing tokenizer. The fts3 module calls this method
- ** exactly once for each successful call to xCreate().
- */
- int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
-
- /*
- ** Create a tokenizer cursor to tokenize an input buffer. The caller
- ** is responsible for ensuring that the input buffer remains valid
- ** until the cursor is closed (using the xClose() method).
- */
- int (*xOpen)(
- sqlite3_tokenizer *pTokenizer, /* Tokenizer object */
- const char *pInput, int nBytes, /* Input buffer */
- sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */
- );
-
- /*
- ** Destroy an existing tokenizer cursor. The fts3 module calls this
- ** method exactly once for each successful call to xOpen().
- */
- int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
-
- /*
- ** Retrieve the next token from the tokenizer cursor pCursor. This
- ** method should either return SQLITE_OK and set the values of the
- ** "OUT" variables identified below, or SQLITE_DONE to indicate that
- ** the end of the buffer has been reached, or an SQLite error code.
- **
- ** *ppToken should be set to point at a buffer containing the
- ** normalized version of the token (i.e. after any case-folding and/or
- ** stemming has been performed). *pnBytes should be set to the length
- ** of this buffer in bytes. The input text that generated the token is
- ** identified by the byte offsets returned in *piStartOffset and
- ** *piEndOffset. *piStartOffset should be set to the index of the first
- ** byte of the token in the input buffer. *piEndOffset should be set
- ** to the index of the first byte just past the end of the token in
- ** the input buffer.
- **
- ** The buffer *ppToken is set to point at is managed by the tokenizer
- ** implementation. It is only required to be valid until the next call
- ** to xNext() or xClose().
- */
- /* TODO(shess) current implementation requires pInput to be
- ** nul-terminated. This should either be fixed, or pInput/nBytes
- ** should be converted to zInput.
- */
- int (*xNext)(
- sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */
- const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */
- int *piStartOffset, /* OUT: Byte offset of token in input buffer */
- int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
- int *piPosition /* OUT: Number of tokens returned before this one */
- );
-};
-
-struct sqlite3_tokenizer {
- const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
- /* Tokenizer implementations will typically add additional fields */
-};
-
-struct sqlite3_tokenizer_cursor {
- sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
- /* Tokenizer implementations will typically add additional fields */
-};
-
-#endif /* _FTS3_TOKENIZER_H_ */
-
-/************** End of fts3_tokenizer.h **************************************/
-/************** Continuing where we left off in fts3_expr.h ******************/
-
-/*
-** The following describes the syntax supported by the fts3 MATCH
-** operator in a similar format to that used by the lemon parser
-** generator. This module does not use actually lemon, it uses a
-** custom parser.
-**
-** query ::= andexpr (OR andexpr)*.
-**
-** andexpr ::= notexpr (AND? notexpr)*.
-**
-** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
-** notexpr ::= LP query RP.
-**
-** nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
-**
-** distance_opt ::= .
-** distance_opt ::= / INTEGER.
-**
-** phrase ::= TOKEN.
-** phrase ::= COLUMN:TOKEN.
-** phrase ::= "TOKEN TOKEN TOKEN...".
-*/
-
-typedef struct Fts3Expr Fts3Expr;
-typedef struct Fts3Phrase Fts3Phrase;
-
-/*
-** A "phrase" is a sequence of one or more tokens that must match in
-** sequence. A single token is the base case and the most common case.
-** For a sequence of tokens contained in "...", nToken will be the number
-** of tokens in the string.
-*/
-struct Fts3Phrase {
- int nToken; /* Number of tokens in the phrase */
- int iColumn; /* Index of column this phrase must match */
- int isNot; /* Phrase prefixed by unary not (-) operator */
- struct PhraseToken {
- char *z; /* Text of the token */
- int n; /* Number of bytes in buffer pointed to by z */
- int isPrefix; /* True if token ends in with a "*" character */
- } aToken[1]; /* One entry for each token in the phrase */
-};
-
-/*
-** A tree of these objects forms the RHS of a MATCH operator.
-*/
-struct Fts3Expr {
- int eType; /* One of the FTSQUERY_XXX values defined below */
- int nNear; /* Valid if eType==FTSQUERY_NEAR */
- Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
- Fts3Expr *pLeft; /* Left operand */
- Fts3Expr *pRight; /* Right operand */
- Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
-};
-
-SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int,
- const char *, int, Fts3Expr **);
-SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *);
-
-/*
-** Candidate values for Fts3Query.eType. Note that the order of the first
-** four values is in order of precedence when parsing expressions. For
-** example, the following:
-**
-** "a OR b AND c NOT d NEAR e"
-**
-** is equivalent to:
-**
-** "a OR (b AND (c NOT (d NEAR e)))"
-*/
-#define FTSQUERY_NEAR 1
-#define FTSQUERY_NOT 2
-#define FTSQUERY_AND 3
-#define FTSQUERY_OR 4
-#define FTSQUERY_PHRASE 5
-
-#ifdef SQLITE_TEST
-SQLITE_PRIVATE void sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
-#endif
-
-/************** End of fts3_expr.h *******************************************/
-/************** Continuing where we left off in fts3.c ***********************/
-/************** Include fts3_hash.h in the middle of fts3.c ******************/
-/************** Begin file fts3_hash.h ***************************************/
-/*
-** 2001 September 22
-**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
-**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This is the header file for the generic hash-table implemenation
-** used in SQLite. We've modified it slightly to serve as a standalone
-** hash table implementation for the full-text indexing module.
-**
-*/
-#ifndef _FTS3_HASH_H_
-#define _FTS3_HASH_H_
-
-/* Forward declarations of structures. */
-typedef struct fts3Hash fts3Hash;
-typedef struct fts3HashElem fts3HashElem;
-
-/* A complete hash table is an instance of the following structure.
-** The internals of this structure are intended to be opaque -- client
-** code should not attempt to access or modify the fields of this structure
-** directly. Change this structure only by using the routines below.
-** However, many of the "procedures" and "functions" for modifying and
-** accessing this structure are really macros, so we can't really make
-** this structure opaque.
-*/
-struct fts3Hash {
- char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
- char copyKey; /* True if copy of key made on insert */
- int count; /* Number of entries in this table */
- fts3HashElem *first; /* The first element of the array */
- int htsize; /* Number of buckets in the hash table */
- struct _fts3ht { /* the hash table */
- int count; /* Number of entries with this hash */
- fts3HashElem *chain; /* Pointer to first entry with this hash */
- } *ht;
-};
-
-/* Each element in the hash table is an instance of the following
-** structure. All elements are stored on a single doubly-linked list.
-**
-** Again, this structure is intended to be opaque, but it can't really
-** be opaque because it is used by macros.
-*/
-struct fts3HashElem {
- fts3HashElem *next, *prev; /* Next and previous elements in the table */
- void *data; /* Data associated with this element */
- void *pKey; int nKey; /* Key associated with this element */
-};
-
-/*
-** There are 2 different modes of operation for a hash table:
-**
-** FTS3_HASH_STRING pKey points to a string that is nKey bytes long
-** (including the null-terminator, if any). Case
-** is respected in comparisons.
-**
-** FTS3_HASH_BINARY pKey points to binary data nKey bytes long.
-** memcmp() is used to compare keys.
-**
-** A copy of the key is made if the copyKey parameter to fts3HashInit is 1.
-*/
-#define FTS3_HASH_STRING 1
-#define FTS3_HASH_BINARY 2
-
-/*
-** Access routines. To delete, insert a NULL pointer.
-*/
-SQLITE_PRIVATE void sqlite3Fts3HashInit(fts3Hash*, int keytype, int copyKey);
-SQLITE_PRIVATE void *sqlite3Fts3HashInsert(fts3Hash*, const void *pKey, int nKey, void *pData);
-SQLITE_PRIVATE void *sqlite3Fts3HashFind(const fts3Hash*, const void *pKey, int nKey);
-SQLITE_PRIVATE void sqlite3Fts3HashClear(fts3Hash*);
-
-/*
-** Shorthand for the functions above
-*/
-#define fts3HashInit sqlite3Fts3HashInit
-#define fts3HashInsert sqlite3Fts3HashInsert
-#define fts3HashFind sqlite3Fts3HashFind
-#define fts3HashClear sqlite3Fts3HashClear
-
-/*
-** Macros for looping over all elements of a hash table. The idiom is
-** like this:
-**
-** fts3Hash h;
-** fts3HashElem *p;
-** ...
-** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
-** SomeStructure *pData = fts3HashData(p);
-** // do something with pData
-** }
-*/
-#define fts3HashFirst(H) ((H)->first)
-#define fts3HashNext(E) ((E)->next)
-#define fts3HashData(E) ((E)->data)
-#define fts3HashKey(E) ((E)->pKey)
-#define fts3HashKeysize(E) ((E)->nKey)
-
-/*
-** Number of entries in a hash table
-*/
-#define fts3HashCount(H) ((H)->count)
-
-#endif /* _FTS3_HASH_H_ */
-
-/************** End of fts3_hash.h *******************************************/
-/************** Continuing where we left off in fts3.c ***********************/
+#include "fts3Int.h"
+
+
#ifndef SQLITE_CORE
SQLITE_EXTENSION_INIT1
#endif
+
/* TODO(shess) MAN, this thing needs some refactoring. At minimum, it
** would be nice to order the file better, perhaps something along the
** lines of:
@@ -98005,29 +97798,10 @@
#if 0
# define FTSTRACE(A) printf A; fflush(stdout)
#else
# define FTSTRACE(A)
#endif
-
-/* It is not safe to call isspace(), tolower(), or isalnum() on
-** hi-bit-set characters. This is the same solution used in the
-** tokenizer.
-*/
-/* TODO(shess) The snippet-generation code should be using the
-** tokenizer-generated tokens rather than doing its own local
-** tokenization.
-*/
-/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
-static int safe_isspace(char c){
- return (c&0x80)==0 ? isspace(c) : 0;
-}
-static int safe_tolower(char c){
- return (c&0x80)==0 ? tolower(c) : c;
-}
-static int safe_isalnum(char c){
- return (c&0x80)==0 ? isalnum(c) : 0;
-}
typedef enum DocListType {
DL_DOCIDS, /* docids only */
DL_POSITIONS, /* docids + positions */
DL_POSITIONS_OFFSETS /* docids + positions + offsets */
@@ -98050,15 +97824,10 @@
POS_END = 0, /* end of this position list */
POS_COLUMN, /* followed by new column number */
POS_BASE
};
-/* MERGE_COUNT controls how often we merge segments (see comment at
-** top of file).
-*/
-#define MERGE_COUNT 16
-
/* utility functions */
/* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single
** record to prevent errors of the form:
**
@@ -98073,7006 +97842,2268 @@
# define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b)))
#else
# define SCRAMBLE(b)
#endif
-/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */
-#define VARINT_MAX 10
-
-/* Write a 64-bit variable-length integer to memory starting at p[0].
- * The length of data written will be between 1 and VARINT_MAX bytes.
- * The number of bytes written is returned. */
-static int fts3PutVarint(char *p, sqlite_int64 v){
+/*
+** Write a 64-bit variable-length integer to memory starting at p[0].
+** The length of data written will be between 1 and FTS3_VARINT_MAX bytes.
+** The number of bytes written is returned.
+*/
+SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){
unsigned char *q = (unsigned char *) p;
sqlite_uint64 vu = v;
do{
*q++ = (unsigned char) ((vu & 0x7f) | 0x80);
vu >>= 7;
}while( vu!=0 );
q[-1] &= 0x7f; /* turn off high bit in final byte */
- assert( q - (unsigned char *)p <= VARINT_MAX );
+ assert( q - (unsigned char *)p <= FTS3_VARINT_MAX );
return (int) (q - (unsigned char *)p);
}
-/* Read a 64-bit variable-length integer from memory starting at p[0].
- * Return the number of bytes read, or 0 on error.
- * The value is stored in *v. */
-static int fts3GetVarint(const char *p, sqlite_int64 *v){
+/*
+** Read a 64-bit variable-length integer from memory starting at p[0].
+** Return the number of bytes read, or 0 on error.
+** The value is stored in *v.
+*/
+SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){
const unsigned char *q = (const unsigned char *) p;
sqlite_uint64 x = 0, y = 1;
while( (*q & 0x80) == 0x80 ){
x += y * (*q++ & 0x7f);
y <<= 7;
- if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */
+ if( q - (unsigned char *)p >= FTS3_VARINT_MAX ){ /* bad data */
assert( 0 );
return 0;
}
}
x += y * (*q++);
*v = (sqlite_int64) x;
return (int) (q - (unsigned char *)p);
}
-static int fts3GetVarint32(const char *p, int *pi){
+/*
+** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a
+** 32-bit integer before it is returned.
+*/
+SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){
sqlite_int64 i;
- int ret = fts3GetVarint(p, &i);
+ int ret = sqlite3Fts3GetVarint(p, &i);
*pi = (int) i;
assert( *pi==i );
return ret;
}
-/*******************************************************************/
-/* DataBuffer is used to collect data into a buffer in piecemeal
-** fashion. It implements the usual distinction between amount of
-** data currently stored (nData) and buffer capacity (nCapacity).
-**
-** dataBufferInit - create a buffer with given initial capacity.
-** dataBufferReset - forget buffer's data, retaining capacity.
-** dataBufferDestroy - free buffer's data.
-** dataBufferSwap - swap contents of two buffers.
-** dataBufferExpand - expand capacity without adding data.
-** dataBufferAppend - append data.
-** dataBufferAppend2 - append two pieces of data at once.
-** dataBufferReplace - replace buffer's data.
-*/
-typedef struct DataBuffer {
- char *pData; /* Pointer to malloc'ed buffer. */
- int nCapacity; /* Size of pData buffer. */
- int nData; /* End of data loaded into pData. */
-} DataBuffer;
-
-static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
- assert( nCapacity>=0 );
- pBuffer->nData = 0;
- pBuffer->nCapacity = nCapacity;
- pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
-}
-static void dataBufferReset(DataBuffer *pBuffer){
- pBuffer->nData = 0;
-}
-static void dataBufferDestroy(DataBuffer *pBuffer){
- if( pBuffer->pData!=NULL ) sqlite3_free(pBuffer->pData);
- SCRAMBLE(pBuffer);
-}
-static void dataBufferSwap(DataBuffer *pBuffer1, DataBuffer *pBuffer2){
- DataBuffer tmp = *pBuffer1;
- *pBuffer1 = *pBuffer2;
- *pBuffer2 = tmp;
-}
-static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
- assert( nAddCapacity>0 );
- /* TODO(shess) Consider expanding more aggressively. Note that the
- ** underlying malloc implementation may take care of such things for
- ** us already.
- */
- if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
- pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
- pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
- }
-}
-static void dataBufferAppend(DataBuffer *pBuffer,
- const char *pSource, int nSource){
- assert( nSource>0 && pSource!=NULL );
- dataBufferExpand(pBuffer, nSource);
- memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
- pBuffer->nData += nSource;
-}
-static void dataBufferAppend2(DataBuffer *pBuffer,
- const char *pSource1, int nSource1,
- const char *pSource2, int nSource2){
- assert( nSource1>0 && pSource1!=NULL );
- assert( nSource2>0 && pSource2!=NULL );
- dataBufferExpand(pBuffer, nSource1+nSource2);
- memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
- memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
- pBuffer->nData += nSource1+nSource2;
-}
-static void dataBufferReplace(DataBuffer *pBuffer,
- const char *pSource, int nSource){
- dataBufferReset(pBuffer);
- dataBufferAppend(pBuffer, pSource, nSource);
-}
-
-/* StringBuffer is a null-terminated version of DataBuffer. */
-typedef struct StringBuffer {
- DataBuffer b; /* Includes null terminator. */
-} StringBuffer;
-
-static void initStringBuffer(StringBuffer *sb){
- dataBufferInit(&sb->b, 100);
- dataBufferReplace(&sb->b, "", 1);
-}
-static int stringBufferLength(StringBuffer *sb){
- return sb->b.nData-1;
-}
-static char *stringBufferData(StringBuffer *sb){
- return sb->b.pData;
-}
-static void stringBufferDestroy(StringBuffer *sb){
- dataBufferDestroy(&sb->b);
-}
-
-static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){
- assert( sb->b.nData>0 );
- if( nFrom>0 ){
- sb->b.nData--;
- dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
- }
-}
-static void append(StringBuffer *sb, const char *zFrom){
- nappend(sb, zFrom, strlen(zFrom));
-}
-
-/* Append a list of strings separated by commas. */
-static void appendList(StringBuffer *sb, int nString, char **azString){
+/*
+** Return the number of bytes required to store the value passed as the
+** first argument in varint form.
+*/
+SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){
+ int i = 0;
+ do{
+ i++;
+ v >>= 7;
+ }while( v!=0 );
+ return i;
+}
+
+/*
+** Convert an SQL-style quoted string into a normal string by removing
+** the quote characters. The conversion is done in-place. If the
+** input does not begin with a quote character, then this routine
+** is a no-op.
+**
+** Examples:
+**
+** "abc" becomes abc
+** 'xyz' becomes xyz
+** [pqr] becomes pqr
+** `mno` becomes mno
+*/
+SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){
+ int quote;
+ int i, j;
+
+ quote = z[0];
+ switch( quote ){
+ case '\'': break;
+ case '"': break;
+ case '`': break; /* For MySQL compatibility */
+ case '[': quote = ']'; break; /* For MS SqlServer compatibility */
+ default: return;
+ }
+ for(i=1, j=0; z[i]; i++){
+ if( z[i]==quote ){
+ if( z[i+1]==quote ){
+ z[j++] = quote;
+ i++;
+ }else{
+ z[j++] = 0;
+ break;
+ }
+ }else{
+ z[j++] = z[i];
+ }
+ }
+}
+
+static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){
+ sqlite3_int64 iVal;
+ *pp += sqlite3Fts3GetVarint(*pp, &iVal);
+ *pVal += iVal;
+}
+
+static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){
+ if( *pp>=pEnd ){
+ *pp = 0;
+ }else{
+ fts3GetDeltaVarint(pp, pVal);
+ }
+}
+
+
+/*
+** The Fts3Cursor.eType member is always set to one of the following.
+*/
+#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
+#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
+#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
+
+static Fts3Table *cursor_vtab(Fts3Cursor *c){
+ return (Fts3Table *) c->base.pVtab;
+}
+
+/*
+** The xDisconnect() virtual table method.
+*/
+static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ int i;
+
+ assert( p->nPendingData==0 );
+
+ /* Free any prepared statements held */
+ for(i=0; i<SizeofArray(p->aStmt); i++){
+ sqlite3_finalize(p->aStmt[i]);
+ }
+ for(i=0; i<p->nLeavesStmt; i++){
+ sqlite3_finalize(p->aLeavesStmt[i]);
+ }
+ sqlite3_free(p->zSelectLeaves);
+ sqlite3_free(p->aLeavesStmt);
+
+ /* Invoke the tokenizer destructor to free the tokenizer. */
+ p->pTokenizer->pModule->xDestroy(p->pTokenizer);
+
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+/*
+** The xDestroy() virtual table method.
+*/
+static int fts3DestroyMethod(sqlite3_vtab *pVtab){
+ int rc; /* Return code */
+ Fts3Table *p = (Fts3Table *)pVtab;
+
+ /* Create a script to drop the underlying three storage tables. */
+ char *zSql = sqlite3_mprintf(
+ "DROP TABLE IF EXISTS %Q.'%q_content';"
+ "DROP TABLE IF EXISTS %Q.'%q_segments';"
+ "DROP TABLE IF EXISTS %Q.'%q_segdir';",
+ p->zDb, p->zName, p->zDb, p->zName, p->zDb, p->zName
+ );
+
+ /* If malloc has failed, set rc to SQLITE_NOMEM. Otherwise, try to
+ ** execute the SQL script created above.
+ */
+ if( zSql ){
+ rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
+ sqlite3_free(zSql);
+ }else{
+ rc = SQLITE_NOMEM;
+ }
+
+ /* If everything has worked, invoke fts3DisconnectMethod() to free the
+ ** memory associated with the Fts3Table structure and return SQLITE_OK.
+ ** Otherwise, return an SQLite error code.
+ */
+ return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc);
+}
+
+
+/*
+** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table
+** passed as the first argument. This is done as part of the xConnect()
+** and xCreate() methods.
+*/
+static int fts3DeclareVtab(Fts3Table *p){
+ int i; /* Iterator variable */
+ int rc; /* Return code */
+ char *zSql; /* SQL statement passed to declare_vtab() */
+ char *zCols; /* List of user defined columns */
+
+ /* Create a list of user columns for the virtual table */
+ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
+ for(i=1; zCols && i<p->nColumn; i++){
+ zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]);
+ }
+
+ /* Create the whole "CREATE TABLE" statement to pass to SQLite */
+ zSql = sqlite3_mprintf(
+ "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
+ );
+
+ if( !zCols || !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_declare_vtab(p->db, zSql);
+ }
+
+ sqlite3_free(zSql);
+ sqlite3_free(zCols);
+ return rc;
+}
+
+/*
+** Create the backing store tables (%_content, %_segments and %_segdir)
+** required by the FTS3 table passed as the only argument. This is done
+** as part of the vtab xCreate() method.
+*/
+static int fts3CreateTables(Fts3Table *p){
+ int rc; /* Return code */
+ int i; /* Iterator variable */
+ char *zContentCols; /* Columns of %_content table */
+ char *zSql; /* SQL script to create required tables */
+
+ /* Create a list of user columns for the content table */
+ zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
+ for(i=0; zContentCols && i<p->nColumn; i++){
+ char *z = p->azColumn[i];
+ zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z);
+ }
+
+ /* Create the whole SQL script */
+ zSql = sqlite3_mprintf(
+ "CREATE TABLE %Q.'%q_content'(%s);"
+ "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);"
+ "CREATE TABLE %Q.'%q_segdir'("
+ "level INTEGER,"
+ "idx INTEGER,"
+ "start_block INTEGER,"
+ "leaves_end_block INTEGER,"
+ "end_block INTEGER,"
+ "root BLOB,"
+ "PRIMARY KEY(level, idx)"
+ ");",
+ p->zDb, p->zName, zContentCols, p->zDb, p->zName, p->zDb, p->zName
+ );
+
+ /* Unless a malloc() failure has occurred, execute the SQL script to
+ ** create the tables used to store data for this FTS3 virtual table.
+ */
+ if( zContentCols==0 || zSql==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
+ }
+
+ sqlite3_free(zSql);
+ sqlite3_free(zContentCols);
+ return rc;
+}
+
+/*
+** This function is the implementation of both the xConnect and xCreate
+** methods of the FTS3 virtual table.
+**
+** The argv[] array contains the following:
+**
+** argv[0] -> module name
+** argv[1] -> database name
+** argv[2] -> table name
+** argv[...] -> "column name" fields...
+*/
+int fts3InitVtab(
+ int isCreate, /* True for xCreate, false for xConnect */
+ sqlite3 *db, /* The SQLite database connection */
+ void *pAux, /* Hash table containing tokenizers */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
+ char **pzErr /* Write any error message here */
+){
+ Fts3Hash *pHash = (Fts3Hash *)pAux;
+ Fts3Table *p; /* Pointer to allocated vtab */
+ int rc; /* Return code */
int i;
- for(i=0; i<nString; ++i){
- if( i>0 ) append(sb, ", ");
- append(sb, azString[i]);
- }
-}
-
-static int endsInWhiteSpace(StringBuffer *p){
- return stringBufferLength(p)>0 &&
- safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
-}
-
-/* If the StringBuffer ends in something other than white space, add a
-** single space character to the end.
-*/
-static void appendWhiteSpace(StringBuffer *p){
- if( stringBufferLength(p)==0 ) return;
- if( !endsInWhiteSpace(p) ) append(p, " ");
-}
-
-/* Remove white space from the end of the StringBuffer */
-static void trimWhiteSpace(StringBuffer *p){
- while( endsInWhiteSpace(p) ){
- p->b.pData[--p->b.nData-1] = '\0';
- }
-}
-
-/*******************************************************************/
-/* DLReader is used to read document elements from a doclist. The
-** current docid is cached, so dlrDocid() is fast. DLReader does not
-** own the doclist buffer.
-**
-** dlrAtEnd - true if there's no more data to read.
-** dlrDocid - docid of current document.
-** dlrDocData - doclist data for current document (including docid).
-** dlrDocDataBytes - length of same.
-** dlrAllDataBytes - length of all remaining data.
-** dlrPosData - position data for current document.
-** dlrPosDataLen - length of pos data for current document (incl POS_END).
-** dlrStep - step to current document.
-** dlrInit - initial for doclist of given type against given data.
-** dlrDestroy - clean up.
-**
-** Expected usage is something like:
-**
-** DLReader reader;
-** dlrInit(&reader, pData, nData);
-** while( !dlrAtEnd(&reader) ){
-** // calls to dlrDocid() and kin.
-** dlrStep(&reader);
-** }
-** dlrDestroy(&reader);
-*/
-typedef struct DLReader {
- DocListType iType;
- const char *pData;
- int nData;
-
- sqlite_int64 iDocid;
- int nElement;
-} DLReader;
-
-static int dlrAtEnd(DLReader *pReader){
- assert( pReader->nData>=0 );
- return pReader->nData==0;
-}
-static sqlite_int64 dlrDocid(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->iDocid;
-}
-static const char *dlrDocData(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->pData;
-}
-static int dlrDocDataBytes(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->nElement;
-}
-static int dlrAllDataBytes(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
- return pReader->nData;
-}
-/* TODO(shess) Consider adding a field to track iDocid varint length
-** to make these two functions faster. This might matter (a tiny bit)
-** for queries.
-*/
-static const char *dlrPosData(DLReader *pReader){
- sqlite_int64 iDummy;
- int n = fts3GetVarint(pReader->pData, &iDummy);
- assert( !dlrAtEnd(pReader) );
- return pReader->pData+n;
-}
-static int dlrPosDataLen(DLReader *pReader){
- sqlite_int64 iDummy;
- int n = fts3GetVarint(pReader->pData, &iDummy);
- assert( !dlrAtEnd(pReader) );
- return pReader->nElement-n;
-}
-static void dlrStep(DLReader *pReader){
- assert( !dlrAtEnd(pReader) );
-
- /* Skip past current doclist element. */
- assert( pReader->nElement<=pReader->nData );
- pReader->pData += pReader->nElement;
- pReader->nData -= pReader->nElement;
-
- /* If there is more data, read the next doclist element. */
- if( pReader->nData!=0 ){
- sqlite_int64 iDocidDelta;
- int iDummy, n = fts3GetVarint(pReader->pData, &iDocidDelta);
- pReader->iDocid += iDocidDelta;
- if( pReader->iType>=DL_POSITIONS ){
- assert( n<pReader->nData );
- while( 1 ){
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- assert( n<=pReader->nData );
- if( iDummy==POS_END ) break;
- if( iDummy==POS_COLUMN ){
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- assert( n<pReader->nData );
- }else if( pReader->iType==DL_POSITIONS_OFFSETS ){
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- n += fts3GetVarint32(pReader->pData+n, &iDummy);
- assert( n<pReader->nData );
- }
- }
- }
- pReader->nElement = n;
- assert( pReader->nElement<=pReader->nData );
- }
-}
-static void dlrInit(DLReader *pReader, DocListType iType,
- const char *pData, int nData){
- assert( pData!=NULL && nData!=0 );
- pReader->iType = iType;
- pReader->pData = pData;
- pReader->nData = nData;
- pReader->nElement = 0;
- pReader->iDocid = 0;
-
- /* Load the first element's data. There must be a first element. */
- dlrStep(pReader);
-}
-static void dlrDestroy(DLReader *pReader){
- SCRAMBLE(pReader);
-}
-
-#ifndef NDEBUG
-/* Verify that the doclist can be validly decoded. Also returns the
-** last docid found because it is convenient in other assertions for
-** DLWriter.
-*/
-static void docListValidate(DocListType iType, const char *pData, int nData,
- sqlite_int64 *pLastDocid){
- sqlite_int64 iPrevDocid = 0;
- assert( nData>0 );
- assert( pData!=0 );
- assert( pData+nData>pData );
- while( nData!=0 ){
- sqlite_int64 iDocidDelta;
- int n = fts3GetVarint(pData, &iDocidDelta);
- iPrevDocid += iDocidDelta;
- if( iType>DL_DOCIDS ){
- int iDummy;
+ int nByte;
+ int iCol;
+ int nString = 0;
+ int nCol = 0;
+ char *zCsr;
+ int nDb;
+ int nName;
+
+ const char *zTokenizer = 0;
+ sqlite3_tokenizer *pTokenizer; /* Tokenizer for this table */
+
+ nDb = strlen(argv[1]) + 1;
+ nName = strlen(argv[2]) + 1;
+ for(i=3; i<argc; i++){
+ char const *z = argv[i];
+ rc = sqlite3Fts3InitTokenizer(pHash, z, &pTokenizer, &zTokenizer, pzErr);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ if( z!=zTokenizer ){
+ nString += strlen(z) + 1;
+ }
+ }
+ nCol = argc - 3 - (zTokenizer!=0);
+ if( zTokenizer==0 ){
+ rc = sqlite3Fts3InitTokenizer(pHash, 0, &pTokenizer, 0, pzErr);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ assert( pTokenizer );
+ }
+
+ /* Allocate and populate the Fts3Table structure. */
+ nByte = sizeof(Fts3Table) + /* Fts3Table */
+ nCol * sizeof(char *) + /* azColumn */
+ nName + /* zName */
+ nDb + /* zDb */
+ nString; /* Space for azColumn strings */
+ p = (Fts3Table*)sqlite3_malloc(nByte);
+ if( p==0 ){
+ rc = SQLITE_NOMEM;
+ goto fts3_init_out;
+ }
+ memset(p, 0, nByte);
+
+ p->db = db;
+ p->nColumn = nCol;
+ p->nPendingData = 0;
+ p->azColumn = (char **)&p[1];
+ p->pTokenizer = pTokenizer;
+ zCsr = (char *)&p->azColumn[nCol];
+
+ fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1);
+
+ /* Fill in the zName and zDb fields of the vtab structure. */
+ p->zName = zCsr;
+ memcpy(zCsr, argv[2], nName);
+ zCsr += nName;
+ p->zDb = zCsr;
+ memcpy(zCsr, argv[1], nDb);
+ zCsr += nDb;
+
+ /* Fill in the azColumn array */
+ iCol = 0;
+ for(i=3; i<argc; i++){
+ if( argv[i]!=zTokenizer ){
+ char *z;
+ int n;
+ z = (char *)sqlite3Fts3NextToken(argv[i], &n);
+ memcpy(zCsr, z, n);
+ zCsr[n] = '\0';
+ sqlite3Fts3Dequote(zCsr);
+ p->azColumn[iCol++] = zCsr;
+ zCsr += n+1;
+ assert( zCsr <= &((char *)p)[nByte] );
+ }
+ }
+
+ /* If this is an xCreate call, create the underlying tables in the
+ ** database. TODO: For xConnect(), it could verify that said tables exist.
+ */
+ if( isCreate ){
+ rc = fts3CreateTables(p);
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+ }
+
+ rc = fts3DeclareVtab(p);
+ if( rc!=SQLITE_OK ) goto fts3_init_out;
+
+ *ppVTab = &p->base;
+
+fts3_init_out:
+ if( rc!=SQLITE_OK ){
+ if( p ) fts3DisconnectMethod((sqlite3_vtab *)p);
+ else if( pTokenizer ) pTokenizer->pModule->xDestroy(pTokenizer);
+ }
+ return rc;
+}
+
+/*
+** The xConnect() and xCreate() methods for the virtual table. All the
+** work is done in function fts3InitVtab().
+*/
+static int fts3ConnectMethod(
+ sqlite3 *db, /* Database connection */
+ void *pAux, /* Pointer to tokenizer hash table */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
+}
+static int fts3CreateMethod(
+ sqlite3 *db, /* Database connection */
+ void *pAux, /* Pointer to tokenizer hash table */
+ int argc, /* Number of elements in argv array */
+ const char * const *argv, /* xCreate/xConnect argument array */
+ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
+ char **pzErr /* OUT: sqlite3_malloc'd error message */
+){
+ return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
+}
+
+/*
+** Implementation of the xBestIndex method for FTS3 tables. There
+** are three possible strategies, in order of preference:
+**
+** 1. Direct lookup by rowid or docid.
+** 2. Full-text search using a MATCH operator on a non-docid column.
+** 3. Linear scan of %_content table.
+*/
+static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
+ Fts3Table *p = (Fts3Table *)pVTab;
+ int i; /* Iterator variable */
+ int iCons = -1; /* Index of constraint to use */
+
+ /* By default use a full table scan. This is an expensive option,
+ ** so search through the constraints to see if a more efficient
+ ** strategy is possible.
+ */
+ pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
+ pInfo->estimatedCost = 500000;
+ for(i=0; i<pInfo->nConstraint; i++){
+ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
+ if( pCons->usable==0 ) continue;
+
+ /* A direct lookup on the rowid or docid column. This is the best
+ ** strategy in all cases. Assign a cost of 1.0 and return early.
+ */
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ
+ && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 )
+ ){
+ pInfo->idxNum = FTS3_DOCID_SEARCH;
+ pInfo->estimatedCost = 1.0;
+ iCons = i;
+ break;
+ }
+
+ /* A MATCH constraint. Use a full-text search.
+ **
+ ** If there is more than one MATCH constraint available, use the first
+ ** one encountered. If there is both a MATCH constraint and a direct
+ ** rowid/docid lookup, prefer the rowid/docid strategy.
+ */
+ if( iCons<0
+ && pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH
+ && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn
+ ){
+ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn;
+ pInfo->estimatedCost = 2.0;
+ iCons = i;
+ }
+ }
+
+ if( iCons>=0 ){
+ pInfo->aConstraintUsage[iCons].argvIndex = 1;
+ pInfo->aConstraintUsage[iCons].omit = 1;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of xOpen method.
+*/
+static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
+ sqlite3_vtab_cursor *pCsr; /* Allocated cursor */
+
+ /* Allocate a buffer large enough for an Fts3Cursor structure. If the
+ ** allocation succeeds, zero it and return SQLITE_OK. Otherwise,
+ ** if the allocation fails, return SQLITE_NOMEM.
+ */
+ *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor));
+ if( !pCsr ){
+ return SQLITE_NOMEM;
+ }
+ memset(pCsr, 0, sizeof(Fts3Cursor));
+ return SQLITE_OK;
+}
+
+/****************************************************************/
+/****************************************************************/
+/****************************************************************/
+/****************************************************************/
+
+
+/*
+** Close the cursor. For additional information see the documentation
+** on the xClose method of the virtual table interface.
+*/
+static int fulltextClose(sqlite3_vtab_cursor *pCursor){
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+ sqlite3_finalize(pCsr->pStmt);
+ sqlite3Fts3ExprFree(pCsr->pExpr);
+ sqlite3_free(pCsr->aDoclist);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+static int fts3CursorSeek(Fts3Cursor *pCsr){
+ if( pCsr->isRequireSeek ){
+ pCsr->isRequireSeek = 0;
+ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId);
+ if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){
+ return SQLITE_OK;
+ }else{
+ int rc;
+ pCsr->isEof = 1;
+ if( SQLITE_OK==(rc = sqlite3_reset(pCsr->pStmt)) ){
+ rc = SQLITE_ERROR;
+ }
+ return rc;
+ }
+ }else{
+ return SQLITE_OK;
+ }
+}
+
+static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
+ int rc = SQLITE_OK; /* Return code */
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+
+ if( pCsr->aDoclist==0 ){
+ if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){
+ pCsr->isEof = 1;
+ rc = sqlite3_reset(pCsr->pStmt);
+ }
+ }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){
+ pCsr->isEof = 1;
+ }else{
+ sqlite3_reset(pCsr->pStmt);
+ fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId);
+ pCsr->isRequireSeek = 1;
+ }
+ return rc;
+}
+
+
+/*
+** The buffer pointed to by argument zNode (size nNode bytes) contains the
+** root node of a b-tree segment. The segment is guaranteed to be at least
+** one level high (i.e. the root node is not also a leaf). If successful,
+** this function locates the leaf node of the segment that may contain the
+** term specified by arguments zTerm and nTerm and writes its block number
+** to *piLeaf.
+**
+** It is possible that the returned leaf node does not contain the specified
+** term. However, if the segment does contain said term, it is stored on
+** the identified leaf node. Because this function only inspects interior
+** segment nodes (and never loads leaf nodes into memory), it is not possible
+** to be sure.
+**
+** If an error occurs, an error code other than SQLITE_OK is returned.
+*/
+static int fts3SelectLeaf(
+ Fts3Table *p, /* Virtual table handle */
+ const char *zTerm, /* Term to select leaves for */
+ int nTerm, /* Size of term zTerm in bytes */
+ const char *zNode, /* Buffer containing segment interior node */
+ int nNode, /* Size of buffer at zNode */
+ sqlite3_int64 *piLeaf /* Selected leaf node */
+){
+ int rc = SQLITE_OK; /* Return code */
+ const char *zCsr = zNode; /* Cursor to iterate through node */
+ const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
+ char *zBuffer = 0; /* Buffer to load terms into */
+ int nAlloc = 0; /* Size of allocated buffer */
+
+ while( 1 ){
+ int isFirstTerm = 1; /* True when processing first term on page */
+ int iHeight; /* Height of this node in tree */
+ sqlite3_int64 iChild; /* Block id of child node to descend to */
+ int nBlock; /* Size of child node in bytes */
+
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight);
+ zCsr += sqlite3Fts3GetVarint(zCsr, &iChild);
+
+ while( zCsr<zEnd ){
+ int cmp; /* memcmp() result */
+ int nSuffix; /* Size of term suffix */
+ int nPrefix = 0; /* Size of term prefix */
+ int nBuffer; /* Total term size */
+
+ /* Load the next term on the node into zBuffer */
+ if( !isFirstTerm ){
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix);
+ }
+ isFirstTerm = 0;
+ zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix);
+ if( nPrefix+nSuffix>nAlloc ){
+ char *zNew;
+ nAlloc = (nPrefix+nSuffix) * 2;
+ zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+ if( !zNew ){
+ sqlite3_free(zBuffer);
+ return SQLITE_NOMEM;
+ }
+ zBuffer = zNew;
+ }
+ memcpy(&zBuffer[nPrefix], zCsr, nSuffix);
+ nBuffer = nPrefix + nSuffix;
+ zCsr += nSuffix;
+
+ /* Compare the term we are searching for with the term just loaded from
+ ** the interior node. If the specified term is greater than or equal
+ ** to the term from the interior node, then all terms on the sub-tree
+ ** headed by node iChild are smaller than zTerm. No need to search
+ ** iChild.
+ **
+ ** If the interior node term is larger than the specified term, then
+ ** the tree headed by iChild may contain the specified term.
+ */
+ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer));
+ if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break;
+ iChild++;
+ };
+
+ /* If (iHeight==1), the children of this interior node are leaves. The
+ ** specified term may be present on leaf node iChild.
+ */
+ if( iHeight==1 ){
+ *piLeaf = iChild;
+ break;
+ }
+
+ /* Descend to interior node iChild. */
+ rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
+ if( rc!=SQLITE_OK ) break;
+ zEnd = &zCsr[nBlock];
+ }
+ sqlite3_free(zBuffer);
+ return rc;
+}
+
+static void fts3PutDeltaVarint(
+ char **pp,
+ sqlite3_int64 *piPrev,
+ sqlite3_int64 iVal
+){
+ assert( iVal-*piPrev > 0 );
+ *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev);
+ *piPrev = iVal;
+}
+
+static void fts3PoslistCopy(char **pp, char **ppPoslist){
+ char *pEnd = *ppPoslist;
+ char c = 0;
+ while( *pEnd | c ) c = *pEnd++ & 0x80;
+ pEnd++;
+ if( pp ){
+ int n = pEnd - *ppPoslist;
+ char *p = *pp;
+ memcpy(p, *ppPoslist, n);
+ p += n;
+ *pp = p;
+ }
+ *ppPoslist = pEnd;
+}
+
+static void fts3ColumnlistCopy(char **pp, char **ppPoslist){
+ char *pEnd = *ppPoslist;
+ char c = 0;
+ while( 0xFE & (*pEnd | c) ) c = *pEnd++ & 0x80;
+ if( pp ){
+ int n = pEnd - *ppPoslist;
+ char *p = *pp;
+ memcpy(p, *ppPoslist, n);
+ p += n;
+ *pp = p;
+ }
+ *ppPoslist = pEnd;
+}
+
+/*
+**
+*/
+static void fts3PoslistMerge(
+ char **pp, /* Output buffer */
+ char **pp1, /* Left input list */
+ char **pp2 /* Right input list */
+){
+ char *p = *pp;
+ char *p1 = *pp1;
+ char *p2 = *pp2;
+
+ while( *p1 && *p2 ){
+ int iCol1 = 0;
+ int iCol2 = 0;
+ if( *p1==0x01 ) sqlite3Fts3GetVarint32(&p1[1], &iCol1);
+ if( *p2==0x01 ) sqlite3Fts3GetVarint32(&p2[1], &iCol2);
+
+ if( iCol1==iCol2 ){
+ sqlite3_int64 i1 = 0;
+ sqlite3_int64 i2 = 0;
+ sqlite3_int64 iPrev = 0;
+ if( iCol1!=0 ){
+ int n;
+ *p++ = 0x01;
+ n = sqlite3Fts3PutVarint(p, iCol1);
+ p += n;
+ p1 += 1 + n;
+ p2 += 1 + n;
+ }
+ while( (*p1&0xFE) || (*p2&0xFE) ){
+ if( i1==i2 ){
+ fts3GetDeltaVarint(&p1, &i1); i1 -= 2;
+ fts3GetDeltaVarint(&p2, &i2); i2 -= 2;
+ }else if( i1<i2 ){
+ fts3GetDeltaVarint(&p1, &i1); i1 -= 2;
+ }else{
+ fts3GetDeltaVarint(&p2, &i2); i2 -= 2;
+ }
+ fts3PutDeltaVarint(&p, &iPrev, (i1<i2 ? i1 : i2) + 2); iPrev -= 2;
+ if( 0==(*p1&0xFE) ) i1 = 0x7FFFFFFF;
+ if( 0==(*p2&0xFE) ) i2 = 0x7FFFFFFF;
+ }
+ }else if( iCol1<iCol2 ){
+ fts3ColumnlistCopy(&p, &p1);
+ }else{
+ fts3ColumnlistCopy(&p, &p2);
+ }
+ }
+
+ *p++ = '\0';
+ *pp = p;
+ *pp1 = p1 + 1;
+ *pp2 = p2 + 1;
+}
+
+/*
+** nToken==1 searches for adjacent positions.
+*/
+static int fts3PoslistPhraseMerge(
+ char **pp, /* Output buffer */
+ int nToken, /* Maximum difference in token positions */
+ int isSaveLeft, /* Save the left position */
+ char **pp1, /* Left input list */
+ char **pp2 /* Right input list */
+){
+ char *p = (pp ? *pp : 0);
+ char *p1 = *pp1;
+ char *p2 = *pp2;
+
+ int iCol1 = 0;
+ int iCol2 = 0;
+ assert( *p1!=0 && *p2!=0 );
+ if( *p1==0x01 ){
+ p1++;
+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
+ }
+ if( *p2==0x01 ){
+ p2++;
+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
+ }
+
+ while( 1 ){
+ if( iCol1==iCol2 ){
+ char *pSave = p;
+ sqlite3_int64 iPrev = 0;
+ sqlite3_int64 iPos1 = 0;
+ sqlite3_int64 iPos2 = 0;
+
+ if( pp && iCol1 ){
+ *p++ = 0x01;
+ p += sqlite3Fts3PutVarint(p, iCol1);
+ }
+
+ assert( *p1!=0x00 && *p2!=0x00 && *p1!=0x01 && *p2!=0x01 );
+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
+
while( 1 ){
- n += fts3GetVarint32(pData+n, &iDummy);
- if( iDummy==POS_END ) break;
- if( iDummy==POS_COLUMN ){
- n += fts3GetVarint32(pData+n, &iDummy);
- }else if( iType>DL_POSITIONS ){
- n += fts3GetVarint32(pData+n, &iDummy);
- n += fts3GetVarint32(pData+n, &iDummy);
- }
- assert( n<=nData );
- }
- }
- assert( n<=nData );
- pData += n;
- nData -= n;
- }
- if( pLastDocid ) *pLastDocid = iPrevDocid;
-}
-#define ASSERT_VALID_DOCLIST(i, p, n, o) docListValidate(i, p, n, o)
-#else
-#define ASSERT_VALID_DOCLIST(i, p, n, o) assert( 1 )
-#endif
-
-/*******************************************************************/
-/* DLWriter is used to write doclist data to a DataBuffer. DLWriter
-** always appends to the buffer and does not own it.
-**
-** dlwInit - initialize to write a given type doclistto a buffer.
-** dlwDestroy - clear the writer's memory. Does not free buffer.
-** dlwAppend - append raw doclist data to buffer.
-** dlwCopy - copy next doclist from reader to writer.
-** dlwAdd - construct doclist element and append to buffer.
-** Only apply dlwAdd() to DL_DOCIDS doclists (else use PLWriter).
-*/
-typedef struct DLWriter {
- DocListType iType;
- DataBuffer *b;
- sqlite_int64 iPrevDocid;
-#ifndef NDEBUG
- int has_iPrevDocid;
-#endif
-} DLWriter;
-
-static void dlwInit(DLWriter *pWriter, DocListType iType, DataBuffer *b){
- pWriter->b = b;
- pWriter->iType = iType;
- pWriter->iPrevDocid = 0;
-#ifndef NDEBUG
- pWriter->has_iPrevDocid = 0;
-#endif
-}
-static void dlwDestroy(DLWriter *pWriter){
- SCRAMBLE(pWriter);
-}
-/* iFirstDocid is the first docid in the doclist in pData. It is
-** needed because pData may point within a larger doclist, in which
-** case the first item would be delta-encoded.
-**
-** iLastDocid is the final docid in the doclist in pData. It is
-** needed to create the new iPrevDocid for future delta-encoding. The
-** code could decode the passed doclist to recreate iLastDocid, but
-** the only current user (docListMerge) already has decoded this
+ if( iPos2>iPos1 && iPos2<=iPos1+nToken ){
+ sqlite3_int64 iSave;
+ if( !pp ){
+ fts3PoslistCopy(0, &p2);
+ fts3PoslistCopy(0, &p1);
+ *pp1 = p1;
+ *pp2 = p2;
+ return 1;
+ }
+ iSave = isSaveLeft ? iPos1 : iPos2;
+ fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2;
+ pSave = 0;
+ }
+ if( iPos2<=iPos1 ){
+ if( (*p2&0xFE)==0 ) break;
+ fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
+ }else{
+ if( (*p1&0xFE)==0 ) break;
+ fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
+ }
+ }
+ if( pSave && pp ){
+ p = pSave;
+ }
+
+ fts3ColumnlistCopy(0, &p1);
+ fts3ColumnlistCopy(0, &p2);
+ assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 );
+ if( 0==*p1 || 0==*p2 ) break;
+
+ p1++;
+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
+ p2++;
+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
+ }
+
+ /* Advance pointer p1 or p2 (whichever corresponds to the smaller of
+ ** iCol1 and iCol2) so that it points to either the 0x00 that marks the
+ ** end of the position list, or the 0x01 that precedes the next
+ ** column-number in the position list.
+ */
+ else if( iCol1<iCol2 ){
+ fts3ColumnlistCopy(0, &p1);
+ if( 0==*p1 ) break;
+ p1++;
+ p1 += sqlite3Fts3GetVarint32(p1, &iCol1);
+ }else{
+ fts3ColumnlistCopy(0, &p2);
+ if( 0==*p2 ) break;
+ p2++;
+ p2 += sqlite3Fts3GetVarint32(p2, &iCol2);
+ }
+ }
+
+ fts3PoslistCopy(0, &p2);
+ fts3PoslistCopy(0, &p1);
+ *pp1 = p1;
+ *pp2 = p2;
+ if( !pp || *pp==p ){
+ return 0;
+ }
+ *p++ = 0x00;
+ *pp = p;
+ return 1;
+}
+
+static int fts3PoslistNearMerge(
+ char **pp, /* Output buffer */
+ char *aTmp, /* Temporary buffer space */
+ int nRight, /* Maximum difference in token positions */
+ int nLeft, /* Maximum difference in token positions */
+ char **pp1, /* Left input list */
+ char **pp2 /* Right input list */
+){
+ char *p1 = *pp1;
+ char *p2 = *pp2;
+
+ if( !pp ){
+ if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1;
+ *pp1 = p1;
+ *pp2 = p2;
+ return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1);
+ }else{
+ char *pTmp1 = aTmp;
+ char *pTmp2;
+ char *aTmp2;
+ int res = 1;
+
+ fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2);
+ aTmp2 = pTmp2 = pTmp1;
+ *pp1 = p1;
+ *pp2 = p2;
+ fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1);
+ if( pTmp1!=aTmp && pTmp2!=aTmp2 ){
+ fts3PoslistMerge(pp, &aTmp, &aTmp2);
+ }else if( pTmp1!=aTmp ){
+ fts3PoslistCopy(pp, &aTmp);
+ }else if( pTmp2!=aTmp2 ){
+ fts3PoslistCopy(pp, &aTmp2);
+ }else{
+ res = 0;
+ }
+
+ return res;
+ }
+}
+
+/*
+** Values that may be used as the first parameter to fts3DoclistMerge().
+*/
+#define MERGE_NOT 2 /* D + D -> D */
+#define MERGE_AND 3 /* D + D -> D */
+#define MERGE_OR 4 /* D + D -> D */
+#define MERGE_POS_OR 5 /* P + P -> P */
+#define MERGE_PHRASE 6 /* P + P -> D */
+#define MERGE_POS_PHRASE 7 /* P + P -> P */
+#define MERGE_NEAR 8 /* P + P -> D */
+#define MERGE_POS_NEAR 9 /* P + P -> P */
+
+/*
+** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2
+** (size n2 bytes). The output is written to pre-allocated buffer aBuffer,
+** which is guaranteed to be large enough to hold the results. The number
+** of bytes written to aBuffer is stored in *pnBuffer before returning.
+**
+** If successful, SQLITE_OK is returned. Otherwise, if a malloc error
+** occurs while allocating a temporary buffer as part of the merge operation,
+** SQLITE_NOMEM is returned.
+*/
+static int fts3DoclistMerge(
+ int mergetype, /* One of the MERGE_XXX constants */
+ int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
+ int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */
+ char *aBuffer, /* Pre-allocated output buffer */
+ int *pnBuffer, /* OUT: Bytes written to aBuffer */
+ char *a1, /* Buffer containing first doclist */
+ int n1, /* Size of buffer a1 */
+ char *a2, /* Buffer containing second doclist */
+ int n2 /* Size of buffer a2 */
+){
+ sqlite3_int64 i1 = 0;
+ sqlite3_int64 i2 = 0;
+ sqlite3_int64 iPrev = 0;
+
+ char *p = aBuffer;
+ char *p1 = a1;
+ char *p2 = a2;
+ char *pEnd1 = &a1[n1];
+ char *pEnd2 = &a2[n2];
+
+ assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR
+ || mergetype==MERGE_AND || mergetype==MERGE_NOT
+ || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
+ || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR
+ );
+
+ if( !aBuffer ){
+ return SQLITE_NOMEM;
+ }
+
+ /* Read the first docid from each doclist */
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+
+ switch( mergetype ){
+ case MERGE_OR:
+ case MERGE_POS_OR:
+ while( p1 || p2 ){
+ if( p2 && p1 && i1==i2 ){
+ fts3PutDeltaVarint(&p, &iPrev, i1);
+ if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }else if( !p2 || (p1 && i1<i2) ){
+ fts3PutDeltaVarint(&p, &iPrev, i1);
+ if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ }else{
+ fts3PutDeltaVarint(&p, &iPrev, i2);
+ if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }
+ }
+ break;
+
+ case MERGE_AND:
+ while( p1 && p2 ){
+ if( i1==i2 ){
+ fts3PutDeltaVarint(&p, &iPrev, i1);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }else if( i1<i2 ){
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ }else{
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }
+ }
+ break;
+
+ case MERGE_NOT:
+ while( p1 ){
+ if( p2 && i1==i2 ){
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }else if( !p2 || i1<i2 ){
+ fts3PutDeltaVarint(&p, &iPrev, i1);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ }else{
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }
+ }
+ break;
+
+ case MERGE_POS_PHRASE:
+ case MERGE_PHRASE: {
+ char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p);
+ while( p1 && p2 ){
+ if( i1==i2 ){
+ char *pSave = p;
+ sqlite3_int64 iPrevSave = iPrev;
+ fts3PutDeltaVarint(&p, &iPrev, i1);
+ if( 0==fts3PoslistPhraseMerge(ppPos, 1, 0, &p1, &p2) ){
+ p = pSave;
+ iPrev = iPrevSave;
+ }
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }else if( i1<i2 ){
+ fts3PoslistCopy(0, &p1);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ }else{
+ fts3PoslistCopy(0, &p2);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }
+ }
+ break;
+ }
+
+ case MERGE_POS_NEAR:
+ case MERGE_NEAR: {
+ char *aTmp = 0;
+ char **ppPos = 0;
+ if( mergetype==MERGE_POS_NEAR ){
+ ppPos = &p;
+ aTmp = sqlite3_malloc(2*(n1+n2));
+ if( !aTmp ){
+ return SQLITE_NOMEM;
+ }
+ }
+
+ while( p1 && p2 ){
+ if( i1==i2 ){
+ char *pSave = p;
+ sqlite3_int64 iPrevSave = iPrev;
+ fts3PutDeltaVarint(&p, &iPrev, i1);
+
+ if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){
+ iPrev = iPrevSave;
+ p = pSave;
+ }
+
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }else if( i1<i2 ){
+ fts3PoslistCopy(0, &p1);
+ fts3GetDeltaVarint2(&p1, pEnd1, &i1);
+ }else{
+ fts3PoslistCopy(0, &p2);
+ fts3GetDeltaVarint2(&p2, pEnd2, &i2);
+ }
+ }
+ sqlite3_free(aTmp);
+ break;
+ }
+
+ default:
+ assert(!"Invalid mergetype value passed to fts3DoclistMerge()");
+ }
+
+ *pnBuffer = (p-aBuffer);
+ return SQLITE_OK;
+}
+
+/*
+** A pointer to an instance of this structure is used as the context
+** argument to sqlite3Fts3SegReaderIterate()
+*/
+typedef struct TermSelect TermSelect;
+struct TermSelect {
+ int isReqPos;
+ char *aOutput; /* Malloc'd output buffer */
+ int nOutput; /* Size of output in bytes */
+};
+
+/*
+** This function is used as the sqlite3Fts3SegReaderIterate() callback when
+** querying the full-text index for a doclist associated with a term or
+** term-prefix.
+*/
+static int fts3TermSelectCb(
+ Fts3Table *p, /* Virtual table object */
+ void *pContext, /* Pointer to TermSelect structure */
+ char *zTerm,
+ int nTerm,
+ char *aDoclist,
+ int nDoclist
+){
+ TermSelect *pTS = (TermSelect *)pContext;
+ int nNew = pTS->nOutput + nDoclist;
+ char *aNew = sqlite3_malloc(nNew);
+
+ if( !aNew ){
+ return SQLITE_NOMEM;
+ }
+
+ if( pTS->nOutput==0 ){
+ /* If this is the first term selected, copy the doclist to the output
+ ** buffer using memcpy(). TODO: Add a way to transfer control of the
+ ** aDoclist buffer from the caller so as to avoid the memcpy().
+ */
+ memcpy(aNew, aDoclist, nDoclist);
+ }else{
+ /* The output buffer is not empty. Merge doclist aDoclist with the
+ ** existing output. This can only happen with prefix-searches (as
+ ** searches for exact terms return exactly one doclist).
+ */
+ int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR);
+ fts3DoclistMerge(mergetype, 0, 0,
+ aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist
+ );
+ }
+
+ sqlite3_free(pTS->aOutput);
+ pTS->aOutput = aNew;
+ pTS->nOutput = nNew;
+
+ return SQLITE_OK;
+}
+
+/*
+** This function retreives the doclist for the specified term (or term
+** prefix) from the database.
+**
+** The returned doclist may be in one of two formats, depending on the
+** value of parameter isReqPos. If isReqPos is zero, then the doclist is
+** a sorted list of delta-compressed docids. If isReqPos is non-zero,
+** then the returned list is in the same format as is stored in the
+** database without the found length specifier at the start of on-disk
+** doclists.
+*/
+static int fts3TermSelect(
+ Fts3Table *p, /* Virtual table handle */
+ int iColumn, /* Column to query (or -ve for all columns) */
+ const char *zTerm, /* Term to query for */
+ int nTerm, /* Size of zTerm in bytes */
+ int isPrefix, /* True for a prefix search */
+ int isReqPos, /* True to include position lists in output */
+ int *pnOut, /* OUT: Size of buffer at *ppOut */
+ char **ppOut /* OUT: Malloced result buffer */
+){
+ int i;
+ TermSelect tsc;
+ Fts3SegFilter filter; /* Segment term filter configuration */
+ Fts3SegReader **apSegment = 0; /* Array of segments to read data from */
+ int nSegment = 0; /* Size of apSegment array */
+ int nAlloc = 0; /* Allocated size of segment array */
+ int rc; /* Return code */
+ sqlite3_stmt *pStmt; /* SQL statement to scan %_segdir table */
+ int iAge = 0; /* Used to assign ages to segments */
+
+ /* Loop through the entire %_segdir table. For each segment, create a
+ ** Fts3SegReader to iterate through the subset of the segment leaves
+ ** that may contain a term that matches zTerm/nTerm. For non-prefix
+ ** searches, this is always a single leaf. For prefix searches, this
+ ** may be a contiguous block of leaves.
+ **
+ ** The code in this loop does not actually load any leaves into memory
+ ** (unless the root node happens to be a leaf). It simply examines the
+ ** b-tree structure to determine which leaves need to be inspected.
+ */
+ rc = sqlite3Fts3AllSegdirs(p, &pStmt);
+ while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
+ Fts3SegReader *pNew = 0;
+ int nRoot = sqlite3_column_bytes(pStmt, 4);
+ char const *zRoot = sqlite3_column_blob(pStmt, 4);
+ if( sqlite3_column_int64(pStmt, 1)==0 ){
+ /* The entire segment is stored on the root node (which must be a
+ ** leaf). Do not bother inspecting any data in this case, just
+ ** create a Fts3SegReader to scan the single leaf.
+ */
+ rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew);
+ }else{
+ sqlite3_int64 i1;
+ rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1);
+ if( rc==SQLITE_OK ){
+ sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2);
+ rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew);
+ }
+ }
+ iAge++;
+
+ /* If a new Fts3SegReader was allocated, add it to the apSegment array. */
+ assert( (rc==SQLITE_OK)==(pNew!=0) );
+ if( pNew ){
+ if( nSegment==nAlloc ){
+ nAlloc += 16;
+ Fts3SegReader **pArray = (Fts3SegReader **)sqlite3_realloc(
+ apSegment, nAlloc*sizeof(Fts3SegReader *)
+ );
+ if( !pArray ){
+ sqlite3Fts3SegReaderFree(p, pNew);
+ rc = SQLITE_NOMEM;
+ goto finished;
+ }
+ apSegment = pArray;
+ }
+ apSegment[nSegment++] = pNew;
+ }
+ }
+ if( rc!=SQLITE_DONE ){
+ assert( rc!=SQLITE_OK );
+ goto finished;
+ }
+
+ memset(&tsc, 0, sizeof(TermSelect));
+ tsc.isReqPos = isReqPos;
+
+ filter.flags = FTS3_SEGMENT_IGNORE_EMPTY
+ | (isPrefix ? FTS3_SEGMENT_PREFIX : 0)
+ | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0)
+ | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0);
+ filter.iCol = iColumn;
+ filter.zTerm = zTerm;
+ filter.nTerm = nTerm;
+
+ rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter,
+ fts3TermSelectCb, (void *)&tsc
+ );
+
+ if( rc==SQLITE_OK ){
+ *ppOut = tsc.aOutput;
+ *pnOut = tsc.nOutput;
+ }else{
+ sqlite3_free(tsc.aOutput);
+ }
+
+finished:
+ sqlite3_reset(pStmt);
+ for(i=0; i<nSegment; i++){
+ sqlite3Fts3SegReaderFree(p, apSegment[i]);
+ }
+ sqlite3_free(apSegment);
+ return rc;
+}
+
+
+/*
+** Return a DocList corresponding to the phrase *pPhrase.
+*/
+static int fts3PhraseSelect(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3Phrase *pPhrase, /* Phrase to return a doclist for */
+ int isReqPos, /* True if output should contain positions */
+ char **paOut, /* OUT: Pointer to malloc'd result buffer */
+ int *pnOut /* OUT: Size of buffer at *paOut */
+){
+ char *pOut = 0;
+ int nOut = 0;
+ int rc = SQLITE_OK;
+ int ii;
+ int iCol = pPhrase->iColumn;
+ int isTermPos = (pPhrase->nToken>1 || isReqPos);
+
+ assert( p->nPendingData==0 );
+
+ for(ii=0; ii<pPhrase->nToken; ii++){
+ struct PhraseToken *pTok = &pPhrase->aToken[ii];
+ char *z = pTok->z; /* Next token of the phrase */
+ int n = pTok->n; /* Size of z in bytes */
+ int isPrefix = pTok->isPrefix;/* True if token is a prefix */
+ char *pList; /* Pointer to token doclist */
+ int nList; /* Size of buffer at pList */
+
+ rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList);
+ if( rc!=SQLITE_OK ) break;
+
+ if( ii==0 ){
+ pOut = pList;
+ nOut = nList;
+ }else{
+ /* Merge the new term list and the current output. If this is the
+ ** last term in the phrase, and positions are not required in the
+ ** output of this function, the positions can be dropped as part
+ ** of this merge. Either way, the result of this merge will be
+ ** smaller than nList bytes. The code in fts3DoclistMerge() is written
+ ** so that it is safe to use pList as the output as well as an input
+ ** in this case.
+ */
+ int mergetype = MERGE_POS_PHRASE;
+ if( ii==pPhrase->nToken-1 && !isReqPos ){
+ mergetype = MERGE_PHRASE;
+ }
+ fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList);
+ sqlite3_free(pOut);
+ pOut = pList;
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ *paOut = pOut;
+ *pnOut = nOut;
+ }else{
+ sqlite3_free(pOut);
+ }
+ return rc;
+}
+
+/*
+** Evaluate the full-text expression pExpr against fts3 table pTab. Store
+** the resulting doclist in *paOut and *pnOut.
+*/
+static int evalFts3Expr(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3Expr *pExpr, /* Parsed fts3 expression */
+ char **paOut, /* OUT: Pointer to malloc'd result buffer */
+ int *pnOut /* OUT: Size of buffer at *paOut */
+){
+ int rc = SQLITE_OK; /* Return code */
+
+ /* Zero the output parameters. */
+ *paOut = 0;
+ *pnOut = 0;
+
+ if( pExpr ){
+ if( pExpr->eType==FTSQUERY_PHRASE ){
+ int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR);
+ rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut);
+ }else{
+ char *aLeft;
+ char *aRight;
+ int nLeft;
+ int nRight;
+
+ if( SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight))
+ && SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft))
+ ){
+ switch( pExpr->eType ){
+ case FTSQUERY_NEAR: {
+ Fts3Expr *pLeft;
+ Fts3Expr *pRight;
+ int mergetype = MERGE_NEAR;
+ int nParam1;
+ int nParam2;
+ char *aBuffer;
+
+ if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
+ mergetype = MERGE_POS_NEAR;
+ }
+ pLeft = pExpr->pLeft;
+ while( pLeft->eType==FTSQUERY_NEAR ){
+ pLeft=pLeft->pRight;
+ }
+ pRight = pExpr->pRight;
+ assert( pRight->eType==FTSQUERY_PHRASE );
+ assert( pLeft->eType==FTSQUERY_PHRASE );
+
+ nParam1 = pExpr->nNear+1;
+ nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2;
+ aBuffer = sqlite3_malloc(nLeft+nRight+1);
+ rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer,
+ pnOut, aLeft, nLeft, aRight, nRight
+ );
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(aBuffer);
+ }else{
+ *paOut = aBuffer;
+ }
+ sqlite3_free(aLeft);
+ break;
+ }
+
+ case FTSQUERY_OR: {
+ /* Allocate a buffer for the output. The maximum size is the
+ ** sum of the sizes of the two input buffers. The +1 term is
+ ** so that a buffer of zero bytes is never allocated - this can
+ ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM.
+ */
+ char *aBuffer = sqlite3_malloc(nRight+nLeft+1);
+ rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut,
+ aLeft, nLeft, aRight, nRight
+ );
+ *paOut = aBuffer;
+ sqlite3_free(aLeft);
+ break;
+ }
+
+ case FTSQUERY_AND:
+ case FTSQUERY_NOT: {
+ assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND );
+ fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut,
+ aLeft, nLeft, aRight, nRight
+ );
+ *paOut = aLeft;
+ break;
+ }
+ }
+ }
+ sqlite3_free(aRight);
+ }
+ }
+
+ return rc;
+}
+
+/*
+** This is the xFilter interface for the virtual table. See
+** the virtual table xFilter method documentation for additional
** information.
-*/
-/* TODO(shess) This has become just a helper for docListMerge.
-** Consider a refactor to make this cleaner.
-*/
-static void dlwAppend(DLWriter *pWriter,
- const char *pData, int nData,
- sqlite_int64 iFirstDocid, sqlite_int64 iLastDocid){
- sqlite_int64 iDocid = 0;
- char c[VARINT_MAX];
- int nFirstOld, nFirstNew; /* Old and new varint len of first docid. */
-#ifndef NDEBUG
- sqlite_int64 iLastDocidDelta;
-#endif
-
- /* Recode the initial docid as delta from iPrevDocid. */
- nFirstOld = fts3GetVarint(pData, &iDocid);
- assert( nFirstOld<nData || (nFirstOld==nData && pWriter->iType==DL_DOCIDS) );
- nFirstNew = fts3PutVarint(c, iFirstDocid-pWriter->iPrevDocid);
-
- /* Verify that the incoming doclist is valid AND that it ends with
- ** the expected docid. This is essential because we'll trust this
- ** docid in future delta-encoding.
- */
- ASSERT_VALID_DOCLIST(pWriter->iType, pData, nData, &iLastDocidDelta);
- assert( iLastDocid==iFirstDocid-iDocid+iLastDocidDelta );
-
- /* Append recoded initial docid and everything else. Rest of docids
- ** should have been delta-encoded from previous initial docid.
- */
- if( nFirstOld<nData ){
- dataBufferAppend2(pWriter->b, c, nFirstNew,
- pData+nFirstOld, nData-nFirstOld);
- }else{
- dataBufferAppend(pWriter->b, c, nFirstNew);
- }
- pWriter->iPrevDocid = iLastDocid;
-}
-static void dlwCopy(DLWriter *pWriter, DLReader *pReader){
- dlwAppend(pWriter, dlrDocData(pReader), dlrDocDataBytes(pReader),
- dlrDocid(pReader), dlrDocid(pReader));
-}
-static void dlwAdd(DLWriter *pWriter, sqlite_int64 iDocid){
- char c[VARINT_MAX];
- int n = fts3PutVarint(c, iDocid-pWriter->iPrevDocid);
-
- /* Docids must ascend. */
- assert( !pWriter->has_iPrevDocid || iDocid>pWriter->iPrevDocid );
- assert( pWriter->iType==DL_DOCIDS );
-
- dataBufferAppend(pWriter->b, c, n);
- pWriter->iPrevDocid = iDocid;
-#ifndef NDEBUG
- pWriter->has_iPrevDocid = 1;
-#endif
-}
-
-/*******************************************************************/
-/* PLReader is used to read data from a document's position list. As
-** the caller steps through the list, data is cached so that varints
-** only need to be decoded once.
-**
-** plrInit, plrDestroy - create/destroy a reader.
-** plrColumn, plrPosition, plrStartOffset, plrEndOffset - accessors
-** plrAtEnd - at end of stream, only call plrDestroy once true.
-** plrStep - step to the next element.
-*/
-typedef struct PLReader {
- /* These refer to the next position's data. nData will reach 0 when
- ** reading the last position, so plrStep() signals EOF by setting
- ** pData to NULL.
- */
- const char *pData;
- int nData;
-
- DocListType iType;
- int iColumn; /* the last column read */
- int iPosition; /* the last position read */
- int iStartOffset; /* the last start offset read */
- int iEndOffset; /* the last end offset read */
-} PLReader;
-
-static int plrAtEnd(PLReader *pReader){
- return pReader->pData==NULL;
-}
-static int plrColumn(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iColumn;
-}
-static int plrPosition(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iPosition;
-}
-static int plrStartOffset(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iStartOffset;
-}
-static int plrEndOffset(PLReader *pReader){
- assert( !plrAtEnd(pReader) );
- return pReader->iEndOffset;
-}
-static void plrStep(PLReader *pReader){
- int i, n;
-
- assert( !plrAtEnd(pReader) );
-
- if( pReader->nData==0 ){
- pReader->pData = NULL;
- return;
- }
-
- n = fts3GetVarint32(pReader->pData, &i);
- if( i==POS_COLUMN ){
- n += fts3GetVarint32(pReader->pData+n, &pReader->iColumn);
- pReader->iPosition = 0;
- pReader->iStartOffset = 0;
- n += fts3GetVarint32(pReader->pData+n, &i);
- }
- /* Should never see adjacent column changes. */
- assert( i!=POS_COLUMN );
-
- if( i==POS_END ){
- pReader->nData = 0;
- pReader->pData = NULL;
- return;
- }
-
- pReader->iPosition += i-POS_BASE;
- if( pReader->iType==DL_POSITIONS_OFFSETS ){
- n += fts3GetVarint32(pReader->pData+n, &i);
- pReader->iStartOffset += i;
- n += fts3GetVarint32(pReader->pData+n, &i);
- pReader->iEndOffset = pReader->iStartOffset+i;
- }
- assert( n<=pReader->nData );
- pReader->pData += n;
- pReader->nData -= n;
-}
-
-static void plrInit(PLReader *pReader, DLReader *pDLReader){
- pReader->pData = dlrPosData(pDLReader);
- pReader->nData = dlrPosDataLen(pDLReader);
- pReader->iType = pDLReader->iType;
- pReader->iColumn = 0;
- pReader->iPosition = 0;
- pReader->iStartOffset = 0;
- pReader->iEndOffset = 0;
- plrStep(pReader);
-}
-static void plrDestroy(PLReader *pReader){
- SCRAMBLE(pReader);
-}
-
-/*******************************************************************/
-/* PLWriter is used in constructing a document's position list. As a
-** convenience, if iType is DL_DOCIDS, PLWriter becomes a no-op.
-** PLWriter writes to the associated DLWriter's buffer.
-**
-** plwInit - init for writing a document's poslist.
-** plwDestroy - clear a writer.
-** plwAdd - append position and offset information.
-** plwCopy - copy next position's data from reader to writer.
-** plwTerminate - add any necessary doclist terminator.
-**
-** Calling plwAdd() after plwTerminate() may result in a corrupt
-** doclist.
-*/
-/* TODO(shess) Until we've written the second item, we can cache the
-** first item's information. Then we'd have three states:
-**
-** - initialized with docid, no positions.
-** - docid and one position.
-** - docid and multiple positions.
-**
-** Only the last state needs to actually write to dlw->b, which would
-** be an improvement in the DLCollector case.
-*/
-typedef struct PLWriter {
- DLWriter *dlw;
-
- int iColumn; /* the last column written */
- int iPos; /* the last position written */
- int iOffset; /* the last start offset written */
-} PLWriter;
-
-/* TODO(shess) In the case where the parent is reading these values
-** from a PLReader, we could optimize to a copy if that PLReader has
-** the same type as pWriter.
-*/
-static void plwAdd(PLWriter *pWriter, int iColumn, int iPos,
- int iStartOffset, int iEndOffset){
- /* Worst-case space for POS_COLUMN, iColumn, iPosDelta,
- ** iStartOffsetDelta, and iEndOffsetDelta.
- */
- char c[5*VARINT_MAX];
- int n = 0;
-
- /* Ban plwAdd() after plwTerminate(). */
- assert( pWriter->iPos!=-1 );
-
- if( pWriter->dlw->iType==DL_DOCIDS ) return;
-
- if( iColumn!=pWriter->iColumn ){
- n += fts3PutVarint(c+n, POS_COLUMN);
- n += fts3PutVarint(c+n, iColumn);
- pWriter->iColumn = iColumn;
- pWriter->iPos = 0;
- pWriter->iOffset = 0;
- }
- assert( iPos>=pWriter->iPos );
- n += fts3PutVarint(c+n, POS_BASE+(iPos-pWriter->iPos));
- pWriter->iPos = iPos;
- if( pWriter->dlw->iType==DL_POSITIONS_OFFSETS ){
- assert( iStartOffset>=pWriter->iOffset );
- n += fts3PutVarint(c+n, iStartOffset-pWriter->iOffset);
- pWriter->iOffset = iStartOffset;
- assert( iEndOffset>=iStartOffset );
- n += fts3PutVarint(c+n, iEndOffset-iStartOffset);
- }
- dataBufferAppend(pWriter->dlw->b, c, n);
-}
-static void plwCopy(PLWriter *pWriter, PLReader *pReader){
- plwAdd(pWriter, plrColumn(pReader), plrPosition(pReader),
- plrStartOffset(pReader), plrEndOffset(pReader));
-}
-static void plwInit(PLWriter *pWriter, DLWriter *dlw, sqlite_int64 iDocid){
- char c[VARINT_MAX];
- int n;
-
- pWriter->dlw = dlw;
-
- /* Docids must ascend. */
- assert( !pWriter->dlw->has_iPrevDocid || iDocid>pWriter->dlw->iPrevDocid );
- n = fts3PutVarint(c, iDocid-pWriter->dlw->iPrevDocid);
- dataBufferAppend(pWriter->dlw->b, c, n);
- pWriter->dlw->iPrevDocid = iDocid;
-#ifndef NDEBUG
- pWriter->dlw->has_iPrevDocid = 1;
-#endif
-
- pWriter->iColumn = 0;
- pWriter->iPos = 0;
- pWriter->iOffset = 0;
-}
-/* TODO(shess) Should plwDestroy() also terminate the doclist? But
-** then plwDestroy() would no longer be just a destructor, it would
-** also be doing work, which isn't consistent with the overall idiom.
-** Another option would be for plwAdd() to always append any necessary
-** terminator, so that the output is always correct. But that would
-** add incremental work to the common case with the only benefit being
-** API elegance. Punt for now.
-*/
-static void plwTerminate(PLWriter *pWriter){
- if( pWriter->dlw->iType>DL_DOCIDS ){
- char c[VARINT_MAX];
- int n = fts3PutVarint(c, POS_END);
- dataBufferAppend(pWriter->dlw->b, c, n);
- }
-#ifndef NDEBUG
- /* Mark as terminated for assert in plwAdd(). */
- pWriter->iPos = -1;
-#endif
-}
-static void plwDestroy(PLWriter *pWriter){
- SCRAMBLE(pWriter);
-}
-
-/*******************************************************************/
-/* DLCollector wraps PLWriter and DLWriter to provide a
-** dynamically-allocated doclist area to use during tokenization.
-**
-** dlcNew - malloc up and initialize a collector.
-** dlcDelete - destroy a collector and all contained items.
-** dlcAddPos - append position and offset information.
-** dlcAddDoclist - add the collected doclist to the given buffer.
-** dlcNext - terminate the current document and open another.
-*/
-typedef struct DLCollector {
- DataBuffer b;
- DLWriter dlw;
- PLWriter plw;
-} DLCollector;
-
-/* TODO(shess) This could also be done by calling plwTerminate() and
-** dataBufferAppend(). I tried that, expecting nominal performance
-** differences, but it seemed to pretty reliably be worth 1% to code
-** it this way. I suspect it is the incremental malloc overhead (some
-** percentage of the plwTerminate() calls will cause a realloc), so
-** this might be worth revisiting if the DataBuffer implementation
-** changes.
-*/
-static void dlcAddDoclist(DLCollector *pCollector, DataBuffer *b){
- if( pCollector->dlw.iType>DL_DOCIDS ){
- char c[VARINT_MAX];
- int n = fts3PutVarint(c, POS_END);
- dataBufferAppend2(b, pCollector->b.pData, pCollector->b.nData, c, n);
- }else{
- dataBufferAppend(b, pCollector->b.pData, pCollector->b.nData);
- }
-}
-static void dlcNext(DLCollector *pCollector, sqlite_int64 iDocid){
- plwTerminate(&pCollector->plw);
- plwDestroy(&pCollector->plw);
- plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
-}
-static void dlcAddPos(DLCollector *pCollector, int iColumn, int iPos,
- int iStartOffset, int iEndOffset){
- plwAdd(&pCollector->plw, iColumn, iPos, iStartOffset, iEndOffset);
-}
-
-static DLCollector *dlcNew(sqlite_int64 iDocid, DocListType iType){
- DLCollector *pCollector = sqlite3_malloc(sizeof(DLCollector));
- dataBufferInit(&pCollector->b, 0);
- dlwInit(&pCollector->dlw, iType, &pCollector->b);
- plwInit(&pCollector->plw, &pCollector->dlw, iDocid);
- return pCollector;
-}
-static void dlcDelete(DLCollector *pCollector){
- plwDestroy(&pCollector->plw);
- dlwDestroy(&pCollector->dlw);
- dataBufferDestroy(&pCollector->b);
- SCRAMBLE(pCollector);
- sqlite3_free(pCollector);
-}
-
-
-/* Copy the doclist data of iType in pData/nData into *out, trimming
-** unnecessary data as we go. Only columns matching iColumn are
-** copied, all columns copied if iColumn is -1. Elements with no
-** matching columns are dropped. The output is an iOutType doclist.
-*/
-/* NOTE(shess) This code is only valid after all doclists are merged.
-** If this is run before merges, then doclist items which represent
-** deletion will be trimmed, and will thus not effect a deletion
-** during the merge.
-*/
-static void docListTrim(DocListType iType, const char *pData, int nData,
- int iColumn, DocListType iOutType, DataBuffer *out){
- DLReader dlReader;
- DLWriter dlWriter;
-
- assert( iOutType<=iType );
-
- dlrInit(&dlReader, iType, pData, nData);
- dlwInit(&dlWriter, iOutType, out);
-
- while( !dlrAtEnd(&dlReader) ){
- PLReader plReader;
- PLWriter plWriter;
- int match = 0;
-
- plrInit(&plReader, &dlReader);
-
- while( !plrAtEnd(&plReader) ){
- if( iColumn==-1 || plrColumn(&plReader)==iColumn ){
- if( !match ){
- plwInit(&plWriter, &dlWriter, dlrDocid(&dlReader));
- match = 1;
- }
- plwAdd(&plWriter, plrColumn(&plReader), plrPosition(&plReader),
- plrStartOffset(&plReader), plrEndOffset(&plReader));
- }
- plrStep(&plReader);
- }
- if( match ){
- plwTerminate(&plWriter);
- plwDestroy(&plWriter);
- }
-
- plrDestroy(&plReader);
- dlrStep(&dlReader);
- }
- dlwDestroy(&dlWriter);
- dlrDestroy(&dlReader);
-}
-
-/* Used by docListMerge() to keep doclists in the ascending order by
-** docid, then ascending order by age (so the newest comes first).
-*/
-typedef struct OrderedDLReader {
- DLReader *pReader;
-
- /* TODO(shess) If we assume that docListMerge pReaders is ordered by
- ** age (which we do), then we could use pReader comparisons to break
- ** ties.
- */
- int idx;
-} OrderedDLReader;
-
-/* Order eof to end, then by docid asc, idx desc. */
-static int orderedDLReaderCmp(OrderedDLReader *r1, OrderedDLReader *r2){
- if( dlrAtEnd(r1->pReader) ){
- if( dlrAtEnd(r2->pReader) ) return 0; /* Both atEnd(). */
- return 1; /* Only r1 atEnd(). */
- }
- if( dlrAtEnd(r2->pReader) ) return -1; /* Only r2 atEnd(). */
-
- if( dlrDocid(r1->pReader)<dlrDocid(r2->pReader) ) return -1;
- if( dlrDocid(r1->pReader)>dlrDocid(r2->pReader) ) return 1;
-
- /* Descending on idx. */
- return r2->idx-r1->idx;
-}
-
-/* Bubble p[0] to appropriate place in p[1..n-1]. Assumes that
-** p[1..n-1] is already sorted.
-*/
-/* TODO(shess) Is this frequent enough to warrant a binary search?
-** Before implementing that, instrument the code to check. In most
-** current usage, I expect that p[0] will be less than p[1] a very
-** high proportion of the time.
-*/
-static void orderedDLReaderReorder(OrderedDLReader *p, int n){
- while( n>1 && orderedDLReaderCmp(p, p+1)>0 ){
- OrderedDLReader tmp = p[0];
- p[0] = p[1];
- p[1] = tmp;
- n--;
- p++;
- }
-}
-
-/* Given an array of doclist readers, merge their doclist elements
-** into out in sorted order (by docid), dropping elements from older
-** readers when there is a duplicate docid. pReaders is assumed to be
-** ordered by age, oldest first.
-*/
-/* TODO(shess) nReaders must be <= MERGE_COUNT. This should probably
-** be fixed.
-*/
-static void docListMerge(DataBuffer *out,
- DLReader *pReaders, int nReaders){
- OrderedDLReader readers[MERGE_COUNT];
- DLWriter writer;
- int i, n;
- const char *pStart = 0;
- int nStart = 0;
- sqlite_int64 iFirstDocid = 0, iLastDocid = 0;
-
- assert( nReaders>0 );
- if( nReaders==1 ){
- dataBufferAppend(out, dlrDocData(pReaders), dlrAllDataBytes(pReaders));
- return;
- }
-
- assert( nReaders<=MERGE_COUNT );
- n = 0;
- for(i=0; i<nReaders; i++){
- assert( pReaders[i].iType==pReaders[0].iType );
- readers[i].pReader = pReaders+i;
- readers[i].idx = i;
- n += dlrAllDataBytes(&pReaders[i]);
- }
- /* Conservatively size output to sum of inputs. Output should end
- ** up strictly smaller than input.
- */
- dataBufferExpand(out, n);
-
- /* Get the readers into sorted order. */
- while( i-->0 ){
- orderedDLReaderReorder(readers+i, nReaders-i);
- }
-
- dlwInit(&writer, pReaders[0].iType, out);
- while( !dlrAtEnd(readers[0].pReader) ){
- sqlite_int64 iDocid = dlrDocid(readers[0].pReader);
-
- /* If this is a continuation of the current buffer to copy, extend
- ** that buffer. memcpy() seems to be more efficient if it has a
- ** lots of data to copy.
- */
- if( dlrDocData(readers[0].pReader)==pStart+nStart ){
- nStart += dlrDocDataBytes(readers[0].pReader);
- }else{
- if( pStart!=0 ){
- dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
- }
- pStart = dlrDocData(readers[0].pReader);
- nStart = dlrDocDataBytes(readers[0].pReader);
- iFirstDocid = iDocid;
- }
- iLastDocid = iDocid;
- dlrStep(readers[0].pReader);
-
- /* Drop all of the older elements with the same docid. */
- for(i=1; i<nReaders &&
- !dlrAtEnd(readers[i].pReader) &&
- dlrDocid(readers[i].pReader)==iDocid; i++){
- dlrStep(readers[i].pReader);
- }
-
- /* Get the readers back into order. */
- while( i-->0 ){
- orderedDLReaderReorder(readers+i, nReaders-i);
- }
- }
-
- /* Copy over any remaining elements. */
- if( nStart>0 ) dlwAppend(&writer, pStart, nStart, iFirstDocid, iLastDocid);
- dlwDestroy(&writer);
-}
-
-/* Helper function for posListUnion(). Compares the current position
-** between left and right, returning as standard C idiom of <0 if
-** left<right, >0 if left>right, and 0 if left==right. "End" always
-** compares greater.
-*/
-static int posListCmp(PLReader *pLeft, PLReader *pRight){
- assert( pLeft->iType==pRight->iType );
- if( pLeft->iType==DL_DOCIDS ) return 0;
-
- if( plrAtEnd(pLeft) ) return plrAtEnd(pRight) ? 0 : 1;
- if( plrAtEnd(pRight) ) return -1;
-
- if( plrColumn(pLeft)<plrColumn(pRight) ) return -1;
- if( plrColumn(pLeft)>plrColumn(pRight) ) return 1;
-
- if( plrPosition(pLeft)<plrPosition(pRight) ) return -1;
- if( plrPosition(pLeft)>plrPosition(pRight) ) return 1;
- if( pLeft->iType==DL_POSITIONS ) return 0;
-
- if( plrStartOffset(pLeft)<plrStartOffset(pRight) ) return -1;
- if( plrStartOffset(pLeft)>plrStartOffset(pRight) ) return 1;
-
- if( plrEndOffset(pLeft)<plrEndOffset(pRight) ) return -1;
- if( plrEndOffset(pLeft)>plrEndOffset(pRight) ) return 1;
-
+**
+** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against
+** the %_content table.
+**
+** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry
+** in the %_content table.
+**
+** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The
+** column on the left-hand side of the MATCH operator is column
+** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand
+** side of the MATCH operator.
+*/
+/* TODO(shess) Upgrade the cursor initialization and destruction to
+** account for fts3FilterMethod() being called multiple times on the
+** same cursor. The current solution is very fragile. Apply fix to
+** fts3 as appropriate.
+*/
+static int fts3FilterMethod(
+ sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
+ int idxNum, /* Strategy index */
+ const char *idxStr, /* Unused */
+ int nVal, /* Number of elements in apVal */
+ sqlite3_value **apVal /* Arguments for the indexing scheme */
+){
+ const char *azSql[] = {
+ "SELECT * FROM %Q.'%q_content' WHERE docid = ?", /* non-full-table-scan */
+ "SELECT * FROM %Q.'%q_content'", /* full-table-scan */
+ };
+ int rc; /* Return code */
+ char *zSql; /* SQL statement used to access %_content */
+ Fts3Table *p = (Fts3Table *)pCursor->pVtab;
+ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
+
+ assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
+ assert( nVal==0 || nVal==1 );
+ assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );
+
+ /* In case the cursor has been used before, clear it now. */
+ sqlite3_finalize(pCsr->pStmt);
+ sqlite3_free(pCsr->aDoclist);
+ memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
+
+ /* Compile a SELECT statement for this cursor. For a full-table-scan, the
+ ** statement loops through all rows of the %_content table. For a
+ ** full-text query or docid lookup, the statement retrieves a single
+ ** row by docid.
+ */
+ zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0);
+ sqlite3_free(zSql);
+ }
+ if( rc!=SQLITE_OK ) return rc;
+ pCsr->eType = idxNum;
+
+ if( idxNum==FTS3_DOCID_SEARCH ){
+ rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
+ }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){
+ int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
+ const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
+
+ rc = sqlite3Fts3PendingTermsFlush(p);
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
+ iCol, zQuery, -1, &pCsr->pExpr
+ );
+ if( rc!=SQLITE_OK ) return rc;
+
+ rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist);
+ pCsr->pNextId = pCsr->aDoclist;
+ pCsr->iPrevId = 0;
+ }
+
+ if( rc!=SQLITE_OK ) return rc;
+ return fts3NextMethod(pCursor);
+}
+
+/*
+** This is the xEof method of the virtual table. SQLite calls this
+** routine to find out if it has reached the end of a result set.
+*/
+static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){
+ return ((Fts3Cursor *)pCursor)->isEof;
+}
+
+/*
+** This is the xColumn method of the virtual table. The SQLite
+** core calls this method during a query when it needs the value
+** of a column from the virtual table. This method needs to use
+** one of the sqlite3_result_*() routines to store the requested
+** value back in the pContext.
+*/
+static int fts3ColumnMethod(sqlite3_vtab_cursor *pCursor,
+ sqlite3_context *pContext, int idxCol){
+ Fts3Cursor *c = (Fts3Cursor *) pCursor;
+ Fts3Table *v = cursor_vtab(c);
+ int rc = fts3CursorSeek(c);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ if( idxCol<v->nColumn ){
+ sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
+ sqlite3_result_value(pContext, pVal);
+ }else if( idxCol==v->nColumn ){
+ /* The extra column whose name is the same as the table.
+ ** Return a blob which is a pointer to the cursor
+ */
+ sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT);
+ }else if( idxCol==v->nColumn+1 ){
+ /* The docid column, which is an alias for rowid. */
+ sqlite3_value *pVal = sqlite3_column_value(c->pStmt, 0);
+ sqlite3_result_value(pContext, pVal);
+ }
+ return SQLITE_OK;
+}
+
+/*
+** This is the xRowid method. The SQLite core calls this routine to
+** retrieve the rowid for the current row of the result set. fts3
+** exposes %_content.docid as the rowid for the virtual table. The
+** rowid should be written to *pRowid.
+*/
+static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
+ if( pCsr->aDoclist ){
+ *pRowid = pCsr->iPrevId;
+ }else{
+ *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
+ }
+ return SQLITE_OK;
+}
+
+/*
+** This function is the implementation of the xUpdate callback used by
+** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
+** inserted, updated or deleted.
+*/
+static int fts3UpdateMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ int nArg, /* Size of argument array */
+ sqlite3_value **apVal, /* Array of arguments */
+ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
+){
+ return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid);
+}
+
+/*
+** Implementation of xSync() method. Flush the contents of the pending-terms
+** hash-table to the database.
+*/
+static int fts3SyncMethod(sqlite3_vtab *pVtab){
+ return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab);
+}
+
+/*
+** Implementation of xBegin() method. This is a no-op.
+*/
+static int fts3BeginMethod(sqlite3_vtab *pVtab){
+ assert( ((Fts3Table *)pVtab)->nPendingData==0 );
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of xCommit() method. This is a no-op. The contents of
+** the pending-terms hash-table have already been flushed into the database
+** by fts3SyncMethod().
+*/
+static int fts3CommitMethod(sqlite3_vtab *pVtab){
+ assert( ((Fts3Table *)pVtab)->nPendingData==0 );
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of xRollback(). Discard the contents of the pending-terms
+** hash-table. Any changes made to the database are reverted by SQLite.
+*/
+static int fts3RollbackMethod(sqlite3_vtab *pVtab){
+ sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab);
+ return SQLITE_OK;
+}
+
+/*
+** Helper function used by the implementation of the overloaded snippet(),
+** offsets() and optimize() SQL functions.
+**
+** If the value passed as the third argument is a blob of size
+** sizeof(Fts3Cursor*), then the blob contents are copied to the
+** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error
+** message is written to context pContext and SQLITE_ERROR returned. The
+** string passed via zFunc is used as part of the error message.
+*/
+static int fts3FunctionArg(
+ sqlite3_context *pContext, /* SQL function call context */
+ const char *zFunc, /* Function name */
+ sqlite3_value *pVal, /* argv[0] passed to function */
+ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */
+){
+ Fts3Cursor *pRet;
+ if( sqlite3_value_type(pVal)!=SQLITE_BLOB
+ && sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *)
+ ){
+ char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc);
+ sqlite3_result_error(pContext, zErr, -1);
+ sqlite3_free(zErr);
+ return SQLITE_ERROR;
+ }
+ memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *));
+ *ppCsr = pRet;
+ return SQLITE_OK;
+}
+
+/*
+** Implementation of the snippet() function for FTS3
+*/
+static void fts3SnippetFunc(
+ sqlite3_context *pContext,
+ int argc,
+ sqlite3_value **argv
+){
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
+ const char *zStart = "<b>";
+ const char *zEnd = "</b>";
+ const char *zEllipsis = "<b>...</b>";
+
+ if( argc<1 || argc>4 ) return;
+ if( fts3FunctionArg(pContext, "snippet", argv[0], &pCsr) ) return;
+
+ switch( argc ){
+ case 4: zEllipsis = (const char*)sqlite3_value_text(argv[3]);
+ case 3: zEnd = (const char*)sqlite3_value_text(argv[2]);
+ case 2: zStart = (const char*)sqlite3_value_text(argv[1]);
+ }
+
+ sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis);
+}
+
+/*
+** Implementation of the offsets() function for FTS3
+*/
+static void fts3OffsetsFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of argument array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
+
+ assert( nVal==1 );
+ if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return;
+ assert( pCsr );
+ sqlite3Fts3Offsets(pContext, pCsr);
+}
+
+/*
+** Implementation of the special optimize() function for FTS3. This
+** function merges all segments in the database to a single segment.
+** Example usage is:
+**
+** SELECT optimize(t) FROM t LIMIT 1;
+**
+** where 't' is the name of an FTS3 table.
+*/
+static void fts3OptimizeFunc(
+ sqlite3_context *pContext, /* SQLite function call context */
+ int nVal, /* Size of argument array */
+ sqlite3_value **apVal /* Array of arguments */
+){
+ int rc; /* Return code */
+ Fts3Table *p; /* Virtual table handle */
+ Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */
+
+ assert( nVal==1 );
+ if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return;
+ p = (Fts3Table *)pCursor->base.pVtab;
+ assert( p );
+
+ rc = sqlite3Fts3Optimize(p);
+
+ switch( rc ){
+ case SQLITE_OK:
+ sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
+ break;
+ case SQLITE_DONE:
+ sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC);
+ break;
+ default:
+ sqlite3_result_error_code(pContext, rc);
+ break;
+ }
+}
+
+/*
+** This routine implements the xFindFunction method for the FTS3
+** virtual table.
+*/
+static int fts3FindFunctionMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ int nArg, /* Number of SQL function arguments */
+ const char *zName, /* Name of SQL function */
+ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
+ void **ppArg /* Unused */
+){
+ struct Overloaded {
+ const char *zName;
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
+ } aOverload[] = {
+ { "snippet", fts3SnippetFunc },
+ { "offsets", fts3OffsetsFunc },
+ { "optimize", fts3OptimizeFunc },
+ };
+ int i; /* Iterator variable */
+ for(i=0; i<SizeofArray(aOverload); i++){
+ if( strcmp(zName, aOverload[i].zName)==0 ){
+ *pxFunc = aOverload[i].xFunc;
+ return 1;
+ }
+ }
+
+ /* No function of the specified name was found. Return 0. */
return 0;
}
-/* Write the union of position lists in pLeft and pRight to pOut.
-** "Union" in this case meaning "All unique position tuples". Should
-** work with any doclist type, though both inputs and the output
-** should be the same type.
-*/
-static void posListUnion(DLReader *pLeft, DLReader *pRight, DLWriter *pOut){
- PLReader left, right;
- PLWriter writer;
-
- assert( dlrDocid(pLeft)==dlrDocid(pRight) );
- assert( pLeft->iType==pRight->iType );
- assert( pLeft->iType==pOut->iType );
-
- plrInit(&left, pLeft);
- plrInit(&right, pRight);
- plwInit(&writer, pOut, dlrDocid(pLeft));
-
- while( !plrAtEnd(&left) || !plrAtEnd(&right) ){
- int c = posListCmp(&left, &right);
- if( c<0 ){
- plwCopy(&writer, &left);
- plrStep(&left);
- }else if( c>0 ){
- plwCopy(&writer, &right);
- plrStep(&right);
- }else{
- plwCopy(&writer, &left);
- plrStep(&left);
- plrStep(&right);
- }
- }
-
- plwTerminate(&writer);
- plwDestroy(&writer);
- plrDestroy(&left);
- plrDestroy(&right);
-}
-
-/* Write the union of doclists in pLeft and pRight to pOut. For
-** docids in common between the inputs, the union of the position
-** lists is written. Inputs and outputs are always type DL_DEFAULT.
-*/
-static void docListUnion(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 ){
- if( nRight!=0) dataBufferAppend(pOut, pRight, nRight);
- return;
- }
- if( nRight==0 ){
- dataBufferAppend(pOut, pLeft, nLeft);
- return;
- }
-
- dlrInit(&left, DL_DEFAULT, pLeft, nLeft);
- dlrInit(&right, DL_DEFAULT, pRight, nRight);
- dlwInit(&writer, DL_DEFAULT, pOut);
-
- while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
- if( dlrAtEnd(&right) ){
- dlwCopy(&writer, &left);
- dlrStep(&left);
- }else if( dlrAtEnd(&left) ){
- dlwCopy(&writer, &right);
- dlrStep(&right);
- }else if( dlrDocid(&left)<dlrDocid(&right) ){
- dlwCopy(&writer, &left);
- dlrStep(&left);
- }else if( dlrDocid(&left)>dlrDocid(&right) ){
- dlwCopy(&writer, &right);
- dlrStep(&right);
- }else{
- posListUnion(&left, &right, &writer);
- dlrStep(&left);
- dlrStep(&right);
- }
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-
-/*
-** This function is used as part of the implementation of phrase and
-** NEAR matching.
-**
-** pLeft and pRight are DLReaders positioned to the same docid in
-** lists of type DL_POSITION. This function writes an entry to the
-** DLWriter pOut for each position in pRight that is less than
-** (nNear+1) greater (but not equal to or smaller) than a position
-** in pLeft. For example, if nNear is 0, and the positions contained
-** by pLeft and pRight are:
-**
-** pLeft: 5 10 15 20
-** pRight: 6 9 17 21
-**
-** then the docid is added to pOut. If pOut is of type DL_POSITIONS,
-** then a positionids "6" and "21" are also added to pOut.
-**
-** If boolean argument isSaveLeft is true, then positionids are copied
-** from pLeft instead of pRight. In the example above, the positions "5"
-** and "20" would be added instead of "6" and "21".
-*/
-static void posListPhraseMerge(
- DLReader *pLeft,
- DLReader *pRight,
- int nNear,
- int isSaveLeft,
- DLWriter *pOut
-){
- PLReader left, right;
- PLWriter writer;
- int match = 0;
-
- assert( dlrDocid(pLeft)==dlrDocid(pRight) );
- assert( pOut->iType!=DL_POSITIONS_OFFSETS );
-
- plrInit(&left, pLeft);
- plrInit(&right, pRight);
-
- while( !plrAtEnd(&left) && !plrAtEnd(&right) ){
- if( plrColumn(&left)<plrColumn(&right) ){
- plrStep(&left);
- }else if( plrColumn(&left)>plrColumn(&right) ){
- plrStep(&right);
- }else if( plrPosition(&left)>=plrPosition(&right) ){
- plrStep(&right);
- }else{
- if( (plrPosition(&right)-plrPosition(&left))<=(nNear+1) ){
- if( !match ){
- plwInit(&writer, pOut, dlrDocid(pLeft));
- match = 1;
- }
- if( !isSaveLeft ){
- plwAdd(&writer, plrColumn(&right), plrPosition(&right), 0, 0);
- }else{
- plwAdd(&writer, plrColumn(&left), plrPosition(&left), 0, 0);
- }
- plrStep(&right);
- }else{
- plrStep(&left);
- }
- }
- }
-
- if( match ){
- plwTerminate(&writer);
- plwDestroy(&writer);
- }
-
- plrDestroy(&left);
- plrDestroy(&right);
-}
-
-/*
-** Compare the values pointed to by the PLReaders passed as arguments.
-** Return -1 if the value pointed to by pLeft is considered less than
-** the value pointed to by pRight, +1 if it is considered greater
-** than it, or 0 if it is equal. i.e.
-**
-** (*pLeft - *pRight)
-**
-** A PLReader that is in the EOF condition is considered greater than
-** any other. If neither argument is in EOF state, the return value of
-** plrColumn() is used. If the plrColumn() values are equal, the
-** comparison is on the basis of plrPosition().
-*/
-static int plrCompare(PLReader *pLeft, PLReader *pRight){
- assert(!plrAtEnd(pLeft) || !plrAtEnd(pRight));
-
- if( plrAtEnd(pRight) || plrAtEnd(pLeft) ){
- return (plrAtEnd(pRight) ? -1 : 1);
- }
- if( plrColumn(pLeft)!=plrColumn(pRight) ){
- return ((plrColumn(pLeft)<plrColumn(pRight)) ? -1 : 1);
- }
- if( plrPosition(pLeft)!=plrPosition(pRight) ){
- return ((plrPosition(pLeft)<plrPosition(pRight)) ? -1 : 1);
- }
- return 0;
-}
-
-/* We have two doclists with positions: pLeft and pRight. Depending
-** on the value of the nNear parameter, perform either a phrase
-** intersection (if nNear==0) or a NEAR intersection (if nNear>0)
-** and write the results into pOut.
-**
-** A phrase intersection means that two documents only match
-** if pLeft.iPos+1==pRight.iPos.
-**
-** A NEAR intersection means that two documents only match if
-** (abs(pLeft.iPos-pRight.iPos)<nNear).
-**
-** If a NEAR intersection is requested, then the nPhrase argument should
-** be passed the number of tokens in the two operands to the NEAR operator
-** combined. For example:
-**
-** Query syntax nPhrase
-** ------------------------------------
-** "A B C" NEAR "D E" 5
-** A NEAR B 2
-**
-** iType controls the type of data written to pOut. If iType is
-** DL_POSITIONS, the positions are those from pRight.
-*/
-static void docListPhraseMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- int nNear, /* 0 for a phrase merge, non-zero for a NEAR merge */
- int nPhrase, /* Number of tokens in left+right operands to NEAR */
- DocListType iType, /* Type of doclist to write to pOut */
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 || nRight==0 ) return;
-
- assert( iType!=DL_POSITIONS_OFFSETS );
-
- dlrInit(&left, DL_POSITIONS, pLeft, nLeft);
- dlrInit(&right, DL_POSITIONS, pRight, nRight);
- dlwInit(&writer, iType, pOut);
-
- while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
- if( dlrDocid(&left)<dlrDocid(&right) ){
- dlrStep(&left);
- }else if( dlrDocid(&right)<dlrDocid(&left) ){
- dlrStep(&right);
- }else{
- if( nNear==0 ){
- posListPhraseMerge(&left, &right, 0, 0, &writer);
+/*
+** Implementation of FTS3 xRename method. Rename an fts3 table.
+*/
+static int fts3RenameMethod(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ const char *zName /* New name of table */
+){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ int rc = SQLITE_NOMEM; /* Return Code */
+ char *zSql; /* SQL script to run to rename tables */
+
+ zSql = sqlite3_mprintf(
+ "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
+ "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
+ "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
+ , p->zDb, p->zName, zName
+ , p->zDb, p->zName, zName
+ , p->zDb, p->zName, zName
+ );
+ if( zSql ){
+ rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
+ sqlite3_free(zSql);
+ }
+ return rc;
+}
+
+static const sqlite3_module fts3Module = {
+ /* iVersion */ 0,
+ /* xCreate */ fts3CreateMethod,
+ /* xConnect */ fts3ConnectMethod,
+ /* xBestIndex */ fts3BestIndexMethod,
+ /* xDisconnect */ fts3DisconnectMethod,
+ /* xDestroy */ fts3DestroyMethod,
+ /* xOpen */ fts3OpenMethod,
+ /* xClose */ fulltextClose,
+ /* xFilter */ fts3FilterMethod,
+ /* xNext */ fts3NextMethod,
+ /* xEof */ fts3EofMethod,
+ /* xColumn */ fts3ColumnMethod,
+ /* xRowid */ fts3RowidMethod,
+ /* xUpdate */ fts3UpdateMethod,
+ /* xBegin */ fts3BeginMethod,
+ /* xSync */ fts3SyncMethod,
+ /* xCommit */ fts3CommitMethod,
+ /* xRollback */ fts3RollbackMethod,
+ /* xFindFunction */ fts3FindFunctionMethod,
+ /* xRename */ fts3RenameMethod,
+};
+
+/*
+** This function is registered as the module destructor (called when an
+** FTS3 enabled database connection is closed). It frees the memory
+** allocated for the tokenizer hash table.
+*/
+static void hashDestroy(void *p){
+ Fts3Hash *pHash = (Fts3Hash *)p;
+ sqlite3Fts3HashClear(pHash);
+ sqlite3_free(pHash);
+}
+
+/*
+** The fts3 built-in tokenizers - "simple" and "porter" - are implemented
+** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following
+** two forward declarations are for functions declared in these files
+** used to retrieve the respective implementations.
+**
+** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
+** to by the argument to point a the "simple" tokenizer implementation.
+** Function ...PorterTokenizerModule() sets *pModule to point to the
+** porter tokenizer/stemmer implementation.
+*/
+SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+
+/*
+** Initialise the fts3 extension. If this extension is built as part
+** of the sqlite library, then this function is called directly by
+** SQLite. If fts3 is built as a dynamically loadable extension, this
+** function is called by the sqlite3_extension_init() entry point.
+*/
+SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){
+ int rc = SQLITE_OK;
+ Fts3Hash *pHash = 0;
+ const sqlite3_tokenizer_module *pSimple = 0;
+ const sqlite3_tokenizer_module *pPorter = 0;
+ const sqlite3_tokenizer_module *pIcu = 0;
+
+ sqlite3Fts3SimpleTokenizerModule(&pSimple);
+ sqlite3Fts3PorterTokenizerModule(&pPorter);
+#ifdef SQLITE_ENABLE_ICU
+ sqlite3Fts3IcuTokenizerModule(&pIcu);
+#endif
+
+ /* Allocate and initialise the hash-table used to store tokenizers. */
+ pHash = sqlite3_malloc(sizeof(Fts3Hash));
+ if( !pHash ){
+ rc = SQLITE_NOMEM;
+ }else{
+ sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
+ }
+
+ /* Load the built-in tokenizers into the hash table */
+ if( rc==SQLITE_OK ){
+ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
+ || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
+ || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
+ ){
+ rc = SQLITE_NOMEM;
+ }
+ }
+
+#ifdef SQLITE_TEST
+ sqlite3Fts3ExprInitTestInterface(db);
+#endif
+
+ /* Create the virtual table wrapper around the hash-table and overload
+ ** the two scalar functions. If this is successful, register the
+ ** module with sqlite.
+ */
+ if( SQLITE_OK==rc
+ && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
+ && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
+ ){
+ return sqlite3_create_module_v2(
+ db, "fts3", &fts3Module, (void *)pHash, hashDestroy
+ );
+ }
+
+ /* An error has occurred. Delete the hash table and return the error code. */
+ assert( rc!=SQLITE_OK );
+ if( pHash ){
+ sqlite3Fts3HashClear(pHash);
+ sqlite3_free(pHash);
+ }
+ return rc;
+}
+
+#if !SQLITE_CORE
+SQLITE_API int sqlite3_extension_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ const sqlite3_api_routines *pApi
+){
+ SQLITE_EXTENSION_INIT2(pApi)
+ return sqlite3Fts3Init(db);
+}
+#endif
+
+#endif
+
+/************** End of fts3.c ************************************************/
+/************** Begin file fts3_expr.c ***************************************/
+/*
+** 2008 Nov 28
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This module contains code that implements a parser for fts3 query strings
+** (the right-hand argument to the MATCH operator). Because the supported
+** syntax is relatively simple, the whole tokenizer/parser system is
+** hand-coded.
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+/*
+** By default, this module parses the legacy syntax that has been
+** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS
+** is defined, then it uses the new syntax. The differences between
+** the new and the old syntaxes are:
+**
+** a) The new syntax supports parenthesis. The old does not.
+**
+** b) The new syntax supports the AND and NOT operators. The old does not.
+**
+** c) The old syntax supports the "-" token qualifier. This is not
+** supported by the new syntax (it is replaced by the NOT operator).
+**
+** d) When using the old syntax, the OR operator has a greater precedence
+** than an implicit AND. When using the new, both implicity and explicit
+** AND operators have a higher precedence than OR.
+**
+** If compiled with SQLITE_TEST defined, then this module exports the
+** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable
+** to zero causes the module to use the old syntax. If it is set to
+** non-zero the new syntax is activated. This is so both syntaxes can
+** be tested using a single build of testfixture.
+**
+** The following describes the syntax supported by the fts3 MATCH
+** operator in a similar format to that used by the lemon parser
+** generator. This module does not use actually lemon, it uses a
+** custom parser.
+**
+** query ::= andexpr (OR andexpr)*.
+**
+** andexpr ::= notexpr (AND? notexpr)*.
+**
+** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*.
+** notexpr ::= LP query RP.
+**
+** nearexpr ::= phrase (NEAR distance_opt nearexpr)*.
+**
+** distance_opt ::= .
+** distance_opt ::= / INTEGER.
+**
+** phrase ::= TOKEN.
+** phrase ::= COLUMN:TOKEN.
+** phrase ::= "TOKEN TOKEN TOKEN...".
+*/
+
+#ifdef SQLITE_TEST
+SQLITE_API int sqlite3_fts3_enable_parentheses = 0;
+#else
+# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
+# define sqlite3_fts3_enable_parentheses 1
+# else
+# define sqlite3_fts3_enable_parentheses 0
+# endif
+#endif
+
+/*
+** Default span for NEAR operators.
+*/
+#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
+
+
+typedef struct ParseContext ParseContext;
+struct ParseContext {
+ sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
+ const char **azCol; /* Array of column names for fts3 table */
+ int nCol; /* Number of entries in azCol[] */
+ int iDefaultCol; /* Default column to query */
+ sqlite3_context *pCtx; /* Write error message here */
+ int nNest; /* Number of nested brackets */
+};
+
+/*
+** This function is equivalent to the standard isspace() function.
+**
+** The standard isspace() can be awkward to use safely, because although it
+** is defined to accept an argument of type int, its behaviour when passed
+** an integer that falls outside of the range of the unsigned char type
+** is undefined (and sometimes, "undefined" means segfault). This wrapper
+** is defined to accept an argument of type char, and always returns 0 for
+** any values that fall outside of the range of the unsigned char type (i.e.
+** negative values).
+*/
+static int fts3isspace(char c){
+ return (c&0x80)==0 ? isspace(c) : 0;
+}
+
+/*
+** Extract the next token from buffer z (length n) using the tokenizer
+** and other information (column names etc.) in pParse. Create an Fts3Expr
+** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
+** single token and set *ppExpr to point to it. If the end of the buffer is
+** reached before a token is found, set *ppExpr to zero. It is the
+** responsibility of the caller to eventually deallocate the allocated
+** Fts3Expr structure (if any) by passing it to sqlite3_free().
+**
+** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation
+** fails.
+*/
+static int getNextToken(
+ ParseContext *pParse, /* fts3 query parse context */
+ int iCol, /* Value for Fts3Phrase.iColumn */
+ const char *z, int n, /* Input string */
+ Fts3Expr **ppExpr, /* OUT: expression */
+ int *pnConsumed /* OUT: Number of bytes consumed */
+){
+ sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ int rc;
+ sqlite3_tokenizer_cursor *pCursor;
+ Fts3Expr *pRet = 0;
+ int nConsumed = 0;
+
+ rc = pModule->xOpen(pTokenizer, z, n, &pCursor);
+ if( rc==SQLITE_OK ){
+ const char *zToken;
+ int nToken, iStart, iEnd, iPosition;
+ int nByte; /* total space to allocate */
+
+ pCursor->pTokenizer = pTokenizer;
+ rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
+
+ if( rc==SQLITE_OK ){
+ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
+ pRet = (Fts3Expr *)sqlite3_malloc(nByte);
+ if( !pRet ){
+ rc = SQLITE_NOMEM;
}else{
- /* This case occurs when two terms (simple terms or phrases) are
- * connected by a NEAR operator, span (nNear+1). i.e.
- *
- * '"terrible company" NEAR widget'
- */
- DataBuffer one = {0, 0, 0};
- DataBuffer two = {0, 0, 0};
-
- DLWriter dlwriter2;
- DLReader dr1 = {0, 0, 0, 0, 0};
- DLReader dr2 = {0, 0, 0, 0, 0};
-
- dlwInit(&dlwriter2, iType, &one);
- posListPhraseMerge(&right, &left, nNear-3+nPhrase, 1, &dlwriter2);
- dlwInit(&dlwriter2, iType, &two);
- posListPhraseMerge(&left, &right, nNear-1, 0, &dlwriter2);
-
- if( one.nData) dlrInit(&dr1, iType, one.pData, one.nData);
- if( two.nData) dlrInit(&dr2, iType, two.pData, two.nData);
-
- if( !dlrAtEnd(&dr1) || !dlrAtEnd(&dr2) ){
- PLReader pr1 = {0};
- PLReader pr2 = {0};
-
- PLWriter plwriter;
- plwInit(&plwriter, &writer, dlrDocid(dlrAtEnd(&dr1)?&dr2:&dr1));
-
- if( one.nData ) plrInit(&pr1, &dr1);
- if( two.nData ) plrInit(&pr2, &dr2);
- while( !plrAtEnd(&pr1) || !plrAtEnd(&pr2) ){
- int iCompare = plrCompare(&pr1, &pr2);
- switch( iCompare ){
- case -1:
- plwCopy(&plwriter, &pr1);
- plrStep(&pr1);
- break;
- case 1:
- plwCopy(&plwriter, &pr2);
- plrStep(&pr2);
- break;
- case 0:
- plwCopy(&plwriter, &pr1);
- plrStep(&pr1);
- plrStep(&pr2);
- break;
- }
- }
- plwTerminate(&plwriter);
- }
- dataBufferDestroy(&one);
- dataBufferDestroy(&two);
- }
- dlrStep(&left);
- dlrStep(&right);
- }
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-
-/* We have two DL_DOCIDS doclists: pLeft and pRight.
-** Write the intersection of these two doclists into pOut as a
-** DL_DOCIDS doclist.
-*/
-static void docListAndMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 || nRight==0 ) return;
-
- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
- dlrInit(&right, DL_DOCIDS, pRight, nRight);
- dlwInit(&writer, DL_DOCIDS, pOut);
-
- while( !dlrAtEnd(&left) && !dlrAtEnd(&right) ){
- if( dlrDocid(&left)<dlrDocid(&right) ){
- dlrStep(&left);
- }else if( dlrDocid(&right)<dlrDocid(&left) ){
- dlrStep(&right);
- }else{
- dlwAdd(&writer, dlrDocid(&left));
- dlrStep(&left);
- dlrStep(&right);
- }
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-
-/* We have two DL_DOCIDS doclists: pLeft and pRight.
-** Write the union of these two doclists into pOut as a
-** DL_DOCIDS doclist.
-*/
-static void docListOrMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 ){
- if( nRight!=0 ) dataBufferAppend(pOut, pRight, nRight);
- return;
- }
- if( nRight==0 ){
- dataBufferAppend(pOut, pLeft, nLeft);
- return;
- }
-
- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
- dlrInit(&right, DL_DOCIDS, pRight, nRight);
- dlwInit(&writer, DL_DOCIDS, pOut);
-
- while( !dlrAtEnd(&left) || !dlrAtEnd(&right) ){
- if( dlrAtEnd(&right) ){
- dlwAdd(&writer, dlrDocid(&left));
- dlrStep(&left);
- }else if( dlrAtEnd(&left) ){
- dlwAdd(&writer, dlrDocid(&right));
- dlrStep(&right);
- }else if( dlrDocid(&left)<dlrDocid(&right) ){
- dlwAdd(&writer, dlrDocid(&left));
- dlrStep(&left);
- }else if( dlrDocid(&right)<dlrDocid(&left) ){
- dlwAdd(&writer, dlrDocid(&right));
- dlrStep(&right);
- }else{
- dlwAdd(&writer, dlrDocid(&left));
- dlrStep(&left);
- dlrStep(&right);
- }
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-
-/* We have two DL_DOCIDS doclists: pLeft and pRight.
-** Write into pOut as DL_DOCIDS doclist containing all documents that
-** occur in pLeft but not in pRight.
-*/
-static void docListExceptMerge(
- const char *pLeft, int nLeft,
- const char *pRight, int nRight,
- DataBuffer *pOut /* Write the combined doclist here */
-){
- DLReader left, right;
- DLWriter writer;
-
- if( nLeft==0 ) return;
- if( nRight==0 ){
- dataBufferAppend(pOut, pLeft, nLeft);
- return;
- }
-
- dlrInit(&left, DL_DOCIDS, pLeft, nLeft);
- dlrInit(&right, DL_DOCIDS, pRight, nRight);
- dlwInit(&writer, DL_DOCIDS, pOut);
-
- while( !dlrAtEnd(&left) ){
- while( !dlrAtEnd(&right) && dlrDocid(&right)<dlrDocid(&left) ){
- dlrStep(&right);
- }
- if( dlrAtEnd(&right) || dlrDocid(&left)<dlrDocid(&right) ){
- dlwAdd(&writer, dlrDocid(&left));
- }
- dlrStep(&left);
- }
-
- dlrDestroy(&left);
- dlrDestroy(&right);
- dlwDestroy(&writer);
-}
-
-static char *string_dup_n(const char *s, int n){
- char *str = sqlite3_malloc(n + 1);
- memcpy(str, s, n);
- str[n] = '\0';
- return str;
-}
-
-/* Duplicate a string; the caller must free() the returned string.
- * (We don't use strdup() since it is not part of the standard C library and
- * may not be available everywhere.) */
-static char *string_dup(const char *s){
- return string_dup_n(s, strlen(s));
-}
-
-/* Format a string, replacing each occurrence of the % character with
- * zDb.zName. This may be more convenient than sqlite_mprintf()
- * when one string is used repeatedly in a format string.
- * The caller must free() the returned string. */
-static char *string_format(const char *zFormat,
- const char *zDb, const char *zName){
- const char *p;
- size_t len = 0;
- size_t nDb = strlen(zDb);
- size_t nName = strlen(zName);
- size_t nFullTableName = nDb+1+nName;
- char *result;
- char *r;
-
- /* first compute length needed */
- for(p = zFormat ; *p ; ++p){
- len += (*p=='%' ? nFullTableName : 1);
- }
- len += 1; /* for null terminator */
-
- r = result = sqlite3_malloc(len);
- for(p = zFormat; *p; ++p){
- if( *p=='%' ){
- memcpy(r, zDb, nDb);
- r += nDb;
- *r++ = '.';
- memcpy(r, zName, nName);
- r += nName;
- } else {
- *r++ = *p;
- }
- }
- *r++ = '\0';
- assert( r == result + len );
- return result;
-}
-
-static int sql_exec(sqlite3 *db, const char *zDb, const char *zName,
- const char *zFormat){
- char *zCommand = string_format(zFormat, zDb, zName);
- int rc;
- FTSTRACE(("FTS3 sql: %s\n", zCommand));
- rc = sqlite3_exec(db, zCommand, NULL, 0, NULL);
- sqlite3_free(zCommand);
- return rc;
-}
-
-static int sql_prepare(sqlite3 *db, const char *zDb, const char *zName,
- sqlite3_stmt **ppStmt, const char *zFormat){
- char *zCommand = string_format(zFormat, zDb, zName);
- int rc;
- FTSTRACE(("FTS3 prepare: %s\n", zCommand));
- rc = sqlite3_prepare_v2(db, zCommand, -1, ppStmt, NULL);
- sqlite3_free(zCommand);
- return rc;
-}
-
-/* end utility functions */
-
-/* Forward reference */
-typedef struct fulltext_vtab fulltext_vtab;
-
-/*
-** An instance of the following structure keeps track of generated
-** matching-word offset information and snippets.
-*/
-typedef struct Snippet {
- int nMatch; /* Total number of matches */
- int nAlloc; /* Space allocated for aMatch[] */
- struct snippetMatch { /* One entry for each matching term */
- char snStatus; /* Status flag for use while constructing snippets */
- short int iCol; /* The column that contains the match */
- short int iTerm; /* The index in Query.pTerms[] of the matching term */
- int iToken; /* The index of the matching document token */
- short int nByte; /* Number of bytes in the term */
- int iStart; /* The offset to the first character of the term */
- } *aMatch; /* Points to space obtained from malloc */
- char *zOffset; /* Text rendering of aMatch[] */
- int nOffset; /* strlen(zOffset) */
- char *zSnippet; /* Snippet text */
- int nSnippet; /* strlen(zSnippet) */
-} Snippet;
-
-
-typedef enum QueryType {
- QUERY_GENERIC, /* table scan */
- QUERY_DOCID, /* lookup by docid */
- QUERY_FULLTEXT /* QUERY_FULLTEXT + [i] is a full-text search for column i*/
-} QueryType;
-
-typedef enum fulltext_statement {
- CONTENT_INSERT_STMT,
- CONTENT_SELECT_STMT,
- CONTENT_UPDATE_STMT,
- CONTENT_DELETE_STMT,
- CONTENT_EXISTS_STMT,
-
- BLOCK_INSERT_STMT,
- BLOCK_SELECT_STMT,
- BLOCK_DELETE_STMT,
- BLOCK_DELETE_ALL_STMT,
-
- SEGDIR_MAX_INDEX_STMT,
- SEGDIR_SET_STMT,
- SEGDIR_SELECT_LEVEL_STMT,
- SEGDIR_SPAN_STMT,
- SEGDIR_DELETE_STMT,
- SEGDIR_SELECT_SEGMENT_STMT,
- SEGDIR_SELECT_ALL_STMT,
- SEGDIR_DELETE_ALL_STMT,
- SEGDIR_COUNT_STMT,
-
- MAX_STMT /* Always at end! */
-} fulltext_statement;
-
-/* These must exactly match the enum above. */
-/* TODO(shess): Is there some risk that a statement will be used in two
-** cursors at once, e.g. if a query joins a virtual table to itself?
-** If so perhaps we should move some of these to the cursor object.
-*/
-static const char *const fulltext_zStatement[MAX_STMT] = {
- /* CONTENT_INSERT */ NULL, /* generated in contentInsertStatement() */
- /* CONTENT_SELECT */ NULL, /* generated in contentSelectStatement() */
- /* CONTENT_UPDATE */ NULL, /* generated in contentUpdateStatement() */
- /* CONTENT_DELETE */ "delete from %_content where docid = ?",
- /* CONTENT_EXISTS */ "select docid from %_content limit 1",
-
- /* BLOCK_INSERT */
- "insert into %_segments (blockid, block) values (null, ?)",
- /* BLOCK_SELECT */ "select block from %_segments where blockid = ?",
- /* BLOCK_DELETE */ "delete from %_segments where blockid between ? and ?",
- /* BLOCK_DELETE_ALL */ "delete from %_segments",
-
- /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
- /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
- /* SEGDIR_SELECT_LEVEL */
- "select start_block, leaves_end_block, root from %_segdir "
- " where level = ? order by idx",
- /* SEGDIR_SPAN */
- "select min(start_block), max(end_block) from %_segdir "
- " where level = ? and start_block <> 0",
- /* SEGDIR_DELETE */ "delete from %_segdir where level = ?",
-
- /* NOTE(shess): The first three results of the following two
- ** statements must match.
- */
- /* SEGDIR_SELECT_SEGMENT */
- "select start_block, leaves_end_block, root from %_segdir "
- " where level = ? and idx = ?",
- /* SEGDIR_SELECT_ALL */
- "select start_block, leaves_end_block, root from %_segdir "
- " order by level desc, idx asc",
- /* SEGDIR_DELETE_ALL */ "delete from %_segdir",
- /* SEGDIR_COUNT */ "select count(*), ifnull(max(level),0) from %_segdir",
-};
-
-/*
-** A connection to a fulltext index is an instance of the following
-** structure. The xCreate and xConnect methods create an instance
-** of this structure and xDestroy and xDisconnect free that instance.
-** All other methods receive a pointer to the structure as one of their
-** arguments.
-*/
-struct fulltext_vtab {
- sqlite3_vtab base; /* Base class used by SQLite core */
- sqlite3 *db; /* The database connection */
- const char *zDb; /* logical database name */
- const char *zName; /* virtual table name */
- int nColumn; /* number of columns in virtual table */
- char **azColumn; /* column names. malloced */
- char **azContentColumn; /* column names in content table; malloced */
- sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
-
- /* Precompiled statements which we keep as long as the table is
- ** open.
- */
- sqlite3_stmt *pFulltextStatements[MAX_STMT];
-
- /* Precompiled statements used for segment merges. We run a
- ** separate select across the leaf level of each tree being merged.
- */
- sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT];
- /* The statement used to prepare pLeafSelectStmts. */
-#define LEAF_SELECT \
- "select block from %_segments where blockid between ? and ? order by blockid"
-
- /* These buffer pending index updates during transactions.
- ** nPendingData estimates the memory size of the pending data. It
- ** doesn't include the hash-bucket overhead, nor any malloc
- ** overhead. When nPendingData exceeds kPendingThreshold, the
- ** buffer is flushed even before the transaction closes.
- ** pendingTerms stores the data, and is only valid when nPendingData
- ** is >=0 (nPendingData<0 means pendingTerms has not been
- ** initialized). iPrevDocid is the last docid written, used to make
- ** certain we're inserting in sorted order.
- */
- int nPendingData;
-#define kPendingThreshold (1*1024*1024)
- sqlite_int64 iPrevDocid;
- fts3Hash pendingTerms;
-};
-
-/*
-** When the core wants to do a query, it create a cursor using a
-** call to xOpen. This structure is an instance of a cursor. It
-** is destroyed by xClose.
-*/
-typedef struct fulltext_cursor {
- sqlite3_vtab_cursor base; /* Base class used by SQLite core */
- QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */
- sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
- int eof; /* True if at End Of Results */
- Fts3Expr *pExpr; /* Parsed MATCH query string */
- Snippet snippet; /* Cached snippet for the current row */
- int iColumn; /* Column being searched */
- DataBuffer result; /* Doclist results from fulltextQuery */
- DLReader reader; /* Result reader if result not empty */
-} fulltext_cursor;
-
-static fulltext_vtab *cursor_vtab(fulltext_cursor *c){
- return (fulltext_vtab *) c->base.pVtab;
-}
-
-static const sqlite3_module fts3Module; /* forward declaration */
-
-/* Return a dynamically generated statement of the form
- * insert into %_content (docid, ...) values (?, ...)
- */
-static const char *contentInsertStatement(fulltext_vtab *v){
- StringBuffer sb;
- int i;
-
- initStringBuffer(&sb);
- append(&sb, "insert into %_content (docid, ");
- appendList(&sb, v->nColumn, v->azContentColumn);
- append(&sb, ") values (?");
- for(i=0; i<v->nColumn; ++i)
- append(&sb, ", ?");
- append(&sb, ")");
- return stringBufferData(&sb);
-}
-
-/* Return a dynamically generated statement of the form
- * select <content columns> from %_content where docid = ?
- */
-static const char *contentSelectStatement(fulltext_vtab *v){
- StringBuffer sb;
- initStringBuffer(&sb);
- append(&sb, "SELECT ");
- appendList(&sb, v->nColumn, v->azContentColumn);
- append(&sb, " FROM %_content WHERE docid = ?");
- return stringBufferData(&sb);
-}
-
-/* Return a dynamically generated statement of the form
- * update %_content set [col_0] = ?, [col_1] = ?, ...
- * where docid = ?
- */
-static const char *contentUpdateStatement(fulltext_vtab *v){
- StringBuffer sb;
- int i;
-
- initStringBuffer(&sb);
- append(&sb, "update %_content set ");
- for(i=0; i<v->nColumn; ++i) {
- if( i>0 ){
- append(&sb, ", ");
- }
- append(&sb, v->azContentColumn[i]);
- append(&sb, " = ?");
- }
- append(&sb, " where docid = ?");
- return stringBufferData(&sb);
-}
-
-/* Puts a freshly-prepared statement determined by iStmt in *ppStmt.
-** If the indicated statement has never been prepared, it is prepared
-** and cached, otherwise the cached version is reset.
-*/
-static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
- sqlite3_stmt **ppStmt){
- assert( iStmt<MAX_STMT );
- if( v->pFulltextStatements[iStmt]==NULL ){
- const char *zStmt;
- int rc;
- switch( iStmt ){
- case CONTENT_INSERT_STMT:
- zStmt = contentInsertStatement(v); break;
- case CONTENT_SELECT_STMT:
- zStmt = contentSelectStatement(v); break;
- case CONTENT_UPDATE_STMT:
- zStmt = contentUpdateStatement(v); break;
- default:
- zStmt = fulltext_zStatement[iStmt];
- }
- rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt],
- zStmt);
- if( zStmt != fulltext_zStatement[iStmt]) sqlite3_free((void *) zStmt);
- if( rc!=SQLITE_OK ) return rc;
- } else {
- int rc = sqlite3_reset(v->pFulltextStatements[iStmt]);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- *ppStmt = v->pFulltextStatements[iStmt];
- return SQLITE_OK;
-}
-
-/* Like sqlite3_step(), but convert SQLITE_DONE to SQLITE_OK and
-** SQLITE_ROW to SQLITE_ERROR. Useful for statements like UPDATE,
-** where we expect no results.
-*/
-static int sql_single_step(sqlite3_stmt *s){
- int rc = sqlite3_step(s);
- return (rc==SQLITE_DONE) ? SQLITE_OK : rc;
-}
-
-/* Like sql_get_statement(), but for special replicated LEAF_SELECT
-** statements. idx -1 is a special case for an uncached version of
-** the statement (used in the optimize implementation).
-*/
-/* TODO(shess) Write version for generic statements and then share
-** that between the cached-statement functions.
-*/
-static int sql_get_leaf_statement(fulltext_vtab *v, int idx,
- sqlite3_stmt **ppStmt){
- assert( idx>=-1 && idx<MERGE_COUNT );
- if( idx==-1 ){
- return sql_prepare(v->db, v->zDb, v->zName, ppStmt, LEAF_SELECT);
- }else if( v->pLeafSelectStmts[idx]==NULL ){
- int rc = sql_prepare(v->db, v->zDb, v->zName, &v->pLeafSelectStmts[idx],
- LEAF_SELECT);
- if( rc!=SQLITE_OK ) return rc;
- }else{
- int rc = sqlite3_reset(v->pLeafSelectStmts[idx]);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- *ppStmt = v->pLeafSelectStmts[idx];
- return SQLITE_OK;
-}
-
-/* insert into %_content (docid, ...) values ([docid], [pValues])
-** If the docid contains SQL NULL, then a unique docid will be
-** generated.
-*/
-static int content_insert(fulltext_vtab *v, sqlite3_value *docid,
- sqlite3_value **pValues){
- sqlite3_stmt *s;
- int i;
- int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_value(s, 1, docid);
- if( rc!=SQLITE_OK ) return rc;
-
- for(i=0; i<v->nColumn; ++i){
- rc = sqlite3_bind_value(s, 2+i, pValues[i]);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- return sql_single_step(s);
-}
-
-/* update %_content set col0 = pValues[0], col1 = pValues[1], ...
- * where docid = [iDocid] */
-static int content_update(fulltext_vtab *v, sqlite3_value **pValues,
- sqlite_int64 iDocid){
- sqlite3_stmt *s;
- int i;
- int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- for(i=0; i<v->nColumn; ++i){
- rc = sqlite3_bind_value(s, 1+i, pValues[i]);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- rc = sqlite3_bind_int64(s, 1+v->nColumn, iDocid);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-
-static void freeStringArray(int nString, const char **pString){
- int i;
-
- for (i=0 ; i < nString ; ++i) {
- if( pString[i]!=NULL ) sqlite3_free((void *) pString[i]);
- }
- sqlite3_free((void *) pString);
-}
-
-/* select * from %_content where docid = [iDocid]
- * The caller must delete the returned array and all strings in it.
- * null fields will be NULL in the returned array.
- *
- * TODO: Perhaps we should return pointer/length strings here for consistency
- * with other code which uses pointer/length. */
-static int content_select(fulltext_vtab *v, sqlite_int64 iDocid,
- const char ***pValues){
- sqlite3_stmt *s;
- const char **values;
- int i;
- int rc;
-
- *pValues = NULL;
-
- rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iDocid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc!=SQLITE_ROW ) return rc;
-
- values = (const char **) sqlite3_malloc(v->nColumn * sizeof(const char *));
- for(i=0; i<v->nColumn; ++i){
- if( sqlite3_column_type(s, i)==SQLITE_NULL ){
- values[i] = NULL;
- }else{
- values[i] = string_dup((char*)sqlite3_column_text(s, i));
- }
- }
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
+ memset(pRet, 0, nByte);
+ pRet->eType = FTSQUERY_PHRASE;
+ pRet->pPhrase = (Fts3Phrase *)&pRet[1];
+ pRet->pPhrase->nToken = 1;
+ pRet->pPhrase->iColumn = iCol;
+ pRet->pPhrase->aToken[0].n = nToken;
+ pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1];
+ memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
+
+ if( iEnd<n && z[iEnd]=='*' ){
+ pRet->pPhrase->aToken[0].isPrefix = 1;
+ iEnd++;
+ }
+ if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
+ pRet->pPhrase->isNot = 1;
+ }
+ }
+ nConsumed = iEnd;
+ }
+
+ pModule->xClose(pCursor);
+ }
+
+ *pnConsumed = nConsumed;
+ *ppExpr = pRet;
+ return rc;
+}
+
+
+/*
+** Enlarge a memory allocation. If an out-of-memory allocation occurs,
+** then free the old allocation.
+*/
+void *fts3ReallocOrFree(void *pOrig, int nNew){
+ void *pRet = sqlite3_realloc(pOrig, nNew);
+ if( !pRet ){
+ sqlite3_free(pOrig);
+ }
+ return pRet;
+}
+
+/*
+** Buffer zInput, length nInput, contains the contents of a quoted string
+** that appeared as part of an fts3 query expression. Neither quote character
+** is included in the buffer. This function attempts to tokenize the entire
+** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE
+** containing the results.
+**
+** If successful, SQLITE_OK is returned and *ppExpr set to point at the
+** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory
+** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set
+** to 0.
+*/
+static int getNextString(
+ ParseContext *pParse, /* fts3 query parse context */
+ const char *zInput, int nInput, /* Input string */
+ Fts3Expr **ppExpr /* OUT: expression */
+){
+ sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ int rc;
+ Fts3Expr *p = 0;
+ sqlite3_tokenizer_cursor *pCursor = 0;
+ char *zTemp = 0;
+ int nTemp = 0;
+
+ rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
+ if( rc==SQLITE_OK ){
+ int ii;
+ pCursor->pTokenizer = pTokenizer;
+ for(ii=0; rc==SQLITE_OK; ii++){
+ const char *zToken;
+ int nToken, iBegin, iEnd, iPos;
+ rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
+ if( rc==SQLITE_OK ){
+ int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
+ p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken));
+ zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
+ if( !p || !zTemp ){
+ goto no_mem;
+ }
+ if( ii==0 ){
+ memset(p, 0, nByte);
+ p->pPhrase = (Fts3Phrase *)&p[1];
+ }
+ p->pPhrase = (Fts3Phrase *)&p[1];
+ p->pPhrase->nToken = ii+1;
+ p->pPhrase->aToken[ii].n = nToken;
+ memcpy(&zTemp[nTemp], zToken, nToken);
+ nTemp += nToken;
+ if( iEnd<nInput && zInput[iEnd]=='*' ){
+ p->pPhrase->aToken[ii].isPrefix = 1;
+ }else{
+ p->pPhrase->aToken[ii].isPrefix = 0;
+ }
+ }
+ }
+
+ pModule->xClose(pCursor);
+ pCursor = 0;
+ }
+
if( rc==SQLITE_DONE ){
- *pValues = values;
- return SQLITE_OK;
- }
-
- freeStringArray(v->nColumn, values);
- return rc;
-}
-
-/* delete from %_content where docid = [iDocid ] */
-static int content_delete(fulltext_vtab *v, sqlite_int64 iDocid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iDocid);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-
-/* Returns SQLITE_ROW if any rows exist in %_content, SQLITE_DONE if
-** no rows exist, and any error in case of failure.
-*/
-static int content_exists(fulltext_vtab *v){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, CONTENT_EXISTS_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc!=SQLITE_ROW ) return rc;
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_ROW;
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- return rc;
-}
-
-/* insert into %_segments values ([pData])
-** returns assigned blockid in *piBlockid
-*/
-static int block_insert(fulltext_vtab *v, const char *pData, int nData,
- sqlite_int64 *piBlockid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, BLOCK_INSERT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_blob(s, 1, pData, nData, SQLITE_STATIC);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
-
- /* blockid column is an alias for rowid. */
- *piBlockid = sqlite3_last_insert_rowid(v->db);
- return SQLITE_OK;
-}
-
-/* delete from %_segments
-** where blockid between [iStartBlockid] and [iEndBlockid]
-**
-** Deletes the range of blocks, inclusive, used to delete the blocks
-** which form a segment.
-*/
-static int block_delete(fulltext_vtab *v,
- sqlite_int64 iStartBlockid, sqlite_int64 iEndBlockid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, BLOCK_DELETE_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iStartBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 2, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-
-/* Returns SQLITE_ROW with *pidx set to the maximum segment idx found
-** at iLevel. Returns SQLITE_DONE if there are no segments at
-** iLevel. Otherwise returns an error.
-*/
-static int segdir_max_index(fulltext_vtab *v, int iLevel, int *pidx){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_MAX_INDEX_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- /* Should always get at least one row due to how max() works. */
- if( rc==SQLITE_DONE ) return SQLITE_DONE;
- if( rc!=SQLITE_ROW ) return rc;
-
- /* NULL means that there were no inputs to max(). */
- if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- return rc;
- }
-
- *pidx = sqlite3_column_int(s, 0);
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
- return SQLITE_ROW;
-}
-
-/* insert into %_segdir values (
-** [iLevel], [idx],
-** [iStartBlockid], [iLeavesEndBlockid], [iEndBlockid],
-** [pRootData]
-** )
-*/
-static int segdir_set(fulltext_vtab *v, int iLevel, int idx,
- sqlite_int64 iStartBlockid,
- sqlite_int64 iLeavesEndBlockid,
- sqlite_int64 iEndBlockid,
- const char *pRootData, int nRootData){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_SET_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 2, idx);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 3, iStartBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 4, iLeavesEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 5, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_blob(s, 6, pRootData, nRootData, SQLITE_STATIC);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-
-/* Queries %_segdir for the block span of the segments in level
-** iLevel. Returns SQLITE_DONE if there are no blocks for iLevel,
-** SQLITE_ROW if there are blocks, else an error.
-*/
-static int segdir_span(fulltext_vtab *v, int iLevel,
- sqlite_int64 *piStartBlockid,
- sqlite_int64 *piEndBlockid){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_SPAN_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_DONE; /* Should never happen */
- if( rc!=SQLITE_ROW ) return rc;
-
- /* This happens if all segments at this level are entirely inline. */
- if( SQLITE_NULL==sqlite3_column_type(s, 0) ){
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- int rc2 = sqlite3_step(s);
- if( rc2==SQLITE_ROW ) return SQLITE_ERROR;
- return rc2;
- }
-
- *piStartBlockid = sqlite3_column_int64(s, 0);
- *piEndBlockid = sqlite3_column_int64(s, 1);
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
- return SQLITE_ROW;
-}
-
-/* Delete the segment blocks and segment directory records for all
-** segments at iLevel.
-*/
-static int segdir_delete(fulltext_vtab *v, int iLevel){
- sqlite3_stmt *s;
- sqlite_int64 iStartBlockid, iEndBlockid;
- int rc = segdir_span(v, iLevel, &iStartBlockid, &iEndBlockid);
- if( rc!=SQLITE_ROW && rc!=SQLITE_DONE ) return rc;
-
- if( rc==SQLITE_ROW ){
- rc = block_delete(v, iStartBlockid, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- /* Delete the segment directory itself. */
- rc = sql_get_statement(v, SEGDIR_DELETE_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-
-/* Delete entire fts index, SQLITE_OK on success, relevant error on
-** failure.
-*/
-static int segdir_delete_all(fulltext_vtab *v){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_DELETE_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sql_single_step(s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sql_get_statement(v, BLOCK_DELETE_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- return sql_single_step(s);
-}
-
-/* Returns SQLITE_OK with *pnSegments set to the number of entries in
-** %_segdir and *piMaxLevel set to the highest level which has a
-** segment. Otherwise returns the SQLite error which caused failure.
-*/
-static int segdir_count(fulltext_vtab *v, int *pnSegments, int *piMaxLevel){
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_COUNT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- /* TODO(shess): This case should not be possible? Should stronger
- ** measures be taken if it happens?
- */
- if( rc==SQLITE_DONE ){
- *pnSegments = 0;
- *piMaxLevel = 0;
- return SQLITE_OK;
- }
- if( rc!=SQLITE_ROW ) return rc;
-
- *pnSegments = sqlite3_column_int(s, 0);
- *piMaxLevel = sqlite3_column_int(s, 1);
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_OK;
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- return rc;
-}
-
-/* TODO(shess) clearPendingTerms() is far down the file because
-** writeZeroSegment() is far down the file because LeafWriter is far
-** down the file. Consider refactoring the code to move the non-vtab
-** code above the vtab code so that we don't need this forward
-** reference.
-*/
-static int clearPendingTerms(fulltext_vtab *v);
-
-/*
-** Free the memory used to contain a fulltext_vtab structure.
-*/
-static void fulltext_vtab_destroy(fulltext_vtab *v){
- int iStmt, i;
-
- FTSTRACE(("FTS3 Destroy %p\n", v));
- for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){
- if( v->pFulltextStatements[iStmt]!=NULL ){
- sqlite3_finalize(v->pFulltextStatements[iStmt]);
- v->pFulltextStatements[iStmt] = NULL;
- }
- }
-
- for( i=0; i<MERGE_COUNT; i++ ){
- if( v->pLeafSelectStmts[i]!=NULL ){
- sqlite3_finalize(v->pLeafSelectStmts[i]);
- v->pLeafSelectStmts[i] = NULL;
- }
- }
-
- if( v->pTokenizer!=NULL ){
- v->pTokenizer->pModule->xDestroy(v->pTokenizer);
- v->pTokenizer = NULL;
- }
-
- clearPendingTerms(v);
-
- sqlite3_free(v->azColumn);
- for(i = 0; i < v->nColumn; ++i) {
- sqlite3_free(v->azContentColumn[i]);
- }
- sqlite3_free(v->azContentColumn);
- sqlite3_free(v);
-}
-
-/*
-** Token types for parsing the arguments to xConnect or xCreate.
-*/
-#define TOKEN_EOF 0 /* End of file */
-#define TOKEN_SPACE 1 /* Any kind of whitespace */
-#define TOKEN_ID 2 /* An identifier */
-#define TOKEN_STRING 3 /* A string literal */
-#define TOKEN_PUNCT 4 /* A single punctuation character */
-
-/*
-** If X is a character that can be used in an identifier then
-** ftsIdChar(X) will be true. Otherwise it is false.
-**
-** For ASCII, any character with the high-order bit set is
-** allowed in an identifier. For 7-bit characters,
-** isFtsIdChar[X] must be 1.
-**
-** Ticket #1066. the SQL standard does not allow '$' in the
-** middle of identfiers. But many SQL implementations do.
-** SQLite will allow '$' in identifiers for compatibility.
-** But the feature is undocumented.
-*/
-static const char isFtsIdChar[] = {
-/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
- 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
- 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
- 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
-};
-#define ftsIdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isFtsIdChar[c-0x20]))
-
-
-/*
-** Return the length of the token that begins at z[0].
-** Store the token type in *tokenType before returning.
-*/
-static int ftsGetToken(const char *z, int *tokenType){
- int i, c;
- switch( *z ){
- case 0: {
- *tokenType = TOKEN_EOF;
- return 0;
- }
- case ' ': case '\t': case '\n': case '\f': case '\r': {
- for(i=1; safe_isspace(z[i]); i++){}
- *tokenType = TOKEN_SPACE;
- return i;
- }
- case '`':
- case '\'':
- case '"': {
- int delim = z[0];
- for(i=1; (c=z[i])!=0; i++){
- if( c==delim ){
- if( z[i+1]==delim ){
- i++;
- }else{
- break;
+ int jj;
+ char *zNew;
+ int nNew = 0;
+ int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
+ nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(struct PhraseToken);
+ p = fts3ReallocOrFree(p, nByte + nTemp);
+ if( !p ){
+ goto no_mem;
+ }
+ if( zTemp ){
+ zNew = &(((char *)p)[nByte]);
+ memcpy(zNew, zTemp, nTemp);
+ }else{
+ memset(p, 0, nByte+nTemp);
+ }
+ p->pPhrase = (Fts3Phrase *)&p[1];
+ for(jj=0; jj<p->pPhrase->nToken; jj++){
+ p->pPhrase->aToken[jj].z = &zNew[nNew];
+ nNew += p->pPhrase->aToken[jj].n;
+ }
+ sqlite3_free(zTemp);
+ p->eType = FTSQUERY_PHRASE;
+ p->pPhrase->iColumn = pParse->iDefaultCol;
+ rc = SQLITE_OK;
+ }
+
+ *ppExpr = p;
+ return rc;
+no_mem:
+
+ if( pCursor ){
+ pModule->xClose(pCursor);
+ }
+ sqlite3_free(zTemp);
+ sqlite3_free(p);
+ *ppExpr = 0;
+ return SQLITE_NOMEM;
+}
+
+/*
+** Function getNextNode(), which is called by fts3ExprParse(), may itself
+** call fts3ExprParse(). So this forward declaration is required.
+*/
+static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);
+
+/*
+** The output variable *ppExpr is populated with an allocated Fts3Expr
+** structure, or set to 0 if the end of the input buffer is reached.
+**
+** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
+** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
+** If SQLITE_ERROR is returned, pContext is populated with an error message.
+*/
+static int getNextNode(
+ ParseContext *pParse, /* fts3 query parse context */
+ const char *z, int n, /* Input string */
+ Fts3Expr **ppExpr, /* OUT: expression */
+ int *pnConsumed /* OUT: Number of bytes consumed */
+){
+ static const struct Fts3Keyword {
+ char z[4]; /* Keyword text */
+ unsigned char n; /* Length of the keyword */
+ unsigned char parenOnly; /* Only valid in paren mode */
+ unsigned char eType; /* Keyword code */
+ } aKeyword[] = {
+ { "OR" , 2, 0, FTSQUERY_OR },
+ { "AND", 3, 1, FTSQUERY_AND },
+ { "NOT", 3, 1, FTSQUERY_NOT },
+ { "NEAR", 4, 0, FTSQUERY_NEAR }
+ };
+ int ii;
+ int iCol;
+ int iColLen;
+ int rc;
+ Fts3Expr *pRet = 0;
+
+ const char *zInput = z;
+ int nInput = n;
+
+ /* Skip over any whitespace before checking for a keyword, an open or
+ ** close bracket, or a quoted string.
+ */
+ while( nInput>0 && fts3isspace(*zInput) ){
+ nInput--;
+ zInput++;
+ }
+ if( nInput==0 ){
+ return SQLITE_DONE;
+ }
+
+ /* See if we are dealing with a keyword. */
+ for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){
+ const struct Fts3Keyword *pKey = &aKeyword[ii];
+
+ if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){
+ continue;
+ }
+
+ if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){
+ int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
+ int nKey = pKey->n;
+ char cNext;
+
+ /* If this is a "NEAR" keyword, check for an explicit nearness. */
+ if( pKey->eType==FTSQUERY_NEAR ){
+ assert( nKey==4 );
+ if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){
+ nNear = 0;
+ for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){
+ nNear = nNear * 10 + (zInput[nKey] - '0');
}
}
}
- *tokenType = TOKEN_STRING;
- return i + (c!=0);
- }
- case '[': {
- for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
- *tokenType = TOKEN_ID;
- return i;
- }
- default: {
- if( !ftsIdChar(*z) ){
- break;
- }
- for(i=1; ftsIdChar(z[i]); i++){}
- *tokenType = TOKEN_ID;
- return i;
- }
- }
- *tokenType = TOKEN_PUNCT;
- return 1;
-}
-
-/*
-** A token extracted from a string is an instance of the following
-** structure.
-*/
-typedef struct FtsToken {
- const char *z; /* Pointer to token text. Not '\000' terminated */
- short int n; /* Length of the token text in bytes. */
-} FtsToken;
-
-/*
-** Given a input string (which is really one of the argv[] parameters
-** passed into xConnect or xCreate) split the string up into tokens.
-** Return an array of pointers to '\000' terminated strings, one string
-** for each non-whitespace token.
-**
-** The returned array is terminated by a single NULL pointer.
-**
-** Space to hold the returned array is obtained from a single
-** malloc and should be freed by passing the return value to free().
-** The individual strings within the token list are all a part of
-** the single memory allocation and will all be freed at once.
-*/
-static char **tokenizeString(const char *z, int *pnToken){
- int nToken = 0;
- FtsToken *aToken = sqlite3_malloc( strlen(z) * sizeof(aToken[0]) );
- int n = 1;
- int e, i;
- int totalSize = 0;
- char **azToken;
- char *zCopy;
- while( n>0 ){
- n = ftsGetToken(z, &e);
- if( e!=TOKEN_SPACE ){
- aToken[nToken].z = z;
- aToken[nToken].n = n;
- nToken++;
- totalSize += n+1;
- }
- z += n;
- }
- azToken = (char**)sqlite3_malloc( nToken*sizeof(char*) + totalSize );
- zCopy = (char*)&azToken[nToken];
- nToken--;
- for(i=0; i<nToken; i++){
- azToken[i] = zCopy;
- n = aToken[i].n;
- memcpy(zCopy, aToken[i].z, n);
- zCopy[n] = 0;
- zCopy += n+1;
- }
- azToken[nToken] = 0;
- sqlite3_free(aToken);
- *pnToken = nToken;
- return azToken;
-}
-
-/*
-** Convert an SQL-style quoted string into a normal string by removing
-** the quote characters. The conversion is done in-place. If the
-** input does not begin with a quote character, then this routine
-** is a no-op.
-**
-** Examples:
-**
-** "abc" becomes abc
-** 'xyz' becomes xyz
-** [pqr] becomes pqr
-** `mno` becomes mno
-*/
-static void dequoteString(char *z){
- int quote;
- int i, j;
- if( z==0 ) return;
- quote = z[0];
- switch( quote ){
- case '\'': break;
- case '"': break;
- case '`': break; /* For MySQL compatibility */
- case '[': quote = ']'; break; /* For MS SqlServer compatibility */
- default: return;
- }
- for(i=1, j=0; z[i]; i++){
- if( z[i]==quote ){
- if( z[i+1]==quote ){
- z[j++] = quote;
- i++;
- }else{
- z[j++] = 0;
- break;
- }
- }else{
- z[j++] = z[i];
- }
- }
-}
-
-/*
-** The input azIn is a NULL-terminated list of tokens. Remove the first
-** token and all punctuation tokens. Remove the quotes from
-** around string literal tokens.
-**
-** Example:
-**
-** input: tokenize chinese ( 'simplifed' , 'mixed' )
-** output: chinese simplifed mixed
-**
-** Another example:
-**
-** input: delimiters ( '[' , ']' , '...' )
-** output: [ ] ...
-*/
-static void tokenListToIdList(char **azIn){
- int i, j;
- if( azIn ){
- for(i=0, j=-1; azIn[i]; i++){
- if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){
- dequoteString(azIn[i]);
- if( j>=0 ){
- azIn[j] = azIn[i];
- }
- j++;
- }
- }
- azIn[j] = 0;
- }
-}
-
-
-/*
-** Find the first alphanumeric token in the string zIn. Null-terminate
-** this token. Remove any quotation marks. And return a pointer to
-** the result.
-*/
-static char *firstToken(char *zIn, char **pzTail){
- int n, ttype;
- while(1){
- n = ftsGetToken(zIn, &ttype);
- if( ttype==TOKEN_SPACE ){
- zIn += n;
- }else if( ttype==TOKEN_EOF ){
- *pzTail = zIn;
- return 0;
- }else{
- zIn[n] = 0;
- *pzTail = &zIn[1];
- dequoteString(zIn);
- return zIn;
- }
- }
- /*NOTREACHED*/
-}
-
-/* Return true if...
-**
-** * s begins with the string t, ignoring case
-** * s is longer than t
-** * The first character of s beyond t is not a alphanumeric
-**
-** Ignore leading space in *s.
-**
-** To put it another way, return true if the first token of
-** s[] is t[].
-*/
-static int startsWith(const char *s, const char *t){
- while( safe_isspace(*s) ){ s++; }
- while( *t ){
- if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0;
- }
- return *s!='_' && !safe_isalnum(*s);
-}
-
-/*
-** An instance of this structure defines the "spec" of a
-** full text index. This structure is populated by parseSpec
-** and use by fulltextConnect and fulltextCreate.
-*/
-typedef struct TableSpec {
- const char *zDb; /* Logical database name */
- const char *zName; /* Name of the full-text index */
- int nColumn; /* Number of columns to be indexed */
- char **azColumn; /* Original names of columns to be indexed */
- char **azContentColumn; /* Column names for %_content */
- char **azTokenizer; /* Name of tokenizer and its arguments */
-} TableSpec;
-
-/*
-** Reclaim all of the memory used by a TableSpec
-*/
-static void clearTableSpec(TableSpec *p) {
- sqlite3_free(p->azColumn);
- sqlite3_free(p->azContentColumn);
- sqlite3_free(p->azTokenizer);
-}
-
-/* Parse a CREATE VIRTUAL TABLE statement, which looks like this:
- *
- * CREATE VIRTUAL TABLE email
- * USING fts3(subject, body, tokenize mytokenizer(myarg))
- *
- * We return parsed information in a TableSpec structure.
- *
- */
-static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv,
- char**pzErr){
- int i, n;
- char *z, *zDummy;
- char **azArg;
- const char *zTokenizer = 0; /* argv[] entry describing the tokenizer */
-
- assert( argc>=3 );
- /* Current interface:
- ** argv[0] - module name
- ** argv[1] - database name
- ** argv[2] - table name
- ** argv[3..] - columns, optionally followed by tokenizer specification
- ** and snippet delimiters specification.
- */
-
- /* Make a copy of the complete argv[][] array in a single allocation.
- ** The argv[][] array is read-only and transient. We can write to the
- ** copy in order to modify things and the copy is persistent.
- */
- CLEAR(pSpec);
- for(i=n=0; i<argc; i++){
- n += strlen(argv[i]) + 1;
- }
- azArg = sqlite3_malloc( sizeof(char*)*argc + n );
- if( azArg==0 ){
- return SQLITE_NOMEM;
- }
- z = (char*)&azArg[argc];
- for(i=0; i<argc; i++){
- azArg[i] = z;
- strcpy(z, argv[i]);
- z += strlen(z)+1;
- }
-
- /* Identify the column names and the tokenizer and delimiter arguments
- ** in the argv[][] array.
- */
- pSpec->zDb = azArg[1];
- pSpec->zName = azArg[2];
- pSpec->nColumn = 0;
- pSpec->azColumn = azArg;
- zTokenizer = "tokenize simple";
- for(i=3; i<argc; ++i){
- if( startsWith(azArg[i],"tokenize") ){
- zTokenizer = azArg[i];
- }else{
- z = azArg[pSpec->nColumn] = firstToken(azArg[i], &zDummy);
- pSpec->nColumn++;
- }
- }
- if( pSpec->nColumn==0 ){
- azArg[0] = "content";
- pSpec->nColumn = 1;
- }
-
- /*
- ** Construct the list of content column names.
- **
- ** Each content column name will be of the form cNNAAAA
- ** where NN is the column number and AAAA is the sanitized
- ** column name. "sanitized" means that special characters are
- ** converted to "_". The cNN prefix guarantees that all column
- ** names are unique.
- **
- ** The AAAA suffix is not strictly necessary. It is included
- ** for the convenience of people who might examine the generated
- ** %_content table and wonder what the columns are used for.
- */
- pSpec->azContentColumn = sqlite3_malloc( pSpec->nColumn * sizeof(char *) );
- if( pSpec->azContentColumn==0 ){
- clearTableSpec(pSpec);
- return SQLITE_NOMEM;
- }
- for(i=0; i<pSpec->nColumn; i++){
- char *p;
- pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]);
- for (p = pSpec->azContentColumn[i]; *p ; ++p) {
- if( !safe_isalnum(*p) ) *p = '_';
- }
- }
-
- /*
- ** Parse the tokenizer specification string.
- */
- pSpec->azTokenizer = tokenizeString(zTokenizer, &n);
- tokenListToIdList(pSpec->azTokenizer);
-
- return SQLITE_OK;
-}
-
-/*
-** Generate a CREATE TABLE statement that describes the schema of
-** the virtual table. Return a pointer to this schema string.
-**
-** Space is obtained from sqlite3_mprintf() and should be freed
-** using sqlite3_free().
-*/
-static char *fulltextSchema(
- int nColumn, /* Number of columns */
- const char *const* azColumn, /* List of columns */
- const char *zTableName /* Name of the table */
-){
- int i;
- char *zSchema, *zNext;
- const char *zSep = "(";
- zSchema = sqlite3_mprintf("CREATE TABLE x");
- for(i=0; i<nColumn; i++){
- zNext = sqlite3_mprintf("%s%s%Q", zSchema, zSep, azColumn[i]);
- sqlite3_free(zSchema);
- zSchema = zNext;
- zSep = ",";
- }
- zNext = sqlite3_mprintf("%s,%Q HIDDEN", zSchema, zTableName);
- sqlite3_free(zSchema);
- zSchema = zNext;
- zNext = sqlite3_mprintf("%s,docid HIDDEN)", zSchema);
- sqlite3_free(zSchema);
- return zNext;
-}
-
-/*
-** Build a new sqlite3_vtab structure that will describe the
-** fulltext index defined by spec.
-*/
-static int constructVtab(
- sqlite3 *db, /* The SQLite database connection */
- fts3Hash *pHash, /* Hash table containing tokenizers */
- TableSpec *spec, /* Parsed spec information from parseSpec() */
- sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
- char **pzErr /* Write any error message here */
-){
- int rc;
- int n;
- fulltext_vtab *v = 0;
- const sqlite3_tokenizer_module *m = NULL;
- char *schema;
-
- char const *zTok; /* Name of tokenizer to use for this fts table */
- int nTok; /* Length of zTok, including nul terminator */
-
- v = (fulltext_vtab *) sqlite3_malloc(sizeof(fulltext_vtab));
- if( v==0 ) return SQLITE_NOMEM;
- CLEAR(v);
- /* sqlite will initialize v->base */
- v->db = db;
- v->zDb = spec->zDb; /* Freed when azColumn is freed */
- v->zName = spec->zName; /* Freed when azColumn is freed */
- v->nColumn = spec->nColumn;
- v->azContentColumn = spec->azContentColumn;
- spec->azContentColumn = 0;
- v->azColumn = spec->azColumn;
- spec->azColumn = 0;
-
- if( spec->azTokenizer==0 ){
- return SQLITE_NOMEM;
- }
-
- zTok = spec->azTokenizer[0];
- if( !zTok ){
- zTok = "simple";
- }
- nTok = strlen(zTok)+1;
-
- m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zTok, nTok);
- if( !m ){
- *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]);
- rc = SQLITE_ERROR;
- goto err;
- }
-
- for(n=0; spec->azTokenizer[n]; n++){}
- if( n ){
- rc = m->xCreate(n-1, (const char*const*)&spec->azTokenizer[1],
- &v->pTokenizer);
- }else{
- rc = m->xCreate(0, 0, &v->pTokenizer);
- }
- if( rc!=SQLITE_OK ) goto err;
- v->pTokenizer->pModule = m;
-
- /* TODO: verify the existence of backing tables foo_content, foo_term */
-
- schema = fulltextSchema(v->nColumn, (const char*const*)v->azColumn,
- spec->zName);
- rc = sqlite3_declare_vtab(db, schema);
- sqlite3_free(schema);
- if( rc!=SQLITE_OK ) goto err;
-
- memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements));
-
- /* Indicate that the buffer is not live. */
- v->nPendingData = -1;
-
- *ppVTab = &v->base;
- FTSTRACE(("FTS3 Connect %p\n", v));
-
- return rc;
-
-err:
- fulltext_vtab_destroy(v);
- return rc;
-}
-
-static int fulltextConnect(
- sqlite3 *db,
- void *pAux,
- int argc, const char *const*argv,
- sqlite3_vtab **ppVTab,
- char **pzErr
-){
- TableSpec spec;
- int rc = parseSpec(&spec, argc, argv, pzErr);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr);
- clearTableSpec(&spec);
- return rc;
-}
-
-/* The %_content table holds the text of each document, with
-** the docid column exposed as the SQLite rowid for the table.
-*/
-/* TODO(shess) This comment needs elaboration to match the updated
-** code. Work it into the top-of-file comment at that time.
-*/
-static int fulltextCreate(sqlite3 *db, void *pAux,
- int argc, const char * const *argv,
- sqlite3_vtab **ppVTab, char **pzErr){
- int rc;
- TableSpec spec;
- StringBuffer schema;
- FTSTRACE(("FTS3 Create\n"));
-
- rc = parseSpec(&spec, argc, argv, pzErr);
- if( rc!=SQLITE_OK ) return rc;
-
- initStringBuffer(&schema);
- append(&schema, "CREATE TABLE %_content(");
- append(&schema, " docid INTEGER PRIMARY KEY,");
- appendList(&schema, spec.nColumn, spec.azContentColumn);
- append(&schema, ")");
- rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema));
- stringBufferDestroy(&schema);
- if( rc!=SQLITE_OK ) goto out;
-
- rc = sql_exec(db, spec.zDb, spec.zName,
- "create table %_segments("
- " blockid INTEGER PRIMARY KEY,"
- " block blob"
- ");"
- );
- if( rc!=SQLITE_OK ) goto out;
-
- rc = sql_exec(db, spec.zDb, spec.zName,
- "create table %_segdir("
- " level integer,"
- " idx integer,"
- " start_block integer,"
- " leaves_end_block integer,"
- " end_block integer,"
- " root blob,"
- " primary key(level, idx)"
- ");");
- if( rc!=SQLITE_OK ) goto out;
-
- rc = constructVtab(db, (fts3Hash *)pAux, &spec, ppVTab, pzErr);
-
-out:
- clearTableSpec(&spec);
- return rc;
-}
-
-/* Decide how to handle an SQL query. */
-static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
- fulltext_vtab *v = (fulltext_vtab *)pVTab;
- int i;
- FTSTRACE(("FTS3 BestIndex\n"));
-
- for(i=0; i<pInfo->nConstraint; ++i){
- const struct sqlite3_index_constraint *pConstraint;
- pConstraint = &pInfo->aConstraint[i];
- if( pConstraint->usable ) {
- if( (pConstraint->iColumn==-1 || pConstraint->iColumn==v->nColumn+1) &&
- pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
- pInfo->idxNum = QUERY_DOCID; /* lookup by docid */
- FTSTRACE(("FTS3 QUERY_DOCID\n"));
- } else if( pConstraint->iColumn>=0 && pConstraint->iColumn<=v->nColumn &&
- pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){
- /* full-text search */
- pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn;
- FTSTRACE(("FTS3 QUERY_FULLTEXT %d\n", pConstraint->iColumn));
- } else continue;
-
- pInfo->aConstraintUsage[i].argvIndex = 1;
- pInfo->aConstraintUsage[i].omit = 1;
-
- /* An arbitrary value for now.
- * TODO: Perhaps docid matches should be considered cheaper than
- * full-text searches. */
- pInfo->estimatedCost = 1.0;
-
- return SQLITE_OK;
- }
- }
- pInfo->idxNum = QUERY_GENERIC;
- return SQLITE_OK;
-}
-
-static int fulltextDisconnect(sqlite3_vtab *pVTab){
- FTSTRACE(("FTS3 Disconnect %p\n", pVTab));
- fulltext_vtab_destroy((fulltext_vtab *)pVTab);
- return SQLITE_OK;
-}
-
-static int fulltextDestroy(sqlite3_vtab *pVTab){
- fulltext_vtab *v = (fulltext_vtab *)pVTab;
- int rc;
-
- FTSTRACE(("FTS3 Destroy %p\n", pVTab));
- rc = sql_exec(v->db, v->zDb, v->zName,
- "drop table if exists %_content;"
- "drop table if exists %_segments;"
- "drop table if exists %_segdir;"
- );
- if( rc!=SQLITE_OK ) return rc;
-
- fulltext_vtab_destroy((fulltext_vtab *)pVTab);
- return SQLITE_OK;
-}
-
-static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
- fulltext_cursor *c;
-
- c = (fulltext_cursor *) sqlite3_malloc(sizeof(fulltext_cursor));
- if( c ){
- memset(c, 0, sizeof(fulltext_cursor));
- /* sqlite will initialize c->base */
- *ppCursor = &c->base;
- FTSTRACE(("FTS3 Open %p: %p\n", pVTab, c));
- return SQLITE_OK;
- }else{
- return SQLITE_NOMEM;
- }
-}
-
-/* Free all of the dynamically allocated memory held by the
-** Snippet
-*/
-static void snippetClear(Snippet *p){
- sqlite3_free(p->aMatch);
- sqlite3_free(p->zOffset);
- sqlite3_free(p->zSnippet);
- CLEAR(p);
-}
-
-/*
-** Append a single entry to the p->aMatch[] log.
-*/
-static void snippetAppendMatch(
- Snippet *p, /* Append the entry to this snippet */
- int iCol, int iTerm, /* The column and query term */
- int iToken, /* Matching token in document */
- int iStart, int nByte /* Offset and size of the match */
-){
- int i;
- struct snippetMatch *pMatch;
- if( p->nMatch+1>=p->nAlloc ){
- p->nAlloc = p->nAlloc*2 + 10;
- p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
- if( p->aMatch==0 ){
- p->nMatch = 0;
- p->nAlloc = 0;
- return;
- }
- }
- i = p->nMatch++;
- pMatch = &p->aMatch[i];
- pMatch->iCol = iCol;
- pMatch->iTerm = iTerm;
- pMatch->iToken = iToken;
- pMatch->iStart = iStart;
- pMatch->nByte = nByte;
-}
-
-/*
-** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
-*/
-#define FTS3_ROTOR_SZ (32)
-#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1)
-
-/*
-** Function to iterate through the tokens of a compiled expression.
-**
-** Except, skip all tokens on the right-hand side of a NOT operator.
-** This function is used to find tokens as part of snippet and offset
-** generation and we do nt want snippets and offsets to report matches
-** for tokens on the RHS of a NOT.
-*/
-static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){
- Fts3Expr *p = *ppExpr;
- int iToken = *piToken;
- if( iToken<0 ){
- /* In this case the expression p is the root of an expression tree.
- ** Move to the first token in the expression tree.
- */
- while( p->pLeft ){
- p = p->pLeft;
- }
- iToken = 0;
- }else{
- assert(p && p->eType==FTSQUERY_PHRASE );
- if( iToken<(p->pPhrase->nToken-1) ){
- iToken++;
- }else{
- iToken = 0;
- while( p->pParent && p->pParent->pLeft!=p ){
- assert( p->pParent->pRight==p );
- p = p->pParent;
- }
- p = p->pParent;
- if( p ){
- assert( p->pRight!=0 );
- p = p->pRight;
- while( p->pLeft ){
- p = p->pLeft;
- }
- }
- }
- }
-
- *ppExpr = p;
- *piToken = iToken;
- return p?1:0;
-}
-
-/*
-** Return TRUE if the expression node pExpr is located beneath the
-** RHS of a NOT operator.
-*/
-static int fts3ExprBeneathNot(Fts3Expr *p){
- Fts3Expr *pParent;
- while( p ){
- pParent = p->pParent;
- if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){
- return 1;
- }
- p = pParent;
- }
- return 0;
-}
-
-/*
-** Add entries to pSnippet->aMatch[] for every match that occurs against
-** document zDoc[0..nDoc-1] which is stored in column iColumn.
-*/
-static void snippetOffsetsOfColumn(
- fulltext_cursor *pCur, /* The fulltest search cursor */
- Snippet *pSnippet, /* The Snippet object to be filled in */
- int iColumn, /* Index of fulltext table column */
- const char *zDoc, /* Text of the fulltext table column */
- int nDoc /* Length of zDoc in bytes */
-){
- const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */
- sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */
- sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */
- fulltext_vtab *pVtab; /* The full text index */
- int nColumn; /* Number of columns in the index */
- int i, j; /* Loop counters */
- int rc; /* Return code */
- unsigned int match, prevMatch; /* Phrase search bitmasks */
- const char *zToken; /* Next token from the tokenizer */
- int nToken; /* Size of zToken */
- int iBegin, iEnd, iPos; /* Offsets of beginning and end */
-
- /* The following variables keep a circular buffer of the last
- ** few tokens */
- unsigned int iRotor = 0; /* Index of current token */
- int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */
- int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */
-
- pVtab = cursor_vtab(pCur);
- nColumn = pVtab->nColumn;
- pTokenizer = pVtab->pTokenizer;
- pTModule = pTokenizer->pModule;
- rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
- if( rc ) return;
- pTCursor->pTokenizer = pTokenizer;
-
- prevMatch = 0;
- while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){
- Fts3Expr *pIter = pCur->pExpr;
- int iIter = -1;
- iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin;
- iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin;
- match = 0;
- for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){
- int nPhrase; /* Number of tokens in current phrase */
- struct PhraseToken *pToken; /* Current token */
- int iCol; /* Column index */
-
- if( fts3ExprBeneathNot(pIter) ) continue;
- nPhrase = pIter->pPhrase->nToken;
- pToken = &pIter->pPhrase->aToken[iIter];
- iCol = pIter->pPhrase->iColumn;
- if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
- if( pToken->n>nToken ) continue;
- if( !pToken->isPrefix && pToken->n<nToken ) continue;
- assert( pToken->n<=nToken );
- if( memcmp(pToken->z, zToken, pToken->n) ) continue;
- if( iIter>0 && (prevMatch & (1<<i))==0 ) continue;
- match |= 1<<i;
- if( i==(FTS3_ROTOR_SZ-2) || nPhrase==iIter+1 ){
- for(j=nPhrase-1; j>=0; j--){
- int k = (iRotor-j) & FTS3_ROTOR_MASK;
- snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j,
- iRotorBegin[k], iRotorLen[k]);
- }
- }
- }
- prevMatch = match<<1;
- iRotor++;
- }
- pTModule->xClose(pTCursor);
-}
-
-/*
-** Remove entries from the pSnippet structure to account for the NEAR
-** operator. When this is called, pSnippet contains the list of token
-** offsets produced by treating all NEAR operators as AND operators.
-** This function removes any entries that should not be present after
-** accounting for the NEAR restriction. For example, if the queried
-** document is:
-**
-** "A B C D E A"
-**
-** and the query is:
-**
-** A NEAR/0 E
-**
-** then when this function is called the Snippet contains token offsets
-** 0, 4 and 5. This function removes the "0" entry (because the first A
-** is not near enough to an E).
-**
-** When this function is called, the value pointed to by parameter piLeft is
-** the integer id of the left-most token in the expression tree headed by
-** pExpr. This function increments *piLeft by the total number of tokens
-** in the expression tree headed by pExpr.
-**
-** Return 1 if any trimming occurs. Return 0 if no trimming is required.
-*/
-static int trimSnippetOffsets(
- Fts3Expr *pExpr, /* The search expression */
- Snippet *pSnippet, /* The set of snippet offsets to be trimmed */
- int *piLeft /* Index of left-most token in pExpr */
-){
- if( pExpr ){
- if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){
- return 1;
- }
-
- switch( pExpr->eType ){
- case FTSQUERY_PHRASE:
- *piLeft += pExpr->pPhrase->nToken;
- break;
- case FTSQUERY_NEAR: {
- /* The right-hand-side of a NEAR operator is always a phrase. The
- ** left-hand-side is either a phrase or an expression tree that is
- ** itself headed by a NEAR operator. The following initializations
- ** set local variable iLeft to the token number of the left-most
- ** token in the right-hand phrase, and iRight to the right most
- ** token in the same phrase. For example, if we had:
- **
- ** <col> MATCH '"abc def" NEAR/2 "ghi jkl"'
- **
- ** then iLeft will be set to 2 (token number of ghi) and nToken will
- ** be set to 4.
- */
- Fts3Expr *pLeft = pExpr->pLeft;
- Fts3Expr *pRight = pExpr->pRight;
- int iLeft = *piLeft;
- int nNear = pExpr->nNear;
- int nToken = pRight->pPhrase->nToken;
- int jj, ii;
- if( pLeft->eType==FTSQUERY_NEAR ){
- pLeft = pLeft->pRight;
- }
- assert( pRight->eType==FTSQUERY_PHRASE );
- assert( pLeft->eType==FTSQUERY_PHRASE );
- nToken += pLeft->pPhrase->nToken;
-
- for(ii=0; ii<pSnippet->nMatch; ii++){
- struct snippetMatch *p = &pSnippet->aMatch[ii];
- if( p->iTerm==iLeft ){
- int isOk = 0;
- /* Snippet ii is an occurence of query term iLeft in the document.
- ** It occurs at position (p->iToken) of the document. We now
- ** search for an instance of token (iLeft-1) somewhere in the
- ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within
- ** the set of snippetMatch structures. If one is found, proceed.
- ** If one cannot be found, then remove snippets ii..(ii+N-1)
- ** from the matching snippets, where N is the number of tokens
- ** in phrase pRight->pPhrase.
- */
- for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
- struct snippetMatch *p2 = &pSnippet->aMatch[jj];
- if( p2->iTerm==(iLeft-1) ){
- if( p2->iToken>=(p->iToken-nNear-1)
- && p2->iToken<(p->iToken+nNear+nToken)
- ){
- isOk = 1;
- }
- }
- }
- if( !isOk ){
- int kk;
- for(kk=0; kk<pRight->pPhrase->nToken; kk++){
- pSnippet->aMatch[kk+ii].iTerm = -2;
- }
- return 1;
- }
- }
- if( p->iTerm==(iLeft-1) ){
- int isOk = 0;
- for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
- struct snippetMatch *p2 = &pSnippet->aMatch[jj];
- if( p2->iTerm==iLeft ){
- if( p2->iToken<=(p->iToken+nNear+1)
- && p2->iToken>(p->iToken-nNear-nToken)
- ){
- isOk = 1;
- }
- }
- }
- if( !isOk ){
- int kk;
- for(kk=0; kk<pLeft->pPhrase->nToken; kk++){
- pSnippet->aMatch[ii-kk].iTerm = -2;
- }
- return 1;
- }
- }
- }
- break;
- }
- }
-
- if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){
- return 1;
- }
- }
- return 0;
-}
-
-/*
-** Compute all offsets for the current row of the query.
-** If the offsets have already been computed, this routine is a no-op.
-*/
-static void snippetAllOffsets(fulltext_cursor *p){
- int nColumn;
- int iColumn, i;
- int iFirst, iLast;
- int iTerm = 0;
- fulltext_vtab *pFts = cursor_vtab(p);
-
- if( p->snippet.nMatch || p->pExpr==0 ){
- return;
- }
- nColumn = pFts->nColumn;
- iColumn = (p->iCursorType - QUERY_FULLTEXT);
- if( iColumn<0 || iColumn>=nColumn ){
- /* Look for matches over all columns of the full-text index */
- iFirst = 0;
- iLast = nColumn-1;
- }else{
- /* Look for matches in the iColumn-th column of the index only */
- iFirst = iColumn;
- iLast = iColumn;
- }
- for(i=iFirst; i<=iLast; i++){
- const char *zDoc;
- int nDoc;
- zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1);
- nDoc = sqlite3_column_bytes(p->pStmt, i+1);
- snippetOffsetsOfColumn(p, &p->snippet, i, zDoc, nDoc);
- }
-
- while( trimSnippetOffsets(p->pExpr, &p->snippet, &iTerm) ){
- iTerm = 0;
- }
-}
-
-/*
-** Convert the information in the aMatch[] array of the snippet
-** into the string zOffset[0..nOffset-1]. This string is used as
-** the return of the SQL offsets() function.
-*/
-static void snippetOffsetText(Snippet *p){
- int i;
- int cnt = 0;
- StringBuffer sb;
- char zBuf[200];
- if( p->zOffset ) return;
- initStringBuffer(&sb);
- for(i=0; i<p->nMatch; i++){
- struct snippetMatch *pMatch = &p->aMatch[i];
- if( pMatch->iTerm>=0 ){
- /* If snippetMatch.iTerm is less than 0, then the match was
- ** discarded as part of processing the NEAR operator (see the
- ** trimSnippetOffsetsForNear() function for details). Ignore
- ** it in this case
+
+ /* At this point this is probably a keyword. But for that to be true,
+ ** the next byte must contain either whitespace, an open or close
+ ** parenthesis, a quote character, or EOF.
*/
- zBuf[0] = ' ';
- sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
- pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
- append(&sb, zBuf);
- cnt++;
- }
- }
- p->zOffset = stringBufferData(&sb);
- p->nOffset = stringBufferLength(&sb);
-}
-
-/*
-** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
-** of matching words some of which might be in zDoc. zDoc is column
-** number iCol.
-**
-** iBreak is suggested spot in zDoc where we could begin or end an
-** excerpt. Return a value similar to iBreak but possibly adjusted
-** to be a little left or right so that the break point is better.
-*/
-static int wordBoundary(
- int iBreak, /* The suggested break point */
- const char *zDoc, /* Document text */
- int nDoc, /* Number of bytes in zDoc[] */
- struct snippetMatch *aMatch, /* Matching words */
- int nMatch, /* Number of entries in aMatch[] */
- int iCol /* The column number for zDoc[] */
-){
- int i;
- if( iBreak<=10 ){
- return 0;
- }
- if( iBreak>=nDoc-10 ){
- return nDoc;
- }
- for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
- while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
- if( i<nMatch ){
- if( aMatch[i].iStart<iBreak+10 ){
- return aMatch[i].iStart;
- }
- if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
- return aMatch[i-1].iStart;
- }
- }
- for(i=1; i<=10; i++){
- if( safe_isspace(zDoc[iBreak-i]) ){
- return iBreak - i + 1;
- }
- if( safe_isspace(zDoc[iBreak+i]) ){
- return iBreak + i + 1;
- }
- }
- return iBreak;
-}
-
-
-
-/*
-** Allowed values for Snippet.aMatch[].snStatus
-*/
-#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
-#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
-
-/*
-** Generate the text of a snippet.
-*/
-static void snippetText(
- fulltext_cursor *pCursor, /* The cursor we need the snippet for */
- const char *zStartMark, /* Markup to appear before each match */
- const char *zEndMark, /* Markup to appear after each match */
- const char *zEllipsis /* Ellipsis mark */
-){
- int i, j;
- struct snippetMatch *aMatch;
- int nMatch;
- int nDesired;
- StringBuffer sb;
- int tailCol;
- int tailOffset;
- int iCol;
- int nDoc;
- const char *zDoc;
- int iStart, iEnd;
- int tailEllipsis = 0;
- int iMatch;
-
-
- sqlite3_free(pCursor->snippet.zSnippet);
- pCursor->snippet.zSnippet = 0;
- aMatch = pCursor->snippet.aMatch;
- nMatch = pCursor->snippet.nMatch;
- initStringBuffer(&sb);
-
- for(i=0; i<nMatch; i++){
- aMatch[i].snStatus = SNIPPET_IGNORE;
- }
- nDesired = 0;
- for(i=0; i<FTS3_ROTOR_SZ; i++){
- for(j=0; j<nMatch; j++){
- if( aMatch[j].iTerm==i ){
- aMatch[j].snStatus = SNIPPET_DESIRED;
- nDesired++;
- break;
- }
- }
- }
-
- iMatch = 0;
- tailCol = -1;
- tailOffset = 0;
- for(i=0; i<nMatch && nDesired>0; i++){
- if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
- nDesired--;
- iCol = aMatch[i].iCol;
- zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
- nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
- iStart = aMatch[i].iStart - 40;
- iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
- if( iStart<=10 ){
- iStart = 0;
- }
- if( iCol==tailCol && iStart<=tailOffset+20 ){
- iStart = tailOffset;
- }
- if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){
- trimWhiteSpace(&sb);
- appendWhiteSpace(&sb);
- append(&sb, zEllipsis);
- appendWhiteSpace(&sb);
- }
- iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
- iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
- if( iEnd>=nDoc-10 ){
- iEnd = nDoc;
- tailEllipsis = 0;
- }else{
- tailEllipsis = 1;
- }
- while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
- while( iStart<iEnd ){
- while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
- && aMatch[iMatch].iCol<=iCol ){
- iMatch++;
- }
- if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
- && aMatch[iMatch].iCol==iCol ){
- nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
- iStart = aMatch[iMatch].iStart;
- append(&sb, zStartMark);
- nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
- append(&sb, zEndMark);
- iStart += aMatch[iMatch].nByte;
- for(j=iMatch+1; j<nMatch; j++){
- if( aMatch[j].iTerm==aMatch[iMatch].iTerm
- && aMatch[j].snStatus==SNIPPET_DESIRED ){
- nDesired--;
- aMatch[j].snStatus = SNIPPET_IGNORE;
- }
- }
- }else{
- nappend(&sb, &zDoc[iStart], iEnd - iStart);
- iStart = iEnd;
- }
- }
- tailCol = iCol;
- tailOffset = iEnd;
- }
- trimWhiteSpace(&sb);
- if( tailEllipsis ){
- appendWhiteSpace(&sb);
- append(&sb, zEllipsis);
- }
- pCursor->snippet.zSnippet = stringBufferData(&sb);
- pCursor->snippet.nSnippet = stringBufferLength(&sb);
-}
-
-
-/*
-** Close the cursor. For additional information see the documentation
-** on the xClose method of the virtual table interface.
-*/
-static int fulltextClose(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- FTSTRACE(("FTS3 Close %p\n", c));
- sqlite3_finalize(c->pStmt);
- sqlite3Fts3ExprFree(c->pExpr);
- snippetClear(&c->snippet);
- if( c->result.nData!=0 ){
- dlrDestroy(&c->reader);
- }
- dataBufferDestroy(&c->result);
- sqlite3_free(c);
- return SQLITE_OK;
-}
-
-static int fulltextNext(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- int rc;
-
- FTSTRACE(("FTS3 Next %p\n", pCursor));
- snippetClear(&c->snippet);
- if( c->iCursorType < QUERY_FULLTEXT ){
- /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
- rc = sqlite3_step(c->pStmt);
- switch( rc ){
- case SQLITE_ROW:
- c->eof = 0;
- return SQLITE_OK;
- case SQLITE_DONE:
- c->eof = 1;
- return SQLITE_OK;
- default:
- c->eof = 1;
- return rc;
- }
- } else { /* full-text query */
- rc = sqlite3_reset(c->pStmt);
- if( rc!=SQLITE_OK ) return rc;
-
- if( c->result.nData==0 || dlrAtEnd(&c->reader) ){
- c->eof = 1;
- return SQLITE_OK;
- }
- rc = sqlite3_bind_int64(c->pStmt, 1, dlrDocid(&c->reader));
- dlrStep(&c->reader);
- if( rc!=SQLITE_OK ) return rc;
- /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */
- rc = sqlite3_step(c->pStmt);
- if( rc==SQLITE_ROW ){ /* the case we expect */
- c->eof = 0;
- return SQLITE_OK;
- }
- /* an error occurred; abort */
- return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
- }
-}
-
-
-/* TODO(shess) If we pushed LeafReader to the top of the file, or to
-** another file, term_select() could be pushed above
-** docListOfTerm().
-*/
-static int termSelect(fulltext_vtab *v, int iColumn,
- const char *pTerm, int nTerm, int isPrefix,
- DocListType iType, DataBuffer *out);
-
-/*
-** Return a DocList corresponding to the phrase *pPhrase.
-**
-** The resulting DL_DOCIDS doclist is stored in pResult, which is
-** overwritten.
-*/
-static int docListOfPhrase(
- fulltext_vtab *pTab, /* The full text index */
- Fts3Phrase *pPhrase, /* Phrase to return a doclist corresponding to */
- DocListType eListType, /* Either DL_DOCIDS or DL_POSITIONS */
- DataBuffer *pResult /* Write the result here */
-){
- int ii;
- int rc = SQLITE_OK;
- int iCol = pPhrase->iColumn;
- DocListType eType = eListType;
- assert( eType==DL_POSITIONS || eType==DL_DOCIDS );
- if( pPhrase->nToken>1 ){
- eType = DL_POSITIONS;
- }
-
- /* This code should never be called with buffered updates. */
- assert( pTab->nPendingData<0 );
-
- for(ii=0; rc==SQLITE_OK && ii<pPhrase->nToken; ii++){
- DataBuffer tmp;
- struct PhraseToken *p = &pPhrase->aToken[ii];
- rc = termSelect(pTab, iCol, p->z, p->n, p->isPrefix, eType, &tmp);
- if( rc==SQLITE_OK ){
- if( ii==0 ){
- *pResult = tmp;
- }else{
- DataBuffer res = *pResult;
- dataBufferInit(pResult, 0);
- if( ii==(pPhrase->nToken-1) ){
- eType = eListType;
- }
- docListPhraseMerge(
- res.pData, res.nData, tmp.pData, tmp.nData, 0, 0, eType, pResult
- );
- dataBufferDestroy(&res);
- dataBufferDestroy(&tmp);
- }
- }
- }
-
- return rc;
-}
-
-/*
-** Evaluate the full-text expression pExpr against fts3 table pTab. Write
-** the results into pRes.
-*/
-static int evalFts3Expr(
- fulltext_vtab *pTab, /* Fts3 Virtual table object */
- Fts3Expr *pExpr, /* Parsed fts3 expression */
- DataBuffer *pRes /* OUT: Write results of the expression here */
-){
- int rc = SQLITE_OK;
-
- /* Initialize the output buffer. If this is an empty query (pExpr==0),
- ** this is all that needs to be done. Empty queries produce empty
- ** result sets.
- */
- dataBufferInit(pRes, 0);
-
- if( pExpr ){
- if( pExpr->eType==FTSQUERY_PHRASE ){
- DocListType eType = DL_DOCIDS;
- if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
- eType = DL_POSITIONS;
- }
- rc = docListOfPhrase(pTab, pExpr->pPhrase, eType, pRes);
- }else{
- DataBuffer lhs;
- DataBuffer rhs;
-
- dataBufferInit(&rhs, 0);
- if( SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pLeft, &lhs))
- && SQLITE_OK==(rc = evalFts3Expr(pTab, pExpr->pRight, &rhs))
+ cNext = zInput[nKey];
+ if( fts3isspace(cNext)
+ || cNext=='"' || cNext=='(' || cNext==')' || cNext==0
){
- switch( pExpr->eType ){
- case FTSQUERY_NEAR: {
- int nToken;
- Fts3Expr *pLeft;
- DocListType eType = DL_DOCIDS;
- if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){
- eType = DL_POSITIONS;
- }
- pLeft = pExpr->pLeft;
- while( pLeft->eType==FTSQUERY_NEAR ){
- pLeft=pLeft->pRight;
- }
- assert( pExpr->pRight->eType==FTSQUERY_PHRASE );
- assert( pLeft->eType==FTSQUERY_PHRASE );
- nToken = pLeft->pPhrase->nToken + pExpr->pRight->pPhrase->nToken;
- docListPhraseMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,
- pExpr->nNear+1, nToken, eType, pRes
- );
- break;
- }
- case FTSQUERY_NOT: {
- docListExceptMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData,pRes);
- break;
- }
- case FTSQUERY_AND: {
- docListAndMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes);
- break;
- }
- case FTSQUERY_OR: {
- docListOrMerge(lhs.pData, lhs.nData, rhs.pData, rhs.nData, pRes);
- break;
- }
- }
- }
- dataBufferDestroy(&lhs);
- dataBufferDestroy(&rhs);
- }
- }
-
- return rc;
-}
-
-/* TODO(shess) Refactor the code to remove this forward decl. */
-static int flushPendingTerms(fulltext_vtab *v);
-
-/* Perform a full-text query using the search expression in
-** zInput[0..nInput-1]. Return a list of matching documents
-** in pResult.
-**
-** Queries must match column iColumn. Or if iColumn>=nColumn
-** they are allowed to match against any column.
-*/
-static int fulltextQuery(
- fulltext_vtab *v, /* The full text index */
- int iColumn, /* Match against this column by default */
- const char *zInput, /* The query string */
- int nInput, /* Number of bytes in zInput[] */
- DataBuffer *pResult, /* Write the result doclist here */
- Fts3Expr **ppExpr /* Put parsed query string here */
-){
- int rc;
-
- /* TODO(shess) Instead of flushing pendingTerms, we could query for
- ** the relevant term and merge the doclist into what we receive from
- ** the database. Wait and see if this is a common issue, first.
- **
- ** A good reason not to flush is to not generate update-related
- ** error codes from here.
- */
-
- /* Flush any buffered updates before executing the query. */
- rc = flushPendingTerms(v);
- if( rc!=SQLITE_OK ){
- return rc;
- }
-
- /* Parse the query passed to the MATCH operator. */
- rc = sqlite3Fts3ExprParse(v->pTokenizer,
- v->azColumn, v->nColumn, iColumn, zInput, nInput, ppExpr
- );
- if( rc!=SQLITE_OK ){
- assert( 0==(*ppExpr) );
- return rc;
- }
-
- return evalFts3Expr(v, *ppExpr, pResult);
-}
-
-/*
-** This is the xFilter interface for the virtual table. See
-** the virtual table xFilter method documentation for additional
-** information.
-**
-** If idxNum==QUERY_GENERIC then do a full table scan against
-** the %_content table.
-**
-** If idxNum==QUERY_DOCID then do a docid lookup for a single entry
-** in the %_content table.
-**
-** If idxNum>=QUERY_FULLTEXT then use the full text index. The
-** column on the left-hand side of the MATCH operator is column
-** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand
-** side of the MATCH operator.
-*/
-/* TODO(shess) Upgrade the cursor initialization and destruction to
-** account for fulltextFilter() being called multiple times on the
-** same cursor. The current solution is very fragile. Apply fix to
-** fts3 as appropriate.
-*/
-static int fulltextFilter(
- sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
- int idxNum, const char *idxStr, /* Which indexing scheme to use */
- int argc, sqlite3_value **argv /* Arguments for the indexing scheme */
-){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- fulltext_vtab *v = cursor_vtab(c);
- int rc;
-
- FTSTRACE(("FTS3 Filter %p\n",pCursor));
-
- /* If the cursor has a statement that was not prepared according to
- ** idxNum, clear it. I believe all calls to fulltextFilter with a
- ** given cursor will have the same idxNum , but in this case it's
- ** easy to be safe.
- */
- if( c->pStmt && c->iCursorType!=idxNum ){
- sqlite3_finalize(c->pStmt);
- c->pStmt = NULL;
- }
-
- /* Get a fresh statement appropriate to idxNum. */
- /* TODO(shess): Add a prepared-statement cache in the vt structure.
- ** The cache must handle multiple open cursors. Easier to cache the
- ** statement variants at the vt to reduce malloc/realloc/free here.
- ** Or we could have a StringBuffer variant which allowed stack
- ** construction for small values.
- */
- if( !c->pStmt ){
- StringBuffer sb;
- initStringBuffer(&sb);
- append(&sb, "SELECT docid, ");
- appendList(&sb, v->nColumn, v->azContentColumn);
- append(&sb, " FROM %_content");
- if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?");
- rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt,
- stringBufferData(&sb));
- stringBufferDestroy(&sb);
- if( rc!=SQLITE_OK ) return rc;
- c->iCursorType = idxNum;
- }else{
- sqlite3_reset(c->pStmt);
- assert( c->iCursorType==idxNum );
- }
-
- switch( idxNum ){
- case QUERY_GENERIC:
- break;
-
- case QUERY_DOCID:
- rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0]));
- if( rc!=SQLITE_OK ) return rc;
- break;
-
- default: /* full-text search */
- {
- int iCol = idxNum-QUERY_FULLTEXT;
- const char *zQuery = (const char *)sqlite3_value_text(argv[0]);
- assert( idxNum<=QUERY_FULLTEXT+v->nColumn);
- assert( argc==1 );
- if( c->result.nData!=0 ){
- /* This case happens if the same cursor is used repeatedly. */
- dlrDestroy(&c->reader);
- dataBufferReset(&c->result);
- }else{
- dataBufferInit(&c->result, 0);
- }
- rc = fulltextQuery(v, iCol, zQuery, -1, &c->result, &c->pExpr);
- if( rc!=SQLITE_OK ) return rc;
- if( c->result.nData!=0 ){
- dlrInit(&c->reader, DL_DOCIDS, c->result.pData, c->result.nData);
- }
- break;
- }
- }
-
- return fulltextNext(pCursor);
-}
-
-/* This is the xEof method of the virtual table. The SQLite core
-** calls this routine to find out if it has reached the end of
-** a query's results set.
-*/
-static int fulltextEof(sqlite3_vtab_cursor *pCursor){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- return c->eof;
-}
-
-/* This is the xColumn method of the virtual table. The SQLite
-** core calls this method during a query when it needs the value
-** of a column from the virtual table. This method needs to use
-** one of the sqlite3_result_*() routines to store the requested
-** value back in the pContext.
-*/
-static int fulltextColumn(sqlite3_vtab_cursor *pCursor,
- sqlite3_context *pContext, int idxCol){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
- fulltext_vtab *v = cursor_vtab(c);
-
- if( idxCol<v->nColumn ){
- sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1);
- sqlite3_result_value(pContext, pVal);
- }else if( idxCol==v->nColumn ){
- /* The extra column whose name is the same as the table.
- ** Return a blob which is a pointer to the cursor
- */
- sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT);
- }else if( idxCol==v->nColumn+1 ){
- /* The docid column, which is an alias for rowid. */
- sqlite3_value *pVal = sqlite3_column_value(c->pStmt, 0);
- sqlite3_result_value(pContext, pVal);
- }
- return SQLITE_OK;
-}
-
-/* This is the xRowid method. The SQLite core calls this routine to
-** retrieve the rowid for the current row of the result set. fts3
-** exposes %_content.docid as the rowid for the virtual table. The
-** rowid should be written to *pRowid.
-*/
-static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
- fulltext_cursor *c = (fulltext_cursor *) pCursor;
-
- *pRowid = sqlite3_column_int64(c->pStmt, 0);
- return SQLITE_OK;
-}
-
-/* Add all terms in [zText] to pendingTerms table. If [iColumn] > 0,
-** we also store positions and offsets in the hash table using that
-** column number.
-*/
-static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
- const char *zText, int iColumn){
- sqlite3_tokenizer *pTokenizer = v->pTokenizer;
- sqlite3_tokenizer_cursor *pCursor;
- const char *pToken;
- int nTokenBytes;
- int iStartOffset, iEndOffset, iPosition;
- int rc;
-
- rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor);
- if( rc!=SQLITE_OK ) return rc;
-
- pCursor->pTokenizer = pTokenizer;
- while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
- &pToken, &nTokenBytes,
- &iStartOffset, &iEndOffset,
- &iPosition)) ){
- DLCollector *p;
- int nData; /* Size of doclist before our update. */
-
- /* Positions can't be negative; we use -1 as a terminator
- * internally. Token can't be NULL or empty. */
- if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){
- rc = SQLITE_ERROR;
- break;
- }
-
- p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes);
- if( p==NULL ){
- nData = 0;
- p = dlcNew(iDocid, DL_DEFAULT);
- fts3HashInsert(&v->pendingTerms, pToken, nTokenBytes, p);
-
- /* Overhead for our hash table entry, the key, and the value. */
- v->nPendingData += sizeof(struct fts3HashElem)+sizeof(*p)+nTokenBytes;
- }else{
- nData = p->b.nData;
- if( p->dlw.iPrevDocid!=iDocid ) dlcNext(p, iDocid);
- }
- if( iColumn>=0 ){
- dlcAddPos(p, iColumn, iPosition, iStartOffset, iEndOffset);
- }
-
- /* Accumulate data added by dlcNew or dlcNext, and dlcAddPos. */
- v->nPendingData += p->b.nData-nData;
- }
-
- /* TODO(shess) Check return? Should this be able to cause errors at
- ** this point? Actually, same question about sqlite3_finalize(),
- ** though one could argue that failure there means that the data is
- ** not durable. *ponder*
- */
- pTokenizer->pModule->xClose(pCursor);
- if( SQLITE_DONE == rc ) return SQLITE_OK;
- return rc;
-}
-
-/* Add doclists for all terms in [pValues] to pendingTerms table. */
-static int insertTerms(fulltext_vtab *v, sqlite_int64 iDocid,
- sqlite3_value **pValues){
- int i;
- for(i = 0; i < v->nColumn ; ++i){
- char *zText = (char*)sqlite3_value_text(pValues[i]);
- int rc = buildTerms(v, iDocid, zText, i);
- if( rc!=SQLITE_OK ) return rc;
- }
- return SQLITE_OK;
-}
-
-/* Add empty doclists for all terms in the given row's content to
-** pendingTerms.
-*/
-static int deleteTerms(fulltext_vtab *v, sqlite_int64 iDocid){
- const char **pValues;
- int i, rc;
-
- /* TODO(shess) Should we allow such tables at all? */
- if( DL_DEFAULT==DL_DOCIDS ) return SQLITE_ERROR;
-
- rc = content_select(v, iDocid, &pValues);
- if( rc!=SQLITE_OK ) return rc;
-
- for(i = 0 ; i < v->nColumn; ++i) {
- rc = buildTerms(v, iDocid, pValues[i], -1);
- if( rc!=SQLITE_OK ) break;
- }
-
- freeStringArray(v->nColumn, pValues);
- return SQLITE_OK;
-}
-
-/* TODO(shess) Refactor the code to remove this forward decl. */
-static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid);
-
-/* Insert a row into the %_content table; set *piDocid to be the ID of the
-** new row. Add doclists for terms to pendingTerms.
-*/
-static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestDocid,
- sqlite3_value **pValues, sqlite_int64 *piDocid){
- int rc;
-
- rc = content_insert(v, pRequestDocid, pValues); /* execute an SQL INSERT */
- if( rc!=SQLITE_OK ) return rc;
-
- /* docid column is an alias for rowid. */
- *piDocid = sqlite3_last_insert_rowid(v->db);
- rc = initPendingTerms(v, *piDocid);
- if( rc!=SQLITE_OK ) return rc;
-
- return insertTerms(v, *piDocid, pValues);
-}
-
-/* Delete a row from the %_content table; add empty doclists for terms
-** to pendingTerms.
-*/
-static int index_delete(fulltext_vtab *v, sqlite_int64 iRow){
- int rc = initPendingTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = deleteTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
-
- return content_delete(v, iRow); /* execute an SQL DELETE */
-}
-
-/* Update a row in the %_content table; add delete doclists to
-** pendingTerms for old terms not in the new data, add insert doclists
-** to pendingTerms for terms in the new data.
-*/
-static int index_update(fulltext_vtab *v, sqlite_int64 iRow,
- sqlite3_value **pValues){
- int rc = initPendingTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Generate an empty doclist for each term that previously appeared in this
- * row. */
- rc = deleteTerms(v, iRow);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */
- if( rc!=SQLITE_OK ) return rc;
-
- /* Now add positions for terms which appear in the updated row. */
- return insertTerms(v, iRow, pValues);
-}
-
-/*******************************************************************/
-/* InteriorWriter is used to collect terms and block references into
-** interior nodes in %_segments. See commentary at top of file for
-** format.
-*/
-
-/* How large interior nodes can grow. */
-#define INTERIOR_MAX 2048
-
-/* Minimum number of terms per interior node (except the root). This
-** prevents large terms from making the tree too skinny - must be >0
-** so that the tree always makes progress. Note that the min tree
-** fanout will be INTERIOR_MIN_TERMS+1.
-*/
-#define INTERIOR_MIN_TERMS 7
-#if INTERIOR_MIN_TERMS<1
-# error INTERIOR_MIN_TERMS must be greater than 0.
-#endif
-
-/* ROOT_MAX controls how much data is stored inline in the segment
-** directory.
-*/
-/* TODO(shess) Push ROOT_MAX down to whoever is writing things. It's
-** only here so that interiorWriterRootInfo() and leafWriterRootInfo()
-** can both see it, but if the caller passed it in, we wouldn't even
-** need a define.
-*/
-#define ROOT_MAX 1024
-#if ROOT_MAX<VARINT_MAX*2
-# error ROOT_MAX must have enough space for a header.
-#endif
-
-/* InteriorBlock stores a linked-list of interior blocks while a lower
-** layer is being constructed.
-*/
-typedef struct InteriorBlock {
- DataBuffer term; /* Leftmost term in block's subtree. */
- DataBuffer data; /* Accumulated data for the block. */
- struct InteriorBlock *next;
-} InteriorBlock;
-
-static InteriorBlock *interiorBlockNew(int iHeight, sqlite_int64 iChildBlock,
- const char *pTerm, int nTerm){
- InteriorBlock *block = sqlite3_malloc(sizeof(InteriorBlock));
- char c[VARINT_MAX+VARINT_MAX];
- int n;
-
- if( block ){
- memset(block, 0, sizeof(*block));
- dataBufferInit(&block->term, 0);
- dataBufferReplace(&block->term, pTerm, nTerm);
-
- n = fts3PutVarint(c, iHeight);
- n += fts3PutVarint(c+n, iChildBlock);
- dataBufferInit(&block->data, INTERIOR_MAX);
- dataBufferReplace(&block->data, c, n);
- }
- return block;
-}
-
-#ifndef NDEBUG
-/* Verify that the data is readable as an interior node. */
-static void interiorBlockValidate(InteriorBlock *pBlock){
- const char *pData = pBlock->data.pData;
- int nData = pBlock->data.nData;
- int n, iDummy;
- sqlite_int64 iBlockid;
-
- assert( nData>0 );
- assert( pData!=0 );
- assert( pData+nData>pData );
-
- /* Must lead with height of node as a varint(n), n>0 */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n<nData );
- pData += n;
- nData -= n;
-
- /* Must contain iBlockid. */
- n = fts3GetVarint(pData, &iBlockid);
- assert( n>0 );
- assert( n<=nData );
- pData += n;
- nData -= n;
-
- /* Zero or more terms of positive length */
- if( nData!=0 ){
- /* First term is not delta-encoded. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0);
- assert( n+iDummy<=nData );
- pData += n+iDummy;
- nData -= n+iDummy;
-
- /* Following terms delta-encoded. */
- while( nData!=0 ){
- /* Length of shared prefix. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>=0 );
- assert( n<nData );
- pData += n;
- nData -= n;
-
- /* Length and data of distinct suffix. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0);
- assert( n+iDummy<=nData );
- pData += n+iDummy;
- nData -= n+iDummy;
- }
- }
-}
-#define ASSERT_VALID_INTERIOR_BLOCK(x) interiorBlockValidate(x)
-#else
-#define ASSERT_VALID_INTERIOR_BLOCK(x) assert( 1 )
-#endif
-
-typedef struct InteriorWriter {
- int iHeight; /* from 0 at leaves. */
- InteriorBlock *first, *last;
- struct InteriorWriter *parentWriter;
-
- DataBuffer term; /* Last term written to block "last". */
- sqlite_int64 iOpeningChildBlock; /* First child block in block "last". */
-#ifndef NDEBUG
- sqlite_int64 iLastChildBlock; /* for consistency checks. */
-#endif
-} InteriorWriter;
-
-/* Initialize an interior node where pTerm[nTerm] marks the leftmost
-** term in the tree. iChildBlock is the leftmost child block at the
-** next level down the tree.
-*/
-static void interiorWriterInit(int iHeight, const char *pTerm, int nTerm,
- sqlite_int64 iChildBlock,
- InteriorWriter *pWriter){
- InteriorBlock *block;
- assert( iHeight>0 );
- CLEAR(pWriter);
-
- pWriter->iHeight = iHeight;
- pWriter->iOpeningChildBlock = iChildBlock;
-#ifndef NDEBUG
- pWriter->iLastChildBlock = iChildBlock;
-#endif
- block = interiorBlockNew(iHeight, iChildBlock, pTerm, nTerm);
- pWriter->last = pWriter->first = block;
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
- dataBufferInit(&pWriter->term, 0);
-}
-
-/* Append the child node rooted at iChildBlock to the interior node,
-** with pTerm[nTerm] as the leftmost term in iChildBlock's subtree.
-*/
-static void interiorWriterAppend(InteriorWriter *pWriter,
- const char *pTerm, int nTerm,
- sqlite_int64 iChildBlock){
- char c[VARINT_MAX+VARINT_MAX];
- int n, nPrefix = 0;
-
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
-
- /* The first term written into an interior node is actually
- ** associated with the second child added (the first child was added
- ** in interiorWriterInit, or in the if clause at the bottom of this
- ** function). That term gets encoded straight up, with nPrefix left
- ** at 0.
- */
- if( pWriter->term.nData==0 ){
- n = fts3PutVarint(c, nTerm);
- }else{
- while( nPrefix<pWriter->term.nData &&
- pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
- nPrefix++;
- }
-
- n = fts3PutVarint(c, nPrefix);
- n += fts3PutVarint(c+n, nTerm-nPrefix);
- }
-
-#ifndef NDEBUG
- pWriter->iLastChildBlock++;
-#endif
- assert( pWriter->iLastChildBlock==iChildBlock );
-
- /* Overflow to a new block if the new term makes the current block
- ** too big, and the current block already has enough terms.
- */
- if( pWriter->last->data.nData+n+nTerm-nPrefix>INTERIOR_MAX &&
- iChildBlock-pWriter->iOpeningChildBlock>INTERIOR_MIN_TERMS ){
- pWriter->last->next = interiorBlockNew(pWriter->iHeight, iChildBlock,
- pTerm, nTerm);
- pWriter->last = pWriter->last->next;
- pWriter->iOpeningChildBlock = iChildBlock;
- dataBufferReset(&pWriter->term);
- }else{
- dataBufferAppend2(&pWriter->last->data, c, n,
- pTerm+nPrefix, nTerm-nPrefix);
- dataBufferReplace(&pWriter->term, pTerm, nTerm);
- }
- ASSERT_VALID_INTERIOR_BLOCK(pWriter->last);
-}
-
-/* Free the space used by pWriter, including the linked-list of
-** InteriorBlocks, and parentWriter, if present.
-*/
-static int interiorWriterDestroy(InteriorWriter *pWriter){
- InteriorBlock *block = pWriter->first;
-
- while( block!=NULL ){
- InteriorBlock *b = block;
- block = block->next;
- dataBufferDestroy(&b->term);
- dataBufferDestroy(&b->data);
- sqlite3_free(b);
- }
- if( pWriter->parentWriter!=NULL ){
- interiorWriterDestroy(pWriter->parentWriter);
- sqlite3_free(pWriter->parentWriter);
- }
- dataBufferDestroy(&pWriter->term);
- SCRAMBLE(pWriter);
- return SQLITE_OK;
-}
-
-/* If pWriter can fit entirely in ROOT_MAX, return it as the root info
-** directly, leaving *piEndBlockid unchanged. Otherwise, flush
-** pWriter to %_segments, building a new layer of interior nodes, and
-** recursively ask for their root into.
-*/
-static int interiorWriterRootInfo(fulltext_vtab *v, InteriorWriter *pWriter,
- char **ppRootInfo, int *pnRootInfo,
- sqlite_int64 *piEndBlockid){
- InteriorBlock *block = pWriter->first;
- sqlite_int64 iBlockid = 0;
- int rc;
-
- /* If we can fit the segment inline */
- if( block==pWriter->last && block->data.nData<ROOT_MAX ){
- *ppRootInfo = block->data.pData;
- *pnRootInfo = block->data.nData;
- return SQLITE_OK;
- }
-
- /* Flush the first block to %_segments, and create a new level of
- ** interior node.
- */
- ASSERT_VALID_INTERIOR_BLOCK(block);
- rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- *piEndBlockid = iBlockid;
-
- pWriter->parentWriter = sqlite3_malloc(sizeof(*pWriter->parentWriter));
- interiorWriterInit(pWriter->iHeight+1,
- block->term.pData, block->term.nData,
- iBlockid, pWriter->parentWriter);
-
- /* Flush additional blocks and append to the higher interior
- ** node.
- */
- for(block=block->next; block!=NULL; block=block->next){
- ASSERT_VALID_INTERIOR_BLOCK(block);
- rc = block_insert(v, block->data.pData, block->data.nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- *piEndBlockid = iBlockid;
-
- interiorWriterAppend(pWriter->parentWriter,
- block->term.pData, block->term.nData, iBlockid);
- }
-
- /* Parent node gets the chance to be the root. */
- return interiorWriterRootInfo(v, pWriter->parentWriter,
- ppRootInfo, pnRootInfo, piEndBlockid);
-}
-
-/****************************************************************/
-/* InteriorReader is used to read off the data from an interior node
-** (see comment at top of file for the format).
-*/
-typedef struct InteriorReader {
- const char *pData;
- int nData;
-
- DataBuffer term; /* previous term, for decoding term delta. */
-
- sqlite_int64 iBlockid;
-} InteriorReader;
-
-static void interiorReaderDestroy(InteriorReader *pReader){
- dataBufferDestroy(&pReader->term);
- SCRAMBLE(pReader);
-}
-
-/* TODO(shess) The assertions are great, but what if we're in NDEBUG
-** and the blob is empty or otherwise contains suspect data?
-*/
-static void interiorReaderInit(const char *pData, int nData,
- InteriorReader *pReader){
- int n, nTerm;
-
- /* Require at least the leading flag byte */
- assert( nData>0 );
- assert( pData[0]!='\0' );
-
- CLEAR(pReader);
-
- /* Decode the base blockid, and set the cursor to the first term. */
- n = fts3GetVarint(pData+1, &pReader->iBlockid);
- assert( 1+n<=nData );
- pReader->pData = pData+1+n;
- pReader->nData = nData-(1+n);
-
- /* A single-child interior node (such as when a leaf node was too
- ** large for the segment directory) won't have any terms.
- ** Otherwise, decode the first term.
- */
- if( pReader->nData==0 ){
- dataBufferInit(&pReader->term, 0);
- }else{
- n = fts3GetVarint32(pReader->pData, &nTerm);
- dataBufferInit(&pReader->term, nTerm);
- dataBufferReplace(&pReader->term, pReader->pData+n, nTerm);
- assert( n+nTerm<=pReader->nData );
- pReader->pData += n+nTerm;
- pReader->nData -= n+nTerm;
- }
-}
-
-static int interiorReaderAtEnd(InteriorReader *pReader){
- return pReader->term.nData==0;
-}
-
-static sqlite_int64 interiorReaderCurrentBlockid(InteriorReader *pReader){
- return pReader->iBlockid;
-}
-
-static int interiorReaderTermBytes(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
- return pReader->term.nData;
-}
-static const char *interiorReaderTerm(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
- return pReader->term.pData;
-}
-
-/* Step forward to the next term in the node. */
-static void interiorReaderStep(InteriorReader *pReader){
- assert( !interiorReaderAtEnd(pReader) );
-
- /* If the last term has been read, signal eof, else construct the
- ** next term.
- */
- if( pReader->nData==0 ){
- dataBufferReset(&pReader->term);
- }else{
- int n, nPrefix, nSuffix;
-
- n = fts3GetVarint32(pReader->pData, &nPrefix);
- n += fts3GetVarint32(pReader->pData+n, &nSuffix);
-
- /* Truncate the current term and append suffix data. */
- pReader->term.nData = nPrefix;
- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
-
- assert( n+nSuffix<=pReader->nData );
- pReader->pData += n+nSuffix;
- pReader->nData -= n+nSuffix;
- }
- pReader->iBlockid++;
-}
-
-/* Compare the current term to pTerm[nTerm], returning strcmp-style
-** results. If isPrefix, equality means equal through nTerm bytes.
-*/
-static int interiorReaderTermCmp(InteriorReader *pReader,
- const char *pTerm, int nTerm, int isPrefix){
- const char *pReaderTerm = interiorReaderTerm(pReader);
- int nReaderTerm = interiorReaderTermBytes(pReader);
- int c, n = nReaderTerm<nTerm ? nReaderTerm : nTerm;
-
- if( n==0 ){
- if( nReaderTerm>0 ) return -1;
- if( nTerm>0 ) return 1;
- return 0;
- }
-
- c = memcmp(pReaderTerm, pTerm, n);
- if( c!=0 ) return c;
- if( isPrefix && n==nTerm ) return 0;
- return nReaderTerm - nTerm;
-}
-
-/****************************************************************/
-/* LeafWriter is used to collect terms and associated doclist data
-** into leaf blocks in %_segments (see top of file for format info).
-** Expected usage is:
-**
-** LeafWriter writer;
-** leafWriterInit(0, 0, &writer);
-** while( sorted_terms_left_to_process ){
-** // data is doclist data for that term.
-** rc = leafWriterStep(v, &writer, pTerm, nTerm, pData, nData);
-** if( rc!=SQLITE_OK ) goto err;
-** }
-** rc = leafWriterFinalize(v, &writer);
-**err:
-** leafWriterDestroy(&writer);
-** return rc;
-**
-** leafWriterStep() may write a collected leaf out to %_segments.
-** leafWriterFinalize() finishes writing any buffered data and stores
-** a root node in %_segdir. leafWriterDestroy() frees all buffers and
-** InteriorWriters allocated as part of writing this segment.
-**
-** TODO(shess) Document leafWriterStepMerge().
-*/
-
-/* Put terms with data this big in their own block. */
-#define STANDALONE_MIN 1024
-
-/* Keep leaf blocks below this size. */
-#define LEAF_MAX 2048
-
-typedef struct LeafWriter {
- int iLevel;
- int idx;
- sqlite_int64 iStartBlockid; /* needed to create the root info */
- sqlite_int64 iEndBlockid; /* when we're done writing. */
-
- DataBuffer term; /* previous encoded term */
- DataBuffer data; /* encoding buffer */
-
- /* bytes of first term in the current node which distinguishes that
- ** term from the last term of the previous node.
- */
- int nTermDistinct;
-
- InteriorWriter parentWriter; /* if we overflow */
- int has_parent;
-} LeafWriter;
-
-static void leafWriterInit(int iLevel, int idx, LeafWriter *pWriter){
- CLEAR(pWriter);
- pWriter->iLevel = iLevel;
- pWriter->idx = idx;
-
- dataBufferInit(&pWriter->term, 32);
-
- /* Start out with a reasonably sized block, though it can grow. */
- dataBufferInit(&pWriter->data, LEAF_MAX);
-}
-
-#ifndef NDEBUG
-/* Verify that the data is readable as a leaf node. */
-static void leafNodeValidate(const char *pData, int nData){
- int n, iDummy;
-
- if( nData==0 ) return;
- assert( nData>0 );
- assert( pData!=0 );
- assert( pData+nData>pData );
-
- /* Must lead with a varint(0) */
- n = fts3GetVarint32(pData, &iDummy);
- assert( iDummy==0 );
- assert( n>0 );
- assert( n<nData );
- pData += n;
- nData -= n;
-
- /* Leading term length and data must fit in buffer. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<nData );
- pData += n+iDummy;
- nData -= n+iDummy;
-
- /* Leading term's doclist length and data must fit. */
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<=nData );
- ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
- pData += n+iDummy;
- nData -= n+iDummy;
-
- /* Verify that trailing terms and doclists also are readable. */
- while( nData!=0 ){
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>=0 );
- assert( n<nData );
- pData += n;
- nData -= n;
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<nData );
- pData += n+iDummy;
- nData -= n+iDummy;
-
- n = fts3GetVarint32(pData, &iDummy);
- assert( n>0 );
- assert( iDummy>0 );
- assert( n+iDummy>0 );
- assert( n+iDummy<=nData );
- ASSERT_VALID_DOCLIST(DL_DEFAULT, pData+n, iDummy, NULL);
- pData += n+iDummy;
- nData -= n+iDummy;
- }
-}
-#define ASSERT_VALID_LEAF_NODE(p, n) leafNodeValidate(p, n)
-#else
-#define ASSERT_VALID_LEAF_NODE(p, n) assert( 1 )
-#endif
-
-/* Flush the current leaf node to %_segments, and adding the resulting
-** blockid and the starting term to the interior node which will
-** contain it.
-*/
-static int leafWriterInternalFlush(fulltext_vtab *v, LeafWriter *pWriter,
- int iData, int nData){
- sqlite_int64 iBlockid = 0;
- const char *pStartingTerm;
- int nStartingTerm, rc, n;
-
- /* Must have the leading varint(0) flag, plus at least some
- ** valid-looking data.
- */
- assert( nData>2 );
- assert( iData>=0 );
- assert( iData+nData<=pWriter->data.nData );
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData+iData, nData);
-
- rc = block_insert(v, pWriter->data.pData+iData, nData, &iBlockid);
- if( rc!=SQLITE_OK ) return rc;
- assert( iBlockid!=0 );
-
- /* Reconstruct the first term in the leaf for purposes of building
- ** the interior node.
- */
- n = fts3GetVarint32(pWriter->data.pData+iData+1, &nStartingTerm);
- pStartingTerm = pWriter->data.pData+iData+1+n;
- assert( pWriter->data.nData>iData+1+n+nStartingTerm );
- assert( pWriter->nTermDistinct>0 );
- assert( pWriter->nTermDistinct<=nStartingTerm );
- nStartingTerm = pWriter->nTermDistinct;
-
- if( pWriter->has_parent ){
- interiorWriterAppend(&pWriter->parentWriter,
- pStartingTerm, nStartingTerm, iBlockid);
- }else{
- interiorWriterInit(1, pStartingTerm, nStartingTerm, iBlockid,
- &pWriter->parentWriter);
- pWriter->has_parent = 1;
- }
-
- /* Track the span of this segment's leaf nodes. */
- if( pWriter->iEndBlockid==0 ){
- pWriter->iEndBlockid = pWriter->iStartBlockid = iBlockid;
- }else{
- pWriter->iEndBlockid++;
- assert( iBlockid==pWriter->iEndBlockid );
- }
-
- return SQLITE_OK;
-}
-static int leafWriterFlush(fulltext_vtab *v, LeafWriter *pWriter){
- int rc = leafWriterInternalFlush(v, pWriter, 0, pWriter->data.nData);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Re-initialize the output buffer. */
- dataBufferReset(&pWriter->data);
-
- return SQLITE_OK;
-}
-
-/* Fetch the root info for the segment. If the entire leaf fits
-** within ROOT_MAX, then it will be returned directly, otherwise it
-** will be flushed and the root info will be returned from the
-** interior node. *piEndBlockid is set to the blockid of the last
-** interior or leaf node written to disk (0 if none are written at
-** all).
-*/
-static int leafWriterRootInfo(fulltext_vtab *v, LeafWriter *pWriter,
- char **ppRootInfo, int *pnRootInfo,
- sqlite_int64 *piEndBlockid){
- /* we can fit the segment entirely inline */
- if( !pWriter->has_parent && pWriter->data.nData<ROOT_MAX ){
- *ppRootInfo = pWriter->data.pData;
- *pnRootInfo = pWriter->data.nData;
- *piEndBlockid = 0;
- return SQLITE_OK;
- }
-
- /* Flush remaining leaf data. */
- if( pWriter->data.nData>0 ){
- int rc = leafWriterFlush(v, pWriter);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- /* We must have flushed a leaf at some point. */
- assert( pWriter->has_parent );
-
- /* Tenatively set the end leaf blockid as the end blockid. If the
- ** interior node can be returned inline, this will be the final
- ** blockid, otherwise it will be overwritten by
- ** interiorWriterRootInfo().
- */
- *piEndBlockid = pWriter->iEndBlockid;
-
- return interiorWriterRootInfo(v, &pWriter->parentWriter,
- ppRootInfo, pnRootInfo, piEndBlockid);
-}
-
-/* Collect the rootInfo data and store it into the segment directory.
-** This has the effect of flushing the segment's leaf data to
-** %_segments, and also flushing any interior nodes to %_segments.
-*/
-static int leafWriterFinalize(fulltext_vtab *v, LeafWriter *pWriter){
- sqlite_int64 iEndBlockid;
- char *pRootInfo;
- int rc, nRootInfo;
-
- rc = leafWriterRootInfo(v, pWriter, &pRootInfo, &nRootInfo, &iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Don't bother storing an entirely empty segment. */
- if( iEndBlockid==0 && nRootInfo==0 ) return SQLITE_OK;
-
- return segdir_set(v, pWriter->iLevel, pWriter->idx,
- pWriter->iStartBlockid, pWriter->iEndBlockid,
- iEndBlockid, pRootInfo, nRootInfo);
-}
-
-static void leafWriterDestroy(LeafWriter *pWriter){
- if( pWriter->has_parent ) interiorWriterDestroy(&pWriter->parentWriter);
- dataBufferDestroy(&pWriter->term);
- dataBufferDestroy(&pWriter->data);
-}
-
-/* Encode a term into the leafWriter, delta-encoding as appropriate.
-** Returns the length of the new term which distinguishes it from the
-** previous term, which can be used to set nTermDistinct when a node
-** boundary is crossed.
-*/
-static int leafWriterEncodeTerm(LeafWriter *pWriter,
- const char *pTerm, int nTerm){
- char c[VARINT_MAX+VARINT_MAX];
- int n, nPrefix = 0;
-
- assert( nTerm>0 );
- while( nPrefix<pWriter->term.nData &&
- pTerm[nPrefix]==pWriter->term.pData[nPrefix] ){
- nPrefix++;
- /* Failing this implies that the terms weren't in order. */
- assert( nPrefix<nTerm );
- }
-
- if( pWriter->data.nData==0 ){
- /* Encode the node header and leading term as:
- ** varint(0)
- ** varint(nTerm)
- ** char pTerm[nTerm]
- */
- n = fts3PutVarint(c, '\0');
- n += fts3PutVarint(c+n, nTerm);
- dataBufferAppend2(&pWriter->data, c, n, pTerm, nTerm);
- }else{
- /* Delta-encode the term as:
- ** varint(nPrefix)
- ** varint(nSuffix)
- ** char pTermSuffix[nSuffix]
- */
- n = fts3PutVarint(c, nPrefix);
- n += fts3PutVarint(c+n, nTerm-nPrefix);
- dataBufferAppend2(&pWriter->data, c, n, pTerm+nPrefix, nTerm-nPrefix);
- }
- dataBufferReplace(&pWriter->term, pTerm, nTerm);
-
- return nPrefix+1;
-}
-
-/* Used to avoid a memmove when a large amount of doclist data is in
-** the buffer. This constructs a node and term header before
-** iDoclistData and flushes the resulting complete node using
-** leafWriterInternalFlush().
-*/
-static int leafWriterInlineFlush(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- int iDoclistData){
- char c[VARINT_MAX+VARINT_MAX];
- int iData, n = fts3PutVarint(c, 0);
- n += fts3PutVarint(c+n, nTerm);
-
- /* There should always be room for the header. Even if pTerm shared
- ** a substantial prefix with the previous term, the entire prefix
- ** could be constructed from earlier data in the doclist, so there
- ** should be room.
- */
- assert( iDoclistData>=n+nTerm );
-
- iData = iDoclistData-(n+nTerm);
- memcpy(pWriter->data.pData+iData, c, n);
- memcpy(pWriter->data.pData+iData+n, pTerm, nTerm);
-
- return leafWriterInternalFlush(v, pWriter, iData, pWriter->data.nData-iData);
-}
-
-/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
-** %_segments.
-*/
-static int leafWriterStepMerge(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- DLReader *pReaders, int nReaders){
- char c[VARINT_MAX+VARINT_MAX];
- int iTermData = pWriter->data.nData, iDoclistData;
- int i, nData, n, nActualData, nActual, rc, nTermDistinct;
-
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
- nTermDistinct = leafWriterEncodeTerm(pWriter, pTerm, nTerm);
-
- /* Remember nTermDistinct if opening a new node. */
- if( iTermData==0 ) pWriter->nTermDistinct = nTermDistinct;
-
- iDoclistData = pWriter->data.nData;
-
- /* Estimate the length of the merged doclist so we can leave space
- ** to encode it.
- */
- for(i=0, nData=0; i<nReaders; i++){
- nData += dlrAllDataBytes(&pReaders[i]);
- }
- n = fts3PutVarint(c, nData);
- dataBufferAppend(&pWriter->data, c, n);
-
- docListMerge(&pWriter->data, pReaders, nReaders);
- ASSERT_VALID_DOCLIST(DL_DEFAULT,
- pWriter->data.pData+iDoclistData+n,
- pWriter->data.nData-iDoclistData-n, NULL);
-
- /* The actual amount of doclist data at this point could be smaller
- ** than the length we encoded. Additionally, the space required to
- ** encode this length could be smaller. For small doclists, this is
- ** not a big deal, we can just use memmove() to adjust things.
- */
- nActualData = pWriter->data.nData-(iDoclistData+n);
- nActual = fts3PutVarint(c, nActualData);
- assert( nActualData<=nData );
- assert( nActual<=n );
-
- /* If the new doclist is big enough for force a standalone leaf
- ** node, we can immediately flush it inline without doing the
- ** memmove().
- */
- /* TODO(shess) This test matches leafWriterStep(), which does this
- ** test before it knows the cost to varint-encode the term and
- ** doclist lengths. At some point, change to
- ** pWriter->data.nData-iTermData>STANDALONE_MIN.
- */
- if( nTerm+nActualData>STANDALONE_MIN ){
- /* Push leaf node from before this term. */
- if( iTermData>0 ){
- rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
- if( rc!=SQLITE_OK ) return rc;
-
- pWriter->nTermDistinct = nTermDistinct;
- }
-
- /* Fix the encoded doclist length. */
- iDoclistData += n - nActual;
- memcpy(pWriter->data.pData+iDoclistData, c, nActual);
-
- /* Push the standalone leaf node. */
- rc = leafWriterInlineFlush(v, pWriter, pTerm, nTerm, iDoclistData);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Leave the node empty. */
- dataBufferReset(&pWriter->data);
-
- return rc;
- }
-
- /* At this point, we know that the doclist was small, so do the
- ** memmove if indicated.
- */
- if( nActual<n ){
- memmove(pWriter->data.pData+iDoclistData+nActual,
- pWriter->data.pData+iDoclistData+n,
- pWriter->data.nData-(iDoclistData+n));
- pWriter->data.nData -= n-nActual;
- }
-
- /* Replace written length with actual length. */
- memcpy(pWriter->data.pData+iDoclistData, c, nActual);
-
- /* If the node is too large, break things up. */
- /* TODO(shess) This test matches leafWriterStep(), which does this
- ** test before it knows the cost to varint-encode the term and
- ** doclist lengths. At some point, change to
- ** pWriter->data.nData>LEAF_MAX.
- */
- if( iTermData+nTerm+nActualData>LEAF_MAX ){
- /* Flush out the leading data as a node */
- rc = leafWriterInternalFlush(v, pWriter, 0, iTermData);
- if( rc!=SQLITE_OK ) return rc;
-
- pWriter->nTermDistinct = nTermDistinct;
-
- /* Rebuild header using the current term */
- n = fts3PutVarint(pWriter->data.pData, 0);
- n += fts3PutVarint(pWriter->data.pData+n, nTerm);
- memcpy(pWriter->data.pData+n, pTerm, nTerm);
- n += nTerm;
-
- /* There should always be room, because the previous encoding
- ** included all data necessary to construct the term.
- */
- assert( n<iDoclistData );
- /* So long as STANDALONE_MIN is half or less of LEAF_MAX, the
- ** following memcpy() is safe (as opposed to needing a memmove).
- */
- assert( 2*STANDALONE_MIN<=LEAF_MAX );
- assert( n+pWriter->data.nData-iDoclistData<iDoclistData );
- memcpy(pWriter->data.pData+n,
- pWriter->data.pData+iDoclistData,
- pWriter->data.nData-iDoclistData);
- pWriter->data.nData -= iDoclistData-n;
- }
- ASSERT_VALID_LEAF_NODE(pWriter->data.pData, pWriter->data.nData);
-
- return SQLITE_OK;
-}
-
-/* Push pTerm[nTerm] along with the doclist data to the leaf layer of
-** %_segments.
-*/
-/* TODO(shess) Revise writeZeroSegment() so that doclists are
-** constructed directly in pWriter->data.
-*/
-static int leafWriterStep(fulltext_vtab *v, LeafWriter *pWriter,
- const char *pTerm, int nTerm,
- const char *pData, int nData){
- int rc;
- DLReader reader;
-
- dlrInit(&reader, DL_DEFAULT, pData, nData);
- rc = leafWriterStepMerge(v, pWriter, pTerm, nTerm, &reader, 1);
- dlrDestroy(&reader);
-
- return rc;
-}
-
-
-/****************************************************************/
-/* LeafReader is used to iterate over an individual leaf node. */
-typedef struct LeafReader {
- DataBuffer term; /* copy of current term. */
-
- const char *pData; /* data for current term. */
- int nData;
-} LeafReader;
-
-static void leafReaderDestroy(LeafReader *pReader){
- dataBufferDestroy(&pReader->term);
- SCRAMBLE(pReader);
-}
-
-static int leafReaderAtEnd(LeafReader *pReader){
- return pReader->nData<=0;
-}
-
-/* Access the current term. */
-static int leafReaderTermBytes(LeafReader *pReader){
- return pReader->term.nData;
-}
-static const char *leafReaderTerm(LeafReader *pReader){
- assert( pReader->term.nData>0 );
- return pReader->term.pData;
-}
-
-/* Access the doclist data for the current term. */
-static int leafReaderDataBytes(LeafReader *pReader){
- int nData;
- assert( pReader->term.nData>0 );
- fts3GetVarint32(pReader->pData, &nData);
- return nData;
-}
-static const char *leafReaderData(LeafReader *pReader){
- int n, nData;
- assert( pReader->term.nData>0 );
- n = fts3GetVarint32(pReader->pData, &nData);
- return pReader->pData+n;
-}
-
-static void leafReaderInit(const char *pData, int nData,
- LeafReader *pReader){
- int nTerm, n;
-
- assert( nData>0 );
- assert( pData[0]=='\0' );
-
- CLEAR(pReader);
-
- /* Read the first term, skipping the header byte. */
- n = fts3GetVarint32(pData+1, &nTerm);
- dataBufferInit(&pReader->term, nTerm);
- dataBufferReplace(&pReader->term, pData+1+n, nTerm);
-
- /* Position after the first term. */
- assert( 1+n+nTerm<nData );
- pReader->pData = pData+1+n+nTerm;
- pReader->nData = nData-1-n-nTerm;
-}
-
-/* Step the reader forward to the next term. */
-static void leafReaderStep(LeafReader *pReader){
- int n, nData, nPrefix, nSuffix;
- assert( !leafReaderAtEnd(pReader) );
-
- /* Skip previous entry's data block. */
- n = fts3GetVarint32(pReader->pData, &nData);
- assert( n+nData<=pReader->nData );
- pReader->pData += n+nData;
- pReader->nData -= n+nData;
-
- if( !leafReaderAtEnd(pReader) ){
- /* Construct the new term using a prefix from the old term plus a
- ** suffix from the leaf data.
- */
- n = fts3GetVarint32(pReader->pData, &nPrefix);
- n += fts3GetVarint32(pReader->pData+n, &nSuffix);
- assert( n+nSuffix<pReader->nData );
- pReader->term.nData = nPrefix;
- dataBufferAppend(&pReader->term, pReader->pData+n, nSuffix);
-
- pReader->pData += n+nSuffix;
- pReader->nData -= n+nSuffix;
- }
-}
-
-/* strcmp-style comparison of pReader's current term against pTerm.
-** If isPrefix, equality means equal through nTerm bytes.
-*/
-static int leafReaderTermCmp(LeafReader *pReader,
- const char *pTerm, int nTerm, int isPrefix){
- int c, n = pReader->term.nData<nTerm ? pReader->term.nData : nTerm;
- if( n==0 ){
- if( pReader->term.nData>0 ) return -1;
- if(nTerm>0 ) return 1;
- return 0;
- }
-
- c = memcmp(pReader->term.pData, pTerm, n);
- if( c!=0 ) return c;
- if( isPrefix && n==nTerm ) return 0;
- return pReader->term.nData - nTerm;
-}
-
-
-/****************************************************************/
-/* LeavesReader wraps LeafReader to allow iterating over the entire
-** leaf layer of the tree.
-*/
-typedef struct LeavesReader {
- int idx; /* Index within the segment. */
-
- sqlite3_stmt *pStmt; /* Statement we're streaming leaves from. */
- int eof; /* we've seen SQLITE_DONE from pStmt. */
-
- LeafReader leafReader; /* reader for the current leaf. */
- DataBuffer rootData; /* root data for inline. */
-} LeavesReader;
-
-/* Access the current term. */
-static int leavesReaderTermBytes(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderTermBytes(&pReader->leafReader);
-}
-static const char *leavesReaderTerm(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderTerm(&pReader->leafReader);
-}
-
-/* Access the doclist data for the current term. */
-static int leavesReaderDataBytes(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderDataBytes(&pReader->leafReader);
-}
-static const char *leavesReaderData(LeavesReader *pReader){
- assert( !pReader->eof );
- return leafReaderData(&pReader->leafReader);
-}
-
-static int leavesReaderAtEnd(LeavesReader *pReader){
- return pReader->eof;
-}
-
-/* loadSegmentLeaves() may not read all the way to SQLITE_DONE, thus
-** leaving the statement handle open, which locks the table.
-*/
-/* TODO(shess) This "solution" is not satisfactory. Really, there
-** should be check-in function for all statement handles which
-** arranges to call sqlite3_reset(). This most likely will require
-** modification to control flow all over the place, though, so for now
-** just punt.
-**
-** Note the the current system assumes that segment merges will run to
-** completion, which is why this particular probably hasn't arisen in
-** this case. Probably a brittle assumption.
-*/
-static int leavesReaderReset(LeavesReader *pReader){
- return sqlite3_reset(pReader->pStmt);
-}
-
-static void leavesReaderDestroy(LeavesReader *pReader){
- /* If idx is -1, that means we're using a non-cached statement
- ** handle in the optimize() case, so we need to release it.
- */
- if( pReader->pStmt!=NULL && pReader->idx==-1 ){
- sqlite3_finalize(pReader->pStmt);
- }
- leafReaderDestroy(&pReader->leafReader);
- dataBufferDestroy(&pReader->rootData);
- SCRAMBLE(pReader);
-}
-
-/* Initialize pReader with the given root data (if iStartBlockid==0
-** the leaf data was entirely contained in the root), or from the
-** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
-*/
-static int leavesReaderInit(fulltext_vtab *v,
- int idx,
- sqlite_int64 iStartBlockid,
- sqlite_int64 iEndBlockid,
- const char *pRootData, int nRootData,
- LeavesReader *pReader){
- CLEAR(pReader);
- pReader->idx = idx;
-
- dataBufferInit(&pReader->rootData, 0);
- if( iStartBlockid==0 ){
- /* Entire leaf level fit in root data. */
- dataBufferReplace(&pReader->rootData, pRootData, nRootData);
- leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
- &pReader->leafReader);
- }else{
- sqlite3_stmt *s;
- int rc = sql_get_leaf_statement(v, idx, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iStartBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 2, iEndBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ){
- pReader->eof = 1;
- return SQLITE_OK;
- }
- if( rc!=SQLITE_ROW ) return rc;
-
- pReader->pStmt = s;
- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
- sqlite3_column_bytes(pReader->pStmt, 0),
- &pReader->leafReader);
- }
- return SQLITE_OK;
-}
-
-/* Step the current leaf forward to the next term. If we reach the
-** end of the current leaf, step forward to the next leaf block.
-*/
-static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
- assert( !leavesReaderAtEnd(pReader) );
- leafReaderStep(&pReader->leafReader);
-
- if( leafReaderAtEnd(&pReader->leafReader) ){
- int rc;
- if( pReader->rootData.pData ){
- pReader->eof = 1;
- return SQLITE_OK;
- }
- rc = sqlite3_step(pReader->pStmt);
- if( rc!=SQLITE_ROW ){
- pReader->eof = 1;
- return rc==SQLITE_DONE ? SQLITE_OK : rc;
- }
- leafReaderDestroy(&pReader->leafReader);
- leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
- sqlite3_column_bytes(pReader->pStmt, 0),
- &pReader->leafReader);
- }
- return SQLITE_OK;
-}
-
-/* Order LeavesReaders by their term, ignoring idx. Readers at eof
-** always sort to the end.
-*/
-static int leavesReaderTermCmp(LeavesReader *lr1, LeavesReader *lr2){
- if( leavesReaderAtEnd(lr1) ){
- if( leavesReaderAtEnd(lr2) ) return 0;
- return 1;
- }
- if( leavesReaderAtEnd(lr2) ) return -1;
-
- return leafReaderTermCmp(&lr1->leafReader,
- leavesReaderTerm(lr2), leavesReaderTermBytes(lr2),
- 0);
-}
-
-/* Similar to leavesReaderTermCmp(), with additional ordering by idx
-** so that older segments sort before newer segments.
-*/
-static int leavesReaderCmp(LeavesReader *lr1, LeavesReader *lr2){
- int c = leavesReaderTermCmp(lr1, lr2);
- if( c!=0 ) return c;
- return lr1->idx-lr2->idx;
-}
-
-/* Assume that pLr[1]..pLr[nLr] are sorted. Bubble pLr[0] into its
-** sorted position.
-*/
-static void leavesReaderReorder(LeavesReader *pLr, int nLr){
- while( nLr>1 && leavesReaderCmp(pLr, pLr+1)>0 ){
- LeavesReader tmp = pLr[0];
- pLr[0] = pLr[1];
- pLr[1] = tmp;
- nLr--;
- pLr++;
- }
-}
-
-/* Initializes pReaders with the segments from level iLevel, returning
-** the number of segments in *piReaders. Leaves pReaders in sorted
-** order.
-*/
-static int leavesReadersInit(fulltext_vtab *v, int iLevel,
- LeavesReader *pReaders, int *piReaders){
- sqlite3_stmt *s;
- int i, rc = sql_get_statement(v, SEGDIR_SELECT_LEVEL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int(s, 1, iLevel);
- if( rc!=SQLITE_OK ) return rc;
-
- i = 0;
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- sqlite_int64 iStart = sqlite3_column_int64(s, 0);
- sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
- const char *pRootData = sqlite3_column_blob(s, 2);
- int nRootData = sqlite3_column_bytes(s, 2);
-
- assert( i<MERGE_COUNT );
- rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
- &pReaders[i]);
- if( rc!=SQLITE_OK ) break;
-
- i++;
- }
- if( rc!=SQLITE_DONE ){
- while( i-->0 ){
- leavesReaderDestroy(&pReaders[i]);
- }
- return rc;
- }
-
- *piReaders = i;
-
- /* Leave our results sorted by term, then age. */
- while( i-- ){
- leavesReaderReorder(pReaders+i, *piReaders-i);
- }
- return SQLITE_OK;
-}
-
-/* Merge doclists from pReaders[nReaders] into a single doclist, which
-** is written to pWriter. Assumes pReaders is ordered oldest to
-** newest.
-*/
-/* TODO(shess) Consider putting this inline in segmentMerge(). */
-static int leavesReadersMerge(fulltext_vtab *v,
- LeavesReader *pReaders, int nReaders,
- LeafWriter *pWriter){
- DLReader dlReaders[MERGE_COUNT];
- const char *pTerm = leavesReaderTerm(pReaders);
- int i, nTerm = leavesReaderTermBytes(pReaders);
-
- assert( nReaders<=MERGE_COUNT );
-
- for(i=0; i<nReaders; i++){
- dlrInit(&dlReaders[i], DL_DEFAULT,
- leavesReaderData(pReaders+i),
- leavesReaderDataBytes(pReaders+i));
- }
-
- return leafWriterStepMerge(v, pWriter, pTerm, nTerm, dlReaders, nReaders);
-}
-
-/* Forward ref due to mutual recursion with segdirNextIndex(). */
-static int segmentMerge(fulltext_vtab *v, int iLevel);
-
-/* Put the next available index at iLevel into *pidx. If iLevel
-** already has MERGE_COUNT segments, they are merged to a higher
-** level to make room.
-*/
-static int segdirNextIndex(fulltext_vtab *v, int iLevel, int *pidx){
- int rc = segdir_max_index(v, iLevel, pidx);
- if( rc==SQLITE_DONE ){ /* No segments at iLevel. */
- *pidx = 0;
- }else if( rc==SQLITE_ROW ){
- if( *pidx==(MERGE_COUNT-1) ){
- rc = segmentMerge(v, iLevel);
- if( rc!=SQLITE_OK ) return rc;
- *pidx = 0;
- }else{
- (*pidx)++;
- }
- }else{
- return rc;
- }
- return SQLITE_OK;
-}
-
-/* Merge MERGE_COUNT segments at iLevel into a new segment at
-** iLevel+1. If iLevel+1 is already full of segments, those will be
-** merged to make room.
-*/
-static int segmentMerge(fulltext_vtab *v, int iLevel){
- LeafWriter writer;
- LeavesReader lrs[MERGE_COUNT];
- int i, rc, idx = 0;
-
- /* Determine the next available segment index at the next level,
- ** merging as necessary.
- */
- rc = segdirNextIndex(v, iLevel+1, &idx);
- if( rc!=SQLITE_OK ) return rc;
-
- /* TODO(shess) This assumes that we'll always see exactly
- ** MERGE_COUNT segments to merge at a given level. That will be
- ** broken if we allow the developer to request preemptive or
- ** deferred merging.
- */
- memset(&lrs, '\0', sizeof(lrs));
- rc = leavesReadersInit(v, iLevel, lrs, &i);
- if( rc!=SQLITE_OK ) return rc;
- assert( i==MERGE_COUNT );
-
- leafWriterInit(iLevel+1, idx, &writer);
-
- /* Since leavesReaderReorder() pushes readers at eof to the end,
- ** when the first reader is empty, all will be empty.
- */
- while( !leavesReaderAtEnd(lrs) ){
- /* Figure out how many readers share their next term. */
- for(i=1; i<MERGE_COUNT && !leavesReaderAtEnd(lrs+i); i++){
- if( 0!=leavesReaderTermCmp(lrs, lrs+i) ) break;
- }
-
- rc = leavesReadersMerge(v, lrs, i, &writer);
- if( rc!=SQLITE_OK ) goto err;
-
- /* Step forward those that were merged. */
- while( i-->0 ){
- rc = leavesReaderStep(v, lrs+i);
- if( rc!=SQLITE_OK ) goto err;
-
- /* Reorder by term, then by age. */
- leavesReaderReorder(lrs+i, MERGE_COUNT-i);
- }
- }
-
- for(i=0; i<MERGE_COUNT; i++){
- leavesReaderDestroy(&lrs[i]);
- }
-
- rc = leafWriterFinalize(v, &writer);
- leafWriterDestroy(&writer);
- if( rc!=SQLITE_OK ) return rc;
-
- /* Delete the merged segment data. */
- return segdir_delete(v, iLevel);
-
- err:
- for(i=0; i<MERGE_COUNT; i++){
- leavesReaderDestroy(&lrs[i]);
- }
- leafWriterDestroy(&writer);
- return rc;
-}
-
-/* Accumulate the union of *acc and *pData into *acc. */
-static void docListAccumulateUnion(DataBuffer *acc,
- const char *pData, int nData) {
- DataBuffer tmp = *acc;
- dataBufferInit(acc, tmp.nData+nData);
- docListUnion(tmp.pData, tmp.nData, pData, nData, acc);
- dataBufferDestroy(&tmp);
-}
-
-/* TODO(shess) It might be interesting to explore different merge
-** strategies, here. For instance, since this is a sorted merge, we
-** could easily merge many doclists in parallel. With some
-** comprehension of the storage format, we could merge all of the
-** doclists within a leaf node directly from the leaf node's storage.
-** It may be worthwhile to merge smaller doclists before larger
-** doclists, since they can be traversed more quickly - but the
-** results may have less overlap, making them more expensive in a
-** different way.
-*/
-
-/* Scan pReader for pTerm/nTerm, and merge the term's doclist over
-** *out (any doclists with duplicate docids overwrite those in *out).
-** Internal function for loadSegmentLeaf().
-*/
-static int loadSegmentLeavesInt(fulltext_vtab *v, LeavesReader *pReader,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- /* doclist data is accumulated into pBuffers similar to how one does
- ** increment in binary arithmetic. If index 0 is empty, the data is
- ** stored there. If there is data there, it is merged and the
- ** results carried into position 1, with further merge-and-carry
- ** until an empty position is found.
- */
- DataBuffer *pBuffers = NULL;
- int nBuffers = 0, nMaxBuffers = 0, rc;
-
- assert( nTerm>0 );
-
- for(rc=SQLITE_OK; rc==SQLITE_OK && !leavesReaderAtEnd(pReader);
- rc=leavesReaderStep(v, pReader)){
- /* TODO(shess) Really want leavesReaderTermCmp(), but that name is
- ** already taken to compare the terms of two LeavesReaders. Think
- ** on a better name. [Meanwhile, break encapsulation rather than
- ** use a confusing name.]
- */
- int c = leafReaderTermCmp(&pReader->leafReader, pTerm, nTerm, isPrefix);
- if( c>0 ) break; /* Past any possible matches. */
- if( c==0 ){
- const char *pData = leavesReaderData(pReader);
- int iBuffer, nData = leavesReaderDataBytes(pReader);
-
- /* Find the first empty buffer. */
- for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
- if( 0==pBuffers[iBuffer].nData ) break;
- }
-
- /* Out of buffers, add an empty one. */
- if( iBuffer==nBuffers ){
- if( nBuffers==nMaxBuffers ){
- DataBuffer *p;
- nMaxBuffers += 20;
-
- /* Manual realloc so we can handle NULL appropriately. */
- p = sqlite3_malloc(nMaxBuffers*sizeof(*pBuffers));
- if( p==NULL ){
- rc = SQLITE_NOMEM;
- break;
- }
-
- if( nBuffers>0 ){
- assert(pBuffers!=NULL);
- memcpy(p, pBuffers, nBuffers*sizeof(*pBuffers));
- sqlite3_free(pBuffers);
- }
- pBuffers = p;
- }
- dataBufferInit(&(pBuffers[nBuffers]), 0);
- nBuffers++;
- }
-
- /* At this point, must have an empty at iBuffer. */
- assert(iBuffer<nBuffers && pBuffers[iBuffer].nData==0);
-
- /* If empty was first buffer, no need for merge logic. */
- if( iBuffer==0 ){
- dataBufferReplace(&(pBuffers[0]), pData, nData);
- }else{
- /* pAcc is the empty buffer the merged data will end up in. */
- DataBuffer *pAcc = &(pBuffers[iBuffer]);
- DataBuffer *p = &(pBuffers[0]);
-
- /* Handle position 0 specially to avoid need to prime pAcc
- ** with pData/nData.
- */
- dataBufferSwap(p, pAcc);
- docListAccumulateUnion(pAcc, pData, nData);
-
- /* Accumulate remaining doclists into pAcc. */
- for(++p; p<pAcc; ++p){
- docListAccumulateUnion(pAcc, p->pData, p->nData);
-
- /* dataBufferReset() could allow a large doclist to blow up
- ** our memory requirements.
- */
- if( p->nCapacity<1024 ){
- dataBufferReset(p);
- }else{
- dataBufferDestroy(p);
- dataBufferInit(p, 0);
- }
- }
- }
- }
- }
-
- /* Union all the doclists together into *out. */
- /* TODO(shess) What if *out is big? Sigh. */
- if( rc==SQLITE_OK && nBuffers>0 ){
- int iBuffer;
- for(iBuffer=0; iBuffer<nBuffers; ++iBuffer){
- if( pBuffers[iBuffer].nData>0 ){
- if( out->nData==0 ){
- dataBufferSwap(out, &(pBuffers[iBuffer]));
- }else{
- docListAccumulateUnion(out, pBuffers[iBuffer].pData,
- pBuffers[iBuffer].nData);
- }
- }
- }
- }
-
- while( nBuffers-- ){
- dataBufferDestroy(&(pBuffers[nBuffers]));
- }
- if( pBuffers!=NULL ) sqlite3_free(pBuffers);
-
- return rc;
-}
-
-/* Call loadSegmentLeavesInt() with pData/nData as input. */
-static int loadSegmentLeaf(fulltext_vtab *v, const char *pData, int nData,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- LeavesReader reader;
- int rc;
-
- assert( nData>1 );
- assert( *pData=='\0' );
- rc = leavesReaderInit(v, 0, 0, 0, pData, nData, &reader);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
- leavesReaderReset(&reader);
- leavesReaderDestroy(&reader);
- return rc;
-}
-
-/* Call loadSegmentLeavesInt() with the leaf nodes from iStartLeaf to
-** iEndLeaf (inclusive) as input, and merge the resulting doclist into
-** out.
-*/
-static int loadSegmentLeaves(fulltext_vtab *v,
- sqlite_int64 iStartLeaf, sqlite_int64 iEndLeaf,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- int rc;
- LeavesReader reader;
-
- assert( iStartLeaf<=iEndLeaf );
- rc = leavesReaderInit(v, 0, iStartLeaf, iEndLeaf, NULL, 0, &reader);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = loadSegmentLeavesInt(v, &reader, pTerm, nTerm, isPrefix, out);
- leavesReaderReset(&reader);
- leavesReaderDestroy(&reader);
- return rc;
-}
-
-/* Taking pData/nData as an interior node, find the sequence of child
-** nodes which could include pTerm/nTerm/isPrefix. Note that the
-** interior node terms logically come between the blocks, so there is
-** one more blockid than there are terms (that block contains terms >=
-** the last interior-node term).
-*/
-/* TODO(shess) The calling code may already know that the end child is
-** not worth calculating, because the end may be in a later sibling
-** node. Consider whether breaking symmetry is worthwhile. I suspect
-** it is not worthwhile.
-*/
-static void getChildrenContaining(const char *pData, int nData,
- const char *pTerm, int nTerm, int isPrefix,
- sqlite_int64 *piStartChild,
- sqlite_int64 *piEndChild){
- InteriorReader reader;
-
- assert( nData>1 );
- assert( *pData!='\0' );
- interiorReaderInit(pData, nData, &reader);
-
- /* Scan for the first child which could contain pTerm/nTerm. */
- while( !interiorReaderAtEnd(&reader) ){
- if( interiorReaderTermCmp(&reader, pTerm, nTerm, 0)>0 ) break;
- interiorReaderStep(&reader);
- }
- *piStartChild = interiorReaderCurrentBlockid(&reader);
-
- /* Keep scanning to find a term greater than our term, using prefix
- ** comparison if indicated. If isPrefix is false, this will be the
- ** same blockid as the starting block.
- */
- while( !interiorReaderAtEnd(&reader) ){
- if( interiorReaderTermCmp(&reader, pTerm, nTerm, isPrefix)>0 ) break;
- interiorReaderStep(&reader);
- }
- *piEndChild = interiorReaderCurrentBlockid(&reader);
-
- interiorReaderDestroy(&reader);
-
- /* Children must ascend, and if !prefix, both must be the same. */
- assert( *piEndChild>=*piStartChild );
- assert( isPrefix || *piStartChild==*piEndChild );
-}
-
-/* Read block at iBlockid and pass it with other params to
-** getChildrenContaining().
-*/
-static int loadAndGetChildrenContaining(
- fulltext_vtab *v,
- sqlite_int64 iBlockid,
- const char *pTerm, int nTerm, int isPrefix,
- sqlite_int64 *piStartChild, sqlite_int64 *piEndChild
-){
- sqlite3_stmt *s = NULL;
- int rc;
-
- assert( iBlockid!=0 );
- assert( pTerm!=NULL );
- assert( nTerm!=0 ); /* TODO(shess) Why not allow this? */
- assert( piStartChild!=NULL );
- assert( piEndChild!=NULL );
-
- rc = sql_get_statement(v, BLOCK_SELECT_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_bind_int64(s, 1, iBlockid);
- if( rc!=SQLITE_OK ) return rc;
-
- rc = sqlite3_step(s);
- if( rc==SQLITE_DONE ) return SQLITE_ERROR;
- if( rc!=SQLITE_ROW ) return rc;
-
- getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
- pTerm, nTerm, isPrefix, piStartChild, piEndChild);
-
- /* We expect only one row. We must execute another sqlite3_step()
- * to complete the iteration; otherwise the table will remain
- * locked. */
- rc = sqlite3_step(s);
- if( rc==SQLITE_ROW ) return SQLITE_ERROR;
- if( rc!=SQLITE_DONE ) return rc;
-
- return SQLITE_OK;
-}
-
-/* Traverse the tree represented by pData[nData] looking for
-** pTerm[nTerm], placing its doclist into *out. This is internal to
-** loadSegment() to make error-handling cleaner.
-*/
-static int loadSegmentInt(fulltext_vtab *v, const char *pData, int nData,
- sqlite_int64 iLeavesEnd,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- /* Special case where root is a leaf. */
- if( *pData=='\0' ){
- return loadSegmentLeaf(v, pData, nData, pTerm, nTerm, isPrefix, out);
- }else{
- int rc;
- sqlite_int64 iStartChild, iEndChild;
-
- /* Process pData as an interior node, then loop down the tree
- ** until we find the set of leaf nodes to scan for the term.
- */
- getChildrenContaining(pData, nData, pTerm, nTerm, isPrefix,
- &iStartChild, &iEndChild);
- while( iStartChild>iLeavesEnd ){
- sqlite_int64 iNextStart, iNextEnd;
- rc = loadAndGetChildrenContaining(v, iStartChild, pTerm, nTerm, isPrefix,
- &iNextStart, &iNextEnd);
- if( rc!=SQLITE_OK ) return rc;
-
- /* If we've branched, follow the end branch, too. */
- if( iStartChild!=iEndChild ){
- sqlite_int64 iDummy;
- rc = loadAndGetChildrenContaining(v, iEndChild, pTerm, nTerm, isPrefix,
- &iDummy, &iNextEnd);
- if( rc!=SQLITE_OK ) return rc;
- }
-
- assert( iNextStart<=iNextEnd );
- iStartChild = iNextStart;
- iEndChild = iNextEnd;
- }
- assert( iStartChild<=iLeavesEnd );
- assert( iEndChild<=iLeavesEnd );
-
- /* Scan through the leaf segments for doclists. */
- return loadSegmentLeaves(v, iStartChild, iEndChild,
- pTerm, nTerm, isPrefix, out);
- }
-}
-
-/* Call loadSegmentInt() to collect the doclist for pTerm/nTerm, then
-** merge its doclist over *out (any duplicate doclists read from the
-** segment rooted at pData will overwrite those in *out).
-*/
-/* TODO(shess) Consider changing this to determine the depth of the
-** leaves using either the first characters of interior nodes (when
-** ==1, we're one level above the leaves), or the first character of
-** the root (which will describe the height of the tree directly).
-** Either feels somewhat tricky to me.
-*/
-/* TODO(shess) The current merge is likely to be slow for large
-** doclists (though it should process from newest/smallest to
-** oldest/largest, so it may not be that bad). It might be useful to
-** modify things to allow for N-way merging. This could either be
-** within a segment, with pairwise merges across segments, or across
-** all segments at once.
-*/
-static int loadSegment(fulltext_vtab *v, const char *pData, int nData,
- sqlite_int64 iLeavesEnd,
- const char *pTerm, int nTerm, int isPrefix,
- DataBuffer *out){
- DataBuffer result;
- int rc;
-
- assert( nData>1 );
-
- /* This code should never be called with buffered updates. */
- assert( v->nPendingData<0 );
-
- dataBufferInit(&result, 0);
- rc = loadSegmentInt(v, pData, nData, iLeavesEnd,
- pTerm, nTerm, isPrefix, &result);
- if( rc==SQLITE_OK && result.nData>0 ){
- if( out->nData==0 ){
- DataBuffer tmp = *out;
- *out = result;
- result = tmp;
- }else{
- DataBuffer merged;
- DLReader readers[2];
-
- dlrInit(&readers[0], DL_DEFAULT, out->pData, out->nData);
- dlrInit(&readers[1], DL_DEFAULT, result.pData, result.nData);
- dataBufferInit(&merged, out->nData+result.nData);
- docListMerge(&merged, readers, 2);
- dataBufferDestroy(out);
- *out = merged;
- dlrDestroy(&readers[0]);
- dlrDestroy(&readers[1]);
- }
- }
- dataBufferDestroy(&result);
- return rc;
-}
-
-/* Scan the database and merge together the posting lists for the term
-** into *out.
-*/
-static int termSelect(
- fulltext_vtab *v,
- int iColumn,
- const char *pTerm, int nTerm, /* Term to query for */
- int isPrefix, /* True for a prefix search */
- DocListType iType,
- DataBuffer *out /* Write results here */
-){
- DataBuffer doclist;
- sqlite3_stmt *s;
- int rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) return rc;
-
- /* This code should never be called with buffered updates. */
- assert( v->nPendingData<0 );
-
- dataBufferInit(&doclist, 0);
- dataBufferInit(out, 0);
-
- /* Traverse the segments from oldest to newest so that newer doclist
- ** elements for given docids overwrite older elements.
- */
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- const char *pData = sqlite3_column_blob(s, 2);
- const int nData = sqlite3_column_bytes(s, 2);
- const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
- rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
- &doclist);
- if( rc!=SQLITE_OK ) goto err;
- }
- if( rc==SQLITE_DONE ){
- if( doclist.nData!=0 ){
- /* TODO(shess) The old term_select_all() code applied the column
- ** restrict as we merged segments, leading to smaller buffers.
- ** This is probably worthwhile to bring back, once the new storage
- ** system is checked in.
- */
- if( iColumn==v->nColumn) iColumn = -1;
- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
- iColumn, iType, out);
- }
- rc = SQLITE_OK;
- }
-
- err:
- dataBufferDestroy(&doclist);
- return rc;
-}
-
-/****************************************************************/
-/* Used to hold hashtable data for sorting. */
-typedef struct TermData {
- const char *pTerm;
- int nTerm;
- DLCollector *pCollector;
-} TermData;
-
-/* Orders TermData elements in strcmp fashion ( <0 for less-than, 0
-** for equal, >0 for greater-than).
-*/
-static int termDataCmp(const void *av, const void *bv){
- const TermData *a = (const TermData *)av;
- const TermData *b = (const TermData *)bv;
- int n = a->nTerm<b->nTerm ? a->nTerm : b->nTerm;
- int c = memcmp(a->pTerm, b->pTerm, n);
- if( c!=0 ) return c;
- return a->nTerm-b->nTerm;
-}
-
-/* Order pTerms data by term, then write a new level 0 segment using
-** LeafWriter.
-*/
-static int writeZeroSegment(fulltext_vtab *v, fts3Hash *pTerms){
- fts3HashElem *e;
- int idx, rc, i, n;
- TermData *pData;
- LeafWriter writer;
- DataBuffer dl;
-
- /* Determine the next index at level 0, merging as necessary. */
- rc = segdirNextIndex(v, 0, &idx);
- if( rc!=SQLITE_OK ) return rc;
-
- n = fts3HashCount(pTerms);
- pData = sqlite3_malloc(n*sizeof(TermData));
-
- for(i = 0, e = fts3HashFirst(pTerms); e; i++, e = fts3HashNext(e)){
- assert( i<n );
- pData[i].pTerm = fts3HashKey(e);
- pData[i].nTerm = fts3HashKeysize(e);
- pData[i].pCollector = fts3HashData(e);
- }
- assert( i==n );
-
- /* TODO(shess) Should we allow user-defined collation sequences,
- ** here? I think we only need that once we support prefix searches.
- */
- if( n>1 ) qsort(pData, n, sizeof(*pData), termDataCmp);
-
- /* TODO(shess) Refactor so that we can write directly to the segment
- ** DataBuffer, as happens for segment merges.
- */
- leafWriterInit(0, idx, &writer);
- dataBufferInit(&dl, 0);
- for(i=0; i<n; i++){
- dataBufferReset(&dl);
- dlcAddDoclist(pData[i].pCollector, &dl);
- rc = leafWriterStep(v, &writer,
- pData[i].pTerm, pData[i].nTerm, dl.pData, dl.nData);
- if( rc!=SQLITE_OK ) goto err;
- }
- rc = leafWriterFinalize(v, &writer);
-
- err:
- dataBufferDestroy(&dl);
- sqlite3_free(pData);
- leafWriterDestroy(&writer);
- return rc;
-}
-
-/* If pendingTerms has data, free it. */
-static int clearPendingTerms(fulltext_vtab *v){
- if( v->nPendingData>=0 ){
- fts3HashElem *e;
- for(e=fts3HashFirst(&v->pendingTerms); e; e=fts3HashNext(e)){
- dlcDelete(fts3HashData(e));
- }
- fts3HashClear(&v->pendingTerms);
- v->nPendingData = -1;
- }
- return SQLITE_OK;
-}
-
-/* If pendingTerms has data, flush it to a level-zero segment, and
-** free it.
-*/
-static int flushPendingTerms(fulltext_vtab *v){
- if( v->nPendingData>=0 ){
- int rc = writeZeroSegment(v, &v->pendingTerms);
- if( rc==SQLITE_OK ) clearPendingTerms(v);
- return rc;
- }
- return SQLITE_OK;
-}
-
-/* If pendingTerms is "too big", or docid is out of order, flush it.
-** Regardless, be certain that pendingTerms is initialized for use.
-*/
-static int initPendingTerms(fulltext_vtab *v, sqlite_int64 iDocid){
- /* TODO(shess) Explore whether partially flushing the buffer on
- ** forced-flush would provide better performance. I suspect that if
- ** we ordered the doclists by size and flushed the largest until the
- ** buffer was half empty, that would let the less frequent terms
- ** generate longer doclists.
- */
- if( iDocid<=v->iPrevDocid || v->nPendingData>kPendingThreshold ){
- int rc = flushPendingTerms(v);
- if( rc!=SQLITE_OK ) return rc;
- }
- if( v->nPendingData<0 ){
- fts3HashInit(&v->pendingTerms, FTS3_HASH_STRING, 1);
- v->nPendingData = 0;
- }
- v->iPrevDocid = iDocid;
- return SQLITE_OK;
-}
-
-/* This function implements the xUpdate callback; it is the top-level entry
- * point for inserting, deleting or updating a row in a full-text table. */
-static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg,
- sqlite_int64 *pRowid){
- fulltext_vtab *v = (fulltext_vtab *) pVtab;
- int rc;
-
- FTSTRACE(("FTS3 Update %p\n", pVtab));
-
- if( nArg<2 ){
- rc = index_delete(v, sqlite3_value_int64(ppArg[0]));
- if( rc==SQLITE_OK ){
- /* If we just deleted the last row in the table, clear out the
- ** index data.
- */
- rc = content_exists(v);
- if( rc==SQLITE_ROW ){
- rc = SQLITE_OK;
- }else if( rc==SQLITE_DONE ){
- /* Clear the pending terms so we don't flush a useless level-0
- ** segment when the transaction closes.
- */
- rc = clearPendingTerms(v);
- if( rc==SQLITE_OK ){
- rc = segdir_delete_all(v);
- }
- }
- }
- } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){
- /* An update:
- * ppArg[0] = old rowid
- * ppArg[1] = new rowid
- * ppArg[2..2+v->nColumn-1] = values
- * ppArg[2+v->nColumn] = value for magic column (we ignore this)
- * ppArg[2+v->nColumn+1] = value for docid
- */
- sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]);
- if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER ||
- sqlite3_value_int64(ppArg[1]) != rowid ){
- rc = SQLITE_ERROR; /* we don't allow changing the rowid */
- }else if( sqlite3_value_type(ppArg[2+v->nColumn+1]) != SQLITE_INTEGER ||
- sqlite3_value_int64(ppArg[2+v->nColumn+1]) != rowid ){
- rc = SQLITE_ERROR; /* we don't allow changing the docid */
- }else{
- assert( nArg==2+v->nColumn+2);
- rc = index_update(v, rowid, &ppArg[2]);
- }
- } else {
- /* An insert:
- * ppArg[1] = requested rowid
- * ppArg[2..2+v->nColumn-1] = values
- * ppArg[2+v->nColumn] = value for magic column (we ignore this)
- * ppArg[2+v->nColumn+1] = value for docid
- */
- sqlite3_value *pRequestDocid = ppArg[2+v->nColumn+1];
- assert( nArg==2+v->nColumn+2);
- if( SQLITE_NULL != sqlite3_value_type(pRequestDocid) &&
- SQLITE_NULL != sqlite3_value_type(ppArg[1]) ){
- /* TODO(shess) Consider allowing this to work if the values are
- ** identical. I'm inclined to discourage that usage, though,
- ** given that both rowid and docid are special columns. Better
- ** would be to define one or the other as the default winner,
- ** but should it be fts3-centric (docid) or SQLite-centric
- ** (rowid)?
- */
- rc = SQLITE_ERROR;
- }else{
- if( SQLITE_NULL == sqlite3_value_type(pRequestDocid) ){
- pRequestDocid = ppArg[1];
- }
- rc = index_insert(v, pRequestDocid, &ppArg[2], pRowid);
- }
- }
-
- return rc;
-}
-
-static int fulltextSync(sqlite3_vtab *pVtab){
- FTSTRACE(("FTS3 xSync()\n"));
- return flushPendingTerms((fulltext_vtab *)pVtab);
-}
-
-static int fulltextBegin(sqlite3_vtab *pVtab){
- fulltext_vtab *v = (fulltext_vtab *) pVtab;
- FTSTRACE(("FTS3 xBegin()\n"));
-
- /* Any buffered updates should have been cleared by the previous
- ** transaction.
- */
- assert( v->nPendingData<0 );
- return clearPendingTerms(v);
-}
-
-static int fulltextCommit(sqlite3_vtab *pVtab){
- fulltext_vtab *v = (fulltext_vtab *) pVtab;
- FTSTRACE(("FTS3 xCommit()\n"));
-
- /* Buffered updates should have been cleared by fulltextSync(). */
- assert( v->nPendingData<0 );
- return clearPendingTerms(v);
-}
-
-static int fulltextRollback(sqlite3_vtab *pVtab){
- FTSTRACE(("FTS3 xRollback()\n"));
- return clearPendingTerms((fulltext_vtab *)pVtab);
-}
-
-/*
-** Implementation of the snippet() function for FTS3
-*/
-static void snippetFunc(
- sqlite3_context *pContext,
- int argc,
- sqlite3_value **argv
-){
- fulltext_cursor *pCursor;
- if( argc<1 ) return;
- if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1);
- }else{
- const char *zStart = "<b>";
- const char *zEnd = "</b>";
- const char *zEllipsis = "<b>...</b>";
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- if( argc>=2 ){
- zStart = (const char*)sqlite3_value_text(argv[1]);
- if( argc>=3 ){
- zEnd = (const char*)sqlite3_value_text(argv[2]);
- if( argc>=4 ){
- zEllipsis = (const char*)sqlite3_value_text(argv[3]);
- }
- }
- }
- snippetAllOffsets(pCursor);
- snippetText(pCursor, zStart, zEnd, zEllipsis);
- sqlite3_result_text(pContext, pCursor->snippet.zSnippet,
- pCursor->snippet.nSnippet, SQLITE_STATIC);
- }
-}
-
-/*
-** Implementation of the offsets() function for FTS3
-*/
-static void snippetOffsetsFunc(
- sqlite3_context *pContext,
- int argc,
- sqlite3_value **argv
-){
- fulltext_cursor *pCursor;
- if( argc<1 ) return;
- if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to offsets",-1);
- }else{
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- snippetAllOffsets(pCursor);
- snippetOffsetText(&pCursor->snippet);
- sqlite3_result_text(pContext,
- pCursor->snippet.zOffset, pCursor->snippet.nOffset,
- SQLITE_STATIC);
- }
-}
-
-/* OptLeavesReader is nearly identical to LeavesReader, except that
-** where LeavesReader is geared towards the merging of complete
-** segment levels (with exactly MERGE_COUNT segments), OptLeavesReader
-** is geared towards implementation of the optimize() function, and
-** can merge all segments simultaneously. This version may be
-** somewhat less efficient than LeavesReader because it merges into an
-** accumulator rather than doing an N-way merge, but since segment
-** size grows exponentially (so segment count logrithmically) this is
-** probably not an immediate problem.
-*/
-/* TODO(shess): Prove that assertion, or extend the merge code to
-** merge tree fashion (like the prefix-searching code does).
-*/
-/* TODO(shess): OptLeavesReader and LeavesReader could probably be
-** merged with little or no loss of performance for LeavesReader. The
-** merged code would need to handle >MERGE_COUNT segments, and would
-** also need to be able to optionally optimize away deletes.
-*/
-typedef struct OptLeavesReader {
- /* Segment number, to order readers by age. */
- int segment;
- LeavesReader reader;
-} OptLeavesReader;
-
-static int optLeavesReaderAtEnd(OptLeavesReader *pReader){
- return leavesReaderAtEnd(&pReader->reader);
-}
-static int optLeavesReaderTermBytes(OptLeavesReader *pReader){
- return leavesReaderTermBytes(&pReader->reader);
-}
-static const char *optLeavesReaderData(OptLeavesReader *pReader){
- return leavesReaderData(&pReader->reader);
-}
-static int optLeavesReaderDataBytes(OptLeavesReader *pReader){
- return leavesReaderDataBytes(&pReader->reader);
-}
-static const char *optLeavesReaderTerm(OptLeavesReader *pReader){
- return leavesReaderTerm(&pReader->reader);
-}
-static int optLeavesReaderStep(fulltext_vtab *v, OptLeavesReader *pReader){
- return leavesReaderStep(v, &pReader->reader);
-}
-static int optLeavesReaderTermCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
- return leavesReaderTermCmp(&lr1->reader, &lr2->reader);
-}
-/* Order by term ascending, segment ascending (oldest to newest), with
-** exhausted readers to the end.
-*/
-static int optLeavesReaderCmp(OptLeavesReader *lr1, OptLeavesReader *lr2){
- int c = optLeavesReaderTermCmp(lr1, lr2);
- if( c!=0 ) return c;
- return lr1->segment-lr2->segment;
-}
-/* Bubble pLr[0] to appropriate place in pLr[1..nLr-1]. Assumes that
-** pLr[1..nLr-1] is already sorted.
-*/
-static void optLeavesReaderReorder(OptLeavesReader *pLr, int nLr){
- while( nLr>1 && optLeavesReaderCmp(pLr, pLr+1)>0 ){
- OptLeavesReader tmp = pLr[0];
- pLr[0] = pLr[1];
- pLr[1] = tmp;
- nLr--;
- pLr++;
- }
-}
-
-/* optimize() helper function. Put the readers in order and iterate
-** through them, merging doclists for matching terms into pWriter.
-** Returns SQLITE_OK on success, or the SQLite error code which
-** prevented success.
-*/
-static int optimizeInternal(fulltext_vtab *v,
- OptLeavesReader *readers, int nReaders,
- LeafWriter *pWriter){
- int i, rc = SQLITE_OK;
- DataBuffer doclist, merged, tmp;
-
- /* Order the readers. */
- i = nReaders;
- while( i-- > 0 ){
- optLeavesReaderReorder(&readers[i], nReaders-i);
- }
-
- dataBufferInit(&doclist, LEAF_MAX);
- dataBufferInit(&merged, LEAF_MAX);
-
- /* Exhausted readers bubble to the end, so when the first reader is
- ** at eof, all are at eof.
- */
- while( !optLeavesReaderAtEnd(&readers[0]) ){
-
- /* Figure out how many readers share the next term. */
- for(i=1; i<nReaders && !optLeavesReaderAtEnd(&readers[i]); i++){
- if( 0!=optLeavesReaderTermCmp(&readers[0], &readers[i]) ) break;
- }
-
- /* Special-case for no merge. */
- if( i==1 ){
- /* Trim deletions from the doclist. */
- dataBufferReset(&merged);
- docListTrim(DL_DEFAULT,
- optLeavesReaderData(&readers[0]),
- optLeavesReaderDataBytes(&readers[0]),
- -1, DL_DEFAULT, &merged);
- }else{
- DLReader dlReaders[MERGE_COUNT];
- int iReader, nReaders;
-
- /* Prime the pipeline with the first reader's doclist. After
- ** one pass index 0 will reference the accumulated doclist.
- */
- dlrInit(&dlReaders[0], DL_DEFAULT,
- optLeavesReaderData(&readers[0]),
- optLeavesReaderDataBytes(&readers[0]));
- iReader = 1;
-
- assert( iReader<i ); /* Must execute the loop at least once. */
- while( iReader<i ){
- /* Merge 16 inputs per pass. */
- for( nReaders=1; iReader<i && nReaders<MERGE_COUNT;
- iReader++, nReaders++ ){
- dlrInit(&dlReaders[nReaders], DL_DEFAULT,
- optLeavesReaderData(&readers[iReader]),
- optLeavesReaderDataBytes(&readers[iReader]));
- }
-
- /* Merge doclists and swap result into accumulator. */
- dataBufferReset(&merged);
- docListMerge(&merged, dlReaders, nReaders);
- tmp = merged;
- merged = doclist;
- doclist = tmp;
-
- while( nReaders-- > 0 ){
- dlrDestroy(&dlReaders[nReaders]);
- }
-
- /* Accumulated doclist to reader 0 for next pass. */
- dlrInit(&dlReaders[0], DL_DEFAULT, doclist.pData, doclist.nData);
- }
-
- /* Destroy reader that was left in the pipeline. */
- dlrDestroy(&dlReaders[0]);
-
- /* Trim deletions from the doclist. */
- dataBufferReset(&merged);
- docListTrim(DL_DEFAULT, doclist.pData, doclist.nData,
- -1, DL_DEFAULT, &merged);
- }
-
- /* Only pass doclists with hits (skip if all hits deleted). */
- if( merged.nData>0 ){
- rc = leafWriterStep(v, pWriter,
- optLeavesReaderTerm(&readers[0]),
- optLeavesReaderTermBytes(&readers[0]),
- merged.pData, merged.nData);
- if( rc!=SQLITE_OK ) goto err;
- }
-
- /* Step merged readers to next term and reorder. */
- while( i-- > 0 ){
- rc = optLeavesReaderStep(v, &readers[i]);
- if( rc!=SQLITE_OK ) goto err;
-
- optLeavesReaderReorder(&readers[i], nReaders-i);
- }
- }
-
- err:
- dataBufferDestroy(&doclist);
- dataBufferDestroy(&merged);
- return rc;
-}
-
-/* Implement optimize() function for FTS3. optimize(t) merges all
-** segments in the fts index into a single segment. 't' is the magic
-** table-named column.
-*/
-static void optimizeFunc(sqlite3_context *pContext,
- int argc, sqlite3_value **argv){
- fulltext_cursor *pCursor;
- if( argc>1 ){
- sqlite3_result_error(pContext, "excess arguments to optimize()",-1);
- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- sqlite3_result_error(pContext, "illegal first argument to optimize",-1);
- }else{
- fulltext_vtab *v;
- int i, rc, iMaxLevel;
- OptLeavesReader *readers;
- int nReaders;
- LeafWriter writer;
- sqlite3_stmt *s;
-
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- v = cursor_vtab(pCursor);
-
- /* Flush any buffered updates before optimizing. */
- rc = flushPendingTerms(v);
- if( rc!=SQLITE_OK ) goto err;
-
- rc = segdir_count(v, &nReaders, &iMaxLevel);
- if( rc!=SQLITE_OK ) goto err;
- if( nReaders==0 || nReaders==1 ){
- sqlite3_result_text(pContext, "Index already optimal", -1,
- SQLITE_STATIC);
- return;
- }
-
- rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
- if( rc!=SQLITE_OK ) goto err;
-
- readers = sqlite3_malloc(nReaders*sizeof(readers[0]));
- if( readers==NULL ) goto err;
-
- /* Note that there will already be a segment at this position
- ** until we call segdir_delete() on iMaxLevel.
- */
- leafWriterInit(iMaxLevel, 0, &writer);
-
- i = 0;
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- sqlite_int64 iStart = sqlite3_column_int64(s, 0);
- sqlite_int64 iEnd = sqlite3_column_int64(s, 1);
- const char *pRootData = sqlite3_column_blob(s, 2);
- int nRootData = sqlite3_column_bytes(s, 2);
-
- assert( i<nReaders );
- rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
- &readers[i].reader);
- if( rc!=SQLITE_OK ) break;
-
- readers[i].segment = i;
- i++;
- }
-
- /* If we managed to successfully read them all, optimize them. */
- if( rc==SQLITE_DONE ){
- assert( i==nReaders );
- rc = optimizeInternal(v, readers, nReaders, &writer);
- }
-
- while( i-- > 0 ){
- leavesReaderDestroy(&readers[i].reader);
- }
- sqlite3_free(readers);
-
- /* If we've successfully gotten to here, delete the old segments
- ** and flush the interior structure of the new segment.
- */
- if( rc==SQLITE_OK ){
- for( i=0; i<=iMaxLevel; i++ ){
- rc = segdir_delete(v, i);
- if( rc!=SQLITE_OK ) break;
- }
-
- if( rc==SQLITE_OK ) rc = leafWriterFinalize(v, &writer);
- }
-
- leafWriterDestroy(&writer);
-
- if( rc!=SQLITE_OK ) goto err;
-
- sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC);
- return;
-
- /* TODO(shess): Error-handling needs to be improved along the
- ** lines of the dump_ functions.
- */
- err:
- {
- char buf[512];
- sqlite3_snprintf(sizeof(buf), buf, "Error in optimize: %s",
- sqlite3_errmsg(sqlite3_context_db_handle(pContext)));
- sqlite3_result_error(pContext, buf, -1);
- }
- }
-}
-
-#ifdef SQLITE_TEST
-/* Generate an error of the form "<prefix>: <msg>". If msg is NULL,
-** pull the error from the context's db handle.
-*/
-static void generateError(sqlite3_context *pContext,
- const char *prefix, const char *msg){
- char buf[512];
- if( msg==NULL ) msg = sqlite3_errmsg(sqlite3_context_db_handle(pContext));
- sqlite3_snprintf(sizeof(buf), buf, "%s: %s", prefix, msg);
- sqlite3_result_error(pContext, buf, -1);
-}
-
-/* Helper function to collect the set of terms in the segment into
-** pTerms. The segment is defined by the leaf nodes between
-** iStartBlockid and iEndBlockid, inclusive, or by the contents of
-** pRootData if iStartBlockid is 0 (in which case the entire segment
-** fit in a leaf).
-*/
-static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s,
- fts3Hash *pTerms){
- const sqlite_int64 iStartBlockid = sqlite3_column_int64(s, 0);
- const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
- const char *pRootData = sqlite3_column_blob(s, 2);
- const int nRootData = sqlite3_column_bytes(s, 2);
- LeavesReader reader;
- int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
- pRootData, nRootData, &reader);
- if( rc!=SQLITE_OK ) return rc;
-
- while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
- const char *pTerm = leavesReaderTerm(&reader);
- const int nTerm = leavesReaderTermBytes(&reader);
- void *oldValue = sqlite3Fts3HashFind(pTerms, pTerm, nTerm);
- void *newValue = (void *)((char *)oldValue+1);
-
- /* From the comment before sqlite3Fts3HashInsert in fts3_hash.c,
- ** the data value passed is returned in case of malloc failure.
- */
- if( newValue==sqlite3Fts3HashInsert(pTerms, pTerm, nTerm, newValue) ){
- rc = SQLITE_NOMEM;
- }else{
- rc = leavesReaderStep(v, &reader);
- }
- }
-
- leavesReaderDestroy(&reader);
- return rc;
-}
-
-/* Helper function to build the result string for dump_terms(). */
-static int generateTermsResult(sqlite3_context *pContext, fts3Hash *pTerms){
- int iTerm, nTerms, nResultBytes, iByte;
- char *result;
- TermData *pData;
- fts3HashElem *e;
-
- /* Iterate pTerms to generate an array of terms in pData for
- ** sorting.
- */
- nTerms = fts3HashCount(pTerms);
- assert( nTerms>0 );
- pData = sqlite3_malloc(nTerms*sizeof(TermData));
- if( pData==NULL ) return SQLITE_NOMEM;
-
- nResultBytes = 0;
- for(iTerm = 0, e = fts3HashFirst(pTerms); e; iTerm++, e = fts3HashNext(e)){
- nResultBytes += fts3HashKeysize(e)+1; /* Term plus trailing space */
- assert( iTerm<nTerms );
- pData[iTerm].pTerm = fts3HashKey(e);
- pData[iTerm].nTerm = fts3HashKeysize(e);
- pData[iTerm].pCollector = fts3HashData(e); /* unused */
- }
- assert( iTerm==nTerms );
-
- assert( nResultBytes>0 ); /* nTerms>0, nResultsBytes must be, too. */
- result = sqlite3_malloc(nResultBytes);
- if( result==NULL ){
- sqlite3_free(pData);
- return SQLITE_NOMEM;
- }
-
- if( nTerms>1 ) qsort(pData, nTerms, sizeof(*pData), termDataCmp);
-
- /* Read the terms in order to build the result. */
- iByte = 0;
- for(iTerm=0; iTerm<nTerms; ++iTerm){
- memcpy(result+iByte, pData[iTerm].pTerm, pData[iTerm].nTerm);
- iByte += pData[iTerm].nTerm;
- result[iByte++] = ' ';
- }
- assert( iByte==nResultBytes );
- assert( result[nResultBytes-1]==' ' );
- result[nResultBytes-1] = '\0';
-
- /* Passes away ownership of result. */
- sqlite3_result_text(pContext, result, nResultBytes-1, sqlite3_free);
- sqlite3_free(pData);
- return SQLITE_OK;
-}
-
-/* Implements dump_terms() for use in inspecting the fts3 index from
-** tests. TEXT result containing the ordered list of terms joined by
-** spaces. dump_terms(t, level, idx) dumps the terms for the segment
-** specified by level, idx (in %_segdir), while dump_terms(t) dumps
-** all terms in the index. In both cases t is the fts table's magic
-** table-named column.
-*/
-static void dumpTermsFunc(
- sqlite3_context *pContext,
- int argc, sqlite3_value **argv
-){
- fulltext_cursor *pCursor;
- if( argc!=3 && argc!=1 ){
- generateError(pContext, "dump_terms", "incorrect arguments");
- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- generateError(pContext, "dump_terms", "illegal first argument");
- }else{
- fulltext_vtab *v;
- fts3Hash terms;
- sqlite3_stmt *s = NULL;
- int rc;
-
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- v = cursor_vtab(pCursor);
-
- /* If passed only the cursor column, get all segments. Otherwise
- ** get the segment described by the following two arguments.
- */
- if( argc==1 ){
- rc = sql_get_statement(v, SEGDIR_SELECT_ALL_STMT, &s);
- }else{
- rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[1]));
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[2]));
- }
- }
- }
-
- if( rc!=SQLITE_OK ){
- generateError(pContext, "dump_terms", NULL);
- return;
- }
-
- /* Collect the terms for each segment. */
- sqlite3Fts3HashInit(&terms, FTS3_HASH_STRING, 1);
- while( (rc = sqlite3_step(s))==SQLITE_ROW ){
- rc = collectSegmentTerms(v, s, &terms);
- if( rc!=SQLITE_OK ) break;
- }
-
- if( rc!=SQLITE_DONE ){
- sqlite3_reset(s);
- generateError(pContext, "dump_terms", NULL);
- }else{
- const int nTerms = fts3HashCount(&terms);
- if( nTerms>0 ){
- rc = generateTermsResult(pContext, &terms);
- if( rc==SQLITE_NOMEM ){
- generateError(pContext, "dump_terms", "out of memory");
- }else{
- assert( rc==SQLITE_OK );
- }
- }else if( argc==3 ){
- /* The specific segment asked for could not be found. */
- generateError(pContext, "dump_terms", "segment not found");
- }else{
- /* No segments found. */
- /* TODO(shess): It should be impossible to reach this. This
- ** case can only happen for an empty table, in which case
- ** SQLite has no rows to call this function on.
- */
- sqlite3_result_null(pContext);
- }
- }
- sqlite3Fts3HashClear(&terms);
- }
-}
-
-/* Expand the DL_DEFAULT doclist in pData into a text result in
-** pContext.
-*/
-static void createDoclistResult(sqlite3_context *pContext,
- const char *pData, int nData){
- DataBuffer dump;
- DLReader dlReader;
-
- assert( pData!=NULL && nData>0 );
-
- dataBufferInit(&dump, 0);
- dlrInit(&dlReader, DL_DEFAULT, pData, nData);
- for( ; !dlrAtEnd(&dlReader); dlrStep(&dlReader) ){
- char buf[256];
- PLReader plReader;
-
- plrInit(&plReader, &dlReader);
- if( DL_DEFAULT==DL_DOCIDS || plrAtEnd(&plReader) ){
- sqlite3_snprintf(sizeof(buf), buf, "[%lld] ", dlrDocid(&dlReader));
- dataBufferAppend(&dump, buf, strlen(buf));
- }else{
- int iColumn = plrColumn(&plReader);
-
- sqlite3_snprintf(sizeof(buf), buf, "[%lld %d[",
- dlrDocid(&dlReader), iColumn);
- dataBufferAppend(&dump, buf, strlen(buf));
-
- for( ; !plrAtEnd(&plReader); plrStep(&plReader) ){
- if( plrColumn(&plReader)!=iColumn ){
- iColumn = plrColumn(&plReader);
- sqlite3_snprintf(sizeof(buf), buf, "] %d[", iColumn);
- assert( dump.nData>0 );
- dump.nData--; /* Overwrite trailing space. */
- assert( dump.pData[dump.nData]==' ');
- dataBufferAppend(&dump, buf, strlen(buf));
- }
- if( DL_DEFAULT==DL_POSITIONS_OFFSETS ){
- sqlite3_snprintf(sizeof(buf), buf, "%d,%d,%d ",
- plrPosition(&plReader),
- plrStartOffset(&plReader), plrEndOffset(&plReader));
- }else if( DL_DEFAULT==DL_POSITIONS ){
- sqlite3_snprintf(sizeof(buf), buf, "%d ", plrPosition(&plReader));
- }else{
- assert( NULL=="Unhandled DL_DEFAULT value");
- }
- dataBufferAppend(&dump, buf, strlen(buf));
- }
- plrDestroy(&plReader);
-
- assert( dump.nData>0 );
- dump.nData--; /* Overwrite trailing space. */
- assert( dump.pData[dump.nData]==' ');
- dataBufferAppend(&dump, "]] ", 3);
- }
- }
- dlrDestroy(&dlReader);
-
- assert( dump.nData>0 );
- dump.nData--; /* Overwrite trailing space. */
- assert( dump.pData[dump.nData]==' ');
- dump.pData[dump.nData] = '\0';
- assert( dump.nData>0 );
-
- /* Passes ownership of dump's buffer to pContext. */
- sqlite3_result_text(pContext, dump.pData, dump.nData, sqlite3_free);
- dump.pData = NULL;
- dump.nData = dump.nCapacity = 0;
-}
-
-/* Implements dump_doclist() for use in inspecting the fts3 index from
-** tests. TEXT result containing a string representation of the
-** doclist for the indicated term. dump_doclist(t, term, level, idx)
-** dumps the doclist for term from the segment specified by level, idx
-** (in %_segdir), while dump_doclist(t, term) dumps the logical
-** doclist for the term across all segments. The per-segment doclist
-** can contain deletions, while the full-index doclist will not
-** (deletions are omitted).
-**
-** Result formats differ with the setting of DL_DEFAULTS. Examples:
-**
-** DL_DOCIDS: [1] [3] [7]
-** DL_POSITIONS: [1 0[0 4] 1[17]] [3 1[5]]
-** DL_POSITIONS_OFFSETS: [1 0[0,0,3 4,23,26] 1[17,102,105]] [3 1[5,20,23]]
-**
-** In each case the number after the outer '[' is the docid. In the
-** latter two cases, the number before the inner '[' is the column
-** associated with the values within. For DL_POSITIONS the numbers
-** within are the positions, for DL_POSITIONS_OFFSETS they are the
-** position, the start offset, and the end offset.
-*/
-static void dumpDoclistFunc(
- sqlite3_context *pContext,
- int argc, sqlite3_value **argv
-){
- fulltext_cursor *pCursor;
- if( argc!=2 && argc!=4 ){
- generateError(pContext, "dump_doclist", "incorrect arguments");
- }else if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ||
- sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){
- generateError(pContext, "dump_doclist", "illegal first argument");
- }else if( sqlite3_value_text(argv[1])==NULL ||
- sqlite3_value_text(argv[1])[0]=='\0' ){
- generateError(pContext, "dump_doclist", "empty second argument");
- }else{
- const char *pTerm = (const char *)sqlite3_value_text(argv[1]);
- const int nTerm = strlen(pTerm);
- fulltext_vtab *v;
- int rc;
- DataBuffer doclist;
-
- memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor));
- v = cursor_vtab(pCursor);
-
- dataBufferInit(&doclist, 0);
-
- /* termSelect() yields the same logical doclist that queries are
- ** run against.
- */
- if( argc==2 ){
- rc = termSelect(v, v->nColumn, pTerm, nTerm, 0, DL_DEFAULT, &doclist);
- }else{
- sqlite3_stmt *s = NULL;
-
- /* Get our specific segment's information. */
- rc = sql_get_statement(v, SEGDIR_SELECT_SEGMENT_STMT, &s);
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 1, sqlite3_value_int(argv[2]));
- if( rc==SQLITE_OK ){
- rc = sqlite3_bind_int(s, 2, sqlite3_value_int(argv[3]));
- }
- }
-
- if( rc==SQLITE_OK ){
- rc = sqlite3_step(s);
-
- if( rc==SQLITE_DONE ){
- dataBufferDestroy(&doclist);
- generateError(pContext, "dump_doclist", "segment not found");
- return;
- }
-
- /* Found a segment, load it into doclist. */
- if( rc==SQLITE_ROW ){
- const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
- const char *pData = sqlite3_column_blob(s, 2);
- const int nData = sqlite3_column_bytes(s, 2);
-
- /* loadSegment() is used by termSelect() to load each
- ** segment's data.
- */
- rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, 0,
- &doclist);
- if( rc==SQLITE_OK ){
- rc = sqlite3_step(s);
-
- /* Should not have more than one matching segment. */
- if( rc!=SQLITE_DONE ){
- sqlite3_reset(s);
- dataBufferDestroy(&doclist);
- generateError(pContext, "dump_doclist", "invalid segdir");
- return;
- }
- rc = SQLITE_OK;
- }
- }
- }
-
- sqlite3_reset(s);
- }
-
- if( rc==SQLITE_OK ){
- if( doclist.nData>0 ){
- createDoclistResult(pContext, doclist.pData, doclist.nData);
- }else{
- /* TODO(shess): This can happen if the term is not present, or
- ** if all instances of the term have been deleted and this is
- ** an all-index dump. It may be interesting to distinguish
- ** these cases.
- */
- sqlite3_result_text(pContext, "", 0, SQLITE_STATIC);
- }
- }else if( rc==SQLITE_NOMEM ){
- /* Handle out-of-memory cases specially because if they are
- ** generated in fts3 code they may not be reflected in the db
- ** handle.
- */
- /* TODO(shess): Handle this more comprehensively.
- ** sqlite3ErrStr() has what I need, but is internal.
- */
- generateError(pContext, "dump_doclist", "out of memory");
- }else{
- generateError(pContext, "dump_doclist", NULL);
- }
-
- dataBufferDestroy(&doclist);
- }
-}
-#endif
-
-/*
-** This routine implements the xFindFunction method for the FTS3
-** virtual table.
-*/
-static int fulltextFindFunction(
- sqlite3_vtab *pVtab,
- int nArg,
- const char *zName,
- void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
- void **ppArg
-){
- if( strcmp(zName,"snippet")==0 ){
- *pxFunc = snippetFunc;
- return 1;
- }else if( strcmp(zName,"offsets")==0 ){
- *pxFunc = snippetOffsetsFunc;
- return 1;
- }else if( strcmp(zName,"optimize")==0 ){
- *pxFunc = optimizeFunc;
- return 1;
-#ifdef SQLITE_TEST
- /* NOTE(shess): These functions are present only for testing
- ** purposes. No particular effort is made to optimize their
- ** execution or how they build their results.
- */
- }else if( strcmp(zName,"dump_terms")==0 ){
- /* fprintf(stderr, "Found dump_terms\n"); */
- *pxFunc = dumpTermsFunc;
- return 1;
- }else if( strcmp(zName,"dump_doclist")==0 ){
- /* fprintf(stderr, "Found dump_doclist\n"); */
- *pxFunc = dumpDoclistFunc;
- return 1;
-#endif
- }
- return 0;
-}
-
-/*
-** Rename an fts3 table.
-*/
-static int fulltextRename(
- sqlite3_vtab *pVtab,
- const char *zName
-){
- fulltext_vtab *p = (fulltext_vtab *)pVtab;
- int rc = SQLITE_NOMEM;
- char *zSql = sqlite3_mprintf(
- "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';"
- "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';"
- "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';"
- , p->zDb, p->zName, zName
- , p->zDb, p->zName, zName
- , p->zDb, p->zName, zName
- );
- if( zSql ){
- rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
- sqlite3_free(zSql);
- }
- return rc;
-}
-
-static const sqlite3_module fts3Module = {
- /* iVersion */ 0,
- /* xCreate */ fulltextCreate,
- /* xConnect */ fulltextConnect,
- /* xBestIndex */ fulltextBestIndex,
- /* xDisconnect */ fulltextDisconnect,
- /* xDestroy */ fulltextDestroy,
- /* xOpen */ fulltextOpen,
- /* xClose */ fulltextClose,
- /* xFilter */ fulltextFilter,
- /* xNext */ fulltextNext,
- /* xEof */ fulltextEof,
- /* xColumn */ fulltextColumn,
- /* xRowid */ fulltextRowid,
- /* xUpdate */ fulltextUpdate,
- /* xBegin */ fulltextBegin,
- /* xSync */ fulltextSync,
- /* xCommit */ fulltextCommit,
- /* xRollback */ fulltextRollback,
- /* xFindFunction */ fulltextFindFunction,
- /* xRename */ fulltextRename,
-};
-
-static void hashDestroy(void *p){
- fts3Hash *pHash = (fts3Hash *)p;
- sqlite3Fts3HashClear(pHash);
- sqlite3_free(pHash);
-}
-
-/*
-** The fts3 built-in tokenizers - "simple" and "porter" - are implemented
-** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following
-** two forward declarations are for functions declared in these files
-** used to retrieve the respective implementations.
-**
-** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
-** to by the argument to point a the "simple" tokenizer implementation.
-** Function ...PorterTokenizerModule() sets *pModule to point to the
-** porter tokenizer/stemmer implementation.
-*/
-SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
-SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
-SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
-
-SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, fts3Hash *, const char *);
-
-/*
-** Initialise the fts3 extension. If this extension is built as part
-** of the sqlite library, then this function is called directly by
-** SQLite. If fts3 is built as a dynamically loadable extension, this
-** function is called by the sqlite3_extension_init() entry point.
-*/
-SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){
- int rc = SQLITE_OK;
- fts3Hash *pHash = 0;
- const sqlite3_tokenizer_module *pSimple = 0;
- const sqlite3_tokenizer_module *pPorter = 0;
- const sqlite3_tokenizer_module *pIcu = 0;
-
- sqlite3Fts3SimpleTokenizerModule(&pSimple);
- sqlite3Fts3PorterTokenizerModule(&pPorter);
-#ifdef SQLITE_ENABLE_ICU
- sqlite3Fts3IcuTokenizerModule(&pIcu);
-#endif
-
- /* Allocate and initialise the hash-table used to store tokenizers. */
- pHash = sqlite3_malloc(sizeof(fts3Hash));
- if( !pHash ){
- rc = SQLITE_NOMEM;
- }else{
- sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
- }
-
- /* Load the built-in tokenizers into the hash table */
- if( rc==SQLITE_OK ){
- if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
- || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter)
- || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
- ){
- rc = SQLITE_NOMEM;
- }
- }
-
-#ifdef SQLITE_TEST
- sqlite3Fts3ExprInitTestInterface(db);
-#endif
-
- /* Create the virtual table wrapper around the hash-table and overload
- ** the two scalar functions. If this is successful, register the
- ** module with sqlite.
- */
- if( SQLITE_OK==rc
- && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", -1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", -1))
-#ifdef SQLITE_TEST
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_terms", -1))
- && SQLITE_OK==(rc = sqlite3_overload_function(db, "dump_doclist", -1))
-#endif
- ){
- return sqlite3_create_module_v2(
- db, "fts3", &fts3Module, (void *)pHash, hashDestroy
- );
- }
-
- /* An error has occurred. Delete the hash table and return the error code. */
- assert( rc!=SQLITE_OK );
- if( pHash ){
- sqlite3Fts3HashClear(pHash);
- sqlite3_free(pHash);
- }
- return rc;
-}
-
-#if !SQLITE_CORE
-SQLITE_API int sqlite3_extension_init(
- sqlite3 *db,
- char **pzErrMsg,
- const sqlite3_api_routines *pApi
-){
- SQLITE_EXTENSION_INIT2(pApi)
- return sqlite3Fts3Init(db);
-}
-#endif
-
-#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
-
-/************** End of fts3.c ************************************************/
-/************** Begin file fts3_expr.c ***************************************/
-/*
-** 2008 Nov 28
-**
-** The author disclaims copyright to this source code. In place of
-** a legal notice, here is a blessing:
-**
-** May you do good and not evil.
-** May you find forgiveness for yourself and forgive others.
-** May you share freely, never taking more than you give.
-**
-******************************************************************************
-**
-** This module contains code that implements a parser for fts3 query strings
-** (the right-hand argument to the MATCH operator). Because the supported
-** syntax is relatively simple, the whole tokenizer/parser system is
-** hand-coded. The public interface to this module is declared in source
-** code file "fts3_expr.h".
-*/
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
-
-/*
-** By default, this module parses the legacy syntax that has been
-** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS
-** is defined, then it uses the new syntax. The differences between
-** the new and the old syntaxes are:
-**
-** a) The new syntax supports parenthesis. The old does not.
-**
-** b) The new syntax supports the AND and NOT operators. The old does not.
-**
-** c) The old syntax supports the "-" token qualifier. This is not
-** supported by the new syntax (it is replaced by the NOT operator).
-**
-** d) When using the old syntax, the OR operator has a greater precedence
-** than an implicit AND. When using the new, both implicity and explicit
-** AND operators have a higher precedence than OR.
-**
-** If compiled with SQLITE_TEST defined, then this module exports the
-** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable
-** to zero causes the module to use the old syntax. If it is set to
-** non-zero the new syntax is activated. This is so both syntaxes can
-** be tested using a single build of testfixture.
-*/
-#ifdef SQLITE_TEST
-SQLITE_API int sqlite3_fts3_enable_parentheses = 0;
-#else
-# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS
-# define sqlite3_fts3_enable_parentheses 1
-# else
-# define sqlite3_fts3_enable_parentheses 0
-# endif
-#endif
-
-/*
-** Default span for NEAR operators.
-*/
-#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10
-
-
-typedef struct ParseContext ParseContext;
-struct ParseContext {
- sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
- const char **azCol; /* Array of column names for fts3 table */
- int nCol; /* Number of entries in azCol[] */
- int iDefaultCol; /* Default column to query */
- sqlite3_context *pCtx; /* Write error message here */
- int nNest; /* Number of nested brackets */
-};
-
-/*
-** This function is equivalent to the standard isspace() function.
-**
-** The standard isspace() can be awkward to use safely, because although it
-** is defined to accept an argument of type int, its behaviour when passed
-** an integer that falls outside of the range of the unsigned char type
-** is undefined (and sometimes, "undefined" means segfault). This wrapper
-** is defined to accept an argument of type char, and always returns 0 for
-** any values that fall outside of the range of the unsigned char type (i.e.
-** negative values).
-*/
-static int fts3isspace(char c){
- return (c&0x80)==0 ? isspace(c) : 0;
-}
-
-/*
-** Extract the next token from buffer z (length n) using the tokenizer
-** and other information (column names etc.) in pParse. Create an Fts3Expr
-** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
-** single token and set *ppExpr to point to it. If the end of the buffer is
-** reached before a token is found, set *ppExpr to zero. It is the
-** responsibility of the caller to eventually deallocate the allocated
-** Fts3Expr structure (if any) by passing it to sqlite3_free().
-**
-** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation
-** fails.
-*/
-static int getNextToken(
- ParseContext *pParse, /* fts3 query parse context */
- int iCol, /* Value for Fts3Phrase.iColumn */
- const char *z, int n, /* Input string */
- Fts3Expr **ppExpr, /* OUT: expression */
- int *pnConsumed /* OUT: Number of bytes consumed */
-){
- sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
- sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
- int rc;
- sqlite3_tokenizer_cursor *pCursor;
- Fts3Expr *pRet = 0;
- int nConsumed = 0;
-
- rc = pModule->xOpen(pTokenizer, z, n, &pCursor);
- if( rc==SQLITE_OK ){
- const char *zToken;
- int nToken, iStart, iEnd, iPosition;
- int nByte; /* total space to allocate */
-
- pCursor->pTokenizer = pTokenizer;
- rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);
-
- if( rc==SQLITE_OK ){
- nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
- pRet = (Fts3Expr *)sqlite3_malloc(nByte);
- if( !pRet ){
- rc = SQLITE_NOMEM;
- }else{
- memset(pRet, 0, nByte);
- pRet->eType = FTSQUERY_PHRASE;
- pRet->pPhrase = (Fts3Phrase *)&pRet[1];
- pRet->pPhrase->nToken = 1;
- pRet->pPhrase->iColumn = iCol;
- pRet->pPhrase->aToken[0].n = nToken;
- pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1];
- memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken);
-
- if( iEnd<n && z[iEnd]=='*' ){
- pRet->pPhrase->aToken[0].isPrefix = 1;
- iEnd++;
- }
- if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
- pRet->pPhrase->isNot = 1;
- }
- }
- nConsumed = iEnd;
- }
-
- pModule->xClose(pCursor);
- }
-
- *pnConsumed = nConsumed;
- *ppExpr = pRet;
- return rc;
-}
-
-
-/*
-** Enlarge a memory allocation. If an out-of-memory allocation occurs,
-** then free the old allocation.
-*/
-void *fts3ReallocOrFree(void *pOrig, int nNew){
- void *pRet = sqlite3_realloc(pOrig, nNew);
- if( !pRet ){
- sqlite3_free(pOrig);
- }
- return pRet;
-}
-
-/*
-** Buffer zInput, length nInput, contains the contents of a quoted string
-** that appeared as part of an fts3 query expression. Neither quote character
-** is included in the buffer. This function attempts to tokenize the entire
-** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE
-** containing the results.
-**
-** If successful, SQLITE_OK is returned and *ppExpr set to point at the
-** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory
-** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set
-** to 0.
-*/
-static int getNextString(
- ParseContext *pParse, /* fts3 query parse context */
- const char *zInput, int nInput, /* Input string */
- Fts3Expr **ppExpr /* OUT: expression */
-){
- sqlite3_tokenizer *pTokenizer = pParse->pTokenizer;
- sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
- int rc;
- Fts3Expr *p = 0;
- sqlite3_tokenizer_cursor *pCursor = 0;
- char *zTemp = 0;
- int nTemp = 0;
-
- rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
- if( rc==SQLITE_OK ){
- int ii;
- pCursor->pTokenizer = pTokenizer;
- for(ii=0; rc==SQLITE_OK; ii++){
- const char *zToken;
- int nToken, iBegin, iEnd, iPos;
- rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
- if( rc==SQLITE_OK ){
- int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
- p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken));
- zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
- if( !p || !zTemp ){
- goto no_mem;
- }
- if( ii==0 ){
- memset(p, 0, nByte);
- p->pPhrase = (Fts3Phrase *)&p[1];
- }
- p->pPhrase = (Fts3Phrase *)&p[1];
- p->pPhrase->nToken = ii+1;
- p->pPhrase->aToken[ii].n = nToken;
- memcpy(&zTemp[nTemp], zToken, nToken);
- nTemp += nToken;
- if( iEnd<nInput && zInput[iEnd]=='*' ){
- p->pPhrase->aToken[ii].isPrefix = 1;
- }else{
- p->pPhrase->aToken[ii].isPrefix = 0;
- }
- }
- }
-
- pModule->xClose(pCursor);
- pCursor = 0;
- }
-
- if( rc==SQLITE_DONE ){
- int jj;
- char *zNew;
- int nNew = 0;
- int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
- nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(struct PhraseToken);
- p = fts3ReallocOrFree(p, nByte + nTemp);
- if( !p ){
- goto no_mem;
- }
- if( zTemp ){
- zNew = &(((char *)p)[nByte]);
- memcpy(zNew, zTemp, nTemp);
- }else{
- memset(p, 0, nByte+nTemp);
- }
- p->pPhrase = (Fts3Phrase *)&p[1];
- for(jj=0; jj<p->pPhrase->nToken; jj++){
- p->pPhrase->aToken[jj].z = &zNew[nNew];
- nNew += p->pPhrase->aToken[jj].n;
- }
- sqlite3_free(zTemp);
- p->eType = FTSQUERY_PHRASE;
- p->pPhrase->iColumn = pParse->iDefaultCol;
- rc = SQLITE_OK;
- }
-
- *ppExpr = p;
- return rc;
-no_mem:
-
- if( pCursor ){
- pModule->xClose(pCursor);
- }
- sqlite3_free(zTemp);
- sqlite3_free(p);
- *ppExpr = 0;
- return SQLITE_NOMEM;
-}
-
-/*
-** Function getNextNode(), which is called by fts3ExprParse(), may itself
-** call fts3ExprParse(). So this forward declaration is required.
-*/
-static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);
-
-/*
-** The output variable *ppExpr is populated with an allocated Fts3Expr
-** structure, or set to 0 if the end of the input buffer is reached.
-**
-** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
-** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
-** If SQLITE_ERROR is returned, pContext is populated with an error message.
-*/
-static int getNextNode(
- ParseContext *pParse, /* fts3 query parse context */
- const char *z, int n, /* Input string */
- Fts3Expr **ppExpr, /* OUT: expression */
- int *pnConsumed /* OUT: Number of bytes consumed */
-){
- static const struct Fts3Keyword {
- char z[4]; /* Keyword text */
- unsigned char n; /* Length of the keyword */
- unsigned char parenOnly; /* Only valid in paren mode */
- unsigned char eType; /* Keyword code */
- } aKeyword[] = {
- { "OR" , 2, 0, FTSQUERY_OR },
- { "AND", 3, 1, FTSQUERY_AND },
- { "NOT", 3, 1, FTSQUERY_NOT },
- { "NEAR", 4, 0, FTSQUERY_NEAR }
- };
- int ii;
- int iCol;
- int iColLen;
- int rc;
- Fts3Expr *pRet = 0;
-
- const char *zInput = z;
- int nInput = n;
-
- /* Skip over any whitespace before checking for a keyword, an open or
- ** close bracket, or a quoted string.
- */
- while( nInput>0 && fts3isspace(*zInput) ){
- nInput--;
- zInput++;
- }
- if( nInput==0 ){
- return SQLITE_DONE;
- }
-
- /* See if we are dealing with a keyword. */
- for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){
- const struct Fts3Keyword *pKey = &aKeyword[ii];
-
- if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){
- continue;
- }
-
- if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){
- int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM;
- int nKey = pKey->n;
- char cNext;
-
- /* If this is a "NEAR" keyword, check for an explicit nearness. */
- if( pKey->eType==FTSQUERY_NEAR ){
- assert( nKey==4 );
- if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){
- nNear = 0;
- for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){
- nNear = nNear * 10 + (zInput[nKey] - '0');
- }
- }
- }
-
- /* At this point this is probably a keyword. But for that to be true,
- ** the next byte must contain either whitespace, an open or close
- ** parenthesis, a quote character, or EOF.
- */
- cNext = zInput[nKey];
- if( fts3isspace(cNext)
- || cNext=='"' || cNext=='(' || cNext==')' || cNext==0
- ){
pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr));
+ if( !pRet ){
+ return SQLITE_NOMEM;
+ }
memset(pRet, 0, sizeof(Fts3Expr));
pRet->eType = pKey->eType;
pRet->nNear = nNear;
*ppExpr = pRet;
*pnConsumed = (zInput - z) + nKey;
@@ -105643,10 +100674,125 @@
** SQLite (in which case SQLITE_ENABLE_FTS3 is defined).
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+/************** Include fts3_hash.h in the middle of fts3_hash.c *************/
+/************** Begin file fts3_hash.h ***************************************/
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the header file for the generic hash-table implemenation
+** used in SQLite. We've modified it slightly to serve as a standalone
+** hash table implementation for the full-text indexing module.
+**
+*/
+#ifndef _FTS3_HASH_H_
+#define _FTS3_HASH_H_
+
+/* Forward declarations of structures. */
+typedef struct Fts3Hash Fts3Hash;
+typedef struct Fts3HashElem Fts3HashElem;
+
+/* A complete hash table is an instance of the following structure.
+** The internals of this structure are intended to be opaque -- client
+** code should not attempt to access or modify the fields of this structure
+** directly. Change this structure only by using the routines below.
+** However, many of the "procedures" and "functions" for modifying and
+** accessing this structure are really macros, so we can't really make
+** this structure opaque.
+*/
+struct Fts3Hash {
+ char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */
+ char copyKey; /* True if copy of key made on insert */
+ int count; /* Number of entries in this table */
+ Fts3HashElem *first; /* The first element of the array */
+ int htsize; /* Number of buckets in the hash table */
+ struct _fts3ht { /* the hash table */
+ int count; /* Number of entries with this hash */
+ Fts3HashElem *chain; /* Pointer to first entry with this hash */
+ } *ht;
+};
+
+/* Each element in the hash table is an instance of the following
+** structure. All elements are stored on a single doubly-linked list.
+**
+** Again, this structure is intended to be opaque, but it can't really
+** be opaque because it is used by macros.
+*/
+struct Fts3HashElem {
+ Fts3HashElem *next, *prev; /* Next and previous elements in the table */
+ void *data; /* Data associated with this element */
+ void *pKey; int nKey; /* Key associated with this element */
+};
+
+/*
+** There are 2 different modes of operation for a hash table:
+**
+** FTS3_HASH_STRING pKey points to a string that is nKey bytes long
+** (including the null-terminator, if any). Case
+** is respected in comparisons.
+**
+** FTS3_HASH_BINARY pKey points to binary data nKey bytes long.
+** memcmp() is used to compare keys.
+**
+** A copy of the key is made if the copyKey parameter to fts3HashInit is 1.
+*/
+#define FTS3_HASH_STRING 1
+#define FTS3_HASH_BINARY 2
+
+/*
+** Access routines. To delete, insert a NULL pointer.
+*/
+SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash*, int keytype, int copyKey);
+SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData);
+SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey);
+SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*);
+
+/*
+** Shorthand for the functions above
+*/
+#define fts3HashInit sqlite3Fts3HashInit
+#define fts3HashInsert sqlite3Fts3HashInsert
+#define fts3HashFind sqlite3Fts3HashFind
+#define fts3HashClear sqlite3Fts3HashClear
+
+/*
+** Macros for looping over all elements of a hash table. The idiom is
+** like this:
+**
+** Fts3Hash h;
+** Fts3HashElem *p;
+** ...
+** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){
+** SomeStructure *pData = fts3HashData(p);
+** // do something with pData
+** }
+*/
+#define fts3HashFirst(H) ((H)->first)
+#define fts3HashNext(E) ((E)->next)
+#define fts3HashData(E) ((E)->data)
+#define fts3HashKey(E) ((E)->pKey)
+#define fts3HashKeysize(E) ((E)->nKey)
+
+/*
+** Number of entries in a hash table
+*/
+#define fts3HashCount(H) ((H)->count)
+
+#endif /* _FTS3_HASH_H_ */
+
+/************** End of fts3_hash.h *******************************************/
+/************** Continuing where we left off in fts3_hash.c ******************/
/*
** Malloc and Free functions
*/
static void *fts3HashMalloc(int n){
@@ -105668,11 +100814,11 @@
** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass
** determines what kind of key the hash table will use. "copyKey" is
** true if the hash table should make its own private copy of keys and
** false if it should just use the supplied pointer.
*/
-SQLITE_PRIVATE void sqlite3Fts3HashInit(fts3Hash *pNew, int keyClass, int copyKey){
+SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, int keyClass, int copyKey){
assert( pNew!=0 );
assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY );
pNew->keyClass = keyClass;
pNew->copyKey = copyKey;
pNew->first = 0;
@@ -105683,21 +100829,21 @@
/* Remove all entries from a hash table. Reclaim all memory.
** Call this routine to delete a hash table or to reset a hash table
** to the empty state.
*/
-SQLITE_PRIVATE void sqlite3Fts3HashClear(fts3Hash *pH){
- fts3HashElem *elem; /* For looping over all elements of the table */
+SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){
+ Fts3HashElem *elem; /* For looping over all elements of the table */
assert( pH!=0 );
elem = pH->first;
pH->first = 0;
fts3HashFree(pH->ht);
pH->ht = 0;
pH->htsize = 0;
while( elem ){
- fts3HashElem *next_elem = elem->next;
+ Fts3HashElem *next_elem = elem->next;
if( pH->copyKey && elem->pKey ){
fts3HashFree(elem->pKey);
}
fts3HashFree(elem);
elem = next_elem;
@@ -105776,15 +100922,15 @@
}
/* Link an element into the hash table
*/
static void fts3HashInsertElement(
- fts3Hash *pH, /* The complete hash table */
+ Fts3Hash *pH, /* The complete hash table */
struct _fts3ht *pEntry, /* The entry into which pNew is inserted */
- fts3HashElem *pNew /* The element to be inserted */
-){
- fts3HashElem *pHead; /* First element already in pEntry */
+ Fts3HashElem *pNew /* The element to be inserted */
+){
+ Fts3HashElem *pHead; /* First element already in pEntry */
pHead = pEntry->chain;
if( pHead ){
pNew->next = pHead;
pNew->prev = pHead->prev;
if( pHead->prev ){ pHead->prev->next = pNew; }
@@ -105803,13 +100949,13 @@
/* Resize the hash table so that it cantains "new_size" buckets.
** "new_size" must be a power of 2. The hash table might fail
** to resize if sqliteMalloc() fails.
*/
-static void fts3Rehash(fts3Hash *pH, int new_size){
+static void fts3Rehash(Fts3Hash *pH, int new_size){
struct _fts3ht *new_ht; /* The new hash table */
- fts3HashElem *elem, *next_elem; /* For looping over existing elements */
+ Fts3HashElem *elem, *next_elem; /* For looping over existing elements */
int (*xHash)(const void*,int); /* The hash function */
assert( (new_size & (new_size-1))==0 );
new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) );
if( new_ht==0 ) return;
@@ -105826,17 +100972,17 @@
/* This function (for internal use only) locates an element in an
** hash table that matches the given key. The hash for this key has
** already been computed and is passed as the 4th parameter.
*/
-static fts3HashElem *fts3FindElementByHash(
- const fts3Hash *pH, /* The pH to be searched */
+static Fts3HashElem *fts3FindElementByHash(
+ const Fts3Hash *pH, /* The pH to be searched */
const void *pKey, /* The key we are searching for */
int nKey,
int h /* The hash for this key. */
){
- fts3HashElem *elem; /* Used to loop thru the element list */
+ Fts3HashElem *elem; /* Used to loop thru the element list */
int count; /* Number of elements left to test */
int (*xCompare)(const void*,int,const void*,int); /* comparison function */
if( pH->ht ){
struct _fts3ht *pEntry = &pH->ht[h];
@@ -105855,12 +101001,12 @@
/* Remove a single entry from the hash table given a pointer to that
** element and a hash on the element's key.
*/
static void fts3RemoveElementByHash(
- fts3Hash *pH, /* The pH containing "elem" */
- fts3HashElem* elem, /* The element to be removed from the pH */
+ Fts3Hash *pH, /* The pH containing "elem" */
+ Fts3HashElem* elem, /* The element to be removed from the pH */
int h /* Hash value for the element */
){
struct _fts3ht *pEntry;
if( elem->prev ){
elem->prev->next = elem->next;
@@ -105892,13 +101038,13 @@
/* Attempt to locate an element of the hash table pH with a key
** that matches pKey,nKey. Return the data for this element if it is
** found, or NULL if there is no match.
*/
-SQLITE_PRIVATE void *sqlite3Fts3HashFind(const fts3Hash *pH, const void *pKey, int nKey){
+SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){
int h; /* A hash on key */
- fts3HashElem *elem; /* The element that matches key */
+ Fts3HashElem *elem; /* The element that matches key */
int (*xHash)(const void*,int); /* The hash function */
if( pH==0 || pH->ht==0 ) return 0;
xHash = ftsHashFunction(pH->keyClass);
assert( xHash!=0 );
@@ -105922,19 +101068,19 @@
**
** If the "data" parameter to this function is NULL, then the
** element corresponding to "key" is removed from the hash table.
*/
SQLITE_PRIVATE void *sqlite3Fts3HashInsert(
- fts3Hash *pH, /* The hash table to insert into */
+ Fts3Hash *pH, /* The hash table to insert into */
const void *pKey, /* The key */
int nKey, /* Number of bytes in the key */
void *data /* The data */
){
int hraw; /* Raw hash value of the key */
int h; /* the hash of the key modulo hash table size */
- fts3HashElem *elem; /* Used to loop thru the element list */
- fts3HashElem *new_elem; /* New element added to the pH */
+ Fts3HashElem *elem; /* Used to loop thru the element list */
+ Fts3HashElem *new_elem; /* New element added to the pH */
int (*xHash)(const void*,int); /* The hash function */
assert( pH!=0 );
xHash = ftsHashFunction(pH->keyClass);
assert( xHash!=0 );
@@ -105957,11 +101103,11 @@
if( pH->htsize==0 ){
pH->count = 0;
return data;
}
}
- new_elem = (fts3HashElem*)fts3HashMalloc( sizeof(fts3HashElem) );
+ new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) );
if( new_elem==0 ) return data;
if( pH->copyKey && pKey!=0 ){
new_elem->pKey = fts3HashMalloc( nKey );
if( new_elem->pKey==0 ){
fts3HashFree(new_elem);
@@ -106014,10 +101160,162 @@
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+/************** Include fts3_tokenizer.h in the middle of fts3_porter.c ******/
+/************** Begin file fts3_tokenizer.h **********************************/
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search. There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions. This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _FTS3_TOKENIZER_H_
+#define _FTS3_TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+
+/*
+** Structures used by the tokenizer interface. When a new tokenizer
+** implementation is registered, the caller provides a pointer to
+** an sqlite3_tokenizer_module containing pointers to the callback
+** functions that make up an implementation.
+**
+** When an fts3 table is created, it passes any arguments passed to
+** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
+** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
+** implementation. The xCreate() function in turn returns an
+** sqlite3_tokenizer structure representing the specific tokenizer to
+** be used for the fts3 table (customized by the tokenizer clause arguments).
+**
+** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
+** method is called. It returns an sqlite3_tokenizer_cursor object
+** that may be used to tokenize a specific input buffer based on
+** the tokenization rules supplied by a specific sqlite3_tokenizer
+** object.
+*/
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+
+struct sqlite3_tokenizer_module {
+
+ /*
+ ** Structure version. Should always be set to 0.
+ */
+ int iVersion;
+
+ /*
+ ** Create a new tokenizer. The values in the argv[] array are the
+ ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
+ ** TABLE statement that created the fts3 table. For example, if
+ ** the following SQL is executed:
+ **
+ ** CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2)
+ **
+ ** then argc is set to 2, and the argv[] array contains pointers
+ ** to the strings "arg1" and "arg2".
+ **
+ ** This method should return either SQLITE_OK (0), or an SQLite error
+ ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
+ ** to point at the newly created tokenizer structure. The generic
+ ** sqlite3_tokenizer.pModule variable should not be initialised by
+ ** this callback. The caller will do so.
+ */
+ int (*xCreate)(
+ int argc, /* Size of argv array */
+ const char *const*argv, /* Tokenizer argument strings */
+ sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */
+ );
+
+ /*
+ ** Destroy an existing tokenizer. The fts3 module calls this method
+ ** exactly once for each successful call to xCreate().
+ */
+ int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+ /*
+ ** Create a tokenizer cursor to tokenize an input buffer. The caller
+ ** is responsible for ensuring that the input buffer remains valid
+ ** until the cursor is closed (using the xClose() method).
+ */
+ int (*xOpen)(
+ sqlite3_tokenizer *pTokenizer, /* Tokenizer object */
+ const char *pInput, int nBytes, /* Input buffer */
+ sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */
+ );
+
+ /*
+ ** Destroy an existing tokenizer cursor. The fts3 module calls this
+ ** method exactly once for each successful call to xOpen().
+ */
+ int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+
+ /*
+ ** Retrieve the next token from the tokenizer cursor pCursor. This
+ ** method should either return SQLITE_OK and set the values of the
+ ** "OUT" variables identified below, or SQLITE_DONE to indicate that
+ ** the end of the buffer has been reached, or an SQLite error code.
+ **
+ ** *ppToken should be set to point at a buffer containing the
+ ** normalized version of the token (i.e. after any case-folding and/or
+ ** stemming has been performed). *pnBytes should be set to the length
+ ** of this buffer in bytes. The input text that generated the token is
+ ** identified by the byte offsets returned in *piStartOffset and
+ ** *piEndOffset. *piStartOffset should be set to the index of the first
+ ** byte of the token in the input buffer. *piEndOffset should be set
+ ** to the index of the first byte just past the end of the token in
+ ** the input buffer.
+ **
+ ** The buffer *ppToken is set to point at is managed by the tokenizer
+ ** implementation. It is only required to be valid until the next call
+ ** to xNext() or xClose().
+ */
+ /* TODO(shess) current implementation requires pInput to be
+ ** nul-terminated. This should either be fixed, or pInput/nBytes
+ ** should be converted to zInput.
+ */
+ int (*xNext)(
+ sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */
+ const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */
+ int *piStartOffset, /* OUT: Byte offset of token in input buffer */
+ int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */
+ int *piPosition /* OUT: Number of tokens returned before this one */
+ );
+};
+
+struct sqlite3_tokenizer {
+ const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */
+ /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+ sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */
+ /* Tokenizer implementations will typically add additional fields */
+};
+
+#endif /* _FTS3_TOKENIZER_H_ */
+
+/************** End of fts3_tokenizer.h **************************************/
+/************** Continuing where we left off in fts3_porter.c ****************/
/*
** Class derived from sqlite3_tokenizer
*/
typedef struct porter_tokenizer {
@@ -106682,18 +101980,18 @@
static void scalarFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
- fts3Hash *pHash;
+ Fts3Hash *pHash;
void *pPtr = 0;
const unsigned char *zName;
int nName;
assert( argc==1 || argc==2 );
- pHash = (fts3Hash *)sqlite3_user_data(context);
+ pHash = (Fts3Hash *)sqlite3_user_data(context);
zName = sqlite3_value_text(argv[0]);
nName = sqlite3_value_bytes(argv[0])+1;
if( argc==2 ){
@@ -106719,10 +102017,132 @@
}
}
sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
}
+
+static int fts3IsIdChar(char c){
+ static const char isFtsIdChar[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
+ 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
+ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
+ };
+ return (c&0x80 || isFtsIdChar[(int)(c)]);
+}
+
+SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){
+ const char *z1;
+ const char *z2 = 0;
+
+ /* Find the start of the next token. */
+ z1 = zStr;
+ while( z2==0 ){
+ switch( *z1 ){
+ case '\0': return 0; /* No more tokens here */
+ case '\'':
+ case '"':
+ case '`': {
+ z2 = &z1[1];
+ while( *z2 && (z2[0]!=*z1 || z2[1]==*z1) ) z2++;
+ if( *z2 ) z2++;
+ break;
+ }
+ case '[':
+ z2 = &z1[1];
+ while( *z2 && z2[0]!=']' ) z2++;
+ if( *z2 ) z2++;
+ break;
+
+ default:
+ if( fts3IsIdChar(*z1) ){
+ z2 = &z1[1];
+ while( fts3IsIdChar(*z2) ) z2++;
+ }else{
+ z1++;
+ }
+ }
+ }
+
+ *pn = (z2-z1);
+ return z1;
+}
+
+SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(
+ Fts3Hash *pHash, /* Tokenizer hash table */
+ const char *zArg, /* Possible tokenizer specification */
+ sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */
+ const char **pzTokenizer, /* OUT: Set to zArg if is tokenizer */
+ char **pzErr /* OUT: Set to malloced error message */
+){
+ int rc;
+ char *z = (char *)zArg;
+ int n;
+ char *zCopy;
+ char *zEnd; /* Pointer to nul-term of zCopy */
+ sqlite3_tokenizer_module *m;
+
+ if( !z ){
+ zCopy = sqlite3_mprintf("simple");
+ }else{
+ while( (*z&0x80) && isspace(*z) ) z++;
+ if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){
+ return SQLITE_OK;
+ }
+ zCopy = sqlite3_mprintf("%s", &z[8]);
+ *pzTokenizer = zArg;
+ }
+ if( !zCopy ){
+ return SQLITE_NOMEM;
+ }
+
+ zEnd = &zCopy[strlen(zCopy)];
+
+ z = (char *)sqlite3Fts3NextToken(zCopy, &n);
+ z[n] = '\0';
+ sqlite3Fts3Dequote(z);
+
+ m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, strlen(z)+1);
+ if( !m ){
+ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
+ rc = SQLITE_ERROR;
+ }else{
+ char const **aArg = 0;
+ int iArg = 0;
+ z = &z[n+1];
+ while( z<zEnd && (z = (char *)sqlite3Fts3NextToken(z, &n)) ){
+ int nNew = sizeof(char *)*(iArg+1);
+ char const **aNew = (const char **)sqlite3_realloc(aArg, nNew);
+ if( !aNew ){
+ sqlite3_free(zCopy);
+ sqlite3_free(aArg);
+ return SQLITE_NOMEM;
+ }
+ aArg = aNew;
+ aArg[iArg++] = z;
+ z[n] = '\0';
+ sqlite3Fts3Dequote(z);
+ z = &z[n+1];
+ }
+ rc = m->xCreate(iArg, aArg, ppTok);
+ assert( rc!=SQLITE_OK || *ppTok );
+ if( rc!=SQLITE_OK ){
+ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
+ }else{
+ (*ppTok)->pModule = m;
+ }
+ sqlite3_free(aArg);
+ }
+
+ sqlite3_free(zCopy);
+ return rc;
+}
+
#ifdef SQLITE_TEST
/*
@@ -106754,11 +102174,11 @@
static void testFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
- fts3Hash *pHash;
+ Fts3Hash *pHash;
sqlite3_tokenizer_module *p;
sqlite3_tokenizer *pTokenizer = 0;
sqlite3_tokenizer_cursor *pCsr = 0;
const char *zErr = 0;
@@ -106787,11 +102207,11 @@
if( argc==3 ){
zArg = (const char *)sqlite3_value_text(argv[1]);
}
- pHash = (fts3Hash *)sqlite3_user_data(context);
+ pHash = (Fts3Hash *)sqlite3_user_data(context);
p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1);
if( !p ){
char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
sqlite3_result_error(context, zErr, -1);
@@ -106956,11 +102376,11 @@
** The third argument to this function, zName, is used as the name
** of both the scalar and, if created, the virtual table.
*/
SQLITE_PRIVATE int sqlite3Fts3InitHashTable(
sqlite3 *db,
- fts3Hash *pHash,
+ Fts3Hash *pHash,
const char *zName
){
int rc = SQLITE_OK;
void *p = (void *)pHash;
const int any = SQLITE_ANY;
@@ -107219,10 +102639,2911 @@
}
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
/************** End of fts3_tokenizer1.c *************************************/
+/************** Begin file fts3_write.c **************************************/
+/*
+** 2009 Oct 23
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file is part of the SQLite FTS3 extension module. Specifically,
+** this file contains code to insert, update and delete rows from FTS3
+** tables. It also contains code to merge FTS3 b-tree segments. Some
+** of the sub-routines used to merge segments are also used by the query
+** code in fts3.c.
+*/
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+
+#define INTERIOR_MAX 2048 /* Soft limit for segment node size */
+#define LEAF_MAX 2048 /* Soft limit for segment leaf size */
+
+typedef struct PendingList PendingList;
+typedef struct SegmentNode SegmentNode;
+typedef struct SegmentWriter SegmentWriter;
+
+/*
+** Data structure used while accumulating terms in the pending-terms hash
+** table. The hash table entry maps from term (a string) to a malloced
+** instance of this structure.
+*/
+struct PendingList {
+ int nData;
+ char *aData;
+ int nSpace;
+ sqlite3_int64 iLastDocid;
+ sqlite3_int64 iLastCol;
+ sqlite3_int64 iLastPos;
+};
+
+/*
+** An instance of this structure is used to iterate through the terms on
+** a contiguous set of segment b-tree leaf nodes. Although the details of
+** this structure are only manipulated by code in this file, opaque handles
+** of type Fts3SegReader* are also used by code in fts3.c to iterate through
+** terms when querying the full-text index. See functions:
+**
+** sqlite3Fts3SegReaderNew()
+** sqlite3Fts3SegReaderFree()
+** sqlite3Fts3SegReaderIterate()
+*/
+struct Fts3SegReader {
+ int iIdx; /* Index within level */
+ sqlite3_int64 iStartBlock;
+ sqlite3_int64 iEndBlock;
+ sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */
+ char *aNode; /* Pointer to node data (or NULL) */
+ int nNode; /* Size of buffer at aNode (or 0) */
+ int nTermAlloc; /* Allocated size of zTerm buffer */
+
+ /* Variables set by fts3SegReaderNext(). These may be read directly
+ ** by the caller. They are valid from the time SegmentReaderNew() returns
+ ** until SegmentReaderNext() returns something other than SQLITE_OK
+ ** (i.e. SQLITE_DONE).
+ */
+ int nTerm; /* Number of bytes in current term */
+ char *zTerm; /* Pointer to current term */
+ char *aDoclist; /* Pointer to doclist of current entry */
+ int nDoclist; /* Size of doclist in current entry */
+
+ /* The following variables are used to iterate through the current doclist */
+ char *pOffsetList;
+ sqlite3_int64 iDocid;
+};
+
+/*
+** An instance of this structure is used to create a segment b-tree in the
+** database. The internal details of this type are only accessed by the
+** following functions:
+**
+** fts3SegWriterAdd()
+** fts3SegWriterFlush()
+** fts3SegWriterFree()
+*/
+struct SegmentWriter {
+ SegmentNode *pTree; /* Pointer to interior tree structure */
+ sqlite3_int64 iFirst; /* First slot in %_segments written */
+ sqlite3_int64 iFree; /* Next free slot in %_segments */
+ char *zTerm; /* Pointer to previous term buffer */
+ int nTerm; /* Number of bytes in zTerm */
+ int nMalloc; /* Size of malloc'd buffer at zMalloc */
+ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */
+ int nSize; /* Size of allocation at aData */
+ int nData; /* Bytes of data in aData */
+ char *aData; /* Pointer to block from malloc() */
+};
+
+/*
+** Type SegmentNode is used by the following three functions to create
+** the interior part of the segment b+-tree structures (everything except
+** the leaf nodes). These functions and type are only ever used by code
+** within the fts3SegWriterXXX() family of functions described above.
+**
+** fts3NodeAddTerm()
+** fts3NodeWrite()
+** fts3NodeFree()
+*/
+struct SegmentNode {
+ SegmentNode *pParent; /* Parent node (or NULL for root node) */
+ SegmentNode *pRight; /* Pointer to right-sibling */
+ SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */
+ int nEntry; /* Number of terms written to node so far */
+ char *zTerm; /* Pointer to previous term buffer */
+ int nTerm; /* Number of bytes in zTerm */
+ int nMalloc; /* Size of malloc'd buffer at zMalloc */
+ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */
+ int nData; /* Bytes of valid data so far */
+ char *aData; /* Node data */
+};
+
+/*
+** Valid values for the second argument to fts3SqlStmt().
+*/
+#define SQL_DELETE_CONTENT 0
+#define SQL_IS_EMPTY 1
+#define SQL_DELETE_ALL_CONTENT 2
+#define SQL_DELETE_ALL_SEGMENTS 3
+#define SQL_DELETE_ALL_SEGDIR 4
+#define SQL_SELECT_CONTENT_BY_ROWID 5
+#define SQL_NEXT_SEGMENT_INDEX 6
+#define SQL_INSERT_SEGMENTS 7
+#define SQL_NEXT_SEGMENTS_ID 8
+#define SQL_INSERT_SEGDIR 9
+#define SQL_SELECT_LEVEL 10
+#define SQL_SELECT_ALL_LEVEL 11
+#define SQL_SELECT_LEVEL_COUNT 12
+#define SQL_SELECT_SEGDIR_COUNT_MAX 13
+#define SQL_DELETE_SEGDIR_BY_LEVEL 14
+#define SQL_DELETE_SEGMENTS_RANGE 15
+#define SQL_CONTENT_INSERT 16
+#define SQL_GET_BLOCK 17
+
+/*
+** This function is used to obtain an SQLite prepared statement handle
+** for the statement identified by the second argument. If successful,
+** *pp is set to the requested statement handle and SQLITE_OK returned.
+** Otherwise, an SQLite error code is returned and *pp is set to 0.
+**
+** If argument apVal is not NULL, then it must point to an array with
+** at least as many entries as the requested statement has bound
+** parameters. The values are bound to the statements parameters before
+** returning.
+*/
+static int fts3SqlStmt(
+ Fts3Table *p, /* Virtual table handle */
+ int eStmt, /* One of the SQL_XXX constants above */
+ sqlite3_stmt **pp, /* OUT: Statement handle */
+ sqlite3_value **apVal /* Values to bind to statement */
+){
+ const char *azSql[] = {
+/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?",
+/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)",
+/* 2 */ "DELETE FROM %Q.'%q_content'",
+/* 3 */ "DELETE FROM %Q.'%q_segments'",
+/* 4 */ "DELETE FROM %Q.'%q_segdir'",
+/* 5 */ "SELECT * FROM %Q.'%q_content' WHERE rowid=?",
+/* 6 */ "SELECT coalesce(max(idx)+1, 0) FROM %Q.'%q_segdir' WHERE level=?",
+/* 7 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)",
+/* 8 */ "SELECT coalesce(max(blockid)+1, 1) FROM %Q.'%q_segments'",
+/* 9 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)",
+
+ /* Return segments in order from oldest to newest.*/
+/* 10 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
+ "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC",
+/* 11 */ "SELECT idx, start_block, leaves_end_block, end_block, root "
+ "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC",
+
+/* 12 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?",
+/* 13 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'",
+
+/* 14 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?",
+/* 15 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?",
+/* 16 */ "INSERT INTO %Q.'%q_content' VALUES(%z)",
+/* 17 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?",
+ };
+ int rc = SQLITE_OK;
+ sqlite3_stmt *pStmt;
+
+ assert( SizeofArray(azSql)==SizeofArray(p->aStmt) );
+ assert( eStmt<SizeofArray(azSql) && eStmt>=0 );
+
+ pStmt = p->aStmt[eStmt];
+ if( !pStmt ){
+ char *zSql;
+ if( eStmt==SQL_CONTENT_INSERT ){
+ int i; /* Iterator variable */
+ char *zVarlist; /* The "?, ?, ..." string */
+ zVarlist = (char *)sqlite3_malloc(2*p->nColumn+2);
+ if( !zVarlist ){
+ *pp = 0;
+ return SQLITE_NOMEM;
+ }
+ zVarlist[0] = '?';
+ zVarlist[p->nColumn*2+1] = '\0';
+ for(i=1; i<=p->nColumn; i++){
+ zVarlist[i*2-1] = ',';
+ zVarlist[i*2] = '?';
+ }
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, zVarlist);
+ }else{
+ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName);
+ }
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL);
+ sqlite3_free(zSql);
+ assert( rc==SQLITE_OK || pStmt==0 );
+ p->aStmt[eStmt] = pStmt;
+ }
+ }
+ if( apVal ){
+ int i;
+ int nParam = sqlite3_bind_parameter_count(pStmt);
+ for(i=0; rc==SQLITE_OK && i<nParam; i++){
+ rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
+ }
+ }
+ *pp = pStmt;
+ return rc;
+}
+
+/*
+** Similar to fts3SqlStmt(). Except, after binding the parameters in
+** array apVal[] to the SQL statement identified by eStmt, the statement
+** is executed.
+**
+** Returns SQLITE_OK if the statement is successfully executed, or an
+** SQLite error code otherwise.
+*/
+static int fts3SqlExec(Fts3Table *p, int eStmt, sqlite3_value **apVal){
+ sqlite3_stmt *pStmt;
+ int rc = fts3SqlStmt(p, eStmt, &pStmt, apVal);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ }
+ return rc;
+}
+
+
+/*
+** Read a single block from the %_segments table. If the specified block
+** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO
+** etc.) occurs, return the appropriate SQLite error code.
+**
+** Otherwise, if successful, set *pzBlock to point to a buffer containing
+** the block read from the database, and *pnBlock to the size of the read
+** block in bytes.
+**
+** WARNING: The returned buffer is only valid until the next call to
+** sqlite3Fts3ReadBlock().
+*/
+SQLITE_PRIVATE int sqlite3Fts3ReadBlock(
+ Fts3Table *p,
+ sqlite3_int64 iBlock,
+ char const **pzBlock,
+ int *pnBlock
+){
+ sqlite3_stmt *pStmt;
+ int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ sqlite3_reset(pStmt);
+
+ sqlite3_bind_int64(pStmt, 1, iBlock);
+ rc = sqlite3_step(pStmt);
+ if( rc!=SQLITE_ROW ){
+ return SQLITE_CORRUPT;
+ }
+
+ *pnBlock = sqlite3_column_bytes(pStmt, 0);
+ *pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
+ if( !*pzBlock ){
+ return SQLITE_NOMEM;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Set *ppStmt to a statement handle that may be used to iterate through
+** all rows in the %_segdir table, from oldest to newest. If successful,
+** return SQLITE_OK. If an error occurs while preparing the statement,
+** return an SQLite error code.
+**
+** There is only ever one instance of this SQL statement compiled for
+** each FTS3 table.
+**
+** The statement returns the following columns from the %_segdir table:
+**
+** 0: idx
+** 1: start_block
+** 2: leaves_end_block
+** 3: end_block
+** 4: root
+*/
+SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table *p, sqlite3_stmt **ppStmt){
+ return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0);
+}
+
+
+/*
+** Append a single varint to a PendingList buffer. SQLITE_OK is returned
+** if successful, or an SQLite error code otherwise.
+**
+** This function also serves to allocate the PendingList structure itself.
+** For example, to create a new PendingList structure containing two
+** varints:
+**
+** PendingList *p = 0;
+** fts3PendingListAppendVarint(&p, 1);
+** fts3PendingListAppendVarint(&p, 2);
+*/
+static int fts3PendingListAppendVarint(
+ PendingList **pp, /* IN/OUT: Pointer to PendingList struct */
+ sqlite3_int64 i /* Value to append to data */
+){
+ PendingList *p = *pp;
+
+ /* Allocate or grow the PendingList as required. */
+ if( !p ){
+ p = sqlite3_malloc(sizeof(*p) + 100);
+ if( !p ){
+ return SQLITE_NOMEM;
+ }
+ p->nSpace = 100;
+ p->aData = (char *)&p[1];
+ p->nData = 0;
+ }
+ else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){
+ int nNew = p->nSpace * 2;
+ p = sqlite3_realloc(p, sizeof(*p) + nNew);
+ if( !p ){
+ sqlite3_free(*pp);
+ *pp = 0;
+ return SQLITE_NOMEM;
+ }
+ p->nSpace = nNew;
+ p->aData = (char *)&p[1];
+ }
+
+ /* Append the new serialized varint to the end of the list. */
+ p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i);
+ p->aData[p->nData] = '\0';
+ *pp = p;
+ return SQLITE_OK;
+}
+
+/*
+** Add a docid/column/position entry to a PendingList structure. Non-zero
+** is returned if the structure is sqlite3_realloced as part of adding
+** the entry. Otherwise, zero.
+**
+** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning.
+** Zero is always returned in this case. Otherwise, if no OOM error occurs,
+** it is set to SQLITE_OK.
+*/
+static int fts3PendingListAppend(
+ PendingList **pp, /* IN/OUT: PendingList structure */
+ sqlite3_int64 iDocid, /* Docid for entry to add */
+ sqlite3_int64 iCol, /* Column for entry to add */
+ sqlite3_int64 iPos, /* Position of term for entry to add */
+ int *pRc /* OUT: Return code */
+){
+ PendingList *p = *pp;
+ int rc = SQLITE_OK;
+
+ assert( !p || p->iLastDocid<=iDocid );
+
+ if( !p || p->iLastDocid!=iDocid ){
+ sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0);
+ if( p ){
+ assert( p->nData<p->nSpace );
+ assert( p->aData[p->nData]==0 );
+ p->nData++;
+ }
+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){
+ goto pendinglistappend_out;
+ }
+ p->iLastCol = -1;
+ p->iLastPos = 0;
+ p->iLastDocid = iDocid;
+ }
+ if( iCol>0 && p->iLastCol!=iCol ){
+ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1))
+ || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol))
+ ){
+ goto pendinglistappend_out;
+ }
+ p->iLastCol = iCol;
+ p->iLastPos = 0;
+ }
+ if( iCol>=0 ){
+ assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) );
+ rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos);
+ p->iLastPos = iPos;
+ }
+
+ pendinglistappend_out:
+ *pRc = rc;
+ if( p!=*pp ){
+ *pp = p;
+ return 1;
+ }
+ return 0;
+}
+
+/*
+** Tokenize the nul-terminated string zText and add all tokens to the
+** pending-terms hash-table. The docid used is that currently stored in
+** p->iPrevDocid, and the column is specified by argument iCol.
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
+*/
+static int fts3PendingTermsAdd(Fts3Table *p, const char *zText, int iCol){
+ int rc;
+ int iStart;
+ int iEnd;
+ int iPos;
+
+ char const *zToken;
+ int nToken;
+
+ sqlite3_tokenizer *pTokenizer = p->pTokenizer;
+ sqlite3_tokenizer_module const *pModule = pTokenizer->pModule;
+ sqlite3_tokenizer_cursor *pCsr;
+ int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
+ const char**,int*,int*,int*,int*);
+
+ assert( pTokenizer && pModule );
+
+ rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ pCsr->pTokenizer = pTokenizer;
+
+ xNext = pModule->xNext;
+ while( SQLITE_OK==rc
+ && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos))
+ ){
+ PendingList *pList;
+
+ /* Positions cannot be negative; we use -1 as a terminator internally.
+ ** Tokens must have a non-zero length.
+ */
+ if( iPos<0 || !zToken || nToken<=0 ){
+ rc = SQLITE_ERROR;
+ break;
+ }
+
+ pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken);
+ if( pList ){
+ p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem));
+ }
+ if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){
+ if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){
+ /* Malloc failed while inserting the new entry. This can only
+ ** happen if there was no previous entry for this token.
+ */
+ assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) );
+ sqlite3_free(pList);
+ rc = SQLITE_NOMEM;
+ }
+ }
+ if( rc==SQLITE_OK ){
+ p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem));
+ }
+ }
+
+ pModule->xClose(pCsr);
+ return (rc==SQLITE_DONE ? SQLITE_OK : rc);
+}
+
+/*
+** Calling this function indicates that subsequent calls to
+** fts3PendingTermsAdd() are to add term/position-list pairs for the
+** contents of the document with docid iDocid.
+*/
+static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){
+ /* TODO(shess) Explore whether partially flushing the buffer on
+ ** forced-flush would provide better performance. I suspect that if
+ ** we ordered the doclists by size and flushed the largest until the
+ ** buffer was half empty, that would let the less frequent terms
+ ** generate longer doclists.
+ */
+ if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){
+ int rc = sqlite3Fts3PendingTermsFlush(p);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ p->iPrevDocid = iDocid;
+ return SQLITE_OK;
+}
+
+SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){
+ Fts3HashElem *pElem;
+ for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
+ sqlite3_free(fts3HashData(pElem));
+ }
+ fts3HashClear(&p->pendingTerms);
+ p->nPendingData = 0;
+}
+
+/*
+** This function is called by the xUpdate() method as part of an INSERT
+** operation. It adds entries for each term in the new record to the
+** pendingTerms hash table.
+**
+** Argument apVal is the same as the similarly named argument passed to
+** fts3InsertData(). Parameter iDocid is the docid of the new row.
+*/
+static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal){
+ int i; /* Iterator variable */
+ for(i=2; i<p->nColumn+2; i++){
+ const char *zText = (const char *)sqlite3_value_text(apVal[i]);
+ if( zText ){
+ int rc = fts3PendingTermsAdd(p, zText, i-2);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+ }
+ }
+ return SQLITE_OK;
+}
+
+/*
+** This function is called by the xUpdate() method for an INSERT operation.
+** The apVal parameter is passed a copy of the apVal argument passed by
+** SQLite to the xUpdate() method. i.e:
+**
+** apVal[0] Not used for INSERT.
+** apVal[1] rowid
+** apVal[2] Left-most user-defined column
+** ...
+** apVal[p->nColumn+1] Right-most user-defined column
+** apVal[p->nColumn+2] Hidden column with same name as table
+** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid)
+*/
+static int fts3InsertData(
+ Fts3Table *p, /* Full-text table */
+ sqlite3_value **apVal, /* Array of values to insert */
+ sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */
+){
+ int rc; /* Return code */
+ int i; /* Iterator variable */
+ sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */
+
+ /* Locate the statement handle used to insert data into the %_content
+ ** table. The SQL for this statement is:
+ **
+ ** INSERT INTO %_content VALUES(?, ?, ?, ...)
+ **
+ ** The statement features N '?' variables, where N is the number of user
+ ** defined columns in the FTS3 table, plus one for the docid field.
+ */
+ rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ /* There is a quirk here. The users INSERT statement may have specified
+ ** a value for the "rowid" field, for the "docid" field, or for both.
+ ** Which is a problem, since "rowid" and "docid" are aliases for the
+ ** same value. For example:
+ **
+ ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2);
+ **
+ ** In FTS3, if a non-NULL docid value is specified, it is the value
+ ** inserted. Otherwise, the rowid value is used.
+ */
+ if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){
+ rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+
+ /* Execute the statement to insert the record. Set *piDocid to the
+ ** new docid value.
+ */
+ sqlite3_step(pContentInsert);
+ rc = sqlite3_reset(pContentInsert);
+
+ *piDocid = sqlite3_last_insert_rowid(p->db);
+ return rc;
+}
+
+
+
+/*
+** Remove all data from the FTS3 table. Clear the hash table containing
+** pending terms.
+*/
+static int fts3DeleteAll(Fts3Table *p){
+ int rc; /* Return code */
+
+ /* Discard the contents of the pending-terms hash table. */
+ sqlite3Fts3PendingTermsClear(p);
+
+ /* Delete everything from the %_content, %_segments and %_segdir tables. */
+ rc = fts3SqlExec(p, SQL_DELETE_ALL_CONTENT, 0);
+ if( rc==SQLITE_OK ){
+ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGMENTS, 0);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0);
+ }
+ return rc;
+}
+
+/*
+** The first element in the apVal[] array is assumed to contain the docid
+** (an integer) of a row about to be deleted. Remove all terms from the
+** full-text index.
+*/
+static int fts3DeleteTerms(Fts3Table *p, sqlite3_value **apVal){
+ int rc;
+ sqlite3_stmt *pSelect;
+
+ rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal);
+ if( rc==SQLITE_OK ){
+ if( SQLITE_ROW==sqlite3_step(pSelect) ){
+ int i;
+ for(i=1; i<=p->nColumn; i++){
+ const char *zText = (const char *)sqlite3_column_text(pSelect, i);
+ rc = fts3PendingTermsAdd(p, zText, -1);
+ if( rc!=SQLITE_OK ){
+ sqlite3_reset(pSelect);
+ return rc;
+ }
+ }
+ }
+ }
+
+ return sqlite3_reset(pSelect);
+}
+
+/*
+** Forward declaration to account for the circular dependency between
+** functions fts3SegmentMerge() and fts3AllocateSegdirIdx().
+*/
+static int fts3SegmentMerge(Fts3Table *, int);
+
+/*
+** This function allocates a new level iLevel index in the segdir table.
+** Usually, indexes are allocated within a level sequentially starting
+** with 0, so the allocated index is one greater than the value returned
+** by:
+**
+** SELECT max(idx) FROM %_segdir WHERE level = :iLevel
+**
+** However, if there are already FTS3_MERGE_COUNT indexes at the requested
+** level, they are merged into a single level (iLevel+1) segment and the
+** allocated index is 0.
+**
+** If successful, *piIdx is set to the allocated index slot and SQLITE_OK
+** returned. Otherwise, an SQLite error code is returned.
+*/
+static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){
+ int rc; /* Return Code */
+ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */
+ int iNext; /* Result of query pNextIdx */
+
+ /* Set variable iNext to the next available segdir index at level iLevel. */
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pNextIdx, 1, iLevel);
+ if( SQLITE_ROW==sqlite3_step(pNextIdx) ){
+ iNext = sqlite3_column_int64(pNextIdx, 0);
+ }
+ rc = sqlite3_reset(pNextIdx);
+ }
+
+ if( rc==SQLITE_OK ){
+ /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already
+ ** full, merge all segments in level iLevel into a single iLevel+1
+ ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise,
+ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext.
+ */
+ if( iNext>=FTS3_MERGE_COUNT ){
+ rc = fts3SegmentMerge(p, iLevel);
+ *piIdx = 0;
+ }else{
+ *piIdx = iNext;
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Move the iterator passed as the first argument to the next term in the
+** segment. If successful, SQLITE_OK is returned. If there is no next term,
+** SQLITE_DONE. Otherwise, an SQLite error code.
+*/
+static int fts3SegReaderNext(Fts3SegReader *pReader){
+ char *pNext; /* Cursor variable */
+ int nPrefix; /* Number of bytes in term prefix */
+ int nSuffix; /* Number of bytes in term suffix */
+
+ if( !pReader->aDoclist ){
+ pNext = pReader->aNode;
+ }else{
+ pNext = &pReader->aDoclist[pReader->nDoclist];
+ }
+
+ if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){
+ int rc;
+ if( !pReader->pStmt ){
+ pReader->aNode = 0;
+ return SQLITE_OK;
+ }
+ rc = sqlite3_step(pReader->pStmt);
+ if( rc!=SQLITE_ROW ){
+ pReader->aNode = 0;
+ return (rc==SQLITE_DONE ? SQLITE_OK : rc);
+ }
+ pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0);
+ pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0);
+ pNext = pReader->aNode;
+ }
+
+ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix);
+ pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix);
+
+ if( nPrefix+nSuffix>pReader->nTermAlloc ){
+ int nNew = (nPrefix+nSuffix)*2;
+ char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pReader->zTerm = zNew;
+ pReader->nTermAlloc = nNew;
+ }
+ memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix);
+ pReader->nTerm = nPrefix+nSuffix;
+ pNext += nSuffix;
+ pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist);
+ assert( pNext<&pReader->aNode[pReader->nNode] );
+ pReader->aDoclist = pNext;
+ pReader->pOffsetList = 0;
+ return SQLITE_OK;
+}
+
+/*
+** Set the SegReader to point to the first docid in the doclist associated
+** with the current term.
+*/
+static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){
+ int n;
+ assert( pReader->aDoclist );
+ assert( !pReader->pOffsetList );
+ n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid);
+ pReader->pOffsetList = &pReader->aDoclist[n];
+}
+
+/*
+** Advance the SegReader to point to the next docid in the doclist
+** associated with the current term.
+**
+** If arguments ppOffsetList and pnOffsetList are not NULL, then
+** *ppOffsetList is set to point to the first column-offset list
+** in the doclist entry (i.e. immediately past the docid varint).
+** *pnOffsetList is set to the length of the set of column-offset
+** lists, not including the nul-terminator byte. For example:
+*/
+static void fts3SegReaderNextDocid(
+ Fts3SegReader *pReader,
+ char **ppOffsetList,
+ int *pnOffsetList
+){
+ char *p = pReader->pOffsetList;
+ char c = 0;
+
+ /* Pointer p currently points at the first byte of an offset list. The
+ ** following two lines advance it to point one byte past the end of
+ ** the same offset list.
+ */
+ while( *p | c ) c = *p++ & 0x80;
+ p++;
+
+ /* If required, populate the output variables with a pointer to and the
+ ** size of the previous offset-list.
+ */
+ if( ppOffsetList ){
+ *ppOffsetList = pReader->pOffsetList;
+ *pnOffsetList = p - pReader->pOffsetList - 1;
+ }
+
+ /* If there are no more entries in the doclist, set pOffsetList to
+ ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and
+ ** Fts3SegReader.pOffsetList to point to the next offset list before
+ ** returning.
+ */
+ if( p>=&pReader->aDoclist[pReader->nDoclist] ){
+ pReader->pOffsetList = 0;
+ }else{
+ sqlite3_int64 iDelta;
+ pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta);
+ pReader->iDocid += iDelta;
+ }
+}
+
+/*
+** Free all allocations associated with the iterator passed as the
+** second argument.
+*/
+SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){
+ if( pReader ){
+ if( pReader->pStmt ){
+ /* Move the leaf-range SELECT statement to the aLeavesStmt[] array,
+ ** so that it can be reused when required by another query.
+ */
+ assert( p->nLeavesStmt<p->nLeavesTotal );
+ sqlite3_reset(pReader->pStmt);
+ p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt;
+ }
+ sqlite3_free(pReader->zTerm);
+ sqlite3_free(pReader);
+ }
+}
+
+/*
+** Allocate a new SegReader object.
+*/
+SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(
+ Fts3Table *p, /* Virtual table handle */
+ int iAge, /* Segment "age". */
+ sqlite3_int64 iStartLeaf, /* First leaf to traverse */
+ sqlite3_int64 iEndLeaf, /* Final leaf to traverse */
+ sqlite3_int64 iEndBlock, /* Final block of segment */
+ const char *zRoot, /* Buffer containing root node */
+ int nRoot, /* Size of buffer containing root node */
+ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */
+){
+ int rc = SQLITE_OK; /* Return code */
+ Fts3SegReader *pReader; /* Newly allocated SegReader object */
+ int nExtra = 0; /* Bytes to allocate segment root node */
+
+ if( iStartLeaf==0 ){
+ nExtra = nRoot;
+ }
+
+ pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra);
+ if( !pReader ){
+ return SQLITE_NOMEM;
+ }
+ memset(pReader, 0, sizeof(Fts3SegReader));
+ pReader->iStartBlock = iStartLeaf;
+ pReader->iIdx = iAge;
+ pReader->iEndBlock = iEndBlock;
+
+ if( nExtra ){
+ /* The entire segment is stored in the root node. */
+ pReader->aNode = (char *)&pReader[1];
+ pReader->nNode = nRoot;
+ memcpy(pReader->aNode, zRoot, nRoot);
+ }else{
+ sqlite3_stmt *pStmt;
+
+ /* If the text of the SQL statement to iterate through a contiguous
+ ** set of entries in the %_segments table has not yet been composed,
+ ** compose it now.
+ */
+ if( !p->zSelectLeaves ){
+ p->zSelectLeaves = sqlite3_mprintf(
+ "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? "
+ "ORDER BY blockid", p->zDb, p->zName
+ );
+ if( !p->zSelectLeaves ){
+ rc = SQLITE_NOMEM;
+ goto finished;
+ }
+ }
+
+ /* If there are no free statements in the aLeavesStmt[] array, prepare
+ ** a new statement now. Otherwise, reuse a prepared statement from
+ ** aLeavesStmt[].
+ */
+ if( p->nLeavesStmt==0 ){
+ if( p->nLeavesTotal==p->nLeavesAlloc ){
+ int nNew = p->nLeavesAlloc + 16;
+ sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc(
+ p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *)
+ );
+ if( !aNew ){
+ rc = SQLITE_NOMEM;
+ goto finished;
+ }
+ p->nLeavesAlloc = nNew;
+ p->aLeavesStmt = aNew;
+ }
+ rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0);
+ if( rc!=SQLITE_OK ){
+ goto finished;
+ }
+ p->nLeavesTotal++;
+ }else{
+ pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt];
+ }
+
+ /* Bind the start and end leaf blockids to the prepared SQL statement. */
+ sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf);
+ sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf);
+ }
+ rc = fts3SegReaderNext(pReader);
+
+ finished:
+ if( rc==SQLITE_OK ){
+ *ppReader = pReader;
+ }else{
+ sqlite3Fts3SegReaderFree(p, pReader);
+ }
+ return rc;
+}
+
+
+/*
+** The second argument to this function is expected to be a statement of
+** the form:
+**
+** SELECT
+** idx, -- col 0
+** start_block, -- col 1
+** leaves_end_block, -- col 2
+** end_block, -- col 3
+** root -- col 4
+** FROM %_segdir ...
+**
+** This function allocates and initializes a Fts3SegReader structure to
+** iterate through the terms stored in the segment identified by the
+** current row that pStmt is pointing to.
+**
+** If successful, the Fts3SegReader is left pointing to the first term
+** in the segment and SQLITE_OK is returned. Otherwise, an SQLite error
+** code is returned.
+*/
+static int fts3SegReaderNew(
+ Fts3Table *p, /* Virtual table handle */
+ sqlite3_stmt *pStmt, /* See above */
+ int iAge, /* Segment "age". */
+ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */
+){
+ return sqlite3Fts3SegReaderNew(p, iAge,
+ sqlite3_column_int64(pStmt, 1),
+ sqlite3_column_int64(pStmt, 2),
+ sqlite3_column_int64(pStmt, 3),
+ sqlite3_column_blob(pStmt, 4),
+ sqlite3_column_bytes(pStmt, 4),
+ ppReader
+ );
+}
+
+/*
+** Compare the entries pointed to by two Fts3SegReader structures.
+** Comparison is as follows:
+**
+** 1) EOF is greater than not EOF.
+**
+** 2) The current terms (if any) are compared with memcmp(). If one
+** term is a prefix of another, the longer term is considered the
+** larger.
+**
+** 3) By segment age. An older segment is considered larger.
+*/
+static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
+ int rc;
+ if( pLhs->aNode && pRhs->aNode ){
+ int rc2 = pLhs->nTerm - pRhs->nTerm;
+ if( rc2<0 ){
+ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm);
+ }else{
+ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm);
+ }
+ if( rc==0 ){
+ rc = rc2;
+ }
+ }else{
+ rc = (pLhs->aNode==0) - (pRhs->aNode==0);
+ }
+ if( rc==0 ){
+ rc = pRhs->iIdx - pLhs->iIdx;
+ }
+ assert( rc!=0 );
+ return rc;
+}
+
+/*
+** A different comparison function for SegReader structures. In this
+** version, it is assumed that each SegReader points to an entry in
+** a doclist for identical terms. Comparison is made as follows:
+**
+** 1) EOF (end of doclist in this case) is greater than not EOF.
+**
+** 2) By current docid.
+**
+** 3) By segment age. An older segment is considered larger.
+*/
+static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){
+ int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0);
+ if( rc==0 ){
+ if( pLhs->iDocid==pRhs->iDocid ){
+ rc = pRhs->iIdx - pLhs->iIdx;
+ }else{
+ rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1;
+ }
+ }
+ assert( pLhs->aNode && pRhs->aNode );
+ return rc;
+}
+
+/*
+** Compare the term that the Fts3SegReader object passed as the first argument
+** points to with the term specified by arguments zTerm and nTerm.
+**
+** If the pSeg iterator is already at EOF, return 0. Otherwise, return
+** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are
+** equal, or +ve if the pSeg term is greater than zTerm/nTerm.
+*/
+static int fts3SegReaderTermCmp(
+ Fts3SegReader *pSeg, /* Segment reader object */
+ const char *zTerm, /* Term to compare to */
+ int nTerm /* Size of term zTerm in bytes */
+){
+ int res = 0;
+ if( pSeg->aNode ){
+ if( pSeg->nTerm>nTerm ){
+ res = memcmp(pSeg->zTerm, zTerm, nTerm);
+ }else{
+ res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm);
+ }
+ if( res==0 ){
+ res = pSeg->nTerm-nTerm;
+ }
+ }
+ return res;
+}
+
+/*
+** Argument apSegment is an array of nSegment elements. It is known that
+** the final (nSegment-nSuspect) members are already in sorted order
+** (according to the comparison function provided). This function shuffles
+** the array around until all entries are in sorted order.
+*/
+static void fts3SegReaderSort(
+ Fts3SegReader **apSegment, /* Array to sort entries of */
+ int nSegment, /* Size of apSegment array */
+ int nSuspect, /* Unsorted entry count */
+ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */
+){
+ int i; /* Iterator variable */
+
+ assert( nSuspect<=nSegment );
+
+ if( nSuspect==nSegment ) nSuspect--;
+ for(i=nSuspect-1; i>=0; i--){
+ int j;
+ for(j=i; j<(nSegment-1); j++){
+ Fts3SegReader *pTmp;
+ if( xCmp(apSegment[j], apSegment[j+1])<0 ) break;
+ pTmp = apSegment[j+1];
+ apSegment[j+1] = apSegment[j];
+ apSegment[j] = pTmp;
+ }
+ }
+
+#ifndef NDEBUG
+ /* Check that the list really is sorted now. */
+ for(i=0; i<(nSuspect-1); i++){
+ assert( xCmp(apSegment[i], apSegment[i+1])<0 );
+ }
+#endif
+}
+
+/*
+** Insert a record into the %_segments table.
+*/
+static int fts3WriteSegment(
+ Fts3Table *p, /* Virtual table handle */
+ sqlite3_int64 iBlock, /* Block id for new block */
+ char *z, /* Pointer to buffer containing block data */
+ int n /* Size of buffer z in bytes */
+){
+ sqlite3_stmt *pStmt;
+ int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int64(pStmt, 1, iBlock);
+ rc = sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ }
+ }
+ return rc;
+}
+
+/*
+** Insert a record into the %_segdir table.
+*/
+static int fts3WriteSegdir(
+ Fts3Table *p, /* Virtual table handle */
+ int iLevel, /* Value for "level" field */
+ int iIdx, /* Value for "idx" field */
+ sqlite3_int64 iStartBlock, /* Value for "start_block" field */
+ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */
+ sqlite3_int64 iEndBlock, /* Value for "end_block" field */
+ char *zRoot, /* Blob value for "root" field */
+ int nRoot /* Number of bytes in buffer zRoot */
+){
+ sqlite3_stmt *pStmt;
+ int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pStmt, 1, iLevel);
+ sqlite3_bind_int(pStmt, 2, iIdx);
+ sqlite3_bind_int64(pStmt, 3, iStartBlock);
+ sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
+ sqlite3_bind_int64(pStmt, 5, iEndBlock);
+ rc = sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
+ if( rc==SQLITE_OK ){
+ sqlite3_step(pStmt);
+ rc = sqlite3_reset(pStmt);
+ }
+ }
+ return rc;
+}
+
+/*
+** Return the size of the common prefix (if any) shared by zPrev and
+** zNext, in bytes. For example,
+**
+** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3
+** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2
+** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0
+*/
+static int fts3PrefixCompress(
+ const char *zPrev, /* Buffer containing previous term */
+ int nPrev, /* Size of buffer zPrev in bytes */
+ const char *zNext, /* Buffer containing next term */
+ int nNext /* Size of buffer zNext in bytes */
+){
+ int n;
+ for(n=0; n<nPrev && zPrev[n]==zNext[n]; n++);
+ return n;
+}
+
+/*
+** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger
+** (according to memcmp) than the previous term.
+*/
+static int fts3NodeAddTerm(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */
+ int isCopyTerm, /* True if zTerm/nTerm is transient */
+ const char *zTerm, /* Pointer to buffer containing term */
+ int nTerm /* Size of term in bytes */
+){
+ SegmentNode *pTree = *ppTree;
+ int rc;
+ SegmentNode *pNew;
+
+ /* First try to append the term to the current node. Return early if
+ ** this is possible.
+ */
+ if( pTree ){
+ int nData = pTree->nData; /* Current size of node in bytes */
+ int nReq = nData; /* Required space after adding zTerm */
+ int nPrefix; /* Number of bytes of prefix compression */
+ int nSuffix; /* Suffix length */
+
+ nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm);
+ nSuffix = nTerm-nPrefix;
+
+ nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix;
+ if( nReq<=INTERIOR_MAX || !pTree->zTerm ){
+
+ if( nReq>INTERIOR_MAX ){
+ /* An unusual case: this is the first term to be added to the node
+ ** and the static node buffer (INTERIOR_MAX bytes) is not large
+ ** enough. Use a separately malloced buffer instead This wastes
+ ** INTERIOR_MAX bytes, but since this scenario only comes about when
+ ** the database contain two terms that share a prefix of almost 2KB,
+ ** this is not expected to be a serious problem.
+ */
+ assert( pTree->aData==(char *)&pTree[1] );
+ pTree->aData = (char *)sqlite3_malloc(nReq);
+ if( !pTree->aData ){
+ return SQLITE_NOMEM;
+ }
+ }
+
+ if( pTree->zTerm ){
+ /* There is no prefix-length field for first term in a node */
+ nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix);
+ }
+
+ nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix);
+ memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix);
+ pTree->nData = nData + nSuffix;
+ pTree->nEntry++;
+
+ if( isCopyTerm ){
+ if( pTree->nMalloc<nTerm ){
+ char *zNew = sqlite3_realloc(pTree->zMalloc, nTerm*2);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pTree->nMalloc = nTerm*2;
+ pTree->zMalloc = zNew;
+ }
+ pTree->zTerm = pTree->zMalloc;
+ memcpy(pTree->zTerm, zTerm, nTerm);
+ pTree->nTerm = nTerm;
+ }else{
+ pTree->zTerm = (char *)zTerm;
+ pTree->nTerm = nTerm;
+ }
+ return SQLITE_OK;
+ }
+ }
+
+ /* If control flows to here, it was not possible to append zTerm to the
+ ** current node. Create a new node (a right-sibling of the current node).
+ ** If this is the first node in the tree, the term is added to it.
+ **
+ ** Otherwise, the term is not added to the new node, it is left empty for
+ ** now. Instead, the term is inserted into the parent of pTree. If pTree
+ ** has no parent, one is created here.
+ */
+ pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + INTERIOR_MAX);
+ if( !pNew ){
+ return SQLITE_NOMEM;
+ }
+ memset(pNew, 0, sizeof(SegmentNode));
+ pNew->nData = 1 + FTS3_VARINT_MAX;
+ pNew->aData = (char *)&pNew[1];
+
+ if( pTree ){
+ SegmentNode *pParent = pTree->pParent;
+ rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm);
+ if( pTree->pParent==0 ){
+ pTree->pParent = pParent;
+ }
+ pTree->pRight = pNew;
+ pNew->pLeftmost = pTree->pLeftmost;
+ pNew->pParent = pParent;
+ pNew->zMalloc = pTree->zMalloc;
+ pNew->nMalloc = pTree->nMalloc;
+ pTree->zMalloc = 0;
+ }else{
+ pNew->pLeftmost = pNew;
+ rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm);
+ }
+
+ *ppTree = pNew;
+ return rc;
+}
+
+/*
+** Helper function for fts3NodeWrite().
+*/
+static int fts3TreeFinishNode(
+ SegmentNode *pTree,
+ int iHeight,
+ sqlite3_int64 iLeftChild
+){
+ int nStart;
+ assert( iHeight>=1 && iHeight<128 );
+ nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild);
+ pTree->aData[nStart] = (char)iHeight;
+ sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild);
+ return nStart;
+}
+
+/*
+** Write the buffer for the segment node pTree and all of its peers to the
+** database. Then call this function recursively to write the parent of
+** pTree and its peers to the database.
+**
+** Except, if pTree is a root node, do not write it to the database. Instead,
+** set output variables *paRoot and *pnRoot to contain the root node.
+**
+** If successful, SQLITE_OK is returned and output variable *piLast is
+** set to the largest blockid written to the database (or zero if no
+** blocks were written to the db). Otherwise, an SQLite error code is
+** returned.
+*/
+static int fts3NodeWrite(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentNode *pTree, /* SegmentNode handle */
+ int iHeight, /* Height of this node in tree */
+ sqlite3_int64 iLeaf, /* Block id of first leaf node */
+ sqlite3_int64 iFree, /* Block id of next free slot in %_segments */
+ sqlite3_int64 *piLast, /* OUT: Block id of last entry written */
+ char **paRoot, /* OUT: Data for root node */
+ int *pnRoot /* OUT: Size of root node in bytes */
+){
+ int rc = SQLITE_OK;
+
+ if( !pTree->pParent ){
+ /* Root node of the tree. */
+ int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf);
+ *piLast = iFree-1;
+ *pnRoot = pTree->nData - nStart;
+ *paRoot = &pTree->aData[nStart];
+ }else{
+ SegmentNode *pIter;
+ sqlite3_int64 iNextFree = iFree;
+ sqlite3_int64 iNextLeaf = iLeaf;
+ for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){
+ int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf);
+ int nWrite = pIter->nData - nStart;
+
+ rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite);
+ iNextFree++;
+ iNextLeaf += (pIter->nEntry+1);
+ }
+ if( rc==SQLITE_OK ){
+ assert( iNextLeaf==iFree );
+ rc = fts3NodeWrite(
+ p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot
+ );
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Free all memory allocations associated with the tree pTree.
+*/
+static void fts3NodeFree(SegmentNode *pTree){
+ if( pTree ){
+ SegmentNode *p = pTree->pLeftmost;
+ fts3NodeFree(p->pParent);
+ while( p ){
+ SegmentNode *pRight = p->pRight;
+ if( p->aData!=(char *)&p[1] ){
+ sqlite3_free(p->aData);
+ }
+ assert( pRight==0 || p->zMalloc==0 );
+ sqlite3_free(p->zMalloc);
+ sqlite3_free(p);
+ p = pRight;
+ }
+ }
+}
+
+/*
+** Add a term to the segment being constructed by the SegmentWriter object
+** *ppWriter. When adding the first term to a segment, *ppWriter should
+** be passed NULL. This function will allocate a new SegmentWriter object
+** and return it via the input/output variable *ppWriter in this case.
+**
+** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code.
+*/
+static int fts3SegWriterAdd(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */
+ int isCopyTerm, /* True if buffer zTerm must be copied */
+ const char *zTerm, /* Pointer to buffer containing term */
+ int nTerm, /* Size of term in bytes */
+ const char *aDoclist, /* Pointer to buffer containing doclist */
+ int nDoclist /* Size of doclist in bytes */
+){
+ int nPrefix; /* Size of term prefix in bytes */
+ int nSuffix; /* Size of term suffix in bytes */
+ int nReq; /* Number of bytes required on leaf page */
+ int nData;
+ SegmentWriter *pWriter = *ppWriter;
+
+ if( !pWriter ){
+ int rc;
+ sqlite3_stmt *pStmt;
+
+ /* Allocate the SegmentWriter structure */
+ pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter));
+ if( !pWriter ) return SQLITE_NOMEM;
+ memset(pWriter, 0, sizeof(SegmentWriter));
+ *ppWriter = pWriter;
+
+ /* Allocate a buffer in which to accumulate data */
+ pWriter->aData = (char *)sqlite3_malloc(LEAF_MAX);
+ if( !pWriter->aData ) return SQLITE_NOMEM;
+ pWriter->nSize = LEAF_MAX;
+
+ /* Find the next free blockid in the %_segments table */
+ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ pWriter->iFree = sqlite3_column_int64(pStmt, 0);
+ pWriter->iFirst = pWriter->iFree;
+ }
+ rc = sqlite3_reset(pStmt);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+ nData = pWriter->nData;
+
+ nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm);
+ nSuffix = nTerm-nPrefix;
+
+ /* Figure out how many bytes are required by this new entry */
+ nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */
+ sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */
+ nSuffix + /* Term suffix */
+ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */
+ nDoclist; /* Doclist data */
+
+ if( nData>0 && nData+nReq>LEAF_MAX ){
+ int rc;
+
+ /* The current leaf node is full. Write it out to the database. */
+ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData);
+ if( rc!=SQLITE_OK ) return rc;
+
+ /* Add the current term to the interior node tree. The term added to
+ ** the interior tree must:
+ **
+ ** a) be greater than the largest term on the leaf node just written
+ ** to the database (still available in pWriter->zTerm), and
+ **
+ ** b) be less than or equal to the term about to be added to the new
+ ** leaf node (zTerm/nTerm).
+ **
+ ** In other words, it must be the prefix of zTerm 1 byte longer than
+ ** the common prefix (if any) of zTerm and pWriter->zTerm.
+ */
+ assert( nPrefix<nTerm );
+ rc = fts3NodeAddTerm(p, &pWriter->pTree, isCopyTerm, zTerm, nPrefix+1);
+ if( rc!=SQLITE_OK ) return rc;
+
+ nData = 0;
+ pWriter->nTerm = 0;
+
+ nPrefix = 0;
+ nSuffix = nTerm;
+ nReq = 1 + /* varint containing prefix size */
+ sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */
+ nTerm + /* Term suffix */
+ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */
+ nDoclist; /* Doclist data */
+ }
+
+ /* If the buffer currently allocated is too small for this entry, realloc
+ ** the buffer to make it large enough.
+ */
+ if( nReq>pWriter->nSize ){
+ char *aNew = sqlite3_realloc(pWriter->aData, nReq);
+ if( !aNew ) return SQLITE_NOMEM;
+ pWriter->aData = aNew;
+ pWriter->nSize = nReq;
+ }
+ assert( nData+nReq<=pWriter->nSize );
+
+ /* Append the prefix-compressed term and doclist to the buffer. */
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix);
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix);
+ memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix);
+ nData += nSuffix;
+ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist);
+ memcpy(&pWriter->aData[nData], aDoclist, nDoclist);
+ pWriter->nData = nData + nDoclist;
+
+ /* Save the current term so that it can be used to prefix-compress the next.
+ ** If the isCopyTerm parameter is true, then the buffer pointed to by
+ ** zTerm is transient, so take a copy of the term data. Otherwise, just
+ ** store a copy of the pointer.
+ */
+ if( isCopyTerm ){
+ if( nTerm>pWriter->nMalloc ){
+ char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2);
+ if( !zNew ){
+ return SQLITE_NOMEM;
+ }
+ pWriter->nMalloc = nTerm*2;
+ pWriter->zMalloc = zNew;
+ pWriter->zTerm = zNew;
+ }
+ assert( pWriter->zTerm==pWriter->zMalloc );
+ memcpy(pWriter->zTerm, zTerm, nTerm);
+ }else{
+ pWriter->zTerm = (char *)zTerm;
+ }
+ pWriter->nTerm = nTerm;
+
+ return SQLITE_OK;
+}
+
+/*
+** Flush all data associated with the SegmentWriter object pWriter to the
+** database. This function must be called after all terms have been added
+** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is
+** returned. Otherwise, an SQLite error code.
+*/
+static int fts3SegWriterFlush(
+ Fts3Table *p, /* Virtual table handle */
+ SegmentWriter *pWriter, /* SegmentWriter to flush to the db */
+ int iLevel, /* Value for 'level' column of %_segdir */
+ int iIdx /* Value for 'idx' column of %_segdir */
+){
+ int rc; /* Return code */
+ if( pWriter->pTree ){
+ sqlite3_int64 iLast; /* Largest block id written to database */
+ sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */
+ char *zRoot; /* Pointer to buffer containing root node */
+ int nRoot; /* Size of buffer zRoot */
+
+ iLastLeaf = pWriter->iFree;
+ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData);
+ if( rc==SQLITE_OK ){
+ rc = fts3NodeWrite(p, pWriter->pTree, 1,
+ pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3WriteSegdir(
+ p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot);
+ }
+ }else{
+ /* The entire tree fits on the root node. Write it to the segdir table. */
+ rc = fts3WriteSegdir(
+ p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData);
+ }
+ return rc;
+}
+
+/*
+** Release all memory held by the SegmentWriter object passed as the
+** first argument.
+*/
+static void fts3SegWriterFree(SegmentWriter *pWriter){
+ if( pWriter ){
+ sqlite3_free(pWriter->aData);
+ sqlite3_free(pWriter->zMalloc);
+ fts3NodeFree(pWriter->pTree);
+ sqlite3_free(pWriter);
+ }
+}
+
+/*
+** The first value in the apVal[] array is assumed to contain an integer.
+** This function tests if there exist any documents with docid values that
+** are different from that integer. i.e. if deleting the document with docid
+** apVal[0] would mean the FTS3 table were empty.
+**
+** If successful, *pisEmpty is set to true if the table is empty except for
+** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an
+** error occurs, an SQLite error code is returned.
+*/
+static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){
+ sqlite3_stmt *pStmt;
+ int rc;
+ rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal);
+ if( rc==SQLITE_OK ){
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ *pisEmpty = sqlite3_column_int(pStmt, 0);
+ }
+ rc = sqlite3_reset(pStmt);
+ }
+ return rc;
+}
+
+/*
+** Set *pnSegment to the number of segments of level iLevel in the database.
+**
+** Return SQLITE_OK if successful, or an SQLite error code if not.
+*/
+static int fts3SegmentCount(Fts3Table *p, int iLevel, int *pnSegment){
+ sqlite3_stmt *pStmt;
+ int rc;
+
+ assert( iLevel>=0 );
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_COUNT, &pStmt, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ sqlite3_bind_int(pStmt, 1, iLevel);
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ *pnSegment = sqlite3_column_int(pStmt, 0);
+ }
+ return sqlite3_reset(pStmt);
+}
+
+/*
+** Set *pnSegment to the total number of segments in the database. Set
+** *pnMax to the largest segment level in the database (segment levels
+** are stored in the 'level' column of the %_segdir table).
+**
+** Return SQLITE_OK if successful, or an SQLite error code if not.
+*/
+static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){
+ sqlite3_stmt *pStmt;
+ int rc;
+
+ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ if( SQLITE_ROW==sqlite3_step(pStmt) ){
+ *pnSegment = sqlite3_column_int(pStmt, 0);
+ *pnMax = sqlite3_column_int(pStmt, 1);
+ }
+ return sqlite3_reset(pStmt);
+}
+
+/*
+** This function is used after merging multiple segments into a single large
+** segment to delete the old, now redundant, segment b-trees. Specifically,
+** it:
+**
+** 1) Deletes all %_segments entries for the segments associated with
+** each of the SegReader objects in the array passed as the third
+** argument, and
+**
+** 2) deletes all %_segdir entries with level iLevel, or all %_segdir
+** entries regardless of level if (iLevel<0).
+**
+** SQLITE_OK is returned if successful, otherwise an SQLite error code.
+*/
+static int fts3DeleteSegdir(
+ Fts3Table *p, /* Virtual table handle */
+ int iLevel, /* Level of %_segdir entries to delete */
+ Fts3SegReader **apSegment, /* Array of SegReader objects */
+ int nReader /* Size of array apSegment */
+){
+ int rc; /* Return Code */
+ int i; /* Iterator variable */
+ sqlite3_stmt *pDelete; /* SQL statement to delete rows */
+
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0);
+ for(i=0; rc==SQLITE_OK && i<nReader; i++){
+ Fts3SegReader *pSegment = apSegment[i];
+ if( pSegment->iStartBlock ){
+ sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock);
+ sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock);
+ sqlite3_step(pDelete);
+ rc = sqlite3_reset(pDelete);
+ }
+ }
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ if( iLevel>=0 ){
+ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0);
+ if( rc==SQLITE_OK ){
+ sqlite3_bind_int(pDelete, 1, iLevel);
+ sqlite3_step(pDelete);
+ rc = sqlite3_reset(pDelete);
+ }
+ }else{
+ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0);
+ }
+
+ return rc;
+}
+
+/*
+** When this function is called, buffer *ppList (size *pnList bytes) contains
+** a position list that may (or may not) feature multiple columns. This
+** function adjusts the pointer *ppList and the length *pnList so that they
+** identify the subset of the position list that corresponds to column iCol.
+**
+** If there are no entries in the input position list for column iCol, then
+** *pnList is set to zero before returning.
+*/
+static void fts3ColumnFilter(
+ int iCol, /* Column to filter on */
+ char **ppList, /* IN/OUT: Pointer to position list */
+ int *pnList /* IN/OUT: Size of buffer *ppList in bytes */
+){
+ char *pList = *ppList;
+ int nList = *pnList;
+ char *pEnd = &pList[nList];
+ int iCurrent = 0;
+ char *p = pList;
+
+ assert( iCol>=0 );
+ while( 1 ){
+ char c = 0;
+ while( p<pEnd && (c | *p)&0xFE ) c = *p++ & 0x80;
+
+ if( iCol==iCurrent ){
+ nList = (p - pList);
+ break;
+ }
+
+ nList -= (p - pList);
+ pList = p;
+ if( nList==0 ){
+ break;
+ }
+ p = &pList[1];
+ p += sqlite3Fts3GetVarint32(p, &iCurrent);
+ }
+
+ *ppList = pList;
+ *pnList = nList;
+}
+
+/*
+** sqlite3Fts3SegReaderIterate() callback used when merging multiple
+** segments to create a single, larger segment.
+*/
+static int fts3MergeCallback(
+ Fts3Table *p,
+ void *pContext,
+ char *zTerm,
+ int nTerm,
+ char *aDoclist,
+ int nDoclist
+){
+ SegmentWriter **ppW = (SegmentWriter **)pContext;
+ return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist);
+}
+
+/*
+** This function is used to iterate through a contiguous set of terms
+** stored in the full-text index. It merges data contained in one or
+** more segments to support this.
+**
+** The second argument is passed an array of pointers to SegReader objects
+** allocated with sqlite3Fts3SegReaderNew(). This function merges the range
+** of terms selected by each SegReader. If a single term is present in
+** more than one segment, the associated doclists are merged. For each
+** term and (possibly merged) doclist in the merged range, the callback
+** function xFunc is invoked with its arguments set as follows.
+**
+** arg 0: Copy of 'p' parameter passed to this function
+** arg 1: Copy of 'pContext' parameter passed to this function
+** arg 2: Pointer to buffer containing term
+** arg 3: Size of arg 2 buffer in bytes
+** arg 4: Pointer to buffer containing doclist
+** arg 5: Size of arg 2 buffer in bytes
+**
+** The 4th argument to this function is a pointer to a structure of type
+** Fts3SegFilter, defined in fts3Int.h. The contents of this structure
+** further restrict the range of terms that callbacks are made for and
+** modify the behaviour of this function. See comments above structure
+** definition for details.
+*/
+SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate(
+ Fts3Table *p, /* Virtual table handle */
+ Fts3SegReader **apSegment, /* Array of Fts3SegReader objects */
+ int nSegment, /* Size of apSegment array */
+ Fts3SegFilter *pFilter, /* Restrictions on range of iteration */
+ int (*xFunc)(Fts3Table *, void *, char *, int, char *, int), /* Callback */
+ void *pContext /* Callback context (2nd argument) */
+){
+ int i; /* Iterator variable */
+ char *aBuffer = 0; /* Buffer to merge doclists in */
+ int nAlloc = 0; /* Allocated size of aBuffer buffer */
+ int rc = SQLITE_OK; /* Return code */
+
+ int isIgnoreEmpty = (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY);
+ int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS);
+ int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER);
+ int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX);
+
+ /* If the Fts3SegFilter defines a specific term (or term prefix) to search
+ ** for, then advance each segment iterator until it points to a term of
+ ** equal or greater value than the specified term. This prevents many
+ ** unnecessary merge/sort operations for the case where single segment
+ ** b-tree leaf nodes contain more than one term.
+ */
+ if( pFilter->zTerm ){
+ int nTerm = pFilter->nTerm;
+ const char *zTerm = pFilter->zTerm;
+ for(i=0; i<nSegment; i++){
+ Fts3SegReader *pSeg = apSegment[i];
+ while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
+ rc = fts3SegReaderNext(pSeg);
+ if( rc!=SQLITE_OK ) goto finished;
+ }
+ }
+ }
+
+ fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp);
+ while( apSegment[0]->aNode ){
+ int nTerm = apSegment[0]->nTerm;
+ char *zTerm = apSegment[0]->zTerm;
+ int nMerge = 1;
+
+ /* If this is a prefix-search, and if the term that apSegment[0] points
+ ** to does not share a suffix with pFilter->zTerm/nTerm, then all
+ ** required callbacks have been made. In this case exit early.
+ **
+ ** Similarly, if this is a search for an exact match, and the first term
+ ** of segment apSegment[0] is not a match, exit early.
+ */
+ if( pFilter->zTerm ){
+ if( nTerm<pFilter->nTerm
+ || (!isPrefix && nTerm>pFilter->nTerm)
+ || memcmp(zTerm, pFilter->zTerm, pFilter->nTerm)
+ ){
+ goto finished;
+ }
+ }
+
+ while( nMerge<nSegment
+ && apSegment[nMerge]->aNode
+ && apSegment[nMerge]->nTerm==nTerm
+ && 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm)
+ ){
+ nMerge++;
+ }
+
+ if( nMerge==1 && !isIgnoreEmpty && !isColFilter && isRequirePos ){
+ Fts3SegReader *p0 = apSegment[0];
+ rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist);
+ if( rc!=SQLITE_OK ) goto finished;
+ }else{
+ int nDoclist = 0; /* Size of doclist */
+ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */
+
+ /* The current term of the first nMerge entries in the array
+ ** of Fts3SegReader objects is the same. The doclists must be merged
+ ** and a single term added to the new segment.
+ */
+ for(i=0; i<nMerge; i++){
+ fts3SegReaderFirstDocid(apSegment[i]);
+ }
+ fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp);
+ while( apSegment[0]->pOffsetList ){
+ int j; /* Number of segments that share a docid */
+ char *pList;
+ int nList;
+ int nByte;
+ sqlite3_int64 iDocid = apSegment[0]->iDocid;
+ fts3SegReaderNextDocid(apSegment[0], &pList, &nList);
+ j = 1;
+ while( j<nMerge
+ && apSegment[j]->pOffsetList
+ && apSegment[j]->iDocid==iDocid
+ ){
+ fts3SegReaderNextDocid(apSegment[j], 0, 0);
+ j++;
+ }
+
+ if( isColFilter ){
+ fts3ColumnFilter(pFilter->iCol, &pList, &nList);
+ }
+
+ if( !isIgnoreEmpty || nList>0 ){
+ nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0);
+ if( nDoclist+nByte>nAlloc ){
+ char *aNew;
+ nAlloc = nDoclist+nByte*2;
+ aNew = sqlite3_realloc(aBuffer, nAlloc);
+ if( !aNew ){
+ rc = SQLITE_NOMEM;
+ goto finished;
+ }
+ aBuffer = aNew;
+ }
+ nDoclist += sqlite3Fts3PutVarint(&aBuffer[nDoclist], iDocid-iPrev);
+ iPrev = iDocid;
+ if( isRequirePos ){
+ memcpy(&aBuffer[nDoclist], pList, nList);
+ nDoclist += nList;
+ aBuffer[nDoclist++] = '\0';
+ }
+ }
+
+ fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp);
+ }
+
+ if( nDoclist>0 ){
+ rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist);
+ if( rc!=SQLITE_OK ) goto finished;
+ }
+ }
+
+ /* If there is a term specified to filter on, and this is not a prefix
+ ** search, return now. The callback that corresponds to the required
+ ** term (if such a term exists in the index) has already been made.
+ */
+ if( pFilter->zTerm && !isPrefix ){
+ goto finished;
+ }
+
+ for(i=0; i<nMerge; i++){
+ rc = fts3SegReaderNext(apSegment[i]);
+ if( rc!=SQLITE_OK ) goto finished;
+ }
+ fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp);
+ }
+
+ finished:
+ sqlite3_free(aBuffer);
+ return rc;
+}
+
+/*
+** Merge all level iLevel segments in the database into a single
+** iLevel+1 segment. Or, if iLevel<0, merge all segments into a
+** single segment with a level equal to the numerically largest level
+** currently present in the database.
+**
+** If this function is called with iLevel<0, but there is only one
+** segment in the database, SQLITE_DONE is returned immediately.
+** Otherwise, if successful, SQLITE_OK is returned. If an error occurs,
+** an SQLite error code is returned.
+*/
+static int fts3SegmentMerge(Fts3Table *p, int iLevel){
+ int i; /* Iterator variable */
+ int rc; /* Return code */
+ int iIdx; /* Index of new segment */
+ int iNewLevel; /* Level to create new segment at */
+ sqlite3_stmt *pStmt;
+ SegmentWriter *pWriter = 0;
+ int nSegment = 0; /* Number of segments being merged */
+ Fts3SegReader **apSegment = 0; /* Array of Segment iterators */
+ Fts3SegFilter filter; /* Segment term filter condition */
+
+ if( iLevel<0 ){
+ /* This call is to merge all segments in the database to a single
+ ** segment. The level of the new segment is equal to the the numerically
+ ** greatest segment level currently present in the database. The index
+ ** of the new segment is always 0.
+ */
+ iIdx = 0;
+ rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel);
+ if( nSegment==1 ){
+ return SQLITE_DONE;
+ }
+ }else{
+ /* This call is to merge all segments at level iLevel. Find the next
+ ** available segment index at level iLevel+1. The call to
+ ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to
+ ** a single iLevel+2 segment if necessary.
+ */
+ iNewLevel = iLevel+1;
+ rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx);
+ if( rc!=SQLITE_OK ) return rc;
+ rc = fts3SegmentCount(p, iLevel, &nSegment);
+ }
+ if( rc!=SQLITE_OK ) return rc;
+ assert( nSegment>0 );
+ assert( iNewLevel>=0 );
+
+ /* Allocate space for an array of pointers to segment iterators. */
+ apSegment = (Fts3SegReader**)sqlite3_malloc(sizeof(Fts3SegReader *)*nSegment);
+ if( !apSegment ){
+ return SQLITE_NOMEM;
+ }
+ memset(apSegment, 0, sizeof(Fts3SegReader *)*nSegment);
+
+ /* Allocate a Fts3SegReader structure for each segment being merged. A
+ ** Fts3SegReader stores the state data required to iterate through all
+ ** entries on all leaves of a single segment.
+ */
+ assert( SQL_SELECT_LEVEL+1==SQL_SELECT_ALL_LEVEL);
+ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL+(iLevel<0), &pStmt, 0);
+ if( rc!=SQLITE_OK ) goto finished;
+ sqlite3_bind_int(pStmt, 1, iLevel);
+ for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){
+ rc = fts3SegReaderNew(p, pStmt, i, &apSegment[i]);
+ if( rc!=SQLITE_OK ){
+ goto finished;
+ }
+ }
+ rc = sqlite3_reset(pStmt);
+ pStmt = 0;
+ if( rc!=SQLITE_OK ) goto finished;
+
+ memset(&filter, 0, sizeof(Fts3SegFilter));
+ filter.flags = FTS3_SEGMENT_REQUIRE_POS;
+ filter.flags |= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0);
+ rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment,
+ &filter, fts3MergeCallback, (void *)&pWriter
+ );
+ if( rc!=SQLITE_OK ) goto finished;
+
+ rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment);
+ if( rc==SQLITE_OK ){
+ rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
+ }
+
+ finished:
+ fts3SegWriterFree(pWriter);
+ if( apSegment ){
+ for(i=0; i<nSegment; i++){
+ sqlite3Fts3SegReaderFree(p, apSegment[i]);
+ }
+ sqlite3_free(apSegment);
+ }
+ sqlite3_reset(pStmt);
+ return rc;
+}
+
+/*
+** This is a comparison function used as a qsort() callback when sorting
+** an array of pending terms by term. This occurs as part of flushing
+** the contents of the pending-terms hash table to the database.
+*/
+static int qsortCompare(const void *lhs, const void *rhs){
+ char *z1 = fts3HashKey(*(Fts3HashElem **)lhs);
+ char *z2 = fts3HashKey(*(Fts3HashElem **)rhs);
+ int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs);
+ int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs);
+
+ int n = (n1<n2 ? n1 : n2);
+ int c = memcmp(z1, z2, n);
+ if( c==0 ){
+ c = n1 - n2;
+ }
+ return c;
+}
+
+
+/*
+** Flush the contents of pendingTerms to a level 0 segment.
+*/
+SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
+ Fts3HashElem *pElem;
+ int idx, rc, i;
+ Fts3HashElem **apElem; /* Array of pointers to hash elements */
+ int nElem; /* Number of terms in new segment */
+ SegmentWriter *pWriter = 0; /* Used to write the segment */
+
+ /* Find the number of terms that will make up the new segment. If there
+ ** are no terms, return early (do not bother to write an empty segment).
+ */
+ nElem = fts3HashCount(&p->pendingTerms);
+ if( nElem==0 ){
+ assert( p->nPendingData==0 );
+ return SQLITE_OK;
+ }
+
+ /* Determine the next index at level 0, merging as necessary. */
+ rc = fts3AllocateSegdirIdx(p, 0, &idx);
+ if( rc!=SQLITE_OK ){
+ return rc;
+ }
+
+ apElem = sqlite3_malloc(nElem*sizeof(Fts3HashElem *));
+ if( !apElem ){
+ return SQLITE_NOMEM;
+ }
+
+ i = 0;
+ for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){
+ apElem[i++] = pElem;
+ }
+ assert( i==nElem );
+
+ /* TODO(shess) Should we allow user-defined collation sequences,
+ ** here? I think we only need that once we support prefix searches.
+ ** Also, should we be using qsort()?
+ */
+ if( nElem>1 ){
+ qsort(apElem, nElem, sizeof(Fts3HashElem *), qsortCompare);
+ }
+
+
+ /* Write the segment tree into the database. */
+ for(i=0; rc==SQLITE_OK && i<nElem; i++){
+ const char *z = fts3HashKey(apElem[i]);
+ int n = fts3HashKeysize(apElem[i]);
+ PendingList *pList = fts3HashData(apElem[i]);
+ rc = fts3SegWriterAdd(p, &pWriter, 0, z, n, pList->aData, pList->nData+1);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3SegWriterFlush(p, pWriter, 0, idx);
+ }
+
+ /* Free all allocated resources before returning */
+ fts3SegWriterFree(pWriter);
+ sqlite3_free(apElem);
+ sqlite3Fts3PendingTermsClear(p);
+ return rc;
+}
+
+/*
+** This function does the work for the xUpdate method of FTS3 virtual
+** tables.
+*/
+SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(
+ sqlite3_vtab *pVtab, /* FTS3 vtab object */
+ int nArg, /* Size of argument array */
+ sqlite3_value **apVal, /* Array of arguments */
+ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
+){
+ Fts3Table *p = (Fts3Table *)pVtab;
+ int rc = SQLITE_OK; /* Return Code */
+ int isRemove = 0; /* True for an UPDATE or DELETE */
+ sqlite3_int64 iRemove; /* Rowid removed by UPDATE or DELETE */
+
+ /* If this is a DELETE or UPDATE operation, remove the old record. */
+ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
+ int isEmpty;
+ rc = fts3IsEmpty(p, apVal, &isEmpty);
+ if( rc==SQLITE_OK ){
+ if( isEmpty ){
+ /* Deleting this row means the whole table is empty. In this case
+ ** delete the contents of all three tables and throw away any
+ ** data in the pendingTerms hash table.
+ */
+ rc = fts3DeleteAll(p);
+ }else{
+ isRemove = 1;
+ iRemove = sqlite3_value_int64(apVal[0]);
+ rc = fts3PendingTermsDocid(p, iRemove);
+ if( rc==SQLITE_OK ){
+ rc = fts3DeleteTerms(p, apVal);
+ if( rc==SQLITE_OK ){
+ rc = fts3SqlExec(p, SQL_DELETE_CONTENT, apVal);
+ }
+ }
+ }
+ }
+ }
+
+ /* If this is an INSERT or UPDATE operation, insert the new record. */
+ if( nArg>1 && rc==SQLITE_OK ){
+ rc = fts3InsertData(p, apVal, pRowid);
+ if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){
+ rc = fts3PendingTermsDocid(p, *pRowid);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fts3InsertTerms(p, apVal);
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Flush any data in the pending-terms hash table to disk. If successful,
+** merge all segments in the database (including the new segment, if
+** there was any data to flush) into a single segment.
+*/
+SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){
+ int rc = sqlite3Fts3PendingTermsFlush(p);
+ if( rc==SQLITE_OK ){
+ rc = fts3SegmentMerge(p, -1);
+ }
+ return rc;
+}
+
+#endif
+
+/************** End of fts3_write.c ******************************************/
+/************** Begin file fts3_snippet.c ************************************/
+/*
+** 2009 Oct 23
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+*/
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
+
+
+typedef struct Snippet Snippet;
+
+/*
+** An instance of the following structure keeps track of generated
+** matching-word offset information and snippets.
+*/
+struct Snippet {
+ int nMatch; /* Total number of matches */
+ int nAlloc; /* Space allocated for aMatch[] */
+ struct snippetMatch { /* One entry for each matching term */
+ char snStatus; /* Status flag for use while constructing snippets */
+ short int iCol; /* The column that contains the match */
+ short int iTerm; /* The index in Query.pTerms[] of the matching term */
+ int iToken; /* The index of the matching document token */
+ short int nByte; /* Number of bytes in the term */
+ int iStart; /* The offset to the first character of the term */
+ } *aMatch; /* Points to space obtained from malloc */
+ char *zOffset; /* Text rendering of aMatch[] */
+ int nOffset; /* strlen(zOffset) */
+ char *zSnippet; /* Snippet text */
+ int nSnippet; /* strlen(zSnippet) */
+};
+
+
+/* It is not safe to call isspace(), tolower(), or isalnum() on
+** hi-bit-set characters. This is the same solution used in the
+** tokenizer.
+*/
+/* TODO(shess) The snippet-generation code should be using the
+** tokenizer-generated tokens rather than doing its own local
+** tokenization.
+*/
+/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */
+static int safe_isspace(char c){
+ return (c&0x80)==0 ? isspace(c) : 0;
+}
+static int safe_isalnum(char c){
+ return (c&0x80)==0 ? isalnum(c) : 0;
+}
+
+/*******************************************************************/
+/* DataBuffer is used to collect data into a buffer in piecemeal
+** fashion. It implements the usual distinction between amount of
+** data currently stored (nData) and buffer capacity (nCapacity).
+**
+** dataBufferInit - create a buffer with given initial capacity.
+** dataBufferReset - forget buffer's data, retaining capacity.
+** dataBufferSwap - swap contents of two buffers.
+** dataBufferExpand - expand capacity without adding data.
+** dataBufferAppend - append data.
+** dataBufferAppend2 - append two pieces of data at once.
+** dataBufferReplace - replace buffer's data.
+*/
+typedef struct DataBuffer {
+ char *pData; /* Pointer to malloc'ed buffer. */
+ int nCapacity; /* Size of pData buffer. */
+ int nData; /* End of data loaded into pData. */
+} DataBuffer;
+
+static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){
+ assert( nCapacity>=0 );
+ pBuffer->nData = 0;
+ pBuffer->nCapacity = nCapacity;
+ pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity);
+}
+static void dataBufferReset(DataBuffer *pBuffer){
+ pBuffer->nData = 0;
+}
+static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){
+ assert( nAddCapacity>0 );
+ /* TODO(shess) Consider expanding more aggressively. Note that the
+ ** underlying malloc implementation may take care of such things for
+ ** us already.
+ */
+ if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){
+ pBuffer->nCapacity = pBuffer->nData+nAddCapacity;
+ pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity);
+ }
+}
+static void dataBufferAppend(DataBuffer *pBuffer,
+ const char *pSource, int nSource){
+ assert( nSource>0 && pSource!=NULL );
+ dataBufferExpand(pBuffer, nSource);
+ memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource);
+ pBuffer->nData += nSource;
+}
+static void dataBufferAppend2(DataBuffer *pBuffer,
+ const char *pSource1, int nSource1,
+ const char *pSource2, int nSource2){
+ assert( nSource1>0 && pSource1!=NULL );
+ assert( nSource2>0 && pSource2!=NULL );
+ dataBufferExpand(pBuffer, nSource1+nSource2);
+ memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1);
+ memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2);
+ pBuffer->nData += nSource1+nSource2;
+}
+static void dataBufferReplace(DataBuffer *pBuffer,
+ const char *pSource, int nSource){
+ dataBufferReset(pBuffer);
+ dataBufferAppend(pBuffer, pSource, nSource);
+}
+
+
+/* StringBuffer is a null-terminated version of DataBuffer. */
+typedef struct StringBuffer {
+ DataBuffer b; /* Includes null terminator. */
+} StringBuffer;
+
+static void initStringBuffer(StringBuffer *sb){
+ dataBufferInit(&sb->b, 100);
+ dataBufferReplace(&sb->b, "", 1);
+}
+static int stringBufferLength(StringBuffer *sb){
+ return sb->b.nData-1;
+}
+static char *stringBufferData(StringBuffer *sb){
+ return sb->b.pData;
+}
+
+static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){
+ assert( sb->b.nData>0 );
+ if( nFrom>0 ){
+ sb->b.nData--;
+ dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1);
+ }
+}
+static void append(StringBuffer *sb, const char *zFrom){
+ nappend(sb, zFrom, strlen(zFrom));
+}
+
+static int endsInWhiteSpace(StringBuffer *p){
+ return stringBufferLength(p)>0 &&
+ safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]);
+}
+
+/* If the StringBuffer ends in something other than white space, add a
+** single space character to the end.
+*/
+static void appendWhiteSpace(StringBuffer *p){
+ if( stringBufferLength(p)==0 ) return;
+ if( !endsInWhiteSpace(p) ) append(p, " ");
+}
+
+/* Remove white space from the end of the StringBuffer */
+static void trimWhiteSpace(StringBuffer *p){
+ while( endsInWhiteSpace(p) ){
+ p->b.pData[--p->b.nData-1] = '\0';
+ }
+}
+
+
+/*
+** Release all memory associated with the Snippet structure passed as
+** an argument.
+*/
+static void fts3SnippetFree(Snippet *p){
+ sqlite3_free(p->aMatch);
+ sqlite3_free(p->zOffset);
+ sqlite3_free(p->zSnippet);
+ sqlite3_free(p);
+}
+
+/*
+** Append a single entry to the p->aMatch[] log.
+*/
+static void snippetAppendMatch(
+ Snippet *p, /* Append the entry to this snippet */
+ int iCol, int iTerm, /* The column and query term */
+ int iToken, /* Matching token in document */
+ int iStart, int nByte /* Offset and size of the match */
+){
+ int i;
+ struct snippetMatch *pMatch;
+ if( p->nMatch+1>=p->nAlloc ){
+ p->nAlloc = p->nAlloc*2 + 10;
+ p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) );
+ if( p->aMatch==0 ){
+ p->nMatch = 0;
+ p->nAlloc = 0;
+ return;
+ }
+ }
+ i = p->nMatch++;
+ pMatch = &p->aMatch[i];
+ pMatch->iCol = iCol;
+ pMatch->iTerm = iTerm;
+ pMatch->iToken = iToken;
+ pMatch->iStart = iStart;
+ pMatch->nByte = nByte;
+}
+
+/*
+** Sizing information for the circular buffer used in snippetOffsetsOfColumn()
+*/
+#define FTS3_ROTOR_SZ (32)
+#define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1)
+
+/*
+** Function to iterate through the tokens of a compiled expression.
+**
+** Except, skip all tokens on the right-hand side of a NOT operator.
+** This function is used to find tokens as part of snippet and offset
+** generation and we do nt want snippets and offsets to report matches
+** for tokens on the RHS of a NOT.
+*/
+static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){
+ Fts3Expr *p = *ppExpr;
+ int iToken = *piToken;
+ if( iToken<0 ){
+ /* In this case the expression p is the root of an expression tree.
+ ** Move to the first token in the expression tree.
+ */
+ while( p->pLeft ){
+ p = p->pLeft;
+ }
+ iToken = 0;
+ }else{
+ assert(p && p->eType==FTSQUERY_PHRASE );
+ if( iToken<(p->pPhrase->nToken-1) ){
+ iToken++;
+ }else{
+ iToken = 0;
+ while( p->pParent && p->pParent->pLeft!=p ){
+ assert( p->pParent->pRight==p );
+ p = p->pParent;
+ }
+ p = p->pParent;
+ if( p ){
+ assert( p->pRight!=0 );
+ p = p->pRight;
+ while( p->pLeft ){
+ p = p->pLeft;
+ }
+ }
+ }
+ }
+
+ *ppExpr = p;
+ *piToken = iToken;
+ return p?1:0;
+}
+
+/*
+** Return TRUE if the expression node pExpr is located beneath the
+** RHS of a NOT operator.
+*/
+static int fts3ExprBeneathNot(Fts3Expr *p){
+ Fts3Expr *pParent;
+ while( p ){
+ pParent = p->pParent;
+ if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){
+ return 1;
+ }
+ p = pParent;
+ }
+ return 0;
+}
+
+/*
+** Add entries to pSnippet->aMatch[] for every match that occurs against
+** document zDoc[0..nDoc-1] which is stored in column iColumn.
+*/
+static void snippetOffsetsOfColumn(
+ Fts3Cursor *pCur, /* The fulltest search cursor */
+ Snippet *pSnippet, /* The Snippet object to be filled in */
+ int iColumn, /* Index of fulltext table column */
+ const char *zDoc, /* Text of the fulltext table column */
+ int nDoc /* Length of zDoc in bytes */
+){
+ const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */
+ sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */
+ sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */
+ Fts3Table *pVtab; /* The full text index */
+ int nColumn; /* Number of columns in the index */
+ int i, j; /* Loop counters */
+ int rc; /* Return code */
+ unsigned int match, prevMatch; /* Phrase search bitmasks */
+ const char *zToken; /* Next token from the tokenizer */
+ int nToken; /* Size of zToken */
+ int iBegin, iEnd, iPos; /* Offsets of beginning and end */
+
+ /* The following variables keep a circular buffer of the last
+ ** few tokens */
+ unsigned int iRotor = 0; /* Index of current token */
+ int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */
+ int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */
+
+ pVtab = (Fts3Table *)pCur->base.pVtab;
+ nColumn = pVtab->nColumn;
+ pTokenizer = pVtab->pTokenizer;
+ pTModule = pTokenizer->pModule;
+ rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor);
+ if( rc ) return;
+ pTCursor->pTokenizer = pTokenizer;
+
+ prevMatch = 0;
+ while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){
+ Fts3Expr *pIter = pCur->pExpr;
+ int iIter = -1;
+ iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin;
+ iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin;
+ match = 0;
+ for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){
+ int nPhrase; /* Number of tokens in current phrase */
+ struct PhraseToken *pToken; /* Current token */
+ int iCol; /* Column index */
+
+ if( fts3ExprBeneathNot(pIter) ) continue;
+ nPhrase = pIter->pPhrase->nToken;
+ pToken = &pIter->pPhrase->aToken[iIter];
+ iCol = pIter->pPhrase->iColumn;
+ if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue;
+ if( pToken->n>nToken ) continue;
+ if( !pToken->isPrefix && pToken->n<nToken ) continue;
+ assert( pToken->n<=nToken );
+ if( memcmp(pToken->z, zToken, pToken->n) ) continue;
+ if( iIter>0 && (prevMatch & (1<<i))==0 ) continue;
+ match |= 1<<i;
+ if( i==(FTS3_ROTOR_SZ-2) || nPhrase==iIter+1 ){
+ for(j=nPhrase-1; j>=0; j--){
+ int k = (iRotor-j) & FTS3_ROTOR_MASK;
+ snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j,
+ iRotorBegin[k], iRotorLen[k]);
+ }
+ }
+ }
+ prevMatch = match<<1;
+ iRotor++;
+ }
+ pTModule->xClose(pTCursor);
+}
+
+/*
+** Remove entries from the pSnippet structure to account for the NEAR
+** operator. When this is called, pSnippet contains the list of token
+** offsets produced by treating all NEAR operators as AND operators.
+** This function removes any entries that should not be present after
+** accounting for the NEAR restriction. For example, if the queried
+** document is:
+**
+** "A B C D E A"
+**
+** and the query is:
+**
+** A NEAR/0 E
+**
+** then when this function is called the Snippet contains token offsets
+** 0, 4 and 5. This function removes the "0" entry (because the first A
+** is not near enough to an E).
+**
+** When this function is called, the value pointed to by parameter piLeft is
+** the integer id of the left-most token in the expression tree headed by
+** pExpr. This function increments *piLeft by the total number of tokens
+** in the expression tree headed by pExpr.
+**
+** Return 1 if any trimming occurs. Return 0 if no trimming is required.
+*/
+static int trimSnippetOffsets(
+ Fts3Expr *pExpr, /* The search expression */
+ Snippet *pSnippet, /* The set of snippet offsets to be trimmed */
+ int *piLeft /* Index of left-most token in pExpr */
+){
+ if( pExpr ){
+ if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){
+ return 1;
+ }
+
+ switch( pExpr->eType ){
+ case FTSQUERY_PHRASE:
+ *piLeft += pExpr->pPhrase->nToken;
+ break;
+ case FTSQUERY_NEAR: {
+ /* The right-hand-side of a NEAR operator is always a phrase. The
+ ** left-hand-side is either a phrase or an expression tree that is
+ ** itself headed by a NEAR operator. The following initializations
+ ** set local variable iLeft to the token number of the left-most
+ ** token in the right-hand phrase, and iRight to the right most
+ ** token in the same phrase. For example, if we had:
+ **
+ ** <col> MATCH '"abc def" NEAR/2 "ghi jkl"'
+ **
+ ** then iLeft will be set to 2 (token number of ghi) and nToken will
+ ** be set to 4.
+ */
+ Fts3Expr *pLeft = pExpr->pLeft;
+ Fts3Expr *pRight = pExpr->pRight;
+ int iLeft = *piLeft;
+ int nNear = pExpr->nNear;
+ int nToken = pRight->pPhrase->nToken;
+ int jj, ii;
+ if( pLeft->eType==FTSQUERY_NEAR ){
+ pLeft = pLeft->pRight;
+ }
+ assert( pRight->eType==FTSQUERY_PHRASE );
+ assert( pLeft->eType==FTSQUERY_PHRASE );
+ nToken += pLeft->pPhrase->nToken;
+
+ for(ii=0; ii<pSnippet->nMatch; ii++){
+ struct snippetMatch *p = &pSnippet->aMatch[ii];
+ if( p->iTerm==iLeft ){
+ int isOk = 0;
+ /* Snippet ii is an occurence of query term iLeft in the document.
+ ** It occurs at position (p->iToken) of the document. We now
+ ** search for an instance of token (iLeft-1) somewhere in the
+ ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within
+ ** the set of snippetMatch structures. If one is found, proceed.
+ ** If one cannot be found, then remove snippets ii..(ii+N-1)
+ ** from the matching snippets, where N is the number of tokens
+ ** in phrase pRight->pPhrase.
+ */
+ for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
+ struct snippetMatch *p2 = &pSnippet->aMatch[jj];
+ if( p2->iTerm==(iLeft-1) ){
+ if( p2->iToken>=(p->iToken-nNear-1)
+ && p2->iToken<(p->iToken+nNear+nToken)
+ ){
+ isOk = 1;
+ }
+ }
+ }
+ if( !isOk ){
+ int kk;
+ for(kk=0; kk<pRight->pPhrase->nToken; kk++){
+ pSnippet->aMatch[kk+ii].iTerm = -2;
+ }
+ return 1;
+ }
+ }
+ if( p->iTerm==(iLeft-1) ){
+ int isOk = 0;
+ for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){
+ struct snippetMatch *p2 = &pSnippet->aMatch[jj];
+ if( p2->iTerm==iLeft ){
+ if( p2->iToken<=(p->iToken+nNear+1)
+ && p2->iToken>(p->iToken-nNear-nToken)
+ ){
+ isOk = 1;
+ }
+ }
+ }
+ if( !isOk ){
+ int kk;
+ for(kk=0; kk<pLeft->pPhrase->nToken; kk++){
+ pSnippet->aMatch[ii-kk].iTerm = -2;
+ }
+ return 1;
+ }
+ }
+ }
+ break;
+ }
+ }
+
+ if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/*
+** Compute all offsets for the current row of the query.
+** If the offsets have already been computed, this routine is a no-op.
+*/
+static int snippetAllOffsets(Fts3Cursor *pCsr, Snippet **ppSnippet){
+ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab;
+ int nColumn;
+ int iColumn, i;
+ int iFirst, iLast;
+ int iTerm = 0;
+ Snippet *pSnippet;
+
+ if( pCsr->pExpr==0 ){
+ return SQLITE_OK;
+ }
+
+ pSnippet = (Snippet *)sqlite3_malloc(sizeof(Snippet));
+ *ppSnippet = pSnippet;
+ if( !pSnippet ){
+ return SQLITE_NOMEM;
+ }
+ memset(pSnippet, 0, sizeof(Snippet));
+
+ nColumn = p->nColumn;
+ iColumn = (pCsr->eType - 2);
+ if( iColumn<0 || iColumn>=nColumn ){
+ /* Look for matches over all columns of the full-text index */
+ iFirst = 0;
+ iLast = nColumn-1;
+ }else{
+ /* Look for matches in the iColumn-th column of the index only */
+ iFirst = iColumn;
+ iLast = iColumn;
+ }
+ for(i=iFirst; i<=iLast; i++){
+ const char *zDoc;
+ int nDoc;
+ zDoc = (const char*)sqlite3_column_text(pCsr->pStmt, i+1);
+ nDoc = sqlite3_column_bytes(pCsr->pStmt, i+1);
+ snippetOffsetsOfColumn(pCsr, pSnippet, i, zDoc, nDoc);
+ }
+
+ while( trimSnippetOffsets(pCsr->pExpr, pSnippet, &iTerm) ){
+ iTerm = 0;
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** Convert the information in the aMatch[] array of the snippet
+** into the string zOffset[0..nOffset-1]. This string is used as
+** the return of the SQL offsets() function.
+*/
+static void snippetOffsetText(Snippet *p){
+ int i;
+ int cnt = 0;
+ StringBuffer sb;
+ char zBuf[200];
+ if( p->zOffset ) return;
+ initStringBuffer(&sb);
+ for(i=0; i<p->nMatch; i++){
+ struct snippetMatch *pMatch = &p->aMatch[i];
+ if( pMatch->iTerm>=0 ){
+ /* If snippetMatch.iTerm is less than 0, then the match was
+ ** discarded as part of processing the NEAR operator (see the
+ ** trimSnippetOffsetsForNear() function for details). Ignore
+ ** it in this case
+ */
+ zBuf[0] = ' ';
+ sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d",
+ pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte);
+ append(&sb, zBuf);
+ cnt++;
+ }
+ }
+ p->zOffset = stringBufferData(&sb);
+ p->nOffset = stringBufferLength(&sb);
+}
+
+/*
+** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set
+** of matching words some of which might be in zDoc. zDoc is column
+** number iCol.
+**
+** iBreak is suggested spot in zDoc where we could begin or end an
+** excerpt. Return a value similar to iBreak but possibly adjusted
+** to be a little left or right so that the break point is better.
+*/
+static int wordBoundary(
+ int iBreak, /* The suggested break point */
+ const char *zDoc, /* Document text */
+ int nDoc, /* Number of bytes in zDoc[] */
+ struct snippetMatch *aMatch, /* Matching words */
+ int nMatch, /* Number of entries in aMatch[] */
+ int iCol /* The column number for zDoc[] */
+){
+ int i;
+ if( iBreak<=10 ){
+ return 0;
+ }
+ if( iBreak>=nDoc-10 ){
+ return nDoc;
+ }
+ for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){}
+ while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; }
+ if( i<nMatch ){
+ if( aMatch[i].iStart<iBreak+10 ){
+ return aMatch[i].iStart;
+ }
+ if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){
+ return aMatch[i-1].iStart;
+ }
+ }
+ for(i=1; i<=10; i++){
+ if( safe_isspace(zDoc[iBreak-i]) ){
+ return iBreak - i + 1;
+ }
+ if( safe_isspace(zDoc[iBreak+i]) ){
+ return iBreak + i + 1;
+ }
+ }
+ return iBreak;
+}
+
+
+
+/*
+** Allowed values for Snippet.aMatch[].snStatus
+*/
+#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */
+#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */
+
+/*
+** Generate the text of a snippet.
+*/
+static void snippetText(
+ Fts3Cursor *pCursor, /* The cursor we need the snippet for */
+ Snippet *pSnippet,
+ const char *zStartMark, /* Markup to appear before each match */
+ const char *zEndMark, /* Markup to appear after each match */
+ const char *zEllipsis /* Ellipsis mark */
+){
+ int i, j;
+ struct snippetMatch *aMatch;
+ int nMatch;
+ int nDesired;
+ StringBuffer sb;
+ int tailCol;
+ int tailOffset;
+ int iCol;
+ int nDoc;
+ const char *zDoc;
+ int iStart, iEnd;
+ int tailEllipsis = 0;
+ int iMatch;
+
+
+ sqlite3_free(pSnippet->zSnippet);
+ pSnippet->zSnippet = 0;
+ aMatch = pSnippet->aMatch;
+ nMatch = pSnippet->nMatch;
+ initStringBuffer(&sb);
+
+ for(i=0; i<nMatch; i++){
+ aMatch[i].snStatus = SNIPPET_IGNORE;
+ }
+ nDesired = 0;
+ for(i=0; i<FTS3_ROTOR_SZ; i++){
+ for(j=0; j<nMatch; j++){
+ if( aMatch[j].iTerm==i ){
+ aMatch[j].snStatus = SNIPPET_DESIRED;
+ nDesired++;
+ break;
+ }
+ }
+ }
+
+ iMatch = 0;
+ tailCol = -1;
+ tailOffset = 0;
+ for(i=0; i<nMatch && nDesired>0; i++){
+ if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue;
+ nDesired--;
+ iCol = aMatch[i].iCol;
+ zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1);
+ nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1);
+ iStart = aMatch[i].iStart - 40;
+ iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol);
+ if( iStart<=10 ){
+ iStart = 0;
+ }
+ if( iCol==tailCol && iStart<=tailOffset+20 ){
+ iStart = tailOffset;
+ }
+ if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){
+ trimWhiteSpace(&sb);
+ appendWhiteSpace(&sb);
+ append(&sb, zEllipsis);
+ appendWhiteSpace(&sb);
+ }
+ iEnd = aMatch[i].iStart + aMatch[i].nByte + 40;
+ iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol);
+ if( iEnd>=nDoc-10 ){
+ iEnd = nDoc;
+ tailEllipsis = 0;
+ }else{
+ tailEllipsis = 1;
+ }
+ while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; }
+ while( iStart<iEnd ){
+ while( iMatch<nMatch && aMatch[iMatch].iStart<iStart
+ && aMatch[iMatch].iCol<=iCol ){
+ iMatch++;
+ }
+ if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd
+ && aMatch[iMatch].iCol==iCol ){
+ nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart);
+ iStart = aMatch[iMatch].iStart;
+ append(&sb, zStartMark);
+ nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte);
+ append(&sb, zEndMark);
+ iStart += aMatch[iMatch].nByte;
+ for(j=iMatch+1; j<nMatch; j++){
+ if( aMatch[j].iTerm==aMatch[iMatch].iTerm
+ && aMatch[j].snStatus==SNIPPET_DESIRED ){
+ nDesired--;
+ aMatch[j].snStatus = SNIPPET_IGNORE;
+ }
+ }
+ }else{
+ nappend(&sb, &zDoc[iStart], iEnd - iStart);
+ iStart = iEnd;
+ }
+ }
+ tailCol = iCol;
+ tailOffset = iEnd;
+ }
+ trimWhiteSpace(&sb);
+ if( tailEllipsis ){
+ appendWhiteSpace(&sb);
+ append(&sb, zEllipsis);
+ }
+ pSnippet->zSnippet = stringBufferData(&sb);
+ pSnippet->nSnippet = stringBufferLength(&sb);
+}
+
+SQLITE_PRIVATE void sqlite3Fts3Offsets(
+ sqlite3_context *pCtx, /* SQLite function call context */
+ Fts3Cursor *pCsr /* Cursor object */
+){
+ Snippet *p; /* Snippet structure */
+ int rc = snippetAllOffsets(pCsr, &p);
+ snippetOffsetText(p);
+ sqlite3_result_text(pCtx, p->zOffset, p->nOffset, SQLITE_TRANSIENT);
+ fts3SnippetFree(p);
+}
+
+SQLITE_PRIVATE void sqlite3Fts3Snippet(
+ sqlite3_context *pCtx, /* SQLite function call context */
+ Fts3Cursor *pCsr, /* Cursor object */
+ const char *zStart, /* Snippet start text - "<b>" */
+ const char *zEnd, /* Snippet end text - "</b>" */
+ const char *zEllipsis /* Snippet ellipsis text - "<b>...</b>" */
+){
+ Snippet *p; /* Snippet structure */
+ int rc = snippetAllOffsets(pCsr, &p);
+ snippetText(pCsr, p, zStart, zEnd, zEllipsis);
+ sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT);
+ fts3SnippetFree(p);
+}
+
+#endif
+
+/************** End of fts3_snippet.c ****************************************/
/************** Begin file rtree.c *******************************************/
/*
** 2001 September 15
**
** The author disclaims copyright to this source code. In place of
@@ -107233,12 +105554,10 @@
** May you share freely, never taking more than you give.
**
*************************************************************************
** This file contains code for implementations of the r-tree and r*-tree
** algorithms packaged as an SQLite virtual table module.
-**
-** $Id: rtree.c,v 1.14 2009/08/06 18:36:47 danielk1977 Exp $
*/
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE)
/*
Modified src/sqlite3.h from [da93a40cb3] to [d1070060b5].
@@ -117,13 +117,13 @@
** [sqlite3_libversion_number()], [sqlite3_sourceid()],
** [sqlite_version()] and [sqlite_source_id()].
**
** Requirements: [H10011] [H10014]
*/
-#define SQLITE_VERSION "3.6.20"
-#define SQLITE_VERSION_NUMBER 3006020
-#define SQLITE_SOURCE_ID "2009-11-04 23:02:52 f1c09acaca3e205acf5b077c9b2d0fe35f035c1e"
+#define SQLITE_VERSION "3.6.21"
+#define SQLITE_VERSION_NUMBER 3006021
+#define SQLITE_SOURCE_ID "2009-11-23 13:17:27 39214aee6553db76309851e7aa74fcc02d4f59b7"
/*
** CAPI3REF: Run-Time Library Version Numbers {H10020} <S60100>
** KEYWORDS: sqlite3_version
**