SHA1 Hash:	791a513c289ded077964c0c1332e089a0e65587d
Date:	2008-05-18 17:18:41
User:	drh
Comment:	Improvements to clone performance. Skip cross-linking during the clone the automatically rebuild after the clone. Fixes to rebuild to make sure all artifacts are crosslinked.
Timelines:	ancestors | descendants | both | trunk
Other Links:	files | ZIP archive | manifest

• branch=trunk inherited from [a28c83647d]
• sym-trunk inherited from [a28c83647d]

Modified src/db.c from [d2925f32ae] to [b2f4759aa8].

@@ -50,10 +50,11 @@
 ** structure.
 */
 struct Stmt {
   Blob sql;               /* The SQL for this statement */
   sqlite3_stmt *pStmt;    /* The results of sqlite3_prepare() */
+  Stmt *pNext, *pPrev;    /* List of all unfinalized statements */
 };
 #endif /* INTERFACE */
 
 /*
 ** Call this routine when a database error occurs.
@@ -82,10 +83,11 @@
 static int nCommitHook = 0; /* Number of commit hooks */
 static struct sCommitHook {
   int (*xHook)(void);  /* Functions to call at db_end_transaction() */
   int sequence;        /* Call functions in sequence order */
 } aHook[5];
+static Stmt *pAllStmt = 0;  /* List of all unfinalized statements */
 
 /*
 ** This routine is called by the SQLite commit-hook mechanism
 ** just prior to each omit.  All this routine does is verify
 ** that nBegin really is zero.  That insures that transactions
@@ -164,35 +166,24 @@
   aHook[nCommitHook].xHook = x;
   nCommitHook++;
 }
 
 /*
-** Prepare or reprepare the sqlite3 statement from the raw SQL text.
-*/
-static void reprepare(Stmt *pStmt){
-  sqlite3_stmt *pNew;
-  if( sqlite3_prepare(g.db, blob_buffer(&pStmt->sql), -1, &pNew, 0)!=0 ){
-    db_err("%s\n%s", blob_str(&pStmt->sql), sqlite3_errmsg(g.db));
-  }
-  if( pStmt->pStmt ){
-    sqlite3_transfer_bindings(pStmt->pStmt, pNew);
-    sqlite3_finalize(pStmt->pStmt);
-  }
-  pStmt->pStmt = pNew;
-}
-
-/*
 ** Prepare a Stmt.  Assume that the Stmt is previously uninitialized.
 ** If the input string contains multiple SQL statements, only the first
 ** one is processed.  All statements beyond the first are silently ignored.
 */
 int db_vprepare(Stmt *pStmt, const char *zFormat, va_list ap){
+  char *zSql;
   blob_zero(&pStmt->sql);
   blob_vappendf(&pStmt->sql, zFormat, ap);
   va_end(ap);
-  pStmt->pStmt = 0;
-  reprepare(pStmt);
+  zSql = blob_str(&pStmt->sql);
+  if( sqlite3_prepare_v2(g.db, zSql, -1, &pStmt->pStmt, 0)!=0 ){
+    db_err("%s\n%s", zSql, sqlite3_errmsg(g.db));
+  }
+  pStmt->pNext = pStmt->pPrev = 0;
   return 0;
 }
 int db_prepare(Stmt *pStmt, const char *zFormat, ...){
   int rc;
   va_list ap;
@@ -205,10 +196,14 @@
   int rc = SQLITE_OK;
   if( blob_size(&pStmt->sql)==0 ){
     va_list ap;
     va_start(ap, zFormat);
     rc = db_vprepare(pStmt, zFormat, ap);
+    pStmt->pNext = pAllStmt;
+    pStmt->pPrev = 0;
+    if( pAllStmt ) pAllStmt->pPrev = pStmt;
+    pAllStmt = pStmt;
     va_end(ap);
   }
   return rc;
 }
 
@@ -258,23 +253,12 @@
 /*
 ** Step the SQL statement.  Return either SQLITE_ROW or an error code
 ** or SQLITE_OK if the statement finishes successfully.
 */
 int db_step(Stmt *pStmt){
-  int rc = SQLITE_OK;
-  int limit = 3;
-  while( limit-- ){
-    rc = sqlite3_step(pStmt->pStmt);
-    if( rc==SQLITE_ERROR ){
-      rc = sqlite3_reset(pStmt->pStmt);
-    }
-    if( rc==SQLITE_SCHEMA ){
-      reprepare(pStmt);
-    }else{
-      break;
-    }
-  }
+  int rc;
+  rc = sqlite3_step(pStmt->pStmt);
   return rc;
 }
 
 /*
 ** Reset or finalize a statement.
@@ -287,10 +271,21 @@
 int db_finalize(Stmt *pStmt){
   int rc;
   blob_reset(&pStmt->sql);
   rc = sqlite3_finalize(pStmt->pStmt);
   db_check_result(rc);
+  pStmt->pStmt = 0;
+  if( pStmt->pNext ){
+    pStmt->pNext->pPrev = pStmt->pPrev;
+  }
+  if( pStmt->pPrev ){
+    pStmt->pPrev->pNext = pStmt->pNext;
+  }else if( pAllStmt==pStmt ){
+    pAllStmt = pStmt->pNext;
+  }
+  pStmt->pNext = 0;
+  pStmt->pPrev = 0;
   return rc;
 }
 
 /*
 ** Return the rowid of the most recent insert
@@ -714,10 +709,13 @@
 /*
 ** Close the database connection.
 */
 void db_close(void){
   if( g.db==0 ) return;
+  while( pAllStmt ){
+    db_finalize(pAllStmt);
+  }
   g.repositoryOpen = 0;
   g.localOpen = 0;
   g.configOpen = 0;
   sqlite3_close(g.db);
   g.db = 0;

Modified src/rebuild.c from [51a3d6cdc6] to [bcc65378b8].

@@ -68,15 +68,18 @@
 ** Variables used for progress information
 */
 static int totalSize;       /* Total number of artifacts to process */
 static int processCnt;      /* Number processed so far */
 static int ttyOutput;       /* Do progress output */
+static Bag bagDone;         /* Bag of records rebuilt */
 
 /*
 ** Called after each artifact is processed
 */
-static void rebuild_step_done(void){
+static void rebuild_step_done(rid){
+  assert( bag_find(&bagDone, rid)==0 );
+  bag_insert(&bagDone, rid);
   if( ttyOutput ){
     processCnt++;
     printf("%d (%d%%)...\r", processCnt, (processCnt*100/totalSize));
     fflush(stdout);
   }
@@ -103,11 +106,14 @@
 
   /* Find all children of artifact rid */
   db_prepare(&q1, "SELECT rid FROM delta WHERE srcid=%d", rid);
   bag_init(&children);
   while( db_step(&q1)==SQLITE_ROW ){
-    bag_insert(&children, db_column_int(&q1, 0));
+    int cid = db_column_int(&q1, 0);
+    if( !bag_find(&bagDone, cid) ){
+      bag_insert(&children, cid);
+    }
   }
   nChild = bag_count(&children);
   db_finalize(&q1);
 
   /* Crosslink the artifact */
@@ -143,11 +149,11 @@
       db_finalize(&q2);
       blob_reset(pUse);
     }
   }
   bag_clear(&children);
-  rebuild_step_done();
+  rebuild_step_done(rid);
 }
 
 /*
 ** Core function to rebuild the infomration in the derived tables of a
 ** fossil repository from the blobs. This function is shared between
@@ -163,10 +169,11 @@
 int rebuild_db(int randomize, int doOut){
   Stmt s;
   int errCnt = 0;
   char *zTable;
 
+  bag_init(&bagDone);
   ttyOutput = doOut;
   processCnt = 0;
   db_multi_exec(zSchemaUpdates);
   for(;;){
     zTable = db_text(0,
@@ -204,13 +211,29 @@
     int size = db_column_int(&s, 1);
     if( size>=0 ){
       Blob content;
       content_get(rid, &content);
       rebuild_step(rid, size, &content);
+    }
+  }
+  db_finalize(&s);
+  db_prepare(&s,
+     "SELECT rid, size FROM blob"
+     " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)"
+  );
+  while( db_step(&s)==SQLITE_ROW ){
+    int rid = db_column_int(&s, 0);
+    int size = db_column_int(&s, 1);
+    if( size>=0 ){
+      if( !bag_find(&bagDone, rid) ){
+        Blob content;
+        content_get(rid, &content);
+        rebuild_step(rid, size, &content);
+      }
     }else{
       db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
-      rebuild_step_done();
+      rebuild_step_done(rid);
     }
   }
   db_finalize(&s);
   if( ttyOutput ){
     printf("\n");