dbda8d6ce9 2007-07-21 drh: /* dbda8d6ce9 2007-07-21 drh: ** Copyright (c) 2007 D. Richard Hipp dbda8d6ce9 2007-07-21 drh: ** dbda8d6ce9 2007-07-21 drh: ** This program is free software; you can redistribute it and/or dbda8d6ce9 2007-07-21 drh: ** modify it under the terms of the GNU General Public dbda8d6ce9 2007-07-21 drh: ** License version 2 as published by the Free Software Foundation. dbda8d6ce9 2007-07-21 drh: ** dbda8d6ce9 2007-07-21 drh: ** This program is distributed in the hope that it will be useful, dbda8d6ce9 2007-07-21 drh: ** but WITHOUT ANY WARRANTY; without even the implied warranty of dbda8d6ce9 2007-07-21 drh: ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU dbda8d6ce9 2007-07-21 drh: ** General Public License for more details. dbda8d6ce9 2007-07-21 drh: ** dbda8d6ce9 2007-07-21 drh: ** You should have received a copy of the GNU General Public dbda8d6ce9 2007-07-21 drh: ** License along with this library; if not, write to the dbda8d6ce9 2007-07-21 drh: ** Free Software Foundation, Inc., 59 Temple Place - Suite 330, dbda8d6ce9 2007-07-21 drh: ** Boston, MA 02111-1307, USA. dbda8d6ce9 2007-07-21 drh: ** dbda8d6ce9 2007-07-21 drh: ** Author contact information: dbda8d6ce9 2007-07-21 drh: ** drh@hwaci.com dbda8d6ce9 2007-07-21 drh: ** http://www.hwaci.com/drh/ dbda8d6ce9 2007-07-21 drh: ** dbda8d6ce9 2007-07-21 drh: ******************************************************************************* dbda8d6ce9 2007-07-21 drh: ** dbda8d6ce9 2007-07-21 drh: ** This file contains code used to rebuild the database. dbda8d6ce9 2007-07-21 drh: */ dbda8d6ce9 2007-07-21 drh: #include "config.h" dbda8d6ce9 2007-07-21 drh: #include "rebuild.h" dbda8d6ce9 2007-07-21 drh: #include <assert.h> dbda8d6ce9 2007-07-21 drh: e00384d26d 2007-08-29 aku: /* 62f37c9722 2007-11-26 drh: ** Schema changes 62f37c9722 2007-11-26 drh: */ 62f37c9722 2007-11-26 drh: static const char zSchemaUpdates[] = 62f37c9722 2007-11-26 drh: @ -- Index on the delta table 62f37c9722 2007-11-26 drh: @ -- f088412c49 2007-11-28 drh: @ CREATE INDEX IF NOT EXISTS delta_i1 ON delta(srcid); 62f37c9722 2007-11-26 drh: @ 62f37c9722 2007-11-26 drh: @ -- Artifacts that should not be processed are identified in the 62f37c9722 2007-11-26 drh: @ -- "shun" table. Artifacts that are control-file forgeries or 525cc35bf3 2008-05-17 drh: @ -- spam or artifacts whose contents violate administrative policy 525cc35bf3 2008-05-17 drh: @ -- can be shunned in order to prevent them from contaminating 62f37c9722 2007-11-26 drh: @ -- the repository. 62f37c9722 2007-11-26 drh: @ -- 525cc35bf3 2008-05-17 drh: @ -- Shunned artifacts do not exist in the blob table. Hence they 525cc35bf3 2008-05-17 drh: @ -- have not artifact ID (rid) and we thus must store their full 525cc35bf3 2008-05-17 drh: @ -- UUID. 525cc35bf3 2008-05-17 drh: @ -- 62f37c9722 2007-11-26 drh: @ CREATE TABLE IF NOT EXISTS shun(uuid UNIQUE); 525cc35bf3 2008-05-17 drh: @ 525cc35bf3 2008-05-17 drh: @ -- Artifacts that should not be pushed are stored in the "private" 525cc35bf3 2008-05-17 drh: @ -- table. 525cc35bf3 2008-05-17 drh: @ -- 525cc35bf3 2008-05-17 drh: @ CREATE TABLE IF NOT EXISTS private(rid INTEGER PRIMARY KEY); 62f37c9722 2007-11-26 drh: @ 62f37c9722 2007-11-26 drh: @ -- An entry in this table describes a database query that generates a 62f37c9722 2007-11-26 drh: @ -- table of tickets. 62f37c9722 2007-11-26 drh: @ -- 62f37c9722 2007-11-26 drh: @ CREATE TABLE IF NOT EXISTS reportfmt( 62f37c9722 2007-11-26 drh: @ rn integer primary key, -- Report number 62f37c9722 2007-11-26 drh: @ owner text, -- Owner of this report format (not used) 62f37c9722 2007-11-26 drh: @ title text, -- Title of this report 62f37c9722 2007-11-26 drh: @ cols text, -- A color-key specification 62f37c9722 2007-11-26 drh: @ sqlcode text -- An SQL SELECT statement for this report 62f37c9722 2007-11-26 drh: @ ); 62f37c9722 2007-11-26 drh: ; 62f37c9722 2007-11-26 drh: 62f37c9722 2007-11-26 drh: /* 61ddd63b72 2008-03-06 drh: ** Variables used for progress information 61ddd63b72 2008-03-06 drh: */ 61ddd63b72 2008-03-06 drh: static int totalSize; /* Total number of artifacts to process */ 61ddd63b72 2008-03-06 drh: static int processCnt; /* Number processed so far */ 61ddd63b72 2008-03-06 drh: static int ttyOutput; /* Do progress output */ 791a513c28 2008-05-18 drh: static Bag bagDone; /* Bag of records rebuilt */ 61ddd63b72 2008-03-06 drh: 61ddd63b72 2008-03-06 drh: /* 61ddd63b72 2008-03-06 drh: ** Called after each artifact is processed 61ddd63b72 2008-03-06 drh: */ 791a513c28 2008-05-18 drh: static void rebuild_step_done(rid){ cfb1341ae3 2008-06-02 drh: /* assert( bag_find(&bagDone, rid)==0 ); */ 791a513c28 2008-05-18 drh: bag_insert(&bagDone, rid); 61ddd63b72 2008-03-06 drh: if( ttyOutput ){ 61ddd63b72 2008-03-06 drh: processCnt++; 61ddd63b72 2008-03-06 drh: printf("%d (%d%%)...\r", processCnt, (processCnt*100/totalSize)); 61ddd63b72 2008-03-06 drh: fflush(stdout); 61ddd63b72 2008-03-06 drh: } 61ddd63b72 2008-03-06 drh: } dbda8d6ce9 2007-07-21 drh: dbda8d6ce9 2007-07-21 drh: /* 61ddd63b72 2008-03-06 drh: ** Rebuild cross-referencing information for the artifact 6458f020fc 2008-05-14 drh: ** rid with content pBase and all of its descendants. This 61ddd63b72 2008-03-06 drh: ** routine clears the content buffer before returning. dbda8d6ce9 2007-07-21 drh: */ 5b74febbcc 2008-05-16 drh: static void rebuild_step(int rid, int size, Blob *pBase){ 61ddd63b72 2008-03-06 drh: Stmt q1; 61ddd63b72 2008-03-06 drh: Bag children; 61ddd63b72 2008-03-06 drh: Blob copy; 61ddd63b72 2008-03-06 drh: Blob *pUse; 61ddd63b72 2008-03-06 drh: int nChild, i, cid; 61ddd63b72 2008-03-06 drh: 5b74febbcc 2008-05-16 drh: /* Fix up the "blob.size" field if needed. */ 5b74febbcc 2008-05-16 drh: if( size!=blob_size(pBase) ){ 5b74febbcc 2008-05-16 drh: db_multi_exec( 5b74febbcc 2008-05-16 drh: "UPDATE blob SET size=%d WHERE rid=%d", blob_size(pBase), rid 5b74febbcc 2008-05-16 drh: ); 5b74febbcc 2008-05-16 drh: } 5b74febbcc 2008-05-16 drh: 61ddd63b72 2008-03-06 drh: /* Find all children of artifact rid */ 61ddd63b72 2008-03-06 drh: db_prepare(&q1, "SELECT rid FROM delta WHERE srcid=%d", rid); 61ddd63b72 2008-03-06 drh: bag_init(&children); 61ddd63b72 2008-03-06 drh: while( db_step(&q1)==SQLITE_ROW ){ 791a513c28 2008-05-18 drh: int cid = db_column_int(&q1, 0); 791a513c28 2008-05-18 drh: if( !bag_find(&bagDone, cid) ){ 791a513c28 2008-05-18 drh: bag_insert(&children, cid); 791a513c28 2008-05-18 drh: } 61ddd63b72 2008-03-06 drh: } 61ddd63b72 2008-03-06 drh: nChild = bag_count(&children); 61ddd63b72 2008-03-06 drh: db_finalize(&q1); 61ddd63b72 2008-03-06 drh: 61ddd63b72 2008-03-06 drh: /* Crosslink the artifact */ 61ddd63b72 2008-03-06 drh: if( nChild==0 ){ 61ddd63b72 2008-03-06 drh: pUse = pBase; 61ddd63b72 2008-03-06 drh: }else{ 61ddd63b72 2008-03-06 drh: blob_copy(©, pBase); 61ddd63b72 2008-03-06 drh: pUse = © 61ddd63b72 2008-03-06 drh: } 61ddd63b72 2008-03-06 drh: manifest_crosslink(rid, pUse); 61ddd63b72 2008-03-06 drh: blob_reset(pUse); dbda8d6ce9 2007-07-21 drh: 61ddd63b72 2008-03-06 drh: /* Call all children recursively */ 61ddd63b72 2008-03-06 drh: for(cid=bag_first(&children), i=1; cid; cid=bag_next(&children, cid), i++){ 61ddd63b72 2008-03-06 drh: Stmt q2; 61ddd63b72 2008-03-06 drh: int sz; 61ddd63b72 2008-03-06 drh: if( nChild==i ){ 61ddd63b72 2008-03-06 drh: pUse = pBase; 61ddd63b72 2008-03-06 drh: }else{ 61ddd63b72 2008-03-06 drh: blob_copy(©, pBase); 61ddd63b72 2008-03-06 drh: pUse = © 61ddd63b72 2008-03-06 drh: } 61ddd63b72 2008-03-06 drh: db_prepare(&q2, "SELECT content, size FROM blob WHERE rid=%d", cid); 61ddd63b72 2008-03-06 drh: if( db_step(&q2)==SQLITE_ROW && (sz = db_column_int(&q2,1))>=0 ){ 61ddd63b72 2008-03-06 drh: Blob delta; 61ddd63b72 2008-03-06 drh: db_ephemeral_blob(&q2, 0, &delta); 61ddd63b72 2008-03-06 drh: blob_uncompress(&delta, &delta); 61ddd63b72 2008-03-06 drh: blob_delta_apply(pUse, &delta, pUse); 61ddd63b72 2008-03-06 drh: blob_reset(&delta); 61ddd63b72 2008-03-06 drh: db_finalize(&q2); 5b74febbcc 2008-05-16 drh: rebuild_step(cid, sz, pUse); 61ddd63b72 2008-03-06 drh: }else{ 61ddd63b72 2008-03-06 drh: db_finalize(&q2); 61ddd63b72 2008-03-06 drh: blob_reset(pUse); 61ddd63b72 2008-03-06 drh: } dbda8d6ce9 2007-07-21 drh: } 61ddd63b72 2008-03-06 drh: bag_clear(&children); 791a513c28 2008-05-18 drh: rebuild_step_done(rid); 61ddd63b72 2008-03-06 drh: } 61ddd63b72 2008-03-06 drh: 61ddd63b72 2008-03-06 drh: /* e00384d26d 2007-08-29 aku: ** Core function to rebuild the infomration in the derived tables of a e00384d26d 2007-08-29 aku: ** fossil repository from the blobs. This function is shared between e00384d26d 2007-08-29 aku: ** 'rebuild_database' ('rebuild') and 'reconstruct_cmd' e00384d26d 2007-08-29 aku: ** ('reconstruct'), both of which have to regenerate this information e00384d26d 2007-08-29 aku: ** from scratch. ce1c1a2907 2007-09-21 drh: ** ce1c1a2907 2007-09-21 drh: ** If the randomize parameter is true, then the BLOBs are deliberately ce1c1a2907 2007-09-21 drh: ** extracted in a random order. This feature is used to test the ce1c1a2907 2007-09-21 drh: ** ability of fossil to accept records in any order and still ce1c1a2907 2007-09-21 drh: ** construct a sane repository. e00384d26d 2007-08-29 aku: */ 61ddd63b72 2008-03-06 drh: int rebuild_db(int randomize, int doOut){ e00384d26d 2007-08-29 aku: Stmt s; e00384d26d 2007-08-29 aku: int errCnt = 0; e00384d26d 2007-08-29 aku: char *zTable; e00384d26d 2007-08-29 aku: 791a513c28 2008-05-18 drh: bag_init(&bagDone); 61ddd63b72 2008-03-06 drh: ttyOutput = doOut; 61ddd63b72 2008-03-06 drh: processCnt = 0; 62f37c9722 2007-11-26 drh: db_multi_exec(zSchemaUpdates); dbda8d6ce9 2007-07-21 drh: for(;;){ dbda8d6ce9 2007-07-21 drh: zTable = db_text(0, dbda8d6ce9 2007-07-21 drh: "SELECT name FROM sqlite_master" dbda8d6ce9 2007-07-21 drh: " WHERE type='table'" 525cc35bf3 2008-05-17 drh: " AND name NOT IN ('blob','delta','rcvfrom','user'," 55342eb9fb 2008-05-17 drh: "'config','shun','private','reportfmt')" 525cc35bf3 2008-05-17 drh: ); dbda8d6ce9 2007-07-21 drh: if( zTable==0 ) break; dbda8d6ce9 2007-07-21 drh: db_multi_exec("DROP TABLE %Q", zTable); dbda8d6ce9 2007-07-21 drh: free(zTable); dbda8d6ce9 2007-07-21 drh: } dbda8d6ce9 2007-07-21 drh: db_multi_exec(zRepositorySchema2); fb358ca492 2007-11-24 drh: ticket_create_table(0); a48474bc75 2008-05-29 drh: shun_artifacts(); dbda8d6ce9 2007-07-21 drh: 525cc35bf3 2008-05-17 drh: db_multi_exec( 525cc35bf3 2008-05-17 drh: "INSERT INTO unclustered" 525cc35bf3 2008-05-17 drh: " SELECT rid FROM blob EXCEPT SELECT rid FROM private" 525cc35bf3 2008-05-17 drh: ); 70d5cc86b7 2007-10-05 drh: db_multi_exec( 70d5cc86b7 2007-10-05 drh: "DELETE FROM unclustered" 70d5cc86b7 2007-10-05 drh: " WHERE rid IN (SELECT rid FROM shun JOIN blob USING(uuid))" 70d5cc86b7 2007-10-05 drh: ); ba486fec5a 2007-09-03 drh: db_multi_exec( ba486fec5a 2007-09-03 drh: "DELETE FROM config WHERE name IN ('remote-code', 'remote-maxid')" ba486fec5a 2007-09-03 drh: ); 61ddd63b72 2008-03-06 drh: totalSize = db_int(0, "SELECT count(*) FROM blob"); ce1c1a2907 2007-09-21 drh: db_prepare(&s, 61ddd63b72 2008-03-06 drh: "SELECT rid, size FROM blob" 61ddd63b72 2008-03-06 drh: " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)" 61ddd63b72 2008-03-06 drh: " AND NOT EXISTS(SELECT 1 FROM delta WHERE rid=blob.rid)" ce1c1a2907 2007-09-21 drh: ); 73bddaebb9 2007-08-09 drh: while( db_step(&s)==SQLITE_ROW ){ 73bddaebb9 2007-08-09 drh: int rid = db_column_int(&s, 0); 73bddaebb9 2007-08-09 drh: int size = db_column_int(&s, 1); 73bddaebb9 2007-08-09 drh: if( size>=0 ){ 73bddaebb9 2007-08-09 drh: Blob content; 73bddaebb9 2007-08-09 drh: content_get(rid, &content); 5b74febbcc 2008-05-16 drh: rebuild_step(rid, size, &content); 791a513c28 2008-05-18 drh: } 791a513c28 2008-05-18 drh: } 791a513c28 2008-05-18 drh: db_finalize(&s); 791a513c28 2008-05-18 drh: db_prepare(&s, 791a513c28 2008-05-18 drh: "SELECT rid, size FROM blob" 791a513c28 2008-05-18 drh: " WHERE NOT EXISTS(SELECT 1 FROM shun WHERE uuid=blob.uuid)" 791a513c28 2008-05-18 drh: ); dbda8d6ce9 2007-07-21 drh: while( db_step(&s)==SQLITE_ROW ){ dbda8d6ce9 2007-07-21 drh: int rid = db_column_int(&s, 0); 791a513c28 2008-05-18 drh: int size = db_column_int(&s, 1); 791a513c28 2008-05-18 drh: if( size>=0 ){ 791a513c28 2008-05-18 drh: if( !bag_find(&bagDone, rid) ){ 791a513c28 2008-05-18 drh: Blob content; 791a513c28 2008-05-18 drh: content_get(rid, &content); 791a513c28 2008-05-18 drh: rebuild_step(rid, size, &content); 791a513c28 2008-05-18 drh: } 73bddaebb9 2007-08-09 drh: }else{ e1c1877c99 2007-09-08 drh: db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid); 791a513c28 2008-05-18 drh: rebuild_step_done(rid); 73bddaebb9 2007-08-09 drh: } 73bddaebb9 2007-08-09 drh: } e38fc922a5 2007-10-11 drh: db_finalize(&s); e38fc922a5 2007-10-11 drh: if( ttyOutput ){ e38fc922a5 2007-10-11 drh: printf("\n"); dbda8d6ce9 2007-07-21 drh: } e00384d26d 2007-08-29 aku: return errCnt; e00384d26d 2007-08-29 aku: } 6607844a01 2007-08-18 drh: 6607844a01 2007-08-18 drh: /* 6607844a01 2007-08-18 drh: ** COMMAND: rebuild 6607844a01 2007-08-18 drh: ** 6607844a01 2007-08-18 drh: ** Usage: %fossil rebuild REPOSITORY 6607844a01 2007-08-18 drh: ** 6607844a01 2007-08-18 drh: ** Reconstruct the named repository database from the core 6607844a01 2007-08-18 drh: ** records. Run this command after updating the fossil 6607844a01 2007-08-18 drh: ** executable in a way that changes the database schema. 6607844a01 2007-08-18 drh: */ 6607844a01 2007-08-18 drh: void rebuild_database(void){ 6607844a01 2007-08-18 drh: int forceFlag; ce1c1a2907 2007-09-21 drh: int randomizeFlag; e00384d26d 2007-08-29 aku: int errCnt; dbda8d6ce9 2007-07-21 drh: 6607844a01 2007-08-18 drh: forceFlag = find_option("force","f",0)!=0; ce1c1a2907 2007-09-21 drh: randomizeFlag = find_option("randomize", 0, 0)!=0; 6607844a01 2007-08-18 drh: if( g.argc!=3 ){ 6607844a01 2007-08-18 drh: usage("REPOSITORY-FILENAME"); 6607844a01 2007-08-18 drh: } 6607844a01 2007-08-18 drh: db_open_repository(g.argv[2]); 6607844a01 2007-08-18 drh: db_begin_transaction(); 61ddd63b72 2008-03-06 drh: ttyOutput = 1; e38fc922a5 2007-10-11 drh: errCnt = rebuild_db(randomizeFlag, 1); dbda8d6ce9 2007-07-21 drh: if( errCnt && !forceFlag ){ dbda8d6ce9 2007-07-21 drh: printf("%d errors. Rolling back changes. Use --force to force a commit.\n", dbda8d6ce9 2007-07-21 drh: errCnt); dbda8d6ce9 2007-07-21 drh: db_end_transaction(1); dbda8d6ce9 2007-07-21 drh: }else{ dbda8d6ce9 2007-07-21 drh: db_end_transaction(0); dbda8d6ce9 2007-07-21 drh: } dbda8d6ce9 2007-07-21 drh: }
File Annotation