Overview
| SHA1 Hash: | 73bddaebb9ec5d6eea26b46f56d2b82b68420f01 |
|---|---|
| Date: | 2007-08-09 17:42:59 |
| User: | drh |
| Comment: | The delta compress on xfer is working better now, but still needs work. |
| Timelines: | ancestors | descendants | both | trunk |
| Other Links: | files | ZIP archive | manifest |
Tags And Properties
- branch=trunk inherited from [a28c83647d]
- sym-trunk inherited from [a28c83647d]
Changes
[hide diffs]Modified src/content.c from [f6a48a05e1] to [9b11b55b6f].
@@ -196,10 +196,11 @@
g.rcvid, size, rid
);
blob_compress(pBlob, &cmpr);
db_bind_blob(&s1, ":data", &cmpr);
db_exec(&s1);
+ db_multi_exec("DELETE FROM phantom WHERE rid=%d", rid);
}else{
/* We are creating a new entry */
db_prepare(&s1,
"INSERT INTO blob(rcvid,size,uuid,content)"
"VALUES(%d,%d,'%s',:data)",
@@ -209,10 +210,13 @@
blob_compress(pBlob, &cmpr);
db_bind_blob(&s1, ":data", &cmpr);
}
db_exec(&s1);
rid = db_last_insert_rowid();
+ if( !pBlob ){
+ db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
+ }
}
/* Finish the transaction and cleanup */
db_finalize(&s1);
Modified src/delta.c from [ee93661e27] to [001257c491].
@@ -333,11 +333,11 @@
/* Begin scanning the target file and generating copy commands and
** literal sections of the delta.
*/
base = 0; /* We have already generated everything before zOut[base] */
- while( base<lenOut-NHASH ){
+ while( base+NHASH<lenOut ){
int iSrc, iBlock;
unsigned int bestCnt, bestOfst, bestLitsz;
hash_init(&h, &zOut[base]);
i = 0; /* Trying to match a landmark against zOut[base+i] */
bestCnt = 0;
Modified src/rebuild.c from [6ab5a0bcde] to [6b83ecc26a].
@@ -60,17 +60,22 @@
db_multi_exec("DROP TABLE %Q", zTable);
free(zTable);
}
db_multi_exec(zRepositorySchema2);
- db_prepare(&s, "SELECT rid FROM blob");
+ db_prepare(&s, "SELECT rid, size FROM blob");
while( db_step(&s)==SQLITE_ROW ){
int rid = db_column_int(&s, 0);
- Blob content;
- content_get(rid, &content);
- manifest_crosslink(rid, &content);
- blob_reset(&content);
+ int size = db_column_int(&s, 1);
+ if( size>=0 ){
+ Blob content;
+ content_get(rid, &content);
+ manifest_crosslink(rid, &content);
+ blob_reset(&content);
+ }else{
+ db_multi_exec("INSERT INTO phantom VALUES(%d)", rid);
+ }
}
if( errCnt && !forceFlag ){
printf("%d errors. Rolling back changes. Use --force to force a commit.\n",
errCnt);
Modified src/schema.c from [468ea581be] to [14ace4fdfb].
@@ -135,10 +135,12 @@ @ fid INTEGER REFERENCES blob, -- Changed file ID in this manifest @ fnid INTEGER REFERENCES filename -- Name of the file @ ); @ CREATE INDEX mlink_i1 ON mlink(mid); @ CREATE INDEX mlink_i2 ON mlink(fnid); +@ CREATE INDEX mlink_i3 ON mlink(fid); +@ CREATE INDEX mlink_i4 ON mlink(pid); @ @ -- Parent/child linkages @ -- @ CREATE TABLE plink( @ pid INTEGER REFERENCES blob, -- Parent manifest @@ -160,13 +162,15 @@ @ comment TEXT @ ); @ CREATE INDEX event_i1 ON event(mtime); @ CREATE INDEX event_i2 ON event(objid); @ -@ -- Make sure reading DELTA by SRCID is efficient +@ -- A record of phantoms @ -- -@ CREATE INDEX IF NOT EXISTS delta_srcid ON delta(srcid, rid); +@ CREATE TABLE phantom( +@ rid INTEGER PRIMARY KEY -- Record ID of the phantom +@ ); @ @ -- Aggregated ticket information @ -- @ CREATE TABLE tkt( @ tktid INTEGER PRIMARY KEY, -- Internal ticket ID
Modified src/xfer.c from [eec14d5874] to [78443669a5].
@@ -34,51 +34,79 @@
** Return the integer record ID of the similar record. Or return
** 0 if none is found.
*/
static int similar_record(int rid, int traceFlag){
int inCnt, outCnt;
+ int i;
Stmt q;
int queue[100];
-
-return 0;
-
- db_prepare(&q,
+ static const char *azQuery[] = {
+ /* Scan the delta table first */
"SELECT srcid, EXISTS(SELECT 1 FROM onremote WHERE rid=srcid)"
" FROM delta"
" WHERE rid=:x"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=srcid)"
" UNION ALL "
"SELECT rid, EXISTS(SELECT 1 FROM onremote WHERE rid=delta.rid)"
" FROM delta"
" WHERE srcid=:x"
- );
- queue[0] = rid;
- inCnt = 1;
- outCnt = 0;
- while( outCnt<inCnt ){
- int xid = queue[outCnt%64];
- outCnt++;
- db_bind_int(&q, ":x", xid);
- if( traceFlag ) printf("xid=%d\n", xid);
- while( db_step(&q)==SQLITE_ROW ){
- int nid = db_column_int(&q, 0);
- int hit = db_column_int(&q, 1);
- if( traceFlag ) printf("nid=%d hit=%d\n", nid, hit);
- if( hit ){
- db_finalize(&q);
- return nid;
- }
- if( inCnt<sizeof(queue)/sizeof(queue[0]) ){
- int i;
- for(i=0; i<inCnt && queue[i]!=nid; i++){}
- if( i>=inCnt ){
- queue[inCnt++] = nid;
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=delta.rid)",
+
+ /* Then the plink table */
+ "SELECT pid, EXISTS(SELECT 1 FROM onremote WHERE rid=pid)"
+ " FROM plink"
+ " WHERE cid=:x"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
+ " UNION ALL "
+ "SELECT cid, EXISTS(SELECT 1 FROM onremote WHERE rid=cid)"
+ " FROM plink"
+ " WHERE pid=:x"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=cid)",
+
+ /* Finally the mlink table */
+ "SELECT pid, EXISTS(SELECT 1 FROM onremote WHERE rid=pid)"
+ " FROM mlink"
+ " WHERE fid=:x AND pid>0"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
+ " UNION ALL "
+ "SELECT fid, EXISTS(SELECT 1 FROM onremote WHERE rid=fid)"
+ " FROM mlink"
+ " WHERE pid=:x AND fid>0"
+ " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=fid)",
+ };
+
+ for(i=0; i<sizeof(azQuery)/sizeof(azQuery[0]); i++){
+ db_prepare(&q, azQuery[i]);
+ queue[0] = rid;
+ inCnt = 1;
+ outCnt = 0;
+ if( traceFlag ) printf("PASS %d\n", i+1);
+ while( outCnt<inCnt ){
+ int xid = queue[outCnt%64];
+ outCnt++;
+ db_bind_int(&q, ":x", xid);
+ if( traceFlag ) printf("xid=%d\n", xid);
+ while( db_step(&q)==SQLITE_ROW ){
+ int nid = db_column_int(&q, 0);
+ int hit = db_column_int(&q, 1);
+ if( traceFlag ) printf("nid=%d hit=%d\n", nid, hit);
+ if( hit ){
+ db_finalize(&q);
+ return nid;
+ }
+ if( inCnt<sizeof(queue)/sizeof(queue[0]) ){
+ int i;
+ for(i=0; i<inCnt && queue[i]!=nid; i++){}
+ if( i>=inCnt ){
+ queue[inCnt++] = nid;
+ }
}
}
- }
- db_reset(&q);
- }
- db_finalize(&q);
+ db_reset(&q);
+ }
+ db_finalize(&q);
+ }
return 0;
}
/*
** COMMAND: test-similar-record
@@ -172,43 +200,37 @@
if( blob_size(&uuid)==0 ){
return 0;
}
content_get(rid, &content);
- srcid = similar_record(rid, 0);
- if( srcid ){
- Blob src, delta;
- Blob srcuuid;
- content_get(srcid, &src);
- blob_delta_create(&src, &content, &delta);
- blob_reset(&src);
- blob_reset(&content);
- blob_zero(&srcuuid);
- db_blob(&srcuuid, "SELECT uuid FROM blob WHERE rid=%d", srcid);
- size = blob_size(&delta);
- if( pOut ){
- blob_appendf(pOut, "file %b %b %d\n", &uuid, &srcuuid, size);
- blob_append(pOut, blob_buffer(&delta), size);
- }else{
- cgi_printf("file %b %b %d\n", &uuid, &srcuuid, size);
- cgi_append_content(blob_buffer(&delta), size);
- }
- blob_reset(&delta);
- blob_reset(&srcuuid);
- blob_reset(&uuid);
+ if( blob_size(&content)>100 ){
+ srcid = similar_record(rid, 0);
+ if( srcid ){
+ Blob src;
+ content_get(srcid, &src);
+ if( blob_size(&src)>100 ){
+ Blob delta;
+ blob_delta_create(&src, &content, &delta);
+ blob_reset(&content);
+ content = delta;
+ blob_append(&uuid, " ", 1);
+ blob_append(&content, "\n", 1);
+ db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d", srcid);
+ }
+ blob_reset(&src);
+ }
+ }
+ size = blob_size(&content);
+ if( pOut ){
+ blob_appendf(pOut, "file %b %d\n", &uuid, size);
+ blob_append(pOut, blob_buffer(&content), size);
}else{
- size = blob_size(&content);
- if( pOut ){
- blob_appendf(pOut, "file %b %d\n", &uuid, size);
- blob_append(pOut, blob_buffer(&content), size);
- }else{
- cgi_printf("file %b %d\n", &uuid, size);
- cgi_append_content(blob_buffer(&content), size);
- }
- blob_reset(&content);
- blob_reset(&uuid);
+ cgi_printf("file %b %d\n", &uuid, size);
+ cgi_append_content(blob_buffer(&content), size);
}
+ blob_reset(&content);
+ blob_reset(&uuid);
db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
return size;
}
@@ -467,11 +489,11 @@
}
isPull = 1;
@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
db_multi_exec(
"INSERT OR IGNORE INTO pending(rid) "
- "SELECT rid FROM blob WHERE size>=0"
+ "SELECT mid FROM mlink JOIN blob ON mid=rid"
);
}else
/* login USER NONCE SIGNATURE
**
@@ -495,14 +517,22 @@
}
/* The input message has now been processed. Generate a reply. */
if( isPush ){
Stmt q;
- db_prepare(&q, "SELECT uuid FROM blob WHERE size<0");
- while( db_step(&q)==SQLITE_ROW ){
+ int nReq = 0;
+ db_prepare(&q, "SELECT uuid, rid FROM phantom JOIN blob USING (rid)");
+ while( db_step(&q)==SQLITE_ROW && nReq++ < 200 ){
const char *zUuid = db_column_text(&q, 0);
+ int rid = db_column_int(&q, 1);
+ int xid = similar_record(rid, 0);
@ gimme %s(zUuid)
+ if( xid ){
+ char *zXUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", xid);
+ @ igot %s(zXUuid);
+ free(zXUuid);
+ }
}
db_finalize(&q);
}
if( isPull ){
send_all_pending(0);
@@ -622,15 +652,21 @@
if( pullFlag ){
/* Send gimme message for every phantom that we hold.
*/
Stmt q;
- db_prepare(&q, "SELECT uuid FROM blob WHERE size<0");
- while( db_step(&q)==SQLITE_ROW ){
+ db_prepare(&q, "SELECT uuid, rid FROM phantom JOIN blob USING (rid)");
+ while( db_step(&q)==SQLITE_ROW && nReq<200 ){
const char *zUuid = db_column_text(&q, 0);
+ int rid = db_column_int(&q, 1);
+ int xid = similar_record(rid, 0);
blob_appendf(&send,"gimme %s\n", zUuid);
nReq++;
+ if( xid ){
+ blob_appendf(&send, "igot %z\n",
+ db_text(0, "SELECT uuid FROM blob WHERE rid=%d", xid));
+ }
}
db_finalize(&q);
}
if( pushFlag ){
@@ -716,11 +752,11 @@
if( db_changes()>0 ){
go = 1;
}
}
if( pullFlag && !go &&
- db_exists("SELECT 1 FROM blob WHERE rid=%d AND size<0", rid) ){
+ db_exists("SELECT 1 FROM phantom WHERE rid=%d", rid) ){
go = 1;
}
}else if( pullFlag ){
go = 1;
content_put(0, blob_str(&aToken[1]));