Overview
| SHA1 Hash: | eea381f416beab14a5356f36f1c9108ba9843b92 |
|---|---|
| Date: | 2007-08-09 03:19:18 |
| User: | drh |
| Comment: | Progress toward delta compression on the xfer protocol. The compression works well. But the client is not telling the server what files it has so the server does not have anything to delta against. |
| Timelines: | ancestors | descendants | both | trunk |
| Other Links: | files | ZIP archive | manifest |
Tags And Properties
- branch=trunk inherited from [a28c83647d]
- sym-trunk inherited from [a28c83647d]
Changes
[hide diffs]Modified src/rebuild.c from [04451053b1] to [6ab5a0bcde].
@@ -46,10 +46,13 @@
usage("REPOSITORY-FILENAME");
}
errCnt = 0;
db_open_repository(g.argv[2]);
db_begin_transaction();
+ db_multi_exec(
+ "CREATE INDEX IF NOT EXISTS delta_i1 ON delta(srcid);"
+ );
for(;;){
zTable = db_text(0,
"SELECT name FROM sqlite_master"
" WHERE type='table'"
" AND name NOT IN ('blob','delta','rcvfrom','user','config')");
Modified src/schema.c from [39376154b8] to [468ea581be].
@@ -80,10 +80,11 @@ @ ); @ CREATE TABLE delta( @ rid INTEGER PRIMARY KEY, -- Record ID @ srcid INTEGER NOT NULL REFERENCES blob -- Record holding source document @ ); +@ CREATE INDEX delta_i1 ON delta(srcid); @ @ -- Whenever new blobs are received into the repository, an entry @ -- in this table records the source of the blob. @ -- @ CREATE TABLE rcvfrom(
Modified src/xfer.c from [3a9a49107f] to [338dc3ed99].
@@ -25,10 +25,83 @@
*/
#include "config.h"
#include "xfer.h"
/*
+** Try to locate a record that is similar to rid and is a likely
+** candidate for delta against rid. The similar record must be
+** referenced in the onremote table.
+**
+** Return the integer record ID of the similar record. Or return
+** 0 if none is found.
+*/
+static int similar_record(int rid, int traceFlag){
+ int inCnt, outCnt;
+ Stmt q;
+ int queue[100];
+
+ db_prepare(&q,
+ "SELECT srcid, EXISTS(SELECT 1 FROM onremote WHERE rid=srcid)"
+ " FROM delta"
+ " WHERE rid=:x"
+ " UNION ALL "
+ "SELECT rid, EXISTS(SELECT 1 FROM onremote WHERE rid=delta.rid)"
+ " FROM delta"
+ " WHERE srcid=:x"
+ );
+ queue[0] = rid;
+ inCnt = 1;
+ outCnt = 0;
+ while( outCnt<inCnt ){
+ int xid = queue[outCnt%64];
+ outCnt++;
+ db_bind_int(&q, ":x", xid);
+ if( traceFlag ) printf("xid=%d\n", xid);
+ while( db_step(&q)==SQLITE_ROW ){
+ int nid = db_column_int(&q, 0);
+ int hit = db_column_int(&q, 1);
+ if( traceFlag ) printf("nid=%d hit=%d\n", nid, hit);
+ if( hit ){
+ db_finalize(&q);
+ return nid;
+ }
+ if( inCnt<sizeof(queue)/sizeof(queue[0]) ){
+ int i;
+ for(i=0; i<inCnt && queue[i]!=nid; i++){}
+ if( i>=inCnt ){
+ queue[inCnt++] = nid;
+ }
+ }
+ }
+ db_reset(&q);
+ }
+ db_finalize(&q);
+ return 0;
+}
+
+/*
+** COMMAND: test-similar-record
+*/
+void test_similar_record(void){
+ int i;
+ if( g.argc<4 ){
+ usage("SRC ONREMOTE...");
+ }
+ db_must_be_within_tree();
+ db_multi_exec(
+ "CREATE TEMP TABLE onremote(rid INTEGER PRIMARY KEY);"
+ );
+ for(i=3; i<g.argc; i++){
+ int rid = name_to_rid(g.argv[i]);
+ printf("%s -> %d\n", g.argv[i], rid);
+ db_multi_exec("INSERT INTO onremote VALUES(%d)", rid);
+ }
+ printf("similar: %d\n", similar_record(name_to_rid(g.argv[2]), 1));
+}
+
+
+/*
** The aToken[0..nToken-1] blob array is a parse of a "file" line
** message. This routine finishes parsing that message and does
** a record insert of the file.
**
** The file line is in one of the following two forms:
@@ -87,43 +160,50 @@
** to pOut. Otherwise, append the text to the CGI output.
*/
static int send_file(int rid, Blob *pOut){
Blob content, uuid;
int size;
-
-#if 0
-SELECT srcid FROM delta
- WHERE rid=%d
- AND EXISTS(SELECT 1 FROM onremote WHERE rid=srcid)
-UNION ALL
-SELECT id FROM delta
- WHERE srcid=%d
- AND EXISTS(SELECT 1 FROM onremote WHERE rid=delta.rid)
-LIMIT 1
-#endif
-
- /* TODO:
- ** Check for related files in the onremote TEMP table. If related
- ** files are found, then send a delta rather than the whole file.
- */
+ int srcid;
+
blob_zero(&uuid);
db_blob(&uuid, "SELECT uuid FROM blob WHERE rid=%d AND size>=0", rid);
if( blob_size(&uuid)==0 ){
return 0;
}
content_get(rid, &content);
- size = blob_size(&content);
- if( pOut ){
- blob_appendf(pOut, "file %b %d\n", &uuid, size);
- blob_append(pOut, blob_buffer(&content), size);
+
+ srcid = similar_record(rid, 0);
+ if( srcid ){
+ Blob src, delta;
+ Blob srcuuid;
+ content_get(srcid, &src);
+ blob_delta_create(&src, &content, &delta);
+ blob_reset(&src);
+ blob_reset(&content);
+ blob_zero(&srcuuid);
+ db_blob(&srcuuid, "SELECT uuid FROM blob WHERE rid=%d", srcid);
+ size = blob_size(&delta);
+ if( pOut ){
+ blob_appendf(pOut, "file %b %b %d\n", &uuid, &srcuuid, size);
+ blob_append(pOut, blob_buffer(&delta), size);
+ }else{
+ cgi_printf("file %b %b %d\n", &uuid, &srcuuid, size);
+ cgi_append_content(blob_buffer(&delta), size);
+ }
}else{
- cgi_printf("file %b %d\n", &uuid, size);
- cgi_append_content(blob_buffer(&content), size);
- }
- blob_reset(&content);
- blob_reset(&uuid);
+ size = blob_size(&content);
+ if( pOut ){
+ blob_appendf(pOut, "file %b %d\n", &uuid, size);
+ blob_append(pOut, blob_buffer(&content), size);
+ }else{
+ cgi_printf("file %b %d\n", &uuid, size);
+ cgi_append_content(blob_buffer(&content), size);
+ }
+ blob_reset(&content);
+ blob_reset(&uuid);
+ }
db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
return size;
}
@@ -134,33 +214,11 @@
int iRidSent = 0;
int sent = 0;
int nSent = 0;
int maxSize = db_get_int("http-msg-size", 500000);
Stmt q;
-#if 0
- db_multi_exec(
- "CREATE TEMP TABLE priority(rid INTEGER PRIMARY KEY);"
- "INSERT INTO priority"
- " SELECT srcid FROM delta"
- " WHERE EXISTS(SELECT 1 FROM onremote WHERE onremote.rid=delta.rid)"
- " AND EXISTS(SELECT 1 FROM pending WHERE delta.srcid=pending.rid);"
- "INSERT OR IGNORE INTO priority"
- " SELECT rid FROM delta"
- " WHERE EXISTS(SELECT 1 FROM onremote WHERE onremote.rid=delta.srcid)"
- " AND EXISTS(SELECT 1 FROM pending WHERE delta.rid=pending.rid);"
- );
- while( sent<maxSize && (rid = db_int(0, "SELECT rid FROM priority"))!=0 ){
- sent += send_file(rid, pOut);
- db_multi_exec(
- "INSERT OR IGNORE INTO priority"
- " SELECT srcid FROM delta WHERE rid=%d"
- " UNION ALL"
- " SELECT rid FROM delta WHERE srcid=%d",
- rid, rid
- );
- }
-#endif
+
db_prepare(&q, "SELECT rid FROM pending ORDER BY rid");
while( db_step(&q)==SQLITE_ROW ){
int rid = db_column_int(&q, 0);
if( sent<maxSize ){
sent += send_file(rid, pOut);
@@ -186,13 +244,10 @@
*/
if( nSent>0 ){
db_multi_exec("DELETE FROM pending WHERE rid <= %d", iRidSent);
}
-#if 0
- db_multi_exec("DROP TABLE priority");
-#endif
return nSent;
}
/*
@@ -592,12 +647,12 @@
}
db_finalize(&q);
}
/* Exchange messages with the server */
- printf("Send: %d files, %d requests, %d other messages\n",
- nFile, nReq, nMsg);
+ printf("Send: %3d files, %3d requests, %3d other msgs, %8d bytes\n",
+ nFile, nReq, nMsg, blob_size(&send));
nFileSend = nFile;
nFile = nReq = nMsg = 0;
http_exchange(&send, &recv);
blob_reset(&send);
@@ -704,13 +759,13 @@
if( blob_size(&errmsg) ){
fossil_fatal("%b", &errmsg);
}
blobarray_reset(aToken, nToken);
}
- blob_reset(&recv);
- printf("Received: %d files, %d requests, %d other messages\n",
- nFile, nReq, nMsg);
+ printf("Received: %3d files, %3d requests, %3d other msgs, %8d bytes\n",
+ nFile, nReq, nMsg, blob_size(&recv));
+ blob_reset(&recv);
if( nFileSend + nFile==0 ){
nNoFileCycle++;
if( nNoFileCycle>1 ){
go = 0;
}