Overview
| SHA1 Hash: | e1c1877c99bf13c0ace4c6f0ba74c9e304aca372 |
|---|---|
| Date: | 2007-09-08 16:01:28 |
| User: | drh |
| Comment: | Sync using clusters appears to work. More testing is needed before we go live. |
| Timelines: | ancestors | descendants | both | trunk |
| Other Links: | files | ZIP archive | manifest |
Tags And Properties
- branch=trunk inherited from [a28c83647d]
- sym-trunk inherited from [a28c83647d]
Changes
[hide diffs]Modified ideas.txt from [084c43a653] to [44635deb96].
@@ -34,12 +34,12 @@
* Header ends with a blank line. wiki content follows.
Cluster format:
- "Cluster"
- (UUID\n)+
+ M+ uuid
+ Z manifest-cksum
* Cluster generated in server mode only.
* Embargo cluster that reference phantoms or other embargoed clusters.
* Never send or ihave an embargoed cluster
@@ -67,10 +67,11 @@
------
* Server receives file message
* Server creates phantoms for unknown ihaves
* Server sends gimme messages for all phantoms
------
+ * Client clears its unsent table
* For each gimme message add an entry to wanted
* Halt if the wanted table is empty
Details on new pull algorithm:
Modified src/checkin.c from [e53057ad70] to [2b31bba6d0].
@@ -391,10 +391,11 @@
nrid = content_put(&content, 0, 0);
if( rid>0 ){
content_deltify(rid, nrid, 0);
}
db_multi_exec("UPDATE vfile SET mrid=%d, rid=%d WHERE id=%d", nrid,nrid,id);
+ db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid);
}
db_finalize(&q);
/* Create the manifest */
blob_zero(&manifest);
@@ -451,10 +452,11 @@
free(zManifestFile);
nvid = content_put(&manifest, 0, 0);
if( nvid==0 ){
fossil_panic("trouble committing manifest: %s", g.zErrMsg);
}
+ db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nvid);
manifest_crosslink(nvid, &manifest);
content_deltify(vid, nvid, 0);
zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", nvid);
printf("New_Version: %s\n", zUuid);
zManifestFile = mprintf("%smanifest.uuid", g.zLocalRoot);
Modified src/content.c from [60a9520d47] to [52f1a41cd5].
@@ -233,10 +233,17 @@
/* If the srcId is specified, then the data we just added is
** really a delta. Record this fact in the delta table.
*/
if( srcId ){
db_multi_exec("REPLACE INTO delta(rid,srcid) VALUES(%d,%d)", rid, srcId);
+ }
+
+ /* Add the element to the unclustered table if it is not a
+ ** a phantom
+ */
+ if( pBlob ){
+ db_multi_exec("INSERT OR IGNORE INTO unclustered VALUES(%d)", rid);
}
/* Finish the transaction and cleanup */
db_finalize(&s1);
db_end_transaction(0);
Modified src/manifest.c from [b1ae7fa07b] to [3b8a4242be].
@@ -349,18 +349,15 @@
"VALUES('ci',%.17g,%d,%Q,%Q)",
m.rDate, rid, m.zUser, m.zComment
);
}
for(i=0; i<m.nCChild; i++){
- static Stmt dc;
- db_static_prepare(&dc,
- "DELETE FROM unclustered WHERE rid ="
- " (SELECT rid FROM blob WHERE uuid=:u)"
- );
- db_bind_text(&dc, ":u", m.azCChild[i]);
- db_step(&dc);
- db_reset(&dc);
+ int rid;
+ rid = uuid_to_rid(m.azCChild[i], 1);
+ if( rid>0 ){
+ db_multi_exec("DELETE FROM unclustered WHERE rid=%d", rid);
+ }
}
db_end_transaction(0);
manifest_clear(&m);
return 1;
}
Modified src/rebuild.c from [fab565baa9] to [45c4994360].
@@ -66,11 +66,11 @@
Blob content;
content_get(rid, &content);
manifest_crosslink(rid, &content);
blob_reset(&content);
}else{
- db_multi_exec("INSERT INTO phantom VALUES(%d)", rid);
+ db_multi_exec("INSERT OR IGNORE INTO phantom VALUES(%d)", rid);
}
}
return errCnt;
}
Modified src/xfer.c from [f732b233d1] to [bebb266bee].
@@ -234,105 +234,11 @@
remote_has(rid);
blob_reset(&uuid);
}
/*
-** Send the file identified by mid and pUuid. If that file happens
-** to be a manifest, then also send all of the associated content
-** files for that manifest. If the file is not a manifest, then this
-** routine is the equivalent of send_file().
-*/
-static void send_manifest(Xfer *pXfer, int mid, Blob *pUuid, int srcId){
- Stmt q2;
- send_file(pXfer, mid, pUuid, srcId);
- db_prepare(&q2,
- "SELECT pid, uuid, fid FROM mlink, blob"
- " WHERE rid=fid AND mid=%d",
- mid
- );
- while( db_step(&q2)==SQLITE_ROW ){
- int pid, fid;
- Blob uuid;
- pid = db_column_int(&q2, 0);
- db_ephemeral_blob(&q2, 1, &uuid);
- fid = db_column_int(&q2, 2);
- send_file(pXfer, fid, &uuid, pid);
- }
- db_finalize(&q2);
-}
-
-/*
-** This routine runs when either client or server is notified that
-** the other side thinks rid is a leaf manifest. If we hold
-** children of rid, then send them over to the other side.
-*/
-static void leaf_response(Xfer *pXfer, int rid){
- Stmt q1;
- db_prepare(&q1,
- "SELECT cid, uuid FROM plink, blob"
- " WHERE blob.rid=plink.cid"
- " AND plink.pid=%d",
- rid
- );
- while( db_step(&q1)==SQLITE_ROW ){
- Blob uuid;
- int cid;
-
- cid = db_column_int(&q1, 0);
- db_ephemeral_blob(&q1, 1, &uuid);
- send_manifest(pXfer, cid, &uuid, rid);
- if( blob_size(pXfer->pOut)<pXfer->mxSend ){
- leaf_response(pXfer, cid);
- }
- }
-}
-
-/*
-** Sent a leaf message for every leaf.
-*/
-static void send_leaves(Xfer *pXfer){
- Stmt q;
- db_prepare(&q,
- "SELECT uuid FROM blob WHERE rid IN"
- " (SELECT cid FROM plink EXCEPT SELECT pid FROM plink)"
- );
- while( db_step(&q)==SQLITE_ROW ){
- const char *zUuid = db_column_text(&q, 0);
- blob_appendf(pXfer->pOut, "leaf %s\n", zUuid);
- }
- db_finalize(&q);
-}
-
-/*
-** Sent leaf content for every leaf that is not found in the
-** onremote table. This is intended to send leaf content for
-** every leaf that is unknown on the remote end.
-**
-** In addition, we might send "igot" messages for a few generations of
-** parents of the unknown leaves. This will speed the transmission
-** of new branches.
-*/
-static void send_unknown_leaf_content(Xfer *pXfer){
- Stmt q1;
- db_prepare(&q1,
- "SELECT rid, uuid FROM blob WHERE rid IN"
- " (SELECT cid FROM plink EXCEPT SELECT pid FROM plink)"
- " AND NOT EXISTS(SELECT 1 FROM onremote WHERE rid=blob.rid)"
- );
- while( db_step(&q1)==SQLITE_ROW ){
- Blob uuid;
- int cid;
-
- cid = db_column_int(&q1, 0);
- db_ephemeral_blob(&q1, 1, &uuid);
- send_manifest(pXfer, cid, &uuid, 0);
- }
- db_finalize(&q1);
-}
-
-/*
-** Sen a gimme message for every phantom.
+** Send a gimme message for every phantom.
*/
static void request_phantoms(Xfer *pXfer){
Stmt q;
db_prepare(&q, "SELECT uuid FROM phantom JOIN blob USING(rid)");
while( db_step(&q)==SQLITE_ROW ){
@@ -396,10 +302,69 @@
}
}
db_reset(&q);
}
+/*
+** Send the content of all files in the unsent table.
+**
+** This is really just an optimization. If you clear the
+** unsent table, all the right files will still get transferred.
+** It just might require an extra round trip or two.
+*/
+static void send_unsent(Xfer *pXfer){
+ Stmt q;
+ db_prepare(&q, "SELECT rid FROM unsent");
+ while( db_step(&q)==SQLITE_ROW ){
+ int rid = db_column_int(&q, 0);
+ send_file(pXfer, rid, 0, 0);
+ }
+ db_finalize(&q);
+ db_multi_exec("DELETE FROM unsent");
+}
+
+/*
+** Check to see if the number of unclustered entries is greater than
+** 100 and if it is, form a new cluster.
+*/
+static void create_cluster(void){
+ Blob cluster, cksum;
+ Stmt q;
+ int rid;
+ if( db_int(0, "SELECT count(*) FROM unclustered")<10 ){
+ return;
+ }
+ blob_zero(&cluster);
+ db_prepare(&q, "SELECT uuid FROM unclustered JOIN blob USING(rid)"
+ " ORDER BY 1");
+ while( db_step(&q)==SQLITE_ROW ){
+ blob_appendf(&cluster, "M %s\n", db_column_text(&q, 0));
+ }
+ db_finalize(&q);
+ md5sum_blob(&cluster, &cksum);
+ blob_appendf(&cluster, "Z %b\n", &cksum);
+ blob_reset(&cksum);
+ db_multi_exec("DELETE FROM unclustered");
+ content_put(&cluster, 0, 0);
+ blob_reset(&cluster);
+}
+
+/*
+** Send an igot message for every entry in unclustered table.
+** Return the number of messages sent.
+*/
+static int send_unclustered(Xfer *pXfer){
+ Stmt q;
+ int cnt = 0;
+ db_prepare(&q, "SELECT uuid FROM unclustered JOIN blob USING(rid)");
+ while( db_step(&q)==SQLITE_ROW ){
+ blob_appendf(pXfer->pOut, "igot %s\n", db_column_text(&q, 0));
+ cnt++;
+ }
+ db_finalize(&q);
+ return cnt;
+}
/*
** If this variable is set, disable login checks. Used for debugging
** only.
*/
@@ -454,22 +419,20 @@
}
}else
/* gimme UUID
**
- ** Client is requesting a file. If the file is a manifest,
- ** the server can assume that the client also needs all content
- ** files associated with that manifest.
+ ** Client is requesting a file. Send it.
*/
if( blob_eq(&xfer.aToken[0], "gimme")
&& xfer.nToken==2
&& blob_is_uuid(&xfer.aToken[1])
){
if( isPull ){
int rid = rid_from_uuid(&xfer.aToken[1], 0);
if( rid ){
- send_manifest(&xfer, rid, &xfer.aToken[1], 0);
+ send_file(&xfer, rid, &xfer.aToken[1], 0);
}
}
}else
/* igot UUID
@@ -483,32 +446,10 @@
if( isPush ){
rid_from_uuid(&xfer.aToken[1], 1);
}
}else
-
- /* leaf UUID
- **
- ** Client announces that it has a particular manifest. If
- ** the server has children of this leaf, then send those
- ** children back to the client. If the server lacks this
- ** leaf, request it.
- */
- if( xfer.nToken==2
- && blob_eq(&xfer.aToken[0], "leaf")
- && blob_is_uuid(&xfer.aToken[1])
- ){
- int rid = rid_from_uuid(&xfer.aToken[1], 0);
- if( rid ){
- remote_has(rid);
- if( isPull ){
- leaf_response(&xfer, rid);
- }
- }else if( isPush ){
- content_put(0, blob_str(&xfer.aToken[1]), 0);
- }
- }else
/* pull SERVERCODE PROJECTCODE
** push SERVERCODE PROJECTCODE
**
** The client wants either send or receive. The server should
@@ -557,38 +498,28 @@
cgi_reset_content();
@ error not\sauthorized\sto\swrite
nErr++;
break;
}
- send_leaves(&xfer);
isPush = 1;
}
}else
/* clone
**
** The client knows nothing. Tell all.
*/
if( blob_eq(&xfer.aToken[0], "clone") ){
- int rootid;
login_check_credentials();
if( !g.okClone ){
cgi_reset_content();
@ error not\sauthorized\sto\sclone
nErr++;
break;
}
isPull = 1;
@ push %s(db_get("server-code", "x")) %s(db_get("project-code", "x"))
- rootid = db_int(0,
- "SELECT pid FROM plink AS a"
- " WHERE NOT EXISTS(SELECT 1 FROM plink WHERE cid=a.pid)"
- );
- if( rootid ){
- send_file(&xfer, rootid, 0, -1);
- leaf_response(&xfer, rootid);
- }
}else
/* login USER NONCE SIGNATURE
**
** Check for a valid login. This has to happen before anything else.
@@ -614,11 +545,12 @@
}
if( isPush ){
request_phantoms(&xfer);
}
if( isPull ){
- send_unknown_leaf_content(&xfer);
+ create_cluster();
+ send_unclustered(&xfer);
}
db_end_transaction(0);
}
/*
@@ -716,11 +648,14 @@
/* Generate gimme messages for phantoms and leaf messages
** for all leaves.
*/
if( pullFlag ){
request_phantoms(&xfer);
- send_leaves(&xfer);
+ }
+ if( pushFlag ){
+ send_unsent(&xfer);
+ nMsg += send_unclustered(&xfer);
}
/* Exchange messages with the server */
nFileSend = xfer.nFileSent + xfer.nDeltaSent;
printf("Send: %10d bytes, %3d messages, %3d files (%d+%d)\n",
@@ -773,11 +708,11 @@
&& blob_is_uuid(&xfer.aToken[1])
){
nMsg++;
if( pushFlag ){
int rid = rid_from_uuid(&xfer.aToken[1], 0);
- send_manifest(&xfer, rid, &xfer.aToken[1], 0);
+ send_file(&xfer, rid, &xfer.aToken[1], 0);
}
}else
/* igot UUID
**
@@ -800,34 +735,10 @@
newPhantom = 1;
}
}
if( rid==0 ){
rid = rid_from_uuid(&xfer.aToken[1], 0);
- }
- remote_has(rid);
- }else
-
-
- /* leaf UUID
- **
- ** Server announces that it has a particular manifest. Send
- ** any children of this leaf that we have if we are pushing.
- ** Make the leaf a phantom if we are pulling. Remember that the
- ** remote end has the specified UUID.
- */
- if( xfer.nToken==2
- && blob_eq(&xfer.aToken[0], "leaf")
- && blob_is_uuid(&xfer.aToken[1])
- ){
- int rid = rid_from_uuid(&xfer.aToken[1], 0);
- nMsg++;
- if( pushFlag && rid ){
- leaf_response(&xfer, rid);
- }
- if( pullFlag && rid==0 ){
- rid = content_put(0, blob_str(&xfer.aToken[1]), 0);
- newPhantom = 1;
}
remote_has(rid);
}else