Overview
| SHA1 Hash: | 8c828207a2fecfffeb043708ae069e2fa9777066 |
|---|---|
| Date: | 2007-08-27 00:04:32 |
| User: | drh |
| Comment: | Give an error if an attempt is made to merge, update, or checkout against an incomplete baseline - one that contains phantoms. Update the xfer protocol to converge on a stable synchronization faster and (hopeful) not quit until the sync is complete. |
| Timelines: | ancestors | descendants | both | trunk |
| Other Links: | files | ZIP archive | manifest |
Tags And Properties
- branch=trunk inherited from [a28c83647d]
- sym-trunk inherited from [a28c83647d]
Changes
[hide diffs]Modified ideas.txt from [e0433ea2b1] to [ff113a4540].
@@ -101,10 +101,13 @@
V* (+|-) tag uuid -- + to set, - to clear.
Z manifest-cksum
-- Must have at least one B or V.
-- Tag "hidden" means do not sync
-- Tag "closed" means do not display as a leaf
+ * A cluster
+ M+ uuid
+ Z manifest-cksum
* Complete set of cards in a manifest files:
A filename uuid
B (+|-)branch-tag uuid
C comment
D date-time
Modified src/vfile.c from [fdfe66efd6] to [aea4942364].
@@ -55,10 +55,29 @@
}
return rid;
}
/*
+** Verify that an object is not a phantom. If the object is
+** a phantom, output an error message and quick.
+*/
+void vfile_verify_not_phantom(int rid, const char *zFilename){
+ if( db_int(-1, "SELECT size FROM blob WHERE rid=%d", rid)<0 ){
+ if( zFilename ){
+ fossil_fatal("content missing for %s", zFilename);
+ }else{
+ char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid);
+ if( zUuid ){
+ fossil_fatal("content missing for [%.10s]", zUuid);
+ }else{
+ fossil_panic("bad object id: %d", rid);
+ }
+ }
+ }
+}
+
+/*
** Build a catalog of all files in a baseline.
** We scan the baseline file for lines of the form:
**
** F NAME UUID
**
@@ -69,10 +88,11 @@
char *zName, *zUuid;
Stmt ins;
Blob line, token, name, uuid;
int seenHeader = 0;
db_begin_transaction();
+ vfile_verify_not_phantom(vid, 0);
db_multi_exec("DELETE FROM vfile WHERE vid=%d", vid);
db_prepare(&ins,
"INSERT INTO vfile(vid,rid,mrid,pathname) "
" VALUES(:vid,:id,:id,:name)");
db_bind_int(&ins, ":vid", vid);
@@ -90,10 +110,11 @@
if( blob_token(&line, &uuid)==0 ) break;
zName = blob_str(&name);
defossilize(zName);
zUuid = blob_str(&uuid);
rid = uuid_to_rid(zUuid, 0);
+ vfile_verify_not_phantom(rid, zName);
if( rid>0 && file_is_simple_pathname(zName) ){
db_bind_int(&ins, ":id", rid);
db_bind_text(&ins, ":name", zName);
db_step(&ins);
db_reset(&ins);
Modified src/xfer.c from [caacb8871c] to [f732b233d1].
@@ -63,10 +63,17 @@
rid = content_put(0, blob_str(pUuid), 0);
}
return rid;
}
+/*
+** Remember that the other side of the connection already has a copy
+** of the file rid.
+*/
+static void remote_has(int rid){
+ db_multi_exec("INSERT OR IGNORE INTO onremote VALUES(%d)", rid);
+}
/*
** The aToken[0..nToken-1] blob array is a parse of a "file" line
** message. This routine finishes parsing that message and does
** a record insert of the file.
@@ -126,10 +133,11 @@
if( rid==0 ){
blob_appendf(&pXfer->err, "%s", g.zErrMsg);
}else{
manifest_crosslink(rid, &content);
}
+ remote_has(rid);
}
/*
** Try to send a file as a delta. If successful, return the number
** of bytes in the delta. If not, return zero.
@@ -221,21 +229,46 @@
blob_append(pXfer->pOut, blob_buffer(&content), size);
pXfer->nFileSent++;
}else{
pXfer->nDeltaSent++;
}
- db_multi_exec("INSERT INTO onremote VALUES(%d)", rid);
+ remote_has(rid);
blob_reset(&uuid);
}
/*
+** Send the file identified by mid and pUuid. If that file happens
+** to be a manifest, then also send all of the associated content
+** files for that manifest. If the file is not a manifest, then this
+** routine is the equivalent of send_file().
+*/
+static void send_manifest(Xfer *pXfer, int mid, Blob *pUuid, int srcId){
+ Stmt q2;
+ send_file(pXfer, mid, pUuid, srcId);
+ db_prepare(&q2,
+ "SELECT pid, uuid, fid FROM mlink, blob"
+ " WHERE rid=fid AND mid=%d",
+ mid
+ );
+ while( db_step(&q2)==SQLITE_ROW ){
+ int pid, fid;
+ Blob uuid;
+ pid = db_column_int(&q2, 0);
+ db_ephemeral_blob(&q2, 1, &uuid);
+ fid = db_column_int(&q2, 2);
+ send_file(pXfer, fid, &uuid, pid);
+ }
+ db_finalize(&q2);
+}
+
+/*
** This routine runs when either client or server is notified that
-** the other side things rid is a leaf manifest. If we hold
+** the other side thinks rid is a leaf manifest. If we hold
** children of rid, then send them over to the other side.
*/
static void leaf_response(Xfer *pXfer, int rid){
- Stmt q1, q2;
+ Stmt q1;
db_prepare(&q1,
"SELECT cid, uuid FROM plink, blob"
" WHERE blob.rid=plink.cid"
" AND plink.pid=%d",
rid
@@ -244,24 +277,11 @@
Blob uuid;
int cid;
cid = db_column_int(&q1, 0);
db_ephemeral_blob(&q1, 1, &uuid);
- send_file(pXfer, cid, &uuid, rid);
- db_prepare(&q2,
- "SELECT pid, uuid, fid FROM mlink, blob"
- " WHERE rid=fid AND mid=%d",
- cid
- );
- while( db_step(&q2)==SQLITE_ROW ){
- int pid, fid;
- pid = db_column_int(&q2, 0);
- db_ephemeral_blob(&q2, 1, &uuid);
- fid = db_column_int(&q2, 2);
- send_file(pXfer, fid, &uuid, pid);
- }
- db_finalize(&q2);
+ send_manifest(pXfer, cid, &uuid, rid);
if( blob_size(pXfer->pOut)<pXfer->mxSend ){
leaf_response(pXfer, cid);
}
}
}
@@ -278,10 +298,37 @@
while( db_step(&q)==SQLITE_ROW ){
const char *zUuid = db_column_text(&q, 0);
blob_appendf(pXfer->pOut, "leaf %s\n", zUuid);
}
db_finalize(&q);
+}
+
+/*
+** Sent leaf content for every leaf that is not found in the
+** onremote table. This is intended to send leaf content for
+** every leaf that is unknown on the remote end.
+**
+** In addition, we might send "igot" messages for a few generations of
+** parents of the unknown leaves. This will speed the transmission
+** of new branches.
+*/
+static void send_unknown_leaf_content(Xfer *pXfer){
+ Stmt q1;
+ db_prepare(&q1,
+ "SELECT rid, uuid FROM blob WHERE rid IN"
+ " (SELECT cid FROM plink EXCEPT SELECT pid FROM plink)"
+ " AND NOT EXISTS(SELECT 1 FROM onremote WHERE rid=blob.rid)"
+ );
+ while( db_step(&q1)==SQLITE_ROW ){
+ Blob uuid;
+ int cid;
+
+ cid = db_column_int(&q1, 0);
+ db_ephemeral_blob(&q1, 1, &uuid);
+ send_manifest(pXfer, cid, &uuid, 0);
+ }
+ db_finalize(&q1);
}
/*
** Sen a gimme message for every phantom.
*/
@@ -407,27 +454,29 @@
}
}else
/* gimme UUID
**
- ** Client is requesting a file
+ ** Client is requesting a file. If the file is a manifest,
+ ** the server can assume that the client also needs all content
+ ** files associated with that manifest.
*/
if( blob_eq(&xfer.aToken[0], "gimme")
&& xfer.nToken==2
&& blob_is_uuid(&xfer.aToken[1])
){
if( isPull ){
int rid = rid_from_uuid(&xfer.aToken[1], 0);
if( rid ){
- send_file(&xfer, rid, &xfer.aToken[1], 0);
+ send_manifest(&xfer, rid, &xfer.aToken[1], 0);
}
}
}else
/* igot UUID
**
- ** Client announces that it has a particular file
+ ** Client announces that it has a particular file.
*/
if( xfer.nToken==2
&& blob_eq(&xfer.aToken[0], "igot")
&& blob_is_uuid(&xfer.aToken[1])
){
@@ -447,22 +496,26 @@
if( xfer.nToken==2
&& blob_eq(&xfer.aToken[0], "leaf")
&& blob_is_uuid(&xfer.aToken[1])
){
int rid = rid_from_uuid(&xfer.aToken[1], 0);
- if( isPull && rid ){
- leaf_response(&xfer, rid);
- }
- if( isPush && !rid ){
+ if( rid ){
+ remote_has(rid);
+ if( isPull ){
+ leaf_response(&xfer, rid);
+ }
+ }else if( isPush ){
content_put(0, blob_str(&xfer.aToken[1]), 0);
}
}else
/* pull SERVERCODE PROJECTCODE
** push SERVERCODE PROJECTCODE
**
- ** The client wants either send or receive
+ ** The client wants either send or receive. The server should
+ ** verify that the project code matches and that the server code
+ ** does not match.
*/
if( xfer.nToken==3
&& (blob_eq(&xfer.aToken[0], "pull") || blob_eq(&xfer.aToken[0], "push"))
&& blob_is_uuid(&xfer.aToken[1])
&& blob_is_uuid(&xfer.aToken[2])
@@ -537,10 +590,11 @@
}else
/* login USER NONCE SIGNATURE
**
** Check for a valid login. This has to happen before anything else.
+ ** The client can send multiple logins. Permissions are cumulative.
*/
if( blob_eq(&xfer.aToken[0], "login")
&& xfer.nToken==4
){
if( disableLogin ){
@@ -558,10 +612,13 @@
}
blobarray_reset(xfer.aToken, xfer.nToken);
}
if( isPush ){
request_phantoms(&xfer);
+ }
+ if( isPull ){
+ send_unknown_leaf_content(&xfer);
}
db_end_transaction(0);
}
/*
@@ -697,53 +754,67 @@
xfer.nToken = blob_tokenize(&xfer.line, xfer.aToken, count(xfer.aToken));
/* file UUID SIZE \n CONTENT
** file UUID DELTASRC SIZE \n CONTENT
**
- ** Receive a file transmitted from the other side
+ ** Receive a file transmitted from the server.
*/
if( blob_eq(&xfer.aToken[0],"file") ){
xfer_accept_file(&xfer);
}else
/* gimme UUID
**
- ** Server is requesting a file
+ ** Server is requesting a file. If the file is a manifest, assume
+ ** that the server will also want to know all of the content files
+ ** associated with the manifest and send those too.
*/
if( blob_eq(&xfer.aToken[0], "gimme")
&& xfer.nToken==2
&& blob_is_uuid(&xfer.aToken[1])
){
nMsg++;
if( pushFlag ){
int rid = rid_from_uuid(&xfer.aToken[1], 0);
- send_file(&xfer, rid, &xfer.aToken[1], 0);
+ send_manifest(&xfer, rid, &xfer.aToken[1], 0);
}
}else
/* igot UUID
**
- ** Server announces that it has a particular file
+ ** Server announces that it has a particular file. If this is
+ ** not a file that we have and we are pulling, then create a
+ ** phantom to cause this file to be requested on the next cycle.
+ ** Always remember that the server has this file so that we do
+ ** not transmit it by accident.
*/
if( xfer.nToken==2
&& blob_eq(&xfer.aToken[0], "igot")
&& blob_is_uuid(&xfer.aToken[1])
){
+ int rid = 0;
nMsg++;
if( pullFlag ){
if( !db_exists("SELECT 1 FROM blob WHERE uuid='%b' AND size>=0",
&xfer.aToken[1]) ){
- content_put(0, blob_str(&xfer.aToken[1]), 0);
+ rid = content_put(0, blob_str(&xfer.aToken[1]), 0);
newPhantom = 1;
}
}
+ if( rid==0 ){
+ rid = rid_from_uuid(&xfer.aToken[1], 0);
+ }
+ remote_has(rid);
}else
/* leaf UUID
**
- ** Server announces that it has a particular manifest
+ ** Server announces that it has a particular manifest. Send
+ ** any children of this leaf that we have if we are pushing.
+ ** Make the leaf a phantom if we are pulling. Remember that the
+ ** remote end has the specified UUID.
*/
if( xfer.nToken==2
&& blob_eq(&xfer.aToken[0], "leaf")
&& blob_is_uuid(&xfer.aToken[1])
){
@@ -751,19 +822,21 @@
nMsg++;
if( pushFlag && rid ){
leaf_response(&xfer, rid);
}
if( pullFlag && rid==0 ){
- content_put(0, blob_str(&xfer.aToken[1]), 0);
+ rid = content_put(0, blob_str(&xfer.aToken[1]), 0);
newPhantom = 1;
}
+ remote_has(rid);
}else
/* push SERVERCODE PRODUCTCODE
**
- ** Should only happen in response to a clone.
+ ** Should only happen in response to a clone. This message tells
+ ** the client what product to use for the new database.
*/
if( blob_eq(&xfer.aToken[0],"push")
&& xfer.nToken==3
&& cloneFlag
&& blob_is_uuid(&xfer.aToken[1])