Differences From:

File tools/cvs2fossil/lib/c2f_prev.tcl part of check-in [f46458d5bd] - Reworked the basic structure of pass InitCSets to keep memory consumption down. Now incremental creates, breaks, saves, and releases changesets, instead of piling them on before saving all at the end. Memory tracking confirms that this changes the accumulating mountain into a near-constant usage, with the expected spikes from the breaking. by aku on 2008-02-17 02:06:19. [view]

To:

File tools/cvs2fossil/lib/c2f_prev.tcl part of check-in [c2ad73ed92] - Added high-level logging for memory tracing to the code breaking the preliminary changesets. First runs indicate that the DEPC array becomes so very large, caused by a high amount of indirect dependencies (several hundred). by aku on 2008-02-21 05:13:14. [view]

@@ -204,9 +204,15 @@
 	# Map:  POS   revision id      -> position in list.
 	#       CROSS position in list -> number of dependencies crossing it
 	#       DEPC  dependency       -> positions it crosses
 	# List: RANGE Of the positions itself.
+	# Map:  DELTA position in list -> time delta between its revision
+	#                                 and the next, if any.
 	# A dependency is a single-element map parent -> child
+
+	# InitializeBreakState initializes their contents after
+	# upvar'ing them from this scope. It uses the information in
+	# DEPENDENCIES to do so.
 
 	InitializeBreakState $myitems
 
 	set fragments {}
@@ -800,8 +806,11 @@
 
 	# First we create a map of positions to make it easier to
 	# determine whether a dependency crosses a particular index.
 
+	log write 14 csets {IBS: #rev [llength $revisions]}
+	log write 14 csets {IBS: pos map, cross counter}
+
 	array set pos   {}
 	array set cross {}
 	array set depc  {}
 	set range       {}
@@ -811,8 +820,10 @@
 	    set pos($rev) $n
 	    set cross($n) 0
 	    incr n
 	}
+
+	log write 14 csets {IBS: pos/[array size pos], cross/[array size cross]}
 
 	# Secondly we count the crossings per position, by iterating
 	# over the recorded internal dependencies.
 
@@ -824,8 +835,10 @@
 	# Note 2: start == end is not possible. It indicates a
 	#         self-dependency due to the uniqueness of positions,
 	#         and that is something we have ruled out already, see
 	#         'rev internalsuccessors'.
+
+	log write 14 csets {IBS: cross counter filling, pos/cross map}
 
 	foreach {rid children} [array get dependencies] {
 	    foreach child $children {
 		set dkey    [list $rid $child]
@@ -849,9 +862,14 @@
 		set depc($dkey) $crosses
 	    }
 	}
 
+	log write 14 csets {IBS: pos/[array size pos], cross/[array size cross], depc/[array size depc] (for [llength $revisions])}
+	log write 14 csets {IBS: timestamps, deltas}
+
 	InitializeDeltas $revisions
+
+	log write 14 csets {IBS: delta [array size delta]}
 	return
     }
 
     proc InitializeDeltas {revisions} {
@@ -870,8 +888,10 @@
 	    WHERE R.rid IN $theset
 	}]] {
 	    set stamp($rid) $time
 	}
+
+	log write 14 csets {IBS: stamp [array size stamp]}
 
 	set n 0
 	foreach rid [lrange $revisions 0 end-1] rnext [lrange $revisions 1 end] {
 	    set delta($n) [expr {$stamp($rnext) - $stamp($rid)}]
@@ -1082,9 +1102,9 @@
     typemethod internalsuccessors {dv revisions} {
 	upvar 1 $dv dependencies
 	set theset ('[join $revisions {','}]')
 
-	log write 14 cset internalsuccessors
+	log write 14 csets internalsuccessors
 
 	# See 'successors' below for the main explanation of
 	# the various cases. This piece is special in that it
 	# restricts the successors we look for to the same set of
@@ -1145,9 +1165,11 @@
 	# We allow revisions to be far apart in time in the same
 	# changeset, but in turn need the pseudo-dependencies to
 	# handle this.
 
-	log write 14 cset pseudo-internalsuccessors
+	log write 14 csets {internal  [array size dep]}
+	log write 14 csets {collected [array size dependencies]}
+	log write 14 csets pseudo-internalsuccessors
 
 	array set fids {}
 	foreach {rid fid} [state run [subst -nocommands -nobackslashes {
 	    SELECT R.rid, R.fid
@@ -1154,8 +1176,9 @@
             FROM   revision R
             WHERE  R.rid IN $theset
 	}]] { lappend fids($fid) $rid }
 
+	set groups {}
 	foreach {fid rids} [array get fids] {
 	    if {[llength $rids] < 2} continue
 	    foreach a $rids {
 		foreach b $rids {
@@ -1166,11 +1189,16 @@
 		    set dep($a,$b) .
 		    set dep($b,$a) .
 		}
 	    }
+	    set n [llength $rids]
+	    lappend groups [list $n [expr {($n*$n-$n)/2}]]
 	}
 
-	log write 14 cset complete
+	log write 14 csets {pseudo    [array size fids] ([lsort -index 0 -decreasing -integer $groups])}
+	log write 14 csets {internal  [array size dep]}
+	log write 14 csets {collected [array size dependencies]}
+	log write 14 csets complete
 	return
     }
 
     # result = 4-list (itemtype itemid nextitemtype nextitemid ...)