Overview
| SHA1 Hash: | f46458d5bdc221859c6b009de9b8e5dcbbc2534a |
|---|---|
| Date: | 2008-02-17 02:06:19 |
| User: | aku |
| Comment: | Reworked the basic structure of pass InitCSets to keep memory consumption down. Now incremental creates, breaks, saves, and releases changesets, instead of piling them on before saving all at the end. Memory tracking confirms that this changes the accumulating mountain into a near-constant usage, with the expected spikes from the breaking. |
| Timelines: | ancestors | descendants | both | trunk |
| Other Links: | files | ZIP archive | manifest |
Tags And Properties
- branch=trunk inherited from [a28c83647d]
- sym-trunk inherited from [a28c83647d]
Changes
[hide diffs]Modified tools/cvs2fossil/getmemoryseries.tcl from [d3a2537d7a] to [8e7cb19203].
@@ -1,11 +1,11 @@
#!/bin/bash
# -*- tcl -*- \
exec tclsh "$0" ${1+"$@"}
package require csv
-foreach {in outbasic outmarker plot} $argv break
+foreach {in outbasic outmarker plot outbasicold} $argv break
set in [open $in r]
set ba [open $outbasic w]
set mr [open $outmarker w]
@@ -42,10 +42,19 @@
# Generate gnuplot control file for the series
set f [open $plot w]
puts $f ""
puts $f "plot \"$outbasic\" using 1:2 title 'Memory' with steps, \\"
puts $f " \"$outbasic\" using 1:3 title 'Max Memory' with steps"
+puts $f "pause -1"
+puts $f ""
+close $f
+
+# Generate gnuplot control file for comparison of series
+set f [open ${plot}-compare w]
+puts $f ""
+puts $f "plot \"$outbasicold\" using 1:2 title 'Memory Old' with steps, \\"
+puts $f " \"$outbasic\" using 1:2 title 'Memory New' with steps"
puts $f "pause -1"
puts $f ""
close $f
exit
Modified tools/cvs2fossil/lib/c2f_pinitcsets.tcl from [9999ef81cc] to [8351f6b640].
@@ -125,18 +125,24 @@
typemethod run {} {
# Pass manager interface. Executed to perform the
# functionality of the pass.
state transaction {
- CreateRevisionChangesets ; # Group file revisions into csets.
- BreakInternalDependencies ; # Split the csets based on internal conflicts.
- CreateSymbolChangesets ; # Create csets for tags and branches.
- PersistTheChangesets
+ CreateRevisionChangesets ; # Group file revisions into
+ # preliminary csets and split
+ # them based on internal
+ # conflicts.
+ CreateSymbolChangesets ; # Create csets for tags and
+ # branches.
}
repository printcsetstatistics
integrity changesets
+
+ # Load the changesets for use by the next passes.
+ project::rev load ::vc::fossil::import::cvs::repository
+ project::rev loadcounter
return
}
typemethod discard {} {
# Pass manager interface. Executed for all passes after the
@@ -168,11 +174,18 @@
# We order the revisions here by time, this will help the
# later passes (avoids joins later to get at the ordering
# info).
- set n 0
+ # The changesets made from these groups are immediately
+ # inspected for internal conflicts and any such are broken by
+ # splitting the problematic changeset into multiple
+ # fragments. The results are changesets which have no internal
+ # dependencies, only external ones.
+
+ set n 0
+ set nx 0
set lastmeta {}
set lastproject {}
set revisions {}
@@ -196,12 +209,15 @@
if {[llength $revisions]} {
incr n
set p [repository projectof $lastproject]
log write 14 initcsets meta_cset_begin
mem::mark
- project::rev %AUTO% $p rev $lastmeta $revisions
+ set cset [project::rev %AUTO% $p rev $lastmeta $revisions]
log write 14 initcsets meta_cset_done
+ $cset breakinternaldependencies nx
+ $cset persist
+ $cset destroy
mem::mark
set revisions {}
}
set lastmeta $mid
set lastproject $pid
@@ -212,19 +228,26 @@
if {[llength $revisions]} {
incr n
set p [repository projectof $lastproject]
log write 14 initcsets meta_cset_begin
mem::mark
- project::rev %AUTO% $p rev $lastmeta $revisions
+ set cset [project::rev %AUTO% $p rev $lastmeta $revisions]
log write 14 initcsets meta_cset_done
+ $cset breakinternaldependencies nx
+ $cset persist
+ $cset destroy
mem::mark
}
log write 14 initcsets meta_done
mem::mark
- log write 4 initcsets "Created [nsp $n {revision changeset}]"
+ log write 4 initcsets "Created and saved [nsp $n {revision changeset}]"
+ log write 4 initcsets "Created and saved [nsp $nx {additional revision changeset}]"
+
+ mem::mark
+ log write 4 initcsets Ok.
return
}
proc CreateSymbolChangesets {} {
log write 3 initcsets {Create changesets based on symbols}
@@ -251,12 +274,14 @@
}] {
if {$lastsymbol != $sid} {
if {[llength $tags]} {
incr n
set p [repository projectof $lastproject]
- project::rev %AUTO% $p sym::tag $lastsymbol $tags
+ set cset [project::rev %AUTO% $p sym::tag $lastsymbol $tags]
set tags {}
+ $cset persist
+ $cset destroy
}
set lastsymbol $sid
set lastproject $pid
}
lappend tags $tid
@@ -263,11 +288,13 @@
}
if {[llength $tags]} {
incr n
set p [repository projectof $lastproject]
- project::rev %AUTO% $p sym::tag $lastsymbol $tags
+ set cset [project::rev %AUTO% $p sym::tag $lastsymbol $tags]
+ $cset persist
+ $cset destroy
}
set lastsymbol {}
set lasproject {}
set branches {}
@@ -280,12 +307,14 @@
}] {
if {$lastsymbol != $sid} {
if {[llength $branches]} {
incr n
set p [repository projectof $lastproject]
- project::rev %AUTO% $p sym::branch $lastsymbol $branches
+ set cset [project::rev %AUTO% $p sym::branch $lastsymbol $branches]
set branches {}
+ $cset persist
+ $cset destroy
}
set lastsymbol $sid
set lastproject $pid
}
lappend branches $bid
@@ -292,50 +321,17 @@
}
if {[llength $branches]} {
incr n
set p [repository projectof $lastproject]
- project::rev %AUTO% $p sym::branch $lastsymbol $branches
+ set cset [project::rev %AUTO% $p sym::branch $lastsymbol $branches]
+ $cset persist
+ $cset destroy
}
- log write 4 initcsets "Created [nsp $n {symbol changeset}]"
- mem::mark
- return
- }
-
- proc BreakInternalDependencies {} {
- # This code operates on the revision changesets created by
- # 'CreateRevisionChangesets'. As such it has to follow after
- # it, before the symbol changesets are made. The changesets
- # are inspected for internal conflicts and any such are broken
- # by splitting the problematic changeset into multiple
- # fragments. The results are changesets which have no internal
- # dependencies, only external ones.
-
- log write 3 initcsets {Break internal dependencies}
- mem::mark
- set old [llength [project::rev all]]
-
- foreach cset [project::rev all] {
- $cset breakinternaldependencies
- }
-
- set n [expr {[llength [project::rev all]] - $old}]
- log write 4 initcsets "Created [nsp $n {additional revision changeset}]"
- log write 4 initcsets Ok.
- mem::mark
- return
- }
-
- proc PersistTheChangesets {} {
- log write 3 initcsets "Saving [nsp [llength [project::rev all]] {initial changeset}] to the persistent state"
-
- foreach cset [project::rev all] {
- $cset persist
- }
-
- log write 4 initcsets Ok.
+ log write 4 initcsets "Created and saved [nsp $n {symbol changeset}]"
+ mem::mark
return
}
# # ## ### ##### ######## #############
## Configuration
Modified tools/cvs2fossil/lib/c2f_prev.tcl from [433b16c0c0] to [cc210d2578].
@@ -62,10 +62,25 @@
log write 8 csets {MAP+ item <$key> $self = [$self str]}
}
return
}
+ destructor {
+ # The main thing is to keep track of the itemmap and remove
+ # the object from it. The lists of changesets (mychangesets,
+ # mytchangesets) are not maintained (= reduced), for the
+ # moment. We may be able to get rid of this entirely, at least
+ # for (de)construction and pass InitCSets.
+
+ foreach iid $myitems {
+ set key [list $mytype $iid]
+ unset myitemmap($key)
+ log write 8 csets {MAP- item <$key> $self = [$self str]}
+ }
+ return
+ }
+
method str {} {
set str "<"
set detail ""
if {[$mytypeobj bysymbol]} {
set detail " '[state one {
@@ -132,11 +147,12 @@
method nextmap {} {
$mytypeobj successors tmp $myitems
return [array get tmp]
}
- method breakinternaldependencies {} {
+ method breakinternaldependencies {cv} {
+ upvar 1 $cv counter
log write 14 csets {[$self str] BID}
vc::tools::mem::mark
##
## NOTE: This method, maybe in conjunction with its caller
## seems to be a memory hog, especially for large
@@ -288,10 +304,11 @@
foreach fragment [lrange $fragments 1 end] {
Border $fragment s e
integrity assert {$laste == ($s - 1)} {Bad fragment border <$laste | $s>, gap or overlap}
set new [$type %AUTO% $myproject $mytype $mysrcid [lrange $myitems $s $e]]
+ incr counter
log write 4 csets "Breaking [$self str ] @ $laste, new [$new str], cutting $breaks($laste)"
set laste $e
}