@@ -404,7 +404,7 @@ func (c *gcControllerState) startCycle(markStartTime int64, procs int, trigger g
404404dedicatedMarkWorkersNeeded := int64 (totalUtilizationGoal + 0.5 )
405405utilError := float64 (dedicatedMarkWorkersNeeded )/ totalUtilizationGoal - 1
406406const maxUtilError = 0.3
407- if utilError < - maxUtilError || utilError > maxUtilError {
407+ if utilError < - maxUtilError || maxUtilError < utilError {
408408// Rounding put us more than 30% off our goal. With
409409// gcBackgroundUtilization of 25%, this happens for
410410// GOMAXPROCS<=3 or GOMAXPROCS=6. Enable fractional
@@ -536,9 +536,7 @@ func (c *gcControllerState) revise() {
536536// This maintains the invariant that we use no more memory than the next GC cycle
537537// will anyway.
538538hardGoal := int64 ((1.0 + float64 (gcPercent )/ 100.0 )* float64 (heapGoal ))
539- if extHeapGoal > hardGoal {
540- extHeapGoal = hardGoal
541- }
539+ extHeapGoal = min (extHeapGoal ,hardGoal )
542540heapGoal = extHeapGoal
543541}
544542if int64 (live )> heapGoal {
@@ -560,17 +558,16 @@ func (c *gcControllerState) revise() {
560558// slowly in the soft regime and not at all in the hard
561559// regime.
562560scanWorkRemaining := scanWorkExpected - work
563- if scanWorkRemaining < 1000 {
564- // We set a somewhat arbitrary lower bound on
565- // remaining scan work since if we aim a little high,
566- // we can miss by a little.
567- //
568- // We *do* need to enforce that this is at least 1,
569- // since marking is racy and double-scanning objects
570- // may legitimately make the remaining scan work
571- // negative, even in the hard goal regime.
572- scanWorkRemaining = 1000
573- }
561+
562+ // We set a somewhat arbitrary lower bound on
563+ // remaining scan work since if we aim a little high,
564+ // we can miss by a little.
565+ //
566+ // We *do* need to enforce that this is at least 1,
567+ // since marking is racy and double-scanning objects
568+ // may legitimately make the remaining scan work
569+ // negative, even in the hard goal regime.
570+ scanWorkRemaining = max (scanWorkRemaining ,1000 )
574571
575572// Compute the heap distance remaining.
576573heapRemaining := heapGoal - int64 (live )
@@ -666,9 +663,7 @@ func (c *gcControllerState) endCycle(now int64, procs int, userForced bool) {
666663oldConsMark := c .consMark
667664c .consMark = currentConsMark
668665for i := range c .lastConsMark {
669- if c .lastConsMark [i ]> c .consMark {
670- c .consMark = c .lastConsMark [i ]
671- }
666+ c .consMark = max (c .consMark ,c .lastConsMark [i ])
672667}
673668copy (c .lastConsMark [:],c .lastConsMark [1 :])
674669c .lastConsMark [len (c .lastConsMark )- 1 ]= currentConsMark
@@ -979,7 +974,7 @@ func (c *gcControllerState) addScannableStack(pp *p, amount int64) {
979974return
980975}
981976pp .maxStackScanDelta += amount
982- if pp .maxStackScanDelta >= maxStackScanSlack || pp . maxStackScanDelta <= - maxStackScanSlack {
977+ if pp .maxStackScanDelta <= - maxStackScanSlack || maxStackScanSlack <= pp . maxStackScanDelta {
983978c .maxStackScan .Add (pp .maxStackScanDelta )
984979pp .maxStackScanDelta = 0
985980}
@@ -1004,43 +999,44 @@ func (c *gcControllerState) heapGoalInternal() (goal, minTrigger uint64) {
1004999goal = c .gcPercentHeapGoal .Load ()
10051000
10061001// Check if the memory-limit-based goal is smaller, and if so, pick that.
1007- if newGoal := c .memoryLimitHeapGoal ();newGoal < goal {
1008- goal = newGoal
1009- } else {
1010- // We're not limited by the memory limit goal, so perform a series of
1011- //adjustments that might move the goal forward in avariety of circumstances.
1012-
1013- sweepDistTrigger := c . sweepDistMinTrigger . Load ()
1014- if sweepDistTrigger > goal {
1015- // Set the goal to maintain a minimum sweep distance since
1016- // thelast call tocommit. Note that we never want to do this
1017- //if we're in the memory limit regime, because it could push
1018- // thegoal up.
1019- goal = sweepDistTrigger
1020- }
1021- // Since we ignore the sweep distance trigger in the memory
1022- //limit regime, weneed to ensure we don't propagate it to
1023- //the trigger, because it could cause a violation of the
1024- //invariant that the trigger < goal.
1025- minTrigger = sweepDistTrigger
1026-
1027- // Ensure that the heap goal is at least a little larger than
1028- //the point at which we triggered. This may not be the case if GC
1029- //start is delayed or if the allocation that pushed gcController.heapLive
1030- //over trigger islarge or if thetrigger is really close to
1031- //GOGC. Assist isproportional to this distance, so enforce a
1032- //minimum distance, even if it means going over the GOGC goal
1033- //by a tiny bit.
1034- //
1035- // Ignore this if we're in the memory limit regime: we'd prefer to
1036- //have the GC respond hard about how close we are to the goal than to
1037- //push thegoal back in such a manner that it could cause us to exceed
1038- // thememory limit.
1039- const minRunway = 64 << 10
1040- if c . triggered != ^ uint64 ( 0 ) && goal < c . triggered + minRunway {
1041- goal = c .triggered + minRunway
1042- }
1002+ if limitGoal := c .memoryLimitHeapGoal ();limitGoal < goal {
1003+ goal = limitGoal
1004+ return
1005+ }
1006+ //We're not limited by thememory limit goal, so perform aseries of
1007+ // adjustments that might move the goal forward in a variety of circumstances.
1008+
1009+ sweepDistTrigger := c . sweepDistMinTrigger . Load ()
1010+
1011+ //Set thegoal tomaintain a minimum sweep distance since
1012+ //the last call to commit. Note that we never want to do this
1013+ //if we're in thememory limit regime, because it could push
1014+ // the goal up.
1015+ goal = max ( goal , sweepDistTrigger )
1016+
1017+ //Since weignore the sweep distance trigger in the memory
1018+ //limit regime, we need to ensure we don't propagate it to
1019+ // the trigger, because it could cause a violation of the
1020+ // invariant that the trigger < goal.
1021+ minTrigger = sweepDistTrigger
1022+
1023+ //Ensure that the heap goal is at least a little larger than
1024+ //the point at which we triggered. This may not be the case if GC
1025+ //start isdelayed or if theallocation that pushed gcController.heapLive
1026+ //over trigger islarge or if the trigger is really close to
1027+ //GOGC. Assist is proportional to this distance, so enforce a
1028+ //minimum distance, even if it means going over the GOGC goal
1029+ // by a tiny bit.
1030+ //
1031+ //Ignore this if we're in the memory limit regime: we'd prefer to
1032+ //have theGC respond hard about how close we are to the goal than to
1033+ //push thegoal back in such a manner that it could cause us to exceed
1034+ // the memory limit.
1035+ const minRunway = 64 << 10
1036+ if c .triggered != ^ uint64 ( 0 ) {
1037+ goal = max ( goal , c . triggered + minRunway )
10431038}
1039+
10441040return
10451041}
10461042
@@ -1142,20 +1138,17 @@ func (c *gcControllerState) memoryLimitHeapGoal() uint64 {
11421138// the impact of scavenging at allocation time in response to a high allocation rate
11431139// when GOGC=off. See issue #57069. Also, be careful about small limits.
11441140headroom := goal / 100 * memoryLimitHeapGoalHeadroomPercent
1145- if headroom < memoryLimitMinHeapGoalHeadroom {
1146- // Set a fixed minimum to deal with the particularly large effect pacing inaccuracies
1147- // have for smaller heaps.
1148- headroom = memoryLimitMinHeapGoalHeadroom
1149- }
1141+ // Set a fixed minimum to deal with the particularly large effect pacing inaccuracies
1142+ // have for smaller heaps.
1143+ headroom = max (headroom ,memoryLimitMinHeapGoalHeadroom )
1144+
11501145if goal < headroom || goal - headroom < headroom {
11511146goal = headroom
11521147}else {
11531148goal = goal - headroom
11541149}
11551150// Don't let us go below the live heap. A heap goal below the live heap doesn't make sense.
1156- if goal < c .heapMarked {
1157- goal = c .heapMarked
1158- }
1151+ goal = max (goal ,c .heapMarked )
11591152return goal
11601153}
11611154
@@ -1207,9 +1200,7 @@ func (c *gcControllerState) trigger() (uint64, uint64) {
12071200
12081201// heapMarked is our absolute minimum, and it's possible the trigger
12091202// bound we get from heapGoalinternal is less than that.
1210- if minTrigger < c .heapMarked {
1211- minTrigger = c .heapMarked
1212- }
1203+ minTrigger = max (minTrigger ,c .heapMarked )
12131204
12141205// If we let the trigger go too low, then if the application
12151206// is allocating very rapidly we might end up in a situation
@@ -1219,9 +1210,7 @@ func (c *gcControllerState) trigger() (uint64, uint64) {
12191210// saying that we're OK using more CPU during the GC to prevent
12201211// this growth in RSS.
12211212triggerLowerBound := ((goal - c .heapMarked )/ triggerRatioDen )* minTriggerRatioNum + c .heapMarked
1222- if minTrigger < triggerLowerBound {
1223- minTrigger = triggerLowerBound
1224- }
1213+ minTrigger = max (minTrigger ,triggerLowerBound )
12251214
12261215// For small heaps, set the max trigger point at maxTriggerRatio of the way
12271216// from the live heap to the heap goal. This ensures we always have *some*
@@ -1298,9 +1287,7 @@ func (c *gcControllerState) commit(isSweepDone bool) {
12981287}
12991288// Apply the minimum heap size here. It's defined in terms of gcPercent
13001289// and is only updated by functions that call commit.
1301- if gcPercentHeapGoal < c .heapMinimum {
1302- gcPercentHeapGoal = c .heapMinimum
1303- }
1290+ gcPercentHeapGoal = max (gcPercentHeapGoal ,c .heapMinimum )
13041291c .gcPercentHeapGoal .Store (gcPercentHeapGoal )
13051292
13061293// Compute the amount of runway we want the GC to have by using our
@@ -1326,7 +1313,9 @@ func (c *gcControllerState) commit(isSweepDone bool) {
13261313// Furthermore, by setting the runway so that CPU resources are divided
13271314// this way, assuming that the cons/mark ratio is correct, we make that
13281315// division a reality.
1329- c .runway .Store (uint64 ((c .consMark * (1 - gcGoalUtilization )/ (gcGoalUtilization ))* float64 (c .lastHeapScan + c .lastStackScan .Load ()+ c .globalsScan .Load ())))
1316+ c .runway .Store (uint64 (
1317+ (c .consMark * (1 - gcGoalUtilization )/ (gcGoalUtilization ))*
1318+ float64 (c .lastHeapScan + c .lastStackScan .Load ()+ c .globalsScan .Load ())))
13301319}
13311320
13321321// setGCPercent updates gcPercent. commit must be called after.