Visitar URL original
Increase production burst ceilings without changing baseline capacity… · modelcontextprotocol/registry@70234c4 · GitHub
Skip to content

Commit 70234c4

Browse files
rdimitrovrdimitrov
authored
Increase production burst ceilings without changing baseline capacity (#1793)
Increase opt-in production burst ceilings from 8 to 16 ingress replicas and from 5 to 7 worker nodes. Keep minimum capacity, ingress CPU reservation and target, application scaling, connection pools, and database placement unchanged. The deployment guard validates only the two ceiling changes and rejects ingress Deployment changes in this profile. Added profile-boundary and saved-plan scope tests; existing deployment tests pass. A private refreshed production preview is required before merge.
1 parent 5c3de5c commit 70234c4

8 files changed

Lines changed: 99 additions & 12 deletions

File tree

‎deploy/Pulumi.gcpProd.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,13 @@ config:
99
mcp-registry:nodeMachineType: e2-standard-4
1010
mcp-registry:nodeAutoscaling: "true"
1111
mcp-registry:nodeMinCount: "3"
12-
mcp-registry:nodeMaxCount: "6"
12+
mcp-registry:nodeMaxCount: "8"
1313
mcp-registry:burstAutoscaling: "true"
1414
mcp-registry:coordinatedCapacity: "true"
1515
mcp-registry:isolatedDatabase: "true"
1616
mcp-registry:productionAppScaling: enabled
1717
mcp-registry:productionIngressScaling: "true"
18+
mcp-registry:productionBurstHeadroom: "true"
1819
mcp-registry:productionInfrastructureOnly: "true"
1920
mcp-registry:registryCPURequest: "1200m"
2021
mcp-registry:postgresCPURequest: "3"

‎deploy/README.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -968,3 +968,14 @@ migration 018 and the derived columns: older writers invalidate derived bytes
968968
through its trigger. Recheck coverage before re-enabling after an older writer
969969
has run. Rolling back the application does not require a database downgrade,
970970
node-pool deletion, or volume replacement.
971+
972+
### Additional production burst headroom
973+
974+
`productionBurstHeadroom: true` extends the enabled production ingress profile
975+
to 3–16 ingress replicas and 2–7 workers (`nodeMaxCount: "8"` includes the
976+
protected database node). The 800m ingress reservation, 70% target, application
977+
2–4 HPA, pools, and minimum node count remain unchanged. The private apply
978+
guard permits only the ingress HPA maximum and worker maximum to change.
979+
This adds on-demand headroom, not a latency guarantee or an immediate node
980+
allocation. Validate placement, origin pressure, and recovery during real
981+
traffic. App scaling beyond four requires a separate database budget review.

‎deploy/pkg/k8s/burst_headroom_test.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package k8s //nolint:testpackage // Verify environment and resource boundaries.
2+
3+
import (
4+
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
5+
"testing"
6+
)
7+
8+
func TestProductionBurstHeadroomBoundaries(t *testing.T) {
9+
for _, tc := range []struct {
10+
key, value string
11+
invalid bool
12+
}{
13+
{"environment", "prod", false}, {"environment", "staging", true},
14+
{"productionIngressScaling", "false", true}, {"nodeMaxCount", "6", true},
15+
{"nodeMinCount", "4", true}, {"registryCPURequest", "800m", true},
16+
} {
17+
t.Run(tc.key+tc.value, func(t *testing.T) {
18+
cfg := map[string]string{}
19+
values := map[string]string{"productionBurstHeadroom": "true", "productionIngressScaling": "true", "productionInfrastructureOnly": "true", "productionAppScaling": "enabled", "coordinatedCapacity": "true", "isolatedDatabase": "true", "burstAutoscaling": "true", "provider": "gcp", "environment": "prod", "nodeMachineType": "e2-standard-4", "registryCPURequest": "1200m", "postgresCPURequest": "3", "postgresCPULimit": "3", "nodeMinCount": "3", "nodeMaxCount": "8", "imageTag": "1.8.7"}
20+
values[tc.key] = tc.value
21+
for k, v := range values {
22+
cfg["mcp-registry:"+k] = v
23+
}
24+
err := pulumi.RunErr(ValidateCapacityConfig, pulumi.WithMocks("test", "prod", &capacityMonitor{t: t}), func(i *pulumi.RunInfo) { i.Config = cfg })
25+
if (err != nil) != tc.invalid {
26+
t.Fatalf("unexpected validation: %v", err)
27+
}
28+
})
29+
}
30+
}

‎deploy/pkg/k8s/capacity.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ func ValidateCapacityConfig(ctx *pulumi.Context) error {
3434
infraOnly := c.GetBool("productionInfrastructureOnly")
3535
phase := c.Get("productionAppScaling")
3636
ingressScaling := c.GetBool("productionIngressScaling")
37+
burstHeadroom := c.GetBool("productionBurstHeadroom")
38+
if burstHeadroom && !ingressScaling {
39+
return fmt.Errorf("productionBurstHeadroom requires productionIngressScaling")
40+
}
3741
if ingressScaling && (!infraOnly || c.Get("environment") != "prod" || phase != "enabled") {
3842
return fmt.Errorf("productionIngressScaling requires enabled isolated production app scaling")
3943
}
@@ -79,6 +83,9 @@ func ValidateCapacityConfig(ctx *pulumi.Context) error {
7983
}
8084
if ingressScaling {
8185
nodeMax = "6"
86+
if burstHeadroom {
87+
nodeMax = "8"
88+
}
8289
}
8390
for key, expected := range map[string]string{"registryCPURequest": appCPU, "postgresCPURequest": dbCPU, "postgresCPULimit": "3", "nodeMinCount": nodeMin, "nodeMaxCount": nodeMax} {
8491
if c.Get(key) != expected {

‎deploy/pkg/k8s/ingress.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo
100100
if conf.GetBool("productionIngressScaling") {
101101
ingressCPURequest = "800m"
102102
autoscaling["maxReplicas"] = pulumi.Int(8)
103+
if conf.GetBool("productionBurstHeadroom") {
104+
autoscaling["maxReplicas"] = pulumi.Int(16)
105+
}
103106
// Retain burst headroom and the 560m target with the prior reservation.
104107
autoscaling["targetCPUUtilizationPercentage"] = pulumi.Int(70)
105108
}

‎deploy/pkg/providers/gcp/node_scaling.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ func readNodeScaling(ctx *pulumi.Context) (nodeScalingConfig, error) {
3232
}
3333
idle := c.GetBool("stagingIdleCapacity")
3434
ingressScaling := c.GetBool("productionIngressScaling")
35+
if c.GetBool("productionBurstHeadroom") && !ingressScaling {
36+
return result, fmt.Errorf("productionBurstHeadroom requires productionIngressScaling")
37+
}
3538
if ingressScaling && (!c.GetBool("isolatedDatabase") || !c.GetBool("productionInfrastructureOnly") || c.Get("environment") != "prod" || c.Get("productionAppScaling") != "enabled") {
3639
return result, fmt.Errorf("productionIngressScaling requires enabled isolated production app scaling")
3740
}
@@ -50,6 +53,9 @@ func readNodeScaling(ctx *pulumi.Context) (nodeScalingConfig, error) {
5053
}
5154
if ingressScaling {
5255
maximum = 6
56+
if c.GetBool("productionBurstHeadroom") {
57+
maximum = 8
58+
}
5359
}
5460
if !c.GetBool("coordinatedCapacity") || (c.Get("environment") != "staging" && !productionInfra) || !result.enabled || result.minimum != minimum || result.maximum != maximum {
5561
return result, fmt.Errorf("isolatedDatabase requires an explicitly enabled coordinatedCapacity profile and total node bounds %d–%d", minimum, maximum)

‎deploy/scripts/production_apply.py‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -131,8 +131,9 @@ def excluded_unchanged_rbac(step, previous):
131131
and before.get("protect", False) == after.get("protect", False))
132132

133133

134-
def validate_preview(preview, phase, completed_hooks=None, ingress_scaling=False):
134+
def validate_preview(preview, phase, completed_hooks=None, ingress_scaling=False, burst_headroom=False):
135135
require(phase in ("off", "prepare", "enabled"), "scaling-phase")
136+
require(not burst_headroom or (ingress_scaling and phase == "enabled"), "headroom-profile")
136137
require(preview.get("steps") and isinstance(preview.get("changeSummary"), dict), "preview-incomplete")
137138
require(not preview.get("maybeCorrupt"), "preview-state")
138139
require(not any(d.get("severity") == "error" for d in preview.get("diagnostics", [])), "preview-diagnostics")
@@ -141,7 +142,7 @@ def validate_preview(preview, phase, completed_hooks=None, ingress_scaling=False
141142
if completed_hooks is not None and (excluded_completed_hook(step, completed_hooks) or excluded_unchanged_rbac(step, completed_hooks)):
142143
continue
143144
try:
144-
label = validate_step(step, phase, ingress_scaling)
145+
label = validate_step(step, phase, ingress_scaling, burst_headroom)
145146
if label:
146147
labels.append(label)
147148
except Blocked as error:
@@ -150,7 +151,7 @@ def validate_preview(preview, phase, completed_hooks=None, ingress_scaling=False
150151
return sorted(set(labels))
151152

152153

153-
def validate_step(step, phase, ingress_scaling=False):
154+
def validate_step(step, phase, ingress_scaling=False, burst_headroom=False):
154155
op, urn = step["op"], step["urn"]
155156
if op in ("same", "refresh", "read"):
156157
return None
@@ -163,7 +164,7 @@ def validate_step(step, phase, ingress_scaling=False):
163164
require(phase == "enabled", "ingress-scaling-phase")
164165
require(urn not in (APP, HPA), "ingress-promotion-app-change")
165166
if urn in (INGRESS_DEPLOYMENT, INGRESS_HPA):
166-
return validate_ingress_step(step)
167+
return validate_ingress_step(step, burst_headroom)
167168
if urn not in (APP, WORKERS, HPA):
168169
# Fixed categories only: never echo URNs, property values, or diagnostics.
169170
category = {
@@ -214,8 +215,8 @@ def validate_step(step, phase, ingress_scaling=False):
214215
elif urn == WORKERS:
215216
require(op == "update" and old and new, "worker-operation")
216217
before, after = copy.deepcopy(old["inputs"]), copy.deepcopy(new["inputs"])
217-
require(before["autoscaling"].pop("maxNodeCount") in ((4, 5) if ingress_scaling else (3, 4)), "worker-existing-maximum")
218-
require(after["autoscaling"].pop("maxNodeCount") == (5 if ingress_scaling else (3 if phase == "off" else 4)), "worker-target-maximum")
218+
require(before["autoscaling"].pop("maxNodeCount") in ((5, 7) if burst_headroom else ((4, 5) if ingress_scaling else (3, 4))), "worker-existing-maximum")
219+
require(after["autoscaling"].pop("maxNodeCount") == (7 if burst_headroom else (5 if ingress_scaling else (3 if phase == "off" else 4))), "worker-target-maximum")
219220
require(before == after, "worker-unrelated-input-change")
220221
return "worker autoscaling ceiling"
221222
else:
@@ -228,12 +229,13 @@ def validate_step(step, phase, ingress_scaling=False):
228229
return "bounded application autoscaler"
229230

230231

231-
def validate_ingress_step(step):
232+
def validate_ingress_step(step, burst_headroom=False):
232233
old, new = step.get("oldState"), step.get("newState")
233234
require(step["op"] == "update" and old and new, "ingress-operation")
234235
require(all(old.get(k) == new.get(k) for k in ("provider", "parent", "protect")), "ingress-identity")
235236
before, after = copy.deepcopy(old["inputs"]), copy.deepcopy(new["inputs"])
236237
if step["urn"] == INGRESS_DEPLOYMENT:
238+
require(not burst_headroom, "headroom-must-not-change-ingress-deployment")
237239
# HPA owns replica count; refreshed old/new values must still agree.
238240
require(before["spec"]["replicas"] == after["spec"]["replicas"], "ingress-replica-ownership")
239241
for value, desired in ((before, False), (after, True)):
@@ -253,7 +255,9 @@ def validate_ingress_step(step):
253255
metric = metrics[0]["resource"]
254256
require(metric["name"] == "cpu" and metric["target"]["type"] == "Utilization", "ingress-hpa-cpu")
255257
percent = metric["target"].pop("averageUtilization")
256-
require((maximum, percent) == (8, 70) if desired else (maximum, percent) in ((6, 70), (8, 56), (8, 70)), "ingress-hpa-bounds")
258+
expected = (16, 70) if burst_headroom else (8, 70)
259+
allowed = ((8, 70), (16, 70)) if burst_headroom else ((6, 70), (8, 56), (8, 70))
260+
require((maximum, percent) == expected if desired else (maximum, percent) in allowed, "ingress-hpa-bounds")
257261
label = "bounded ingress autoscaler"
258262
require(before == after, "ingress-unrelated-input-change")
259263
return label
@@ -423,6 +427,8 @@ def main(mode):
423427
config = json.loads(quiet(["pulumi", "config", "--json", "--stack", "gcpProd", "--non-interactive"], directory, "config", env).read_text())
424428
phase = config.get("mcp-registry:productionAppScaling", {}).get("value", "off")
425429
ingress_scaling = config.get("mcp-registry:productionIngressScaling", {}).get("value", "false") == "true"
430+
burst_headroom = config.get("mcp-registry:productionBurstHeadroom", {}).get("value", "false") == "true"
431+
require(not burst_headroom or ingress_scaling, "headroom-requires-ingress-profile")
426432
quiet([sys.executable, "scripts/check_production_scaling.py"], directory, "preflight", env)
427433
print("Read-only transition preflight passed.", flush=True)
428434
exported = directory / "before.json"
@@ -431,7 +437,7 @@ def main(mode):
431437
require(all(urn in previous for urn in (STACK, APP, WORKERS)), "existing-production-stack")
432438
full_plan = directory / "full-plan.json"
433439
full = quiet(["pulumi", "preview", "--refresh", "--json", "--save-plan", str(full_plan), "--stack", "gcpProd", "--non-interactive"], directory, "full-preview", env)
434-
validate_preview(parse_preview_events(full.read_text()), phase, completed_hooks=previous, ingress_scaling=ingress_scaling)
440+
validate_preview(parse_preview_events(full.read_text()), phase, completed_hooks=previous, ingress_scaling=ingress_scaling, burst_headroom=burst_headroom)
435441
print("Full change scope validation passed.", flush=True)
436442
# Never recreate TTL-expired setup jobs as a side effect of scaling.
437443
# The full preview above still blocks every other unrelated change.
@@ -447,13 +453,13 @@ def main(mode):
447453
full_graph = decode_plan(full_plan.read_text())
448454
target_graph = decode_plan(plan.read_text())
449455
changed = dependency_mismatch_labels(full_graph, target_graph)
450-
labels = validate_preview(parse_preview_events(result.read_text()), phase, ingress_scaling=ingress_scaling)
456+
labels = validate_preview(parse_preview_events(result.read_text()), phase, ingress_scaling=ingress_scaling, burst_headroom=burst_headroom)
451457
print("Full versus targeted dependency comparison: " + (",".join(changed) if changed else "identical"), flush=True)
452458
reconciled, dependency_changes = reconcile_plan_dependencies(full_graph, target_graph)
453459
if dependency_changes:
454460
confirmed_plan = directory / "confirmed-full-plan.json"
455461
confirmed = quiet(["pulumi", "preview", "--refresh", "--json", "--save-plan", str(confirmed_plan), "--stack", "gcpProd", "--non-interactive"], directory, "confirm-preview", env)
456-
validate_preview(parse_preview_events(confirmed.read_text()), phase, completed_hooks=previous, ingress_scaling=ingress_scaling)
462+
validate_preview(parse_preview_events(confirmed.read_text()), phase, completed_hooks=previous, ingress_scaling=ingress_scaling, burst_headroom=burst_headroom)
457463
confirmation_graph = decode_plan(confirmed_plan.read_text())
458464
for urn in dependency_changes:
459465
require(dependency_signature(full_graph["resourcePlans"][urn]["goal"]) == dependency_signature(confirmation_graph["resourcePlans"][urn]["goal"]), "unstable-full-dependency-graph-" + resource_kind_label(urn))

‎deploy/scripts/test_production_apply.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,5 +248,28 @@ def test_blocks_unrelated_and_incomplete_changes(self):
248248
validate_preview(preview(changes), "enabled", ingress_scaling=True)
249249

250250

251+
class PrivateBurstHeadroom(PrivateIngressPromotion):
252+
def headroom(self):
253+
changes = self.fixtures()[1:]
254+
for change in changes:
255+
change['oldState'] = copy.deepcopy(change['newState'])
256+
changes[0]['newState']['inputs']['spec']['maxReplicas'] = 16
257+
changes[1]['newState']['inputs']['autoscaling']['maxNodeCount'] = 7
258+
return changes
259+
260+
def test_headroom_preserves_minimum_and_all_other_inputs(self):
261+
changes = self.headroom()
262+
self.assertEqual(validate_preview(preview(changes), 'enabled', ingress_scaling=True, burst_headroom=True), ['bounded ingress autoscaler', 'worker autoscaling ceiling'])
263+
for index, field, value in [(0, 'minReplicas', 4), (0, 'maxReplicas', 17)]:
264+
invalid = copy.deepcopy(changes)
265+
invalid[index]['newState']['inputs']['spec'][field] = value
266+
with self.assertRaises(Blocked):
267+
validate_preview(preview(invalid), 'enabled', ingress_scaling=True, burst_headroom=True)
268+
with self.assertRaises(Blocked):
269+
validate_preview(preview(changes + self.fixtures()[:1]), 'enabled', ingress_scaling=True, burst_headroom=True)
270+
with self.assertRaises(Blocked):
271+
validate_preview(preview(changes), 'enabled', ingress_scaling=True)
272+
273+
251274
if __name__ == "__main__":
252275
unittest.main()

0 commit comments

Comments
 (0)