Repository navigation
Expand file tree
/
Copy pathmerge_spark_pr.py
More file actions
executable file
·2914 lines (2528 loc) · 119 KB
/
Copy pathmerge_spark_pr.py
File metadata and controls
executable file
·2914 lines (2528 loc) · 119 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python3
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Utility for creating well-formed pull request merges and pushing them to Apache
# Spark.
# usage: ./merge_spark_pr.py (see config env vars below)
#
# Rolling integration branches branch-M.x (e.g. branch-4.x; later branch-5.x after 5.0.0)
# absorb work for the next minors on major M. Maintenance branches branch-M.N are cut from
# branch-M.x. Cherry-pick prompts sort by semver via semver_branch_rank (branch-M.x before branch-M.N).
#
# Fix Version defaults when resolving JIRA aim for the minimal set of versions that tells users
# which Spark releases will contain the commit, leveraging the Upstream-First backporting policy
# (cherry-picks flow master -> branch-M.x -> branch-M.N). master contributes the greatest
# unreleased N.0.0, but is omitted when any branch-M.x is in the merge set (a cherry-pick to
# branch-M.x has already landed on master). Each branch-M.x contributes that major's greatest
# unreleased minor.0, but is omitted when a sibling branch-M.N contributes M.N.0 (M.N.0 still
# unreleased means branch-M.N is still a subset of branch-M.x). Patch merges on branch-M.N use
# unreleased versions starting with M.N.
# This utility assumes you already have a local Spark git folder and that you
# have added remotes corresponding to both (i) the github apache Spark
# mirror and (ii) the apache git repo.
import argparse
import json
import os
import re
import subprocess
import sys
import traceback
from typing import List, Optional
from urllib.error import HTTPError
from urllib.request import Request, urlopen
# Shared with dev/pr_merge_status.py so the two committer tools agree on where a PR landed.
# Importable because Python puts this script's own directory first on sys.path.
from spark_merge_footer import branches_with_merge_footer as _branches_with_merge_footer
from spark_merge_footer import has_merge_footer
try:
import jira.client
JIRA_IMPORTED = True
except ImportError:
JIRA_IMPORTED = False
# Location of your Spark git development area
SPARK_HOME = os.environ.get("SPARK_HOME", os.getcwd())
# Remote name which points to the Github site
PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache-github")
# Remote name which points to Apache git
PUSH_REMOTE_NAME = os.environ.get("PUSH_REMOTE_NAME", "apache")
# ASF JIRA username
JIRA_USERNAME = os.environ.get("JIRA_USERNAME", "")
# ASF JIRA password
JIRA_PASSWORD = os.environ.get("JIRA_PASSWORD", "")
# ASF JIRA access token
# If it is configured, username and password are dismissed
# Go to https://issues.apache.org/jira/secure/ViewProfile.jspa -> Personal Access Tokens for
# your own token management.
JIRA_ACCESS_TOKEN = os.environ.get("JIRA_ACCESS_TOKEN")
# Connection timeout (in seconds) for the JIRA client. Raise this if the default is too
# short for your network, e.g. when the TLS handshake goes through a slow proxy.
JIRA_CONNECT_TIMEOUT = float(os.environ.get("JIRA_CONNECT_TIMEOUT", "3.05"))
# OAuth key used for issuing requests against the GitHub API. If this is not defined, then requests
# will be unauthenticated. You should only need to configure this if you find yourself regularly
# exceeding your IP's unauthenticated request rate limit. You can create an OAuth key at
# https://github.com/settings/tokens. This script only requires the "public_repo" scope.
GITHUB_OAUTH_KEY = os.environ.get("GITHUB_OAUTH_KEY")
# Setting the DRY_RUN env var to any non-empty value makes the --dry-run/-n flag default to on
# (see build_arg_parser); the flag is the primary interface and can also turn it on explicitly.
# Consistent with the SKIP_VERSION_CHECK env toggle above. In a dry run every read-only step and
# local git op still runs -- fetching the PR, JIRA lookup, project_versions, JIRA/GitHub token
# validation, and the local squash-merge and cherry-picks on the throwaway PR_TOOL_* branches so
# conflicts and the computed merge hash stay realistic -- but every outbound effect is routed
# through a DryRun* client (see Git/GitHub/Jira below) that logs a "DRY-RUN: would ..." line
# instead of executing it: the git push to PUSH_REMOTE_NAME, the GitHub PR close/comment, and all
# JIRA writes (component and fixVersion updates, assignment, and resolve/reopen transitions).
DRY_RUN_ENV = bool(os.environ.get("DRY_RUN"))
GITHUB_BASE = "https://github.com/apache/spark/pull"
GITHUB_COMMIT_BASE = "https://github.com/apache/spark/commit"
GITHUB_API_BASE = "https://api.github.com/repos/apache/spark"
JIRA_BASE = "https://issues.apache.org/jira/browse"
JIRA_API_BASE = "https://issues.apache.org/jira"
# Prefix added to temporary branches
BRANCH_PREFIX = "PR_TOOL"
# Set to a truthy value to skip the check that compares this script against apache/spark master.
# Intended for committers iterating on the merge script itself, where a local diff is expected.
SKIP_VERSION_CHECK = os.environ.get("SKIP_VERSION_CHECK", "")
# Path of this script relative to the repo root, used to fetch the canonical copy from master.
MERGE_SCRIPT_REPO_PATH = "dev/merge_spark_pr.py"
def semver_branch_rank(name):
"""
Sort key for maintenance branches: higher merges / cherry-picks first.
Goals:
- Numeric Spark branches branch-M.N sort by (M, minor), not lexicographically
(branch-4.11 is newer than branch-4.2).
- Rolling integration branches branch-M.x rank before branch-M.N for the same M so they are
suggested first among targets on major M.
>>> semver_branch_rank("branch-5.x") > semver_branch_rank("branch-5.0") > semver_branch_rank("branch-4.11")
True
>>> semver_branch_rank("branch-5.x") > semver_branch_rank("branch-5.2")
True
>>> semver_branch_rank("branch-4.x") > semver_branch_rank("branch-4.2")
True
>>> semver_branch_rank("branch-4.2") > semver_branch_rank("branch-4.11")
False
"""
integration = re.match(r"^branch-(\d+)\.x$", name)
if integration:
return int(integration.group(1)), float("inf")
matched = re.match(r"^branch-(\d+)\.(\d+)$", name)
if matched:
return int(matched.group(1)), int(matched.group(2))
return (-1, -1)
def parse_version(name: str) -> Optional[tuple[int, int, int]]:
"""Parse a dotted ``x.y.z`` version name into an ``(int, int, int)`` tuple.
Returns None for anything that is not a plain three-part numeric version, so the
result can be compared directly and non-version names are skipped by callers.
>>> parse_version("4.4.0")
(4, 4, 0)
>>> parse_version("4.10.2")
(4, 10, 2)
>>> parse_version("branch-4.x") is None
True
>>> parse_version("4.4") is None
True
"""
m = re.match(r"^(\d+)\.(\d+)\.(\d+)$", name)
return tuple(int(g) for g in m.groups()) if m else None
def _semver_max_version(names):
"""
Highest dotted version by numeric semver (SPARK Fix Version naming).
>>> _semver_max_version(["4.2.1", "4.3.0"])
'4.3.0'
>>> _semver_max_version(["5.0.0", "6.0.0"])
'6.0.0'
"""
parsed = [(parse_version(n), n) for n in names]
parsed = [(t, n) for t, n in parsed if t is not None]
if not parsed:
return None
return max(parsed)[1]
def _integration_major_from_branch(branch_name):
"""If branch-name is branch-M.x, return integer M; else None."""
m = re.match(r"^branch-(\d+)\.x$", branch_name)
return int(m.group(1)) if m else None
def _first_integration_major(merge_branches):
"""First branch-M.x on merge_branches in list order (deterministic).
>>> _first_integration_major(["master", "branch-5.x", "branch-4.x"])
5
>>> _first_integration_major(["master", "branch-4.2"]) is None
True
"""
for ref in merge_branches:
mj = _integration_major_from_branch(ref)
if mj is not None:
return mj
return None
def _sort_version_names_lex_desc(names):
"""Same ordering as resolve_jira_issue (reverse lexicographic on Fix Version name)."""
return sorted(names, reverse=True)
def compute_merge_default_fix_versions(merge_branches, unreleased_version_names):
"""
Build suggested SPARK Fix Version names from merged git refs (no JIRA I/O).
unreleased_version_names: x.y.z strings marked unreleased in JIRA.
Returns (default_version_names, warning_messages).
The output is the minimal set of Fix Versions that tells users which Spark releases will
contain the commit, leveraging the Upstream-First backporting policy (cherry-picks flow
master -> branch-M.x -> branch-M.N):
- master contributes the greatest unreleased N.0.0;
- branch-M.x contributes that major's greatest unreleased minor.0;
- branch-M.N contributes its greatest unreleased M.N.patch.
Redundant entries are then suppressed: master's N.0.0 is dropped when any branch-M.x is in
the merge set (a cherry-pick to branch-M.x has already landed on master); branch-M.x's
minor.0 is dropped when a branch-M.N for the same major contributes M.N.0 (M.N.0 still
unreleased means branch-M.N is still a subset of branch-M.x).
>>> compute_merge_default_fix_versions(["master"], ["4.3.0", "5.0.0", "6.0.0"])[0]
['6.0.0']
>>> compute_merge_default_fix_versions(
... ["master", "branch-4.x"], ["6.0.0", "5.0.0", "4.3.0", "4.2.1"])[0]
['4.3.0']
>>> compute_merge_default_fix_versions(
... ["master", "branch-5.x"],
... ["7.0.0", "6.0.0", "5.2.0", "5.1.2", "5.1.1", "5.0.1"],
... )[0]
['5.2.0']
>>> compute_merge_default_fix_versions(
... ["master", "branch-3.5"],
... ["6.0.0", "4.1.0", "3.5.2", "3.5.1", "3.5.0"],
... )[0]
['6.0.0', '3.5.2']
>>> compute_merge_default_fix_versions(
... ["master", "branch-1.1", "branch-1.0"],
... ["2.0.0", "1.1.1", "1.1.0", "1.0.1", "1.0.0"],
... )[0]
['2.0.0', '1.1.1', '1.0.1']
>>> d, w = compute_merge_default_fix_versions(["master"], ["4.3.0", "4.2.1"])
>>> (d, len(w), "N.0.0" in w[0])
([], 1, True)
>>> compute_merge_default_fix_versions(["branch-4.x"], ["4.3.0"])
(['4.3.0'], [])
>>> d, w = compute_merge_default_fix_versions(["branch-4.99"], ["4.3.0"])
>>> d == [] and len(w) == 1 and "branch-4.99" in w[0]
True
>>> compute_merge_default_fix_versions(
... ["master", "branch-5.x", "branch-4.x"], ["5.2.0", "4.3.0", "6.0.0"]
... )[0]
['5.2.0', '4.3.0']
>>> compute_merge_default_fix_versions(
... ["master", "branch-4.x", "branch-4.x"], ["6.0.0", "4.3.0"]
... )[0]
['4.3.0']
>>> compute_merge_default_fix_versions(
... ["master", "branch-4.x", "branch-4.2"], ["5.0.0", "4.3.0", "4.2.0"]
... )[0]
['4.2.0']
>>> compute_merge_default_fix_versions(
... ["master", "branch-4.x", "branch-4.2"], ["5.0.0", "4.3.0", "4.2.1"]
... )[0]
['4.3.0', '4.2.1']
>>> compute_merge_default_fix_versions(
... ["master", "branch-4.2"], ["5.0.0", "4.2.0"]
... )[0]
['5.0.0', '4.2.0']
>>> compute_merge_default_fix_versions(
... ["master", "branch-4.2"], ["5.0.0", "4.2.1"]
... )[0]
['5.0.0', '4.2.1']
"""
names = _sort_version_names_lex_desc(list(unreleased_version_names))
has_integration_branch = any(
_integration_major_from_branch(b) is not None for b in merge_branches
)
# Collect each merge branch's candidate Fix Version. We track (branch, version) so we can
# later suppress redundant entries (master, branch-M.x) based on which branches contributed.
contributions = []
warnings = []
for b in merge_branches:
if b == "master":
majors = [n for n in names if re.match(r"^\d+\.0\.0$", n)]
chosen = _semver_max_version(majors)
if chosen:
contributions.append((b, chosen))
else:
warnings.append(
"No unreleased N.0.0 Fix Version found in JIRA for master; "
"enter comma-separated Fix Version(s) manually when prompted."
)
continue
line_major = _integration_major_from_branch(b)
if line_major is not None:
line_versions = [n for n in names if re.match(r"^%s\.\d+\.\d+$" % line_major, n)]
chosen = _semver_max_version(line_versions)
if chosen:
contributions.append((b, chosen))
else:
warnings.append(
"Could not infer an unreleased Spark %s (minor.maintenance) fix version "
"for branch-%s.x merge; enter version(s) manually when prompted."
% (line_major, line_major)
)
continue
prefix = b.replace("branch-", "") + "."
found_versions = [n for n in names if n.startswith(prefix)]
chosen = _semver_max_version(found_versions)
if chosen:
contributions.append((b, chosen))
else:
warnings.append(
"Target version for %s is not found on JIRA, it may be archived or "
"not created. Skipping it." % b
)
# Majors where a release branch's M.N.0 contribution makes the sibling branch-M.x's
# minor.0 redundant: an unreleased M.N.0 means branch-M.N has not diverged from branch-M.x
# yet, so anything that's on branch-M.N is also on branch-M.x.
suppress_integration_majors = set()
for b, v in contributions:
if b == "master" or _integration_major_from_branch(b) is not None:
continue
m = re.match(r"^(\d+)\.\d+\.0$", v)
if m:
suppress_integration_majors.add(int(m.group(1)))
def keep(item):
b, _ = item
if b == "master":
return not has_integration_branch
major = _integration_major_from_branch(b)
if major is not None:
return major not in suppress_integration_majors
return True
filtered = [item for item in contributions if keep(item)]
return list(dict.fromkeys(v for _, v in filtered)), warnings
def compute_revert_fix_versions(merge_branches, unreleased_version_names, existing_version_names):
"""Suggest recorded Fix Versions to remove for the branches receiving a revert.
Infer each branch separately so that redundant versions recorded by separate merge
runs are included in the removal candidates.
>>> versions = ["5.0.0", "4.3.0", "4.2.0", "3.5.2"]
>>> compute_revert_fix_versions(
... ["master", "branch-4.x", "branch-4.2"], versions, versions)
(['5.0.0', '4.3.0', '4.2.0'], [])
>>> compute_revert_fix_versions(["master"], versions, versions)
(['5.0.0'], [])
>>> compute_revert_fix_versions(["branch-4.2"], versions, ["4.3.0", "3.5.2"])
([], [])
>>> compute_revert_fix_versions(["branch-4.x"], versions, ["4.3.0", "4.2.0"])
(['4.3.0'], [])
>>> compute_revert_fix_versions(
... ["branch-4.2"], ["4.2.1", "4.20.0"], ["4.2.1", "4.20.0"])
(['4.2.1'], [])
"""
candidates = set()
warnings = []
for branch in merge_branches:
versions, branch_warnings = compute_merge_default_fix_versions(
[branch], unreleased_version_names
)
candidates.update(versions)
warnings.extend(branch_warnings)
return [v for v in existing_version_names if v in candidates], warnings
def additional_fix_versions(inferred_versions, existing_versions):
"""Return inferred Fix Versions not already present on a JIRA issue.
Existing versions are preserved separately when the issue is updated, so this only
identifies the additions needed after a later backport.
>>> additional_fix_versions(["4.4.0"], ["5.0.0"])
['4.4.0']
>>> additional_fix_versions(["4.4.0"], ["5.0.0", "4.4.0"])
[]
>>> additional_fix_versions(["4.4.0", "4.3.4"], ["5.0.0", "4.4.0"])
['4.3.4']
"""
existing = set(existing_versions)
return [version for version in inferred_versions if version not in existing]
def fix_version_additions(inferred_versions, existing_versions):
"""Return (additions, all_inferred_present).
An empty ``additions`` list is ambiguous on its own: either the issue already carries
every inferred version, or nothing was inferred. Only the first means there is nothing
to do; the second still needs the committer prompted.
>>> fix_version_additions(["4.3.0"], ["5.0.0"])
(['4.3.0'], False)
>>> fix_version_additions(["5.0.0"], ["5.0.0"])
([], True)
>>> fix_version_additions([], ["5.0.0"])
([], False)
"""
additions = additional_fix_versions(inferred_versions, existing_versions)
return additions, bool(inferred_versions) and not additions
def parse_version_list(raw: str) -> list[str]:
"""Split a comma-separated version string into trimmed, de-duplicated names.
Surrounding whitespace on each entry is stripped and empty entries are dropped, so a
blank or all-separator string yields no names. Input order is preserved. Shared by the
Fix Version and Affects Version prompts so both parse committer input the same way.
>>> parse_version_list("5.0.0, 4.3.0")
['5.0.0', '4.3.0']
>>> parse_version_list(" 4.4.0 , , 4.4.0 ")
['4.4.0']
>>> parse_version_list(" ")
[]
"""
names = [segment.strip() for segment in raw.split(",")]
return list(dict.fromkeys(n for n in names if n))
def fix_versions_from_input(raw_input, default_fix_versions):
"""Resolve the Fix Version prompt's raw input into a list of version names.
Blank falls back to the inferred default. With no default to fall back on, blank means
skip: the empty list, not [""], which no known version can match.
>>> fix_versions_from_input("4.2.2", "")
['4.2.2']
>>> fix_versions_from_input("", "5.0.0")
['5.0.0']
>>> fix_versions_from_input("5.0.0, 4.3.0", "")
['5.0.0', '4.3.0']
>>> fix_versions_from_input("", "")
[]
>>> fix_versions_from_input(" ", "")
[]
"""
if raw_input == "":
raw_input = default_fix_versions
return parse_version_list(raw_input)
def fix_precedes_affects(fix_version_names: list[str], affects_version_names: list[str]) -> bool:
"""Whether the earliest fix version is below the earliest recorded Affects Version.
True means the affected floor sits above a fixed release, so the ticket omits a
version the fix reaches -- either too-high on a fresh resolve (fixed 4.4.0, affects
only 5.0.0) or a backport to an earlier line (affects 4.4.0, backport adds fix 4.3.x).
Non-``x.y.z`` names and empty lists compare as absent, so they never trigger a prompt.
>>> fix_precedes_affects(["4.4.0"], ["5.0.0"])
True
>>> fix_precedes_affects(["4.4.0"], ["4.3.0"])
False
>>> fix_precedes_affects(["4.4.0"], ["4.4.0"])
False
>>> fix_precedes_affects(["4.4.0", "4.3.1"], ["4.4.0"])
True
>>> fix_precedes_affects(["4.4.0"], ["4.10.0"])
True
>>> fix_precedes_affects([], ["5.0.0"])
False
"""
fix = [t for t in map(parse_version, fix_version_names) if t is not None]
affects = [t for t in map(parse_version, affects_version_names) if t is not None]
if not fix or not affects:
return False
return min(fix) < min(affects)
def suggest_affects_version(
fix_version_names: list[str], unreleased_version_names: list[str]
) -> str:
"""Suggest the earliest fix version only if it is an unreleased major or minor release.
An unreleased x.y.0 can identify affected development builds. A patch fix version
does not identify the earlier release affected by the bug.
>>> unreleased = ["5.0.0", "4.4.0", "4.2.1"]
>>> suggest_affects_version(["4.4.0"], unreleased)
'4.4.0'
>>> suggest_affects_version(["5.0.0"], unreleased)
'5.0.0'
>>> suggest_affects_version(["5.0.0", "4.4.0"], unreleased)
'4.4.0'
>>> suggest_affects_version(["4.2.1"], unreleased)
''
>>> suggest_affects_version(["4.4.0", "4.2.1"], unreleased)
''
>>> suggest_affects_version(["4.3.0", "4.4.0"], unreleased)
''
>>> suggest_affects_version(["4.10.0", "4.9.0"], ["4.10.0", "4.9.0"])
'4.9.0'
>>> suggest_affects_version([], unreleased)
''
>>> suggest_affects_version(["unknown", "4.4.0"], unreleased)
''
"""
versions = [(parse_version(name), name) for name in fix_version_names]
if not versions or any(version is None for version, _ in versions):
return ""
version, name = min(versions)
return name if version[2] == 0 and name in unreleased_version_names else ""
def red(text):
return "\033[91m%s\033[0m" % text
def bold(text):
return "\033[1m%s\033[0m" % text
def print_error(msg):
print(red(msg))
def bold_input(prompt) -> str:
return input("\033[1m%s\033[0m" % prompt)
def get_input(prompt, options, bold=True, ignore_case=True):
"""
Get input from the user until a valid answer is provided.
Args:
prompt: The prompt to display to the user.
options:
* A dictionary of "option: accepted answer" to choose from.
* A list of options.
* A regex pattern.
bold: Whether to use bold formatting for the prompt.
ignore_case: Whether to ignore case when comparing the answer to the options.
Returns:
If options is provided - the valid answer from the user.
If regex is provided
* If no group is specified - the whole matched string.
* If one group is specified - the first group.
* If multiple groups are specified - a tuple of the groups.
"""
# Normalize options
if isinstance(options, (list, tuple)):
options = {option: [option] for option in options}
if isinstance(options, dict):
for option, acceptable_answer in options.items():
if ignore_case:
options[option] = [answer.lower() for answer in acceptable_answer]
while True:
if bold:
answer = bold_input(prompt)
else:
answer = input(prompt)
answer = answer.strip()
if ignore_case:
answer = answer.lower()
if isinstance(options, str):
if (m := re.match(options, answer)) is not None:
groups = m.groups()
if len(groups) == 0:
return m.group(0)
if len(groups) == 1:
return groups[0]
else:
return groups
else:
for option, acceptable_answer in options.items():
if answer in acceptable_answer:
return option
def get_json(url):
try:
request = Request(url)
if GITHUB_OAUTH_KEY:
request.add_header("Authorization", "token %s" % GITHUB_OAUTH_KEY)
return json.load(urlopen(request))
except HTTPError as e:
if "X-RateLimit-Remaining" in e.headers and e.headers["X-RateLimit-Remaining"] == "0":
print_error(
"Exceeded the GitHub API rate limit; see the instructions in "
+ "dev/merge_spark_pr.py to configure an OAuth token for making authenticated "
+ "GitHub requests."
)
elif e.code == 401:
print_error(
"GITHUB_OAUTH_KEY is invalid or expired. Please regenerate a new one with "
+ "at least the 'public_repo' scope on https://github.com/settings/tokens and "
+ "update your local settings before you try again."
)
else:
print_error("Unable to fetch URL, exiting: %s" % url)
sys.exit(-1)
def merge_commit_candidates(pr_events):
"""Split `pr_events` into (closed_commits, referenced_commits), each oldest-first.
Ordered by time so that a PR reopened and merged again yields its latest merge last.
>>> merge_commit_candidates([{"event": "closed", "commit_id": "a", "created_at": "t2"},
... {"event": "referenced", "commit_id": "b", "created_at": "t1"}])
(['a'], ['b'])
>>> merge_commit_candidates([{"event": "closed", "commit_id": None, "created_at": "t1"}])
([], [])
>>> merge_commit_candidates([{"event": "referenced", "commit_id": "c", "created_at": "t2"},
... {"event": "referenced", "commit_id": "b", "created_at": "t1"}])
([], ['b', 'c'])
"""
def commits_of(event_name):
matched = [e for e in pr_events if e["event"] == event_name and e["commit_id"] is not None]
return [e["commit_id"] for e in sorted(matched, key=lambda x: x["created_at"])]
return commits_of("closed"), commits_of("referenced")
def find_merge_commit(pr_num, pr_events):
"""Return (hash, message) of the commit that merged `pr_num`, or (None, None).
GitHub attributes the merge commit to the `closed` event only when that commit lands
on the default branch (master), because the "Closes #N" keyword in the commit message
is what closes the PR and the keyword is honored only there. A PR merged into any
other branch -- e.g. one opened against a rolling branch-M.x -- is instead closed by
this script through the API, and that `closed` event carries no commit, so the merge
survives only as a `referenced` event. Prefer the `closed` commit, which GitHub itself
linked; otherwise fall back to `referenced` events, which are also raised by any commit
merely mentioning the PR, so confirm each against the merge footer `merge_pr` generates.
"""
def message_of(commit_hash):
return get_json("%s/commits/%s" % (GITHUB_API_BASE, commit_hash))["commit"]["message"]
closed_commits, referenced_commits = merge_commit_candidates(pr_events)
if closed_commits:
return closed_commits[-1], message_of(closed_commits[-1])
for commit_hash in reversed(referenced_commits):
message = message_of(commit_hash)
if has_merge_footer(message, pr_num):
return commit_hash, message
return None, None
class GitHub:
"""GitHub REST writes used by the merge script -- the single seam for PR mutations.
Reads go through the module-level get_json(); only the two mutations (closing the PR and
posting a comment) live here, so DryRunGitHub can override them without touching any read
path. main() constructs GitHub() normally and DryRunGitHub() for a dry run.
"""
def close_pr(self, pr_num):
url = "%s/pulls/%s" % (GITHUB_API_BASE, pr_num)
data = json.dumps({"state": "closed"}).encode("utf-8")
request = Request(url, data=data, method="PATCH")
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/vnd.github+json")
if GITHUB_OAUTH_KEY:
request.add_header("Authorization", "token %s" % GITHUB_OAUTH_KEY)
try:
return json.load(urlopen(request))
except HTTPError as e:
print_error("Failed to close PR #%s: HTTP %s %s" % (pr_num, e.code, e.reason))
return None
def comment(self, pr_num, body):
# Posting a comment is a write and needs auth; without a token, skip rather than 401.
if not GITHUB_OAUTH_KEY:
print_error("GITHUB_OAUTH_KEY is not set; skipping the comment.")
return None
url = "%s/issues/%s/comments" % (GITHUB_API_BASE, pr_num)
data = json.dumps({"body": body}).encode("utf-8")
request = Request(url, data=data, method="POST")
request.add_header("Content-Type", "application/json")
request.add_header("Accept", "application/vnd.github+json")
request.add_header("Authorization", "token %s" % GITHUB_OAUTH_KEY)
try:
return json.load(urlopen(request))
except HTTPError as e:
print_error("Failed to comment on PR #%s: HTTP %s %s" % (pr_num, e.code, e.reason))
return None
class DryRunGitHub(GitHub):
"""Logs the intended PR close/comment instead of calling the GitHub API."""
def close_pr(self, pr_num):
print("DRY-RUN: would close PR #%s via the GitHub API." % pr_num)
return None
def comment(self, pr_num, body):
print("DRY-RUN: would post the following comment on PR #%s:\n%s" % (pr_num, body))
return None
# Set in main(): GitHub() normally, DryRunGitHub() for a dry run. Module-level default so the
# top-level except handler and any early failure path have a usable client.
github = GitHub()
def post_merge_comment(pr_num, merged_commits):
"""Post a comment on the PR recording every branch the change landed on and a
link to the resulting commit, so the merge is traceable from the PR page.
``merged_commits`` is an ordered list of (branch, commit_hash) pairs, the merge
sink first followed by each cherry-pick target.
"""
if not merged_commits:
return
lines = [
"- merged into %s %s/%s" % (ref, GITHUB_COMMIT_BASE, commit_hash)
for ref, commit_hash in merged_commits
]
summary = "**Merge Summary:**\n" + "\n".join(lines)
attribution = "*Posted by `merge_spark_pr.py`*"
body = "%s\n\n%s" % (summary, attribution)
print(
"\n%s\n\n%s\n%s"
% (bold("Posting merge comment on PR #%s:" % pr_num), bold(summary), attribution)
)
github.comment(pr_num, body)
def fail(msg):
print_error(msg)
clean_up()
sys.exit(-1)
class Git:
"""Runs the merge-flow git commands for the merge script.
This is the seam for the merge/backport flow, not every git invocation:
check_script_up_to_date() shells out to git merge-base directly. ``run`` executes read-only
and local commands (fetch, checkout, merge, commit, cherry-pick, rev-parse, config, branch
bookkeeping); those are safe even in a dry run because the merge and cherry-picks happen on
the throwaway PR_TOOL_* branches that clean_up removes. ``push`` is the only command that
mutates the shared repo at PUSH_REMOTE_NAME, so it is the one method DryRunGit overrides;
everything else it inherits and runs for real, keeping conflict detection and the computed
merge hash realistic. main() constructs Git() normally and DryRunGit() for a dry run.
"""
def run(self, cmd):
print(cmd)
if isinstance(cmd, list):
return subprocess.check_output(cmd).decode("utf-8")
else:
return subprocess.check_output(cmd.split(" ")).decode("utf-8")
def push(self, remote, local_ref, remote_ref):
return self.run("git push %s %s:%s" % (remote, local_ref, remote_ref))
class DryRunGit(Git):
"""Logs the intended push instead of updating the remote; all other git runs for real."""
def push(self, remote, local_ref, remote_ref):
print("DRY-RUN: would push %s to %s:%s" % (local_ref, remote, remote_ref))
return ""
# Set in main(): Git() normally, DryRunGit() for a dry run. Module-level default so clean_up and
# the top-level except handler have a usable client even if main() fails early.
git = Git()
class SkipCherryPick(Exception):
"""Signals that the committer declined to resolve a conflicting cherry-pick.
A backport conflict is routine, and by the time one is hit the merge into the target
branch (and any earlier cherry-picks) has already been pushed. Declining it must skip
only that one branch -- letting the caller offer another branch and, crucially, still
resolve the JIRA -- instead of aborting the whole merge the way a hard `fail()` would.
"""
def continue_maybe(prompt, cherry=False):
if get_input(f"{prompt} (y/N): ", ["y", "n", ""]) != "y":
if cherry:
try:
git.run("git cherry-pick --abort")
except Exception:
print_error("Unable to abort and get back to the state before cherry-pick")
print("Skipping this cherry-pick; the merge continues.")
clean_up()
raise SkipCherryPick()
fail("Okay, exiting")
def clean_up():
if "original_head" in globals():
print("Restoring head pointer to %s" % original_head)
git.run("git checkout %s" % original_head)
branches = git.run("git branch").replace(" ", "").split("\n")
for branch in list(filter(lambda x: x.startswith(BRANCH_PREFIX), branches)):
print("Deleting local branch %s" % branch)
git.run("git branch -D %s" % branch)
# merge the requested PR and return the merge hash
def merge_pr(pr_num, target_ref, title, body, pr_repo_desc, pr_author, co_authors):
pr_branch_name = "%s_MERGE_PR_%s" % (BRANCH_PREFIX, pr_num)
target_branch_name = "%s_MERGE_PR_%s_%s" % (BRANCH_PREFIX, pr_num, target_ref.upper())
git.run("git fetch %s pull/%s/head:%s" % (PR_REMOTE_NAME, pr_num, pr_branch_name))
git.run("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, target_ref, target_branch_name))
git.run("git checkout %s" % target_branch_name)
had_conflicts = False
try:
git.run(["git", "merge", pr_branch_name, "--squash"])
except Exception as e:
msg = "Error merging: %s\nWould you like to manually fix-up this merge?" % e
continue_maybe(msg)
msg = "Okay, please fix any conflicts and 'git add' conflicting files... Finished?"
continue_maybe(msg)
had_conflicts = True
# Use PR author as the primary author for proper GitHub attribution
primary_author = bold_input(
'Enter primary author in the format of "name <email>" [%s]: ' % pr_author
)
if primary_author == "":
primary_author = pr_author
merge_message_flags = []
merge_message_flags += ["-m", title]
if body is not None:
# We remove @ symbols from the body to avoid triggering e-mails
# to people every time someone creates a public fork of Spark.
merge_message_flags += ["-m", body.replace("@", "")]
committer_name = git.run("git config --get user.name").strip()
committer_email = git.run("git config --get user.email").strip()
if had_conflicts:
message = "This patch had conflicts when merged, resolved by\nCommitter: %s <%s>" % (
committer_name,
committer_email,
)
merge_message_flags += ["-m", message]
# The string "Closes #%s" string is required for GitHub to correctly close the PR
merge_message_flags += ["-m", "Closes #%s from %s." % (pr_num, pr_repo_desc)]
authors = "Authored-by:" if len(co_authors) == 0 else "Lead-authored-by:"
authors += " %s" % primary_author
if len(co_authors) > 0:
authors += "\n" + "\n".join(["Co-authored-by: %s" % a for a in co_authors])
authors += "\n" + "Signed-off-by: %s <%s>" % (committer_name, committer_email)
merge_message_flags += ["-m", authors]
git.run(["git", "commit", '--author="%s"' % primary_author] + merge_message_flags)
continue_maybe(
"Merge complete (local ref %s). Push to %s?" % (target_branch_name, PUSH_REMOTE_NAME)
)
try:
git.push(PUSH_REMOTE_NAME, target_branch_name, target_ref)
except Exception as e:
clean_up()
print_error("Exception while pushing: %s" % e)
merge_hash = git.run("git rev-parse %s" % target_branch_name).strip()
clean_up()
print("Pull request #%s merged!" % pr_num)
print("Merge hash: %s" % merge_hash)
return merge_hash
def _do_cherry_pick(pr_num, merge_hash, pick_ref):
"""Cherry-pick `merge_hash` onto `pick_ref` and push.
Returns the (pushed ref, pushed commit hash) pair. Raises `SkipCherryPick` if the
cherry-pick conflicts and the committer declines to resolve it, after attempting to
abort the cherry-pick and restore the working tree.
"""
pick_branch_name = "%s_PICK_PR_%s_%s" % (BRANCH_PREFIX, pr_num, pick_ref.upper())
git.run("git fetch %s %s:%s" % (PUSH_REMOTE_NAME, pick_ref, pick_branch_name))
git.run("git checkout %s" % pick_branch_name)
pick_head = git.run("git rev-parse HEAD").strip()
try:
git.run(
[
"git",
"-c",
"commit.cleanup=scissors",
"cherry-pick",
"-sx",
merge_hash,
]
)
except Exception as e:
msg = "Error cherry-picking: %s\nWould you like to manually fix-up this merge?" % e
continue_maybe(msg, True)
msg = "Okay, please fix any conflicts and 'git add' conflicting files... Finished?"
continue_maybe(msg, True)
if git.run("git rev-parse HEAD").strip() == pick_head:
# Important to use `scissors` and `--edit` otherwise git will strip lines starting
# with `#` when calling `--continue`. See: https://github.com/apache/spark/pull/58214
git.run(
[
"git",
"-c",
"commit.cleanup=scissors",
"cherry-pick",
"--continue",
"--edit",
]
)
else:
print("Cherry-pick already completed manually; continuing with the backport.")
continue_maybe(
"Pick complete (local ref %s). Push to %s?" % (pick_branch_name, PUSH_REMOTE_NAME)
)
try:
git.push(PUSH_REMOTE_NAME, pick_branch_name, pick_ref)
except Exception as e:
fail("Exception while pushing: %s" % e)
pick_hash = git.run("git rev-parse %s" % pick_branch_name).strip()
clean_up()
print("Pull request #%s picked into %s!" % (pr_num, pick_ref))
print("Pick hash: %s" % pick_hash)
return pick_ref, pick_hash
def branches_with_merge_footer(pr_num, branch_names):
"""Release branches from `branch_names` that already carry `pr_num`'s merge footer.
Thin wrapper over the shared reader in `spark_merge_footer`, adding this script's own
policy: a git failure here must not abort a merge that may already have pushed, so it
warns and reports nothing rather than exiting. Per that module's refresh policy no fetch
is issued, so a backport not yet fetched into PUSH_REMOTE_NAME's tracking refs is simply
not reported -- the committer is still prompted and can type any branch.
"""
try:
landed = _branches_with_merge_footer(
pr_num, PUSH_REMOTE_NAME, lambda args: git.run(["git"] + args)
)
except Exception as e:
print_error("Could not scan for existing backports of #%s (%s)." % (pr_num, e))
return []
# Keep branch_names' newest-first order, and drop anything not a known release branch.
return [b for b in branch_names if b in landed]
def default_pick_branch(branch_names, already_picked):
"""Highest-ranked release branch that has not already received the change, or None.
`branch_names` is ordered newest-first (see `semver_branch_rank`) and `already_picked`
holds the branches the change is known to be on, so the prompt never defaults to a
branch where the cherry-pick would come up empty. Returns None when every known branch
already has it, so callers can say so instead of offering an empty pick.
>>> default_pick_branch(["branch-4.x", "branch-4.3", "branch-4.2"], ("branch-4.x",))
'branch-4.3'
>>> default_pick_branch(["branch-4.x", "branch-4.3"], ())
'branch-4.x'
>>> default_pick_branch(["branch-4.x"], ("branch-4.x",)) is None
True
"""
remaining = [b for b in branch_names if b not in already_picked]
return remaining[0] if remaining else None