Visitar URL original
Compare any two indexes by stonebig · Pull Request #2105 · winpython/winpython · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 11 additions & 23 deletions .github/scripts/changelog_files.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""File a cycle's build output into changelogs/, and write the histories.
"""File a cycle's build output into changelogs/.

python .github/scripts/changelog_files.py <downloaded metadata> changelogs

Expand All @@ -8,10 +8,13 @@
The fourth, hashes_<winpyver>.md, describes the binaries of one build rather
than the release, and stays out.

The _History.md companions are then written here rather than shipped from the
build, because a history is a comparison against the *previous* release, and
only a checkout of the repository has that to compare against. Ordering is
`wppm.diff`'s job; this decides what to hand it.
No _History.md companion is written. A history compares a release against the
one immediately before it, and that chain carries no meaning: flavors are not
stable across cycles -- one may appear while another goes away -- and people
upgrade about once a year rather than every cycle, so a diff against the
predecessor answers a question almost nobody asks. What is useful is comparing
two package indexes of the reader's own choosing, which `wppm -diff` does
against any two of the files this script files.

"""
import re
Expand All @@ -26,7 +29,6 @@
# which installs none -- nothing at all.
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))

from wppm import diff # noqa: E402 the path above has to be set first
from wppm.diff import version # noqa: E402 packaging, or pip's vendored copy

# WinPythonslim-64bit-3.15.0.5b1.md -- the flavor may be empty, and the version
Expand All @@ -42,7 +44,7 @@ def parse_changelog_name(name: str):
try:
version.parse(match.group("version"))
except version.InvalidVersion:
return None # the _History companions land here, as they should
return None # anything whose tail is not a version is not an index
return match.group("flavor"), int(match.group("arch")), match.group("version")


Expand Down Expand Up @@ -78,22 +80,8 @@ def main(argv: list[str]) -> None:
raise SystemExit(f"{source} held no changelog, lock file or requirements")
for name in filed:
print(f"filed {name}")

# every package index has to be in place before any history is written: a
# history reads the index of the release it compares against, which for the
# second flavor of a cycle may well be the one just copied
histories = 0
for name in filed:
parsed = parse_changelog_name(name)
if not parsed:
continue
flavor, architecture, ver = parsed
previous = diff.find_previous_version(ver, changelogs, flavor, architecture)
diff.write_changelog(ver, None, changelogs, flavor, architecture)
histories += 1
against = "nothing earlier" if previous == ver else previous
print(f"history WinPython{flavor}-{architecture}bit-{ver} vs {against}")
print(f"\n{len(filed)} file(s) filed, {histories} history file(s) written")
indexes = sum(1 for name in filed if parse_changelog_name(name))
print(f"\n{len(filed)} file(s) filed, {indexes} package index(es)")


if __name__ == "__main__":
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/build_winpython_cycle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ jobs:
merge-multiple: true
path: release_metadata

- name: File the changelogs and write the histories
- name: File the changelogs
run: python .github/scripts/changelog_files.py release_metadata changelogs

- name: Open the changelog pull request
Expand Down Expand Up @@ -271,5 +271,5 @@ jobs:
else
gh pr create --base "$BASE" --head "$branch" \
--title "Changelogs for $TITLE" \
--body "Package indexes, lock files and requirements for every leg of \`$TAG\`, with the \`_History.md\` companions written against the previous release. Filed by the build that produced them, rather than copied by hand."
--body "Package indexes, lock files and requirements for every leg of \`$TAG\`, filed by the build that produced them rather than copied by hand. Compare any two of them with \`wppm -diff <a>.md <b>.md\`."
fi
25 changes: 12 additions & 13 deletions tests/test_changelog_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
and has never been kept there. Getting that wrong is quiet: the wrong file is
committed and nothing complains, so the selection is pinned here.

The `_History.md` companions are written from the checkout rather than shipped
by the build, because a history compares against the *previous* release, which
only the repository has.
No `_History.md` companion is written. A history compares a release against
the one immediately before it, and that chain carries no meaning: flavors come
and go between cycles, and readers upgrade about once a year rather than every
cycle. The comparison that is wanted is between two indexes of the reader's own
choosing -- `wppm -diff a.md b.md` -- so the absence is pinned here too.
"""
import importlib.util
import os
Expand Down Expand Up @@ -146,28 +148,25 @@ def test_wppm_comes_from_the_checkout(self, cycle, tmp_path):
assert proc.returncode == 0, proc.stderr
assert "decoy" not in proc.stderr

def test_files_the_three_and_writes_the_history(self, cycle, tmp_path):
def test_files_the_three_and_nothing_else(self, cycle, tmp_path):
source, changelogs = cycle
proc = run_script(source, changelogs, cwd=tmp_path)
assert proc.returncode == 0, proc.stderr
landed = sorted(p.name for p in changelogs.iterdir())
assert landed == [
"WinPythonslim-64bit-3.15.0.4.md", # was already there
"WinPythonslim-64bit-3.15.0.5b1.md", # filed
"WinPythonslim-64bit-3.15.0.5b1_History.md", # written
"WinPythonslim-64bit-3.15.0.4.md", # was already there
"WinPythonslim-64bit-3.15.0.5b1.md", # filed
"pylock.64-3_15_0_5slimb1.toml",
"requir.64-3_15_0_5slimb1.txt",
]

def test_the_history_names_the_release_it_compares_against(self, cycle, tmp_path):
def test_no_history_is_written_against_the_previous_release(self, cycle, tmp_path):
"""Deliberate: see the module docstring. Not an oversight to restore."""
source, changelogs = cycle
proc = run_script(source, changelogs, cwd=tmp_path)
assert proc.returncode == 0, proc.stderr
history = (changelogs / "WinPythonslim-64bit-3.15.0.5b1_History.md").read_text(
encoding="utf-8"
)
assert "since version 3.15.0.4slim" in history
assert "3.15.0.5b1slim" in history
assert not list(changelogs.glob("*_History.md"))
assert "history" not in proc.stdout.lower()

def test_an_empty_metadata_directory_is_an_error(self, tmp_path):
"""Silence here would commit nothing and call it a success."""
Expand Down
14 changes: 11 additions & 3 deletions winpython/build_winpython.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,15 +328,23 @@ def main():
)]
run_command(cmd, shell=True)
shutil.copyfile (winpydirbase.parent / mdn, changelog_dir / mdn)


# What moved since the previous build of this flavor -- for the person
# running the build, who wants to see it before publishing anything. It
# stays beside the build output and is NOT copied into changelogs/: the
# archive keeps the package indexes, and a diff against the immediately
# previous release is not a useful thing to publish, since flavors come and
# go between cycles and readers upgrade about once a year. Anyone wanting a
# comparison picks their own two indexes with `wppm -diff <a>.md <b>.md`.
# The previous index is still read from changelog_dir, which holds them all.
cmd = [str(target_python), "-X", "utf8", "-c",
(
"from wppm import diff;"
f"result = diff.compare_package_indexes('{changelog_version}', searchdir=r'{changelog_dir}', flavor=r'{args.flavor}', architecture={args.arch});"
f"open(r'{winpydirbase.parent / out}', 'w', encoding='utf-8').write(result)"
f"open(r'{winpydirbase.parent / out}', 'w', encoding='utf-8').write(result)"
)]
run_command(cmd, check=False)
shutil.copyfile (winpydirbase.parent / out, changelog_dir / out)
logging.info(f"changes since the previous {args.flavor or 'plain'} build: {winpydirbase.parent / out}")

if args.create_installer != "":
log_section("🙏 Step 8: Create Installer")
Expand Down
43 changes: 31 additions & 12 deletions wppm/diff.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,17 +94,31 @@ def compare_markdown_section_pairs(md1, md2, header_pairs, label1="Input1", labe
text += f"\n## {label1} [{h1}] vs {label2} [{h2}]\n\n{diff}\n"
return text

def index_title(file1, file2):
"""## Changes from <first index> to <second index>.

Comparing two files says nothing about which two, and the direction
matters: every upgrade line is written "old -> new", so a reader who has
the pair the wrong way round sees every upgrade as a downgrade. The version
mode gets this from the versions it was handed; the file mode has only the
names, so it uses them, without directory or extension.
"""
name = lambda f: Path(f).stem
return f"## Changes from {name(file1)} to {name(file2)}\n"


def compare_files(file1, file2, mode="full", header1=None, header2=None, header_pairs=None):
with open(file1, encoding=utils.guess_encoding(file1)[0]) as f1, \
open(file2, encoding=utils.guess_encoding(file2)[0]) as f2:
md1, md2 = f1.read(), f2.read()
if mode == "full":
result = ""
result = index_title(file1, file2) + "\n"
for k in PackageIndex.HEADERS:
result += compare_markdown_sections(md1, md2, k, k, file1, file2) + "\n"
return result
elif mode == "section":
return compare_markdown_sections(md1, md2, header1, header2, file1, file2)
return (index_title(file1, file2) + "\n"
+ compare_markdown_sections(md1, md2, header1, header2, file1, file2))
elif mode == "pairs":
return compare_markdown_section_pairs(md1, md2, header_pairs, file1, file2)
else:
Expand Down Expand Up @@ -158,7 +172,6 @@ def compare_package_indexes(version2, version1=None, searchdir=None, flavor="",
flavor1 = flavor1 or flavor
md1 = load_version_markdown(version1, searchdir, flavor1, architecture)
md2 = load_version_markdown(version2, searchdir, flavor, architecture)
result = f"# WinPython {architecture}bit {version2}{flavor} vs {version1}{flavor1}\n"
result = (
f"## History of changes for WinPython-{architecture}bit {version2 + flavor}\r\n\r\n"
f"The following changes were made to WinPython-{architecture}bit distribution since version {version1 + flavor1}.\n\n\n"
Expand Down Expand Up @@ -193,19 +206,25 @@ def write_changelog(version2, version1=None, searchdir=None, flavor="", architec
shutil.copyfile(output_file, Path(basedir) / output_file.name)

def print_usage():
# "python diff.py" cannot work -- the relative imports need the package --
# so every line says how it is really run
print("Usage:")
print(" python diff.py file1.md file2.md")
print(" - Compare all sections of two markdown files.")
print(" python diff.py file1.md file2.md --section header1 header2")
print(" wppm -diff file1.md file2.md (or: python -m wppm.diff file1.md file2.md)")
print(" - Compare all sections of two package indexes, in any order or flavor.")
print(" python -m wppm.diff file1.md file2.md --section header1 header2")
print(" - Compare section 'header1' of file1 with section 'header2' of file2.")
print(" python diff.py file1.md file2.md --pairs header1a header2a [header1b header2b ...]")
print(" - Compare pairs of sections. Example: python diff.py f1.md f2.md --pairs python wheelhouse tools tools")
print(" python diff.py <version2> <version1> [searchdir] [flavor] [architecture]")
print(" - Compare WinPython markdown changelogs by version (historical mode).")
print(" python diff.py --write-changelog <version2> <version1> [searchdir] [flavor] [architecture] [basedir]")
print(" - Write changelog between version1 and version2 to file (and optionally copy to basedir).")
print(" python -m wppm.diff file1.md file2.md --pairs header1a header2a [header1b header2b ...]")
print(" - Compare pairs of sections. Example: --pairs python wheelhouse tools tools")
print(" python -m wppm.diff <version2> <version1> [searchdir] [flavor] [architecture]")
print(" - Compare WinPython package indexes by version, from a changelogs directory.")
print(" python -m wppm.diff --write-changelog <version2> <version1> [searchdir] [flavor] [architecture] [basedir]")
print(" - Write that comparison to a file (and optionally copy it to basedir).")

if __name__ == "__main__":
# every upgraded package prints an arrow, and a Windows console is cp1252,
# so the default encoding turned "compare two releases" into a traceback
if hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
args = sys.argv
if len(args) >= 3 and all(arg.lower().endswith('.md') for arg in args[1:3]):
file1, file2 = args[1], args[2]
Expand Down
13 changes: 13 additions & 0 deletions wppm/wppm.py
Original file line number Diff line number Diff line change
Expand Up @@ -345,6 +345,8 @@ def main(test=False):
parser.add_argument("-ls", "--list", action="store_true", help="list installed packages matching [optional] expression: wppm -ls, wppm -ls pand")
parser.add_argument("-lsa", dest="all", action="store_true",help=f"list details of packages matching [optional] expression: wppm -lsa pandas -l1")
parser.add_argument("-md", dest="markdown", action="store_true",help=f"markdown summary of the installation")
parser.add_argument("-diff", dest="diff", nargs=2, metavar=("INDEX1", "INDEX2"), default=None,
help="compare two package indexes, whichever two you choose: wppm -diff WinPythonslim-64bit-3.13.5.0.md WinPythonslim-64bit-3.14.7.0.md\nany two releases or flavors, in either direction -- they need not be consecutive")
parser.add_argument("-p",dest="pipdown",action="store_true",help="show Package (!= missing) dependencies of the given package[option], [.]=all: wppm -p pandas[.]")
parser.add_argument("-r", dest="pipup", action="store_true", help=f"show Reverse (!= constraining) dependancies of the given package[option]: wppm -r pytest![test]")
parser.add_argument("-tl", "--top-level", action="store_true", help="keep only the entries no other entry pulls in, sorted: wppm -tl, wppm requirements.txt -tl -v\nwith -p or -r, start the tree from them instead of every package: wppm -tl -p")
Expand All @@ -355,6 +357,17 @@ def main(test=False):
parser.add_argument("-u", "--uninstall", action="store_true", help="uninstall package (use pip for more features)")

args = parser.parse_args()
if args.diff:
# no distribution involved: this reads two files. The indexes are in
# the changelogs/ directory of the winpython repository, one per
# release and flavor, and any two of them can be compared.
# utf-8 here and not just errors="replace": an upgrade is written
# "1.2.3 → 1.2.4", and replacing that arrow with "?" is most of the
# output on a page whose whole subject is upgrades.
if sys.stdout and hasattr(sys.stdout, "reconfigure"):
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
print(diff.compare_files(*args.diff))
sys.exit()
targetpython = None
if args.target and args.target != sys.prefix:
targetpython = args.target if args.target.lower().endswith('.exe') else str(Path(args.target) / 'python.exe')
Expand Down
Loading