Visitar URL original
[WIP] Various build system improvements by indygreg · Pull Request #101093 · python/cpython · GitHub
Skip to content

[WIP] Various build system improvements #101093

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
indygreg wants to merge 6 commits into
base: main
Choose a base branch
from
Draft

[WIP] Various build system improvements #101093

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8eb0b50
gh-104523: inline minimal PGO rules
indygreg Jan 16, 2023
f215844
gh-xxxx: overhaul build rules for optimized binaries
indygreg Jan 16, 2023
c8759fd
gh-xxxx: move BOLT arguments to configure
indygreg Jan 16, 2023
02418fd
gh-xxxx: run BOLT instrumented binary with `$(RUNSHARED)`
indygreg Jan 16, 2023
a212332
gh-xxxx: enable BOLT optimization of libpython
indygreg Jan 16, 2023
9e50ee0
gh-xxxx: expose _PYTHON_HOST_PLATFORM as a settable variable
indygreg Jan 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
gh-xxxx: enable BOLT optimization of libpython 
Before, we only supported running BOLT on the main `python` binary.
If a shared library was in play, it wouldn't be optimized. That was
leaving a ton of optimization opportunities on the floor.

This commit adds support for running BOLT on libpython.

Functionality is disabled by default because BOLT asserts on LLVM 15,
which is the latest LLVM. I've built LLVM tip and it is able to
process libpython just fine. So it is known to work.
  • Loading branch information
@indygreg
indygreg committed May 16, 2023
commit a212332e85c4e6454460301dd3d84108dfd9f7d0
30 changes: 18 additions & 12 deletions Makefile.pre.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -722,7 +722,7 @@ profile-pgo-apply-stamp: profile-pgo-analyze-stamp
# passes.

# List of binaries that BOLT runs on.
BOLT_BINARIES = $(BUILDPYTHON)
BOLT_BINARIES = @BOLT_BINARIES@

BOLT_INSTRUMENT_FLAGS ?= @BOLT_INSTRUMENT_FLAGS@
BOLT_APPLY_FLAGS ?= @BOLT_APPLY_FLAGS@
Expand All @@ -734,8 +734,6 @@ clean-bolt:
find . -name '*.bolt_inst' -exec rm -f {} ';'
# The data files they produce.
find . -name '*.fdata' -exec rm -f {} ';'
# Copied of binaries before BOLT application.
find . -name '*.prebolt' -exec rm -f {} ';'

# BOLTs dependencies are a bit wonky.
#
Expand All @@ -753,28 +751,36 @@ profile-bolt-prebuild-stamp: @MAKE_BOLT_NATIVE_DEPENDENCY@

profile-bolt-instrument-stamp: profile-bolt-prebuild-stamp
for bin in $(BOLT_BINARIES); do \
if [ -e "$${bin}.prebolt" ]; then \
echo "Restoring pre-BOLT binary $${bin}.prebolt"; \
prebolt="$${bin}.prebolt"; \
if [ -e "$${prebolt}" ]; then \
echo "Restoring pre-BOLT binary $${prebolt}"; \
mv "$${bin}.prebolt" "$${bin}"; \
fi \
fi; \
cp "$${bin}" "$${prebolt}"; \
done
# Ensure prior BOLT state is purged.
$(MAKE) clean-bolt
@LLVM_BOLT@ ./$(BUILDPYTHON) -instrument -instrumentation-file-append-pid -instrumentation-file=$(abspath $(BUILDPYTHON).bolt) -o $(BUILDPYTHON).bolt_inst $(BOLT_INSTRUMENT_FLAGS)
for bin in $(BOLT_BINARIES); do \
@LLVM_BOLT@ $${bin} -instrument -instrumentation-file-append-pid -instrumentation-file=$(abspath $${bin}.bolt) -o $${bin}.bolt_inst $(BOLT_INSTRUMENT_FLAGS); \
mv "$${bin}.bolt_inst" "$${bin}"; \
done
touch $@

profile-bolt-run-stamp: profile-bolt-instrument-stamp
$(RUNSHARED) ./$(BUILDPYTHON).bolt_inst $(PROFILE_TASK) || true
$(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true
touch $@

profile-bolt-analyze-stamp: profile-bolt-run-stamp
@MERGE_FDATA@ $(BUILDPYTHON).*.fdata > $(BUILDPYTHON).fdata
for bin in $(BOLT_BINARIES); do \
@MERGE_FDATA@ $${bin}.*.fdata > $${bin}.fdata; \
done
touch $@

profile-bolt-apply-stamp: profile-bolt-analyze-stamp
@LLVM_BOLT@ ./$(BUILDPYTHON) -o $(BUILDPYTHON).bolt -data=$(BUILDPYTHON).fdata $(BOLT_APPLY_FLAGS)
mv $(BUILDPYTHON) $(BUILDPYTHON).prebolt
mv $(BUILDPYTHON).bolt $(BUILDPYTHON)
for bin in $(BOLT_BINARIES); do \
@LLVM_BOLT@ "$${bin}.prebolt" -o "$${bin}.bolt" -data="$${bin}.fdata" $(BOLT_APPLY_FLAGS); \
mv "$${bin}.bolt" "$${bin}"; \
done
touch $@

# End of profile-based optimization rules.
Expand Down
27 changes: 27 additions & 0 deletions configure
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 17 additions & 0 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2083,6 +2083,23 @@ if test "$Py_BOLT" = 'true' ; then
fi
fi

# Enable BOLT optimizations of libpython. Optional for now due to known
# crashes on LLVM 15. Seems to be fixed in LLVM 16.
AC_SUBST(BOLT_BINARIES)
BOLT_BINARIES='$(BUILDPYTHON)'

AC_MSG_CHECKING(for --with-bolt-libpython)
AC_ARG_WITH(bolt_libpython,
AS_HELP_STRING([--with-bolt-libpython], [enable BOLT optimization of libpython (WARNING: known to crash BOLT)]),
[with_bolt_libpython="yes"],
[with_bolt_libpython="no"])
AC_MSG_RESULT($with_bolt_libpython)

if test "${enable_shared}" = "yes" -a "${with_bolt_libpython}" = "yes"
then
BOLT_BINARIES="${BOLT_BINARIES} \$(INSTSONAME)"
fi

AC_ARG_VAR(BOLT_INSTRUMENT_FLAGS, Arguments to llvm-bolt when instrumenting binaries)
AC_MSG_CHECKING(BOLT_INSTRUMENT_FLAGS)
if test -z "${BOLT_INSTRUMENT_FLAGS}"
Expand Down