bpo-40280: Add Tools/wasm with helpers for cross building (GH-29984)

Makefile.pre.in

@@ -830,6 +830,22 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
 	else true; \
 	fi
 
+# wasm32-emscripten build
+# wasm assets directory is relative to current build dir, e.g. "./usr/local".
+# --preload-file turns a relative asset path into an absolute path.
+WASM_ASSETS_DIR=".$(prefix)"
+WASM_STDLIB="$(WASM_ASSETS_DIR)/local/lib/python$(VERSION)/os.py"
+
+$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
+ pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py
+	$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
+ --builddir . --prefix $(prefix)
+
+python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB)
+	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \
+ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \
+ -s ASSERTIONS=1 --preload-file $(WASM_ASSETS_DIR)
+
 ##########################################################################
 # Build static libmpdec.a
 LIBMPDEC_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBMPDEC_CFLAGS@
@@ -938,6 +954,7 @@ Makefile Modules/config.c: Makefile.pre \
 	$(SHELL) $(MAKESETUP) -c $(srcdir)/Modules/config.c.in \
  -s Modules \
  Modules/Setup.local \
+ @MODULES_SETUP_STDLIB@ \
  $(srcdir)/Modules/Setup.bootstrap \
  $(srcdir)/Modules/Setup
 	@mv config.c Modules
@@ -2379,6 +2396,7 @@ clean-retain-profile: pycremoval
 	-rm -f pybuilddir.txt
 	-rm -f Lib/lib2to3/*Grammar*.pickle
 	-rm -f _bootstrap_python
+	-rm -f python.html python.js python.data
 	-rm -f Programs/_testembed Programs/_freeze_module
 	-rm -f Python/deepfreeze/*.[co]
 	-rm -f Python/frozen_modules/*.h

Misc/NEWS.d/next/Build/2021-12-13-21-03-52.bpo-40280.b7NG4Y.rst

@@ -0,0 +1 @@
+A new directory ``Tools/wasm`` contains WebAssembly-related helpers like ``config.site`` override for wasm32-emscripten, wasm assets generator to bundle the stdlib, and a README.

Tools/wasm/README.md

@@ -0,0 +1,55 @@
+# Python WebAssembly (WASM) build
+
+This directory contains configuration and helpers to facilitate cross
+compilation of CPython to WebAssembly (WASM).
+
+## wasm32-emscripten build
+
+Cross compiling to wasm32-emscripten platform needs the [Emscripten](https://emscripten.org/)
+tool chain and a build Python interpreter.
+All commands below are relative to a repository checkout.
+
+### Compile a build Python interpreter
+
+```shell
+mkdir -p builddir/build
+pushd builddir/build
+../../configure -C
+make -j$(nproc)
+popd
+```
+
+### Fetch and build additional emscripten ports
+
+```shell
+embuilder build zlib
+```
+
+### Cross compile to wasm32-emscripten
+
+```shell
+mkdir -p builddir/emscripten
+pushd builddir/emscripten
+
+CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
+ emconfigure ../../configure -C \
+ --host=wasm32-unknown-emscripten \
+ --build=$(../../config.guess) \
+ --with-build-python=$(pwd)/../build/python
+
+emmake make -j$(nproc) python.html
+```
+
+### Test in browser
+
+Serve `python.html` with a local webserver and open the file in a browser.
+
+```shell
+emrun python.html
+```
+
+or
+
+```shell
+python3 -m http.server
+```

Tools/wasm/config.site-wasm32-emscripten

@@ -0,0 +1,70 @@
+# config.site override for cross compiling to wasm32-emscripten platform
+#
+# CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \
+# emconfigure ./configure --host=wasm32-unknown-emscripten --build=...
+#
+# Written by Christian Heimes <christian@python.org>
+# Partly based on pyodide's pyconfig.undefs.h file.
+#
+
+# cannot be detected in cross builds
+ac_cv_buggy_getaddrinfo=no
+
+# Emscripten has no /dev/pt*
+ac_cv_file__dev_ptmx=no
+ac_cv_file__dev_ptc=no
+
+# dummy readelf, Emscripten build does not need readelf.
+ac_cv_prog_ac_ct_READELF=true
+
+# new undefined symbols / unsupported features
+ac_cv_func_posix_spawn=no
+ac_cv_func_posix_spawnp=no
+ac_cv_func_eventfd=no
+ac_cv_func_memfd_create=no
+ac_cv_func_prlimit=no
+
+# unsupported syscall, https://github.com/emscripten-core/emscripten/issues/13393
+ac_cv_func_shutdown=no
+
+# breaks build, see https://github.com/ethanhs/python-wasm/issues/16
+ac_cv_lib_bz2_BZ2_bzCompress=no
+
+# The rest is based on pyodide
+# https://github.com/pyodide/pyodide/blob/main/cpython/pyconfig.undefs.h
+
+ac_cv_func_epoll=no
+ac_cv_func_epoll_create1=no
+ac_cv_header_linux_vm_sockets_h=no
+ac_cv_func_socketpair=no
+ac_cv_func_utimensat=no
+ac_cv_func_sigaction=no
+
+# Untested syscalls in emscripten
+ac_cv_func_openat=no
+ac_cv_func_mkdirat=no
+ac_cv_func_fchownat=no
+ac_cv_func_renameat=no
+ac_cv_func_linkat=no
+ac_cv_func_symlinkat=no
+ac_cv_func_readlinkat=no
+ac_cv_func_fchmodat=no
+ac_cv_func_dup3=no
+
+# Syscalls not implemented in emscripten
+ac_cv_func_preadv2=no
+ac_cv_func_preadv=no
+ac_cv_func_pwritev2=no
+ac_cv_func_pwritev=no
+ac_cv_func_pipe2=no
+ac_cv_func_nice=no
+
+# Syscalls that resulted in a segfault
+ac_cv_func_utimensat=no
+ac_cv_header_sys_ioctl_h=no
+
+# sockets are supported, but only in non-blocking mode
+# ac_cv_header_sys_socket_h=no
+
+# Unsupported functionality
+#undef HAVE_PTHREAD_H

Tools/wasm/wasm_assets.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python
+"""Create a WASM asset bundle directory structure.
+
+The WASM asset bundles are pre-loaded by the final WASM build. The bundle
+contains:
+
+- a stripped down, pyc-only stdlib zip file, e.g. {PREFIX}/lib/python311.zip
+- os.py as marker module {PREFIX}/lib/python3.11/os.py
+- empty lib-dynload directory, to make sure it is copied into the bundle {PREFIX}/lib/python3.11/lib-dynload/.empty
+"""
+
+import argparse
+import pathlib
+import shutil
+import sys
+import zipfile
+
+# source directory
+SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
+SRCDIR_LIB = SRCDIR / "Lib"
+
+# sysconfig data relative to build dir.
+SYSCONFIGDATA_GLOB = "build/lib.*/_sysconfigdata_*.py"
+
+# Library directory relative to $(prefix).
+WASM_LIB = pathlib.PurePath("lib")
+WASM_STDLIB_ZIP = (
+ WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip"
+)
+WASM_STDLIB = (
+ WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}"
+)
+WASM_DYNLOAD = WASM_STDLIB / "lib-dynload"
+
+
+# Don't ship large files / packages that are not particularly useful at
+# the moment.
+OMIT_FILES = (
+ # regression tests
+ "test/",
+ # user interfaces: TK, curses
+ "curses/",
+ "idlelib/",
+ "tkinter/",
+ "turtle.py",
+ "turtledemo/",
+ # package management
+ "ensurepip/",
+ "venv/",
+ # build system
+ "distutils/",
+ "lib2to3/",
+ # concurrency
+ "concurrent/",
+ "multiprocessing/",
+ # deprecated
+ "asyncore.py",
+ "asynchat.py",
+ # Synchronous network I/O and protocols are not supported; for example,
+ # socket.create_connection() raises an exception:
+ # "BlockingIOError: [Errno 26] Operation in progress".
+ "cgi.py",
+ "cgitb.py",
+ "email/",
+ "ftplib.py",
+ "http/",
+ "imaplib.py",
+ "nntplib.py",
+ "poplib.py",
+ "smtpd.py",
+ "smtplib.py",
+ "socketserver.py",
+ "telnetlib.py",
+ "urllib/",
+ "wsgiref/",
+ "xmlrpc/",
+ # dbm / gdbm
+ "dbm/",
+ # other platforms
+ "_aix_support.py",
+ "_bootsubprocess.py",
+ "_osx_support.py",
+ # webbrowser
+ "antigravity.py",
+ "webbrowser.py",
+ # ctypes
+ "ctypes/",
+ # Pure Python implementations of C extensions
+ "_pydecimal.py",
+ "_pyio.py",
+ # Misc unused or large files
+ "pydoc_data/",
+ "msilib/",
+)
+
+# regression test sub directories
+OMIT_SUBDIRS = (
+ "ctypes/test/",
+ "tkinter/test/",
+ "unittest/test/",
+)
+
+
+OMIT_ABSOLUTE = {SRCDIR_LIB / name for name in OMIT_FILES}
+OMIT_SUBDIRS_ABSOLUTE = tuple(str(SRCDIR_LIB / name) for name in OMIT_SUBDIRS)
+
+
+def filterfunc(name: str) -> bool:
+ return not name.startswith(OMIT_SUBDIRS_ABSOLUTE)
+
+
+def create_stdlib_zip(
+ args: argparse.Namespace, compression: int = zipfile.ZIP_DEFLATED, *, optimize: int = 0
+) -> None:
+ sysconfig_data = list(args.builddir.glob(SYSCONFIGDATA_GLOB))
+ if not sysconfig_data:
+ raise ValueError("No sysconfigdata file found")
+
+ with zipfile.PyZipFile(
+ args.wasm_stdlib_zip, mode="w", compression=compression, optimize=0
+ ) as pzf:
+ for entry in sorted(args.srcdir_lib.iterdir()):
+ if entry.name == "__pycache__":
+ continue
+ if entry in OMIT_ABSOLUTE:
+ continue
+ if entry.name.endswith(".py") or entry.is_dir():
+ # writepy() writes .pyc files (bytecode).
+ pzf.writepy(entry, filterfunc=filterfunc)
+ for entry in sysconfig_data:
+ pzf.writepy(entry)
+
+
+def path(val: str) -> pathlib.Path:
+ return pathlib.Path(val).absolute()
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+ "--builddir",
+ help="absolute build directory",
+ default=pathlib.Path(".").absolute(),
+ type=path,
+)
+parser.add_argument(
+ "--prefix", help="install prefix", default=pathlib.Path("/usr/local"), type=path
+)
+
+
+def main():
+ args = parser.parse_args()
+
+ relative_prefix = args.prefix.relative_to(pathlib.Path("/"))
+ args.srcdir = SRCDIR
+ args.srcdir_lib = SRCDIR_LIB
+ args.wasm_root = args.builddir / relative_prefix
+ args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP
+ args.wasm_stdlib = args.wasm_root / WASM_STDLIB
+ args.wasm_dynload = args.wasm_root / WASM_DYNLOAD
+
+ # Empty, unused directory for dynamic libs, but required for site initialization.
+ args.wasm_dynload.mkdir(parents=True, exist_ok=True)
+ marker = args.wasm_dynload / ".empty"
+ marker.touch()
+ # os.py is a marker for finding the correct lib directory.
+ shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib)
+ # The rest of stdlib that's useful in a WASM context.
+ create_stdlib_zip(args)
+ size = round(args.wasm_stdlib_zip.stat().st_size / 1024 ** 2, 2)
+ parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n")
+
+
+if __name__ == "__main__":
+ main()