build_libtcod.py

#!/usr/bin/env python3
"""Parse and compile libtcod and SDL sources for CFFI."""
from __future__ import annotations

import contextlib
import glob
import os
import platform
import re
import sys
from pathlib import Path
from typing import Any, Iterable, Iterator

from cffi import FFI

sys.path.append(str(Path(__file__).parent)) # Allow importing local modules.

import build_sdl # noqa: E402

Py_LIMITED_API = 0x03060000

HEADER_PARSE_PATHS = ("tcod/", "libtcod/src/libtcod/")
HEADER_PARSE_EXCLUDES = ("gl2_ext_.h", "renderer_gl_internal.h", "event.h")

BIT_SIZE, LINKAGE = platform.architecture()

# Regular expressions to parse the headers for cffi.
RE_COMMENT = re.compile(r"\s*/\*.*?\*/|\s*//*?$", re.DOTALL | re.MULTILINE)
RE_CPLUSPLUS = re.compile(r"#ifdef __cplusplus.*?#endif.*?$", re.DOTALL | re.MULTILINE)
RE_PREPROCESSOR = re.compile(r"(?!#define\s+\w+\s+\d+$)#.*?(?<!\\)$", re.DOTALL | re.MULTILINE)
RE_INCLUDE = re.compile(r'#include "([^"]*)"')
RE_TAGS = re.compile(
 r"TCODLIB_C?API|TCOD_PUBLIC|TCOD_NODISCARD|TCOD_DEPRECATED_NOMESSAGE|TCOD_DEPRECATED_ENUM"
 r"|(TCOD_DEPRECATED\(\".*?\"\))"
 r"|(TCOD_DEPRECATED|TCODLIB_FORMAT)\([^)]*\)|__restrict"
)
RE_VAFUNC = re.compile(r"^[^;]*\([^;]*va_list.*\);", re.MULTILINE)
RE_INLINE = re.compile(r"(^.*?inline.*?\(.*?\))\s*\{.*?\}$", re.DOTALL | re.MULTILINE)


class ParsedHeader:
 """Header manager class for parsing headers.

 Holds parsed sources and keeps information needed to resolve header order.
 """

 # Class dictionary of all parsed headers.
 all_headers: dict[Path, ParsedHeader] = {}

 def __init__(self, path: Path) -> None:
 """Initialize and organize a header file."""
 self.path = path = path.resolve(True)
 directory = path.parent
 depends = set()
 header = self.path.read_text(encoding="utf-8")
 header = RE_COMMENT.sub("", header)
 header = RE_CPLUSPLUS.sub("", header)
 for dependency in RE_INCLUDE.findall(header):
 depends.add((directory / str(dependency)).resolve(True))
 header = RE_PREPROCESSOR.sub("", header)
 header = RE_TAGS.sub("", header)
 header = RE_VAFUNC.sub("", header)
 header = RE_INLINE.sub(r"\1;", header)
 self.header = header.strip()
 self.depends = frozenset(depends)
 self.all_headers[self.path] = self

 def parsed_depends(self) -> Iterator[ParsedHeader]:
 """Return dependencies excluding ones that were not loaded."""
 for dep in self.depends:
 with contextlib.suppress(KeyError):
 yield self.all_headers[dep]

 def __str__(self) -> str:
 """Return useful info on this object."""
 return "Parsed harder at '{}'\n Depends on: {}".format(
 self.path,
 "\n\t".join(str(d) for d in self.depends),
 )

 def __repr__(self) -> str:
 """Return the representation of this object."""
 return f"ParsedHeader({self.path!r})"


def walk_includes(directory: str) -> Iterator[ParsedHeader]:
 """Parse all the include files in a directory and subdirectories."""
 for path, _dirs, files in os.walk(directory):
 for file in files:
 if file in HEADER_PARSE_EXCLUDES:
 continue
 if file.endswith(".h"):
 yield ParsedHeader(Path(path, file).resolve(True))


def resolve_dependencies(
 includes: Iterable[ParsedHeader],
) -> list[ParsedHeader]:
 """Sort headers by their correct include order."""
 unresolved = set(includes)
 resolved: set[ParsedHeader] = set()
 result = []
 while unresolved:
 for item in unresolved:
 if frozenset(item.parsed_depends()).issubset(resolved):
 resolved.add(item)
 result.append(item)
 if not unresolved & resolved:
 msg = (
 "Could not resolve header load order."
 "\nPossible cyclic dependency with the unresolved headers:"
 f"\n{unresolved}"
 )
 raise RuntimeError(msg)
 unresolved -= resolved
 return result


def parse_includes() -> list[ParsedHeader]:
 """Collect all parsed header files and return them.

 Reads HEADER_PARSE_PATHS and HEADER_PARSE_EXCLUDES.
 """
 includes: list[ParsedHeader] = []
 for dirpath in HEADER_PARSE_PATHS:
 includes.extend(walk_includes(dirpath))
 return resolve_dependencies(includes)


def walk_sources(directory: str) -> Iterator[str]:
 """Iterate over the C sources of a directory recursively."""
 for path, _dirs, files in os.walk(directory):
 for source in files:
 if source.endswith(".c"):
 yield str(Path(path, source))


includes = parse_includes()

module_name = "tcod._libtcod"
include_dirs: list[str] = [
 ".",
 "libtcod/src/vendor/",
 "libtcod/src/vendor/utf8proc",
 "libtcod/src/vendor/zlib/",
 *build_sdl.include_dirs,
]

extra_compile_args: list[str] = [*build_sdl.extra_compile_args]
extra_link_args: list[str] = [*build_sdl.extra_link_args]
sources: list[str] = []

libraries: list[str] = [*build_sdl.libraries]
library_dirs: list[str] = [*build_sdl.library_dirs]
define_macros: list[tuple[str, Any]] = [("Py_LIMITED_API", Py_LIMITED_API)]

sources += walk_sources("tcod/")
sources += walk_sources("libtcod/src/libtcod/")
sources += ["libtcod/src/vendor/stb.c"]
sources += ["libtcod/src/vendor/lodepng.c"]
sources += ["libtcod/src/vendor/utf8proc/utf8proc.c"]
sources += glob.glob("libtcod/src/vendor/zlib/*.c")

if sys.platform == "win32":
 libraries += ["User32"]
 define_macros.append(("TCODLIB_API", ""))
 define_macros.append(("_CRT_SECURE_NO_WARNINGS", None))

if sys.platform in ["win32", "darwin"]:
 include_dirs.append("libtcod/src/zlib/")


if sys.platform == "darwin":
 # Fix "implicit declaration of function 'close'" in zlib.
 define_macros.append(("HAVE_UNISTD_H", 1))


tdl_build = os.environ.get("TDL_BUILD", "RELEASE").upper()

MSVC_CFLAGS = {"DEBUG": ["/Od"], "RELEASE": ["/GL", "/O2", "/GS-", "/wd4996"]}
MSVC_LDFLAGS: dict[str, list[str]] = {"DEBUG": [], "RELEASE": ["/LTCG"]}
GCC_CFLAGS = {
 "DEBUG": ["-std=c99", "-Og", "-g", "-fPIC"],
 "RELEASE": [
 "-std=c99",
 "-flto",
 "-O3",
 "-g",
 "-fPIC",
 "-Wno-deprecated-declarations",
 "-Wno-discarded-qualifiers", # Ignore discarded restrict qualifiers.
 ],
}

if sys.platform == "win32" and "--compiler=mingw32" not in sys.argv:
 extra_compile_args.extend(MSVC_CFLAGS[tdl_build])
 extra_link_args.extend(MSVC_LDFLAGS[tdl_build])
else:
 extra_compile_args.extend(GCC_CFLAGS[tdl_build])
 extra_link_args.extend(GCC_CFLAGS[tdl_build])

ffi = FFI()
ffi.cdef(build_sdl.get_cdef())
for include in includes:
 try:
 ffi.cdef(include.header)
 except Exception:
 # Print the source, for debugging.
 print(f"Error with: {include.path}")
 for i, line in enumerate(include.header.split("\n"), 1):
 print("%03i %s" % (i, line))
 raise
ffi.cdef(
 """
#define TCOD_COMPILEDVERSION ...
"""
)
ffi.set_source(
 module_name,
 "#include <tcod/cffi.h>\n#include <SDL.h>",
 include_dirs=include_dirs,
 library_dirs=library_dirs,
 sources=sources,
 libraries=libraries,
 extra_compile_args=extra_compile_args,
 extra_link_args=extra_link_args,
 define_macros=define_macros,
 py_limited_api=True,
)

CONSTANT_MODULE_HEADER = '''"""Constants from the libtcod C API.

This module is auto-generated by `build_libtcod.py`.
"""
from tcod.color import Color

'''

EVENT_CONSTANT_MODULE_HEADER = '''"""Event constants from SDL's C API.

This module is auto-generated by `build_libtcod.py`.
"""
'''


def find_sdl_attrs(prefix: str) -> Iterator[tuple[str, int | str | Any]]:
 """Return names and values from `tcod.lib`.

 `prefix` is used to filter out which names to copy.
 """
 from tcod._libtcod import lib # type: ignore

 if prefix.startswith("SDL_"):
 name_starts_at = 4
 elif prefix.startswith("SDL"):
 name_starts_at = 3
 else:
 name_starts_at = 0
 for attr in dir(lib):
 if attr.startswith(prefix):
 yield attr[name_starts_at:], getattr(lib, attr)


def parse_sdl_attrs(prefix: str, all_names: list[str] | None) -> tuple[str, str]:
 """Return the name/value pairs, and the final dictionary string for the library attributes with `prefix`.

 Append matching names to the `all_names` list.
 """
 names = []
 lookup = []
 for name, value in sorted(find_sdl_attrs(prefix), key=lambda item: item[1]):
 if name == "KMOD_RESERVED":
 continue
 if all_names is not None:
 all_names.append(name)
 names.append(f"{name} = {value}")
 lookup.append(f'{value}: "{name}"')
 return "\n".join(names), "{{\n {},\n}}".format(",\n ".join(lookup))


EXCLUDE_CONSTANTS = [
 "TCOD_MAJOR_VERSION",
 "TCOD_MINOR_VERSION",
 "TCOD_PATCHLEVEL",
 "TCOD_COMPILEDVERSION",
 "TCOD_PATHFINDER_MAX_DIMENSIONS",
 "TCOD_KEY_TEXT_SIZE",
 "TCOD_NOISE_MAX_DIMENSIONS",
 "TCOD_NOISE_MAX_OCTAVES",
 "TCOD_FALLBACK_FONT_SIZE",
]

EXCLUDE_CONSTANT_PREFIXES = [
 "TCOD_E_",
 "TCOD_HEAP_",
 "TCOD_LEX_",
 "TCOD_CHARMAP_",
 "TCOD_LOG_",
]


def update_module_all(filename: Path, new_all: str) -> None:
 """Update the __all__ of a file with the constants from new_all."""
 RE_CONSTANTS_ALL = re.compile(
 r"(.*# --- From constants.py ---).*(# --- End constants.py ---.*)",
 re.DOTALL,
 )
 match = RE_CONSTANTS_ALL.match(filename.read_text(encoding="utf-8"))
 assert match, f"Can't determine __all__ subsection in {filename}!"
 header, footer = match.groups()
 filename.write_text(f"{header}\n {new_all},\n {footer}", encoding="utf-8")


def generate_enums(prefix: str) -> Iterator[str]:
 """Generate attribute assignments suitable for a Python enum."""
 for symbol, value in sorted(find_sdl_attrs(prefix), key=lambda item: item[1]):
 _, name = symbol.split("_", 1)
 if name.isdigit():
 name = f"N{name}"
 if name in "IOl": # Ignore ambiguous variable name warnings.
 yield f"{name} = {value} # noqa: E741"
 else:
 yield f"{name} = {value}"


def write_library_constants() -> None:
 """Write libtcod constants into the tcod.constants module."""
 import tcod.color
 from tcod._libtcod import ffi, lib

 with Path("tcod/constants.py").open("w", encoding="utf-8") as f:
 all_names = []
 f.write(CONSTANT_MODULE_HEADER)
 for name in dir(lib):
 # To exclude specific names use either EXCLUDE_CONSTANTS or
 # EXCLUDE_CONSTANT_PREFIXES before editing this.
 if name.endswith("_"):
 continue
 if name in EXCLUDE_CONSTANTS:
 continue
 if any(name.startswith(prefix) for prefix in EXCLUDE_CONSTANT_PREFIXES):
 continue
 value = getattr(lib, name)
 if name[:5] == "TCOD_":
 if name.isupper(): # const names
 f.write(f"{name[5:]} = {value!r}\n")
 all_names.append(name[5:])
 elif name.startswith("FOV"): # fov const names
 f.write(f"{name} = {value!r}\n")
 all_names.append(name)
 elif name[:6] == "TCODK_": # key name
 f.write(f"KEY_{name[6:]} = {value!r}\n")
 all_names.append(f"KEY_{name[6:]}")

 f.write("\n# --- colors ---\n")
 for name in dir(lib):
 if name[:5] != "TCOD_":
 continue
 value = getattr(lib, name)
 if not isinstance(value, ffi.CData):
 continue
 if ffi.typeof(value) != ffi.typeof("TCOD_color_t"):
 continue
 color = tcod.color.Color._new_from_cdata(value)
 f.write(f"{name[5:]} = {color!r}\n")

 all_names_merged = ",\n ".join(f'"{name}"' for name in all_names)
 f.write(f"\n__all__ = [\n {all_names_merged},\n]\n")
 update_module_all(Path("tcod/__init__.py"), all_names_merged)
 update_module_all(Path("tcod/libtcodpy.py"), all_names_merged)

 with Path("tcod/event_constants.py").open("w", encoding="utf-8") as f:
 all_names = []
 f.write(EVENT_CONSTANT_MODULE_HEADER)
 f.write("\n# --- SDL scancodes ---\n")
 f.write(f"""{parse_sdl_attrs("SDL_SCANCODE", None)[0]}\n""")

 f.write("\n# --- SDL keyboard symbols ---\n")
 f.write(f"""{parse_sdl_attrs("SDLK", None)[0]}\n""")

 f.write("\n# --- SDL keyboard modifiers ---\n")
 f.write("{}\n_REVERSE_MOD_TABLE = {}\n".format(*parse_sdl_attrs("KMOD", all_names)))

 f.write("\n# --- SDL wheel ---\n")
 f.write("{}\n_REVERSE_WHEEL_TABLE = {}\n".format(*parse_sdl_attrs("SDL_MOUSEWHEEL", all_names)))
 all_names_merged = ",\n ".join(f'"{name}"' for name in all_names)
 f.write(f"\n__all__ = [\n {all_names_merged},\n]\n")

 event_py = Path("tcod/event.py").read_text(encoding="utf-8")

 event_py = re.sub(
 r"(?<=# --- SDL scancodes ---\n ).*?(?=\n # --- end ---\n)",
 "\n ".join(generate_enums("SDL_SCANCODE")),
 event_py,
 flags=re.DOTALL,
 )
 event_py = re.sub(
 r"(?<=# --- SDL keyboard symbols ---\n ).*?(?=\n # --- end ---\n)",
 "\n ".join(generate_enums("SDLK")),
 event_py,
 flags=re.DOTALL,
 )

 Path("tcod/event.py").write_text(event_py, encoding="utf-8")


if __name__ == "__main__":
 write_library_constants()