Skip to content

gh-91719: Add pycore_opcode.h internal header file#91906

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 2 commits into from
Apr 25, 2022
Merged

gh-91719: Add pycore_opcode.h internal header file #91906

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d0af188
gh-91719: Add pycore_opcode.h internal header file
vstinner Apr 25, 2022
75a9bcf
Update PCbuild/regen.targets
vstinner Apr 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
581 changes: 581 additions & 0 deletions Include/internal/pycore_opcode.h
View file
Open in desktop

Large diffs are not rendered by default.

566 changes: 2 additions & 564 deletions Include/opcode.h
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 3 additions & 1 deletion Makefile.pre.in
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1311,8 +1311,10 @@ regen-opcode:
# using Tools/scripts/generate_opcode_h.py
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \
$(srcdir)/Lib/opcode.py \
$(srcdir)/Include/opcode.h.new
$(srcdir)/Include/opcode.h.new \
$(srcdir)/Include/internal/pycore_opcode.h.new
$(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new
$(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new

.PHONY: regen-token
regen-token:
Expand Down
1 change: 1 addition & 0 deletions Objects/codeobject.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -5,6 +5,7 @@
#include "structmember.h" // PyMemberDef
#include "pycore_code.h" // _PyCodeConstructor
#include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs
#include "pycore_opcode.h" // _PyOpcode_Deopt
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_tuple.h" // _PyTuple_ITEMS()
#include "clinic/codeobject.c.h"
Expand Down
5 changes: 3 additions & 2 deletions Objects/frameobject.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,10 +2,11 @@

#include "Python.h"
#include "pycore_ceval.h" // _PyEval_BuiltinsFromGlobals()
#include "pycore_moduleobject.h" // _PyModule_GetDict()
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
#include "pycore_code.h" // CO_FAST_LOCAL, etc.
#include "pycore_function.h" // _PyFunction_FromConstructor()
#include "pycore_moduleobject.h" // _PyModule_GetDict()
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
#include "pycore_opcode.h" // _PyOpcode_Caches

#include "frameobject.h" // PyFrameObject
#include "pycore_frame.h"
Expand Down
4 changes: 2 additions & 2 deletions Objects/genobject.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,12 +3,12 @@
#include "Python.h"
#include "pycore_call.h" // _PyObject_CallNoArgs()
#include "pycore_ceval.h" // _PyEval_EvalFrame()
#include "pycore_frame.h" // _PyInterpreterFrame
#include "pycore_genobject.h" // struct _Py_async_gen_state
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
#include "pycore_opcode.h" // _PyOpcode_Deopt
#include "pycore_pyerrors.h" // _PyErr_ClearExcState()
#include "pycore_pystate.h" // _PyThreadState_GET()
#include "pycore_frame.h" // _PyInterpreterFrame
#include "frameobject.h" // PyFrameObject
#include "structmember.h" // PyMemberDef
#include "opcode.h" // SEND

Expand Down
4 changes: 2 additions & 2 deletions PCbuild/regen.targets
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -14,7 +14,7 @@
<Argument>-C</Argument>
</_ASTOutputs>
<_OpcodeSources Include="$(PySourcePath)Tools\scripts\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" />
<_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Python\opcode_targets.h" />
<_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Include\internal\pycore_opcode.h;$(PySourcePath)Python\opcode_targets.h" />
<_TokenSources Include="$(PySourcePath)Grammar\Tokens" />
<_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc">
<Format>rst</Format>
Expand DownExpand Up@@ -59,7 +59,7 @@
Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)"
DependsOnTargets="FindPythonForBuild">
<Message Text="Regenerate @(_OpcodeOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
<Exec Command="$(PythonForBuild) Tools\scripts\generate_opcode_h.py Lib\opcode.py Include\opcode.h"
<Exec Command="$(PythonForBuild) Tools\scripts\generate_opcode_h.py Lib\opcode.py Include\opcode.h Include\internal\pycore_opcode.h"
WorkingDirectory="$(PySourcePath)" />
<Exec Command="$(PythonForBuild) Python\makeopcodetargets.py Python\opcode_targets.h"
WorkingDirectory="$(PySourcePath)" />
Expand Down
1 change: 1 addition & 0 deletions Python/ceval.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -15,6 +15,7 @@
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_object.h" // _PyObject_GC_TRACK()
#include "pycore_moduleobject.h" // PyModuleObject
#include "pycore_opcode.h" // EXTRA_CASES

@gvanrossumgvanrossumApr 26, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know why you call out EXTRA_CASES in the comment -- this file uses several other tables defined there as well.

@vstinnervstinnerApr 27, 2022

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When I add an #include, I put the same of a single symbol get from this #include, even if many other symbols are used. I only do that to check time to time if the include is still used: here I just have check if EXTRA_CASES is still used in the file. Otherwise, for each #include, I have to remove the line, try to build, and check if the C file still builds successfully.

@gvanrossumgvanrossumApr 27, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would be nice if there was tooling that did this instead. Without the comment.

#include "pycore_pyerrors.h" // _PyErr_Fetch()
#include "pycore_pylifecycle.h" // _PyErr_Print()
#include "pycore_pymem.h" // _PyMem_IsPtrFreed()
Expand Down
11 changes: 6 additions & 5 deletions Python/compile.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,17 +23,18 @@

#include <stdbool.h>

// Need _PyOpcode_RelativeJump of pycore_opcode.h
#define NEED_OPCODE_TABLES

#include "Python.h"
#include "pycore_ast.h" // _PyAST_GetDocString()
#include "pycore_compile.h" // _PyFuture_FromAST()
#include "pycore_code.h" // _PyCode_New()
#include "pycore_pymem.h" // _PyMem_IsPtrFreed()
#include "pycore_compile.h" // _PyFuture_FromAST()
#include "pycore_long.h" // _PyLong_GetZero()
#include "pycore_opcode.h" // _PyOpcode_Caches
#include "pycore_pymem.h" // _PyMem_IsPtrFreed()
#include "pycore_symtable.h" // PySTEntryObject

#define NEED_OPCODE_TABLES
#include "opcode.h" // EXTENDED_ARG


#define DEFAULT_BLOCK_SIZE 16
#define DEFAULT_CODE_SIZE 128
Expand Down
2 changes: 1 addition & 1 deletion Python/specialize.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@
#include "pycore_long.h"
#include "pycore_moduleobject.h"
#include "pycore_object.h"
#include "opcode.h"
#include "pycore_opcode.h" // _PyOpcode_Caches
#include "structmember.h" // struct PyMemberDef, T_OFFSET_EX

#include <stdlib.h> // rand()
Expand Down
87 changes: 59 additions & 28 deletions Tools/scripts/generate_opcode_h.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,12 +3,16 @@
import sys
import tokenize

header = """
/* Auto-generated by Tools/scripts/generate_opcode_h.py from Lib/opcode.py */
SCRIPT_NAME = "Tools/scripts/generate_opcode_h.py"
PYTHON_OPCODE = "Lib/opcode.py"

header = f"""
// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE}

#ifndef Py_OPCODE_H
#define Py_OPCODE_H
#ifdef __cplusplus
extern "C" {
extern "C" {{
#endif


Expand All@@ -28,6 +32,29 @@
#endif /* !Py_OPCODE_H */
"""

internal_header = f"""
// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE}

#ifndef Py_INTERNAL_OPCODE_H
#define Py_INTERNAL_OPCODE_H
#ifdef __cplusplus
extern "C" {{
#endif

#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif

#include "opcode.h"
""".lstrip()

internal_footer = """
#ifdef __cplusplus
}
#endif
#endif // !Py_INTERNAL_OPCODE_H
"""

DEFINE = "#define {:<38} {:>3}\n"

UINT32_MASK = (1<<32)-1
Expand All@@ -43,7 +70,7 @@ def write_int_array_from_ops(name, ops, out):
assert bits == 0
out.write(f"}};\n")

def main(opcode_py, outfile='Include/opcode.h'):
def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/internal/pycore_opcode.h'):
opcode = {}
if hasattr(tokenize, 'open'):
fp = tokenize.open(opcode_py) # Python 3.2+
Expand DownExpand Up@@ -75,8 +102,10 @@ def main(opcode_py, outfile='Include/opcode.h'):
opname_including_specialized[255] = 'DO_TRACING'
used[255] = True

with open(outfile, 'w') as fobj:
with (open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj):
fobj.write(header)
iobj.write(internal_header)

for name in opname:
if name in opmap:
fobj.write(DEFINE.format(name, opmap[name]))
Expand All@@ -86,28 +115,29 @@ def main(opcode_py, outfile='Include/opcode.h'):
for name, op in specialized_opmap.items():
fobj.write(DEFINE.format(name, op))

fobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
fobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n")
fobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], fobj)
write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], fobj)
iobj.write("\nextern const uint8_t _PyOpcode_Caches[256];\n")
iobj.write("\nextern const uint8_t _PyOpcode_Deopt[256];\n")
iobj.write("\n#ifdef NEED_OPCODE_TABLES\n")
write_int_array_from_ops("_PyOpcode_RelativeJump", opcode['hasjrel'], iobj)
write_int_array_from_ops("_PyOpcode_Jump", opcode['hasjrel'] + opcode['hasjabs'], iobj)

fobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n")
iobj.write("\nconst uint8_t _PyOpcode_Caches[256] = {\n")
for i, entries in enumerate(opcode["_inline_cache_entries"]):
if entries:
fobj.write(f" [{opname[i]}] = {entries},\n")
fobj.write("};\n")
iobj.write(f" [{opname[i]}] = {entries},\n")
iobj.write("};\n")

deoptcodes = {}
for basic in opmap:
deoptcodes[basic] = basic
for basic, family in opcode["_specializations"].items():
for specialized in family:
deoptcodes[specialized] = basic
fobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n")
iobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n")
for opt, deopt in sorted(deoptcodes.items()):
fobj.write(f" [{opt}] = {deopt},\n")
fobj.write("};\n")
fobj.write("#endif /* OPCODE_TABLES */\n")
iobj.write(f" [{opt}] = {deopt},\n")
iobj.write("};\n")
iobj.write("#endif // NEED_OPCODE_TABLES\n")

fobj.write("\n")
fobj.write("#define HAS_CONST(op) (false\\")
Expand All@@ -119,28 +149,29 @@ def main(opcode_py, outfile='Include/opcode.h'):
for i, (op, _) in enumerate(opcode["_nb_ops"]):
fobj.write(DEFINE.format(op, i))

fobj.write("\n")
fobj.write("#ifdef Py_DEBUG\n")
fobj.write("static const char *const _PyOpcode_OpName[256] = {\n")
iobj.write("\n")
iobj.write("#ifdef Py_DEBUG\n")
iobj.write("static const char *const _PyOpcode_OpName[256] = {\n")
for op, name in enumerate(opname_including_specialized):
if name[0] != "<":
op = name
fobj.write(f''' [{op}] = "{name}",\n''')
fobj.write("};\n")
fobj.write("#endif\n")
iobj.write(f''' [{op}] = "{name}",\n''')
iobj.write("};\n")
iobj.write("#endif\n")

fobj.write("\n")
fobj.write("#define EXTRA_CASES \\\n")
iobj.write("\n")
iobj.write("#define EXTRA_CASES \\\n")
for i, flag in enumerate(used):
if not flag:
fobj.write(f" case {i}: \\\n")
fobj.write(" ;\n")
iobj.write(f" case {i}: \\\n")
iobj.write(" ;\n")

fobj.write(footer)
iobj.write(internal_footer)


print(f"{outfile} regenerated from {opcode_py}")


if __name__ == '__main__':
main(sys.argv[1], sys.argv[2])
main(sys.argv[1], sys.argv[2], sys.argv[3])