diff --git a/eng/azure-pipelines.yml b/eng/azure-pipelines.yml index 2cb66672430..3ae2b131527 100644 --- a/eng/azure-pipelines.yml +++ b/eng/azure-pipelines.yml @@ -40,7 +40,7 @@ stages: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 container: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-webassembly-20220317214646-1ad56e8 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-webassembly-20220531132048-00a561c steps: - bash: | ./build.sh --ci --restore --build --configuration $(_BuildConfig) /p:TargetOS=Browser /p:TargetArchitecture=wasm $(_InternalBuildArgs) @@ -62,7 +62,7 @@ stages: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 container: - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-webassembly-20220317214646-1ad56e8 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-webassembly-20220531132048-00a561c steps: - bash: | ./build.sh --ci --restore --build --configuration $(_BuildConfig) /p:TargetOS=Browser /p:TargetArchitecture=wasm /p:WasmEnableThreads=true $(_InternalBuildArgs) diff --git a/eng/icu.proj b/eng/icu.proj index 9bc24b0e59e..da08c33e49d 100644 --- a/eng/icu.proj +++ b/eng/icu.proj @@ -24,8 +24,6 @@ succeed, we need to provide a patched version of that file. Ideally, fixing upstream would be best, but until then, patch. --> - diff --git a/eng/patches/emcc.py b/eng/patches/emcc.py index 8379d88ad74..976e13a9bc2 100644 --- a/eng/patches/emcc.py +++ b/eng/patches/emcc.py @@ -151,7 +151,7 @@ ] VALID_ENVIRONMENTS = ('web', 'webview', 'worker', 'node', 'shell') -SIMD_INTEL_FEATURE_TOWER = ['-msse', '-msse2', '-msse3', '-mssse3', '-msse4.1', '-msse4.2', '-mavx'] +SIMD_INTEL_FEATURE_TOWER = ['-msse', '-msse2', '-msse3', '-mssse3', '-msse4.1', '-msse4.2', '-msse4', '-mavx'] SIMD_NEON_FLAGS = ['-mfpu=neon'] COMPILE_ONLY_FLAGS = {'--default-obj-ext'} LINK_ONLY_FLAGS = { @@ -267,7 +267,6 @@ def __init__(self): self.use_preload_plugins = False self.default_object_extension = '.o' self.valid_abspaths = [] - self.cfi = False # Specifies the line ending format to use for all generated text files. # Defaults to using the native EOL on each platform (\r\n on Windows, \n on # Linux & MacOS) @@ -306,10 +305,10 @@ def setup_environment_settings(): (settings.ENVIRONMENT_MAY_BE_NODE and settings.USE_PTHREADS) if not settings.ENVIRONMENT_MAY_BE_WORKER and settings.PROXY_TO_WORKER: - exit_with_error('If you specify --proxy-to-worker and specify a "-s ENVIRONMENT=" directive, it must include "worker" as a target! (Try e.g. -s ENVIRONMENT=web,worker)') + exit_with_error('If you specify --proxy-to-worker and specify a "-sENVIRONMENT=" directive, it must include "worker" as a target! (Try e.g. -sENVIRONMENT=web,worker)') if not settings.ENVIRONMENT_MAY_BE_WORKER and settings.SHARED_MEMORY: - exit_with_error('When building with multithreading enabled and a "-s ENVIRONMENT=" directive is specified, it must include "worker" as a target! (Try e.g. -s ENVIRONMENT=web,worker)') + exit_with_error('When building with multithreading enabled and a "-sENVIRONMENT=" directive is specified, it must include "worker" as a target! (Try e.g. -sENVIRONMENT=web,worker)') def minify_whitespace(): @@ -376,25 +375,17 @@ def apply_settings(user_settings): else: value = value.replace('\\', '\\\\') - existing = getattr(settings, user_key, None) - expect_list = type(existing) == list + expected_type = settings.types.get(key) - if filename and expect_list and value.strip()[0] != '[': + if filename and expected_type == list and value.strip()[0] != '[': # Prefer simpler one-line-per value parser value = parse_symbol_list_file(value) else: try: - value = parse_value(value, expect_list) + value = parse_value(value, expected_type) except Exception as e: exit_with_error('a problem occurred in evaluating the content after a "-s", specifically "%s=%s": %s', key, value, str(e)) - # Do some basic type checking by comparing to the existing settings. - # Sadly we can't do this generically in the SettingsManager since there are settings - # that so change types internally over time. - # We only currently worry about lists vs non-lists. - if expect_list != (type(value) == list): - exit_with_error('setting `%s` expects `%s` but got `%s`' % (user_key, type(existing), type(value))) - setattr(settings, user_key, value) if key == 'EXPORTED_FUNCTIONS': @@ -522,27 +513,30 @@ def cxx_to_c_compiler(cxx): return os.path.join(dirname, basename) -def get_binaryen_passes(): +def should_run_binaryen_optimizer(): # run the binaryen optimizer in -O2+. in -O0 we don't need it obviously, while # in -O1 we don't run it as the LLVM optimizer has been run, and it does the # great majority of the work; not running the binaryen optimizer in that case # keeps -O1 mostly-optimized while compiling quickly and without rewriting # DWARF etc. - run_binaryen_optimizer = settings.OPT_LEVEL >= 2 + return settings.OPT_LEVEL >= 2 + +def get_binaryen_passes(): passes = [] + optimizing = should_run_binaryen_optimizer() # safe heap must run before post-emscripten, so post-emscripten can apply the sbrk ptr if settings.SAFE_HEAP: passes += ['--safe-heap'] if settings.MEMORY64 == 2: passes += ['--memory64-lowering'] - if run_binaryen_optimizer: + if optimizing: passes += ['--post-emscripten'] - if run_binaryen_optimizer: + if optimizing: passes += [building.opt_level_to_str(settings.OPT_LEVEL, settings.SHRINK_LEVEL)] # when optimizing, use the fact that low memory is never used (1024 is a # hardcoded value in the binaryen pass) - if run_binaryen_optimizer and settings.GLOBAL_BASE >= 1024: + if optimizing and settings.GLOBAL_BASE >= 1024: passes += ['--low-memory-unused'] if settings.AUTODEBUG: # adding '--flatten' here may make these even more effective @@ -595,7 +589,7 @@ def check_human_readable_list(items): # the one exception is dynamic linking of a side module: the main module is ok # as it is loaded first, but the side module may be assigned memory that was # previously used. - if run_binaryen_optimizer and not settings.SIDE_MODULE: + if optimizing and not settings.SIDE_MODULE: passes += ['--zero-filled-memory'] if settings.BINARYEN_EXTRA_PASSES: @@ -799,10 +793,11 @@ def array_contains_any_of(hay, needles): if array_contains_any_of(user_args, SIMD_INTEL_FEATURE_TOWER[4:]): cflags += ['-D__SSE4_1__=1'] + # Handle both -msse4.2 and its alias -msse4. if array_contains_any_of(user_args, SIMD_INTEL_FEATURE_TOWER[5:]): cflags += ['-D__SSE4_2__=1'] - if array_contains_any_of(user_args, SIMD_INTEL_FEATURE_TOWER[6:]): + if array_contains_any_of(user_args, SIMD_INTEL_FEATURE_TOWER[7:]): cflags += ['-D__AVX__=1'] if array_contains_any_of(user_args, SIMD_NEON_FLAGS): @@ -833,6 +828,9 @@ def get_cflags(user_args): if settings.SHARED_MEMORY: cflags.append('-D__EMSCRIPTEN_SHARED_MEMORY__=1') + if settings.WASM_WORKERS: + cflags.append('-D__EMSCRIPTEN_WASM_WORKERS__=1') + if not settings.STRICT: # The preprocessor define EMSCRIPTEN is deprecated. Don't pass it to code # in strict mode. Code should use the define __EMSCRIPTEN__ instead. @@ -937,11 +935,9 @@ def in_temp(name): def dedup_list(lst): - rtn = [] - for item in lst: - if item not in rtn: - rtn.append(item) - return rtn + # Since we require python 3.6, that ordering of dictionaries is guaranteed + # to be insertion order so we can use 'dict' here but not 'set'. + return list(dict.fromkeys(lst)) def move_file(src, dst): @@ -994,6 +990,8 @@ def package_files(options, target): if not settings.ENVIRONMENT_MAY_BE_NODE: file_args.append('--no-node') if options.embed_files: + if settings.MEMORY64: + file_args += ['--wasm64'] object_file = in_temp('embedded_files.o') file_args += ['--obj-output=' + object_file] rtn.append(object_file) @@ -1040,7 +1038,7 @@ def run(args): # Strip args[0] (program name) args = args[1:] - misc_temp_files = shared.configuration.get_temp_files() + misc_temp_files = shared.get_temp_files() # Handle some global flags @@ -1424,11 +1422,17 @@ def phase_setup(options, state, newargs, user_settings): if settings.MAIN_MODULE or settings.SIDE_MODULE: settings.RELOCATABLE = 1 - if settings.USE_PTHREADS: + # Pthreads and Wasm Workers require targeting shared Wasm memory (SAB). + if settings.USE_PTHREADS or settings.WASM_WORKERS: settings.SHARED_MEMORY = 1 - if settings.SHARED_MEMORY and '-pthread' not in newargs: + if settings.USE_PTHREADS and '-pthread' not in newargs: newargs += ['-pthread'] + elif settings.SHARED_MEMORY: + if '-matomics' not in newargs: + newargs += ['-matomics'] + if '-mbulk-memory' not in newargs: + newargs += ['-mbulk-memory'] if 'DISABLE_EXCEPTION_CATCHING' in user_settings and 'EXCEPTION_CATCHING_ALLOWED' in user_settings: # If we get here then the user specified both DISABLE_EXCEPTION_CATCHING and EXCEPTION_CATCHING_ALLOWED @@ -1445,6 +1449,11 @@ def phase_setup(options, state, newargs, user_settings): if settings.DISABLE_EXCEPTION_THROWING and not settings.DISABLE_EXCEPTION_CATCHING: exit_with_error("DISABLE_EXCEPTION_THROWING was set (probably from -fno-exceptions) but is not compatible with enabling exception catching (DISABLE_EXCEPTION_CATCHING=0). If you don't want exceptions, set DISABLE_EXCEPTION_CATCHING to 1; if you do want exceptions, don't link with -fno-exceptions") + if settings.MEMORY64: + default_setting(user_settings, 'SUPPORT_LONGJMP', 0) + if settings.SUPPORT_LONGJMP: + exit_with_error('MEMORY64 is not compatible with SUPPORT_LONGJMP') + # SUPPORT_LONGJMP=1 means the default SjLj handling mechanism, currently # 'emscripten' if settings.SUPPORT_LONGJMP == 1: @@ -1467,6 +1476,95 @@ def phase_setup(options, state, newargs, user_settings): return (newargs, input_files) +def setup_pthreads(target): + if settings.RELOCATABLE: + # phtreads + dyanmic linking has certain limitations + if settings.SIDE_MODULE: + diagnostics.warning('experimental', '-sSIDE_MODULE + pthreads is experimental') + elif settings.MAIN_MODULE: + diagnostics.warning('experimental', '-sMAIN_MODULE + pthreads is experimental') + elif settings.LINKABLE: + diagnostics.warning('experimental', '-sLINKABLE + pthreads is experimental') + if settings.ALLOW_MEMORY_GROWTH: + diagnostics.warning('pthreads-mem-growth', 'USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code slowly, see https://github.com/WebAssembly/design/issues/1271') + + # Functions needs to be exported from the module since they are used in worker.js + settings.REQUIRED_EXPORTS += [ + 'emscripten_dispatch_to_thread_', + '_emscripten_thread_free_data', + 'emscripten_main_browser_thread_id', + 'emscripten_main_thread_process_queued_calls', + 'emscripten_run_in_main_runtime_thread_js', + 'emscripten_stack_set_limits', + ] + + if settings.MAIN_MODULE: + settings.REQUIRED_EXPORTS += ['_emscripten_thread_sync_code', '__dl_seterr'] + + settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += [ + '$exitOnMainThread', + ] + # Some symbols are required by worker.js. + # Because emitDCEGraph only considers the main js file, and not worker.js + # we have explicitly mark these symbols as user-exported so that they will + # kept alive through DCE. + # TODO: Find a less hacky way to do this, perhaps by also scanning worker.js + # for roots. + worker_imports = [ + '__emscripten_thread_init', + '__emscripten_thread_exit', + '__emscripten_thread_crashed', + '__emscripten_tls_init', + '_pthread_self', + 'executeNotifiedProxyingQueue', + ] + settings.EXPORTED_FUNCTIONS += worker_imports + building.user_requested_exports.update(worker_imports) + + # set location of worker.js + settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + '.worker.js' + + # memalign is used to ensure allocated thread stacks are aligned. + settings.REQUIRED_EXPORTS += ['emscripten_builtin_memalign'] + + if settings.MINIMAL_RUNTIME: + building.user_requested_exports.add('exit') + + if settings.PROXY_TO_PTHREAD: + settings.REQUIRED_EXPORTS += ['emscripten_proxy_main'] + + # All proxying async backends will need this. + if settings.WASMFS: + settings.REQUIRED_EXPORTS += ['emscripten_proxy_finish'] + + # pthread stack setup and other necessary utilities + def include_and_export(name): + settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$' + name] + settings.EXPORTED_FUNCTIONS += [name] + + include_and_export('establishStackSpace') + include_and_export('invokeEntryPoint') + include_and_export('PThread') + if not settings.MINIMAL_RUNTIME: + # keepRuntimeAlive does not apply to MINIMAL_RUNTIME. + settings.EXPORTED_RUNTIME_METHODS += ['keepRuntimeAlive', 'ExitStatus', 'wasmMemory'] + + if settings.MODULARIZE: + if not settings.EXPORT_ES6 and settings.EXPORT_NAME == 'Module': + exit_with_error('pthreads + MODULARIZE currently require you to set -sEXPORT_NAME=Something (see settings.js) to Something != Module, so that the .worker.js file can work') + + # MODULARIZE+USE_PTHREADS mode requires extra exports out to Module so that worker.js + # can access them: + + # general threading variables: + settings.EXPORTED_RUNTIME_METHODS += ['PThread'] + + # To keep code size to minimum, MINIMAL_RUNTIME does not utilize the global ExitStatus + # object, only regular runtime has it. + if not settings.MINIMAL_RUNTIME: + settings.EXPORTED_RUNTIME_METHODS += ['ExitStatus'] + + @ToolchainProfiler.profile_block('linker_setup') def phase_linker_setup(options, state, newargs, user_settings): autoconf = os.environ.get('EMMAKEN_JUST_CONFIGURE') or 'conftest.c' in state.orig_args @@ -1539,8 +1637,8 @@ def phase_linker_setup(options, state, newargs, user_settings): diagnostics.warning('deprecated', 'EXTRA_EXPORTED_RUNTIME_METHODS is deprecated, please use EXPORTED_RUNTIME_METHODS instead') settings.EXPORTED_RUNTIME_METHODS += settings.EXTRA_EXPORTED_RUNTIME_METHODS - # If no output format was sepecific we try to imply the format based on - # the output filename extension. + # If no output format was specified we try to deduce the format based on + # the output filename extension if not options.oformat and (options.relocatable or (options.shared and not settings.SIDE_MODULE)): # Until we have a better story for actually producing runtime shared libraries # we support a compatibility mode where shared libraries are actually just @@ -1638,7 +1736,7 @@ def phase_linker_setup(options, state, newargs, user_settings): # Note the exports the user requested building.user_requested_exports.update(settings.EXPORTED_FUNCTIONS) - # -s ASSERTIONS=1 implies basic stack overflow checks, and ASSERTIONS=2 + # -sASSERTIONS implies basic stack overflow checks, and ASSERTIONS=2 # implies full stack overflow checks. if settings.ASSERTIONS: # However, we don't set this default in PURE_WASI, or when we are linking without standard @@ -1708,14 +1806,17 @@ def phase_linker_setup(options, state, newargs, user_settings): if settings.MINIMAL_RUNTIME_STREAMING_WASM_COMPILATION and settings.MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION: exit_with_error('MINIMAL_RUNTIME_STREAMING_WASM_COMPILATION and MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION are mutually exclusive!') - if options.emrun and settings.MINIMAL_RUNTIME: - exit_with_error('--emrun is not compatible with MINIMAL_RUNTIME') + if options.emrun: + if settings.MINIMAL_RUNTIME: + exit_with_error('--emrun is not compatible with MINIMAL_RUNTIME') + if options.oformat != OFormat.HTML: + exit_with_error('--emrun is only compatible with html output') if options.use_closure_compiler: - settings.USE_CLOSURE_COMPILER = options.use_closure_compiler + settings.USE_CLOSURE_COMPILER = 1 if settings.CLOSURE_WARNINGS not in ['quiet', 'warn', 'error']: - exit_with_error('Invalid option -s CLOSURE_WARNINGS=%s specified! Allowed values are "quiet", "warn" or "error".' % settings.CLOSURE_WARNINGS) + exit_with_error('Invalid option -sCLOSURE_WARNINGS=%s specified! Allowed values are "quiet", "warn" or "error".' % settings.CLOSURE_WARNINGS) # Include dynCall() function by default in DYNCALLS builds in classic runtime; in MINIMAL_RUNTIME, must add this explicitly. if settings.DYNCALLS and not settings.MINIMAL_RUNTIME: @@ -1724,6 +1825,10 @@ def phase_linker_setup(options, state, newargs, user_settings): if not settings.BOOTSTRAPPING_STRUCT_INFO: # Include the internal library function since they are used by runtime functions. settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$getWasmTableEntry', '$setWasmTableEntry'] + if settings.SAFE_HEAP: + settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$getValue_safe', '$setValue_safe'] + if not settings.MINIMAL_RUNTIME: + settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$getValue', '$setValue'] if settings.MAIN_MODULE: assert not settings.SIDE_MODULE @@ -1755,7 +1860,7 @@ def phase_linker_setup(options, state, newargs, user_settings): if settings.USE_PTHREADS: settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += [ - '$registerTlsInit', + '$registerTLSInit', ] if settings.RELOCATABLE: @@ -1847,7 +1952,7 @@ def phase_linker_setup(options, state, newargs, user_settings): if settings.MODULARIZE: if settings.PROXY_TO_WORKER: - exit_with_error('-s MODULARIZE=1 is not compatible with --proxy-to-worker (if you want to run in a worker with -s MODULARIZE=1, you likely want to do the worker side setup manually)') + exit_with_error('-sMODULARIZE is not compatible with --proxy-to-worker (if you want to run in a worker with -sMODULARIZE, you likely want to do the worker side setup manually)') # in MINIMAL_RUNTIME we may not need to emit the Promise code, as the # HTML output creates a singleton instance, and it does so without the # Promise. However, in Pthreads mode the Promise is used for worker @@ -1909,7 +2014,7 @@ def phase_linker_setup(options, state, newargs, user_settings): exit_with_error('MIN_WEBGL_VERSION must be smaller or equal to MAX_WEBGL_VERSION!') if not settings.GL_SUPPORT_SIMPLE_ENABLE_EXTENSIONS and settings.GL_SUPPORT_AUTOMATIC_ENABLE_EXTENSIONS: - exit_with_error('-s GL_SUPPORT_SIMPLE_ENABLE_EXTENSIONS=0 only makes sense with -s GL_SUPPORT_AUTOMATIC_ENABLE_EXTENSIONS=0!') + exit_with_error('-sGL_SUPPORT_SIMPLE_ENABLE_EXTENSIONS=0 only makes sense with -sGL_SUPPORT_AUTOMATIC_ENABLE_EXTENSIONS=0!') if settings.WASMFS: state.forced_stdlibs.append('libwasmfs') @@ -1917,6 +2022,12 @@ def phase_linker_setup(options, state, newargs, user_settings): settings.SYSCALLS_REQUIRE_FILESYSTEM = 0 settings.JS_LIBRARIES.append((0, 'library_wasmfs.js')) settings.REQUIRED_EXPORTS += ['_wasmfs_read_file'] + if settings.MAIN_MODULE: + # Dynamic library support uses JS API internals, so include it all + # TODO: rewriting more of the dynamic linking support code into wasm could + # avoid this. also, after we remove the old FS, we could write a + # more specific API for wasmfs/dynamic linking integration perhaps + settings.FORCE_FILESYSTEM = 1 if settings.FORCE_FILESYSTEM: # Add exports for the JS API. Like the old JS FS, WasmFS by default # includes just what JS parts it actually needs, and FORCE_FILESYSTEM is @@ -1928,6 +2039,7 @@ def phase_linker_setup(options, state, newargs, user_settings): '_wasmfs_chdir', '_wasmfs_symlink', '_wasmfs_chmod', + '_wasmfs_identify', ] # Explicitly drop linking in a malloc implementation if program is not using any dynamic allocation calls. @@ -1990,12 +2102,11 @@ def phase_linker_setup(options, state, newargs, user_settings): ] if not settings.STANDALONE_WASM and (settings.EXIT_RUNTIME or settings.ASSERTIONS): - # We use __stdio_exit to shut down musl's stdio subsystems and flush - # streams on exit. - # We only include it if the runtime is exitable, or when ASSERTIONS + # to flush streams on FS exit, we need to be able to call fflush + # we only include it if the runtime is exitable, or when ASSERTIONS # (ASSERTIONS will check that streams do not need to be flushed, # helping people see when they should have enabled EXIT_RUNTIME) - settings.EXPORT_IF_DEFINED += ['__stdio_exit'] + settings.EXPORT_IF_DEFINED += ['fflush'] if settings.SUPPORT_ERRNO: # so setErrNo JS library function can report errno back to C @@ -2017,54 +2128,29 @@ def phase_linker_setup(options, state, newargs, user_settings): default_setting(user_settings, 'ABORTING_MALLOC', 0) if settings.USE_PTHREADS: - if settings.ALLOW_MEMORY_GROWTH: - diagnostics.warning('pthreads-mem-growth', 'USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code slowly, see https://github.com/WebAssembly/design/issues/1271') + setup_pthreads(target) settings.JS_LIBRARIES.append((0, 'library_pthread.js')) - # Functions needs to be exported from the module since they are used in worker.js - settings.REQUIRED_EXPORTS += [ - 'emscripten_dispatch_to_thread_', - '_emscripten_thread_free_data', - '_emscripten_allow_main_runtime_queued_calls', - 'emscripten_main_browser_thread_id', - 'emscripten_main_thread_process_queued_calls', - 'emscripten_run_in_main_runtime_thread_js', - 'emscripten_stack_set_limits', - 'emscripten_sync_run_in_main_thread_2', - 'emscripten_sync_run_in_main_thread_4', - ] - - if settings.MAIN_MODULE: - settings.REQUIRED_EXPORTS += ['_emscripten_thread_sync_code', '__dl_seterr'] - - settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += [ - '$exitOnMainThread', - ] - # Some symbols are required by worker.js. - # Because emitDCEGraph only considers the main js file, and not worker.js - # we have explicitly mark these symbols as user-exported so that they will - # kept alive through DCE. - # TODO: Find a less hacky way to do this, perhaps by also scanning worker.js - # for roots. - worker_imports = [ - '__emscripten_thread_init', - '__emscripten_thread_exit', - '__emscripten_thread_crashed', - '_emscripten_tls_init', - '_emscripten_current_thread_process_queued_calls', - '_pthread_self', - ] - settings.EXPORTED_FUNCTIONS += worker_imports - building.user_requested_exports.update(worker_imports) - - # set location of worker.js - settings.PTHREAD_WORKER_FILE = unsuffixed_basename(target) + '.worker.js' else: + if settings.PROXY_TO_PTHREAD: + exit_with_error('-sPROXY_TO_PTHREAD requires -sUSE_PTHREADS to work!') settings.JS_LIBRARIES.append((0, 'library_pthread_stub.js')) # TODO: Move this into the library JS file once it becomes possible. # See https://github.com/emscripten-core/emscripten/pull/15982 if settings.INCLUDE_FULL_LIBRARY and not settings.DISABLE_EXCEPTION_CATCHING: - settings.EXPORTED_FUNCTIONS += ['_emscripten_format_exception', '_free'] + settings.EXPORTED_FUNCTIONS += ['___get_exception_message', '_free'] + + if settings.WASM_WORKERS: + # TODO: After #15982 is resolved, these dependencies can be declared in library_wasm_worker.js + # instead of having to record them here. + wasm_worker_imports = ['_emscripten_wasm_worker_initialize'] + settings.EXPORTED_FUNCTIONS += wasm_worker_imports + building.user_requested_exports.update(wasm_worker_imports) + settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['_wasm_worker_initializeRuntime'] + # set location of Wasm Worker bootstrap JS file + if settings.WASM_WORKERS == 1: + settings.WASM_WORKER_FILE = unsuffixed(os.path.basename(target)) + '.ww.js' + settings.JS_LIBRARIES.append((0, shared.path_from_root('src', 'library_wasm_worker.js'))) if settings.FORCE_FILESYSTEM and not settings.MINIMAL_RUNTIME: # when the filesystem is forced, we export by default methods that filesystem usage @@ -2075,63 +2161,20 @@ def phase_linker_setup(options, state, newargs, user_settings): 'FS_createPath', 'FS_createDataFile', 'FS_createPreloadedFile', - 'FS_createLazyFile', - 'FS_createDevice', 'FS_unlink' ] + if not settings.WASMFS: + # The old FS has some functionality that WasmFS lacks. + settings.EXPORTED_RUNTIME_METHODS += [ + 'FS_createLazyFile', + 'FS_createDevice' + ] settings.EXPORTED_RUNTIME_METHODS += [ 'addRunDependency', 'removeRunDependency', ] - if settings.USE_PTHREADS: - # memalign is used to ensure allocated thread stacks are aligned. - settings.REQUIRED_EXPORTS += ['emscripten_builtin_memalign'] - - if settings.MINIMAL_RUNTIME: - building.user_requested_exports.add('exit') - - if settings.PROXY_TO_PTHREAD: - settings.REQUIRED_EXPORTS += ['emscripten_proxy_main'] - - # pthread stack setup and other necessary utilities - def include_and_export(name): - settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$' + name] - settings.EXPORTED_FUNCTIONS += [name] - - include_and_export('establishStackSpace') - include_and_export('invokeEntryPoint') - if not settings.MINIMAL_RUNTIME: - # keepRuntimeAlive does not apply to MINIMAL_RUNTIME. - settings.EXPORTED_RUNTIME_METHODS += ['keepRuntimeAlive'] - - if settings.MODULARIZE: - if not settings.EXPORT_ES6 and settings.EXPORT_NAME == 'Module': - exit_with_error('pthreads + MODULARIZE currently require you to set -s EXPORT_NAME=Something (see settings.js) to Something != Module, so that the .worker.js file can work') - - # MODULARIZE+USE_PTHREADS mode requires extra exports out to Module so that worker.js - # can access them: - - # general threading variables: - settings.EXPORTED_RUNTIME_METHODS += ['PThread'] - - # To keep code size to minimum, MINIMAL_RUNTIME does not utilize the global ExitStatus - # object, only regular runtime has it. - if not settings.MINIMAL_RUNTIME: - settings.EXPORTED_RUNTIME_METHODS += ['ExitStatus'] - - if settings.RELOCATABLE: - # phtreads + dyanmic linking has certain limitations - if settings.SIDE_MODULE: - diagnostics.warning('experimental', '-s SIDE_MODULE + pthreads is experimental') - elif settings.MAIN_MODULE: - diagnostics.warning('experimental', '-s MAIN_MODULE + pthreads is experimental') - elif settings.LINKABLE: - diagnostics.warning('experimental', '-s LINKABLE + pthreads is experimental') - elif settings.PROXY_TO_PTHREAD: - exit_with_error('-s PROXY_TO_PTHREAD=1 requires -s USE_PTHREADS to work!') - def check_memory_setting(setting): if settings[setting] % webassembly.WASM_PAGE_SIZE != 0: exit_with_error(f'{setting} must be a multiple of WebAssembly page size (64KiB), was {settings[setting]}') @@ -2199,7 +2242,7 @@ def check_memory_setting(setting): if settings.MODULARIZE and not (settings.EXPORT_ES6 and not settings.SINGLE_FILE) and \ settings.EXPORT_NAME == 'Module' and options.oformat == OFormat.HTML and \ (options.shell_path == utils.path_from_root('src/shell.html') or options.shell_path == utils.path_from_root('src/shell_minimal.html')): - exit_with_error(f'Due to collision in variable name "Module", the shell file "{options.shell_path}" is not compatible with build options "-s MODULARIZE=1 -s EXPORT_NAME=Module". Either provide your own shell file, change the name of the export to something else to avoid the name collision. (see https://github.com/emscripten-core/emscripten/issues/7950 for details)') + exit_with_error(f'Due to collision in variable name "Module", the shell file "{options.shell_path}" is not compatible with build options "-sMODULARIZE -sEXPORT_NAME=Module". Either provide your own shell file, change the name of the export to something else to avoid the name collision. (see https://github.com/emscripten-core/emscripten/issues/7950 for details)') if settings.STANDALONE_WASM: if settings.MINIMAL_RUNTIME: @@ -2212,6 +2255,17 @@ def check_memory_setting(setting): if settings.SHARED_MEMORY or settings.RELOCATABLE or settings.ASYNCIFY_LAZY_LOAD_CODE or settings.WASM2JS: settings.IMPORTED_MEMORY = 1 + # Any "pointers" passed to JS will now be i64's, in both modes. + # Also turn off minifying, which clashes with instrumented functions in preamble.js + if settings.MEMORY64: + if settings.RELOCATABLE: + exit_with_error('MEMORY64 is not compatible with dynamic linking') + if not settings.DISABLE_EXCEPTION_CATCHING: + exit_with_error('MEMORY64 is not compatible with DISABLE_EXCEPTION_CATCHING=0') + settings.WASM_BIGINT = 1 + settings.MINIFY_WASM_IMPORTS_AND_EXPORTS = 0 + settings.MINIFY_WASM_IMPORTED_MODULES = 0 + if settings.WASM_BIGINT: settings.LEGALIZE_JS_FFI = 0 @@ -2458,8 +2512,13 @@ def get_full_import_name(name): settings.ASYNCIFY_IMPORTS = [get_full_import_name(i) for i in settings.ASYNCIFY_IMPORTS] - if settings.WASM2JS and settings.GENERATE_SOURCE_MAP: - exit_with_error('wasm2js does not support source maps yet (debug in wasm for now)') + if settings.WASM2JS: + if settings.GENERATE_SOURCE_MAP: + exit_with_error('wasm2js does not support source maps yet (debug in wasm for now)') + if settings.WASM_BIGINT: + exit_with_error('wasm2js does not support WASM_BIGINT') + if settings.MEMORY64: + exit_with_error('wasm2js does not support MEMORY64') if settings.NODE_CODE_CACHING: if settings.WASM_ASYNC_COMPILATION: @@ -2478,14 +2537,6 @@ def get_full_import_name(name): 'emscripten_stack_get_base', 'emscripten_stack_get_end'] - # Any "pointers" passed to JS will now be i64's, in both modes. - # Also turn off minifying, which clashes with instrumented functions in preamble.js - if settings.MEMORY64: - if user_settings.get('WASM_BIGINT') == '0': - exit_with_error('MEMORY64 is not compatible with WASM_BIGINT=0') - settings.WASM_BIGINT = 1 - settings.MINIFY_WASM_IMPORTS_AND_EXPORTS = 0 - # check if we can address the 2GB mark and higher: either if we start at # 2GB, or if we allow growth to either any amount or to 2GB or more. if settings.INITIAL_MEMORY > 2 * 1024 * 1024 * 1024 or \ @@ -2503,6 +2554,19 @@ def get_full_import_name(name): if settings.WASMFS: settings.LINK_AS_CXX = True + # Some settings make no sense when not linking as C++ + if not settings.LINK_AS_CXX: + cxx_only_settings = [ + 'DEMANGLE_SUPPORT', + 'EXCEPTION_DEBUG', + 'DISABLE_EXCEPTION_CATCHING', + 'EXCEPTION_CATCHING_ALLOWED', + 'DISABLE_EXCEPTION_THROWING', + ] + for setting in cxx_only_settings: + if setting in user_settings: + diagnostics.warning('linkflags', 'setting `%s` is not meaningful unless linking as C++', setting) + # Export tag objects which are likely needed by the native code, but which are # currently not reported in the metadata of wasm-emscripten-finalize if settings.RELOCATABLE: @@ -2511,6 +2575,9 @@ def get_full_import_name(name): if settings.SUPPORT_LONGJMP == 'wasm': settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.append('__c_longjmp') + if settings.EXCEPTION_HANDLING: + settings.REQUIRED_EXPORTS += ['__trap'] + return target, wasm_target @@ -2672,9 +2739,9 @@ def compile_source_file(i, input_file): @ToolchainProfiler.profile_block('calculate system libraries') def phase_calculate_system_libraries(state, linker_arguments, linker_inputs, newargs): extra_files_to_link = [] - # link in ports and system libraries, if necessary + # Link in ports and system libraries, if necessary if not settings.SIDE_MODULE: - # Ports are always linked into the main module, never the size module. + # Ports are always linked into the main module, never the side module. extra_files_to_link += ports.get_libs(settings) all_linker_inputs = [f for _, f in sorted(linker_inputs)] + extra_files_to_link extra_files_to_link += system_libs.calculate(all_linker_inputs, newargs, forced=state.forced_stdlibs) @@ -2805,8 +2872,8 @@ def phase_final_emitting(options, state, target, wasm_target, memfile): # src = re.sub(r'\n+[ \n]*\n+', '\n', src) # write_file(final_js, src) + target_dir = os.path.dirname(os.path.abspath(target)) if settings.USE_PTHREADS: - target_dir = os.path.dirname(os.path.abspath(target)) worker_output = os.path.join(target_dir, settings.PTHREAD_WORKER_FILE) contents = shared.read_and_preprocess(utils.path_from_root('src/worker.js'), expand_macros=True) write_file(worker_output, contents) @@ -2816,6 +2883,17 @@ def phase_final_emitting(options, state, target, wasm_target, memfile): minified_worker = building.acorn_optimizer(worker_output, ['minifyWhitespace'], return_output=True) write_file(worker_output, minified_worker) + # Deploy the Wasm Worker bootstrap file as an output file (*.ww.js) + if settings.WASM_WORKERS == 1: + worker_output = os.path.join(target_dir, settings.WASM_WORKER_FILE) + with open(worker_output, 'w') as f: + f.write(shared.read_and_preprocess(shared.path_from_root('src', 'wasm_worker.js'), expand_macros=True)) + + # Minify the wasm_worker.js file in optimized builds + if (settings.OPT_LEVEL >= 1 or settings.SHRINK_LEVEL >= 1) and not settings.DEBUG_LEVEL: + minified_worker = building.acorn_optimizer(worker_output, ['minifyWhitespace'], return_output=True) + open(worker_output, 'w').write(minified_worker) + # track files that will need native eols generated_text_files_with_native_eols = [] @@ -2824,13 +2902,13 @@ def phase_final_emitting(options, state, target, wasm_target, memfile): module_export_name_substitution() - # Run a final regex pass to clean up items that were not possible to optimize by Closure, or unoptimalities that were left behind + # Run a final optimization pass to clean up items that were not possible to optimize by Closure, or unoptimalities that were left behind # by processing steps that occurred after Closure. - if settings.MINIMAL_RUNTIME == 2 and settings.USE_CLOSURE_COMPILER and settings.DEBUG_LEVEL == 0 and not settings.SINGLE_FILE: - # Process .js runtime file. Note that we need to handle the license text - # here, so that it will not confuse the hacky script. - js_manipulation.handle_license(final_js) - shared.run_process([shared.PYTHON, utils.path_from_root('tools/hacky_postprocess_around_closure_limitations.py'), final_js]) + if settings.MINIMAL_RUNTIME == 2 and settings.USE_CLOSURE_COMPILER and settings.DEBUG_LEVEL == 0: + shared.run_js_tool(utils.path_from_root('tools/unsafe_optimizations.js'), [final_js, '-o', final_js], cwd=utils.path_from_root('.')) + # Finally, rerun Closure compile with simple optimizations. It will be able to further minify the code. (n.b. it would not be safe + # to run in advanced mode) + final_js = building.closure_compiler(final_js, pretty=False, advanced=False, extra_closure_args=options.closure_args) # Unmangle previously mangled `import.meta` references in both main code and libraries. # See also: `preprocess` in parseTools.js. @@ -2967,7 +3045,7 @@ def consume_arg_file(): options.requested_level = 2 settings.SHRINK_LEVEL = 2 settings_changes.append('INLINING_LIMIT=1') - settings.OPT_LEVEL = validate_arg_level(options.requested_level, 3, 'Invalid optimization level: ' + arg, clamp=True) + settings.OPT_LEVEL = validate_arg_level(options.requested_level, 3, 'invalid optimization level: ' + arg, clamp=True) elif check_arg('--js-opts'): logger.warning('--js-opts ignored when using llvm backend') consume_arg() @@ -3016,14 +3094,14 @@ def consume_arg_file(): requested_level = strip_prefix(arg, '-g') or '3' if is_int(requested_level): # the -gX value is the debug level (-g1, -g2, etc.) - settings.DEBUG_LEVEL = validate_arg_level(requested_level, 4, 'Invalid debug level: ' + arg) + settings.DEBUG_LEVEL = validate_arg_level(requested_level, 4, 'invalid debug level: ' + arg) # if we don't need to preserve LLVM debug info, do not keep this flag # for clang if settings.DEBUG_LEVEL < 3: - newargs[i] = '' + newargs[i] = '-g0' else: - # for 3+, report -g to clang as -g4 etc. are not accepted - newargs[i] = '-g' + # for 3+, report -g3 to clang as -g4 etc. are not accepted + newargs[i] = '-g3' if settings.DEBUG_LEVEL == 4: settings.GENERATE_SOURCE_MAP = 1 diagnostics.warning('deprecated', 'please replace -g4 with -gsource-map') @@ -3153,8 +3231,8 @@ def consume_arg_file(): options.default_object_extension = consume_arg() if not options.default_object_extension.startswith('.'): options.default_object_extension = '.' + options.default_object_extension - elif arg == '-fsanitize=cfi': - options.cfi = True + elif arg.startswith('-fsanitize=cfi'): + exit_with_error('emscripten does not currently support -fsanitize=cfi') elif check_arg('--output_eol'): style = consume_arg() if style.lower() == 'windows': @@ -3180,7 +3258,7 @@ def consume_arg_file(): else: value = '1' if key in settings.keys(): - exit_with_error(f'{arg}: cannot change built-in settings values with a -jsD directive. Pass -s {key}={value} instead!') + exit_with_error(f'{arg}: cannot change built-in settings values with a -jsD directive. Pass -s{key}={value} instead!') user_js_defines += [(key, value)] newargs[i] = '' elif check_flag('-shared'): @@ -3259,6 +3337,11 @@ def phase_binaryen(target, options, wasm_target): # the only reason we need intermediate debug info, we can stop keeping it if settings.ASYNCIFY: intermediate_debug_info -= 1 + # currently binaryen's DWARF support will limit some optimizations; warn on + # that. see https://github.com/emscripten-core/emscripten/issues/15269 + dwarf_info = settings.DEBUG_LEVEL >= 3 + if dwarf_info: + diagnostics.warning('limited-postlink-optimizations', 'running limited binaryen optimizations because DWARF info requested (or indirectly required)') building.run_wasm_opt(wasm_target, wasm_target, args=passes, @@ -3353,7 +3436,7 @@ def preprocess_wasm2js_script(): symbols_file=symbols_file, symbols_file_js=symbols_file_js) - shared.configuration.get_temp_files().note(wasm2js) + shared.get_temp_files().note(wasm2js) if settings.WASM == 2: safe_copy(wasm2js, wasm2js_template) @@ -3376,7 +3459,7 @@ def preprocess_wasm2js_script(): save_intermediate_with_wasm('symbolmap', wasm_target) if settings.DEBUG_LEVEL >= 3 and settings.SEPARATE_DWARF and os.path.exists(wasm_target): - building.emit_debug_on_side(wasm_target, settings.SEPARATE_DWARF) + building.emit_debug_on_side(wasm_target) if settings.WASM2C: wasm2c.do_wasm2c(wasm_target) @@ -3475,7 +3558,7 @@ def modularize(): exports["%(EXPORT_NAME)s"] = %(EXPORT_NAME)s; ''' % {'EXPORT_NAME': settings.EXPORT_NAME}) - shared.configuration.get_temp_files().note(final_js) + shared.get_temp_files().note(final_js) save_intermediate('modularized') @@ -3492,10 +3575,10 @@ def module_export_name_substitution(): src = re.sub(r'{\s*[\'"]?__EMSCRIPTEN_PRIVATE_MODULE_EXPORT_NAME_SUBSTITUTION__[\'"]?:\s*1\s*}', replacement, src) # For Node.js and other shell environments, create an unminified Module object so that # loading external .asm.js file that assigns to Module['asm'] works even when Closure is used. - if settings.MINIMAL_RUNTIME and (shared.target_environment_may_be('node') or shared.target_environment_may_be('shell')): + if settings.MINIMAL_RUNTIME and not settings.MODULARIZE and (shared.target_environment_may_be('node') or shared.target_environment_may_be('shell')): src = 'if(typeof Module==="undefined"){var Module={};}\n' + src write_file(final_js, src) - shared.configuration.get_temp_files().note(final_js) + shared.get_temp_files().note(final_js) save_intermediate('module_export_name_substitution') @@ -3837,7 +3920,7 @@ def parse_symbol_list_file(contents): return [v.strip() for v in values] -def parse_value(text, expect_list): +def parse_value(text, expected_type): # Note that using response files can introduce whitespace, if the file # has a newline at the end. For that reason, we rstrip() in relevant # places here. @@ -3892,7 +3975,7 @@ def parse_string_list(text): return [] return parse_string_list_members(text) - if expect_list or (text and text[0] == '['): + if expected_type == list or (text and text[0] == '['): # if json parsing fails, we fall back to our own parser, which can handle a few # simpler syntaxes try: @@ -3900,6 +3983,12 @@ def parse_string_list(text): except ValueError: return parse_string_list(text) + if expected_type == float: + try: + return float(text) + except ValueError: + pass + try: if text.startswith('0x'): base = 16 @@ -3914,13 +4003,13 @@ def validate_arg_level(level_string, max_level, err_msg, clamp=False): try: level = int(level_string) except ValueError: - raise Exception(err_msg) + exit_with_error(err_msg) if clamp: if level > max_level: logger.warning("optimization level '-O" + level_string + "' is not supported; using '-O" + str(max_level) + "' instead") level = max_level if not 0 <= level <= max_level: - raise Exception(err_msg) + exit_with_error(err_msg) return level diff --git a/eng/patches/emcmake.py b/eng/patches/emcmake.py deleted file mode 100644 index b16f565c1c8..00000000000 --- a/eng/patches/emcmake.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2016 The Emscripten Authors. All rights reserved. -# Emscripten is available under two separate licenses, the MIT license and the -# University of Illinois/NCSA Open Source License. Both these licenses can be -# found in the LICENSE file. - -import sys -from tools import shared -from tools import config -from tools import utils -from subprocess import CalledProcessError - - -# -# Main run() function -# -def run(): - if len(sys.argv) < 2 or sys.argv[1] in ('--version', '--help'): - print('''\ -emcmake is a helper for cmake, setting various environment -variables so that emcc etc. are used. Typical usage: - - emcmake cmake [FLAGS] -''', file=sys.stderr) - return 1 - - args = sys.argv[1:] - - def has_substr(args, substr): - return any(substr in s for s in args) - - # Append the Emscripten toolchain file if the user didn't specify one. - if not has_substr(args, '-DCMAKE_TOOLCHAIN_FILE'): - args.append('-DCMAKE_TOOLCHAIN_FILE=' + utils.path_from_root('cmake/Modules/Platform/Emscripten.cmake')) - - if not has_substr(args, '-DCMAKE_CROSSCOMPILING_EMULATOR'): - node_js = config.NODE_JS[0] - # In order to allow cmake to run code built with pthreads we need to pass some extra flags to node. - # Note that we also need --experimental-wasm-bulk-memory which is true by default and hence not added here - # See https://github.com/emscripten-core/emscripten/issues/15522 - args.append(f'-DCMAKE_CROSSCOMPILING_EMULATOR={node_js};--experimental-wasm-threads') - - # On Windows specify MinGW Makefiles or ninja if we have them and no other - # toolchain was specified, to keep CMake from pulling in a native Visual - # Studio, or Unix Makefiles. - if utils.WINDOWS and not any(arg.startswith('-G') for arg in args): - if utils.which('mingw32-make'): - args += ['-G', 'MinGW Makefiles'] - elif utils.which('ninja'): - args += ['-G', 'Ninja'] - else: - print('emcmake: no compatible cmake generator found; Please install ninja or mingw32-make, or specify a generator explicitly using -G', file=sys.stderr) - return 1 - - print('configure: ' + shared.shlex_join(args), file=sys.stderr) - try: - shared.check_call(args) - return 0 - except CalledProcessError as e: - return e.returncode - - -if __name__ == '__main__': - sys.exit(run())