Custom alloc, using stable Rust compilers, and more

.github/workflows/ci.yaml

@@ -13,7 +13,7 @@ jobs:
  arch: [arm, arm64, ppc64le, riscv64, x86_64]
  toolchain: [gcc, clang, llvm]
  config: [debug, release]
- rustc: [2021-05-29]
+ rustc: [2021-06-23]
  output: [src] # [src, build]
  install: [rustup] # [rustup, standalone]
  sysroot: [common] # [common, custom]
@@ -30,30 +30,30 @@ jobs:
 
  # A few independent combinations to avoid exploding the matrix:
  # - The other option for `output`.
- # - Different nightlies for `rustc`.
+ # - Different releases for `rustc`.
  # - The other three (`install`, `sysroot`) combinations
  # (they are interrelated, so the cross-product needs to be tested)
  include:
  - arch: arm64
  toolchain: gcc
  config: debug
- rustc: 2021-05-29
+ rustc: 2021-06-23
  output: build
  install: rustup
  sysroot: custom
 
  - arch: ppc64le
  toolchain: clang
  config: release
- rustc: 2021-05-29
+ rustc: 2021-06-23
  output: build
  install: standalone
  sysroot: common
 
  - arch: x86_64
  toolchain: llvm
  config: debug
- rustc: 2021-05-29
+ rustc: 2021-06-23
  output: build
  install: standalone
  sysroot: custom
@@ -178,12 +178,14 @@ jobs:
  # Setup: rustc
  - if: matrix.install == 'rustup'
  run: |
- rustup default nightly-${{ matrix.rustc }}
+ rustup default beta-${{ matrix.rustc }}
  rustup component add rustfmt
  - if: matrix.install == 'standalone'
  run: |
- curl https://static.rust-lang.org/dist/${{ matrix.rustc }}/rust-nightly-x86_64-unknown-linux-gnu.tar.gz | tar xz
- rust-nightly-x86_64-unknown-linux-gnu/install.sh --without=rust-docs --prefix=$HOME/rustc
+ # FIXME: there is no beta equivalent for https://static.rust-lang.org/dist/channel-rust-x.yy.z.toml,
+ # so just hardcode the URL/date for the moment, since we will moving to a stable release soon anyway.
+ curl https://static.rust-lang.org/dist/${{ matrix.rustc }}/rust-beta-x86_64-unknown-linux-gnu.tar.gz | tar xz
+ rust-beta-x86_64-unknown-linux-gnu/install.sh --without=rust-docs --prefix=$HOME/rustc
  echo $HOME/rustc/bin >> $GITHUB_PATH
 
  # Setup: rustc native libs
@@ -402,7 +404,7 @@ jobs:
  - run: make ${{ env.MAKE_ARCH }} ${{ env.MAKE_CROSS_COMPILE }} ${{ env.MAKE_TOOLCHAIN }} ${{ env.MAKE_OUTPUT }} ${{ env.MAKE_SYSROOT }} -j3 rusttest
 
  # Formatting
- - run: make rustfmtcheck
+ - run: make ${{ env.MAKE_ARCH }} ${{ env.MAKE_CROSS_COMPILE }} ${{ env.MAKE_TOOLCHAIN }} ${{ env.MAKE_OUTPUT }} ${{ env.MAKE_SYSROOT }} -j3 rustfmtcheck
 
  # View changes to ccache
  - run: ccache -s

.rustfmt.toml

@@ -1,11 +1,11 @@
 edition = "2018"
-format_code_in_doc_comments = true
 newline_style = "Unix"
 
 # Unstable options that help catching some mistakes in formatting and that we may want to enable
 # when they become stable.
 #
 # They are kept here since they are useful to run from time to time.
+#format_code_in_doc_comments = true
 #reorder_impl_items = true
 #comment_width = 100
 #wrap_comments = true

Documentation/process/changes.rst

@@ -26,12 +26,17 @@ running a Linux kernel. Also, not all tools are necessary on all
 systems; obviously, if you don't have any PC Card hardware, for example,
 you probably needn't concern yourself with pcmciautils.
 
+Furthermore, note that newer versions of the Rust toolchain may or may not work
+because, for the moment, we depend on some unstable features. Thus, unless you
+know what you are doing, use the exact version listed here. Please see
+:ref:`Documentation/rust/quick-start.rst <rust_quick_start>` for details.
+
 ====================== =============== ========================================
  Program Minimal version Command to check the version
 ====================== =============== ========================================
 GNU C 4.9 gcc --version
 Clang/LLVM (optional) 10.0.1 clang --version
-rustc (optional) nightly  rustc --version
+rustc (optional) 1.54.0-beta.1 rustc --version
 bindgen (optional) 0.56.0 bindgen --version
 GNU make 3.81 make --version
 binutils 2.23 ld -v
@@ -57,7 +62,6 @@ iptables 1.4.2 iptables -V
 openssl & libcrypto 1.0.0 openssl version
 bc 1.06.95 bc --version
 Sphinx\ [#f1]_ 1.3 sphinx-build --version
-rustdoc (optional) nightly rustdoc --version
 ====================== =============== ========================================
 
 .. [#f1] Sphinx is needed only to build the Kernel documentation

Documentation/rust/quick-start.rst

@@ -23,13 +23,13 @@ at the time of writing, they are likely to not be recent enough.
 rustc
 *****
 
-A recent *nightly* Rust toolchain (with, at least, ``rustc``) is required,
-e.g. ``nightly-2021-05-29``. Our goal is to use a stable toolchain as soon
-as possible, but for the moment we depend on a handful of nightly features.
+A particular version (`1.54.0-beta.1`) of the Rust compiler is required.
+Newer versions may or may not work because, for the moment, we depend on
+some unstable Rust features.
 
 If you are using ``rustup``, run::
 
- rustup default nightly-2021-05-29
+ rustup default beta-2021-06-23
 
 Otherwise, fetch a standalone installer or install ``rustup`` from:
 
@@ -125,16 +125,30 @@ the component manually::
 The standalone installers also come with ``clippy``.
 
 
+cargo
+*****
+
+``cargo`` is the Rust native build system. It is currently required to run
+the tests (``rusttest`` target) since we use it to build a custom standard
+library that contains the facilities provided by our custom ``alloc``.
+
+If you are using ``rustup``, all the profiles already install the tool,
+so you should be good to go. The standalone installers also include ``cargo``.
+
+
 rustdoc
 *******
 
-If you install the ``rustdoc`` tool, then you will be able to generate pretty
-HTML documentation for Rust code, including for the libraries (crates) inside
-``rust/`` that are used by the rest of the kernel (for details, please see
-:ref:`Documentation/rust/docs.rst <rust_docs>`).
+``rustdoc`` is the documentation tool for Rust. It generates pretty HTML
+documentation for Rust code (for details, please see
+:ref:`Documentation/rust/docs.rst <rust_docs>`.
+
+``rustdoc`` is also able to test the examples provided in documented Rust code
+(called doctests or documentation tests). We use this feature, thus ``rustdoc``
+is required to run the tests (``rusttest`` target).
 
 If you are using ``rustup``, all the profiles already install the tool,
-so you should be good to go. The standalone installers also come with ``rustdoc``.
+so you should be good to go. The standalone installers also include ``rustdoc``.
 
 
 rust-analyzer

Makefile

@@ -460,9 +460,12 @@ READELF = $(CROSS_COMPILE)readelf
 STRIP = $(CROSS_COMPILE)strip
 endif
 RUSTC = rustc
+RUSTC_BOOTSTRAP = 1
+RUSTDOC = rustdoc
 RUSTFMT = rustfmt
 CLIPPY_DRIVER	= clippy-driver
 BINDGEN = bindgen
+CARGO = cargo
 PAHOLE = pahole
 RESOLVE_BTFIDS	= $(objtree)/tools/bpf/resolve_btfids/resolve_btfids
 LEX = flex
@@ -524,7 +527,7 @@ KBUILD_RUSTCFLAGS := --emit=dep-info,obj,metadata --edition=2018 \
  -Cpanic=abort -Cembed-bitcode=n -Clto=n -Crpath=n \
  -Cforce-unwind-tables=n -Ccodegen-units=1 \
  -Zbinary_dep_depinfo=y -Zsymbol-mangling-version=v0 \
- -D unsafe_op_in_unsafe_fn
+ -Dunsafe_op_in_unsafe_fn -Wmissing_docs
 KBUILD_AFLAGS_KERNEL :=
 KBUILD_CFLAGS_KERNEL :=
 KBUILD_RUSTCFLAGS_KERNEL :=
@@ -542,13 +545,13 @@ else
 	RUSTC_OR_CLIPPY_QUIET := RUSTC
 	RUSTC_OR_CLIPPY = $(RUSTC)
 endif
-export RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY
 
 ifdef RUST_LIB_SRC
 	export RUST_LIB_SRC
 endif
 
-export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC RUSTC BINDGEN
+export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
+export RUSTC RUSTC_BOOTSTRAP RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN CARGO
 export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL
 export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
 export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD
@@ -1596,7 +1599,7 @@ endif # CONFIG_MODULES
 # Directories & files removed with 'make clean'
 CLEAN_FILES += include/ksym vmlinux.symvers modules-only.symvers \
  modules.builtin modules.builtin.modinfo modules.nsdeps \
- compile_commands.json .thinlto-cache
+ compile_commands.json .thinlto-cache rust/test rust/doc
 
 # Directories & files removed with 'make mrproper'
 MRPROPER_FILES += include/config include/generated \
@@ -1726,7 +1729,7 @@ help:
 	@echo ' rustdoc - Generate Rust documentation'
 	@echo ' (requires kernel .config)'
 	@echo ' rusttest - Runs the Rust tests'
-	@echo ' (requires kernel .config)'
+	@echo ' (requires kernel .config; downloads external repos)'
 	@echo ' rust-analyzer - Generate rust-project.json rust-analyzer support file'
 	@echo ' (requires kernel .config)'
 	@echo ''
@@ -1820,11 +1823,21 @@ rusttest: prepare0
 # Formatting targets
 PHONY += rustfmt rustfmtcheck
 
+# We skip `rust/alloc` since we want to minimize the diff w.r.t. upstream.
+#
+# We match using absolute paths since `find` does not resolve them
+# when matching, which is a problem when e.g. `srctree` is `..`.
+# We `grep` afterwards in order to remove the directory entry itself.
 rustfmt:
-	find $(srctree) -type f -name '*.rs' | xargs $(RUSTFMT)
-
-rustfmtcheck:
-	find $(srctree) -type f -name '*.rs' | xargs $(RUSTFMT) --check
+	$(Q)find $(abs_srctree) -type f -name '*.rs' \
+ -o -path $(abs_srctree)/rust/alloc -prune \
+ -o -path $(abs_objtree)/rust/test -prune \
+ | grep -Fv $(abs_srctree)/rust/alloc \
+ | grep -Fv $(abs_objtree)/rust/test \
+ | xargs $(RUSTFMT) $(rustfmt_flags)
+
+rustfmtcheck: rustfmt_flags = --check
+rustfmtcheck: rustfmt
 
 # IDE support targets
 PHONY += rust-analyzer

drivers/android/process.rs

@@ -614,12 +614,11 @@ impl Process {
  for _ in 0..page_count {
  let page = Pages::<0>::new()?;
  page.insert_page(vma, address)?;
- pages.push(page);
+ pages.try_push(page)?;
  address += 1 << bindings::PAGE_SHIFT;
  }
 
- // TODO: This allocates memory.
- let arc = Arc::from(pages);
+ let arc = Arc::try_from_vec(pages)?;
 
  // Save pages for later.
  let mut inner = self.inner.lock();