rustc: Allow changing the default allocator

This commit is an implementation of [RFC 1183][rfc] which allows swapping out
the default allocator on nightly Rust. No new stable surface area should be
added as a part of this commit.

[rfc]: rust-lang/rfcs#1183

Two new attributes have been added to the compiler:

* `#![needs_allocator]` - this is used by liballoc (and likely only liballoc) to
  indicate that it requires an allocator crate to be in scope.
* `#![allocator]` - this is a indicator that the crate is an allocator which can
  satisfy the `needs_allocator` attribute above.

The ABI of the allocator crate is defined to be a set of symbols that implement
the standard Rust allocation/deallocation functions. The symbols are not
currently checked for exhaustiveness or typechecked. There are also a number of
restrictions on these crates:

* An allocator crate cannot transitively depend on a crate that is flagged as
  needing an allocator (e.g. allocator crates can't depend on liballoc).
* There can only be one explicitly linked allocator in a final image.
* If no allocator is explicitly requested one will be injected on behalf of the
  compiler. Binaries and Rust dylibs will use jemalloc by default where
  available and staticlibs/other dylibs will use the system allocator by
  default.

Two allocators are provided by the distribution by default, `alloc_system` and
`alloc_jemalloc` which operate as advertised.

Closes rust-lang#27389

Author: alexcrichton

mk/crates.mk

@@ -52,23 +52,23 @@
 TARGET_CRATES := libc std flate arena term \
                  serialize getopts collections test rand \
                  log graphviz core rbml alloc \
-                 rustc_unicode rustc_bitflags
+                 rustc_unicode rustc_bitflags \
+		 alloc_system
 RUSTC_CRATES := rustc rustc_typeck rustc_borrowck rustc_resolve rustc_driver \
                 rustc_trans rustc_back rustc_llvm rustc_privacy rustc_lint \
                 rustc_data_structures
 HOST_CRATES := syntax $(RUSTC_CRATES) rustdoc fmt_macros
-CRATES := $(TARGET_CRATES) $(HOST_CRATES)
 TOOLS := compiletest rustdoc rustc rustbook error-index-generator
 
 DEPS_core :=
 DEPS_libc := core
 DEPS_rustc_unicode := core
-DEPS_alloc := core libc native:jemalloc
+DEPS_alloc := core libc alloc_system
 DEPS_std := core libc rand alloc collections rustc_unicode \
 	native:rust_builtin native:backtrace \
-	rustc_bitflags
+	alloc_system
 DEPS_graphviz := std
-DEPS_syntax := std term serialize log fmt_macros arena libc
+DEPS_syntax := std term serialize log fmt_macros arena libc rustc_bitflags
 DEPS_rustc_driver := arena flate getopts graphviz libc rustc rustc_back rustc_borrowck \
                      rustc_typeck rustc_resolve log syntax serialize rustc_llvm \
 		     rustc_trans rustc_privacy rustc_lint
@@ -82,7 +82,7 @@ DEPS_rustc_privacy := rustc log syntax
 DEPS_rustc_lint := rustc log syntax
 DEPS_rustc := syntax flate arena serialize getopts rbml \
               log graphviz rustc_llvm rustc_back rustc_data_structures
-DEPS_rustc_llvm := native:rustllvm libc std
+DEPS_rustc_llvm := native:rustllvm libc std rustc_bitflags
 DEPS_rustc_back := std syntax rustc_llvm flate log libc
 DEPS_rustc_data_structures := std log serialize
 DEPS_rustdoc := rustc rustc_driver native:hoedown serialize getopts \
@@ -102,6 +102,7 @@ DEPS_test := std getopts serialize rbml term native:rust_test_helpers
 DEPS_rand := core
 DEPS_log := std
 DEPS_fmt_macros = std
+DEPS_alloc_system := core libc
 
 TOOL_DEPS_compiletest := test getopts
 TOOL_DEPS_rustdoc := rustdoc
@@ -121,14 +122,26 @@ ONLY_RLIB_rand := 1
 ONLY_RLIB_collections := 1
 ONLY_RLIB_rustc_unicode := 1
 ONLY_RLIB_rustc_bitflags := 1
+ONLY_RLIB_alloc_system := 1
 
 # Documented-by-default crates
 DOC_CRATES := std alloc collections core libc rustc_unicode
 
+ifeq ($(CFG_DISABLE_JEMALLOC),)
+TARGET_CRATES += alloc_jemalloc
+DEPS_std += alloc_jemalloc
+DEPS_alloc_jemalloc := core libc native:jemalloc
+ONLY_RLIB_alloc_jemalloc := 1
+else
+RUSTFLAGS_rustc_back := --cfg disable_jemalloc
+endif
+
 ################################################################################
 # You should not need to edit below this line
 ################################################################################
 
+CRATES := $(TARGET_CRATES) $(HOST_CRATES)
+
 # This macro creates some simple definitions for each crate being built, just
 # some munging of all of the parameters above.
 #

mk/rt.mk

@@ -184,8 +184,6 @@ $$(JEMALLOC_LOCAL_$(1)): $$(JEMALLOC_DEPS) $$(MKFILE_DEPS)
 		EXTRA_CFLAGS="-g1 -ffunction-sections -fdata-sections"
 	$$(Q)$$(MAKE) -C "$$(JEMALLOC_BUILD_DIR_$(1))" build_lib_static
 
-ifeq ($$(CFG_DISABLE_JEMALLOC),)
-RUSTFLAGS_alloc := --cfg jemalloc
 ifeq ($(1),$$(CFG_BUILD))
 ifneq ($$(CFG_JEMALLOC_ROOT),)
 $$(JEMALLOC_LIB_$(1)): $$(CFG_JEMALLOC_ROOT)/libjemalloc_pic.a
@@ -199,10 +197,6 @@ else
 $$(JEMALLOC_LIB_$(1)): $$(JEMALLOC_LOCAL_$(1))
 	$$(Q)cp $$< $$@
 endif
-else
-$$(JEMALLOC_LIB_$(1)): $$(MKFILE_DEPS)
-	$$(Q)touch $$@
-endif
 
 ################################################################################
 # compiler-rt

mk/tests.mk

@@ -22,7 +22,8 @@ $(eval $(call RUST_CRATE,coretest))
 DEPS_collectionstest :=
 $(eval $(call RUST_CRATE,collectionstest))
 
-TEST_TARGET_CRATES = $(filter-out core rustc_unicode,$(TARGET_CRATES)) \
+TEST_TARGET_CRATES = $(filter-out core rustc_unicode alloc_system \
+		     		  alloc_jemalloc,$(TARGET_CRATES)) \
 			collectionstest coretest
 TEST_DOC_CRATES = $(DOC_CRATES)
 TEST_HOST_CRATES = $(filter-out rustc_typeck rustc_borrowck rustc_resolve \