Skip to content

Fix cross-compilation issues for aws-lc-sys #689

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Dec 1, 2022
Merged

Fix cross-compilation issues for aws-lc-sys #689

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a84d75b
move binding generation out of aws-lc-sys
samuel40791765 Nov 18, 2022
10bc306
backport cmake dependency removal and add option removals
samuel40791765 Nov 22, 2022
28cd662
update Readme for aws-lc-sys crate generation
samuel40791765 Nov 23, 2022
3d712c9
leverage bindings generation in aws-lc-sys for pre-generation
samuel40791765 Nov 29, 2022
daff023
warn when prebuilt bindings aren't available
samuel40791765 Nov 30, 2022
bc1e095
apply cfg_aliases and clear up feature names
samuel40791765 Nov 30, 2022
91e1db4
small fixes for internal_generation
samuel40791765 Dec 1, 2022
aba6f1d
Merge branch 'main' into pre-generate
justsmth Dec 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion bindings/rust/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,23 @@ Low-level AWS-LC bindings crate for Rust.
### How it works
`aws-lc-sys` uses `bindgen` to generate Rust compatibility shims for the targeted platform. It is important to generate it for the correct platform because `bindgen` uses LLVM information for alignment which varies depending on architecture.

### Installation

In order to generate the `aws-lc-sys` crate, you need to have the following installed:

* Rust - installable with [rustup](https://rustup.rs/)
* libclang
* docker
* C compilation tools to build AWS-LC. We pull in [generated-src](https://github.com/awslabs/aws-lc/tree/main/generated-src) into `aws-lc-sys`, so Go and Perl aren't needed (for non-FIPS).
* CMake3 or above. [`cmake-rs`](https://docs.rs/cmake/latest/cmake/) appends build options after the path, which isn't supported in older versions of cmake.

### To Use
The `aws-lc-sys` create can be built by running the [generate.sh](./generate/generate.sh) script.
The script requires the [docker images](../../tests/ci/docker_images/rust) to
be [built](../../tests/ci/docker_images/rust/build_images.sh) and available locally.
be [built](../../tests/ci/docker_images/rust/build_images.sh) and available locally. Bindings for `macos-x86_64` are generated on the native MacOS x86_64 platform. Bindings are generated for `linux-x86_64`, `linux-aarch64`, `linux-x86` using the corresponding docker images.

```
./bindings/rust/generate/generate.sh
```

See AWS-LC build documentation for more details: https://github.com/awslabs/aws-lc/blob/main/BUILDING.md
10 changes: 7 additions & 3 deletions bindings/rust/aws-lc-sys-template/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ version = "__AWS_LC_SYS_VERSION__"
authors = ["AWS-LC"]
edition = "2021"
repository = "https://github.com/awslabs/aws-lc"
license = "Apache-2.0"
license = "Apache-2.0 OR ISC"
rust-version = "1.57.0"
include = [
"build.rs",
Expand All @@ -25,14 +25,18 @@ include = [

[features]
asan = []
generate_bindings = ["bindgen"] # Generate the bindings on the targetted platform as a fallback mechanism.
internal_generate = ["bindgen"] # Only for internal use, this line is removed prior to publishing.

[build-dependencies]
bindgen = "0.61"
cmake = "0.1.48"
bindgen = { version = "0.61", optional = true }
regex = "1"
dunce = "1.0"
cfg_aliases = "0.1.1"

[dependencies]
libc = "0.2"

[dev-dependencies]
openssl = { version = "0.10", features = ["vendored"] }
openssl = { version = "0.10", features = ["vendored"] }
247 changes: 67 additions & 180 deletions bindings/rust/aws-lc-sys-template/build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,99 +15,19 @@
// SPDX-License-Identifier: Apache-2.0 OR ISC
// Modifications Copyright Amazon.com, Inc. or its affiliates. See GitHub history for details.

use bindgen::callbacks::ParseCallbacks;
use std::ffi::{OsStr, OsString};
use std::fmt::Debug;
use cfg_aliases::cfg_aliases;
use std::ffi::OsStr;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::{env, fs};

#[derive(Debug)]
struct StripPrefixCallback {
remove_prefix: Option<String>,
}

impl StripPrefixCallback {
fn new(prefix: &str) -> StripPrefixCallback {
StripPrefixCallback {
remove_prefix: Some(prefix.to_string()),
}
}
}

impl ParseCallbacks for StripPrefixCallback {
fn generated_name_override(&self, name: &str) -> Option<String> {
self.remove_prefix.as_ref().and_then(|s| {
let prefix = format!("{}_", s);
name.strip_prefix(prefix.as_str()).map(String::from)
})
}
}
#[cfg(feature = "bindgen")]
mod build_bindgen;

fn get_include_path(manifest_dir: &Path) -> PathBuf {
pub(crate) fn get_include_path(manifest_dir: &Path) -> PathBuf {
manifest_dir.join("deps").join("aws-lc").join("include")
}

fn prepare_clang_args(manifest_dir: &Path, build_prefix: Option<&str>) -> Vec<String> {
let mut clang_args: Vec<String> = vec![
"-I".to_string(),
get_include_path(manifest_dir).display().to_string(),
];

if let Some(prefix) = build_prefix {
clang_args.push(format!("-DBORINGSSL_PREFIX={}", prefix));
}

clang_args
}

const COPYRIGHT: &str = r#"
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0 OR ISC
"#;

const PRELUDE: &str = r#"
#![allow(unused_imports, non_camel_case_types, non_snake_case, non_upper_case_globals, improper_ctypes)]
"#;

fn prepare_bindings_builder(manifest_dir: &Path, build_prefix: Option<&str>) -> bindgen::Builder {
let clang_args = prepare_clang_args(manifest_dir, build_prefix);

let mut builder = bindgen::Builder::default()
.derive_copy(true)
.derive_debug(true)
.derive_default(true)
.derive_eq(true)
.allowlist_file(".*/openssl/[^/]+\\.h")
.allowlist_file(".*/rust_wrapper\\.h")
.default_enum_style(bindgen::EnumVariation::NewType {
is_bitfield: false,
is_global: false,
})
.default_macro_constant_type(bindgen::MacroTypeVariation::Signed)
.generate_comments(true)
.fit_macro_constants(false)
.size_t_is_usize(true)
.layout_tests(true)
.prepend_enum_name(true)
.rustfmt_bindings(true)
.clang_args(clang_args)
.raw_line(COPYRIGHT)
.raw_line(PRELUDE)
.header(
get_include_path(manifest_dir)
.join("rust_wrapper.h")
.display()
.to_string(),
);

if let Some(ps) = build_prefix {
builder = builder.parse_callbacks(Box::new(StripPrefixCallback::new(ps)));
}

builder
}

const AWS_LC_PATH: &str = "deps/aws-lc";

#[allow(dead_code)]
Expand Down Expand Up @@ -179,6 +99,15 @@ fn prefix_string() -> String {
format!("aws_lc_{}", VERSION.to_string().replace('.', "_"))
}

#[cfg(feature = "internal_generate")]
fn target_platform_bindings_string() -> String {
format!(
"{}_{}_bindings.rs",
std::env::consts::OS,
std::env::consts::ARCH
)
}

fn test_command(executable: &OsStr, args: &[&OsStr]) -> bool {
if let Ok(output) = Command::new(executable).args(args).output() {
return output.status.success();
Expand All @@ -196,114 +125,70 @@ fn find_cmake_command() -> Option<&'static OsStr> {
}
}

fn prepare_cmake_args(
aws_lc_path: &Path,
build_prefix: Option<&str>,
) -> Result<Vec<OsString>, &'static str> {
let mut args: Vec<OsString> = vec![
OsString::from("-DBUILD_TESTING=OFF"),
OsString::from("-DBUILD_LIBSSL=OFF"),
OsString::from("-DDISABLE_PERL=ON"),
OsString::from("-DDISABLE_GO=ON"),
aws_lc_path.as_os_str().into(),
];
fn get_cmake_config() -> cmake::Config {
let pwd = env::current_dir().unwrap();

cmake::Config::new(pwd.join(AWS_LC_PATH))
}

fn prepare_cmake_build(build_prefix: Option<&str>) -> cmake::Config {
let mut cmake_cfg = get_cmake_config();

let opt_level = env::var("OPT_LEVEL").unwrap_or_else(|_| "0".to_string());
if opt_level.eq("0") {
args.push(OsString::from("-DCMAKE_BUILD_TYPE=debug"));
} else if opt_level.eq("1") || opt_level.eq("2") {
args.push(OsString::from("-DCMAKE_BUILD_TYPE=relwithdebinfo"));
} else {
args.push(OsString::from("-DCMAKE_BUILD_TYPE=release"));
if opt_level.ne("0") {
if opt_level.eq("1") || opt_level.eq("2") {
cmake_cfg.define("CMAKE_BUILD_TYPE", "relwithdebinfo");
} else {
cmake_cfg.define("CMAKE_BUILD_TYPE", "release");
}
}

if let Some(symbol_prefix) = build_prefix {
args.push(OsString::from(format!(
"-DBORINGSSL_PREFIX={}",
symbol_prefix
)));
let include_path = aws_lc_path.join("include");
args.push(OsString::from(format!(
"-DBORINGSSL_PREFIX_HEADERS={}",
include_path.display()
)));
cmake_cfg.define("BORINGSSL_PREFIX", symbol_prefix);
let pwd = env::current_dir().unwrap();
let include_path = pwd.join(AWS_LC_PATH).join("include");
cmake_cfg.define(
"BORINGSSL_PREFIX_HEADERS",
include_path.display().to_string(),
);
}

// Build flags that minimize our crate size.
cmake_cfg.define("BUILD_TESTING", "OFF");
cmake_cfg.define("BUILD_LIBSSL", "OFF");
// Build flags that minimize our dependencies.
cmake_cfg.define("DISABLE_PERL", "ON");
cmake_cfg.define("DISABLE_GO", "ON");

if cfg!(feature = "asan") {
env::set_var("CC", "/usr/bin/clang");
env::set_var("CXX", "/usr/bin/clang++");
env::set_var("ASM", "/usr/bin/clang");

args.push(OsString::from("-DASAN=1"));
cmake_cfg.define("ASAN", "1");
}

Ok(args)
}

fn load_env_var(var_name: &str) -> String {
env::var(var_name).unwrap_or_else(|_| panic!("Missing Environment variable: {}", var_name))
cmake_cfg
}

fn build_aws_lc() -> Result<PathBuf, &'static str> {
let cmake_cmd: &OsStr;
if let Some(cmd) = find_cmake_command() {
// This can be a let-else as-of Rust 1.65
cmake_cmd = cmd;
} else {
return Err("Missing dependency: cmake");
};

let pwd = env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
let aws_lc_path = pwd.join(AWS_LC_PATH);

let out_dir = PathBuf::from(load_env_var("OUT_DIR"));
let build_dir = out_dir.join("build");
if !build_dir.exists() {
fs::create_dir_all(build_dir.clone()).unwrap();
}
fn build_aws_lc() -> PathBuf {
let mut cmake_cfg = prepare_cmake_build(Some(&prefix_string()));

let mut setup_command = Command::new(cmake_cmd);

setup_command.args(prepare_cmake_args(&aws_lc_path, Some(&prefix_string()))?);
setup_command.current_dir(&build_dir);
println!("running: {:?}", setup_command);
let output = setup_command
.output()
.map_err(|_| "cmake setup command failed")?;

if !output.status.success() {
eprintln!("{}", String::from_utf8(output.stderr).unwrap());
return Err("cmake setup command unsuccessful");
}

let mut build_command = Command::new(cmake_cmd);
build_command.args(&[
OsString::from("--build"),
OsString::from("."),
OsString::from("--target"),
OsString::from("crypto"),
]);
build_command.current_dir(&build_dir);

println!("running: {:?}", build_command);
let output = build_command
.output()
.map_err(|_| "cmake build command failed")?;
if !output.status.success() {
eprintln!("-- stdout --");
eprintln!("{}", String::from_utf8(output.stdout).unwrap());
eprintln!("-- stderr --");
eprintln!("{}", String::from_utf8(output.stderr).unwrap());
return Err("cmake build command unsuccessful.");
}

Ok(out_dir)
cmake_cfg.build_target("crypto").build()
}

fn main() -> Result<(), String> {
use crate::OutputLib::Crypto;
use crate::OutputLibType::Static;

cfg_aliases! {
linux_x86_bindings: { all(not(feature = "bindgen"), target_os = "linux", target_arch = "x86") },
linux_x86_64_bindings: { all(not(feature = "bindgen"), target_os = "linux", target_arch = "x86_64") },
linux_aarch64_bindings: { all(not(feature = "bindgen"), target_os = "linux", target_arch = "aarch64") },
macos_x86_64_bindings: { all(not(feature = "bindgen"), target_os = "macos", target_arch = "x86_64") },
not_pregenerated: { not(any(linux_x86_bindings, linux_aarch64_bindings, linux_x86_64_bindings, macos_x86_64_bindings)) },
}

let mut missing_dependency = false;

if let Some(cmake_cmd) = find_cmake_command() {
Expand All @@ -321,16 +206,18 @@ fn main() -> Result<(), String> {
let manifest_dir = dunce::canonicalize(Path::new(&manifest_dir)).unwrap();
let prefix = prefix_string();

let bindings_file = manifest_dir.join("src").join("bindings.rs");

let builder = prepare_bindings_builder(&manifest_dir, Some(&prefix));
let bindings = builder.generate().expect("Unable to generate bindings.");
bindings
.write_to_file(bindings_file)
.expect("Unable to write bindings to file.");

let aws_lc_dir = build_aws_lc()?;

#[cfg(any(feature = "bindgen", not_pregenerated))]
build_bindgen::generate_bindings(&manifest_dir, Some(&prefix), "bindings.rs")
.expect("Unable to generate bindings.");
#[cfg(feature = "internal_generate")]
build_bindgen::generate_bindings(
&manifest_dir,
Some(&prefix),
&target_platform_bindings_string().to_string(),
)
.expect("Unable to generate bindings.");

let aws_lc_dir = build_aws_lc();
let lib_file = Crypto.locate_file(&aws_lc_dir, Static, None);
let prefixed_lib_file = Crypto.locate_file(&aws_lc_dir, Static, Some(&prefix));
fs::rename(lib_file, prefixed_lib_file).expect("Unexpected error: Library not found");
Expand Down
Loading