diff --git a/site/src/github.rs b/site/src/github.rs
index b41569570..c606f1e75 100644
--- a/site/src/github.rs
+++ b/site/src/github.rs
@@ -29,30 +29,63 @@ pub async fn unroll_rollup(
     previous_master: &str,
     rollup_pr_number: u32,
 ) -> Result<(), String> {
+    let commit_link = |sha: &str| format!("https://github.com/rust-lang-ci/rust/commit/{sha}");
+
     let format_commit = |s: &str, truncate: bool| {
         let display = truncate.then(|| s.split_at(10).0).unwrap_or(s);
-        format!("[{display}](https://github.com/rust-lang-ci/rust/commit/{s})")
+        format!("[{display}]({})", commit_link(s))
     };
-    let mapping = enqueue_unrolled_try_builds(ci_client, rollup_merges, previous_master)
-        .await?
+
+    // Sort rolled up commits by their PR number in ascending order, so that they have the
+    // same ordering as in the rollup PR description.
+    let mut unrolled_builds: Vec<UnrolledCommit> =
+        enqueue_unrolled_try_builds(ci_client, rollup_merges, previous_master).await?;
+    // The number should really be an integer, but if not, we will just sort the "non-integer" PRs
+    // first.
+    unrolled_builds.sort_by_cached_key(|commit| commit.original_pr_number.parse::<u64>().ok());
+
+    let mapping = unrolled_builds
         .into_iter()
         .fold(String::new(), |mut string, c| {
             use std::fmt::Write;
             let commit = c
                 .sha
                 .as_deref()
-                .map(|s| format_commit(s, false))
+                .map(|s| {
+                    // Format the SHA as a code block to make it easy to copy-paste verbatim
+                    let link = commit_link(s);
+                    format!("`{s}` ([link]({link}))")
+                })
                 .unwrap_or_else(|| {
                     let head = format_commit(&c.rolled_up_head, true);
                     format!("❌ conflicts merging '{head}' into previous master ❌")
                 });
-            writeln!(&mut string, "|#{pr}|{commit}|", pr = c.original_pr_number).unwrap();
+            let message = c
+                .rollup_merge
+                .message
+                .split('\n')
+                // Skip over "Rollup merge of ..." and an empty line
+                .nth(2)
+                .map(|m| {
+                    if m.len() <= 60 {
+                        m.to_string()
+                    } else {
+                        format!("{}…", m.split_at(59).0)
+                    }
+                })
+                .unwrap_or_else(|| format!("#{}", c.original_pr_number));
+            writeln!(
+                &mut string,
+                "|#{pr}|{message}|{commit}|",
+                pr = c.original_pr_number
+            )
+            .unwrap();
             string
         });
     let previous_master = format_commit(previous_master, true);
     let msg =
         format!("📌 Perf builds for each rolled up PR:\n\n\
-        |PR# | Perf Build Sha|\n|----|:-----:|\n\
+        |PR# | Message | Perf Build Sha|\n|----|:-----:|\n\
         {mapping}\n\n*previous master*: {previous_master}\n\nIn the case of a perf regression, \
         run the following command for each PR you suspect might be the cause: `@rust-timer build $SHA`\n\
         {COMMENT_MARK_ROLLUP}");
@@ -108,7 +141,7 @@ async fn enqueue_unrolled_try_builds<'a>(
             .merge_branch(
                 "perf-tmp",
                 &rolled_up_head,
-                &format!("Unrolled build for #{original_pr_number}"),
+                &format!("Unrolled build for #{original_pr_number}\n{}", rollup_merge.message),
             )
             .await
             .map_err(|e| {