diff --git a/dev/merge_spark_pr.py b/dev/merge_spark_pr.py
index 40a02cba82820..03f8fc28938e8 100755
--- a/dev/merge_spark_pr.py
+++ b/dev/merge_spark_pr.py
@@ -93,10 +93,11 @@ def merge_pr(pr_num, target_ref):
     '--pretty=format:%an <%ae>']).split("\n")
   distinct_authors = sorted(set(commit_authors), key=lambda x: commit_authors.count(x), reverse=True)
   primary_author = distinct_authors[0]
-  commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name]).split("\n\n")
+  commits = run_cmd(['git', 'log', 'HEAD..%s' % pr_branch_name, 
+    '--pretty=format:%h [%an] %s']).split("\n\n")
 
-  merge_message = "Merge pull request #%s from %s. Closes #%s.\n\n%s\n\n%s" % (
-    pr_num, pr_repo_desc, pr_num, title, body)
+  merge_message = "Merge pull request #%s from %s.\n\n%s\n\n%s" % (
+    pr_num, pr_repo_desc, title, body)
   merge_message_parts = merge_message.split("\n\n")
   merge_message_flags = []
 
@@ -104,7 +105,8 @@ def merge_pr(pr_num, target_ref):
     merge_message_flags = merge_message_flags + ["-m", p]
   authors = "\n".join(["Author: %s" % a for a in distinct_authors])
   merge_message_flags = merge_message_flags + ["-m", authors]
-  merge_message_flags = merge_message_flags + ["-m", "== Merge branch commits =="]
+  merge_message_flags = merge_message_flags + [
+    "-m", "Closes #%s and squashes the following commits:" % pr_num]
   for c in commits:
     merge_message_flags = merge_message_flags + ["-m", c]