[PATCH] R13(B02) 64-bit opensolaris support

[Paul Fisher]

I have finally had a chance to test 64-bit R13 in our environment and on 
opensolaris.  The --enable-m64-build option added in R13 has helped the 
size of the changes, but unfortunately I still need the patch at the end 
of this mail to build a fully working 64-bit environment.

When configuring/build like so:

$ CC=gcc LD=gcc CFLAGS=-O2 LIBS=-lmtmalloc \
   ./configure --enable-threads --enable-smp-support \
     --enable-kernel-poll --enable-hipe \
     --disable-megaco-flex-scanner-drvlineno --without-unixodbc \
     --enable-m64-build

The following issues exist:

1) --enable-m64-build disables hipe unconditionally
To my knowledge HiPE works on amd64 solaris 5.11, so disabling this no 
matter what seems wrong.

2) DED_LDFLAGS is set for old solaris ld and not gnu ld or gcc
The erts/configure(.in) should detect if gcc is being specified as the 
linker and set the DED_LDFLAGS appropriately, even though DED_LD gets 
set to $(CC) unconditionally.  Things need to be sensitive to the 
options differences.

3) erl_interface configure.in does not understand --enable-m64-build
The option needs to be added and the CFLAGS twiddled in the same way as 
erts/configure.in.

Here is the patch against R13B02 Aug 12 snapshot that I ended up with (I 
included the changes to the generated files too, just for convenience):

--- otp_src_R13B02/erts/configure	2009-08-11 13:01:22.000000000 -0500
+++ otp_src_R13B02/erts/configure.new	2009-08-13 15:54:04.243098019 -0500
@@ -2807,8 +2807,10 @@
     { (exit 1); exit 1; }; }
  	fi
  fi
-if test X${enable_darwin_64bit} = Xyes -o X${enable_m64_build} = Xyes; then
+if test X${enable_darwin_64bit} = Xyes; then
  	enable_hipe=no
+fi
+if test X${enable_darwin_64bit} = Xyes -o X${enable_m64_build} = Xyes; then
  	case $CFLAGS in
  		*-m64*)
  			;;
@@ -20897,9 +20899,16 @@
  		DED_LD_FLAG_RUNTIME_LIBRARY_PATH=
  	;;
  	solaris2*|sysv4*)
-		DED_LDFLAGS="-G"
-		if test X${enable_m64_build} = Xyes; then
-			DED_LDFLAGS="-64 $DED_LDFLAGS"
+		if test "x$GCC" = "xyes"; then
+			DED_LD=gcc
+			DED_LDFLAGS="-shared"
+			DED_LDFLAGS_64="-m64"
+		else
+			DED_LDFLAGS="-G"
+			DED_LDFLAGS_64="-64"
+		fi
+		if test "X${enable_m64_build}" = "Xyes"; then
+			DED_LDFLAGS="$DED_LDFLAGS_64 $DED_LDFLAGS"
  		fi
  	;;
  	aix4*)
--- otp_src_R13B02/erts/configure.in	2009-06-05 07:53:07.000000000 -0500
+++ otp_src_R13B02/erts/configure.in.new	2009-08-13 15:54:04.242383252 -0500
@@ -291,8 +291,10 @@
  		AC_MSG_ERROR([--enable-darwin-64bit only supported on x86 host])
  	fi
  fi
-if test X${enable_darwin_64bit} = Xyes -o X${enable_m64_build} = Xyes; then
+if test X${enable_darwin_64bit} = Xyes; then
  	enable_hipe=no
+fi
+if test X${enable_darwin_64bit} = Xyes -o X${enable_m64_build} = Xyes; then
  	case $CFLAGS in
  		*-m64*)
  			;;
@@ -2924,9 +2926,17 @@
  		DED_LD_FLAG_RUNTIME_LIBRARY_PATH=
  	;;
  	solaris2*|sysv4*)
-		DED_LDFLAGS="-G"
-		if test X${enable_m64_build} = Xyes; then
-			DED_LDFLAGS="-64 $DED_LDFLAGS"
+		if test "x$GCC" = "xyes"; then
+			DED_LD=gcc
+			DED_LDFLAGS="-shared"
+			DED_LDFLAGS_64="-m64"
+		
+		elif
+			DED_LDFLAGS="-G"
+			DED_LDFLAGS_64="-m64"
+		fi
+		if test "X${enable_m64_build}" = "Xyes"; then
+			DED_LDFLAGS="$DED_LDFLAGS_64 $DED_LDFLAGS"
  		fi
  	;;
  	aix4*)
--- otp_src_R13B02/lib/erl_interface/configure	2009-08-11 
13:01:14.000000000 -0500
+++ otp_src_R13B02/lib/erl_interface/configure.new	2009-08-14 
08:20:24.804342800 -0500
@@ -1440,6 +1440,27 @@
     threads_disabled=no
  fi;

+# Check whether --enable-m64-build or --disable-m64-build was given.
+if test "${enable_m64_build+set}" = set; then
+  enableval="$enable_m64_build"
+   case "$enableval" in
+    no) enable_m64_build=no ;;
+    *)  enable_m64_build=yes ;;
+  esac
+else
+   enable_m64_build=no
+fi;
+
+if test X${enable_m64_build} = Xyes; then
+	case $CFLAGS in
+		*-m64*)
+			;;
+		*)
+			CFLAGS="-m64 $CFLAGS"
+			;;
+	esac
+fi
+

  ac_ext=c
  ac_cpp='$CPP $CPPFLAGS'
--- otp_src_R13B02/lib/erl_interface/configure.in	2009-08-14 
08:24:02.056578225 -0500
+++ otp_src_R13B02/lib/erl_interface/configure.in.new	2009-08-14 
08:25:16.412178147 -0500
@@ -87,6 +87,24 @@
    esac ],
  [ threads_disabled=no ])

+AC_ARG_ENABLE(m64-build,
+[  --enable-m64-build    build 64bit binaries using the -m64 flag to 
(g)cc],
+[ case "$enableval" in
+    no) enable_m64_build=no ;;
+    *)  enable_m64_build=yes ;;
+  esac
+],enable_m64_build=no)
+
+if test X${enable_m64_build} = Xyes; then
+	case $CFLAGS in
+		*-m64*)
+			;;
+		*)
+			CFLAGS="-m64 $CFLAGS"
+			;;
+	esac
+fi
+
  dnl ----------------------------------------------------------------------
  dnl Checks for programs
  dnl ----------------------------------------------------------------------