Prefer build-time endianness detection (libarchive#2464)

Endianness is easy to determine at runtime, but detecting this a single
time and then reusing the cached result might require API changes.

However we can use compile-time detection for some known compiler macros
without API changes fairly easily. Let's start by enabling this for
Clang and GCC.

Author: mostynb

libarchive/archive_string.c

@@ -154,7 +154,6 @@ static int archive_wstring_append_from_mbs_in_codepage(
     struct archive_string_conv *);
 static int archive_string_append_from_wcs_in_codepage(struct archive_string *,
     const wchar_t *, size_t, struct archive_string_conv *);
-static int is_big_endian(void);
 static int strncat_in_codepage(struct archive_string *, const void *,
     size_t, struct archive_string_conv *);
 static int win_strncat_from_utf16be(struct archive_string *, const void *,
@@ -199,6 +198,27 @@ static int archive_string_normalize_D(struct archive_string *, const void *,
 static int archive_string_append_unicode(struct archive_string *,
     const void *, size_t, struct archive_string_conv *);
 
+#if defined __LITTLE_ENDIAN__
+  #define IS_BIG_ENDIAN 0
+#elif defined __BIG_ENDIAN__
+  #define IS_BIG_ENDIAN 1
+#elif defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+  #define IS_BIG_ENDIAN 0
+#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+  #define IS_BIG_ENDIAN 1
+#else
+// Detect endianness at runtime.
+static int
+is_big_endian(void)
+{
+	uint16_t d = 1;
+
+	return (archive_be16dec(&d) == 1);
+}
+
+#define IS_BIG_ENDIAN is_big_endian()
+#endif
+
 static struct archive_string *
 archive_string_append(struct archive_string *as, const char *p, size_t s)
 {
@@ -485,7 +505,7 @@ archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
 		struct archive_string u16;
 		int saved_flag = sc->flag;/* save current flag. */
 
-		if (is_big_endian())
+		if (IS_BIG_ENDIAN)
 			sc->flag |= SCONV_TO_UTF16BE;
 		else
 			sc->flag |= SCONV_TO_UTF16LE;
@@ -523,14 +543,14 @@ archive_wstring_append_from_mbs_in_codepage(struct archive_wstring *dest,
 		    dest->length + count + 1))
 			return (-1);
 		wmemcpy(dest->s + dest->length, (const wchar_t *)s, count);
-		if ((sc->flag & SCONV_FROM_UTF16BE) && !is_big_endian()) {
+		if ((sc->flag & SCONV_FROM_UTF16BE) && !IS_BIG_ENDIAN) {
 			uint16_t *u16 = (uint16_t *)(dest->s + dest->length);
 			int b;
 			for (b = 0; b < count; b++) {
 				uint16_t val = archive_le16dec(u16+b);
 				archive_be16enc(u16+b, val);
 			}
-		} else if ((sc->flag & SCONV_FROM_UTF16LE) && is_big_endian()) {
+		} else if ((sc->flag & SCONV_FROM_UTF16LE) && IS_BIG_ENDIAN) {
 			uint16_t *u16 = (uint16_t *)(dest->s + dest->length);
 			int b;
 			for (b = 0; b < count; b++) {
@@ -3538,7 +3558,7 @@ win_strncat_from_utf16(struct archive_string *as, const void *_p, size_t bytes,
 
 	archive_string_init(&tmp);
 	if (be) {
-		if (is_big_endian()) {
+		if (IS_BIG_ENDIAN) {
 			u16 = _p;
 		} else {
 			if (archive_string_ensure(&tmp, bytes+2) == NULL)
@@ -3551,7 +3571,7 @@ win_strncat_from_utf16(struct archive_string *as, const void *_p, size_t bytes,
 			u16 = tmp.s;
 		}
 	} else {
-		if (!is_big_endian()) {
+		if (!IS_BIG_ENDIAN) {
 			u16 = _p;
 		} else {
 			if (archive_string_ensure(&tmp, bytes+2) == NULL)
@@ -3605,14 +3625,6 @@ win_strncat_from_utf16le(struct archive_string *as, const void *_p,
 	return (win_strncat_from_utf16(as, _p, bytes, sc, 0));
 }
 
-static int
-is_big_endian(void)
-{
-	uint16_t d = 1;
-
-	return (archive_be16dec(&d) == 1);
-}
-
 /*
  * Convert a current locale string to UTF-16BE/LE and copy the result.
  * Return -1 if conversion fails.
@@ -3673,7 +3685,7 @@ win_strncat_to_utf16(struct archive_string *as16, const void *_p,
 	if (count == 0)
 		return (-1);
 
-	if (is_big_endian()) {
+	if (IS_BIG_ENDIAN) {
 		if (!bigendian) {
 			while (count > 0) {
 				uint16_t v = archive_be16dec(u16);