From 53f860e80162b09c44b48f207342c1452289072c Mon Sep 17 00:00:00 2001

From: Stefan Liebler <stli@linux.vnet.ibm.com>

Date: Mon, 7 Nov 2016 15:37:29 +0100

Subject: [PATCH 04/17] S390: Optimize builtin iconv-modules.

Upstream commit 3b704e26b33e35d99de920f8462d8e438f89be39

This patch introduces a s390 specific gconv_simple.c file which provides

optimized versions for z13 with vector instructions, which will be chosen at

runtime via ifunc.

The optimized conversions can convert between internal and ascii, ucs4, ucs4le,

ucs2, ucs2le.

If the build-environment lacks vector support, then iconv/gconv_simple.c

is used wihtout any change. Otherwise iconvdata/gconv_simple.c is used to create

conversion loop routines without vector instructions as fallback, if vector

instructions aren't available at runtime.

ChangeLog:

	* sysdeps/s390/multiarch/gconv_simple.c: New File.

	* sysdeps/s390/multiarch/Makefile (sysdep_routines): Add gconv_simple.

---

 sysdeps/s390/multiarch/Makefile       |    4 +

 sysdeps/s390/multiarch/gconv_simple.c | 1266 +++++++++++++++++++++++++++++++++

 2 files changed, 1270 insertions(+)

 create mode 100644 sysdeps/s390/multiarch/gconv_simple.c

diff --git a/sysdeps/s390/multiarch/Makefile b/sysdeps/s390/multiarch/Makefile

index 11ad2b9..24949cd 100644

--- a/sysdeps/s390/multiarch/Makefile

+++ b/sysdeps/s390/multiarch/Makefile

@@ -52,3 +52,7 @@ $(move-if-change) $(@:stmp=T) $(@:stmp=h)

 touch $@

 endef

 endif

+

+ifeq ($(subdir),iconv)

+sysdep_routines += gconv_simple

+endif

diff --git a/sysdeps/s390/multiarch/gconv_simple.c b/sysdeps/s390/multiarch/gconv_simple.c

new file mode 100644

index 0000000..dc53a48

--- /dev/null

+++ b/sysdeps/s390/multiarch/gconv_simple.c

@@ -0,0 +1,1266 @@

+/* Simple transformations functions - s390 version.

+   Copyright (C) 2016 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#if defined HAVE_S390_VX_ASM_SUPPORT

+# include <ifunc-resolve.h>

+

+# if defined HAVE_S390_VX_GCC_SUPPORT

+#  define ASM_CLOBBER_VR(NR) , NR

+# else

+#  define ASM_CLOBBER_VR(NR)

+# endif

+

+# define ICONV_C_NAME(NAME) __##NAME##_c

+# define ICONV_VX_NAME(NAME) __##NAME##_vx

+# define ICONV_VX_IFUNC(FUNC)						\

+  extern __typeof (ICONV_C_NAME (FUNC)) __##FUNC;			\

+  s390_vx_libc_ifunc (__##FUNC)						\

+  int FUNC (struct __gconv_step *step, struct __gconv_step_data *data,	\

+	    const unsigned char **inptrp, const unsigned char *inend,	\

+	    unsigned char **outbufstart, size_t *irreversible,		\

+	    int do_flush, int consume_incomplete)			\

+  {									\

+    return __##FUNC (step, data, inptrp, inend,outbufstart,		\

+		     irreversible, do_flush, consume_incomplete);	\

+  }

+# define ICONV_VX_SINGLE(NAME)						\

+  static __typeof (NAME##_single) __##NAME##_vx_single __attribute__((alias(#NAME "_single")));

+

+/* Generate the transformations which are used, if the target machine does not

+   support vector instructions.  */

+# define __gconv_transform_ascii_internal		\

+  ICONV_C_NAME (__gconv_transform_ascii_internal)

+# define __gconv_transform_internal_ascii		\

+  ICONV_C_NAME (__gconv_transform_internal_ascii)

+# define __gconv_transform_internal_ucs4le		\

+  ICONV_C_NAME (__gconv_transform_internal_ucs4le)

+# define __gconv_transform_ucs4_internal		\

+  ICONV_C_NAME (__gconv_transform_ucs4_internal)

+# define __gconv_transform_ucs4le_internal		\

+  ICONV_C_NAME (__gconv_transform_ucs4le_internal)

+# define __gconv_transform_ucs2_internal		\

+  ICONV_C_NAME (__gconv_transform_ucs2_internal)

+# define __gconv_transform_ucs2reverse_internal		\

+  ICONV_C_NAME (__gconv_transform_ucs2reverse_internal)

+# define __gconv_transform_internal_ucs2		\

+  ICONV_C_NAME (__gconv_transform_internal_ucs2)

+# define __gconv_transform_internal_ucs2reverse		\

+  ICONV_C_NAME (__gconv_transform_internal_ucs2reverse)

+

+

+# include <iconv/gconv_simple.c>

+

+# undef __gconv_transform_ascii_internal

+# undef __gconv_transform_internal_ascii

+# undef __gconv_transform_internal_ucs4le

+# undef __gconv_transform_ucs4_internal

+# undef __gconv_transform_ucs4le_internal

+# undef __gconv_transform_ucs2_internal

+# undef __gconv_transform_ucs2reverse_internal

+# undef __gconv_transform_internal_ucs2

+# undef __gconv_transform_internal_ucs2reverse

+

+/* Now define the functions with vector support.  */

+# if defined __s390x__

+#  define CONVERT_32BIT_SIZE_T(REG)

+# else

+#  define CONVERT_32BIT_SIZE_T(REG) "llgfr %" #REG ",%" #REG "\n\t"

+# endif

+

+/* Convert from ISO 646-IRV to the internal (UCS4-like) format.  */

+# define DEFINE_INIT		0

+# define DEFINE_FINI		0

+# define MIN_NEEDED_FROM	1

+# define MIN_NEEDED_TO		4

+# define FROM_DIRECTION		1

+# define FROM_LOOP		ICONV_VX_NAME (ascii_internal_loop)

+# define TO_LOOP		ICONV_VX_NAME (ascii_internal_loop) /* This is not used.  */

+# define FUNCTION_NAME		ICONV_VX_NAME (__gconv_transform_ascii_internal)

+# define ONE_DIRECTION		1

+

+# define MIN_NEEDED_INPUT	MIN_NEEDED_FROM

+# define MIN_NEEDED_OUTPUT	MIN_NEEDED_TO

+# define LOOPFCT		FROM_LOOP

+# define BODY_ORIG_ERROR						\

+    /* The value is too large.  We don't try transliteration here since \

+       this is not an error because of the lack of possibilities to	\

+       represent the result.  This is a genuine bug in the input since	\

+       ASCII does not allow such values.  */				\

+    STANDARD_FROM_LOOP_ERR_HANDLER (1);

+

+# define BODY_ORIG							\

+  {									\

+    if (__glibc_unlikely (*inptr > '\x7f'))				\

+      {									\

+	BODY_ORIG_ERROR							\

+      }									\

+    else								\

+      {									\

+	/* It's an one byte sequence.  */				\

+	*((uint32_t *) outptr) = *inptr++;				\

+	outptr += sizeof (uint32_t);					\

+      }									\

+  }

+# define BODY								\

+  {									\

+    size_t len = inend - inptr;						\

+    if (len > (outend - outptr) / 4)					\

+      len = (outend - outptr) / 4;					\

+    size_t loop_count, tmp;						\

+    __asm__ volatile (".machine push\n\t"				\

+		      ".machine \"z13\"\n\t"				\

+		      ".machinemode \"zarch_nohighgprs\"\n\t"		\

+		      CONVERT_32BIT_SIZE_T ([R_LEN])			\

+		      "    vrepib %%v30,0x7f\n\t" /* For compare > 0x7f.  */ \

+		      "    srlg %[R_LI],%[R_LEN],4\n\t"			\

+		      "    vrepib %%v31,0x20\n\t"			\

+		      "    clgije %[R_LI],0,1f\n\t"			\

+		      "0:  \n\t" /* Handle 16-byte blocks.  */		\

+		      "    vl %%v16,0(%[R_IN])\n\t"			\

+		      /* Checking for values > 0x7f.  */		\

+		      "    vstrcbs %%v17,%%v16,%%v30,%%v31\n\t"		\

+		      "    jno 10f\n\t"					\

+		      /* Enlarge to UCS4.  */				\

+		      "    vuplhb %%v17,%%v16\n\t"			\

+		      "    vupllb %%v18,%%v16\n\t"			\

+		      "    vuplhh %%v19,%%v17\n\t"			\

+		      "    vupllh %%v20,%%v17\n\t"			\

+		      "    vuplhh %%v21,%%v18\n\t"			\

+		      "    vupllh %%v22,%%v18\n\t"			\

+		      /* Store 64bytes to buf_out.  */			\

+		      "    vstm %%v19,%%v22,0(%[R_OUT])\n\t"		\

+		      "    la %[R_IN],16(%[R_IN])\n\t"			\

+		      "    la %[R_OUT],64(%[R_OUT])\n\t"		\

+		      "    brctg %[R_LI],0b\n\t"			\

+		      "    lghi %[R_LI],15\n\t"				\

+		      "    ngr %[R_LEN],%[R_LI]\n\t"			\

+		      "    je 20f\n\t" /* Jump away if no remaining bytes.  */ \

+		      /* Handle remaining bytes.  */			\

+		      "1: aghik %[R_LI],%[R_LEN],-1\n\t"		\

+		      "    jl 20f\n\t" /* Jump away if no remaining bytes.  */ \

+		      "    vll %%v16,%[R_LI],0(%[R_IN])\n\t"		\

+		      /* Checking for values > 0x7f.  */		\

+		      "    vstrcbs %%v17,%%v16,%%v30,%%v31\n\t"		\

+		      "    vlgvb %[R_TMP],%%v17,7\n\t"			\

+		      "    clr %[R_TMP],%[R_LI]\n\t"			\

+		      "    locrh %[R_TMP],%[R_LEN]\n\t"			\

+		      "    locghih %[R_LEN],0\n\t"			\

+		      "    j 12f\n\t"					\

+		      "10:\n\t"						\

+		      /* Found a value > 0x7f.				\

+			 Store the preceding chars.  */			\

+		      "    vlgvb %[R_TMP],%%v17,7\n\t"			\

+		      "12: la %[R_IN],0(%[R_TMP],%[R_IN])\n\t"		\

+		      "    sllk %[R_TMP],%[R_TMP],2\n\t"		\

+		      "    ahi %[R_TMP],-1\n\t"				\

+		      "    jl 20f\n\t"					\

+		      "    lgr %[R_LI],%[R_TMP]\n\t"			\

+		      "    vuplhb %%v17,%%v16\n\t"			\

+		      "    vuplhh %%v19,%%v17\n\t"			\

+		      "    vstl %%v19,%[R_LI],0(%[R_OUT])\n\t"		\

+		      "    ahi %[R_LI],-16\n\t"				\

+		      "    jl 11f\n\t"					\

+		      "    vupllh %%v20,%%v17\n\t"			\

+		      "    vstl %%v20,%[R_LI],16(%[R_OUT])\n\t"		\

+		      "    ahi %[R_LI],-16\n\t"				\

+		      "    jl 11f\n\t"					\

+		      "    vupllb %%v18,%%v16\n\t"			\

+		      "    vuplhh %%v21,%%v18\n\t"			\

+		      "    vstl %%v21,%[R_LI],32(%[R_OUT])\n\t"		\

+		      "    ahi %[R_LI],-16\n\t"				\

+		      "    jl 11f\n\t"					\

+		      "    vupllh %%v22,%%v18\n\t"			\

+		      "    vstl %%v22,%[R_LI],48(%[R_OUT])\n\t"		\

+		      "11:\n\t"						\

+		      "    la %[R_OUT],1(%[R_TMP],%[R_OUT])\n\t"	\

+		      "20:\n\t"						\

+		      ".machine pop"					\

+		      : /* outputs */ [R_OUT] "+a" (outptr)		\

+			, [R_IN] "+a" (inptr)				\

+			, [R_LEN] "+d" (len)				\

+			, [R_LI] "=d" (loop_count)			\

+			, [R_TMP] "=a" (tmp)				\

+		      : /* inputs */					\

+		      : /* clobber list*/ "memory", "cc"		\

+			ASM_CLOBBER_VR ("v16") ASM_CLOBBER_VR ("v17")	\

+			ASM_CLOBBER_VR ("v18") ASM_CLOBBER_VR ("v19")	\

+			ASM_CLOBBER_VR ("v20") ASM_CLOBBER_VR ("v21")	\

+			ASM_CLOBBER_VR ("v22") ASM_CLOBBER_VR ("v30")	\

+			ASM_CLOBBER_VR ("v31")				\

+		      );						\

+    if (len > 0)							\

+      {									\

+	/* Found an invalid character at the next input byte.  */	\

+	BODY_ORIG_ERROR							\

+      }									\

+  }

+

+# define LOOP_NEED_FLAGS

+# include <iconv/loop.c>

+# include <iconv/skeleton.c>

+# undef BODY_ORIG

+# undef BODY_ORIG_ERROR

+ICONV_VX_IFUNC (__gconv_transform_ascii_internal)

+

+/* Convert from the internal (UCS4-like) format to ISO 646-IRV.  */

+# define DEFINE_INIT		0

+# define DEFINE_FINI		0

+# define MIN_NEEDED_FROM	4

+# define MIN_NEEDED_TO		1

+# define FROM_DIRECTION		1

+# define FROM_LOOP		ICONV_VX_NAME (internal_ascii_loop)

+# define TO_LOOP		ICONV_VX_NAME (internal_ascii_loop) /* This is not used.  */

+# define FUNCTION_NAME		ICONV_VX_NAME (__gconv_transform_internal_ascii)

+# define ONE_DIRECTION		1

+

+# define MIN_NEEDED_INPUT	MIN_NEEDED_FROM

+# define MIN_NEEDED_OUTPUT	MIN_NEEDED_TO

+# define LOOPFCT		FROM_LOOP

+# define BODY_ORIG_ERROR						\

+  UNICODE_TAG_HANDLER (*((const uint32_t *) inptr), 4);			\

+  STANDARD_TO_LOOP_ERR_HANDLER (4);

+

+# define BODY_ORIG							\

+  {									\

+    if (__glibc_unlikely (*((const uint32_t *) inptr) > 0x7f))		\

+      {									\

+	BODY_ORIG_ERROR							\

+      }									\

+    else								\

+      {									\

+	/* It's an one byte sequence.  */				\

+	*outptr++ = *((const uint32_t *) inptr);			\

+	inptr += sizeof (uint32_t);					\

+      }									\

+  }

+# define BODY								\

+  {									\

+    size_t len = (inend - inptr) / 4;					\

+    if (len > outend - outptr)						\

+      len = outend - outptr;						\

+    size_t loop_count, tmp, tmp2;					\

+    __asm__ volatile (".machine push\n\t"				\

+		      ".machine \"z13\"\n\t"				\

+		      ".machinemode \"zarch_nohighgprs\"\n\t"		\

+		      CONVERT_32BIT_SIZE_T ([R_LEN])			\

+		      /* Setup to check for ch > 0x7f.  */		\

+		      "    vzero %%v21\n\t"				\

+		      "    srlg %[R_LI],%[R_LEN],4\n\t"			\

+		      "    vleih %%v21,8192,0\n\t"  /* element 0:   >  */ \

+		      "    vleih %%v21,-8192,2\n\t" /* element 1: =<>  */ \

+		      "    vleif %%v20,127,0\n\t"   /* element 0: 127  */ \

+		      "    lghi %[R_TMP],0\n\t"				\

+		      "    clgije %[R_LI],0,1f\n\t"			\

+		      "0:\n\t"						\

+		      "    vlm %%v16,%%v19,0(%[R_IN])\n\t"		\

+		      /* Shorten to byte values.  */			\

+		      "    vpkf %%v23,%%v16,%%v17\n\t"			\

+		      "    vpkf %%v24,%%v18,%%v19\n\t"			\

+		      "    vpkh %%v23,%%v23,%%v24\n\t"			\

+		      /* Checking for values > 0x7f.  */		\

+		      "    vstrcfs %%v22,%%v16,%%v20,%%v21\n\t"		\

+		      "    jno 10f\n\t"					\

+		      "    vstrcfs %%v22,%%v17,%%v20,%%v21\n\t"		\

+		      "    jno 11f\n\t"					\

+		      "    vstrcfs %%v22,%%v18,%%v20,%%v21\n\t"		\

+		      "    jno 12f\n\t"					\

+		      "    vstrcfs %%v22,%%v19,%%v20,%%v21\n\t"		\

+		      "    jno 13f\n\t"					\

+		      /* Store 16bytes to outptr.  */			\

+		      "    vst %%v23,0(%[R_OUT])\n\t"			\

+		      "    la %[R_IN],64(%[R_IN])\n\t"			\

+		      "    la %[R_OUT],16(%[R_OUT])\n\t"		\

+		      "    brctg %[R_LI],0b\n\t"			\

+		      "    lghi %[R_LI],15\n\t"				\

+		      "    ngr %[R_LEN],%[R_LI]\n\t"			\

+		      "    je 20f\n\t" /* Jump away if no remaining bytes.  */ \

+		      /* Handle remaining bytes.  */			\

+		      "1: sllg %[R_LI],%[R_LEN],2\n\t"			\

+		      "    aghi %[R_LI],-1\n\t"				\

+		      "    jl 20f\n\t" /* Jump away if no remaining bytes.  */ \

+		      /* Load remaining 1...63 bytes.  */		\

+		      "    vll %%v16,%[R_LI],0(%[R_IN])\n\t"		\

+		      "    ahi %[R_LI],-16\n\t"				\

+		      "    jl 2f\n\t"					\

+		      "    vll %%v17,%[R_LI],16(%[R_IN])\n\t"		\

+		      "    ahi %[R_LI],-16\n\t"				\

+		      "    jl 2f\n\t"					\

+		      "    vll %%v18,%[R_LI],32(%[R_IN])\n\t"		\

+		      "    ahi %[R_LI],-16\n\t"				\

+		      "    jl 2f\n\t"					\

+		      "    vll %%v19,%[R_LI],48(%[R_IN])\n\t"		\

+		      "2:\n\t"						\

+		      /* Shorten to byte values.  */			\

+		      "    vpkf %%v23,%%v16,%%v17\n\t"			\

+		      "    vpkf %%v24,%%v18,%%v19\n\t"			\

+		      "    vpkh %%v23,%%v23,%%v24\n\t"			\

+		      "    sllg %[R_LI],%[R_LEN],2\n\t"			\

+		      "    aghi %[R_LI],-16\n\t"			\

+		      "    jl 3f\n\t" /* v16 is not fully loaded.  */	\

+		      "    vstrcfs %%v22,%%v16,%%v20,%%v21\n\t"		\

+		      "    jno 10f\n\t"					\

+		      "    aghi %[R_LI],-16\n\t"			\

+		      "    jl 4f\n\t" /* v17 is not fully loaded.  */	\

+		      "    vstrcfs %%v22,%%v17,%%v20,%%v21\n\t"		\

+		      "    jno 11f\n\t"					\

+		      "    aghi %[R_LI],-16\n\t"			\

+		      "    jl 5f\n\t" /* v18 is not fully loaded.  */	\

+		      "    vstrcfs %%v22,%%v18,%%v20,%%v21\n\t"		\

+		      "    jno 12f\n\t"					\

+		      "    aghi %[R_LI],-16\n\t"			\

+		      /* v19 is not fully loaded. */			\

+		      "    lghi %[R_TMP],12\n\t"			\

+		      "    vstrcfs %%v22,%%v19,%%v20,%%v21\n\t"		\

+		      "6: vlgvb %[R_I],%%v22,7\n\t"			\

+		      "    aghi %[R_LI],16\n\t"				\

+		      "    clrjl %[R_I],%[R_LI],14f\n\t"		\

+		      "    lgr %[R_I],%[R_LEN]\n\t"			\

+		      "    lghi %[R_LEN],0\n\t"				\

+		      "    j 15f\n\t"					\

+		      "3: vstrcfs %%v22,%%v16,%%v20,%%v21\n\t"		\

+		      "    j 6b\n\t"					\

+		      "4: vstrcfs %%v22,%%v17,%%v20,%%v21\n\t"		\

+		      "    lghi %[R_TMP],4\n\t"				\

+		      "    j 6b\n\t"					\

+		      "5: vstrcfs %%v22,%%v17,%%v20,%%v21\n\t"		\

+		      "    lghi %[R_TMP],8\n\t"				\

+		      "    j 6b\n\t"					\

+		      /* Found a value > 0x7f.  */			\

+		      "13: ahi %[R_TMP],4\n\t"				\

+		      "12: ahi %[R_TMP],4\n\t"				\

+		      "11: ahi %[R_TMP],4\n\t"				\

+		      "10: vlgvb %[R_I],%%v22,7\n\t"			\

+		      "14: srlg %[R_I],%[R_I],2\n\t"			\

+		      "    agr %[R_I],%[R_TMP]\n\t"			\

+		      "    je 20f\n\t"					\

+		      /* Store characters before invalid one...  */	\

+		      "15: aghi %[R_I],-1\n\t"				\

+		      "    vstl %%v23,%[R_I],0(%[R_OUT])\n\t"		\

+		      /* ... and update pointers.  */			\

+		      "    la %[R_OUT],1(%[R_I],%[R_OUT])\n\t"		\

+		      "    sllg %[R_I],%[R_I],2\n\t"			\

+		      "    la %[R_IN],4(%[R_I],%[R_IN])\n\t"		\

+		      "20:\n\t"						\

+		      ".machine pop"					\

+		      : /* outputs */ [R_OUT] "+a" (outptr)		\

+			, [R_IN] "+a" (inptr)				\

+			, [R_LEN] "+d" (len)				\

+			, [R_LI] "=d" (loop_count)			\

+			, [R_I] "=a" (tmp2)				\

+			, [R_TMP] "=d" (tmp)				\

+		      : /* inputs */					\

+		      : /* clobber list*/ "memory", "cc"		\

+			ASM_CLOBBER_VR ("v16") ASM_CLOBBER_VR ("v17")	\

+			ASM_CLOBBER_VR ("v18") ASM_CLOBBER_VR ("v19")	\

+			ASM_CLOBBER_VR ("v20") ASM_CLOBBER_VR ("v21")	\

+			ASM_CLOBBER_VR ("v22") ASM_CLOBBER_VR ("v23")	\

+			ASM_CLOBBER_VR ("v24")				\

+		      );						\

+    if (len > 0)							\

+      {									\

+	/* Found an invalid character > 0x7f at next character.  */	\

+	BODY_ORIG_ERROR							\

+      }									\

+  }

+# define LOOP_NEED_FLAGS

+# include <iconv/loop.c>

+# include <iconv/skeleton.c>

+# undef BODY_ORIG

+# undef BODY_ORIG_ERROR

+ICONV_VX_IFUNC (__gconv_transform_internal_ascii)

+

+

+/* Convert from internal UCS4 to UCS4 little endian form.  */

+# define DEFINE_INIT		0

+# define DEFINE_FINI		0

+# define MIN_NEEDED_FROM	4

+# define MIN_NEEDED_TO		4

+# define FROM_DIRECTION		1

+# define FROM_LOOP		ICONV_VX_NAME (internal_ucs4le_loop)

+# define TO_LOOP		ICONV_VX_NAME (internal_ucs4le_loop) /* This is not used.  */

+# define FUNCTION_NAME		ICONV_VX_NAME (__gconv_transform_internal_ucs4le)

+# define ONE_DIRECTION		0

+

+static inline int

+__attribute ((always_inline))

+ICONV_VX_NAME (internal_ucs4le_loop) (struct __gconv_step *step,

+				      struct __gconv_step_data *step_data,

+				      const unsigned char **inptrp,

+				      const unsigned char *inend,

+				      unsigned char **outptrp,

+				      unsigned char *outend,

+				      size_t *irreversible)

+{

+  const unsigned char *inptr = *inptrp;

+  unsigned char *outptr = *outptrp;

+  int result;

+  size_t len = MIN (inend - inptr, outend - outptr) / 4;

+  size_t loop_count;

+  __asm__ volatile (".machine push\n\t"

+		    ".machine \"z13\"\n\t"

+		    ".machinemode \"zarch_nohighgprs\"\n\t"

+		    CONVERT_32BIT_SIZE_T ([R_LEN])

+		    "    bras %[R_LI],1f\n\t"

+		    /* Vector permute mask:  */

+		    "    .long 0x03020100,0x7060504,0x0B0A0908,0x0F0E0D0C\n\t"

+		    "1:  vl %%v20,0(%[R_LI])\n\t"

+		    /* Process 64byte (16char) blocks.  */

+		    "    srlg %[R_LI],%[R_LEN],4\n\t"

+		    "    clgije %[R_LI],0,10f\n\t"

+		    "0:  vlm %%v16,%%v19,0(%[R_IN])\n\t"

+		    "    vperm %%v16,%%v16,%%v16,%%v20\n\t"

+		    "    vperm %%v17,%%v17,%%v17,%%v20\n\t"

+		    "    vperm %%v18,%%v18,%%v18,%%v20\n\t"

+		    "    vperm %%v19,%%v19,%%v19,%%v20\n\t"

+		    "    vstm %%v16,%%v19,0(%[R_OUT])\n\t"

+		    "    la %[R_IN],64(%[R_IN])\n\t"

+		    "    la %[R_OUT],64(%[R_OUT])\n\t"

+		    "    brctg %[R_LI],0b\n\t"

+		    "    llgfr %[R_LEN],%[R_LEN]\n\t"

+		    "    nilf %[R_LEN],15\n\t"

+		    /* Process 16byte (4char) blocks.  */

+		    "10: srlg %[R_LI],%[R_LEN],2\n\t"

+		    "    clgije %[R_LI],0,20f\n\t"

+		    "11: vl %%v16,0(%[R_IN])\n\t"

+		    "    vperm %%v16,%%v16,%%v16,%%v20\n\t"

+		    "    vst %%v16,0(%[R_OUT])\n\t"

+		    "    la %[R_IN],16(%[R_IN])\n\t"

+		    "    la %[R_OUT],16(%[R_OUT])\n\t"

+		    "    brctg %[R_LI],11b\n\t"

+		    "    nill %[R_LEN],3\n\t"

+		    /* Process <16bytes.  */

+		    "20: sll %[R_LEN],2\n\t"

+		    "    ahi %[R_LEN],-1\n\t"

+		    "    jl 30f\n\t"

+		    "    vll %%v16,%[R_LEN],0(%[R_IN])\n\t"

+		    "    vperm %%v16,%%v16,%%v16,%%v20\n\t"

+		    "    vstl %%v16,%[R_LEN],0(%[R_OUT])\n\t"

+		    "    la %[R_IN],1(%[R_LEN],%[R_IN])\n\t"

+		    "    la %[R_OUT],1(%[R_LEN],%[R_OUT])\n\t"

+		    "30: \n\t"

+		    ".machine pop"

+		    : /* outputs */ [R_OUT] "+a" (outptr)

+		      , [R_IN] "+a" (inptr)

+		      , [R_LI] "=a" (loop_count)

+		      , [R_LEN] "+a" (len)

+		    : /* inputs */

+		    : /* clobber list*/ "memory", "cc"

+		      ASM_CLOBBER_VR ("v16") ASM_CLOBBER_VR ("v17")

+		      ASM_CLOBBER_VR ("v18") ASM_CLOBBER_VR ("v19")

+		      ASM_CLOBBER_VR ("v20")

+		    );

+  *inptrp = inptr;

+  *outptrp = outptr;

+

+  /* Determine the status.  */

+  if (*inptrp == inend)

+    result = __GCONV_EMPTY_INPUT;

+  else if (*outptrp + 4 > outend)

+    result = __GCONV_FULL_OUTPUT;

+  else

+    result = __GCONV_INCOMPLETE_INPUT;

+

+  return result;

+}

+

+ICONV_VX_SINGLE (internal_ucs4le_loop)

+# include <iconv/skeleton.c>

+ICONV_VX_IFUNC (__gconv_transform_internal_ucs4le)

+

+

+/* Transform from UCS4 to the internal, UCS4-like format.  Unlike

+   for the other direction we have to check for correct values here.  */

+# define DEFINE_INIT		0

+# define DEFINE_FINI		0

+# define MIN_NEEDED_FROM	4

+# define MIN_NEEDED_TO		4

+# define FROM_DIRECTION		1

+# define FROM_LOOP		ICONV_VX_NAME (ucs4_internal_loop)

+# define TO_LOOP		ICONV_VX_NAME (ucs4_internal_loop) /* This is not used.  */

+# define FUNCTION_NAME		ICONV_VX_NAME (__gconv_transform_ucs4_internal)

+# define ONE_DIRECTION		0

+

+

+static inline int

+__attribute ((always_inline))

+ICONV_VX_NAME (ucs4_internal_loop) (struct __gconv_step *step,

+				    struct __gconv_step_data *step_data,

+				    const unsigned char **inptrp,

+				    const unsigned char *inend,

+				    unsigned char **outptrp,

+				    unsigned char *outend,

+				    size_t *irreversible)

+{

+  int flags = step_data->__flags;

+  const unsigned char *inptr = *inptrp;

+  unsigned char *outptr = *outptrp;

+  int result;

+  size_t len, loop_count;

+  do

+    {

+      len = MIN (inend - inptr, outend - outptr) / 4;

+      __asm__ volatile (".machine push\n\t"

+			".machine \"z13\"\n\t"

+			".machinemode \"zarch_nohighgprs\"\n\t"

+			CONVERT_32BIT_SIZE_T ([R_LEN])

+			/* Setup to check for ch > 0x7fffffff.  */

+			"    larl %[R_LI],9f\n\t"

+			"    vlm %%v20,%%v21,0(%[R_LI])\n\t"

+			"    srlg %[R_LI],%[R_LEN],2\n\t"

+			"    clgije %[R_LI],0,1f\n\t"

+			/* Process 16byte (4char) blocks.  */

+			"0:  vl %%v16,0(%[R_IN])\n\t"

+			"    vstrcfs %%v22,%%v16,%%v20,%%v21\n\t"

+			"    jno 10f\n\t"

+			"    vst %%v16,0(%[R_OUT])\n\t"

+			"    la %[R_IN],16(%[R_IN])\n\t"

+			"    la %[R_OUT],16(%[R_OUT])\n\t"

+			"    brctg %[R_LI],0b\n\t"

+			"    llgfr %[R_LEN],%[R_LEN]\n\t"

+			"    nilf %[R_LEN],3\n\t"

+			/* Process <16bytes.  */

+			"1:  sll %[R_LEN],2\n\t"

+			"    ahik %[R_LI],%[R_LEN],-1\n\t"

+			"    jl 20f\n\t" /* No further bytes available.  */

+			"    vll %%v16,%[R_LI],0(%[R_IN])\n\t"

+			"    vstrcfs %%v22,%%v16,%%v20,%%v21\n\t"

+			"    vlgvb %[R_LI],%%v22,7\n\t"

+			"    clr %[R_LI],%[R_LEN]\n\t"

+			"    locgrhe %[R_LI],%[R_LEN]\n\t"

+			"    locghihe %[R_LEN],0\n\t"

+			"    j 11f\n\t"

+			/* v20: Vector string range compare values.  */

+			"9:  .long 0x7fffffff,0x0,0x0,0x0\n\t"

+			/* v21: Vector string range compare control-bits.

+			   element 0: >; element 1: =<> (always true)  */

+			"    .long 0x20000000,0xE0000000,0x0,0x0\n\t"

+			/* Found a value > 0x7fffffff.  */

+			"10: vlgvb %[R_LI],%%v22,7\n\t"

+			/* Store characters before invalid one.  */

+			"11: aghi %[R_LI],-1\n\t"

+			"    jl 20f\n\t"

+			"    vstl %%v16,%[R_LI],0(%[R_OUT])\n\t"

+			"    la %[R_IN],1(%[R_LI],%[R_IN])\n\t"

+			"    la %[R_OUT],1(%[R_LI],%[R_OUT])\n\t"

+			"20:\n\t"

+			".machine pop"

+			: /* outputs */ [R_OUT] "+a" (outptr)

+			  , [R_IN] "+a" (inptr)

+			  , [R_LI] "=a" (loop_count)

+			  , [R_LEN] "+d" (len)

+			: /* inputs */

+			: /* clobber list*/ "memory", "cc"

+			  ASM_CLOBBER_VR ("v16") ASM_CLOBBER_VR ("v20")

+			  ASM_CLOBBER_VR ("v21") ASM_CLOBBER_VR ("v22")

+			);

+      if (len > 0)

+	{

+	  /* The value is too large.  We don't try transliteration here since

+	     this is not an error because of the lack of possibilities to

+	     represent the result.  This is a genuine bug in the input since

+	     UCS4 does not allow such values.  */

+	  if (irreversible == NULL)

+	    /* We are transliterating, don't try to correct anything.  */

+	    return __GCONV_ILLEGAL_INPUT;

+

+	  if (flags & __GCONV_IGNORE_ERRORS)

+	    {

+	      /* Just ignore this character.  */

+	      ++*irreversible;

+	      inptr += 4;

+	      continue;

+	    }

+

+	  *inptrp = inptr;

+	  *outptrp = outptr;

+	  return __GCONV_ILLEGAL_INPUT;

+	}

+    }

+  while (len > 0);

+

+  *inptrp = inptr;

+  *outptrp = outptr;

+

+  /* Determine the status.  */

+  if (*inptrp == inend)

+    result = __GCONV_EMPTY_INPUT;

+  else if (*outptrp + 4 > outend)

+    result = __GCONV_FULL_OUTPUT;

+  else

+    result = __GCONV_INCOMPLETE_INPUT;

+

+  return result;

+}

+

+ICONV_VX_SINGLE (ucs4_internal_loop)

+# include <iconv/skeleton.c>

+ICONV_VX_IFUNC (__gconv_transform_ucs4_internal)

+

+

+/* Transform from UCS4-LE to the internal encoding.  */

+# define DEFINE_INIT		0

+# define DEFINE_FINI		0

+# define MIN_NEEDED_FROM	4

+# define MIN_NEEDED_TO		4

+# define FROM_DIRECTION		1

+# define FROM_LOOP		ICONV_VX_NAME (ucs4le_internal_loop)

+# define TO_LOOP		ICONV_VX_NAME (ucs4le_internal_loop) /* This is not used.  */

+# define FUNCTION_NAME		ICONV_VX_NAME (__gconv_transform_ucs4le_internal)

+# define ONE_DIRECTION		0

+

+static inline int

+__attribute ((always_inline))

+ICONV_VX_NAME (ucs4le_internal_loop) (struct __gconv_step *step,

+				      struct __gconv_step_data *step_data,

+				      const unsigned char **inptrp,

+				      const unsigned char *inend,

+				      unsigned char **outptrp,

+				      unsigned char *outend,

+				      size_t *irreversible)

+{

+  int flags = step_data->__flags;

+  const unsigned char *inptr = *inptrp;

+  unsigned char *outptr = *outptrp;

+  int result;

+  size_t len, loop_count;

+  do

+    {

+      len = MIN (inend - inptr, outend - outptr) / 4;

+      __asm__ volatile (".machine push\n\t"

+			".machine \"z13\"\n\t"

+			".machinemode \"zarch_nohighgprs\"\n\t"

+			CONVERT_32BIT_SIZE_T ([R_LEN])

+			/* Setup to check for ch > 0x7fffffff.  */

+			"    larl %[R_LI],9f\n\t"

+			"    vlm %%v20,%%v22,0(%[R_LI])\n\t"

+			"    srlg %[R_LI],%[R_LEN],2\n\t"

+			"    clgije %[R_LI],0,1f\n\t"

+			/* Process 16byte (4char) blocks.  */

+			"0:  vl %%v16,0(%[R_IN])\n\t"

+			"    vperm %%v16,%%v16,%%v16,%%v22\n\t"

+			"    vstrcfs %%v23,%%v16,%%v20,%%v21\n\t"

+			"    jno 10f\n\t"

+			"    vst %%v16,0(%[R_OUT])\n\t"

+			"    la %[R_IN],16(%[R_IN])\n\t"

+			"    la %[R_OUT],16(%[R_OUT])\n\t"

+			"    brctg %[R_LI],0b\n\t"

+			"    llgfr %[R_LEN],%[R_LEN]\n\t"

+			"    nilf %[R_LEN],3\n\t"

+			/* Process <16bytes.  */

+			"1:  sll %[R_LEN],2\n\t"

+			"    ahik %[R_LI],%[R_LEN],-1\n\t"

+			"    jl 20f\n\t" /* No further bytes available.  */

+			"    vll %%v16,%[R_LI],0(%[R_IN])\n\t"

+			"    vperm %%v16,%%v16,%%v16,%%v22\n\t"

+			"    vstrcfs %%v23,%%v16,%%v20,%%v21\n\t"

+			"    vlgvb %[R_LI],%%v23,7\n\t"

+			"    clr %[R_LI],%[R_LEN]\n\t"

+			"    locgrhe %[R_LI],%[R_LEN]\n\t"

+			"    locghihe %[R_LEN],0\n\t"

+			"    j 11f\n\t"