From 7cb28f3e21ff0c9658fad3d021e5a5548e1e49ae Mon Sep 17 00:00:00 2001

From: Rajalakshmi Srinivasaraghavan <raji@linux.vnet.ibm.com>

Date: Wed, 20 Apr 2016 23:10:42 +0530

Subject: [PATCH] powerpc: strcasestr optmization for power8

This patch optimizes strcasestr function for power >= 8 systems.  The average

improvement of this optimization is ~40% and compares 16 bytes at a time

using vector instructions.  This patch is tested on powerpc64 and powerpc64le.

(cherry picked from commit e413b14e18ac635b5683ab7bbb1c901f79d1b06b)

---

 ChangeLog                                          |  15 +

 sysdeps/powerpc/locale-defines.sym                 |   4 +

 sysdeps/powerpc/powerpc64/multiarch/Makefile       |   3 +-

 .../powerpc/powerpc64/multiarch/ifunc-impl-list.c  |   8 +

 .../powerpc64/multiarch/strcasestr-power8.S        |  49 ++

 .../powerpc/powerpc64/multiarch/strcasestr-ppc64.c |  34 ++

 sysdeps/powerpc/powerpc64/multiarch/strcasestr.c   |  37 ++

 sysdeps/powerpc/powerpc64/power8/Makefile          |   3 +

 .../powerpc/powerpc64/power8/strcasestr-ppc64.c    |  29 ++

 sysdeps/powerpc/powerpc64/power8/strcasestr.S      | 531 +++++++++++++++++++++

 10 files changed, 712 insertions(+), 1 deletion(-)

 create mode 100644 sysdeps/powerpc/powerpc64/multiarch/strcasestr-power8.S

 create mode 100644 sysdeps/powerpc/powerpc64/multiarch/strcasestr-ppc64.c

 create mode 100644 sysdeps/powerpc/powerpc64/multiarch/strcasestr.c

 create mode 100644 sysdeps/powerpc/powerpc64/power8/Makefile

 create mode 100644 sysdeps/powerpc/powerpc64/power8/strcasestr-ppc64.c

 create mode 100644 sysdeps/powerpc/powerpc64/power8/strcasestr.S

diff --git a/ChangeLog b/ChangeLog

index e7ea58a..6677ea2 100644

diff --git a/sysdeps/powerpc/locale-defines.sym b/sysdeps/powerpc/locale-defines.sym

index af64b92..5c5379c 100644

--- a/sysdeps/powerpc/locale-defines.sym

+++ b/sysdeps/powerpc/locale-defines.sym

@@ -3,3 +3,7 @@

 --

 LOCALE_CTYPE_TOLOWER	offsetof (struct __locale_struct, __ctype_tolower)

+LOCALE_CTYPE_TOUPPER	offsetof (struct __locale_struct, __ctype_toupper)

+_NL_CTYPE_NONASCII_CASE

+LOCALE_DATA_VALUES	offsetof (struct __locale_data, values)

+SIZEOF_VALUES		sizeof (((struct __locale_data *) 0)->values[0])

diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile

index 57abe8f..7f70ceb 100644

--- a/sysdeps/powerpc/powerpc64/multiarch/Makefile

+++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile

@@ -20,6 +20,7 @@ sysdep_routines += memcpy-power7 memcpy-a2 memcpy-power6 memcpy-cell \

                   mempcpy-power7 mempcpy-ppc64 memchr-power7 memchr-ppc64 \

                   memrchr-power7 memrchr-ppc64 rawmemchr-power7 \

                   stpcpy-power8 stpcpy-power7 stpcpy-ppc64 \

+                  strcasestr-power8 strcasestr-ppc64 \

                   strcat-power8 strcat-power7 strcat-ppc64 \

                   strcmp-power8 strcmp-power7 strcmp-ppc64 \

                   strcpy-power8 strcpy-power7 strcpy-ppc64 \

diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

index 583885c..994e852 100644

--- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

+++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

@@ -341,6 +341,14 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,

              IFUNC_IMPL_ADD (array, i, strstr, 1,

                              __strstr_ppc))

+  /* Support sysdeps/powerpc/powerpc64/multiarch/strcasestr.c.  */

+  IFUNC_IMPL (i, name, strcasestr,

+             IFUNC_IMPL_ADD (array, i, strcasestr,

+                             hwcap2 & PPC_FEATURE2_ARCH_2_07,

+                             __strcasestr_power8)

+             IFUNC_IMPL_ADD (array, i, strcasestr, 1,

+                             __strcasestr_ppc))

+

   /* Support sysdeps/powerpc/powerpc64/multiarch/wcschr.c.  */

   IFUNC_IMPL (i, name, wcschr,

              IFUNC_IMPL_ADD (array, i, wcschr,

diff --git a/sysdeps/powerpc/powerpc64/multiarch/strcasestr-power8.S b/sysdeps/powerpc/powerpc64/multiarch/strcasestr-power8.S

new file mode 100644

index 0000000..c77ff9f

--- /dev/null

+++ b/sysdeps/powerpc/powerpc64/multiarch/strcasestr-power8.S

@@ -0,0 +1,49 @@

+/* Optimized strcasestr implementation for POWER8.

+   Copyright (C) 2016 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#include <sysdep.h>

+

+#undef EALIGN

+#define EALIGN(name, alignt, words)				\

+  .section ".text";						\

+  ENTRY_2(__strcasestr_power8)					\

+  .align ALIGNARG(alignt);					\

+  EALIGN_W_##words;						\

+  BODY_LABEL(__strcasestr_power8):				\

+  cfi_startproc;						\

+  LOCALENTRY(__strcasestr_power8)

+

+#undef END

+#define END(name)						\

+  cfi_endproc;							\

+  TRACEBACK(__strcasestr_power8)				\

+  END_2(__strcasestr_power8)

+

+#undef libc_hidden_builtin_def

+#define libc_hidden_builtin_def(name)

+

+/* The following definitions are used in strcasestr optimization.  */

+

+/* strlen is used to calculate len of r4.  */

+#define STRLEN __strlen_power8

+/* strnlen is used to check if len of r3 is more than r4.  */

+#define STRNLEN __strnlen_power7

+/* strchr is used to check if first char of r4 is present in r3.  */

+#define STRCHR __strchr_power7

+

+#include <sysdeps/powerpc/powerpc64/power8/strcasestr.S>

diff --git a/sysdeps/powerpc/powerpc64/multiarch/strcasestr-ppc64.c b/sysdeps/powerpc/powerpc64/multiarch/strcasestr-ppc64.c

new file mode 100644

index 0000000..7f7bb9e

--- /dev/null

+++ b/sysdeps/powerpc/powerpc64/multiarch/strcasestr-ppc64.c

@@ -0,0 +1,34 @@

+/* PowerPC64 default implementation of strcasestr.

+   Copyright (C) 2016 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#include <string.h>

+

+#define STRCASESTR  __strcasestr_ppc

+#if IS_IN (libc) && defined(SHARED)

+# undef libc_hidden_builtin_def

+# define libc_hidden_builtin_def(name) \

+  __hidden_ver1(__strcasestr_ppc, __GI_strcasestr, __strcasestr_ppc);

+#endif

+

+

+#undef weak_alias

+#define weak_alias(a,b)

+

+extern __typeof (strcasestr) __strcasestr_ppc attribute_hidden;

+

+#include <string/strcasestr.c>

diff --git a/sysdeps/powerpc/powerpc64/multiarch/strcasestr.c b/sysdeps/powerpc/powerpc64/multiarch/strcasestr.c

new file mode 100644

index 0000000..17ba188

--- /dev/null

+++ b/sysdeps/powerpc/powerpc64/multiarch/strcasestr.c

@@ -0,0 +1,37 @@

+/* Multiple versions of strcasestr.

+   Copyright (C) 2016 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#if IS_IN (libc)

+# include <string.h>

+# include <shlib-compat.h>

+# include "init-arch.h"

+

+extern __typeof (__strcasestr) __strcasestr_ppc attribute_hidden;

+extern __typeof (__strcasestr) __strcasestr_power8 attribute_hidden;

+

+/* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle

+   ifunc symbol properly.  */

+libc_ifunc (__strcasestr,

+		(hwcap2 & PPC_FEATURE2_ARCH_2_07)

+		? __strcasestr_power8

+		: __strcasestr_ppc);

+

+weak_alias (__strcasestr, strcasestr)

+#else

+#include <string/strcasestr.c>

+#endif

diff --git a/sysdeps/powerpc/powerpc64/power8/Makefile b/sysdeps/powerpc/powerpc64/power8/Makefile

new file mode 100644

index 0000000..71a5952

--- /dev/null

+++ b/sysdeps/powerpc/powerpc64/power8/Makefile

@@ -0,0 +1,3 @@

+ifeq ($(subdir),string)

+sysdep_routines += strcasestr-ppc64

+endif

diff --git a/sysdeps/powerpc/powerpc64/power8/strcasestr-ppc64.c b/sysdeps/powerpc/powerpc64/power8/strcasestr-ppc64.c

new file mode 100644

index 0000000..09a07b0

--- /dev/null

+++ b/sysdeps/powerpc/powerpc64/power8/strcasestr-ppc64.c

@@ -0,0 +1,29 @@

+/* Optimized strcasestr implementation for PowerPC64/POWER8.

+   Copyright (C) 2016 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#include <string.h>

+

+#define STRCASESTR __strcasestr_ppc

+#undef libc_hidden_builtin_def

+#define libc_hidden_builtin_def(__name)

+

+#undef weak_alias

+#define weak_alias(a,b)

+extern __typeof (strcasestr) __strcasestr_ppc attribute_hidden;

+

+#include <string/strcasestr.c>

diff --git a/sysdeps/powerpc/powerpc64/power8/strcasestr.S b/sysdeps/powerpc/powerpc64/power8/strcasestr.S

new file mode 100644

index 0000000..24b2b76

--- /dev/null

+++ b/sysdeps/powerpc/powerpc64/power8/strcasestr.S

@@ -0,0 +1,531 @@

+/* Optimized strcasestr implementation for PowerPC64/POWER8.

+   Copyright (C) 2016 Free Software Foundation, Inc.

+   This file is part of the GNU C Library.

+

+   The GNU C Library is free software; you can redistribute it and/or

+   modify it under the terms of the GNU Lesser General Public

+   License as published by the Free Software Foundation; either

+   version 2.1 of the License, or (at your option) any later version.

+

+   The GNU C Library is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+   Lesser General Public License for more details.

+

+   You should have received a copy of the GNU Lesser General Public

+   License along with the GNU C Library; if not, see

+   <http://www.gnu.org/licenses/>.  */

+

+#include <sysdep.h>

+#include <locale-defines.h>

+

+/* Char * [r3] strcasestr (char *s [r3], char * pat[r4])  */

+

+/* The performance gain is obtained by comparing 16 bytes.  */

+

+/* When the first char of r4 is hit ITERATIONS times in r3

+   fallback to default.  */

+#define ITERATIONS	64

+

+#ifndef STRLEN

+/* For builds without IFUNC support, local calls should be made to internal

+   GLIBC symbol (created by libc_hidden_builtin_def).  */

+# ifdef SHARED

+#  define STRLEN   __GI_strlen

+# else

+#  define STRLEN   strlen

+# endif

+#endif

+

+#ifndef STRNLEN

+/* For builds without IFUNC support, local calls should be made to internal

+   GLIBC symbol (created by libc_hidden_builtin_def).  */

+# ifdef SHARED

+#  define STRNLEN   __GI_strnlen

+# else

+#  define STRNLEN    __strnlen

+# endif

+#endif

+

+#ifndef STRCHR

+# ifdef SHARED

+#  define STRCHR   __GI_strchr

+# else

+#  define STRCHR   strchr

+# endif

+#endif

+

+/* Convert 16 bytes of v4 and reg to lowercase and compare.  */

+#define TOLOWER(reg)     \

+	vcmpgtub	v6, v4, v1; \

+	vcmpgtub	v7, v2, v4; \

+	vand	v8, v7, v6; \

+	vand	v8, v8, v3; \

+	vor	v4, v8, v4; \

+	vcmpgtub	v6, reg, v1; \

+	vcmpgtub	v7, v2, reg; \

+	vand	v8, v7, v6; \

+	vand	v8, v8, v3; \

+	vor	reg, v8, reg; \

+	vcmpequb.	v6, reg, v4;

+

+/* TODO: change these to the actual instructions when the minimum required

+   binutils allows it.  */

+#ifdef _ARCH_PWR8

+#define VCLZD_V8_v7	vclzd	v8, v7;

+#else

+#define VCLZD_V8_v7	.long	0x11003fc2

+#endif

+

+#define	FRAMESIZE	(FRAME_MIN_SIZE+48)

+/* TODO: change this to .machine power8 when the minimum required binutils

+   allows it.  */

+	.machine  power7

+EALIGN (strcasestr, 4, 0)

+	CALL_MCOUNT 2

+	mflr	r0			/* Load link register LR to r0.  */

+	std	r31, -8(r1)		/* Save callers register r31.  */

+	std	r30, -16(r1)		/* Save callers register r30.  */

+	std	r29, -24(r1)		/* Save callers register r29.  */

+	std	r28, -32(r1)		/* Save callers register r28.  */

+	std	r27, -40(r1)		/* Save callers register r27.  */

+	std	r0, 16(r1)		/* Store the link register.  */

+	cfi_offset(r31, -8)

+	cfi_offset(r30, -16)

+	cfi_offset(r29, -24)

+	cfi_offset(r28, -32)

+	cfi_offset(r27, -40)

+	cfi_offset(lr, 16)

+	stdu	r1, -FRAMESIZE(r1)	/* Create the stack frame.  */

+	cfi_adjust_cfa_offset(FRAMESIZE)

+

+	dcbt	0, r3

+	dcbt	0, r4

+	cmpdi	cr7, r3, 0		/* Input validation.  */

+	beq	cr7, L(retnull)

+	cmpdi	cr7, r4, 0

+	beq	cr7, L(retnull)

+

+	mr	r29, r3

+	mr	r30, r4

+	/* Load first byte from r4 and check if its null.  */

+	lbz	r6, 0(r4)

+	cmpdi	cr7, r6, 0

+	beq	cr7, L(ret_r3)

+

+	ld	r10, __libc_tsd_LOCALE@got@tprel(r2)

+	add	r9, r10, __libc_tsd_LOCALE@tls

+	ld	r9, 0(r9)

+	ld	r9, LOCALE_CTYPE_TOUPPER(r9)

+	sldi	r10, r6, 2		/* Convert to upper case.  */

+	lwzx	r28, r9, r10

+

+	ld	r10, __libc_tsd_LOCALE@got@tprel(r2)

+	add	r11, r10, __libc_tsd_LOCALE@tls

+	ld	r11, 0(r11)

+	ld	r11, LOCALE_CTYPE_TOLOWER(r11)

+	sldi	r10, r6, 2              /* Convert to lower case.  */

+	lwzx	r27, r11, r10

+

+	/* Check if the first char is present.  */

+	mr	r4, r27

+	bl	STRCHR

+	nop

+	mr	r5, r3

+	mr	r3, r29

+	mr	r29, r5

+	mr	r4, r28

+	bl	STRCHR

+	nop

+	cmpdi	cr7, r29, 0

+	beq	cr7, L(firstpos)

+	cmpdi	cr7, r3, 0

+	beq	cr7, L(skipcheck)

+	cmpw	cr7, r3, r29

+	ble 	cr7, L(firstpos)

+	/* Move r3 to the first occurence.  */

+L(skipcheck):

+	mr	r3, r29

+L(firstpos):

+	mr	r29, r3

+

+	sldi	r9, r27, 8

+	or	r28, r9, r28

+	/* Reg r27 is used to count the number of iterations.  */

+	li	r27, 0

+	/* If first char of search str is not present.  */

+	cmpdi	cr7, r3, 0

+	ble	cr7, L(end)

+

+	/* Find the length of pattern.  */

+	mr	r3, r30

+	bl	STRLEN

+	nop

+

+	cmpdi	cr7, r3, 0	/* If search str is null.  */

+	beq	cr7, L(ret_r3)

+

+	mr	r31, r3

+	mr	r4, r3

+	mr	r3, r29

+	bl	STRNLEN

+	nop

+

+	cmpd	cr7, r3, r31 	/* If len(r3) < len(r4).  */

+	blt	cr7, L(retnull)

+

+	mr	r3, r29

+

+	/* Locales not matching ASCII for single bytes.  */

+	ld	r10, __libc_tsd_LOCALE@got@tprel(r2)

+	add	r9, r10, __libc_tsd_LOCALE@tls

+	ld	r9, 0(r9)

+	ld	r7, 0(r9)

+	addi	r7, r7, LOCALE_DATA_VALUES+_NL_CTYPE_NONASCII_CASE*SIZEOF_VALUES

+	lwz	r8, 0(r7)

+	cmpdi	cr7, r8, 1

+	beq	cr7, L(bytebybyte)

+

+	/* If len(r4) < 16 handle byte by byte.  */

+	/* For shorter strings we will not use vector registers.  */

+	cmpdi	cr7, r31, 16

+	blt	cr7, L(bytebybyte)

+

+	/* Comparison values used for TOLOWER.  */

+	/* Load v1 = 64('A' - 1), v2 = 91('Z' + 1), v3 = 32 in each byte.  */

+	vspltish	v0, 0

+	vspltisb	v5, 2

+	vspltisb	v4, 4

+	vsl	v3, v5, v4

+	vaddubm	v1, v3, v3

+	vspltisb	v5, 15

+	vaddubm	v2, v5, v5

+	vaddubm	v2, v1, v2

+	vspltisb	v4, -3

+	vaddubm	v2, v2, v4

+

+	/*

+	1. Load 16 bytes from r3 and r4

+	2. Check if there is null, If yes, proceed byte by byte path.

+	3. Else,Convert both to lowercase and compare.

+	4. If they are same proceed to 1.

+	5. If they dont match, find if first char of r4 is present in the

+	   loaded 16 byte of r3.

+	6. If yes, move position, load next 16 bytes of r3 and proceed to 2.

+	*/

+

+	mr	r8, r3		/* Save r3 for future use.  */

+	mr	r4, r30		/* Restore r4.  */

+	clrldi	r10, r4, 60

+	lvx	v5, 0, r4	/* Load 16 bytes from r4.  */

+	cmpdi	cr7, r10, 0

+	beq	cr7, L(begin2)

+	/* If r4 is unaligned, load another 16 bytes.  */

+#ifdef __LITTLE_ENDIAN__

+	lvsr	v7, 0, r4

+#else

+	lvsl	v7, 0, r4

+#endif

+	addi	r5, r4, 16

+	lvx	v9, 0, r5

+#ifdef __LITTLE_ENDIAN__

+	vperm	v5, v9, v5, v7

+#else

+	vperm	v5, v5, v9, v7

+#endif

+L(begin2):

+	lvx	v4, 0, r3

+	vcmpequb.	v7, v0, v4	/* Check for null.  */

+	beq	cr6, L(nullchk6)

+	b	L(trailcheck)

+

+        .align  4

+L(nullchk6):

+	clrldi	r10, r3, 60

+	cmpdi	cr7, r10, 0

+	beq	cr7, L(next16)

+#ifdef __LITTLE_ENDIAN__

+	lvsr	v7, 0, r3

+#else

+	lvsl	v7, 0, r3

+#endif

+	addi	r5, r3, 16

+	/* If r3 is unaligned, load another 16 bytes.  */

+	lvx	v10, 0, r5

+#ifdef __LITTLE_ENDIAN__

+	vperm	v4, v10, v4, v7

+#else

+	vperm	v4, v4, v10, v7

+#endif

+L(next16):

+	vcmpequb.	v6, v0, v5	/* Check for null.  */

+	beq	cr6, L(nullchk)

+	b	L(trailcheck)

+

+	.align	4

+L(nullchk):

+	vcmpequb.	v6, v0, v4

+	beq	cr6, L(nullchk1)

+	b	L(retnull)

+

+	.align	4

+L(nullchk1):

+	/* Convert both v3 and v4 to lower.  */

+	TOLOWER(v5)

+	/* If both are same, branch to match.  */

+	blt	cr6, L(match)

+	/* Find if the first char is present in next 15 bytes.  */

+#ifdef __LITTLE_ENDIAN__

+	vspltb	v6, v5, 15

+	vsldoi	v7, v0, v4, 15

+#else

+	vspltb	v6, v5, 0

+	vspltisb	v7, 8

+	vslo	v7, v4, v7

+#endif

+	vcmpequb	v7, v6, v7

+	vcmpequb.	v6, v0, v7

+	/* Shift r3 by 16 bytes and proceed.  */

+	blt	cr6, L(shift16)

+	VCLZD_V8_v7

+#ifdef __LITTLE_ENDIAN__

+	vspltb	v6, v8, 15

+#else

+	vspltb	v6, v8, 7

+#endif

+	vcmpequb.	v6, v6, v1

+	/* Shift r3 by 8  bytes and proceed.  */

+	blt	cr6, L(shift8)

+	b	L(begin)

+

+	.align	4

+L(match):

+	/* There is a match of 16 bytes, check next bytes.  */

+	cmpdi	cr7, r31, 16

+	mr	r29, r3

+	beq	cr7, L(ret_r3)

+

+L(secondmatch):

+	addi	r3, r3, 16

+	addi	r4, r4, 16

+	/* Load next 16 bytes of r3 and r4 and compare.  */

+	clrldi	r10, r4, 60

+	cmpdi	cr7, r10, 0

+	beq	cr7, L(nextload)

+	/* Handle unaligned case.  */

+	vor	v6, v9, v9

+	vcmpequb.	v7, v0, v6

+	beq	cr6, L(nullchk2)

+	b	L(trailcheck)

+

+	.align	4

+L(nullchk2):

+#ifdef __LITTLE_ENDIAN__

+	lvsr	v7, 0, r4

+#else

+	lvsl	v7, 0, r4

+#endif

+	addi	r5, r4, 16

+	/* If r4 is unaligned, load another 16 bytes.  */

+	lvx	v9, 0, r5

+#ifdef __LITTLE_ENDIAN__

+	vperm	v11, v9, v6, v7

+#else

+	vperm	v11, v6, v9, v7

+#endif

+	b	L(compare)

+

+	.align	4

+L(nextload):

+	lvx	v11, 0, r4

+L(compare):

+	vcmpequb.	v7, v0, v11

+	beq	cr6, L(nullchk3)

+	b	L(trailcheck)

+

+	.align	4

+L(nullchk3):

+	clrldi	r10, r3, 60

+	cmpdi 	cr7, r10, 0

+	beq 	cr7, L(nextload1)

+	/* Handle unaligned case.  */

+	vor	v4, v10, v10

+	vcmpequb.	v7, v0, v4

+	beq	cr6, L(nullchk4)

+	b	L(retnull)

+

+	.align	4

+L(nullchk4):

+#ifdef __LITTLE_ENDIAN__

+	lvsr	v7, 0, r3

+#else

+	lvsl	v7, 0, r3

+#endif

+	addi	r5, r3, 16

+	/* If r3 is unaligned, load another 16 bytes.  */

+	lvx	v10, 0, r5

+#ifdef __LITTLE_ENDIAN__

+	vperm	v4, v10, v4, v7

+#else

+	vperm	v4, v4, v10, v7

+#endif

+	b	L(compare1)

+

+	.align	4

+L(nextload1):

+	lvx	v4, 0, r3

+L(compare1):

+	vcmpequb.	v7, v0, v4

+	beq	cr6, L(nullchk5)

+	b	L(retnull)

+

+	.align	4

+L(nullchk5):

+	/* Convert both v3 and v4 to lower.  */

+	TOLOWER(v11)

+	/* If both are same, branch to secondmatch.  */

+	blt 	cr6, L(secondmatch)

+	/* Continue the search.  */

+        b	L(begin)

+

+	.align	4

+L(trailcheck):

+	ld	r10, __libc_tsd_LOCALE@got@tprel(r2)

+	add	r11, r10, __libc_tsd_LOCALE@tls

+	ld	r11, 0(r11)

+	ld	r11, LOCALE_CTYPE_TOLOWER(r11)

+L(loop2):

+	lbz	r5, 0(r3)               /* Load byte from r3.  */

+	lbz	r6, 0(r4)               /* Load next byte from r4.  */

+	cmpdi 	cr7, r6, 0              /* Is it null?  */

+	beq 	cr7, L(updater3)

+	cmpdi 	cr7, r5, 0              /* Is it null?  */

+	beq 	cr7, L(retnull)         /* If yes, return.  */

+	addi	r3, r3, 1

+	addi	r4, r4, 1               /* Increment r4.  */

+	sldi	r10, r5, 2              /* Convert to lower case.  */

+	lwzx	r10, r11, r10

+	sldi	r7, r6, 2               /* Convert to lower case.  */

+	lwzx	r7, r11, r7

+	cmpw	cr7, r7, r10            /* Compare with byte from r4.  */

+	bne	cr7, L(begin)

+	b	L(loop2)

+

+	.align	4

+L(shift8):

+	addi	r8, r8, 7

+	b	L(begin)

+	.align	4

+L(shift16):

+	addi	r8, r8, 15

+	.align	4

+L(begin):

+	addi	r8, r8, 1

+	mr	r3, r8

+	/* When our iterations exceed ITERATIONS,fall back to default.  */

+	addi	r27, r27, 1

+	cmpdi	cr7, r27, ITERATIONS

+	beq	cr7, L(default)

+	mr	r4, r30         /* Restore r4.  */

+	b	L(begin2)

+

+	/* Handling byte by byte.  */

+	.align	4

+L(loop1):

+	mr	r3, r8

+	addi	r27, r27, 1

+	cmpdi	cr7, r27, ITERATIONS

+	beq	cr7, L(default)

+	mr	r29, r8

+	srdi	r4, r28, 8

+	/* Check if the first char is present.  */

+	bl	STRCHR

+	nop

+	mr	r5, r3

+	mr	r3, r29

+	mr	r29, r5

+	sldi	r4, r28, 56

+	srdi	r4, r4, 56

+	bl	STRCHR

+	nop

+	cmpdi	cr7, r29, 0

+	beq	cr7, L(nextpos)

+	cmpdi	cr7, r3, 0

+	beq	cr7, L(skipcheck1)

+	cmpw	cr7, r3, r29

+	ble 	cr7, L(nextpos)

+	/* Move r3 to first occurence.  */

+L(skipcheck1):