To: vim_dev@googlegroups.com
Subject: Patch 7.4.001
Fcc: outbox
From: Bram Moolenaar <Bram@moolenaar.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
------------
Patch 7.4.001
Problem: Character classes such as [a-z] to not react to 'ignorecase'.
Breaks man page highlighting. (Mario Grgic)
Solution: Add separate items for classes that react to 'ignorecase'. Clean
up logic handling character classes. Add more tests.
Files: src/regexp_nfa.c, src/testdir/test64.in, src/testdir/test64.ok
*** ../vim-7.4.000/src/regexp_nfa.c 2013-08-01 18:27:51.000000000 +0200
--- src/regexp_nfa.c 2013-08-14 11:49:50.000000000 +0200
***************
*** 29,34 ****
--- 29,37 ----
# define NFA_REGEXP_DEBUG_LOG "nfa_regexp_debug.log"
#endif
+ /* Added to NFA_ANY - NFA_NUPPER_IC to include a NL. */
+ #define NFA_ADD_NL 31
+
enum
{
NFA_SPLIT = -1024,
***************
*** 183,188 ****
--- 186,198 ----
NFA_NLOWER, /* Match non-lowercase char */
NFA_UPPER, /* Match uppercase char */
NFA_NUPPER, /* Match non-uppercase char */
+ NFA_LOWER_IC, /* Match [a-z] */
+ NFA_NLOWER_IC, /* Match [^a-z] */
+ NFA_UPPER_IC, /* Match [A-Z] */
+ NFA_NUPPER_IC, /* Match [^A-Z] */
+
+ NFA_FIRST_NL = NFA_ANY + NFA_ADD_NL,
+ NFA_LAST_NL = NFA_NUPPER_IC + NFA_ADD_NL,
NFA_CURSOR, /* Match cursor pos */
NFA_LNUM, /* Match line number */
***************
*** 199,207 ****
NFA_MARK_LT, /* Match < mark */
NFA_VISUAL, /* Match Visual area */
- NFA_FIRST_NL = NFA_ANY + ADD_NL,
- NFA_LAST_NL = NFA_NUPPER + ADD_NL,
-
/* Character classes [:alnum:] etc */
NFA_CLASS_ALNUM,
NFA_CLASS_ALPHA,
--- 209,214 ----
***************
*** 578,583 ****
--- 585,592 ----
* On failure, return 0 (=FAIL)
* Start points to the first char of the range, while end should point
* to the closing brace.
+ * Keep in mind that 'ignorecase' applies at execution time, thus [a-z] may
+ * need to be interpreted as [a-zA-Z].
*/
static int
nfa_recognize_char_class(start, end, extra_newl)
***************
*** 681,687 ****
return FAIL;
if (newl == TRUE)
! extra_newl = ADD_NL;
switch (config)
{
--- 690,696 ----
return FAIL;
if (newl == TRUE)
! extra_newl = NFA_ADD_NL;
switch (config)
{
***************
*** 710,722 ****
case CLASS_not | CLASS_az | CLASS_AZ:
return extra_newl + NFA_NALPHA;
case CLASS_az:
! return extra_newl + NFA_LOWER;
case CLASS_not | CLASS_az:
! return extra_newl + NFA_NLOWER;
case CLASS_AZ:
! return extra_newl + NFA_UPPER;
case CLASS_not | CLASS_AZ:
! return extra_newl + NFA_NUPPER;
}
return FAIL;
}
--- 719,731 ----
case CLASS_not | CLASS_az | CLASS_AZ:
return extra_newl + NFA_NALPHA;
case CLASS_az:
! return extra_newl + NFA_LOWER_IC;
case CLASS_not | CLASS_az:
! return extra_newl + NFA_NLOWER_IC;
case CLASS_AZ:
! return extra_newl + NFA_UPPER_IC;
case CLASS_not | CLASS_AZ:
! return extra_newl + NFA_NUPPER_IC;
}
return FAIL;
}
***************
*** 914,920 ****
break;
}
! extra = ADD_NL;
/* "\_[" is collection plus newline */
if (c == '[')
--- 923,929 ----
break;
}
! extra = NFA_ADD_NL;
/* "\_[" is collection plus newline */
if (c == '[')
***************
*** 970,976 ****
}
#endif
EMIT(nfa_classcodes[p - classchars]);
! if (extra == ADD_NL)
{
EMIT(NFA_NEWL);
EMIT(NFA_OR);
--- 979,985 ----
}
#endif
EMIT(nfa_classcodes[p - classchars]);
! if (extra == NFA_ADD_NL)
{
EMIT(NFA_NEWL);
EMIT(NFA_OR);
***************
*** 1240,1260 ****
{
/*
* Try to reverse engineer character classes. For example,
! * recognize that [0-9] stands for \d and [A-Za-z_] with \h,
* and perform the necessary substitutions in the NFA.
*/
result = nfa_recognize_char_class(regparse, endp,
! extra == ADD_NL);
if (result != FAIL)
{
! if (result >= NFA_DIGIT && result <= NFA_NUPPER)
! EMIT(result);
! else /* must be char class + newline */
{
! EMIT(result - ADD_NL);
EMIT(NFA_NEWL);
EMIT(NFA_OR);
}
regparse = endp;
mb_ptr_adv(regparse);
return OK;
--- 1249,1269 ----
{
/*
* Try to reverse engineer character classes. For example,
! * recognize that [0-9] stands for \d and [A-Za-z_] for \h,
* and perform the necessary substitutions in the NFA.
*/
result = nfa_recognize_char_class(regparse, endp,
! extra == NFA_ADD_NL);
if (result != FAIL)
{
! if (result >= NFA_FIRST_NL && result <= NFA_LAST_NL)
{
! EMIT(result - NFA_ADD_NL);
EMIT(NFA_NEWL);
EMIT(NFA_OR);
}
+ else
+ EMIT(result);
regparse = endp;
mb_ptr_adv(regparse);
return OK;
***************
*** 1504,1510 ****
* collection, add an OR below. But not for negated
* range. */
if (!negated)
! extra = ADD_NL;
}
else
{
--- 1513,1519 ----
* collection, add an OR below. But not for negated
* range. */
if (!negated)
! extra = NFA_ADD_NL;
}
else
{
***************
*** 1537,1543 ****
EMIT(NFA_END_COLL);
/* \_[] also matches \n but it's not negated */
! if (extra == ADD_NL)
{
EMIT(reg_string ? NL : NFA_NEWL);
EMIT(NFA_OR);
--- 1546,1552 ----
EMIT(NFA_END_COLL);
/* \_[] also matches \n but it's not negated */
! if (extra == NFA_ADD_NL)
{
EMIT(reg_string ? NL : NFA_NEWL);
EMIT(NFA_OR);
***************
*** 2011,2017 ****
if (c >= NFA_FIRST_NL && c <= NFA_LAST_NL)
{
addnl = TRUE;
! c -= ADD_NL;
}
STRCPY(code, "");
--- 2020,2026 ----
if (c >= NFA_FIRST_NL && c <= NFA_LAST_NL)
{
addnl = TRUE;
! c -= NFA_ADD_NL;
}
STRCPY(code, "");
***************
*** 2217,2222 ****
--- 2226,2235 ----
case NFA_NLOWER:STRCPY(code, "NFA_NLOWER"); break;
case NFA_UPPER: STRCPY(code, "NFA_UPPER"); break;
case NFA_NUPPER:STRCPY(code, "NFA_NUPPER"); break;
+ case NFA_LOWER_IC: STRCPY(code, "NFA_LOWER_IC"); break;
+ case NFA_NLOWER_IC: STRCPY(code, "NFA_NLOWER_IC"); break;
+ case NFA_UPPER_IC: STRCPY(code, "NFA_UPPER_IC"); break;
+ case NFA_NUPPER_IC: STRCPY(code, "NFA_NUPPER_IC"); break;
default:
STRCPY(code, "CHAR(x)");
***************
*** 2687,2692 ****
--- 2700,2709 ----
case NFA_NLOWER:
case NFA_UPPER:
case NFA_NUPPER:
+ case NFA_LOWER_IC:
+ case NFA_NLOWER_IC:
+ case NFA_UPPER_IC:
+ case NFA_NUPPER_IC:
/* possibly non-ascii */
#ifdef FEAT_MBYTE
if (has_mbyte)
***************
*** 3841,3846 ****
--- 3858,3867 ----
case NFA_NLOWER:
case NFA_UPPER:
case NFA_NUPPER:
+ case NFA_LOWER_IC:
+ case NFA_NLOWER_IC:
+ case NFA_UPPER_IC:
+ case NFA_NUPPER_IC:
case NFA_START_COLL:
case NFA_START_NEG_COLL:
case NFA_NEWL:
***************
*** 5872,5877 ****
--- 5893,5920 ----
ADD_STATE_IF_MATCH(t->state);
break;
+ case NFA_LOWER_IC: /* [a-z] */
+ result = ri_lower(curc) || (ireg_ic && ri_upper(curc));
+ ADD_STATE_IF_MATCH(t->state);
+ break;
+
+ case NFA_NLOWER_IC: /* [^a-z] */
+ result = curc != NUL
+ && !(ri_lower(curc) || (ireg_ic && ri_upper(curc)));
+ ADD_STATE_IF_MATCH(t->state);
+ break;
+
+ case NFA_UPPER_IC: /* [A-Z] */
+ result = ri_upper(curc) || (ireg_ic && ri_lower(curc));
+ ADD_STATE_IF_MATCH(t->state);
+ break;
+
+ case NFA_NUPPER_IC: /* ^[A-Z] */
+ result = curc != NUL
+ && !(ri_upper(curc) || (ireg_ic && ri_lower(curc)));
+ ADD_STATE_IF_MATCH(t->state);
+ break;
+
case NFA_BACKREF1:
case NFA_BACKREF2:
case NFA_BACKREF3:
*** ../vim-7.4.000/src/testdir/test64.in 2013-08-01 17:45:33.000000000 +0200
--- src/testdir/test64.in 2013-08-14 11:50:11.000000000 +0200
***************
*** 289,303 ****
:call add(tl, [2, '.a\%$', " a\n "])
:call add(tl, [2, '.a\%$', " a\n_a", "_a"])
:"
! :"""" Test recognition of some character classes
! :call add(tl, [2, '[0-9]', '8', '8'])
! :call add(tl, [2, '[^0-9]', '8'])
! :call add(tl, [2, '[0-9a-fA-F]*', '0a7', '0a7'])
! :call add(tl, [2, '[^0-9A-Fa-f]\+', '0a7'])
! :call add(tl, [2, '[a-z_A-Z0-9]\+', 'aso_sfoij', 'aso_sfoij'])
! :call add(tl, [2, '[a-z]', 'a', 'a'])
! :call add(tl, [2, '[a-zA-Z]', 'a', 'a'])
! :call add(tl, [2, '[A-Z]', 'a'])
:call add(tl, [2, '\C[^A-Z]\+', 'ABCOIJDEOIFNSD jsfoij sa', ' jsfoij sa'])
:"
:"""" Tests for \z features
--- 289,317 ----
:call add(tl, [2, '.a\%$', " a\n "])
:call add(tl, [2, '.a\%$', " a\n_a", "_a"])
:"
! :"""" Test recognition of character classes
! :call add(tl, [2, '[0-7]\+', 'x0123456789x', '01234567'])
! :call add(tl, [2, '[^0-7]\+', '0a;X+% 897', 'a;X+% 89'])
! :call add(tl, [2, '[0-9]\+', 'x0123456789x', '0123456789'])
! :call add(tl, [2, '[^0-9]\+', '0a;X+% 9', 'a;X+% '])
! :call add(tl, [2, '[0-9a-fA-F]\+', 'x0189abcdefg', '0189abcdef'])
! :call add(tl, [2, '[^0-9A-Fa-f]\+', '0189g;X+% ab', 'g;X+% '])
! :call add(tl, [2, '[a-z_A-Z0-9]\+', ';+aso_SfOij ', 'aso_SfOij'])
! :call add(tl, [2, '[^a-z_A-Z0-9]\+', 'aSo_;+% sfOij', ';+% '])
! :call add(tl, [2, '[a-z_A-Z]\+', '0abyz_ABYZ;', 'abyz_ABYZ'])
! :call add(tl, [2, '[^a-z_A-Z]\+', 'abAB_09;+% yzYZ', '09;+% '])
! :call add(tl, [2, '[a-z]\+', '0abcxyz1', 'abcxyz'])
! :call add(tl, [2, '[a-z]\+', 'AabxyzZ', 'abxyz'])
! :call add(tl, [2, '[^a-z]\+', 'a;X09+% x', ';X09+% '])
! :call add(tl, [2, '[^a-z]\+', 'abX0;%yz', 'X0;%'])
! :call add(tl, [2, '[a-zA-Z]\+', '0abABxzXZ9', 'abABxzXZ'])
! :call add(tl, [2, '[^a-zA-Z]\+', 'ab09_;+ XZ', '09_;+ '])
! :call add(tl, [2, '[A-Z]\+', 'aABXYZz', 'ABXYZ'])
! :call add(tl, [2, '[^A-Z]\+', 'ABx0;%YZ', 'x0;%'])
! :call add(tl, [2, '[a-z]\+\c', '0abxyzABXYZ;', 'abxyzABXYZ'])
! :call add(tl, [2, '[A-Z]\+\c', '0abABxzXZ9', 'abABxzXZ'])
! :call add(tl, [2, '\c[^a-z]\+', 'ab09_;+ XZ', '09_;+ '])
! :call add(tl, [2, '\c[^A-Z]\+', 'ab09_;+ XZ', '09_;+ '])
:call add(tl, [2, '\C[^A-Z]\+', 'ABCOIJDEOIFNSD jsfoij sa', ' jsfoij sa'])
:"
:"""" Tests for \z features
*** ../vim-7.4.000/src/testdir/test64.ok 2013-08-01 18:28:56.000000000 +0200
--- src/testdir/test64.ok 2013-08-14 11:50:37.000000000 +0200
***************
*** 650,679 ****
OK 0 - .a\%$
OK 1 - .a\%$
OK 2 - .a\%$
! OK 0 - [0-9]
! OK 1 - [0-9]
! OK 2 - [0-9]
! OK 0 - [^0-9]
! OK 1 - [^0-9]
! OK 2 - [^0-9]
! OK 0 - [0-9a-fA-F]*
! OK 1 - [0-9a-fA-F]*
! OK 2 - [0-9a-fA-F]*
OK 0 - [^0-9A-Fa-f]\+
OK 1 - [^0-9A-Fa-f]\+
OK 2 - [^0-9A-Fa-f]\+
OK 0 - [a-z_A-Z0-9]\+
OK 1 - [a-z_A-Z0-9]\+
OK 2 - [a-z_A-Z0-9]\+
! OK 0 - [a-z]
! OK 1 - [a-z]
! OK 2 - [a-z]
! OK 0 - [a-zA-Z]
! OK 1 - [a-zA-Z]
! OK 2 - [a-zA-Z]
! OK 0 - [A-Z]
! OK 1 - [A-Z]
! OK 2 - [A-Z]
OK 0 - \C[^A-Z]\+
OK 1 - \C[^A-Z]\+
OK 2 - \C[^A-Z]\+
--- 650,721 ----
OK 0 - .a\%$
OK 1 - .a\%$
OK 2 - .a\%$
! OK 0 - [0-7]\+
! OK 1 - [0-7]\+
! OK 2 - [0-7]\+
! OK 0 - [^0-7]\+
! OK 1 - [^0-7]\+
! OK 2 - [^0-7]\+
! OK 0 - [0-9]\+
! OK 1 - [0-9]\+
! OK 2 - [0-9]\+
! OK 0 - [^0-9]\+
! OK 1 - [^0-9]\+
! OK 2 - [^0-9]\+
! OK 0 - [0-9a-fA-F]\+
! OK 1 - [0-9a-fA-F]\+
! OK 2 - [0-9a-fA-F]\+
OK 0 - [^0-9A-Fa-f]\+
OK 1 - [^0-9A-Fa-f]\+
OK 2 - [^0-9A-Fa-f]\+
OK 0 - [a-z_A-Z0-9]\+
OK 1 - [a-z_A-Z0-9]\+
OK 2 - [a-z_A-Z0-9]\+
! OK 0 - [^a-z_A-Z0-9]\+
! OK 1 - [^a-z_A-Z0-9]\+
! OK 2 - [^a-z_A-Z0-9]\+
! OK 0 - [a-z_A-Z]\+
! OK 1 - [a-z_A-Z]\+
! OK 2 - [a-z_A-Z]\+
! OK 0 - [^a-z_A-Z]\+
! OK 1 - [^a-z_A-Z]\+
! OK 2 - [^a-z_A-Z]\+
! OK 0 - [a-z]\+
! OK 1 - [a-z]\+
! OK 2 - [a-z]\+
! OK 0 - [a-z]\+
! OK 1 - [a-z]\+
! OK 2 - [a-z]\+
! OK 0 - [^a-z]\+
! OK 1 - [^a-z]\+
! OK 2 - [^a-z]\+
! OK 0 - [^a-z]\+
! OK 1 - [^a-z]\+
! OK 2 - [^a-z]\+
! OK 0 - [a-zA-Z]\+
! OK 1 - [a-zA-Z]\+
! OK 2 - [a-zA-Z]\+
! OK 0 - [^a-zA-Z]\+
! OK 1 - [^a-zA-Z]\+
! OK 2 - [^a-zA-Z]\+
! OK 0 - [A-Z]\+
! OK 1 - [A-Z]\+
! OK 2 - [A-Z]\+
! OK 0 - [^A-Z]\+
! OK 1 - [^A-Z]\+
! OK 2 - [^A-Z]\+
! OK 0 - [a-z]\+\c
! OK 1 - [a-z]\+\c
! OK 2 - [a-z]\+\c
! OK 0 - [A-Z]\+\c
! OK 1 - [A-Z]\+\c
! OK 2 - [A-Z]\+\c
! OK 0 - \c[^a-z]\+
! OK 1 - \c[^a-z]\+
! OK 2 - \c[^a-z]\+
! OK 0 - \c[^A-Z]\+
! OK 1 - \c[^A-Z]\+
! OK 2 - \c[^A-Z]\+
OK 0 - \C[^A-Z]\+
OK 1 - \C[^A-Z]\+
OK 2 - \C[^A-Z]\+
*** ../vim-7.4.000/src/version.c 2013-08-10 13:29:20.000000000 +0200
--- src/version.c 2013-08-14 11:54:57.000000000 +0200
***************
*** 729,730 ****
--- 729,732 ----
{ /* Add new patch number below this line */
+ /**/
+ 1,
/**/
--
How many light bulbs does it take to change a person?
/// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\
/// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\
\\\ an exciting new programming language -- http://www.Zimbu.org ///
\\\ help me help AIDS victims -- http://ICCF-Holland.org ///