patch 7.4.1142

Problem:    Cannot define keyword characters for a syntax file.
Solution:   Add the ":syn iskeyword" command. (Christian Brabandt)

Author: brammool

runtime/doc/options.txt

@@ -1,4 +1,4 @@
-*options.txt*	For Vim version 7.4.  Last change: 2016 Jan 09
+*options.txt*	For Vim version 7.4.  Last change: 2016 Jan 19
 
 
 		  VIM REFERENCE MANUAL	  by Bram Moolenaar
@@ -4489,6 +4489,8 @@ A jump table for the options with a short description can be found at |Q_op|.
 	'*', '"' and '|' (so that CTRL-] on a command finds the help for that
 	command).
 	When the 'lisp' option is on the '-' character is always included.
+	This option also influences syntax highlighting, unless the syntax
+	uses |:syn-iskeyword|.
 	NOTE: This option is set to the Vi default value when 'compatible' is
 	set and to the Vim default value when 'compatible' is reset.
 

runtime/doc/syntax.txt

@@ -1,4 +1,4 @@
-*syntax.txt*	For Vim version 7.4.  Last change: 2015 Dec 19
+*syntax.txt*	For Vim version 7.4.  Last change: 2016 Jan 19
 
 
 		  VIM REFERENCE MANUAL	  by Bram Moolenaar
@@ -3438,6 +3438,32 @@ SPELL CHECKING						*:syn-spell*
 
 	To activate spell checking the 'spell' option must be set.
 
+SYNTAX ISKEYWORD SETTING				*:syn-iskeyword*
+
+:sy[ntax] iskeyword [clear | {option}]
+	This defines the keyword characters.  It's like the 'iskeyword' option
+	for but only applies to syntax highlighting.
+
+	clear:		Syntax specific iskeyword setting is disabled and the
+			buffer-local 'iskeyword' setting is used.
+	{option}        Set the syntax 'iskeyword' option to a new value. 
+
+	Example: >
+  :syntax iskeyword @,48-57,192-255,$,_
+<
+	This would set the syntax specific iskeyword option to include all
+	alphabetic characters, plus the numeric characters, all accented
+	characters and also includes the "_" and the "$".
+
+	If no argument is given, the current value will be output.
+
+	Setting this option influences what |/\k| matches in syntax patterns
+	and also determines where |:syn-keywords| will be checked for a new
+	match.
+
+	It is recommended when writing syntax files, to use this command
+	to the correct value for the specific syntax language and not change
+	the 'iskeyword' option.
 
 DEFINING KEYWORDS					*:syn-keyword*
 
@@ -3469,6 +3495,7 @@ DEFINING KEYWORDS					*:syn-keyword*
 	isn't, the keyword will never be recognized.
 	Multi-byte characters can also be used.  These do not have to be in
 	'iskeyword'.
+	See |:syn-iskeyword| for defining syntax specific iskeyword settings.
 
 	A keyword always has higher priority than a match or region, the
 	keyword is used if more than one item matches.	Keywords do not nest

src/buffer.c

@@ -1955,6 +1955,7 @@ free_buf_options(buf, free_p_ff)
     clear_string_option(&buf->b_p_nf);
 #ifdef FEAT_SYN_HL
     clear_string_option(&buf->b_p_syn);
+    clear_string_option(&buf->b_s.b_syn_isk);
 #endif
 #ifdef FEAT_SPELL
     clear_string_option(&buf->b_s.b_p_spc);

src/option.c

@@ -5494,6 +5494,7 @@ check_buf_options(buf)
 #endif
 #ifdef FEAT_SYN_HL
     check_string_option(&buf->b_p_syn);
+    check_string_option(&buf->b_s.b_syn_isk);
 #endif
 #ifdef FEAT_SPELL
     check_string_option(&buf->b_s.b_p_spc);
@@ -10821,6 +10822,7 @@ buf_copy_options(buf, flags)
 	    /* Don't copy 'syntax', it must be set */
 	    buf->b_p_syn = empty_option;
 	    buf->b_p_smc = p_smc;
+	    buf->b_s.b_syn_isk = empty_option;
 #endif
 #ifdef FEAT_SPELL
 	    buf->b_s.b_p_spc = vim_strsave(p_spc);

src/structs.h

@@ -1362,6 +1362,8 @@ typedef struct {
 #if !defined(FEAT_SYN_HL) && !defined(FEAT_SPELL)
     int		dummy;
 #endif
+    char_u	b_syn_chartab[32];	/* syntax iskeyword option */
+    char_u	*b_syn_isk;		/* iskeyword option */
 } synblock_T;
 
 

src/syntax.c

@@ -376,6 +376,8 @@ static int	current_line_id = 0;	/* unique number for current line */
 #define CUR_STATE(idx)	((stateitem_T *)(current_state.ga_data))[idx]
 
 static void syn_sync __ARGS((win_T *wp, linenr_T lnum, synstate_T *last_valid));
+static void save_chartab(char_u *chartab);
+static void restore_chartab(char_u *chartab);
 static int syn_match_linecont __ARGS((linenr_T lnum));
 static void syn_start_line __ARGS((void));
 static void syn_update_ends __ARGS((int startofline));
@@ -458,6 +460,7 @@ static void add_keyword __ARGS((char_u *name, int id, int flags, short *cont_in_
 static char_u *get_group_name __ARGS((char_u *arg, char_u **name_end));
 static char_u *get_syn_options __ARGS((char_u *arg, syn_opt_arg_T *opt, int *conceal_char));
 static void syn_cmd_include __ARGS((exarg_T *eap, int syncing));
+static void syn_cmd_iskeyword __ARGS((exarg_T *eap, int syncing));
 static void syn_cmd_keyword __ARGS((exarg_T *eap, int syncing));
 static void syn_cmd_match __ARGS((exarg_T *eap, int syncing));
 static void syn_cmd_region __ARGS((exarg_T *eap, int syncing));
@@ -984,6 +987,24 @@ syn_sync(wp, start_lnum, last_valid)
     validate_current_state();
 }
 
+    static void
+save_chartab(char_u *chartab)
+{
+    if (syn_block->b_syn_isk != empty_option)
+    {
+	mch_memmove(chartab, syn_buf->b_chartab, (size_t)32);
+	mch_memmove(syn_buf->b_chartab, syn_win->w_s->b_syn_chartab,
+								  (size_t)32);
+    }
+}
+
+    static void
+restore_chartab(char_u *chartab)
+{
+    if (syn_win->w_s->b_syn_isk != empty_option)
+	mch_memmove(syn_buf->b_chartab, chartab, (size_t)32);
+}
+
 /*
  * Return TRUE if the line-continuation pattern matches in line "lnum".
  */
@@ -993,14 +1014,18 @@ syn_match_linecont(lnum)
 {
     regmmatch_T regmatch;
     int r;
+    char_u	buf_chartab[32];  /* chartab array for syn iskyeyword */
 
     if (syn_block->b_syn_linecont_prog != NULL)
     {
+	/* use syntax iskeyword option */
+	save_chartab(buf_chartab);
 	regmatch.rmm_ic = syn_block->b_syn_linecont_ic;
 	regmatch.regprog = syn_block->b_syn_linecont_prog;
 	r = syn_regexec(&regmatch, lnum, (colnr_T)0,
 				IF_SYN_TIME(&syn_block->b_syn_linecont_time));
 	syn_block->b_syn_linecont_prog = regmatch.regprog;
+	restore_chartab(buf_chartab);
 	return r;
     }
     return FALSE;
@@ -1891,6 +1916,7 @@ syn_current_attr(syncing, displaying, can_spell, keep_state)
     lpos_T	pos;
     int		lc_col;
     reg_extmatch_T *cur_extmatch = NULL;
+    char_u	buf_chartab[32];  /* chartab array for syn iskyeyword */
     char_u	*line;		/* current line.  NOTE: becomes invalid after
 				   looking for a pattern match! */
 
@@ -1945,6 +1971,9 @@ syn_current_attr(syncing, displaying, can_spell, keep_state)
      * avoid matching the same item in the same position twice. */
     ga_init2(&zero_width_next_ga, (int)sizeof(int), 10);
 
+    /* use syntax iskeyword option */
+    save_chartab(buf_chartab);
+
     /*
      * Repeat matching keywords and patterns, to find contained items at the
      * same column.  This stops when there are no extra matches at the current
@@ -1956,6 +1985,7 @@ syn_current_attr(syncing, displaying, can_spell, keep_state)
 	keep_next_list = FALSE;
 	syn_id = 0;
 
+
 	/*
 	 * 1. Check for a current state.
 	 *    Only when there is no current state, or if the current state may
@@ -2309,6 +2339,8 @@ syn_current_attr(syncing, displaying, can_spell, keep_state)
 
     } while (found_match);
 
+    restore_chartab(buf_chartab);
+
     /*
      * Use attributes from the current state, if within its highlighting.
      * If not, use attributes from the current-but-one state, etc.
@@ -2915,6 +2947,7 @@ find_endpos(idx, startpos, m_endpos, hl_endpos, flagsp, end_endpos,
     lpos_T	pos;
     char_u	*line;
     int		had_match = FALSE;
+    char_u	buf_chartab[32];  /* chartab array for syn option iskyeyword */
 
     /* just in case we are invoked for a keyword */
     if (idx < 0)
@@ -2961,6 +2994,10 @@ find_endpos(idx, startpos, m_endpos, hl_endpos, flagsp, end_endpos,
     matchcol = startpos->col;	/* start looking for a match at sstart */
     start_idx = idx;		/* remember the first END pattern. */
     best_regmatch.startpos[0].col = 0;		/* avoid compiler warning */
+
+    /* use syntax iskeyword option */
+    save_chartab(buf_chartab);
+
     for (;;)
     {
 	/*
@@ -3117,6 +3154,8 @@ find_endpos(idx, startpos, m_endpos, hl_endpos, flagsp, end_endpos,
     if (!had_match)
 	m_endpos->lnum = 0;
 
+    restore_chartab(buf_chartab);
+
     /* Remove external matches. */
     unref_extmatch(re_extmatch_in);
     re_extmatch_in = NULL;
@@ -3481,6 +3520,57 @@ syn_cmd_spell(eap, syncing)
     redraw_win_later(curwin, NOT_VALID);
 }
 
+/*
+ * Handle ":syntax iskeyword" command.
+ */
+    static void
+syn_cmd_iskeyword(eap, syncing)
+    exarg_T	*eap;
+    int		syncing UNUSED;
+{
+    char_u	*arg = eap->arg;
+    char_u	save_chartab[32];
+    char_u	*save_isk;
+
+    if (eap->skip)
+	return;
+
+    arg = skipwhite(arg);
+    if (*arg == NUL)
+    {
+	MSG_PUTS("\n");
+	MSG_PUTS(_("syntax iskeyword "));
+	if (curwin->w_s->b_syn_isk != empty_option)
+	    msg_outtrans(curwin->w_s->b_syn_isk);
+	else
+	    msg_outtrans((char_u *)"not set");
+    }
+    else
+    {
+	if (STRNICMP(arg, "clear", 5) == 0)
+	{
+	    mch_memmove(curwin->w_s->b_syn_chartab, curbuf->b_chartab,
+								  (size_t)32);
+	    clear_string_option(&curwin->w_s->b_syn_isk);
+	}
+	else
+	{
+	    mch_memmove(save_chartab, curbuf->b_chartab, (size_t)32);
+	    save_isk = curbuf->b_p_isk;
+	    curbuf->b_p_isk = vim_strsave(arg);
+
+	    buf_init_chartab(curbuf, FALSE);
+	    mch_memmove(curwin->w_s->b_syn_chartab, curbuf->b_chartab,
+								  (size_t)32);
+	    mch_memmove(curbuf->b_chartab, save_chartab, (size_t)32);
+	    clear_string_option(&curwin->w_s->b_syn_isk);
+	    curwin->w_s->b_syn_isk = curbuf->b_p_isk;
+	    curbuf->b_p_isk = save_isk;
+	}
+    }
+    redraw_win_later(curwin, NOT_VALID);
+}
+
 /*
  * Clear all syntax info for one buffer.
  */
@@ -3523,6 +3613,7 @@ syntax_clear(block)
 #ifdef FEAT_FOLDING
     block->b_syn_folditems = 0;
 #endif
+    clear_string_option(&block->b_syn_isk);
 
     /* free the stored states */
     syn_stack_free_all(block);
@@ -3569,8 +3660,9 @@ syntax_sync_clear()
     curwin->w_s->b_syn_linecont_prog = NULL;
     vim_free(curwin->w_s->b_syn_linecont_pat);
     curwin->w_s->b_syn_linecont_pat = NULL;
+    clear_string_option(&curwin->w_s->b_syn_isk);
 
-    syn_stack_free_all(curwin->w_s);		/* Need to recompute all syntax. */
+    syn_stack_free_all(curwin->w_s);	/* Need to recompute all syntax. */
 }
 
 /*
@@ -3777,6 +3869,7 @@ syn_cmd_reset(eap, syncing)
     eap->nextcmd = check_nextcmd(eap->arg);
     if (!eap->skip)
     {
+	clear_string_option(&curwin->w_s->b_syn_isk);
 	set_internal_string_var((char_u *)"syntax_cmd", (char_u *)"reset");
 	do_cmdline_cmd((char_u *)"runtime! syntax/syncolor.vim");
 	do_unlet((char_u *)"g:syntax_cmd", TRUE);
@@ -6253,6 +6346,7 @@ static struct subcommand subcommands[] =
     {"conceal",		syn_cmd_conceal},
     {"enable",		syn_cmd_enable},
     {"include",		syn_cmd_include},
+    {"iskeyword",	syn_cmd_iskeyword},
     {"keyword",		syn_cmd_keyword},
     {"list",		syn_cmd_list},
     {"manual",		syn_cmd_manual},
@@ -6331,6 +6425,7 @@ ex_ownsyntax(eap)
 	clear_string_option(&curwin->w_s->b_p_spf);
 	clear_string_option(&curwin->w_s->b_p_spl);
 #endif
+	clear_string_option(&curwin->w_s->b_syn_isk);
     }
 
     /* save value of b:current_syntax */

src/testdir/Make_all.mak

@@ -175,6 +175,7 @@ NEW_TESTS = test_arglist.res \
 	    test_increment.res \
 	    test_perl.res \
 	    test_quickfix.res \
+	    test_syntax.res \
 	    test_viminfo.res \
 	    test_viml.res \
 	    test_alot.res

src/testdir/test_syntax.vim

@@ -0,0 +1,67 @@
+" Test for syntax and syntax iskeyword option
+
+if !has("syntax")
+  finish
+endif
+
+func GetSyntaxItem(pat)
+  let c = ''
+  let a = ['a', getreg('a'), getregtype('a')]
+  0
+  redraw!
+  call search(a:pat, 'W')
+  let synid = synID(line('.'), col('.'), 1)
+  while synid == synID(line('.'), col('.'), 1)
+    norm! v"ay
+    " stop at whitespace
+    if @a =~# '\s'
+      break
+    endif
+    let c .= @a
+    norm! l
+  endw
+  call call('setreg', a)
+  0
+  return c
+endfunc
+
+func Test_syn_iskeyword()
+  new
+  call setline(1, [
+	\ 'CREATE TABLE FOOBAR(',
+	\ '    DLTD_BY VARCHAR2(100)',
+	\ ');',
+  	\ ''])
+
+  syntax on
+  set ft=sql
+  syn match SYN /C\k\+\>/
+  hi link SYN ErrorMsg
+  call assert_equal('DLTD_BY', GetSyntaxItem('DLTD'))
+  /\<D\k\+\>/:norm! ygn
+  call assert_equal('DLTD_BY', @0)
+  redir @c
+  syn iskeyword
+  redir END
+  call assert_equal("\nsyntax iskeyword not set", @c)
+
+  syn iskeyword @,48-57,_,192-255
+  redir @c
+  syn iskeyword
+  redir END
+  call assert_equal("\nsyntax iskeyword @,48-57,_,192-255", @c)
+
+  setlocal isk-=_
+  call assert_equal('DLTD_BY', GetSyntaxItem('DLTD'))
+  /\<D\k\+\>/:norm! ygn
+  let b2=@0
+  call assert_equal('DLTD', @0)
+
+  syn iskeyword clear
+  redir @c
+  syn iskeyword
+  redir END
+  call assert_equal("\nsyntax iskeyword not set", @c)
+
+  quit!
+endfunc