To: vim_dev@googlegroups.com

Subject: Patch 7.4.609

Fcc: outbox

From: Bram Moolenaar <Bram@moolenaar.net>

Mime-Version: 1.0

Content-Type: text/plain; charset=UTF-8

Content-Transfer-Encoding: 8bit

------------

Patch 7.4.609

Problem:    For complicated list and dict use the garbage collector can run

	    out of stack space.

Solution:   Use a stack of dicts and lists to be marked, thus making it

	    iterative instead of recursive. (Ben Fritz)

Files:	    src/eval.c, src/if_lua.c, src/if_py_both.h, src/if_python.c,

	    src/if_python3.c, src/proto/eval.pro, src/proto/if_lua.pro,

	    src/proto/if_python.pro, src/proto/if_python3.pro, src/structs.h

*** ../vim-7.4.608/src/eval.c	2015-01-27 15:18:55.152333309 +0100

--- src/eval.c	2015-02-03 12:41:59.468525906 +0100

***************

*** 93,99 ****

      char_u	*ll_newkey;	/* New key for Dict in alloc. mem or NULL. */

  } lval_T;

- 

  static char *e_letunexp	= N_("E18: Unexpected characters in :let");

  static char *e_listidx = N_("E684: list index out of range: %ld");

  static char *e_undefvar = N_("E121: Undefined variable: %s");

--- 93,98 ----

***************

*** 6811,6816 ****

--- 6810,6816 ----

  garbage_collect()

  {

      int		copyID;

+     int		abort = FALSE;

      buf_T	*buf;

      win_T	*wp;

      int		i;

***************

*** 6841,6922 ****

       * the item is referenced elsewhere the funccal must not be freed. */

      for (fc = previous_funccal; fc != NULL; fc = fc->caller)

      {

! 	set_ref_in_ht(&fc->l_vars.dv_hashtab, copyID + 1);

! 	set_ref_in_ht(&fc->l_avars.dv_hashtab, copyID + 1);

      }

      /* script-local variables */

      for (i = 1; i <= ga_scripts.ga_len; ++i)

! 	set_ref_in_ht(&SCRIPT_VARS(i), copyID);

      /* buffer-local variables */

      for (buf = firstbuf; buf != NULL; buf = buf->b_next)

! 	set_ref_in_item(&buf->b_bufvar.di_tv, copyID);

      /* window-local variables */

      FOR_ALL_TAB_WINDOWS(tp, wp)

! 	set_ref_in_item(&wp->w_winvar.di_tv, copyID);

  #ifdef FEAT_AUTOCMD

      if (aucmd_win != NULL)

! 	set_ref_in_item(&aucmd_win->w_winvar.di_tv, copyID);

  #endif

  #ifdef FEAT_WINDOWS

      /* tabpage-local variables */

      for (tp = first_tabpage; tp != NULL; tp = tp->tp_next)

! 	set_ref_in_item(&tp->tp_winvar.di_tv, copyID);

  #endif

      /* global variables */

!     set_ref_in_ht(&globvarht, copyID);

      /* function-local variables */

      for (fc = current_funccal; fc != NULL; fc = fc->caller)

      {

! 	set_ref_in_ht(&fc->l_vars.dv_hashtab, copyID);

! 	set_ref_in_ht(&fc->l_avars.dv_hashtab, copyID);

      }

      /* v: vars */

!     set_ref_in_ht(&vimvarht, copyID);

  #ifdef FEAT_LUA

!     set_ref_in_lua(copyID);

  #endif

  #ifdef FEAT_PYTHON

!     set_ref_in_python(copyID);

  #endif

  #ifdef FEAT_PYTHON3

!     set_ref_in_python3(copyID);

  #endif

!     /*

!      * 2. Free lists and dictionaries that are not referenced.

!      */

!     did_free = free_unref_items(copyID);

! 

!     /*

!      * 3. Check if any funccal can be freed now.

!      */

!     for (pfc = &previous_funccal; *pfc != NULL; )

      {

! 	if (can_free_funccal(*pfc, copyID))

  	{

! 	    fc = *pfc;

! 	    *pfc = fc->caller;

! 	    free_funccal(fc, TRUE);

! 	    did_free = TRUE;

! 	    did_free_funccal = TRUE;

  	}

! 	else

! 	    pfc = &(*pfc)->caller;

      }

-     if (did_free_funccal)

- 	/* When a funccal was freed some more items might be garbage

- 	 * collected, so run again. */

- 	(void)garbage_collect();

      return did_free;

  }

--- 6841,6935 ----

       * the item is referenced elsewhere the funccal must not be freed. */

      for (fc = previous_funccal; fc != NULL; fc = fc->caller)

      {

! 	abort = abort || set_ref_in_ht(&fc->l_vars.dv_hashtab, copyID + 1,

! 									NULL);

! 	abort = abort || set_ref_in_ht(&fc->l_avars.dv_hashtab, copyID + 1,

! 									NULL);

      }

      /* script-local variables */

      for (i = 1; i <= ga_scripts.ga_len; ++i)

! 	abort = abort || set_ref_in_ht(&SCRIPT_VARS(i), copyID, NULL);

      /* buffer-local variables */

      for (buf = firstbuf; buf != NULL; buf = buf->b_next)

! 	abort = abort || set_ref_in_item(&buf->b_bufvar.di_tv, copyID,

! 								  NULL, NULL);

      /* window-local variables */

      FOR_ALL_TAB_WINDOWS(tp, wp)

! 	abort = abort || set_ref_in_item(&wp->w_winvar.di_tv, copyID,

! 								  NULL, NULL);

  #ifdef FEAT_AUTOCMD

      if (aucmd_win != NULL)

! 	abort = abort || set_ref_in_item(&aucmd_win->w_winvar.di_tv, copyID,

! 								  NULL, NULL);

  #endif

  #ifdef FEAT_WINDOWS

      /* tabpage-local variables */

      for (tp = first_tabpage; tp != NULL; tp = tp->tp_next)

! 	abort = abort || set_ref_in_item(&tp->tp_winvar.di_tv, copyID,

! 								  NULL, NULL);

  #endif

      /* global variables */

!     abort = abort || set_ref_in_ht(&globvarht, copyID, NULL);

      /* function-local variables */

      for (fc = current_funccal; fc != NULL; fc = fc->caller)

      {

! 	abort = abort || set_ref_in_ht(&fc->l_vars.dv_hashtab, copyID, NULL);

! 	abort = abort || set_ref_in_ht(&fc->l_avars.dv_hashtab, copyID, NULL);

      }

      /* v: vars */

!     abort = abort || set_ref_in_ht(&vimvarht, copyID, NULL);

  #ifdef FEAT_LUA

!     abort = abort || set_ref_in_lua(copyID);

  #endif

  #ifdef FEAT_PYTHON

!     abort = abort || set_ref_in_python(copyID);

  #endif

  #ifdef FEAT_PYTHON3

!     abort = abort || set_ref_in_python3(copyID);

  #endif

!     if (!abort)

      {

! 	/*

! 	 * 2. Free lists and dictionaries that are not referenced.

! 	 */

! 	did_free = free_unref_items(copyID);

! 

! 	/*

! 	 * 3. Check if any funccal can be freed now.

! 	 */

! 	for (pfc = &previous_funccal; *pfc != NULL; )

  	{

! 	    if (can_free_funccal(*pfc, copyID))

! 	    {

! 		fc = *pfc;

! 		*pfc = fc->caller;

! 		free_funccal(fc, TRUE);

! 		did_free = TRUE;

! 		did_free_funccal = TRUE;

! 	    }

! 	    else

! 		pfc = &(*pfc)->caller;

  	}

! 	if (did_free_funccal)

! 	    /* When a funccal was freed some more items might be garbage

! 	     * collected, so run again. */

! 	    (void)garbage_collect();

!     }

!     else if (p_verbose > 0)

!     {

! 	verb_msg((char_u *)_("Not enough memory to set references, garbage collection aborted!"));

      }

      return did_free;

  }

***************

*** 6976,7023 ****

  /*

   * Mark all lists and dicts referenced through hashtab "ht" with "copyID".

   */

!     void

! set_ref_in_ht(ht, copyID)

!     hashtab_T	*ht;

!     int		copyID;

  {

      int		todo;

      hashitem_T	*hi;

!     todo = (int)ht->ht_used;

!     for (hi = ht->ht_array; todo > 0; ++hi)

! 	if (!HASHITEM_EMPTY(hi))

  	{

! 	    --todo;

! 	    set_ref_in_item(&HI2DI(hi)->di_tv, copyID);

  	}

  }

  /*

   * Mark all lists and dicts referenced through list "l" with "copyID".

   */

!     void

! set_ref_in_list(l, copyID)

      list_T	*l;

      int		copyID;

  {

!     listitem_T *li;

!     for (li = l->lv_first; li != NULL; li = li->li_next)

! 	set_ref_in_item(&li->li_tv, copyID);

  }

  /*

   * Mark all lists and dicts referenced through typval "tv" with "copyID".

   */

!     void

! set_ref_in_item(tv, copyID)

!     typval_T	*tv;

!     int		copyID;

  {

      dict_T	*dd;

      list_T	*ll;

      switch (tv->v_type)

      {

--- 6989,7100 ----

  /*

   * Mark all lists and dicts referenced through hashtab "ht" with "copyID".

+  * "list_stack" is used to add lists to be marked.  Can be NULL.

+  *

+  * Returns TRUE if setting references failed somehow.

   */

!     int

! set_ref_in_ht(ht, copyID, list_stack)

!     hashtab_T	    *ht;

!     int		    copyID;

!     list_stack_T    **list_stack;

  {

      int		todo;

+     int		abort = FALSE;

      hashitem_T	*hi;

+     hashtab_T	*cur_ht;

+     ht_stack_T	*ht_stack = NULL;

+     ht_stack_T	*tempitem;

!     cur_ht = ht;

!     for (;;)

!     {

! 	if (!abort)

  	{

! 	    /* Mark each item in the hashtab.  If the item contains a hashtab

! 	     * it is added to ht_stack, if it contains a list it is added to

! 	     * list_stack. */

! 	    todo = (int)cur_ht->ht_used;

! 	    for (hi = cur_ht->ht_array; todo > 0; ++hi)

! 		if (!HASHITEM_EMPTY(hi))

! 		{

! 		    --todo;

! 		    abort = abort || set_ref_in_item(&HI2DI(hi)->di_tv, copyID,

! 						       &ht_stack, list_stack);

! 		}

  	}

+ 

+ 	if (ht_stack == NULL)

+ 	    break;

+ 

+ 	/* take an item from the stack */

+ 	cur_ht = ht_stack->ht;

+ 	tempitem = ht_stack;

+ 	ht_stack = ht_stack->prev;

+ 	free(tempitem);

+     }

+ 

+     return abort;

  }

  /*

   * Mark all lists and dicts referenced through list "l" with "copyID".

+  * "ht_stack" is used to add hashtabs to be marked.  Can be NULL.

+  *

+  * Returns TRUE if setting references failed somehow.

   */

!     int

! set_ref_in_list(l, copyID, ht_stack)

      list_T	*l;

      int		copyID;

+     ht_stack_T	**ht_stack;

  {

!     listitem_T	 *li;

!     int		 abort = FALSE;

!     list_T	 *cur_l;

!     list_stack_T *list_stack = NULL;

!     list_stack_T *tempitem;

! 

!     cur_l = l;

!     for (;;)

!     {

! 	if (!abort)

! 	    /* Mark each item in the list.  If the item contains a hashtab

! 	     * it is added to ht_stack, if it contains a list it is added to

! 	     * list_stack. */

! 	    for (li = cur_l->lv_first; !abort && li != NULL; li = li->li_next)

! 		abort = abort || set_ref_in_item(&li->li_tv, copyID,

! 						       ht_stack, &list_stack);

! 	if (list_stack == NULL)

! 	    break;

! 

! 	/* take an item from the stack */

! 	cur_l = list_stack->list;

! 	tempitem = list_stack;

! 	list_stack = list_stack->prev;

! 	free(tempitem);

!     }

!     return abort;

  }

  /*

   * Mark all lists and dicts referenced through typval "tv" with "copyID".

+  * "list_stack" is used to add lists to be marked.  Can be NULL.

+  * "ht_stack" is used to add hashtabs to be marked.  Can be NULL.

+  *

+  * Returns TRUE if setting references failed somehow.

   */

!     int

! set_ref_in_item(tv, copyID, ht_stack, list_stack)

!     typval_T	    *tv;

!     int		    copyID;

!     ht_stack_T	    **ht_stack;

!     list_stack_T    **list_stack;

  {

      dict_T	*dd;

      list_T	*ll;

+     int		abort = FALSE;

      switch (tv->v_type)

      {

***************

*** 7027,7033 ****

  	    {

  		/* Didn't see this dict yet. */

  		dd->dv_copyID = copyID;

! 		set_ref_in_ht(&dd->dv_hashtab, copyID);

  	    }

  	    break;

--- 7104,7126 ----

  	    {

  		/* Didn't see this dict yet. */

  		dd->dv_copyID = copyID;

! 		if (ht_stack == NULL)

! 		{

! 		    abort = set_ref_in_ht(&dd->dv_hashtab, copyID, list_stack);

! 		}

! 		else

! 		{

! 		    ht_stack_T *newitem = (ht_stack_T*)malloc(

! 							  sizeof(ht_stack_T));

! 		    if (newitem == NULL)

! 			abort = TRUE;

! 		    else

! 		    {

! 			newitem->ht = &dd->dv_hashtab;

! 			newitem->prev = *ht_stack;

! 			*ht_stack = newitem;

! 		    }

! 		}

  	    }

  	    break;

***************

*** 7037,7047 ****

  	    {

  		/* Didn't see this list yet. */

  		ll->lv_copyID = copyID;

! 		set_ref_in_list(ll, copyID);

  	    }

  	    break;

      }

!     return;

  }

  /*

--- 7130,7156 ----

  	    {

  		/* Didn't see this list yet. */

  		ll->lv_copyID = copyID;

! 		if (list_stack == NULL)

! 		{

! 		    abort = set_ref_in_list(ll, copyID, ht_stack);

! 		}

! 		else

! 		{

! 		    list_stack_T *newitem = (list_stack_T*)malloc(

! 							sizeof(list_stack_T));

! 		    if (newitem == NULL)

! 			abort = TRUE;

! 		    else

! 		    {

! 			newitem->list = ll;

! 			newitem->prev = *list_stack;

! 			*list_stack = newitem;

! 		    }

! 		}

  	    }

  	    break;

      }

!     return abort;

  }

  /*

*** ../vim-7.4.608/src/if_lua.c	2014-05-07 17:31:32.473182497 +0200

--- src/if_lua.c	2015-02-03 12:45:41.978204997 +0100

***************

*** 1523,1534 ****

      static int

  luaV_setref (lua_State *L)

  {

!     int copyID = lua_tointeger(L, 1);

!     typval_T tv;

      luaV_getfield(L, LUAVIM_LIST);

      luaV_getfield(L, LUAVIM_DICT);

      lua_pushnil(L);

!     while (lua_next(L, lua_upvalueindex(1)) != 0) /* traverse cache table */

      {

  	lua_getmetatable(L, -1);

  	if (lua_rawequal(L, -1, 2)) /* list? */

--- 1523,1536 ----

      static int

  luaV_setref (lua_State *L)

  {

!     int		copyID = lua_tointeger(L, 1);

!     int		abort = FALSE;

!     typval_T	tv;

! 

      luaV_getfield(L, LUAVIM_LIST);

      luaV_getfield(L, LUAVIM_DICT);

      lua_pushnil(L);

!     while (!abort && lua_next(L, lua_upvalueindex(1)) != 0) /* traverse cache table */

      {

  	lua_getmetatable(L, -1);

  	if (lua_rawequal(L, -1, 2)) /* list? */

***************

*** 1542,1550 ****

  	    tv.vval.v_dict = (dict_T *) lua_touserdata(L, 4); /* key */

  	}

  	lua_pop(L, 2); /* metatable and value */

! 	set_ref_in_item(&tv, copyID);

      }

!     return 0;

  }

      static int

--- 1544,1552 ----

  	    tv.vval.v_dict = (dict_T *) lua_touserdata(L, 4); /* key */

  	}

  	lua_pop(L, 2); /* metatable and value */

! 	abort = set_ref_in_item(&tv, copyID, NULL, NULL);

      }

!     lua_pushinteger(L, abort);

  }

      static int

***************

*** 1770,1782 ****

      lua_call(L, 3, 0);

  }

!     void

  set_ref_in_lua (int copyID)

  {

!     if (!lua_isopen()) return;

!     luaV_getfield(L, LUAVIM_SETREF);

!     lua_pushinteger(L, copyID);

!     lua_call(L, 1, 0);

  }

  #endif

--- 1772,1794 ----

      lua_call(L, 3, 0);

  }

!     int

  set_ref_in_lua (int copyID)

  {

!     int aborted = 0;

! 

!     if (lua_isopen())

!     {

! 	luaV_getfield(L, LUAVIM_SETREF);

! 	/* call the function with 1 arg, getting 1 result back */

! 	lua_pushinteger(L, copyID);

! 	lua_call(L, 1, 1);

! 	/* get the result */

! 	aborted = lua_tointeger(L, -1);

! 	/* pop result off the stack */

! 	lua_pop(L, 1);

!     }

!     return aborted;

  }

  #endif

*** ../vim-7.4.608/src/if_py_both.h	2014-12-17 14:45:56.095854545 +0100

--- src/if_py_both.h	2015-02-03 12:46:46.629530177 +0100

***************

*** 5502,5535 ****

      PyErr_Clear();

  }

!     static void

  set_ref_in_py(const int copyID)

  {

      pylinkedlist_T	*cur;

      dict_T	*dd;

      list_T	*ll;

      if (lastdict != NULL)

! 	for(cur = lastdict ; cur != NULL ; cur = cur->pll_prev)

  	{

  	    dd = ((DictionaryObject *) (cur->pll_obj))->dict;

  	    if (dd->dv_copyID != copyID)

  	    {

  		dd->dv_copyID = copyID;

! 		set_ref_in_ht(&dd->dv_hashtab, copyID);

  	    }

  	}

      if (lastlist != NULL)

! 	for(cur = lastlist ; cur != NULL ; cur = cur->pll_prev)

  	{

  	    ll = ((ListObject *) (cur->pll_obj))->list;

  	    if (ll->lv_copyID != copyID)

  	    {

  		ll->lv_copyID = copyID;

! 		set_ref_in_list(ll, copyID);

  	    }

  	}

  }

      static int

--- 5502,5542 ----

      PyErr_Clear();

  }

!     static int

  set_ref_in_py(const int copyID)

  {

      pylinkedlist_T	*cur;

      dict_T	*dd;

      list_T	*ll;

+     int		abort = FALSE;

      if (lastdict != NULL)

!     {

! 	for(cur = lastdict ; !abort && cur != NULL ; cur = cur->pll_prev)

  	{

  	    dd = ((DictionaryObject *) (cur->pll_obj))->dict;

  	    if (dd->dv_copyID != copyID)

  	    {

  		dd->dv_copyID = copyID;

! 		abort = abort || set_ref_in_ht(&dd->dv_hashtab, copyID, NULL);

  	    }

  	}

+     }

      if (lastlist != NULL)

!     {

! 	for(cur = lastlist ; !abort && cur != NULL ; cur = cur->pll_prev)

  	{

  	    ll = ((ListObject *) (cur->pll_obj))->list;

  	    if (ll->lv_copyID != copyID)

  	    {

  		ll->lv_copyID = copyID;

! 		abort = abort || set_ref_in_list(ll, copyID, NULL);

  	    }

  	}

+     }

+ 

+     return abort;

  }

      static int

*** ../vim-7.4.608/src/if_python.c	2014-07-23 16:56:56.587876204 +0200

--- src/if_python.c	2015-02-03 12:05:13.471422753 +0100

***************

*** 1567,1574 ****

  }

  #endif /* Python 1.4 */

!     void

  set_ref_in_python (int copyID)

  {

!     set_ref_in_py(copyID);

  }

--- 1567,1574 ----

  }

  #endif /* Python 1.4 */

!     int

  set_ref_in_python (int copyID)

  {

!     return set_ref_in_py(copyID);

  }

*** ../vim-7.4.608/src/if_python3.c	2014-03-30 16:11:37.180530823 +0200

--- src/if_python3.c	2015-02-03 12:05:13.471422753 +0100

***************

*** 1649,1656 ****

      }

  }

!     void

  set_ref_in_python3 (int copyID)

  {

!     set_ref_in_py(copyID);

  }

--- 1649,1656 ----

      }

  }

!     int

  set_ref_in_python3 (int copyID)

  {

!     int set_ref_in_py(copyID);

  }

*** ../vim-7.4.608/src/proto/eval.pro	2014-12-17 14:36:10.363090985 +0100

--- src/proto/eval.pro	2015-02-03 12:47:58.472780049 +0100

***************

*** 62,70 ****

  void list_insert __ARGS((list_T *l, listitem_T *ni, listitem_T *item));

  void vimlist_remove __ARGS((list_T *l, listitem_T *item, listitem_T *item2));

  int garbage_collect __ARGS((void));

! void set_ref_in_ht __ARGS((hashtab_T *ht, int copyID));

! void set_ref_in_list __ARGS((list_T *l, int copyID));

! void set_ref_in_item __ARGS((typval_T *tv, int copyID));

  dict_T *dict_alloc __ARGS((void));

  void dict_unref __ARGS((dict_T *d));

  void dict_free __ARGS((dict_T *d, int recurse));

--- 62,70 ----

  void list_insert __ARGS((list_T *l, listitem_T *ni, listitem_T *item));

  void vimlist_remove __ARGS((list_T *l, listitem_T *item, listitem_T *item2));

  int garbage_collect __ARGS((void));

! int set_ref_in_ht __ARGS((hashtab_T *ht, int copyID, list_stack_T **list_stack));

! int set_ref_in_list __ARGS((list_T *l, int copyID, ht_stack_T **ht_stack));

! int set_ref_in_item __ARGS((typval_T *tv, int copyID, ht_stack_T **ht_stack, list_stack_T **list_stack));

  dict_T *dict_alloc __ARGS((void));

  void dict_unref __ARGS((dict_T *d));

  void dict_free __ARGS((dict_T *d, int recurse));

*** ../vim-7.4.608/src/proto/if_lua.pro	2012-04-05 16:41:35.000000000 +0200

--- src/proto/if_lua.pro	2015-02-03 12:05:13.475422711 +0100

***************

*** 7,11 ****

  void lua_buffer_free __ARGS((buf_T *buf));

  void lua_window_free __ARGS((win_T *win));

  void do_luaeval __ARGS((char_u *str, typval_T *arg, typval_T *rettv));

! void set_ref_in_lua __ARGS((int copyID));

  /* vim: set ft=c : */

--- 7,11 ----

  void lua_buffer_free __ARGS((buf_T *buf));

  void lua_window_free __ARGS((win_T *win));

  void do_luaeval __ARGS((char_u *str, typval_T *arg, typval_T *rettv));

! int set_ref_in_lua __ARGS((int copyID));

  /* vim: set ft=c : */

*** ../vim-7.4.608/src/proto/if_python.pro	2013-08-10 13:37:15.000000000 +0200

--- src/proto/if_python.pro	2015-02-03 12:48:02.936733431 +0100

***************

*** 9,13 ****

  void python_window_free __ARGS((win_T *win));

  void python_tabpage_free __ARGS((tabpage_T *tab));

  void do_pyeval __ARGS((char_u *str, typval_T *rettv));

! void set_ref_in_python __ARGS((int copyID));

  /* vim: set ft=c : */

--- 9,13 ----

  void python_window_free __ARGS((win_T *win));