Add proper memory management to tokenizers

src/g_objdef.c

@@ -220,7 +220,7 @@ static G_odarg G_ObjDef_getArgs(G_dodst *st, char const *arg)
       #define CaseN(c, t, f, ...) \
          case c: \
             a.argv[a.argc++].t = \
-               f(st->expect(tok_number, "number")->text, __VA_ARGS__); \
+               f(st->expect(tok_number, "number")->textV, __VA_ARGS__); \
             break
       CaseN('u', u, strtoui,   NULL, 0);
       CaseN('i', i, strtoi,    NULL, 0);
@@ -248,17 +248,15 @@ static G_odarg G_ObjDef_getArgs(G_dodst *st, char const *arg)
 //
 static void G_ObjDef_getEntAnim(G_dodst *st, G_etdcl *decl)
 {
-   char alias[M_countof(decl->name)];
+   char const *alias = st->expect(tok_identi, "alias name")->textV;
-
-   M_strbufcpy(alias, st->expect(tok_identi, "alias name")->text);
    st->expect(tok_eq, "'='");
 
-   G_andcl *an = G_animsmap.find(st->expect(tok_identi, "anim name")->text);
+   G_andcl *an = G_animsmap.find(st->expect(tok_identi, "anim name")->textV);
    if(an == NULL)             st->throw("anim not defined");
    if(decl->anim.find(alias)) st->throw("alias redefinition");
 
    M_Vec_grow(decl->anims, 1);
-   G_manim *anim = &decl->animsV[decl->animsC++];
+   G_manim *anim = &M_Vec_next(decl->anims);
    M_strbufcpy(anim->name, alias);
    anim->keyhash = M_StrHash(anim->name);
    anim->anim = (G_anima){an->frameV, an->frameC};
@@ -272,7 +270,7 @@ static void G_ObjDef_getEntProp(G_dodst *st, G_entty *type, char const *id)
 {
    if(strcmp(id, "task") == 0)
    {
-      G_ofdcl *fn = G_objfuncs.find(st->expect(tok_identi, "function")->text);
+      G_ofdcl *fn = G_objfuncs.find(st->expect(tok_identi, "function")->textV);
       if(!fn) st->throw("invalid function name");
       type->task = fn->fptr;
    }
@@ -300,7 +298,7 @@ static void G_ObjDef_getEntProp(G_dodst *st, G_entty *type, char const *id)
    else if(strcmp(id, "sprite") == 0)
    {
       type->sprite = DGE_Texture_Get(
-         M_StrCreate(st->expect(tok_string, "sprite name")->text));
+         M_StrCreate(st->expect(tok_string, "sprite name")->textV));
    }
 
    // TODO: change this when access through rvalues is fixed
@@ -325,13 +323,13 @@ static M_texid G_ObjDef_getFrameSprite(char const *fmt, char frame, int i)
 
    for(; *fmt; fmt++)
    {
-      if(*fmt != '%') {M_Vec_grow(buf, 1); bufV[bufC++] = *fmt; continue;}
+      if(*fmt != '%') {M_Vec_grow(buf, 1); M_Vec_next(buf) = *fmt; continue;}
 
       switch(*++fmt)
       {
       default:
-      case '%': M_Vec_grow(buf, 1); bufV[bufC++] =  *fmt; break;
+      case '%': M_Vec_grow(buf, 1); M_Vec_next(buf) =  *fmt; break;
-      case 'c': M_Vec_grow(buf, 1); bufV[bufC++] = frame; break;
+      case 'c': M_Vec_grow(buf, 1); M_Vec_next(buf) = frame; break;
       case 'i':
          M_Vec_grow(buf, 20);
          bufC += snprintf(&bufV[bufC], 20, "%i", i);
@@ -351,10 +349,9 @@ static M_texid G_ObjDef_getFrameSprite(char const *fmt, char frame, int i)
 //
 static void G_ObjDef_parseFrameDef(G_dodst *st)
 {
-   char name[32], frame[32], fmt[32];
+   char const *name  = st->expect(tok_identi, "name")->textV;
-   M_strbufcpy(name,  st->expect(tok_identi, "name")->text);
+   char const *frame = st->expect(tok_identi, "frames")->textV;
-   M_strbufcpy(frame, st->expect(tok_identi, "frames")->text);
+   char const *fmt   = st->expect(tok_string, "format")->textV;
-   M_strbufcpy(fmt,   st->expect(tok_string, "format")->text);
 
    for(int i = 0; frame[i]; i++)
    {
@@ -367,7 +364,7 @@ static void G_ObjDef_parseFrameDef(G_dodst *st)
       if(!decl)
       {
          M_Vec_grow(G_frvec, 1);
-         *(decl = &G_frvecV[G_frvecC++]) = (G_frdcl){};
+         *(decl = &M_Vec_next(G_frvec)) = (G_frdcl){};
          M_strbufcpy(decl->name, key);
          decl->keyhash = M_StrHash(decl->name);
          G_frametab.insert(decl);
@@ -382,17 +379,16 @@ static void G_ObjDef_parseFrameDef(G_dodst *st)
 //
 static void G_ObjDef_parseAnim(G_dodst *st, G_andcl *decl)
 {
-   char name[32], frames[32];
+   char const *name   = st->expect(tok_identi, "name")->textV;
-   M_strbufcpy(name,   st->expect(tok_identi, "name")->text);
+   char const *frames = st->expect(tok_identi, "frame")->textV;
-   M_strbufcpy(frames, st->expect(tok_identi, "frame")->text);
+   int time = strtoi(st->expect(tok_number, "time")->textV, NULL, 0);
-   int time = strtoi(st->expect(tok_number, "time")->text, NULL, 0);
 
    M_Vec_growN(decl->frame, strlen(frames), 0);
    for(char const *frame = frames; *frame; frame++) {
       char    *key = M_StrFmt("%s%c", name, *frame);
       G_frdcl *fr  = G_frametab.find(key);
       if(!fr) st->throw(M_StrFmt("undefined frame '%s'", key));
-      decl->frameV[decl->frameC++] = (G_frame){time, fr->sprite};
+      M_Vec_next(decl->frame) = (G_frame){time, fr->sprite};
    }
 }
 
@@ -404,11 +400,11 @@ static void G_ObjDef_parseAnimDef(G_dodst *st)
    bool ins = true;
    G_andcl *decl;
 
-   __with(char const *name = st->expect(tok_identi, "anim name")->text;)
+   __with(char const *name = st->expect(tok_identi, "anim name")->textV;)
    {
       if((decl = G_animsmap.find(name)) == NULL) {
          M_Vec_grow(G_anvec, 1);
-         *(decl = &G_anvecV[G_anvecC++]) = (G_andcl){};
+         *(decl = &M_Vec_next(G_anvec)) = (G_andcl){};
          M_strbufcpy(decl->name, name);
       } else {
          ins = false;
@@ -434,11 +430,11 @@ static void G_ObjDef_parseEntDef(G_dodst *st)
    bool ins = true;
    G_etdcl *decl;
 
-   __with(char const *name = st->expect(tok_identi, "entity name")->text;)
+   __with(char const *name = st->expect(tok_identi, "entity name")->textV;)
    {
       if((decl = G_enttypes.find(name)) == NULL) {
          M_Vec_grow(G_etvec, 1);
-         *(decl = &G_etvecV[G_etvecC++]) = (G_etdcl){};
+         *(decl = &M_Vec_next(G_etvec)) = (G_etdcl){};
          M_strbufcpy(decl->name, name);
          G_anims_ctor(&decl->anim, 8, 8);
       } else {
@@ -448,7 +444,7 @@ static void G_ObjDef_parseEntDef(G_dodst *st)
 
    if(st->drop(tok_lt)) {
       G_etdcl *base =
-         G_enttypes.find(st->expect(tok_identi, "base entity name")->text);
+         G_enttypes.find(st->expect(tok_identi, "base entity name")->textV);
       if(!base) st->throw("base type not found");
       decl->type = base->type;
    }
@@ -459,10 +455,10 @@ static void G_ObjDef_parseEntDef(G_dodst *st)
       switch(tok->type)
    {
    case tok_identi:
-      if(strcmp(tok->text, "anim") == 0)
+      if(strcmp(tok->textV, "anim") == 0)
          G_ObjDef_getEntAnim(st, decl);
       else
-         G_ObjDef_getEntProp(st, &decl->type, tok->text);
+         G_ObjDef_getEntProp(st, &decl->type, tok->textV);
       break;
    case tok_semico:
       continue;
@@ -506,7 +502,7 @@ void G_ObjDef_Load(char const *fname)
       goto done;
    }
 
-   st.tb.toks = calloc(G_dodTokEnd, sizeof(M_token));
+   st.tb.ctor();
 
    if(setjmp(st.ejmp) == 1)
       goto done;
@@ -515,14 +511,14 @@ void G_ObjDef_Load(char const *fname)
       switch(tok->type)
    {
    case tok_identi:
-      if(strcmp(tok->text, "entity") == 0)
+      if(strcmp(tok->textV, "entity") == 0)
          G_ObjDef_parseEntDef(&st);
-      else if(strcmp(tok->text, "anim") == 0)
+      else if(strcmp(tok->textV, "anim") == 0)
          G_ObjDef_parseAnimDef(&st);
-      else if(strcmp(tok->text, "frame") == 0)
+      else if(strcmp(tok->textV, "frame") == 0)
          G_ObjDef_parseFrameDef(&st);
-      else if(strcmp(tok->text, "include") == 0)
+      else if(strcmp(tok->textV, "include") == 0)
-         G_ObjDef_Load(st.expect(tok_string, "string")->text);
+         G_ObjDef_Load(st.expect(tok_string, "string")->textV);
       else
          st.throw("unknown identifier in toplevel");
       continue;
@@ -533,7 +529,7 @@ void G_ObjDef_Load(char const *fname)
    printf("ObjDef: '%s' loaded.\n", fname);
 
 done:
-   free(st.tb.toks);
+   st.tb.dtor();
    fclose(st.fp);
 }
 
@@ -544,7 +540,7 @@ void G_ObjDef_LoadFunc(char const *name, DGE_CallbackType fptr)
 {
    M_Vec_grow(G_ofvec, 1);
    G_ofvecV[G_ofvecC] = (G_ofdcl){fptr, name, M_StrHash(name)};
-   G_objfuncs.insert(&G_ofvecV[G_ofvecC++]);
+   G_objfuncs.insert(&M_Vec_next(G_ofvec));
 }
 
 //

src/m_darray.h

@@ -20,4 +20,6 @@
 
 #define M_Vec_grow(vec, n) M_Vec_growN(vec, n, 8)
 
+#define M_Vec_next(vec) ((vec##V)[(vec##C)++])
+
 #endif

src/m_tokbuf.c

@@ -2,9 +2,29 @@
 #include "m_tokbuf.h"
 
 #include <string.h>
+#include <stdlib.h>
 
 // Extern Functions ----------------------------------------------------------|
 
+//
+// M_TokBuf_Ctor
+//
+void M_TokBuf_Ctor(M_tkbuf *tb)
+{
+   tb->toks = calloc(tb->bend, sizeof(M_token));
+}
+
+//
+// M_TokBuf_Dtor
+//
+void M_TokBuf_Dtor(M_tkbuf *tb)
+{
+   for(int i = 0; i < tb->bend; i++)
+      free(tb->toks[i].textV);
+
+   free(tb->toks);
+}
+
 //
 // M_TokBuf_Get
 //
@@ -14,6 +34,9 @@ M_token *M_TokBuf_Get(M_tkbuf *tb)
    if(++tb->tpos < tb->tend)
       return &tb->toks[tb->tpos];
 
+   for(int i = 0; i < tb->tend - tb->bbeg; i++)
+      free(tb->toks[i].textV);
+
    memmove(&tb->toks[0], &tb->toks[tb->tend - tb->bbeg],
       sizeof(M_token) * tb->bbeg);
 

src/m_tokbuf.h

@@ -6,6 +6,8 @@
 
 // Extern Functions ----------------------------------------------------------|
 
+void M_TokBuf_Ctor(struct M_tkbuf *tb);
+void M_TokBuf_Dtor(struct M_tkbuf *tb);
 M_token *M_TokBuf_Get(struct M_tkbuf *tb);
 M_token *M_TokBuf_UnGet(struct M_tkbuf *tb);
 M_token *M_TokBuf_ReGet(struct M_tkbuf *tb);
@@ -22,6 +24,8 @@ enum M_tkprc
 
 typedef struct M_tkbuf
 {
+   __prop ctor  {call: M_TokBuf_Ctor (this)}
+   __prop dtor  {call: M_TokBuf_Dtor (this)}
    __prop get   {call: M_TokBuf_Get  (this)}
    __prop unget {call: M_TokBuf_UnGet(this)}
    __prop reget {call: M_TokBuf_ReGet(this)}

src/m_token.c

@@ -1,14 +1,18 @@
 // Copyright © 2017 Project Golan, all rights reserved.
 #include "m_token.h"
+#include "m_darray.h"
 
 #include <ctype.h>
+#include <stdlib.h>
 
-#define M_Tk_getPredStr(s, fn) \
+#define M_Tk_getPredStr(fn) \
    do { \
-      int i; \
+      for(; fn(ch); ch = fgetc(fp)) { \
-      for(i = (s); i < M_countof(tok->text)-1 && fn(ch); i++, ch = fgetc(fp)) \
+         M_Vec_grow(tok->text, 1); \
-         tok->text[i] = ch; \
+         M_Vec_next(tok->text) = ch; \
-      tok->text[i] = '\0'; \
+      } \
+      M_Vec_grow(tok->text, 1); \
+      M_Vec_next(tok->text) = '\0'; \
       ungetc(ch, fp); \
    } while(0)
 
@@ -24,7 +28,11 @@ void M_Tk_Parse(FILE *fp, M_token *tok)
 {
    if(!tok) return;
 
-   tok->text[0] = '\0';
+   if(tok->textV) {
+      free(tok->textV);
+      tok->textV = NULL;
+      tok->textC = tok->textS = 0;
+   }
 
    if(!fp || feof(fp)) {
       tok->type = tok_eof;
@@ -88,7 +96,7 @@ begin:;
       {
          #define incmt(ch) ((ch) != '\n' && !feof(fp))
          ch = fgetc(fp);
-         M_Tk_getPredStr(0, incmt);
+         M_Tk_getPredStr(incmt);
          tok->type = tok_cmtlin;
          #undef incmt
       }
@@ -112,9 +120,12 @@ begin:;
    case '"':  tok->type = tok_string; goto string;
    string: {
       int i, beg;
-      for(i = 0, beg = ch; (ch = fgetc(fp)) != beg && !feof(fp); i++)
+      for(i = 0, beg = ch; (ch = fgetc(fp)) != beg && !feof(fp);) {
-         tok->text[i] = ch;
+         M_Vec_grow(tok->text, 1);
-      tok->text[i] = '\0';
+         M_Vec_next(tok->text) = ch;
+      }
+      M_Vec_grow(tok->text, 1);
+      M_Vec_next(tok->text) = '\0';
       return;
    }
    }
@@ -127,20 +138,22 @@ begin:;
    }
    else if(isdigit(ch) || ch == '.' || ch == '-')
    {
-      tok->text[0] = ch;
+      M_Vec_grow(tok->text, 1);
+      M_Vec_next(tok->text) = ch;
       ch = fgetc(fp);
-      M_Tk_getPredStr(1, M_Tk_isnum);
+      M_Tk_getPredStr(M_Tk_isnum);
       tok->type = tok_number;
    }
    else if(M_Tk_isidenti(ch))
    {
-      M_Tk_getPredStr(0, M_Tk_isidenti);
+      M_Tk_getPredStr(M_Tk_isidenti);
       tok->type = tok_identi;
    }
    else
    {
-      tok->text[0] = ch;
+      M_Vec_grow(tok->text, 2);
-      tok->text[1] = '\0';
+      M_Vec_next(tok->text) = ch;
+      M_Vec_next(tok->text) = '\0';
       tok->type = tok_chrseq;
    }
 }

src/m_token.h

@@ -3,6 +3,7 @@
 #define m_token_h
 
 #include "m_types.h"
+#include "m_darray.h"
 
 #include <stdio.h>
 
@@ -90,7 +91,7 @@ typedef struct M_token
 {
    M_tokty type;
    int     line;
-   char    text[1024];
+   M_Vec_decl(char, text);
 } M_token;
 
 // Extern Functions ----------------------------------------------------------|