marrub
/
LoveToken
Archived
1
0
Fork 0
Code Releases Activity

Merge 283e091c18 into dcb3534546

pull/2/merge
Benjamin Moir 2015-08-31 14:52:08 +00:00
parent dcb3534546 283e091c18
commit ad3d850bd3
3 changed files with 229 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lua/tokenizer.lua
View File

@ -63,8 +63,10 @@ LT_BOOL LT_Assert(LT_BOOL assertion, const char *fmt, ...);
LT_AssertInfo LT_CheckAssert(void);
LT_BOOL LT_OpenFile(const char *filePath);
LT_BOOL LT_OpenString(const char *str);
void LT_SetPos(int newPos);
void LT_CloseFile(void);
void LT_CloseString(void);
char *LT_ReadNumber(void);
void LT_ReadString(LT_Token *tk, char term);
@ -79,7 +81,9 @@ local pReturn
function tokenizer:init(initInfo, filePath)
loveToken.LT_Init(initInfo)
loveToken.LT_OpenFile(filePath)
if (filePath ~= nil) then
loveToken.LT_OpenFile(filePath)
end
end
function tokenizer:assert(assertion, str)
@ -102,10 +106,20 @@ function tokenizer:openFile(filePath)
return pReturn
end
function tokenizer:openString(str)
pReturn = loveToken.LT_OpenString(str)
tokenizer:checkError()
return pReturn
end
function tokenizer:closeFile()
loveToken.LT_CloseFile()
end
function tokenizer:closeString()
loveToken.LT_CloseString()
end
function tokenizer:quit()
loveToken.LT_CloseFile()
loveToken.LT_Quit()

259
src/lt.c
View File

@ -64,7 +64,10 @@ typedef struct
static LT_GarbageList *gbHead, *gbRover;
static FILE *parseFile;
static const char *parseStr;
static LT_Config cfg;
static int parseMode;
static int parseStrPos;
#ifndef LT_NO_ICONV
static iconv_t icDesc;
@ -397,6 +400,9 @@ void LT_Quit()
#endif
LT_CloseFile();
LT_CloseString();
parseMode = LT_PARSE_NONE;
#ifndef __GDCC__
gbRover = gbHead;
@ -428,10 +434,14 @@ LT_BOOL LT_Assert(LT_BOOL assertion, const char *fmt, ...)
char *asBuffer = LT_Alloc(512);
int place;
if(parseFile != NULL)
if(parseMode == LT_PARSE_FILE && parseFile != NULL)
{
place = (int)ftell(parseFile);
}
else if (parseMode == LT_PARSE_STR && parseStr != NULL)
{
place = parseStrPos;
}
else
{
place = -1;
@ -475,26 +485,41 @@ LT_BOOL LT_OpenFile(const char *filePath)
LT_BOOL LT_OpenFile(__str filePath)
#endif
{
parseMode = LT_PARSE_FILE;
parseFile = fopen(filePath, "r");
if(parseFile == NULL)
{
LT_Assert(LT_TRUE, "LT_OpenFile: %s", strerror(errno));
if (LT_Assert(parseFile == NULL, "LT_OpenFile: %s", strerror(errno)));
return LT_FALSE;
}
return LT_TRUE;
}
LT_BOOL LT_OpenString(const char *str)
{
parseMode = LT_PARSE_STR;
parseStr = str;
parseStrPos = 0;
if (LT_Assert(parseStr == NULL, "LT_OpenString: %s", strerror(errno)))
return LT_FALSE;
return LT_TRUE;
}
void LT_SetPos(int newPos)
{
#ifndef __GDCC__
if(fseek(parseFile, newPos, SEEK_SET) != 0)
if(parseMode == LT_PARSE_FILE && fseek(parseFile, newPos, SEEK_SET) != 0)
{
LT_Assert(ferror(parseFile), "LT_SetPos: %s", strerror(errno));
}
else if (parseMode == LT_PARSE_STR)
{
parseStrPos = newPos;
}
#else
fseek(parseFile, newPos, SEEK_SET);
if (parseMode == LT_PARSE_FILE)
fseek(parseFile, newPos, SEEK_SET);
else if (parseMode == LT_PARSE_STR)
parseStrPos = newPos;
#endif
}
@ -504,6 +529,12 @@ void LT_CloseFile()
{
fclose(parseFile);
}
parseMode = LT_PARSE_NONE;
}
void LT_CloseString()
{
parseMode = LT_PARSE_NONE;
}
char *LT_ReadNumber()
@ -514,11 +545,17 @@ char *LT_ReadNumber()
while(c != EOF)
{
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(!isalnum(c) && c != '.')
{
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
break;
}
@ -551,11 +588,14 @@ void LT_ReadString(LT_Token *tk, char term)
{
size_t i = 0, strBlocks = 1;
char *str = LT_Alloc(TOKEN_STR_BLOCK_LENGTH);
int c;
int c = 0;
while(LT_TRUE)
{
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == term)
{
@ -576,7 +616,10 @@ void LT_ReadString(LT_Token *tk, char term)
if(c == '\\' && cfg.escapeChars)
{
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(LT_Assert(c == EOF || c == '\n', "LT_ReadString: Unterminated string literal"))
{
@ -643,7 +686,11 @@ char *LT_Escaper(char *str, size_t pos, char escape)
i = 0;
while(!exitloop)
{
int c = fgetc(parseFile);
int c = 0;
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
switch(c)
{
@ -665,7 +712,10 @@ char *LT_Escaper(char *str, size_t pos, char escape)
case 'f': case 'F': i = i * 16 + 0xF; break;
default:
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
str[pos] = i;
exitloop = LT_TRUE;
break;
@ -695,12 +745,18 @@ char *LT_Escaper(char *str, size_t pos, char escape)
case '6': i = i * 8 + 06; break;
case '7': i = i * 8 + 07; break;
default:
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
str[pos] = i;
return str;
}
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
}
str[pos] = i;
@ -719,28 +775,44 @@ char *LT_Escaper(char *str, size_t pos, char escape)
LT_Token LT_GetToken()
{
LT_Token tk = { 0 };
int c = fgetc(parseFile);
int c = 0;
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == EOF)
{
tk.token = LT_TkNames[TOK_EOF];
tk.pos = ftell(parseFile);
if (parseMode == LT_PARSE_FILE)
tk.pos = ftell(parseFile);
else if (parseMode == LT_PARSE_STR)
tk.pos = parseStrPos;
return tk;
}
while(isspace(c) && c != '\n')
{
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == EOF) // [marrub] This could have caused issues if there was whitespace before EOF.
{
tk.token = LT_TkNames[TOK_EOF];
tk.pos = ftell(parseFile);
if (parseMode == LT_PARSE_FILE)
tk.pos = ftell(parseFile);
else if (parseMode == LT_PARSE_STR)
tk.pos = parseStrPos;
return tk;
}
}
tk.pos = ftell(parseFile) - 1;
if (parseMode == LT_PARSE_FILE)
tk.pos = ftell(parseFile) - 1;
else if (parseMode == LT_PARSE_STR)
tk.pos = parseStrPos - 1;
switch(c)
{
@ -764,8 +836,10 @@ LT_Token LT_GetToken()
// but sometimes I really do care about my sanity. And wrists.
#define DoubleTokDef(ch, t1, t2) \
case ch: \
c = fgetc(parseFile); \
\
if (parseMode == LT_PARSE_FILE) \
c = fgetc(parseFile); \
else if (parseMode == LT_PARSE_STR) \
c = parseStr[++parseStrPos]; \
if(c == ch) \
{ \
tk.token = LT_TkNames[t2]; \
@ -773,7 +847,10 @@ LT_Token LT_GetToken()
else \
{ \
tk.token = LT_TkNames[t1]; \
ungetc(c, parseFile); \
if (parseMode == LT_PARSE_FILE) \
ungetc(c, parseFile); \
else if (parseMode == LT_PARSE_STR) \
c = parseStr[--parseStrPos]; \
} \
\
return tk;
@ -787,7 +864,10 @@ LT_Token LT_GetToken()
// [marrub] Special god damn snowflakes
case '>':
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '=')
{
@ -800,12 +880,18 @@ LT_Token LT_GetToken()
else
{
tk.token = LT_TkNames[TOK_CmpGT];
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
}
return tk;
case '<':
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '=')
{
@ -822,12 +908,18 @@ LT_Token LT_GetToken()
else
{
tk.token = LT_TkNames[TOK_CmpLT];
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
}
return tk;
case '!':
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '=')
{
@ -836,12 +928,18 @@ LT_Token LT_GetToken()
else
{
tk.token = LT_TkNames[TOK_Not];
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
}
return tk;
case '~':
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '=')
{
@ -849,7 +947,10 @@ LT_Token LT_GetToken()
}
else
{
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
tk.token = LT_TkNames[TOK_ChrSeq];
tk.string = LT_Alloc(2);
tk.string[0] = c;
@ -861,7 +962,10 @@ LT_Token LT_GetToken()
return tk;
// [zombie] extra tokens
case '/':
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '/')
{
@ -878,12 +982,18 @@ LT_Token LT_GetToken()
else
{
tk.token = LT_TkNames[TOK_Div];
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
}
return tk;
case '*':
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '/')
{
@ -896,12 +1006,18 @@ LT_Token LT_GetToken()
else
{
tk.token = LT_TkNames[TOK_Mul];
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
}
return tk;
case '-':
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '-')
{
@ -914,12 +1030,18 @@ LT_Token LT_GetToken()
else
{
tk.token = LT_TkNames[TOK_Sub];
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
}
return tk;
case '+':
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if (c == '/')
{
@ -932,7 +1054,10 @@ LT_Token LT_GetToken()
else
{
tk.token = LT_TkNames[TOK_Add];
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
}
return tk;
@ -983,7 +1108,10 @@ LT_Token LT_GetToken()
if(isdigit(c))
{
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
tk.token = LT_TkNames[TOK_Number];
tk.string = LT_ReadNumber();
@ -1004,7 +1132,10 @@ LT_Token LT_GetToken()
str[i++] = c;
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
}
str[i++] = '\0'; // [marrub] Completely forgot this line earlier. Really screwed up everything.
@ -1016,7 +1147,10 @@ LT_Token LT_GetToken()
}
#endif
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
tk.token = LT_TkNames[TOK_Identi];
tk.string = LT_SetGarbage(LT_ReAlloc(str, i));
@ -1037,12 +1171,15 @@ char *LT_ReadLiteral()
{
size_t i = 0;
size_t strBlocks = 1;
int c;
int c = 0;
char *str = LT_Alloc(TOKEN_STR_BLOCK_LENGTH);
while(LT_TRUE)
{
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '\r' || c == '\n' || c == EOF) break;
if(i >= (TOKEN_STR_BLOCK_LENGTH * strBlocks))
@ -1060,24 +1197,44 @@ char *LT_ReadLiteral()
void LT_SkipWhite()
{
char c = fgetc(parseFile);
char c = 0;
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
while(isspace(c) && c != EOF)
{
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
}
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
}
void LT_SkipWhite2()
{
char c = fgetc(parseFile);
char c = 0;
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
while(isspace(c) && c != EOF && c != '\r' && c != '\n')
{
c = fgetc(parseFile);
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
}
ungetc(c, parseFile);
if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
}

6
src/lt.h
View File

@ -64,6 +64,10 @@ THE SOFTWARE.
#define LT_TRUE 1
#define LT_FALSE 0
#define LT_PARSE_NONE 0
#define LT_PARSE_FILE 1
#define LT_PARSE_STR 2
enum
{
TOK_Colon, TOK_Comma, TOK_Div, TOK_Mod, TOK_Mul,
@ -146,8 +150,10 @@ LT_DLLEXPORT LT_BOOL LT_EXPORT LT_OpenFile(const char *filePath);
#else
LT_DLLEXPORT LT_BOOL LT_EXPORT LT_OpenFile(__str filePath);
#endif
LT_DLLEXPORT LT_BOOL LT_EXPORT LT_OpenString(const char *str);
LT_DLLEXPORT void LT_EXPORT LT_SetPos(int newPos);
LT_DLLEXPORT void LT_EXPORT LT_CloseFile(void);
LT_DLLEXPORT void LT_EXPORT LT_CloseString(void);
LT_DLLEXPORT char *LT_EXPORT LT_ReadNumber(void);
LT_DLLEXPORT void LT_EXPORT LT_ReadString(LT_Token *tk, char term);