marrub/LoveToken
Archived
1
0
Fork 0
Code Releases Activity

Merge 283e091c18 into dcb3534546

Browse Source
This commit is contained in:
Benjamin Moir 2015-08-31 14:52:08 +00:00
commit ad3d850bd3
3 changed files with 229 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lua/tokenizer.lua
View File

@ -63,8 +63,10 @@ LT_BOOL LT_Assert(LT_BOOL assertion, const char *fmt, ...);
LT_AssertInfo LT_CheckAssert(void); LT_AssertInfo LT_CheckAssert(void);
LT_BOOL LT_OpenFile(const char *filePath); LT_BOOL LT_OpenFile(const char *filePath);
LT_BOOL LT_OpenString(const char *str);
void LT_SetPos(int newPos); void LT_SetPos(int newPos);
void LT_CloseFile(void); void LT_CloseFile(void);
void LT_CloseString(void);
char *LT_ReadNumber(void); char *LT_ReadNumber(void);
void LT_ReadString(LT_Token *tk, char term); void LT_ReadString(LT_Token *tk, char term);
@ -79,7 +81,9 @@ local pReturn
function tokenizer:init(initInfo, filePath) function tokenizer:init(initInfo, filePath)
loveToken.LT_Init(initInfo) loveToken.LT_Init(initInfo)
loveToken.LT_OpenFile(filePath) if (filePath ~= nil) then
loveToken.LT_OpenFile(filePath)
end
end end
function tokenizer:assert(assertion, str) function tokenizer:assert(assertion, str)
@ -102,10 +106,20 @@ function tokenizer:openFile(filePath)
return pReturn return pReturn
end end
function tokenizer:openString(str)
pReturn = loveToken.LT_OpenString(str)
tokenizer:checkError()
return pReturn
end
function tokenizer:closeFile() function tokenizer:closeFile()
loveToken.LT_CloseFile() loveToken.LT_CloseFile()
end end
function tokenizer:closeString()
loveToken.LT_CloseString()
end
function tokenizer:quit() function tokenizer:quit()
loveToken.LT_CloseFile() loveToken.LT_CloseFile()
loveToken.LT_Quit() loveToken.LT_Quit()

259
src/lt.c
View File

@ -64,7 +64,10 @@ typedef struct
static LT_GarbageList *gbHead, *gbRover; static LT_GarbageList *gbHead, *gbRover;
static FILE *parseFile; static FILE *parseFile;
static const char *parseStr;
static LT_Config cfg; static LT_Config cfg;
static int parseMode;
static int parseStrPos;
#ifndef LT_NO_ICONV #ifndef LT_NO_ICONV
static iconv_t icDesc; static iconv_t icDesc;
@ -397,6 +400,9 @@ void LT_Quit()
#endif #endif
LT_CloseFile(); LT_CloseFile();
LT_CloseString();
parseMode = LT_PARSE_NONE;
#ifndef __GDCC__ #ifndef __GDCC__
gbRover = gbHead; gbRover = gbHead;
@ -428,10 +434,14 @@ LT_BOOL LT_Assert(LT_BOOL assertion, const char *fmt, ...)
char *asBuffer = LT_Alloc(512); char *asBuffer = LT_Alloc(512);
int place; int place;
if(parseFile != NULL) if(parseMode == LT_PARSE_FILE && parseFile != NULL)
{ {
place = (int)ftell(parseFile); place = (int)ftell(parseFile);
} }
else if (parseMode == LT_PARSE_STR && parseStr != NULL)
{
place = parseStrPos;
}
else else
{ {
place = -1; place = -1;
@ -475,26 +485,41 @@ LT_BOOL LT_OpenFile(const char *filePath)
LT_BOOL LT_OpenFile(__str filePath) LT_BOOL LT_OpenFile(__str filePath)
#endif #endif
{ {
parseMode = LT_PARSE_FILE;
parseFile = fopen(filePath, "r"); parseFile = fopen(filePath, "r");
if(parseFile == NULL) if (LT_Assert(parseFile == NULL, "LT_OpenFile: %s", strerror(errno)));
{
LT_Assert(LT_TRUE, "LT_OpenFile: %s", strerror(errno));
return LT_FALSE; return LT_FALSE;
} return LT_TRUE;
}
LT_BOOL LT_OpenString(const char *str)
{
parseMode = LT_PARSE_STR;
parseStr = str;
parseStrPos = 0;
if (LT_Assert(parseStr == NULL, "LT_OpenString: %s", strerror(errno)))
return LT_FALSE;
return LT_TRUE; return LT_TRUE;
} }
void LT_SetPos(int newPos) void LT_SetPos(int newPos)
{ {
#ifndef __GDCC__ #ifndef __GDCC__
if(fseek(parseFile, newPos, SEEK_SET) != 0) if(parseMode == LT_PARSE_FILE && fseek(parseFile, newPos, SEEK_SET) != 0)
{ {
LT_Assert(ferror(parseFile), "LT_SetPos: %s", strerror(errno)); LT_Assert(ferror(parseFile), "LT_SetPos: %s", strerror(errno));
} }
else if (parseMode == LT_PARSE_STR)
{
parseStrPos = newPos;
}
#else #else
fseek(parseFile, newPos, SEEK_SET); if (parseMode == LT_PARSE_FILE)
fseek(parseFile, newPos, SEEK_SET);
else if (parseMode == LT_PARSE_STR)
parseStrPos = newPos;
#endif #endif
} }
@ -504,6 +529,12 @@ void LT_CloseFile()
{ {
fclose(parseFile); fclose(parseFile);
} }
parseMode = LT_PARSE_NONE;
}
void LT_CloseString()
{
parseMode = LT_PARSE_NONE;
} }
char *LT_ReadNumber() char *LT_ReadNumber()
@ -514,11 +545,17 @@ char *LT_ReadNumber()
while(c != EOF) while(c != EOF)
{ {
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(!isalnum(c) && c != '.') if(!isalnum(c) && c != '.')
{ {
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
break; break;
} }
@ -551,11 +588,14 @@ void LT_ReadString(LT_Token *tk, char term)
{ {
size_t i = 0, strBlocks = 1; size_t i = 0, strBlocks = 1;
char *str = LT_Alloc(TOKEN_STR_BLOCK_LENGTH); char *str = LT_Alloc(TOKEN_STR_BLOCK_LENGTH);
int c; int c = 0;
while(LT_TRUE) while(LT_TRUE)
{ {
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == term) if(c == term)
{ {
@ -576,7 +616,10 @@ void LT_ReadString(LT_Token *tk, char term)
if(c == '\\' && cfg.escapeChars) if(c == '\\' && cfg.escapeChars)
{ {
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(LT_Assert(c == EOF || c == '\n', "LT_ReadString: Unterminated string literal")) if(LT_Assert(c == EOF || c == '\n', "LT_ReadString: Unterminated string literal"))
{ {
@ -643,7 +686,11 @@ char *LT_Escaper(char *str, size_t pos, char escape)
i = 0; i = 0;
while(!exitloop) while(!exitloop)
{ {
int c = fgetc(parseFile); int c = 0;
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
switch(c) switch(c)
{ {
@ -665,7 +712,10 @@ char *LT_Escaper(char *str, size_t pos, char escape)
case 'f': case 'F': i = i * 16 + 0xF; break; case 'f': case 'F': i = i * 16 + 0xF; break;
default: default:
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
str[pos] = i; str[pos] = i;
exitloop = LT_TRUE; exitloop = LT_TRUE;
break; break;
@ -695,12 +745,18 @@ char *LT_Escaper(char *str, size_t pos, char escape)
case '6': i = i * 8 + 06; break; case '6': i = i * 8 + 06; break;
case '7': i = i * 8 + 07; break; case '7': i = i * 8 + 07; break;
default: default:
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
str[pos] = i; str[pos] = i;
return str; return str;
} }
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
} }
str[pos] = i; str[pos] = i;
@ -719,28 +775,44 @@ char *LT_Escaper(char *str, size_t pos, char escape)
LT_Token LT_GetToken() LT_Token LT_GetToken()
{ {
LT_Token tk = { 0 }; LT_Token tk = { 0 };
int c = fgetc(parseFile); int c = 0;
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == EOF) if(c == EOF)
{ {
tk.token = LT_TkNames[TOK_EOF]; tk.token = LT_TkNames[TOK_EOF];
tk.pos = ftell(parseFile); if (parseMode == LT_PARSE_FILE)
tk.pos = ftell(parseFile);
else if (parseMode == LT_PARSE_STR)
tk.pos = parseStrPos;
return tk; return tk;
} }
while(isspace(c) && c != '\n') while(isspace(c) && c != '\n')
{ {
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == EOF) // [marrub] This could have caused issues if there was whitespace before EOF. if(c == EOF) // [marrub] This could have caused issues if there was whitespace before EOF.
{ {
tk.token = LT_TkNames[TOK_EOF]; tk.token = LT_TkNames[TOK_EOF];
tk.pos = ftell(parseFile); if (parseMode == LT_PARSE_FILE)
tk.pos = ftell(parseFile);
else if (parseMode == LT_PARSE_STR)
tk.pos = parseStrPos;
return tk; return tk;
} }
} }
tk.pos = ftell(parseFile) - 1; if (parseMode == LT_PARSE_FILE)
tk.pos = ftell(parseFile) - 1;
else if (parseMode == LT_PARSE_STR)
tk.pos = parseStrPos - 1;
switch(c) switch(c)
{ {
@ -764,8 +836,10 @@ LT_Token LT_GetToken()
// but sometimes I really do care about my sanity. And wrists. // but sometimes I really do care about my sanity. And wrists.
#define DoubleTokDef(ch, t1, t2) \ #define DoubleTokDef(ch, t1, t2) \
case ch: \ case ch: \
c = fgetc(parseFile); \ if (parseMode == LT_PARSE_FILE) \
\ c = fgetc(parseFile); \
else if (parseMode == LT_PARSE_STR) \
c = parseStr[++parseStrPos]; \
if(c == ch) \ if(c == ch) \
{ \ { \
tk.token = LT_TkNames[t2]; \ tk.token = LT_TkNames[t2]; \
@ -773,7 +847,10 @@ LT_Token LT_GetToken()
else \ else \
{ \ { \
tk.token = LT_TkNames[t1]; \ tk.token = LT_TkNames[t1]; \
ungetc(c, parseFile); \ if (parseMode == LT_PARSE_FILE) \
ungetc(c, parseFile); \
else if (parseMode == LT_PARSE_STR) \
c = parseStr[--parseStrPos]; \
} \ } \
\ \
return tk; return tk;
@ -787,7 +864,10 @@ LT_Token LT_GetToken()
// [marrub] Special god damn snowflakes // [marrub] Special god damn snowflakes
case '>': case '>':
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '=') if(c == '=')
{ {
@ -800,12 +880,18 @@ LT_Token LT_GetToken()
else else
{ {
tk.token = LT_TkNames[TOK_CmpGT]; tk.token = LT_TkNames[TOK_CmpGT];
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
} }
return tk; return tk;
case '<': case '<':
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '=') if(c == '=')
{ {
@ -822,12 +908,18 @@ LT_Token LT_GetToken()
else else
{ {
tk.token = LT_TkNames[TOK_CmpLT]; tk.token = LT_TkNames[TOK_CmpLT];
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
} }
return tk; return tk;
case '!': case '!':
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '=') if(c == '=')
{ {
@ -836,12 +928,18 @@ LT_Token LT_GetToken()
else else
{ {
tk.token = LT_TkNames[TOK_Not]; tk.token = LT_TkNames[TOK_Not];
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
} }
return tk; return tk;
case '~': case '~':
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '=') if(c == '=')
{ {
@ -849,7 +947,10 @@ LT_Token LT_GetToken()
} }
else else
{ {
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
tk.token = LT_TkNames[TOK_ChrSeq]; tk.token = LT_TkNames[TOK_ChrSeq];
tk.string = LT_Alloc(2); tk.string = LT_Alloc(2);
tk.string[0] = c; tk.string[0] = c;
@ -861,7 +962,10 @@ LT_Token LT_GetToken()
return tk; return tk;
// [zombie] extra tokens // [zombie] extra tokens
case '/': case '/':
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '/') if(c == '/')
{ {
@ -878,12 +982,18 @@ LT_Token LT_GetToken()
else else
{ {
tk.token = LT_TkNames[TOK_Div]; tk.token = LT_TkNames[TOK_Div];
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
} }
return tk; return tk;
case '*': case '*':
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '/') if(c == '/')
{ {
@ -896,12 +1006,18 @@ LT_Token LT_GetToken()
else else
{ {
tk.token = LT_TkNames[TOK_Mul]; tk.token = LT_TkNames[TOK_Mul];
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
} }
return tk; return tk;
case '-': case '-':
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '-') if(c == '-')
{ {
@ -914,12 +1030,18 @@ LT_Token LT_GetToken()
else else
{ {
tk.token = LT_TkNames[TOK_Sub]; tk.token = LT_TkNames[TOK_Sub];
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
} }
return tk; return tk;
case '+': case '+':
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if (c == '/') if (c == '/')
{ {
@ -932,7 +1054,10 @@ LT_Token LT_GetToken()
else else
{ {
tk.token = LT_TkNames[TOK_Add]; tk.token = LT_TkNames[TOK_Add];
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
} }
return tk; return tk;
@ -983,7 +1108,10 @@ LT_Token LT_GetToken()
if(isdigit(c)) if(isdigit(c))
{ {
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
tk.token = LT_TkNames[TOK_Number]; tk.token = LT_TkNames[TOK_Number];
tk.string = LT_ReadNumber(); tk.string = LT_ReadNumber();
@ -1004,7 +1132,10 @@ LT_Token LT_GetToken()
str[i++] = c; str[i++] = c;
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
} }
str[i++] = '\0'; // [marrub] Completely forgot this line earlier. Really screwed up everything. str[i++] = '\0'; // [marrub] Completely forgot this line earlier. Really screwed up everything.
@ -1016,7 +1147,10 @@ LT_Token LT_GetToken()
} }
#endif #endif
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
tk.token = LT_TkNames[TOK_Identi]; tk.token = LT_TkNames[TOK_Identi];
tk.string = LT_SetGarbage(LT_ReAlloc(str, i)); tk.string = LT_SetGarbage(LT_ReAlloc(str, i));
@ -1037,12 +1171,15 @@ char *LT_ReadLiteral()
{ {
size_t i = 0; size_t i = 0;
size_t strBlocks = 1; size_t strBlocks = 1;
int c; int c = 0;
char *str = LT_Alloc(TOKEN_STR_BLOCK_LENGTH); char *str = LT_Alloc(TOKEN_STR_BLOCK_LENGTH);
while(LT_TRUE) while(LT_TRUE)
{ {
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
if(c == '\r' || c == '\n' || c == EOF) break; if(c == '\r' || c == '\n' || c == EOF) break;
if(i >= (TOKEN_STR_BLOCK_LENGTH * strBlocks)) if(i >= (TOKEN_STR_BLOCK_LENGTH * strBlocks))
@ -1060,24 +1197,44 @@ char *LT_ReadLiteral()
void LT_SkipWhite() void LT_SkipWhite()
{ {
char c = fgetc(parseFile); char c = 0;
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
while(isspace(c) && c != EOF) while(isspace(c) && c != EOF)
{ {
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
} }
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
} }
void LT_SkipWhite2() void LT_SkipWhite2()
{ {
char c = fgetc(parseFile); char c = 0;
if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
while(isspace(c) && c != EOF && c != '\r' && c != '\n') while(isspace(c) && c != EOF && c != '\r' && c != '\n')
{ {
c = fgetc(parseFile); if (parseMode == LT_PARSE_FILE)
c = fgetc(parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[++parseStrPos];
} }
ungetc(c, parseFile); if (parseMode == LT_PARSE_FILE)
ungetc(c, parseFile);
else if (parseMode == LT_PARSE_STR)
c = parseStr[--parseStrPos];
} }

6
src/lt.h
View File

@ -64,6 +64,10 @@ THE SOFTWARE.
#define LT_TRUE 1 #define LT_TRUE 1
#define LT_FALSE 0 #define LT_FALSE 0
#define LT_PARSE_NONE 0
#define LT_PARSE_FILE 1
#define LT_PARSE_STR 2
enum enum
{ {
TOK_Colon, TOK_Comma, TOK_Div, TOK_Mod, TOK_Mul, TOK_Colon, TOK_Comma, TOK_Div, TOK_Mod, TOK_Mul,
@ -146,8 +150,10 @@ LT_DLLEXPORT LT_BOOL LT_EXPORT LT_OpenFile(const char *filePath);
#else #else
LT_DLLEXPORT LT_BOOL LT_EXPORT LT_OpenFile(__str filePath); LT_DLLEXPORT LT_BOOL LT_EXPORT LT_OpenFile(__str filePath);
#endif #endif
LT_DLLEXPORT LT_BOOL LT_EXPORT LT_OpenString(const char *str);
LT_DLLEXPORT void LT_EXPORT LT_SetPos(int newPos); LT_DLLEXPORT void LT_EXPORT LT_SetPos(int newPos);
LT_DLLEXPORT void LT_EXPORT LT_CloseFile(void); LT_DLLEXPORT void LT_EXPORT LT_CloseFile(void);
LT_DLLEXPORT void LT_EXPORT LT_CloseString(void);
LT_DLLEXPORT char *LT_EXPORT LT_ReadNumber(void); LT_DLLEXPORT char *LT_EXPORT LT_ReadNumber(void);
LT_DLLEXPORT void LT_EXPORT LT_ReadString(LT_Token *tk, char term); LT_DLLEXPORT void LT_EXPORT LT_ReadString(LT_Token *tk, char term);