|
@ -13,18 +13,45 @@ |
|
|
* it also tokenizes things like number, string, and symbol literals |
|
|
* it also tokenizes things like number, string, and symbol literals |
|
|
*/ |
|
|
*/ |
|
|
|
|
|
|
|
|
static const token_t nulltok = {.token_type=EMPTY, {.null_token=false}}; |
|
|
static const token_t nulltok = { |
|
|
|
|
|
.token_type=EMPTY, |
|
|
|
|
|
{ |
|
|
|
|
|
.null_token=false |
|
|
|
|
|
} |
|
|
|
|
|
}; |
|
|
|
|
|
|
|
|
static const token_t whitespace_tok = {.token_type=WSPACE, .token={.whitespace=true } }; |
|
|
static const token_t whitespace_tok = { |
|
|
|
|
|
.token_type=WSPACE, |
|
|
|
|
|
.token= { |
|
|
|
|
|
.whitespace=true |
|
|
|
|
|
} |
|
|
|
|
|
}; |
|
|
|
|
|
|
|
|
static const token_t quote_tok = {.token_type=QUOTE, .token={.quote=true} }; |
|
|
static const token_t quote_tok = { |
|
|
|
|
|
.token_type=QUOTE, |
|
|
|
|
|
.token= { |
|
|
|
|
|
.quote=true |
|
|
|
|
|
} |
|
|
|
|
|
}; |
|
|
|
|
|
|
|
|
static const token_t left_paren = {.token_type=PAREN, .token={.parenthesis="("} }; |
|
|
static const token_t left_paren = { |
|
|
|
|
|
.token_type = PAREN, |
|
|
|
|
|
.token = { |
|
|
|
|
|
.parenthesis="(" |
|
|
|
|
|
} |
|
|
|
|
|
}; |
|
|
|
|
|
|
|
|
static const token_t right_paren = {.token_type=PAREN, .token={.parenthesis=")"} }; |
|
|
static const token_t right_paren = { |
|
|
|
|
|
.token_type = PAREN, |
|
|
|
|
|
.token = { |
|
|
|
|
|
.parenthesis=")" |
|
|
|
|
|
} |
|
|
|
|
|
}; |
|
|
|
|
|
|
|
|
static inline char * |
|
|
static inline char * |
|
|
string_head(uint32_t n, char *in, char *out) { |
|
|
string_head(uint32_t n, |
|
|
|
|
|
char *in, |
|
|
|
|
|
char *out) { |
|
|
/* out must be large enough to store the number of characters
|
|
|
/* out must be large enough to store the number of characters
|
|
|
* you want to select from in, plus a byte for the null terminator |
|
|
* you want to select from in, plus a byte for the null terminator |
|
|
*/ |
|
|
*/ |
|
@ -44,7 +71,8 @@ string_head(uint32_t n, char *in, char *out) { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
static inline token_t |
|
|
static inline token_t |
|
|
make_token(token_val_t val, tok_t toktype) { |
|
|
make_token(token_val_t val, |
|
|
|
|
|
tok_t toktype) { |
|
|
token_t result; |
|
|
token_t result; |
|
|
result.token_type = toktype; |
|
|
result.token_type = toktype; |
|
|
result.token = val; |
|
|
result.token = val; |
|
@ -52,7 +80,8 @@ make_token(token_val_t val, tok_t toktype) { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
bool |
|
|
bool |
|
|
push_token(token_stream *tokens, token_t token) { |
|
|
push_token(token_stream *tokens, |
|
|
|
|
|
token_t token) { |
|
|
/*
|
|
|
/*
|
|
|
* Check if tokens points to NULL |
|
|
* Check if tokens points to NULL |
|
|
*/ |
|
|
*/ |
|
@ -120,7 +149,9 @@ peek_token(token_stream *tokens) { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
static inline uint32_t |
|
|
static inline uint32_t |
|
|
match_int(source_t source, uint32_t begin, const uint32_t length) { |
|
|
match_int(source_t source, |
|
|
|
|
|
uint32_t begin, |
|
|
|
|
|
const uint32_t length) { |
|
|
/* Return false if there is no match
|
|
|
/* Return false if there is no match
|
|
|
* otherwise return the position of the end of the match + 1 |
|
|
* otherwise return the position of the end of the match + 1 |
|
|
*/ |
|
|
*/ |
|
@ -144,7 +175,9 @@ match_int(source_t source, uint32_t begin, const uint32_t length) { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
static inline uint32_t |
|
|
static inline uint32_t |
|
|
match_float(source_t source, uint32_t begin, const uint32_t length) { |
|
|
match_float(source_t source, |
|
|
|
|
|
uint32_t begin, |
|
|
|
|
|
const uint32_t length) { |
|
|
/* Return false if there is no match
|
|
|
/* Return false if there is no match
|
|
|
* otherwise: |
|
|
* otherwise: |
|
|
* if there is a leading decimal point and then a valid int match: |
|
|
* if there is a leading decimal point and then a valid int match: |
|
@ -194,7 +227,9 @@ match_float(source_t source, uint32_t begin, const uint32_t length) { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
static inline uint32_t |
|
|
static inline uint32_t |
|
|
match_identifier(source_t source, uint32_t begin, const uint32_t length) { |
|
|
match_identifier(source_t source, |
|
|
|
|
|
uint32_t begin, |
|
|
|
|
|
const uint32_t length) { |
|
|
|
|
|
|
|
|
/* Return false if there is no match
|
|
|
/* Return false if there is no match
|
|
|
* if there is a match for any characters that are not: |
|
|
* if there is a match for any characters that are not: |
|
@ -225,7 +260,9 @@ match_identifier(source_t source, uint32_t begin, const uint32_t length) { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
static inline uint32_t |
|
|
static inline uint32_t |
|
|
match_symbol(source_t source, uint32_t begin, const uint32_t length) { |
|
|
match_symbol(source_t source, |
|
|
|
|
|
uint32_t begin, |
|
|
|
|
|
const uint32_t length) { |
|
|
uint32_t i; |
|
|
uint32_t i; |
|
|
assert(source != NULL); |
|
|
assert(source != NULL); |
|
|
assert(length > 0); |
|
|
assert(length > 0); |
|
@ -256,7 +293,9 @@ extract_token(uint32_t position, |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
token_stream |
|
|
token_stream |
|
|
tokenize(source_t source, uint32_t begin, const uint32_t length) { |
|
|
tokenize(source_t source, |
|
|
|
|
|
uint32_t begin, |
|
|
|
|
|
const uint32_t length) { |
|
|
/*
|
|
|
/*
|
|
|
* Remember to free everything from this struct |
|
|
* Remember to free everything from this struct |
|
|
* for example, token_stack.tokens will not necessarily be |
|
|
* for example, token_stack.tokens will not necessarily be |
|
@ -397,7 +436,8 @@ tokenize(source_t source, uint32_t begin, const uint32_t length) { |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
int |
|
|
int |
|
|
free_token(const void *key, const void *val) { |
|
|
free_token(const void *key, |
|
|
|
|
|
const void *val) { |
|
|
/* silence warnings about unused parameters, key and val point to the same data*/ |
|
|
/* silence warnings about unused parameters, key and val point to the same data*/ |
|
|
(void)key; |
|
|
(void)key; |
|
|
free((char *)val); |
|
|
free((char *)val); |
|
|