From 1fd19a37d10895f6d1e2fec25b6117047a80f9ab Mon Sep 17 00:00:00 2001 From: nisstyre56 Date: Thu, 7 Aug 2014 17:22:42 -0400 Subject: [PATCH] add string field --- tokenize.c | 12 ++++++------ tokenize.py | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tokenize.c b/tokenize.c index fd20396..4aaa1a7 100644 --- a/tokenize.c +++ b/tokenize.c @@ -102,7 +102,7 @@ push_token(token_stream *tokens, /* We've reached the maximum stack size * So we must try to increase that by GROWTH_SIZE */ - token_t *new_tokens = xrealloc(tokens->tokens, sizeof(token_t) * (max + GROWTH_SIZE)); + token_t *new_tokens = xrealloc(tokens->tokens, sizeof (token_t) * (max + GROWTH_SIZE)); if (!new_tokens) { printf("Could not allocate enough memory for the token stack\n"); exit(EXIT_FAILURE); @@ -311,7 +311,7 @@ tokenize(source_t source, const char *current_token_val; token_stream token_stack; token_val_t current_token; - token_t *tokens = xcalloc(STACK_SIZE, sizeof(token_t)); + token_t *tokens = xcalloc(STACK_SIZE, sizeof (token_t)); hsh_HashTable token_memo = hsh_create(NULL, NULL); @@ -353,7 +353,7 @@ tokenize(source_t source, else { source[position] = lookahead; assert(position > begin); - current_token_val = xcalloc(((position - begin) + 1), sizeof(char)); + current_token_val = xcalloc(((position - begin) + 1), sizeof (char)); CHECK(current_token_val); extract_token(position, begin, source, current_token_val); hsh_insert(token_stack.memo, current_token_val, current_token_val); @@ -374,7 +374,7 @@ tokenize(source_t source, assert(position <= length); source[position] = lookahead; - current_token_val = xcalloc(((position - begin) + 1), sizeof(char)); + current_token_val = xcalloc(((position - begin) + 1), sizeof (char)); CHECK(current_token_val); extract_token(position, begin, source, current_token_val); hsh_insert(token_stack.memo, current_token_val, current_token_val); @@ -395,7 +395,7 @@ tokenize(source_t source, assert(position <= length); source[position] = lookahead; - current_token_val = xcalloc(((position - begin) + 1), sizeof(char)); + current_token_val = xcalloc(((position - begin) + 1), sizeof (char)); CHECK(current_token_val); extract_token(position, begin, source, current_token_val); hsh_insert(token_stack.memo, current_token_val, current_token_val); @@ -421,7 +421,7 @@ tokenize(source_t source, assert(position <= length); source[position] = lookahead; - current_token_val = xcalloc(((position - begin) + 1), sizeof(char)); + current_token_val = xcalloc(((position - begin) + 1), sizeof (char)); CHECK(current_token_val); extract_token(position, begin, source, current_token_val); hsh_insert(token_stack.memo, current_token_val, current_token_val); diff --git a/tokenize.py b/tokenize.py index e7612c3..51aa84f 100644 --- a/tokenize.py +++ b/tokenize.py @@ -18,6 +18,7 @@ class TokenValT(Union): ("integer", c_char_p), ("floating", c_char_p), ("parenthesis", c_char_p), + ("string", c_char_p), ("quote", c_bool), ("whitespace", c_bool), ("null_token", c_bool)] @@ -49,4 +50,4 @@ def tokenize(source): line = " '''' a b" xs = list(tokenize(line)) -print xs +print(xs)