Merge pull request #5015 from rmosolgo/remove-prev-token

Remove previous_token tracking and Token class

graphql-c_parser/ext/graphql_c_parser_ext/lexer.rl

@@ -198,7 +198,6 @@ typedef struct Meta {
   char *query_cstr;
   char *pe;
   VALUE tokens;
-  VALUE previous_token;
   int dedup_identifiers;
   int reject_numbers_followed_by_names;
   int preceeded_by_number;
@@ -289,12 +288,13 @@ void emit(TokenType tt, char *ts, char *te, Meta *meta) {
       if (meta->reject_numbers_followed_by_names && meta->preceeded_by_number) {
         VALUE mGraphQL = rb_const_get_at(rb_cObject, rb_intern("GraphQL"));
         VALUE mCParser = rb_const_get_at(mGraphQL, rb_intern("CParser"));
+        VALUE prev_token = rb_ary_entry(meta->tokens, -1);
         VALUE exception = rb_funcall(
             mCParser, rb_intern("prepare_number_name_parse_error"), 5,
             LONG2NUM(meta->line),
             LONG2NUM(meta->col),
             rb_str_new_cstr(meta->query_cstr),
-            rb_ary_entry(meta->previous_token, 3),
+            rb_ary_entry(prev_token, 3),
             rb_utf8_str_new(ts, te - ts)
         );
         rb_exc_raise(exception);
@@ -373,21 +373,18 @@ void emit(TokenType tt, char *ts, char *te, Meta *meta) {
       }
     }
 
-    VALUE token = rb_ary_new_from_args(6,
+    VALUE token = rb_ary_new_from_args(5,
       token_sym,
       rb_int2inum(meta->line),
       rb_int2inum(meta->col),
       token_content,
-      meta->previous_token,
       INT2FIX(200 + (int)tt)
     );
 
-    // COMMENTs are retained as `previous_token` but aren't pushed to the normal token list
     if (tt != COMMENT) {
       rb_ary_push(meta->tokens, token);
     }
     meta->preceeded_by_number = this_token_is_number;
-    meta->previous_token = token;
   }
   // Bump the column counter for the next token
   meta->col += te - ts;
@@ -403,7 +400,7 @@ VALUE tokenize(VALUE query_rbstr, int fstring_identifiers, int reject_numbers_fo
   char *ts = 0;
   char *te = 0;
   VALUE tokens = rb_ary_new();
-  struct Meta meta_s = {1, 1, p, pe, tokens, Qnil, fstring_identifiers, reject_numbers_followed_by_names, 0, max_tokens, 0};
+  struct Meta meta_s = {1, 1, p, pe, tokens, fstring_identifiers, reject_numbers_followed_by_names, 0, max_tokens, 0};
   Meta *meta = &meta_s;
 
   %% write init;

graphql-c_parser/ext/graphql_c_parser_ext/parser.c

@@ -3250,7 +3250,7 @@ int yylex (YYSTYPE *lvalp, VALUE parser, VALUE filename) {
     return YYEOF;
   }
   rb_ivar_set(parser, rb_intern("@next_token_index"), INT2FIX(next_token_idx + 1));
-  VALUE token_type_rb_int = rb_ary_entry(next_token, 5);
+  VALUE token_type_rb_int = rb_ary_entry(next_token, 4);
   int next_token_type = FIX2INT(token_type_rb_int);
   if (next_token_type == 241) { // BAD_UNICODE_ESCAPE
     VALUE mGraphQL = rb_const_get_at(rb_cObject, rb_intern("GraphQL"));

graphql-c_parser/ext/graphql_c_parser_ext/parser.y

@@ -874,7 +874,7 @@ int yylex (YYSTYPE *lvalp, VALUE parser, VALUE filename) {
     return YYEOF;
   }
   rb_ivar_set(parser, rb_intern("@next_token_index"), INT2FIX(next_token_idx + 1));
-  VALUE token_type_rb_int = rb_ary_entry(next_token, 5);
+  VALUE token_type_rb_int = rb_ary_entry(next_token, 4);
   int next_token_type = FIX2INT(token_type_rb_int);
   if (next_token_type == 241) { // BAD_UNICODE_ESCAPE
     VALUE mGraphQL = rb_const_get_at(rb_cObject, rb_intern("GraphQL"));

graphql-c_parser/lib/graphql/c_parser.rb

@@ -75,7 +75,6 @@ def self.tokenize(graphql_string, intern_identifiers: false, max_tokens: nil)
               1,
               1,
               graphql_string,
-              nil, # prev token
               241 # BAD_UNICODE_ESCAPE in lexer.rl
             ]
           ]

lib/graphql/language.rb

@@ -9,7 +9,6 @@
 require "graphql/language/cache"
 require "graphql/language/parser"
 require "graphql/language/static_visitor"
-require "graphql/language/token"
 require "graphql/language/visitor"
 require "graphql/language/definition_slice"
 require "strscan"

lib/graphql/language/lexer.rb

@@ -345,17 +345,14 @@ def self.replace_escaped_characters_in_place(raw_string)
       def self.tokenize(string)
         lexer = GraphQL::Language::Lexer.new(string)
         tokens = []
-        prev_token = nil
         while (token_name = lexer.advance)
           new_token = [
             token_name,
             lexer.line_number,
             lexer.column_number,
             lexer.debug_token_value(token_name),
-            prev_token,
           ]
           tokens << new_token
-          prev_token = new_token
         end
         tokens
       end

spec/graphql/language/clexer_spec.rb

@@ -12,9 +12,8 @@ def assert_bad_unicode(string, _message = nil)
 
     it "makes tokens like the other lexer" do
       str = "{ f1(type: \"str\") ...F2 }\nfragment F2 on SomeType { f2 }"
-      # Don't include prev_token here
       tokens = GraphQL.scan_with_c(str).map { |t| [*t.first(4), t[3].encoding] }
-      old_tokens = GraphQL.scan_with_ruby(str).map { |t| [*t.first(4), t[3].encoding] }
+      old_tokens = GraphQL.scan_with_ruby(str).map { |t| [*t, t[3].encoding] }
 
       assert_equal [
         [:LCURLY, 1, 1, "{", Encoding::UTF_8],

spec/graphql/language/lexer_examples.rb

@@ -21,14 +21,6 @@ def col
       self[2]
     end
 
-    def prev_token
-      self[4]
-    end
-
-    def previous_token
-      self[4]
-    end
-
     def inspect
       "(#{name} #{value.inspect} [#{line}:#{col}])"
     end
@@ -64,10 +56,6 @@ def self.included(child_mod)
           assert_equal Encoding::UTF_8, tokens[2].value.encoding
         end
 
-        it "keeps track of previous_token" do
-          assert_equal tokens[0], tokens[1].prev_token
-        end
-
         it "handles integers with a leading zero" do
           tokens = subject.tokenize("{ a(id: 04) }")
           assert_equal :INT, tokens[5].name
@@ -165,11 +153,6 @@ def self.included(child_mod)
           assert_bad_unicode(text2, 'Bad unicode escape in "\\xED\\xB0\\x80\\xED\\xBC\\xAC"')
         end
 
-        it "clears the previous_token between runs" do
-          tok_2 = subject.tokenize(query_string)
-          assert_nil tok_2[0].prev_token
-        end
-
         it "counts string position properly" do
           tokens = subject.tokenize('{ a(b: "c")}')
           str_token = tokens[5]