merged.

Pascal J. Bourguignon [2015-07-16 20:15]
merged.
Filename
common-lisp/parser/generate-scanner.lisp
common-lisp/parser/packages.lisp
common-lisp/parser/scanner.lisp
languages/c11/com.informatimago.languages.c11.asd
languages/c11/context.lisp
languages/c11/packages.lisp
languages/c11/parser.lisp
languages/c11/scanner.lisp
languages/cpp/cpp.lisp
languages/cpp/packages.lisp
languages/cpp/token.lisp
diff --git a/common-lisp/parser/generate-scanner.lisp b/common-lisp/parser/generate-scanner.lisp
index 151f02e..8e5be6c 100644
--- a/common-lisp/parser/generate-scanner.lisp
+++ b/common-lisp/parser/generate-scanner.lisp
@@ -174,14 +174,16 @@
            (lit-nan-terminals-regexp
              (format nil "^(~{~A~^|~})"
                      (mapcar (function regexp-quote-extended)  nan-terminals)))
-           (pname (package-name *package*)))
+           (kind-pname   (package-name *package*))
+           (symbol-pname (package-name *package*)))

       `(progn

          (defclass ,name  (,superclass)
            ()
            (:default-initargs :spaces ,spaces
-                              :token-package (load-time-value (find-package ,pname))))
+                              :token-kind-package   (load-time-value (find-package ,kind-pname))
+                              :token-symbol-package (load-time-value (find-package ,symbol-pname))))

          (defmethod scan-next-token ((scanner ,name) &optional parser-data)
            "RETURN: (scanner-current-token scanner)"
@@ -252,15 +254,7 @@
                          :format-control "Invalid character ~S at position: ~D~%"
                          :format-arguments (list (aref (scanner-buffer scanner) pos)
                                                  (scanner-column scanner)))))))
-           (setf (scanner-current-token scanner)
-                 (make-instance 'token
-                                :line  (scanner-line scanner)
-                                :column (scanner-column scanner)
-                                :text (scanner-current-text scanner)
-                                :kind (etypecase (scanner-current-token scanner)
-                                        (string (intern (scanner-current-token scanner)
-                                                        (scanner-token-package scanner)))
-                                        (symbol (scanner-current-token scanner))))))))))
+           (setf (scanner-current-token scanner) (make-current-token scanner)))))))


 (defmacro define-scanner (name &key
@@ -279,6 +273,10 @@ RETURN:         NAME
 SUPERCLASS:     The name of the superclass of the class NAME. Should
                 be a subclass of SCANNER.

+TOKEN-CLASS-NAME:
+                The name of the class of tokens. Should TOKEN or a
+                subclass of TOKEN.
+
 TERMINALS:      A list of couples (name-of-terminals
                 regexp-of-terminal) or strings containing the literal terminal..

diff --git a/common-lisp/parser/packages.lisp b/common-lisp/parser/packages.lisp
index 045fdad..75996aa 100644
--- a/common-lisp/parser/packages.lisp
+++ b/common-lisp/parser/packages.lisp
@@ -6,7 +6,7 @@
 ;;;;USER-INTERFACE:     NONE
 ;;;;DESCRIPTION
 ;;;;
-;;;;    Defines the packages.
+;;;;    Defines the packages for the abstract scanner and parsers.
 ;;;;
 ;;;;AUTHORS
 ;;;;    <PJB> Pascal J. Bourguignon <pjb@informatimago.com>
@@ -16,7 +16,7 @@
 ;;;;LEGAL
 ;;;;    AGPL3
 ;;;;
-;;;;    Copyright Pascal J. Bourguignon 2015 - 2015
+;;;;    Copyright Pascal J. Bourguignon 2004 - 2015
 ;;;;
 ;;;;    This program is free software: you can redistribute it and/or modify
 ;;;;    it under the terms of the GNU Affero General Public License as published by
@@ -45,11 +45,15 @@
    "DEFINE-SCANNER"
    ;; TOKEN:
    "TOKEN" "TOKEN-KIND" "TOKEN-TEXT" "TOKEN-LINE" "TOKEN-COLUMN"
+   "KIND" "TEXT" "LINE" "COLUMN"
+   "WORD-EQUAL"
    "*SPACE*"
    ;; SCANNER:
    "SCANNER" "SCANNER-CURRENT-TOKEN"
    "SCANNER-SOURCE" "SCANNER-LINE" "SCANNER-COLUMN" "SCANNER-STATE"
    "SCANNER-SPACES" "SCANNER-TAB-WIDTH"
+   "SCANNER-TOKEN-KIND-PACKAGE"
+   "SCANNER-CURRENT-TOKEN"
    ;; SCANNER-ERROR condition:
    "SCANNER-ERROR" "SCANNER-ERROR-LINE" "SCANNER-ERROR-COLUMN"
    "SCANNER-ERROR-STATE" "SCANNER-ERROR-CURRENT-TOKEN"
@@ -57,18 +61,14 @@
    "SCANNER-ERROR-FORMAT-CONTROL" "SCANNER-ERROR-FORMAT-ARGUMENTS"
    "SCANNER-ERROR-INVALID-CHARACTER"
    ;; SCANNER methods:
-   "SKIP-SPACES" "SCAN-NEXT-TOKEN"
+   "SKIP-SPACES" "SCAN-NEXT-TOKEN" "MAKE-CURRENT-TOKEN"
    ;; PEEK-STREAM methods specialized on SCANNER:
    "NEXTCHAR" "UNGETCHAR" "GETCHAR"
+   ;; BUFFERED-SCANNER methods:
    "BUFFERED-SCANNER" "SCANNER-BUFFER" "SCANNER-CURRENT-TEXT"
-   "ADVANCE-LINE"
-   "SCANNER-END-OF-SOURCE-P"
-   "SCANNER-END-OF-LINE-P"
-   "SCANNER-CURRENT-TOKEN"
-   "ACCEPT"
-   "WORD-EQUAL"
-   "GENERATE-SCANNER" "READLINE"
-   "SCANNER-TOKEN-PACKAGE" "ADVANCE-LINE")
+   "ADVANCE-LINE"  "READLINE"
+   "SCANNER-END-OF-SOURCE-P" "SCANNER-END-OF-LINE-P"
+   "ACCEPT" "GENERATE-SCANNER")
   (:documentation
    "
 An abstract scanner class.
@@ -80,7 +80,7 @@ License:

     AGPL3

-    Copyright Pascal J. Bourguignon 2004 - 2013
+    Copyright Pascal J. Bourguignon 2004 - 2015

     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU Affero General Public License as published by
@@ -97,6 +97,7 @@ License:
     If not, see <http://www.gnu.org/licenses/>
 "))

+
 (defpackage "COM.INFORMATIMAGO.COMMON-LISP.PARSER.PARSER"
   (:use "COMMON-LISP"
         "COM.INFORMATIMAGO.COMMON-LISP.CESARUM.UTILITY"
@@ -120,7 +121,7 @@ License:

     AGPL3

-    Copyright Pascal J. Bourguignon 2004 - 2012
+    Copyright Pascal J. Bourguignon 2004 - 2015

     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU Affero General Public License as published by
@@ -136,8 +137,6 @@ License:
     along with this program.
     If not, see <http://www.gnu.org/licenses/>

-
-
 "))

 ;;;; THE END ;;;;
diff --git a/common-lisp/parser/scanner.lisp b/common-lisp/parser/scanner.lisp
index 505ccfd..eeab8d6 100644
--- a/common-lisp/parser/scanner.lisp
+++ b/common-lisp/parser/scanner.lisp
@@ -71,7 +71,7 @@
    (text       :accessor token-text
                :initarg :text
                :initform ""
-               :type     string)
+               :type     (or null string))
    (column     :accessor token-column
                :initarg :column
                :initform 1
@@ -165,44 +165,44 @@


 (defclass scanner ()
-  ((source        :initarg :source
-                  :reader scanner-source
-                  :documentation "The source can be a PEEK-STREAM, a STREAM, or a STRING.")
-   (stream        :type peek-stream
-                  :reader scanner-stream
-                  :documentation "The source is wrapped into this PEEK-STREAM.
+  ((source               :initarg       :source
+                         :reader        scanner-source
+                         :documentation "The source can be a PEEK-STREAM, a STREAM, or a STRING.")
+   (stream               :type          peek-stream
+                         :reader        scanner-stream
+                         :documentation "The source is wrapped into this PEEK-STREAM.
 Subclasses may use scanner-stream to read from the source.")
-   (line          :initarg :line
-                  :accessor scanner-line
-                  :type integer
-                  :initform 0
-                  :documentation "The number of the current line. First line is line number 1.")
-   (column        :initarg :column
-                  :accessor scanner-column
-                  :type integer
-                  :initform 1
-                  :documentation "The number of the current column. First column is column number 1.")
-   (state         :initarg :state
-                  :accessor scanner-state
-                  :initform nil
-                  :documentation "The state of the scanner.")
-   (spaces        :initarg :spaces
-                  :accessor scanner-spaces
-                  :type string
-                  :initform *spaces*
-                  :documentation "A string containing the characters considered space by SKIP-SPACES.")
-   (tab-width     :initarg :tab-width
-                  :accessor scanner-tab-width
-                  :type fixnum
-                  :initform 8
-                  :documentation "TAB aligns to column number modulo TAB-WIDTH.")
-   (current-token :accessor scanner-current-token
-                  :initform nil
-                  :documentation "The last token read.")
-   (token-package :accessor scanner-token-package
-                  :initform (load-time-value (find-package "COM.INFORMATIMAGO.COMMON-LISP.PARSER.SCANNER"))
-                  :initarg :token-package
-                  :documentation "The package where the token-kind symbols are interned in."))
+   (line                 :initarg       :line
+                         :accessor      scanner-line
+                         :type          integer
+                         :initform      0
+                         :documentation "The number of the current line. First line is line number 1.")
+   (column               :initarg       :column
+                         :accessor      scanner-column
+                         :type          integer
+                         :initform      1
+                         :documentation "The number of the current column. First column is column number 1.")
+   (state                :initarg       :state
+                         :accessor      scanner-state
+                         :initform      nil
+                         :documentation "The state of the scanner.")
+   (spaces               :initarg       :spaces
+                         :accessor      scanner-spaces
+                         :type          string
+                         :initform      *spaces*
+                         :documentation "A string containing the characters considered space by SKIP-SPACES.")
+   (tab-width            :initarg       :tab-width
+                         :accessor      scanner-tab-width
+                         :type          fixnum
+                         :initform      8
+                         :documentation "TAB aligns to column number modulo TAB-WIDTH.")
+   (current-token        :accessor      scanner-current-token
+                         :initform      nil
+                         :documentation "The last token read.")
+   (token-kind-package   :accessor      scanner-token-kind-package
+                         :initform      (load-time-value (find-package "COM.INFORMATIMAGO.COMMON-LISP.PARSER.SCANNER"))
+                         :initarg       :token-kind-package
+                         :documentation "The package where the token-kind symbols are interned in."))
   (:documentation "An abstract scanner."))


@@ -227,20 +227,17 @@ RETURN:         SCANNER
   (:documentation   "
 DO: Skips over the spaces in the input stream. Updates line and column slots.
 RETURN: line; column
-"))
+")
+  (:method ((scanner scanner))
+    (loop
+      :for ch = (getchar scanner)
+      :while (and ch (find ch (scanner-spaces scanner)))
+      :finally (when ch
+                 (ungetchar scanner ch))
+               (return (values (scanner-line   scanner)
+                               (scanner-column scanner))))))
+

-(defmethod skip-spaces ((scanner scanner))
-  "
-DO: Skips over the spaces in the input stream. Updates line and column slots.
-RETURN: line; column
-"
-  (loop
-    :for ch = (getchar scanner)
-    :while (and ch (find ch (scanner-spaces scanner)))
-    :finally (when ch
-               (ungetchar scanner ch))
-             (return (values (scanner-line   scanner)
-                             (scanner-column scanner)))))


 (defgeneric scan-next-token (scanner &optional parser-data)
@@ -332,4 +329,18 @@ RETURN:       (scanner-current-token scanner).
     (incf (scanner-line scanner))
     (setf (scanner-column scanner) 1)))

+
+(defgeneric make-current-token (scanner)
+  (:documentation "Makes an instance of the TOKEN class or a subclass
+thereof, filled with the current information in the scanner object.")
+  (:method ((scanner scanner))
+    (make-instance 'token
+                   :line  (scanner-line scanner)
+                   :column (scanner-column scanner)
+                   :text (scanner-current-text scanner)
+                   :kind (etypecase (scanner-current-token scanner)
+                           (string (intern (scanner-current-token scanner)
+                                           (scanner-token-kind-package scanner)))
+                           (symbol (scanner-current-token scanner))))))
+
 ;;;; THE END ;;;;
diff --git a/languages/c11/com.informatimago.languages.c11.asd b/languages/c11/com.informatimago.languages.c11.asd
index 082f832..e5176b0 100644
--- a/languages/c11/com.informatimago.languages.c11.asd
+++ b/languages/c11/com.informatimago.languages.c11.asd
@@ -55,8 +55,13 @@
                "com.informatimago.languages.cpp"
                "yacc")
   :components ((:file "packages"        :depends-on  ())
-               (:file "read-yacc"       :depends-on  ("packages"))
-               (:file "parser"          :depends-on  ("packages" "read-yacc")))
+               (:file "context"         :depends-on  ("packages"))
+               (:file "scanner"         :depends-on  ("packages" "context"))
+               (:file "read-yacc"       :depends-on  ("packages" ))
+               (:file "parser"          :depends-on  ("packages"
+                                                      "read-yacc"
+                                                      "context"
+                                                      "scanner")))
   :in-order-to ((asdf:test-op (asdf:test-op "com.informatimago.languages.c11.test"))))

 ;;;; THE END ;;;;
diff --git a/languages/c11/context.lisp b/languages/c11/context.lisp
new file mode 100644
index 0000000..3ce97ef
--- /dev/null
+++ b/languages/c11/context.lisp
@@ -0,0 +1,102 @@
+;;;; -*- mode:lisp;coding:utf-8 -*-
+;;;;**************************************************************************
+;;;;FILE:               context.lisp
+;;;;LANGUAGE:           Common-Lisp
+;;;;SYSTEM:             Common-Lisp
+;;;;USER-INTERFACE:     NONE
+;;;;DESCRIPTION
+;;;;
+;;;;    C11 compilation context.
+;;;;
+;;;;AUTHORS
+;;;;    <PJB> Pascal J. Bourguignon <pjb@informatimago.com>
+;;;;MODIFICATIONS
+;;;;    2015-07-12 <PJB> Created.
+;;;;BUGS
+;;;;LEGAL
+;;;;    AGPL3
+;;;;
+;;;;    Copyright Pascal J. Bourguignon 2015 - 2015
+;;;;
+;;;;    This program is free software: you can redistribute it and/or modify
+;;;;    it under the terms of the GNU Affero General Public License as published by
+;;;;    the Free Software Foundation, either version 3 of the License, or
+;;;;    (at your option) any later version.
+;;;;
+;;;;    This program is distributed in the hope that it will be useful,
+;;;;    but WITHOUT ANY WARRANTY; without even the implied warranty of
+;;;;    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;;;;    GNU Affero General Public License for more details.
+;;;;
+;;;;    You should have received a copy of the GNU Affero General Public License
+;;;;    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+;;;;**************************************************************************
+(in-package "COM.INFORMATIMAGO.LANGUAGES.C11.PARSER")
+(eval-when (:compile-toplevel :load-toplevel :execute)
+  (setf *readtable* (copy-readtable nil)))
+
+(defclass context ()
+  ((c-identifiers-package :initarg :c-identifiers-package
+                          :initform (load-time-value (find-package "COM.INFORMATIMAGO.LANGUAGES.C11.C"))
+                          :accessor context-c-identifiers-package)
+   (typedefs              :initarg :typedefs
+                          :initform (make-hash-table :test (function equal))
+                          :documentation "Maps identifier symbols to typedef declarations.
+The scanner uses it to detect typedef_name tokens."
+                          :reader context-typedefs)
+   (functions             :initarg :functions
+                          :initform (make-hash-table :test (function equal))
+                          :documentation "Maps identifier symbols to function declarations.
+The scanner uses it to detect func_name tokens."
+                          :reader context-functions)
+   (enumeration-constants :initarg :enum-constants
+                          :initform (make-hash-table :test (function equal))
+                          :documentation "Maps identifier symbols to enumeration constant declarations.
+The scanner uses it to detect enumeration_constant tokens."
+                          :reader context-enumeration-constants)))
+
+(defvar *context* nil
+  "The C11 parser context.")
+
+(defgeneric typedef-name-p (context token)
+  (:method ((context null) token)
+    nil))
+
+(defgeneric function-name-p (context token)
+  (:method ((context null) token)
+    nil))
+
+(defgeneric enumeration-constant-name-p (context token)
+  (:method ((context null) token)
+    nil))
+
+(defun identifier-in-table-p (context table name)
+  (and (eq '|identifier| (token-kind name))
+       (gethash (token-symbol name) table)))
+
+(defun enter-into-table (context table kind name definition)
+  (assert (eq kind (token-kind name)) (name))
+  (setf (gethash (token-symbol name) table) definition))
+
+(defmethod typedef-name-p ((context context) token)
+  (identifier-in-table-p context (context-typedefs context) token))
+
+(defmethod function-name-p ((context context) token)
+  (identifier-in-table-p context (context-functions context) token))
+
+(defmethod enumeration-constant-name-p ((context context) token)
+  (identifier-in-table-p context (context-enumeration-constants context) token))
+
+(defgeneric enter-typedef (context name &optional definition)
+  (:method ((context context) name &optional (definition t))
+    (enter-into-table context (context-typedefs context) '|typedef_name| name definition)))
+
+(defgeneric enter-function (context name &optional definition)
+  (:method ((context context) name &optional (definition t))
+    (enter-into-table context (context-functions context) '|func_name| name definition)))
+
+(defgeneric enter-enumeration-constant (context name &optional definition)
+  (:method ((context context) name &optional (definition t))
+    (enter-into-table context (context-enumeration-constants context) '|enumeration_constant| name definition)))
+
+;;;; THE END ;;;;
diff --git a/languages/c11/packages.lisp b/languages/c11/packages.lisp
index b559863..72e0ca0 100644
--- a/languages/c11/packages.lisp
+++ b/languages/c11/packages.lisp
@@ -32,9 +32,6 @@
 ;;;;    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ;;;;**************************************************************************

-(defpackage "COM.INFORMATIMAGO.LANGUAGES.C11"
-  (:use "COMMON-LISP"))
-
 (defpackage "COM.INFORMATIMAGO.LANGUAGES.C11.PARSER"
   (:use "COMMON-LISP"
         "YACC"
@@ -49,4 +46,8 @@
   (:shadowing-import-from "COM.INFORMATIMAGO.COMMON-LISP.REGEXP.REGEXP"
                           "SPLIT-STRING"))

+(defpackage "COM.INFORMATIMAGO.LANGUAGES.C11.C"
+  (:use)
+  (:documentation "Default package where the C identifiers are interned."))
+
 ;;;; THE END ;;;;
diff --git a/languages/c11/parser.lisp b/languages/c11/parser.lisp
index 4dec58d..7008b83 100644
--- a/languages/c11/parser.lisp
+++ b/languages/c11/parser.lisp
@@ -37,240 +37,527 @@

 #|
 (defmethod accept ((scanner buffered-scanner) token)
-  (if (word-equal token (scanner-current-token scanner))
-      (prog1 (list (token-kind (scanner-current-token scanner))
-                   (scanner-current-text scanner)
-                   (scanner-column scanner))
-        (scan-next-token scanner))
-      (error 'unexpected-token-error
-             :line   (scanner-line   scanner)
-             :column (scanner-column scanner)
-             :state  (scanner-state  scanner)
-             :current-token (scanner-current-token scanner)
-             :scanner scanner
-             :expected-token token
-             :format-control "Expected ~S, not ~A (~S)~%"
-             :format-arguments (list
-                                token
-                                (scanner-current-token scanner)
-                                (scanner-current-text scanner)))))
+(if (word-equal token (scanner-current-token scanner))
+(prog1 (list (token-kind (scanner-current-token scanner))
+(scanner-current-text scanner)
+(scanner-column scanner))
+(scan-next-token scanner))
+(error 'unexpected-token-error
+:line   (scanner-line   scanner)
+:column (scanner-column scanner)
+:state  (scanner-state  scanner)
+:current-token (scanner-current-token scanner)
+:scanner scanner
+:expected-token token
+:format-control "Expected ~S, not ~A (~S)~%"
+:format-arguments (list
+token
+(scanner-current-token scanner)
+(scanner-current-text scanner)))))

 |#

-#-(and) ; not yet.
+
+(defun c-declaration (specifiers init-declarators semicolon)
+  (print (list 'specifiers  specifiers))
+  (print (list 'init-declarators init-declarators))
+  (list specifiers init-declarators semicolon))
+
+;; #-(and) ; not yet.
 (DEFINE-PARSER *C11-PARSER*
+
   (:START-SYMBOL |translation_unit|)
-  (:TERMINALS (
-               ALIGNAS ALIGNOF ATOMIC GENERIC NORETURN STATIC_ASSERT THREAD_LOCAL CASE
-               DEFAULT IF ELSE SWITCH WHILE DO FOR GOTO CONTINUE BREAK RETURN STRUCT UNION
-               ENUM ELLIPSIS COMPLEX IMAGINARY BOOL CHAR SHORT INT LONG SIGNED UNSIGNED
-               FLOAT DOUBLE VOID CONST RESTRICT VOLATILE TYPEDEF EXTERN STATIC AUTO
-               REGISTER INLINE TYPEDEF_NAME ENUMERATION_CONSTANT XOR_ASSIGN OR_ASSIGN
-               SUB_ASSIGN LEFT_ASSIGN RIGHT_ASSIGN AND_ASSIGN AND_OP OR_OP MUL_ASSIGN
-               DIV_ASSIGN MOD_ASSIGN ADD_ASSIGN PTR_OP INC_OP DEC_OP LEFT_OP RIGHT_OP
-               LE_OP GE_OP EQ_OP NE_OP IDENTIFIER I_CONSTANT F_CONSTANT STRING_LITERAL
-               FUNC_NAME SIZEOF))
-  (|primary_expression| IDENTIFIER |constant| |string| (\( |expression| \))
-                        |generic_selection|)
+
+  (:TERMINALS
+   (
+    |identifier| |typedef_name| |func_name| |string_literal|
+                 |i_constant| |f_constant|
+                 |enumeration_constant|
+
+                 |alignas| |alignof| |atomic| |generic| |noreturn| |static_assert|
+                 |thread_local| |case| |default| |if| |else| |switch| |while| |do|
+                 |for| |goto| |continue| |break| |return| |struct| |union| |enum|
+                 |...| |complex| |imaginary| |bool| |char| |short| |int| |long|
+                 |signed| |unsigned| |float| |double| |void| |const| |restrict|
+                 |volatile| |typedef| |extern| |static| |auto| |register| |inline|
+                 |sizeof|
+
+                 ^= \|= -= <<= >>= &= && |\|\|| *= /= %= += -> ++ -- << >>
+                 <= >= == !=))
+
+
+  ;; renaming terminals:
+
+  (IDENTIFIER |identifier|)
+  (TYPEDEF_NAME |typedef_name|)
+  (FUNC_NAME |func_name|)
+  (ENUMERATION_CONSTANT |enumeration_constant|)
+
+  (STRING_LITERAL |string_literal|)
+  (I_CONSTANT     |i_constant|)
+  (F_CONSTANT     |f_constant|)
+
   (|constant| I_CONSTANT F_CONSTANT ENUMERATION_CONSTANT)
-  (|enumeration_constant| IDENTIFIER)
   (|string| STRING_LITERAL FUNC_NAME)
+
+
+
+  (ALIGNAS |alignas|)
+  (ALIGNOF |alignof|)
+  (ATOMIC |atomic|)
+  (GENERIC |generic|)
+  (NORETURN |noreturn|)
+  (STATIC_ASSERT |static_assert|)
+  (THREAD_LOCAL |thread_local|)
+  (CASE |case|)
+  (DEFAULT |default|)
+  (IF |if|)
+  (ELSE |else|)
+  (SWITCH |switch|)
+  (WHILE |while|)
+  (DO |do|)
+  (FOR |for|)
+  (GOTO |goto|)
+  (CONTINUE |continue|)
+  (BREAK |break|)
+  (RETURN |return|)
+  (STRUCT |struct|)
+  (UNION |union|)
+  (ENUM |enum|)
+  (ELLIPSIS |...|)
+  (COMPLEX |complex|)
+  (IMAGINARY |imaginary|)
+  (BOOL |bool|)
+  (CHAR |char|)
+  (SHORT |short|)
+  (INT |int|)
+  (LONG |long|)
+  (SIGNED |signed|)
+  (UNSIGNED |unsigned|)
+  (FLOAT |float|)
+  (DOUBLE |double|)
+  (VOID |void|)
+  (CONST |const|)
+  (RESTRICT |restrict|)
+  (VOLATILE |volatile|)
+  (TYPEDEF |typedef|)
+  (EXTERN |extern|)
+  (STATIC |static|)
+  (AUTO |auto|)
+  (REGISTER |register|)
+  (INLINE |inline|)
+  (SIZEOF |sizeof|)
+
+
+  (XOR_ASSIGN  |^=|)
+  (OR_ASSIGN   \|=)
+  (SUB_ASSIGN  |-=|)
+  (LEFT_ASSIGN |<<=|)
+  (RIGHT_ASSIGN |>>=|)
+  (AND_ASSIGN |&=|)
+  (AND_OP |&&|)
+  (OR_OP \|\|)
+  (MUL_ASSIGN |*=|)
+  (DIV_ASSIGN |/=|)
+  (MOD_ASSIGN |%=|)
+  (ADD_ASSIGN |+=|)
+  (PTR_OP |->|)
+  (INC_OP |++|)
+  (DEC_OP |--|)
+  (LEFT_OP |<<|)
+  (RIGHT_OP |>>|)
+  (LE_OP |<=|)
+  (GE_OP |>=|)
+  (EQ_OP |==|)
+  (NE_OP |!=|)
+
+  ;; productions:
+
+  (|primary_expression|
+   IDENTIFIER
+   |constant|
+   |string|
+   (\( |expression| \))
+   |generic_selection|)
+
   (|generic_selection|
    (GENERIC \( |assignment_expression| \, |generic_assoc_list| \)))
-  (|generic_assoc_list| |generic_association|
-                        (|generic_assoc_list| \, |generic_association|))
-  (|generic_association| (|type_name| \: |assignment_expression|)
-                         (DEFAULT \: |assignment_expression|))
-  (|postfix_expression| |primary_expression|
-                        (|postfix_expression| \[ |expression| \]) (|postfix_expression| \( \))
-                        (|postfix_expression| \( |argument_expression_list| \))
-                        (|postfix_expression| |.| IDENTIFIER)
-                        (|postfix_expression| PTR_OP IDENTIFIER) (|postfix_expression| INC_OP)
-                        (|postfix_expression| DEC_OP) (\( |type_name| \) { |initializer_list| })
-                        (\( |type_name| \) { |initializer_list| \, }))
-  (|argument_expression_list| |assignment_expression|
-                              (|argument_expression_list| \, |assignment_expression|))
-  (|unary_expression| |postfix_expression| (INC_OP |unary_expression|)
-                      (DEC_OP |unary_expression|) (|unary_operator| |cast_expression|)
-                      (SIZEOF |unary_expression|) (SIZEOF \( |type_name| \))
-                      (ALIGNOF \( |type_name| \)))
+
+  (|generic_assoc_list|
+   |generic_association|
+   (|generic_assoc_list| \, |generic_association|))
+
+  (|generic_association|
+   (|type_name| \: |assignment_expression|)
+   (DEFAULT \: |assignment_expression|))
+
+  (|postfix_expression|
+   |primary_expression|
+   (|postfix_expression| \[ |expression| \])
+   (|postfix_expression| \( \))
+   (|postfix_expression| \( |argument_expression_list| \))
+   (|postfix_expression| |.| IDENTIFIER)
+   (|postfix_expression| PTR_OP IDENTIFIER)
+   (|postfix_expression| INC_OP)
+   (|postfix_expression| DEC_OP)
+   (\( |type_name| \) { |initializer_list| })
+   (\( |type_name| \) { |initializer_list| \, }))
+
+  (|argument_expression_list|
+   |assignment_expression|
+   (|argument_expression_list| \, |assignment_expression|))
+
+  (|unary_expression|
+   |postfix_expression|
+   (INC_OP |unary_expression|)
+   (DEC_OP |unary_expression|)
+   (|unary_operator| |cast_expression|)
+   (SIZEOF |unary_expression|)
+   (SIZEOF \( |type_name| \))
+   (ALIGNOF \( |type_name| \)))
+
   (|unary_operator| & * + - ~ !)
-  (|cast_expression| |unary_expression| (\( |type_name| \) |cast_expression|))
-  (|multiplicative_expression| |cast_expression|
-                               (|multiplicative_expression| * |cast_expression|)
-                               (|multiplicative_expression| / |cast_expression|)
-                               (|multiplicative_expression| % |cast_expression|))
-  (|additive_expression| |multiplicative_expression|
-                         (|additive_expression| + |multiplicative_expression|)
-                         (|additive_expression| - |multiplicative_expression|))
-  (|shift_expression| |additive_expression|
-                      (|shift_expression| LEFT_OP |additive_expression|)
-                      (|shift_expression| RIGHT_OP |additive_expression|))
-  (|relational_expression| |shift_expression|
-                           (|relational_expression| < |shift_expression|)
-                           (|relational_expression| > |shift_expression|)
-                           (|relational_expression| LE_OP |shift_expression|)
-                           (|relational_expression| GE_OP |shift_expression|))
-  (|equality_expression| |relational_expression|
-                         (|equality_expression| EQ_OP |relational_expression|)
-                         (|equality_expression| NE_OP |relational_expression|))
-  (|and_expression| |equality_expression|
-                    (|and_expression| & |equality_expression|))
-  (|exclusive_or_expression| |and_expression|
-                             (|exclusive_or_expression| ^ |and_expression|))
-  (|inclusive_or_expression| |exclusive_or_expression|
-                             (|inclusive_or_expression| \| |exclusive_or_expression|))
-  (|logical_and_expression| |inclusive_or_expression|
-                            (|logical_and_expression| AND_OP |inclusive_or_expression|))
-  (|logical_or_expression| |logical_and_expression|
-                           (|logical_or_expression| OR_OP |logical_and_expression|))
-  (|conditional_expression| |logical_or_expression|
-                            (|logical_or_expression| ? |expression| \: |conditional_expression|))
-  (|assignment_expression| |conditional_expression|
-                           (|unary_expression| |assignment_operator| |assignment_expression|))
+
+  (|cast_expression|
+   |unary_expression|
+   (\( |type_name| \) |cast_expression|))
+
+  (|multiplicative_expression|
+   |cast_expression|
+   (|multiplicative_expression| * |cast_expression|)
+   (|multiplicative_expression| / |cast_expression|)
+   (|multiplicative_expression| % |cast_expression|))
+
+  (|additive_expression|
+   |multiplicative_expression|
+   (|additive_expression| + |multiplicative_expression|)
+   (|additive_expression| - |multiplicative_expression|))
+
+  (|shift_expression|
+   |additive_expression|
+   (|shift_expression| LEFT_OP |additive_expression|)
+   (|shift_expression| RIGHT_OP |additive_expression|))
+
+  (|relational_expression|
+   |shift_expression|
+   (|relational_expression| < |shift_expression|)
+   (|relational_expression| > |shift_expression|)
+   (|relational_expression| LE_OP |shift_expression|)
+   (|relational_expression| GE_OP |shift_expression|))
+
+  (|equality_expression|
+   |relational_expression|
+   (|equality_expression| EQ_OP |relational_expression|)
+   (|equality_expression| NE_OP |relational_expression|))
+
+  (|and_expression|
+   |equality_expression|
+   (|and_expression| & |equality_expression|))
+
+  (|exclusive_or_expression|
+   |and_expression|
+   (|exclusive_or_expression| ^ |and_expression|))
+
+  (|inclusive_or_expression|
+   |exclusive_or_expression|
+   (|inclusive_or_expression| \| |exclusive_or_expression|))
+
+  (|logical_and_expression|
+   |inclusive_or_expression|
+   (|logical_and_expression| AND_OP |inclusive_or_expression|))
+
+  (|logical_or_expression|
+   |logical_and_expression|
+   (|logical_or_expression| OR_OP |logical_and_expression|))
+
+  (|conditional_expression|
+   |logical_or_expression|
+   (|logical_or_expression| ? |expression| \: |conditional_expression|))
+
+  (|assignment_expression|
+   |conditional_expression|
+   (|unary_expression| |assignment_operator| |assignment_expression|))
+
   (|assignment_operator| = MUL_ASSIGN DIV_ASSIGN MOD_ASSIGN ADD_ASSIGN
-                         SUB_ASSIGN LEFT_ASSIGN RIGHT_ASSIGN AND_ASSIGN XOR_ASSIGN OR_ASSIGN)
-  (|expression| |assignment_expression|
-                (|expression| \, |assignment_expression|))
-  (|constant_expression| |conditional_expression|)
-  (|declaration| (|declaration_specifiers| \;)
-                 (|declaration_specifiers| |init_declarator_list| \;)
-                 |static_assert_declaration|)
+                         SUB_ASSIGN LEFT_ASSIGN RIGHT_ASSIGN AND_ASSIGN
+                         XOR_ASSIGN OR_ASSIGN)
+
+  (|expression|
+   |assignment_expression|
+   (|expression| \, |assignment_expression|))
+
+  (|constant_expression|
+   |conditional_expression|)
+
+  (|declaration|
+   (|declaration_specifiers| \;)
+   (|declaration_specifiers| |init_declarator_list| \; #'c-declaration)
+   |static_assert_declaration|)
+
   (|declaration_specifiers|
    (|storage_class_specifier| |declaration_specifiers|)
-   |storage_class_specifier| (|type_specifier| |declaration_specifiers|)
-   |type_specifier| (|type_qualifier| |declaration_specifiers|)
-   |type_qualifier| (|function_specifier| |declaration_specifiers|)
-   |function_specifier| (|alignment_specifier| |declaration_specifiers|)
+   |storage_class_specifier|
+   (|type_specifier| |declaration_specifiers|)
+   |type_specifier|
+   (|type_qualifier| |declaration_specifiers|)
+   |type_qualifier|
+   (|function_specifier| |declaration_specifiers|)
+   |function_specifier|
+   (|alignment_specifier| |declaration_specifiers|)
    |alignment_specifier|)
-  (|init_declarator_list| |init_declarator|
-                          (|init_declarator_list| \, |init_declarator|))
-  (|init_declarator| (|declarator| = |initializer|) |declarator|)
+
+  (|init_declarator_list|
+   |init_declarator|
+   (|init_declarator_list| \, |init_declarator|))
+
+  (|init_declarator|
+   (|declarator| = |initializer|)
+   |declarator|)
+
   (|storage_class_specifier| TYPEDEF EXTERN STATIC THREAD_LOCAL AUTO REGISTER)
+
   (|type_specifier| VOID CHAR SHORT INT LONG FLOAT DOUBLE SIGNED UNSIGNED BOOL
                     COMPLEX IMAGINARY |atomic_type_specifier| |struct_or_union_specifier|
                     |enum_specifier| TYPEDEF_NAME)
+
   (|struct_or_union_specifier|
    (|struct_or_union| { |struct_declaration_list| })
    (|struct_or_union| IDENTIFIER { |struct_declaration_list| })
    (|struct_or_union| IDENTIFIER))
+
   (|struct_or_union| STRUCT UNION)
-  (|struct_declaration_list| |struct_declaration|
-                             (|struct_declaration_list| |struct_declaration|))
-  (|struct_declaration| (|specifier_qualifier_list| \;)
-                        (|specifier_qualifier_list| |struct_declarator_list| \;)
-                        |static_assert_declaration|)
-  (|specifier_qualifier_list| (|type_specifier| |specifier_qualifier_list|)
-                              |type_specifier| (|type_qualifier| |specifier_qualifier_list|)
-                              |type_qualifier|)
-  (|struct_declarator_list| |struct_declarator|
-                            (|struct_declarator_list| \, |struct_declarator|))
-  (|struct_declarator| (\: |constant_expression|)
-                       (|declarator| \: |constant_expression|) |declarator|)
-  (|enum_specifier| (ENUM { |enumerator_list| })
-                    (ENUM { |enumerator_list| \, }) (ENUM IDENTIFIER { |enumerator_list| })
-                    (ENUM IDENTIFIER { |enumerator_list| \, }) (ENUM IDENTIFIER))
-  (|enumerator_list| |enumerator| (|enumerator_list| \, |enumerator|))
-  (|enumerator| (|enumeration_constant| = |constant_expression|)
-                |enumeration_constant|)
-  (|atomic_type_specifier| (ATOMIC \( |type_name| \)))
+
+  (|struct_declaration_list|
+   |struct_declaration|
+   (|struct_declaration_list| |struct_declaration|))
+
+  (|struct_declaration|
+   (|specifier_qualifier_list| \;)
+   (|specifier_qualifier_list| |struct_declarator_list| \;)
+   |static_assert_declaration|)
+
+  (|specifier_qualifier_list|
+   (|type_specifier| |specifier_qualifier_list|)
+   |type_specifier|
+   (|type_qualifier| |specifier_qualifier_list|)
+   |type_qualifier|)
+
+  (|struct_declarator_list|
+   |struct_declarator|
+   (|struct_declarator_list| \, |struct_declarator|))
+
+  (|struct_declarator|
+   (\: |constant_expression|)
+   (|declarator| \: |constant_expression|)
+   |declarator|)
+
+  (|enum_specifier|
+   (ENUM { |enumerator_list| })
+   (ENUM { |enumerator_list| \, })
+   (ENUM IDENTIFIER { |enumerator_list| })
+   (ENUM IDENTIFIER { |enumerator_list| \, })
+   (ENUM IDENTIFIER))
+
+  (|enumerator_list|
+   |enumerator|
+   (|enumerator_list| \, |enumerator|))
+
+  (|enumerator|
+   (|enumeration_constant| = |constant_expression|)
+   |enumeration_constant|)
+
+  (|atomic_type_specifier|
+   (ATOMIC \( |type_name| \)))
+
   (|type_qualifier| CONST RESTRICT VOLATILE ATOMIC)
+
   (|function_specifier| INLINE NORETURN)
-  (|alignment_specifier| (ALIGNAS \( |type_name| \))
-                         (ALIGNAS \( |constant_expression| \)))
-  (|declarator| (|pointer| |direct_declarator|) |direct_declarator|)
-  (|direct_declarator| IDENTIFIER (\( |declarator| \))
-                       (|direct_declarator| \[ \]) (|direct_declarator| \[ * \])
-                       (|direct_declarator| \[ STATIC |type_qualifier_list| |assignment_expression|
-                                                     \])
-                       (|direct_declarator| \[ STATIC |assignment_expression| \])
-                       (|direct_declarator| \[ |type_qualifier_list| * \])
-                       (|direct_declarator| \[ |type_qualifier_list| STATIC |assignment_expression|
-                                                                    \])
-                       (|direct_declarator| \[ |type_qualifier_list| |assignment_expression| \])
-                       (|direct_declarator| \[ |type_qualifier_list| \])
-                       (|direct_declarator| \[ |assignment_expression| \])
-                       (|direct_declarator| \( |parameter_type_list| \))
-                       (|direct_declarator| \( \)) (|direct_declarator| \( |identifier_list| \)))
-  (|pointer| (* |type_qualifier_list| |pointer|) (* |type_qualifier_list|)
-             (* |pointer|) *)
-  (|type_qualifier_list| |type_qualifier|
-                         (|type_qualifier_list| |type_qualifier|))
-  (|parameter_type_list| (|parameter_list| \, ELLIPSIS) |parameter_list|)
-  (|parameter_list| |parameter_declaration|
-                    (|parameter_list| \, |parameter_declaration|))
-  (|parameter_declaration| (|declaration_specifiers| |declarator|)
-                           (|declaration_specifiers| |abstract_declarator|) |declaration_specifiers|)
-  (|identifier_list| IDENTIFIER (|identifier_list| \, IDENTIFIER))
-  (|type_name| (|specifier_qualifier_list| |abstract_declarator|)
-               |specifier_qualifier_list|)
-  (|abstract_declarator| (|pointer| |direct_abstract_declarator|) |pointer|
-                         |direct_abstract_declarator|)
-  (|direct_abstract_declarator| (\( |abstract_declarator| \)) (\[ \]) (\[ * \])
-                                (\[ STATIC |type_qualifier_list| |assignment_expression| \])
-                                (\[ STATIC |assignment_expression| \])
-                                (\[ |type_qualifier_list| STATIC |assignment_expression| \])
-                                (\[ |type_qualifier_list| |assignment_expression| \])
-                                (\[ |type_qualifier_list| \]) (\[ |assignment_expression| \])
-                                (|direct_abstract_declarator| \[ \]) (|direct_abstract_declarator| \[ * \])
-                                (|direct_abstract_declarator| \[ STATIC |type_qualifier_list|
-                                                                       |assignment_expression| \])
-                                (|direct_abstract_declarator| \[ STATIC |assignment_expression| \])
-                                (|direct_abstract_declarator| \[ |type_qualifier_list|
-                                                                |assignment_expression| \])
-                                (|direct_abstract_declarator| \[ |type_qualifier_list| STATIC
-                                                                                      |assignment_expression| \])
-                                (|direct_abstract_declarator| \[ |type_qualifier_list| \])
-                                (|direct_abstract_declarator| \[ |assignment_expression| \]) (\( \))
-                                (\( |parameter_type_list| \)) (|direct_abstract_declarator| \( \))
-                                (|direct_abstract_declarator| \( |parameter_type_list| \)))
-  (|initializer| ({ |initializer_list| }) ({ |initializer_list| \, })
-                 |assignment_expression|)
-  (|initializer_list| (|designation| |initializer|) |initializer|
-                      (|initializer_list| \, |designation| |initializer|)
-                      (|initializer_list| \, |initializer|))
-  (|designation| (|designator_list| =))
-  (|designator_list| |designator| (|designator_list| |designator|))
-  (|designator| (\[ |constant_expression| \]) (|.| IDENTIFIER))
+
+  (|alignment_specifier|
+   (ALIGNAS \( |type_name| \))
+   (ALIGNAS \( |constant_expression| \)))
+
+  (|declarator|
+   (|pointer| |direct_declarator|)
+   |direct_declarator|)
+
+  (|direct_declarator|
+   IDENTIFIER
+   (\( |declarator| \))
+   (|direct_declarator| \[ \])
+   (|direct_declarator| \[ * \])
+   (|direct_declarator| \[ STATIC |assignment_expression| \])
+   (|direct_declarator| \[ |type_qualifier_list| * \])
+   (|direct_declarator| \[ |type_qualifier_list| STATIC |assignment_expression| \])
+   (|direct_declarator| \[ |type_qualifier_list| |assignment_expression| \])
+   (|direct_declarator| \[ |type_qualifier_list| \])
+   (|direct_declarator| \[ |assignment_expression| \])
+   (|direct_declarator| \( |parameter_type_list| \))
+   (|direct_declarator| \( \))
+   (|direct_declarator| \( |identifier_list| \)))
+
+  (|pointer|
+   (* |type_qualifier_list| |pointer|)
+   (* |type_qualifier_list|)
+   (* |pointer|)
+   *)
+
+  (|type_qualifier_list|
+   |type_qualifier|
+   (|type_qualifier_list| |type_qualifier|))
+
+  (|parameter_type_list|
+   (|parameter_list| \, ELLIPSIS)
+   |parameter_list|)
+
+  (|parameter_list|
+   |parameter_declaration|
+   (|parameter_list| \, |parameter_declaration|))
+
+  (|parameter_declaration|
+   (|declaration_specifiers| |declarator|)
+   (|declaration_specifiers| |abstract_declarator|)
+   |declaration_specifiers|)
+
+  (|identifier_list|
+   IDENTIFIER
+   (|identifier_list| \, IDENTIFIER))
+
+  (|type_name|
+   (|specifier_qualifier_list| |abstract_declarator|)
+   |specifier_qualifier_list|)
+
+  (|abstract_declarator|
+   (|pointer| |direct_abstract_declarator|)
+   |pointer|
+   |direct_abstract_declarator|)
+
+  (|direct_abstract_declarator|
+   (\( |abstract_declarator| \))
+   (\[ \])
+   (\[ * \])
+   (\[ STATIC |type_qualifier_list| |assignment_expression| \])
+   (\[ STATIC |assignment_expression| \])
+   (\[ |type_qualifier_list| STATIC |assignment_expression| \])
+   (\[ |type_qualifier_list| |assignment_expression| \])
+   (\[ |type_qualifier_list| \])
+   (\[ |assignment_expression| \])
+   (|direct_abstract_declarator| \[ \])
+   (|direct_abstract_declarator| \[ * \])
+   (|direct_abstract_declarator| \[ STATIC |type_qualifier_list| |assignment_expression| \])
+   (|direct_abstract_declarator| \[ STATIC |assignment_expression| \])
+   (|direct_abstract_declarator| \[ |type_qualifier_list| |assignment_expression| \])
+   (|direct_abstract_declarator| \[ |type_qualifier_list| STATIC |assignment_expression| \])
+   (|direct_abstract_declarator| \[ |type_qualifier_list| \])
+   (|direct_abstract_declarator| \[ |assignment_expression| \])
+   (\( \))
+   (\( |parameter_type_list| \))
+   (|direct_abstract_declarator| \( \))
+   (|direct_abstract_declarator| \( |parameter_type_list| \)))
+
+  (|initializer|
+   ({ |initializer_list| })
+   ({ |initializer_list| \, })
+   |assignment_expression|)
+
+  (|initializer_list|
+   (|designation| |initializer|)
+   |initializer|
+   (|initializer_list| \, |designation| |initializer|)
+   (|initializer_list| \, |initializer|))
+
+  (|designation|
+   (|designator_list| =))
+
+  (|designator_list|
+   |designator|
+   (|designator_list| |designator|))
+
+  (|designator|
+   (\[ |constant_expression| \])
+   (|.| IDENTIFIER))
+
   (|static_assert_declaration|
    (STATIC_ASSERT \( |constant_expression| \, STRING_LITERAL \) \;))
-  (|statement| |labeled_statement| |compound_statement| |expression_statement|
-               |selection_statement| |iteration_statement| |jump_statement|)
-  (|labeled_statement| (IDENTIFIER \: |statement|)
-                       (CASE |constant_expression| \: |statement|) (DEFAULT \: |statement|))
-  (|compound_statement| ({ }) ({ |block_item_list| }))
-  (|block_item_list| |block_item| (|block_item_list| |block_item|))
-  (|block_item| |declaration| |statement|)
-  (|expression_statement| \; (|expression| \;))
-  (|selection_statement| (IF \( |expression| \) |statement| ELSE |statement|)
-                         (IF \( |expression| \) |statement|) (SWITCH \( |expression| \) |statement|))
-  (|iteration_statement| (WHILE \( |expression| \) |statement|)
-                         (DO |statement| WHILE  \( |expression| \) \;)
-                         (FOR \( |expression_statement| |expression_statement| \) |statement|)
-                         (FOR \( |expression_statement| |expression_statement| |expression| \)
-                           |statement|)
-                         (FOR \( |declaration| |expression_statement| \) |statement|)
-                         (FOR \( |declaration| |expression_statement| |expression| \) |statement|))
-  (|jump_statement| (GOTO IDENTIFIER \;) (CONTINUE \;) (BREAK \;) (RETURN \;)
-                    (RETURN |expression| \;))
-  (|translation_unit| |external_declaration|
-                      (|translation_unit| |external_declaration|))
-  (|external_declaration| |function_definition| |declaration|)
+
+  (|statement|
+   |labeled_statement|
+   |compound_statement|
+   |expression_statement|
+   |selection_statement|
+   |iteration_statement|
+   |jump_statement|)
+
+  (|labeled_statement|
+   (IDENTIFIER \: |statement|)
+   (CASE |constant_expression| \: |statement|)
+   (DEFAULT \: |statement|))
+
+  (|compound_statement|
+   ({ })
+   ({ |block_item_list| }))
+
+  (|block_item_list|
+   |block_item|
+   (|block_item_list| |block_item|))
+
+  (|block_item|
+   |declaration|
+   |statement|)
+
+  (|expression_statement|
+   \;
+   (|expression| \;))
+
+  (|selection_statement|
+   (IF \( |expression| \) |statement| ELSE |statement|)
+   (IF \( |expression| \) |statement|)
+   (SWITCH \( |expression| \) |statement|))
+
+  (|iteration_statement|
+   (WHILE \( |expression| \) |statement|)
+   (DO |statement| WHILE  \( |expression| \) \;)
+   (FOR \( |expression_statement| |expression_statement| \) |statement|)
+   (FOR \( |expression_statement| |expression_statement| |expression| \) |statement|)
+   (FOR \( |declaration| |expression_statement| \) |statement|)
+   (FOR \( |declaration| |expression_statement| |expression| \) |statement|))
+
+  (|jump_statement|
+   (GOTO IDENTIFIER \;)
+   (CONTINUE \;)
+   (BREAK \;)
+   (RETURN \;)
+   (RETURN |expression| \;))
+
+  (|translation_unit|
+   |external_declaration|
+   (|translation_unit| |external_declaration|))
+
+  (|external_declaration|
+   |function_definition|
+   |declaration|)
+
   (|function_definition|
-   (|declaration_specifiers| |declarator| |declaration_list|
-                             |compound_statement|)
+   (|declaration_specifiers| |declarator| |declaration_list| |compound_statement|)
    (|declaration_specifiers| |declarator| |compound_statement|))
-  (|declaration_list| |declaration| (|declaration_list| |declaration|)))
+
+  (|declaration_list|
+   |declaration|
+   (|declaration_list| |declaration|)))


 (defun make-list-lexer (tokens)
   (lambda ()
     (if tokens
         (let ((token (pop tokens)))
-          (values (token-kind token) token))
+          (values (setf (token-kind token) (compute-token-kind token))
+                  token))
         (values nil nil))))

-;; (parse-with-lexer (make-list-lexer *tc*) *c11-parser*)
+#-(and)
+(let ((*context* (make-instance 'context)))
+  (values (parse-with-lexer (make-list-lexer *tc*) *c11-parser*)
+          *context*))


 ;;;; THE END ;;;;
diff --git a/languages/c11/scanner.lisp b/languages/c11/scanner.lisp
index a412e95..62e5681 100644
--- a/languages/c11/scanner.lisp
+++ b/languages/c11/scanner.lisp
@@ -33,6 +33,8 @@
 ;;;;**************************************************************************
 (in-package "COM.INFORMATIMAGO.LANGUAGES.C11.PARSER")

+
+#-(and) ; we use the cpp-scanner.
 (define-scanner c11-scanner
   :terminals  (
                "!" "!=" "%" "%=" "%>" "&" "&&" "&=" "(" ")"
@@ -89,22 +91,57 @@

 (defparameter *c11-regexp-tokens*
   ;; order matters
-  '((lchar      "L?'(\\.|[^\\'])+'")
-    (str        "L?\"(\\.|[^\\\"])*\"")
-    (identifier "[a-zA-Z_$][a-zA-Z_$0-9]*")
-    (flt1       "[0-9]+[Ee][-+]?[0-9]+[fFlL]?")
-    (flt2       "[0-9]*\\.[0-9]+([Ee][-+]?[0-9]+)?[fFlL]?")
-    (flt3       "[0-9]+\\.[0-9]*([Ee][-+]?[0-9]+)?[fFlL]?")
-    (hex        "0[xX][0-9A-Fa-f]+[uUlL]*")
-    (oct        "0[0-7]+[uUlL]*")
-    (dec        "[0-9]+[uUlL]*")))
+  '((|string_literal|
+     (str        "L?\"(\\.|[^\\\"])*\""))
+    (|i_constant|
+     (lchar      "L?'(\\.|[^\\'])+'"))
+    (|identifier|
+     (identifier "[a-zA-Z_$][a-zA-Z_$0-9]*"))
+    (|f_constant|
+     (flt1       "[0-9]+[Ee][-+]?[0-9]+[fFlL]?")
+     (flt2       "[0-9]*\\.[0-9]+([Ee][-+]?[0-9]+)?[fFlL]?")
+     (flt3       "[0-9]+\\.[0-9]*([Ee][-+]?[0-9]+)?[fFlL]?"))
+    (|i_constant|
+     (hex        "0[xX][0-9A-Fa-f]+[uUlL]*")
+     (oct        "0[0-7]+[uUlL]*")
+     (dec        "[0-9]+[uUlL]*"))))

 (defun compute-token-kind (token)
   (let ((text  (token-text token)))
     (or (gethash text *c11-literal-tokens-map*)
-        (first (find-if (lambda (entry)
-                          (string-match (format nil "^~A$" (second entry)) text))
-                        *c11-regexp-tokens*)))))
+        (let ((kind (first (find-if (lambda (entry)
+                                 (some (lambda (regexp)
+                                         (string-match (format nil "^~A$" (second regexp)) text))
+                                       (rest entry)))
+                                    *c11-regexp-tokens*))))
+          (if (eq kind '|identifier|)
+              (cond
+                ((typedef-name-p              *context* token) '|typedef_name|)
+                ((function-name-p             *context* token) '|func_name|)
+                ((enumeration-constant-name-p *context* token) '|enumeration_constant|)
+                (t kind))
+              kind)))))
+
+;; (untrace compute-token-kind)
+#-(and)
+(defparameter *tc*
+  (mapcar (lambda (token)
+            (setf (token-kind token) (compute-token-kind token))
+            token)
+          (reduce (function append)
+                  (reverse (com.informatimago.languages.cpp::context-output-lines
+                            (let ((*identifier-package*
+                                    (load-time-value (find-package "COM.INFORMATIMAGO.LANGUAGES.C11.C"))))
+                              (cpp-e "/Users/pjb/src/public/lisp/languages/cpp/tests/emacs.c"
+                                     :trace-includes t
+                                     :defines '("__GNUC__" "4" "__STDC__" "1" "__x86_64__" "1")
+                                     :includes '("/Users/pjb/src/macosx/emacs-24.5/src/")
+                                     :include-bracket-directories '("/Users/pjb/src/macosx/emacs-24.5/src/"
+                                                                    "/Users/pjb/src/macosx/emacs-24.5/lib/"
+                                                                    "/Users/pjb/src/macosx/gcc-4.9.2/gcc/ginclude/"
+                                                                    "/usr/include/")
+                                     :write-processed-lines nil))))
+                  :initial-value '())))


 #-(and) (
@@ -130,18 +167,23 @@
                    collect (print token))))

          (defparameter *tc*
-           (reduce (function append)
-                   (reverse (com.informatimago.languages.cpp::context-output-lines
-                             (cpp-e "/Users/pjb/src/public/lisp/languages/cpp/tests/emacs.c"
-                                    :trace-includes t
-                                    :defines '("__GNUC__" "4" "__STDC__" "1" "__x86_64__" "1")
-                                    :includes '("/Users/pjb/src/macosx/emacs-24.5/src/")
-                                    :include-bracket-directories '("/Users/pjb/src/macosx/emacs-24.5/src/"
-                                                                   "/Users/pjb/src/macosx/emacs-24.5/lib/"
-                                                                   "/Users/pjb/src/macosx/gcc-4.9.2/gcc/ginclude/"
-                                                                   "/usr/include/")
-                                    :write-processed-lines nil)))
-                   :initial-value '()))
+           (mapcar (lambda (token)
+                     (setf (token-kind token) (compute-token-kind token))
+                     token)
+                   (reduce (function append)
+                           (reverse (com.informatimago.languages.cpp::context-output-lines
+                                     (let ((*identifier-package*
+                                             (load-time-value (find-package "COM.INFORMATIMAGO.LANGUAGES.C11.C"))))
+                                       (cpp-e "/Users/pjb/src/public/lisp/languages/cpp/tests/emacs.c"
+                                              :trace-includes t
+                                              :defines '("__GNUC__" "4" "__STDC__" "1" "__x86_64__" "1")
+                                              :includes '("/Users/pjb/src/macosx/emacs-24.5/src/")
+                                              :include-bracket-directories '("/Users/pjb/src/macosx/emacs-24.5/src/"
+                                                                             "/Users/pjb/src/macosx/emacs-24.5/lib/"
+                                                                             "/Users/pjb/src/macosx/gcc-4.9.2/gcc/ginclude/"
+                                                                             "/usr/include/")
+                                              :write-processed-lines nil))))
+                           :initial-value '())))

          (dolist (token *tc*)
            (setf (token-kind token) (compute-token-kind token)))
diff --git a/languages/cpp/cpp.lisp b/languages/cpp/cpp.lisp
index cceb808..9ebb313 100644
--- a/languages/cpp/cpp.lisp
+++ b/languages/cpp/cpp.lisp
@@ -236,7 +236,7 @@
 ;;; --------------------


-(defun skip-spaces (text start)
+(defun skip-spaces-in-text (text start)
   (loop
     :while (and (< start (length text))
                 (whitespacep (aref text start)))
@@ -244,7 +244,7 @@
   start)

 (defun skip-spaces-but-one (text start)
-  (let ((start (skip-spaces text start)))
+  (let ((start (skip-spaces-in-text text start)))
     (when (and (plusp start)
                (whitespacep (aref text (1- start))))
       (decf start))
@@ -400,10 +400,10 @@ RETURN: the token text; the end position."
       :with record-space-token := nil ; we track #define to detect the same in NAME ( vs. NAME(
       :do (setf start (ecase record-space-token
                         ((nil)
-                         (skip-spaces text start))
+                         (skip-spaces-in-text text start))
                         ((:before-name)
                          (setf record-space-token :after-name)
-                         (skip-spaces text start))
+                         (skip-spaces-in-text text start))
                         (:after-name
                          (setf record-space-token nil)
                          (skip-spaces-but-one text start))))
diff --git a/languages/cpp/packages.lisp b/languages/cpp/packages.lisp
index 562bdb7..a86b609 100644
--- a/languages/cpp/packages.lisp
+++ b/languages/cpp/packages.lisp
@@ -46,10 +46,12 @@
                           "COPY-STREAM")
   (:import-from "ALEXANDRIA" "PLIST-ALIST")
   (:export "PROCESS-TOPLEVEL-FILE"
-           "TOKEN" "TOKEN-LINE" "TOKEN-COLUMN" "TOKEN-FILE"
+           "CPP-TOKEN" "TOKEN-LINE" "TOKEN-COLUMN" "TOKEN-FILE"
            "TOKEN-TEXT" "IDENTIFIER-TOKEN" "NUMBER-TOKEN" "PUNCTUATION-TOKEN"
            "OTHER-TOKEN"
-           "TOKEN-STRING"
+           "TOKEN-STRING" "TOKEN-SYMBOL"
+           "*IDENTIFIER-PACKAGE*"
+           "IDENTIFIER" "NUMBER" "STRING-LITERAL" "CHARACTER-LITERAL" "PUNCTUATION"
            "ENVIRONMENT-MACRO-DEFINITION"
            "ENVIRONMENT-MACRO-DEFINEDP"
            "ENVIRONMENT-MACRO-UNDEFINE"
diff --git a/languages/cpp/token.lisp b/languages/cpp/token.lisp
index 2ccc7b2..dd99808 100644
--- a/languages/cpp/token.lisp
+++ b/languages/cpp/token.lisp
@@ -33,11 +33,7 @@
 ;;;;**************************************************************************
 (in-package "COM.INFORMATIMAGO.LANGUAGES.CPP")

-
-(defclass cpp-token (token)
-  ((file   :initform "-" :initarg :file   :accessor token-file)))
-
-
+;;; ------------------------------------------------------------

 (defstruct (numbered-line
             (:type list)
@@ -61,30 +57,65 @@
 (defmethod token-text ((numbered-line cons))
   (line-text numbered-line))

+;;; ------------------------------------------------------------

+(defclass cpp-token (token)
+  ((file   :initform "-" :initarg :file   :accessor token-file)))

-(defmacro define-token-class (name)
+(defmacro define-token-class (name &optional slots)
   (let ((class-name (intern (concatenate 'string (string name) (string '-token)))))
     `(progn
-       (defclass ,class-name   (cpp-token) ())
+       (defclass ,class-name (cpp-token) ,slots)
        (defun ,(intern (concatenate 'string (string name) (string '-p))) (object)
          (typep object ',class-name))
        (defmethod print-object ((self ,class-name) stream)
          (print-unreadable-object (self stream :identity nil :type t)
            (let ((*print-circle* nil))
-            (format stream "~A:~A:~A: ~S"
-                    (token-file self) (token-line self) (token-column self) (token-text self))))
+             (format stream "~A:~A:~A: ~S"
+                     (token-file self) (token-line self) (token-column self) (token-text self))))
          self)
        (defun ,(intern (concatenate 'string (string 'make-) (string name))) (text &optional (column 0) (line 0) (file "-"))
          (make-instance ',class-name :text text :column column :line line :file file)))))

-(define-token-class identifier)
+(defgeneric token-symbol (token)
+  (:method ((token token))
+    (token-kind token)))
+
+(define-token-class identifier
+    ((symbol :initarg :symbol
+             :initform nil
+             :accessor token-symbol)))
 (define-token-class number)
 (define-token-class string-literal)
 (define-token-class character-literal)
 (define-token-class punctuation)
 (define-token-class other)

+(defvar *identifier-package* (load-time-value (find-package #.(package-name *package*)))
+  "Package where to intern token-symbol of identifiers.")
+
+(defmethod initialize-instance :after ((token identifier-token) &key &allow-other-keys)
+  (setf (slot-value token 'kind)   'identifier
+        (slot-value token 'symbol) (intern (slot-value token 'text) *identifier-package*)
+        (slot-value token 'text) nil))
+(defmethod token-text ((token identifier-token))
+  (symbol-name (token-symbol token)))
+
+(defmethod initialize-instance :after ((token punctuation-token) &key &allow-other-keys)
+  (setf (slot-value token 'kind) 'punctuation))
+(defmethod initialize-instance :after ((token number-token) &key &allow-other-keys)
+  (setf (slot-value token 'kind) 'number))
+(defmethod initialize-instance :after ((token string-literal-token) &key &allow-other-keys)
+  (setf (slot-value token 'kind) 'string-literal))
+(defmethod initialize-instance :after ((token character-literal-token) &key &allow-other-keys)
+  (setf (slot-value token 'kind) 'character-literal))
+
+(defun identifierp (token)
+  (typep token 'identifier-token))
+
+(defun number-token-p (token)
+  (typep token 'number-token))
+

 (defun pseudo-token (file lino)
   (make-other "" 0 lino file))
@@ -150,12 +181,6 @@
 (define-punctuation-predicate op-colon-p       ":")


-(defun identifierp (token)
-  (typep token 'identifier-token))
-
-(defun number-token-p (token)
-  (typep token 'number-token))
-

 (define-condition cpp-error (simple-error)
   ())
@@ -206,8 +231,4 @@
           :format-control format-control
           :format-arguments  format-arguments)))

-
-
-
-
 ;;;; THE END ;;;;
ViewGit