.. comment: -*- mode:rst; coding:utf-8 -*-


We need to get tokens.

::

    tu -> tokens -> cursors
    tu -> cursors

annotate with tokens ::

    /* id token_spelling(id token,id tu){ */
    /*     return(cxstring(clang_getTokenSpelling(unbox_translation_unit(tu),unbox_token(token))));} */
    /* id token_location(id token,id tu){ */
    /*     return(location(clang_getTokenLocation(unbox_translation_unit(tu),unbox_token(token))));} */
    /* id token_range(id token,id tu){ */
    /*     return(range(clang_getTokenExtent(unbox_translation_unit(tu),unbox_token(token))));} */

    void annotate_cursors_with_tokens(id tucu,id table){
        CXTranslationUnit tu=clang_Cursor_getTranslationUnit(unbox_cursor(tucu));
        CXSourceRange range=clang_getCursorExtent(unbox_cursor(tucu));
        unsigned numTokens=0;
        CXToken* tokens=0;
        clang_tokenize(tu,range,&tokens,&numTokens);
        CXCursor* cursors=memory_allocate(__FUNCTION__,sizeof(*cursors),numTokens);
        clang_annotateTokens(tu,tokens,numTokens,cursors);
        DOTIMES(i,numTokens,
                { id key=cursor(cursors[i]);
                  id entry=hashtable_get(table,key);
                  if(not(null(entry))){
                      id kind=tokenSymbol(clang_getTokenKind(tokens[i]));
                      id spelling=cxstring(clang_getTokenSpelling(tu,tokens[i]));
                      hashtable_set(table,key,list(2,first(entry),
                                                   cons(first(second(entry)),
                                                        cons(KW(token),
                                                             cons(list(3,SY(token),kind,spelling),
                                                                  rest(second(entry)))))));}});}


    annotate_cursors_with_tokens(tucu,table);