# HG changeset patch
# User Mike Becker <universe@uap-core.de>
# Date 1471962842 -7200
# Node ID c39ecbbca7c0237416fef2833a76bc7060d474e5
# Parent  534a4ef4143dff506e27aad65cb89575298b5f5c
words (token) are now stored as sstr_t

diff -r 534a4ef4143d -r c39ecbbca7c0 src/c2html.c
--- a/src/c2html.c	Tue Aug 23 15:55:02 2016 +0200
+++ b/src/c2html.c	Tue Aug 23 16:34:02 2016 +0200
@@ -48,7 +48,7 @@
 }
 
 /* TODO: remove this workaround after refactoring highlighter structure */
-static void plainparseline(char *src, UcxBuffer *dest, HighlighterData* hltr) {
+static void plainparseline(char *src, UcxBuffer *dest, int* x) {
     size_t dp = 0;
     char *buf = dest->space + dest->pos;
     while (*src && *src != '\n') {
@@ -84,8 +84,7 @@
 
     /* process lines */
     size_t lineno = 0;
-    HighlighterData highlighter_data;
-    memset(&highlighter_data, 0, sizeof(HighlighterData));
+    int multiline_comment = 0;
     
     UCX_FOREACH(sourceline, in) {
         /* increase line number and clean line buffer */
@@ -100,7 +99,7 @@
         }
         
         /* process code line */
-        highlighter(sourceline->data, line, &highlighter_data);
+        highlighter(sourceline->data, line, &multiline_comment);
         
         /* write code line */
         out(line->space, 1, line->size, stream);
@@ -174,7 +173,7 @@
         printhelp();
         return EXIT_FAILURE;
     } else {
-        /* Configure highlighter */
+        /* Choose highlighter */
         highlighter_func hltr = NULL;
         switch (sourcetype) {
             case SOURCE_C:
diff -r 534a4ef4143d -r c39ecbbca7c0 src/ccodegen.c
--- a/src/ccodegen.c	Tue Aug 23 15:55:02 2016 +0200
+++ b/src/ccodegen.c	Tue Aug 23 16:34:02 2016 +0200
@@ -42,18 +42,27 @@
 #define memcpy_const(darr,doff,str) memcpy(&(darr[doff]), str, sizeof(str)-1); \
                                     dp += sizeof(str)-1
 
-void cparseline(char *src, UcxBuffer *destbuf, HighlighterData *hltr) {
+void cparseline(char *src, UcxBuffer *destbuf, int *multiline_comment) {
     /* TODO: workaround for using old code with UcxBuffer */
     char *dest = destbuf->space + destbuf->pos;
 
-    memset(hltr->word, 0, WORDBUF_SIZE);
-    size_t wp = 0, ifp = 0, sp = (size_t)-1, dp = 0;
+    /* TODO: try to replace these buffers */
+    char wordbuf[WORDBUF_SIZE];
+    sstr_t word;
+    word.ptr = wordbuf; word.length = 0;
+    
+    char includefilebuf[512];
+    sstr_t includefile;
+    includefile.ptr = includefilebuf;
+    includefile.length = 0;
+    
+    size_t sp = (size_t)-1, dp = 0;
     int isstring = 0, iscomment = 0, isinclude = 0, parseinclude = 0;
     char quote = '\0';
     int isescaping = 0;
     
     /* continue a multi line comment highlighting */
-    if (hltr->iscommentml) {
+    if (*multiline_comment) {
         iscomment = 1;
         memcpy_const(dest, dp, "<span class=\"c2html-comment\">");
     }
@@ -65,14 +74,14 @@
         
         /* comments */
         if (!isstring && c == '/') {
-            if (hltr->iscommentml && sp > 0 && src[sp-1] == '*') {
+            if (*multiline_comment && sp > 0 && src[sp-1] == '*') {
                 iscomment = 0;
-                hltr->iscommentml = 0;
+                *multiline_comment = 0;
                 memcpy_const(dest, dp, "/</span>");
                 continue;
             } else if (!iscomment && (src[sp+1] == '/' || src[sp+1] == '*')) {
                 iscomment = 1;
-                hltr->iscommentml = (src[sp+1] == '*');
+                *multiline_comment = (src[sp+1] == '*');
                 memcpy_const(dest, dp, "<span class=\"c2html-comment\">");
             }
         }
@@ -90,8 +99,8 @@
                 if (parseinclude) {
                     dest[dp++] = '\"';
                     dest[dp++] = '>';
-                    memcpy(&(dest[dp]), hltr->includefile, ifp);
-                    dp += ifp;
+                    memcpy(&(dest[dp]), includefile.ptr, includefile.length);
+                    dp += includefile.length;
 
                     dp = writeescapedchar(dest, dp, c);
                     memcpy_const(dest, dp, "</a>");
@@ -100,8 +109,8 @@
                     memcpy_const(dest, dp,
                         "<a class=\"c2html-userinclude\" href=");
                     dp = writeescapedchar(dest, dp, c);
-                    ifp = 0;
-                    hltr->includefile[ifp++] = '\"';
+                    includefile.length = 0;
+                    includefile.ptr[includefile.length++] = '\"';
                     parseinclude = 1;
                 }
             } else if (c == '>') {
@@ -109,7 +118,7 @@
                 memcpy_const(dest, dp, "</span>");
             } else {
                 if (parseinclude) {
-                    hltr->includefile[ifp++] = c;
+                    includefile.ptr[includefile.length++] = c;
                 }
                 dp = writeescapedchar(dest, dp, c);
             }
@@ -135,46 +144,43 @@
                 if (isstring) {
                     dp = writeescapedchar(dest, dp, c);
                 } else if (!iswordcharacter(c)) {
-                    /* interpret word int_t */
-                    if (wp > 0 && wp < WORDBUF_SIZE) {
+                    if (word.length > 0 && word.length < WORDBUF_SIZE) {
                         int closespan = 1;
-                        if (check_keyword(hltr->word, ckeywords)) {
+                        sstr_t typesuffix = ST("_t");
+                        if (check_keyword(word, ckeywords)) {
                             memcpy_const(dest, dp, 
                                 "<span class=\"c2html-keyword\">");
-                        } else if (hltr->word[wp-2] == '_'
-                                && hltr->word[wp-1] == 't') {
+                        } else if (sstrsuffix(word, typesuffix)) {
                             memcpy_const(dest, dp, 
                                 "<span class=\"c2html-type\">");
-                        } else if (hltr->word[0] == '#') {
-                            isinclude = !strncmp(
-                                "#include", hltr->word, WORDBUF_SIZE);
+                        } else if (word.ptr[0] == '#') {
+                            isinclude = !sstrcmp(word, S("#include"));
                             memcpy_const(dest, dp, 
                                 "<span class=\"c2html-directive\">");
-                        } else if (check_capsonly(hltr->word, wp)) {
+                        } else if (check_capsonly(word)) {
                             memcpy_const(dest, dp, 
                                 "<span class=\"c2html-macroconst\">");
                         } else {
                             closespan = 0;
                         }
-                        for (int i = 0 ; i < wp ; i++) {
-                            dp = writeescapedchar(dest, dp, hltr->word[i]);
+                        for (int i = 0 ; i < word.length ; i++) {
+                            dp = writeescapedchar(dest, dp, word.ptr[i]);
                         }
                         if (closespan) {
                             memcpy_const(dest, dp, "</span>");
                         }
                     }
-                    memset(hltr->word, 0, WORDBUF_SIZE);
-                    wp = 0;
+                    word.length = 0;
                     dp = writeescapedchar(dest, dp, c);
                 } else {
                     /* read word */
-                    if (wp < WORDBUF_SIZE) {
-                        hltr->word[wp++] = c;
-                    } else if (wp == WORDBUF_SIZE) {
+                    if (word.length < WORDBUF_SIZE) {
+                        word.ptr[word.length++] = c;
+                    } else if (word.length == WORDBUF_SIZE) {
                         for (int i = 0 ; i < WORDBUF_SIZE ; i++) {
-                            dp = writeescapedchar(dest, dp, hltr->word[i]);
+                            dp = writeescapedchar(dest, dp, word.ptr[i]);
                         }
-                        wp++;
+                        word.length++;
                         dp = writeescapedchar(dest, dp, c);
                     } else {
                         dp = writeescapedchar(dest, dp, c);
diff -r 534a4ef4143d -r c39ecbbca7c0 src/ccodegen.h
--- a/src/ccodegen.h	Tue Aug 23 15:55:02 2016 +0200
+++ b/src/ccodegen.h	Tue Aug 23 16:34:02 2016 +0200
@@ -41,7 +41,7 @@
 
 int check_ctype(char *word, size_t len);
 int check_cdirective(char *word);
-void cparseline(char *src, UcxBuffer *dest, HighlighterData *hltr);
+void cparseline(char *src, UcxBuffer *dest, int *mlc);
 
 #ifdef	__cplusplus
 }
diff -r 534a4ef4143d -r c39ecbbca7c0 src/codegens.c
--- a/src/codegens.c	Tue Aug 23 15:55:02 2016 +0200
+++ b/src/codegens.c	Tue Aug 23 16:34:02 2016 +0200
@@ -44,18 +44,19 @@
     return dp;
 }
 
-int check_keyword(char *word, const char** keywords) {
+int check_keyword(sstr_t word, const char** keywords) {
     for (int i = 0 ; keywords[i] ; i++) {
-        if (strncmp(keywords[i], word, WORDBUF_SIZE) == 0) {
+        if (sstrcmp(word, sstr((char*)keywords[i])) == 0) {
             return 1;
         }
     }
     return 0;
 }
 
-int check_capsonly(char *word, size_t wp) {
-    for (size_t i = 0 ; i < wp ; i++) {
-        if (!isupper(word[i]) && !isdigit(word[i]) && word[i] != '_') {
+int check_capsonly(sstr_t word) {
+    for (size_t i = 0 ; i < word.length ; i++) {
+        if (!isupper(word.ptr[i]) && !isdigit(word.ptr[i])
+                && word.ptr[i] != '_') {
             return 0;
         }
     }
diff -r 534a4ef4143d -r c39ecbbca7c0 src/codegens.h
--- a/src/codegens.h	Tue Aug 23 15:55:02 2016 +0200
+++ b/src/codegens.h	Tue Aug 23 16:34:02 2016 +0200
@@ -31,26 +31,21 @@
 #define	CODEGENS_H
 
 #include <stdio.h>
+#include "ucx/string.h"
 #include "ucx/buffer.h"
 
 #ifdef	__cplusplus
 extern "C" {
 #endif
 
-#define WORDBUF_SIZE 64
+#define WORDBUF_SIZE 256
 
-typedef struct {
-    int iscommentml;
-    char word[WORDBUF_SIZE];
-    char includefile[FILENAME_MAX];
-} HighlighterData;
-
-typedef void(*highlighter_func)(char*,UcxBuffer*,HighlighterData*);
+typedef void(*highlighter_func)(char*,UcxBuffer*,int*);
 
 #define iswordcharacter(c) (isalnum(c) || c=='_' || c=='#' || c=='@')
 size_t writeescapedchar(char *dest, size_t dp, char c);
-int check_keyword(char *word, const char** keywords);
-int check_capsonly(char *word, size_t wp);
+int check_keyword(sstr_t word, const char** keywords);
+int check_capsonly(sstr_t word);
 
 
 #ifdef	__cplusplus
diff -r 534a4ef4143d -r c39ecbbca7c0 src/javacodegen.c
--- a/src/javacodegen.c	Tue Aug 23 15:55:02 2016 +0200
+++ b/src/javacodegen.c	Tue Aug 23 16:34:02 2016 +0200
@@ -44,17 +44,21 @@
 #define memcpy_const(darr,doff,str) memcpy(&(darr[doff]), str, sizeof(str)-1); \
                                     dp += sizeof(str)-1
 
-void jparseline(char *src, UcxBuffer *destbuf, HighlighterData *hltr) {
+void jparseline(char *src, UcxBuffer *destbuf, int *multiline_comment) {
     /* TODO: workaround for using old code with UcxBuffer */
     char *dest = destbuf->space + destbuf->pos;
     
-    memset(hltr->word, 0, WORDBUF_SIZE);
-    size_t wp = 0, sp = (size_t)-1, dp = 0;
+    /* TODO: try to replace this buffer */
+    char wordbuf[WORDBUF_SIZE];
+    sstr_t word;
+    word.ptr = wordbuf; word.length = 0;
+    
+    size_t sp = (size_t)-1, dp = 0;
     int isstring = 0, iscomment = 0, isimport = 0;
     char quote = '\0';
     int isescaping = 0;
 
-    if (hltr->iscommentml) {
+    if (*multiline_comment) {
         iscomment = 1;
         memcpy_const(dest, dp, "<span class=\"c2html-comment\">");
     }
@@ -66,14 +70,14 @@
         
         /* comments */
         if (!isstring && c == '/') {
-            if (hltr->iscommentml && sp > 0 && src[sp-1] == '*') {
+            if (*multiline_comment && sp > 0 && src[sp-1] == '*') {
                 iscomment = 0;
-                hltr->iscommentml = 0;
+                *multiline_comment = 0;
                 memcpy_const(dest, dp, "/</span>");
                 continue;
             } else if (!iscomment && (src[sp+1] == '/' || src[sp+1] == '*')) {
                 iscomment = 1;
-                hltr->iscommentml = (src[sp+1] == '*');
+                *multiline_comment = (src[sp+1] == '*');
                 memcpy_const(dest, dp, "<span class=\"c2html-comment\">");
             }
         }
@@ -85,7 +89,7 @@
             }
             dp = writeescapedchar(dest, dp, c);
         } else if (isimport) {
-            // TODO: local imports
+            /* TODO: local imports */
         } else {
             /* strings */
             if (!isescaping && (c == '\'' || c == '\"')) {
@@ -108,43 +112,41 @@
                 if (isstring) {
                     dp = writeescapedchar(dest, dp, c);
                 } else if (!iswordcharacter(c)) {
-                    /* interpret word int_t */
-                    if (wp > 0 && wp < WORDBUF_SIZE) {
+                    if (word.length > 0 && word.length < WORDBUF_SIZE) {
                         int closespan = 1;
-                        if (check_keyword(hltr->word, jkeywords)) {
+                        if (check_keyword(word, jkeywords)) {
                             memcpy_const(dest, dp, 
                                 "<span class=\"c2html-keyword\">");
-                        } else if (isupper(hltr->word[0])) {
+                        } else if (isupper(word.ptr[0])) {
                             memcpy_const(dest, dp, 
                                 "<span class=\"c2html-type\">");
-                        } else if (hltr->word[0] == '@') {
+                        } else if (word.ptr[0] == '@') {
                             memcpy_const(dest, dp, 
                                 "<span class=\"c2html-directive\">");
-                        } else if (check_capsonly(hltr->word, wp)) {
+                        } else if (check_capsonly(word)) {
                             memcpy_const(dest, dp, 
                                 "<span class=\"c2html-macroconst\">");
                         } else {
                             closespan = 0;
                         }
-                        for (int i = 0 ; i < wp ; i++) {
-                            dp = writeescapedchar(dest, dp, hltr->word[i]);
+                        for (int i = 0 ; i < word.length ; i++) {
+                            dp = writeescapedchar(dest, dp, word.ptr[i]);
                         }
                         if (closespan) {
                             memcpy_const(dest, dp, "</span>");
                         }
                     }
-                    memset(hltr->word, 0, WORDBUF_SIZE);
-                    wp = 0;
+                    word.length = 0;
                     dp = writeescapedchar(dest, dp, c);
                 } else {
                     /* read word */
-                    if (wp < WORDBUF_SIZE) {
-                        hltr->word[wp++] = c;
-                    } else if (wp == WORDBUF_SIZE) {
+                    if (word.length < WORDBUF_SIZE) {
+                        word.ptr[word.length++] = c;
+                    } else if (word.length == WORDBUF_SIZE) {
                         for (int i = 0 ; i < WORDBUF_SIZE ; i++) {
-                            dp = writeescapedchar(dest, dp, hltr->word[i]);
+                            dp = writeescapedchar(dest, dp, word.ptr[i]);
                         }
-                        wp++;
+                        word.length++;
                         dp = writeescapedchar(dest, dp, c);
                     } else {
                         dp = writeescapedchar(dest, dp, c);
diff -r 534a4ef4143d -r c39ecbbca7c0 src/javacodegen.h
--- a/src/javacodegen.h	Tue Aug 23 15:55:02 2016 +0200
+++ b/src/javacodegen.h	Tue Aug 23 16:34:02 2016 +0200
@@ -41,7 +41,7 @@
 
 int check_jtype(char *word, size_t len);
 int check_jdirective(char *word);
-void jparseline(char *src, UcxBuffer *dest, HighlighterData *hltr);
+void jparseline(char *src, UcxBuffer *dest, int *mlc);
 
 
 #ifdef	__cplusplus