Initial population

git-svn-id: https://svn.o-hand.com/repos/poky@2 311d38ba-8fff-0310-9ca6-ca027cbcb966

5 files changed, 706 insertions, 0 deletions

diff --git a/bitbake/lib/bb/parse/parse_c/bitbakeparser.l b/bitbake/lib/bb/parse/parse_c/bitbakeparser.l
new file mode 100644
index 0000000000..ee4ce14839
--- /dev/null
+++ b/bitbake/lib/bb/parse/parse_c/bitbakeparser.l
@@ -0,0 +1,288 @@
+/* bbf.flex 
+
+   written by Marc Singer
+   6 January 2005
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License as
+   published by the Free Software Foundation; either version 2 of the
+   License, or (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+   USA.
+
+   DESCRIPTION
+   -----------
+
+   flex lexer specification for a BitBake input file parser.
+
+   Unfortunately, flex doesn't welcome comments within the rule sets.
+   I say unfortunately because this lexer is unreasonably complex and
+   comments would make the code much easier to comprehend.
+
+   The BitBake grammar is not regular.  In order to interpret all
+   of the available input files, the lexer maintains much state as it
+   parses.  There are places where this lexer will emit tokens that
+   are invalid.  The parser will tend to catch these. 
+
+   The lexer requires C++ at the moment.  The only reason for this has
+   to do with a very small amount of managed state.  Producing a C
+   lexer should be a reasonably easy task as long as the %reentrant
+   option is used.
+
+
+   NOTES
+   -----
+
+   o RVALUES.  There are three kinds of RVALUES.  There are unquoted
+     values, double quote enclosed strings, and single quote 
+     strings.  Quoted strings may contain unescaped quotes (of either
+     type), *and* any type may span more than one line by using a
+     continuation '\' at the end of the line.  This requires us to
+     recognize all types of values with a single expression.
+     Moreover, the only reason to quote a value is to include
+     trailing or leading whitespace.  Whitespace within a value is
+     preserved, ugh.
+
+   o CLASSES.  C_ patterns define classes.  Classes ought not include
+     a repitition operator, instead letting the reference to the class
+     define the repitition count.
+
+     C_SS - symbol start
+     C_SB - symbol body
+     C_SP - whitespace
+
+*/
+
+%option never-interactive
+%option yylineno
+%option noyywrap
+%option reentrant stack
+
+
+%{
+
+#include "token.h"
+#include "lexer.h"
+#include <ctype.h>
+
+extern void *bbparseAlloc(void *(*mallocProc)(size_t));
+extern void bbparseFree(void *p, void (*freeProc)(void*));
+extern void *bbparseAlloc(void *(*mallocProc)(size_t));
+extern void *bbparse(void*, int, token_t, lex_t*);
+extern void bbparseTrace(FILE *TraceFILE, char *zTracePrompt);
+
+//static const char* rgbInput;
+//static size_t cbInput;
+
+
+int lineError;
+int errorParse;
+
+enum {
+  errorNone = 0,
+  errorUnexpectedInput,
+  errorUnsupportedFeature, 
+};
+
+#define YY_EXTRA_TYPE lex_t*
+
+        /* Read from buffer */
+#define YY_INPUT(buf,result,max_size) \
+ { yyextra->input(buf, &result, max_size); }
+
+//#define YY_DECL static size_t yylex ()
+
+#define ERROR(e) \
+  do { lineError = yylineno; errorParse = e; yyterminate (); } while (0)
+
+static const char* fixup_escapes (const char* sz);
+
+%}
+
+
+C_SP            [ \t]
+COMMENT         #.*\n
+OP_ASSIGN       "="
+OP_IMMEDIATE    ":="
+OP_PREPEND      "=+"
+OP_APPEND       "+="
+OP_COND         "?="
+B_OPEN          "{"
+B_CLOSE         "}"
+
+K_ADDTASK       "addtask"
+K_ADDHANDLER    "addhandler"
+K_AFTER         "after"
+K_BEFORE        "before"
+K_DEF           "def"
+K_INCLUDE       "include"
+K_INHERIT       "inherit"
+K_PYTHON        "python"
+K_FAKEROOT      "fakeroot"
+K_EXPORT        "export"
+K_EXPORT_FUNC   "EXPORT_FUNCTIONS"
+
+STRING          \"([^\n\r]|"\\\n")*\"
+SSTRING         \'([^\n\r]|"\\\n")*\'
+VALUE           ([^'" \t\n])|([^'" \t\n]([^\n]|(\\\n))*[^'" \t\n])
+
+C_SS            [a-zA-Z_]
+C_SB            [a-zA-Z0-9_+-.]
+REF             $\{{C_SS}{C_SB}*\}
+SYMBOL          {C_SS}{C_SB}*
+VARIABLE        $?{C_SS}({C_SB}*|{REF})*(\[[a-zA-Z0-9_]*\])?
+FILENAME        ([a-zA-Z_./]|{REF})(([-+a-zA-Z0-9_./]*)|{REF})*
+
+PROC            \({C_SP}*\)
+
+%s S_DEF
+%s S_DEF_ARGS
+%s S_DEF_BODY
+%s S_FUNC
+%s S_INCLUDE
+%s S_INHERIT
+%s S_PROC
+%s S_RVALUE
+%s S_TASK
+
+%%
+
+{OP_APPEND}                     { BEGIN S_RVALUE;
+                                  yyextra->accept (T_OP_APPEND); }
+{OP_PREPEND}                    { BEGIN S_RVALUE;
+                                  yyextra->accept (T_OP_PREPEND); }
+{OP_IMMEDIATE}                  { BEGIN S_RVALUE;
+                                  yyextra->accept (T_OP_IMMEDIATE); }
+{OP_ASSIGN}                     { BEGIN S_RVALUE;
+                                  yyextra->accept (T_OP_ASSIGN); }
+{OP_COND}                       { BEGIN S_RVALUE;
+                                  yyextra->accept (T_OP_COND); }
+
+<S_RVALUE>\\\n{C_SP}*           {  }
+<S_RVALUE>{STRING}              { BEGIN INITIAL;
+                                  size_t cb = yyleng;
+                                  while (cb && isspace (yytext[cb - 1]))
+                                      --cb;
+                                  yytext[cb - 1] = 0;
+                                  yyextra->accept (T_STRING, yytext + 1); }
+<S_RVALUE>{SSTRING}             { BEGIN INITIAL;
+                                  size_t cb = yyleng;
+                                  while (cb && isspace (yytext[cb - 1]))
+                                      --cb;
+                                  yytext[cb - 1] = 0;
+                                  yyextra->accept (T_STRING, yytext + 1); }
+
+<S_RVALUE>{VALUE}               { ERROR (errorUnexpectedInput);  }
+<S_RVALUE>{C_SP}*\n+            { BEGIN INITIAL;
+                                  yyextra->accept (T_STRING, NULL); }
+
+{K_INCLUDE}                     { BEGIN S_INCLUDE;
+                                  yyextra->accept (T_INCLUDE); }
+{K_INHERIT}                     { BEGIN S_INHERIT;
+                                  yyextra->accept (T_INHERIT); }
+{K_ADDTASK}                     { BEGIN S_TASK;
+                                  yyextra->accept (T_ADDTASK); }
+{K_ADDHANDLER}                  { yyextra->accept (T_ADDHANDLER); }
+{K_EXPORT_FUNC}                 { BEGIN S_FUNC;
+                                  yyextra->accept (T_EXPORT_FUNC); }
+<S_TASK>{K_BEFORE}              { yyextra->accept (T_BEFORE); }
+<S_TASK>{K_AFTER}               { yyextra->accept (T_AFTER); }
+<INITIAL>{K_EXPORT}             { yyextra->accept (T_EXPORT); }
+
+<INITIAL>{K_FAKEROOT}           { yyextra->accept (T_FAKEROOT); }
+<INITIAL>{K_PYTHON}             { yyextra->accept (T_PYTHON); }
+{PROC}{C_SP}*{B_OPEN}{C_SP}*\n*  { BEGIN S_PROC;
+                                  yyextra->accept (T_PROC_OPEN); }
+<S_PROC>{B_CLOSE}{C_SP}*\n*      { BEGIN INITIAL;
+                                  yyextra->accept (T_PROC_CLOSE); }
+<S_PROC>([^}][^\n]*)?\n*        { yyextra->accept (T_PROC_BODY, yytext); }
+
+{K_DEF}                         { BEGIN S_DEF; }
+<S_DEF>{SYMBOL}                 { BEGIN S_DEF_ARGS;
+                                  yyextra->accept (T_SYMBOL, yytext); }
+<S_DEF_ARGS>[^\n:]*:            { yyextra->accept (T_DEF_ARGS, yytext); }
+<S_DEF_ARGS>{C_SP}*\n           { BEGIN S_DEF_BODY; }
+<S_DEF_BODY>{C_SP}+[^\n]*\n     { yyextra->accept (T_DEF_BODY, yytext); }
+<S_DEF_BODY>\n                  { yyextra->accept (T_DEF_BODY, yytext); }
+<S_DEF_BODY>.                   { BEGIN INITIAL; unput (yytext[0]); }
+
+{COMMENT}                       { }
+
+<INITIAL>{SYMBOL}               { yyextra->accept (T_SYMBOL, yytext); }
+<INITIAL>{VARIABLE}             { yyextra->accept (T_VARIABLE, yytext); }
+
+<S_TASK>{SYMBOL}                { yyextra->accept (T_TSYMBOL, yytext); }
+<S_FUNC>{SYMBOL}                { yyextra->accept (T_FSYMBOL, yytext); }
+<S_INHERIT>{SYMBOL}             { yyextra->accept (T_ISYMBOL, yytext); }
+<S_INCLUDE>{FILENAME}           { BEGIN INITIAL;
+                                  yyextra->accept (T_ISYMBOL, yytext); }
+
+<S_TASK>\n                      { BEGIN INITIAL; }
+<S_FUNC>\n                      { BEGIN INITIAL; }
+<S_INHERIT>\n                   { BEGIN INITIAL; }
+
+[ \t\r\n]                       /* Insignificant whitespace */
+
+.                               { ERROR (errorUnexpectedInput); }
+
+                                /* Check for premature termination */
+<<EOF>>                         { return T_EOF; }
+
+%%
+
+void lex_t::accept (int token, const char* sz) 
+{
+    token_t t;
+    memset (&t, 0, sizeof (t));
+    t.copyString(sz);
+
+    /* tell lemon to parse the token */
+    parse (parser, token, t, this);
+}
+
+int lex_t::line ()const
+{
+    return yyget_lineno (scanner);
+}
+
+const char* lex_t::filename ()const
+{
+    return m_fileName;
+}
+
+void parse (MappedFile* mf)
+{
+    void* parser = bbparseAlloc (malloc);
+    yyscan_t scanner;
+    lex_t lex;
+
+    yylex_init (&scanner);
+
+    lex.parser = parser;
+    lex.scanner = scanner;
+    lex.mf = mf;
+    lex.rgbInput = mf->m_rgb;
+    lex.cbInput = mf->m_cb;
+    lex.parse = bbparse;
+    yyset_extra (&lex, scanner);
+
+
+    int result = yylex (scanner);
+
+    lex.accept (0);
+    bbparseTrace (NULL, NULL);
+
+    if (result != T_EOF)
+        WARNING ("premature end of file\n");
+
+    yylex_destroy (scanner);
+    bbparseFree (parser, free);
+}
diff --git a/bitbake/lib/bb/parse/parse_c/bitbakeparser.py b/bitbake/lib/bb/parse/parse_c/bitbakeparser.py
new file mode 100644
index 0000000000..ed7b13eef9
--- /dev/null
+++ b/bitbake/lib/bb/parse/parse_c/bitbakeparser.py
@@ -0,0 +1,133 @@
+"""
+
+BitBake C Parser Python Code
+
+Copyright (C) 2005 Holger Hans Peter Freyther
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+
+__version__ = "0xdeadbeef"
+
+class CParser:
+    """
+    The C-based Parser for Bitbake
+    """
+    def __init__(self, data, type):
+        """
+        Constructor
+        """
+        self._data = data
+
+    def _syntax_error(self, file, line):
+        """
+        lemon/flex reports an syntax error to us and we will
+        raise an exception
+        """
+        pass
+
+    def _export(self, data):
+        """
+        EXPORT VAR = "MOO"
+        we will now export VAR
+        """
+        pass
+
+    def _assign(self, key, value):
+        """
+        VAR = "MOO"
+        we will assign moo to VAR
+        """
+        pass
+
+    def _assign(self, key, value):
+        """
+        """
+        pass
+
+    def _append(self, key, value):
+        """
+        VAR += "MOO"
+        we will append " MOO" to var
+        """
+        pass
+
+    def _prepend(self, key, value):
+        """
+        VAR =+ "MOO"
+        we will prepend "MOO " to var
+        """
+        pass
+
+    def _immediate(self, key, value):
+        """
+        VAR := "MOO ${CVSDATE}"
+        we will assign immediately and expand vars
+        """
+        pass
+
+    def _conditional(self, key, value):
+        """
+        """
+        pass
+
+    def _add_task(self, task, before = None, after = None):
+        """
+        """
+        pass
+
+    def _include(self, file):
+        """
+        """
+        pass
+
+    def _inherit(self, file):
+        """
+        """
+        pass
+
+    def _shell_procedure(self, name, body):
+        """
+        """
+        pass
+
+    def _python_procedure(self, name, body):
+        """
+        """
+        pass
+
+    def _fakeroot_procedure(self, name, body):
+        """
+        """
+        pass
+
+    def _def_procedure(self, a, b, c):
+        """
+        """
+        pass
+
+    def _export_func(self, name):
+        """
+        """
+        pass
+
+    def _add_handler(self, handler):
+        """
+        """
+        pass
diff --git a/bitbake/lib/bb/parse/parse_c/bitbakeparser.y b/bitbake/lib/bb/parse/parse_c/bitbakeparser.y
new file mode 100644
index 0000000000..4bc81a913a
--- /dev/null
+++ b/bitbake/lib/bb/parse/parse_c/bitbakeparser.y
@@ -0,0 +1,161 @@
+/* bbp.lemon 
+
+   written by Marc Singer
+   6 January 2005
+
+   This program is free software; you can redistribute it and/or
+   modify it under the terms of the GNU General Public License as
+   published by the Free Software Foundation; either version 2 of the
+   License, or (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+   USA.
+
+   DESCRIPTION
+   -----------
+
+   lemon parser specification file for a BitBake input file parser.
+
+   Most of the interesting shenanigans are done in the lexer.  The
+   BitBake grammar is not regular.  In order to emit tokens that
+   the parser can properly interpret in LALR fashion, the lexer
+   manages the interpretation state.  This is why there are ISYMBOLs,
+   SYMBOLS, and TSYMBOLS.
+
+   This parser was developed by reading the limited available
+   documentation for BitBake and by analyzing the available BB files.
+   There is no assertion of correctness to be made about this parser.
+
+*/
+
+%token_type {token_t}
+%name bbparse
+%token_prefix   T_
+%extra_argument  {lex_t* lex}
+
+%include {
+#include "token.h"
+}
+
+
+%token_destructor { $$.release_this (); }
+
+%syntax_error     { printf ("%s:%d: syntax error\n",
+                    lex->filename (), lex->line ()); }
+
+program ::= statements.
+
+statements ::= statements statement.
+statements ::= .
+
+variable(r) ::= SYMBOL(s).
+        { r.assignString( s.string() );
+          s.assignString( 0 );
+          s.release_this(); }
+variable(r) ::= VARIABLE(v).
+        {
+          r.assignString( v.string() );
+          v.assignString( 0 );
+          v.release_this(); }
+
+statement ::= EXPORT variable(s) OP_ASSIGN STRING(v).
+        { e_assign( s.string(), v.string() );
+          e_export( s.string() );
+          s.release_this(); v.release_this(); }
+statement ::= EXPORT variable(s) OP_IMMEDIATE STRING(v).
+        { e_immediate (s.string(), v.string() );
+          e_export( s.string() );
+          s.release_this(); v.release_this(); }
+statement ::= EXPORT variable(s) OP_COND STRING(v).
+        { e_cond( s.string(), v.string() );
+          s.release_this(); v.release_this(); }
+
+statement ::= variable(s) OP_ASSIGN STRING(v).
+        { e_assign( s.string(), v.string() );
+          s.release_this(); v.release_this(); }
+statement ::= variable(s) OP_PREPEND STRING(v).
+        { e_prepend( s.string(), v.string() );
+          s.release_this(); v.release_this(); }
+statement ::= variable(s) OP_APPEND STRING(v).
+        { e_append( s.string() , v.string() );
+          s.release_this(); v.release_this(); }
+statement ::= variable(s) OP_IMMEDIATE STRING(v).
+        { e_immediate( s.string(), v.string() );
+          s.release_this(); v.release_this(); }
+statement ::= variable(s) OP_COND STRING(v).
+        { e_cond( s.string(), v.string() );
+          s.release_this(); v.release_this(); }
+
+task ::= TSYMBOL(t) BEFORE TSYMBOL(b) AFTER  TSYMBOL(a).
+        { e_addtask( t.string(), b.string(), a.string() );
+          t.release_this(); b.release_this(); a.release_this(); }
+task ::= TSYMBOL(t) AFTER  TSYMBOL(a) BEFORE TSYMBOL(b).
+        { e_addtask( t.string(), b.string(), a.string());
+          t.release_this(); a.release_this(); b.release_this(); }
+task ::= TSYMBOL(t).
+        { e_addtask( t.string(), NULL, NULL);
+          t.release_this();}
+task ::= TSYMBOL(t) BEFORE TSYMBOL(b).
+        { e_addtask( t.string(), b.string(), NULL);
+          t.release_this(); b.release_this(); }
+task ::= TSYMBOL(t) AFTER  TSYMBOL(a).
+        { e_addtask( t.string(), NULL, a.string());
+          t.release_this(); a.release_this(); }
+tasks ::= tasks task.
+tasks ::= task.
+statement ::= ADDTASK tasks.
+
+statement ::= ADDHANDLER SYMBOL(s).
+        { e_addhandler( s.string()); s.release_this (); }
+
+func ::= FSYMBOL(f). { e_export_func(f.string()); f.release_this(); }
+funcs ::= funcs func.
+funcs ::= func.
+statement ::= EXPORT_FUNC funcs.
+
+inherit ::= ISYMBOL(i). { e_inherit(i.string() ); i.release_this (); }
+inherits ::= inherits inherit.
+inherits ::= inherit.
+statement ::= INHERIT inherits.
+
+statement ::= INCLUDE ISYMBOL(i).
+        { e_include(i.string() ); i.release_this(); }
+
+proc_body(r) ::= proc_body(l) PROC_BODY(b). 
+        { /* concatenate body lines */
+          r.assignString( token_t::concatString(l.string(), b.string()) );
+          l.release_this ();
+          b.release_this ();
+        }
+proc_body(b) ::= . { b.assignString(0); }
+statement ::= variable(p) PROC_OPEN proc_body(b) PROC_CLOSE.
+        { e_proc( p.string(), b.string() );
+          p.release_this(); b.release_this(); }
+statement ::= PYTHON SYMBOL(p) PROC_OPEN proc_body(b) PROC_CLOSE.
+        { e_proc_python (p.string(), b.string() );
+          p.release_this(); b.release_this(); }
+statement ::= PYTHON PROC_OPEN proc_body(b) PROC_CLOSE.
+        { e_proc_python( NULL, b.string());
+          b.release_this (); }
+
+statement ::= FAKEROOT SYMBOL(p) PROC_OPEN proc_body(b) PROC_CLOSE.
+        { e_proc_fakeroot(p.string(), b.string() );
+          p.release_this (); b.release_this (); }
+
+def_body(r) ::= def_body(l) DEF_BODY(b).
+        { /* concatenate body lines */
+          r.assignString( token_t::concatString(l.string(), b.string());
+          l.release_this (); b.release_this ();
+        }
+def_body(b) ::= . { b.sz = 0; }
+statement ::= SYMBOL(p) DEF_ARGS(a) def_body(b).
+        { e_def( p.string(), a.string(), b.string());
+          p.release_this(); a.release_this(); b.release_this(); }
+
diff --git a/bitbake/lib/bb/parse/parse_c/lexer.h b/bitbake/lib/bb/parse/parse_c/lexer.h
new file mode 100644
index 0000000000..1edf72dcf5
--- /dev/null
+++ b/bitbake/lib/bb/parse/parse_c/lexer.h
@@ -0,0 +1,41 @@
+/*
+Copyright (C) 2005 Holger Hans Peter Freyther
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+*/
+
+#ifndef LEXER_H
+#define LEXER_H
+
+struct lex_t {
+    void *parser;
+    void *scanner;
+    void* (*parse)(void*, int, token_t, lex_t*);
+
+    void accept(int token, const char* string = 0);
+    void input(char *buf, int *result, int_max_size);
+    int  line()const;
+    const char* filename()const;
+private:
+    const char* m_fileName;
+};
+
+
+#endif
diff --git a/bitbake/lib/bb/parse/parse_c/token.h b/bitbake/lib/bb/parse/parse_c/token.h
new file mode 100644
index 0000000000..2351fda6b5
--- /dev/null
+++ b/bitbake/lib/bb/parse/parse_c/token.h
@@ -0,0 +1,83 @@
+/*
+Copyright (C) 2005 Holger Hans Peter Freyther
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+*/
+
+#ifndef TOKEN_H
+#define TOKEN_H
+
+#define PURE_METHOD
+
+struct token_t {
+    const char* string()const PURE_METHOD;
+
+    static char* concatString(const char* l, const char* r);
+    void assignString(const char* str);
+    void copyString(const char* str);
+
+    void release_this();
+
+private:
+    char  *m_string;
+    size_t m_stringLen;
+};
+
+inline const char* token_t::string()const
+{
+    return m_string;
+}
+
+/*
+ * append str to the current string
+ */
+inline char* token_t::concatString(const char* l, const char* r)
+{
+    size_t cb = (l ? strlen (l) : 0) + strlen (r) + 1;
+    r_sz = new char[cb];
+    *r_sz = 0;
+    if (l) strcat (r_sz, l);
+    strcat (r_sz, r);
+
+    return r_sz;
+}
+
+inline void token_t::assignString(const char* str)
+{
+    m_string = str;
+    m_stringLen = str ? strlen(str) : 0;
+}
+
+inline void token_t::copyString(const char* str)
+{
+    if( str ) {
+        m_stringLen = strlen(str);
+        m_string = new char[m_stringLen+1];
+        strcpy(m_string, str)
+    }
+}
+
+inline void token_t::release_this()
+{
+    delete m_string;
+    m_string = 0;
+}
+
+#endif