Improved robustness of the emoji feature

Changes:
- Use of `@emoji name` instead of `:name:`
- Support only GitHub emojis (i.e. without spaces or special characters in the name)
- Provided script to download images for LaTeX support.
- XML output now has <emoji> tag with name an unicode sequence.

1 files changed, 14 insertions, 46 deletions

diff --git a/src/doctokenizer.l b/src/doctokenizer.l
index 4dc5fad..bd1242d 100644
--- a/src/doctokenizer.l
+++ b/src/doctokenizer.l
@@ -126,7 +126,6 @@ const char *tokToString(int token)
     case TK_RCSTAG:      return "TK_RCSTAG";
     case TK_URL:         return "TK_URL";
     case TK_COMMAND_BS:  return "TK_COMMAND_BS";
-    case TK_EMOJI:       return "TK_EMOJI";
   }
   return "ERROR";
 }
@@ -343,7 +342,6 @@ CMD       ("\\"|"@")
 WS        [ \t\r\n]
 NONWS     [^ \t\r\n]
 BLANK     [ \t\r]
-EMOJI     (":"[a-z_A-Z0-9\x80-\xFF"'+()&\*\.!,#-]":"|":"[a-z_A-Z0-9\x80-\xFF"'+()&\*\.!,#-][a-z_A-Z0-9\x80-\xFF "'+()&\*\.!,#-]*[a-z_A-Z0-9\x80-\xFF"'+()&\*\.!,#-]":")
 ID        "$"?[a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF]*
 LABELID   [a-z_A-Z\x80-\xFF][a-z_A-Z0-9\x80-\xFF\-]*
 PHPTYPE   [\\:a-z_A-Z0-9\x80-\xFF\-]+
@@ -462,6 +460,7 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 %x St_SetScopeEnd
 %x St_Options
 %x St_Block
+%x St_Emoji
 
 %x St_Sections
 %s St_SecLabel1
@@ -689,17 +688,6 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          g_token->name = yytext;
                          return TK_SYMBOL;
                        }
-<St_Para,St_Text>{EMOJI}/[^:] { /* emoji symbol */ 
-                         if (g_fileName == CiteConsts::fileName)
-                         {
-                           REJECT;
-                         }
-                         else
-                         {
-                           g_token->name = yytext;
-                           return TK_EMOJI;
-                         }
-                       }
 
   /********* patterns for linkable words ******************/
 
@@ -946,17 +934,6 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          g_token->name = yytext;
   		         return TK_SYMBOL;
                        }
-<St_TitleN>{EMOJI}/[^:]     { /* emoji */
-                         if (g_fileName == CiteConsts::fileName)
-                         {
-                           REJECT;
-                         }
-                         else
-                         {
-                           g_token->name = yytext;
-                           return TK_EMOJI;
-                         }
-                       }
 <St_TitleN>{HTMLTAG}   {
   		       }
 <St_TitleN>{SPCMD1}    |   
@@ -993,17 +970,6 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          g_token->name = yytext;
   		         return TK_SYMBOL;
                        }
-<St_TitleQ>{EMOJI}/[^:]  { /* emoji */
-                         if (g_fileName == CiteConsts::fileName)
-                         {
-                           REJECT;
-                         }
-                         else
-                         {
-                           g_token->name = yytext;
-                           return TK_EMOJI;
-                         }
-                       }
 <St_TitleQ>{SPCMD1}    |   
 <St_TitleQ>{SPCMD2}    { /* special command */ 
                          g_token->name = yytext+1;
@@ -1143,17 +1109,6 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
                          g_token->name = yytext;
   		         return TK_SYMBOL;
                        }
-<St_Ref2>{EMOJI}/[^:]    { /* emoji */
-                         if (g_fileName == CiteConsts::fileName)
-                         {
-                           REJECT;
-                         }
-                         else
-                         {
-                           g_token->name = yytext;
-                           return TK_EMOJI;
-                         }
-                       }
 <St_Ref2>{SPCMD1}      |   
 <St_Ref2>{SPCMD2}      { /* special command */ 
                          g_token->name = yytext+1;
@@ -1232,6 +1187,13 @@ REFWORD_NOCV   {FILEMASK}|{LABELID}|{REFWORD2_NOCV}|{REFWORD3}|{REFWORD4_NOCV}
 <St_Block>"]"          {
                          return TK_WORD;
                        }
+<St_Emoji>[:0-9_a-z+-]+  {
+                         g_token->name=yytext;
+                         return TK_WORD;
+                       }
+<St_Emoji>.            {
+                         return 0;
+                       }
 <St_File>{FILEMASK}    {
   			 g_token->name = yytext;
 			 return TK_WORD;  
@@ -1606,6 +1568,12 @@ void doctokenizerYYsetStateBlock()
   BEGIN(St_Block);
 }
 
+void doctokenizerYYsetStateEmoji()
+{
+  g_token->name="";
+  BEGIN(St_Emoji);
+}
+
 void doctokenizerYYcleanup()
 {
   yy_delete_buffer( YY_CURRENT_BUFFER );