qdoc: Added support for different source and output character encodings.

Previously, qdoc assumed Latin1 (ISO-8859-1) for source code and other
documentation, and wrote out XHTML with the same encoding. This change
adds additional configuration options (sourceencoding, outputencoding,
naturallanguage) that enable translated documentation in non-Latin1
encodings to be built with qdoc.

To be reviewed before merge into the master branch.

Reviewed-by: Trust Me

1 files changed, 77 insertions, 60 deletions

diff --git a/tools/qdoc3/htmlgenerator.cpp b/tools/qdoc3/htmlgenerator.cpp
index a3cdae6..21b358a 100644
--- a/tools/qdoc3/htmlgenerator.cpp
+++ b/tools/qdoc3/htmlgenerator.cpp
@@ -54,6 +54,7 @@
 #include <qdebug.h>
 #include <qlist.h>
 #include <qiterator.h>
+#include <qtextcodec.h>
 
 QT_BEGIN_NAMESPACE
 
@@ -266,6 +267,15 @@ void HtmlGenerator::initializeGenerator(const Config &config)
 
     projectUrl = config.getString(CONFIG_URL);
 
+    outputEncoding = config.getString(CONFIG_OUTPUTENCODING);
+    if (outputEncoding.isEmpty())
+        outputEncoding = QLatin1String("ISO-8859-1");
+    outputCodec = QTextCodec::codecForName(outputEncoding.toLocal8Bit());
+
+    naturalLanguage = config.getString(CONFIG_NATURALLANGUAGE);
+    if (naturalLanguage.isEmpty())
+        naturalLanguage = QLatin1String("en");
+
     QSet<QString> editionNames = config.subVars(CONFIG_EDITION);
     QSet<QString>::ConstIterator edition = editionNames.begin();
     while (edition != editionNames.end()) {
@@ -431,11 +441,11 @@ int HtmlGenerator::generateAtom(const Atom *atom,
                 endLink();
             }
             else {
-                out() << protect(atom->string());
+                out() << protectEnc(atom->string());
             }
         }
         else {
-            out() << protect(atom->string());
+            out() << protectEnc(atom->string());
         }
         break;
     case Atom::BaseName:
@@ -483,7 +493,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
     case Atom::C:
         out() << formattingLeftMap()[ATOM_FORMATTING_TELETYPE];
         if (inLink) {
-            out() << protect(plainCode(atom->string()));
+            out() << protectEnc(plainCode(atom->string()));
         }
         else {
             out() << highlightedCode(atom->string(), marker, relative);
@@ -516,7 +526,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
         // fallthrough
     case Atom::CodeBad:
         out() << "<pre><font color=\"#404040\">"
-              << trimmedTrailing(protect(plainCode(indent(codeIndent,atom->string()))))
+              << trimmedTrailing(protectEnc(plainCode(indent(codeIndent,atom->string()))))
               << "</font></pre>\n";
 	break;
     case Atom::FootnoteLeft:
@@ -768,7 +778,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
                         out() << "<a name=\""
                               << Doc::canonicalTitle((*s).name)
                               << "\"></a>\n";
-                        out() << "<h3>" << protect((*s).name) << "</h3>\n";
+                        out() << "<h3>" << protectEnc((*s).name) << "</h3>\n";
                         if (idx == Class)
                             generateCompactList(0, marker, ncmap.value(), QString("Q"));
                         else if (idx == MemberFunction) {
@@ -792,7 +802,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
                                       << linkForNode(pmap.key(), 0)
                                       << "\">";
                                 QStringList pieces = fullName(pmap.key(), 0, marker).split("::");
-                                out() << protect(pieces.last());
+                                out() << protectEnc(pieces.last());
                                 out() << "</a>"  << ":</p>\n";
 
                                 generateSection(nlist, 0, marker, CodeMarker::Summary);
@@ -820,12 +830,12 @@ int HtmlGenerator::generateAtom(const Atom *atom,
                 out() << "<p align=\"center\">";
             if (fileName.isEmpty()) {
                 out() << "<font color=\"red\">[Missing image "
-                      << protect(atom->string()) << "]</font>";
+                      << protectEnc(atom->string()) << "]</font>";
             }
             else {
-                out() << "<img src=\"" << protect(fileName) << "\"";
+                out() << "<img src=\"" << protectEnc(fileName) << "\"";
                 if (!text.isEmpty())
-                    out() << " alt=\"" << protect(text) << "\"";
+                    out() << " alt=\"" << protectEnc(text) << "\"";
                 out() << " />";
                 helpProjectWriter->addExtraFile(fileName);
             }
@@ -923,7 +933,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
             // ### Trenton
 
             out() << "<tr><td valign=\"top\"><tt>"
-                  << protect(plainCode(marker->markedUpEnumValue(atom->next()->string(),
+                  << protectEnc(plainCode(marker->markedUpEnumValue(atom->next()->string(),
                                                                  relative)))
                   << "</tt></td><td align=\"center\" valign=\"top\">";
 
@@ -936,7 +946,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
             if (itemValue.isEmpty())
                 out() << "?";
             else
-                out() << "<tt>" << protect(itemValue) << "</tt>";
+                out() << "<tt>" << protectEnc(itemValue) << "</tt>";
 
             skipAhead = 1;
         }
@@ -1052,7 +1062,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
             generateLink(atom, relative, marker);
         }
         else {
-            out() << protect(atom->string());
+            out() << protectEnc(atom->string());
         }
         break;
     case Atom::TableLeft:
@@ -1166,7 +1176,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
         out() << "<font color=\"red\"><b>&lt;Missing HTML&gt;</b></font>";
         break;
     case Atom::UnknownCommand:
-        out() << "<font color=\"red\"><b><code>\\" << protect(atom->string())
+        out() << "<font color=\"red\"><b><code>\\" << protectEnc(atom->string())
               << "</code></b></font>";
         break;
 #ifdef QDOC_QML
@@ -1295,7 +1305,7 @@ void HtmlGenerator::generateClassLikeNode(const InnerNode *inner,
                 out() << "<a name=\""
                       << registerRef((*s).name.toLower())
                       << "\"></a>\n";
-                out() << "<h2>" << protect((*s).name) << "</h2>\n";
+                out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
                 generateSection(s->members, inner, marker, CodeMarker::Summary);
             }
             if (!s->reimpMembers.isEmpty()) {
@@ -1304,7 +1314,7 @@ void HtmlGenerator::generateClassLikeNode(const InnerNode *inner,
                 out() << "<a name=\""
                       << registerRef(name.toLower())
                       << "\"></a>\n";
-                out() << "<h2>" << protect(name) << "</h2>\n";
+                out() << "<h2>" << protectEnc(name) << "</h2>\n";
                 generateSection(s->reimpMembers, inner, marker, CodeMarker::Summary);
             }
 
@@ -1343,7 +1353,7 @@ void HtmlGenerator::generateClassLikeNode(const InnerNode *inner,
     s = sections.begin();
     while (s != sections.end()) {
         out() << "<hr />\n";
-        out() << "<h2>" << protect((*s).name) << "</h2>\n";
+        out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
 
         NodeList::ConstIterator m = (*s).members.begin();
         while (m != (*s).members.end()) {
@@ -1513,7 +1523,7 @@ void HtmlGenerator::generateFakeNode(const FakeNode *fake, CodeMarker *marker)
         s = sections.begin();
         while (s != sections.end()) {
             out() << "<a name=\"" << registerRef((*s).name) << "\"></a>\n";
-            out() << "<h2>" << protect((*s).name) << "</h2>\n";
+            out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
             generateQmlSummary(*s,fake,marker);
             ++s;
         }
@@ -1529,7 +1539,7 @@ void HtmlGenerator::generateFakeNode(const FakeNode *fake, CodeMarker *marker)
         sections = marker->qmlSections(qml_cn,CodeMarker::Detailed);
         s = sections.begin();
         while (s != sections.end()) {
-            out() << "<h2>" << protect((*s).name) << "</h2>\n";
+            out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
             NodeList::ConstIterator m = (*s).members.begin();
             while (m != (*s).members.end()) {
                 generateDetailedQmlMember(*m, fake, marker);
@@ -1549,7 +1559,7 @@ void HtmlGenerator::generateFakeNode(const FakeNode *fake, CodeMarker *marker)
     s = sections.begin();
     while (s != sections.end()) {
         out() << "<a name=\"" << registerRef((*s).name) << "\"></a>\n";
-        out() << "<h2>" << protect((*s).name) << "</h2>\n";
+        out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
         generateSectionList(*s, fake, marker, CodeMarker::Summary);
         ++s;
     }
@@ -1578,7 +1588,7 @@ void HtmlGenerator::generateFakeNode(const FakeNode *fake, CodeMarker *marker)
     s = sections.begin();
     while (s != sections.end()) {
         out() << "<hr />\n";
-        out() << "<h2>" << protect((*s).name) << "</h2>\n";
+        out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
 
         NodeList::ConstIterator m = (*s).members.begin();
         while (m != (*s).members.end()) {
@@ -1624,11 +1634,11 @@ void HtmlGenerator::generateHeader(const QString& title,
                                    CodeMarker *marker,
                                    bool mainPage)
 {
-    out() << "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n";
+    out() << QString("<?xml version=\"1.0\" encoding=\"%1\"?>\n").arg(outputEncoding);
 
     out() << "<!DOCTYPE html\n"
-             "    PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"DTD/xhtml1-strict.dtd\">\n"
-             "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">\n";
+             "    PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"DTD/xhtml1-strict.dtd\">\n";
+    out() << QString("<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"%1\" lang=\"%1\">\n").arg(naturalLanguage);
 
     QString shortVersion;
     if ((project != "Qtopia") && (project != "Qt Extended")) {
@@ -1648,7 +1658,9 @@ void HtmlGenerator::generateHeader(const QString& title,
     }
 
     out() << "<head>\n"
-             "  <title>" << shortVersion << protect(title) << "</title>\n";
+             "  <title>" << shortVersion << protectEnc(title) << "</title>\n";
+    out() << QString("<meta http-equiv=\"Content-type\" content=\"text/html; charset=%1\" />").arg(outputEncoding);
+
     if (!style.isEmpty())
         out() << "    <style type=\"text/css\">" << style << "</style>\n";
 
@@ -1657,8 +1669,8 @@ void HtmlGenerator::generateHeader(const QString& title,
         QMapIterator<QString, QString> i(metaMap);
         while (i.hasNext()) {
             i.next();
-            out() << "    <meta name=\"" << protect(i.key()) << "\" contents=\""
-                  << protect(i.value()) << "\" />\n";
+            out() << "    <meta name=\"" << protectEnc(i.key()) << "\" contents=\""
+                  << protectEnc(i.value()) << "\" />\n";
         }
     }
 
@@ -1682,9 +1694,9 @@ void HtmlGenerator::generateHeader(const QString& title,
 
             navigationLinks += "[Previous: <a href=\"" + anchorPair.first + "\">";
             if (linkPair.first == linkPair.second && !anchorPair.second.isEmpty())
-                navigationLinks += protect(anchorPair.second);
+                navigationLinks += protectEnc(anchorPair.second);
             else
-                navigationLinks += protect(linkPair.second);
+                navigationLinks += protectEnc(linkPair.second);
             navigationLinks += "</a>]\n";
         }
         if (node->links().contains(Node::ContentsLink)) {
@@ -1700,9 +1712,9 @@ void HtmlGenerator::generateHeader(const QString& title,
 
             navigationLinks += "[<a href=\"" + anchorPair.first + "\">";
             if (linkPair.first == linkPair.second && !anchorPair.second.isEmpty())
-                navigationLinks += protect(anchorPair.second);
+                navigationLinks += protectEnc(anchorPair.second);
             else
-                navigationLinks += protect(linkPair.second);
+                navigationLinks += protectEnc(linkPair.second);
             navigationLinks += "</a>]\n";
         }
         if (node->links().contains(Node::NextLink)) {
@@ -1718,9 +1730,9 @@ void HtmlGenerator::generateHeader(const QString& title,
 
             navigationLinks += "[Next: <a href=\"" + anchorPair.first + "\">";
             if (linkPair.first == linkPair.second && !anchorPair.second.isEmpty())
-                navigationLinks += protect(anchorPair.second);
+                navigationLinks += protectEnc(anchorPair.second);
             else
-                navigationLinks += protect(linkPair.second);
+                navigationLinks += protectEnc(linkPair.second);
             navigationLinks += "</a>]\n";
         }
         if (node->links().contains(Node::IndexLink)) {
@@ -1771,7 +1783,7 @@ void HtmlGenerator::generateTitle(const QString& title,
                                   const Node *relative,
                                   CodeMarker *marker)
 {
-    out() << "<h1 class=\"title\">" << protect(title);
+    out() << "<h1 class=\"title\">" << protectEnc(title);
     if (!subTitle.isEmpty()) {
         out() << "<br />";
         if (subTitleSize == SmallSubTitle)
@@ -2009,18 +2021,18 @@ QString HtmlGenerator::generateLowStatusMemberFile(const InnerNode *inner,
 
     out() << "<p><ul><li><a href=\""
           << linkForNode(inner, 0) << "\">"
-          << protect(inner->name())
+          << protectEnc(inner->name())
           << " class reference</a></li></ul></p>\n";
 
     for (i = 0; i < sections.size(); ++i) {
-        out() << "<h2>" << protect(sections.at(i).name) << "</h2>\n";
+        out() << "<h2>" << protectEnc(sections.at(i).name) << "</h2>\n";
         generateSectionList(sections.at(i), inner, marker, CodeMarker::Summary);
     }
 
     sections = marker->sections(inner, CodeMarker::Detailed, status);
     for (i = 0; i < sections.size(); ++i) {
         out() << "<hr />\n";
-        out() << "<h2>" << protect(sections.at(i).name) << "</h2>\n";
+        out() << "<h2>" << protectEnc(sections.at(i).name) << "</h2>\n";
 
         NodeList::ConstIterator m = sections.at(i).members.begin();
         while (m != sections.at(i).members.end()) {
@@ -2113,7 +2125,7 @@ void HtmlGenerator::generateAnnotatedList(const Node *relative,
         }
         else {
             out() << "<td>";
-            out() << protect(node->doc().briefText().toString());
+            out() << protectEnc(node->doc().briefText().toString());
             out() << "</td>";
         }
         out() << "</tr>\n";
@@ -2315,7 +2327,7 @@ void HtmlGenerator::generateCompactList(const Node *relative,
                         << linkForNode(it.value(), relative)
                         << "\">";
                     QStringList pieces = fullName(it.value(), relative, marker).split("::");
-                    out() << protect(pieces.last());
+                    out() << protectEnc(pieces.last());
                     out() << "</a>";
                     if (pieces.size() > 1) {
                         out() << " (";
@@ -2357,7 +2369,7 @@ void HtmlGenerator::generateFunctionIndex(const Node *relative,
 #else
         out() << "<p>";
 #endif
-        out() << protect(f.key()) << ":";
+        out() << protectEnc(f.key()) << ":";
 
         currentLetter = f.key()[0].unicode();
         while (islower(currentLetter) && currentLetter >= nextLetter) {
@@ -2411,7 +2423,7 @@ void HtmlGenerator::generateLegaleseList(const Node *relative,
     QString marked = marker->markedUpSynopsis(node, relative, style);
     QRegExp templateTag("(<[^@>]*>)");
     if (marked.indexOf(templateTag) != -1) {
-        QString contents = protect(marked.mid(templateTag.pos(1),
+        QString contents = protectEnc(marked.mid(templateTag.pos(1),
                                               templateTag.cap(1).length()));
         marked.replace(templateTag.pos(1), templateTag.cap(1).length(),
                         contents);
@@ -2450,7 +2462,7 @@ void HtmlGenerator::generateQmlItem(const Node *node,
     QString marked = marker->markedUpQmlItem(node,summary);
     QRegExp templateTag("(<[^@>]*>)");
     if (marked.indexOf(templateTag) != -1) {
-        QString contents = protect(marked.mid(templateTag.pos(1),
+        QString contents = protectEnc(marked.mid(templateTag.pos(1),
                                               templateTag.cap(1).length()));
         marked.replace(templateTag.pos(1), templateTag.cap(1).length(),
                         contents);
@@ -2556,7 +2568,7 @@ void HtmlGenerator::generateOverviewList(const Node *relative, CodeMarker * /* m
             const FakeNode *groupNode = groupTitlesMap[groupTitle];
             out() << QString("<h3><a href=\"%1\">%2</a></h3>\n").arg(
                         linkForNode(groupNode, relative)).arg(
-                        protect(groupNode->fullTitle()));
+                        protectEnc(groupNode->fullTitle()));
 
             if (fakeNodeMap[groupNode].count() == 0)
                 continue;
@@ -2568,7 +2580,7 @@ void HtmlGenerator::generateOverviewList(const Node *relative, CodeMarker * /* m
                 if (title.startsWith("The "))
                     title.remove(0, 4);
                 out() << "<li><a href=\"" << linkForNode(fakeNode, relative) << "\">"
-                      << protect(title) << "</a></li>\n";
+                      << protectEnc(title) << "</a></li>\n";
             }
             out() << "</ul>\n";
         }
@@ -2582,7 +2594,7 @@ void HtmlGenerator::generateOverviewList(const Node *relative, CodeMarker * /* m
             if (title.startsWith("The "))
                 title.remove(0, 4);
             out() << "<li><a href=\"" << linkForNode(fakeNode, relative) << "\">"
-                  << protect(title) << "</a></li>\n";
+                  << protectEnc(title) << "</a></li>\n";
         }
         out() << "</ul>\n";
     }
@@ -2745,7 +2757,7 @@ void HtmlGenerator::generateSectionInheritedList(const Section& section,
         }
         out() << " inherited from <a href=\"" << fileName((*p).first)
               << "#" << HtmlGenerator::cleanRef(section.name.toLower()) << "\">"
-              << protect(marker->plainFullName((*p).first, relative))
+              << protectEnc(marker->plainFullName((*p).first, relative))
               << "</a></li>\n";
         ++p;
     }
@@ -2760,7 +2772,7 @@ void HtmlGenerator::generateSynopsis(const Node *node,
     QString marked = marker->markedUpSynopsis(node, relative, style);
     QRegExp templateTag("(<[^@>]*>)");
     if (marked.indexOf(templateTag) != -1) {
-        QString contents = protect(marked.mid(templateTag.pos(1),
+        QString contents = protectEnc(marked.mid(templateTag.pos(1),
                                               templateTag.cap(1).length()));
         marked.replace(templateTag.pos(1), templateTag.cap(1).length(),
                         contents);
@@ -3029,7 +3041,7 @@ void HtmlGenerator::generateSectionInheritedList(const Section& section,
         }
         out() << " inherited from <a href=\"" << fileName((*p).first)
               << "#" << HtmlGenerator::cleanRef(section.name.toLower()) << "\">"
-              << protect(marker->plainFullName((*p).first, relative))
+              << protectEnc(marker->plainFullName((*p).first, relative))
               << "</a></li>\n";
         ++p;
     }
@@ -3043,7 +3055,7 @@ void HtmlGenerator::generateSynopsis(const Node *node,
     QString marked = marker->markedUpSynopsis(node, relative, style);
     QRegExp templateTag("(<[^@>]*>)");
     if (marked.indexOf(templateTag) != -1) {
-        QString contents = protect(marked.mid(templateTag.pos(1),
+        QString contents = protectEnc(marked.mid(templateTag.pos(1),
                                               templateTag.cap(1).length()));
         marked.replace(templateTag.pos(1), templateTag.cap(1).length(),
                         contents);
@@ -3241,7 +3253,7 @@ void HtmlGenerator::generateLink(const Atom* atom,
     if (funcLeftParen.indexIn(atom->string()) != -1 && marker->recognizeLanguage("Cpp")) {
         // hack for C++: move () outside of link
         int k = funcLeftParen.pos(1);
-        out() << protect(atom->string().left(k));
+        out() << protectEnc(atom->string().left(k));
         if (link.isEmpty()) {
             if (showBrokenLinks)
                 out() << "</i>";
@@ -3249,7 +3261,7 @@ void HtmlGenerator::generateLink(const Atom* atom,
             out() << "</a>";
         }
         inLink = false;
-        out() << protect(atom->string().mid(k));
+        out() << protectEnc(atom->string().mid(k));
     } else if (marker->recognizeLanguage("Java")) {
 	// hack for Java: remove () and use <tt> when appropriate
         bool func = atom->string().endsWith("()");
@@ -3257,13 +3269,13 @@ void HtmlGenerator::generateLink(const Atom* atom,
         if (tt)
             out() << "<tt>";
         if (func) {
-            out() << protect(atom->string().left(atom->string().length() - 2));
+            out() << protectEnc(atom->string().left(atom->string().length() - 2));
         } else {
-            out() << protect(atom->string());
+            out() << protectEnc(atom->string());
         }
         out() << "</tt>";
     } else {
-        out() << protect(atom->string());
+        out() << protectEnc(atom->string());
     }
 }
 
@@ -3337,7 +3349,12 @@ QString HtmlGenerator::registerRef(const QString& ref)
     return clean;
 }
 
-QString HtmlGenerator::protect(const QString& string)
+QString HtmlGenerator::protectEnc(const QString &string)
+{
+    return protect(string, outputEncoding);
+}
+
+QString HtmlGenerator::protect(const QString &string, const QString &outputEncoding)
 {
 #define APPEND(x) \
     if (html.isEmpty()) { \
@@ -3360,7 +3377,7 @@ QString HtmlGenerator::protect(const QString& string)
             APPEND("&gt;");
         } else if (ch == QLatin1Char('"')) {
             APPEND("&quot;");
-        } else if (ch.unicode() > 0x007F
+        } else if ((outputEncoding == "ISO-8859-1" && ch.unicode() > 0x007F)
                    || (ch == QLatin1Char('*') && i + 1 < n && string.at(i) == QLatin1Char('/'))
                    || (ch == QLatin1Char('.') && i > 2 && string.at(i - 2) == QLatin1Char('.'))) {
             // we escape '*/' and the last dot in 'e.g.' and 'i.e.' for the Javadoc generator
@@ -3490,7 +3507,7 @@ QString HtmlGenerator::refForNode(const Node *node)
         ref = node->name() + "-var";
         break;
     case Node::Target:
-        return protect(node->name());
+        return protectEnc(node->name());
     }
     return registerRef(ref);
 }
@@ -3564,7 +3581,7 @@ void HtmlGenerator::generateFullName(const Node *apparentNode,
         }
     }
     out() << "\">";
-    out() << protect(fullName(apparentNode, relative, marker));
+    out() << protectEnc(fullName(apparentNode, relative, marker));
     out() << "</a>";
 }
 
@@ -3625,12 +3642,12 @@ void HtmlGenerator::generateDetailedMember(const Node *node,
     else if (node->type() == Node::Enum) {
         const EnumNode *enume = static_cast<const EnumNode *>(node);
         if (enume->flagsType()) {
-            out() << "<p>The " << protect(enume->flagsType()->name())
+            out() << "<p>The " << protectEnc(enume->flagsType()->name())
                   << " type is a typedef for "
                   << "<a href=\"qflags.html\">QFlags</a>&lt;"
-                  << protect(enume->name())
+                  << protectEnc(enume->name())
                   << "&gt;. It stores an OR combination of "
-                  << protect(enume->name())
+                  << protectEnc(enume->name())
                   << " values.</p>\n";
         }
     }