qdoc: Added support for different source and output character encodings.

Previously, qdoc assumed Latin1 (ISO-8859-1) for source code and other
documentation, and wrote out XHTML with the same encoding. This change
adds additional configuration options (sourceencoding, outputencoding,
naturallanguage) that enable translated documentation in non-Latin1
encodings to be built with qdoc.

To be reviewed before merge into the master branch.

Reviewed-by: Trust Me

10 files changed, 125 insertions, 84 deletions

diff --git a/tools/qdoc3/config.h b/tools/qdoc3/config.h
index 5e7e6f1..6f23469 100644
--- a/tools/qdoc3/config.h
+++ b/tools/qdoc3/config.h
@@ -140,8 +140,10 @@ class Config
 #define CONFIG_INDEXES                  "indexes"
 #define CONFIG_LANGUAGE                 "language"
 #define CONFIG_MACRO                    "macro"
+#define CONFIG_NATURALLANGUAGE          "naturallanguage"
 #define CONFIG_OBSOLETELINKS            "obsoletelinks"
 #define CONFIG_OUTPUTDIR                "outputdir"
+#define CONFIG_OUTPUTENCODING           "outputencoding"
 #define CONFIG_OUTPUTLANGUAGE           "outputlanguage"
 #define CONFIG_OUTPUTFORMATS            "outputformats"
 #define CONFIG_PROJECT                  "project"
@@ -150,6 +152,7 @@ class Config
 #define CONFIG_SLOW                     "slow"
 #define CONFIG_SHOWINTERNAL             "showinternal"
 #define CONFIG_SOURCEDIRS               "sourcedirs"
+#define CONFIG_SOURCEENCODING           "sourceencoding"
 #define CONFIG_SOURCES                  "sources"
 #define CONFIG_SPURIOUS                 "spurious"
 #define CONFIG_STYLESHEETS              "stylesheets"
diff --git a/tools/qdoc3/cppcodeparser.cpp b/tools/qdoc3/cppcodeparser.cpp
index 9b6a516..129fa7d 100644
--- a/tools/qdoc3/cppcodeparser.cpp
+++ b/tools/qdoc3/cppcodeparser.cpp
@@ -280,8 +280,8 @@ void CppCodeParser::parseHeaderFile(const Location& location,
                                     const QString& filePath,
                                     Tree *tree)
 {
-    FILE *in = fopen(QFile::encodeName(filePath), "r");
-    if (!in) {
+    QFile in(filePath);
+    if (!in.open(QIODevice::ReadOnly)) {
         location.error(tr("Cannot open C++ header file '%1'").arg(filePath));
         return;
     }
@@ -294,7 +294,7 @@ void CppCodeParser::parseHeaderFile(const Location& location,
     matchDeclList(tree->root());
     if (!fileTokenizer.version().isEmpty())
         tree->setVersion(fileTokenizer.version());
-    fclose(in);
+    in.close();
 
     if (fileLocation.fileName() == "qiterator.h")
         parseQiteratorDotH(location, filePath);
@@ -311,8 +311,8 @@ void CppCodeParser::parseSourceFile(const Location& location,
                                     const QString& filePath,
                                     Tree *tree)
 {
-    FILE *in = fopen(QFile::encodeName(filePath), "r");
-    if (!in) {
+    QFile in(filePath);
+    if (!in.open(QIODevice::ReadOnly)) {
         location.error(tr("Cannot open C++ source file '%1' (%2)").arg(filePath).arg(strerror(errno)));
         return;
     }
@@ -324,7 +324,7 @@ void CppCodeParser::parseSourceFile(const Location& location,
     readToken();
     usedNamespaces.clear();
     matchDocsAndStuff();
-    fclose(in);
+    in.close();
 }
 
 /*!
diff --git a/tools/qdoc3/htmlgenerator.cpp b/tools/qdoc3/htmlgenerator.cpp
index a3cdae6..21b358a 100644
--- a/tools/qdoc3/htmlgenerator.cpp
+++ b/tools/qdoc3/htmlgenerator.cpp
@@ -54,6 +54,7 @@
 #include <qdebug.h>
 #include <qlist.h>
 #include <qiterator.h>
+#include <qtextcodec.h>
 
 QT_BEGIN_NAMESPACE
 
@@ -266,6 +267,15 @@ void HtmlGenerator::initializeGenerator(const Config &config)
 
     projectUrl = config.getString(CONFIG_URL);
 
+    outputEncoding = config.getString(CONFIG_OUTPUTENCODING);
+    if (outputEncoding.isEmpty())
+        outputEncoding = QLatin1String("ISO-8859-1");
+    outputCodec = QTextCodec::codecForName(outputEncoding.toLocal8Bit());
+
+    naturalLanguage = config.getString(CONFIG_NATURALLANGUAGE);
+    if (naturalLanguage.isEmpty())
+        naturalLanguage = QLatin1String("en");
+
     QSet<QString> editionNames = config.subVars(CONFIG_EDITION);
     QSet<QString>::ConstIterator edition = editionNames.begin();
     while (edition != editionNames.end()) {
@@ -431,11 +441,11 @@ int HtmlGenerator::generateAtom(const Atom *atom,
                 endLink();
             }
             else {
-                out() << protect(atom->string());
+                out() << protectEnc(atom->string());
             }
         }
         else {
-            out() << protect(atom->string());
+            out() << protectEnc(atom->string());
         }
         break;
     case Atom::BaseName:
@@ -483,7 +493,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
     case Atom::C:
         out() << formattingLeftMap()[ATOM_FORMATTING_TELETYPE];
         if (inLink) {
-            out() << protect(plainCode(atom->string()));
+            out() << protectEnc(plainCode(atom->string()));
         }
         else {
             out() << highlightedCode(atom->string(), marker, relative);
@@ -516,7 +526,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
         // fallthrough
     case Atom::CodeBad:
         out() << "<pre><font color=\"#404040\">"
-              << trimmedTrailing(protect(plainCode(indent(codeIndent,atom->string()))))
+              << trimmedTrailing(protectEnc(plainCode(indent(codeIndent,atom->string()))))
               << "</font></pre>\n";
 	break;
     case Atom::FootnoteLeft:
@@ -768,7 +778,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
                         out() << "<a name=\""
                               << Doc::canonicalTitle((*s).name)
                               << "\"></a>\n";
-                        out() << "<h3>" << protect((*s).name) << "</h3>\n";
+                        out() << "<h3>" << protectEnc((*s).name) << "</h3>\n";
                         if (idx == Class)
                             generateCompactList(0, marker, ncmap.value(), QString("Q"));
                         else if (idx == MemberFunction) {
@@ -792,7 +802,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
                                       << linkForNode(pmap.key(), 0)
                                       << "\">";
                                 QStringList pieces = fullName(pmap.key(), 0, marker).split("::");
-                                out() << protect(pieces.last());
+                                out() << protectEnc(pieces.last());
                                 out() << "</a>"  << ":</p>\n";
 
                                 generateSection(nlist, 0, marker, CodeMarker::Summary);
@@ -820,12 +830,12 @@ int HtmlGenerator::generateAtom(const Atom *atom,
                 out() << "<p align=\"center\">";
             if (fileName.isEmpty()) {
                 out() << "<font color=\"red\">[Missing image "
-                      << protect(atom->string()) << "]</font>";
+                      << protectEnc(atom->string()) << "]</font>";
             }
             else {
-                out() << "<img src=\"" << protect(fileName) << "\"";
+                out() << "<img src=\"" << protectEnc(fileName) << "\"";
                 if (!text.isEmpty())
-                    out() << " alt=\"" << protect(text) << "\"";
+                    out() << " alt=\"" << protectEnc(text) << "\"";
                 out() << " />";
                 helpProjectWriter->addExtraFile(fileName);
             }
@@ -923,7 +933,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
             // ### Trenton
 
             out() << "<tr><td valign=\"top\"><tt>"
-                  << protect(plainCode(marker->markedUpEnumValue(atom->next()->string(),
+                  << protectEnc(plainCode(marker->markedUpEnumValue(atom->next()->string(),
                                                                  relative)))
                   << "</tt></td><td align=\"center\" valign=\"top\">";
 
@@ -936,7 +946,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
             if (itemValue.isEmpty())
                 out() << "?";
             else
-                out() << "<tt>" << protect(itemValue) << "</tt>";
+                out() << "<tt>" << protectEnc(itemValue) << "</tt>";
 
             skipAhead = 1;
         }
@@ -1052,7 +1062,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
             generateLink(atom, relative, marker);
         }
         else {
-            out() << protect(atom->string());
+            out() << protectEnc(atom->string());
         }
         break;
     case Atom::TableLeft:
@@ -1166,7 +1176,7 @@ int HtmlGenerator::generateAtom(const Atom *atom,
         out() << "<font color=\"red\"><b>&lt;Missing HTML&gt;</b></font>";
         break;
     case Atom::UnknownCommand:
-        out() << "<font color=\"red\"><b><code>\\" << protect(atom->string())
+        out() << "<font color=\"red\"><b><code>\\" << protectEnc(atom->string())
               << "</code></b></font>";
         break;
 #ifdef QDOC_QML
@@ -1295,7 +1305,7 @@ void HtmlGenerator::generateClassLikeNode(const InnerNode *inner,
                 out() << "<a name=\""
                       << registerRef((*s).name.toLower())
                       << "\"></a>\n";
-                out() << "<h2>" << protect((*s).name) << "</h2>\n";
+                out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
                 generateSection(s->members, inner, marker, CodeMarker::Summary);
             }
             if (!s->reimpMembers.isEmpty()) {
@@ -1304,7 +1314,7 @@ void HtmlGenerator::generateClassLikeNode(const InnerNode *inner,
                 out() << "<a name=\""
                       << registerRef(name.toLower())
                       << "\"></a>\n";
-                out() << "<h2>" << protect(name) << "</h2>\n";
+                out() << "<h2>" << protectEnc(name) << "</h2>\n";
                 generateSection(s->reimpMembers, inner, marker, CodeMarker::Summary);
             }
 
@@ -1343,7 +1353,7 @@ void HtmlGenerator::generateClassLikeNode(const InnerNode *inner,
     s = sections.begin();
     while (s != sections.end()) {
         out() << "<hr />\n";
-        out() << "<h2>" << protect((*s).name) << "</h2>\n";
+        out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
 
         NodeList::ConstIterator m = (*s).members.begin();
         while (m != (*s).members.end()) {
@@ -1513,7 +1523,7 @@ void HtmlGenerator::generateFakeNode(const FakeNode *fake, CodeMarker *marker)
         s = sections.begin();
         while (s != sections.end()) {
             out() << "<a name=\"" << registerRef((*s).name) << "\"></a>\n";
-            out() << "<h2>" << protect((*s).name) << "</h2>\n";
+            out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
             generateQmlSummary(*s,fake,marker);
             ++s;
         }
@@ -1529,7 +1539,7 @@ void HtmlGenerator::generateFakeNode(const FakeNode *fake, CodeMarker *marker)
         sections = marker->qmlSections(qml_cn,CodeMarker::Detailed);
         s = sections.begin();
         while (s != sections.end()) {
-            out() << "<h2>" << protect((*s).name) << "</h2>\n";
+            out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
             NodeList::ConstIterator m = (*s).members.begin();
             while (m != (*s).members.end()) {
                 generateDetailedQmlMember(*m, fake, marker);
@@ -1549,7 +1559,7 @@ void HtmlGenerator::generateFakeNode(const FakeNode *fake, CodeMarker *marker)
     s = sections.begin();
     while (s != sections.end()) {
         out() << "<a name=\"" << registerRef((*s).name) << "\"></a>\n";
-        out() << "<h2>" << protect((*s).name) << "</h2>\n";
+        out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
         generateSectionList(*s, fake, marker, CodeMarker::Summary);
         ++s;
     }
@@ -1578,7 +1588,7 @@ void HtmlGenerator::generateFakeNode(const FakeNode *fake, CodeMarker *marker)
     s = sections.begin();
     while (s != sections.end()) {
         out() << "<hr />\n";
-        out() << "<h2>" << protect((*s).name) << "</h2>\n";
+        out() << "<h2>" << protectEnc((*s).name) << "</h2>\n";
 
         NodeList::ConstIterator m = (*s).members.begin();
         while (m != (*s).members.end()) {
@@ -1624,11 +1634,11 @@ void HtmlGenerator::generateHeader(const QString& title,
                                    CodeMarker *marker,
                                    bool mainPage)
 {
-    out() << "<?xml version=\"1.0\" encoding=\"iso-8859-1\"?>\n";
+    out() << QString("<?xml version=\"1.0\" encoding=\"%1\"?>\n").arg(outputEncoding);
 
     out() << "<!DOCTYPE html\n"
-             "    PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"DTD/xhtml1-strict.dtd\">\n"
-             "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">\n";
+             "    PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"DTD/xhtml1-strict.dtd\">\n";
+    out() << QString("<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"%1\" lang=\"%1\">\n").arg(naturalLanguage);
 
     QString shortVersion;
     if ((project != "Qtopia") && (project != "Qt Extended")) {
@@ -1648,7 +1658,9 @@ void HtmlGenerator::generateHeader(const QString& title,
     }
 
     out() << "<head>\n"
-             "  <title>" << shortVersion << protect(title) << "</title>\n";
+             "  <title>" << shortVersion << protectEnc(title) << "</title>\n";
+    out() << QString("<meta http-equiv=\"Content-type\" content=\"text/html; charset=%1\" />").arg(outputEncoding);
+
     if (!style.isEmpty())
         out() << "    <style type=\"text/css\">" << style << "</style>\n";
 
@@ -1657,8 +1669,8 @@ void HtmlGenerator::generateHeader(const QString& title,
         QMapIterator<QString, QString> i(metaMap);
         while (i.hasNext()) {
             i.next();
-            out() << "    <meta name=\"" << protect(i.key()) << "\" contents=\""
-                  << protect(i.value()) << "\" />\n";
+            out() << "    <meta name=\"" << protectEnc(i.key()) << "\" contents=\""
+                  << protectEnc(i.value()) << "\" />\n";
         }
     }
 
@@ -1682,9 +1694,9 @@ void HtmlGenerator::generateHeader(const QString& title,
 
             navigationLinks += "[Previous: <a href=\"" + anchorPair.first + "\">";
             if (linkPair.first == linkPair.second && !anchorPair.second.isEmpty())
-                navigationLinks += protect(anchorPair.second);
+                navigationLinks += protectEnc(anchorPair.second);
             else
-                navigationLinks += protect(linkPair.second);
+                navigationLinks += protectEnc(linkPair.second);
             navigationLinks += "</a>]\n";
         }
         if (node->links().contains(Node::ContentsLink)) {
@@ -1700,9 +1712,9 @@ void HtmlGenerator::generateHeader(const QString& title,
 
             navigationLinks += "[<a href=\"" + anchorPair.first + "\">";
             if (linkPair.first == linkPair.second && !anchorPair.second.isEmpty())
-                navigationLinks += protect(anchorPair.second);
+                navigationLinks += protectEnc(anchorPair.second);
             else
-                navigationLinks += protect(linkPair.second);
+                navigationLinks += protectEnc(linkPair.second);
             navigationLinks += "</a>]\n";
         }
         if (node->links().contains(Node::NextLink)) {
@@ -1718,9 +1730,9 @@ void HtmlGenerator::generateHeader(const QString& title,
 
             navigationLinks += "[Next: <a href=\"" + anchorPair.first + "\">";
             if (linkPair.first == linkPair.second && !anchorPair.second.isEmpty())
-                navigationLinks += protect(anchorPair.second);
+                navigationLinks += protectEnc(anchorPair.second);
             else
-                navigationLinks += protect(linkPair.second);
+                navigationLinks += protectEnc(linkPair.second);
             navigationLinks += "</a>]\n";
         }
         if (node->links().contains(Node::IndexLink)) {
@@ -1771,7 +1783,7 @@ void HtmlGenerator::generateTitle(const QString& title,
                                   const Node *relative,
                                   CodeMarker *marker)
 {
-    out() << "<h1 class=\"title\">" << protect(title);
+    out() << "<h1 class=\"title\">" << protectEnc(title);
     if (!subTitle.isEmpty()) {
         out() << "<br />";
         if (subTitleSize == SmallSubTitle)
@@ -2009,18 +2021,18 @@ QString HtmlGenerator::generateLowStatusMemberFile(const InnerNode *inner,
 
     out() << "<p><ul><li><a href=\""
           << linkForNode(inner, 0) << "\">"
-          << protect(inner->name())
+          << protectEnc(inner->name())
           << " class reference</a></li></ul></p>\n";
 
     for (i = 0; i < sections.size(); ++i) {
-        out() << "<h2>" << protect(sections.at(i).name) << "</h2>\n";
+        out() << "<h2>" << protectEnc(sections.at(i).name) << "</h2>\n";
         generateSectionList(sections.at(i), inner, marker, CodeMarker::Summary);
     }
 
     sections = marker->sections(inner, CodeMarker::Detailed, status);
     for (i = 0; i < sections.size(); ++i) {
         out() << "<hr />\n";
-        out() << "<h2>" << protect(sections.at(i).name) << "</h2>\n";
+        out() << "<h2>" << protectEnc(sections.at(i).name) << "</h2>\n";
 
         NodeList::ConstIterator m = sections.at(i).members.begin();
         while (m != sections.at(i).members.end()) {
@@ -2113,7 +2125,7 @@ void HtmlGenerator::generateAnnotatedList(const Node *relative,
         }
         else {
             out() << "<td>";
-            out() << protect(node->doc().briefText().toString());
+            out() << protectEnc(node->doc().briefText().toString());
             out() << "</td>";
         }
         out() << "</tr>\n";
@@ -2315,7 +2327,7 @@ void HtmlGenerator::generateCompactList(const Node *relative,
                         << linkForNode(it.value(), relative)
                         << "\">";
                     QStringList pieces = fullName(it.value(), relative, marker).split("::");
-                    out() << protect(pieces.last());
+                    out() << protectEnc(pieces.last());
                     out() << "</a>";
                     if (pieces.size() > 1) {
                         out() << " (";
@@ -2357,7 +2369,7 @@ void HtmlGenerator::generateFunctionIndex(const Node *relative,
 #else
         out() << "<p>";
 #endif
-        out() << protect(f.key()) << ":";
+        out() << protectEnc(f.key()) << ":";
 
         currentLetter = f.key()[0].unicode();
         while (islower(currentLetter) && currentLetter >= nextLetter) {
@@ -2411,7 +2423,7 @@ void HtmlGenerator::generateLegaleseList(const Node *relative,
     QString marked = marker->markedUpSynopsis(node, relative, style);
     QRegExp templateTag("(<[^@>]*>)");
     if (marked.indexOf(templateTag) != -1) {
-        QString contents = protect(marked.mid(templateTag.pos(1),
+        QString contents = protectEnc(marked.mid(templateTag.pos(1),
                                               templateTag.cap(1).length()));
         marked.replace(templateTag.pos(1), templateTag.cap(1).length(),
                         contents);
@@ -2450,7 +2462,7 @@ void HtmlGenerator::generateQmlItem(const Node *node,
     QString marked = marker->markedUpQmlItem(node,summary);
     QRegExp templateTag("(<[^@>]*>)");
     if (marked.indexOf(templateTag) != -1) {
-        QString contents = protect(marked.mid(templateTag.pos(1),
+        QString contents = protectEnc(marked.mid(templateTag.pos(1),
                                               templateTag.cap(1).length()));
         marked.replace(templateTag.pos(1), templateTag.cap(1).length(),
                         contents);
@@ -2556,7 +2568,7 @@ void HtmlGenerator::generateOverviewList(const Node *relative, CodeMarker * /* m
             const FakeNode *groupNode = groupTitlesMap[groupTitle];
             out() << QString("<h3><a href=\"%1\">%2</a></h3>\n").arg(
                         linkForNode(groupNode, relative)).arg(
-                        protect(groupNode->fullTitle()));
+                        protectEnc(groupNode->fullTitle()));
 
             if (fakeNodeMap[groupNode].count() == 0)
                 continue;
@@ -2568,7 +2580,7 @@ void HtmlGenerator::generateOverviewList(const Node *relative, CodeMarker * /* m
                 if (title.startsWith("The "))
                     title.remove(0, 4);
                 out() << "<li><a href=\"" << linkForNode(fakeNode, relative) << "\">"
-                      << protect(title) << "</a></li>\n";
+                      << protectEnc(title) << "</a></li>\n";
             }
             out() << "</ul>\n";
         }
@@ -2582,7 +2594,7 @@ void HtmlGenerator::generateOverviewList(const Node *relative, CodeMarker * /* m
             if (title.startsWith("The "))
                 title.remove(0, 4);
             out() << "<li><a href=\"" << linkForNode(fakeNode, relative) << "\">"
-                  << protect(title) << "</a></li>\n";
+                  << protectEnc(title) << "</a></li>\n";
         }
         out() << "</ul>\n";
     }
@@ -2745,7 +2757,7 @@ void HtmlGenerator::generateSectionInheritedList(const Section& section,
         }
         out() << " inherited from <a href=\"" << fileName((*p).first)
               << "#" << HtmlGenerator::cleanRef(section.name.toLower()) << "\">"
-              << protect(marker->plainFullName((*p).first, relative))
+              << protectEnc(marker->plainFullName((*p).first, relative))
               << "</a></li>\n";
         ++p;
     }
@@ -2760,7 +2772,7 @@ void HtmlGenerator::generateSynopsis(const Node *node,
     QString marked = marker->markedUpSynopsis(node, relative, style);
     QRegExp templateTag("(<[^@>]*>)");
     if (marked.indexOf(templateTag) != -1) {
-        QString contents = protect(marked.mid(templateTag.pos(1),
+        QString contents = protectEnc(marked.mid(templateTag.pos(1),
                                               templateTag.cap(1).length()));
         marked.replace(templateTag.pos(1), templateTag.cap(1).length(),
                         contents);
@@ -3029,7 +3041,7 @@ void HtmlGenerator::generateSectionInheritedList(const Section& section,
         }
         out() << " inherited from <a href=\"" << fileName((*p).first)
               << "#" << HtmlGenerator::cleanRef(section.name.toLower()) << "\">"
-              << protect(marker->plainFullName((*p).first, relative))
+              << protectEnc(marker->plainFullName((*p).first, relative))
               << "</a></li>\n";
         ++p;
     }
@@ -3043,7 +3055,7 @@ void HtmlGenerator::generateSynopsis(const Node *node,
     QString marked = marker->markedUpSynopsis(node, relative, style);
     QRegExp templateTag("(<[^@>]*>)");
     if (marked.indexOf(templateTag) != -1) {
-        QString contents = protect(marked.mid(templateTag.pos(1),
+        QString contents = protectEnc(marked.mid(templateTag.pos(1),
                                               templateTag.cap(1).length()));
         marked.replace(templateTag.pos(1), templateTag.cap(1).length(),
                         contents);
@@ -3241,7 +3253,7 @@ void HtmlGenerator::generateLink(const Atom* atom,
     if (funcLeftParen.indexIn(atom->string()) != -1 && marker->recognizeLanguage("Cpp")) {
         // hack for C++: move () outside of link
         int k = funcLeftParen.pos(1);
-        out() << protect(atom->string().left(k));
+        out() << protectEnc(atom->string().left(k));
         if (link.isEmpty()) {
             if (showBrokenLinks)
                 out() << "</i>";
@@ -3249,7 +3261,7 @@ void HtmlGenerator::generateLink(const Atom* atom,
             out() << "</a>";
         }
         inLink = false;
-        out() << protect(atom->string().mid(k));
+        out() << protectEnc(atom->string().mid(k));
     } else if (marker->recognizeLanguage("Java")) {
 	// hack for Java: remove () and use <tt> when appropriate
         bool func = atom->string().endsWith("()");
@@ -3257,13 +3269,13 @@ void HtmlGenerator::generateLink(const Atom* atom,
         if (tt)
             out() << "<tt>";
         if (func) {
-            out() << protect(atom->string().left(atom->string().length() - 2));
+            out() << protectEnc(atom->string().left(atom->string().length() - 2));
         } else {
-            out() << protect(atom->string());
+            out() << protectEnc(atom->string());
         }
         out() << "</tt>";
     } else {
-        out() << protect(atom->string());
+        out() << protectEnc(atom->string());
     }
 }
 
@@ -3337,7 +3349,12 @@ QString HtmlGenerator::registerRef(const QString& ref)
     return clean;
 }
 
-QString HtmlGenerator::protect(const QString& string)
+QString HtmlGenerator::protectEnc(const QString &string)
+{
+    return protect(string, outputEncoding);
+}
+
+QString HtmlGenerator::protect(const QString &string, const QString &outputEncoding)
 {
 #define APPEND(x) \
     if (html.isEmpty()) { \
@@ -3360,7 +3377,7 @@ QString HtmlGenerator::protect(const QString& string)
             APPEND("&gt;");
         } else if (ch == QLatin1Char('"')) {
             APPEND("&quot;");
-        } else if (ch.unicode() > 0x007F
+        } else if ((outputEncoding == "ISO-8859-1" && ch.unicode() > 0x007F)
                    || (ch == QLatin1Char('*') && i + 1 < n && string.at(i) == QLatin1Char('/'))
                    || (ch == QLatin1Char('.') && i > 2 && string.at(i - 2) == QLatin1Char('.'))) {
             // we escape '*/' and the last dot in 'e.g.' and 'i.e.' for the Javadoc generator
@@ -3490,7 +3507,7 @@ QString HtmlGenerator::refForNode(const Node *node)
         ref = node->name() + "-var";
         break;
     case Node::Target:
-        return protect(node->name());
+        return protectEnc(node->name());
     }
     return registerRef(ref);
 }
@@ -3564,7 +3581,7 @@ void HtmlGenerator::generateFullName(const Node *apparentNode,
         }
     }
     out() << "\">";
-    out() << protect(fullName(apparentNode, relative, marker));
+    out() << protectEnc(fullName(apparentNode, relative, marker));
     out() << "</a>";
 }
 
@@ -3625,12 +3642,12 @@ void HtmlGenerator::generateDetailedMember(const Node *node,
     else if (node->type() == Node::Enum) {
         const EnumNode *enume = static_cast<const EnumNode *>(node);
         if (enume->flagsType()) {
-            out() << "<p>The " << protect(enume->flagsType()->name())
+            out() << "<p>The " << protectEnc(enume->flagsType()->name())
                   << " type is a typedef for "
                   << "<a href=\"qflags.html\">QFlags</a>&lt;"
-                  << protect(enume->name())
+                  << protectEnc(enume->name())
                   << "&gt;. It stores an OR combination of "
-                  << protect(enume->name())
+                  << protectEnc(enume->name())
                   << " values.</p>\n";
         }
     }
diff --git a/tools/qdoc3/htmlgenerator.h b/tools/qdoc3/htmlgenerator.h
index 19a7c78..5ed26e0 100644
--- a/tools/qdoc3/htmlgenerator.h
+++ b/tools/qdoc3/htmlgenerator.h
@@ -104,7 +104,8 @@ class HtmlGenerator : public PageGenerator
     virtual QString format();
     virtual void generateTree(const Tree *tree, CodeMarker *marker);
 
-    static QString protect(const QString& string);
+    QString protectEnc(const QString &string);
+    static QString protect(const QString &string, const QString &encoding = "ISO-8859-1");
     static QString cleanRef(const QString& ref);
     static QString sinceTitle(int i) { return sinceTitles[i]; }
 
diff --git a/tools/qdoc3/pagegenerator.cpp b/tools/qdoc3/pagegenerator.cpp
index a001c10..e637109 100644
--- a/tools/qdoc3/pagegenerator.cpp
+++ b/tools/qdoc3/pagegenerator.cpp
@@ -176,7 +176,7 @@ void PageGenerator::beginSubPage(const Location& location,
 	location.fatal(tr("Cannot open output file '%1'")
 			.arg(outFile->fileName()));
     QTextStream *out = new QTextStream(outFile);
-    out->setCodec("ISO-8859-1");
+    out->setCodec(outputCodec);
     outStreamStack.push(out);
 }
 
diff --git a/tools/qdoc3/pagegenerator.h b/tools/qdoc3/pagegenerator.h
index db24edd..20a6c3c 100644
--- a/tools/qdoc3/pagegenerator.h
+++ b/tools/qdoc3/pagegenerator.h
@@ -54,6 +54,8 @@
 
 QT_BEGIN_NAMESPACE
 
+class QTextCodec;
+
 class ClassNode;
 class InnerNode;
 class NamespaceNode;
@@ -76,6 +78,10 @@ class PageGenerator : public Generator
     virtual void generateInnerNode(const InnerNode *node, CodeMarker *marker);
     QTextStream& out();
 
+    QString naturalLanguage;
+    QString outputEncoding;
+    QTextCodec *outputCodec;
+
  private:
     QStack<QTextStream *> outStreamStack;
 };
diff --git a/tools/qdoc3/qsakernelparser.cpp b/tools/qdoc3/qsakernelparser.cpp
index 9320a17..8f12eda 100644
--- a/tools/qdoc3/qsakernelparser.cpp
+++ b/tools/qdoc3/qsakernelparser.cpp
@@ -70,8 +70,8 @@ void QsaKernelParser::parseSourceFile( const Location& location,
 				       const QString& filePath,
 				       Tree * /* tree */ )
 {
-    FILE *in = fopen( QFile::encodeName(filePath), "r" );
-    if ( in == 0 ) {
+    QFile in(filePath);
+    if (!in.open(QIODevice::ReadOnly)) {
         location.error( tr("Cannot open QSA kernel file '%1'").arg(filePath) );
         return;
     }
@@ -171,7 +171,7 @@ void QsaKernelParser::parseSourceFile( const Location& location,
 	    readToken();
 	}
     }
-    fclose( in );
+    in.close();
 }
 
 void QsaKernelParser::doneParsingSourceFiles( Tree * /* tree */ )
diff --git a/tools/qdoc3/qscodeparser.cpp b/tools/qdoc3/qscodeparser.cpp
index 8b3bc98..3b8bc1a 100644
--- a/tools/qdoc3/qscodeparser.cpp
+++ b/tools/qdoc3/qscodeparser.cpp
@@ -151,8 +151,8 @@ void QsCodeParser::parseHeaderFile(const Location& location,
 {
     qsTre = tree;
 
-    FILE *in = fopen(QFile::encodeName(filePath), "r");
-    if (in == 0) {
+    QFile in(filePath);
+    if (!in.open(QIODevice::ReadOnly)) {
 	location.error(tr("Cannot open Qt Script class list '%1'")
 			.arg(filePath));
 	return;
@@ -175,7 +175,7 @@ void QsCodeParser::parseHeaderFile(const Location& location,
 	}
 	tok = fileTokenizer.getToken();
     }
-    fclose(in);
+    in.close();
 }
 
 void QsCodeParser::parseSourceFile(const Location& location,
diff --git a/tools/qdoc3/tokenizer.cpp b/tools/qdoc3/tokenizer.cpp
index b391759..7c10de6 100644
--- a/tools/qdoc3/tokenizer.cpp
+++ b/tools/qdoc3/tokenizer.cpp
@@ -47,6 +47,7 @@
 #include <qhash.h>
 #include <qregexp.h>
 #include <qstring.h>
+#include <qtextcodec.h>
 
 #include <ctype.h>
 #include <string.h>
@@ -97,6 +98,8 @@ static QRegExp *definedX = 0;
 static QRegExp *defines = 0;
 static QRegExp *falsehoods = 0;
 
+static QTextCodec *sourceCodec = 0;
+
 /*
   This function is a perfect hash function for the 37 keywords of C99
   (with a hash table size of 512). It should perform well on our
@@ -118,13 +121,10 @@ static void insertKwordIntoHash(const char *s, int number)
     kwordHashTable[k] = number;
 }
 
-Tokenizer::Tokenizer(const Location& loc, FILE *in)
+Tokenizer::Tokenizer(const Location& loc, QFile &in)
 {
     init();
-    QFile file;
-    file.open(in, QIODevice::ReadOnly);
-    yyIn = file.readAll();
-    file.close();
+    yyIn = in.readAll();
     yyPos = 0;
     start(loc);
 }
@@ -483,6 +483,11 @@ void Tokenizer::initialize(const Config &config)
 {
     QString versionSym = config.getString(CONFIG_VERSIONSYM);
 
+    QString sourceEncoding = config.getString(CONFIG_SOURCEENCODING);
+    if (sourceEncoding.isEmpty())
+        sourceEncoding = QLatin1String("ISO-8859-1");
+    sourceCodec = QTextCodec::codecForName(sourceEncoding.toLocal8Bit());
+
     comment = new QRegExp("/(?:\\*.*\\*/|/.*\n|/[^\n]*$)");
     comment->setMinimal(true);
     versionX = new QRegExp("$cannot possibly match^");
@@ -750,4 +755,14 @@ bool Tokenizer::isTrue(const QString &condition)
         return !falsehoods->exactMatch(t);
 }
 
+QString Tokenizer::lexeme() const
+{
+    return sourceCodec->toUnicode(yyLex);
+}
+
+QString Tokenizer::previousLexeme() const
+{
+    return sourceCodec->toUnicode(yyPrevLex);
+}
+
 QT_END_NAMESPACE
diff --git a/tools/qdoc3/tokenizer.h b/tools/qdoc3/tokenizer.h
index 519cffb..f55d2ef 100644
--- a/tools/qdoc3/tokenizer.h
+++ b/tools/qdoc3/tokenizer.h
@@ -46,11 +46,10 @@
 #ifndef TOKENIZER_H
 #define TOKENIZER_H
 
+#include <qfile.h>
 #include <qstack.h>
 #include <qstring.h>
 
-#include <stdio.h>
-
 #include "location.h"
 
 QT_BEGIN_NAMESPACE
@@ -99,7 +98,7 @@ class Tokenizer
 {
  public:
     Tokenizer(const Location& loc, const QByteArray &in);
-    Tokenizer(const Location& loc, FILE *in);
+    Tokenizer(const Location& loc, QFile &file);
 
     ~Tokenizer();
 
@@ -108,8 +107,8 @@ class Tokenizer
     bool parsingFnOrMacro() const { return parsingMacro; }
 
     const Location &location() const { return yyTokLoc; }
-    QString previousLexeme() const { return QString(yyPrevLex); }
-    QString lexeme() const { return QString(yyLex); }
+    QString previousLexeme() const;
+    QString lexeme() const;
     QString version() const { return yyVersion; }
     int braceDepth() const { return yyBraceDepth; }
     int parenDepth() const { return yyParenDepth; }