CPack Documentation extraction from CMake script begins to work

 - Enhance extract doc parser. Seems robust now. The legacy
   module documentation parser works as before ignoring
   the new markup.

 - Proof of concept for CPack (generic), CPack RPM and CPack Deb
   generator for macro and variables.
   Try cpack --help-command and cpack --help-variables

9 files changed, 226 insertions, 169 deletions

diff --git a/Source/CPack/cmCPackDocumentMacros.cxx b/Source/CPack/cmCPackDocumentMacros.cxx
index 1ff0351..8752e56 100644
--- a/Source/CPack/cmCPackDocumentMacros.cxx
+++ b/Source/CPack/cmCPackDocumentMacros.cxx
@@ -3,76 +3,14 @@
 void cmCPackDocumentMacros::GetMacrosDocumentation(
         std::vector<cmDocumentationEntry>& v)
 {
-    cmDocumentationEntry e("cpack_add_component",
-            "Describes a CPack installation component "
-            "named by the COMPONENT argument to a CMake INSTALL command.",
-            "       cpack_add_component(compname\n"
-            "                          [DISPLAY_NAME name]\n"
-            "                          [DESCRIPTION description]\n"
-            "                          [HIDDEN | REQUIRED | DISABLED ]\n"
-            "                          [GROUP group]\n"
-            "                          [DEPENDS comp1 comp2 ... ]\n"
-            "                          [INSTALL_TYPES type1 type2 ... ]\n"
-            "                          [DOWNLOADED]\n"
-            "                          [ARCHIVE_FILE filename])\n"
-            "\n"
-            "The cmake_add_component command describes an installation"
-            "component, which the user can opt to install or remove as part of"
-            " the graphical installation process. compname is the name of the "
-            "component, as provided to the COMPONENT argument of one or more "
-            "CMake INSTALL commands."
-            "\n"
-          "DISPLAY_NAME is the displayed name of the component, used in "
-          "graphical installers to display the component name. This value "
-          "can be any string."
-          "\n"
-          "DESCRIPTION is an extended description of the component, used in "
-          "graphical installers to give the user additional information about "
-          "the component. Descriptions can span multiple lines using \"\\n\" "
-          " as the line separator. Typically, these descriptions should be no "
-          "more than a few lines long."
-          "\n"
-          "HIDDEN indicates that this component will be hidden in the "
-          "graphical installer, so that the user cannot directly change "
-          "whether it is installed or not."
-          "\n"
-          "REQUIRED indicates that this component is required, and therefore "
-          "will always be installed. It will be visible in the graphical "
-          "installer, but it cannot be unselected. (Typically, required "
-          "components are shown greyed out)."
-          "\n"
-          "DISABLED indicates that this component should be disabled "
-          "(unselected) by default. The user is free to select this component "
-          "for installation, unless it is also HIDDEN."
-          "\n"
-          "DEPENDS lists the components on which this component depends. If "
-          "this component is selected, then each of the components listed "
-          "must also be selected. The dependency information is encoded "
-          "within the installer itself, so that users cannot install "
-          "inconsistent sets of components."
-          "\n"
-          "GROUP names the component group of which this component is a "
-          "part. If not provided, the component will be a standalone "
-          "component, not part of any component group. Component groups are "
-          "described with the cpack_add_component_group command, detailed"
-          "below."
-          "\n"
-          "INSTALL_TYPES lists the installation types of which this component "
-          "is a part. When one of these installations types is selected, this "
-          "component will automatically be selected. Installation types are"
-          "described with the cpack_add_install_type command, detailed below."
-          "\n"
-          "DOWNLOADED indicates that this component should be downloaded "
-          "on-the-fly by the installer, rather than packaged in with the "
-          "installer itself. For more information, see the "
-          "cpack_configure_downloads command."
-          "\n"
-          "ARCHIVE_FILE provides a name for the archive file created by CPack "
-          "to be used for downloaded components. If not supplied, CPack will "
-          "create a file with some name based on CPACK_PACKAGE_FILE_NAME and "
-          "the name of the component. See cpack_configure_downloads for more "
-          "information."
-);
-
+   // Commented-out example of use
+   //
+   //    cmDocumentationEntry e("cpack_<macro>",
+   //            "Brief Description"
+   //            "which may be on several lines.",
+   //            "Long description in pre-formatted format"
+   //            "                          blah\n"
+   //            "                          blah\n"
+   //);
    //v.push_back(e);
 }
diff --git a/Source/CPack/cmCPackDocumentVariables.cxx b/Source/CPack/cmCPackDocumentVariables.cxx
index ab806d2..8b60d5f 100644
--- a/Source/CPack/cmCPackDocumentVariables.cxx
+++ b/Source/CPack/cmCPackDocumentVariables.cxx
@@ -6,19 +6,19 @@ void cmCPackDocumentVariables::DefineVariables(cmake* cm)
   // Subsection: variables defined/used by cpack,
   // which are common to all CPack generators
 
-  cm->DefineProperty
-      ("CPACK_PACKAGE_VENDOR", cmProperty::VARIABLE,
-       "The name of the package vendor.",
-       "If not specified, defaults to \"Humanity\"."
-       "", false,
-       "Variables common to all CPack generators");
+//  cm->DefineProperty
+//      ("CPACK_PACKAGE_VENDOR", cmProperty::VARIABLE,
+//       "The name of the package vendor.",
+//       "If not specified, defaults to \"Humanity\"."
+//       "", false,
+//       "Variables common to all CPack generators");
 
   // Subsection: variables defined/used by cpack,
   // which are specific to one CPack generator
-  cm->DefineProperty
-      ("CPACK_RPM_PACKAGE_NAME", cmProperty::VARIABLE,
-       "RPM specific package name.",
-       "If not specified, defaults to CPACK_PACKAGE_NAME."
-       "", false,
-       "Variables specific to a CPack generator");
+//  cm->DefineProperty
+//      ("CPACK_RPM_PACKAGE_NAME", cmProperty::VARIABLE,
+//       "RPM specific package name.",
+//       "If not specified, defaults to CPACK_PACKAGE_NAME."
+//       "", false,
+//       "Variables specific to a CPack generator");
 }
diff --git a/Source/CPack/cpack.cxx b/Source/CPack/cpack.cxx
index 8e59fa0..4117971 100644
--- a/Source/CPack/cpack.cxx
+++ b/Source/CPack/cpack.cxx
@@ -303,29 +303,30 @@ int main (int argc, char *argv[])
       help = false;
     }
 
-  if ( parsed && !help )
+  // find out which system cpack is running on, so it can setup the search
+  // paths, so FIND_XXX() commands can be used in scripts
+  // This part is used for cpack documentation lookup as well.
+  cminst.AddCMakePaths();
+  std::string systemFile =
+    globalMF->GetModulesFile("CMakeDetermineSystem.cmake");
+  if (!globalMF->ReadListFile(0, systemFile.c_str()))
     {
-    // find out which system cpack is running on, so it can setup the search
-    // paths, so FIND_XXX() commands can be used in scripts
-    cminst.AddCMakePaths();
-    std::string systemFile =
-      globalMF->GetModulesFile("CMakeDetermineSystem.cmake");
-    if (!globalMF->ReadListFile(0, systemFile.c_str()))
-      {
-      cmCPack_Log(&log, cmCPackLog::LOG_ERROR,
-        "Error reading CMakeDetermineSystem.cmake" << std::endl);
-      return 1;
-      }
+    cmCPack_Log(&log, cmCPackLog::LOG_ERROR,
+      "Error reading CMakeDetermineSystem.cmake" << std::endl);
+    return 1;
+    }
 
-    systemFile =
-      globalMF->GetModulesFile("CMakeSystemSpecificInformation.cmake");
-    if (!globalMF->ReadListFile(0, systemFile.c_str()))
-      {
-      cmCPack_Log(&log, cmCPackLog::LOG_ERROR,
-        "Error reading CMakeSystemSpecificInformation.cmake" << std::endl);
-      return 1;
-      }
+  systemFile =
+    globalMF->GetModulesFile("CMakeSystemSpecificInformation.cmake");
+  if (!globalMF->ReadListFile(0, systemFile.c_str()))
+    {
+    cmCPack_Log(&log, cmCPackLog::LOG_ERROR,
+      "Error reading CMakeSystemSpecificInformation.cmake" << std::endl);
+    return 1;
+    }
 
+  if ( parsed && !help )
+    {
     if ( cmSystemTools::FileExists(cpackConfigFile.c_str()) )
       {
       cpackConfigFile =
@@ -519,43 +520,47 @@ int main (int argc, char *argv[])
     doc.SetSection("Description",cmDocumentationDescription);
     doc.PrependSection("Options",cmDocumentationOptions);
 
+    // statically (in C++ code) defined variables
     cmCPackDocumentVariables::DefineVariables(&cminst);
 
     std::vector<cmDocumentationEntry> commands;
 
-    cminst.AddCMakePaths();
-    std::string systemFile =
-            globalMF->GetModulesFile("CMakeDetermineSystem.cmake");
-    if (!globalMF->ReadListFile(0, systemFile.c_str()))
-    {
-        cmCPack_Log(&log, cmCPackLog::LOG_ERROR,
-                "Error reading CMakeDetermineSystem.cmake" << std::endl);
-        return 1;
-    }
-
-    systemFile =
-            globalMF->GetModulesFile("CMakeSystemSpecificInformation.cmake");
-    if (!globalMF->ReadListFile(0, systemFile.c_str()))
-    {
-        cmCPack_Log(&log, cmCPackLog::LOG_ERROR,
-                "Error reading CMakeSystemSpecificInformation.cmake"
-                << std::endl);
-        return 1;
-    }
-    std::string cpFile = globalMF->GetModulesFile("CPack.cmake");
-    doc.getStructuredDocFromFile(cpFile.c_str(),
-            commands,&cminst,"Variables common to all CPack generators");
-    cpFile = globalMF->GetModulesFile("CPackComponent.cmake");
-    doc.getStructuredDocFromFile(cpFile.c_str(),
-                commands,&cminst,"Variables common to all CPack generators");
-    cpFile = globalMF->GetModulesFile("CPackRPM.cmake");
-        doc.getStructuredDocFromFile(cpFile.c_str(),
-                    commands,&cminst,"Variables specific to a CPack generator");
+    typedef std::pair<std::string,std::string> docModuleSectionPair_t;
+    typedef std::list<docModuleSectionPair_t>  docedModulesList_t;
+    docedModulesList_t docedModList;
+    docModuleSectionPair_t docPair;
+    std::string            docedFile;
+
+    // build the list of files to be parsed for documentation
+    // extraction
+    docPair.first  = "CPack.cmake";
+    docPair.second = "Variables common to all CPack generators";
+    docedModList.push_back(docPair);
+    docPair.first  = "CPackComponent.cmake";
+    docedModList.push_back(docPair);
+    docPair.first  = "CPackRPM.cmake";
+    docPair.second = "Variables specific to a CPack generator";
+    docedModList.push_back(docPair);
+    docPair.first  = "CPackDeb.cmake";
+    docedModList.push_back(docPair);
+
+    // parse the files for documentation.
+    for (docedModulesList_t::iterator it = docedModList.begin();
+         it!= docedModList.end(); ++it)
+      {
+      docedFile = globalMF->GetModulesFile((it->first).c_str());
+      if (docedFile.length()!=0)
+        {
+          doc.GetStructuredDocFromFile(docedFile.c_str(),
+                                       commands,&cminst,(it->second).c_str());
+        }
+     }
 
     std::map<std::string,cmDocumentationSection *> propDocs;
     cminst.GetPropertiesDocumentation(propDocs);
     doc.SetSections(propDocs);
-    cminst.GetCommandDocumentation(commands);
+    cminst.GetCommandDocumentation(commands,true,false);
+    // statically (in C++ code) defined macros/commands
     cmCPackDocumentMacros::GetMacrosDocumentation(commands);
     doc.SetSection("Commands",commands);
 
diff --git a/Source/cmCommand.h b/Source/cmCommand.h
index 7817eb3..e046096 100644
--- a/Source/cmCommand.h
+++ b/Source/cmCommand.h
@@ -111,6 +111,17 @@ public:
     }
 
   /**
+   * This is used to avoid including this command
+   * in documentation. This is mainly used by
+   * cmMacroHelperCommand and cmFunctionHelperCommand
+   * which cannot provide appropriate documentation.
+   */
+  virtual bool ShouldAppearInDocumentation()
+    {
+    return true;
+    }
+
+  /**
    * The name of the command as specified in CMakeList.txt.
    */
   virtual const char* GetName() = 0;
diff --git a/Source/cmDocumentation.cxx b/Source/cmDocumentation.cxx
index b7b8e37..f032592 100644
--- a/Source/cmDocumentation.cxx
+++ b/Source/cmDocumentation.cxx
@@ -757,15 +757,14 @@ static void trim(std::string& s)
   else s.erase(s.begin(), s.end());
 }
 
-int cmDocumentation::getStructuredDocFromFile(
+int cmDocumentation::GetStructuredDocFromFile(
         const char* fname,
         std::vector<cmDocumentationEntry>& commands,
         cmake* cm,
         const char *docSection)
 {
     typedef enum sdoce {
-        SDOC_NONE, SDOC_MACRO,
-        SDOC_PARAM, SDOC_VARIABLE,
+        SDOC_NONE, SDOC_MODULE, SDOC_MACRO, SDOC_FUNCTION, SDOC_VARIABLE,
         SDOC_UNKNOWN} sdoc_t;
     int nbDocItemFound = 0;
     int docCtxIdx      = 0;
@@ -775,15 +774,16 @@ int cmDocumentation::getStructuredDocFromFile(
     std::ifstream fin(fname);
     if(!fin)
       {
-      //std::cerr << "Internal error: can not open script file: <" << fname <<">."<< std::endl;
       return nbDocItemFound;
       }
     std::string name;
     std::string full;
     std::string brief;
     std::string line;
-    bool newCtx  = false;
-    bool inBrief = false;
+    bool newCtx  = false; /* we've just entered ##<beginkey> context */
+    bool inBrief = false; /* we are currently parsing brief desc. */
+    bool inFullFirstParagraph = false; /* we are currently parsing full
+                                          desc. first paragraph */
     brief = "";
     full  = "";
     bool newParagraph = true;
@@ -806,11 +806,24 @@ int cmDocumentation::getStructuredDocFromFile(
                docContextStack[docCtxIdx]=SDOC_VARIABLE;
                newCtx = true;
             }
+            else if (mkword=="function")
+            {
+               docCtxIdx++;
+               docContextStack[docCtxIdx]=SDOC_FUNCTION;
+               newCtx = true;
+            }
+            else if (mkword=="module")
+            {
+               docCtxIdx++;
+               docContextStack[docCtxIdx]=SDOC_MODULE;
+               newCtx = true;
+            }
             else if (mkword.substr(0,3)=="end")
             {
-               ;
                switch (docContextStack[docCtxIdx]) {
                case SDOC_MACRO:
+                   /* for now MACRO and FUNCTION are handled in the same way */
+               case SDOC_FUNCTION:
                    commands.push_back(cmDocumentationEntry(name.c_str(),
                            brief.c_str(),full.c_str()));
                    break;
@@ -821,15 +834,20 @@ int cmDocumentation::getStructuredDocFromFile(
                         full.c_str(),false,
                         docSection);
                    break;
+               case SDOC_MODULE:
+                   /*  not implemented */
+                   break;
+               default:
+                   /* ignore other cases */
+                   break;
                }
                docCtxIdx--;
                newCtx = false;
+               ++nbDocItemFound;
             }
             else
             {
                 // error out unhandled context
-                std::cerr << "Internal error: unknown markup context <"
-                          << mkword <<"> found in:" << fname << std::endl;
                 return nbDocItemFound;
             }
             /* context is set go to next doc line */
@@ -843,47 +861,106 @@ int cmDocumentation::getStructuredDocFromFile(
         //                       followed by a blank line)
         if (newCtx)
         {
-            name = line.substr(1,line.find("-")-1);
-            trim(name);
-            brief = "";
-            line = line.substr(line.find("- ")+1,std::string::npos);
-            inBrief = true;
-            full = "";
+            // no brief (for easy variable definition)
+            if (line.find("-")==std::string::npos)
+            {
+                name = line.substr(1,std::string::npos);
+                trim(name);
+                brief   = "";
+                inBrief = false;
+                full = "";
+            }
+            // here we have a name and brief beginning
+            else
+            {
+                name = line.substr(1,line.find("-")-1);
+                trim(name);
+                // we are parsing the brief context
+                brief = line.substr(line.find("-")+1,std::string::npos);
+                trim(brief);
+                // Brief may already be terminated on the first line
+                if (brief.find('.')!=std::string::npos)
+                {
+                   inBrief = false;
+                   full    = brief.substr(brief.find('.')+1,std::string::npos);
+                   trim(full);
+                   inFullFirstParagraph = true;
+                   brief   = brief.substr(0,brief.find('.'));
+                }
+                // brief is continued on following lines
+                else
+                {
+                    inBrief = true;
+                    full = "";
+                }
+            }
+            newCtx = false;
+            continue;
         }
         // blank line
         if(line.size() <= 2)
         {
-            inBrief = false;
-            full += "\n";
+            if (inBrief) {
+              inBrief = false;
+              full    = "";
+            } else {
+              full += "\n";
+              // the first paragraph of full has ended
+              inFullFirstParagraph = false;
+            }
             newParagraph = true;
         }
-        // brief terminated by .
-        else if (inBrief && line[line.length()-1]=='.')
+        // brief is terminated by '.'
+        else if (inBrief && (line.find('.')!=std::string::npos))
         {
+            /* the brief just ended */
             inBrief = false;
-            brief += line.c_str()+1;
+            std::string endBrief  = line.substr(1,line.find('.'));
+            trim(endBrief);
+            trim(brief);
+            brief  += " " + endBrief;
+            full   += line.substr(line.find('.')+1,std::string::npos);
+            trim(full);
+            inFullFirstParagraph = true;
         }
         // we handle full text or multi-line brief.
         else
           {
           std::string* text;
-          if (inBrief) {
-              text = &brief;
-          } else {
-              text = &full;
-          }
+          if (inBrief)
+            {
+            text = &brief;
+            }
+          else
+            {
+            text = &full;
+            }
           // two spaces
           if(line[1] == ' ' && line[2] == ' ')
             {
-            if(!newParagraph)
+            // there is no "full first paragraph at all."
+            if (line[3] == ' ')
               {
+              inFullFirstParagraph = false;
+              }
 
-                *text += "\n";
-                newParagraph = true;
+            if(!newParagraph && !inFullFirstParagraph)
+              {
+              *text += "\n";
+              newParagraph = true;
               }
-            // Skip #, and leave space for preformatted
-            *text += line.c_str()+1;
-            *text += "\n";
+              // Skip #, and leave space for pre-formatted
+              if (inFullFirstParagraph)
+                {
+                std::string temp = line.c_str()+1;
+                trim(temp);
+                *text += " " + temp;
+                }
+              else
+                {
+                *text += line.c_str()+1;
+                *text += "\n";
+                }
             }
           else if(line[1] == ' ')
             {
@@ -910,7 +987,6 @@ int cmDocumentation::getStructuredDocFromFile(
       /* next line is not the first context line */
       newCtx = false;
       }
-
     return nbDocItemFound;
 }
 
diff --git a/Source/cmDocumentation.h b/Source/cmDocumentation.h
index 2f6e09e..fae409f 100644
--- a/Source/cmDocumentation.h
+++ b/Source/cmDocumentation.h
@@ -133,23 +133,26 @@ public:
   void addCPackStandardDocSections();
 
   /**
-   * Get the documentation of macros and variable documented
+   * Get the documentation of macros, functions and variable documented
    * with CMake structured documentation in a CMake script.
+   * (in fact it may be in any file which follow the structured doc format)
    * Structured documentation begin with
    * ## (double sharp) in column 1 & 2 immediately followed
    * by a markup. Those ## are ignored by the legacy module
    * documentation parser @see CreateSingleModule.
-   * Current markup are ##macro, ##param, ##variable and ##end
-   * which is closing either of the previous ones.
+   * Current markup are ##macro, ##function, ##variable and ##end.
+   * ##end is closing either of the previous ones.
    * @param[in] fname the script file name to be parsed for documentation
    * @param[in,out] commands the vector of command/macros documentation
    *                entry found in the script file.
    * @param[in,out] the cmake object instance to which variable documentation
    *                will be attached (using @see cmake::DefineProperty)
+   * @param[in] the documentation section in which the property will be
+   *            inserted.
    * @return the number of documented items (command and variable)
    *         found in the file.
    */
-  int getStructuredDocFromFile(const char* fname,
+  int GetStructuredDocFromFile(const char* fname,
                                std::vector<cmDocumentationEntry>& commands,
                                cmake* cm,
                                const char *docSection);
diff --git a/Source/cmFunctionCommand.cxx b/Source/cmFunctionCommand.cxx
index ec4fd16..7a80a1c 100644
--- a/Source/cmFunctionCommand.cxx
+++ b/Source/cmFunctionCommand.cxx
@@ -23,6 +23,17 @@ public:
   ~cmFunctionHelperCommand() {};
 
   /**
+   * This is used to avoid including this command
+   * in documentation. This is mainly used by
+   * cmMacroHelperCommand and cmFunctionHelperCommand
+   * which cannot provide appropriate documentation.
+   */
+  virtual bool ShouldAppearInDocumentation()
+    {
+    return false;
+    }
+
+  /**
    * This is a virtual constructor for the command.
    */
   virtual cmCommand* Clone()
diff --git a/Source/cmMacroCommand.cxx b/Source/cmMacroCommand.cxx
index 774f32b..f81a63d 100644
--- a/Source/cmMacroCommand.cxx
+++ b/Source/cmMacroCommand.cxx
@@ -23,6 +23,17 @@ public:
   ~cmMacroHelperCommand() {};
 
   /**
+   * This is used to avoid including this command
+   * in documentation. This is mainly used by
+   * cmMacroHelperCommand and cmFunctionHelperCommand
+   * which cannot provide appropriate documentation.
+   */
+  virtual bool ShouldAppearInDocumentation()
+    {
+    return false;
+    }
+
+  /**
    * This is a virtual constructor for the command.
    */
   virtual cmCommand* Clone()
diff --git a/Source/cmake.cxx b/Source/cmake.cxx
index d691f46..f8e1216 100644
--- a/Source/cmake.cxx
+++ b/Source/cmake.cxx
@@ -2677,7 +2677,9 @@ void cmake::GetCommandDocumentation(std::vector<cmDocumentationEntry>& v,
       j != this->Commands.end(); ++j)
     {
     if (((  withCompatCommands == false) && ( (*j).second->IsDiscouraged()))
-        || ((withCurrentCommands == false) && (!(*j).second->IsDiscouraged())))
+        || ((withCurrentCommands == false) && (!(*j).second->IsDiscouraged()))
+        || (!((*j).second->ShouldAppearInDocumentation()))
+        )
       {
       continue;
       }