diff options
157 files changed, 8913 insertions, 1472 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index c96f68b..ebee14b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -372,12 +372,14 @@ macro (CMAKE_BUILD_UTILITIES) set(ENABLE_LZMA ON CACHE INTERNAL "Enable the use of the system found LZMA library if found") set(ENABLE_ZLIB ON CACHE INTERNAL "Enable the use of the system found ZLIB library if found") set(ENABLE_BZip2 ON CACHE INTERNAL "Enable the use of the system found BZip2 library if found") + set(ENABLE_LIBXML2 OFF CACHE INTERNAL "Enable the use of the system found libxml2 library if found") set(ENABLE_EXPAT OFF CACHE INTERNAL "Enable the use of the system found EXPAT library if found") set(ENABLE_PCREPOSIX OFF CACHE INTERNAL "Enable the use of the system found PCREPOSIX library if found") set(ENABLE_LibGCC OFF CACHE INTERNAL "Enable the use of the system found LibGCC library if found") set(ENABLE_XATTR OFF CACHE INTERNAL "Enable extended attribute support") set(ENABLE_ACL OFF CACHE INTERNAL "Enable ACL support") set(ENABLE_ICONV OFF CACHE INTERNAL "Enable iconv support") + set(ENABLE_CNG OFF CACHE INTERNAL "Enable the use of CNG(Crypto Next Generation)") add_subdirectory(Utilities/cmlibarchive) CMAKE_SET_TARGET_FOLDER(cmlibarchive "Utilities/3rdParty") set(CMAKE_TAR_LIBRARIES cmlibarchive ${BZIP2_LIBRARIES}) diff --git a/Modules/CheckForPthreads.c b/Modules/CheckForPthreads.c index 2732957..344c81b 100644 --- a/Modules/CheckForPthreads.c +++ b/Modules/CheckForPthreads.c @@ -16,8 +16,8 @@ int main(int ac, char*av[]){ pthread_create(&tid[0], 0, runner, (void*)1); pthread_create(&tid[1], 0, runner, (void*)2); -#if defined(__BEOS__) && !defined(__ZETA__) // (no usleep on BeOS 5.) - usleep(1); // for strange behavior on single-processor sun +#if defined(__BEOS__) && !defined(__ZETA__) /* (no usleep on BeOS 5.) */ + usleep(1); /* for strange behavior on single-processor sun */ #endif pthread_join(tid[0], 0); diff --git a/Source/CMakeVersion.cmake b/Source/CMakeVersion.cmake index a5f01d3..97768e1 100644 --- a/Source/CMakeVersion.cmake +++ b/Source/CMakeVersion.cmake @@ -1,5 +1,5 @@ # CMake version number components. set(CMake_VERSION_MAJOR 3) set(CMake_VERSION_MINOR 4) -set(CMake_VERSION_PATCH 20151026) +set(CMake_VERSION_PATCH 20151028) #set(CMake_VERSION_RC 1) diff --git a/Source/cmAddExecutableCommand.cxx b/Source/cmAddExecutableCommand.cxx index 47f6592..a84bb9d 100644 --- a/Source/cmAddExecutableCommand.cxx +++ b/Source/cmAddExecutableCommand.cxx @@ -192,7 +192,7 @@ bool cmAddExecutableCommand this->SetError(e.str()); return false; } - this->Makefile->AddAlias(exename, aliasedTarget); + this->Makefile->AddAlias(exename, aliasedName); return true; } diff --git a/Source/cmAddLibraryCommand.cxx b/Source/cmAddLibraryCommand.cxx index e0adee3..5296cbb 100644 --- a/Source/cmAddLibraryCommand.cxx +++ b/Source/cmAddLibraryCommand.cxx @@ -314,7 +314,7 @@ bool cmAddLibraryCommand this->SetError(e.str()); return false; } - this->Makefile->AddAlias(libName, aliasedTarget); + this->Makefile->AddAlias(libName, aliasedName); return true; } diff --git a/Source/cmArchiveWrite.cxx b/Source/cmArchiveWrite.cxx index 7946950..e62a2ed 100644 --- a/Source/cmArchiveWrite.cxx +++ b/Source/cmArchiveWrite.cxx @@ -18,6 +18,10 @@ #include <cm_libarchive.h> #include "cm_get_date.h" +#ifndef __LA_SSIZE_T +# define __LA_SSIZE_T la_ssize_t +#endif + //---------------------------------------------------------------------------- static std::string cm_archive_error_string(struct archive* a) { diff --git a/Source/cmAuxSourceDirectoryCommand.cxx b/Source/cmAuxSourceDirectoryCommand.cxx index 5f5017d..92ac07d 100644 --- a/Source/cmAuxSourceDirectoryCommand.cxx +++ b/Source/cmAuxSourceDirectoryCommand.cxx @@ -60,10 +60,10 @@ bool cmAuxSourceDirectoryCommand::InitialPass std::string ext = file.substr(dotpos+1); std::string base = file.substr(0, dotpos); // Process only source files - if(!base.empty() - && std::find( this->Makefile->GetSourceExtensions().begin(), - this->Makefile->GetSourceExtensions().end(), ext ) - != this->Makefile->GetSourceExtensions().end() ) + std::vector<std::string> srcExts = + this->Makefile->GetCMakeInstance()->GetSourceExtensions(); + if(!base.empty() && + std::find(srcExts.begin(), srcExts.end(), ext) != srcExts.end()) { std::string fullname = templateDirectory; fullname += "/"; diff --git a/Source/cmComputeLinkInformation.cxx b/Source/cmComputeLinkInformation.cxx index a32bb48..50d8324 100644 --- a/Source/cmComputeLinkInformation.cxx +++ b/Source/cmComputeLinkInformation.cxx @@ -377,9 +377,9 @@ cmComputeLinkInformation // Add the search path entries requested by the user to path ordering. this->OrderLinkerSearchPath - ->AddUserDirectories(this->Target->Target->GetLinkDirectories()); + ->AddUserDirectories(this->Target->GetLinkDirectories()); this->OrderRuntimeSearchPath - ->AddUserDirectories(this->Target->Target->GetLinkDirectories()); + ->AddUserDirectories(this->Target->GetLinkDirectories()); // Set up the implicit link directories. this->LoadImplicitLinkInfo(); @@ -413,7 +413,7 @@ cmComputeLinkInformation // Construct a mask to not bother with this behavior for link // directories already specified by the user. std::vector<std::string> const& dirs = - this->Target->Target->GetLinkDirectories(); + this->Target->GetLinkDirectories(); this->OldLinkDirMask.insert(dirs.begin(), dirs.end()); } diff --git a/Source/cmComputeTargetDepends.cxx b/Source/cmComputeTargetDepends.cxx index 113c989..586b5bf 100644 --- a/Source/cmComputeTargetDepends.cxx +++ b/Source/cmComputeTargetDepends.cxx @@ -382,7 +382,7 @@ void cmComputeTargetDepends::AddTargetDepend( << "\" of target \"" << depender->GetName() << "\" does not exist."; cmListFileBacktrace const* backtrace = - depender->Target->GetUtilityBacktrace(dependee_name); + depender->GetUtilityBacktrace(dependee_name); if(backtrace) { cm->IssueMessage(messageType, e.str(), *backtrace); diff --git a/Source/cmExportBuildFileGenerator.cxx b/Source/cmExportBuildFileGenerator.cxx index a99c50b..dcb2187 100644 --- a/Source/cmExportBuildFileGenerator.cxx +++ b/Source/cmExportBuildFileGenerator.cxx @@ -212,12 +212,8 @@ cmExportBuildFileGenerator properties[prop] = value; } - // Check whether this is a DLL platform. - bool dll_platform = - (mf->IsOn("WIN32") || mf->IsOn("CYGWIN") || mf->IsOn("MINGW")); - // Add the import library for windows DLLs. - if(dll_platform && + if(target->IsDLLPlatform() && (target->GetType() == cmState::SHARED_LIBRARY || target->IsExecutableWithExports()) && mf->GetDefinition("CMAKE_IMPORT_LIBRARY_SUFFIX")) diff --git a/Source/cmExportFileGenerator.cxx b/Source/cmExportFileGenerator.cxx index d52ce35..e8a2e6a 100644 --- a/Source/cmExportFileGenerator.cxx +++ b/Source/cmExportFileGenerator.cxx @@ -879,10 +879,7 @@ cmExportFileGenerator if(target->GetType() == cmState::SHARED_LIBRARY || target->GetType() == cmState::MODULE_LIBRARY) { - // Check whether this is a DLL platform. - bool dll_platform = - (mf->IsOn("WIN32") || mf->IsOn("CYGWIN") || mf->IsOn("MINGW")); - if(!dll_platform) + if(!target->IsDLLPlatform()) { std::string prop; std::string value; diff --git a/Source/cmExportTryCompileFileGenerator.cxx b/Source/cmExportTryCompileFileGenerator.cxx index 1daa67e..83127e7 100644 --- a/Source/cmExportTryCompileFileGenerator.cxx +++ b/Source/cmExportTryCompileFileGenerator.cxx @@ -105,16 +105,18 @@ cmExportTryCompileFileGenerator::PopulateProperties( ImportPropertyMap& properties, std::set<cmGeneratorTarget const*> &emitted) { - cmPropertyMap props = target->Target->GetProperties(); - for(cmPropertyMap::const_iterator i = props.begin(); i != props.end(); ++i) + std::vector<std::string> props = target->GetPropertyKeys(); + for(std::vector<std::string>::const_iterator i = props.begin(); + i != props.end(); ++i) { - properties[i->first] = i->second.GetValue(); - if(i->first.find("IMPORTED_LINK_INTERFACE_LIBRARIES") == 0 - || i->first.find("IMPORTED_LINK_DEPENDENT_LIBRARIES") == 0 - || i->first.find("INTERFACE_LINK_LIBRARIES") == 0) + properties[*i] = target->GetProperty(*i); + + if(i->find("IMPORTED_LINK_INTERFACE_LIBRARIES") == 0 + || i->find("IMPORTED_LINK_DEPENDENT_LIBRARIES") == 0 + || i->find("INTERFACE_LINK_LIBRARIES") == 0) { - std::string evalResult = this->FindTargets(i->first, + std::string evalResult = this->FindTargets(*i, target, emitted); std::vector<std::string> depends; diff --git a/Source/cmExtraCodeBlocksGenerator.cxx b/Source/cmExtraCodeBlocksGenerator.cxx index 597c9d8..9348ef2 100644 --- a/Source/cmExtraCodeBlocksGenerator.cxx +++ b/Source/cmExtraCodeBlocksGenerator.cxx @@ -383,6 +383,9 @@ void cmExtraCodeBlocksGenerator all_files_map_t allFiles; std::vector<std::string> cFiles; + std::vector<std::string> srcExts = + this->GlobalGenerator->GetCMakeInstance()->GetSourceExtensions(); + for (std::vector<cmLocalGenerator*>::const_iterator lg=lgs.begin(); lg!=lgs.end(); lg++) { @@ -420,9 +423,7 @@ void cmExtraCodeBlocksGenerator { std::string srcext = (*si)->GetExtension(); for(std::vector<std::string>::const_iterator - ext = mf->GetSourceExtensions().begin(); - ext != mf->GetSourceExtensions().end(); - ++ext) + ext = srcExts.begin(); ext != srcExts.end(); ++ext) { if (srcext == *ext) { @@ -449,6 +450,9 @@ void cmExtraCodeBlocksGenerator } } + std::vector<std::string> headerExts = + this->GlobalGenerator->GetCMakeInstance()->GetHeaderExtensions(); + // The following loop tries to add header files matching to implementation // files to the project. It does that by iterating over all // C/C++ source files, @@ -468,8 +472,8 @@ void cmExtraCodeBlocksGenerator // check if there's a matching header around for(std::vector<std::string>::const_iterator - ext = mf->GetHeaderExtensions().begin(); - ext != mf->GetHeaderExtensions().end(); + ext = headerExts.begin(); + ext != headerExts.end(); ++ext) { std::string hname=headerBasename; diff --git a/Source/cmExtraCodeLiteGenerator.cxx b/Source/cmExtraCodeLiteGenerator.cxx index f4a6537..67aa157 100644 --- a/Source/cmExtraCodeLiteGenerator.cxx +++ b/Source/cmExtraCodeLiteGenerator.cxx @@ -149,6 +149,11 @@ void cmExtraCodeLiteGenerator // which may have an acompanying header, one for all other files std::string projectType; + std::vector<std::string> srcExts = + this->GlobalGenerator->GetCMakeInstance()->GetSourceExtensions(); + std::vector<std::string> headerExts = + this->GlobalGenerator->GetCMakeInstance()->GetHeaderExtensions(); + std::map<std::string, cmSourceFile*> cFiles; std::set<std::string> otherFiles; for (std::vector<cmLocalGenerator*>::const_iterator lg=lgs.begin(); @@ -207,9 +212,7 @@ void cmExtraCodeLiteGenerator { std::string srcext = (*si)->GetExtension(); for(std::vector<std::string>::const_iterator - ext = mf->GetSourceExtensions().begin(); - ext != mf->GetSourceExtensions().end(); - ++ext) + ext = srcExts.begin(); ext != srcExts.end(); ++ext) { if (srcext == *ext) { @@ -253,8 +256,8 @@ void cmExtraCodeLiteGenerator // check if there's a matching header around for(std::vector<std::string>::const_iterator - ext = mf->GetHeaderExtensions().begin(); - ext != mf->GetHeaderExtensions().end(); + ext = headerExts.begin(); + ext != headerExts.end(); ++ext) { std::string hname=headerBasename; diff --git a/Source/cmGeneratorExpressionNode.cxx b/Source/cmGeneratorExpressionNode.cxx index af72123..32b2f82 100644 --- a/Source/cmGeneratorExpressionNode.cxx +++ b/Source/cmGeneratorExpressionNode.cxx @@ -394,7 +394,7 @@ struct CompilerIdNode : public cmGeneratorExpressionNode if (cmsysString_strcasecmp(parameters.begin()->c_str(), compilerId) == 0) { - switch(context->LG->GetMakefile()->GetPolicyStatus(cmPolicies::CMP0044)) + switch(context->LG->GetPolicyStatus(cmPolicies::CMP0044)) { case cmPolicies::WARN: { @@ -1000,7 +1000,7 @@ static const struct TargetPropertyNode : public cmGeneratorExpressionNode if (propertyName == "LINKER_LANGUAGE") { - if (target->Target->LinkLanguagePropagatesToDependents() && + if (target->LinkLanguagePropagatesToDependents() && dagCheckerParent && (dagCheckerParent->EvaluatingLinkLibraries() || dagCheckerParent->EvaluatingSources())) { @@ -1101,7 +1101,7 @@ static const struct TargetPropertyNode : public cmGeneratorExpressionNode "COMPILE_DEFINITIONS_")) { cmPolicies::PolicyStatus polSt = - context->LG->GetMakefile()->GetPolicyStatus(cmPolicies::CMP0043); + context->LG->GetPolicyStatus(cmPolicies::CMP0043); if (polSt == cmPolicies::WARN || polSt == cmPolicies::OLD) { interfacePropertyName = "INTERFACE_COMPILE_DEFINITIONS"; @@ -1443,7 +1443,7 @@ cmPolicies::PolicyStatus statusForTarget(cmGeneratorTarget const* tgt, #define RETURN_POLICY(POLICY) \ if (strcmp(policy, #POLICY) == 0) \ { \ - return tgt->Target->GetPolicyStatus ## POLICY (); \ + return tgt->GetPolicyStatus ## POLICY (); \ } \ CM_FOR_EACH_TARGET_POLICY(RETURN_POLICY) diff --git a/Source/cmGeneratorTarget.cxx b/Source/cmGeneratorTarget.cxx index fe6b446..1f74eda 100644 --- a/Source/cmGeneratorTarget.cxx +++ b/Source/cmGeneratorTarget.cxx @@ -796,7 +796,7 @@ std::set<cmLinkItem> const& cmGeneratorTarget::GetUtilityItems() const if(!this->UtilityItemsDone) { this->UtilityItemsDone = true; - std::set<std::string> const& utilities = this->Target->GetUtilities(); + std::set<std::string> const& utilities = this->GetUtilities(); for(std::set<std::string>::const_iterator i = utilities.begin(); i != utilities.end(); ++i) { @@ -833,11 +833,34 @@ const char* cmGeneratorTarget::GetLocation(const std::string& config) const return location.c_str(); } +std::vector<cmCustomCommand> const& +cmGeneratorTarget::GetPreBuildCommands() const +{ + return this->Target->GetPreBuildCommands(); +} + +std::vector<cmCustomCommand> const& +cmGeneratorTarget::GetPreLinkCommands() const +{ + return this->Target->GetPreLinkCommands(); +} + +std::vector<cmCustomCommand> const& +cmGeneratorTarget::GetPostBuildCommands() const +{ + return this->Target->GetPostBuildCommands(); +} + bool cmGeneratorTarget::IsImported() const { return this->Target->IsImported(); } +bool cmGeneratorTarget::IsImportedGloballyVisible() const +{ + return this->Target->IsImportedGloballyVisible(); +} + //---------------------------------------------------------------------------- const char* cmGeneratorTarget::GetLocationForBuild() const { @@ -1623,7 +1646,7 @@ cmGeneratorTarget::GetFrameworkDirectory(const std::string& config, if(!rootDir && !this->Makefile->PlatformIsAppleIos()) { fpath += "/Versions/"; - fpath += this->Target->GetFrameworkVersion(); + fpath += this->GetFrameworkVersion(); } return fpath; } @@ -1714,6 +1737,22 @@ cmListFileBacktrace cmGeneratorTarget::GetBacktrace() const return this->Target->GetBacktrace(); } +const std::vector<std::string>&cmGeneratorTarget::GetLinkDirectories() const +{ + return this->Target->GetLinkDirectories(); +} + +const std::set<std::string>& cmGeneratorTarget::GetUtilities() const +{ + return this->Target->GetUtilities(); +} + +const cmListFileBacktrace* +cmGeneratorTarget::GetUtilityBacktrace(const std::string& u) const +{ + return this->Target->GetUtilityBacktrace(u); +} + //---------------------------------------------------------------------------- bool cmGeneratorTarget::HaveWellDefinedOutputFiles() const { @@ -1759,7 +1798,7 @@ public: UNORDERED_SET<std::string>& languages, cmGeneratorTarget const* head): Config(config), Languages(languages), HeadTarget(head), - Makefile(target->Target->GetMakefile()), Target(target) + Target(target) { this->Visited.insert(target); } void Visit(cmLinkItem const& item) @@ -1771,7 +1810,8 @@ public: bool noMessage = false; cmake::MessageType messageType = cmake::FATAL_ERROR; std::stringstream e; - switch(this->Makefile->GetPolicyStatus(cmPolicies::CMP0028)) + switch(this->Target->GetLocalGenerator() + ->GetPolicyStatus(cmPolicies::CMP0028)) { case cmPolicies::WARN: { @@ -1825,7 +1865,6 @@ private: std::string Config; UNORDERED_SET<std::string>& Languages; cmGeneratorTarget const* HeadTarget; - cmMakefile* Makefile; const cmGeneratorTarget* Target; std::set<cmGeneratorTarget const*> Visited; }; @@ -2267,11 +2306,11 @@ cmTargetTraceDependencies // Queue pre-build, pre-link, and post-build rule dependencies. this->CheckCustomCommands( - this->GeneratorTarget->Target->GetPreBuildCommands()); + this->GeneratorTarget->GetPreBuildCommands()); this->CheckCustomCommands( - this->GeneratorTarget->Target->GetPreLinkCommands()); + this->GeneratorTarget->GetPreLinkCommands()); this->CheckCustomCommands( - this->GeneratorTarget->Target->GetPostBuildCommands()); + this->GeneratorTarget->GetPostBuildCommands()); } //---------------------------------------------------------------------------- @@ -3313,7 +3352,7 @@ void cmGeneratorTarget::GetLibraryNames(std::string& name, if(!this->Makefile->PlatformIsAppleIos()) { realName += "Versions/"; - realName += this->Target->GetFrameworkVersion(); + realName += this->GetFrameworkVersion(); realName += "/"; } realName += base; @@ -4217,9 +4256,11 @@ PropertyType checkInterfacePropertyCompatibility(cmGeneratorTarget const* tgt, PropertyType *) { PropertyType propContent = getTypedProperty<PropertyType>(tgt, p); - const bool explicitlySet = tgt->Target->GetProperties() - .find(p) - != tgt->Target->GetProperties().end(); + std::vector<std::string> headPropKeys = tgt->GetPropertyKeys(); + const bool explicitlySet = + std::find(headPropKeys.begin(), headPropKeys.end(), + p) != headPropKeys.end(); + const bool impliedByUse = tgt->IsNullImpliedByLinkLibraries(p); assert((impliedByUse ^ explicitlySet) @@ -4264,9 +4305,11 @@ PropertyType checkInterfacePropertyCompatibility(cmGeneratorTarget const* tgt, cmGeneratorTarget const* theTarget = *li; - const bool ifaceIsSet = theTarget->Target->GetProperties() - .find(interfaceProperty) - != theTarget->Target->GetProperties().end(); + std::vector<std::string> propKeys = theTarget->GetPropertyKeys(); + + const bool ifaceIsSet = + std::find(propKeys.begin(), propKeys.end(), + interfaceProperty) != propKeys.end(); PropertyType ifacePropContent = getTypedProperty<PropertyType>(theTarget, interfaceProperty); @@ -4508,6 +4551,25 @@ void cmGeneratorTarget::GetTargetVersion(bool soversion, } //---------------------------------------------------------------------------- +std::string cmGeneratorTarget::GetFrameworkVersion() const +{ + assert(this->GetType() != cmState::INTERFACE_LIBRARY); + + if(const char* fversion = this->GetProperty("FRAMEWORK_VERSION")) + { + return fversion; + } + else if(const char* tversion = this->GetProperty("VERSION")) + { + return tversion; + } + else + { + return "A"; + } +} + +//---------------------------------------------------------------------------- void cmGeneratorTarget::ComputeVersionedName(std::string& vName, std::string const& prefix, std::string const& base, @@ -4524,6 +4586,19 @@ void cmGeneratorTarget::ComputeVersionedName(std::string& vName, vName += this->Makefile->IsOn("APPLE") ? suffix : std::string(); } +std::vector<std::string> cmGeneratorTarget::GetPropertyKeys() const +{ + cmPropertyMap propsObject = this->Target->GetProperties(); + std::vector<std::string> props; + props.reserve(propsObject.size()); + for (cmPropertyMap::const_iterator it = propsObject.begin(); + it != propsObject.end(); ++it) + { + props.push_back(it->first); + } + return props; +} + //---------------------------------------------------------------------------- void cmGeneratorTarget::ReportPropertyOrigin(const std::string &p, @@ -4717,7 +4792,7 @@ void cmGeneratorTarget::ComputeLinkInterface(const std::string& config, iface.WrongConfigLibraries = impl->WrongConfigLibraries; } - if(this->Target->LinkLanguagePropagatesToDependents()) + if(this->LinkLanguagePropagatesToDependents()) { // Targets using this archive need its language runtime libraries. if(cmLinkImplementation const* impl = @@ -5432,7 +5507,7 @@ void cmGeneratorTarget::ComputeImportInfo(std::string const& desired_config, } // Get the link languages. - if(this->Target->LinkLanguagePropagatesToDependents()) + if(this->LinkLanguagePropagatesToDependents()) { std::string linkProp = "IMPORTED_LINK_INTERFACE_LANGUAGES"; linkProp += suffix; @@ -5490,7 +5565,7 @@ cmGeneratorTarget::GetLinkImplementation(const std::string& config) const } std::string CONFIG = cmSystemTools::UpperCase(config); - cmOptionalLinkImplementation& impl = this->LinkImplMap[CONFIG][this->Target]; + cmOptionalLinkImplementation& impl = this->LinkImplMap[CONFIG][this]; if(!impl.LibrariesDone) { impl.LibrariesDone = true; @@ -5770,7 +5845,7 @@ cmGeneratorTarget::GetLinkImplementationLibrariesInternal( return &hm.begin()->second; } - cmOptionalLinkImplementation& impl = hm[head->Target]; + cmOptionalLinkImplementation& impl = hm[head]; if(!impl.LibrariesDone) { impl.LibrariesDone = true; diff --git a/Source/cmGeneratorTarget.h b/Source/cmGeneratorTarget.h index 544cf0e..da59a98 100644 --- a/Source/cmGeneratorTarget.h +++ b/Source/cmGeneratorTarget.h @@ -31,8 +31,13 @@ public: cmLocalGenerator* GetLocalGenerator() const; bool IsImported() const; + bool IsImportedGloballyVisible() const; const char *GetLocation(const std::string& config) const; + std::vector<cmCustomCommand> const &GetPreBuildCommands() const; + std::vector<cmCustomCommand> const &GetPreLinkCommands() const; + std::vector<cmCustomCommand> const &GetPostBuildCommands() const; + #define DECLARE_TARGET_POLICY(POLICY) \ cmPolicies::PolicyStatus GetPolicyStatus ## POLICY () const \ { return this->PolicyMap.Get(cmPolicies::POLICY); } @@ -53,6 +58,7 @@ public: std::string GetName() const; std::string GetExportName() const; + std::vector<std::string> GetPropertyKeys() const; const char *GetProperty(const std::string& prop) const; bool GetPropertyAsBool(const std::string& prop) const; void GetSourceFiles(std::vector<cmSourceFile*>& files, @@ -161,6 +167,10 @@ public: std::string GetFrameworkDirectory(const std::string& config, bool rootDir) const; + /** Return the framework version string. Undefined if + IsFrameworkOnApple returns false. */ + std::string GetFrameworkVersion() const; + /** @return the Mac CFBundle directory without the base */ std::string GetCFBundleDirectory(const std::string& config, bool contentOnly) const; @@ -176,6 +186,14 @@ public: cmListFileBacktrace GetBacktrace() const; + const std::vector<std::string>& GetLinkDirectories() const; + + std::set<std::string>const& GetUtilities() const; + cmListFileBacktrace const* GetUtilityBacktrace(const std::string& u) const; + + bool LinkLanguagePropagatesToDependents() const + { return this->GetType() == cmState::STATIC_LIBRARY; } + /** Get the macro to define when building sources in this target. If no macro should be defined null is returned. */ const char* GetExportMacro() const; @@ -630,7 +648,7 @@ private: const std::string& config) const; struct HeadToLinkImplementationMap: - public std::map<cmTarget const*, cmOptionalLinkImplementation> {}; + public std::map<cmGeneratorTarget const*, cmOptionalLinkImplementation> {}; typedef std::map<std::string, HeadToLinkImplementationMap> LinkImplMapType; mutable LinkImplMapType LinkImplMap; diff --git a/Source/cmGhsMultiTargetGenerator.cxx b/Source/cmGhsMultiTargetGenerator.cxx index 12115be..18e140e 100644 --- a/Source/cmGhsMultiTargetGenerator.cxx +++ b/Source/cmGhsMultiTargetGenerator.cxx @@ -392,10 +392,10 @@ void cmGhsMultiTargetGenerator::WriteTargetLinkLibraries() void cmGhsMultiTargetGenerator::WriteCustomCommands() { WriteCustomCommandsHelper( - this->GeneratorTarget->Target->GetPreBuildCommands(), + this->GeneratorTarget->GetPreBuildCommands(), cmTarget::PRE_BUILD); WriteCustomCommandsHelper( - this->GeneratorTarget->Target->GetPostBuildCommands(), + this->GeneratorTarget->GetPostBuildCommands(), cmTarget::POST_BUILD); } diff --git a/Source/cmGlobalGenerator.cxx b/Source/cmGlobalGenerator.cxx index d26cc34..3d2db42 100644 --- a/Source/cmGlobalGenerator.cxx +++ b/Source/cmGlobalGenerator.cxx @@ -1208,8 +1208,6 @@ void cmGlobalGenerator::Configure() void cmGlobalGenerator::CreateGenerationObjects(TargetTypes targetTypes) { this->CreateLocalGenerators(); - cmDeleteAll(this->GeneratorTargets); - this->GeneratorTargets.clear(); this->CreateGeneratorTargets(targetTypes); this->ComputeBuildFileGenerators(); } @@ -1583,10 +1581,12 @@ void cmGlobalGenerator::FinalizeTargetCompileInfo() } //---------------------------------------------------------------------------- -void cmGlobalGenerator::CreateGeneratorTargets(TargetTypes targetTypes, - cmLocalGenerator *lg) +void cmGlobalGenerator::CreateGeneratorTargets( + TargetTypes targetTypes, + cmMakefile *mf, + cmLocalGenerator *lg, + std::map<cmTarget*, cmGeneratorTarget*> const& importedMap) { - cmMakefile* mf = lg->GetMakefile(); if (targetTypes == AllTargets) { cmTargets& targets = mf->GetTargets(); @@ -1595,27 +1595,42 @@ void cmGlobalGenerator::CreateGeneratorTargets(TargetTypes targetTypes, { cmTarget* t = &ti->second; cmGeneratorTarget* gt = new cmGeneratorTarget(t, lg); - this->GeneratorTargets[t] = gt; lg->AddGeneratorTarget(gt); } } + std::vector<cmTarget*> itgts = mf->GetImportedTargets(); + for(std::vector<cmTarget*>::const_iterator - j = mf->GetOwnedImportedTargets().begin(); - j != mf->GetOwnedImportedTargets().end(); ++j) + j = itgts.begin(); j != itgts.end(); ++j) { - cmGeneratorTarget* gt = new cmGeneratorTarget(*j, lg); - this->GeneratorTargets[*j] = gt; + lg->AddImportedGeneratorTarget(importedMap.find(*j)->second); } } //---------------------------------------------------------------------------- void cmGlobalGenerator::CreateGeneratorTargets(TargetTypes targetTypes) { + std::map<cmTarget*, cmGeneratorTarget*> importedMap; + for(unsigned int i=0; i < this->Makefiles.size(); ++i) + { + cmMakefile* mf = this->Makefiles[i]; + for(std::vector<cmTarget*>::const_iterator + j = mf->GetOwnedImportedTargets().begin(); + j != mf->GetOwnedImportedTargets().end(); ++j) + { + cmLocalGenerator* lg = this->LocalGenerators[i]; + cmGeneratorTarget* gt = new cmGeneratorTarget(*j, lg); + lg->AddOwnedImportedGeneratorTarget(gt); + importedMap[*j] = gt; + } + } + // Construct per-target generator information. for(unsigned int i=0; i < this->LocalGenerators.size(); ++i) { - this->CreateGeneratorTargets(targetTypes, this->LocalGenerators[i]); + this->CreateGeneratorTargets(targetTypes, this->Makefiles[i], + this->LocalGenerators[i], importedMap); } } @@ -1623,9 +1638,6 @@ void cmGlobalGenerator::CreateGeneratorTargets(TargetTypes targetTypes) //---------------------------------------------------------------------------- void cmGlobalGenerator::ClearGeneratorMembers() { - cmDeleteAll(this->GeneratorTargets); - this->GeneratorTargets.clear(); - cmDeleteAll(this->BuildExportSets); this->BuildExportSets.clear(); @@ -1637,8 +1649,6 @@ void cmGlobalGenerator::ClearGeneratorMembers() this->ExportSets.clear(); this->TargetDependencies.clear(); - this->TotalTargets.clear(); - this->ImportedTargets.clear(); this->ProjectMap.clear(); this->RuleHashes.clear(); this->DirectoryContentMap.clear(); @@ -1646,20 +1656,6 @@ void cmGlobalGenerator::ClearGeneratorMembers() } //---------------------------------------------------------------------------- -cmGeneratorTarget* -cmGlobalGenerator::GetGeneratorTarget(cmTarget const* t) const -{ - cmGeneratorTargetsType::const_iterator ti = this->GeneratorTargets.find(t); - if(ti == this->GeneratorTargets.end()) - { - this->CMakeInstance->IssueMessage( - cmake::INTERNAL_ERROR, "Missing cmGeneratorTarget instance!"); - return 0; - } - return ti->second; -} - -//---------------------------------------------------------------------------- void cmGlobalGenerator::ComputeTargetObjectDirectory(cmGeneratorTarget*) const { } @@ -2169,9 +2165,10 @@ cmGlobalGenerator::FindLocalGenerator(const std::string& start_dir) const } //---------------------------------------------------------------------------- -void cmGlobalGenerator::AddAlias(const std::string& name, cmTarget *tgt) +void cmGlobalGenerator::AddAlias(const std::string& name, + std::string const& tgtName) { - this->AliasTargets[name] = tgt; + this->AliasTargets[name] = tgtName; } //---------------------------------------------------------------------------- @@ -2180,6 +2177,79 @@ bool cmGlobalGenerator::IsAlias(const std::string& name) const return this->AliasTargets.find(name) != this->AliasTargets.end(); } +cmTarget* cmGlobalGenerator::FindTargetImpl(std::string const& name) const +{ + for (unsigned int i = 0; i < this->Makefiles.size(); ++i) + { + cmTargets& tgts = this->Makefiles[i]->GetTargets(); + for (cmTargets::iterator it = tgts.begin(); it != tgts.end(); ++it) + { + if (it->second.GetName() == name) + { + return &it->second; + } + } + } + return 0; +} + +cmGeneratorTarget* +cmGlobalGenerator::FindGeneratorTargetImpl(std::string const& name) const +{ + for (unsigned int i = 0; i < this->LocalGenerators.size(); ++i) + { + std::vector<cmGeneratorTarget*> tgts = + this->LocalGenerators[i]->GetGeneratorTargets(); + for (std::vector<cmGeneratorTarget*>::iterator it = tgts.begin(); + it != tgts.end(); ++it) + { + if ((*it)->GetName() == name) + { + return *it; + } + } + } + return 0; +} + +cmTarget* +cmGlobalGenerator::FindImportedTargetImpl(std::string const& name) const +{ + for (unsigned int i = 0; i < this->Makefiles.size(); ++i) + { + std::vector<cmTarget*> tgts = + this->Makefiles[i]->GetOwnedImportedTargets(); + for (std::vector<cmTarget*>::iterator it = tgts.begin(); + it != tgts.end(); ++it) + { + if ((*it)->GetName() == name && (*it)->IsImportedGloballyVisible()) + { + return *it; + } + } + } + return 0; +} + +cmGeneratorTarget* cmGlobalGenerator::FindImportedGeneratorTargetImpl( + std::string const& name) const +{ + for (unsigned int i = 0; i < this->LocalGenerators.size(); ++i) + { + std::vector<cmGeneratorTarget*> tgts = + this->LocalGenerators[i]->GetImportedGeneratorTargets(); + for (std::vector<cmGeneratorTarget*>::iterator it = tgts.begin(); + it != tgts.end(); ++it) + { + if ((*it)->IsImportedGloballyVisible() && (*it)->GetName() == name) + { + return *it; + } + } + } + return 0; +} + //---------------------------------------------------------------------------- cmTarget* cmGlobalGenerator::FindTarget(const std::string& name, @@ -2187,23 +2257,28 @@ cmGlobalGenerator::FindTarget(const std::string& name, { if (!excludeAliases) { - TargetMap::const_iterator ai = this->AliasTargets.find(name); + std::map<std::string, std::string>::const_iterator ai = + this->AliasTargets.find(name); if (ai != this->AliasTargets.end()) { - return ai->second; + return this->FindTargetImpl(ai->second); } } - TargetMap::const_iterator i = this->TotalTargets.find ( name ); - if ( i != this->TotalTargets.end() ) + if (cmTarget* tgt = this->FindTargetImpl(name)) { - return i->second; + return tgt; } - i = this->ImportedTargets.find(name); - if ( i != this->ImportedTargets.end() ) + return this->FindImportedTargetImpl(name); +} + +cmGeneratorTarget* +cmGlobalGenerator::FindGeneratorTarget(const std::string& name) const +{ + if (cmGeneratorTarget* tgt = this->FindGeneratorTargetImpl(name)) { - return i->second; + return tgt; } - return 0; + return this->FindImportedGeneratorTargetImpl(name); } //---------------------------------------------------------------------------- @@ -2217,8 +2292,7 @@ cmGlobalGenerator::NameResolvesToFramework(const std::string& libname) const if(cmTarget* tgt = this->FindTarget(libname)) { - cmGeneratorTarget* gt = this->GetGeneratorTarget(tgt); - if(gt->IsFrameworkOnApple()) + if(tgt->IsFrameworkOnApple()) { return true; } @@ -2622,18 +2696,6 @@ cmGlobalGenerator::GetTargetDirectDepends(cmGeneratorTarget const* target) return this->TargetDependencies[target]; } -void cmGlobalGenerator::AddTarget(cmTarget* t) -{ - if(t->IsImported()) - { - this->ImportedTargets[t->GetName()] = t; - } - else - { - this->TotalTargets[t->GetName()] = t; - } -} - bool cmGlobalGenerator::IsReservedTarget(std::string const& name) { // The following is a list of targets reserved @@ -2939,17 +3001,20 @@ void cmGlobalGenerator::WriteSummary() fname += "/TargetDirectories.txt"; cmGeneratedFileStream fout(fname.c_str()); - // Generate summary information files for each target. - for(TargetMap::const_iterator ti = - this->TotalTargets.begin(); ti != this->TotalTargets.end(); ++ti) + for (unsigned int i = 0; i < this->LocalGenerators.size(); ++i) { - if ((ti->second)->GetType() == cmState::INTERFACE_LIBRARY) + std::vector<cmGeneratorTarget*> tgts = + this->LocalGenerators[i]->GetGeneratorTargets(); + for (std::vector<cmGeneratorTarget*>::iterator it = tgts.begin(); + it != tgts.end(); ++it) { - continue; + if ((*it)->GetType() == cmState::INTERFACE_LIBRARY) + { + continue; + } + this->WriteSummary(*it); + fout << (*it)->GetSupportDirectory() << "\n"; } - cmGeneratorTarget* gt = this->GetGeneratorTarget(ti->second); - this->WriteSummary(gt); - fout << gt->GetSupportDirectory() << "\n"; } } diff --git a/Source/cmGlobalGenerator.h b/Source/cmGlobalGenerator.h index 18f5329..bc6e17d 100644 --- a/Source/cmGlobalGenerator.h +++ b/Source/cmGlobalGenerator.h @@ -42,10 +42,6 @@ class cmInstallTargetGenerator; class cmInstallFilesGenerator; class cmExportBuildFileGenerator; -typedef std::map<cmTarget const*, - cmGeneratorTarget*, - cmTarget::StrictTargetComparison> cmGeneratorTargetsType; - /** \class cmGlobalGenerator * \brief Responsible for overseeing the generation process for the entire tree * @@ -254,7 +250,9 @@ public: cmTarget* FindTarget(const std::string& name, bool excludeAliases = false) const; - void AddAlias(const std::string& name, cmTarget *tgt); + cmGeneratorTarget* FindGeneratorTarget(const std::string& name) const; + + void AddAlias(const std::string& name, const std::string& tgtName); bool IsAlias(const std::string& name) const; /** Determine if a name resolves to a framework on disk or a built target @@ -280,8 +278,6 @@ public: std::set<std::string> const& GetDirectoryContent(std::string const& dir, bool needDisk = true); - void AddTarget(cmTarget* t); - static bool IsReservedTarget(std::string const& name); virtual const char* GetAllTargetName() const { return "ALL_BUILD"; } @@ -307,14 +303,6 @@ public: TargetDependSet const& GetTargetDirectDepends( const cmGeneratorTarget* target); - /** Get per-target generator information. */ - cmGeneratorTarget* GetGeneratorTarget(cmTarget const*) const; - - void AddGeneratorTarget(cmTarget* t, cmGeneratorTarget* gt) - { - this->GeneratorTargets[t] = gt; - } - const std::map<std::string, std::vector<cmLocalGenerator*> >& GetProjectMap() const {return this->ProjectMap;} @@ -429,19 +417,14 @@ protected: std::map<std::string, cmExportBuildFileGenerator*> BuildExportSets; std::map<std::string, cmExportBuildFileGenerator*> BuildExportExportSets; - // All targets in the entire project. -#if defined(CMAKE_BUILD_WITH_CMAKE) -#ifdef CMake_HAVE_CXX11_UNORDERED_MAP - typedef std::unordered_map<std::string, cmTarget*> TargetMap; -#else - typedef cmsys::hash_map<std::string, cmTarget*> TargetMap; -#endif -#else - typedef std::map<std::string,cmTarget *> TargetMap; -#endif - TargetMap TotalTargets; - TargetMap AliasTargets; - TargetMap ImportedTargets; + std::map<std::string, std::string> AliasTargets; + + cmTarget* FindTargetImpl(std::string const& name) const; + cmTarget* FindImportedTargetImpl(std::string const& name) const; + + cmGeneratorTarget* FindGeneratorTargetImpl(std::string const& name) const; + cmGeneratorTarget* + FindImportedGeneratorTargetImpl(std::string const& name) const; const char* GetPredefinedTargetsFolder(); virtual bool UseFolderProperty(); @@ -487,10 +470,10 @@ private: typedef std::map<cmGeneratorTarget const*, TargetDependSet> TargetDependMap; TargetDependMap TargetDependencies; - // Per-target generator information. - cmGeneratorTargetsType GeneratorTargets; friend class cmake; - void CreateGeneratorTargets(TargetTypes targetTypes, cmLocalGenerator* lg); + void CreateGeneratorTargets(TargetTypes targetTypes, cmMakefile* mf, + cmLocalGenerator* lg, + std::map<cmTarget*, cmGeneratorTarget*> const& importedMap); void CreateGeneratorTargets(TargetTypes targetTypes); void ClearGeneratorMembers(); diff --git a/Source/cmGlobalKdevelopGenerator.cxx b/Source/cmGlobalKdevelopGenerator.cxx index 7c6c48c..018ab24 100644 --- a/Source/cmGlobalKdevelopGenerator.cxx +++ b/Source/cmGlobalKdevelopGenerator.cxx @@ -105,6 +105,9 @@ bool cmGlobalKdevelopGenerator std::set<std::string> files; std::string tmp; + std::vector<std::string> hdrExts = + this->GlobalGenerator->GetCMakeInstance()->GetHeaderExtensions(); + for (std::vector<cmLocalGenerator*>::const_iterator it=lgs.begin(); it!=lgs.end(); it++) { @@ -160,8 +163,7 @@ bool cmGlobalKdevelopGenerator // check if there's a matching header around for(std::vector<std::string>::const_iterator - ext = makefile->GetHeaderExtensions().begin(); - ext != makefile->GetHeaderExtensions().end(); ++ext) + ext = hdrExts.begin(); ext != hdrExts.end(); ++ext) { std::string hname=headerBasename; hname += "."; diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx index 2671f4d..8498e39 100644 --- a/Source/cmGlobalNinjaGenerator.cxx +++ b/Source/cmGlobalNinjaGenerator.cxx @@ -965,7 +965,7 @@ cmGlobalNinjaGenerator if (target->GetType() == cmState::GLOBAL_TARGET) { // Global targets only depend on other utilities, which may not appear in // the TargetDepends set (e.g. "all"). - std::set<std::string> const& utils = target->Target->GetUtilities(); + std::set<std::string> const& utils = target->GetUtilities(); std::copy(utils.begin(), utils.end(), std::back_inserter(outputs)); } else { cmTargetDependSet const& targetDeps = this->GetTargetDirectDepends(target); diff --git a/Source/cmGlobalUnixMakefileGenerator3.cxx b/Source/cmGlobalUnixMakefileGenerator3.cxx index 2fa0598..ce7815d 100644 --- a/Source/cmGlobalUnixMakefileGenerator3.cxx +++ b/Source/cmGlobalUnixMakefileGenerator3.cxx @@ -521,7 +521,7 @@ cmGlobalUnixMakefileGenerator3 cmLocalUnixMakefileGenerator3* lg) { // Only subdirectories need these rules. - if(lg->GetMakefile()->IsRootMakefile()) + if(lg->IsRootMakefile()) { return; } @@ -1082,7 +1082,7 @@ void cmGlobalUnixMakefileGenerator3::WriteHelpRule static_cast<cmLocalUnixMakefileGenerator3 *>(this->LocalGenerators[i]); // for the passed in makefile or if this is the top Makefile wripte out // the targets - if (lg2 == lg || lg->GetMakefile()->IsRootMakefile()) + if (lg2 == lg || lg->IsRootMakefile()) { // for each target Generate the rule files for each target. std::vector<cmGeneratorTarget*> targets = lg2->GetGeneratorTargets(); diff --git a/Source/cmGlobalVisualStudio6Generator.cxx b/Source/cmGlobalVisualStudio6Generator.cxx index bb494f1..5866c0e 100644 --- a/Source/cmGlobalVisualStudio6Generator.cxx +++ b/Source/cmGlobalVisualStudio6Generator.cxx @@ -235,7 +235,7 @@ void cmGlobalVisualStudio6Generator std::string project = target->GetName(); std::string location = expath; this->WriteExternalProject(fout, project.c_str(), - location.c_str(), target->Target->GetUtilities()); + location.c_str(), target->GetUtilities()); } else { diff --git a/Source/cmGlobalVisualStudio7Generator.cxx b/Source/cmGlobalVisualStudio7Generator.cxx index 5473247..f5848ab 100644 --- a/Source/cmGlobalVisualStudio7Generator.cxx +++ b/Source/cmGlobalVisualStudio7Generator.cxx @@ -401,7 +401,7 @@ void cmGlobalVisualStudio7Generator::WriteTargetConfigurations( for(OrderedTargetDependSet::const_iterator tt = projectTargets.begin(); tt != projectTargets.end(); ++tt) { - cmTarget const* target = (*tt)->Target; + cmGeneratorTarget const* target = *tt; if(target->GetType() == cmState::INTERFACE_LIBRARY) { continue; @@ -459,7 +459,7 @@ void cmGlobalVisualStudio7Generator::WriteTargetsToSolution( project.c_str(), location.c_str(), target->GetProperty("VS_PROJECT_TYPE"), - target->Target->GetUtilities()); + target->GetUtilities()); written = true; } else @@ -996,7 +996,8 @@ cmGlobalVisualStudio7Generator std::set<std::string> cmGlobalVisualStudio7Generator::IsPartOfDefaultBuild( std::vector<std::string> const& configs, - OrderedTargetDependSet const& projectTargets, cmTarget const* target) + OrderedTargetDependSet const& projectTargets, + cmGeneratorTarget const* target) { std::set<std::string> activeConfigs; // if it is a utilitiy target then only make it part of the @@ -1011,13 +1012,13 @@ cmGlobalVisualStudio7Generator::IsPartOfDefaultBuild( for(std::vector<std::string>::const_iterator i = configs.begin(); i != configs.end(); ++i) { - const char* propertyValue = target->GetMakefile() + const char* propertyValue = target->Target->GetMakefile() ->GetDefinition("CMAKE_VS_INCLUDE_INSTALL_TO_DEFAULT_BUILD"); cmGeneratorExpression ge; cmsys::auto_ptr<cmCompiledGeneratorExpression> cge = ge.Parse(propertyValue); - cmGeneratorTarget* gt = this->GetGeneratorTarget(target); - if(cmSystemTools::IsOn(cge->Evaluate(gt->GetLocalGenerator(), *i))) + if(cmSystemTools::IsOn(cge->Evaluate(target->GetLocalGenerator(), + *i))) { activeConfigs.insert(*i); } @@ -1029,13 +1030,12 @@ cmGlobalVisualStudio7Generator::IsPartOfDefaultBuild( { return activeConfigs; } - cmGeneratorTarget* gt = this->GetGeneratorTarget(target); // inspect EXCLUDE_FROM_DEFAULT_BUILD[_<CONFIG>] properties for(std::vector<std::string>::const_iterator i = configs.begin(); i != configs.end(); ++i) { const char* propertyValue = - gt->GetFeature("EXCLUDE_FROM_DEFAULT_BUILD", i->c_str()); + target->GetFeature("EXCLUDE_FROM_DEFAULT_BUILD", i->c_str()); if(cmSystemTools::IsOff(propertyValue)) { activeConfigs.insert(*i); @@ -1047,9 +1047,8 @@ cmGlobalVisualStudio7Generator::IsPartOfDefaultBuild( bool cmGlobalVisualStudio7Generator ::IsDependedOn(OrderedTargetDependSet const& projectTargets, - cmTarget const* targetIn) + cmGeneratorTarget const* gtIn) { - cmGeneratorTarget* gtIn = this->GetGeneratorTarget(targetIn); for (OrderedTargetDependSet::const_iterator l = projectTargets.begin(); l != projectTargets.end(); ++l) { diff --git a/Source/cmGlobalVisualStudio7Generator.h b/Source/cmGlobalVisualStudio7Generator.h index 6a0e5e9..fe6db2e 100644 --- a/Source/cmGlobalVisualStudio7Generator.h +++ b/Source/cmGlobalVisualStudio7Generator.h @@ -163,9 +163,9 @@ protected: std::set<std::string> IsPartOfDefaultBuild(std::vector<std::string> const& configs, OrderedTargetDependSet const& projectTargets, - cmTarget const* target); + cmGeneratorTarget const* target); bool IsDependedOn(OrderedTargetDependSet const& projectTargets, - cmTarget const* target); + cmGeneratorTarget const* target); std::map<std::string, std::string> GUIDMap; virtual void WriteFolders(std::ostream& fout); diff --git a/Source/cmGlobalVisualStudio8Generator.cxx b/Source/cmGlobalVisualStudio8Generator.cxx index bdb1b25..5e239b8 100644 --- a/Source/cmGlobalVisualStudio8Generator.cxx +++ b/Source/cmGlobalVisualStudio8Generator.cxx @@ -257,7 +257,6 @@ bool cmGlobalVisualStudio8Generator::AddCheckTarget() cmGeneratorTarget* gt = new cmGeneratorTarget(tgt, lg); lg->AddGeneratorTarget(gt); - this->AddGeneratorTarget(tgt, gt); // Organize in the "predefined targets" folder: // @@ -354,14 +353,18 @@ void cmGlobalVisualStudio8Generator::AddExtraIDETargets() cmGlobalVisualStudio7Generator::AddExtraIDETargets(); if(this->AddCheckTarget()) { - // All targets depend on the build-system check target. - for(TargetMap::const_iterator - ti = this->TotalTargets.begin(); - ti != this->TotalTargets.end(); ++ti) + for (unsigned int i = 0; i < this->LocalGenerators.size(); ++i) { - if(ti->first != CMAKE_CHECK_BUILD_SYSTEM_TARGET) + std::vector<cmGeneratorTarget*> tgts = + this->LocalGenerators[i]->GetGeneratorTargets(); + // All targets depend on the build-system check target. + for(std::vector<cmGeneratorTarget*>::iterator ti = tgts.begin(); + ti != tgts.end(); ++ti) { - ti->second->AddUtility(CMAKE_CHECK_BUILD_SYSTEM_TARGET); + if((*ti)->GetName() != CMAKE_CHECK_BUILD_SYSTEM_TARGET) + { + (*ti)->Target->AddUtility(CMAKE_CHECK_BUILD_SYSTEM_TARGET); + } } } } @@ -464,8 +467,8 @@ bool cmGlobalVisualStudio8Generator::NeedLinkLibraryDependencies( { // Look for utility dependencies that magically link. for(std::set<std::string>::const_iterator ui = - target->Target->GetUtilities().begin(); - ui != target->Target->GetUtilities().end(); ++ui) + target->GetUtilities().begin(); + ui != target->GetUtilities().end(); ++ui) { if(cmGeneratorTarget* depTarget = target->GetLocalGenerator()->FindGeneratorTargetToUse(ui->c_str())) diff --git a/Source/cmGlobalVisualStudioGenerator.cxx b/Source/cmGlobalVisualStudioGenerator.cxx index 0488391..6a1aa29 100644 --- a/Source/cmGlobalVisualStudioGenerator.cxx +++ b/Source/cmGlobalVisualStudioGenerator.cxx @@ -88,7 +88,6 @@ void cmGlobalVisualStudioGenerator::AddExtraIDETargets() cmGeneratorTarget* gt = new cmGeneratorTarget(allBuild, gen[0]); gen[0]->AddGeneratorTarget(gt); - this->AddGeneratorTarget(allBuild, gt); #if 0 // Can't activate this code because we want ALL_BUILD diff --git a/Source/cmGlobalXCodeGenerator.cxx b/Source/cmGlobalXCodeGenerator.cxx index eecc82b..da8c814 100644 --- a/Source/cmGlobalXCodeGenerator.cxx +++ b/Source/cmGlobalXCodeGenerator.cxx @@ -460,7 +460,6 @@ cmGlobalXCodeGenerator::AddExtraTargets(cmLocalGenerator* root, cmGeneratorTarget* allBuildGt = new cmGeneratorTarget(allbuild, root); root->AddGeneratorTarget(allBuildGt); - root->GetGlobalGenerator()->AddGeneratorTarget(allbuild, allBuildGt); // Refer to the main build configuration file for easy editing. std::string listfile = root->GetCurrentSourceDirectory(); @@ -496,7 +495,6 @@ cmGlobalXCodeGenerator::AddExtraTargets(cmLocalGenerator* root, cmGeneratorTarget* checkGt = new cmGeneratorTarget(check, root); root->AddGeneratorTarget(checkGt); - root->GetGlobalGenerator()->AddGeneratorTarget(check, checkGt); } // now make the allbuild depend on all the non-utility targets @@ -1362,12 +1360,17 @@ cmGlobalXCodeGenerator::CreateXCodeTargets(cmLocalGenerator* gen, //---------------------------------------------------------------------------- void cmGlobalXCodeGenerator::ForceLinkerLanguages() { - // This makes sure all targets link using the proper language. - for(TargetMap::const_iterator - ti = this->TotalTargets.begin(); ti != this->TotalTargets.end(); ++ti) + for (unsigned int i = 0; i < this->LocalGenerators.size(); ++i) { - cmGeneratorTarget* gt = this->GetGeneratorTarget(ti->second); - this->ForceLinkerLanguage(gt); + std::vector<cmGeneratorTarget*> tgts = + this->LocalGenerators[i]->GetGeneratorTargets(); + // All targets depend on the build-system check target. + for(std::vector<cmGeneratorTarget*>::const_iterator ti = tgts.begin(); + ti != tgts.end(); ++ti) + { + // This makes sure all targets link using the proper language. + this->ForceLinkerLanguage(*ti); + } } } @@ -1420,7 +1423,7 @@ void cmGlobalXCodeGenerator::ForceLinkerLanguage(cmGeneratorTarget* gtgt) bool cmGlobalXCodeGenerator::IsHeaderFile(cmSourceFile* sf) { const std::vector<std::string>& hdrExts = - this->CurrentMakefile->GetHeaderExtensions(); + this->CMakeInstance->GetHeaderExtensions(); return (std::find(hdrExts.begin(), hdrExts.end(), sf->GetExtension()) != hdrExts.end()); } @@ -1469,11 +1472,11 @@ void cmGlobalXCodeGenerator::CreateCustomCommands(cmXCodeObject* buildPhases, cmGeneratorTarget* gtgt) { std::vector<cmCustomCommand> const & prebuild - = gtgt->Target->GetPreBuildCommands(); + = gtgt->GetPreBuildCommands(); std::vector<cmCustomCommand> const & prelink - = gtgt->Target->GetPreLinkCommands(); + = gtgt->GetPreLinkCommands(); std::vector<cmCustomCommand> postbuild - = gtgt->Target->GetPostBuildCommands(); + = gtgt->GetPostBuildCommands(); if(gtgt->GetType() == cmState::SHARED_LIBRARY && !gtgt->IsFrameworkOnApple()) @@ -2141,7 +2144,7 @@ void cmGlobalXCodeGenerator::CreateBuildSettings(cmGeneratorTarget* gtgt, { if(gtgt->GetPropertyAsBool("FRAMEWORK")) { - std::string fw_version = gtgt->Target->GetFrameworkVersion(); + std::string fw_version = gtgt->GetFrameworkVersion(); buildSettings->AddAttribute("FRAMEWORK_VERSION", this->CreateString(fw_version.c_str())); @@ -2486,13 +2489,13 @@ void cmGlobalXCodeGenerator::CreateBuildSettings(cmGeneratorTarget* gtgt, // put this last so it can override existing settings // Convert "XCODE_ATTRIBUTE_*" properties directly. { - cmPropertyMap const& props = gtgt->Target->GetProperties(); - for(cmPropertyMap::const_iterator i = props.begin(); + std::vector<std::string> const& props = gtgt->GetPropertyKeys(); + for(std::vector<std::string>::const_iterator i = props.begin(); i != props.end(); ++i) { - if(i->first.find("XCODE_ATTRIBUTE_") == 0) + if(i->find("XCODE_ATTRIBUTE_") == 0) { - std::string attribute = i->first.substr(16); + std::string attribute = i->substr(16); // Handle [variant=<config>] condition explicitly here. std::string::size_type beginVariant = attribute.find("[variant="); @@ -2523,7 +2526,7 @@ void cmGlobalXCodeGenerator::CreateBuildSettings(cmGeneratorTarget* gtgt, if (!attribute.empty()) { cmGeneratorExpression ge; - std::string processed = ge.Parse(i->second.GetValue()) + std::string processed = ge.Parse(gtgt->GetProperty(*i)) ->Evaluate(this->CurrentLocalGenerator, configName); buildSettings->AddAttribute(attribute.c_str(), this->CreateString(processed)); diff --git a/Source/cmLocalGenerator.cxx b/Source/cmLocalGenerator.cxx index c37417d..d92cbea 100644 --- a/Source/cmLocalGenerator.cxx +++ b/Source/cmLocalGenerator.cxx @@ -51,6 +51,8 @@ cmLocalGenerator::cmLocalGenerator(cmGlobalGenerator* gg, this->Makefile = makefile; + this->AliasTargets = makefile->GetAliasTargets(); + this->EmitUniversalBinaryFlags = true; this->BackwardsCompatibility = 0; this->BackwardsCompatibilityFinal = false; @@ -60,6 +62,8 @@ cmLocalGenerator::cmLocalGenerator(cmGlobalGenerator* gg, cmLocalGenerator::~cmLocalGenerator() { + cmDeleteAll(this->GeneratorTargets); + cmDeleteAll(this->OwnedImportedGeneratorTargets); } void cmLocalGenerator::IssueMessage(cmake::MessageType t, @@ -453,11 +457,55 @@ void cmLocalGenerator::AddGeneratorTarget(cmGeneratorTarget* gt) this->GeneratorTargets.push_back(gt); } +void cmLocalGenerator::AddImportedGeneratorTarget(cmGeneratorTarget* gt) +{ + this->ImportedGeneratorTargets.push_back(gt); +} + +void cmLocalGenerator::AddOwnedImportedGeneratorTarget(cmGeneratorTarget* gt) +{ + this->OwnedImportedGeneratorTargets.push_back(gt); +} + +struct NamedGeneratorTargetFinder +{ + NamedGeneratorTargetFinder(std::string const& name) + : Name(name) + { + + } + + bool operator()(cmGeneratorTarget* tgt) + { + return tgt->GetName() == this->Name; + } +private: + std::string Name; +}; + cmGeneratorTarget* cmLocalGenerator::FindGeneratorTarget( const std::string& name) const { - return this->GetGlobalGenerator()->GetGeneratorTarget( - this->Makefile->FindTarget(name)); + std::map<std::string, std::string>::const_iterator i = + this->AliasTargets.find(name); + if (i != this->AliasTargets.end()) + { + std::vector<cmGeneratorTarget*>::const_iterator ai = + std::find_if(this->GeneratorTargets.begin(), + this->GeneratorTargets.end(), + NamedGeneratorTargetFinder(i->second)); + return *ai; + } + std::vector<cmGeneratorTarget*>::const_iterator ti = + std::find_if(this->GeneratorTargets.begin(), + this->GeneratorTargets.end(), + NamedGeneratorTargetFinder(name)); + if ( ti != this->GeneratorTargets.end() ) + { + return *ti; + } + + return 0; } //---------------------------------------------------------------------------- @@ -490,6 +538,11 @@ void cmLocalGenerator::ComputeTargetManifest() } } +bool cmLocalGenerator::IsRootMakefile() const +{ + return !this->StateSnapshot.GetBuildsystemDirectoryParent().IsValid(); +} + cmState* cmLocalGenerator::GetState() const { return this->GlobalGenerator->GetCMakeInstance()->GetState(); @@ -1775,11 +1828,21 @@ void cmLocalGenerator::AddLanguageFlags(std::string& flags, cmGeneratorTarget* cmLocalGenerator::FindGeneratorTargetToUse(const std::string& name) const { - if (cmTarget *t = this->Makefile->FindTargetToUse(name)) + std::vector<cmGeneratorTarget*>::const_iterator + imported = std::find_if(this->ImportedGeneratorTargets.begin(), + this->ImportedGeneratorTargets.end(), + NamedGeneratorTargetFinder(name)); + if(imported != this->ImportedGeneratorTargets.end()) { - return this->GetGlobalGenerator()->GetGeneratorTarget(t); + return *imported; } - return 0; + + if(cmGeneratorTarget* t = this->FindGeneratorTarget(name)) + { + return t; + } + + return this->GetGlobalGenerator()->FindGeneratorTarget(name); } //---------------------------------------------------------------------------- @@ -2028,7 +2091,7 @@ AddCompilerRequirementFlag(std::string &flags, } static void AddVisibilityCompileOption(std::string &flags, - cmTarget const* target, + cmGeneratorTarget const* target, cmLocalGenerator *lg, const std::string& lang, std::string* warnCMP0063) @@ -2068,7 +2131,7 @@ static void AddVisibilityCompileOption(std::string &flags, } static void AddInlineVisibilityCompileOption(std::string &flags, - cmTarget const* target, + cmGeneratorTarget const* target, cmLocalGenerator *lg, std::string* warnCMP0063) { @@ -2121,12 +2184,11 @@ void cmLocalGenerator } } - AddVisibilityCompileOption(flags, target->Target, this, lang, pWarnCMP0063); + AddVisibilityCompileOption(flags, target, this, lang, pWarnCMP0063); if(lang == "CXX") { - AddInlineVisibilityCompileOption(flags, target->Target, - this, pWarnCMP0063); + AddInlineVisibilityCompileOption(flags, target, this, pWarnCMP0063); } if (!warnCMP0063.empty() && @@ -2201,7 +2263,7 @@ bool cmLocalGenerator::GetShouldUseOldFlags(bool shared, if (flags && flags != originalFlags) { - switch (this->Makefile->GetPolicyStatus(cmPolicies::CMP0018)) + switch (this->GetPolicyStatus(cmPolicies::CMP0018)) { case cmPolicies::WARN: { @@ -2949,7 +3011,7 @@ bool cmLocalGenerator::NeedBackwardsCompatibility_2_4() { // Check the policy to decide whether to pay attention to this // variable. - switch(this->Makefile->GetPolicyStatus(cmPolicies::CMP0001)) + switch(this->GetPolicyStatus(cmPolicies::CMP0001)) { case cmPolicies::WARN: // WARN is just OLD without warning because user code does not @@ -2976,6 +3038,12 @@ bool cmLocalGenerator::NeedBackwardsCompatibility_2_4() actual_compat <= CMake_VERSION_ENCODE(2, 4, 255)); } +cmPolicies::PolicyStatus +cmLocalGenerator::GetPolicyStatus(cmPolicies::PolicyID id) const +{ + return this->Makefile->GetPolicyStatus(id); +} + //---------------------------------------------------------------------------- bool cmLocalGenerator::CheckDefinition(std::string const& define) const { diff --git a/Source/cmLocalGenerator.h b/Source/cmLocalGenerator.h index 552020b..e2f5519 100644 --- a/Source/cmLocalGenerator.h +++ b/Source/cmLocalGenerator.h @@ -67,6 +67,8 @@ public: */ void ComputeTargetManifest(); + bool IsRootMakefile() const; + ///! Get the makefile for this generator cmMakefile *GetMakefile() { return this->Makefile; } @@ -118,7 +120,14 @@ public: return this->GeneratorTargets; } + const std::vector<cmGeneratorTarget*> &GetImportedGeneratorTargets() const + { + return this->ImportedGeneratorTargets; + } + void AddGeneratorTarget(cmGeneratorTarget* gt); + void AddImportedGeneratorTarget(cmGeneratorTarget* gt); + void AddOwnedImportedGeneratorTarget(cmGeneratorTarget* gt); cmGeneratorTarget* FindGeneratorTarget(const std::string& name) const; cmGeneratorTarget* FindGeneratorTargetToUse(const std::string& name) const; @@ -265,6 +274,8 @@ public: */ bool NeedBackwardsCompatibility_2_4(); + cmPolicies::PolicyStatus GetPolicyStatus(cmPolicies::PolicyID id) const; + cmake* GetCMakeInstance() const; const char* GetSourceDirectory() const; @@ -369,6 +380,9 @@ protected: std::set<cmGeneratorTarget const*> WarnCMP0063; std::vector<cmGeneratorTarget*> GeneratorTargets; + std::vector<cmGeneratorTarget*> ImportedGeneratorTargets; + std::vector<cmGeneratorTarget*> OwnedImportedGeneratorTargets; + std::map<std::string, std::string> AliasTargets; bool EmitUniversalBinaryFlags; diff --git a/Source/cmLocalNinjaGenerator.cxx b/Source/cmLocalNinjaGenerator.cxx index e11571a..ecaa269 100644 --- a/Source/cmLocalNinjaGenerator.cxx +++ b/Source/cmLocalNinjaGenerator.cxx @@ -56,7 +56,7 @@ void cmLocalNinjaGenerator::Generate() #endif // We do that only once for the top CMakeLists.txt file. - if(this->Makefile->IsRootMakefile()) + if(this->IsRootMakefile()) { this->WriteBuildFileTop(); @@ -277,7 +277,7 @@ void cmLocalNinjaGenerator::WriteProcessedMakefile(std::ostream& os) << "# Write statements declared in CMakeLists.txt:" << std::endl << "# " << this->Makefile->GetDefinition("CMAKE_CURRENT_LIST_FILE") << std::endl; - if(this->Makefile->IsRootMakefile()) + if(this->IsRootMakefile()) os << "# Which is the root file." << std::endl; cmGlobalNinjaGenerator::WriteDivider(os); os << std::endl; diff --git a/Source/cmLocalUnixMakefileGenerator3.cxx b/Source/cmLocalUnixMakefileGenerator3.cxx index 3972b41..62fea3d 100644 --- a/Source/cmLocalUnixMakefileGenerator3.cxx +++ b/Source/cmLocalUnixMakefileGenerator3.cxx @@ -253,7 +253,7 @@ void cmLocalUnixMakefileGenerator3::WriteLocalMakefile() return; } // always write the top makefile - if (!this->GetMakefile()->IsRootMakefile()) + if (!this->IsRootMakefile()) { ruleFileStream.SetCopyIfDifferent(true); } @@ -264,7 +264,7 @@ void cmLocalUnixMakefileGenerator3::WriteLocalMakefile() // only write local targets unless at the top Keep track of targets already // listed. std::set<std::string> emittedTargets; - if (!this->GetMakefile()->IsRootMakefile()) + if (!this->IsRootMakefile()) { // write our targets, and while doing it collect up the object // file rules @@ -879,7 +879,7 @@ void cmLocalUnixMakefileGenerator3 std::vector<std::string> no_depends; std::vector<std::string> commands; commands.push_back(runRule); - if(!this->GetMakefile()->IsRootMakefile()) + if(!this->IsRootMakefile()) { this->CreateCDCommand(commands, this->GetBinaryDirectory(), @@ -1777,8 +1777,8 @@ void cmLocalUnixMakefileGenerator3 { text = "Running external command ..."; } - depends.insert(depends.end(), (*glIt)->Target->GetUtilities().begin(), - (*glIt)->Target->GetUtilities().end()); + depends.insert(depends.end(), (*glIt)->GetUtilities().begin(), + (*glIt)->GetUtilities().end()); this->AppendEcho(commands, text, cmLocalUnixMakefileGenerator3::EchoGlobal); @@ -1786,15 +1786,15 @@ void cmLocalUnixMakefileGenerator3 // Global targets store their rules in pre- and post-build commands. this->AppendCustomDepends(depends, - gt->Target->GetPreBuildCommands()); + gt->GetPreBuildCommands()); this->AppendCustomDepends(depends, - gt->Target->GetPostBuildCommands()); + gt->GetPostBuildCommands()); this->AppendCustomCommands(commands, - gt->Target->GetPreBuildCommands(), + gt->GetPreBuildCommands(), gt, cmLocalGenerator::START_OUTPUT); this->AppendCustomCommands(commands, - gt->Target->GetPostBuildCommands(), + gt->GetPostBuildCommands(), gt, cmLocalGenerator::START_OUTPUT); std::string targetName = gt->GetName(); diff --git a/Source/cmLocalVisualStudio6Generator.cxx b/Source/cmLocalVisualStudio6Generator.cxx index d5abf68..cdacb9e 100644 --- a/Source/cmLocalVisualStudio6Generator.cxx +++ b/Source/cmLocalVisualStudio6Generator.cxx @@ -266,21 +266,21 @@ void cmLocalVisualStudio6Generator::WriteDSPFile(std::ostream& fout, // depend only on the previous rule. if ((target->GetType() == cmState::UTILITY || target->GetType() == cmState::GLOBAL_TARGET) && - (!target->Target->GetPreBuildCommands().empty() || - !target->Target->GetPostBuildCommands().empty())) + (!target->GetPreBuildCommands().empty() || + !target->GetPostBuildCommands().empty())) { // Accumulate the dependencies of all the commands. std::vector<std::string> depends; for (std::vector<cmCustomCommand>::const_iterator cr = - target->Target->GetPreBuildCommands().begin(); - cr != target->Target->GetPreBuildCommands().end(); ++cr) + target->GetPreBuildCommands().begin(); + cr != target->GetPreBuildCommands().end(); ++cr) { depends.insert(depends.end(), cr->GetDepends().begin(), cr->GetDepends().end()); } for (std::vector<cmCustomCommand>::const_iterator cr = - target->Target->GetPostBuildCommands().begin(); - cr != target->Target->GetPostBuildCommands().end(); ++cr) + target->GetPostBuildCommands().begin(); + cr != target->GetPostBuildCommands().end(); ++cr) { depends.insert(depends.end(), cr->GetDepends().begin(), cr->GetDepends().end()); @@ -289,14 +289,14 @@ void cmLocalVisualStudio6Generator::WriteDSPFile(std::ostream& fout, // Add the pre- and post-build commands in order. int count = 1; for (std::vector<cmCustomCommand>::const_iterator cr = - target->Target->GetPreBuildCommands().begin(); - cr != target->Target->GetPreBuildCommands().end(); ++cr) + target->GetPreBuildCommands().begin(); + cr != target->GetPreBuildCommands().end(); ++cr) { this->AddUtilityCommandHack(target, count++, depends, *cr); } for (std::vector<cmCustomCommand>::const_iterator cr = - target->Target->GetPostBuildCommands().begin(); - cr != target->Target->GetPostBuildCommands().end(); ++cr) + target->GetPostBuildCommands().begin(); + cr != target->GetPostBuildCommands().end(); ++cr) { this->AddUtilityCommandHack(target, count++, depends, *cr); } @@ -838,8 +838,8 @@ cmLocalVisualStudio6Generator::CreateTargetRules(cmGeneratorTarget *target, // Write the pre-build and pre-link together (VS6 does not support both). event.Start("PreLink"); - event.Write(target->Target->GetPreBuildCommands()); - event.Write(target->Target->GetPreLinkCommands()); + event.Write(target->GetPreBuildCommands()); + event.Write(target->GetPreLinkCommands()); cmsys::auto_ptr<cmCustomCommand> pcc( this->MaybeCreateImplibDir(target, configName, false)); if(pcc.get()) @@ -855,7 +855,7 @@ cmLocalVisualStudio6Generator::CreateTargetRules(cmGeneratorTarget *target, // Write the post-build rules. event.Start("PostBuild"); - event.Write(target->Target->GetPostBuildCommands()); + event.Write(target->GetPostBuildCommands()); event.Finish(); customRuleCode += "# End Special Build Tool\n"; @@ -1052,7 +1052,7 @@ void cmLocalVisualStudio6Generator } std::vector<std::string>::const_iterator i; const std::vector<std::string>& libdirs = - target->Target->GetLinkDirectories(); + target->GetLinkDirectories(); for(i = libdirs.begin(); i != libdirs.end(); ++i) { std::string path = *i; @@ -1112,15 +1112,14 @@ void cmLocalVisualStudio6Generator // Compute the proper name to use to link this library. std::string lib; std::string libDebug; - cmTarget* tgt = this->GlobalGenerator->FindTarget(j->first.c_str()); + cmGeneratorTarget* tgt = + this->GlobalGenerator->FindGeneratorTarget(j->first.c_str()); if(tgt) { - cmGeneratorTarget* gt = - this->GlobalGenerator->GetGeneratorTarget(tgt); lib = cmSystemTools::GetFilenameWithoutExtension - (gt->GetFullName().c_str()); + (tgt->GetFullName().c_str()); libDebug = cmSystemTools::GetFilenameWithoutExtension - (gt->GetFullName("Debug").c_str()); + (tgt->GetFullName("Debug").c_str()); lib += ".lib"; libDebug += ".lib"; } diff --git a/Source/cmLocalVisualStudio7Generator.cxx b/Source/cmLocalVisualStudio7Generator.cxx index ca8d697..ae6a24e 100644 --- a/Source/cmLocalVisualStudio7Generator.cxx +++ b/Source/cmLocalVisualStudio7Generator.cxx @@ -783,10 +783,10 @@ void cmLocalVisualStudio7Generator::WriteConfiguration(std::ostream& fout, intermediateDir += "/"; intermediateDir += configName; - if (target->Target->GetType() < cmState::UTILITY) + if (target->GetType() < cmState::UTILITY) { std::string const& outDir = - target->Target->GetType() == cmState::OBJECT_LIBRARY? + target->GetType() == cmState::OBJECT_LIBRARY? intermediateDir : target->GetDirectory(configName); fout << "\t\t\tOutputDirectory=\"" << this->ConvertToXMLOutputPathSingle(outDir.c_str()) << "\"\n"; @@ -837,7 +837,7 @@ void cmLocalVisualStudio7Generator::WriteConfiguration(std::ostream& fout, if(this->FortranProject) { const char* target_mod_dir = - target->Target->GetProperty("Fortran_MODULE_DIRECTORY"); + target->GetProperty("Fortran_MODULE_DIRECTORY"); std::string modDir; if(target_mod_dir) { @@ -877,7 +877,7 @@ void cmLocalVisualStudio7Generator::WriteConfiguration(std::ostream& fout, targetOptions.OutputFlagMap(fout, "\t\t\t\t"); targetOptions.OutputPreprocessorDefinitions(fout, "\t\t\t\t", "\n", "CXX"); fout << "\t\t\t\tObjectFile=\"$(IntDir)\\\"\n"; - if(target->Target->GetType() <= cmState::OBJECT_LIBRARY) + if(target->GetType() <= cmState::OBJECT_LIBRARY) { // Specify the compiler program database file if configured. std::string pdb = target->GetCompilePDBPath(configName); @@ -2021,7 +2021,7 @@ void cmLocalVisualStudio7Generator const char* tool = this->FortranProject? "VFPreBuildEventTool":"VCPreBuildEventTool"; event.Start(tool); - event.Write(target->Target->GetPreBuildCommands()); + event.Write(target->GetPreBuildCommands()); event.Finish(); // Add pre-link event. @@ -2035,7 +2035,7 @@ void cmLocalVisualStudio7Generator { addedPrelink = true; std::vector<cmCustomCommand> commands = - target->Target->GetPreLinkCommands(); + target->GetPreLinkCommands(); cmGlobalVisualStudioGenerator* gg = static_cast<cmGlobalVisualStudioGenerator*>(this->GlobalGenerator); gg->AddSymbolExportCommand( @@ -2045,7 +2045,7 @@ void cmLocalVisualStudio7Generator } if (!addedPrelink) { - event.Write(target->Target->GetPreLinkCommands()); + event.Write(target->GetPreLinkCommands()); } cmsys::auto_ptr<cmCustomCommand> pcc( this->MaybeCreateImplibDir(target, @@ -2059,7 +2059,7 @@ void cmLocalVisualStudio7Generator // Add post-build event. tool = this->FortranProject? "VFPostBuildEventTool":"VCPostBuildEventTool"; event.Start(tool); - event.Write(target->Target->GetPostBuildCommands()); + event.Write(target->GetPostBuildCommands()); event.Finish(); } @@ -2219,17 +2219,18 @@ void cmLocalVisualStudio7Generator::WriteVCProjFooter( { fout << "\t<Globals>\n"; - cmPropertyMap const& props = target->Target->GetProperties(); - for(cmPropertyMap::const_iterator i = props.begin(); i != props.end(); ++i) + std::vector<std::string> const& props = target->GetPropertyKeys(); + for(std::vector<std::string>::const_iterator i = props.begin(); + i != props.end(); ++i) { - if(i->first.find("VS_GLOBAL_") == 0) + if(i->find("VS_GLOBAL_") == 0) { - std::string name = i->first.substr(10); + std::string name = i->substr(10); if(name != "") { fout << "\t\t<Global\n" << "\t\t\tName=\"" << name << "\"\n" - << "\t\t\tValue=\"" << i->second.GetValue() << "\"\n" + << "\t\t\tValue=\"" << target->GetProperty(*i) << "\"\n" << "\t\t/>\n"; } } diff --git a/Source/cmLocalXCodeGenerator.cxx b/Source/cmLocalXCodeGenerator.cxx index 3b430d2..aec2603 100644 --- a/Source/cmLocalXCodeGenerator.cxx +++ b/Source/cmLocalXCodeGenerator.cxx @@ -51,13 +51,11 @@ void cmLocalXCodeGenerator::Generate() { cmLocalGenerator::Generate(); - cmTargets& targets = this->Makefile->GetTargets(); - for(cmTargets::iterator iter = targets.begin(); + std::vector<cmGeneratorTarget*> targets = this->GetGeneratorTargets(); + for(std::vector<cmGeneratorTarget*>::iterator iter = targets.begin(); iter != targets.end(); ++iter) { - cmTarget* t = &iter->second; - cmGeneratorTarget* gt = this->GlobalGenerator->GetGeneratorTarget(t); - gt->HasMacOSXRpathInstallNameDir(""); + (*iter)->HasMacOSXRpathInstallNameDir(""); } } @@ -66,13 +64,11 @@ void cmLocalXCodeGenerator::GenerateInstallRules() { cmLocalGenerator::GenerateInstallRules(); - cmTargets& targets = this->Makefile->GetTargets(); - for(cmTargets::iterator iter = targets.begin(); + std::vector<cmGeneratorTarget*> targets = this->GetGeneratorTargets(); + for(std::vector<cmGeneratorTarget*>::iterator iter = targets.begin(); iter != targets.end(); ++iter) { - cmTarget* t = &iter->second; - cmGeneratorTarget* gt = this->GlobalGenerator->GetGeneratorTarget(t); - gt->HasMacOSXRpathInstallNameDir(""); + (*iter)->HasMacOSXRpathInstallNameDir(""); } } diff --git a/Source/cmMakefile.cxx b/Source/cmMakefile.cxx index 84ec4fb..4fb4579 100644 --- a/Source/cmMakefile.cxx +++ b/Source/cmMakefile.cxx @@ -57,31 +57,6 @@ cmMakefile::cmMakefile(cmGlobalGenerator* globalGenerator, // Setup the default include complaint regular expression (match nothing). this->ComplainFileRegularExpression = "^$"; - // Source and header file extensions that we can handle - - // Set up a list of source and header extensions - // these are used to find files when the extension - // is not given - // The "c" extension MUST precede the "C" extension. - this->SourceFileExtensions.push_back( "c" ); - this->SourceFileExtensions.push_back( "C" ); - - this->SourceFileExtensions.push_back( "c++" ); - this->SourceFileExtensions.push_back( "cc" ); - this->SourceFileExtensions.push_back( "cpp" ); - this->SourceFileExtensions.push_back( "cxx" ); - this->SourceFileExtensions.push_back( "m" ); - this->SourceFileExtensions.push_back( "M" ); - this->SourceFileExtensions.push_back( "mm" ); - - this->HeaderFileExtensions.push_back( "h" ); - this->HeaderFileExtensions.push_back( "hh" ); - this->HeaderFileExtensions.push_back( "h++" ); - this->HeaderFileExtensions.push_back( "hm" ); - this->HeaderFileExtensions.push_back( "hpp" ); - this->HeaderFileExtensions.push_back( "hxx" ); - this->HeaderFileExtensions.push_back( "in" ); - this->HeaderFileExtensions.push_back( "txx" ); this->DefineFlags = " "; @@ -756,15 +731,26 @@ void cmMakefile::ConfigureFinalPass() "with CMake 2.4 or later. For compatibility with older versions please " "use any CMake 2.8.x release or lower."); } - for (cmTargets::iterator l = this->Targets.begin(); - l != this->Targets.end(); l++) +#if defined(_WIN32) && !defined(__CYGWIN__) + // Do old-style link dependency analysis only for CM_USE_OLD_VS6. + if(this->GetGlobalGenerator()->IsForVS6()) { - if (l->second.GetType() == cmState::INTERFACE_LIBRARY) + for (cmTargets::iterator l = this->Targets.begin(); + l != this->Targets.end(); l++) { - continue; + if (l->second.GetType() == cmState::INTERFACE_LIBRARY) + { + continue; + } + // Erase any cached link information that might have been comptued + // on-demand during the configuration. This ensures that build + // system generation uses up-to-date information even if other cache + // invalidation code in this source file is buggy. + + l->second.AnalyzeLibDependenciesForVS6(*this); } - l->second.FinishConfigure(); } +#endif } //---------------------------------------------------------------------------- @@ -1780,6 +1766,18 @@ const char* cmMakefile::GetCurrentBinaryDirectory() const return this->StateSnapshot.GetDirectory().GetCurrentBinary(); } +std::vector<cmTarget*> cmMakefile::GetImportedTargets() const +{ + std::vector<cmTarget*> tgts; + tgts.reserve(this->ImportedTargets.size()); + for (TargetMap::const_iterator it = this->ImportedTargets.begin(); + it != this->ImportedTargets.end(); ++it) + { + tgts.push_back(it->second); + } + return tgts; +} + //---------------------------------------------------------------------------- void cmMakefile::AddIncludeDirectories(const std::vector<std::string> &incs, bool before) @@ -2050,10 +2048,11 @@ void cmMakefile::AddGlobalLinkInformation(const std::string& name, } -void cmMakefile::AddAlias(const std::string& lname, cmTarget *tgt) +void cmMakefile::AddAlias(const std::string& lname, + std::string const& tgtName) { - this->AliasTargets[lname] = tgt; - this->GetGlobalGenerator()->AddAlias(lname, tgt); + this->AliasTargets[lname] = tgtName; + this->GetGlobalGenerator()->AddAlias(lname, tgtName); } cmTarget* cmMakefile::AddLibrary(const std::string& lname, @@ -2110,7 +2109,6 @@ cmMakefile::AddNewTarget(cmState::TargetType type, const std::string& name) cmTarget& target = it->second; target.SetType(type, name); target.SetMakefile(this); - this->GetGlobalGenerator()->AddTarget(&it->second); return &it->second; } @@ -4039,10 +4037,12 @@ cmTarget* cmMakefile::FindTarget(const std::string& name, { if (!excludeAliases) { - TargetMap::const_iterator i = this->AliasTargets.find(name); + std::map<std::string, std::string>::const_iterator i = + this->AliasTargets.find(name); if (i != this->AliasTargets.end()) { - return i->second; + cmTargets::iterator ai = this->Targets.find(i->second); + return &ai->second; } } cmTargets::iterator i = this->Targets.find( name ); @@ -4197,15 +4197,11 @@ cmMakefile::AddImportedTarget(const std::string& name, // Create the target. cmsys::auto_ptr<cmTarget> target(new cmTarget); target->SetType(type, name); - target->MarkAsImported(); + target->MarkAsImported(global); target->SetMakefile(this); // Add to the set of available imported targets. this->ImportedTargets[name] = target.get(); - if(global) - { - this->GetGlobalGenerator()->AddTarget(target.get()); - } // Transfer ownership to this cmMakefile object. this->ImportedTargetsOwned.push_back(target.get()); diff --git a/Source/cmMakefile.h b/Source/cmMakefile.h index 725448b..f1dd374 100644 --- a/Source/cmMakefile.h +++ b/Source/cmMakefile.h @@ -283,7 +283,7 @@ public: cmTarget* AddLibrary(const std::string& libname, cmState::TargetType type, const std::vector<std::string> &srcs, bool excludeFromAll = false); - void AddAlias(const std::string& libname, cmTarget *tgt); + void AddAlias(const std::string& libname, const std::string& tgt); #if defined(CMAKE_BUILD_WITH_CMAKE) /** @@ -390,6 +390,7 @@ public: { return this->ImportedTargetsOwned; } + std::vector<cmTarget*> GetImportedTargets() const; cmTarget* FindTarget(const std::string& name, bool excludeAliases = false) const; @@ -400,6 +401,11 @@ public: bool excludeAliases = false) const; bool IsAlias(const std::string& name) const; + std::map<std::string, std::string> GetAliasTargets() const + { + return this->AliasTargets; + } + /** * Mark include directories as system directories. */ @@ -425,17 +431,6 @@ public: cmSourceFile* GetOrCreateSource(const std::string& sourceName, bool generated = false); - //@{ - /** - * Return a list of extensions associated with source and header - * files - */ - const std::vector<std::string>& GetSourceExtensions() const - {return this->SourceFileExtensions;} - const std::vector<std::string>& GetHeaderExtensions() const - {return this->HeaderFileExtensions;} - //@} - /** * Given a variable name, return its value (as a string). * If the variable is not found in this makefile instance, the @@ -801,7 +796,7 @@ protected: #else typedef std::map<std::string, cmTarget*> TargetMap; #endif - TargetMap AliasTargets; + std::map<std::string, std::string> AliasTargets; std::vector<cmSourceFile*> SourceFiles; // Tests @@ -823,8 +818,6 @@ protected: std::vector<cmTestGenerator*> TestGenerators; std::string ComplainFileRegularExpression; - std::vector<std::string> SourceFileExtensions; - std::vector<std::string> HeaderFileExtensions; std::string DefineFlags; // Track the value of the computed DEFINITIONS property. diff --git a/Source/cmMakefileExecutableTargetGenerator.cxx b/Source/cmMakefileExecutableTargetGenerator.cxx index f56037f..9e35e4c 100644 --- a/Source/cmMakefileExecutableTargetGenerator.cxx +++ b/Source/cmMakefileExecutableTargetGenerator.cxx @@ -279,11 +279,11 @@ void cmMakefileExecutableTargetGenerator::WriteExecutableRule(bool relink) { this->LocalGenerator ->AppendCustomCommands(commands, - this->GeneratorTarget->Target->GetPreBuildCommands(), + this->GeneratorTarget->GetPreBuildCommands(), this->GeneratorTarget); this->LocalGenerator ->AppendCustomCommands(commands, - this->GeneratorTarget->Target->GetPreLinkCommands(), + this->GeneratorTarget->GetPreLinkCommands(), this->GeneratorTarget); } @@ -450,7 +450,7 @@ void cmMakefileExecutableTargetGenerator::WriteExecutableRule(bool relink) { this->LocalGenerator-> AppendCustomCommands(commands, - this->GeneratorTarget->Target->GetPostBuildCommands(), + this->GeneratorTarget->GetPostBuildCommands(), this->GeneratorTarget); } diff --git a/Source/cmMakefileLibraryTargetGenerator.cxx b/Source/cmMakefileLibraryTargetGenerator.cxx index 0a65641..1923ea4 100644 --- a/Source/cmMakefileLibraryTargetGenerator.cxx +++ b/Source/cmMakefileLibraryTargetGenerator.cxx @@ -114,7 +114,7 @@ void cmMakefileLibraryTargetGenerator::WriteObjectLibraryRules() // Add post-build rules. this->LocalGenerator-> AppendCustomCommands(commands, - this->GeneratorTarget->Target->GetPostBuildCommands(), + this->GeneratorTarget->GetPostBuildCommands(), this->GeneratorTarget); // Depend on the object files. @@ -458,11 +458,11 @@ void cmMakefileLibraryTargetGenerator::WriteLibraryRules { this->LocalGenerator ->AppendCustomCommands(commands, - this->GeneratorTarget->Target->GetPreBuildCommands(), + this->GeneratorTarget->GetPreBuildCommands(), this->GeneratorTarget); this->LocalGenerator ->AppendCustomCommands(commands, - this->GeneratorTarget->Target->GetPreLinkCommands(), + this->GeneratorTarget->GetPreLinkCommands(), this->GeneratorTarget); } @@ -814,7 +814,7 @@ void cmMakefileLibraryTargetGenerator::WriteLibraryRules { this->LocalGenerator-> AppendCustomCommands(commands, - this->GeneratorTarget->Target->GetPostBuildCommands(), + this->GeneratorTarget->GetPostBuildCommands(), this->GeneratorTarget); } diff --git a/Source/cmMakefileUtilityTargetGenerator.cxx b/Source/cmMakefileUtilityTargetGenerator.cxx index d35c3ae..5b62cbf 100644 --- a/Source/cmMakefileUtilityTargetGenerator.cxx +++ b/Source/cmMakefileUtilityTargetGenerator.cxx @@ -67,20 +67,20 @@ void cmMakefileUtilityTargetGenerator::WriteRuleFiles() // Utility targets store their rules in pre- and post-build commands. this->LocalGenerator->AppendCustomDepends - (depends, this->GeneratorTarget->Target->GetPreBuildCommands()); + (depends, this->GeneratorTarget->GetPreBuildCommands()); this->LocalGenerator->AppendCustomDepends - (depends, this->GeneratorTarget->Target->GetPostBuildCommands()); + (depends, this->GeneratorTarget->GetPostBuildCommands()); this->LocalGenerator->AppendCustomCommands - (commands, this->GeneratorTarget->Target->GetPreBuildCommands(), + (commands, this->GeneratorTarget->GetPreBuildCommands(), this->GeneratorTarget); // Depend on all custom command outputs for sources this->DriveCustomCommands(depends); this->LocalGenerator->AppendCustomCommands - (commands, this->GeneratorTarget->Target->GetPostBuildCommands(), + (commands, this->GeneratorTarget->GetPostBuildCommands(), this->GeneratorTarget); // Add dependencies on targets that must be built first. diff --git a/Source/cmNinjaNormalTargetGenerator.cxx b/Source/cmNinjaNormalTargetGenerator.cxx index b533d37..17561b5 100644 --- a/Source/cmNinjaNormalTargetGenerator.cxx +++ b/Source/cmNinjaNormalTargetGenerator.cxx @@ -598,9 +598,9 @@ void cmNinjaNormalTargetGenerator::WriteLinkStatement() } const std::vector<cmCustomCommand> *cmdLists[3] = { - >.Target->GetPreBuildCommands(), - >.Target->GetPreLinkCommands(), - >.Target->GetPostBuildCommands() + >.GetPreBuildCommands(), + >.GetPreLinkCommands(), + >.GetPostBuildCommands() }; std::vector<std::string> preLinkCmdLines, postBuildCmdLines; diff --git a/Source/cmNinjaUtilityTargetGenerator.cxx b/Source/cmNinjaUtilityTargetGenerator.cxx index b2a5334..edc65f0 100644 --- a/Source/cmNinjaUtilityTargetGenerator.cxx +++ b/Source/cmNinjaUtilityTargetGenerator.cxx @@ -33,8 +33,8 @@ void cmNinjaUtilityTargetGenerator::Generate() cmNinjaDeps deps, outputs, util_outputs(1, utilCommandName); const std::vector<cmCustomCommand> *cmdLists[2] = { - &this->GetGeneratorTarget()->Target->GetPreBuildCommands(), - &this->GetGeneratorTarget()->Target->GetPostBuildCommands() + &this->GetGeneratorTarget()->GetPreBuildCommands(), + &this->GetGeneratorTarget()->GetPostBuildCommands() }; bool uses_terminal = false; diff --git a/Source/cmOSXBundleGenerator.cxx b/Source/cmOSXBundleGenerator.cxx index e9dc433..3c4aa00 100644 --- a/Source/cmOSXBundleGenerator.cxx +++ b/Source/cmOSXBundleGenerator.cxx @@ -83,7 +83,7 @@ void cmOSXBundleGenerator::CreateFramework( std::string newoutpath = outpath + "/" + this->GT->GetFrameworkDirectory(this->ConfigName, false); - std::string frameworkVersion = this->GT->Target->GetFrameworkVersion(); + std::string frameworkVersion = this->GT->GetFrameworkVersion(); // Configure the Info.plist file into the Resources directory. this->MacContentFolders->insert("Resources"); diff --git a/Source/cmQtAutoGeneratorInitializer.cxx b/Source/cmQtAutoGeneratorInitializer.cxx index 225c03e..b813c14 100644 --- a/Source/cmQtAutoGeneratorInitializer.cxx +++ b/Source/cmQtAutoGeneratorInitializer.cxx @@ -896,7 +896,6 @@ void cmQtAutoGeneratorInitializer::InitializeAutogenTarget( cmGeneratorTarget* gt = new cmGeneratorTarget(autogenTarget, lg); lg->AddGeneratorTarget(gt); - lg->GetGlobalGenerator()->AddGeneratorTarget(autogenTarget, gt); // Set target folder const char* autogenFolder = makefile->GetState() diff --git a/Source/cmQtAutoGenerators.cxx b/Source/cmQtAutoGenerators.cxx index fc690f8..b16eccd 100644 --- a/Source/cmQtAutoGenerators.cxx +++ b/Source/cmQtAutoGenerators.cxx @@ -528,7 +528,7 @@ bool cmQtAutoGenerators::RunAutogen(cmMakefile* makefile) cmSystemTools::ExpandListArgument(this->Sources, sourceFiles); const std::vector<std::string>& headerExtensions = - makefile->GetHeaderExtensions(); + makefile->GetCMakeInstance()->GetHeaderExtensions(); std::map<std::string, std::vector<std::string> > includedUis; std::map<std::string, std::vector<std::string> > skippedUis; diff --git a/Source/cmSourceFile.cxx b/Source/cmSourceFile.cxx index 37383be..a9ac549 100644 --- a/Source/cmSourceFile.cxx +++ b/Source/cmSourceFile.cxx @@ -167,8 +167,10 @@ bool cmSourceFile::FindFullPath(std::string* error) { tryDirs[0] = ""; } - const std::vector<std::string>& srcExts = mf->GetSourceExtensions(); - const std::vector<std::string>& hdrExts = mf->GetHeaderExtensions(); + const std::vector<std::string>& srcExts = + mf->GetCMakeInstance()->GetSourceExtensions(); + std::vector<std::string> hdrExts = + mf->GetCMakeInstance()->GetHeaderExtensions(); for(const char* const* di = tryDirs; *di; ++di) { std::string tryPath = this->Location.GetDirectory(); diff --git a/Source/cmSourceFileLocation.cxx b/Source/cmSourceFileLocation.cxx index b8d5c02..00d5d6a 100644 --- a/Source/cmSourceFileLocation.cxx +++ b/Source/cmSourceFileLocation.cxx @@ -121,8 +121,10 @@ void cmSourceFileLocation::UpdateExtension(const std::string& name) // The global generator checks extensions of enabled languages. cmGlobalGenerator* gg = this->Makefile->GetGlobalGenerator(); cmMakefile const* mf = this->Makefile; - const std::vector<std::string>& srcExts = mf->GetSourceExtensions(); - const std::vector<std::string>& hdrExts = mf->GetHeaderExtensions(); + const std::vector<std::string>& srcExts = + mf->GetCMakeInstance()->GetSourceExtensions(); + const std::vector<std::string>& hdrExts = + mf->GetCMakeInstance()->GetHeaderExtensions(); if(!gg->GetLanguageFromExtension(ext.c_str()).empty() || std::find(srcExts.begin(), srcExts.end(), ext) != srcExts.end() || std::find(hdrExts.begin(), hdrExts.end(), ext) != hdrExts.end()) @@ -193,12 +195,14 @@ cmSourceFileLocation // disk. One of these must match if loc refers to this source file. std::string const& ext = this->Name.substr(loc.Name.size()+1); cmMakefile const* mf = this->Makefile; - const std::vector<std::string>& srcExts = mf->GetSourceExtensions(); + const std::vector<std::string>& srcExts = + mf->GetCMakeInstance()->GetSourceExtensions(); if(std::find(srcExts.begin(), srcExts.end(), ext) != srcExts.end()) { return true; } - const std::vector<std::string>& hdrExts = mf->GetHeaderExtensions(); + std::vector<std::string> hdrExts = + mf->GetCMakeInstance()->GetHeaderExtensions(); if(std::find(hdrExts.begin(), hdrExts.end(), ext) != hdrExts.end()) { return true; diff --git a/Source/cmSystemTools.cxx b/Source/cmSystemTools.cxx index 2c5aa8a..b2827dc 100644 --- a/Source/cmSystemTools.cxx +++ b/Source/cmSystemTools.cxx @@ -29,6 +29,9 @@ # include "cmArchiveWrite.h" # include "cmLocale.h" # include <cm_libarchive.h> +# ifndef __LA_INT64_T +# define __LA_INT64_T la_int64_t +# endif #endif #include <cmsys/FStream.hxx> #include <cmsys/Terminal.h> diff --git a/Source/cmTarget.cxx b/Source/cmTarget.cxx index da49048..9ea1a34 100644 --- a/Source/cmTarget.cxx +++ b/Source/cmTarget.cxx @@ -62,6 +62,7 @@ cmTarget::cmTarget() this->DLLPlatform = false; this->IsAndroid = false; this->IsImportedTarget = false; + this->ImportedGloballyVisible = false; this->BuildInterfaceIncludesAppended = false; } @@ -312,23 +313,6 @@ cmListFileBacktrace const* cmTarget::GetUtilityBacktrace( } //---------------------------------------------------------------------------- -void cmTarget::FinishConfigure() -{ - // Erase any cached link information that might have been comptued - // on-demand during the configuration. This ensures that build - // system generation uses up-to-date information even if other cache - // invalidation code in this source file is buggy. - -#if defined(_WIN32) && !defined(__CYGWIN__) - // Do old-style link dependency analysis only for CM_USE_OLD_VS6. - if(this->Makefile->GetGlobalGenerator()->IsForVS6()) - { - this->AnalyzeLibDependenciesForVS6(*this->Makefile); - } -#endif -} - -//---------------------------------------------------------------------------- cmListFileBacktrace const& cmTarget::GetBacktrace() const { return this->Backtrace; @@ -1519,9 +1503,10 @@ void cmTarget::CheckProperty(const std::string& prop, } //---------------------------------------------------------------------------- -void cmTarget::MarkAsImported() +void cmTarget::MarkAsImported(bool global) { this->IsImportedTarget = true; + this->ImportedGloballyVisible = global; } //---------------------------------------------------------------------------- @@ -1614,7 +1599,7 @@ const char *cmTarget::GetProperty(const std::string& prop, // CMake time. cmGlobalGenerator* gg = this->Makefile->GetGlobalGenerator(); gg->CreateGenerationObjects(); - cmGeneratorTarget* gt = gg->GetGeneratorTarget(this); + cmGeneratorTarget* gt = gg->FindGeneratorTarget(this->GetName()); this->Properties.SetProperty(propLOCATION, gt->GetLocationForBuild()); } @@ -1639,7 +1624,7 @@ const char *cmTarget::GetProperty(const std::string& prop, { cmGlobalGenerator* gg = this->Makefile->GetGlobalGenerator(); gg->CreateGenerationObjects(); - cmGeneratorTarget* gt = gg->GetGeneratorTarget(this); + cmGeneratorTarget* gt = gg->FindGeneratorTarget(this->GetName()); this->Properties.SetProperty( prop, gt->GetFullPath(configName, false).c_str()); } @@ -1663,7 +1648,7 @@ const char *cmTarget::GetProperty(const std::string& prop, { cmGlobalGenerator* gg = this->Makefile->GetGlobalGenerator(); gg->CreateGenerationObjects(); - cmGeneratorTarget* gt = gg->GetGeneratorTarget(this); + cmGeneratorTarget* gt = gg->FindGeneratorTarget(this->GetName()); this->Properties.SetProperty( prop, gt->GetFullPath(configName, false).c_str()); } @@ -2045,25 +2030,6 @@ void cmTarget::SetPropertyDefault(const std::string& property, } } -//---------------------------------------------------------------------------- -std::string cmTarget::GetFrameworkVersion() const -{ - assert(this->GetType() != cmState::INTERFACE_LIBRARY); - - if(const char* fversion = this->GetProperty("FRAMEWORK_VERSION")) - { - return fversion; - } - else if(const char* tversion = this->GetProperty("VERSION")) - { - return tversion; - } - else - { - return "A"; - } -} - bool cmTarget::GetMappedConfig(std::string const& desired_config, const char** loc, const char** imp, diff --git a/Source/cmTarget.h b/Source/cmTarget.h index f3a45d3..97b0871 100644 --- a/Source/cmTarget.h +++ b/Source/cmTarget.h @@ -74,7 +74,7 @@ public: */ void SetType(cmState::TargetType f, const std::string& name); - void MarkAsImported(); + void MarkAsImported(bool global = false); ///! Set/Get the name of the target const std::string& GetName() const {return this->Name;} @@ -175,9 +175,6 @@ public: std::set<std::string>const& GetUtilities() const { return this->Utilities; } cmListFileBacktrace const* GetUtilityBacktrace(const std::string& u) const; - /** Finalize the target at the end of the Configure step. */ - void FinishConfigure(); - ///! Set/Get a property of this target file void SetProperty(const std::string& prop, const char *value); void AppendProperty(const std::string& prop, const char* value, @@ -188,6 +185,8 @@ public: void CheckProperty(const std::string& prop, cmMakefile* context) const; bool IsImported() const {return this->IsImportedTarget;} + bool IsImportedGloballyVisible() const + { return this->ImportedGloballyVisible; } // Get the properties cmPropertyMap &GetProperties() const { return this->Properties; } @@ -208,10 +207,6 @@ public: /** Return whether this target is an executable Bundle on Apple. */ bool IsAppBundleOnApple() const; - /** Return the framework version string. Undefined if - IsFrameworkOnApple returns false. */ - std::string GetFrameworkVersion() const; - /** Get a backtrace from the creation of the target. */ cmListFileBacktrace const& GetBacktrace() const; @@ -233,9 +228,6 @@ public: std::set<std::string> const & GetSystemIncludeDirectories() const { return this->SystemIncludeDirectories; } - bool LinkLanguagePropagatesToDependents() const - { return this->TargetTypeValue == cmState::STATIC_LIBRARY; } - cmStringRange GetIncludeDirectoriesEntries() const; cmBacktraceRange GetIncludeDirectoriesBacktraces() const; @@ -253,10 +245,11 @@ public: cmStringRange GetLinkImplementationEntries() const; cmBacktraceRange GetLinkImplementationBacktraces() const; - #if defined(_WIN32) && !defined(__CYGWIN__) const LinkLibraryVectorType &GetLinkLibrariesForVS6() const { return this->LinkLibrariesForVS6;} + + void AnalyzeLibDependenciesForVS6( const cmMakefile& mf ); #endif struct StrictTargetComparison { @@ -315,8 +308,6 @@ private: void GatherDependenciesForVS6( const cmMakefile& mf, const LibraryID& lib, DependencyMap& dep_map); - - void AnalyzeLibDependenciesForVS6( const cmMakefile& mf ); #endif const char* GetSuffixVariableInternal(bool implib) const; @@ -358,6 +349,7 @@ private: bool DLLPlatform; bool IsAndroid; bool IsImportedTarget; + bool ImportedGloballyVisible; bool BuildInterfaceIncludesAppended; #if defined(_WIN32) && !defined(__CYGWIN__) bool LinkLibrariesForVS6Analyzed; diff --git a/Source/cmVisualStudio10TargetGenerator.cxx b/Source/cmVisualStudio10TargetGenerator.cxx index 9c9f5ab..541e5f6 100644 --- a/Source/cmVisualStudio10TargetGenerator.cxx +++ b/Source/cmVisualStudio10TargetGenerator.cxx @@ -2814,7 +2814,7 @@ cmVisualStudio10TargetGenerator::WriteEvents(std::string const& configName) { addedPrelink = true; std::vector<cmCustomCommand> commands = - this->GeneratorTarget->Target->GetPreLinkCommands(); + this->GeneratorTarget->GetPreLinkCommands(); this->GlobalGenerator->AddSymbolExportCommand( this->GeneratorTarget, commands, configName); this->WriteEvent("PreLinkEvent", commands, configName); @@ -2823,12 +2823,12 @@ cmVisualStudio10TargetGenerator::WriteEvents(std::string const& configName) if (!addedPrelink) { this->WriteEvent("PreLinkEvent", - this->GeneratorTarget->Target->GetPreLinkCommands(), configName); + this->GeneratorTarget->GetPreLinkCommands(), configName); } this->WriteEvent("PreBuildEvent", - this->GeneratorTarget->Target->GetPreBuildCommands(), configName); + this->GeneratorTarget->GetPreBuildCommands(), configName); this->WriteEvent("PostBuildEvent", - this->GeneratorTarget->Target->GetPostBuildCommands(), configName); + this->GeneratorTarget->GetPostBuildCommands(), configName); } void cmVisualStudio10TargetGenerator::WriteEvent( diff --git a/Source/cmake.cxx b/Source/cmake.cxx index 7268241..d5bc3d2 100644 --- a/Source/cmake.cxx +++ b/Source/cmake.cxx @@ -165,6 +165,30 @@ cmake::cmake() // Make sure we can capture the build tool output. cmSystemTools::EnableVSConsoleOutput(); + + // Set up a list of source and header extensions + // these are used to find files when the extension + // is not given + // The "c" extension MUST precede the "C" extension. + this->SourceFileExtensions.push_back( "c" ); + this->SourceFileExtensions.push_back( "C" ); + + this->SourceFileExtensions.push_back( "c++" ); + this->SourceFileExtensions.push_back( "cc" ); + this->SourceFileExtensions.push_back( "cpp" ); + this->SourceFileExtensions.push_back( "cxx" ); + this->SourceFileExtensions.push_back( "m" ); + this->SourceFileExtensions.push_back( "M" ); + this->SourceFileExtensions.push_back( "mm" ); + + this->HeaderFileExtensions.push_back( "h" ); + this->HeaderFileExtensions.push_back( "hh" ); + this->HeaderFileExtensions.push_back( "h++" ); + this->HeaderFileExtensions.push_back( "hm" ); + this->HeaderFileExtensions.push_back( "hpp" ); + this->HeaderFileExtensions.push_back( "hxx" ); + this->HeaderFileExtensions.push_back( "in" ); + this->HeaderFileExtensions.push_back( "txx" ); } cmake::~cmake() @@ -423,8 +447,8 @@ bool cmake::FindPackage(const std::vector<std::string>& args) (cmSystemTools::GetCurrentWorkingDirectory()); // read in the list file to fill the cache snapshot.SetDefaultDefinitions(); - cmsys::auto_ptr<cmMakefile> mf(new cmMakefile(gg, snapshot)); - cmsys::auto_ptr<cmLocalGenerator> lg(gg->CreateLocalGenerator(mf.get())); + cmMakefile* mf = new cmMakefile(gg, snapshot); + gg->AddMakefile(mf); mf->SetArgcArgv(args); @@ -457,6 +481,8 @@ bool cmake::FindPackage(const std::vector<std::string>& args) std::vector<std::string> includeDirs; cmSystemTools::ExpandListArgument(includes, includeDirs); + gg->CreateGenerationObjects(); + cmLocalGenerator* lg = gg->LocalGenerators[0]; std::string includeFlags = lg->GetIncludeFlags(includeDirs, 0, language); std::string definitions = mf->GetSafeDefinition("PACKAGE_DEFINITIONS"); @@ -486,8 +512,9 @@ bool cmake::FindPackage(const std::vector<std::string>& args) std::string linkPath; std::string flags; std::string linkFlags; - gg->CreateGeneratorTargets(cmGlobalGenerator::AllTargets, lg.get()); - cmGeneratorTarget *gtgt = gg->GetGeneratorTarget(tgt); + gg->CreateGenerationObjects(); + cmGeneratorTarget *gtgt = gg->FindGeneratorTarget(tgt->GetName()); + cmLocalGenerator* lg = gtgt->GetLocalGenerator(); lg->GetTargetFlags(linkLibs, frameworkPath, linkPath, flags, linkFlags, gtgt, false); linkLibs = frameworkPath + linkPath + linkLibs; diff --git a/Source/cmake.h b/Source/cmake.h index 9d28cba..6b0e83f 100644 --- a/Source/cmake.h +++ b/Source/cmake.h @@ -182,6 +182,11 @@ class cmake ///! get the cmCachemManager used by this invocation of cmake cmCacheManager *GetCacheManager() { return this->CacheManager; } + const std::vector<std::string>& GetSourceExtensions() const + {return this->SourceFileExtensions;} + const std::vector<std::string>& GetHeaderExtensions() const + {return this->HeaderFileExtensions;} + /** * Given a variable name, return its value (as a string). */ @@ -391,6 +396,8 @@ private: std::string CheckStampFile; std::string CheckStampList; std::string VSSolutionFile; + std::vector<std::string> SourceFileExtensions; + std::vector<std::string> HeaderFileExtensions; bool ClearBuildSystem; bool DebugTryCompile; cmFileTimeComparison* FileComparison; diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index e2f380c..1b31726 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -71,6 +71,7 @@ add_test(NAME PassingTest COMMAND ${CMAKE_COMMAND} -E echo PassingTestOutput) add_test(NAME FailingTest COMMAND ${CMAKE_COMMAND} -E no_such_command) ]]) + unset(ENV{CTEST_PARALLEL_LEVEL}) run_ctest(TestOutputSize) endfunction() run_TestOutputSize() diff --git a/Tests/RunCMake/ctest_test/TestOutputSize-check.cmake b/Tests/RunCMake/ctest_test/TestOutputSize-check.cmake index 918d242..74ad669 100644 --- a/Tests/RunCMake/ctest_test/TestOutputSize-check.cmake +++ b/Tests/RunCMake/ctest_test/TestOutputSize-check.cmake @@ -4,14 +4,14 @@ if(test_xml_file) if("${test_xml}" MATCHES [[(<Test Status="passed">.*</Test>).*(<Test Status="failed">.*</Test>)]]) set(test_passed "${CMAKE_MATCH_1}") set(test_failed "${CMAKE_MATCH_2}") + if(NOT "${test_passed}" MATCHES [[<Value>PassingTes\.\.\..*10 bytes]]) + set(RunCMake_TEST_FAILED "Test.xml passed test output not truncated at 10 bytes:\n ${test_passed}") + elseif(NOT "${test_failed}" MATCHES [[<Value>CMake Error:\.\.\..*12 bytes]]) + set(RunCMake_TEST_FAILED "Test.xml failed test output not truncated at 12 bytes:\n ${test_failed}") + endif() else() set(RunCMake_TEST_FAILED "Test.xml does not contain a passed then failed test:\n ${test_xml}") endif() - if(NOT "${test_passed}" MATCHES [[<Value>PassingTes\.\.\..*10 bytes]]) - set(RunCMake_TEST_FAILED "Test.xml passed test output not truncated at 10 bytes:\n ${test_passed}") - elseif(NOT "${test_failed}" MATCHES [[<Value>CMake Error:\.\.\..*12 bytes]]) - set(RunCMake_TEST_FAILED "Test.xml failed test output not truncated at 12 bytes:\n ${test_failed}") - endif() else() set(RunCMake_TEST_FAILED "Test.xml not found") endif() diff --git a/Utilities/cmlibarchive/CMakeLists.txt b/Utilities/cmlibarchive/CMakeLists.txt index b150408..00550e2 100644 --- a/Utilities/cmlibarchive/CMakeLists.txt +++ b/Utilities/cmlibarchive/CMakeLists.txt @@ -8,6 +8,9 @@ endif() # On MacOS, prefer MacPorts libraries to system libraries. # I haven't come up with a compelling argument for this to be conditional. list(APPEND CMAKE_PREFIX_PATH /opt/local) +# Enable @rpath in the install name. +# detail in "cmake --help-policy CMP0042" +SET(CMAKE_MACOSX_RPATH ON) # # Version - read from 'version' file. @@ -71,9 +74,12 @@ OPTION(ENABLE_OPENSSL "Enable use of OpenSSL" ON) OPTION(ENABLE_LZMA "Enable the use of the system found LZMA library if found" ON) OPTION(ENABLE_ZLIB "Enable the use of the system found ZLIB library if found" ON) OPTION(ENABLE_BZip2 "Enable the use of the system found BZip2 library if found" ON) +OPTION(ENABLE_LIBXML2 "Enable the use of the system found libxml2 library if found" ON) OPTION(ENABLE_EXPAT "Enable the use of the system found EXPAT library if found" ON) OPTION(ENABLE_PCREPOSIX "Enable the use of the system found PCREPOSIX library if found" ON) OPTION(ENABLE_LibGCC "Enable the use of the system found LibGCC library if found" ON) +# CNG is used for encrypt/decrypt Zip archives on Windows. +OPTION(ENABLE_CNG "Enable the use of CNG(Crypto Next Generation)" ON) OPTION(ENABLE_XATTR "Enable extended attribute support" ON) OPTION(ENABLE_ACL "Enable ACL support" ON) @@ -88,6 +94,8 @@ IF(WIN32) SET(_WIN32_WINNT ${WINVER}) ENDIF(WIN32) +set(HAVE_PTHREAD_H 0) # no threads in CMake + IF("${CMAKE_C_PLATFORM_ID}" MATCHES "^(HP-UX)$") ADD_DEFINITIONS(-D_XOPEN_SOURCE=500) # Ask wchar.h for mbstate_t ENDIF() @@ -273,6 +281,8 @@ IF(BZIP2_FOUND) ENDIF(BZIP2_FOUND) MARK_AS_ADVANCED(CLEAR BZIP2_INCLUDE_DIR) MARK_AS_ADVANCED(CLEAR BZIP2_LIBRARIES) + + # # Find LZMA # @@ -335,6 +345,35 @@ ENDIF(LZO2_FOUND) MARK_AS_ADVANCED(CLEAR LZO2_INCLUDE_DIR) MARK_AS_ADVANCED(CLEAR LZO2_LIBRARY) ENDIF() +IF(0) # CMake does not need LZ4 support in libarchive +# +# Find LZ4 +# +IF (LZ4_INCLUDE_DIR) + # Already in cache, be silent + SET(LZ4_FIND_QUIETLY TRUE) +ENDIF (LZ4_INCLUDE_DIR) + +FIND_PATH(LZ4_INCLUDE_DIR lz4.h) +FIND_LIBRARY(LZ4_LIBRARY NAMES lz4 liblz4) +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(LZ4 DEFAULT_MSG LZ4_LIBRARY LZ4_INCLUDE_DIR) +IF(LZ4_FOUND) + SET(HAVE_LIBLZ4 1) + SET(HAVE_LZ4_H 1) + CMAKE_PUSH_CHECK_STATE() # Save the state of the variables + SET(CMAKE_REQUIRED_INCLUDES ${LZ4_INCLUDE_DIR}) + CHECK_INCLUDE_FILES("lz4hc.h" HAVE_LZ4HC_H) + CMAKE_POP_CHECK_STATE() # Restore the state of the variables + INCLUDE_DIRECTORIES(${LZ4_INCLUDE_DIR}) + LIST(APPEND ADDITIONAL_LIBS ${LZ4_LIBRARY}) + # + # TODO: test for static library. + # +ENDIF(LZ4_FOUND) +MARK_AS_ADVANCED(CLEAR LZ4_INCLUDE_DIR) +MARK_AS_ADVANCED(CLEAR LZ4_LIBRARY) +ENDIF() # # Check headers @@ -384,7 +423,9 @@ LA_CHECK_INCLUDE_FILE("memory.h" HAVE_MEMORY_H) LA_CHECK_INCLUDE_FILE("paths.h" HAVE_PATHS_H) LA_CHECK_INCLUDE_FILE("poll.h" HAVE_POLL_H) LA_CHECK_INCLUDE_FILE("process.h" HAVE_PROCESS_H) +LA_CHECK_INCLUDE_FILE("pthread.h" HAVE_PTHREAD_H) LA_CHECK_INCLUDE_FILE("pwd.h" HAVE_PWD_H) +LA_CHECK_INCLUDE_FILE("readpassphrase.h" HAVE_READPASSPHRASE_H) LA_CHECK_INCLUDE_FILE("regex.h" HAVE_REGEX_H) LA_CHECK_INCLUDE_FILE("signal.h" HAVE_SIGNAL_H) LA_CHECK_INCLUDE_FILE("spawn.h" HAVE_SPAWN_H) @@ -415,6 +456,11 @@ LA_CHECK_INCLUDE_FILE("utime.h" HAVE_UTIME_H) LA_CHECK_INCLUDE_FILE("wchar.h" HAVE_WCHAR_H) LA_CHECK_INCLUDE_FILE("wctype.h" HAVE_WCTYPE_H) LA_CHECK_INCLUDE_FILE("windows.h" HAVE_WINDOWS_H) +IF(ENABLE_CNG) + LA_CHECK_INCLUDE_FILE("Bcrypt.h" HAVE_BCRYPT_H) +ELSE(ENABLE_CNG) + UNSET(HAVE_BCRYPT_H CACHE) +ENDIF(ENABLE_CNG) # Following files need windwos.h, so we should test it after windows.h test. LA_CHECK_INCLUDE_FILE("wincrypt.h" HAVE_WINCRYPT_H) LA_CHECK_INCLUDE_FILE("winioctl.h" HAVE_WINIOCTL_H) @@ -446,6 +492,7 @@ IF(ENABLE_NETTLE) SET(HAVE_NETTLE_SHA_H 1) INCLUDE_DIRECTORIES(${NETTLE_INCLUDE_DIR}) LIST(APPEND ADDITIONAL_LIBS ${NETTLE_LIBRARIES}) + LA_CHECK_INCLUDE_FILE("nettle/pbkdf2.h" HAVE_NETTLE_PBKDF2_H) ENDIF(NETTLE_FOUND) MARK_AS_ADVANCED(CLEAR NETTLE_INCLUDE_DIR) MARK_AS_ADVANCED(CLEAR NETTLE_LIBRARIES) @@ -457,6 +504,9 @@ ENDIF(ENABLE_NETTLE) # IF(ENABLE_OPENSSL AND NOT CMAKE_SYSTEM_NAME MATCHES "Darwin") FIND_PACKAGE(OpenSSL) + IF(OPENSSL_FOUND) + SET(HAVE_LIBCRYPTO 1) + ENDIF(OPENSSL_FOUND) ELSE() SET(OPENSSL_FOUND FALSE) # Override cached value ENDIF() @@ -475,7 +525,7 @@ ENDIF(NOT OPENSSL_FOUND) # # How to prove that CRYPTO functions, which have several names on various -# platforms, just see if archive_crypto.c can compile and link against +# platforms, just see if archive_digest.c can compile and link against # required libraries. # MACRO(CHECK_CRYPTO ALGORITHMS IMPLEMENTATION) @@ -514,7 +564,7 @@ MACRO(CHECK_CRYPTO ALGORITHMS IMPLEMENTATION) ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/confdefs.h) FILE(READ "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/confdefs.h" CONFDEFS_H) - FILE(READ "${CMAKE_CURRENT_SOURCE_DIR}/libarchive/archive_crypto.c" + FILE(READ "${CMAKE_CURRENT_SOURCE_DIR}/libarchive/archive_digest.c" ARCHIVE_CRYPTO_C) SET(SOURCE "${CONFDEFS_H} @@ -664,14 +714,15 @@ ENDMACRO(CHECK_CRYPTO_WIN CRYPTO_LIST) MACRO(CHECK_ICONV LIB TRY_ICONV_CONST) IF(NOT HAVE_ICONV) CMAKE_PUSH_CHECK_STATE() # Save the state of the variables - IF (CMAKE_C_COMPILER_ID STREQUAL "GNU") + IF (CMAKE_C_COMPILER_ID STREQUAL "GNU" OR + CMAKE_C_COMPILER_ID STREQUAL "Clang") # # During checking iconv proto type, we should use -Werror to avoid the # success of iconv detection with a warnig which success is a miss # detection. So this needs for all build mode(even it's a release mode). # SET(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -Werror") - ENDIF (CMAKE_C_COMPILER_ID STREQUAL "GNU") + ENDIF () IF (MSVC) # NOTE: /WX option is the same as gcc's -Werror option. SET(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} /WX") @@ -795,7 +846,11 @@ IF(0) # CMake does not need XML support in libarchive # # Find Libxml2 # -FIND_PACKAGE(LibXml2) +IF(ENABLE_LIBXML2) + FIND_PACKAGE(LibXml2) +ELSE() + SET(LIBXML2_FOUND FALSE) +ENDIF() IF(LIBXML2_FOUND) CMAKE_PUSH_CHECK_STATE() # Save the state of the variables INCLUDE_DIRECTORIES(${LIBXML2_INCLUDE_DIR}) @@ -820,7 +875,11 @@ ELSE(LIBXML2_FOUND) # # Find Expat # - FIND_PACKAGE(EXPAT) + IF(ENABLE_EXPAT) + FIND_PACKAGE(EXPAT) + ELSE() + SET(EXPAT_FOUND FALSE) + ENDIF() IF(EXPAT_FOUND) CMAKE_PUSH_CHECK_STATE() # Save the state of the variables INCLUDE_DIRECTORIES(${EXPAT_INCLUDE_DIR}) @@ -838,7 +897,8 @@ ENDIF() # Check functions # CMAKE_PUSH_CHECK_STATE() # Save the state of the variables -IF (CMAKE_C_COMPILER_ID STREQUAL "GNU") +IF (CMAKE_C_COMPILER_ID STREQUAL "GNU" OR + CMAKE_C_COMPILER_ID STREQUAL "Clang") # # During checking functions, we should use -fno-builtin to avoid the # failure of function detection which failure is an error "conflicting @@ -847,6 +907,7 @@ IF (CMAKE_C_COMPILER_ID STREQUAL "GNU") SET(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -fno-builtin") ENDIF () CHECK_SYMBOL_EXISTS(_CrtSetReportMode "crtdbg.h" HAVE__CrtSetReportMode) +CHECK_FUNCTION_EXISTS_GLIBC(arc4random_buf HAVE_ARC4RANDOM_BUF) CHECK_FUNCTION_EXISTS_GLIBC(chflags HAVE_CHFLAGS) CHECK_FUNCTION_EXISTS_GLIBC(chown HAVE_CHOWN) CHECK_FUNCTION_EXISTS_GLIBC(chroot HAVE_CHROOT) @@ -894,6 +955,7 @@ CHECK_FUNCTION_EXISTS_GLIBC(pipe HAVE_PIPE) CHECK_FUNCTION_EXISTS_GLIBC(poll HAVE_POLL) CHECK_FUNCTION_EXISTS_GLIBC(posix_spawnp HAVE_POSIX_SPAWNP) CHECK_FUNCTION_EXISTS_GLIBC(readlink HAVE_READLINK) +CHECK_FUNCTION_EXISTS_GLIBC(readpassphrase HAVE_READPASSPHRASE) CHECK_FUNCTION_EXISTS_GLIBC(select HAVE_SELECT) CHECK_FUNCTION_EXISTS_GLIBC(setenv HAVE_SETENV) CHECK_FUNCTION_EXISTS_GLIBC(setlocale HAVE_SETLOCALE) @@ -932,6 +994,7 @@ CHECK_FUNCTION_EXISTS(strftime HAVE_STRFTIME) CHECK_FUNCTION_EXISTS(vprintf HAVE_VPRINTF) CHECK_FUNCTION_EXISTS(wmemcmp HAVE_WMEMCMP) CHECK_FUNCTION_EXISTS(wmemcpy HAVE_WMEMCPY) +CHECK_FUNCTION_EXISTS(wmemmove HAVE_WMEMMOVE) CMAKE_POP_CHECK_STATE() # Restore the state of the variables diff --git a/Utilities/cmlibarchive/COPYING b/Utilities/cmlibarchive/COPYING index b258806..93952b7 100644 --- a/Utilities/cmlibarchive/COPYING +++ b/Utilities/cmlibarchive/COPYING @@ -17,12 +17,11 @@ the actual statements in the files are controlling. files for details: libarchive/archive_entry.c libarchive/archive_read_support_filter_compress.c - libarchive/archive_write_set_filter_compress.c + libarchive/archive_write_add_filter_compress.c libarchive/mtree.5 - tar/matching.c * The following source files are in the public domain: - tar/getdate.c + libarchive/archive_getdate.c * The build files---including Makefiles, configure scripts, and auxiliary scripts used as part of the compile process---have diff --git a/Utilities/cmlibarchive/README-CMake.txt b/Utilities/cmlibarchive/README-CMake.txt index 8f3b29b..0a3e34a 100644 --- a/Utilities/cmlibarchive/README-CMake.txt +++ b/Utilities/cmlibarchive/README-CMake.txt @@ -11,7 +11,7 @@ branch, but it is merged into our history. Update libarchive from upstream as follows. Create a local branch to explicitly reference the upstream snapshot branch head: - git branch libarchive-upstream 37f225b7 + git branch libarchive-upstream 1a8c7bc2 Use a temporary directory to checkout the branch: @@ -24,7 +24,7 @@ Use a temporary directory to checkout the branch: Now place the (reduced) libarchive content in this directory. See instructions shown by - git log 37f225b7 + git log 1a8c7bc2 for help extracting the content from the upstream svn repo. Then run the following commands to commit the new version. Substitute the @@ -34,8 +34,8 @@ appropriate date and version number: GIT_AUTHOR_NAME='LibArchive Upstream' \ GIT_AUTHOR_EMAIL='libarchive-discuss@googlegroups.com' \ - GIT_AUTHOR_DATE='Mon Apr 14 19:19:05 2014 -0700' \ - git commit -m 'libarchive 3.1.2-246-ga5a5d28b (reduced)' && + GIT_AUTHOR_DATE='Wed Oct 21 01:47:34 2015 -0700' \ + git commit -m 'libarchive 3.1.2-601-g3bfe5f1 (reduced)' && git commit --amend Edit the commit message to describe the procedure used to obtain the diff --git a/Utilities/cmlibarchive/build/cmake/CreatePkgConfigFile.cmake b/Utilities/cmlibarchive/build/cmake/CreatePkgConfigFile.cmake index f96bbef..fc8529a 100644 --- a/Utilities/cmlibarchive/build/cmake/CreatePkgConfigFile.cmake +++ b/Utilities/cmlibarchive/build/cmake/CreatePkgConfigFile.cmake @@ -27,5 +27,7 @@ CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/pkgconfig/libarchive.pc.in ${CMAKE_CURRENT_SOURCE_DIR}/build/pkgconfig/libarchive.pc @ONLY) # And install it, of course ;). -INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/build/pkgconfig/libarchive.pc - DESTINATION "lib/pkgconfig") +IF(ENABLE_INSTALL) + INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/build/pkgconfig/libarchive.pc + DESTINATION "lib/pkgconfig") +ENDIF() diff --git a/Utilities/cmlibarchive/build/cmake/config.h.in b/Utilities/cmlibarchive/build/cmake/config.h.in index 32a29d0..1abeaaa 100644 --- a/Utilities/cmlibarchive/build/cmake/config.h.in +++ b/Utilities/cmlibarchive/build/cmake/config.h.in @@ -66,7 +66,7 @@ typedef long long int64_t; * Similarly for int32_t */ #if !defined(HAVE_INT32_T) && SIZE_OF_INT == 4 -typedef long int32_t; +typedef int int32_t; #define HAVE_INT32_T #endif @@ -331,9 +331,15 @@ typedef uint64_t uintmax_t; /* True for systems with POSIX ACL support */ #cmakedefine HAVE_ACL_USER 1 +/* Define to 1 if you have the `arc4random_buf' function. */ +#cmakedefine HAVE_ARC4RANDOM_BUF 1 + /* Define to 1 if you have the <attr/xattr.h> header file. */ #cmakedefine HAVE_ATTR_XATTR_H 1 +/* Define to 1 if you have the <Bcrypt.h> header file. */ +#cmakedefine HAVE_BCRYPT_H 1 + /* Define to 1 if you have the <bsdxml.h> header file. */ #cmakedefine HAVE_BSDXML_H 1 @@ -582,12 +588,21 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `bz2' library (-lbz2). */ #cmakedefine HAVE_LIBBZ2 1 +/* Define to 1 if you have the `charset' library (-lcharset). */ +#cmakedefine HAVE_LIBCHARSET 1 + +/* Define to 1 if you have the `crypto' library (-lcrypto). */ +#cmakedefine HAVE_LIBCRYPTO 1 + /* Define to 1 if you have the `expat' library (-lexpat). */ #cmakedefine HAVE_LIBEXPAT 1 /* Define to 1 if you have the `gcc' library (-lgcc). */ #cmakedefine HAVE_LIBGCC 1 +/* Define to 1 if you have the `lz4' library (-llz4). */ +#cmakedefine HAVE_LIBLZ4 1 + /* Define to 1 if you have the `lzma' library (-llzma). */ #cmakedefine HAVE_LIBLZMA 1 @@ -682,6 +697,12 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `lutimes' function. */ #cmakedefine HAVE_LUTIMES 1 +/* Define to 1 if you have the <lz4hc.h> header file. */ +#cmakedefine HAVE_LZ4HC_H 1 + +/* Define to 1 if you have the <lz4.h> header file. */ +#cmakedefine HAVE_LZ4_H 1 + /* Define to 1 if you have the <lzmadec.h> header file. */ #cmakedefine HAVE_LZMADEC_H 1 @@ -721,6 +742,9 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the <nettle/md5.h> header file. */ #cmakedefine HAVE_NETTLE_MD5_H 1 +/* Define to 1 if you have the <nettle/pbkdf2.h> header file. */ +#cmakedefine HAVE_NETTLE_PBKDF2_H 1 + /* Define to 1 if you have the <nettle/ripemd160.h> header file. */ #cmakedefine HAVE_NETTLE_RIPEMD160_H 1 @@ -742,6 +766,9 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `pipe' function. */ #cmakedefine HAVE_PIPE 1 +/* Define to 1 if you have the `PKCS5_PBKDF2_HMAC_SHA1' function. */ +#cmakedefine HAVE_PKCS5_PBKDF2_HMAC_SHA1 1 + /* Define to 1 if you have the `poll' function. */ #cmakedefine HAVE_POLL 1 @@ -754,6 +781,9 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the <process.h> header file. */ #cmakedefine HAVE_PROCESS_H 1 +/* Define to 1 if you have the <pthread.h> header file. */ +#cmakedefine HAVE_PTHREAD_H 1 + /* Define to 1 if you have the <pwd.h> header file. */ #cmakedefine HAVE_PWD_H 1 @@ -766,6 +796,12 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `readlinkat' function. */ #cmakedefine HAVE_READLINKAT 1 +/* Define to 1 if you have the `readpassphrase' function. */ +#cmakedefine HAVE_READPASSPHRASE 1 + +/* Define to 1 if you have the <readpassphrase.h> header file. */ +#cmakedefine HAVE_READPASSPHRASE_H 1 + /* Define to 1 if you have the <regex.h> header file. */ #cmakedefine HAVE_REGEX_H 1 @@ -1021,6 +1057,9 @@ typedef uint64_t uintmax_t; /* Define to 1 if you have the `wmemcpy' function. */ #cmakedefine HAVE_WMEMCPY 1 +/* Define to 1 if you have the `wmemmove' function. */ +#cmakedefine HAVE_WMEMMOVE 1 + /* Define to 1 if you have a working EXT2_IOC_GETFLAGS */ #cmakedefine HAVE_WORKING_EXT2_IOC_GETFLAGS 1 diff --git a/Utilities/cmlibarchive/libarchive/CMakeLists.txt b/Utilities/cmlibarchive/libarchive/CMakeLists.txt index 8908a62..891c728 100644 --- a/Utilities/cmlibarchive/libarchive/CMakeLists.txt +++ b/Utilities/cmlibarchive/libarchive/CMakeLists.txt @@ -18,8 +18,10 @@ SET(libarchive_SOURCES archive_cmdline.c archive_cmdline_private.h archive_crc32.h - archive_crypto.c - archive_crypto_private.h + archive_cryptor.c + archive_cryptor_private.h + archive_digest.c + archive_digest_private.h archive_endian.h archive_entry.c archive_entry.h @@ -32,6 +34,8 @@ SET(libarchive_SOURCES archive_entry_strmode.c archive_entry_xattr.c archive_getdate.c + archive_hmac.c + archive_hmac_private.h archive_match.c archive_options.c archive_options_private.h @@ -44,9 +48,12 @@ SET(libarchive_SOURCES archive_ppmd7.c archive_ppmd7_private.h archive_private.h + archive_random.c + archive_random_private.h archive_rb.c archive_rb.h archive_read.c + archive_read_add_passphrase.c archive_read_append_filter.c archive_read_data_into_fd.c archive_read_disk_entry_from_file.c @@ -68,6 +75,7 @@ SET(libarchive_SOURCES archive_read_support_filter_gzip.c archive_read_support_filter_grzip.c archive_read_support_filter_lrzip.c + archive_read_support_filter_lz4.c archive_read_support_filter_lzop.c archive_read_support_filter_none.c archive_read_support_filter_program.c @@ -87,6 +95,7 @@ SET(libarchive_SOURCES archive_read_support_format_rar.c archive_read_support_format_raw.c archive_read_support_format_tar.c + archive_read_support_format_warc.c archive_read_support_format_xar.c archive_read_support_format_zip.c archive_string.c @@ -113,6 +122,7 @@ SET(libarchive_SOURCES archive_write_add_filter_grzip.c archive_write_add_filter_gzip.c archive_write_add_filter_lrzip.c + archive_write_add_filter_lz4.c archive_write_add_filter_lzop.c archive_write_add_filter_none.c archive_write_add_filter_program.c @@ -124,6 +134,7 @@ SET(libarchive_SOURCES archive_write_set_format_by_name.c archive_write_set_format_cpio.c archive_write_set_format_cpio_newc.c + archive_write_set_format_filter_by_ext.c archive_write_set_format_gnutar.c archive_write_set_format_iso9660.c archive_write_set_format_mtree.c @@ -132,11 +143,15 @@ SET(libarchive_SOURCES archive_write_set_format_shar.c archive_write_set_format_ustar.c archive_write_set_format_v7tar.c + archive_write_set_format_warc.c archive_write_set_format_xar.c archive_write_set_format_zip.c archive_write_set_options.c + archive_write_set_passphrase.c + archive_xxhash.h filter_fork_posix.c filter_fork.h + xxhash.c ) # Man pages @@ -149,12 +164,14 @@ SET(libarchive_MANS archive_entry_stat.3 archive_entry_time.3 archive_read.3 + archive_read_add_passphrase.3 archive_read_disk.3 archive_read_set_options.3 archive_util.3 archive_write.3 archive_write_disk.3 archive_write_set_options.3 + archive_write_set_passphrase.3 cpio.5 libarchive.3 libarchive_internals.3 diff --git a/Utilities/cmlibarchive/libarchive/archive.h b/Utilities/cmlibarchive/libarchive/archive.h index 9cf762d..a8d4c6c 100644 --- a/Utilities/cmlibarchive/libarchive/archive.h +++ b/Utilities/cmlibarchive/libarchive/archive.h @@ -28,6 +28,16 @@ #ifndef ARCHIVE_H_INCLUDED #define ARCHIVE_H_INCLUDED +/* + * The version number is expressed as a single integer that makes it + * easy to compare versions at build time: for version a.b.c, the + * version number is printf("%d%03d%03d",a,b,c). For example, if you + * know your application requires version 2.12.108 or later, you can + * assert that ARCHIVE_VERSION_NUMBER >= 2012108. + */ +/* Note: Compiler will complain if this does not match archive_entry.h! */ +#define ARCHIVE_VERSION_NUMBER 3001002 + #include <sys/stat.h> #include <stddef.h> /* for wchar_t */ #include <stdio.h> /* For FILE * */ @@ -45,32 +55,44 @@ # include <inttypes.h> #endif -/* Borland symbols are case-insensitive. This workaround only works - within CMake because we do not mix compilers. */ -#if defined(__BORLANDC__) -# define archive_read_open_FILE archive_read_open_FILE_ -# define archive_write_open_FILE archive_write_open_FILE_ +/* Get appropriate definitions of 64-bit integer */ +#if !defined(__LA_INT64_T_DEFINED) +/* Older code relied on the __LA_INT64_T macro; after 4.0 we'll switch to the typedef exclusively. */ +# if ARCHIVE_VERSION_NUMBER < 4000000 +#define __LA_INT64_T la_int64_t +# endif +#define __LA_INT64_T_DEFINED +# if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__WATCOMC__) +typedef __int64 la_int64_t; +# else +# include <unistd.h> /* ssize_t */ +# if defined(_SCO_DS) || defined(__osf__) +typedef long long la_int64_t; +# else +typedef int64_t la_int64_t; +# endif +# endif #endif -/* Get appropriate definitions of standard POSIX-style types. */ -/* These should match the types used in 'struct stat' */ -#if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__WATCOMC__) -# define __LA_INT64_T __int64 -# if defined(_SSIZE_T_DEFINED) || defined(_SSIZE_T_) -# define __LA_SSIZE_T ssize_t -# elif defined(_WIN64) -# define __LA_SSIZE_T __int64 -# else -# define __LA_SSIZE_T long +/* The la_ssize_t should match the type used in 'struct stat' */ +#if !defined(__LA_SSIZE_T_DEFINED) +/* Older code relied on the __LA_SSIZE_T macro; after 4.0 we'll switch to the typedef exclusively. */ +# if ARCHIVE_VERSION_NUMBER < 4000000 +#define __LA_SSIZE_T la_ssize_t # endif -#else -# include <unistd.h> /* ssize_t, uid_t, and gid_t */ -# if defined(_SCO_DS) || defined(__osf__) -# define __LA_INT64_T long long +#define __LA_SSIZE_T_DEFINED +# if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__WATCOMC__) +# if defined(_SSIZE_T_DEFINED) || defined(_SSIZE_T_) +typedef ssize_t la_ssize_t; +# elif defined(_WIN64) +typedef __int64 la_ssize_t; +# else +typedef long la_ssize_t; +# endif # else -# define __LA_INT64_T int64_t +# include <unistd.h> /* ssize_t */ +typedef ssize_t la_ssize_t; # endif -# define __LA_SSIZE_T ssize_t #endif /* @@ -119,26 +141,21 @@ extern "C" { * header and library are very different, you should expect some * strangeness. Don't do that. */ - -/* - * The version number is expressed as a single integer that makes it - * easy to compare versions at build time: for version a.b.c, the - * version number is printf("%d%03d%03d",a,b,c). For example, if you - * know your application requires version 2.12.108 or later, you can - * assert that ARCHIVE_VERSION_NUMBER >= 2012108. - */ -/* Note: Compiler will complain if this does not match archive_entry.h! */ -#define ARCHIVE_VERSION_NUMBER 3001002 __LA_DECL int archive_version_number(void); /* * Textual name/version of the library, useful for version displays. */ -#define ARCHIVE_VERSION_STRING "libarchive 3.1.2" +#define ARCHIVE_VERSION_ONLY_STRING "3.1.2" +#define ARCHIVE_VERSION_STRING "libarchive " ARCHIVE_VERSION_ONLY_STRING __LA_DECL const char * archive_version_string(void); /* * Detailed textual name/version of the library and its dependencies. + * This has the form: + * "libarchive x.y.z zlib/a.b.c liblzma/d.e.f ... etc ..." + * the list of libraries described here will vary depending on how + * libarchive was compiled. */ __LA_DECL const char * archive_version_details(void); @@ -182,7 +199,7 @@ struct archive_entry; */ /* Returns pointer and size of next block of data from archive. */ -typedef __LA_SSIZE_T archive_read_callback(struct archive *, +typedef la_ssize_t archive_read_callback(struct archive *, void *_client_data, const void **_buffer); /* Skips at most request bytes from archive and returns the skipped amount. @@ -190,18 +207,18 @@ typedef __LA_SSIZE_T archive_read_callback(struct archive *, * If you do skip fewer bytes than requested, libarchive will invoke your * read callback and discard data as necessary to make up the full skip. */ -typedef __LA_INT64_T archive_skip_callback(struct archive *, - void *_client_data, __LA_INT64_T request); +typedef la_int64_t archive_skip_callback(struct archive *, + void *_client_data, la_int64_t request); /* Seeks to specified location in the file and returns the position. * Whence values are SEEK_SET, SEEK_CUR, SEEK_END from stdio.h. * Return ARCHIVE_FATAL if the seek fails for any reason. */ -typedef __LA_INT64_T archive_seek_callback(struct archive *, - void *_client_data, __LA_INT64_T offset, int whence); +typedef la_int64_t archive_seek_callback(struct archive *, + void *_client_data, la_int64_t offset, int whence); /* Returns size actually written, zero on EOF, -1 on error. */ -typedef __LA_SSIZE_T archive_write_callback(struct archive *, +typedef la_ssize_t archive_write_callback(struct archive *, void *_client_data, const void *_buffer, size_t _length); @@ -217,6 +234,13 @@ typedef int archive_switch_callback(struct archive *, void *_client_data1, void *_client_data2); /* + * Returns a passphrase used for encryption or decryption, NULL on nothing + * to do and give it up. + */ +typedef const char *archive_passphrase_callback(struct archive *, + void *_client_data); + +/* * Codes to identify various stream filters. */ #define ARCHIVE_FILTER_NONE 0 @@ -232,6 +256,7 @@ typedef int archive_switch_callback(struct archive *, void *_client_data1, #define ARCHIVE_FILTER_LRZIP 10 #define ARCHIVE_FILTER_LZOP 11 #define ARCHIVE_FILTER_GRZIP 12 +#define ARCHIVE_FILTER_LZ4 13 #if ARCHIVE_VERSION_NUMBER < 4000000 #define ARCHIVE_COMPRESSION_NONE ARCHIVE_FILTER_NONE @@ -293,6 +318,7 @@ typedef int archive_switch_callback(struct archive *, void *_client_data1, #define ARCHIVE_FORMAT_CAB 0xC0000 #define ARCHIVE_FORMAT_RAR 0xD0000 #define ARCHIVE_FORMAT_7ZIP 0xE0000 +#define ARCHIVE_FORMAT_WARC 0xF0000 /* * Codes returned by archive_read_format_capabilities(). @@ -375,6 +401,7 @@ __LA_DECL int archive_read_support_filter_compress(struct archive *); __LA_DECL int archive_read_support_filter_gzip(struct archive *); __LA_DECL int archive_read_support_filter_grzip(struct archive *); __LA_DECL int archive_read_support_filter_lrzip(struct archive *); +__LA_DECL int archive_read_support_filter_lz4(struct archive *); __LA_DECL int archive_read_support_filter_lzip(struct archive *); __LA_DECL int archive_read_support_filter_lzma(struct archive *); __LA_DECL int archive_read_support_filter_lzop(struct archive *); @@ -402,6 +429,7 @@ __LA_DECL int archive_read_support_format_mtree(struct archive *); __LA_DECL int archive_read_support_format_rar(struct archive *); __LA_DECL int archive_read_support_format_raw(struct archive *); __LA_DECL int archive_read_support_format_tar(struct archive *); +__LA_DECL int archive_read_support_format_warc(struct archive *); __LA_DECL int archive_read_support_format_xar(struct archive *); /* archive_read_support_format_zip() enables both streamable and seekable * zip readers. */ @@ -482,9 +510,9 @@ __LA_DECL int archive_read_open_file(struct archive *, const char *_filename, size_t _block_size) __LA_DEPRECATED; /* Read an archive that's stored in memory. */ __LA_DECL int archive_read_open_memory(struct archive *, - void * buff, size_t size); + const void * buff, size_t size); /* A more involved version that is only used for internal testing. */ -__LA_DECL int archive_read_open_memory2(struct archive *a, void *buff, +__LA_DECL int archive_read_open_memory2(struct archive *a, const void *buff, size_t size, size_t read_size); /* Read an archive that's already open, using the file descriptor. */ __LA_DECL int archive_read_open_fd(struct archive *, int _fd, @@ -505,7 +533,7 @@ __LA_DECL int archive_read_next_header2(struct archive *, * Retrieve the byte offset in UNCOMPRESSED data where last-read * header started. */ -__LA_DECL __LA_INT64_T archive_read_header_position(struct archive *); +__LA_DECL la_int64_t archive_read_header_position(struct archive *); /* * Returns 1 if the archive contains at least one encrypted entry. @@ -534,11 +562,11 @@ __LA_DECL int archive_read_has_encrypted_entries(struct archive *); __LA_DECL int archive_read_format_capabilities(struct archive *); /* Read data from the body of an entry. Similar to read(2). */ -__LA_DECL __LA_SSIZE_T archive_read_data(struct archive *, +__LA_DECL la_ssize_t archive_read_data(struct archive *, void *, size_t); /* Seek within the body of an entry. Similar to lseek(2). */ -__LA_DECL __LA_INT64_T archive_seek_data(struct archive *, __LA_INT64_T, int); +__LA_DECL la_int64_t archive_seek_data(struct archive *, la_int64_t, int); /* * A zero-copy version of archive_read_data that also exposes the file offset @@ -547,7 +575,7 @@ __LA_DECL __LA_INT64_T archive_seek_data(struct archive *, __LA_INT64_T, int); * be strictly increasing and that returned blocks will not overlap. */ __LA_DECL int archive_read_data_block(struct archive *a, - const void **buff, size_t *size, __LA_INT64_T *offset); + const void **buff, size_t *size, la_int64_t *offset); /*- * Some convenience functions that are built on archive_read_data: @@ -577,6 +605,14 @@ __LA_DECL int archive_read_set_option(struct archive *_a, __LA_DECL int archive_read_set_options(struct archive *_a, const char *opts); +/* + * Add a decryption passphrase. + */ +__LA_DECL int archive_read_add_passphrase(struct archive *, const char *); +__LA_DECL int archive_read_set_passphrase_callback(struct archive *, + void *client_data, archive_passphrase_callback *); + + /*- * Convenience function to recreate the current entry (whose header * has just been read) on disk. @@ -629,6 +665,10 @@ __LA_DECL int archive_read_set_options(struct archive *_a, /* Default: Do not use HFS+ compression if it was not compressed. */ /* This has no effect except on Mac OS v10.6 or later. */ #define ARCHIVE_EXTRACT_HFS_COMPRESSION_FORCED (0x8000) +/* Default: Do not reject entries with absolute paths */ +#define ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS (0x10000) +/* Default: Do not clear no-change flags when unlinking object */ +#define ARCHIVE_EXTRACT_CLEAR_NOCHANGE_FFLAGS (0x20000) __LA_DECL int archive_read_extract(struct archive *, struct archive_entry *, int flags); @@ -640,7 +680,7 @@ __LA_DECL void archive_read_extract_set_progress_callback(struct archive *, /* Record the dev/ino of a file that will not be written. This is * generally set to the dev/ino of the archive being read. */ __LA_DECL void archive_read_extract_set_skip_file(struct archive *, - __LA_INT64_T, __LA_INT64_T); + la_int64_t, la_int64_t); /* Close the file and release most resources. */ __LA_DECL int archive_read_close(struct archive *); @@ -679,7 +719,7 @@ __LA_DECL int archive_write_get_bytes_in_last_block(struct archive *); /* The dev/ino of a file that won't be archived. This is used * to avoid recursively adding an archive to itself. */ __LA_DECL int archive_write_set_skip_file(struct archive *, - __LA_INT64_T, __LA_INT64_T); + la_int64_t, la_int64_t); #if ARCHIVE_VERSION_NUMBER < 4000000 __LA_DECL int archive_write_set_compression_bzip2(struct archive *) @@ -710,6 +750,7 @@ __LA_DECL int archive_write_add_filter_compress(struct archive *); __LA_DECL int archive_write_add_filter_grzip(struct archive *); __LA_DECL int archive_write_add_filter_gzip(struct archive *); __LA_DECL int archive_write_add_filter_lrzip(struct archive *); +__LA_DECL int archive_write_add_filter_lz4(struct archive *); __LA_DECL int archive_write_add_filter_lzip(struct archive *); __LA_DECL int archive_write_add_filter_lzma(struct archive *); __LA_DECL int archive_write_add_filter_lzop(struct archive *); @@ -742,8 +783,11 @@ __LA_DECL int archive_write_set_format_shar(struct archive *); __LA_DECL int archive_write_set_format_shar_dump(struct archive *); __LA_DECL int archive_write_set_format_ustar(struct archive *); __LA_DECL int archive_write_set_format_v7tar(struct archive *); +__LA_DECL int archive_write_set_format_warc(struct archive *); __LA_DECL int archive_write_set_format_xar(struct archive *); __LA_DECL int archive_write_set_format_zip(struct archive *); +__LA_DECL int archive_write_set_format_filter_by_ext(struct archive *a, const char *filename); +__LA_DECL int archive_write_set_format_filter_by_ext_def(struct archive *a, const char *filename, const char * def_ext); __LA_DECL int archive_write_zip_set_compression_deflate(struct archive *); __LA_DECL int archive_write_zip_set_compression_store(struct archive *); __LA_DECL int archive_write_open(struct archive *, void *, @@ -768,12 +812,12 @@ __LA_DECL int archive_write_open_memory(struct archive *, */ __LA_DECL int archive_write_header(struct archive *, struct archive_entry *); -__LA_DECL __LA_SSIZE_T archive_write_data(struct archive *, +__LA_DECL la_ssize_t archive_write_data(struct archive *, const void *, size_t); /* This interface is currently only available for archive_write_disk handles. */ -__LA_DECL __LA_SSIZE_T archive_write_data_block(struct archive *, - const void *, size_t, __LA_INT64_T); +__LA_DECL la_ssize_t archive_write_data_block(struct archive *, + const void *, size_t, la_int64_t); __LA_DECL int archive_write_finish_entry(struct archive *); __LA_DECL int archive_write_close(struct archive *); @@ -808,6 +852,13 @@ __LA_DECL int archive_write_set_option(struct archive *_a, __LA_DECL int archive_write_set_options(struct archive *_a, const char *opts); +/* + * Set a encryption passphrase. + */ +__LA_DECL int archive_write_set_passphrase(struct archive *_a, const char *p); +__LA_DECL int archive_write_set_passphrase_callback(struct archive *, + void *client_data, archive_passphrase_callback *); + /*- * ARCHIVE_WRITE_DISK API * @@ -827,7 +878,7 @@ __LA_DECL int archive_write_set_options(struct archive *_a, __LA_DECL struct archive *archive_write_disk_new(void); /* This file will not be overwritten. */ __LA_DECL int archive_write_disk_set_skip_file(struct archive *, - __LA_INT64_T, __LA_INT64_T); + la_int64_t, la_int64_t); /* Set flags to control how the next item gets created. * This accepts a bitmask of ARCHIVE_EXTRACT_XXX flags defined above. */ __LA_DECL int archive_write_disk_set_options(struct archive *, @@ -857,14 +908,14 @@ __LA_DECL int archive_write_disk_set_standard_lookup(struct archive *); */ __LA_DECL int archive_write_disk_set_group_lookup(struct archive *, void * /* private_data */, - __LA_INT64_T (*)(void *, const char *, __LA_INT64_T), + la_int64_t (*)(void *, const char *, la_int64_t), void (* /* cleanup */)(void *)); __LA_DECL int archive_write_disk_set_user_lookup(struct archive *, void * /* private_data */, - __LA_INT64_T (*)(void *, const char *, __LA_INT64_T), + la_int64_t (*)(void *, const char *, la_int64_t), void (* /* cleanup */)(void *)); -__LA_DECL __LA_INT64_T archive_write_disk_gid(struct archive *, const char *, __LA_INT64_T); -__LA_DECL __LA_INT64_T archive_write_disk_uid(struct archive *, const char *, __LA_INT64_T); +__LA_DECL la_int64_t archive_write_disk_gid(struct archive *, const char *, la_int64_t); +__LA_DECL la_int64_t archive_write_disk_uid(struct archive *, const char *, la_int64_t); /* * ARCHIVE_READ_DISK API @@ -885,19 +936,19 @@ __LA_DECL int archive_read_disk_entry_from_file(struct archive *, struct archive_entry *, int /* fd */, const struct stat *); /* Look up gname for gid or uname for uid. */ /* Default implementations are very, very stupid. */ -__LA_DECL const char *archive_read_disk_gname(struct archive *, __LA_INT64_T); -__LA_DECL const char *archive_read_disk_uname(struct archive *, __LA_INT64_T); +__LA_DECL const char *archive_read_disk_gname(struct archive *, la_int64_t); +__LA_DECL const char *archive_read_disk_uname(struct archive *, la_int64_t); /* "Standard" implementation uses getpwuid_r, getgrgid_r and caches the * results for performance. */ __LA_DECL int archive_read_disk_set_standard_lookup(struct archive *); /* You can install your own lookups if you like. */ __LA_DECL int archive_read_disk_set_gname_lookup(struct archive *, void * /* private_data */, - const char *(* /* lookup_fn */)(void *, __LA_INT64_T), + const char *(* /* lookup_fn */)(void *, la_int64_t), void (* /* cleanup_fn */)(void *)); __LA_DECL int archive_read_disk_set_uname_lookup(struct archive *, void * /* private_data */, - const char *(* /* lookup_fn */)(void *, __LA_INT64_T), + const char *(* /* lookup_fn */)(void *, la_int64_t), void (* /* cleanup_fn */)(void *)); /* Start traversal. */ __LA_DECL int archive_read_disk_open(struct archive *, const char *); @@ -927,8 +978,10 @@ __LA_DECL int archive_read_disk_set_atime_restored(struct archive *); /* Default: Skip a mac resource fork file whose prefix is "._" because of * using copyfile. */ #define ARCHIVE_READDISK_MAC_COPYFILE (0x0004) -/* Default: Do not traverse mount points. */ +/* Default: Traverse mount points. */ #define ARCHIVE_READDISK_NO_TRAVERSE_MOUNTS (0x0008) +/* Default: Xattrs are read from disk. */ +#define ARCHIVE_READDISK_NO_XATTR (0x0010) __LA_DECL int archive_read_disk_set_behavior(struct archive *, int flags); @@ -961,7 +1014,7 @@ __LA_DECL int archive_free(struct archive *); * last filter, which is always the pseudo-filter that wraps the * client callbacks. */ __LA_DECL int archive_filter_count(struct archive *); -__LA_DECL __LA_INT64_T archive_filter_bytes(struct archive *, int); +__LA_DECL la_int64_t archive_filter_bytes(struct archive *, int); __LA_DECL int archive_filter_code(struct archive *, int); __LA_DECL const char * archive_filter_name(struct archive *, int); @@ -969,10 +1022,10 @@ __LA_DECL const char * archive_filter_name(struct archive *, int); /* These don't properly handle multiple filters, so are deprecated and * will eventually be removed. */ /* As of libarchive 3.0, this is an alias for archive_filter_bytes(a, -1); */ -__LA_DECL __LA_INT64_T archive_position_compressed(struct archive *) +__LA_DECL la_int64_t archive_position_compressed(struct archive *) __LA_DEPRECATED; /* As of libarchive 3.0, this is an alias for archive_filter_bytes(a, 0); */ -__LA_DECL __LA_INT64_T archive_position_uncompressed(struct archive *) +__LA_DECL la_int64_t archive_position_uncompressed(struct archive *) __LA_DEPRECATED; /* As of libarchive 3.0, this is an alias for archive_filter_name(a, 0); */ __LA_DECL const char *archive_compression_name(struct archive *) @@ -1088,8 +1141,8 @@ __LA_DECL int archive_match_exclude_entry(struct archive *, __LA_DECL int archive_match_owner_excluded(struct archive *, struct archive_entry *); /* Add inclusion uid, gid, uname and gname. */ -__LA_DECL int archive_match_include_uid(struct archive *, __LA_INT64_T); -__LA_DECL int archive_match_include_gid(struct archive *, __LA_INT64_T); +__LA_DECL int archive_match_include_uid(struct archive *, la_int64_t); +__LA_DECL int archive_match_include_gid(struct archive *, la_int64_t); __LA_DECL int archive_match_include_uname(struct archive *, const char *); __LA_DECL int archive_match_include_uname_w(struct archive *, const wchar_t *); @@ -1108,9 +1161,4 @@ __LA_DECL int archive_utility_string_sort(char **); /* These are meaningless outside of this header. */ #undef __LA_DECL -/* These need to remain defined because they're used in the - * callback type definitions. XXX Fix this. This is ugly. XXX */ -/* #undef __LA_INT64_T */ -/* #undef __LA_SSIZE_T */ - #endif /* !ARCHIVE_H_INCLUDED */ diff --git a/Utilities/cmlibarchive/libarchive/archive_cryptor.c b/Utilities/cmlibarchive/libarchive/archive_cryptor.c new file mode 100644 index 0000000..efd350d --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_cryptor.c @@ -0,0 +1,435 @@ +/*- +* Copyright (c) 2014 Michihiro NAKAJIMA +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR +* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "archive_platform.h" + +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#include "archive.h" +#include "archive_cryptor_private.h" + +#ifdef ARCHIVE_CRYPTOR_USE_Apple_CommonCrypto + +static int +pbkdf2_sha1(const char *pw, size_t pw_len, const uint8_t *salt, + size_t salt_len, unsigned rounds, uint8_t *derived_key, + size_t derived_key_len) +{ + CCKeyDerivationPBKDF(kCCPBKDF2, (const char *)pw, + pw_len, salt, salt_len, kCCPRFHmacAlgSHA1, rounds, + derived_key, derived_key_len); + return 0; +} + +#elif defined(_WIN32) && !defined(__CYGWIN__) && defined(HAVE_BCRYPT_H) +#ifdef _MSC_VER +#pragma comment(lib, "Bcrypt.lib") +#endif + +static int +pbkdf2_sha1(const char *pw, size_t pw_len, const uint8_t *salt, + size_t salt_len, unsigned rounds, uint8_t *derived_key, + size_t derived_key_len) +{ + NTSTATUS status; + BCRYPT_ALG_HANDLE hAlg; + + status = BCryptOpenAlgorithmProvider(&hAlg, BCRYPT_SHA1_ALGORITHM, + MS_PRIMITIVE_PROVIDER, BCRYPT_ALG_HANDLE_HMAC_FLAG); + if (!BCRYPT_SUCCESS(status)) + return -1; + + status = BCryptDeriveKeyPBKDF2(hAlg, + (PUCHAR)(uintptr_t)pw, (ULONG)pw_len, + (PUCHAR)(uintptr_t)salt, (ULONG)salt_len, rounds, + (PUCHAR)derived_key, (ULONG)derived_key_len, 0); + + BCryptCloseAlgorithmProvider(hAlg, 0); + + return (BCRYPT_SUCCESS(status)) ? 0: -1; +} + +#elif defined(HAVE_LIBNETTLE) && defined(HAVE_NETTLE_PBKDF2_H) + +static int +pbkdf2_sha1(const char *pw, size_t pw_len, const uint8_t *salt, + size_t salt_len, unsigned rounds, uint8_t *derived_key, + size_t derived_key_len) { + pbkdf2_hmac_sha1((unsigned)pw_len, (const uint8_t *)pw, rounds, + salt_len, salt, derived_key_len, derived_key); + return 0; +} + +#elif defined(HAVE_LIBCRYPTO) && defined(HAVE_PKCS5_PBKDF2_HMAC_SHA1) + +static int +pbkdf2_sha1(const char *pw, size_t pw_len, const uint8_t *salt, + size_t salt_len, unsigned rounds, uint8_t *derived_key, + size_t derived_key_len) { + + PKCS5_PBKDF2_HMAC_SHA1(pw, pw_len, salt, salt_len, rounds, + derived_key_len, derived_key); + return 0; +} + +#else + +/* Stub */ +static int +pbkdf2_sha1(const char *pw, size_t pw_len, const uint8_t *salt, + size_t salt_len, unsigned rounds, uint8_t *derived_key, + size_t derived_key_len) { + (void)pw; /* UNUSED */ + (void)pw_len; /* UNUSED */ + (void)salt; /* UNUSED */ + (void)salt_len; /* UNUSED */ + (void)rounds; /* UNUSED */ + (void)derived_key; /* UNUSED */ + (void)derived_key_len; /* UNUSED */ + return -1; /* UNSUPPORTED */ +} + +#endif + +#ifdef ARCHIVE_CRYPTOR_USE_Apple_CommonCrypto +# if MAC_OS_X_VERSION_MAX_ALLOWED < 1090 +# define kCCAlgorithmAES kCCAlgorithmAES128 +# endif + +static int +aes_ctr_init(archive_crypto_ctx *ctx, const uint8_t *key, size_t key_len) +{ + CCCryptorStatus r; + + ctx->key_len = key_len; + memcpy(ctx->key, key, key_len); + memset(ctx->nonce, 0, sizeof(ctx->nonce)); + ctx->encr_pos = AES_BLOCK_SIZE; + r = CCCryptorCreateWithMode(kCCEncrypt, kCCModeECB, kCCAlgorithmAES, + ccNoPadding, NULL, key, key_len, NULL, 0, 0, 0, &ctx->ctx); + return (r == kCCSuccess)? 0: -1; +} + +static int +aes_ctr_encrypt_counter(archive_crypto_ctx *ctx) +{ + CCCryptorRef ref = ctx->ctx; + CCCryptorStatus r; + + r = CCCryptorReset(ref, NULL); + if (r != kCCSuccess) + return -1; + r = CCCryptorUpdate(ref, ctx->nonce, AES_BLOCK_SIZE, ctx->encr_buf, + AES_BLOCK_SIZE, NULL); + return (r == kCCSuccess)? 0: -1; +} + +static int +aes_ctr_release(archive_crypto_ctx *ctx) +{ + memset(ctx->key, 0, ctx->key_len); + memset(ctx->nonce, 0, sizeof(ctx->nonce)); + return 0; +} + +#elif defined(_WIN32) && !defined(__CYGWIN__) && defined(HAVE_BCRYPT_H) + +static int +aes_ctr_init(archive_crypto_ctx *ctx, const uint8_t *key, size_t key_len) +{ + BCRYPT_ALG_HANDLE hAlg; + BCRYPT_KEY_HANDLE hKey; + DWORD keyObj_len, aes_key_len; + PBYTE keyObj; + ULONG result; + NTSTATUS status; + BCRYPT_KEY_LENGTHS_STRUCT key_lengths; + + ctx->hAlg = NULL; + ctx->hKey = NULL; + ctx->keyObj = NULL; + switch (key_len) { + case 16: aes_key_len = 128; break; + case 24: aes_key_len = 192; break; + case 32: aes_key_len = 256; break; + default: return -1; + } + status = BCryptOpenAlgorithmProvider(&hAlg, BCRYPT_AES_ALGORITHM, + MS_PRIMITIVE_PROVIDER, 0); + if (!BCRYPT_SUCCESS(status)) + return -1; + status = BCryptGetProperty(hAlg, BCRYPT_KEY_LENGTHS, (PUCHAR)&key_lengths, + sizeof(key_lengths), &result, 0); + if (!BCRYPT_SUCCESS(status)) { + BCryptCloseAlgorithmProvider(hAlg, 0); + return -1; + } + if (key_lengths.dwMinLength > aes_key_len + || key_lengths.dwMaxLength < aes_key_len) { + BCryptCloseAlgorithmProvider(hAlg, 0); + return -1; + } + status = BCryptGetProperty(hAlg, BCRYPT_OBJECT_LENGTH, (PUCHAR)&keyObj_len, + sizeof(keyObj_len), &result, 0); + if (!BCRYPT_SUCCESS(status)) { + BCryptCloseAlgorithmProvider(hAlg, 0); + return -1; + } + keyObj = (PBYTE)HeapAlloc(GetProcessHeap(), 0, keyObj_len); + if (keyObj == NULL) { + BCryptCloseAlgorithmProvider(hAlg, 0); + return -1; + } + status = BCryptSetProperty(hAlg, BCRYPT_CHAINING_MODE, + (PUCHAR)BCRYPT_CHAIN_MODE_ECB, sizeof(BCRYPT_CHAIN_MODE_ECB), 0); + if (!BCRYPT_SUCCESS(status)) { + BCryptCloseAlgorithmProvider(hAlg, 0); + HeapFree(GetProcessHeap(), 0, keyObj); + return -1; + } + status = BCryptGenerateSymmetricKey(hAlg, &hKey, + keyObj, keyObj_len, + (PUCHAR)(uintptr_t)key, (ULONG)key_len, 0); + if (!BCRYPT_SUCCESS(status)) { + BCryptCloseAlgorithmProvider(hAlg, 0); + HeapFree(GetProcessHeap(), 0, keyObj); + return -1; + } + + ctx->hAlg = hAlg; + ctx->hKey = hKey; + ctx->keyObj = keyObj; + ctx->keyObj_len = keyObj_len; + ctx->encr_pos = AES_BLOCK_SIZE; + + return 0; +} + +static int +aes_ctr_encrypt_counter(archive_crypto_ctx *ctx) +{ + NTSTATUS status; + ULONG result; + + status = BCryptEncrypt(ctx->hKey, (PUCHAR)ctx->nonce, AES_BLOCK_SIZE, + NULL, NULL, 0, (PUCHAR)ctx->encr_buf, AES_BLOCK_SIZE, + &result, 0); + return BCRYPT_SUCCESS(status) ? 0 : -1; +} + +static int +aes_ctr_release(archive_crypto_ctx *ctx) +{ + + if (ctx->hAlg != NULL) { + BCryptCloseAlgorithmProvider(ctx->hAlg, 0); + ctx->hAlg = NULL; + BCryptDestroyKey(ctx->hKey); + ctx->hKey = NULL; + HeapFree(GetProcessHeap(), 0, ctx->keyObj); + ctx->keyObj = NULL; + } + memset(ctx, 0, sizeof(*ctx)); + return 0; +} + +#elif defined(HAVE_LIBNETTLE) + +static int +aes_ctr_init(archive_crypto_ctx *ctx, const uint8_t *key, size_t key_len) +{ + ctx->key_len = key_len; + memcpy(ctx->key, key, key_len); + memset(ctx->nonce, 0, sizeof(ctx->nonce)); + ctx->encr_pos = AES_BLOCK_SIZE; + memset(&ctx->ctx, 0, sizeof(ctx->ctx)); + return 0; +} + +static int +aes_ctr_encrypt_counter(archive_crypto_ctx *ctx) +{ + aes_set_encrypt_key(&ctx->ctx, ctx->key_len, ctx->key); + aes_encrypt(&ctx->ctx, AES_BLOCK_SIZE, ctx->encr_buf, ctx->nonce); + return 0; +} + +static int +aes_ctr_release(archive_crypto_ctx *ctx) +{ + memset(ctx, 0, sizeof(*ctx)); + return 0; +} + +#elif defined(HAVE_LIBCRYPTO) + +static int +aes_ctr_init(archive_crypto_ctx *ctx, const uint8_t *key, size_t key_len) +{ + + switch (key_len) { + case 16: ctx->type = EVP_aes_128_ecb(); break; + case 24: ctx->type = EVP_aes_192_ecb(); break; + case 32: ctx->type = EVP_aes_256_ecb(); break; + default: ctx->type = NULL; return -1; + } + + ctx->key_len = key_len; + memcpy(ctx->key, key, key_len); + memset(ctx->nonce, 0, sizeof(ctx->nonce)); + ctx->encr_pos = AES_BLOCK_SIZE; + EVP_CIPHER_CTX_init(&ctx->ctx); + return 0; +} + +static int +aes_ctr_encrypt_counter(archive_crypto_ctx *ctx) +{ + int outl = 0; + int r; + + r = EVP_EncryptInit_ex(&ctx->ctx, ctx->type, NULL, ctx->key, NULL); + if (r == 0) + return -1; + r = EVP_EncryptUpdate(&ctx->ctx, ctx->encr_buf, &outl, ctx->nonce, + AES_BLOCK_SIZE); + if (r == 0 || outl != AES_BLOCK_SIZE) + return -1; + return 0; +} + +static int +aes_ctr_release(archive_crypto_ctx *ctx) +{ + EVP_CIPHER_CTX_cleanup(&ctx->ctx); + memset(ctx->key, 0, ctx->key_len); + memset(ctx->nonce, 0, sizeof(ctx->nonce)); + return 0; +} + +#else + +#define ARCHIVE_CRYPTOR_STUB +/* Stub */ +static int +aes_ctr_init(archive_crypto_ctx *ctx, const uint8_t *key, size_t key_len) +{ + (void)ctx; /* UNUSED */ + (void)key; /* UNUSED */ + (void)key_len; /* UNUSED */ + return -1; +} + +static int +aes_ctr_encrypt_counter(archive_crypto_ctx *ctx) +{ + (void)ctx; /* UNUSED */ + return -1; +} + +static int +aes_ctr_release(archive_crypto_ctx *ctx) +{ + (void)ctx; /* UNUSED */ + return 0; +} + +#endif + +#ifdef ARCHIVE_CRYPTOR_STUB +static int +aes_ctr_update(archive_crypto_ctx *ctx, const uint8_t * const in, + size_t in_len, uint8_t * const out, size_t *out_len) +{ + (void)ctx; /* UNUSED */ + (void)in; /* UNUSED */ + (void)in_len; /* UNUSED */ + (void)out; /* UNUSED */ + (void)out_len; /* UNUSED */ + aes_ctr_encrypt_counter(ctx); /* UNUSED */ /* Fix unused function warning */ + return -1; +} + +#else +static void +aes_ctr_increase_counter(archive_crypto_ctx *ctx) +{ + uint8_t *const nonce = ctx->nonce; + int j; + + for (j = 0; j < 8; j++) { + if (++nonce[j]) + break; + } +} + +static int +aes_ctr_update(archive_crypto_ctx *ctx, const uint8_t * const in, + size_t in_len, uint8_t * const out, size_t *out_len) +{ + uint8_t *const ebuf = ctx->encr_buf; + unsigned pos = ctx->encr_pos; + unsigned max = (unsigned)((in_len < *out_len)? in_len: *out_len); + unsigned i; + + for (i = 0; i < max; ) { + if (pos == AES_BLOCK_SIZE) { + aes_ctr_increase_counter(ctx); + if (aes_ctr_encrypt_counter(ctx) != 0) + return -1; + while (max -i >= AES_BLOCK_SIZE) { + for (pos = 0; pos < AES_BLOCK_SIZE; pos++) + out[i+pos] = in[i+pos] ^ ebuf[pos]; + i += AES_BLOCK_SIZE; + aes_ctr_increase_counter(ctx); + if (aes_ctr_encrypt_counter(ctx) != 0) + return -1; + } + pos = 0; + if (i >= max) + break; + } + out[i] = in[i] ^ ebuf[pos++]; + i++; + } + ctx->encr_pos = pos; + *out_len = i; + + return 0; +} +#endif /* ARCHIVE_CRYPTOR_STUB */ + + +const struct archive_cryptor __archive_cryptor = +{ + &pbkdf2_sha1, + &aes_ctr_init, + &aes_ctr_update, + &aes_ctr_release, + &aes_ctr_init, + &aes_ctr_update, + &aes_ctr_release, +}; diff --git a/Utilities/cmlibarchive/libarchive/archive_cryptor_private.h b/Utilities/cmlibarchive/libarchive/archive_cryptor_private.h new file mode 100644 index 0000000..9a96aee --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_cryptor_private.h @@ -0,0 +1,152 @@ +/*- +* Copyright (c) 2014 Michihiro NAKAJIMA +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR +* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef __LIBARCHIVE_BUILD +#error This header is only to be used internally to libarchive. +#endif + +#ifndef ARCHIVE_CRYPTOR_PRIVATE_H_INCLUDED +#define ARCHIVE_CRYPTOR_PRIVATE_H_INCLUDED + +#ifdef __APPLE__ +# include <AvailabilityMacros.h> +# if MAC_OS_X_VERSION_MAX_ALLOWED >= 1080 +# define ARCHIVE_CRYPTOR_USE_Apple_CommonCrypto +# endif +#endif + +#ifdef ARCHIVE_CRYPTOR_USE_Apple_CommonCrypto +#include <CommonCrypto/CommonCryptor.h> +#include <CommonCrypto/CommonKeyDerivation.h> +#define AES_BLOCK_SIZE 16 +#define AES_MAX_KEY_SIZE kCCKeySizeAES256 + +typedef struct { + CCCryptorRef ctx; + uint8_t key[AES_MAX_KEY_SIZE]; + unsigned key_len; + uint8_t nonce[AES_BLOCK_SIZE]; + uint8_t encr_buf[AES_BLOCK_SIZE]; + unsigned encr_pos; +} archive_crypto_ctx; + +#elif defined(_WIN32) && !defined(__CYGWIN__) && defined(HAVE_BCRYPT_H) +#include <Bcrypt.h> + +/* Common in other bcrypt implementations, but missing from VS2008. */ +#ifndef BCRYPT_SUCCESS +#define BCRYPT_SUCCESS(r) ((NTSTATUS)(r) == STATUS_SUCCESS) +#endif + +#define AES_MAX_KEY_SIZE 32 +#define AES_BLOCK_SIZE 16 +typedef struct { + BCRYPT_ALG_HANDLE hAlg; + BCRYPT_KEY_HANDLE hKey; + PBYTE keyObj; + DWORD keyObj_len; + uint8_t nonce[AES_BLOCK_SIZE]; + uint8_t encr_buf[AES_BLOCK_SIZE]; + unsigned encr_pos; +} archive_crypto_ctx; + +#elif defined(HAVE_LIBNETTLE) +#if defined(HAVE_NETTLE_PBKDF2_H) +#include <nettle/pbkdf2.h> +#endif +#include <nettle/aes.h> + +typedef struct { + struct aes_ctx ctx; + uint8_t key[AES_MAX_KEY_SIZE]; + unsigned key_len; + uint8_t nonce[AES_BLOCK_SIZE]; + uint8_t encr_buf[AES_BLOCK_SIZE]; + unsigned encr_pos; +} archive_crypto_ctx; + +#elif defined(HAVE_LIBCRYPTO) +#include <openssl/evp.h> +#define AES_BLOCK_SIZE 16 +#define AES_MAX_KEY_SIZE 32 + +typedef struct { + EVP_CIPHER_CTX ctx; + const EVP_CIPHER *type; + uint8_t key[AES_MAX_KEY_SIZE]; + unsigned key_len; + uint8_t nonce[AES_BLOCK_SIZE]; + uint8_t encr_buf[AES_BLOCK_SIZE]; + unsigned encr_pos; +} archive_crypto_ctx; + +#else + +#define AES_BLOCK_SIZE 16 +#define AES_MAX_KEY_SIZE 32 +typedef int archive_crypto_ctx; + +#endif + +/* defines */ +#define archive_pbkdf2_sha1(pw, pw_len, salt, salt_len, rounds, dk, dk_len)\ + __archive_cryptor.pbkdf2sha1(pw, pw_len, salt, salt_len, rounds, dk, dk_len) + +#define archive_decrypto_aes_ctr_init(ctx, key, key_len) \ + __archive_cryptor.decrypto_aes_ctr_init(ctx, key, key_len) +#define archive_decrypto_aes_ctr_update(ctx, in, in_len, out, out_len) \ + __archive_cryptor.decrypto_aes_ctr_update(ctx, in, in_len, out, out_len) +#define archive_decrypto_aes_ctr_release(ctx) \ + __archive_cryptor.decrypto_aes_ctr_release(ctx) + +#define archive_encrypto_aes_ctr_init(ctx, key, key_len) \ + __archive_cryptor.encrypto_aes_ctr_init(ctx, key, key_len) +#define archive_encrypto_aes_ctr_update(ctx, in, in_len, out, out_len) \ + __archive_cryptor.encrypto_aes_ctr_update(ctx, in, in_len, out, out_len) +#define archive_encrypto_aes_ctr_release(ctx) \ + __archive_cryptor.encrypto_aes_ctr_release(ctx) + +/* Minimal interface to cryptographic functionality for internal use in + * libarchive */ +struct archive_cryptor +{ + /* PKCS5 PBKDF2 HMAC-SHA1 */ + int (*pbkdf2sha1)(const char *pw, size_t pw_len, const uint8_t *salt, + size_t salt_len, unsigned rounds, uint8_t *derived_key, + size_t derived_key_len); + /* AES CTR mode(little endian version) */ + int (*decrypto_aes_ctr_init)(archive_crypto_ctx *, const uint8_t *, size_t); + int (*decrypto_aes_ctr_update)(archive_crypto_ctx *, const uint8_t *, + size_t, uint8_t *, size_t *); + int (*decrypto_aes_ctr_release)(archive_crypto_ctx *); + int (*encrypto_aes_ctr_init)(archive_crypto_ctx *, const uint8_t *, size_t); + int (*encrypto_aes_ctr_update)(archive_crypto_ctx *, const uint8_t *, + size_t, uint8_t *, size_t *); + int (*encrypto_aes_ctr_release)(archive_crypto_ctx *); +}; + +extern const struct archive_cryptor __archive_cryptor; + +#endif diff --git a/Utilities/cmlibarchive/libarchive/archive_crypto.c b/Utilities/cmlibarchive/libarchive/archive_digest.c index 85aba3a..f009d31 100644 --- a/Utilities/cmlibarchive/libarchive/archive_crypto.c +++ b/Utilities/cmlibarchive/libarchive/archive_digest.c @@ -28,7 +28,7 @@ #include "archive_platform.h" #include "archive.h" -#include "archive_crypto_private.h" +#include "archive_digest_private.h" /* In particular, force the configure probe to break if it tries * to test a combination of OpenSSL and libmd. */ @@ -1216,8 +1216,8 @@ __archive_stub_sha512final(archive_sha512_ctx *ctx, void *md) #endif -/* NOTE: Crypto functions are set based on availability and by the following - * order of preference. +/* NOTE: Message Digest functions are set based on availability and by the + * following order of preference. * 1. libc * 2. libc2 * 3. libc3 @@ -1227,7 +1227,7 @@ __archive_stub_sha512final(archive_sha512_ctx *ctx, void *md) * 7. libmd * 8. Windows API */ -const struct archive_crypto __archive_crypto = +const struct archive_digest __archive_digest = { /* MD5 */ #if defined(ARCHIVE_CRYPTO_MD5_LIBC) @@ -1412,7 +1412,7 @@ const struct archive_crypto __archive_crypto = #elif defined(ARCHIVE_CRYPTO_SHA512_NETTLE) &__archive_nettle_sha512init, &__archive_nettle_sha512update, - &__archive_nettle_sha512final, + &__archive_nettle_sha512final #elif defined(ARCHIVE_CRYPTO_SHA512_OPENSSL) &__archive_openssl_sha512init, &__archive_openssl_sha512update, diff --git a/Utilities/cmlibarchive/libarchive/archive_crypto_private.h b/Utilities/cmlibarchive/libarchive/archive_digest_private.h index f8b1fb3..77fad58 100644 --- a/Utilities/cmlibarchive/libarchive/archive_crypto_private.h +++ b/Utilities/cmlibarchive/libarchive/archive_digest_private.h @@ -264,11 +264,11 @@ typedef unsigned char archive_sha512_ctx; #define ARCHIVE_HAS_MD5 #endif #define archive_md5_init(ctx)\ - __archive_crypto.md5init(ctx) + __archive_digest.md5init(ctx) #define archive_md5_final(ctx, md)\ - __archive_crypto.md5final(ctx, md) + __archive_digest.md5final(ctx, md) #define archive_md5_update(ctx, buf, n)\ - __archive_crypto.md5update(ctx, buf, n) + __archive_digest.md5update(ctx, buf, n) #if defined(ARCHIVE_CRYPTO_RMD160_LIBC) ||\ defined(ARCHIVE_CRYPTO_RMD160_NETTLE) ||\ @@ -276,11 +276,11 @@ typedef unsigned char archive_sha512_ctx; #define ARCHIVE_HAS_RMD160 #endif #define archive_rmd160_init(ctx)\ - __archive_crypto.rmd160init(ctx) + __archive_digest.rmd160init(ctx) #define archive_rmd160_final(ctx, md)\ - __archive_crypto.rmd160final(ctx, md) + __archive_digest.rmd160final(ctx, md) #define archive_rmd160_update(ctx, buf, n)\ - __archive_crypto.rmd160update(ctx, buf, n) + __archive_digest.rmd160update(ctx, buf, n) #if defined(ARCHIVE_CRYPTO_SHA1_LIBC) ||\ defined(ARCHIVE_CRYPTO_SHA1_LIBMD) || \ @@ -291,11 +291,11 @@ typedef unsigned char archive_sha512_ctx; #define ARCHIVE_HAS_SHA1 #endif #define archive_sha1_init(ctx)\ - __archive_crypto.sha1init(ctx) + __archive_digest.sha1init(ctx) #define archive_sha1_final(ctx, md)\ - __archive_crypto.sha1final(ctx, md) + __archive_digest.sha1final(ctx, md) #define archive_sha1_update(ctx, buf, n)\ - __archive_crypto.sha1update(ctx, buf, n) + __archive_digest.sha1update(ctx, buf, n) #if defined(ARCHIVE_CRYPTO_SHA256_LIBC) ||\ defined(ARCHIVE_CRYPTO_SHA256_LIBC2) ||\ @@ -308,11 +308,11 @@ typedef unsigned char archive_sha512_ctx; #define ARCHIVE_HAS_SHA256 #endif #define archive_sha256_init(ctx)\ - __archive_crypto.sha256init(ctx) + __archive_digest.sha256init(ctx) #define archive_sha256_final(ctx, md)\ - __archive_crypto.sha256final(ctx, md) + __archive_digest.sha256final(ctx, md) #define archive_sha256_update(ctx, buf, n)\ - __archive_crypto.sha256update(ctx, buf, n) + __archive_digest.sha256update(ctx, buf, n) #if defined(ARCHIVE_CRYPTO_SHA384_LIBC) ||\ defined(ARCHIVE_CRYPTO_SHA384_LIBC2) ||\ @@ -324,11 +324,11 @@ typedef unsigned char archive_sha512_ctx; #define ARCHIVE_HAS_SHA384 #endif #define archive_sha384_init(ctx)\ - __archive_crypto.sha384init(ctx) + __archive_digest.sha384init(ctx) #define archive_sha384_final(ctx, md)\ - __archive_crypto.sha384final(ctx, md) + __archive_digest.sha384final(ctx, md) #define archive_sha384_update(ctx, buf, n)\ - __archive_crypto.sha384update(ctx, buf, n) + __archive_digest.sha384update(ctx, buf, n) #if defined(ARCHIVE_CRYPTO_SHA512_LIBC) ||\ defined(ARCHIVE_CRYPTO_SHA512_LIBC2) ||\ @@ -341,14 +341,14 @@ typedef unsigned char archive_sha512_ctx; #define ARCHIVE_HAS_SHA512 #endif #define archive_sha512_init(ctx)\ - __archive_crypto.sha512init(ctx) + __archive_digest.sha512init(ctx) #define archive_sha512_final(ctx, md)\ - __archive_crypto.sha512final(ctx, md) + __archive_digest.sha512final(ctx, md) #define archive_sha512_update(ctx, buf, n)\ - __archive_crypto.sha512update(ctx, buf, n) + __archive_digest.sha512update(ctx, buf, n) -/* Minimal interface to crypto functionality for internal use in libarchive */ -struct archive_crypto +/* Minimal interface to digest functionality for internal use in libarchive */ +struct archive_digest { /* Message Digest */ int (*md5init)(archive_md5_ctx *ctx); @@ -371,6 +371,6 @@ struct archive_crypto int (*sha512final)(archive_sha512_ctx *, void *); }; -extern const struct archive_crypto __archive_crypto; +extern const struct archive_digest __archive_digest; #endif diff --git a/Utilities/cmlibarchive/libarchive/archive_entry.3 b/Utilities/cmlibarchive/libarchive/archive_entry.3 index f77f385..f5e22af 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry.3 +++ b/Utilities/cmlibarchive/libarchive/archive_entry.3 @@ -131,11 +131,11 @@ be discarded in favor of the new data. .\" .Sh RETURN VALUES .\" .Sh ERRORS .Sh SEE ALSO -.Xr archive 3 , .Xr archive_entry_acl 3 , .Xr archive_entry_paths 3 , .Xr archive_entry_perms 3 , .Xr archive_entry_time 3 +.Xr libarchive 3 , .Sh HISTORY The .Nm libarchive diff --git a/Utilities/cmlibarchive/libarchive/archive_entry.c b/Utilities/cmlibarchive/libarchive/archive_entry.c index 293c701..4ac1966 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry.c +++ b/Utilities/cmlibarchive/libarchive/archive_entry.c @@ -418,6 +418,18 @@ archive_entry_gname(struct archive_entry *entry) return (NULL); } +const char * +archive_entry_gname_utf8(struct archive_entry *entry) +{ + const char *p; + if (archive_mstring_get_utf8(entry->archive, &entry->ae_gname, &p) == 0) + return (p); + if (errno == ENOMEM) + __archive_errx(1, "No memory"); + return (NULL); +} + + const wchar_t * archive_entry_gname_w(struct archive_entry *entry) { @@ -450,6 +462,20 @@ archive_entry_hardlink(struct archive_entry *entry) return (NULL); } +const char * +archive_entry_hardlink_utf8(struct archive_entry *entry) +{ + const char *p; + if ((entry->ae_set & AE_SET_HARDLINK) == 0) + return (NULL); + if (archive_mstring_get_utf8( + entry->archive, &entry->ae_hardlink, &p) == 0) + return (p); + if (errno == ENOMEM) + __archive_errx(1, "No memory"); + return (NULL); +} + const wchar_t * archive_entry_hardlink_w(struct archive_entry *entry) { @@ -536,6 +562,18 @@ archive_entry_pathname(struct archive_entry *entry) return (NULL); } +const char * +archive_entry_pathname_utf8(struct archive_entry *entry) +{ + const char *p; + if (archive_mstring_get_utf8( + entry->archive, &entry->ae_pathname, &p) == 0) + return (p); + if (errno == ENOMEM) + __archive_errx(1, "No memory"); + return (NULL); +} + const wchar_t * archive_entry_pathname_w(struct archive_entry *entry) { @@ -637,6 +675,20 @@ archive_entry_symlink(struct archive_entry *entry) return (NULL); } +const char * +archive_entry_symlink_utf8(struct archive_entry *entry) +{ + const char *p; + if ((entry->ae_set & AE_SET_SYMLINK) == 0) + return (NULL); + if (archive_mstring_get_utf8( + entry->archive, &entry->ae_symlink, &p) == 0) + return (p); + if (errno == ENOMEM) + __archive_errx(1, "No memory"); + return (NULL); +} + const wchar_t * archive_entry_symlink_w(struct archive_entry *entry) { @@ -680,6 +732,17 @@ archive_entry_uname(struct archive_entry *entry) return (NULL); } +const char * +archive_entry_uname_utf8(struct archive_entry *entry) +{ + const char *p; + if (archive_mstring_get_utf8(entry->archive, &entry->ae_uname, &p) == 0) + return (p); + if (errno == ENOMEM) + __archive_errx(1, "No memory"); + return (NULL); +} + const wchar_t * archive_entry_uname_w(struct archive_entry *entry) { @@ -769,6 +832,12 @@ archive_entry_set_gname(struct archive_entry *entry, const char *name) } void +archive_entry_set_gname_utf8(struct archive_entry *entry, const char *name) +{ + archive_mstring_copy_utf8(&entry->ae_gname, name); +} + +void archive_entry_copy_gname(struct archive_entry *entry, const char *name) { archive_mstring_copy_mbs(&entry->ae_gname, name); @@ -825,6 +894,16 @@ archive_entry_set_hardlink(struct archive_entry *entry, const char *target) } void +archive_entry_set_hardlink_utf8(struct archive_entry *entry, const char *target) +{ + archive_mstring_copy_utf8(&entry->ae_hardlink, target); + if (target != NULL) + entry->ae_set |= AE_SET_HARDLINK; + else + entry->ae_set &= ~AE_SET_HARDLINK; +} + +void archive_entry_copy_hardlink(struct archive_entry *entry, const char *target) { archive_mstring_copy_mbs(&entry->ae_hardlink, target); @@ -962,6 +1041,15 @@ archive_entry_set_link(struct archive_entry *entry, const char *target) archive_mstring_copy_mbs(&entry->ae_hardlink, target); } +void +archive_entry_set_link_utf8(struct archive_entry *entry, const char *target) +{ + if (entry->ae_set & AE_SET_SYMLINK) + archive_mstring_copy_utf8(&entry->ae_symlink, target); + else + archive_mstring_copy_utf8(&entry->ae_hardlink, target); +} + /* Set symlink if symlink is already set, else set hardlink. */ void archive_entry_copy_link(struct archive_entry *entry, const char *target) @@ -1052,6 +1140,12 @@ archive_entry_set_pathname(struct archive_entry *entry, const char *name) } void +archive_entry_set_pathname_utf8(struct archive_entry *entry, const char *name) +{ + archive_mstring_copy_utf8(&entry->ae_pathname, name); +} + +void archive_entry_copy_pathname(struct archive_entry *entry, const char *name) { archive_mstring_copy_mbs(&entry->ae_pathname, name); @@ -1152,6 +1246,16 @@ archive_entry_set_symlink(struct archive_entry *entry, const char *linkname) } void +archive_entry_set_symlink_utf8(struct archive_entry *entry, const char *linkname) +{ + archive_mstring_copy_utf8(&entry->ae_symlink, linkname); + if (linkname != NULL) + entry->ae_set |= AE_SET_SYMLINK; + else + entry->ae_set &= ~AE_SET_SYMLINK; +} + +void archive_entry_copy_symlink(struct archive_entry *entry, const char *linkname) { archive_mstring_copy_mbs(&entry->ae_symlink, linkname); @@ -1215,6 +1319,12 @@ archive_entry_set_uname(struct archive_entry *entry, const char *name) } void +archive_entry_set_uname_utf8(struct archive_entry *entry, const char *name) +{ + archive_mstring_copy_utf8(&entry->ae_uname, name); +} + +void archive_entry_copy_uname(struct archive_entry *entry, const char *name) { archive_mstring_copy_mbs(&entry->ae_uname, name); @@ -1629,19 +1739,23 @@ ae_strtofflags(const char *s, unsigned long *setp, unsigned long *clrp) while (*start == '\t' || *start == ' ' || *start == ',') start++; while (*start != '\0') { + size_t length; /* Locate end of token. */ end = start; while (*end != '\0' && *end != '\t' && *end != ' ' && *end != ',') end++; + length = end - start; for (flag = flags; flag->name != NULL; flag++) { - if (memcmp(start, flag->name, end - start) == 0) { + size_t flag_length = strlen(flag->name); + if (length == flag_length + && memcmp(start, flag->name, length) == 0) { /* Matched "noXXXX", so reverse the sense. */ clear |= flag->set; set |= flag->clear; break; - } else if (memcmp(start, flag->name + 2, end - start) - == 0) { + } else if (length == flag_length - 2 + && memcmp(start, flag->name + 2, length) == 0) { /* Matched "XXXX", so don't reverse. */ set |= flag->set; clear |= flag->clear; @@ -1693,19 +1807,23 @@ ae_wcstofflags(const wchar_t *s, unsigned long *setp, unsigned long *clrp) while (*start == L'\t' || *start == L' ' || *start == L',') start++; while (*start != L'\0') { + size_t length; /* Locate end of token. */ end = start; while (*end != L'\0' && *end != L'\t' && *end != L' ' && *end != L',') end++; + length = end - start; for (flag = flags; flag->wname != NULL; flag++) { - if (wmemcmp(start, flag->wname, end - start) == 0) { + size_t flag_length = wcslen(flag->wname); + if (length == flag_length + && wmemcmp(start, flag->wname, length) == 0) { /* Matched "noXXXX", so reverse the sense. */ clear |= flag->set; set |= flag->clear; break; - } else if (wmemcmp(start, flag->wname + 2, end - start) - == 0) { + } else if (length == flag_length - 2 + && wmemcmp(start, flag->wname + 2, length) == 0) { /* Matched "XXXX", so don't reverse. */ set |= flag->set; clear |= flag->clear; diff --git a/Utilities/cmlibarchive/libarchive/archive_entry.h b/Utilities/cmlibarchive/libarchive/archive_entry.h index efc4d73..68d8ab9 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry.h +++ b/Utilities/cmlibarchive/libarchive/archive_entry.h @@ -43,15 +43,25 @@ #include <stddef.h> /* for wchar_t */ #include <time.h> +#if defined(_WIN32) && !defined(__CYGWIN__) +#include <windows.h> +#endif + /* Get a suitable 64-bit integer type. */ -#if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__WATCOMC__) -# define __LA_INT64_T __int64 -#else -#include <unistd.h> -# if defined(_SCO_DS) || defined(__osf__) -# define __LA_INT64_T long long +#if !defined(__LA_INT64_T_DEFINED) +# if ARCHIVE_VERSION_NUMBER < 4000000 +#define __LA_INT64_T la_int64_t +# endif +#define __LA_INT64_T_DEFINED +# if defined(_WIN32) && !defined(__CYGWIN__) && !defined(__WATCOMC__) +typedef __int64 la_int64_t; # else -# define __LA_INT64_T int64_t +#include <unistd.h> +# if defined(_SCO_DS) || defined(__osf__) +typedef long long la_int64_t; +# else +typedef int64_t la_int64_t; +# endif # endif #endif @@ -202,13 +212,15 @@ __LA_DECL void archive_entry_fflags(struct archive_entry *, unsigned long * /* set */, unsigned long * /* clear */); __LA_DECL const char *archive_entry_fflags_text(struct archive_entry *); -__LA_DECL __LA_INT64_T archive_entry_gid(struct archive_entry *); +__LA_DECL la_int64_t archive_entry_gid(struct archive_entry *); __LA_DECL const char *archive_entry_gname(struct archive_entry *); +__LA_DECL const char *archive_entry_gname_utf8(struct archive_entry *); __LA_DECL const wchar_t *archive_entry_gname_w(struct archive_entry *); __LA_DECL const char *archive_entry_hardlink(struct archive_entry *); +__LA_DECL const char *archive_entry_hardlink_utf8(struct archive_entry *); __LA_DECL const wchar_t *archive_entry_hardlink_w(struct archive_entry *); -__LA_DECL __LA_INT64_T archive_entry_ino(struct archive_entry *); -__LA_DECL __LA_INT64_T archive_entry_ino64(struct archive_entry *); +__LA_DECL la_int64_t archive_entry_ino(struct archive_entry *); +__LA_DECL la_int64_t archive_entry_ino64(struct archive_entry *); __LA_DECL int archive_entry_ino_is_set(struct archive_entry *); __LA_DECL __LA_MODE_T archive_entry_mode(struct archive_entry *); __LA_DECL time_t archive_entry_mtime(struct archive_entry *); @@ -216,6 +228,7 @@ __LA_DECL long archive_entry_mtime_nsec(struct archive_entry *); __LA_DECL int archive_entry_mtime_is_set(struct archive_entry *); __LA_DECL unsigned int archive_entry_nlink(struct archive_entry *); __LA_DECL const char *archive_entry_pathname(struct archive_entry *); +__LA_DECL const char *archive_entry_pathname_utf8(struct archive_entry *); __LA_DECL const wchar_t *archive_entry_pathname_w(struct archive_entry *); __LA_DECL __LA_MODE_T archive_entry_perm(struct archive_entry *); __LA_DECL dev_t archive_entry_rdev(struct archive_entry *); @@ -223,13 +236,15 @@ __LA_DECL dev_t archive_entry_rdevmajor(struct archive_entry *); __LA_DECL dev_t archive_entry_rdevminor(struct archive_entry *); __LA_DECL const char *archive_entry_sourcepath(struct archive_entry *); __LA_DECL const wchar_t *archive_entry_sourcepath_w(struct archive_entry *); -__LA_DECL __LA_INT64_T archive_entry_size(struct archive_entry *); +__LA_DECL la_int64_t archive_entry_size(struct archive_entry *); __LA_DECL int archive_entry_size_is_set(struct archive_entry *); __LA_DECL const char *archive_entry_strmode(struct archive_entry *); __LA_DECL const char *archive_entry_symlink(struct archive_entry *); +__LA_DECL const char *archive_entry_symlink_utf8(struct archive_entry *); __LA_DECL const wchar_t *archive_entry_symlink_w(struct archive_entry *); -__LA_DECL __LA_INT64_T archive_entry_uid(struct archive_entry *); +__LA_DECL la_int64_t archive_entry_uid(struct archive_entry *); __LA_DECL const char *archive_entry_uname(struct archive_entry *); +__LA_DECL const char *archive_entry_uname_utf8(struct archive_entry *); __LA_DECL const wchar_t *archive_entry_uname_w(struct archive_entry *); __LA_DECL int archive_entry_is_data_encrypted(struct archive_entry *); __LA_DECL int archive_entry_is_metadata_encrypted(struct archive_entry *); @@ -265,18 +280,21 @@ __LA_DECL const char *archive_entry_copy_fflags_text(struct archive_entry *, const char *); __LA_DECL const wchar_t *archive_entry_copy_fflags_text_w(struct archive_entry *, const wchar_t *); -__LA_DECL void archive_entry_set_gid(struct archive_entry *, __LA_INT64_T); +__LA_DECL void archive_entry_set_gid(struct archive_entry *, la_int64_t); __LA_DECL void archive_entry_set_gname(struct archive_entry *, const char *); +__LA_DECL void archive_entry_set_gname_utf8(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_gname(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_gname_w(struct archive_entry *, const wchar_t *); __LA_DECL int archive_entry_update_gname_utf8(struct archive_entry *, const char *); __LA_DECL void archive_entry_set_hardlink(struct archive_entry *, const char *); +__LA_DECL void archive_entry_set_hardlink_utf8(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_hardlink(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_hardlink_w(struct archive_entry *, const wchar_t *); __LA_DECL int archive_entry_update_hardlink_utf8(struct archive_entry *, const char *); -__LA_DECL void archive_entry_set_ino(struct archive_entry *, __LA_INT64_T); -__LA_DECL void archive_entry_set_ino64(struct archive_entry *, __LA_INT64_T); +__LA_DECL void archive_entry_set_ino(struct archive_entry *, la_int64_t); +__LA_DECL void archive_entry_set_ino64(struct archive_entry *, la_int64_t); __LA_DECL void archive_entry_set_link(struct archive_entry *, const char *); +__LA_DECL void archive_entry_set_link_utf8(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_link(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_link_w(struct archive_entry *, const wchar_t *); __LA_DECL int archive_entry_update_link_utf8(struct archive_entry *, const char *); @@ -285,6 +303,7 @@ __LA_DECL void archive_entry_set_mtime(struct archive_entry *, time_t, long); __LA_DECL void archive_entry_unset_mtime(struct archive_entry *); __LA_DECL void archive_entry_set_nlink(struct archive_entry *, unsigned int); __LA_DECL void archive_entry_set_pathname(struct archive_entry *, const char *); +__LA_DECL void archive_entry_set_pathname_utf8(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_pathname(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_pathname_w(struct archive_entry *, const wchar_t *); __LA_DECL int archive_entry_update_pathname_utf8(struct archive_entry *, const char *); @@ -292,16 +311,18 @@ __LA_DECL void archive_entry_set_perm(struct archive_entry *, __LA_MODE_T); __LA_DECL void archive_entry_set_rdev(struct archive_entry *, dev_t); __LA_DECL void archive_entry_set_rdevmajor(struct archive_entry *, dev_t); __LA_DECL void archive_entry_set_rdevminor(struct archive_entry *, dev_t); -__LA_DECL void archive_entry_set_size(struct archive_entry *, __LA_INT64_T); +__LA_DECL void archive_entry_set_size(struct archive_entry *, la_int64_t); __LA_DECL void archive_entry_unset_size(struct archive_entry *); __LA_DECL void archive_entry_copy_sourcepath(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_sourcepath_w(struct archive_entry *, const wchar_t *); __LA_DECL void archive_entry_set_symlink(struct archive_entry *, const char *); +__LA_DECL void archive_entry_set_symlink_utf8(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_symlink(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_symlink_w(struct archive_entry *, const wchar_t *); __LA_DECL int archive_entry_update_symlink_utf8(struct archive_entry *, const char *); -__LA_DECL void archive_entry_set_uid(struct archive_entry *, __LA_INT64_T); +__LA_DECL void archive_entry_set_uid(struct archive_entry *, la_int64_t); __LA_DECL void archive_entry_set_uname(struct archive_entry *, const char *); +__LA_DECL void archive_entry_set_uname_utf8(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_uname(struct archive_entry *, const char *); __LA_DECL void archive_entry_copy_uname_w(struct archive_entry *, const wchar_t *); __LA_DECL int archive_entry_update_uname_utf8(struct archive_entry *, const char *); @@ -515,7 +536,7 @@ __LA_DECL int archive_entry_xattr_next(struct archive_entry *, __LA_DECL void archive_entry_sparse_clear(struct archive_entry *); __LA_DECL void archive_entry_sparse_add_entry(struct archive_entry *, - __LA_INT64_T /* offset */, __LA_INT64_T /* length */); + la_int64_t /* offset */, la_int64_t /* length */); /* * To retrieve the xattr list, first "reset", then repeatedly ask for the @@ -525,7 +546,7 @@ __LA_DECL void archive_entry_sparse_add_entry(struct archive_entry *, __LA_DECL int archive_entry_sparse_count(struct archive_entry *); __LA_DECL int archive_entry_sparse_reset(struct archive_entry *); __LA_DECL int archive_entry_sparse_next(struct archive_entry *, - __LA_INT64_T * /* offset */, __LA_INT64_T * /* length */); + la_int64_t * /* offset */, la_int64_t * /* length */); /* * Utility to match up hardlinks. diff --git a/Utilities/cmlibarchive/libarchive/archive_entry_acl.3 b/Utilities/cmlibarchive/libarchive/archive_entry_acl.3 index f5c3377..5aff996 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry_acl.3 +++ b/Utilities/cmlibarchive/libarchive/archive_entry_acl.3 @@ -226,8 +226,8 @@ The returned long string is valid until the next call to or .Fn archive_entry_acl_text_w . .Sh SEE ALSO -.Xr archive 3 , .Xr archive_entry 3 +.Xr libarchive 3 , .Sh BUGS .Dv ARCHIVE_ENTRY_ACL_STYLE_EXTRA_ID and diff --git a/Utilities/cmlibarchive/libarchive/archive_entry_paths.3 b/Utilities/cmlibarchive/libarchive/archive_entry_paths.3 index 51c8b8c..fd22cf7 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry_paths.3 +++ b/Utilities/cmlibarchive/libarchive/archive_entry_paths.3 @@ -149,5 +149,5 @@ It doesn't have a corresponding get accessor function. is an alias for .Fn archive_entry_copy_XXX . .Sh SEE ALSO -.Xr archive 3 , .Xr archive_entry 3 +.Xr libarchive 3 , diff --git a/Utilities/cmlibarchive/libarchive/archive_entry_perms.3 b/Utilities/cmlibarchive/libarchive/archive_entry_perms.3 index 5b7b5d9..340c5ea 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry_perms.3 +++ b/Utilities/cmlibarchive/libarchive/archive_entry_perms.3 @@ -194,11 +194,11 @@ every name that is recognized. .Xr strtofflags 3 , which stops parsing at the first unrecognized name.) .Sh SEE ALSO -.Xr archive 3 , .Xr archive_entry 3 , .Xr archive_entry_acl 3 , .Xr archive_read_disk 3 , .Xr archive_write_disk 3 +.Xr libarchive 3 , .Sh BUGS The platform types .Vt uid_t diff --git a/Utilities/cmlibarchive/libarchive/archive_entry_stat.3 b/Utilities/cmlibarchive/libarchive/archive_entry_stat.3 index 84a4ea1..26611e4 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry_stat.3 +++ b/Utilities/cmlibarchive/libarchive/archive_entry_stat.3 @@ -226,7 +226,7 @@ and are used by .Xr archive_entry_linkify 3 to find hardlinks. -The pair of device and inode is suppossed to identify hardlinked files. +The pair of device and inode is supposed to identify hardlinked files. .Pp The device major and minor number can be obtained independently using .Fn archive_entry_devmajor @@ -267,8 +267,8 @@ platforms. Some archive formats use the combined form, while other formats use the split form. .Sh SEE ALSO -.Xr archive 3 , .Xr archive_entry_acl 3 , .Xr archive_entry_perms 3 , .Xr archive_entry_time 3 , +.Xr libarchive 3 , .Xr stat 2 diff --git a/Utilities/cmlibarchive/libarchive/archive_entry_time.3 b/Utilities/cmlibarchive/libarchive/archive_entry_time.3 index 17c658a..1864521 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry_time.3 +++ b/Utilities/cmlibarchive/libarchive/archive_entry_time.3 @@ -113,8 +113,8 @@ The current state can be queried using .Fn XXX_is_set . Unset time fields have a second and nanosecond field of 0. .Sh SEE ALSO -.Xr archive 3 , .Xr archive_entry 3 +.Xr libarchive 3 , .Sh HISTORY The .Nm libarchive diff --git a/Utilities/cmlibarchive/libarchive/archive_entry_xattr.c b/Utilities/cmlibarchive/libarchive/archive_entry_xattr.c index a3efe7c..05eb90f 100644 --- a/Utilities/cmlibarchive/libarchive/archive_entry_xattr.c +++ b/Utilities/cmlibarchive/libarchive/archive_entry_xattr.c @@ -98,7 +98,10 @@ archive_entry_xattr_add_entry(struct archive_entry *entry, /* XXX Error XXX */ return; - xp->name = strdup(name); + if ((xp->name = strdup(name)) == NULL) + /* XXX Error XXX */ + return; + if ((xp->value = malloc(size)) != NULL) { memcpy(xp->value, value, size); xp->size = size; diff --git a/Utilities/cmlibarchive/libarchive/archive_hmac.c b/Utilities/cmlibarchive/libarchive/archive_hmac.c new file mode 100644 index 0000000..36e3e1c --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_hmac.c @@ -0,0 +1,234 @@ +/*- +* Copyright (c) 2014 Michihiro NAKAJIMA +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR +* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "archive_platform.h" + +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#include "archive.h" +#include "archive_hmac_private.h" + +#ifdef ARCHIVE_HMAC_USE_Apple_CommonCrypto + +static int +__hmac_sha1_init(archive_hmac_sha1_ctx *ctx, const uint8_t *key, size_t key_len) +{ + CCHmacInit(ctx, kCCHmacAlgSHA1, key, key_len); + return 0; +} + +static void +__hmac_sha1_update(archive_hmac_sha1_ctx *ctx, const uint8_t *data, + size_t data_len) +{ + CCHmacUpdate(ctx, data, data_len); +} + +static void +__hmac_sha1_final(archive_hmac_sha1_ctx *ctx, uint8_t *out, size_t *out_len) +{ + CCHmacFinal(ctx, out); + *out_len = 20; +} + +static void +__hmac_sha1_cleanup(archive_hmac_sha1_ctx *ctx) +{ + memset(ctx, 0, sizeof(*ctx)); +} + +#elif defined(_WIN32) && !defined(__CYGWIN__) && defined(HAVE_BCRYPT_H) + +static int +__hmac_sha1_init(archive_hmac_sha1_ctx *ctx, const uint8_t *key, size_t key_len) +{ + BCRYPT_ALG_HANDLE hAlg; + BCRYPT_HASH_HANDLE hHash; + DWORD hash_len; + PBYTE hash; + ULONG result; + NTSTATUS status; + + ctx->hAlg = NULL; + status = BCryptOpenAlgorithmProvider(&hAlg, BCRYPT_SHA1_ALGORITHM, + MS_PRIMITIVE_PROVIDER, BCRYPT_ALG_HANDLE_HMAC_FLAG); + if (!BCRYPT_SUCCESS(status)) + return -1; + status = BCryptGetProperty(hAlg, BCRYPT_HASH_LENGTH, (PUCHAR)&hash_len, + sizeof(hash_len), &result, 0); + if (!BCRYPT_SUCCESS(status)) { + BCryptCloseAlgorithmProvider(hAlg, 0); + return -1; + } + hash = (PBYTE)HeapAlloc(GetProcessHeap(), 0, hash_len); + if (hash == NULL) { + BCryptCloseAlgorithmProvider(hAlg, 0); + return -1; + } + status = BCryptCreateHash(hAlg, &hHash, NULL, 0, + (PUCHAR)key, (ULONG)key_len, BCRYPT_HASH_REUSABLE_FLAG); + if (!BCRYPT_SUCCESS(status)) { + BCryptCloseAlgorithmProvider(hAlg, 0); + HeapFree(GetProcessHeap(), 0, hash); + return -1; + } + + ctx->hAlg = hAlg; + ctx->hHash = hHash; + ctx->hash_len = hash_len; + ctx->hash = hash; + + return 0; +} + +static void +__hmac_sha1_update(archive_hmac_sha1_ctx *ctx, const uint8_t *data, + size_t data_len) +{ + BCryptHashData(ctx->hHash, (PUCHAR)(uintptr_t)data, (ULONG)data_len, 0); +} + +static void +__hmac_sha1_final(archive_hmac_sha1_ctx *ctx, uint8_t *out, size_t *out_len) +{ + BCryptFinishHash(ctx->hHash, ctx->hash, ctx->hash_len, 0); + if (ctx->hash_len == *out_len) + memcpy(out, ctx->hash, *out_len); +} + +static void +__hmac_sha1_cleanup(archive_hmac_sha1_ctx *ctx) +{ + if (ctx->hAlg != NULL) { + BCryptCloseAlgorithmProvider(ctx->hAlg, 0); + HeapFree(GetProcessHeap(), 0, ctx->hash); + ctx->hAlg = NULL; + } +} + +#elif defined(HAVE_LIBNETTLE) + +static int +__hmac_sha1_init(archive_hmac_sha1_ctx *ctx, const uint8_t *key, size_t key_len) +{ + hmac_sha1_set_key(ctx, key_len, key); + return 0; +} + +static void +__hmac_sha1_update(archive_hmac_sha1_ctx *ctx, const uint8_t *data, + size_t data_len) +{ + hmac_sha1_update(ctx, data_len, data); +} + +static void +__hmac_sha1_final(archive_hmac_sha1_ctx *ctx, uint8_t *out, size_t *out_len) +{ + hmac_sha1_digest(ctx, (unsigned)*out_len, out); +} + +static void +__hmac_sha1_cleanup(archive_hmac_sha1_ctx *ctx) +{ + memset(ctx, 0, sizeof(*ctx)); +} + +#elif defined(HAVE_LIBCRYPTO) + +static int +__hmac_sha1_init(archive_hmac_sha1_ctx *ctx, const uint8_t *key, size_t key_len) +{ + HMAC_CTX_init(ctx); + HMAC_Init(ctx, key, key_len, EVP_sha1()); + return 0; +} + +static void +__hmac_sha1_update(archive_hmac_sha1_ctx *ctx, const uint8_t *data, + size_t data_len) +{ + HMAC_Update(ctx, data, data_len); +} + +static void +__hmac_sha1_final(archive_hmac_sha1_ctx *ctx, uint8_t *out, size_t *out_len) +{ + unsigned int len = (unsigned int)*out_len; + HMAC_Final(ctx, out, &len); + *out_len = len; +} + +static void +__hmac_sha1_cleanup(archive_hmac_sha1_ctx *ctx) +{ + HMAC_CTX_cleanup(ctx); + memset(ctx, 0, sizeof(*ctx)); +} + +#else + +/* Stub */ +static int +__hmac_sha1_init(archive_hmac_sha1_ctx *ctx, const uint8_t *key, size_t key_len) +{ + (void)ctx;/* UNUSED */ + (void)key;/* UNUSED */ + (void)key_len;/* UNUSED */ + return -1; +} + +static void +__hmac_sha1_update(archive_hmac_sha1_ctx *ctx, const uint8_t *data, + size_t data_len) +{ + (void)ctx;/* UNUSED */ + (void)data;/* UNUSED */ + (void)data_len;/* UNUSED */ +} + +static void +__hmac_sha1_final(archive_hmac_sha1_ctx *ctx, uint8_t *out, size_t *out_len) +{ + (void)ctx;/* UNUSED */ + (void)out;/* UNUSED */ + (void)out_len;/* UNUSED */ +} + +static void +__hmac_sha1_cleanup(archive_hmac_sha1_ctx *ctx) +{ + (void)ctx;/* UNUSED */ +} + +#endif + +const struct archive_hmac __archive_hmac = { + &__hmac_sha1_init, + &__hmac_sha1_update, + &__hmac_sha1_final, + &__hmac_sha1_cleanup, +}; diff --git a/Utilities/cmlibarchive/libarchive/archive_hmac_private.h b/Utilities/cmlibarchive/libarchive/archive_hmac_private.h new file mode 100644 index 0000000..a9fb8ec --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_hmac_private.h @@ -0,0 +1,95 @@ +/*- +* Copyright (c) 2014 Michihiro NAKAJIMA +* All rights reserved. +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR +* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +* IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef __LIBARCHIVE_BUILD +#error This header is only to be used internally to libarchive. +#endif + +#ifndef ARCHIVE_HMAC_PRIVATE_H_INCLUDED +#define ARCHIVE_HMAC_PRIVATE_H_INCLUDED + +#ifdef __APPLE__ +# include <AvailabilityMacros.h> +# if MAC_OS_X_VERSION_MAX_ALLOWED >= 1060 +# define ARCHIVE_HMAC_USE_Apple_CommonCrypto +# endif +#endif + +#ifdef ARCHIVE_HMAC_USE_Apple_CommonCrypto +#include <CommonCrypto/CommonHMAC.h> + +typedef CCHmacContext archive_hmac_sha1_ctx; + +#elif defined(_WIN32) && !defined(__CYGWIN__) && defined(HAVE_BCRYPT_H) +#include <bcrypt.h> + +typedef struct { + BCRYPT_ALG_HANDLE hAlg; + BCRYPT_HASH_HANDLE hHash; + DWORD hash_len; + PBYTE hash; + +} archive_hmac_sha1_ctx; + +#elif defined(HAVE_LIBNETTLE) +#include <nettle/hmac.h> + +typedef struct hmac_sha1_ctx archive_hmac_sha1_ctx; + +#elif defined(HAVE_LIBCRYPTO) +#include <openssl/hmac.h> + +typedef HMAC_CTX archive_hmac_sha1_ctx; + +#else + +typedef int archive_hmac_sha1_ctx; + +#endif + + +/* HMAC */ +#define archive_hmac_sha1_init(ctx, key, key_len)\ + __archive_hmac.__hmac_sha1_init(ctx, key, key_len) +#define archive_hmac_sha1_update(ctx, data, data_len)\ + __archive_hmac.__hmac_sha1_update(ctx, data, data_len) +#define archive_hmac_sha1_final(ctx, out, out_len)\ + __archive_hmac.__hmac_sha1_final(ctx, out, out_len) +#define archive_hmac_sha1_cleanup(ctx)\ + __archive_hmac.__hmac_sha1_cleanup(ctx) + + +struct archive_hmac { + /* HMAC */ + int (*__hmac_sha1_init)(archive_hmac_sha1_ctx *, const uint8_t *, + size_t); + void (*__hmac_sha1_update)(archive_hmac_sha1_ctx *, const uint8_t *, + size_t); + void (*__hmac_sha1_final)(archive_hmac_sha1_ctx *, uint8_t *, size_t *); + void (*__hmac_sha1_cleanup)(archive_hmac_sha1_ctx *); +}; + +extern const struct archive_hmac __archive_hmac; +#endif /* ARCHIVE_HMAC_PRIVATE_H_INCLUDED */ diff --git a/Utilities/cmlibarchive/libarchive/archive_match.c b/Utilities/cmlibarchive/libarchive/archive_match.c index 6fb8644..74aaacb 100644 --- a/Utilities/cmlibarchive/libarchive/archive_match.c +++ b/Utilities/cmlibarchive/libarchive/archive_match.c @@ -580,6 +580,7 @@ add_pattern_from_file(struct archive_match *a, struct match_list *mlist, return (ARCHIVE_FATAL); } r = archive_read_support_format_raw(ar); + r = archive_read_support_format_empty(ar); if (r != ARCHIVE_OK) { archive_copy_error(&(a->archive), ar); archive_read_free(ar); @@ -596,9 +597,13 @@ add_pattern_from_file(struct archive_match *a, struct match_list *mlist, } r = archive_read_next_header(ar, &ae); if (r != ARCHIVE_OK) { - archive_copy_error(&(a->archive), ar); archive_read_free(ar); - return (r); + if (r == ARCHIVE_EOF) { + return (ARCHIVE_OK); + } else { + archive_copy_error(&(a->archive), ar); + return (r); + } } archive_string_init(&as); diff --git a/Utilities/cmlibarchive/libarchive/archive_pathmatch.c b/Utilities/cmlibarchive/libarchive/archive_pathmatch.c index 505252a..619e2b6 100644 --- a/Utilities/cmlibarchive/libarchive/archive_pathmatch.c +++ b/Utilities/cmlibarchive/libarchive/archive_pathmatch.c @@ -394,8 +394,8 @@ __archive_pathmatch(const char *p, const char *s, int flags) if (*p == '/' && *s != '/') return (0); - /* Certain patterns and file names anchor implicitly. */ - if (*p == '*' || *p == '/' || *p == '/') { + /* Certain patterns anchor implicitly. */ + if (*p == '*' || *p == '/') { while (*p == '/') ++p; while (*s == '/') @@ -434,8 +434,8 @@ __archive_pathmatch_w(const wchar_t *p, const wchar_t *s, int flags) if (*p == L'/' && *s != L'/') return (0); - /* Certain patterns and file names anchor implicitly. */ - if (*p == L'*' || *p == L'/' || *p == L'/') { + /* Certain patterns anchor implicitly. */ + if (*p == L'*' || *p == L'/') { while (*p == L'/') ++p; while (*s == L'/') diff --git a/Utilities/cmlibarchive/libarchive/archive_private.h b/Utilities/cmlibarchive/libarchive/archive_private.h index 30d472f..4b4be97 100644 --- a/Utilities/cmlibarchive/libarchive/archive_private.h +++ b/Utilities/cmlibarchive/libarchive/archive_private.h @@ -119,6 +119,23 @@ struct archive { unsigned current_codepage; /* Current ACP(ANSI CodePage). */ unsigned current_oemcp; /* Current OEMCP(OEM CodePage). */ struct archive_string_conv *sconv; + + /* + * Used by archive_read_data() to track blocks and copy + * data to client buffers, filling gaps with zero bytes. + */ + const char *read_data_block; + int64_t read_data_offset; + int64_t read_data_output_offset; + size_t read_data_remaining; + + /* + * Used by formats/filters to determine the amount of data + * requested from a call to archive_read_data(). This is only + * useful when the format/filter has seek support. + */ + char read_data_is_posix_read; + size_t read_data_requested; }; /* Check magic value and state; return(ARCHIVE_FATAL) if it isn't valid. */ @@ -139,6 +156,8 @@ int __archive_mktemp(const char *tmpdir); int __archive_clean(struct archive *); +void __archive_reset_read_data(struct archive *); + #define err_combine(a,b) ((a) < (b) ? (a) : (b)) #if defined(__BORLANDC__) || (defined(_MSC_VER) && _MSC_VER <= 1300) diff --git a/Utilities/cmlibarchive/libarchive/archive_random.c b/Utilities/cmlibarchive/libarchive/archive_random.c new file mode 100644 index 0000000..a20b9b1 --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_random.c @@ -0,0 +1,269 @@ +/*- + * Copyright (c) 2014 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif + +#if !defined(HAVE_ARC4RANDOM_BUF) && (!defined(_WIN32) || defined(__CYGWIN__)) + +#ifdef HAVE_FCNTL +#include <fcntl.h> +#endif +#ifdef HAVE_LIMITS_H +#include <limits.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_SYS_TIME_H +#include <sys/time.h> +#endif +#ifdef HAVE_PTHREAD_H +#include <pthread.h> +#endif + +static void arc4random_buf(void *, size_t); + +#endif /* HAVE_ARC4RANDOM_BUF */ + +#include "archive.h" +#include "archive_random_private.h" + +#if defined(HAVE_WINCRYPT_H) && !defined(__CYGWIN__) +#include <wincrypt.h> +#endif + +#ifndef O_CLOEXEC +#define O_CLOEXEC 0 +#endif + +/* + * Random number generator function. + * This simply calls arc4random_buf function if the platform provides it. + */ + +int +archive_random(void *buf, size_t nbytes) +{ +#if defined(_WIN32) && !defined(__CYGWIN__) + HCRYPTPROV hProv; + BOOL success; + + success = CryptAcquireContext(&hProv, NULL, NULL, PROV_RSA_FULL, + CRYPT_VERIFYCONTEXT); + if (!success && GetLastError() == NTE_BAD_KEYSET) { + success = CryptAcquireContext(&hProv, NULL, NULL, + PROV_RSA_FULL, CRYPT_NEWKEYSET); + } + if (success) { + success = CryptGenRandom(hProv, (DWORD)nbytes, (BYTE*)buf); + CryptReleaseContext(hProv, 0); + if (success) + return ARCHIVE_OK; + } + /* TODO: Does this case really happen? */ + return ARCHIVE_FAILED; +#else + arc4random_buf(buf, nbytes); + return ARCHIVE_OK; +#endif +} + +#if !defined(HAVE_ARC4RANDOM_BUF) && (!defined(_WIN32) || defined(__CYGWIN__)) + +/* $OpenBSD: arc4random.c,v 1.24 2013/06/11 16:59:50 deraadt Exp $ */ +/* + * Copyright (c) 1996, David Mazieres <dm@uun.org> + * Copyright (c) 2008, Damien Miller <djm@openbsd.org> + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * Arc4 random number generator for OpenBSD. + * + * This code is derived from section 17.1 of Applied Cryptography, + * second edition, which describes a stream cipher allegedly + * compatible with RSA Labs "RC4" cipher (the actual description of + * which is a trade secret). The same algorithm is used as a stream + * cipher called "arcfour" in Tatu Ylonen's ssh package. + * + * RC4 is a registered trademark of RSA Laboratories. + */ + +#ifdef __GNUC__ +#define inline __inline +#else /* !__GNUC__ */ +#define inline +#endif /* !__GNUC__ */ + +struct arc4_stream { + uint8_t i; + uint8_t j; + uint8_t s[256]; +}; + +#define RANDOMDEV "/dev/urandom" +#define KEYSIZE 128 +#ifdef HAVE_PTHREAD_H +static pthread_mutex_t arc4random_mtx = PTHREAD_MUTEX_INITIALIZER; +#define _ARC4_LOCK() pthread_mutex_lock(&arc4random_mtx); +#define _ARC4_UNLOCK() pthread_mutex_unlock(&arc4random_mtx); +#else +#define _ARC4_LOCK() +#define _ARC4_UNLOCK() +#endif + +static int rs_initialized; +static struct arc4_stream rs; +static pid_t arc4_stir_pid; +static int arc4_count; + +static inline uint8_t arc4_getbyte(void); +static void arc4_stir(void); + +static inline void +arc4_init(void) +{ + int n; + + for (n = 0; n < 256; n++) + rs.s[n] = n; + rs.i = 0; + rs.j = 0; +} + +static inline void +arc4_addrandom(u_char *dat, int datlen) +{ + int n; + uint8_t si; + + rs.i--; + for (n = 0; n < 256; n++) { + rs.i = (rs.i + 1); + si = rs.s[rs.i]; + rs.j = (rs.j + si + dat[n % datlen]); + rs.s[rs.i] = rs.s[rs.j]; + rs.s[rs.j] = si; + } + rs.j = rs.i; +} + +static void +arc4_stir(void) +{ + int done, fd, i; + struct { + struct timeval tv; + pid_t pid; + u_char rnd[KEYSIZE]; + } rdat; + + if (!rs_initialized) { + arc4_init(); + rs_initialized = 1; + } + done = 0; + fd = open(RANDOMDEV, O_RDONLY | O_CLOEXEC, 0); + if (fd >= 0) { + if (read(fd, &rdat, KEYSIZE) == KEYSIZE) + done = 1; + (void)close(fd); + } + if (!done) { + (void)gettimeofday(&rdat.tv, NULL); + rdat.pid = getpid(); + /* We'll just take whatever was on the stack too... */ + } + + arc4_addrandom((u_char *)&rdat, KEYSIZE); + + /* + * Discard early keystream, as per recommendations in: + * "(Not So) Random Shuffles of RC4" by Ilya Mironov. + */ + for (i = 0; i < 1024; i++) + (void)arc4_getbyte(); + arc4_count = 1600000; +} + +static void +arc4_stir_if_needed(void) +{ + pid_t pid = getpid(); + + if (arc4_count <= 0 || !rs_initialized || arc4_stir_pid != pid) { + arc4_stir_pid = pid; + arc4_stir(); + } +} + +static inline uint8_t +arc4_getbyte(void) +{ + uint8_t si, sj; + + rs.i = (rs.i + 1); + si = rs.s[rs.i]; + rs.j = (rs.j + si); + sj = rs.s[rs.j]; + rs.s[rs.i] = sj; + rs.s[rs.j] = si; + return (rs.s[(si + sj) & 0xff]); +} + +static void +arc4random_buf(void *_buf, size_t n) +{ + u_char *buf = (u_char *)_buf; + _ARC4_LOCK(); + arc4_stir_if_needed(); + while (n--) { + if (--arc4_count <= 0) + arc4_stir(); + buf[n] = arc4_getbyte(); + } + _ARC4_UNLOCK(); +} + +#endif /* !HAVE_ARC4RANDOM_BUF */ diff --git a/Utilities/cmlibarchive/libarchive/archive_random_private.h b/Utilities/cmlibarchive/libarchive/archive_random_private.h new file mode 100644 index 0000000..c414779 --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_random_private.h @@ -0,0 +1,36 @@ +/*- + * Copyright (c) 2014 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __LIBARCHIVE_BUILD +#error This header is only to be used internally to libarchive. +#endif + +#ifndef ARCHIVE_RANDOM_PRIVATE_H_INCLUDED +#define ARCHIVE_RANDOM_PRIVATE_H_INCLUDED + +/* Random number generator. */ +int archive_random(void *buf, size_t nbytes); + +#endif /* ARCHIVE_RANDOM_PRIVATE_H_INCLUDED */ diff --git a/Utilities/cmlibarchive/libarchive/archive_read.3 b/Utilities/cmlibarchive/libarchive/archive_read.3 index a29cc1e..d37e732 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read.3 +++ b/Utilities/cmlibarchive/libarchive/archive_read.3 @@ -130,7 +130,7 @@ which provides a slightly more efficient interface. You may prefer to use the higher-level .Fn archive_read_data_skip , which reads and discards the data for this entry, -.Fn archive_read_data_to_file , +.Fn archive_read_data_into_fd , which copies the data to the provided file descriptor, or .Fn archive_read_extract , which recreates the specified entry on disk and copies data @@ -186,7 +186,7 @@ list_archive(const char *name) free(mydata); } -ssize_t +la_ssize_t myread(struct archive *a, void *client_data, const void **buff) { struct mydata *mydata = client_data; diff --git a/Utilities/cmlibarchive/libarchive/archive_read.c b/Utilities/cmlibarchive/libarchive/archive_read.c index a65b94d..033ed8b 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read.c +++ b/Utilities/cmlibarchive/libarchive/archive_read.c @@ -101,16 +101,17 @@ archive_read_new(void) { struct archive_read *a; - a = (struct archive_read *)malloc(sizeof(*a)); + a = (struct archive_read *)calloc(1, sizeof(*a)); if (a == NULL) return (NULL); - memset(a, 0, sizeof(*a)); a->archive.magic = ARCHIVE_READ_MAGIC; a->archive.state = ARCHIVE_STATE_NEW; a->entry = archive_entry_new2(&a->archive); a->archive.vtable = archive_read_vtable(); + a->passphrases.last = &a->passphrases.first; + return (&a->archive); } @@ -194,10 +195,12 @@ client_skip_proxy(struct archive_read_filter *self, int64_t request) ask = skip_limit; get = (self->archive->client.skipper) (&self->archive->archive, self->data, ask); - if (get == 0) + total += get; + if (get == 0 || get == request) return (total); + if (get > request) + return ARCHIVE_FATAL; request -= get; - total += get; } } else if (self->archive->client.seeker != NULL && request > 64 * 1024) { @@ -230,8 +233,11 @@ client_seek_proxy(struct archive_read_filter *self, int64_t offset, int whence) * other libarchive code that assumes a successful forward * seek means it can also seek backwards. */ - if (self->archive->client.seeker == NULL) + if (self->archive->client.seeker == NULL) { + archive_set_error(&self->archive->archive, ARCHIVE_ERRNO_MISC, + "Current client reader does not support seeking a device"); return (ARCHIVE_FAILED); + } return (self->archive->client.seeker)(&self->archive->archive, self->data, offset, whence); } @@ -454,7 +460,7 @@ archive_read_open1(struct archive *_a) { struct archive_read *a = (struct archive_read *)_a; struct archive_read_filter *filter, *tmp; - int slot, e; + int slot, e = ARCHIVE_OK; unsigned int i; archive_check_magic(_a, ARCHIVE_READ_MAGIC, ARCHIVE_STATE_NEW, @@ -544,13 +550,13 @@ archive_read_open1(struct archive *_a) static int choose_filters(struct archive_read *a) { - int number_bidders, i, bid, best_bid; + int number_bidders, i, bid, best_bid, n; struct archive_read_filter_bidder *bidder, *best_bidder; struct archive_read_filter *filter; ssize_t avail; int r; - for (;;) { + for (n = 0; n < 25; ++n) { number_bidders = sizeof(a->bidders) / sizeof(a->bidders[0]); best_bid = 0; @@ -596,6 +602,9 @@ choose_filters(struct archive_read *a) return (ARCHIVE_FATAL); } } + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Input requires too many filters for decoding"); + return (ARCHIVE_FATAL); } /* @@ -658,16 +667,14 @@ _archive_read_next_header2(struct archive *_a, struct archive_entry *entry) break; } - a->read_data_output_offset = 0; - a->read_data_remaining = 0; - a->read_data_is_posix_read = 0; - a->read_data_requested = 0; + __archive_reset_read_data(&a->archive); + a->data_start_node = a->client.cursor; /* EOF always wins; otherwise return the worst error. */ return (r2 < r1 || r2 == ARCHIVE_EOF) ? r2 : r1; } -int +static int _archive_read_next_header(struct archive *_a, struct archive_entry **entryp) { int ret; @@ -813,7 +820,7 @@ archive_read_format_capabilities(struct archive *_a) ssize_t archive_read_data(struct archive *_a, void *buff, size_t s) { - struct archive_read *a = (struct archive_read *)_a; + struct archive *a = (struct archive *)_a; char *dest; const void *read_buf; size_t bytes_read; @@ -828,7 +835,7 @@ archive_read_data(struct archive *_a, void *buff, size_t s) read_buf = a->read_data_block; a->read_data_is_posix_read = 1; a->read_data_requested = s; - r = _archive_read_data_block(&a->archive, &read_buf, + r = archive_read_data_block(a, &read_buf, &a->read_data_remaining, &a->read_data_offset); a->read_data_block = read_buf; if (r == ARCHIVE_EOF) @@ -843,7 +850,7 @@ archive_read_data(struct archive *_a, void *buff, size_t s) } if (a->read_data_offset < a->read_data_output_offset) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + archive_set_error(a, ARCHIVE_ERRNO_FILE_FORMAT, "Encountered out-of-order sparse blocks"); return (ARCHIVE_RETRY); } @@ -887,6 +894,21 @@ archive_read_data(struct archive *_a, void *buff, size_t s) } /* + * Reset the read_data_* variables, used for starting a new entry. + */ +void __archive_reset_read_data(struct archive * a) +{ + a->read_data_output_offset = 0; + a->read_data_remaining = 0; + a->read_data_is_posix_read = 0; + a->read_data_requested = 0; + + /* extra resets, from rar.c */ + a->read_data_block = NULL; + a->read_data_offset = 0; +} + +/* * Skip over all remaining data in this entry. */ int @@ -953,7 +975,7 @@ _archive_read_data_block(struct archive *_a, if (a->format->read_data == NULL) { archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER, "Internal error: " - "No format_read_data_block function registered"); + "No format->read_data function registered"); return (ARCHIVE_FATAL); } @@ -1040,6 +1062,7 @@ static int _archive_read_free(struct archive *_a) { struct archive_read *a = (struct archive_read *)_a; + struct archive_read_passphrase *p; int i, n; int slots; int r = ARCHIVE_OK; @@ -1077,9 +1100,20 @@ _archive_read_free(struct archive *_a) } } + /* Release passphrase list. */ + p = a->passphrases.first; + while (p != NULL) { + struct archive_read_passphrase *np = p->next; + + /* A passphrase should be cleaned. */ + memset(p->passphrase, 0, strlen(p->passphrase)); + free(p->passphrase); + free(p); + p = np; + } + archive_string_free(&a->archive.error_string); - if (a->entry) - archive_entry_free(a->entry); + archive_entry_free(a->entry); a->archive.magic = 0; __archive_clean(&a->archive); free(a->client.dataset); @@ -1451,6 +1485,8 @@ __archive_read_filter_consume(struct archive_read_filter * filter, { int64_t skipped; + if (request < 0) + return ARCHIVE_FATAL; if (request == 0) return 0; diff --git a/Utilities/cmlibarchive/libarchive/archive_read_add_passphrase.3 b/Utilities/cmlibarchive/libarchive/archive_read_add_passphrase.3 new file mode 100644 index 0000000..8b242ea --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_read_add_passphrase.3 @@ -0,0 +1,74 @@ +.\" Copyright (c) 2014 Michihiro NAKAJIMA +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd September 14, 2014 +.Dt ARCHIVE_READ_ADD_PASSPHRASE 3 +.Os +.Sh NAME +.Nm archive_read_add_passphrase , +.Nm archive_read_set_passphrase_callback +.Nd functions for reading encrypted archives +.Sh LIBRARY +Streaming Archive Library (libarchive, -larchive) +.Sh SYNOPSIS +.In archive.h +.Ft int +.Fo archive_read_add_passphrase +.Fa "struct archive *" +.Fa "const char *passphrase" +.Fc +.Ft int +.Fo archive_read_set_passphrase_callback +.Fa "struct archive *" +.Fa "void *client_data" +.Fa "archive_passphrase_callback *" +.Fc +.Sh DESCRIPTION +.Bl -tag -width indent +.It Fn archive_read_add_passphrase +Register passphrases for reading an encryption archive. +If +.Ar passphrase +is +.Dv NULL +or empty, this function will do nothing and +.Cm ARCHIVE_FAILED +will be returned. +Otherwise, +.Cm ARCHIVE_OK +will be returned. +.It Fn archive_read_set_passphrase_callback +Register callback function that will be invoked to get a passphrase +for decrption after trying all passphrases registered by the +.Fn archive_read_add_passphrase +function failed. +.El +.\" .Sh ERRORS +.Sh SEE ALSO +.Xr tar 1 , +.Xr libarchive 3 , +.Xr archive_read 3 , +.Xr archive_read_set_options 3 diff --git a/Utilities/cmlibarchive/libarchive/archive_read_add_passphrase.c b/Utilities/cmlibarchive/libarchive/archive_read_add_passphrase.c new file mode 100644 index 0000000..f67f1eb --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_read_add_passphrase.c @@ -0,0 +1,186 @@ +/*- + * Copyright (c) 2014 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#include "archive_read_private.h" + +static void +add_passphrase_to_tail(struct archive_read *a, + struct archive_read_passphrase *p) +{ + *a->passphrases.last = p; + a->passphrases.last = &p->next; + p->next = NULL; +} + +static struct archive_read_passphrase * +remove_passphrases_from_head(struct archive_read *a) +{ + struct archive_read_passphrase *p; + + p = a->passphrases.first; + if (p != NULL) + a->passphrases.first = p->next; + return (p); +} + +static void +insert_passphrase_to_head(struct archive_read *a, + struct archive_read_passphrase *p) +{ + p->next = a->passphrases.first; + a->passphrases.first = p; +} + +static struct archive_read_passphrase * +new_read_passphrase(struct archive_read *a, const char *passphrase) +{ + struct archive_read_passphrase *p; + + p = malloc(sizeof(*p)); + if (p == NULL) { + archive_set_error(&a->archive, ENOMEM, + "Can't allocate memory"); + return (NULL); + } + p->passphrase = strdup(passphrase); + if (p->passphrase == NULL) { + free(p); + archive_set_error(&a->archive, ENOMEM, + "Can't allocate memory"); + return (NULL); + } + return (p); +} + +int +archive_read_add_passphrase(struct archive *_a, const char *passphrase) +{ + struct archive_read *a = (struct archive_read *)_a; + struct archive_read_passphrase *p; + + archive_check_magic(_a, ARCHIVE_READ_MAGIC, ARCHIVE_STATE_NEW, + "archive_read_add_passphrase"); + + if (passphrase == NULL || passphrase[0] == '\0') { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Empty passphrase is unacceptable"); + return (ARCHIVE_FAILED); + } + + p = new_read_passphrase(a, passphrase); + if (p == NULL) + return (ARCHIVE_FATAL); + add_passphrase_to_tail(a, p); + + return (ARCHIVE_OK); +} + +int +archive_read_set_passphrase_callback(struct archive *_a, void *client_data, + archive_passphrase_callback *cb) +{ + struct archive_read *a = (struct archive_read *)_a; + + archive_check_magic(_a, ARCHIVE_READ_MAGIC, ARCHIVE_STATE_NEW, + "archive_read_set_passphrase_callback"); + + a->passphrases.callback = cb; + a->passphrases.client_data = client_data; + return (ARCHIVE_OK); +} + +/* + * Call this in advance when you start to get a passphrase for decryption + * for a entry. + */ +void +__archive_read_reset_passphrase(struct archive_read *a) +{ + + a->passphrases.candiate = -1; +} + +/* + * Get a passphrase for decryption. + */ +const char * +__archive_read_next_passphrase(struct archive_read *a) +{ + struct archive_read_passphrase *p; + const char *passphrase; + + if (a->passphrases.candiate < 0) { + /* Count out how many passphrases we have. */ + int cnt = 0; + + for (p = a->passphrases.first; p != NULL; p = p->next) + cnt++; + a->passphrases.candiate = cnt; + p = a->passphrases.first; + } else if (a->passphrases.candiate > 1) { + /* Rotate a passphrase list. */ + a->passphrases.candiate--; + p = remove_passphrases_from_head(a); + add_passphrase_to_tail(a, p); + /* Pick a new passphrase candiate up. */ + p = a->passphrases.first; + } else if (a->passphrases.candiate == 1) { + /* This case is that all cadiates failed to decryption. */ + a->passphrases.candiate = 0; + if (a->passphrases.first->next != NULL) { + /* Rotate a passphrase list. */ + p = remove_passphrases_from_head(a); + add_passphrase_to_tail(a, p); + } + p = NULL; + } else /* There is no passphrase candaite. */ + p = NULL; + + if (p != NULL) + passphrase = p->passphrase; + else if (a->passphrases.callback != NULL) { + /* Get a passphrase through a call-back function + * since we tried all passphrases out or we don't + * have it. */ + passphrase = a->passphrases.callback(&a->archive, + a->passphrases.client_data); + if (passphrase != NULL) { + p = new_read_passphrase(a, passphrase); + if (p == NULL) + return (NULL); + insert_passphrase_to_head(a, p); + a->passphrases.candiate = 1; + } + } else + passphrase = NULL; + + return (passphrase); +} diff --git a/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c b/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c index 017d7c6..3a0d4d6 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_append_filter.c @@ -43,7 +43,7 @@ archive_read_append_filter(struct archive *_a, int code) struct archive_read_filter *filter; struct archive_read *a = (struct archive_read *)_a; - r1 = r2 = (ARCHIVE_OK); + r2 = (ARCHIVE_OK); switch (code) { case ARCHIVE_FILTER_NONE: @@ -85,6 +85,10 @@ archive_read_append_filter(struct archive *_a, int code) strcpy(str, "rpm"); r1 = archive_read_support_filter_rpm(_a); break; + case ARCHIVE_FILTER_LZ4: + strcpy(str, "lz4"); + r1 = archive_read_support_filter_lz4(_a); + break; case ARCHIVE_FILTER_LZIP: strcpy(str, "lzip"); r1 = archive_read_support_filter_lzip(_a); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_data.3 b/Utilities/cmlibarchive/libarchive/archive_read_data.3 index bf0578c..c1bc15d 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_data.3 +++ b/Utilities/cmlibarchive/libarchive/archive_read_data.3 @@ -37,7 +37,7 @@ Streaming Archive Library (libarchive, -larchive) .Sh SYNOPSIS .In archive.h -.Ft ssize_t +.Ft la_ssize_t .Fn archive_read_data "struct archive *" "void *buff" "size_t len" .Ft int .Fo archive_read_data_block diff --git a/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c b/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c index e81cbec..38303aa 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_disk_entry_from_file.c @@ -251,9 +251,11 @@ archive_read_disk_entry_from_file(struct archive *_a, #endif /* HAVE_READLINK || HAVE_READLINKAT */ r = setup_acls(a, entry, &fd); - r1 = setup_xattrs(a, entry, &fd); - if (r1 < r) - r = r1; + if (!a->suppress_xattr) { + r1 = setup_xattrs(a, entry, &fd); + if (r1 < r) + r = r1; + } if (a->enable_copyfile) { r1 = setup_mac_metadata(a, entry, &fd); if (r1 < r) diff --git a/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c b/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c index 94eb5e7..f480539 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_disk_posix.c @@ -356,6 +356,8 @@ static int _archive_read_free(struct archive *); static int _archive_read_close(struct archive *); static int _archive_read_data_block(struct archive *, const void **, size_t *, int64_t *); +static int _archive_read_next_header(struct archive *, + struct archive_entry **); static int _archive_read_next_header2(struct archive *, struct archive_entry *); static const char *trivial_lookup_gname(void *, int64_t gid); @@ -377,6 +379,7 @@ archive_read_disk_vtable(void) av.archive_free = _archive_read_free; av.archive_close = _archive_read_close; av.archive_read_data_block = _archive_read_data_block; + av.archive_read_next_header = _archive_read_next_header; av.archive_read_next_header2 = _archive_read_next_header2; inited = 1; } @@ -459,6 +462,7 @@ archive_read_disk_new(void) a->archive.magic = ARCHIVE_READ_DISK_MAGIC; a->archive.state = ARCHIVE_STATE_NEW; a->archive.vtable = archive_read_disk_vtable(); + a->entry = archive_entry_new2(&a->archive); a->lookup_uname = trivial_lookup_uname; a->lookup_gname = trivial_lookup_gname; a->enable_copyfile = 1; @@ -491,6 +495,7 @@ _archive_read_free(struct archive *_a) if (a->cleanup_uname != NULL && a->lookup_uname_data != NULL) (a->cleanup_uname)(a->lookup_uname_data); archive_string_free(&a->archive.error_string); + archive_entry_free(a->entry); a->archive.magic = 0; __archive_clean(&a->archive); free(a); @@ -609,6 +614,10 @@ archive_read_disk_set_behavior(struct archive *_a, int flags) a->traverse_mount_points = 0; else a->traverse_mount_points = 1; + if (flags & ARCHIVE_READDISK_NO_XATTR) + a->suppress_xattr = 1; + else + a->suppress_xattr = 0; return (r); } @@ -974,7 +983,7 @@ next_entry(struct archive_read_disk *a, struct tree *t, t->initial_filesystem_id = t->current_filesystem_id; if (!a->traverse_mount_points) { if (t->initial_filesystem_id != t->current_filesystem_id) - return (ARCHIVE_RETRY); + descend = 0; } t->descend = descend; @@ -1081,6 +1090,17 @@ next_entry(struct archive_read_disk *a, struct tree *t, } static int +_archive_read_next_header(struct archive *_a, struct archive_entry **entryp) +{ + int ret; + struct archive_read_disk *a = (struct archive_read_disk *)_a; + *entryp = NULL; + ret = _archive_read_next_header2(_a, a->entry); + *entryp = a->entry; + return ret; +} + +static int _archive_read_next_header2(struct archive *_a, struct archive_entry *entry) { struct archive_read_disk *a = (struct archive_read_disk *)_a; @@ -1148,6 +1168,7 @@ _archive_read_next_header2(struct archive *_a, struct archive_entry *entry) break; } + __archive_reset_read_data(&a->archive); return (r); } diff --git a/Utilities/cmlibarchive/libarchive/archive_read_disk_private.h b/Utilities/cmlibarchive/libarchive/archive_read_disk_private.h index e5af16b..2569321 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_disk_private.h +++ b/Utilities/cmlibarchive/libarchive/archive_read_disk_private.h @@ -39,6 +39,9 @@ struct archive_entry; struct archive_read_disk { struct archive archive; + /* Reused by archive_read_next_header() */ + struct archive_entry *entry; + /* * Symlink mode is one of 'L'ogical, 'P'hysical, or 'H'ybrid, * following an old BSD convention. 'L' follows all symlinks, @@ -68,6 +71,8 @@ struct archive_read_disk { int enable_copyfile; /* Set 1 if users request to traverse mount points. */ int traverse_mount_points; + /* Set 1 if users want to suppress xattr information. */ + int suppress_xattr; const char * (*lookup_gname)(void *private, int64_t gid); void (*cleanup_gname)(void *private); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_disk_set_standard_lookup.c b/Utilities/cmlibarchive/libarchive/archive_read_disk_set_standard_lookup.c index 3bc52c7..d6b2d55 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_disk_set_standard_lookup.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_disk_set_standard_lookup.c @@ -83,7 +83,7 @@ static const char * lookup_uname_helper(struct name_cache *, id_t uid); * a simple cache to accelerate such lookups---into the archive_read_disk * object. This is in a separate file because getpwuid()/getgrgid() * can pull in a LOT of library code (including NIS/LDAP functions, which - * pull in DNS resolveers, etc). This can easily top 500kB, which makes + * pull in DNS resolvers, etc). This can easily top 500kB, which makes * it inappropriate for some space-constrained applications. * * Applications that are size-sensitive may want to just use the diff --git a/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c b/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c index 5c0f366..53bd4b8 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_disk_windows.c @@ -288,6 +288,8 @@ static int _archive_read_free(struct archive *); static int _archive_read_close(struct archive *); static int _archive_read_data_block(struct archive *, const void **, size_t *, int64_t *); +static int _archive_read_next_header(struct archive *, + struct archive_entry **); static int _archive_read_next_header2(struct archive *, struct archive_entry *); static const char *trivial_lookup_gname(void *, int64_t gid); @@ -310,6 +312,7 @@ archive_read_disk_vtable(void) av.archive_free = _archive_read_free; av.archive_close = _archive_read_close; av.archive_read_data_block = _archive_read_data_block; + av.archive_read_next_header = _archive_read_next_header; av.archive_read_next_header2 = _archive_read_next_header2; inited = 1; } @@ -393,6 +396,7 @@ archive_read_disk_new(void) a->archive.magic = ARCHIVE_READ_DISK_MAGIC; a->archive.state = ARCHIVE_STATE_NEW; a->archive.vtable = archive_read_disk_vtable(); + a->entry = archive_entry_new2(&a->archive); a->lookup_uname = trivial_lookup_uname; a->lookup_gname = trivial_lookup_gname; a->enable_copyfile = 1; @@ -422,6 +426,7 @@ _archive_read_free(struct archive *_a) if (a->cleanup_uname != NULL && a->lookup_uname_data != NULL) (a->cleanup_uname)(a->lookup_uname_data); archive_string_free(&a->archive.error_string); + archive_entry_free(a->entry); a->archive.magic = 0; free(a); return (r); @@ -945,6 +950,17 @@ next_entry(struct archive_read_disk *a, struct tree *t, } static int +_archive_read_next_header(struct archive *_a, struct archive_entry **entryp) +{ + int ret; + struct archive_read_disk *a = (struct archive_read_disk *)_a; + *entryp = NULL; + ret = _archive_read_next_header2(_a, a->entry); + *entryp = a->entry; + return ret; +} + +static int _archive_read_next_header2(struct archive *_a, struct archive_entry *entry) { struct archive_read_disk *a = (struct archive_read_disk *)_a; @@ -1000,6 +1016,7 @@ _archive_read_next_header2(struct archive *_a, struct archive_entry *entry) break; } + __archive_reset_read_data(&a->archive); return (r); } @@ -1851,8 +1868,6 @@ entry_copy_bhfi(struct archive_entry *entry, const wchar_t *path, break; case L'C': case L'c': if (((p[2] == L'M' || p[2] == L'm' ) && - (p[3] == L'D' || p[3] == L'd' )) || - ((p[2] == L'M' || p[2] == L'm' ) && (p[3] == L'D' || p[3] == L'd' ))) mode |= S_IXUSR | S_IXGRP | S_IXOTH; break; diff --git a/Utilities/cmlibarchive/libarchive/archive_read_extract.c b/Utilities/cmlibarchive/libarchive/archive_read_extract.c index ce76a6c..b7973fa 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_extract.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_extract.c @@ -35,8 +35,6 @@ __FBSDID("$FreeBSD: src/lib/libarchive/archive_read_extract.c,v 1.61 2008/05/26 #include "archive_private.h" #include "archive_read_private.h" -static int archive_read_extract_cleanup(struct archive_read *); - int archive_read_extract(struct archive *_a, struct archive_entry *entry, int flags) { @@ -55,23 +53,8 @@ archive_read_extract(struct archive *_a, struct archive_entry *entry, int flags) return (ARCHIVE_FATAL); } archive_write_disk_set_standard_lookup(extract->ad); - a->cleanup_archive_extract = archive_read_extract_cleanup; } archive_write_disk_set_options(extract->ad, flags); return (archive_read_extract2(&a->archive, entry, extract->ad)); } - -/* - * Cleanup function for archive_extract. - */ -static int -archive_read_extract_cleanup(struct archive_read *a) -{ - int ret = ARCHIVE_OK; - - ret = archive_write_free(a->extract->ad); - free(a->extract); - a->extract = NULL; - return (ret); -} diff --git a/Utilities/cmlibarchive/libarchive/archive_read_extract2.c b/Utilities/cmlibarchive/libarchive/archive_read_extract2.c index 3c65e80..7b2c126 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_extract2.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_extract2.c @@ -42,6 +42,8 @@ __FBSDID("$FreeBSD: src/lib/libarchive/archive_read_extract.c,v 1.61 2008/05/26 #include "archive_read_private.h" static int copy_data(struct archive *ar, struct archive *aw); +static int archive_read_extract_cleanup(struct archive_read *); + /* Retrieve an extract object without initialising the associated * archive_write_disk object. @@ -56,10 +58,27 @@ __archive_read_get_extract(struct archive_read *a) return (NULL); } memset(a->extract, 0, sizeof(*a->extract)); + a->cleanup_archive_extract = archive_read_extract_cleanup; } return (a->extract); } +/* + * Cleanup function for archive_extract. + */ +static int +archive_read_extract_cleanup(struct archive_read *a) +{ + int ret = ARCHIVE_OK; + + if (a->extract->ad != NULL) { + ret = archive_write_free(a->extract->ad); + } + free(a->extract); + a->extract = NULL; + return (ret); +} + int archive_read_extract2(struct archive *_a, struct archive_entry *entry, struct archive *ad) @@ -125,7 +144,7 @@ copy_data(struct archive *ar, struct archive *aw) r = (int)archive_write_data_block(aw, buff, size, offset); if (r < ARCHIVE_WARN) r = ARCHIVE_WARN; - if (r != ARCHIVE_OK) { + if (r < ARCHIVE_OK) { archive_set_error(ar, archive_errno(aw), "%s", archive_error_string(aw)); return (r); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_filter.3 b/Utilities/cmlibarchive/libarchive/archive_read_filter.3 index 8761127..47effac 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_filter.3 +++ b/Utilities/cmlibarchive/libarchive/archive_read_filter.3 @@ -24,7 +24,7 @@ .\" .\" $FreeBSD$ .\" -.Dd February 2, 2012 +.Dd August 14, 2014 .Dt ARCHIVE_READ_FILTER 3 .Os .Sh NAME @@ -32,8 +32,11 @@ .Nm archive_read_support_filter_bzip2 , .Nm archive_read_support_filter_compress , .Nm archive_read_support_filter_gzip , +.Nm archive_read_support_filter_lz4 , .Nm archive_read_support_filter_lzma , .Nm archive_read_support_filter_none , +.Nm archive_read_support_filter_rpm , +.Nm archive_read_support_filter_uu , .Nm archive_read_support_filter_xz , .Nm archive_read_support_filter_program , .Nm archive_read_support_filter_program_signature @@ -52,10 +55,18 @@ Streaming Archive Library (libarchive, -larchive) .Ft int .Fn archive_read_support_filter_gzip "struct archive *" .Ft int +.Fn archive_read_support_filter_lz4 "struct archive *" +.Ft int .Fn archive_read_support_filter_lzma "struct archive *" .Ft int +.Fn archive_read_support_filter_lzop "struct archive *" +.Ft int .Fn archive_read_support_filter_none "struct archive *" .Ft int +.Fn archive_read_support_filter_rpm "struct archive *" +.Ft int +.Fn archive_read_support_filter_uu "struct archive *" +.Ft int .Fn archive_read_support_filter_xz "struct archive *" .Ft int .Fo archive_read_support_filter_program @@ -76,8 +87,12 @@ Streaming Archive Library (libarchive, -larchive) .Fn archive_read_support_filter_bzip2 , .Fn archive_read_support_filter_compress , .Fn archive_read_support_filter_gzip , +.Fn archive_read_support_filter_lz4 , .Fn archive_read_support_filter_lzma , +.Fn archive_read_support_filter_lzop , .Fn archive_read_support_filter_none , +.Fn archive_read_support_filter_rpm , +.Fn archive_read_support_filter_uu , .Fn archive_read_support_filter_xz .Xc Enables auto-detection code and decompression support for the diff --git a/Utilities/cmlibarchive/libarchive/archive_read_open.3 b/Utilities/cmlibarchive/libarchive/archive_read_open.3 index 30a740b..4d8272c 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_open.3 +++ b/Utilities/cmlibarchive/libarchive/archive_read_open.3 @@ -130,14 +130,14 @@ objects can be found in the overview manual page for The callback functions must match the following prototypes: .Bl -item -offset indent .It -.Ft typedef ssize_t +.Ft typedef la_ssize_t .Fo archive_read_callback .Fa "struct archive *" .Fa "void *client_data" .Fa "const void **buffer" .Fc .It -.Ft typedef off_t +.Ft typedef la_int64_t .Fo archive_skip_callback .Fa "struct archive *" .Fa "void *client_data" diff --git a/Utilities/cmlibarchive/libarchive/archive_read_open_fd.c b/Utilities/cmlibarchive/libarchive/archive_read_open_fd.c index e0f95bf..f59cd07 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_open_fd.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_open_fd.c @@ -59,6 +59,7 @@ struct read_fd_data { static int file_close(struct archive *, void *); static ssize_t file_read(struct archive *, void *, const void **buff); +static int64_t file_seek(struct archive *, void *, int64_t request, int); static int64_t file_skip(struct archive *, void *, int64_t request); int @@ -102,6 +103,7 @@ archive_read_open_fd(struct archive *a, int fd, size_t block_size) archive_read_set_read_callback(a, file_read); archive_read_set_skip_callback(a, file_skip); + archive_read_set_seek_callback(a, file_seek); archive_read_set_close_callback(a, file_close); archive_read_set_callback_data(a, mine); return (archive_read_open1(a)); @@ -170,6 +172,33 @@ file_skip(struct archive *a, void *client_data, int64_t request) return (-1); } +/* + * TODO: Store the offset and use it in the read callback. + */ +static int64_t +file_seek(struct archive *a, void *client_data, int64_t request, int whence) +{ + struct read_fd_data *mine = (struct read_fd_data *)client_data; + int64_t r; + + /* We use off_t here because lseek() is declared that way. */ + /* See above for notes about when off_t is less than 64 bits. */ + r = lseek(mine->fd, request, whence); + if (r >= 0) + return r; + + if (errno == ESPIPE) { + archive_set_error(a, errno, + "A file descriptor(%d) is not seekable(PIPE)", mine->fd); + return (ARCHIVE_FAILED); + } else { + /* If the input is corrupted or truncated, fail. */ + archive_set_error(a, errno, + "Error seeking in a file descriptor(%d)", mine->fd); + return (ARCHIVE_FATAL); + } +} + static int file_close(struct archive *a, void *client_data) { diff --git a/Utilities/cmlibarchive/libarchive/archive_read_open_memory.c b/Utilities/cmlibarchive/libarchive/archive_read_open_memory.c index bcc7d6f..ff935a7 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_open_memory.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_open_memory.c @@ -41,9 +41,9 @@ __FBSDID("$FreeBSD: src/lib/libarchive/archive_read_open_memory.c,v 1.6 2007/07/ */ struct read_memory_data { - unsigned char *start; - unsigned char *p; - unsigned char *end; + const unsigned char *start; + const unsigned char *p; + const unsigned char *end; ssize_t read_size; }; @@ -54,7 +54,7 @@ static int64_t memory_read_skip(struct archive *, void *, int64_t request); static ssize_t memory_read(struct archive *, void *, const void **buff); int -archive_read_open_memory(struct archive *a, void *buff, size_t size) +archive_read_open_memory(struct archive *a, const void *buff, size_t size) { return archive_read_open_memory2(a, buff, size, size); } @@ -65,7 +65,7 @@ archive_read_open_memory(struct archive *a, void *buff, size_t size) * test harnesses can exercise block operations inside the library. */ int -archive_read_open_memory2(struct archive *a, void *buff, +archive_read_open_memory2(struct archive *a, const void *buff, size_t size, size_t read_size) { struct read_memory_data *mine; @@ -76,7 +76,7 @@ archive_read_open_memory2(struct archive *a, void *buff, return (ARCHIVE_FATAL); } memset(mine, 0, sizeof(*mine)); - mine->start = mine->p = (unsigned char *)buff; + mine->start = mine->p = (const unsigned char *)buff; mine->end = mine->start + size; mine->read_size = read_size; archive_read_set_open_callback(a, memory_read_open); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_private.h b/Utilities/cmlibarchive/libarchive/archive_read_private.h index 27e203b..9b61a53 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_private.h +++ b/Utilities/cmlibarchive/libarchive/archive_read_private.h @@ -26,8 +26,10 @@ */ #ifndef __LIBARCHIVE_BUILD +#ifndef __LIBARCHIVE_TEST #error This header is only to be used internally to libarchive. #endif +#endif #ifndef ARCHIVE_READ_PRIVATE_H_INCLUDED #define ARCHIVE_READ_PRIVATE_H_INCLUDED @@ -141,6 +143,10 @@ struct archive_read_client { int64_t position; struct archive_read_data_node *dataset; }; +struct archive_read_passphrase { + char *passphrase; + struct archive_read_passphrase *next; +}; struct archive_read_extract { struct archive *ad; /* archive_write_disk object */ @@ -160,28 +166,11 @@ struct archive_read { int64_t skip_file_dev; int64_t skip_file_ino; - /* - * Used by archive_read_data() to track blocks and copy - * data to client buffers, filling gaps with zero bytes. - */ - const char *read_data_block; - int64_t read_data_offset; - int64_t read_data_output_offset; - size_t read_data_remaining; - - /* - * Used by formats/filters to determine the amount of data - * requested from a call to archive_read_data(). This is only - * useful when the format/filter has seek support. - */ - char read_data_is_posix_read; - size_t read_data_requested; - /* Callbacks to open/read/write/close client archive streams. */ struct archive_read_client client; /* Registered filter bidders. */ - struct archive_read_filter_bidder bidders[14]; + struct archive_read_filter_bidder bidders[16]; /* Last filter in chain */ struct archive_read_filter *filter; @@ -225,6 +214,17 @@ struct archive_read { */ struct archive_read_extract *extract; int (*cleanup_archive_extract)(struct archive_read *); + + /* + * Decryption passphrase. + */ + struct { + struct archive_read_passphrase *first; + struct archive_read_passphrase **last; + int candiate; + archive_passphrase_callback *callback; + void *client_data; + } passphrases; }; int __archive_read_register_format(struct archive_read *a, @@ -254,4 +254,11 @@ int __archive_read_program(struct archive_read_filter *, const char *); void __archive_read_free_filters(struct archive_read *); int __archive_read_close_filters(struct archive_read *); struct archive_read_extract *__archive_read_get_extract(struct archive_read *); + + +/* + * Get a decryption passphrase. + */ +void __archive_read_reset_passphrase(struct archive_read *a); +const char * __archive_read_next_passphrase(struct archive_read *a); #endif diff --git a/Utilities/cmlibarchive/libarchive/archive_read_set_options.c b/Utilities/cmlibarchive/libarchive/archive_read_set_options.c index 46678b1..2e2eea6 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_set_options.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_set_options.c @@ -76,14 +76,13 @@ archive_set_format_option(struct archive *_a, const char *m, const char *o, const char *v) { struct archive_read *a = (struct archive_read *)_a; - struct archive_format_descriptor *format; size_t i; int r, rv = ARCHIVE_WARN, matched_modules = 0; for (i = 0; i < sizeof(a->formats)/sizeof(a->formats[0]); i++) { - format = &a->formats[i]; - if (format == NULL || format->options == NULL || - format->name == NULL) + struct archive_format_descriptor *format = &a->formats[i]; + + if (format->options == NULL || format->name == NULL) /* This format does not support option. */ continue; if (m != NULL) { diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c index b778cfb..68c53de 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_all.c @@ -69,6 +69,8 @@ archive_read_support_filter_all(struct archive *a) archive_read_support_filter_lzop(a); /* The decode code always uses "grzip -d" command-line. */ archive_read_support_filter_grzip(a); + /* Lz4 falls back to "lz4 -d" command-line program. */ + archive_read_support_filter_lz4(a); /* Note: We always return ARCHIVE_OK here, even if some of the * above return ARCHIVE_WARN. The intent here is to enable diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_compress.c b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_compress.c index 3f5d1f3..e05132d 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_compress.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_compress.c @@ -185,19 +185,22 @@ compress_bidder_bid(struct archive_read_filter_bidder *self, (void)self; /* UNUSED */ - buffer = __archive_read_filter_ahead(filter, 2, &avail); + /* Shortest valid compress file is 3 bytes. */ + buffer = __archive_read_filter_ahead(filter, 3, &avail); if (buffer == NULL) return (0); bits_checked = 0; + /* First two bytes are the magic value */ if (buffer[0] != 0x1F || buffer[1] != 0x9D) return (0); - bits_checked += 16; - - /* - * TODO: Verify more. - */ + /* Third byte holds compression parameters. */ + if (buffer[2] & 0x20) /* Reserved bit, must be zero. */ + return (0); + if (buffer[2] & 0x40) /* Reserved bit, must be zero. */ + return (0); + bits_checked += 18; return (bits_checked); } @@ -239,7 +242,13 @@ compress_bidder_init(struct archive_read_filter *self) (void)getbits(self, 8); /* Skip first signature byte. */ (void)getbits(self, 8); /* Skip second signature byte. */ + /* Get compression parameters. */ code = getbits(self, 8); + if ((code & 0x1f) > 16) { + archive_set_error(&self->archive->archive, -1, + "Invalid compressed data"); + return (ARCHIVE_FATAL); + } state->maxcode_bits = code & 0x1f; state->maxcode = (1 << state->maxcode_bits); state->use_reset_code = code & 0x80; @@ -368,7 +377,8 @@ next_code(struct archive_read_filter *self) return (next_code(self)); } - if (code > state->free_ent) { + if (code > state->free_ent + || (code == state->free_ent && state->oldcode < 0)) { /* An invalid code is a fatal error. */ archive_set_error(&(self->archive->archive), -1, "Invalid compressed data"); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_lz4.c b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_lz4.c new file mode 100644 index 0000000..db62cb3 --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_lz4.c @@ -0,0 +1,728 @@ +/*- + * Copyright (c) 2014 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" + +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#include <stdio.h> +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#ifdef HAVE_UNISTD_H +#include <unistd.h> +#endif +#ifdef HAVE_LZ4_H +#include <lz4.h> +#endif + +#include "archive.h" +#include "archive_endian.h" +#include "archive_private.h" +#include "archive_read_private.h" +#include "archive_xxhash.h" + +#define LZ4_MAGICNUMBER 0x184d2204 +#define LZ4_SKIPPABLED 0x184d2a50 +#define LZ4_LEGACY 0x184c2102 + +#if defined(HAVE_LIBLZ4) +struct private_data { + enum { SELECT_STREAM, + READ_DEFAULT_STREAM, + READ_DEFAULT_BLOCK, + READ_LEGACY_STREAM, + READ_LEGACY_BLOCK, + } stage; + struct { + unsigned block_independence:1; + unsigned block_checksum:3; + unsigned stream_size:1; + unsigned stream_checksum:1; + unsigned preset_dictionary:1; + int block_maximum_size; + } flags; + int64_t stream_size; + uint32_t dict_id; + char *out_block; + size_t out_block_size; + + /* Bytes read but not yet consumed via __archive_read_consume() */ + size_t unconsumed; + size_t decoded_size; + void *xxh32_state; + + char valid; /* True = decompressor is initialized */ + char eof; /* True = found end of compressed data. */ +}; + +#define LEGACY_BLOCK_SIZE (8 * 1024 * 1024) + +/* Lz4 filter */ +static ssize_t lz4_filter_read(struct archive_read_filter *, const void **); +static int lz4_filter_close(struct archive_read_filter *); +#endif + +/* + * Note that we can detect lz4 archives even if we can't decompress + * them. (In fact, we like detecting them because we can give better + * error messages.) So the bid framework here gets compiled even + * if liblz4 is unavailable. + */ +static int lz4_reader_bid(struct archive_read_filter_bidder *, struct archive_read_filter *); +static int lz4_reader_init(struct archive_read_filter *); +static int lz4_reader_free(struct archive_read_filter_bidder *); +#if defined(HAVE_LIBLZ4) +static ssize_t lz4_filter_read_default_stream(struct archive_read_filter *, + const void **); +static ssize_t lz4_filter_read_legacy_stream(struct archive_read_filter *, + const void **); +#endif + +int +archive_read_support_filter_lz4(struct archive *_a) +{ + struct archive_read *a = (struct archive_read *)_a; + struct archive_read_filter_bidder *reader; + + archive_check_magic(_a, ARCHIVE_READ_MAGIC, + ARCHIVE_STATE_NEW, "archive_read_support_filter_lz4"); + + if (__archive_read_get_bidder(a, &reader) != ARCHIVE_OK) + return (ARCHIVE_FATAL); + + reader->data = NULL; + reader->name = "lz4"; + reader->bid = lz4_reader_bid; + reader->init = lz4_reader_init; + reader->options = NULL; + reader->free = lz4_reader_free; +#if defined(HAVE_LIBLZ4) + return (ARCHIVE_OK); +#else + archive_set_error(_a, ARCHIVE_ERRNO_MISC, + "Using external lz4 program"); + return (ARCHIVE_WARN); +#endif +} + +static int +lz4_reader_free(struct archive_read_filter_bidder *self){ + (void)self; /* UNUSED */ + return (ARCHIVE_OK); +} + +/* + * Test whether we can handle this data. + * + * This logic returns zero if any part of the signature fails. It + * also tries to Do The Right Thing if a very short buffer prevents us + * from verifying as much as we would like. + */ +static int +lz4_reader_bid(struct archive_read_filter_bidder *self, + struct archive_read_filter *filter) +{ + const unsigned char *buffer; + ssize_t avail; + int bits_checked; + uint32_t number; + + (void)self; /* UNUSED */ + + /* Minimal lz4 archive is 11 bytes. */ + buffer = __archive_read_filter_ahead(filter, 11, &avail); + if (buffer == NULL) + return (0); + + /* First four bytes must be LZ4 magic numbers. */ + bits_checked = 0; + if ((number = archive_le32dec(buffer)) == LZ4_MAGICNUMBER) { + unsigned char flag, BD; + + bits_checked += 32; + /* Next follows a stream descriptor. */ + /* Descriptor Flags. */ + flag = buffer[4]; + /* A version number must be "01". */ + if (((flag & 0xc0) >> 6) != 1) + return (0); + /* A reserved bit must be "0". */ + if (flag & 2) + return (0); + bits_checked += 8; + BD = buffer[5]; + /* A block maximum size shuld be more than 3. */ + if (((BD & 0x70) >> 4) < 4) + return (0); + /* Reserved bits must be "0". */ + if (BD & ~0x70) + return (0); + bits_checked += 8; + } else if (number == LZ4_LEGACY) { + bits_checked += 32; + } + + return (bits_checked); +} + +#if !defined(HAVE_LIBLZ4) + +/* + * If we don't have the library on this system, we can't actually do the + * decompression. We can, however, still detect compressed archives + * and emit a useful message. + */ +static int +lz4_reader_init(struct archive_read_filter *self) +{ + int r; + + r = __archive_read_program(self, "lz4 -d -q"); + /* Note: We set the format here even if __archive_read_program() + * above fails. We do, after all, know what the format is + * even if we weren't able to read it. */ + self->code = ARCHIVE_FILTER_LZ4; + self->name = "lz4"; + return (r); +} + + +#else + +/* + * Setup the callbacks. + */ +static int +lz4_reader_init(struct archive_read_filter *self) +{ + struct private_data *state; + + self->code = ARCHIVE_FILTER_LZ4; + self->name = "lz4"; + + state = (struct private_data *)calloc(sizeof(*state), 1); + if (state == NULL) { + archive_set_error(&self->archive->archive, ENOMEM, + "Can't allocate data for lz4 decompression"); + return (ARCHIVE_FATAL); + } + + self->data = state; + state->stage = SELECT_STREAM; + self->read = lz4_filter_read; + self->skip = NULL; /* not supported */ + self->close = lz4_filter_close; + + return (ARCHIVE_OK); +} + +static int +lz4_allocate_out_block(struct archive_read_filter *self) +{ + struct private_data *state = (struct private_data *)self->data; + size_t out_block_size = state->flags.block_maximum_size; + void *out_block; + + if (!state->flags.block_independence) + out_block_size += 64 * 1024; + if (state->out_block_size < out_block_size) { + free(state->out_block); + out_block = (unsigned char *)malloc(out_block_size); + state->out_block_size = out_block_size; + if (out_block == NULL) { + archive_set_error(&self->archive->archive, ENOMEM, + "Can't allocate data for lz4 decompression"); + return (ARCHIVE_FATAL); + } + state->out_block = out_block; + } + if (!state->flags.block_independence) + memset(state->out_block, 0, 64 * 1024); + return (ARCHIVE_OK); +} + +static int +lz4_allocate_out_block_for_legacy(struct archive_read_filter *self) +{ + struct private_data *state = (struct private_data *)self->data; + size_t out_block_size = LEGACY_BLOCK_SIZE; + void *out_block; + + if (state->out_block_size < out_block_size) { + free(state->out_block); + out_block = (unsigned char *)malloc(out_block_size); + state->out_block_size = out_block_size; + if (out_block == NULL) { + archive_set_error(&self->archive->archive, ENOMEM, + "Can't allocate data for lz4 decompression"); + return (ARCHIVE_FATAL); + } + state->out_block = out_block; + } + return (ARCHIVE_OK); +} + +/* + * Return the next block of decompressed data. + */ +static ssize_t +lz4_filter_read(struct archive_read_filter *self, const void **p) +{ + struct private_data *state = (struct private_data *)self->data; + ssize_t ret; + + if (state->eof) { + *p = NULL; + return (0); + } + + __archive_read_filter_consume(self->upstream, state->unconsumed); + state->unconsumed = 0; + + switch (state->stage) { + case SELECT_STREAM: + break; + case READ_DEFAULT_STREAM: + case READ_LEGACY_STREAM: + /* Reading a lz4 stream already failed. */ + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, "Invalid sequence."); + return (ARCHIVE_FATAL); + case READ_DEFAULT_BLOCK: + ret = lz4_filter_read_default_stream(self, p); + if (ret != 0 || state->stage != SELECT_STREAM) + return ret; + break; + case READ_LEGACY_BLOCK: + ret = lz4_filter_read_legacy_stream(self, p); + if (ret != 0 || state->stage != SELECT_STREAM) + return ret; + break; + default: + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, "Program error."); + return (ARCHIVE_FATAL); + break; + } + + while (state->stage == SELECT_STREAM) { + const char *read_buf; + + /* Read a magic number. */ + read_buf = __archive_read_filter_ahead(self->upstream, 4, + NULL); + if (read_buf == NULL) { + state->eof = 1; + *p = NULL; + return (0); + } + uint32_t number = archive_le32dec(read_buf); + __archive_read_filter_consume(self->upstream, 4); + if (number == LZ4_MAGICNUMBER) + return lz4_filter_read_default_stream(self, p); + else if (number == LZ4_LEGACY) + return lz4_filter_read_legacy_stream(self, p); + else if ((number & ~0xF) == LZ4_SKIPPABLED) { + read_buf = __archive_read_filter_ahead( + self->upstream, 4, NULL); + if (read_buf == NULL) { + archive_set_error( + &self->archive->archive, + ARCHIVE_ERRNO_MISC, + "Malformed lz4 data"); + return (ARCHIVE_FATAL); + } + uint32_t skip_bytes = archive_le32dec(read_buf); + __archive_read_filter_consume(self->upstream, + 4 + skip_bytes); + } else { + /* Ignore following unrecognized data. */ + state->eof = 1; + *p = NULL; + return (0); + } + } + state->eof = 1; + *p = NULL; + return (0); +} + +static int +lz4_filter_read_descriptor(struct archive_read_filter *self) +{ + struct private_data *state = (struct private_data *)self->data; + const char *read_buf; + ssize_t bytes_remaining; + ssize_t descriptor_bytes; + unsigned char flag, bd; + unsigned int chsum, chsum_verifier; + + /* Make sure we have 2 bytes for flags. */ + read_buf = __archive_read_filter_ahead(self->upstream, 2, + &bytes_remaining); + if (read_buf == NULL) { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "truncated lz4 input"); + return (ARCHIVE_FATAL); + } + + /* + Parse flags. + */ + flag = (unsigned char)read_buf[0]; + /* Verify version number. */ + if ((flag & 0xc0) != 1<<6) + goto malformed_error; + /* A reserved bit must be zero. */ + if (flag & 0x02) + goto malformed_error; + state->flags.block_independence = (flag & 0x20) != 0; + state->flags.block_checksum = (flag & 0x10)?4:0; + state->flags.stream_size = (flag & 0x08) != 0; + state->flags.stream_checksum = (flag & 0x04) != 0; + state->flags.preset_dictionary = (flag & 0x01) != 0; + + /* BD */ + bd = (unsigned char)read_buf[1]; + /* Reserved bits must be zero. */ + if (bd & 0x8f) + goto malformed_error; + /* Get a maxinum block size. */ + switch (read_buf[1] >> 4) { + case 4: /* 64 KB */ + state->flags.block_maximum_size = 64 * 1024; + break; + case 5: /* 256 KB */ + state->flags.block_maximum_size = 256 * 1024; + break; + case 6: /* 1 MB */ + state->flags.block_maximum_size = 1024 * 1024; + break; + case 7: /* 4 MB */ + state->flags.block_maximum_size = 4 * 1024 * 1024; + break; + default: + goto malformed_error; + } + + /* Read the whole descriptor in a stream block. */ + descriptor_bytes = 3; + if (state->flags.stream_size) + descriptor_bytes += 8; + if (state->flags.preset_dictionary) + descriptor_bytes += 4; + if (bytes_remaining < descriptor_bytes) { + read_buf = __archive_read_filter_ahead(self->upstream, + descriptor_bytes, &bytes_remaining); + if (read_buf == NULL) { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "truncated lz4 input"); + return (ARCHIVE_FATAL); + } + } + /* Check if a descriptor is corrupted */ + chsum = __archive_xxhash.XXH32(read_buf, (int)descriptor_bytes -1, 0); + chsum = (chsum >> 8) & 0xff; + chsum_verifier = read_buf[descriptor_bytes-1] & 0xff; + if (chsum != chsum_verifier) + goto malformed_error; + + __archive_read_filter_consume(self->upstream, descriptor_bytes); + + /* Make sure we have an enough buffer for uncompressed data. */ + if (lz4_allocate_out_block(self) != ARCHIVE_OK) + return (ARCHIVE_FATAL); + if (state->flags.stream_checksum) + state->xxh32_state = __archive_xxhash.XXH32_init(0); + + state->decoded_size = 0; + /* Success */ + return (ARCHIVE_OK); +malformed_error: + archive_set_error(&self->archive->archive, ARCHIVE_ERRNO_MISC, + "malformed lz4 data"); + return (ARCHIVE_FATAL); +} + +static ssize_t +lz4_filter_read_data_block(struct archive_read_filter *self, const void **p) +{ + struct private_data *state = (struct private_data *)self->data; + ssize_t compressed_size; + const char *read_buf; + ssize_t bytes_remaining; + int checksum_size; + ssize_t uncompressed_size; + size_t prefix64k; + + *p = NULL; + + /* Make sure we have 4 bytes for a block size. */ + read_buf = __archive_read_filter_ahead(self->upstream, 4, + &bytes_remaining); + if (read_buf == NULL) + goto truncated_error; + compressed_size = archive_le32dec(read_buf); + if ((compressed_size & ~(1 << 31)) > state->flags.block_maximum_size) + goto malformed_error; + /* A compressed size == 0 means the end of stream blocks. */ + if (compressed_size == 0) { + __archive_read_filter_consume(self->upstream, 4); + return 0; + } + + checksum_size = state->flags.block_checksum; + /* Check if the block is uncompressed. */ + if (compressed_size & (1 << 31)) { + compressed_size &= ~(1 << 31); + uncompressed_size = compressed_size; + } else + uncompressed_size = 0;/* Unknown yet. */ + + /* + Unfortunately, lz4 decompression API requires a whole block + for its decompression speed, so we read a whole block and allocate + a huge buffer used for decoded data. + */ + read_buf = __archive_read_filter_ahead(self->upstream, + 4 + compressed_size + checksum_size, &bytes_remaining); + if (read_buf == NULL) + goto truncated_error; + + /* Optional process, checking a block sum. */ + if (checksum_size) { + unsigned int chsum = __archive_xxhash.XXH32( + read_buf + 4, (int)compressed_size, 0); + unsigned int chsum_block = + archive_le32dec(read_buf + 4 + compressed_size); + if (chsum != chsum_block) + goto malformed_error; + } + + + /* If the block is uncompressed, there is nothing to do. */ + if (uncompressed_size) { + /* Prepare a prefix 64k block for next block. */ + if (!state->flags.block_independence) { + prefix64k = 64 * 1024; + if (uncompressed_size < (ssize_t)prefix64k) { + memcpy(state->out_block + + prefix64k - uncompressed_size, + read_buf + 4, + uncompressed_size); + memset(state->out_block, 0, + prefix64k - uncompressed_size); + } else { + memcpy(state->out_block, + read_buf + 4 + + uncompressed_size - prefix64k, + prefix64k); + } + state->decoded_size = 0; + } + state->unconsumed = 4 + uncompressed_size + checksum_size; + *p = read_buf + 4; + return uncompressed_size; + } + + /* + Decompress a block data. + */ + if (state->flags.block_independence) { + prefix64k = 0; + uncompressed_size = LZ4_decompress_safe(read_buf + 4, + state->out_block, (int)compressed_size, + state->flags.block_maximum_size); + } else { + prefix64k = 64 * 1024; + if (state->decoded_size) { + if (state->decoded_size < prefix64k) { + memmove(state->out_block + + prefix64k - state->decoded_size, + state->out_block + prefix64k, + state->decoded_size); + memset(state->out_block, 0, + prefix64k - state->decoded_size); + } else { + memmove(state->out_block, + state->out_block + state->decoded_size, + prefix64k); + } + } + uncompressed_size = LZ4_decompress_safe_withPrefix64k( + read_buf + 4, + state->out_block + prefix64k, (int)compressed_size, + state->flags.block_maximum_size); + } + + /* Check if an error happend in decompression process. */ + if (uncompressed_size < 0) { + archive_set_error(&(self->archive->archive), + ARCHIVE_ERRNO_MISC, "lz4 decompression failed"); + return (ARCHIVE_FATAL); + } + + state->unconsumed = 4 + compressed_size + checksum_size; + *p = state->out_block + prefix64k; + state->decoded_size = uncompressed_size; + return uncompressed_size; + +malformed_error: + archive_set_error(&self->archive->archive, ARCHIVE_ERRNO_MISC, + "malformed lz4 data"); + return (ARCHIVE_FATAL); +truncated_error: + archive_set_error(&self->archive->archive, ARCHIVE_ERRNO_MISC, + "truncated lz4 input"); + return (ARCHIVE_FATAL); +} + +static ssize_t +lz4_filter_read_default_stream(struct archive_read_filter *self, const void **p) +{ + struct private_data *state = (struct private_data *)self->data; + const char *read_buf; + ssize_t bytes_remaining; + ssize_t ret; + + if (state->stage == SELECT_STREAM) { + state->stage = READ_DEFAULT_STREAM; + /* First, read a desciprtor. */ + if((ret = lz4_filter_read_descriptor(self)) != ARCHIVE_OK) + return (ret); + state->stage = READ_DEFAULT_BLOCK; + } + /* Decompress a block. */ + ret = lz4_filter_read_data_block(self, p); + + /* If the end of block is detected, change the filter status + to read next stream. */ + if (ret == 0 && *p == NULL) + state->stage = SELECT_STREAM; + + /* Optional process, checking a stream sum. */ + if (state->flags.stream_checksum) { + if (state->stage == SELECT_STREAM) { + unsigned int checksum; + unsigned int checksum_stream; + read_buf = __archive_read_filter_ahead(self->upstream, + 4, &bytes_remaining); + if (read_buf == NULL) { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, "truncated lz4 input"); + return (ARCHIVE_FATAL); + } + checksum = archive_le32dec(read_buf); + __archive_read_filter_consume(self->upstream, 4); + checksum_stream = __archive_xxhash.XXH32_digest( + state->xxh32_state); + state->xxh32_state = NULL; + if (checksum != checksum_stream) { + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "lz4 stream cheksum error"); + return (ARCHIVE_FATAL); + } + } else if (ret > 0) + __archive_xxhash.XXH32_update(state->xxh32_state, + *p, (int)ret); + } + return (ret); +} + +static ssize_t +lz4_filter_read_legacy_stream(struct archive_read_filter *self, const void **p) +{ + struct private_data *state = (struct private_data *)self->data; + int compressed; + const char *read_buf; + ssize_t ret; + + *p = NULL; + ret = lz4_allocate_out_block_for_legacy(self); + if (ret != ARCHIVE_OK) + return ret; + + /* Make sure we have 4 bytes for a block size. */ + read_buf = __archive_read_filter_ahead(self->upstream, 4, NULL); + if (read_buf == NULL) { + if (state->stage == SELECT_STREAM) { + state->stage = READ_LEGACY_STREAM; + archive_set_error(&self->archive->archive, + ARCHIVE_ERRNO_MISC, + "truncated lz4 input"); + return (ARCHIVE_FATAL); + } + state->stage = SELECT_STREAM; + return 0; + } + state->stage = READ_LEGACY_BLOCK; + compressed = archive_le32dec(read_buf); + if (compressed > LZ4_COMPRESSBOUND(LEGACY_BLOCK_SIZE)) { + state->stage = SELECT_STREAM; + return 0; + } + + /* Make sure we have a whole block. */ + read_buf = __archive_read_filter_ahead(self->upstream, + 4 + compressed, NULL); + ret = LZ4_decompress_safe(read_buf + 4, state->out_block, + compressed, (int)state->out_block_size); + if (ret < 0) { + archive_set_error(&(self->archive->archive), + ARCHIVE_ERRNO_MISC, "lz4 decompression failed"); + return (ARCHIVE_FATAL); + } + *p = state->out_block; + state->unconsumed = 4 + compressed; + return ret; +} + +/* + * Clean up the decompressor. + */ +static int +lz4_filter_close(struct archive_read_filter *self) +{ + struct private_data *state; + int ret = ARCHIVE_OK; + + state = (struct private_data *)self->data; + free(state->xxh32_state); + free(state->out_block); + free(state); + return (ret); +} + +#endif /* HAVE_LIBLZ4 */ diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_lzop.c b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_lzop.c index 9ef2054..23c18c7 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_lzop.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_lzop.c @@ -242,10 +242,17 @@ consume_header(struct archive_read_filter *self) if (version >= 0x940) { unsigned level = *p++; +#if 0 unsigned default_level[] = {0, 3, 1, 9}; +#endif if (level == 0) /* Method is 1..3 here due to check above. */ +#if 0 /* Avoid an error Clang Static Analyzer claims + "Value stored to 'level' is never read". */ level = default_level[method]; +#else + ;/* NOP */ +#endif else if (level > 9) { archive_set_error(&self->archive->archive, ARCHIVE_ERRNO_MISC, "Invalid level"); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_uu.c b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_uu.c index 471771b..787a619 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_uu.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_uu.c @@ -509,7 +509,7 @@ read_more: return (ARCHIVE_FATAL); } llen = len; - if (nl == 0) { + if ((nl == 0) && (uudecode->state != ST_UUEND)) { /* * Save remaining data which does not contain * NL('\n','\r'). @@ -527,6 +527,7 @@ read_more: self->upstream, ravail); goto read_more; } + used += len; break; } switch (uudecode->state) { diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_xz.c b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_xz.c index a3ad67a..8f48273 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_filter_xz.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_filter_xz.c @@ -627,7 +627,7 @@ lzip_tail(struct archive_read_filter *self) f = __archive_read_filter_ahead(self->upstream, tail, &avail_in); if (f == NULL && avail_in < 0) return (ARCHIVE_FATAL); - if (avail_in < tail) { + if (f == NULL || avail_in < tail) { archive_set_error(&self->archive->archive, ARCHIVE_ERRNO_MISC, "Lzip: Remaining data is less bytes"); return (ARCHIVE_FAILED); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c index 47e0752..f045b8f 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_7zip.c @@ -108,6 +108,7 @@ __FBSDID("$FreeBSD$"); #define kMTime 0x14 #define kAttributes 0x15 #define kEncodedHeader 0x17 +#define kDummy 0x19 struct _7z_digests { unsigned char *defineds; @@ -331,6 +332,11 @@ struct _7zip { int has_encrypted_entries; }; +/* Maximum entry size. This limitation prevents reading intentional + * corrupted 7-zip files on assuming there are not so many entries in + * the files. */ +#define UMAX_ENTRY ARCHIVE_LITERAL_ULL(100000000) + static int archive_read_format_7zip_has_encrypted_entries(struct archive_read *); static int archive_read_support_format_7zip_capabilities(struct archive_read *a); static int archive_read_format_7zip_bid(struct archive_read *, int); @@ -640,7 +646,7 @@ archive_read_format_7zip_read_header(struct archive_read *a, } zip_entry = zip->entry; - if (zip->entries_remaining <= 0) + if (zip->entries_remaining <= 0 || zip_entry == NULL) return ARCHIVE_EOF; --zip->entries_remaining; @@ -1663,7 +1669,7 @@ parse_7zip_uint64(struct archive_read *a, uint64_t *val) mask >>= 1; continue; } - *val += (avail & (mask -1)) << (8 * i); + *val += ((uint64_t)(avail & (mask -1))) << (8 * i); break; } return (0); @@ -1763,7 +1769,7 @@ read_PackInfo(struct archive_read *a, struct _7z_pack_info *pi) return (-1); if (pi->numPackStreams == 0) return (-1); - if (1000000 < pi->numPackStreams) + if (UMAX_ENTRY < pi->numPackStreams) return (-1); /* @@ -1892,12 +1898,12 @@ read_Folder(struct archive_read *a, struct _7z_folder *f) if (parse_7zip_uint64( a, &(f->coders[i].numInStreams)) < 0) return (-1); - if (1000000 < f->coders[i].numInStreams) + if (UMAX_ENTRY < f->coders[i].numInStreams) return (-1); if (parse_7zip_uint64( a, &(f->coders[i].numOutStreams)) < 0) return (-1); - if (1000000 < f->coders[i].numOutStreams) + if (UMAX_ENTRY < f->coders[i].numOutStreams) return (-1); } @@ -1937,11 +1943,11 @@ read_Folder(struct archive_read *a, struct _7z_folder *f) for (i = 0; i < f->numBindPairs; i++) { if (parse_7zip_uint64(a, &(f->bindPairs[i].inIndex)) < 0) return (-1); - if (1000000 < f->bindPairs[i].inIndex) + if (UMAX_ENTRY < f->bindPairs[i].inIndex) return (-1); if (parse_7zip_uint64(a, &(f->bindPairs[i].outIndex)) < 0) return (-1); - if (1000000 < f->bindPairs[i].outIndex) + if (UMAX_ENTRY < f->bindPairs[i].outIndex) return (-1); } @@ -1967,7 +1973,7 @@ read_Folder(struct archive_read *a, struct _7z_folder *f) for (i = 0; i < f->numPackedStreams; i++) { if (parse_7zip_uint64(a, &(f->packedStreams[i])) < 0) return (-1); - if (1000000 < f->packedStreams[i]) + if (UMAX_ENTRY < f->packedStreams[i]) return (-1); } } @@ -2009,8 +2015,8 @@ read_CodersInfo(struct archive_read *a, struct _7z_coders_info *ci) */ if (parse_7zip_uint64(a, &(ci->numFolders)) < 0) goto failed; - if (1000000 < ci->numFolders) - return (-1); + if (UMAX_ENTRY < ci->numFolders) + return (-1); /* * Read External. @@ -2031,9 +2037,18 @@ read_CodersInfo(struct archive_read *a, struct _7z_coders_info *ci) case 1: if (parse_7zip_uint64(a, &(ci->dataStreamIndex)) < 0) return (-1); - if (1000000 < ci->dataStreamIndex) + if (UMAX_ENTRY < ci->dataStreamIndex) return (-1); + if (ci->numFolders > 0) { + archive_set_error(&a->archive, -1, + "Malformed 7-Zip archive"); + goto failed; + } break; + default: + archive_set_error(&a->archive, -1, + "Malformed 7-Zip archive"); + goto failed; } if ((p = header_bytes(a, 1)) == NULL) @@ -2136,7 +2151,7 @@ read_SubStreamsInfo(struct archive_read *a, struct _7z_substream_info *ss, for (i = 0; i < numFolders; i++) { if (parse_7zip_uint64(a, &(f[i].numUnpackStreams)) < 0) return (-1); - if (1000000 < f[i].numUnpackStreams) + if (UMAX_ENTRY < f[i].numUnpackStreams) return (-1); unpack_streams += (size_t)f[i].numUnpackStreams; } @@ -2385,8 +2400,8 @@ read_Header(struct archive_read *a, struct _7z_header_info *h, if (parse_7zip_uint64(a, &(zip->numFiles)) < 0) return (-1); - if (1000000 < zip->numFiles) - return (-1); + if (UMAX_ENTRY < zip->numFiles) + return (-1); zip->entries = calloc((size_t)zip->numFiles, sizeof(*zip->entries)); if (zip->entries == NULL) @@ -2545,6 +2560,9 @@ read_Header(struct archive_read *a, struct _7z_header_info *h, } break; } + case kDummy: + if (ll == 0) + break; default: if (header_bytes(a, ll) == NULL) return (-1); @@ -2684,7 +2702,7 @@ read_Times(struct archive_read *a, struct _7z_header_info *h, int type) if (*p) { if (parse_7zip_uint64(a, &(h->dataIndex)) < 0) goto failed; - if (1000000 < h->dataIndex) + if (UMAX_ENTRY < h->dataIndex) goto failed; } diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_all.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_all.c index 53fe6fa..2127ebd3 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_all.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_all.c @@ -61,6 +61,7 @@ archive_read_support_format_all(struct archive *a) archive_read_support_format_mtree(a); archive_read_support_format_tar(a); archive_read_support_format_xar(a); + archive_read_support_format_warc(a); /* * Install expensive bidders last. By doing them last, we diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_ar.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_ar.c index 82756c9..4b5b66b 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_ar.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_ar.c @@ -180,7 +180,7 @@ _ar_read_header(struct archive_read *a, struct archive_entry *entry, if (strncmp(h + AR_fmag_offset, "`\n", 2) != 0) { archive_set_error(&a->archive, EINVAL, "Incorrect file header signature"); - return (ARCHIVE_WARN); + return (ARCHIVE_FATAL); } /* Copy filename into work buffer. */ @@ -239,8 +239,15 @@ _ar_read_header(struct archive_read *a, struct archive_entry *entry, * and are not terminated in '/', so we don't trim anything * that starts with '/'.) */ - if (filename[0] != '/' && *p == '/') + if (filename[0] != '/' && p > filename && *p == '/') { *p = '\0'; + } + + if (p < filename) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Found entry with empty filename"); + return (ARCHIVE_FATAL); + } /* * '//' is the GNU filename table. @@ -262,12 +269,12 @@ _ar_read_header(struct archive_read *a, struct archive_entry *entry, if (entry_size == 0) { archive_set_error(&a->archive, EINVAL, "Invalid string table"); - return (ARCHIVE_WARN); + return (ARCHIVE_FATAL); } if (ar->strtab != NULL) { archive_set_error(&a->archive, EINVAL, "More than one string tables exist"); - return (ARCHIVE_WARN); + return (ARCHIVE_FATAL); } /* Read the filename table into memory. */ @@ -311,11 +318,11 @@ _ar_read_header(struct archive_read *a, struct archive_entry *entry, */ if (ar->strtab == NULL || number > ar->strtab_size) { archive_set_error(&a->archive, EINVAL, - "Can't find long filename for entry"); + "Can't find long filename for GNU/SVR4 archive entry"); archive_entry_copy_pathname(entry, filename); /* Parse the time, owner, mode, size fields. */ ar_parse_common_header(ar, entry, h); - return (ARCHIVE_WARN); + return (ARCHIVE_FATAL); } archive_entry_copy_pathname(entry, &ar->strtab[(size_t)number]); @@ -573,7 +580,7 @@ bad_string_table: "Invalid string table"); free(ar->strtab); ar->strtab = NULL; - return (ARCHIVE_WARN); + return (ARCHIVE_FATAL); } static uint64_t diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c index 0b69689..c2ca85b 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_cpio.c @@ -198,7 +198,7 @@ static int archive_read_format_cpio_read_data(struct archive_read *, static int archive_read_format_cpio_read_header(struct archive_read *, struct archive_entry *); static int archive_read_format_cpio_skip(struct archive_read *); -static int be4(const unsigned char *); +static int64_t be4(const unsigned char *); static int find_odc_header(struct archive_read *); static int find_newc_header(struct archive_read *); static int header_bin_be(struct archive_read *, struct cpio *, @@ -213,7 +213,7 @@ static int header_afiol(struct archive_read *, struct cpio *, struct archive_entry *, size_t *, size_t *); static int is_octal(const char *, size_t); static int is_hex(const char *, size_t); -static int le4(const unsigned char *); +static int64_t le4(const unsigned char *); static int record_hardlink(struct archive_read *a, struct cpio *cpio, struct archive_entry *entry); @@ -866,8 +866,11 @@ header_bin_le(struct archive_read *a, struct cpio *cpio, /* Read fixed-size portion of header. */ h = __archive_read_ahead(a, bin_header_size, NULL); - if (h == NULL) + if (h == NULL) { + archive_set_error(&a->archive, 0, + "End of file trying to read next cpio header"); return (ARCHIVE_FATAL); + } /* Parse out binary fields. */ header = (const unsigned char *)h; @@ -902,8 +905,11 @@ header_bin_be(struct archive_read *a, struct cpio *cpio, /* Read fixed-size portion of header. */ h = __archive_read_ahead(a, bin_header_size, NULL); - if (h == NULL) + if (h == NULL) { + archive_set_error(&a->archive, 0, + "End of file trying to read next cpio header"); return (ARCHIVE_FATAL); + } /* Parse out binary fields. */ header = (const unsigned char *)h; @@ -946,17 +952,17 @@ archive_read_format_cpio_cleanup(struct archive_read *a) return (ARCHIVE_OK); } -static int +static int64_t le4(const unsigned char *p) { - return ((p[0]<<16) + (p[1]<<24) + (p[2]<<0) + (p[3]<<8)); + return ((p[0] << 16) + (((int64_t)p[1]) << 24) + (p[2] << 0) + (p[3] << 8)); } -static int +static int64_t be4(const unsigned char *p) { - return ((p[0]<<24) + (p[1]<<16) + (p[2]<<8) + (p[3])); + return ((((int64_t)p[0]) << 24) + (p[1] << 16) + (p[2] << 8) + (p[3])); } /* diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c index 2219e61..3628025 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_iso9660.c @@ -387,7 +387,7 @@ static int archive_read_format_iso9660_read_data(struct archive_read *, static int archive_read_format_iso9660_read_data_skip(struct archive_read *); static int archive_read_format_iso9660_read_header(struct archive_read *, struct archive_entry *); -static const char *build_pathname(struct archive_string *, struct file_info *); +static const char *build_pathname(struct archive_string *, struct file_info *, int); static int build_pathname_utf16be(unsigned char *, size_t, size_t *, struct file_info *); #if DEBUG @@ -1225,6 +1225,7 @@ archive_read_format_iso9660_read_header(struct archive_read *a, archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, "Pathname is too long"); + return (ARCHIVE_FATAL); } r = archive_entry_copy_pathname_l(entry, @@ -1247,9 +1248,16 @@ archive_read_format_iso9660_read_header(struct archive_read *a, rd_r = ARCHIVE_WARN; } } else { - archive_string_empty(&iso9660->pathname); - archive_entry_set_pathname(entry, - build_pathname(&iso9660->pathname, file)); + const char *path = build_pathname(&iso9660->pathname, file, 0); + if (path == NULL) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, + "Pathname is too long"); + return (ARCHIVE_FATAL); + } else { + archive_string_empty(&iso9660->pathname); + archive_entry_set_pathname(entry, path); + } } iso9660->entry_bytes_remaining = file->size; @@ -1744,12 +1752,12 @@ parse_file_info(struct archive_read *a, struct file_info *parent, const unsigned char *isodirrec) { struct iso9660 *iso9660; - struct file_info *file; + struct file_info *file, *filep; size_t name_len; const unsigned char *rr_start, *rr_end; const unsigned char *p; size_t dr_len; - uint64_t fsize; + uint64_t fsize, offset; int32_t location; int flags; @@ -1793,6 +1801,16 @@ parse_file_info(struct archive_read *a, struct file_info *parent, return (NULL); } + /* Sanity check that this entry does not create a cycle. */ + offset = iso9660->logical_block_size * (uint64_t)location; + for (filep = parent; filep != NULL; filep = filep->parent) { + if (filep->offset == offset) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Directory structure contains loop"); + return (NULL); + } + } + /* Create a new file entry and copy data from the ISO dir record. */ file = (struct file_info *)calloc(1, sizeof(*file)); if (file == NULL) { @@ -1801,7 +1819,7 @@ parse_file_info(struct archive_read *a, struct file_info *parent, return (NULL); } file->parent = parent; - file->offset = iso9660->logical_block_size * (uint64_t)location; + file->offset = offset; file->size = fsize; file->mtime = isodate7(isodirrec + DR_date_offset); file->ctime = file->atime = file->mtime; @@ -3147,29 +3165,39 @@ static time_t time_from_tm(struct tm *t) { #if HAVE_TIMEGM - /* Use platform timegm() if available. */ - return (timegm(t)); + /* Use platform timegm() if available. */ + return (timegm(t)); #elif HAVE__MKGMTIME64 - return (_mkgmtime64(t)); + return (_mkgmtime64(t)); #else - /* Else use direct calculation using POSIX assumptions. */ - /* First, fix up tm_yday based on the year/month/day. */ - if (mktime(t) == (time_t)-1) - return ((time_t)-1); - /* Then we can compute timegm() from first principles. */ - return (t->tm_sec + t->tm_min * 60 + t->tm_hour * 3600 - + t->tm_yday * 86400 + (t->tm_year - 70) * 31536000 - + ((t->tm_year - 69) / 4) * 86400 - - ((t->tm_year - 1) / 100) * 86400 - + ((t->tm_year + 299) / 400) * 86400); + /* Else use direct calculation using POSIX assumptions. */ + /* First, fix up tm_yday based on the year/month/day. */ + if (mktime(t) == (time_t)-1) + return ((time_t)-1); + /* Then we can compute timegm() from first principles. */ + return (t->tm_sec + + t->tm_min * 60 + + t->tm_hour * 3600 + + t->tm_yday * 86400 + + (t->tm_year - 70) * 31536000 + + ((t->tm_year - 69) / 4) * 86400 + - ((t->tm_year - 1) / 100) * 86400 + + ((t->tm_year + 299) / 400) * 86400); #endif } static const char * -build_pathname(struct archive_string *as, struct file_info *file) +build_pathname(struct archive_string *as, struct file_info *file, int depth) { + // Plain ISO9660 only allows 8 dir levels; if we get + // to 1000, then something is very, very wrong. + if (depth > 1000) { + return NULL; + } if (file->parent != NULL && archive_strlen(&file->parent->name) > 0) { - build_pathname(as, file->parent); + if (build_pathname(as, file->parent, depth + 1) == NULL) { + return NULL; + } archive_strcat(as, "/"); } if (archive_strlen(&file->name) == 0) diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c index b88731a..eff02d8 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_lha.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2008-2012 Michihiro NAKAJIMA + * Copyright (c) 2008-2014 Michihiro NAKAJIMA * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -82,9 +82,6 @@ struct lzh_dec { /* The length how many bytes we can copy decoded code from * the window. */ int copy_len; - /* The remaining bytes that we have not copied decoded data from - * the window to an output buffer. */ - int w_remaining; /* * Bit stream reader. @@ -140,10 +137,10 @@ struct lzh_dec { struct lzh_stream { const unsigned char *next_in; - int64_t avail_in; + int avail_in; int64_t total_in; - unsigned char *next_out; - int64_t avail_out; + const unsigned char *ref_ptr; + int avail_out; int64_t total_out; struct lzh_dec *ds; }; @@ -198,9 +195,6 @@ struct lha { char end_of_entry_cleanup; char entry_is_compressed; - unsigned char *uncompressed_buffer; - size_t uncompressed_buffer_size; - char format_name[64]; struct lzh_stream strm; @@ -214,41 +208,6 @@ struct lha { #define H_LEVEL_OFFSET 20 /* Header Level. */ #define H_SIZE 22 /* Minimum header size. */ -static const uint16_t crc16tbl[256] = { - 0x0000,0xC0C1,0xC181,0x0140,0xC301,0x03C0,0x0280,0xC241, - 0xC601,0x06C0,0x0780,0xC741,0x0500,0xC5C1,0xC481,0x0440, - 0xCC01,0x0CC0,0x0D80,0xCD41,0x0F00,0xCFC1,0xCE81,0x0E40, - 0x0A00,0xCAC1,0xCB81,0x0B40,0xC901,0x09C0,0x0880,0xC841, - 0xD801,0x18C0,0x1980,0xD941,0x1B00,0xDBC1,0xDA81,0x1A40, - 0x1E00,0xDEC1,0xDF81,0x1F40,0xDD01,0x1DC0,0x1C80,0xDC41, - 0x1400,0xD4C1,0xD581,0x1540,0xD701,0x17C0,0x1680,0xD641, - 0xD201,0x12C0,0x1380,0xD341,0x1100,0xD1C1,0xD081,0x1040, - 0xF001,0x30C0,0x3180,0xF141,0x3300,0xF3C1,0xF281,0x3240, - 0x3600,0xF6C1,0xF781,0x3740,0xF501,0x35C0,0x3480,0xF441, - 0x3C00,0xFCC1,0xFD81,0x3D40,0xFF01,0x3FC0,0x3E80,0xFE41, - 0xFA01,0x3AC0,0x3B80,0xFB41,0x3900,0xF9C1,0xF881,0x3840, - 0x2800,0xE8C1,0xE981,0x2940,0xEB01,0x2BC0,0x2A80,0xEA41, - 0xEE01,0x2EC0,0x2F80,0xEF41,0x2D00,0xEDC1,0xEC81,0x2C40, - 0xE401,0x24C0,0x2580,0xE541,0x2700,0xE7C1,0xE681,0x2640, - 0x2200,0xE2C1,0xE381,0x2340,0xE101,0x21C0,0x2080,0xE041, - 0xA001,0x60C0,0x6180,0xA141,0x6300,0xA3C1,0xA281,0x6240, - 0x6600,0xA6C1,0xA781,0x6740,0xA501,0x65C0,0x6480,0xA441, - 0x6C00,0xACC1,0xAD81,0x6D40,0xAF01,0x6FC0,0x6E80,0xAE41, - 0xAA01,0x6AC0,0x6B80,0xAB41,0x6900,0xA9C1,0xA881,0x6840, - 0x7800,0xB8C1,0xB981,0x7940,0xBB01,0x7BC0,0x7A80,0xBA41, - 0xBE01,0x7EC0,0x7F80,0xBF41,0x7D00,0xBDC1,0xBC81,0x7C40, - 0xB401,0x74C0,0x7580,0xB541,0x7700,0xB7C1,0xB681,0x7640, - 0x7200,0xB2C1,0xB381,0x7340,0xB101,0x71C0,0x7080,0xB041, - 0x5000,0x90C1,0x9181,0x5140,0x9301,0x53C0,0x5280,0x9241, - 0x9601,0x56C0,0x5780,0x9741,0x5500,0x95C1,0x9481,0x5440, - 0x9C01,0x5CC0,0x5D80,0x9D41,0x5F00,0x9FC1,0x9E81,0x5E40, - 0x5A00,0x9AC1,0x9B81,0x5B40,0x9901,0x59C0,0x5880,0x9841, - 0x8801,0x48C0,0x4980,0x8941,0x4B00,0x8BC1,0x8A81,0x4A40, - 0x4E00,0x8EC1,0x8F81,0x4F40,0x8D01,0x4DC0,0x4C80,0x8C41, - 0x4400,0x84C1,0x8581,0x4540,0x8701,0x47C0,0x4680,0x8641, - 0x8201,0x42C0,0x4380,0x8341,0x4100,0x81C1,0x8081,0x4040 -}; - static int archive_read_format_lha_bid(struct archive_read *, int); static int archive_read_format_lha_options(struct archive_read *, const char *, const char *); @@ -279,6 +238,7 @@ static int lha_read_data_none(struct archive_read *, const void **, size_t *, int64_t *); static int lha_read_data_lzh(struct archive_read *, const void **, size_t *, int64_t *); +static void lha_crc16_init(void); static uint16_t lha_crc16(uint16_t, const void *, size_t); static int lzh_decode_init(struct lzh_stream *, const char *); static void lzh_decode_free(struct lzh_stream *); @@ -520,6 +480,8 @@ archive_read_format_lha_read_header(struct archive_read *a, const char *signature; int err; + lha_crc16_init(); + a->archive.archive_format = ARCHIVE_FORMAT_LHA; if (a->archive.archive_format_name == NULL) a->archive.archive_format_name = "lha"; @@ -1232,13 +1194,15 @@ lha_read_file_extended_header(struct archive_read *a, struct lha *lha, archive_string_empty(&lha->filename); break; } + if (extdheader[0] == '\0') + goto invalid; archive_strncpy(&lha->filename, (const char *)extdheader, datasize); break; case EXT_DIRECTORY: - if (datasize == 0) + if (datasize == 0 || extdheader[0] == '\0') /* no directory name data. exit this case. */ - break; + goto invalid; archive_strncpy(&lha->dirname, (const char *)extdheader, datasize); @@ -1378,6 +1342,26 @@ invalid: } static int +lha_end_of_entry(struct archive_read *a) +{ + struct lha *lha = (struct lha *)(a->format->data); + int r = ARCHIVE_EOF; + + if (!lha->end_of_entry_cleanup) { + if ((lha->setflag & CRC_IS_SET) && + lha->crc != lha->entry_crc_calculated) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "LHa data CRC error"); + r = ARCHIVE_WARN; + } + + /* End-of-entry cleanup done. */ + lha->end_of_entry_cleanup = 1; + } + return (r); +} + +static int archive_read_format_lha_read_data(struct archive_read *a, const void **buff, size_t *size, int64_t *offset) { @@ -1390,22 +1374,10 @@ archive_read_format_lha_read_data(struct archive_read *a, lha->entry_unconsumed = 0; } if (lha->end_of_entry) { - if (!lha->end_of_entry_cleanup) { - if ((lha->setflag & CRC_IS_SET) && - lha->crc != lha->entry_crc_calculated) { - archive_set_error(&a->archive, - ARCHIVE_ERRNO_MISC, - "LHa data CRC error"); - return (ARCHIVE_WARN); - } - - /* End-of-entry cleanup done. */ - lha->end_of_entry_cleanup = 1; - } *offset = lha->entry_offset; *size = 0; *buff = NULL; - return (ARCHIVE_EOF); + return (lha_end_of_entry(a)); } if (lha->entry_is_compressed) @@ -1477,18 +1449,6 @@ lha_read_data_lzh(struct archive_read *a, const void **buff, ssize_t bytes_avail; int r; - /* If the buffer hasn't been allocated, allocate it now. */ - if (lha->uncompressed_buffer == NULL) { - lha->uncompressed_buffer_size = 64 * 1024; - lha->uncompressed_buffer - = (unsigned char *)malloc(lha->uncompressed_buffer_size); - if (lha->uncompressed_buffer == NULL) { - archive_set_error(&a->archive, ENOMEM, - "No memory for lzh decompression"); - return (ARCHIVE_FATAL); - } - } - /* If we haven't yet read any data, initialize the decompressor. */ if (!lha->decompress_init) { r = lzh_decode_init(&(lha->strm), lha->method); @@ -1534,12 +1494,9 @@ lha_read_data_lzh(struct archive_read *a, const void **buff, if (bytes_avail > lha->entry_bytes_remaining) bytes_avail = (ssize_t)lha->entry_bytes_remaining; - lha->strm.avail_in = bytes_avail; + lha->strm.avail_in = (int)bytes_avail; lha->strm.total_in = 0; - if (lha->strm.avail_out == 0) { - lha->strm.next_out = lha->uncompressed_buffer; - lha->strm.avail_out = lha->uncompressed_buffer_size; - } + lha->strm.avail_out = 0; r = lzh_decode(&(lha->strm), bytes_avail == lha->entry_bytes_remaining); switch (r) { @@ -1556,10 +1513,10 @@ lha_read_data_lzh(struct archive_read *a, const void **buff, lha->entry_unconsumed = lha->strm.total_in; lha->entry_bytes_remaining -= lha->strm.total_in; - if (lha->strm.avail_out == 0 || lha->end_of_entry) { + if (lha->strm.avail_out) { *offset = lha->entry_offset; - *size = lha->strm.next_out - lha->uncompressed_buffer; - *buff = lha->uncompressed_buffer; + *size = lha->strm.avail_out; + *buff = lha->strm.ref_ptr; lha->entry_crc_calculated = lha_crc16(lha->entry_crc_calculated, *buff, *size); lha->entry_offset += *size; @@ -1567,6 +1524,8 @@ lha_read_data_lzh(struct archive_read *a, const void **buff, *offset = lha->entry_offset; *size = 0; *buff = NULL; + if (lha->end_of_entry) + return (lha_end_of_entry(a)); } return (ARCHIVE_OK); } @@ -1611,7 +1570,6 @@ archive_read_format_lha_cleanup(struct archive_read *a) struct lha *lha = (struct lha *)(a->format->data); lzh_decode_free(&(lha->strm)); - free(lha->uncompressed_buffer); archive_string_free(&(lha->dirname)); archive_string_free(&(lha->filename)); archive_string_free(&(lha->uname)); @@ -1702,50 +1660,90 @@ lha_calcsum(unsigned char sum, const void *pp, int offset, size_t size) return (sum); } -#define CRC16(crc, v) do { \ - (crc) = crc16tbl[((crc) ^ v) & 0xFF] ^ ((crc) >> 8); \ -} while (0) +static uint16_t crc16tbl[2][256]; +static void +lha_crc16_init(void) +{ + unsigned int i; + static int crc16init = 0; + + if (crc16init) + return; + crc16init = 1; + + for (i = 0; i < 256; i++) { + unsigned int j; + uint16_t crc = (uint16_t)i; + for (j = 8; j; j--) + crc = (crc >> 1) ^ ((crc & 1) * 0xA001); + crc16tbl[0][i] = crc; + } + + for (i = 0; i < 256; i++) { + crc16tbl[1][i] = (crc16tbl[0][i] >> 8) + ^ crc16tbl[0][crc16tbl[0][i] & 0xff]; + } +} static uint16_t lha_crc16(uint16_t crc, const void *pp, size_t len) { - const unsigned char *buff = (const unsigned char *)pp; - - while (len >= 8) { - CRC16(crc, *buff++); CRC16(crc, *buff++); - CRC16(crc, *buff++); CRC16(crc, *buff++); - CRC16(crc, *buff++); CRC16(crc, *buff++); - CRC16(crc, *buff++); CRC16(crc, *buff++); - len -= 8; + const unsigned char *p = (const unsigned char *)pp; + const uint16_t *buff; + const union { + uint32_t i; + char c[4]; + } u = { 0x01020304 }; + + if (len == 0) + return crc; + + /* Process unaligned address. */ + if (((uintptr_t)p) & (uintptr_t)0x1) { + crc = (crc >> 8) ^ crc16tbl[0][(crc ^ *p++) & 0xff]; + len--; } - switch (len) { - case 7: - CRC16(crc, *buff++); - /* FALL THROUGH */ - case 6: - CRC16(crc, *buff++); - /* FALL THROUGH */ - case 5: - CRC16(crc, *buff++); - /* FALL THROUGH */ - case 4: - CRC16(crc, *buff++); - /* FALL THROUGH */ - case 3: - CRC16(crc, *buff++); - /* FALL THROUGH */ - case 2: - CRC16(crc, *buff++); - /* FALL THROUGH */ - case 1: - CRC16(crc, *buff); - /* FALL THROUGH */ - case 0: - break; + buff = (const uint16_t *)p; + /* + * Modern C compiler such as GCC does not unroll automatically yet + * without unrolling pragma, and Clang is so. So we should + * unroll this loop for its performance. + */ + for (;len >= 8; len -= 8) { + /* This if statement expects compiler optimization will + * remove the stament which will not be executed. */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif +#if defined(_MSC_VER) && _MSC_VER >= 1400 /* Visual Studio */ +# define bswap16(x) _byteswap_ushort(x) +#elif (defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap16)) +# define bswap16(x) __builtin_bswap16(x) +#else +# define bswap16(x) ((((x) >> 8) & 0xff) | ((x) << 8)) +#endif +#define CRC16W do { \ + if(u.c[0] == 1) { /* Big endian */ \ + crc ^= bswap16(*buff); buff++; \ + } else \ + crc ^= *buff++; \ + crc = crc16tbl[1][crc & 0xff] ^ crc16tbl[0][crc >> 8];\ +} while (0) + CRC16W; + CRC16W; + CRC16W; + CRC16W; +#undef CRC16W +#undef bswap16 } - return (crc); -} + p = (const unsigned char *)buff; + for (;len; len--) { + crc = (crc >> 8) ^ crc16tbl[0][(crc ^ *p++) & 0xff]; + } + return crc; +} /* * Initialize LZHUF decoder. @@ -1784,18 +1782,18 @@ lzh_decode_init(struct lzh_stream *strm, const char *method) return (ARCHIVE_FAILED);/* Not supported. */ } ds->error = ARCHIVE_FATAL; - w_size = ds->w_size; - ds->w_size = 1U << w_bits; + /* Expand a window size up to 128 KiB for decompressing process + * performance whatever its original window size is. */ + ds->w_size = 1U << 17; ds->w_mask = ds->w_size -1; - if (ds->w_buff == NULL || w_size != ds->w_size) { - free(ds->w_buff); + if (ds->w_buff == NULL) { ds->w_buff = malloc(ds->w_size); if (ds->w_buff == NULL) return (ARCHIVE_FATAL); } - memset(ds->w_buff, 0x20, ds->w_size); + w_size = 1U << w_bits; + memset(ds->w_buff + ds->w_size - w_size, 0x20, w_size); ds->w_pos = 0; - ds->w_remaining = 0; ds->state = 0; ds->pos_pt_len_size = w_bits + 1; ds->pos_pt_len_bits = (w_bits == 15 || w_bits == 16)? 5: 4; @@ -1882,9 +1880,10 @@ lzh_br_fillup(struct lzh_stream *strm, struct lzh_br *br) int n = CACHE_BITS - br->cache_avail; for (;;) { - switch (n >> 3) { - case 8: - if (strm->avail_in >= 8) { + const int x = n >> 3; + if (strm->avail_in >= x) { + switch (x) { + case 8: br->cache_buffer = ((uint64_t)strm->next_in[0]) << 56 | ((uint64_t)strm->next_in[1]) << 48 | @@ -1898,10 +1897,7 @@ lzh_br_fillup(struct lzh_stream *strm, struct lzh_br *br) strm->avail_in -= 8; br->cache_avail += 8 * 8; return (1); - } - break; - case 7: - if (strm->avail_in >= 7) { + case 7: br->cache_buffer = (br->cache_buffer << 56) | ((uint64_t)strm->next_in[0]) << 48 | @@ -1915,10 +1911,7 @@ lzh_br_fillup(struct lzh_stream *strm, struct lzh_br *br) strm->avail_in -= 7; br->cache_avail += 7 * 8; return (1); - } - break; - case 6: - if (strm->avail_in >= 6) { + case 6: br->cache_buffer = (br->cache_buffer << 48) | ((uint64_t)strm->next_in[0]) << 40 | @@ -1931,14 +1924,13 @@ lzh_br_fillup(struct lzh_stream *strm, struct lzh_br *br) strm->avail_in -= 6; br->cache_avail += 6 * 8; return (1); + case 0: + /* We have enough compressed data in + * the cache buffer.*/ + return (1); + default: + break; } - break; - case 0: - /* We have enough compressed data in - * the cache buffer.*/ - return (1); - default: - break; } if (strm->avail_in == 0) { /* There is not enough compressed data to fill up the @@ -1993,7 +1985,7 @@ static int lzh_decode(struct lzh_stream *strm, int last) { struct lzh_dec *ds = strm->ds; - int64_t avail_in; + int avail_in; int r; if (ds->error) @@ -2010,35 +2002,12 @@ lzh_decode(struct lzh_stream *strm, int last) return (r); } -static int -lzh_copy_from_window(struct lzh_stream *strm, struct lzh_dec *ds) +static void +lzh_emit_window(struct lzh_stream *strm, size_t s) { - size_t copy_bytes; - - if (ds->w_remaining == 0 && ds->w_pos > 0) { - if (ds->w_pos - ds->copy_pos <= strm->avail_out) - copy_bytes = ds->w_pos - ds->copy_pos; - else - copy_bytes = (size_t)strm->avail_out; - memcpy(strm->next_out, - ds->w_buff + ds->copy_pos, copy_bytes); - ds->copy_pos += (int)copy_bytes; - } else { - if (ds->w_remaining <= strm->avail_out) - copy_bytes = ds->w_remaining; - else - copy_bytes = (size_t)strm->avail_out; - memcpy(strm->next_out, - ds->w_buff + ds->w_size - ds->w_remaining, copy_bytes); - ds->w_remaining -= (int)copy_bytes; - } - strm->next_out += copy_bytes; - strm->avail_out -= copy_bytes; - strm->total_out += copy_bytes; - if (strm->avail_out == 0) - return (0); - else - return (1); + strm->ref_ptr = strm->ds->w_buff; + strm->avail_out = (int)s; + strm->total_out += s; } static int @@ -2073,8 +2042,9 @@ lzh_read_blocks(struct lzh_stream *strm, int last) goto failed; } if (ds->w_pos > 0) { - if (!lzh_copy_from_window(strm, ds)) - return (ARCHIVE_OK); + lzh_emit_window(strm, ds->w_pos); + ds->w_pos = 0; + return (ARCHIVE_OK); } /* End of compressed data; we have completely * handled all compressed data. */ @@ -2291,10 +2261,6 @@ lzh_decode_blocks(struct lzh_stream *strm, int last) int lt_max_bits = lt->max_bits, pt_max_bits = pt->max_bits; int state = ds->state; - if (ds->w_remaining > 0) { - if (!lzh_copy_from_window(strm, ds)) - goto next_data; - } for (;;) { switch (state) { case ST_GET_LITERAL: @@ -2349,9 +2315,8 @@ lzh_decode_blocks(struct lzh_stream *strm, int last) w_buff[w_pos] = c; if (++w_pos >= w_size) { w_pos = 0; - ds->w_remaining = w_size; - if (!lzh_copy_from_window(strm, ds)) - goto next_data; + lzh_emit_window(strm, w_size); + goto next_data; } } /* 'c' is the length of a match pattern we have @@ -2429,25 +2394,26 @@ lzh_decode_blocks(struct lzh_stream *strm, int last) d = w_buff + w_pos; s = w_buff + copy_pos; - for (li = 0; li < l; li++) + for (li = 0; li < l-1;) { + d[li] = s[li];li++; + d[li] = s[li];li++; + } + if (li < l) d[li] = s[li]; } - w_pos = (w_pos + l) & w_mask; - if (w_pos == 0) { - ds->w_remaining = w_size; - if (!lzh_copy_from_window(strm, ds)) { - if (copy_len <= l) - state = ST_GET_LITERAL; - else { - state = ST_COPY_DATA; - ds->copy_len = - copy_len - l; - ds->copy_pos = - (copy_pos + l) - & w_mask; - } - goto next_data; + w_pos += l; + if (w_pos == w_size) { + w_pos = 0; + lzh_emit_window(strm, w_size); + if (copy_len <= l) + state = ST_GET_LITERAL; + else { + state = ST_COPY_DATA; + ds->copy_len = copy_len - l; + ds->copy_pos = + (copy_pos + l) & w_mask; } + goto next_data; } if (copy_len <= l) /* A copy of current pattern ended. */ @@ -2507,14 +2473,80 @@ lzh_huffman_free(struct huffman *hf) free(hf->tree); } +static char bitlen_tbl[0x400] = { + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 16, 0 +}; static int lzh_read_pt_bitlen(struct lzh_stream *strm, int start, int end) { struct lzh_dec *ds = strm->ds; - struct lzh_br * br = &(ds->br); + struct lzh_br *br = &(ds->br); int c, i; - for (i = start; i < end;) { + for (i = start; i < end; ) { /* * bit pattern the number we need * 000 -> 0 @@ -2530,17 +2562,13 @@ lzh_read_pt_bitlen(struct lzh_stream *strm, int start, int end) if (!lzh_br_read_ahead(strm, br, 3)) return (i); if ((c = lzh_br_bits(br, 3)) == 7) { - int d; if (!lzh_br_read_ahead(strm, br, 13)) return (i); - d = lzh_br_bits(br, 13); - while (d & 0x200) { - c++; - d <<= 1; - } - if (c > 16) + c = bitlen_tbl[lzh_br_bits(br, 13) & 0x3FF]; + if (c) + lzh_br_consume(br, c - 3); + else return (-1);/* Invalid data. */ - lzh_br_consume(br, c - 3); } else lzh_br_consume(br, 3); ds->pt.bitlen[i++] = c; @@ -2603,7 +2631,7 @@ lzh_make_huffman_table(struct huffman *hf) } } if (maxbits > HTBL_BITS) { - int htbl_max; + unsigned htbl_max; uint16_t *p; diffbits = maxbits - HTBL_BITS; @@ -2647,8 +2675,40 @@ lzh_make_huffman_table(struct huffman *hf) return (0);/* Invalid */ /* Update the table */ p = &(tbl[ptn]); - while (--cnt >= 0) - p[cnt] = (uint16_t)i; + if (cnt > 7) { + uint16_t *pc; + + cnt -= 8; + pc = &p[cnt]; + pc[0] = (uint16_t)i; + pc[1] = (uint16_t)i; + pc[2] = (uint16_t)i; + pc[3] = (uint16_t)i; + pc[4] = (uint16_t)i; + pc[5] = (uint16_t)i; + pc[6] = (uint16_t)i; + pc[7] = (uint16_t)i; + if (cnt > 7) { + cnt -= 8; + memcpy(&p[cnt], pc, + 8 * sizeof(uint16_t)); + pc = &p[cnt]; + while (cnt > 15) { + cnt -= 16; + memcpy(&p[cnt], pc, + 16 * sizeof(uint16_t)); + } + } + if (cnt) + memcpy(p, pc, cnt * sizeof(uint16_t)); + } else { + while (cnt > 1) { + p[--cnt] = (uint16_t)i; + p[--cnt] = (uint16_t)i; + } + if (cnt) + p[--cnt] = (uint16_t)i; + } continue; } @@ -2742,7 +2802,7 @@ lzh_decode_huffman(struct huffman *hf, unsigned rbits) * If it fails, search a huffman tree for. */ c = hf->tbl[rbits >> hf->shift_bits]; - if (c < hf->len_avail) + if (c < hf->len_avail || hf->len_avail == 0) return (c); /* This bit pattern needs to be found out at a huffman tree. */ return (lzh_decode_huffman_tree(hf, rbits, c)); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c index d82d4c1..b5f8e30 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_mtree.c @@ -139,16 +139,19 @@ get_time_t_max(void) #if defined(TIME_T_MAX) return TIME_T_MAX; #else - static time_t t; - time_t a; - if (t == 0) { - a = 1; - while (a > t) { - t = a; - a = a * 2 + 1; - } + /* ISO C allows time_t to be a floating-point type, + but POSIX requires an integer type. The following + should work on any system that follows the POSIX + conventions. */ + if (((time_t)0) < ((time_t)-1)) { + /* Time_t is unsigned */ + return (~(time_t)0); + } else { + /* Time_t is signed. */ + const uintmax_t max_unsigned_time_t = (uintmax_t)(~(time_t)0); + const uintmax_t max_signed_time_t = max_unsigned_time_t >> 1; + return (time_t)max_signed_time_t; } - return t; #endif } @@ -158,20 +161,16 @@ get_time_t_min(void) #if defined(TIME_T_MIN) return TIME_T_MIN; #else - /* 't' will hold the minimum value, which will be zero (if - * time_t is unsigned) or -2^n (if time_t is signed). */ - static int computed; - static time_t t; - time_t a; - if (computed == 0) { - a = (time_t)-1; - while (a < t) { - t = a; - a = a * 2; - } - computed = 1; + if (((time_t)0) < ((time_t)-1)) { + /* Time_t is unsigned */ + return (time_t)0; + } else { + /* Time_t is signed. */ + const uintmax_t max_unsigned_time_t = (uintmax_t)(~(time_t)0); + const uintmax_t max_signed_time_t = max_unsigned_time_t >> 1; + const intmax_t min_signed_time_t = (intmax_t)~max_signed_time_t; + return (time_t)min_signed_time_t; } - return t; #endif } @@ -532,32 +531,34 @@ bid_entry(const char *p, ssize_t len, ssize_t nl, int *last_is_path) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* E0 - EF */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* F0 - FF */ }; - ssize_t ll = len; + ssize_t ll; const char *pp = p; + const char * const pp_end = pp + len; *last_is_path = 0; /* * Skip the path-name which is quoted. */ - while (ll > 0 && *pp != ' ' &&*pp != '\t' && *pp != '\r' && - *pp != '\n') { + for (;pp < pp_end; ++pp) { if (!safe_char[*(const unsigned char *)pp]) { - f = 0; + if (*pp != ' ' && *pp != '\t' && *pp != '\r' + && *pp != '\n') + f = 0; break; } - ++pp; - --ll; - ++f; + f = 1; } + ll = pp_end - pp; + /* If a path-name was not found at the first, try to check - * a mtree format ``NetBSD's mtree -D'' creates, which - * places the path-name at the last. */ + * a mtree format(a.k.a form D) ``NetBSD's mtree -D'' creates, + * which places the path-name at the last. */ if (f == 0) { const char *pb = p + len - nl; int name_len = 0; int slash; - /* Do not accept multi lines for form D. */ + /* The form D accepts only a single line for an entry. */ if (pb-2 >= p && pb[-1] == '\\' && (pb[-2] == ' ' || pb[-2] == '\t')) return (-1); @@ -1056,7 +1057,8 @@ read_header(struct archive_read *a, struct archive_entry *entry) } if (!mtree->this_entry->used) { use_next = 0; - r = parse_file(a, entry, mtree, mtree->this_entry, &use_next); + r = parse_file(a, entry, mtree, mtree->this_entry, + &use_next); if (use_next == 0) return (r); } @@ -1151,8 +1153,8 @@ parse_file(struct archive_read *a, struct archive_entry *entry, mtree->fd = open(path, O_RDONLY | O_BINARY | O_CLOEXEC); __archive_ensure_cloexec_flag(mtree->fd); if (mtree->fd == -1 && - (errno != ENOENT || - archive_strlen(&mtree->contents_name) > 0)) { + (errno != ENOENT || + archive_strlen(&mtree->contents_name) > 0)) { archive_set_error(&a->archive, errno, "Can't open %s", path); r = ARCHIVE_WARN; @@ -1175,76 +1177,79 @@ parse_file(struct archive_read *a, struct archive_entry *entry, } /* - * Check for a mismatch between the type in the specification and - * the type of the contents object on disk. + * Check for a mismatch between the type in the specification + * and the type of the contents object on disk. */ if (st != NULL) { - if ( - ((st->st_mode & S_IFMT) == S_IFREG && - archive_entry_filetype(entry) == AE_IFREG) + if (((st->st_mode & S_IFMT) == S_IFREG && + archive_entry_filetype(entry) == AE_IFREG) #ifdef S_IFLNK - || ((st->st_mode & S_IFMT) == S_IFLNK && - archive_entry_filetype(entry) == AE_IFLNK) + ||((st->st_mode & S_IFMT) == S_IFLNK && + archive_entry_filetype(entry) == AE_IFLNK) #endif #ifdef S_IFSOCK - || ((st->st_mode & S_IFSOCK) == S_IFSOCK && - archive_entry_filetype(entry) == AE_IFSOCK) + ||((st->st_mode & S_IFSOCK) == S_IFSOCK && + archive_entry_filetype(entry) == AE_IFSOCK) #endif #ifdef S_IFCHR - || ((st->st_mode & S_IFMT) == S_IFCHR && - archive_entry_filetype(entry) == AE_IFCHR) + ||((st->st_mode & S_IFMT) == S_IFCHR && + archive_entry_filetype(entry) == AE_IFCHR) #endif #ifdef S_IFBLK - || ((st->st_mode & S_IFMT) == S_IFBLK && - archive_entry_filetype(entry) == AE_IFBLK) + ||((st->st_mode & S_IFMT) == S_IFBLK && + archive_entry_filetype(entry) == AE_IFBLK) #endif - || ((st->st_mode & S_IFMT) == S_IFDIR && - archive_entry_filetype(entry) == AE_IFDIR) + ||((st->st_mode & S_IFMT) == S_IFDIR && + archive_entry_filetype(entry) == AE_IFDIR) #ifdef S_IFIFO - || ((st->st_mode & S_IFMT) == S_IFIFO && - archive_entry_filetype(entry) == AE_IFIFO) + ||((st->st_mode & S_IFMT) == S_IFIFO && + archive_entry_filetype(entry) == AE_IFIFO) #endif - ) { - /* Types match. */ - } else { - /* Types don't match; bail out gracefully. */ - if (mtree->fd >= 0) - close(mtree->fd); - mtree->fd = -1; - if (parsed_kws & MTREE_HAS_OPTIONAL) { - /* It's not an error for an optional entry - to not match disk. */ - *use_next = 1; - } else if (r == ARCHIVE_OK) { - archive_set_error(&a->archive, - ARCHIVE_ERRNO_MISC, - "mtree specification has different type for %s", - archive_entry_pathname(entry)); - r = ARCHIVE_WARN; - } - return r; - } + ) { + /* Types match. */ + } else { + /* Types don't match; bail out gracefully. */ + if (mtree->fd >= 0) + close(mtree->fd); + mtree->fd = -1; + if (parsed_kws & MTREE_HAS_OPTIONAL) { + /* It's not an error for an optional + * entry to not match disk. */ + *use_next = 1; + } else if (r == ARCHIVE_OK) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_MISC, + "mtree specification has different" + " type for %s", + archive_entry_pathname(entry)); + r = ARCHIVE_WARN; + } + return (r); + } } /* - * If there is a contents file on disk, pick some of the metadata - * from that file. For most of these, we only set it from the contents - * if it wasn't already parsed from the specification. + * If there is a contents file on disk, pick some of the + * metadata from that file. For most of these, we only + * set it from the contents if it wasn't already parsed + * from the specification. */ if (st != NULL) { if (((parsed_kws & MTREE_HAS_DEVICE) == 0 || - (parsed_kws & MTREE_HAS_NOCHANGE) != 0) && - (archive_entry_filetype(entry) == AE_IFCHR || - archive_entry_filetype(entry) == AE_IFBLK)) + (parsed_kws & MTREE_HAS_NOCHANGE) != 0) && + (archive_entry_filetype(entry) == AE_IFCHR || + archive_entry_filetype(entry) == AE_IFBLK)) archive_entry_set_rdev(entry, st->st_rdev); - if ((parsed_kws & (MTREE_HAS_GID | MTREE_HAS_GNAME)) == 0 || - (parsed_kws & MTREE_HAS_NOCHANGE) != 0) + if ((parsed_kws & (MTREE_HAS_GID | MTREE_HAS_GNAME)) + == 0 || + (parsed_kws & MTREE_HAS_NOCHANGE) != 0) archive_entry_set_gid(entry, st->st_gid); - if ((parsed_kws & (MTREE_HAS_UID | MTREE_HAS_UNAME)) == 0 || - (parsed_kws & MTREE_HAS_NOCHANGE) != 0) + if ((parsed_kws & (MTREE_HAS_UID | MTREE_HAS_UNAME)) + == 0 || + (parsed_kws & MTREE_HAS_NOCHANGE) != 0) archive_entry_set_uid(entry, st->st_uid); if ((parsed_kws & MTREE_HAS_MTIME) == 0 || - (parsed_kws & MTREE_HAS_NOCHANGE) != 0) { + (parsed_kws & MTREE_HAS_NOCHANGE) != 0) { #if HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC archive_entry_set_mtime(entry, st->st_mtime, st->st_mtimespec.tv_nsec); @@ -1265,23 +1270,24 @@ parse_file(struct archive_read *a, struct archive_entry *entry, #endif } if ((parsed_kws & MTREE_HAS_NLINK) == 0 || - (parsed_kws & MTREE_HAS_NOCHANGE) != 0) + (parsed_kws & MTREE_HAS_NOCHANGE) != 0) archive_entry_set_nlink(entry, st->st_nlink); if ((parsed_kws & MTREE_HAS_PERM) == 0 || - (parsed_kws & MTREE_HAS_NOCHANGE) != 0) + (parsed_kws & MTREE_HAS_NOCHANGE) != 0) archive_entry_set_perm(entry, st->st_mode); if ((parsed_kws & MTREE_HAS_SIZE) == 0 || - (parsed_kws & MTREE_HAS_NOCHANGE) != 0) + (parsed_kws & MTREE_HAS_NOCHANGE) != 0) archive_entry_set_size(entry, st->st_size); archive_entry_set_ino(entry, st->st_ino); archive_entry_set_dev(entry, st->st_dev); - archive_entry_linkify(mtree->resolver, &entry, &sparse_entry); + archive_entry_linkify(mtree->resolver, &entry, + &sparse_entry); } else if (parsed_kws & MTREE_HAS_OPTIONAL) { /* * Couldn't open the entry, stat it or the on-disk type - * didn't match. If this entry is optional, just ignore it - * and read the next header entry. + * didn't match. If this entry is optional, just + * ignore it and read the next header entry. */ *use_next = 1; return ARCHIVE_OK; @@ -1370,7 +1376,7 @@ parse_device(dev_t *pdev, struct archive *a, char *val) "Missing number"); return ARCHIVE_WARN; } - numbers[argc++] = mtree_atol(&p); + numbers[argc++] = (unsigned long)mtree_atol(&p); if (argc > MAX_PACK_ARGS) { archive_set_error(a, ARCHIVE_ERRNO_FILE_FORMAT, "Too many arguments"); @@ -1583,32 +1589,38 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, } case 'c': if (strcmp(val, "char") == 0) { - archive_entry_set_filetype(entry, AE_IFCHR); + archive_entry_set_filetype(entry, + AE_IFCHR); break; } case 'd': if (strcmp(val, "dir") == 0) { - archive_entry_set_filetype(entry, AE_IFDIR); + archive_entry_set_filetype(entry, + AE_IFDIR); break; } case 'f': if (strcmp(val, "fifo") == 0) { - archive_entry_set_filetype(entry, AE_IFIFO); + archive_entry_set_filetype(entry, + AE_IFIFO); break; } if (strcmp(val, "file") == 0) { - archive_entry_set_filetype(entry, AE_IFREG); + archive_entry_set_filetype(entry, + AE_IFREG); break; } case 'l': if (strcmp(val, "link") == 0) { - archive_entry_set_filetype(entry, AE_IFLNK); + archive_entry_set_filetype(entry, + AE_IFLNK); break; } default: archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, - "Unrecognized file type \"%s\"; assuming \"file\"", val); + "Unrecognized file type \"%s\"; " + "assuming \"file\"", val); archive_entry_set_filetype(entry, AE_IFREG); return (ARCHIVE_WARN); } @@ -1635,7 +1647,8 @@ parse_keyword(struct archive_read *a, struct mtree *mtree, } static int -read_data(struct archive_read *a, const void **buff, size_t *size, int64_t *offset) +read_data(struct archive_read *a, const void **buff, size_t *size, + int64_t *offset) { size_t bytes_to_read; ssize_t bytes_read; @@ -1761,6 +1774,10 @@ parse_escapes(char *src, struct mtree_entry *mentry) c = '\v'; ++src; break; + case '\\': + c = '\\'; + ++src; + break; } } *dest++ = c; @@ -1898,14 +1915,14 @@ mtree_atol(char **p) * point to first character of line. */ static ssize_t -readline(struct archive_read *a, struct mtree *mtree, char **start, ssize_t limit) +readline(struct archive_read *a, struct mtree *mtree, char **start, + ssize_t limit) { ssize_t bytes_read; ssize_t total_size = 0; ssize_t find_off = 0; const void *t; - const char *s; - void *p; + void *nl; char *u; /* Accumulate line in a line buffer. */ @@ -1916,11 +1933,10 @@ readline(struct archive_read *a, struct mtree *mtree, char **start, ssize_t limi return (0); if (bytes_read < 0) return (ARCHIVE_FATAL); - s = t; /* Start of line? */ - p = memchr(t, '\n', bytes_read); - /* If we found '\n', trim the read. */ - if (p != NULL) { - bytes_read = 1 + ((const char *)p) - s; + nl = memchr(t, '\n', bytes_read); + /* If we found '\n', trim the read to end exactly there. */ + if (nl != NULL) { + bytes_read = ((const char *)nl) - ((const char *)t) + 1; } if (total_size + bytes_read + 1 > limit) { archive_set_error(&a->archive, @@ -1934,38 +1950,34 @@ readline(struct archive_read *a, struct mtree *mtree, char **start, ssize_t limi "Can't allocate working buffer"); return (ARCHIVE_FATAL); } + /* Append new bytes to string. */ memcpy(mtree->line.s + total_size, t, bytes_read); __archive_read_consume(a, bytes_read); total_size += bytes_read; - /* Null terminate. */ mtree->line.s[total_size] = '\0'; - /* If we found an unescaped '\n', clean up and return. */ + for (u = mtree->line.s + find_off; *u; ++u) { if (u[0] == '\n') { + /* Ends with unescaped newline. */ *start = mtree->line.s; return total_size; - } - if (u[0] == '#') { - if (p == NULL) + } else if (u[0] == '#') { + /* Ends with comment sequence #...\n */ + if (nl == NULL) { + /* But we've not found the \n yet */ break; - *start = mtree->line.s; - return total_size; - } - if (u[0] != '\\') - continue; - if (u[1] == '\\') { - ++u; - continue; - } - if (u[1] == '\n') { - memmove(u, u + 1, - total_size - (u - mtree->line.s) + 1); - --total_size; - ++u; - break; + } + } else if (u[0] == '\\') { + if (u[1] == '\n') { + /* Trim escaped newline. */ + total_size -= 2; + mtree->line.s[total_size] = '\0'; + break; + } else if (u[1] != '\0') { + /* Skip the two-char escape sequence */ + ++u; + } } - if (u[1] == '\0') - break; } find_off = u - mtree->line.s; } diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c index 4c56834..35ac7cf 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_rar.c @@ -186,6 +186,7 @@ struct huffman_code { struct huffman_tree_node *tree; int numentries; + int numallocatedentries; int minlength; int maxlength; int tablesize; @@ -225,6 +226,7 @@ struct rar mode_t mode; char *filename; char *filename_save; + size_t filename_save_size; size_t filename_allocated; /* File header optional entries */ @@ -1000,8 +1002,8 @@ archive_read_format_rar_read_data(struct archive_read *a, const void **buff, rar->bytes_unconsumed = 0; } + *buff = NULL; if (rar->entry_eof || rar->offset_seek >= rar->unp_size) { - *buff = NULL; *size = 0; *offset = rar->offset; if (*offset < rar->unp_size) @@ -1201,10 +1203,8 @@ archive_read_format_rar_seek_data(struct archive_read *a, int64_t offset, ret -= rar->dbo[0].start_offset; /* Always restart reading the file after a seek */ - a->read_data_block = NULL; - a->read_data_offset = 0; - a->read_data_output_offset = 0; - a->read_data_remaining = 0; + __archive_reset_read_data(&a->archive); + rar->bytes_unconsumed = 0; rar->offset = 0; @@ -1530,6 +1530,7 @@ read_header(struct archive_read *a, struct archive_entry *entry, /* Split file in multivolume RAR. No more need to process header. */ if (rar->filename_save && + filename_size == rar->filename_save_size && !memcmp(rar->filename, rar->filename_save, filename_size + 1)) { __archive_read_consume(a, header_size - 7); @@ -1559,6 +1560,7 @@ read_header(struct archive_read *a, struct archive_entry *entry, rar->filename_save = (char*)realloc(rar->filename_save, filename_size + 1); memcpy(rar->filename_save, rar->filename, filename_size + 1); + rar->filename_save_size = filename_size; /* Set info for seeking */ free(rar->dbo); @@ -2406,6 +2408,8 @@ create_code(struct archive_read *a, struct huffman_code *code, { int i, j, codebits = 0, symbolsleft = numsymbols; + code->numentries = 0; + code->numallocatedentries = 0; if (new_node(code) < 0) { archive_set_error(&a->archive, ENOMEM, "Unable to allocate memory for node data."); @@ -2534,11 +2538,17 @@ static int new_node(struct huffman_code *code) { void *new_tree; - - new_tree = realloc(code->tree, (code->numentries + 1) * sizeof(*code->tree)); - if (new_tree == NULL) - return (-1); - code->tree = (struct huffman_tree_node *)new_tree; + if (code->numallocatedentries == code->numentries) { + int new_num_entries = 256; + if (code->numentries > 0) { + new_num_entries = code->numentries * 2; + } + new_tree = realloc(code->tree, new_num_entries * sizeof(*code->tree)); + if (new_tree == NULL) + return (-1); + code->tree = (struct huffman_tree_node *)new_tree; + code->numallocatedentries = new_num_entries; + } code->tree[code->numentries].branches[0] = -1; code->tree[code->numentries].branches[1] = -2; return 1; @@ -2895,8 +2905,8 @@ rar_read_ahead(struct archive_read *a, size_t min, ssize_t *avail) int ret; if (avail) { - if (a->read_data_is_posix_read && *avail > (ssize_t)a->read_data_requested) - *avail = a->read_data_requested; + if (a->archive.read_data_is_posix_read && *avail > (ssize_t)a->archive.read_data_requested) + *avail = a->archive.read_data_requested; if (*avail > rar->bytes_remaining) *avail = (ssize_t)rar->bytes_remaining; if (*avail < 0) diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c index 734424d..01d85cf 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_tar.c @@ -456,6 +456,7 @@ archive_read_format_tar_read_header(struct archive_read *a, static int default_dev; struct tar *tar; const char *p; + const wchar_t *wp; int r; size_t l, unconsumed = 0; @@ -506,27 +507,22 @@ archive_read_format_tar_read_header(struct archive_read *a, } } - if (r == ARCHIVE_OK) { + if (r == ARCHIVE_OK && archive_entry_filetype(entry) == AE_IFREG) { /* * "Regular" entry with trailing '/' is really * directory: This is needed for certain old tar * variants and even for some broken newer ones. */ - const wchar_t *wp; - wp = archive_entry_pathname_w(entry); - if (wp != NULL) { + if ((wp = archive_entry_pathname_w(entry)) != NULL) { l = wcslen(wp); - if (archive_entry_filetype(entry) == AE_IFREG - && wp[l-1] == L'/') + if (l > 0 && wp[l - 1] == L'/') { archive_entry_set_filetype(entry, AE_IFDIR); - } else { - p = archive_entry_pathname(entry); - if (p == NULL) - return (ARCHIVE_FAILED); + } + } else if ((p = archive_entry_pathname(entry)) != NULL) { l = strlen(p); - if (archive_entry_filetype(entry) == AE_IFREG - && p[l-1] == '/') + if (l > 0 && p[l - 1] == '/') { archive_entry_set_filetype(entry, AE_IFDIR); + } } } return (r); @@ -599,13 +595,27 @@ static int archive_read_format_tar_skip(struct archive_read *a) { int64_t bytes_skipped; + int64_t request; + struct sparse_block *p; struct tar* tar; tar = (struct tar *)(a->format->data); - bytes_skipped = __archive_read_consume(a, - tar->entry_bytes_remaining + tar->entry_padding + - tar->entry_bytes_unconsumed); + /* Do not consume the hole of a sparse file. */ + request = 0; + for (p = tar->sparse_list; p != NULL; p = p->next) { + if (!p->hole) { + if (p->remaining >= INT64_MAX - request) { + return ARCHIVE_FATAL; + } + request += p->remaining; + } + } + if (request > tar->entry_bytes_remaining) + request = tar->entry_bytes_remaining; + request += tar->entry_padding + tar->entry_bytes_unconsumed; + + bytes_skipped = __archive_read_consume(a, request); if (bytes_skipped < 0) return (ARCHIVE_FATAL); @@ -2117,6 +2127,10 @@ gnu_add_sparse_entry(struct archive_read *a, struct tar *tar, else tar->sparse_list = p; tar->sparse_last = p; + if (remaining < 0 || offset < 0) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "Malformed sparse map data"); + return (ARCHIVE_FATAL); + } p->offset = offset; p->remaining = remaining; return (ARCHIVE_OK); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_warc.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_warc.c new file mode 100644 index 0000000..57534f3 --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_warc.c @@ -0,0 +1,794 @@ +/*- + * Copyright (c) 2014 Sebastian Freundt + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" +__FBSDID("$FreeBSD$"); + +/** + * WARC is standardised by ISO TC46/SC4/WG12 and currently available as + * ISO 28500:2009. + * For the purposes of this file we used the final draft from: + * http://bibnum.bnf.fr/warc/WARC_ISO_28500_version1_latestdraft.pdf + * + * Todo: + * [ ] real-world warcs can contain resources at endpoints ending in / + * e.g. http://bibnum.bnf.fr/warc/ + * if you're lucky their response contains a Content-Location: header + * pointing to a unix-compliant filename, in the example above it's + * Content-Location: http://bibnum.bnf.fr/warc/index.html + * however, that's not mandated and github for example doesn't follow + * this convention. + * We need a set of archive options to control what to do with + * entries like these, at the moment care is taken to skip them. + * + **/ + +#ifdef HAVE_SYS_STAT_H +#include <sys/stat.h> +#endif +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#ifdef HAVE_LIMITS_H +#include <limits.h> +#endif +#ifdef HAVE_CTYPE_H +#include <ctype.h> +#endif +#ifdef HAVE_TIME_H +#include <time.h> +#endif + +#include "archive.h" +#include "archive_entry.h" +#include "archive_private.h" +#include "archive_read_private.h" + +typedef enum { + WT_NONE, + /* warcinfo */ + WT_INFO, + /* metadata */ + WT_META, + /* resource */ + WT_RSRC, + /* request, unsupported */ + WT_REQ, + /* response, unsupported */ + WT_RSP, + /* revisit, unsupported */ + WT_RVIS, + /* conversion, unsupported */ + WT_CONV, + /* continutation, unsupported at the moment */ + WT_CONT, + /* invalid type */ + LAST_WT +} warc_type_t; + +typedef struct { + size_t len; + const char *str; +} warc_string_t; + +typedef struct { + size_t len; + char *str; +} warc_strbuf_t; + +struct warc_s { + /* content length ahead */ + size_t cntlen; + /* and how much we've processed so far */ + size_t cntoff; + /* and how much we need to consume between calls */ + size_t unconsumed; + + /* string pool */ + warc_strbuf_t pool; + /* previous version */ + unsigned int pver; + /* stringified format name */ + struct archive_string sver; +}; + +static int _warc_bid(struct archive_read *a, int); +static int _warc_cleanup(struct archive_read *a); +static int _warc_read(struct archive_read*, const void**, size_t*, int64_t*); +static int _warc_skip(struct archive_read *a); +static int _warc_rdhdr(struct archive_read *a, struct archive_entry *e); + +/* private routines */ +static unsigned int _warc_rdver(const char buf[10], size_t bsz); +static unsigned int _warc_rdtyp(const char *buf, size_t bsz); +static warc_string_t _warc_rduri(const char *buf, size_t bsz); +static ssize_t _warc_rdlen(const char *buf, size_t bsz); +static time_t _warc_rdrtm(const char *buf, size_t bsz); +static time_t _warc_rdmtm(const char *buf, size_t bsz); +static const char *_warc_find_eoh(const char *buf, size_t bsz); + + +int +archive_read_support_format_warc(struct archive *_a) +{ + struct archive_read *a = (struct archive_read *)_a; + struct warc_s *w; + int r; + + archive_check_magic(_a, ARCHIVE_READ_MAGIC, + ARCHIVE_STATE_NEW, "archive_read_support_format_warc"); + + if ((w = malloc(sizeof(*w))) == NULL) { + archive_set_error(&a->archive, ENOMEM, + "Can't allocate warc data"); + return (ARCHIVE_FATAL); + } + memset(w, 0, sizeof(*w)); + + r = __archive_read_register_format( + a, w, "warc", + _warc_bid, NULL, _warc_rdhdr, _warc_read, + _warc_skip, NULL, _warc_cleanup, NULL, NULL); + + if (r != ARCHIVE_OK) { + free(w); + return (r); + } + return (ARCHIVE_OK); +} + +static int +_warc_cleanup(struct archive_read *a) +{ + struct warc_s *w = a->format->data; + + if (w->pool.len > 0U) { + free(w->pool.str); + } + archive_string_free(&w->sver); + free(w); + a->format->data = NULL; + return (ARCHIVE_OK); +} + +static int +_warc_bid(struct archive_read *a, int best_bid) +{ + const char *hdr; + ssize_t nrd; + unsigned int ver; + + (void)best_bid; /* UNUSED */ + + /* check first line of file, it should be a record already */ + if ((hdr = __archive_read_ahead(a, 12U, &nrd)) == NULL) { + /* no idea what to do */ + return -1; + } else if (nrd < 12) { + /* nah, not for us, our magic cookie is at least 12 bytes */ + return -1; + } + + /* otherwise snarf the record's version number */ + ver = _warc_rdver(hdr, nrd); + if (ver == 0U || ver > 10000U) { + /* oh oh oh, best not to wager ... */ + return -1; + } + + /* otherwise be confident */ + return (64); +} + +static int +_warc_rdhdr(struct archive_read *a, struct archive_entry *entry) +{ +#define HDR_PROBE_LEN (12U) + struct warc_s *w = a->format->data; + unsigned int ver; + const char *buf; + ssize_t nrd; + const char *eoh; + /* for the file name, saves some strndup()'ing */ + warc_string_t fnam; + /* warc record type, not that we really use it a lot */ + warc_type_t ftyp; + /* content-length+error monad */ + ssize_t cntlen; + /* record time is the WARC-Date time we reinterpret it as ctime */ + time_t rtime; + /* mtime is the Last-Modified time which will be the entry's mtime */ + time_t mtime; + +start_over: + /* just use read_ahead() they keep track of unconsumed + * bits and bobs for us; no need to put an extra shift in + * and reproduce that functionality here */ + buf = __archive_read_ahead(a, HDR_PROBE_LEN, &nrd); + + if (nrd < 0) { + /* no good */ + archive_set_error( + &a->archive, ARCHIVE_ERRNO_MISC, + "Bad record header"); + return (ARCHIVE_FATAL); + } else if (buf == NULL) { + /* there should be room for at least WARC/bla\r\n + * must be EOF therefore */ + return (ARCHIVE_EOF); + } + /* looks good so far, try and find the end of the header now */ + eoh = _warc_find_eoh(buf, nrd); + if (eoh == NULL) { + /* still no good, the header end might be beyond the + * probe we've requested, but then again who'd cram + * so much stuff into the header *and* be 28500-compliant */ + archive_set_error( + &a->archive, ARCHIVE_ERRNO_MISC, + "Bad record header"); + return (ARCHIVE_FATAL); + } else if ((ver = _warc_rdver(buf, eoh - buf)) > 10000U) { + /* nawww, I wish they promised backward compatibility + * anyhoo, in their infinite wisdom the 28500 guys might + * come up with something we can't possibly handle so + * best end things here */ + archive_set_error( + &a->archive, ARCHIVE_ERRNO_MISC, + "Unsupported record version"); + return (ARCHIVE_FATAL); + } else if ((cntlen = _warc_rdlen(buf, eoh - buf)) < 0) { + /* nightmare! the specs say content-length is mandatory + * so I don't feel overly bad stopping the reader here */ + archive_set_error( + &a->archive, EINVAL, + "Bad content length"); + return (ARCHIVE_FATAL); + } else if ((rtime = _warc_rdrtm(buf, eoh - buf)) == (time_t)-1) { + /* record time is mandatory as per WARC/1.0, + * so just barf here, fast and loud */ + archive_set_error( + &a->archive, EINVAL, + "Bad record time"); + return (ARCHIVE_FATAL); + } + + /* let the world know we're a WARC archive */ + a->archive.archive_format = ARCHIVE_FORMAT_WARC; + if (ver != w->pver) { + /* stringify this entry's version */ + archive_string_sprintf(&w->sver, + "WARC/%u.%u", ver / 10000, ver % 10000); + /* remember the version */ + w->pver = ver; + } + /* start off with the type */ + ftyp = _warc_rdtyp(buf, eoh - buf); + /* and let future calls know about the content */ + w->cntlen = cntlen; + w->cntoff = 0U; + mtime = 0;/* Avoid compiling error on some platform. */ + + switch (ftyp) { + case WT_RSRC: + case WT_RSP: + /* only try and read the filename in the cases that are + * guaranteed to have one */ + fnam = _warc_rduri(buf, eoh - buf); + /* check the last character in the URI to avoid creating + * directory endpoints as files, see Todo above */ + if (fnam.len == 0 || fnam.str[fnam.len - 1] == '/') { + /* break here for now */ + fnam.len = 0U; + fnam.str = NULL; + break; + } + /* bang to our string pool, so we save a + * malloc()+free() roundtrip */ + if (fnam.len + 1U > w->pool.len) { + w->pool.len = ((fnam.len + 64U) / 64U) * 64U; + w->pool.str = realloc(w->pool.str, w->pool.len); + } + memcpy(w->pool.str, fnam.str, fnam.len); + w->pool.str[fnam.len] = '\0'; + /* let noone else know about the pool, it's a secret, shhh */ + fnam.str = w->pool.str; + + /* snarf mtime or deduce from rtime + * this is a custom header added by our writer, it's quite + * hard to believe anyone else would go through with it + * (apart from being part of some http responses of course) */ + if ((mtime = _warc_rdmtm(buf, eoh - buf)) == (time_t)-1) { + mtime = rtime; + } + break; + default: + fnam.len = 0U; + fnam.str = NULL; + break; + } + + /* now eat some of those delicious buffer bits */ + __archive_read_consume(a, eoh - buf); + + switch (ftyp) { + case WT_RSRC: + case WT_RSP: + if (fnam.len > 0U) { + /* populate entry object */ + archive_entry_set_filetype(entry, AE_IFREG); + archive_entry_copy_pathname(entry, fnam.str); + archive_entry_set_size(entry, cntlen); + archive_entry_set_perm(entry, 0644); + /* rtime is the new ctime, mtime stays mtime */ + archive_entry_set_ctime(entry, rtime, 0L); + archive_entry_set_mtime(entry, mtime, 0L); + break; + } + /* FALLTHROUGH */ + default: + /* consume the content and start over */ + _warc_skip(a); + goto start_over; + } + return (ARCHIVE_OK); +} + +static int +_warc_read(struct archive_read *a, const void **buf, size_t *bsz, int64_t *off) +{ + struct warc_s *w = a->format->data; + const char *rab; + ssize_t nrd; + + if (w->cntoff >= w->cntlen) { + eof: + /* it's our lucky day, no work, we can leave early */ + *buf = NULL; + *bsz = 0U; + *off = w->cntoff + 4U/*for \r\n\r\n separator*/; + w->unconsumed = 0U; + return (ARCHIVE_EOF); + } + + rab = __archive_read_ahead(a, 1U, &nrd); + if (nrd < 0) { + *bsz = 0U; + /* big catastrophe */ + return (int)nrd; + } else if (nrd == 0) { + goto eof; + } else if ((size_t)nrd > w->cntlen - w->cntoff) { + /* clamp to content-length */ + nrd = w->cntlen - w->cntoff; + } + *off = w->cntoff; + *bsz = nrd; + *buf = rab; + + w->cntoff += nrd; + w->unconsumed = (size_t)nrd; + return (ARCHIVE_OK); +} + +static int +_warc_skip(struct archive_read *a) +{ + struct warc_s *w = a->format->data; + + __archive_read_consume(a, w->cntlen + 4U/*\r\n\r\n separator*/); + w->cntlen = 0U; + w->cntoff = 0U; + return (ARCHIVE_OK); +} + + +/* private routines */ +static void* +deconst(const void *c) +{ + return (char *)0x1 + (((const char *)c) - (const char *)0x1); +} + +static char* +xmemmem(const char *hay, const size_t hz_, const char *ndl, const size_t nz) +{ + const char *const eoh = hay + hz_; + const char *const eon = ndl + nz; + const char *hp; + const char *np; + const char *cand; + unsigned int hsum; + unsigned int nsum; + unsigned int eqp; + + /* trivial checks first + * a 0-sized needle is defined to be found anywhere in haystack + * then run strchr() to find a candidate in HAYSTACK (i.e. a portion + * that happens to begin with *NEEDLE) */ + if (nz == 0UL) { + return deconst(hay); + } else if ((hay = memchr(hay, *ndl, hz_)) == NULL) { + /* trivial */ + return NULL; + } + + /* First characters of haystack and needle are the same now. Both are + * guaranteed to be at least one character long. Now computes the sum + * of characters values of needle together with the sum of the first + * needle_len characters of haystack. */ + for (hp = hay + 1U, np = ndl + 1U, hsum = *hay, nsum = *hay, eqp = 1U; + hp < eoh && np < eon; + hsum ^= *hp, nsum ^= *np, eqp &= *hp == *np, hp++, np++); + + /* HP now references the (NZ + 1)-th character. */ + if (np < eon) { + /* haystack is smaller than needle, :O */ + return NULL; + } else if (eqp) { + /* found a match */ + return deconst(hay); + } + + /* now loop through the rest of haystack, + * updating the sum iteratively */ + for (cand = hay; hp < eoh; hp++) { + hsum ^= *cand++; + hsum ^= *hp; + + /* Since the sum of the characters is already known to be + * equal at that point, it is enough to check just NZ - 1 + * characters for equality, + * also CAND is by design < HP, so no need for range checks */ + if (hsum == nsum && memcmp(cand, ndl, nz - 1U) == 0) { + return deconst(cand); + } + } + return NULL; +} + +static int +strtoi_lim(const char *str, const char **ep, int llim, int ulim) +{ + int res = 0; + const char *sp; + /* we keep track of the number of digits via rulim */ + int rulim; + + for (sp = str, rulim = ulim > 10 ? ulim : 10; + res * 10 <= ulim && rulim && *sp >= '0' && *sp <= '9'; + sp++, rulim /= 10) { + res *= 10; + res += *sp - '0'; + } + if (sp == str) { + res = -1; + } else if (res < llim || res > ulim) { + res = -2; + } + *ep = (const char*)sp; + return res; +} + +static time_t +time_from_tm(struct tm *t) +{ +#if HAVE_TIMEGM + /* Use platform timegm() if available. */ + return (timegm(t)); +#elif HAVE__MKGMTIME64 + return (_mkgmtime64(t)); +#else + /* Else use direct calculation using POSIX assumptions. */ + /* First, fix up tm_yday based on the year/month/day. */ + if (mktime(t) == (time_t)-1) + return ((time_t)-1); + /* Then we can compute timegm() from first principles. */ + return (t->tm_sec + + t->tm_min * 60 + + t->tm_hour * 3600 + + t->tm_yday * 86400 + + (t->tm_year - 70) * 31536000 + + ((t->tm_year - 69) / 4) * 86400 + - ((t->tm_year - 1) / 100) * 86400 + + ((t->tm_year + 299) / 400) * 86400); +#endif +} + +static time_t +xstrpisotime(const char *s, char **endptr) +{ +/** like strptime() but strictly for ISO 8601 Zulu strings */ + struct tm tm; + time_t res = (time_t)-1; + + /* make sure tm is clean */ + memset(&tm, 0, sizeof(tm)); + + /* as a courtesy to our callers, and since this is a non-standard + * routine, we skip leading whitespace */ + for (; isspace(*s); s++); + + /* read year */ + if ((tm.tm_year = strtoi_lim(s, &s, 1583, 4095)) < 0 || *s++ != '-') { + goto out; + } + /* read month */ + if ((tm.tm_mon = strtoi_lim(s, &s, 1, 12)) < 0 || *s++ != '-') { + goto out; + } + /* read day-of-month */ + if ((tm.tm_mday = strtoi_lim(s, &s, 1, 31)) < 0 || *s++ != 'T') { + goto out; + } + /* read hour */ + if ((tm.tm_hour = strtoi_lim(s, &s, 0, 23)) < 0 || *s++ != ':') { + goto out; + } + /* read minute */ + if ((tm.tm_min = strtoi_lim(s, &s, 0, 59)) < 0 || *s++ != ':') { + goto out; + } + /* read second */ + if ((tm.tm_sec = strtoi_lim(s, &s, 0, 60)) < 0 || *s++ != 'Z') { + goto out; + } + + /* massage TM to fulfill some of POSIX' contraints */ + tm.tm_year -= 1900; + tm.tm_mon--; + + /* now convert our custom tm struct to a unix stamp using UTC */ + res = time_from_tm(&tm); + +out: + if (endptr != NULL) { + *endptr = deconst(s); + } + return res; +} + +static unsigned int +_warc_rdver(const char buf[10], size_t bsz) +{ + static const char magic[] = "WARC/"; + unsigned int ver; + + (void)bsz; /* UNUSED */ + + if (memcmp(buf, magic, sizeof(magic) - 1U) != 0) { + /* nope */ + return 99999U; + } + /* looks good so far, read the version number for a laugh */ + buf += sizeof(magic) - 1U; + /* most common case gets a quick-check here */ + if (memcmp(buf, "1.0\r\n", 5U) == 0) { + ver = 10000U; + } else { + switch (*buf) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + if (buf[1U] == '.') { + char *on; + + /* set up major version */ + ver = (buf[0U] - '0') * 10000U; + /* minor version, anyone? */ + ver += (strtol(buf + 2U, &on, 10)) * 100U; + /* don't parse anything else */ + if (on > buf + 2U) { + break; + } + } + /* FALLTHROUGH */ + case '9': + default: + /* just make the version ridiculously high */ + ver = 999999U; + break; + } + } + return ver; +} + +static unsigned int +_warc_rdtyp(const char *buf, size_t bsz) +{ + static const char _key[] = "\r\nWARC-Type:"; + const char *const eob = buf + bsz; + const char *val; + + if ((val = xmemmem(buf, bsz, _key, sizeof(_key) - 1U)) == NULL) { + /* no bother */ + return WT_NONE; + } + /* overread whitespace */ + for (val += sizeof(_key) - 1U; val < eob && isspace(*val); val++); + + if (val + 8U > eob) { + ; + } else if (memcmp(val, "resource", 8U) == 0) { + return WT_RSRC; + } else if (memcmp(val, "warcinfo", 8U) == 0) { + return WT_INFO; + } else if (memcmp(val, "metadata", 8U) == 0) { + return WT_META; + } else if (memcmp(val, "request", 7U) == 0) { + return WT_REQ; + } else if (memcmp(val, "response", 8U) == 0) { + return WT_RSP; + } else if (memcmp(val, "conversi", 8U) == 0) { + return WT_CONV; + } else if (memcmp(val, "continua", 8U) == 0) { + return WT_CONT; + } + return WT_NONE; +} + +static warc_string_t +_warc_rduri(const char *buf, size_t bsz) +{ + static const char _key[] = "\r\nWARC-Target-URI:"; + const char *const eob = buf + bsz; + const char *val; + const char *uri; + const char *eol; + warc_string_t res = {0U, NULL}; + + if ((val = xmemmem(buf, bsz, _key, sizeof(_key) - 1U)) == NULL) { + /* no bother */ + return res; + } + /* overread whitespace */ + for (val += sizeof(_key) - 1U; val < eob && isspace(*val); val++); + + /* overread URL designators */ + if ((uri = xmemmem(val, eob - val, "://", 3U)) == NULL) { + /* not touching that! */ + return res; + } else if ((eol = memchr(uri, '\n', eob - uri)) == NULL) { + /* no end of line? :O */ + return res; + } + + /* massage uri to point to after :// */ + uri += 3U; + /* also massage eol to point to the first whitespace + * after the last non-whitespace character before + * the end of the line */ + for (; eol > uri && isspace(eol[-1]); eol--); + + /* now then, inspect the URI */ + if (memcmp(val, "file", 4U) == 0) { + /* perfect, nothing left to do here */ + + } else if (memcmp(val, "http", 4U) == 0 || + memcmp(val, "ftp", 3U) == 0) { + /* overread domain, and the first / */ + while (uri < eol && *uri++ != '/'); + } else { + /* not sure what to do? best to bugger off */ + return res; + } + res.str = uri; + res.len = eol - uri; + return res; +} + +static ssize_t +_warc_rdlen(const char *buf, size_t bsz) +{ + static const char _key[] = "\r\nContent-Length:"; + const char *val; + char *on = NULL; + long int len; + + if ((val = xmemmem(buf, bsz, _key, sizeof(_key) - 1U)) == NULL) { + /* no bother */ + return -1; + } + + /* strtol kindly overreads whitespace for us, so use that */ + val += sizeof(_key) - 1U; + len = strtol(val, &on, 10); + if (on == NULL || !isspace(*on)) { + /* hm, can we trust that number? Best not. */ + return -1; + } + return (size_t)len; +} + +static time_t +_warc_rdrtm(const char *buf, size_t bsz) +{ + static const char _key[] = "\r\nWARC-Date:"; + const char *val; + char *on = NULL; + time_t res; + + if ((val = xmemmem(buf, bsz, _key, sizeof(_key) - 1U)) == NULL) { + /* no bother */ + return (time_t)-1; + } + + /* xstrpisotime() kindly overreads whitespace for us, so use that */ + val += sizeof(_key) - 1U; + res = xstrpisotime(val, &on); + if (on == NULL || !isspace(*on)) { + /* hm, can we trust that number? Best not. */ + return (time_t)-1; + } + return res; +} + +static time_t +_warc_rdmtm(const char *buf, size_t bsz) +{ + static const char _key[] = "\r\nLast-Modified:"; + const char *val; + char *on = NULL; + time_t res; + + if ((val = xmemmem(buf, bsz, _key, sizeof(_key) - 1U)) == NULL) { + /* no bother */ + return (time_t)-1; + } + + /* xstrpisotime() kindly overreads whitespace for us, so use that */ + val += sizeof(_key) - 1U; + res = xstrpisotime(val, &on); + if (on == NULL || !isspace(*on)) { + /* hm, can we trust that number? Best not. */ + return (time_t)-1; + } + return res; +} + +static const char* +_warc_find_eoh(const char *buf, size_t bsz) +{ + static const char _marker[] = "\r\n\r\n"; + const char *hit = xmemmem(buf, bsz, _marker, sizeof(_marker) - 1U); + + if (hit != NULL) { + hit += sizeof(_marker) - 1U; + } + return hit; +} + +/* archive_read_support_format_warc.c ends here */ diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c index 68485d1..bbef99f 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_xar.c @@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$"); #endif #include "archive.h" -#include "archive_crypto_private.h" +#include "archive_digest_private.h" #include "archive_endian.h" #include "archive_entry.h" #include "archive_entry_locale.h" @@ -1107,20 +1107,23 @@ static time_t time_from_tm(struct tm *t) { #if HAVE_TIMEGM - /* Use platform timegm() if available. */ - return (timegm(t)); + /* Use platform timegm() if available. */ + return (timegm(t)); #elif HAVE__MKGMTIME64 - return (_mkgmtime64(t)); + return (_mkgmtime64(t)); #else - /* Else use direct calculation using POSIX assumptions. */ - /* First, fix up tm_yday based on the year/month/day. */ - mktime(t); - /* Then we can compute timegm() from first principles. */ - return (t->tm_sec + t->tm_min * 60 + t->tm_hour * 3600 - + t->tm_yday * 86400 + (t->tm_year - 70) * 31536000 - + ((t->tm_year - 69) / 4) * 86400 - - ((t->tm_year - 1) / 100) * 86400 - + ((t->tm_year + 299) / 400) * 86400); + /* Else use direct calculation using POSIX assumptions. */ + /* First, fix up tm_yday based on the year/month/day. */ + mktime(t); + /* Then we can compute timegm() from first principles. */ + return (t->tm_sec + + t->tm_min * 60 + + t->tm_hour * 3600 + + t->tm_yday * 86400 + + (t->tm_year - 70) * 31536000 + + ((t->tm_year - 69) / 4) * 86400 + - ((t->tm_year - 1) / 100) * 86400 + + ((t->tm_year + 299) / 400) * 86400); #endif } @@ -3189,9 +3192,8 @@ xml2_read_toc(struct archive_read *a) case XML_READER_TYPE_ELEMENT: empty = xmlTextReaderIsEmptyElement(reader); r = xml2_xmlattr_setup(a, &list, reader); - if (r != ARCHIVE_OK) - return (r); - r = xml_start(a, name, &list); + if (r == ARCHIVE_OK) + r = xml_start(a, name, &list); xmlattr_cleanup(&list); if (r != ARCHIVE_OK) return (r); diff --git a/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c b/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c index 5ef2952..62bf5e3 100644 --- a/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c +++ b/Utilities/cmlibarchive/libarchive/archive_read_support_format_zip.c @@ -1,6 +1,6 @@ /*- * Copyright (c) 2004-2013 Tim Kientzle - * Copyright (c) 2011-2012 Michihiro NAKAJIMA + * Copyright (c) 2011-2012,2014 Michihiro NAKAJIMA * Copyright (c) 2013 Konrad Kleine * All rights reserved. * @@ -53,9 +53,12 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_read_support_format_zip.c 201102 #endif #include "archive.h" +#include "archive_digest_private.h" +#include "archive_cryptor_private.h" #include "archive_endian.h" #include "archive_entry.h" #include "archive_entry_locale.h" +#include "archive_hmac_private.h" #include "archive_private.h" #include "archive_rb.h" #include "archive_read_private.h" @@ -64,10 +67,6 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_read_support_format_zip.c 201102 #include "archive_crc32.h" #endif -#if defined(_WIN32) && !defined(__CYGWIN__) -# define snprintf _snprintf -#endif - struct zip_entry { struct archive_rb_node node; struct zip_entry *next; @@ -86,6 +85,25 @@ struct zip_entry { unsigned char compression; unsigned char system; /* From "version written by" */ unsigned char flags; /* Our extra markers. */ + unsigned char decdat;/* Used for Decryption check */ + + /* WinZip AES encryption extra field should be available + * when compression is 99. */ + struct { + /* Vendor version: AE-1 - 0x0001, AE-2 - 0x0002 */ + unsigned vendor; +#define AES_VENDOR_AE_1 0x0001 +#define AES_VENDOR_AE_2 0x0002 + /* AES encryption strength: + * 1 - 128 bits, 2 - 192 bits, 2 - 256 bits. */ + unsigned strength; + /* Actual compression method. */ + unsigned char compression; + } aes_extra; +}; + +struct trad_enc_ctx { + uint32_t keys[3]; }; /* Bits used in zip_flags. */ @@ -101,9 +119,20 @@ struct zip_entry { #define LA_USED_ZIP64 (1 << 0) #define LA_FROM_CENTRAL_DIRECTORY (1 << 1) +/* + * See "WinZip - AES Encryption Information" + * http://www.winzip.com/aes_info.htm + */ +/* Value used in compression method. */ +#define WINZIP_AES_ENCRYPTION 99 +/* Authentication code size. */ +#define AUTH_CODE_SIZE 10 +/**/ +#define MAX_DERIVED_KEY_BUF_SIZE (AES_MAX_KEY_SIZE * 2 + 2) + struct zip { /* Structural information about the archive. */ - char format_name[64]; + struct archive_string format_name; int64_t central_directory_offset; size_t central_directory_entries_total; size_t central_directory_entries_on_this_disk; @@ -127,7 +156,8 @@ struct zip { /* Running CRC32 of the decompressed data */ unsigned long entry_crc32; - unsigned long (*crc32func)(unsigned long, const void *, size_t); + unsigned long (*crc32func)(unsigned long, const void *, + size_t); char ignore_crc32; /* Flags to mark progress of decompression. */ @@ -146,6 +176,38 @@ struct zip { struct archive_string_conv *sconv_utf8; int init_default_conversion; int process_mac_extensions; + + char init_decryption; + + /* Decryption buffer. */ + unsigned char *decrypted_buffer; + unsigned char *decrypted_ptr; + size_t decrypted_buffer_size; + size_t decrypted_bytes_remaining; + size_t decrypted_unconsumed_bytes; + + /* Traditional PKWARE decryption. */ + struct trad_enc_ctx tctx; + char tctx_valid; + + /* WinZip AES decyption. */ + /* Contexts used for AES decryption. */ + archive_crypto_ctx cctx; + char cctx_valid; + archive_hmac_sha1_ctx hctx; + char hctx_valid; + + /* Strong encryption's decryption header information. */ + unsigned iv_size; + unsigned alg_id; + unsigned bit_len; + unsigned flags; + unsigned erd_size; + unsigned v_size; + unsigned v_crc32; + uint8_t *iv; + uint8_t *erd; + uint8_t *v_data; }; /* Many systems define min or MIN, but not all. */ @@ -154,6 +216,106 @@ struct zip { /* ------------------------------------------------------------------------ */ /* + Traditional PKWARE Decryption functions. + */ + +static void +trad_enc_update_keys(struct trad_enc_ctx *ctx, uint8_t c) +{ + uint8_t t; +#define CRC32(c, b) (crc32(c ^ 0xffffffffUL, &b, 1) ^ 0xffffffffUL) + + ctx->keys[0] = CRC32(ctx->keys[0], c); + ctx->keys[1] = (ctx->keys[1] + (ctx->keys[0] & 0xff)) * 134775813L + 1; + t = (ctx->keys[1] >> 24) & 0xff; + ctx->keys[2] = CRC32(ctx->keys[2], t); +#undef CRC32 +} + +static uint8_t +trad_enc_decypt_byte(struct trad_enc_ctx *ctx) +{ + unsigned temp = ctx->keys[2] | 2; + return (uint8_t)((temp * (temp ^ 1)) >> 8) & 0xff; +} + +static void +trad_enc_decrypt_update(struct trad_enc_ctx *ctx, const uint8_t *in, + size_t in_len, uint8_t *out, size_t out_len) +{ + unsigned i, max; + + max = (unsigned)((in_len < out_len)? in_len: out_len); + + for (i = 0; i < max; i++) { + uint8_t t = in[i] ^ trad_enc_decypt_byte(ctx); + out[i] = t; + trad_enc_update_keys(ctx, t); + } +} + +static int +trad_enc_init(struct trad_enc_ctx *ctx, const char *pw, size_t pw_len, + const uint8_t *key, size_t key_len, uint8_t *crcchk) +{ + uint8_t header[12]; + + if (key_len < 12) { + *crcchk = 0xff; + return -1; + } + + ctx->keys[0] = 305419896L; + ctx->keys[1] = 591751049L; + ctx->keys[2] = 878082192L; + + for (;pw_len; --pw_len) + trad_enc_update_keys(ctx, *pw++); + + trad_enc_decrypt_update(ctx, key, 12, header, 12); + /* Return the last byte for CRC check. */ + *crcchk = header[11]; + return 0; +} + +#if 0 +static void +crypt_derive_key_sha1(const void *p, int size, unsigned char *key, + int key_size) +{ +#define MD_SIZE 20 + archive_sha1_ctx ctx; + unsigned char md1[MD_SIZE]; + unsigned char md2[MD_SIZE * 2]; + unsigned char mkb[64]; + int i; + + archive_sha1_init(&ctx); + archive_sha1_update(&ctx, p, size); + archive_sha1_final(&ctx, md1); + + memset(mkb, 0x36, sizeof(mkb)); + for (i = 0; i < MD_SIZE; i++) + mkb[i] ^= md1[i]; + archive_sha1_init(&ctx); + archive_sha1_update(&ctx, mkb, sizeof(mkb)); + archive_sha1_final(&ctx, md2); + + memset(mkb, 0x5C, sizeof(mkb)); + for (i = 0; i < MD_SIZE; i++) + mkb[i] ^= md1[i]; + archive_sha1_init(&ctx); + archive_sha1_update(&ctx, mkb, sizeof(mkb)); + archive_sha1_final(&ctx, md2 + MD_SIZE); + + if (key_size > 32) + key_size = 32; + memcpy(key, md2, key_size); +#undef MD_SIZE +} +#endif + +/* * Common code for streaming or seeking modes. * * Includes code to read local file headers, decompress data @@ -163,9 +325,10 @@ struct zip { static unsigned long real_crc32(unsigned long crc, const void *buff, size_t len) { - return crc32(crc, buff, len); + return crc32(crc, buff, (unsigned int)len); } +/* Used by "ignorecrc32" option to speed up tests. */ static unsigned long fake_crc32(unsigned long crc, const void *buff, size_t len) { @@ -185,33 +348,36 @@ static struct { {3, "reduced-2"}, /* The file is Reduced with compression factor 2 */ {4, "reduced-3"}, /* The file is Reduced with compression factor 3 */ {5, "reduced-4"}, /* The file is Reduced with compression factor 4 */ - {6, "imploded"}, /* The file is Imploded */ - {7, "reserved"}, /* Reserved for Tokenizing compression algorithm */ + {6, "imploded"}, /* The file is Imploded */ + {7, "reserved"}, /* Reserved for Tokenizing compression algorithm */ {8, "deflation"}, /* The file is Deflated */ {9, "deflation-64-bit"}, /* Enhanced Deflating using Deflate64(tm) */ - {10, "ibm-terse"}, /* PKWARE Data Compression Library Imploding (old IBM TERSE) */ + {10, "ibm-terse"},/* PKWARE Data Compression Library Imploding + * (old IBM TERSE) */ {11, "reserved"}, /* Reserved by PKWARE */ - {12, "bzip"}, /* File is compressed using BZIP2 algorithm */ + {12, "bzip"}, /* File is compressed using BZIP2 algorithm */ {13, "reserved"}, /* Reserved by PKWARE */ - {14, "lzma"}, /* LZMA (EFS) */ + {14, "lzma"}, /* LZMA (EFS) */ {15, "reserved"}, /* Reserved by PKWARE */ {16, "reserved"}, /* Reserved by PKWARE */ {17, "reserved"}, /* Reserved by PKWARE */ {18, "ibm-terse-new"}, /* File is compressed using IBM TERSE (new) */ - {19, "ibm-lz777"}, /* IBM LZ77 z Architecture (PFS) */ + {19, "ibm-lz777"},/* IBM LZ77 z Architecture (PFS) */ {97, "wav-pack"}, /* WavPack compressed data */ - {98, "ppmd-1"} /* PPMd version I, Rev 1 */ + {98, "ppmd-1"}, /* PPMd version I, Rev 1 */ + {99, "aes"} /* WinZip AES encryption */ }; static const char * compression_name(const int compression) { - static const int num_compression_methods = sizeof(compression_methods)/sizeof(compression_methods[0]); + static const int num_compression_methods = + sizeof(compression_methods)/sizeof(compression_methods[0]); int i=0; + while(compression >= 0 && i < num_compression_methods) { - if (compression_methods[i].id == compression) { + if (compression_methods[i].id == compression) return compression_methods[i].name; - } i++; } return "??"; @@ -248,15 +414,15 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry) { unsigned offset = 0; - while (offset < extra_length - 4) - { + while (offset < extra_length - 4) { unsigned short headerid = archive_le16dec(p + offset); unsigned short datasize = archive_le16dec(p + offset + 2); + offset += 4; if (offset + datasize > extra_length) break; #ifdef DEBUG - fprintf(stderr, "Header id 0x%x, length %d\n", + fprintf(stderr, "Header id 0x%04x, length %d\n", headerid, datasize); #endif switch (headerid) { @@ -291,6 +457,23 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry) * on which file starts, but we don't handle * multi-volume Zip files. */ break; +#ifdef DEBUG + case 0x0017: + { + /* Strong encryption field. */ + if (archive_le16dec(p + offset) == 2) { + unsigned algId = + archive_le16dec(p + offset + 2); + unsigned bitLen = + archive_le16dec(p + offset + 4); + int flags = + archive_le16dec(p + offset + 6); + fprintf(stderr, "algId=0x%04x, bitLen=%u, " + "flgas=%d\n", algId, bitLen,flags); + } + break; + } +#endif case 0x5455: { /* Extended time field "UT". */ @@ -345,16 +528,34 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry) } break; } - case 0x6c65: + case 0x6c78: { - /* Experimental 'el' field */ + /* Experimental 'xl' field */ /* * Introduced Dec 2013 to provide a way to - * include external file attributes in local file - * header. This provides file type and permission - * information necessary to support full streaming - * extraction. Currently being discussed with - * other Zip developers... subject to change. + * include external file attributes (and other + * fields that ordinarily appear only in + * central directory) in local file header. + * This provides file type and permission + * information necessary to support full + * streaming extraction. Currently being + * discussed with other Zip developers + * ... subject to change. + * + * Format: + * The field starts with a bitmap that specifies + * which additional fields are included. The + * bitmap is variable length and can be extended in + * the future. + * + * n bytes - feature bitmap: first byte has low-order + * 7 bits. If high-order bit is set, a subsequent + * byte holds the next 7 bits, etc. + * + * if bitmap & 1, 2 byte "version made by" + * if bitmap & 2, 2 byte "internal file attributes" + * if bitmap & 4, 4 byte "external file attributes" + * if bitmap * 7, 2 byte comment length + n byte comment */ int bitmap, bitmap_last; @@ -373,7 +574,7 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry) } if (bitmap & 1) { - // 2 byte "version made by" + /* 2 byte "version made by" */ if (datasize < 2) break; zip_entry->system @@ -382,19 +583,19 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry) datasize -= 2; } if (bitmap & 2) { - // 2 byte "internal file attributes" + /* 2 byte "internal file attributes" */ uint32_t internal_attributes; if (datasize < 2) break; internal_attributes = archive_le16dec(p + offset); - // Not used by libarchive at present. + /* Not used by libarchive at present. */ (void)internal_attributes; /* UNUSED */ offset += 2; datasize -= 2; } if (bitmap & 4) { - // 4 byte "external file attributes" + /* 4 byte "external file attributes" */ uint32_t external_attributes; if (datasize < 4) break; @@ -408,7 +609,7 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry) datasize -= 4; } if (bitmap & 8) { - // 2 byte comment length + comment + /* 2 byte comment length + comment */ uint32_t comment_length; if (datasize < 2) break; @@ -419,7 +620,7 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry) if (datasize < comment_length) break; - // Comment is not supported by libarchive + /* Comment is not supported by libarchive */ offset += comment_length; datasize -= comment_length; } @@ -447,7 +648,7 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry) if (datasize >= 1 && p[offset] == 1) {/* version=1 */ if (datasize >= 4) { /* get a uid size. */ - uidsize = p[offset+1]; + uidsize = 0xff & (int)p[offset+1]; if (uidsize == 2) zip_entry->uid = archive_le16dec( @@ -459,7 +660,7 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry) } if (datasize >= (2 + uidsize + 3)) { /* get a gid size. */ - gidsize = p[offset+2+uidsize]; + gidsize = 0xff & (int)p[offset+2+uidsize]; if (gidsize == 2) zip_entry->gid = archive_le16dec( @@ -473,6 +674,19 @@ process_extra(const char *p, size_t extra_length, struct zip_entry* zip_entry) } break; } + case 0x9901: + /* WinZIp AES extra data field. */ + if (p[offset + 2] == 'A' && p[offset + 3] == 'E') { + /* Vendor version. */ + zip_entry->aes_extra.vendor = + archive_le16dec(p + offset); + /* AES encryption strength. */ + zip_entry->aes_extra.strength = p[offset + 4]; + /* Actual compression method. */ + zip_entry->aes_extra.compression = + p[offset + 5]; + } + break; default: break; } @@ -544,9 +758,14 @@ zip_read_local_file_header(struct archive_read *a, struct archive_entry *entry, return ARCHIVE_FATAL; } } + zip->init_decryption = (zip_entry->zip_flags & ZIP_ENCRYPTED); zip_entry->compression = (char)archive_le16dec(p + 8); zip_entry->mtime = zip_time(p + 10); zip_entry->crc32 = archive_le32dec(p + 14); + if (zip_entry->zip_flags & ZIP_LENGTH_AT_END) + zip_entry->decdat = p[11]; + else + zip_entry->decdat = p[17]; zip_entry->compressed_size = archive_le32dec(p + 18); zip_entry->uncompressed_size = archive_le32dec(p + 22); filename_length = archive_le16dec(p + 26); @@ -687,7 +906,15 @@ zip_read_local_file_header(struct archive_read *a, struct archive_entry *entry, archive_entry_set_atime(entry, zip_entry->atime, 0); if ((zip->entry->mode & AE_IFMT) == AE_IFLNK) { - size_t linkname_length = zip_entry->compressed_size; + size_t linkname_length; + + if (zip_entry->compressed_size > 64 * 1024) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Zip file with oversized link entry"); + return ARCHIVE_FATAL; + } + + linkname_length = (size_t)zip_entry->compressed_size; archive_entry_set_size(entry, 0); p = __archive_read_ahead(a, linkname_length, NULL); @@ -696,11 +923,6 @@ zip_read_local_file_header(struct archive_read *a, struct archive_entry *entry, "Truncated Zip file"); return ARCHIVE_FATAL; } - if (__archive_read_consume(a, linkname_length) < 0) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, - "Read error skipping symlink target name"); - return ARCHIVE_FATAL; - } sconv = zip->sconv; if (sconv == NULL && (zip->entry->zip_flags & ZIP_UTF8_NAME)) @@ -735,6 +957,12 @@ zip_read_local_file_header(struct archive_read *a, struct archive_entry *entry, } } zip_entry->uncompressed_size = zip_entry->compressed_size = 0; + + if (__archive_read_consume(a, linkname_length) < 0) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Read error skipping symlink target name"); + return ARCHIVE_FATAL; + } } else if (0 == (zip_entry->zip_flags & ZIP_LENGTH_AT_END) || zip_entry->uncompressed_size > 0) { /* Set the size only if it's meaningful. */ @@ -748,14 +976,51 @@ zip_read_local_file_header(struct archive_read *a, struct archive_entry *entry, zip->end_of_entry = 1; /* Set up a more descriptive format name. */ - snprintf(zip->format_name, sizeof(zip->format_name), "ZIP %d.%d (%s)", + archive_string_sprintf(&zip->format_name, "ZIP %d.%d (%s)", version / 10, version % 10, compression_name(zip->entry->compression)); - a->archive.archive_format_name = zip->format_name; + a->archive.archive_format_name = zip->format_name.s; return (ret); } +static int +check_authentication_code(struct archive_read *a, const void *_p) +{ + struct zip *zip = (struct zip *)(a->format->data); + + /* Check authentication code. */ + if (zip->hctx_valid) { + const void *p; + uint8_t hmac[20]; + size_t hmac_len = 20; + int cmp; + + archive_hmac_sha1_final(&zip->hctx, hmac, &hmac_len); + if (_p == NULL) { + /* Read authentication code. */ + p = __archive_read_ahead(a, AUTH_CODE_SIZE, NULL); + if (p == NULL) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, + "Truncated ZIP file data"); + return (ARCHIVE_FATAL); + } + } else { + p = _p; + } + cmp = memcmp(hmac, p, AUTH_CODE_SIZE); + __archive_read_consume(a, AUTH_CODE_SIZE); + if (cmp != 0) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_MISC, + "ZIP bad Authentication code"); + return (ARCHIVE_WARN); + } + } + return (ARCHIVE_OK); +} + /* * Read "uncompressed" data. There are three cases: * 1) We know the size of the data. This is always true for the @@ -787,6 +1052,7 @@ zip_read_data_none(struct archive_read *a, const void **_buff, struct zip *zip; const char *buff; ssize_t bytes_avail; + int r; (void)offset; /* UNUSED */ @@ -794,10 +1060,13 @@ zip_read_data_none(struct archive_read *a, const void **_buff, if (zip->entry->zip_flags & ZIP_LENGTH_AT_END) { const char *p; + ssize_t grabbing_bytes = 24; + if (zip->hctx_valid) + grabbing_bytes += AUTH_CODE_SIZE; /* Grab at least 24 bytes. */ - buff = __archive_read_ahead(a, 24, &bytes_avail); - if (bytes_avail < 24) { + buff = __archive_read_ahead(a, grabbing_bytes, &bytes_avail); + if (bytes_avail < grabbing_bytes) { /* Zip archives have end-of-archive markers that are longer than this, so a failure to get at least 24 bytes really does indicate a truncated @@ -810,21 +1079,34 @@ zip_read_data_none(struct archive_read *a, const void **_buff, /* Check for a complete PK\007\010 signature, followed * by the correct 4-byte CRC. */ p = buff; + if (zip->hctx_valid) + p += AUTH_CODE_SIZE; if (p[0] == 'P' && p[1] == 'K' && p[2] == '\007' && p[3] == '\010' && (archive_le32dec(p + 4) == zip->entry_crc32 - || zip->ignore_crc32)) { + || zip->ignore_crc32 + || (zip->hctx_valid + && zip->entry->aes_extra.vendor == AES_VENDOR_AE_2))) { if (zip->entry->flags & LA_USED_ZIP64) { zip->entry->crc32 = archive_le32dec(p + 4); - zip->entry->compressed_size = archive_le64dec(p + 8); - zip->entry->uncompressed_size = archive_le64dec(p + 16); + zip->entry->compressed_size = + archive_le64dec(p + 8); + zip->entry->uncompressed_size = + archive_le64dec(p + 16); zip->unconsumed = 24; } else { zip->entry->crc32 = archive_le32dec(p + 4); - zip->entry->compressed_size = archive_le32dec(p + 8); - zip->entry->uncompressed_size = archive_le32dec(p + 12); + zip->entry->compressed_size = + archive_le32dec(p + 8); + zip->entry->uncompressed_size = + archive_le32dec(p + 12); zip->unconsumed = 16; } + if (zip->hctx_valid) { + r = check_authentication_code(a, buff); + if (r != ARCHIVE_OK) + return (r); + } zip->end_of_entry = 1; return (ARCHIVE_OK); } @@ -841,6 +1123,8 @@ zip_read_data_none(struct archive_read *a, const void **_buff, else if (p[3] == '\007') { p += 1; } else if (p[3] == '\010' && p[2] == '\007' && p[1] == 'K' && p[0] == 'P') { + if (zip->hctx_valid) + p -= AUTH_CODE_SIZE; break; } else { p += 4; } } @@ -848,6 +1132,11 @@ zip_read_data_none(struct archive_read *a, const void **_buff, } else { if (zip->entry_bytes_remaining == 0) { zip->end_of_entry = 1; + if (zip->hctx_valid) { + r = check_authentication_code(a, NULL); + if (r != ARCHIVE_OK) + return (r); + } return (ARCHIVE_OK); } /* Grab a bunch of bytes. */ @@ -861,6 +1150,26 @@ zip_read_data_none(struct archive_read *a, const void **_buff, if (bytes_avail > zip->entry_bytes_remaining) bytes_avail = (ssize_t)zip->entry_bytes_remaining; } + if (zip->tctx_valid || zip->cctx_valid) { + size_t dec_size = bytes_avail; + + if (dec_size > zip->decrypted_buffer_size) + dec_size = zip->decrypted_buffer_size; + if (zip->tctx_valid) { + trad_enc_decrypt_update(&zip->tctx, + (const uint8_t *)buff, dec_size, + zip->decrypted_buffer, dec_size); + } else { + size_t dsize = dec_size; + archive_hmac_sha1_update(&zip->hctx, + (const uint8_t *)buff, dec_size); + archive_decrypto_aes_ctr_update(&zip->cctx, + (const uint8_t *)buff, dec_size, + zip->decrypted_buffer, &dsize); + } + bytes_avail = dec_size; + buff = (const char *)zip->decrypted_buffer; + } *size = bytes_avail; zip->entry_bytes_remaining -= bytes_avail; zip->entry_uncompressed_bytes_read += bytes_avail; @@ -902,7 +1211,7 @@ zip_read_data_deflate(struct archive_read *a, const void **buff, { struct zip *zip; ssize_t bytes_avail; - const void *compressed_buff; + const void *compressed_buff, *sp; int r; (void)offset; /* UNUSED */ @@ -931,7 +1240,7 @@ zip_read_data_deflate(struct archive_read *a, const void **buff, * available bytes; asking for more than that forces the * decompressor to combine reads by copying data. */ - compressed_buff = __archive_read_ahead(a, 1, &bytes_avail); + compressed_buff = sp = __archive_read_ahead(a, 1, &bytes_avail); if (0 == (zip->entry->zip_flags & ZIP_LENGTH_AT_END) && bytes_avail > zip->entry_bytes_remaining) { bytes_avail = (ssize_t)zip->entry_bytes_remaining; @@ -942,6 +1251,51 @@ zip_read_data_deflate(struct archive_read *a, const void **buff, return (ARCHIVE_FATAL); } + if (zip->tctx_valid || zip->cctx_valid) { + if (zip->decrypted_bytes_remaining < (size_t)bytes_avail) { + size_t buff_remaining = zip->decrypted_buffer_size + - (zip->decrypted_ptr - zip->decrypted_buffer); + + if (buff_remaining > (size_t)bytes_avail) + buff_remaining = (size_t)bytes_avail; + + if (0 == (zip->entry->zip_flags & ZIP_LENGTH_AT_END) && + zip->entry_bytes_remaining > 0) { + if ((int64_t)(zip->decrypted_bytes_remaining + + buff_remaining) + > zip->entry_bytes_remaining) { + if (zip->entry_bytes_remaining < + (int64_t)zip->decrypted_bytes_remaining) + buff_remaining = 0; + else + buff_remaining = + (size_t)zip->entry_bytes_remaining + - zip->decrypted_bytes_remaining; + } + } + if (buff_remaining > 0) { + if (zip->tctx_valid) { + trad_enc_decrypt_update(&zip->tctx, + compressed_buff, buff_remaining, + zip->decrypted_ptr + + zip->decrypted_bytes_remaining, + buff_remaining); + } else { + size_t dsize = buff_remaining; + archive_decrypto_aes_ctr_update( + &zip->cctx, + compressed_buff, buff_remaining, + zip->decrypted_ptr + + zip->decrypted_bytes_remaining, + &dsize); + } + zip->decrypted_bytes_remaining += buff_remaining; + } + } + bytes_avail = zip->decrypted_bytes_remaining; + compressed_buff = (const char *)zip->decrypted_ptr; + } + /* * A bug in zlib.h: stream.next_in should be marked 'const' * but isn't (the library never alters data through the @@ -974,6 +1328,16 @@ zip_read_data_deflate(struct archive_read *a, const void **buff, /* Consume as much as the compressor actually used. */ bytes_avail = zip->stream.total_in; + if (zip->tctx_valid || zip->cctx_valid) { + zip->decrypted_bytes_remaining -= bytes_avail; + if (zip->decrypted_bytes_remaining == 0) + zip->decrypted_ptr = zip->decrypted_buffer; + else + zip->decrypted_ptr += bytes_avail; + } + /* Calculate compressed data as much as we used.*/ + if (zip->hctx_valid) + archive_hmac_sha1_update(&zip->hctx, sp, bytes_avail); __archive_read_consume(a, bytes_avail); zip->entry_bytes_remaining -= bytes_avail; zip->entry_compressed_bytes_read += bytes_avail; @@ -982,6 +1346,12 @@ zip_read_data_deflate(struct archive_read *a, const void **buff, zip->entry_uncompressed_bytes_read += zip->stream.total_out; *buff = zip->uncompressed_buffer; + if (zip->end_of_entry && zip->hctx_valid) { + r = check_authentication_code(a, NULL); + if (r != ARCHIVE_OK) + return (r); + } + if (zip->end_of_entry && (zip->entry->zip_flags & ZIP_LENGTH_AT_END)) { const char *p; @@ -1015,13 +1385,392 @@ zip_read_data_deflate(struct archive_read *a, const void **buff, #endif static int +read_decryption_header(struct archive_read *a) +{ + struct zip *zip = (struct zip *)(a->format->data); + const char *p; + unsigned int remaining_size; + unsigned int ts; + + /* + * Read an initialization vector data field. + */ + p = __archive_read_ahead(a, 2, NULL); + if (p == NULL) + goto truncated; + ts = zip->iv_size; + zip->iv_size = archive_le16dec(p); + __archive_read_consume(a, 2); + if (ts < zip->iv_size) { + free(zip->iv); + zip->iv = NULL; + } + p = __archive_read_ahead(a, zip->iv_size, NULL); + if (p == NULL) + goto truncated; + if (zip->iv == NULL) { + zip->iv = malloc(zip->iv_size); + if (zip->iv == NULL) + goto nomem; + } + memcpy(zip->iv, p, zip->iv_size); + __archive_read_consume(a, zip->iv_size); + + /* + * Read a size of remaining decryption header field. + */ + p = __archive_read_ahead(a, 14, NULL); + if (p == NULL) + goto truncated; + remaining_size = archive_le32dec(p); + if (remaining_size < 16 || remaining_size > (1 << 18)) + goto corrupted; + + /* Check if format version is supported. */ + if (archive_le16dec(p+4) != 3) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, + "Unsupported encryption format version: %u", + archive_le16dec(p+4)); + return (ARCHIVE_FAILED); + } + + /* + * Read an encryption algorithm field. + */ + zip->alg_id = archive_le16dec(p+6); + switch (zip->alg_id) { + case 0x6601:/* DES */ + case 0x6602:/* RC2 */ + case 0x6603:/* 3DES 168 */ + case 0x6609:/* 3DES 112 */ + case 0x660E:/* AES 128 */ + case 0x660F:/* AES 192 */ + case 0x6610:/* AES 256 */ + case 0x6702:/* RC2 (version >= 5.2) */ + case 0x6720:/* Blowfish */ + case 0x6721:/* Twofish */ + case 0x6801:/* RC4 */ + /* Suuported encryption algorithm. */ + break; + default: + archive_set_error(&a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, + "Unknown encryption algorithm: %u", zip->alg_id); + return (ARCHIVE_FAILED); + } + + /* + * Read a bit length field. + */ + zip->bit_len = archive_le16dec(p+8); + + /* + * Read a flags field. + */ + zip->flags = archive_le16dec(p+10); + switch (zip->flags & 0xf000) { + case 0x0001: /* Password is required to decrypt. */ + case 0x0002: /* Certificates only. */ + case 0x0003: /* Password or certificate required to decrypt. */ + break; + default: + archive_set_error(&a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, + "Unknown encryption flag: %u", zip->flags); + return (ARCHIVE_FAILED); + } + if ((zip->flags & 0xf000) == 0 || + (zip->flags & 0xf000) == 0x4000) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, + "Unknown encryption flag: %u", zip->flags); + return (ARCHIVE_FAILED); + } + + /* + * Read an encrypted random data field. + */ + ts = zip->erd_size; + zip->erd_size = archive_le16dec(p+12); + __archive_read_consume(a, 14); + if ((zip->erd_size & 0xf) != 0 || + (zip->erd_size + 16) > remaining_size || + (zip->erd_size + 16) < zip->erd_size) + goto corrupted; + + if (ts < zip->erd_size) { + free(zip->erd); + zip->erd = NULL; + } + p = __archive_read_ahead(a, zip->erd_size, NULL); + if (p == NULL) + goto truncated; + if (zip->erd == NULL) { + zip->erd = malloc(zip->erd_size); + if (zip->erd == NULL) + goto nomem; + } + memcpy(zip->erd, p, zip->erd_size); + __archive_read_consume(a, zip->erd_size); + + /* + * Read a reserved data field. + */ + p = __archive_read_ahead(a, 4, NULL); + if (p == NULL) + goto truncated; + /* Reserved data size should be zero. */ + if (archive_le32dec(p) != 0) + goto corrupted; + __archive_read_consume(a, 4); + + /* + * Read a password validation data field. + */ + p = __archive_read_ahead(a, 2, NULL); + if (p == NULL) + goto truncated; + ts = zip->v_size; + zip->v_size = archive_le16dec(p); + __archive_read_consume(a, 2); + if ((zip->v_size & 0x0f) != 0 || + (zip->erd_size + zip->v_size + 16) > remaining_size || + (zip->erd_size + zip->v_size + 16) < (zip->erd_size + zip->v_size)) + goto corrupted; + if (ts < zip->v_size) { + free(zip->v_data); + zip->v_data = NULL; + } + p = __archive_read_ahead(a, zip->v_size, NULL); + if (p == NULL) + goto truncated; + if (zip->v_data == NULL) { + zip->v_data = malloc(zip->v_size); + if (zip->v_data == NULL) + goto nomem; + } + memcpy(zip->v_data, p, zip->v_size); + __archive_read_consume(a, zip->v_size); + + p = __archive_read_ahead(a, 4, NULL); + if (p == NULL) + goto truncated; + zip->v_crc32 = archive_le32dec(p); + __archive_read_consume(a, 4); + + /*return (ARCHIVE_OK); + * This is not fully implemnted yet.*/ + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Encrypted file is unsupported"); + return (ARCHIVE_FAILED); +truncated: + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Truncated ZIP file data"); + return (ARCHIVE_FATAL); +corrupted: + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Corrupted ZIP file data"); + return (ARCHIVE_FATAL); +nomem: + archive_set_error(&a->archive, ENOMEM, + "No memory for ZIP decryption"); + return (ARCHIVE_FATAL); +} + +static int +zip_alloc_decryption_buffer(struct archive_read *a) +{ + struct zip *zip = (struct zip *)(a->format->data); + size_t bs = 256 * 1024; + + if (zip->decrypted_buffer == NULL) { + zip->decrypted_buffer_size = bs; + zip->decrypted_buffer = malloc(bs); + if (zip->decrypted_buffer == NULL) { + archive_set_error(&a->archive, ENOMEM, + "No memory for ZIP decryption"); + return (ARCHIVE_FATAL); + } + } + zip->decrypted_ptr = zip->decrypted_buffer; + return (ARCHIVE_OK); +} + +static int +init_traditional_PKWARE_decryption(struct archive_read *a) +{ + struct zip *zip = (struct zip *)(a->format->data); + const void *p; + int retry; + int r; + + if (zip->tctx_valid) + return (ARCHIVE_OK); + + /* + Read the 12 bytes encryption header stored at + the start of the data area. + */ +#define ENC_HEADER_SIZE 12 + if (0 == (zip->entry->zip_flags & ZIP_LENGTH_AT_END) + && zip->entry_bytes_remaining < ENC_HEADER_SIZE) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Truncated Zip encrypted body: only %jd bytes available", + (intmax_t)zip->entry_bytes_remaining); + return (ARCHIVE_FATAL); + } + + p = __archive_read_ahead(a, ENC_HEADER_SIZE, NULL); + if (p == NULL) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Truncated ZIP file data"); + return (ARCHIVE_FATAL); + } + + for (retry = 0;; retry++) { + const char *passphrase; + uint8_t crcchk; + + passphrase = __archive_read_next_passphrase(a); + if (passphrase == NULL) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + (retry > 0)? + "Incorrect passphrase": + "Passphrase required for this entry"); + return (ARCHIVE_FAILED); + } + + /* + * Initialize ctx for Traditional PKWARE Decyption. + */ + r = trad_enc_init(&zip->tctx, passphrase, strlen(passphrase), + p, ENC_HEADER_SIZE, &crcchk); + if (r == 0 && crcchk == zip->entry->decdat) + break;/* The passphrase is OK. */ + if (retry > 10000) { + /* Avoid infinity loop. */ + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Too many incorrect passphrases"); + return (ARCHIVE_FAILED); + } + } + + __archive_read_consume(a, ENC_HEADER_SIZE); + zip->tctx_valid = 1; + if (0 == (zip->entry->zip_flags & ZIP_LENGTH_AT_END)) { + zip->entry_bytes_remaining -= ENC_HEADER_SIZE; + } + /*zip->entry_uncompressed_bytes_read += ENC_HEADER_SIZE;*/ + zip->entry_compressed_bytes_read += ENC_HEADER_SIZE; + zip->decrypted_bytes_remaining = 0; + + return (zip_alloc_decryption_buffer(a)); +#undef ENC_HEADER_SIZE +} + +static int +init_WinZip_AES_decryption(struct archive_read *a) +{ + struct zip *zip = (struct zip *)(a->format->data); + const void *p; + const uint8_t *pv; + size_t key_len, salt_len; + uint8_t derived_key[MAX_DERIVED_KEY_BUF_SIZE]; + int retry; + int r; + + if (zip->cctx_valid || zip->hctx_valid) + return (ARCHIVE_OK); + + switch (zip->entry->aes_extra.strength) { + case 1: salt_len = 8; key_len = 16; break; + case 2: salt_len = 12; key_len = 24; break; + case 3: salt_len = 16; key_len = 32; break; + default: goto corrupted; + } + p = __archive_read_ahead(a, salt_len + 2, NULL); + if (p == NULL) + goto truncated; + + for (retry = 0;; retry++) { + const char *passphrase; + + passphrase = __archive_read_next_passphrase(a); + if (passphrase == NULL) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + (retry > 0)? + "Incorrect passphrase": + "Passphrase required for this entry"); + return (ARCHIVE_FAILED); + } + memset(derived_key, 0, sizeof(derived_key)); + r = archive_pbkdf2_sha1(passphrase, strlen(passphrase), + p, salt_len, 1000, derived_key, key_len * 2 + 2); + if (r != 0) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Decryption is unsupported due to lack of " + "crypto library"); + return (ARCHIVE_FAILED); + } + + /* Check password verification value. */ + pv = ((const uint8_t *)p) + salt_len; + if (derived_key[key_len * 2] == pv[0] && + derived_key[key_len * 2 + 1] == pv[1]) + break;/* The passphrase is OK. */ + if (retry > 10000) { + /* Avoid infinity loop. */ + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Too many incorrect passphrases"); + return (ARCHIVE_FAILED); + } + } + + r = archive_decrypto_aes_ctr_init(&zip->cctx, derived_key, key_len); + if (r != 0) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Decryption is unsupported due to lack of crypto library"); + return (ARCHIVE_FAILED); + } + r = archive_hmac_sha1_init(&zip->hctx, derived_key + key_len, key_len); + if (r != 0) { + archive_decrypto_aes_ctr_release(&zip->cctx); + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Failed to initialize HMAC-SHA1"); + return (ARCHIVE_FAILED); + } + zip->cctx_valid = zip->hctx_valid = 1; + __archive_read_consume(a, salt_len + 2); + zip->entry_bytes_remaining -= salt_len + 2 + AUTH_CODE_SIZE; + if (0 == (zip->entry->zip_flags & ZIP_LENGTH_AT_END) + && zip->entry_bytes_remaining < 0) + goto corrupted; + zip->entry_compressed_bytes_read += salt_len + 2 + AUTH_CODE_SIZE; + zip->decrypted_bytes_remaining = 0; + + zip->entry->compression = zip->entry->aes_extra.compression; + return (zip_alloc_decryption_buffer(a)); + +truncated: + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Truncated ZIP file data"); + return (ARCHIVE_FATAL); +corrupted: + archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + "Corrupted ZIP file data"); + return (ARCHIVE_FATAL); +} + +static int archive_read_format_zip_read_data(struct archive_read *a, const void **buff, size_t *size, int64_t *offset) { int r; struct zip *zip = (struct zip *)(a->format->data); - if (zip->has_encrypted_entries == ARCHIVE_READ_FORMAT_ENCRYPTION_DONT_KNOW) { + if (zip->has_encrypted_entries == + ARCHIVE_READ_FORMAT_ENCRYPTION_DONT_KNOW) { zip->has_encrypted_entries = 0; } @@ -1037,16 +1786,22 @@ archive_read_format_zip_read_data(struct archive_read *a, if (AE_IFREG != (zip->entry->mode & AE_IFMT)) return (ARCHIVE_EOF); - if (zip->entry->zip_flags & (ZIP_ENCRYPTED | ZIP_STRONG_ENCRYPTED)) { - zip->has_encrypted_entries = 1; - archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, - "Encrypted file is unsupported"); - return (ARCHIVE_FAILED); - } - __archive_read_consume(a, zip->unconsumed); zip->unconsumed = 0; + if (zip->init_decryption) { + zip->has_encrypted_entries = 1; + if (zip->entry->zip_flags & ZIP_STRONG_ENCRYPTED) + r = read_decryption_header(a); + else if (zip->entry->compression == WINZIP_AES_ENCRYPTION) + r = init_WinZip_AES_decryption(a); + else + r = init_traditional_PKWARE_decryption(a); + if (r != ARCHIVE_OK) + return (r); + zip->init_decryption = 0; + } + switch(zip->entry->compression) { case 0: /* No compression. */ r = zip_read_data_none(a, buff, size, offset); @@ -1096,7 +1851,9 @@ archive_read_format_zip_read_data(struct archive_read *a, return (ARCHIVE_WARN); } /* Check computed CRC against header */ - if (zip->entry->crc32 != zip->entry_crc32 + if ((!zip->hctx_valid || + zip->entry->aes_extra.vendor != AES_VENDOR_AE_2) && + zip->entry->crc32 != zip->entry_crc32 && !zip->ignore_crc32) { archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "ZIP bad CRC: 0x%lx should be 0x%lx", @@ -1130,6 +1887,15 @@ archive_read_format_zip_cleanup(struct archive_read *a) zip_entry = next_zip_entry; } } + free(zip->decrypted_buffer); + if (zip->cctx_valid) + archive_decrypto_aes_ctr_release(&zip->cctx); + if (zip->hctx_valid) + archive_hmac_sha1_cleanup(&zip->hctx); + free(zip->iv); + free(zip->erd); + free(zip->v_data); + archive_string_free(&zip->format_name); free(zip); (a->format->data) = NULL; return (ARCHIVE_OK); @@ -1217,7 +1983,8 @@ static int archive_read_support_format_zip_capabilities_streamable(struct archive_read * a) { (void)a; /* UNUSED */ - return (ARCHIVE_READ_FORMAT_CAPS_ENCRYPT_DATA | ARCHIVE_READ_FORMAT_CAPS_ENCRYPT_METADATA); + return (ARCHIVE_READ_FORMAT_CAPS_ENCRYPT_DATA | + ARCHIVE_READ_FORMAT_CAPS_ENCRYPT_METADATA); } static int @@ -1275,9 +2042,9 @@ archive_read_format_zip_streamable_read_header(struct archive_read *a, * archive_read_data(), so be it. We'll do the same check there * as well. */ - if (zip->has_encrypted_entries == ARCHIVE_READ_FORMAT_ENCRYPTION_DONT_KNOW) { + if (zip->has_encrypted_entries == + ARCHIVE_READ_FORMAT_ENCRYPTION_DONT_KNOW) zip->has_encrypted_entries = 0; - } /* Make sure we have a zip_entry structure to use. */ if (zip->zip_entries == NULL) { @@ -1291,6 +2058,13 @@ archive_read_format_zip_streamable_read_header(struct archive_read *a, zip->entry = zip->zip_entries; memset(zip->entry, 0, sizeof(struct zip_entry)); + if (zip->cctx_valid) + archive_decrypto_aes_ctr_release(&zip->cctx); + if (zip->hctx_valid) + archive_hmac_sha1_cleanup(&zip->hctx); + zip->tctx_valid = zip->cctx_valid = zip->hctx_valid = 0; + __archive_read_reset_passphrase(a); + /* Search ahead for the next local file header. */ __archive_read_consume(a, zip->unconsumed); zip->unconsumed = 0; @@ -1363,12 +2137,28 @@ archive_read_format_zip_read_data_skip_streamable(struct archive_read *a) if (0 == (zip->entry->zip_flags & ZIP_LENGTH_AT_END) || zip->entry->compressed_size > 0) { /* We know the compressed length, so we can just skip. */ - bytes_skipped = __archive_read_consume(a, zip->entry_bytes_remaining); + bytes_skipped = __archive_read_consume(a, + zip->entry_bytes_remaining); if (bytes_skipped < 0) return (ARCHIVE_FATAL); return (ARCHIVE_OK); } + if (zip->init_decryption) { + int r; + + zip->has_encrypted_entries = 1; + if (zip->entry->zip_flags & ZIP_STRONG_ENCRYPTED) + r = read_decryption_header(a); + else if (zip->entry->compression == WINZIP_AES_ENCRYPTION) + r = init_WinZip_AES_decryption(a); + else + r = init_traditional_PKWARE_decryption(a); + if (r != ARCHIVE_OK) + return (r); + zip->init_decryption = 0; + } + /* We're streaming and we don't know the length. */ /* If the body is compressed and we know the format, we can * find an exact end-of-entry by decompressing it. */ @@ -1406,9 +2196,11 @@ archive_read_format_zip_read_data_skip_streamable(struct archive_read *a) else if (p[3] == '\010' && p[2] == '\007' && p[1] == 'K' && p[0] == 'P') { if (zip->entry->flags & LA_USED_ZIP64) - __archive_read_consume(a, p - buff + 24); + __archive_read_consume(a, + p - buff + 24); else - __archive_read_consume(a, p - buff + 16); + __archive_read_consume(a, + p - buff + 16); return ARCHIVE_OK; } else { p += 4; } } @@ -1427,13 +2219,12 @@ archive_read_support_format_zip_streamable(struct archive *_a) archive_check_magic(_a, ARCHIVE_READ_MAGIC, ARCHIVE_STATE_NEW, "archive_read_support_format_zip"); - zip = (struct zip *)malloc(sizeof(*zip)); + zip = (struct zip *)calloc(1, sizeof(*zip)); if (zip == NULL) { archive_set_error(&a->archive, ENOMEM, "Can't allocate zip data"); return (ARCHIVE_FATAL); } - memset(zip, 0, sizeof(*zip)); /* Streamable reader doesn't support mac extensions. */ zip->process_mac_extensions = 0; @@ -1473,7 +2264,8 @@ static int archive_read_support_format_zip_capabilities_seekable(struct archive_read * a) { (void)a; /* UNUSED */ - return (ARCHIVE_READ_FORMAT_CAPS_ENCRYPT_DATA | ARCHIVE_READ_FORMAT_CAPS_ENCRYPT_METADATA); + return (ARCHIVE_READ_FORMAT_CAPS_ENCRYPT_DATA | + ARCHIVE_READ_FORMAT_CAPS_ENCRYPT_METADATA); } /* @@ -1514,7 +2306,11 @@ read_eocd(struct zip *zip, const char *p, int64_t current_offset) return 32; } -static int +/* + * Examine Zip64 EOCD locator: If it's valid, store the information + * from it. + */ +static void read_zip64_eocd(struct archive_read *a, struct zip *zip, const char *p) { int64_t eocd64_offset; @@ -1524,37 +2320,35 @@ read_zip64_eocd(struct archive_read *a, struct zip *zip, const char *p) /* Central dir must be on first volume. */ if (archive_le32dec(p + 4) != 0) - return 0; + return; /* Must be only a single volume. */ if (archive_le32dec(p + 16) != 1) - return 0; + return; /* Find the Zip64 EOCD record. */ eocd64_offset = archive_le64dec(p + 8); if (__archive_read_seek(a, eocd64_offset, SEEK_SET) < 0) - return 0; + return; if ((p = __archive_read_ahead(a, 56, NULL)) == NULL) - return 0; + return; /* Make sure we can read all of it. */ eocd64_size = archive_le64dec(p + 4) + 12; if (eocd64_size < 56 || eocd64_size > 16384) - return 0; - if ((p = __archive_read_ahead(a, eocd64_size, NULL)) == NULL) - return 0; + return; + if ((p = __archive_read_ahead(a, (size_t)eocd64_size, NULL)) == NULL) + return; /* Sanity-check the EOCD64 */ if (archive_le32dec(p + 16) != 0) /* Must be disk #0 */ - return 0; + return; if (archive_le32dec(p + 20) != 0) /* CD must be on disk #0 */ - return 0; + return; /* CD can't be split. */ if (archive_le64dec(p + 24) != archive_le64dec(p + 32)) - return 0; + return; /* Save the central directory offset for later use. */ zip->central_directory_offset = archive_le64dec(p + 48); - - return 32; } static int @@ -1575,40 +2369,39 @@ archive_read_format_zip_seekable_bid(struct archive_read *a, int best_bid) return 0; /* Search last 16k of file for end-of-central-directory - * record (which starts with PK\005\006) or Zip64 locator - * record (which begins with PK\006\007) */ - tail = zipmin(1024 * 16, file_size); + * record (which starts with PK\005\006) */ + tail = (int)zipmin(1024 * 16, file_size); current_offset = __archive_read_seek(a, -tail, SEEK_END); if (current_offset < 0) return 0; if ((p = __archive_read_ahead(a, (size_t)tail, NULL)) == NULL) return 0; - /* TODO: Rework this to search backwards from the end. We - * normally expect the EOCD record to be at the very end, so - * that should be significantly faster. Tricky part: Make - * sure we still prefer the Zip64 locator if it's present. */ - for (i = 0; i <= tail - 22;) { - switch (p[i + 3]) { - case 'P': i += 3; break; - case 'K': i += 2; break; - case 005: i += 1; break; - case 006: + /* Boyer-Moore search backwards from the end, since we want + * to match the last EOCD in the file (there can be more than + * one if there is an uncompressed Zip archive as a member + * within this Zip archive). */ + for (i = tail - 22; i > 0;) { + switch (p[i]) { + case 'P': if (memcmp(p + i, "PK\005\006", 4) == 0) { - int ret = read_eocd(zip, p + i, current_offset + i); - if (ret > 0) - return (ret); - } - i += 1; /* Look for PK\006\007 next */ - break; - case 007: - if (memcmp(p + i, "PK\006\007", 4) == 0) { - int ret = read_zip64_eocd(a, zip, p + i); - if (ret > 0) + int ret = read_eocd(zip, p + i, + current_offset + i); + if (ret > 0) { + /* Zip64 EOCD locator precedes + * regular EOCD if present. */ + if (i >= 20 + && memcmp(p + i - 20, "PK\006\007", 4) == 0) { + read_zip64_eocd(a, zip, p + i - 20); + } return (ret); + } } - i += 4; + i -= 4; break; - default: i += 4; break; + case 'K': i -= 1; break; + case 005: i -= 2; break; + case 006: i -= 3; break; + default: i -= 4; break; } } return 0; @@ -1760,7 +2553,8 @@ slurp_central_directory(struct archive_read *a, struct zip *zip) } __archive_read_consume(a, i); } - correction = archive_filter_bytes(&a->archive, 0) - zip->central_directory_offset; + correction = archive_filter_bytes(&a->archive, 0) + - zip->central_directory_offset; __archive_rb_tree_init(&zip->tree, &rb_ops); __archive_rb_tree_init(&zip->tree_rsrc, &rb_rsrc_ops); @@ -1795,12 +2589,17 @@ slurp_central_directory(struct archive_read *a, struct zip *zip) zip_entry->system = p[5]; /* version_required = archive_le16dec(p + 6); */ zip_entry->zip_flags = archive_le16dec(p + 8); - if (zip_entry->zip_flags & (ZIP_ENCRYPTED | ZIP_STRONG_ENCRYPTED)){ + if (zip_entry->zip_flags + & (ZIP_ENCRYPTED | ZIP_STRONG_ENCRYPTED)){ zip->has_encrypted_entries = 1; } zip_entry->compression = (char)archive_le16dec(p + 10); zip_entry->mtime = zip_time(p + 12); zip_entry->crc32 = archive_le32dec(p + 16); + if (zip_entry->zip_flags & ZIP_LENGTH_AT_END) + zip_entry->decdat = p[13]; + else + zip_entry->decdat = p[19]; zip_entry->compressed_size = archive_le32dec(p + 20); zip_entry->uncompressed_size = archive_le32dec(p + 24); filename_length = archive_le16dec(p + 28); @@ -1823,9 +2622,11 @@ slurp_central_directory(struct archive_read *a, struct zip *zip) /* We're done with the regular data; get the filename and * extra data. */ __archive_read_consume(a, 46); - if ((p = __archive_read_ahead(a, filename_length + extra_length, NULL)) - == NULL) { - archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT, + p = __archive_read_ahead(a, filename_length + extra_length, + NULL); + if (p == NULL) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, "Truncated ZIP file header"); return ARCHIVE_FATAL; } @@ -1850,26 +2651,30 @@ slurp_central_directory(struct archive_read *a, struct zip *zip) * resource fork file to expose it. */ if (name[filename_length-1] != '/' && (r - name < 3 || r[0] != '.' || r[1] != '_')) { - __archive_rb_tree_insert_node(&zip->tree, - &zip_entry->node); + __archive_rb_tree_insert_node( + &zip->tree, &zip_entry->node); /* Expose its parent directories. */ - expose_parent_dirs(zip, name, filename_length); + expose_parent_dirs(zip, name, + filename_length); } else { /* This file is a resource fork file or * a directory. */ - archive_strncpy(&(zip_entry->rsrcname), name, - filename_length); - __archive_rb_tree_insert_node(&zip->tree_rsrc, - &zip_entry->node); + archive_strncpy(&(zip_entry->rsrcname), + name, filename_length); + __archive_rb_tree_insert_node( + &zip->tree_rsrc, &zip_entry->node); } } else { - /* Generate resource fork name to find its resource - * file at zip->tree_rsrc. */ - archive_strcpy(&(zip_entry->rsrcname), "__MACOSX/"); - archive_strncat(&(zip_entry->rsrcname), name, r - name); + /* Generate resource fork name to find its + * resource file at zip->tree_rsrc. */ + archive_strcpy(&(zip_entry->rsrcname), + "__MACOSX/"); + archive_strncat(&(zip_entry->rsrcname), + name, r - name); archive_strcat(&(zip_entry->rsrcname), "._"); archive_strncat(&(zip_entry->rsrcname), - name + (r - name), filename_length - (r - name)); + name + (r - name), + filename_length - (r - name)); /* Register an entry to RB tree to sort it by * file offset. */ __archive_rb_tree_insert_node(&zip->tree, @@ -2060,9 +2865,9 @@ archive_read_format_zip_seekable_read_header(struct archive_read *a, * archive_read_data(), so be it. We'll do the same check there * as well. */ - if (zip->has_encrypted_entries == ARCHIVE_READ_FORMAT_ENCRYPTION_DONT_KNOW) { + if (zip->has_encrypted_entries == + ARCHIVE_READ_FORMAT_ENCRYPTION_DONT_KNOW) zip->has_encrypted_entries = 0; - } a->archive.archive_format = ARCHIVE_FORMAT_ZIP; if (a->archive.archive_format_name == NULL) @@ -2091,6 +2896,13 @@ archive_read_format_zip_seekable_read_header(struct archive_read *a, else rsrc = NULL; + if (zip->cctx_valid) + archive_decrypto_aes_ctr_release(&zip->cctx); + if (zip->hctx_valid) + archive_hmac_sha1_cleanup(&zip->hctx); + zip->tctx_valid = zip->cctx_valid = zip->hctx_valid = 0; + __archive_read_reset_passphrase(a); + /* File entries are sorted by the header offset, we should mostly * use __archive_read_consume to advance a read point to avoid redundant * data reading. */ @@ -2099,7 +2911,8 @@ archive_read_format_zip_seekable_read_header(struct archive_read *a, __archive_read_consume(a, zip->entry->local_header_offset - offset); else if (offset != zip->entry->local_header_offset) { - __archive_read_seek(a, zip->entry->local_header_offset, SEEK_SET); + __archive_read_seek(a, zip->entry->local_header_offset, + SEEK_SET); } zip->unconsumed = 0; r = zip_read_local_file_header(a, entry, zip); @@ -2137,13 +2950,12 @@ archive_read_support_format_zip_seekable(struct archive *_a) archive_check_magic(_a, ARCHIVE_READ_MAGIC, ARCHIVE_STATE_NEW, "archive_read_support_format_zip_seekable"); - zip = (struct zip *)malloc(sizeof(*zip)); + zip = (struct zip *)calloc(1, sizeof(*zip)); if (zip == NULL) { archive_set_error(&a->archive, ENOMEM, "Can't allocate zip data"); return (ARCHIVE_FATAL); } - memset(zip, 0, sizeof(*zip)); #ifdef HAVE_COPYFILE_H /* Set this by default on Mac OS. */ diff --git a/Utilities/cmlibarchive/libarchive/archive_string.c b/Utilities/cmlibarchive/libarchive/archive_string.c index 87f9288..3d4be82 100644 --- a/Utilities/cmlibarchive/libarchive/archive_string.c +++ b/Utilities/cmlibarchive/libarchive/archive_string.c @@ -71,6 +71,10 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_string.c 201095 2009-12-28 02:33 #define wmemcpy(a,b,i) (wchar_t *)memcpy((a), (b), (i) * sizeof(wchar_t)) #endif +#if !defined(HAVE_WMEMMOVE) && !defined(wmemmove) +#define wmemmove(a,b,i) (wchar_t *)memmove((a), (b), (i) * sizeof(wchar_t)) +#endif + struct archive_string_conv { struct archive_string_conv *next; char *from_charset; @@ -127,12 +131,7 @@ struct archive_string_conv { #define UNICODE_MAX 0x10FFFF #define UNICODE_R_CHAR 0xFFFD /* Replacement character. */ /* Set U+FFFD(Replacement character) in UTF-8. */ -#define UTF8_SET_R_CHAR(outp) do { \ - (outp)[0] = 0xef; \ - (outp)[1] = 0xbf; \ - (outp)[2] = 0xbd; \ -} while (0) -#define UTF8_R_CHAR_SIZE 3 +static const char utf8_replacement_char[] = {0xef, 0xbf, 0xbd}; static struct archive_string_conv *find_sconv_object(struct archive *, const char *, const char *); @@ -203,7 +202,7 @@ archive_string_append(struct archive_string *as, const char *p, size_t s) { if (archive_string_ensure(as, as->length + s + 1) == NULL) return (NULL); - memcpy(as->s + as->length, p, s); + memmove(as->s + as->length, p, s); as->length += s; as->s[as->length] = 0; return (as); @@ -214,7 +213,7 @@ archive_wstring_append(struct archive_wstring *as, const wchar_t *p, size_t s) { if (archive_wstring_ensure(as, as->length + s + 1) == NULL) return (NULL); - wmemcpy(as->s + as->length, p, s); + wmemmove(as->s + as->length, p, s); as->length += s; as->s[as->length] = 0; return (as); @@ -2037,7 +2036,7 @@ iconv_strncat_in_locale(struct archive_string *as, const void *_p, if (sc->flag & (SCONV_TO_UTF8 | SCONV_TO_UTF16)) { size_t rbytes; if (sc->flag & SCONV_TO_UTF8) - rbytes = UTF8_R_CHAR_SIZE; + rbytes = sizeof(utf8_replacement_char); else rbytes = 2; @@ -2053,7 +2052,7 @@ iconv_strncat_in_locale(struct archive_string *as, const void *_p, - as->length - to_size; } if (sc->flag & SCONV_TO_UTF8) - UTF8_SET_R_CHAR(outp); + memcpy(outp, utf8_replacement_char, sizeof(utf8_replacement_char)); else if (sc->flag & SCONV_TO_UTF16BE) archive_be16enc(outp, UNICODE_R_CHAR); else @@ -2202,9 +2201,7 @@ best_effort_strncat_in_locale(struct archive_string *as, const void *_p, size_t length, struct archive_string_conv *sc) { size_t remaining; - char *otp; const uint8_t *itp; - size_t avail; int return_value = 0; /* success */ /* @@ -2223,46 +2220,25 @@ best_effort_strncat_in_locale(struct archive_string *as, const void *_p, * byte sequence 0xEF 0xBD 0xBD, which are code point U+FFFD, * a Replacement Character in Unicode. */ - if (archive_string_ensure(as, as->length + length + 1) == NULL) - return (-1); remaining = length; itp = (const uint8_t *)_p; - otp = as->s + as->length; - avail = as->buffer_length - as->length -1; while (*itp && remaining > 0) { - if (*itp > 127 && (sc->flag & SCONV_TO_UTF8)) { - if (avail < UTF8_R_CHAR_SIZE) { - as->length = otp - as->s; - if (NULL == archive_string_ensure(as, - as->buffer_length + remaining + - UTF8_R_CHAR_SIZE)) - return (-1); - otp = as->s + as->length; - avail = as->buffer_length - as->length -1; + if (*itp > 127) { + // Non-ASCII: Substitute with suitable replacement + if (sc->flag & SCONV_TO_UTF8) { + if (archive_string_append(as, utf8_replacement_char, sizeof(utf8_replacement_char)) == NULL) { + __archive_errx(1, "Out of memory"); + } + } else { + archive_strappend_char(as, '?'); } - /* - * When coping a string in UTF-8, unknown character - * should be U+FFFD (replacement character). - */ - UTF8_SET_R_CHAR(otp); - otp += UTF8_R_CHAR_SIZE; - avail -= UTF8_R_CHAR_SIZE; - itp++; - remaining--; - return_value = -1; - } else if (*itp > 127) { - *otp++ = '?'; - itp++; - remaining--; return_value = -1; } else { - *otp++ = (char)*itp++; - remaining--; + archive_strappend_char(as, *itp); } + ++itp; } - as->length = otp - as->s; - as->s[as->length] = '\0'; return (return_value); } @@ -2488,6 +2464,9 @@ unicode_to_utf8(char *p, size_t remaining, uint32_t uc) { char *_p = p; + /* Invalid Unicode char maps to Replacement character */ + if (uc > UNICODE_MAX) + uc = UNICODE_R_CHAR; /* Translate code point to UTF8 */ if (uc <= 0x7f) { if (remaining == 0) @@ -2504,22 +2483,13 @@ unicode_to_utf8(char *p, size_t remaining, uint32_t uc) *p++ = 0xe0 | ((uc >> 12) & 0x0f); *p++ = 0x80 | ((uc >> 6) & 0x3f); *p++ = 0x80 | (uc & 0x3f); - } else if (uc <= UNICODE_MAX) { + } else { if (remaining < 4) return (0); *p++ = 0xf0 | ((uc >> 18) & 0x07); *p++ = 0x80 | ((uc >> 12) & 0x3f); *p++ = 0x80 | ((uc >> 6) & 0x3f); *p++ = 0x80 | (uc & 0x3f); - } else { - /* - * Undescribed code point should be U+FFFD - * (replacement character). - */ - if (remaining < UTF8_R_CHAR_SIZE) - return (0); - UTF8_SET_R_CHAR(p); - p += UTF8_R_CHAR_SIZE; } return (p - _p); } @@ -3887,7 +3857,7 @@ archive_mstring_get_utf8(struct archive *a, struct archive_mstring *aes, sc = archive_string_conversion_to_charset(a, "UTF-8", 1); if (sc == NULL) return (-1);/* Couldn't allocate memory for sc. */ - r = archive_strncpy_l(&(aes->aes_mbs), aes->aes_mbs.s, + r = archive_strncpy_l(&(aes->aes_utf8), aes->aes_mbs.s, aes->aes_mbs.length, sc); if (a == NULL) free_sconv_object(sc); @@ -4062,6 +4032,19 @@ archive_mstring_copy_wcs(struct archive_mstring *aes, const wchar_t *wcs) } int +archive_mstring_copy_utf8(struct archive_mstring *aes, const char *utf8) +{ + if (utf8 == NULL) { + aes->aes_set = 0; + } + aes->aes_set = AES_SET_UTF8; + archive_string_empty(&(aes->aes_mbs)); + archive_string_empty(&(aes->aes_wcs)); + archive_strncpy(&(aes->aes_utf8), utf8, strlen(utf8)); + return (int)strlen(utf8); +} + +int archive_mstring_copy_wcs_len(struct archive_mstring *aes, const wchar_t *wcs, size_t len) { diff --git a/Utilities/cmlibarchive/libarchive/archive_util.c b/Utilities/cmlibarchive/libarchive/archive_util.c index d136498..5f31edf 100644 --- a/Utilities/cmlibarchive/libarchive/archive_util.c +++ b/Utilities/cmlibarchive/libarchive/archive_util.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2009-2012 Michihiro NAKAJIMA + * Copyright (c) 2009-2012,2014 Michihiro NAKAJIMA * Copyright (c) 2003-2007 Tim Kientzle * All rights reserved. * @@ -54,9 +54,13 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_util.c 201098 2009-12-28 02:58:1 #ifdef HAVE_BZLIB_H #include <cm_bzlib.h> #endif +#ifdef HAVE_LZ4_H +#include <lz4.h> +#endif #include "archive.h" #include "archive_private.h" +#include "archive_random_private.h" #include "archive_string.h" #ifndef O_CLOEXEC @@ -113,7 +117,11 @@ archive_version_details(void) archive_strncat(&str, p, sep - p); } #endif - } +#if defined(HAVE_LZ4_H) && defined(HAVE_LIBLZ4) + archive_string_sprintf(&str, " liblz4/%d.%d.%d", + LZ4_VERSION_MAJOR, LZ4_VERSION_MINOR, LZ4_VERSION_RELEASE); +#endif + } return str.s; } @@ -465,7 +473,6 @@ __archive_mktemp(const char *tmpdir) struct stat st; int fd; char *tp, *ep; - unsigned seed; fd = -1; archive_string_init(&temp_name); @@ -489,21 +496,15 @@ __archive_mktemp(const char *tmpdir) archive_strcat(&temp_name, "XXXXXXXXXX"); ep = temp_name.s + archive_strlen(&temp_name); - fd = open("/dev/random", O_RDONLY | O_CLOEXEC); - __archive_ensure_cloexec_flag(fd); - if (fd < 0) - seed = time(NULL); - else { - if (read(fd, &seed, sizeof(seed)) < 0) - seed = time(NULL); - close(fd); - } do { char *p; p = tp; - while (p < ep) - *p++ = num[((unsigned)rand_r(&seed)) % sizeof(num)]; + archive_random(p, ep - p); + while (p < ep) { + int d = *((unsigned char *)p) % sizeof(num); + *p++ = num[d]; + } fd = open(temp_name.s, O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0600); } while (fd < 0 && errno == EEXIST); @@ -551,62 +552,70 @@ __archive_ensure_cloexec_flag(int fd) static int archive_utility_string_sort_helper(char **strings, unsigned int n) { - unsigned int i, lesser_count, greater_count; - char **lesser, **greater, **tmp, *pivot; - int retval1, retval2; - - /* A list of 0 or 1 elements is already sorted */ - if (n <= 1) - return (ARCHIVE_OK); - - lesser_count = greater_count = 0; - lesser = greater = NULL; - pivot = strings[0]; - for (i = 1; i < n; i++) - { - if (strcmp(strings[i], pivot) < 0) - { - lesser_count++; - tmp = (char **)realloc(lesser, lesser_count * sizeof(char *)); - if (!tmp) - return (ARCHIVE_FATAL); - lesser = tmp; - lesser[lesser_count - 1] = strings[i]; - } - else - { - greater_count++; - tmp = (char **)realloc(greater, greater_count * sizeof(char *)); - if (!tmp) - return (ARCHIVE_FATAL); - greater = tmp; - greater[greater_count - 1] = strings[i]; - } - } - - /* quicksort(lesser) */ - retval1 = archive_utility_string_sort_helper(lesser, lesser_count); - for (i = 0; i < lesser_count; i++) - strings[i] = lesser[i]; - free(lesser); - - /* pivot */ - strings[lesser_count] = pivot; - - /* quicksort(greater) */ - retval2 = archive_utility_string_sort_helper(greater, greater_count); - for (i = 0; i < greater_count; i++) - strings[lesser_count + 1 + i] = greater[i]; - free(greater); - - return (retval1 < retval2) ? retval1 : retval2; + unsigned int i, lesser_count, greater_count; + char **lesser, **greater, **tmp, *pivot; + int retval1, retval2; + + /* A list of 0 or 1 elements is already sorted */ + if (n <= 1) + return (ARCHIVE_OK); + + lesser_count = greater_count = 0; + lesser = greater = NULL; + pivot = strings[0]; + for (i = 1; i < n; i++) + { + if (strcmp(strings[i], pivot) < 0) + { + lesser_count++; + tmp = (char **)realloc(lesser, + lesser_count * sizeof(char *)); + if (!tmp) { + free(greater); + free(lesser); + return (ARCHIVE_FATAL); + } + lesser = tmp; + lesser[lesser_count - 1] = strings[i]; + } + else + { + greater_count++; + tmp = (char **)realloc(greater, + greater_count * sizeof(char *)); + if (!tmp) { + free(greater); + free(lesser); + return (ARCHIVE_FATAL); + } + greater = tmp; + greater[greater_count - 1] = strings[i]; + } + } + + /* quicksort(lesser) */ + retval1 = archive_utility_string_sort_helper(lesser, lesser_count); + for (i = 0; i < lesser_count; i++) + strings[i] = lesser[i]; + free(lesser); + + /* pivot */ + strings[lesser_count] = pivot; + + /* quicksort(greater) */ + retval2 = archive_utility_string_sort_helper(greater, greater_count); + for (i = 0; i < greater_count; i++) + strings[lesser_count + 1 + i] = greater[i]; + free(greater); + + return (retval1 < retval2) ? retval1 : retval2; } int archive_utility_string_sort(char **strings) { - unsigned int size = 0; - while (strings[size] != NULL) - size++; - return archive_utility_string_sort_helper(strings, size); + unsigned int size = 0; + while (strings[size] != NULL) + size++; + return archive_utility_string_sort_helper(strings, size); } diff --git a/Utilities/cmlibarchive/libarchive/archive_windows.h b/Utilities/cmlibarchive/libarchive/archive_windows.h index dc2f241..7c31a16 100644 --- a/Utilities/cmlibarchive/libarchive/archive_windows.h +++ b/Utilities/cmlibarchive/libarchive/archive_windows.h @@ -227,7 +227,9 @@ #define S_IRWXG _S_IRWXG #define S_IXGRP _S_IXGRP #define S_IWGRP _S_IWGRP +#ifndef S_IRGRP #define S_IRGRP _S_IRGRP +#endif #define S_IRWXO _S_IRWXO #define S_IXOTH _S_IXOTH #define S_IWOTH _S_IWOTH diff --git a/Utilities/cmlibarchive/libarchive/archive_write.3 b/Utilities/cmlibarchive/libarchive/archive_write.3 index 228bc2c..376d71d 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write.3 @@ -155,7 +155,7 @@ myopen(struct archive *a, void *client_data) return (ARCHIVE_FATAL); } -ssize_t +la_ssize_t mywrite(struct archive *a, void *client_data, const void *buff, size_t n) { struct mydata *mydata = client_data; @@ -186,8 +186,13 @@ write_archive(const char *outname, const char **filename) a = archive_write_new(); mydata->name = outname; - archive_write_add_filter_gzip(a); - archive_write_set_format_ustar(a); + /* Set archive format and filter according to output file extension. + * If it fails, set default format. Platform depended function. + * See supported formats in archive_write_set_format_filter_by_ext.c */ + if (archive_write_set_format_filter_by_ext(a, outname) != ARCHIVE_OK) { + archive_write_add_filter_gzip(a); + archive_write_set_format_ustar(a); + } archive_write_open(a, mydata, myopen, mywrite, myclose); while (*filename) { stat(*filename, &st); @@ -197,7 +202,7 @@ write_archive(const char *outname, const char **filename) archive_write_header(a, entry); if ((fd = open(*filename, O_RDONLY)) != -1) { len = read(fd, buff, sizeof(buff)); - while ( len > 0 ) { + while (len > 0) { archive_write_data(a, buff, len); len = read(fd, buff, sizeof(buff)); } @@ -213,7 +218,7 @@ int main(int argc, const char **argv) { const char *outname; argv++; - outname = argv++; + outname = *argv++; write_archive(outname, argv); return 0; } diff --git a/Utilities/cmlibarchive/libarchive/archive_write.c b/Utilities/cmlibarchive/libarchive/archive_write.c index 8997193..e3fa335 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write.c +++ b/Utilities/cmlibarchive/libarchive/archive_write.c @@ -444,6 +444,12 @@ archive_write_client_close(struct archive_write_filter *f) /* Clear the close handler myself not to be called again. */ f->close = NULL; a->client_data = NULL; + /* Clear passphrase. */ + if (a->passphrase != NULL) { + memset(a->passphrase, 0, strlen(a->passphrase)); + free(a->passphrase); + a->passphrase = NULL; + } return (ret); } @@ -592,6 +598,11 @@ _archive_write_free(struct archive *_a) /* Release various dynamic buffers. */ free((void *)(uintptr_t)(const void *)a->nulls); archive_string_free(&a->archive.error_string); + if (a->passphrase != NULL) { + /* A passphrase should be cleaned. */ + memset(a->passphrase, 0, strlen(a->passphrase)); + free(a->passphrase); + } a->archive.magic = 0; __archive_clean(&a->archive); free(a); diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c index 81dd683..ad5dc83 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter.c @@ -47,6 +47,7 @@ struct { int code; int (*setter)(struct archive *); } codes[] = { ARCHIVE_FILTER_COMPRESS, archive_write_add_filter_compress }, { ARCHIVE_FILTER_GRZIP, archive_write_add_filter_grzip }, { ARCHIVE_FILTER_LRZIP, archive_write_add_filter_lrzip }, + { ARCHIVE_FILTER_LZ4, archive_write_add_filter_lz4 }, { ARCHIVE_FILTER_LZIP, archive_write_add_filter_lzip }, { ARCHIVE_FILTER_LZMA, archive_write_add_filter_lzma }, { ARCHIVE_FILTER_LZOP, archive_write_add_filter_lzip }, diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c index e4cba4a..eac4011 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_by_name.c @@ -51,6 +51,7 @@ struct { const char *name; int (*setter)(struct archive *); } names[] = { "grzip", archive_write_add_filter_grzip }, { "gzip", archive_write_add_filter_gzip }, { "lrzip", archive_write_add_filter_lrzip }, + { "lz4", archive_write_add_filter_lz4 }, { "lzip", archive_write_add_filter_lzip }, { "lzma", archive_write_add_filter_lzma }, { "lzop", archive_write_add_filter_lzop }, diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_lz4.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_lz4.c new file mode 100644 index 0000000..e23e5e9 --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_lz4.c @@ -0,0 +1,646 @@ +/*- + * Copyright (c) 2014 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" + +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#include <stdio.h> +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#ifdef HAVE_LZ4_H +#include <lz4.h> +#endif +#ifdef HAVE_LZ4HC_H +#include <lz4hc.h> +#endif + +#include "archive.h" +#include "archive_endian.h" +#include "archive_private.h" +#include "archive_write_private.h" +#include "archive_xxhash.h" + +#define LZ4_MAGICNUMBER 0x184d2204 + +struct private_data { + int compression_level; + unsigned header_written:1; + unsigned version_number:1; + unsigned block_independence:1; + unsigned block_checksum:1; + unsigned stream_size:1; + unsigned stream_checksum:1; + unsigned preset_dictionary:1; + unsigned block_maximum_size:3; +#if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2 + int64_t total_in; + char *out; + char *out_buffer; + size_t out_buffer_size; + size_t out_block_size; + char *in; + char *in_buffer_allocated; + char *in_buffer; + size_t in_buffer_size; + size_t block_size; + + void *xxh32_state; + void *lz4_stream; +#else + struct archive_write_program_data *pdata; +#endif +}; + +static int archive_filter_lz4_close(struct archive_write_filter *); +static int archive_filter_lz4_free(struct archive_write_filter *); +static int archive_filter_lz4_open(struct archive_write_filter *); +static int archive_filter_lz4_options(struct archive_write_filter *, + const char *, const char *); +static int archive_filter_lz4_write(struct archive_write_filter *, + const void *, size_t); + +/* + * Add a lz4 compression filter to this write handle. + */ +int +archive_write_add_filter_lz4(struct archive *_a) +{ + struct archive_write *a = (struct archive_write *)_a; + struct archive_write_filter *f = __archive_write_allocate_filter(_a); + struct private_data *data; + + archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC, + ARCHIVE_STATE_NEW, "archive_write_add_filter_lz4"); + + data = calloc(1, sizeof(*data)); + if (data == NULL) { + archive_set_error(&a->archive, ENOMEM, "Out of memory"); + return (ARCHIVE_FATAL); + } + + /* + * Setup default settings. + */ + data->compression_level = 1; + data->version_number = 0x01; + data->block_independence = 1; + data->block_checksum = 0; + data->stream_size = 0; + data->stream_checksum = 1; + data->preset_dictionary = 0; + data->block_maximum_size = 7; + + /* + * Setup a filter setting. + */ + f->data = data; + f->options = &archive_filter_lz4_options; + f->close = &archive_filter_lz4_close; + f->free = &archive_filter_lz4_free; + f->open = &archive_filter_lz4_open; + f->code = ARCHIVE_FILTER_LZ4; + f->name = "lz4"; +#if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2 + return (ARCHIVE_OK); +#else + /* + * We don't have lz4 library, and execute external lz4 program + * instead. + */ + data->pdata = __archive_write_program_allocate(); + if (data->pdata == NULL) { + free(data); + archive_set_error(&a->archive, ENOMEM, "Out of memory"); + return (ARCHIVE_FATAL); + } + data->compression_level = 0; + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Using external lz4 program"); + return (ARCHIVE_WARN); +#endif +} + +/* + * Set write options. + */ +static int +archive_filter_lz4_options(struct archive_write_filter *f, + const char *key, const char *value) +{ + struct private_data *data = (struct private_data *)f->data; + + if (strcmp(key, "compression-level") == 0) { + if (value == NULL || !(value[0] >= '1' && value[0] <= '9') || + value[1] != '\0') + return (ARCHIVE_WARN); + data->compression_level = value[0] - '0'; + return (ARCHIVE_OK); + } + if (strcmp(key, "stream-checksum") == 0) { + data->stream_checksum = value != NULL; + return (ARCHIVE_OK); + } + if (strcmp(key, "block-checksum") == 0) { + data->block_checksum = value != NULL; + return (ARCHIVE_OK); + } + if (strcmp(key, "block-size") == 0) { + if (value == NULL || !(value[0] >= '4' && value[0] <= '7') || + value[1] != '\0') + return (ARCHIVE_WARN); + data->block_maximum_size = value[0] - '0'; + return (ARCHIVE_OK); + } + if (strcmp(key, "block-dependence") == 0) { + data->block_independence = value == NULL; + return (ARCHIVE_OK); + } + + /* Note: The "warn" return is just to inform the options + * supervisor that we didn't handle it. It will generate + * a suitable error if no one used this option. */ + return (ARCHIVE_WARN); +} + +#if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2 +/* Don't compile this if we don't have liblz4. */ + +static int drive_compressor(struct archive_write_filter *, const char *, + size_t); +static int drive_compressor_independence(struct archive_write_filter *, + const char *, size_t); +static int drive_compressor_dependence(struct archive_write_filter *, + const char *, size_t); +static int lz4_write_stream_descriptor(struct archive_write_filter *); +static ssize_t lz4_write_one_block(struct archive_write_filter *, const char *, + size_t); + + +/* + * Setup callback. + */ +static int +archive_filter_lz4_open(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + int ret; + size_t required_size; + static size_t bkmap[] = { 64 * 1024, 256 * 1024, 1 * 1024 * 1024, + 4 * 1024 * 1024 }; + size_t pre_block_size; + + ret = __archive_write_open_filter(f->next_filter); + if (ret != 0) + return (ret); + + if (data->block_maximum_size < 4) + data->block_size = bkmap[0]; + else + data->block_size = bkmap[data->block_maximum_size - 4]; + + required_size = 4 + 15 + 4 + data->block_size + 4 + 4; + if (data->out_buffer_size < required_size) { + size_t bs = required_size, bpb; + free(data->out_buffer); + if (f->archive->magic == ARCHIVE_WRITE_MAGIC) { + /* Buffer size should be a multiple number of + * the of bytes per block for performance. */ + bpb = archive_write_get_bytes_per_block(f->archive); + if (bpb > bs) + bs = bpb; + else if (bpb != 0) { + bs += bpb; + bs -= bs % bpb; + } + } + data->out_block_size = bs; + bs += required_size; + data->out_buffer = malloc(bs); + data->out = data->out_buffer; + data->out_buffer_size = bs; + } + + pre_block_size = (data->block_independence)? 0: 64 * 1024; + if (data->in_buffer_size < data->block_size + pre_block_size) { + free(data->in_buffer_allocated); + data->in_buffer_size = data->block_size; + data->in_buffer_allocated = + malloc(data->in_buffer_size + pre_block_size); + data->in_buffer = data->in_buffer_allocated + pre_block_size; + if (!data->block_independence && data->compression_level >= 3) + data->in_buffer = data->in_buffer_allocated; + data->in = data->in_buffer; + data->in_buffer_size = data->block_size; + } + + if (data->out_buffer == NULL || data->in_buffer_allocated == NULL) { + archive_set_error(f->archive, ENOMEM, + "Can't allocate data for compression buffer"); + return (ARCHIVE_FATAL); + } + + f->write = archive_filter_lz4_write; + + return (ARCHIVE_OK); +} + +/* + * Write data to the out stream. + * + * Returns ARCHIVE_OK if all data written, error otherwise. + */ +static int +archive_filter_lz4_write(struct archive_write_filter *f, + const void *buff, size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + int ret = ARCHIVE_OK; + const char *p; + size_t remaining; + ssize_t size; + + /* If we haven't written a stream descriptor, we have to do it first. */ + if (!data->header_written) { + ret = lz4_write_stream_descriptor(f); + if (ret != ARCHIVE_OK) + return (ret); + data->header_written = 1; + } + + /* Update statistics */ + data->total_in += length; + + p = (const char *)buff; + remaining = length; + while (remaining) { + size_t l; + /* Compress input data to output buffer */ + size = lz4_write_one_block(f, p, remaining); + if (size < ARCHIVE_OK) + return (ARCHIVE_FATAL); + l = data->out - data->out_buffer; + if (l >= data->out_block_size) { + ret = __archive_write_filter(f->next_filter, + data->out_buffer, data->out_block_size); + l -= data->out_block_size; + memcpy(data->out_buffer, + data->out_buffer + data->out_block_size, l); + data->out = data->out_buffer + l; + if (ret < ARCHIVE_WARN) + break; + } + p += size; + remaining -= size; + } + + return (ret); +} + +/* + * Finish the compression. + */ +static int +archive_filter_lz4_close(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + int ret, r1; + + /* Finish compression cycle. */ + ret = (int)lz4_write_one_block(f, NULL, 0); + if (ret >= 0) { + /* + * Write the last block and the end of the stream data. + */ + + /* Write End Of Stream. */ + memset(data->out, 0, 4); data->out += 4; + /* Write Stream checksum if needed. */ + if (data->stream_checksum) { + unsigned int checksum; + checksum = __archive_xxhash.XXH32_digest( + data->xxh32_state); + data->xxh32_state = NULL; + archive_le32enc(data->out, checksum); + data->out += 4; + } + ret = __archive_write_filter(f->next_filter, + data->out_buffer, data->out - data->out_buffer); + } + + r1 = __archive_write_close_filter(f->next_filter); + return (r1 < ret ? r1 : ret); +} + +static int +archive_filter_lz4_free(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + + if (data->lz4_stream != NULL) { + if (data->compression_level < 3) +#if LZ4_VERSION_MINOR >= 3 + LZ4_freeStream(data->lz4_stream); +#else + LZ4_free(data->lz4_stream); +#endif + else + LZ4_freeHC(data->lz4_stream); + } + free(data->out_buffer); + free(data->in_buffer_allocated); + free(data->xxh32_state); + free(data); + f->data = NULL; + return (ARCHIVE_OK); +} + +static int +lz4_write_stream_descriptor(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + uint8_t *sd; + + sd = (uint8_t *)data->out; + /* Write Magic Number. */ + archive_le32enc(&sd[0], LZ4_MAGICNUMBER); + /* FLG */ + sd[4] = (data->version_number << 6) + | (data->block_independence << 5) + | (data->block_checksum << 4) + | (data->stream_size << 3) + | (data->stream_checksum << 2) + | (data->preset_dictionary << 0); + /* BD */ + sd[5] = (data->block_maximum_size << 4); + sd[6] = (__archive_xxhash.XXH32(&sd[4], 2, 0) >> 8) & 0xff; + data->out += 7; + if (data->stream_checksum) + data->xxh32_state = __archive_xxhash.XXH32_init(0); + else + data->xxh32_state = NULL; + return (ARCHIVE_OK); +} + +static ssize_t +lz4_write_one_block(struct archive_write_filter *f, const char *p, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + ssize_t r; + + if (p == NULL) { + /* Compress remaining uncompressed data. */ + if (data->in_buffer == data->in) + return 0; + else { + size_t l = data->in - data->in_buffer; + r = drive_compressor(f, data->in_buffer, l); + if (r == ARCHIVE_OK) + r = (ssize_t)l; + } + } else if ((data->block_independence || data->compression_level < 3) && + data->in_buffer == data->in && length >= data->block_size) { + r = drive_compressor(f, p, data->block_size); + if (r == ARCHIVE_OK) + r = (ssize_t)data->block_size; + } else { + size_t remaining_size = data->in_buffer_size - + (data->in - data->in_buffer); + size_t l = (remaining_size > length)? length: remaining_size; + memcpy(data->in, p, l); + data->in += l; + if (l == remaining_size) { + r = drive_compressor(f, data->in_buffer, + data->block_size); + if (r == ARCHIVE_OK) + r = (ssize_t)l; + data->in = data->in_buffer; + } else + r = (ssize_t)l; + } + + return (r); +} + + +/* + * Utility function to push input data through compressor, writing + * full output blocks as necessary. + * + * Note that this handles both the regular write case (finishing == + * false) and the end-of-archive case (finishing == true). + */ +static int +drive_compressor(struct archive_write_filter *f, const char *p, size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + + if (data->stream_checksum) + __archive_xxhash.XXH32_update(data->xxh32_state, + p, (int)length); + if (data->block_independence) + return drive_compressor_independence(f, p, length); + else + return drive_compressor_dependence(f, p, length); +} + +static int +drive_compressor_independence(struct archive_write_filter *f, const char *p, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + unsigned int outsize; + + if (data->compression_level < 4) + outsize = LZ4_compress_limitedOutput(p, data->out + 4, + (int)length, (int)data->block_size); + else + outsize = LZ4_compressHC2_limitedOutput(p, data->out + 4, + (int)length, (int)data->block_size, + data->compression_level); + + if (outsize) { + /* The buffer is compressed. */ + archive_le32enc(data->out, outsize); + data->out += 4; + } else { + /* The buffer is not compressed. The commpressed size was + * bigger than its uncompressed size. */ + archive_le32enc(data->out, length | 0x80000000); + data->out += 4; + memcpy(data->out, p, length); + outsize = length; + } + data->out += outsize; + if (data->block_checksum) { + unsigned int checksum = + __archive_xxhash.XXH32(data->out - outsize, outsize, 0); + archive_le32enc(data->out, checksum); + data->out += 4; + } + return (ARCHIVE_OK); +} + +static int +drive_compressor_dependence(struct archive_write_filter *f, const char *p, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + int outsize; + + if (data->compression_level < 3) { + if (data->lz4_stream == NULL) { + data->lz4_stream = LZ4_createStream(); + if (data->lz4_stream == NULL) { + archive_set_error(f->archive, ENOMEM, + "Can't allocate data for compression" + " buffer"); + return (ARCHIVE_FATAL); + } + } + outsize = LZ4_compress_limitedOutput_continue( + data->lz4_stream, p, data->out + 4, (int)length, + (int)data->block_size); + } else { + if (data->lz4_stream == NULL) { + data->lz4_stream = + LZ4_createHC(data->in_buffer_allocated); + if (data->lz4_stream == NULL) { + archive_set_error(f->archive, ENOMEM, + "Can't allocate data for compression" + " buffer"); + return (ARCHIVE_FATAL); + } + } + outsize = LZ4_compressHC2_limitedOutput_continue( + data->lz4_stream, p, data->out + 4, (int)length, + (int)data->block_size, data->compression_level); + } + + if (outsize) { + /* The buffer is compressed. */ + archive_le32enc(data->out, outsize); + data->out += 4; + } else { + /* The buffer is not compressed. The commpressed size was + * bigger than its uncompressed size. */ + archive_le32enc(data->out, length | 0x80000000); + data->out += 4; + memcpy(data->out, p, length); + outsize = length; + } + data->out += outsize; + if (data->block_checksum) { + unsigned int checksum = + __archive_xxhash.XXH32(data->out - outsize, outsize, 0); + archive_le32enc(data->out, checksum); + data->out += 4; + } + + if (length == data->block_size) { +#define DICT_SIZE (64 * 1024) + if (data->compression_level < 3) + LZ4_saveDict(data->lz4_stream, + data->in_buffer_allocated, DICT_SIZE); + else { + LZ4_slideInputBufferHC(data->lz4_stream); + data->in_buffer = data->in_buffer_allocated + DICT_SIZE; + } +#undef DICT_SIZE + } + return (ARCHIVE_OK); +} + +#else /* HAVE_LIBLZ4 */ + +static int +archive_filter_lz4_open(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + struct archive_string as; + int r; + + archive_string_init(&as); + archive_strcpy(&as, "lz4 -z -q -q"); + + /* Specify a compression level. */ + if (data->compression_level > 0) { + archive_strcat(&as, " -"); + archive_strappend_char(&as, '0' + data->compression_level); + } + /* Specify a block size. */ + archive_strcat(&as, " -B"); + archive_strappend_char(&as, '0' + data->block_maximum_size); + + if (data->block_checksum) + archive_strcat(&as, " -BX"); + if (data->stream_checksum == 0) + archive_strcat(&as, " -Sx"); + if (data->block_independence == 0) + archive_strcat(&as, " -BD"); + + f->write = archive_filter_lz4_write; + + r = __archive_write_program_open(f, data->pdata, as.s); + archive_string_free(&as); + return (r); +} + +static int +archive_filter_lz4_write(struct archive_write_filter *f, const void *buff, + size_t length) +{ + struct private_data *data = (struct private_data *)f->data; + + return __archive_write_program_write(f, data->pdata, buff, length); +} + +static int +archive_filter_lz4_close(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + + return __archive_write_program_close(f, data->pdata); +} + +static int +archive_filter_lz4_free(struct archive_write_filter *f) +{ + struct private_data *data = (struct private_data *)f->data; + + __archive_write_program_free(data->pdata); + free(data); + return (ARCHIVE_OK); +} + +#endif /* HAVE_LIBLZ4 */ diff --git a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_xz.c b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_xz.c index a566732..4f849ce 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_add_filter_xz.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_add_filter_xz.c @@ -100,6 +100,7 @@ archive_write_add_filter_lzip(struct archive *a) struct private_data { int compression_level; + uint32_t threads; lzma_stream stream; lzma_filter lzmafilters[2]; lzma_options_lzma lzma_opt; @@ -151,6 +152,7 @@ common_setup(struct archive_write_filter *f) } f->data = data; data->compression_level = LZMA_PRESET_DEFAULT; + data->threads = 1; f->open = &archive_compressor_xz_open; f->close = archive_compressor_xz_close; f->free = archive_compressor_xz_free; @@ -221,23 +223,37 @@ archive_compressor_xz_init_stream(struct archive_write_filter *f, { static const lzma_stream lzma_stream_init_data = LZMA_STREAM_INIT; int ret; +#ifdef HAVE_LZMA_STREAM_ENCODER_MT + lzma_mt mt_options; +#endif data->stream = lzma_stream_init_data; data->stream.next_out = data->compressed; data->stream.avail_out = data->compressed_buffer_size; - if (f->code == ARCHIVE_FILTER_XZ) - ret = lzma_stream_encoder(&(data->stream), - data->lzmafilters, LZMA_CHECK_CRC64); - else if (f->code == ARCHIVE_FILTER_LZMA) + if (f->code == ARCHIVE_FILTER_XZ) { +#ifdef HAVE_LZMA_STREAM_ENCODER_MT + if (data->threads != 1) { + bzero(&mt_options, sizeof(mt_options)); + mt_options.threads = data->threads; + mt_options.timeout = 300; + mt_options.filters = data->lzmafilters; + mt_options.check = LZMA_CHECK_CRC64; + ret = lzma_stream_encoder_mt(&(data->stream), + &mt_options); + } else +#endif + ret = lzma_stream_encoder(&(data->stream), + data->lzmafilters, LZMA_CHECK_CRC64); + } else if (f->code == ARCHIVE_FILTER_LZMA) { ret = lzma_alone_encoder(&(data->stream), &data->lzma_opt); - else { /* ARCHIVE_FILTER_LZIP */ + } else { /* ARCHIVE_FILTER_LZIP */ int dict_size = data->lzma_opt.dict_size; int ds, log2dic, wedges; /* Calculate a coded dictionary size */ if (dict_size < (1 << 12) || dict_size > (1 << 27)) { archive_set_error(f->archive, ARCHIVE_ERRNO_MISC, - "Unacceptable dictionary dize for lzip: %d", + "Unacceptable dictionary size for lzip: %d", dict_size); return (ARCHIVE_FATAL); } @@ -373,6 +389,22 @@ archive_compressor_xz_options(struct archive_write_filter *f, if (data->compression_level > 6) data->compression_level = 6; return (ARCHIVE_OK); + } else if (strcmp(key, "threads") == 0) { + if (value == NULL) + return (ARCHIVE_WARN); + data->threads = (int)strtoul(value, NULL, 10); + if (data->threads == 0 && errno != 0) { + data->threads = 1; + return (ARCHIVE_WARN); + } + if (data->threads == 0) { +#ifdef HAVE_LZMA_STREAM_ENCODER_MT + data->threads = lzma_cputhreads(); +#else + data->threads = 1; +#endif + } + return (ARCHIVE_OK); } /* Note: The "warn" return is just to inform the options diff --git a/Utilities/cmlibarchive/libarchive/archive_write_data.3 b/Utilities/cmlibarchive/libarchive/archive_write_data.3 index cfd5cd5..0cdd25f 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_data.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write_data.3 @@ -34,7 +34,7 @@ Streaming Archive Library (libarchive, -larchive) .Sh SYNOPSIS .In archive.h -.Ft ssize_t +.Ft la_ssize_t .Fn archive_write_data "struct archive *" "const void *" "size_t" .Sh DESCRIPTION Write data corresponding to the header just written. @@ -42,8 +42,7 @@ Write data corresponding to the header just written. .\" .Sh RETURN VALUES This function returns the number of bytes actually written, or -.Li -1 -on error. +a negative error code on error. .\" .Sh ERRORS Detailed error codes and textual descriptions are available from the @@ -52,6 +51,15 @@ and .Fn archive_error_string functions. .\" +.Sh BUGS +In libarchive 3.x, this function sometimes returns +zero on success instead of returning the number of bytes written. +Specifically, this occurs when writing to an +.Vt archive_write_disk +handle. +Clients should treat any value less than zero as an error +and consider any non-negative value as success. +.\" .Sh SEE ALSO .Xr tar 1 , .Xr libarchive 3 , diff --git a/Utilities/cmlibarchive/libarchive/archive_write_disk.3 b/Utilities/cmlibarchive/libarchive/archive_write_disk.3 index fa925cc..ba6c970 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_disk.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write_disk.3 @@ -70,9 +70,9 @@ Streaming Archive Library (libarchive, -larchive) .Fc .Ft int .Fn archive_write_header "struct archive *" "struct archive_entry *" -.Ft ssize_t +.Ft la_ssize_t .Fn archive_write_data "struct archive *" "const void *" "size_t" -.Ft ssize_t +.Ft la_ssize_t .Fn archive_write_data_block "struct archive *" "const void *" "size_t size" "int64_t offset" .Ft int .Fn archive_write_finish_entry "struct archive *" @@ -177,10 +177,16 @@ The default is to not refuse such paths. Note that paths ending in .Pa .. always cause an error, regardless of this flag. +.It Cm ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS +Refuse to extract an absolute path. +The default is to not refuse such paths. .It Cm ARCHIVE_EXTRACT_SPARSE Scan data for blocks of NUL bytes and try to recreate them with holes. This results in sparse files, independent of whether the archive format supports or uses them. +.It Cm ARCHIVE_EXTRACT_CLEAR_NOCHANGE_FFLAGS +Before removing a file system object prior to replacing it, clear +platform-specific file flags which might prevent its removal. .El .It Xo .Fn archive_write_disk_set_group_lookup , diff --git a/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c b/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c index a3d350b..450ac75 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_disk_posix.c @@ -343,6 +343,7 @@ static int restore_entry(struct archive_write_disk *); static int set_mac_metadata(struct archive_write_disk *, const char *, const void *, size_t); static int set_xattrs(struct archive_write_disk *); +static int clear_nochange_fflags(struct archive_write_disk *); static int set_fflags(struct archive_write_disk *); static int set_fflags_platform(struct archive_write_disk *, int fd, const char *name, mode_t mode, @@ -1467,10 +1468,14 @@ _archive_write_disk_data_block(struct archive *_a, return (r); if ((size_t)r < size) { archive_set_error(&a->archive, 0, - "Write request too large"); + "Too much data: Truncating file at %ju bytes", (uintmax_t)a->filesize); return (ARCHIVE_WARN); } +#if ARCHIVE_VERSION_NUMBER < 3999000 return (ARCHIVE_OK); +#else + return (size); +#endif } static ssize_t @@ -1842,6 +1847,8 @@ restore_entry(struct archive_write_disk *a) * object is a dir, but that doesn't mean the old * object isn't a dir. */ + if (a->flags & ARCHIVE_EXTRACT_CLEAR_NOCHANGE_FFLAGS) + (void)clear_nochange_fflags(a); if (unlink(a->name) == 0) { /* We removed it, reset cached stat. */ a->pst = NULL; @@ -1869,6 +1876,13 @@ restore_entry(struct archive_write_disk *a) en = create_filesystem_object(a); } + if ((en == ENOENT) && (archive_entry_hardlink(a->entry) != NULL)) { + archive_set_error(&a->archive, en, + "Hard-link target '%s' does not exist.", + archive_entry_hardlink(a->entry)); + return (ARCHIVE_FAILED); + } + if ((en == EISDIR || en == EEXIST) && (a->flags & ARCHIVE_EXTRACT_NO_OVERWRITE)) { /* If we're not overwriting, we're done. */ @@ -1940,6 +1954,8 @@ restore_entry(struct archive_write_disk *a) if (!S_ISDIR(a->st.st_mode)) { /* A non-dir is in the way, unlink it. */ + if (a->flags & ARCHIVE_EXTRACT_CLEAR_NOCHANGE_FFLAGS) + (void)clear_nochange_fflags(a); if (unlink(a->name) != 0) { archive_set_error(&a->archive, errno, "Can't unlink already-existing object"); @@ -1950,6 +1966,8 @@ restore_entry(struct archive_write_disk *a) en = create_filesystem_object(a); } else if (!S_ISDIR(a->mode)) { /* A dir is in the way of a non-dir, rmdir it. */ + if (a->flags & ARCHIVE_EXTRACT_CLEAR_NOCHANGE_FFLAGS) + (void)clear_nochange_fflags(a); if (rmdir(a->name) != 0) { archive_set_error(&a->archive, errno, "Can't replace existing directory with non-directory"); @@ -2505,8 +2523,9 @@ cleanup_pathname_win(struct archive_write_disk *a) /* * Canonicalize the pathname. In particular, this strips duplicate * '/' characters, '.' elements, and trailing '/'. It also raises an - * error for an empty path, a trailing '..' or (if _SECURE_NODOTDOT is - * set) any '..' in the path. + * error for an empty path, a trailing '..', (if _SECURE_NODOTDOT is + * set) any '..' in the path or (if ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS + * is set) if the path is absolute. */ static int cleanup_pathname(struct archive_write_disk *a) @@ -2525,8 +2544,15 @@ cleanup_pathname(struct archive_write_disk *a) cleanup_pathname_win(a); #endif /* Skip leading '/'. */ - if (*src == '/') + if (*src == '/') { + if (a->flags & ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Path is absolute"); + return (ARCHIVE_FAILED); + } + separator = *src++; + } /* Scan the pathname one element at a time. */ for (;;) { @@ -3056,9 +3082,23 @@ set_mode(struct archive_write_disk *a, int mode) * impact. */ if (lchmod(a->name, mode) != 0) { - archive_set_error(&a->archive, errno, - "Can't set permissions to 0%o", (int)mode); - r = ARCHIVE_WARN; + switch (errno) { + case ENOTSUP: + case ENOSYS: +#if ENOTSUP != EOPNOTSUPP + case EOPNOTSUPP: +#endif + /* + * if lchmod is defined but the platform + * doesn't support it, silently ignore + * error + */ + break; + default: + archive_set_error(&a->archive, errno, + "Can't set permissions to 0%o", (int)mode); + r = ARCHIVE_WARN; + } } #endif } else if (!S_ISDIR(a->mode)) { @@ -3159,6 +3199,36 @@ set_fflags(struct archive_write_disk *a) return (ARCHIVE_OK); } +static int +clear_nochange_fflags(struct archive_write_disk *a) +{ + int nochange_flags; + mode_t mode = archive_entry_mode(a->entry); + + /* Hopefully, the compiler will optimize this mess into a constant. */ + nochange_flags = 0; +#ifdef SF_IMMUTABLE + nochange_flags |= SF_IMMUTABLE; +#endif +#ifdef UF_IMMUTABLE + nochange_flags |= UF_IMMUTABLE; +#endif +#ifdef SF_APPEND + nochange_flags |= SF_APPEND; +#endif +#ifdef UF_APPEND + nochange_flags |= UF_APPEND; +#endif +#ifdef EXT2_APPEND_FL + nochange_flags |= EXT2_APPEND_FL; +#endif +#ifdef EXT2_IMMUTABLE_FL + nochange_flags |= EXT2_IMMUTABLE_FL; +#endif + + return (set_fflags_platform(a, a->fd, a->name, mode, 0, nochange_flags)); +} + #if ( defined(HAVE_LCHFLAGS) || defined(HAVE_CHFLAGS) || defined(HAVE_FCHFLAGS) ) && defined(HAVE_STRUCT_STAT_ST_FLAGS) /* @@ -3366,6 +3436,7 @@ copy_xattrs(struct archive_write_disk *a, int tmpfd, int dffd) } for (xattr_i = 0; xattr_i < xattr_size; xattr_i += strlen(xattr_names + xattr_i) + 1) { + char *xattr_val_saved; ssize_t s; int f; @@ -3376,11 +3447,13 @@ copy_xattrs(struct archive_write_disk *a, int tmpfd, int dffd) ret = ARCHIVE_WARN; goto exit_xattr; } + xattr_val_saved = xattr_val; xattr_val = realloc(xattr_val, s); if (xattr_val == NULL) { archive_set_error(&a->archive, ENOMEM, "Failed to get metadata(xattr)"); ret = ARCHIVE_WARN; + free(xattr_val_saved); goto exit_xattr; } s = fgetxattr(tmpfd, xattr_names + xattr_i, xattr_val, s, 0, 0); diff --git a/Utilities/cmlibarchive/libarchive/archive_write_disk_set_standard_lookup.c b/Utilities/cmlibarchive/libarchive/archive_write_disk_set_standard_lookup.c index e79008e..3b868fb 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_disk_set_standard_lookup.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_disk_set_standard_lookup.c @@ -67,7 +67,7 @@ static void cleanup(void *); * a simple cache to accelerate such lookups---into the archive_write_disk * object. This is in a separate file because getpwnam()/getgrnam() * can pull in a LOT of library code (including NIS/LDAP functions, which - * pull in DNS resolveers, etc). This can easily top 500kB, which makes + * pull in DNS resolvers, etc). This can easily top 500kB, which makes * it inappropriate for some space-constrained applications. * * Applications that are size-sensitive may want to just use the @@ -86,6 +86,11 @@ archive_write_disk_set_standard_lookup(struct archive *a) { struct bucket *ucache = malloc(cache_size * sizeof(struct bucket)); struct bucket *gcache = malloc(cache_size * sizeof(struct bucket)); + if (ucache == NULL || gcache == NULL) { + free(ucache); + free(gcache); + return (ARCHIVE_FATAL); + } memset(ucache, 0, cache_size * sizeof(struct bucket)); memset(gcache, 0, cache_size * sizeof(struct bucket)); archive_write_disk_set_group_lookup(a, gcache, lookup_gid, cleanup); diff --git a/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c b/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c index ed62009..800aa89 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_disk_windows.c @@ -330,8 +330,6 @@ file_information(struct archive_write_disk *a, wchar_t *path, break; case L'C': case L'c': if (((p[2] == L'M' || p[2] == L'm' ) && - (p[3] == L'D' || p[3] == L'd' )) || - ((p[2] == L'M' || p[2] == L'm' ) && (p[3] == L'D' || p[3] == L'd' ))) *mode |= S_IXUSR | S_IXGRP | S_IXOTH; break; @@ -1011,7 +1009,11 @@ _archive_write_disk_data_block(struct archive *_a, "Write request too large"); return (ARCHIVE_WARN); } +#if ARCHIVE_VERSION_NUMBER < 3999000 return (ARCHIVE_OK); +#else + return (size); +#endif } static ssize_t diff --git a/Utilities/cmlibarchive/libarchive/archive_write_filter.3 b/Utilities/cmlibarchive/libarchive/archive_write_filter.3 index 3ca248b..83bd2c6 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_filter.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write_filter.3 @@ -24,46 +24,62 @@ .\" .\" $FreeBSD$ .\" -.Dd February 2, 2012 +.Dd August 14, 2014 .Dt ARCHIVE_WRITE_FILTER 3 .Os .Sh NAME +.Nm archive_write_add_filter_b64encode , .Nm archive_write_add_filter_bzip2 , .Nm archive_write_add_filter_compress , .Nm archive_write_add_filter_gzip , +.Nm archive_write_add_filter_lz4 , .Nm archive_write_add_filter_lzip , .Nm archive_write_add_filter_lzma , +.Nm archive_write_add_filter_lzop , .Nm archive_write_add_filter_none , .Nm archive_write_add_filter_program , +.Nm archive_write_add_filter_uuencode , .Nm archive_write_add_filter_xz .Sh LIBRARY Streaming Archive Library (libarchive, -larchive) .Sh SYNOPSIS .In archive.h .Ft int +.Fn archive_write_add_filter_b64encode "struct archive *" +.Ft int .Fn archive_write_add_filter_bzip2 "struct archive *" .Ft int .Fn archive_write_add_filter_compress "struct archive *" .Ft int .Fn archive_write_add_filter_gzip "struct archive *" .Ft int +.Fn archive_write_add_filter_lz4 "struct archive *" +.Ft int .Fn archive_write_add_filter_lzip "struct archive *" .Ft int .Fn archive_write_add_filter_lzma "struct archive *" .Ft int +.Fn archive_write_add_filter_lzop "struct archive *" +.Ft int .Fn archive_write_add_filter_none "struct archive *" .Ft int .Fn archive_write_add_filter_program "struct archive *" "const char * cmd" .Ft int +.Fn archive_write_add_filter_uuencode "struct archive *" +.Ft int .Fn archive_write_add_filter_xz "struct archive *" .Sh DESCRIPTION .Bl -tag -width indent .It Xo +.Fn archive_write_add_filter_b64encode , .Fn archive_write_add_filter_bzip2 , .Fn archive_write_add_filter_compress , .Fn archive_write_add_filter_gzip , +.Fn archive_write_add_filter_lz4 , .Fn archive_write_add_filter_lzip , .Fn archive_write_add_filter_lzma , +.Fn archive_write_add_filter_lzop , +.Fn archive_write_add_filter_uuencode , .Fn archive_write_add_filter_xz , .Xc The resulting archive will be compressed as specified. diff --git a/Utilities/cmlibarchive/libarchive/archive_write_finish_entry.3 b/Utilities/cmlibarchive/libarchive/archive_write_finish_entry.3 index d881c80..c5ef69e 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_finish_entry.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write_finish_entry.3 @@ -41,7 +41,7 @@ Close out the entry just written. In particular, this writes out the final padding required by some formats. Ordinarily, clients never need to call this, as it is called automatically by -.Fn archive_write_next_header +.Fn archive_write_header and .Fn archive_write_close as needed. diff --git a/Utilities/cmlibarchive/libarchive/archive_write_format.3 b/Utilities/cmlibarchive/libarchive/archive_write_format.3 index 39d3006..4bd1163 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_format.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write_format.3 @@ -34,7 +34,9 @@ .Nm archive_write_set_format_raw , .Nm archive_write_set_format_shar , .Nm archive_write_set_format_shar_dump , -.Nm archive_write_set_format_ustar +.Nm archive_write_set_format_ustar , +.Nm archive_write_set_format_filter_by_ext , +.Nm archive_write_set_format_filter_by_ext_def .Nd functions for creating archives .Sh LIBRARY Streaming Archive Library (libarchive, -larchive) @@ -54,6 +56,10 @@ Streaming Archive Library (libarchive, -larchive) .Fn archive_write_set_format_shar_dump "struct archive *" .Ft int .Fn archive_write_set_format_ustar "struct archive *" +.Ft int +.Fn archive_write_set_format_filter_by_ext "struct archive *" "const char *" +.Ft int +.Fn archive_write_set_format_filter_by_ext_def "struct archive *" "const char *" "const char *" .Sh DESCRIPTION These functions set the format that will be used for the archive. .Pp @@ -79,6 +85,14 @@ filenames, linknames, uids, sizes, etc. is the library default; this is the same as pax format, but suppresses the pax extended header for most normal files. In most cases, this will result in ordinary ustar archives. +.Bl -tag -width indent +.It Xo +.Fn archive_write_set_format_filter_by_ext , +.Fn archive_write_set_format_filter_by_ext_def +.Xc +Format and filter for archive can be set automatically, based on output file name extension. +The functions are platform dependent. +Supported extensions: .7z, .zip, .jar, .cpio, .iso, .a, .ar, .tar, .tgz, .tar.gz, .tar.bz2, .tar.xz .\" .Sh RETURN VALUES These functions return diff --git a/Utilities/cmlibarchive/libarchive/archive_write_open.3 b/Utilities/cmlibarchive/libarchive/archive_write_open.3 index 4037248..a52959b 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_open.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write_open.3 @@ -154,7 +154,7 @@ to register an error code and message and return .Cm ARCHIVE_FATAL . .Bl -item -offset indent .It -.Ft typedef ssize_t +.Ft typedef la_ssize_t .Fo archive_write_callback .Fa "struct archive *" .Fa "void *client_data" diff --git a/Utilities/cmlibarchive/libarchive/archive_write_open_filename.c b/Utilities/cmlibarchive/libarchive/archive_write_open_filename.c index 196b770..66e0dfe 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_open_filename.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_open_filename.c @@ -243,7 +243,10 @@ file_close(struct archive *a, void *client_data) struct write_file_data *mine = (struct write_file_data *)client_data; (void)a; /* UNUSED */ - close(mine->fd); + + if (mine->fd >= 0) + close(mine->fd); + archive_mstring_clean(&mine->filename); free(mine); return (ARCHIVE_OK); diff --git a/Utilities/cmlibarchive/libarchive/archive_write_private.h b/Utilities/cmlibarchive/libarchive/archive_write_private.h index e600d54..0c3cc0c 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_private.h +++ b/Utilities/cmlibarchive/libarchive/archive_write_private.h @@ -26,8 +26,10 @@ */ #ifndef __LIBARCHIVE_BUILD +#ifndef __LIBARCHIVE_TEST #error This header is only to be used internally to libarchive. #endif +#endif #ifndef ARCHIVE_WRITE_PRIVATE_H_INCLUDED #define ARCHIVE_WRITE_PRIVATE_H_INCLUDED @@ -116,6 +118,14 @@ struct archive_write { const void *buff, size_t); int (*format_close)(struct archive_write *); int (*format_free)(struct archive_write *); + + + /* + * Encryption passphrase. + */ + char *passphrase; + archive_passphrase_callback *passphrase_callback; + void *passphrase_client_data; }; /* @@ -142,4 +152,9 @@ int __archive_write_program_close(struct archive_write_filter *, struct archive_write_program_data *); int __archive_write_program_write(struct archive_write_filter *, struct archive_write_program_data *, const void *, size_t); + +/* + * Get a encryption passphrase. + */ +const char * __archive_write_get_passphrase(struct archive_write *a); #endif diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format.c index 9055753..744302d 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format.c @@ -57,6 +57,7 @@ struct { int code; int (*setter)(struct archive *); } codes[] = { ARCHIVE_FORMAT_TAR_PAX_RESTRICTED, archive_write_set_format_pax_restricted }, { ARCHIVE_FORMAT_TAR_USTAR, archive_write_set_format_ustar }, + { ARCHIVE_FORMAT_WARC, archive_write_set_format_warc }, { ARCHIVE_FORMAT_XAR, archive_write_set_format_xar }, { ARCHIVE_FORMAT_ZIP, archive_write_set_format_zip }, { 0, NULL } diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_by_name.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_by_name.c index 4f3ce7d..a2ce7c6 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format_by_name.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_by_name.c @@ -70,6 +70,7 @@ struct { const char *name; int (*setter)(struct archive *); } names[] = { "ustar", archive_write_set_format_ustar }, { "v7tar", archive_write_set_format_v7tar }, { "v7", archive_write_set_format_v7tar }, + { "warc", archive_write_set_format_warc }, { "xar", archive_write_set_format_xar }, { "zip", archive_write_set_format_zip }, { NULL, NULL } diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_filter_by_ext.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_filter_by_ext.c new file mode 100644 index 0000000..adec9b2 --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_filter_by_ext.c @@ -0,0 +1,142 @@ +/*- + * Copyright (c) 2003-2007 Tim Kientzle + * Copyright (c) 2015 Okhotnikov Kirill + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" +__FBSDID("$FreeBSD: head/lib/libarchive/archive_write_set_format_by_name.c 201168 2009-12-29 06:15:32Z kientzle $"); + +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#ifdef HAVE_STRING_H +#include <string.h> +#endif + +#include "archive.h" +#include "archive_private.h" + +/* A table that maps names to functions. */ +static +struct { const char *name; int (*format)(struct archive *); int (*filter)(struct archive *); } names[] = +{ + { ".7z", archive_write_set_format_7zip, archive_write_add_filter_none}, + { ".zip", archive_write_set_format_zip, archive_write_add_filter_none}, + { ".jar", archive_write_set_format_zip, archive_write_add_filter_none}, + { ".cpio", archive_write_set_format_cpio, archive_write_add_filter_none}, + { ".iso", archive_write_set_format_iso9660, archive_write_add_filter_none}, +#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__NetBSD__) || defined(__OpenBSD__) + { ".a", archive_write_set_format_ar_bsd, archive_write_add_filter_none}, + { ".ar", archive_write_set_format_ar_bsd, archive_write_add_filter_none}, +#else + { ".a", archive_write_set_format_ar_svr4, archive_write_add_filter_none}, + { ".ar", archive_write_set_format_ar_svr4, archive_write_add_filter_none}, +#endif + { ".tar", archive_write_set_format_pax_restricted, archive_write_add_filter_none}, + { ".tgz", archive_write_set_format_pax_restricted, archive_write_add_filter_gzip}, + { ".tar.gz", archive_write_set_format_pax_restricted, archive_write_add_filter_gzip}, + { ".tar.bz2", archive_write_set_format_pax_restricted, archive_write_add_filter_bzip2}, + { ".tar.xz", archive_write_set_format_pax_restricted, archive_write_add_filter_xz}, + { NULL, NULL, NULL } +}; + +static +int cmpsuff(const char *str, const char *suffix) +{ + size_t length_str, length_suffix; + + if ((str == NULL) || (suffix == NULL)) + return -1; + + length_str = strlen(str); + length_suffix = strlen(suffix); + + if (length_str >= length_suffix) { + return strcmp(str + (length_str - length_suffix), suffix); + } else { + return -1; + } +} + +static int get_array_index(const char *name) +{ + int i; + + for (i = 0; names[i].name != NULL; i++) + { + if (cmpsuff(name, names[i].name) == 0) + return i; + } + return -1; + +} + +int +archive_write_set_format_filter_by_ext(struct archive *a, const char *filename) +{ + int names_index = get_array_index(filename); + + if (names_index >= 0) + { + int format_state = (names[names_index].format)(a); + if (format_state == ARCHIVE_OK) + return ((names[names_index].filter)(a)); + else + return format_state; + } + + archive_set_error(a, EINVAL, "No such format '%s'", filename); + a->state = ARCHIVE_STATE_FATAL; + return (ARCHIVE_FATAL); +} + +int +archive_write_set_format_filter_by_ext_def(struct archive *a, const char *filename, const char * def_ext) +{ + int names_index = get_array_index(filename); + + if (names_index < 0) + names_index = get_array_index(def_ext); + + if (names_index >= 0) + { + int format_state = (names[names_index].format)(a); + if (format_state == ARCHIVE_OK) + return ((names[names_index].filter)(a)); + else + return format_state; + } + + archive_set_error(a, EINVAL, "No such format '%s'", filename); + a->state = ARCHIVE_STATE_FATAL; + return (ARCHIVE_FATAL); +} + + + + diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_iso9660.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_iso9660.c index e722625..576f4c2 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format_iso9660.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_iso9660.c @@ -2719,6 +2719,16 @@ extra_get_record(struct isoent *isoent, int *space, int *off, int *loc) rec->offset = 0; /* Insert `rec` into the tail of isoent->extr_rec_list */ rec->next = NULL; + /* + * Note: testing isoent->extr_rec_list.last == NULL + * here is really unneeded since it has been already + * initialized at isoent_new function but Clang Static + * Analyzer claims that it is dereference of null + * pointer. + */ + if (isoent->extr_rec_list.last == NULL) + isoent->extr_rec_list.last = + &(isoent->extr_rec_list.first); *isoent->extr_rec_list.last = rec; isoent->extr_rec_list.last = &(rec->next); } diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_mtree.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_mtree.c index b0c8018..b686303 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format_mtree.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_mtree.c @@ -35,7 +35,7 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_write_set_format_mtree.c 201171 #include <string.h> #include "archive.h" -#include "archive_crypto_private.h" +#include "archive_digest_private.h" #include "archive_entry.h" #include "archive_private.h" #include "archive_rb.h" @@ -639,7 +639,7 @@ attr_counter_inc(struct attr_counter **top, struct attr_counter *ac, *top = ac; ac->next->prev = ac; } - } else { + } else if (last != NULL) { ac = attr_counter_new(me, last); if (ac == NULL) return (-1); diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_warc.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_warc.c new file mode 100644 index 0000000..80abc6f --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_warc.c @@ -0,0 +1,439 @@ +/*- + * Copyright (c) 2014 Sebastian Freundt + * Author: Sebastian Freundt <devel@fresse.org> + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#include <stdio.h> +#ifdef HAVE_STDLIB_H +#include <stdlib.h> +#endif +#ifdef HAVE_STRING_H +#include <string.h> +#endif +#ifdef HAVE_TIME_H +#include <time.h> +#endif + +#include "archive.h" +#include "archive_entry.h" +#include "archive_entry_locale.h" +#include "archive_private.h" +#include "archive_random_private.h" +#include "archive_write_private.h" + +struct warc_s { + unsigned int omit_warcinfo:1; + + time_t now; + mode_t typ; + unsigned int rng; + /* populated size */ + uint64_t populz; +}; + +static const char warcinfo[] = + "software: libarchive/" ARCHIVE_VERSION_ONLY_STRING "\r\n" + "format: WARC file version 1.0\r\n"; + +typedef enum { + WT_NONE, + /* warcinfo */ + WT_INFO, + /* metadata */ + WT_META, + /* resource */ + WT_RSRC, + /* request, unsupported */ + WT_REQ, + /* response, unsupported */ + WT_RSP, + /* revisit, unsupported */ + WT_RVIS, + /* conversion, unsupported */ + WT_CONV, + /* continutation, unsupported at the moment */ + WT_CONT, + /* invalid type */ + LAST_WT +} warc_type_t; + +typedef struct { + warc_type_t type; + const char *tgturi; + const char *recid; + time_t rtime; + time_t mtime; + const char *cnttyp; + uint64_t cntlen; +} warc_essential_hdr_t; + +typedef struct { + unsigned int u[4U]; +} warc_uuid_t; + +static int _warc_options(struct archive_write*, const char *key, const char *v); +static int _warc_header(struct archive_write *a, struct archive_entry *entry); +static ssize_t _warc_data(struct archive_write *a, const void *buf, size_t sz); +static int _warc_finish_entry(struct archive_write *a); +static int _warc_close(struct archive_write *a); +static int _warc_free(struct archive_write *a); + +/* private routines */ +static ssize_t _popul_ehdr(struct archive_string *t, size_t z, warc_essential_hdr_t); +static int _gen_uuid(warc_uuid_t *tgt); + + +/* + * Set output format to ISO 28500 (aka WARC) format. + */ +int +archive_write_set_format_warc(struct archive *_a) +{ + struct archive_write *a = (struct archive_write *)_a; + struct warc_s *w; + + archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, + ARCHIVE_STATE_NEW, "archive_write_set_format_warc"); + + /* If another format was already registered, unregister it. */ + if (a->format_free != NULL) { + (a->format_free)(a); + } + + w = malloc(sizeof(*w)); + if (w == NULL) { + archive_set_error(&a->archive, ENOMEM, + "Can't allocate warc data"); + return (ARCHIVE_FATAL); + } + /* by default we're emitting a file wide header */ + w->omit_warcinfo = 0U; + /* obtain current time for date fields */ + w->now = time(NULL); + /* reset file type info */ + w->typ = 0; + /* also initialise our rng */ + w->rng = (unsigned int)w->now; + + a->format_data = w; + a->format_name = "WARC/1.0"; + a->format_options = _warc_options; + a->format_write_header = _warc_header; + a->format_write_data = _warc_data; + a->format_close = _warc_close; + a->format_free = _warc_free; + a->format_finish_entry = _warc_finish_entry; + a->archive.archive_format = ARCHIVE_FORMAT_WARC; + a->archive.archive_format_name = "WARC/1.0"; + return (ARCHIVE_OK); +} + + +/* archive methods */ +static int +_warc_options(struct archive_write *a, const char *key, const char *val) +{ + struct warc_s *w = a->format_data; + + if (strcmp(key, "omit-warcinfo") == 0) { + if (val == NULL || strcmp(val, "true") == 0) { + /* great */ + w->omit_warcinfo = 1U; + return (ARCHIVE_OK); + } + } + + /* Note: The "warn" return is just to inform the options + * supervisor that we didn't handle it. It will generate + * a suitable error if no one used this option. */ + return (ARCHIVE_WARN); +} + +static int +_warc_header(struct archive_write *a, struct archive_entry *entry) +{ + struct warc_s *w = a->format_data; + struct archive_string hdr; +#define MAX_HDR_SIZE 512 + + /* check whether warcinfo record needs outputting */ + if (!w->omit_warcinfo) { + warc_essential_hdr_t wi = { + WT_INFO, + /*uri*/NULL, + /*urn*/NULL, + /*rtm*/w->now, + /*mtm*/w->now, + /*cty*/"application/warc-fields", + /*len*/sizeof(warcinfo) - 1U, + }; + ssize_t r; + + archive_string_init(&hdr); + r = _popul_ehdr(&hdr, MAX_HDR_SIZE, wi); + if (r >= 0) { + /* jackpot! */ + /* now also use HDR buffer for the actual warcinfo */ + archive_strncat(&hdr, warcinfo, sizeof(warcinfo) -1); + + /* append end-of-record indicator */ + archive_strncat(&hdr, "\r\n\r\n", 4); + + /* write to output stream */ + __archive_write_output(a, hdr.s, archive_strlen(&hdr)); + } + /* indicate we're done with file header writing */ + w->omit_warcinfo = 1U; + archive_string_free(&hdr); + } + + if (archive_entry_pathname(entry) == NULL) { + archive_set_error(&a->archive, EINVAL, + "Invalid filename"); + return (ARCHIVE_WARN); + } + + w->typ = archive_entry_filetype(entry); + w->populz = 0U; + if (w->typ == AE_IFREG) { + warc_essential_hdr_t rh = { + WT_RSRC, + /*uri*/archive_entry_pathname(entry), + /*urn*/NULL, + /*rtm*/w->now, + /*mtm*/archive_entry_mtime(entry), + /*cty*/NULL, + /*len*/(size_t)archive_entry_size(entry), + }; + ssize_t r; + + archive_string_init(&hdr); + r = _popul_ehdr(&hdr, MAX_HDR_SIZE, rh); + if (r < 0) { + /* don't bother */ + archive_set_error( + &a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, + "cannot archive file"); + return (ARCHIVE_WARN); + } + /* otherwise append to output stream */ + __archive_write_output(a, hdr.s, r); + /* and let subsequent calls to _data() know about the size */ + w->populz = rh.cntlen; + archive_string_free(&hdr); + return (ARCHIVE_OK); + } + /* just resort to erroring as per Tim's advice */ + archive_set_error( + &a->archive, + ARCHIVE_ERRNO_FILE_FORMAT, + "WARC can only process regular files"); + return (ARCHIVE_FAILED); +} + +static ssize_t +_warc_data(struct archive_write *a, const void *buf, size_t len) +{ + struct warc_s *w = a->format_data; + + if (w->typ == AE_IFREG) { + int rc; + + /* never write more bytes than announced */ + if (len > w->populz) { + len = (size_t)w->populz; + } + + /* now then, out we put the whole shebang */ + rc = __archive_write_output(a, buf, len); + if (rc != ARCHIVE_OK) { + return rc; + } + } + return len; +} + +static int +_warc_finish_entry(struct archive_write *a) +{ + static const char _eor[] = "\r\n\r\n"; + struct warc_s *w = a->format_data; + + if (w->typ == AE_IFREG) { + int rc = __archive_write_output(a, _eor, sizeof(_eor) - 1U); + + if (rc != ARCHIVE_OK) { + return rc; + } + } + /* reset type info */ + w->typ = 0; + return (ARCHIVE_OK); +} + +static int +_warc_close(struct archive_write *a) +{ + (void)a; /* UNUSED */ + return (ARCHIVE_OK); +} + +static int +_warc_free(struct archive_write *a) +{ + struct warc_s *w = a->format_data; + + free(w); + a->format_data = NULL; + return (ARCHIVE_OK); +} + + +/* private routines */ +static void +xstrftime(struct archive_string *as, const char *fmt, time_t t) +{ +/** like strftime(3) but for time_t objects */ + struct tm *rt; +#if defined(HAVE_GMTIME_R) || defined(HAVE__GMTIME64_S) + struct tm time; +#endif + char strtime[100]; + size_t len; + +#ifdef HAVE_GMTIME_R + if ((rt = gmtime_r(&t, &time)) == NULL) + return; +#elif defined(HAVE__GMTIME64_S) + _gmtime64_s(&time, &t); +#else + if ((rt = gmtime(&t)) == NULL) + return; +#endif + /* leave the hard yacker to our role model strftime() */ + len = strftime(strtime, sizeof(strtime)-1, fmt, rt); + archive_strncat(as, strtime, len); +} + +static ssize_t +_popul_ehdr(struct archive_string *tgt, size_t tsz, warc_essential_hdr_t hdr) +{ + static const char _ver[] = "WARC/1.0\r\n"; + static const char *_typ[LAST_WT] = { + NULL, "warcinfo", "metadata", "resource", NULL + }; + char std_uuid[48U]; + + if (hdr.type == WT_NONE || hdr.type > WT_RSRC) { + /* brilliant, how exactly did we get here? */ + return -1; + } + + archive_strcpy(tgt, _ver); + + archive_string_sprintf(tgt, "WARC-Type: %s\r\n", _typ[hdr.type]); + + if (hdr.tgturi != NULL) { + /* check if there's a xyz:// */ + static const char _uri[] = ""; + static const char _fil[] = "file://"; + const char *u; + char *chk = strchr(hdr.tgturi, ':'); + + if (chk != NULL && chk[1U] == '/' && chk[2U] == '/') { + /* yep, it's definitely a URI */ + u = _uri; + } else { + /* hm, best to prepend file:// then */ + u = _fil; + } + archive_string_sprintf(tgt, + "WARC-Target-URI: %s%s\r\n", u, hdr.tgturi); + } + + /* record time is usually when the http is sent off, + * just treat the archive writing as such for a moment */ + xstrftime(tgt, "WARC-Date: %Y-%m-%dT%H:%M:%SZ\r\n", hdr.rtime); + + /* while we're at it, record the mtime */ + xstrftime(tgt, "Last-Modified: %Y-%m-%dT%H:%M:%SZ\r\n", hdr.mtime); + + if (hdr.recid == NULL) { + /* generate one, grrrr */ + warc_uuid_t u; + + _gen_uuid(&u); + /* Unfortunately, archive_string_sprintf does not + * handle the minimum number following '%'. + * So we have to use snprintf function here instead + * of archive_string_snprintf function. */ +#if defined(_WIN32) && !defined(__CYGWIN__) +#define snprintf _snprintf +#endif + snprintf( + std_uuid, sizeof(std_uuid), + "<urn:uuid:%08x-%04x-%04x-%04x-%04x%08x>", + u.u[0U], + u.u[1U] >> 16U, u.u[1U] & 0xffffU, + u.u[2U] >> 16U, u.u[2U] & 0xffffU, + u.u[3U]); + hdr.recid = std_uuid; + } + + /* record-id is mandatory, fingers crossed we won't fail */ + archive_string_sprintf(tgt, "WARC-Record-ID: %s\r\n", hdr.recid); + + if (hdr.cnttyp != NULL) { + archive_string_sprintf(tgt, "Content-Type: %s\r\n", hdr.cnttyp); + } + + /* next one is mandatory */ + archive_string_sprintf(tgt, "Content-Length: %ju\r\n", (uintmax_t)hdr.cntlen); + /**/ + archive_strncat(tgt, "\r\n", 2); + + return (archive_strlen(tgt) >= tsz)? -1: (ssize_t)archive_strlen(tgt); +} + +static int +_gen_uuid(warc_uuid_t *tgt) +{ + archive_random(tgt->u, sizeof(tgt->u)); + /* obey uuid version 4 rules */ + tgt->u[1U] &= 0xffff0fffU; + tgt->u[1U] |= 0x4000U; + tgt->u[2U] &= 0x3fffffffU; + tgt->u[2U] |= 0x80000000U; + return 0; +} + +/* archive_write_set_format_warc.c ends here */ diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_xar.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_xar.c index 4cd2b9d..a47c53c 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format_xar.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_xar.c @@ -47,7 +47,7 @@ __FBSDID("$FreeBSD$"); #endif #include "archive.h" -#include "archive_crypto_private.h" +#include "archive_digest_private.h" #include "archive_endian.h" #include "archive_entry.h" #include "archive_entry_locale.h" @@ -114,7 +114,7 @@ enum sumalg { #define MAX_SUM_SIZE 20 #define MD5_NAME "md5" #define SHA1_NAME "sha1" - + enum enctype { NONE, GZIP, @@ -242,6 +242,7 @@ struct xar { enum sumalg opt_sumalg; enum enctype opt_compression; int opt_compression_level; + uint32_t opt_threads; struct chksumwork a_sumwrk; /* archived checksum. */ struct chksumwork e_sumwrk; /* extracted checksum. */ @@ -317,7 +318,7 @@ static int compression_end_bzip2(struct archive *, struct la_zstream *); static int compression_init_encoder_lzma(struct archive *, struct la_zstream *, int); static int compression_init_encoder_xz(struct archive *, - struct la_zstream *, int); + struct la_zstream *, int, int); #if defined(HAVE_LZMA_H) static int compression_code_lzma(struct archive *, struct la_zstream *, enum la_zaction); @@ -380,9 +381,10 @@ archive_write_set_format_xar(struct archive *_a) /* Set default checksum type. */ xar->opt_toc_sumalg = CKSUM_SHA1; xar->opt_sumalg = CKSUM_SHA1; - /* Set default compression type and level. */ + /* Set default compression type, level, and number of threads. */ xar->opt_compression = GZIP; xar->opt_compression_level = 6; + xar->opt_threads = 1; a->format_data = xar; @@ -493,6 +495,26 @@ xar_options(struct archive_write *a, const char *key, const char *value) } return (ARCHIVE_OK); } + if (strcmp(key, "threads") == 0) { + if (value == NULL) + return (ARCHIVE_FAILED); + xar->opt_threads = (int)strtoul(value, NULL, 10); + if (xar->opt_threads == 0 && errno != 0) { + xar->opt_threads = 1; + archive_set_error(&(a->archive), + ARCHIVE_ERRNO_MISC, + "Illegal value `%s'", + value); + return (ARCHIVE_FAILED); + } + if (xar->opt_threads == 0) { +#ifdef HAVE_LZMA_STREAM_ENCODER_MT + xar->opt_threads = lzma_cputhreads(); +#else + xar->opt_threads = 1; +#endif + } + } /* Note: The "warn" return is just to inform the options * supervisor that we didn't handle it. It will generate @@ -805,7 +827,7 @@ xmlwrite_string(struct archive_write *a, xmlTextWriterPtr writer, if (value == NULL) return (ARCHIVE_OK); - + r = xmlTextWriterStartElement(writer, BAD_CAST_CONST(key)); if (r < 0) { archive_set_error(&a->archive, @@ -1875,7 +1897,7 @@ file_cmp_node(const struct archive_rb_node *n1, return (strcmp(f1->basename.s, f2->basename.s)); } - + static int file_cmp_key(const struct archive_rb_node *n, const void *key) { @@ -2154,7 +2176,7 @@ file_gen_utility_names(struct archive_write *a, struct file *file) file->parentdir.length = len; archive_string_copy(&(file->basename), &(file->parentdir)); archive_string_empty(&(file->parentdir)); - file->parentdir.s = '\0'; + *file->parentdir.s = '\0'; return (r); } @@ -2494,7 +2516,7 @@ file_init_hardlinks(struct xar *xar) static const struct archive_rb_tree_ops rb_ops = { file_hd_cmp_node, file_hd_cmp_key, }; - + __archive_rb_tree_init(&(xar->hardlink_rbtree), &rb_ops); } @@ -2848,13 +2870,18 @@ compression_init_encoder_lzma(struct archive *a, static int compression_init_encoder_xz(struct archive *a, - struct la_zstream *lastrm, int level) + struct la_zstream *lastrm, int level, int threads) { static const lzma_stream lzma_init_data = LZMA_STREAM_INIT; lzma_stream *strm; lzma_filter *lzmafilters; lzma_options_lzma lzma_opt; int r; +#ifdef HAVE_LZMA_STREAM_ENCODER_MT + lzma_mt mt_options; +#endif + + (void)threads; /* UNUSED (if multi-threaded LZMA library not avail) */ if (lastrm->valid) compression_end(a, lastrm); @@ -2879,7 +2906,17 @@ compression_init_encoder_xz(struct archive *a, lzmafilters[1].id = LZMA_VLI_UNKNOWN;/* Terminate */ *strm = lzma_init_data; - r = lzma_stream_encoder(strm, lzmafilters, LZMA_CHECK_CRC64); +#ifdef HAVE_LZMA_STREAM_ENCODER_MT + if (threads > 1) { + bzero(&mt_options, sizeof(mt_options)); + mt_options.threads = threads; + mt_options.timeout = 300; + mt_options.filters = lzmafilters; + mt_options.check = LZMA_CHECK_CRC64; + r = lzma_stream_encoder_mt(strm, &mt_options); + } else +#endif + r = lzma_stream_encoder(strm, lzmafilters, LZMA_CHECK_CRC64); switch (r) { case LZMA_OK: lastrm->real_stream = strm; @@ -2979,10 +3016,11 @@ compression_init_encoder_lzma(struct archive *a, } static int compression_init_encoder_xz(struct archive *a, - struct la_zstream *lastrm, int level) + struct la_zstream *lastrm, int level, int threads) { (void) level; /* UNUSED */ + (void) threads; /* UNUSED */ if (lastrm->valid) compression_end(a, lastrm); return (compression_unsupported_encoder(a, lastrm, "xz")); @@ -3015,7 +3053,7 @@ xar_compression_init_encoder(struct archive_write *a) case XZ: r = compression_init_encoder_xz( &(a->archive), &(xar->stream), - xar->opt_compression_level); + xar->opt_compression_level, xar->opt_threads); break; default: r = ARCHIVE_OK; @@ -3178,4 +3216,3 @@ getalgname(enum sumalg sumalg) } #endif /* Support xar format */ - diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_format_zip.c b/Utilities/cmlibarchive/libarchive/archive_write_set_format_zip.c index 3b07e35..0636f46 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_format_zip.c +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_format_zip.c @@ -1,7 +1,7 @@ /*- * Copyright (c) 2008 Anselm Strauss * Copyright (c) 2009 Joerg Sonnenberger - * Copyright (c) 2011-2012 Michihiro NAKAJIMA + * Copyright (c) 2011-2012,2014 Michihiro NAKAJIMA * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -49,20 +49,23 @@ __FBSDID("$FreeBSD: head/lib/libarchive/archive_write_set_format_zip.c 201168 20 #endif #include "archive.h" +#include "archive_cryptor_private.h" #include "archive_endian.h" #include "archive_entry.h" #include "archive_entry_locale.h" +#include "archive_hmac_private.h" #include "archive_private.h" +#include "archive_random_private.h" #include "archive_write_private.h" #ifndef HAVE_ZLIB_H #include "archive_crc32.h" #endif +#define ZIP_ENTRY_FLAG_ENCRYPTED (1<<0) #define ZIP_ENTRY_FLAG_LENGTH_AT_END (1<<3) #define ZIP_ENTRY_FLAG_UTF8_NAME (1 << 11) - enum compression { COMPRESSION_UNSPECIFIED = -1, COMPRESSION_STORE = 0, @@ -75,6 +78,32 @@ enum compression { #define COMPRESSION_DEFAULT COMPRESSION_STORE #endif +enum encryption { + ENCRYPTION_NONE = 0, + ENCRYPTION_TRADITIONAL, /* Traditional PKWARE encryption. */ + ENCRYPTION_WINZIP_AES128, /* WinZIP AES-128 encryption. */ + ENCRYPTION_WINZIP_AES256, /* WinZIP AES-256 encryption. */ +}; + +#define TRAD_HEADER_SIZE 12 +/* + * See "WinZip - AES Encryption Information" + * http://www.winzip.com/aes_info.htm + */ +/* Value used in compression method. */ +#define WINZIP_AES_ENCRYPTION 99 +/* A WinZip AES header size which is stored at the beginning of + * file contents. */ +#define WINZIP_AES128_HEADER_SIZE (8 + 2) +#define WINZIP_AES256_HEADER_SIZE (16 + 2) +/* AES vendor version. */ +#define AES_VENDOR_AE_1 0x0001 +#define AES_VENDOR_AE_2 0x0002 +/* Authentication code size. */ +#define AUTH_CODE_SIZE 10 +/**/ +#define MAX_DERIVED_KEY_BUF_SIZE (AES_MAX_KEY_SIZE * 2 + 2) + struct cd_segment { struct cd_segment *next; size_t buff_size; @@ -82,9 +111,9 @@ struct cd_segment { unsigned char *p; }; -/* Bits used to enable/disable certain experimental features. */ -#define EXPERIMENT_LA 1 -#define EXPERIMENTS_ALL 0xffff +struct trad_enc_ctx { + uint32_t keys[3]; +}; struct zip { @@ -97,9 +126,18 @@ struct zip { struct archive_entry *entry; uint32_t entry_crc32; enum compression entry_compression; + enum encryption entry_encryption; int entry_flags; int entry_uses_zip64; int experiments; + struct trad_enc_ctx tctx; + char tctx_valid; + unsigned char trad_chkdat; + unsigned aes_vendor; + archive_crypto_ctx cctx; + char cctx_valid; + archive_hmac_sha1_ctx hctx; + char hctx_valid; unsigned char *file_header; size_t file_header_extra_offset; @@ -115,18 +153,20 @@ struct zip { struct archive_string_conv *opt_sconv; struct archive_string_conv *sconv_default; enum compression requested_compression; + int deflate_compression_level; int init_default_conversion; + enum encryption encryption_type; #define ZIP_FLAG_AVOID_ZIP64 1 #define ZIP_FLAG_FORCE_ZIP64 2 -#define ZIP_FLAG_EXPERIMENT_EL 4 +#define ZIP_FLAG_EXPERIMENT_xl 4 int flags; #ifdef HAVE_ZLIB_H z_stream stream; +#endif size_t len_buf; unsigned char *buf; -#endif }; /* Don't call this min or MIN, since those are already defined @@ -147,6 +187,13 @@ static size_t path_length(struct archive_entry *); static int write_path(struct archive_entry *, struct archive_write *); static void copy_path(struct archive_entry *, unsigned char *); static struct archive_string_conv *get_sconv(struct archive_write *, struct zip *); +static int trad_enc_init(struct trad_enc_ctx *, const char *, size_t); +static unsigned trad_enc_encrypt_update(struct trad_enc_ctx *, const uint8_t *, + size_t, uint8_t *, size_t); +static int init_traditional_pkware_encryption(struct archive_write *); +static int is_traditional_pkware_encryption_supported(void); +static int init_winzip_aes_encryption(struct archive_write *); +static int is_winzip_aes_encryption_supported(int encryption); static unsigned char * cd_alloc(struct zip *zip, size_t length) @@ -186,7 +233,7 @@ cd_alloc(struct zip *zip, size_t length) static unsigned long real_crc32(unsigned long crc, const void *buff, size_t len) { - return crc32(crc, buff, len); + return crc32(crc, buff, (unsigned int)len); } static unsigned long @@ -227,11 +274,70 @@ archive_write_zip_options(struct archive_write *a, const char *key, ret = ARCHIVE_OK; } return (ret); + } else if (strcmp(key, "compression-level") == 0) { + if (val == NULL || !(val[0] >= '0' && val[0] <= '9') || val[1] != '\0') { + return ARCHIVE_WARN; + } + + if (val[0] == '0') { + zip->requested_compression = COMPRESSION_STORE; + return ARCHIVE_OK; + } else { +#ifdef HAVE_ZLIB_H + zip->requested_compression = COMPRESSION_DEFLATE; + zip->deflate_compression_level = val[0] - '0'; + return ARCHIVE_OK; +#else + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "deflate compression not supported"); +#endif + } + } else if (strcmp(key, "encryption") == 0) { + if (val == NULL) { + zip->encryption_type = ENCRYPTION_NONE; + ret = ARCHIVE_OK; + } else if (val[0] == '1' || strcmp(val, "traditional") == 0 + || strcmp(val, "zipcrypt") == 0 + || strcmp(val, "ZipCrypt") == 0) { + if (is_traditional_pkware_encryption_supported()) { + zip->encryption_type = ENCRYPTION_TRADITIONAL; + ret = ARCHIVE_OK; + } else { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_MISC, + "encryption not supported"); + } + } else if (strcmp(val, "aes128") == 0) { + if (is_winzip_aes_encryption_supported( + ENCRYPTION_WINZIP_AES128)) { + zip->encryption_type = ENCRYPTION_WINZIP_AES128; + ret = ARCHIVE_OK; + } else { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_MISC, + "encryption not supported"); + } + } else if (strcmp(val, "aes256") == 0) { + if (is_winzip_aes_encryption_supported( + ENCRYPTION_WINZIP_AES256)) { + zip->encryption_type = ENCRYPTION_WINZIP_AES256; + ret = ARCHIVE_OK; + } else { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_MISC, + "encryption not supported"); + } + } else { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "%s: unknown encryption '%s'", + a->format_name, val); + } + return (ret); } else if (strcmp(key, "experimental") == 0) { if (val == NULL || val[0] == 0) { - zip->flags &= ~ ZIP_FLAG_EXPERIMENT_EL; + zip->flags &= ~ ZIP_FLAG_EXPERIMENT_xl; } else { - zip->flags |= ZIP_FLAG_EXPERIMENT_EL; + zip->flags |= ZIP_FLAG_EXPERIMENT_xl; } return (ARCHIVE_OK); } else if (strcmp(key, "fakecrc32") == 0) { @@ -357,9 +463,12 @@ archive_write_set_format_zip(struct archive *_a) /* "Unspecified" lets us choose the appropriate compression. */ zip->requested_compression = COMPRESSION_UNSPECIFIED; +#ifdef HAVE_ZLIB_H + zip->deflate_compression_level = Z_DEFAULT_COMPRESSION; +#endif zip->crc32func = real_crc32; -#ifdef HAVE_ZLIB_H + /* A buffer used for both compression and encryption. */ zip->len_buf = 65536; zip->buf = malloc(zip->len_buf); if (zip->buf == NULL) { @@ -368,7 +477,6 @@ archive_write_set_format_zip(struct archive *_a) "Can't allocate compression buffer"); return (ARCHIVE_FATAL); } -#endif a->format_data = zip; a->format_name = "zip"; @@ -400,7 +508,7 @@ static int archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) { unsigned char local_header[32]; - unsigned char local_extra[128]; + unsigned char local_extra[144]; struct zip *zip = a->format_data; unsigned char *e; unsigned char *cd_extra; @@ -409,7 +517,6 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) size_t slink_size = 0; struct archive_string_conv *sconv = get_sconv(a, zip); int ret, ret2 = ARCHIVE_OK; - int64_t size; mode_t type; int version_needed = 10; @@ -425,13 +532,14 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) if (zip->flags & ZIP_FLAG_AVOID_ZIP64) { /* Reject entries over 4GB. */ if (archive_entry_size_is_set(entry) - && (archive_entry_size(entry) > 0xffffffff)) { + && (archive_entry_size(entry) > + ARCHIVE_LITERAL_LL(0xffffffff))) { archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "Files > 4GB require Zip64 extensions"); return ARCHIVE_FAILED; } /* Reject entries if archive is > 4GB. */ - if (zip->written_bytes > 0xffffffff) { + if (zip->written_bytes > ARCHIVE_LITERAL_LL(0xffffffff)) { archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, "Archives > 4GB require Zip64 extensions"); return ARCHIVE_FAILED; @@ -453,11 +561,34 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) zip->entry_flags = 0; zip->entry_uses_zip64 = 0; zip->entry_crc32 = zip->crc32func(0, NULL, 0); + zip->entry_encryption = 0; if (zip->entry != NULL) { archive_entry_free(zip->entry); zip->entry = NULL; } + if (zip->cctx_valid) + archive_encrypto_aes_ctr_release(&zip->cctx); + if (zip->hctx_valid) + archive_hmac_sha1_cleanup(&zip->hctx); + zip->tctx_valid = zip->cctx_valid = zip->hctx_valid = 0; + + if (type == AE_IFREG + &&(!archive_entry_size_is_set(entry) + || archive_entry_size(entry) > 0)) { + switch (zip->encryption_type) { + case ENCRYPTION_TRADITIONAL: + case ENCRYPTION_WINZIP_AES128: + case ENCRYPTION_WINZIP_AES256: + zip->entry_flags |= ZIP_ENTRY_FLAG_ENCRYPTED; + zip->entry_encryption = zip->encryption_type; + break; + default: + break; + } + } + + #if defined(_WIN32) && !defined(__CYGWIN__) /* Make sure the path separators in pahtname, hardlink and symlink * are all slash '/', not the Windows path separator '\'. */ @@ -543,10 +674,11 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) } else if (type != AE_IFREG) { zip->entry_compression = COMPRESSION_STORE; zip->entry_uncompressed_limit = 0; - size = 0; version_needed = 20; } else if (archive_entry_size_is_set(zip->entry)) { - size = archive_entry_size(zip->entry); + int64_t size = archive_entry_size(zip->entry); + int64_t additional_size = 0; + zip->entry_uncompressed_limit = size; zip->entry_compression = zip->requested_compression; if (zip->entry_compression == COMPRESSION_UNSPECIFIED) { @@ -560,8 +692,46 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) zip->entry_uncompressed_size = size; version_needed = 20; } - if ((zip->flags & ZIP_FLAG_FORCE_ZIP64) /* User asked. */ - || (zip->entry_uncompressed_size > ARCHIVE_LITERAL_LL(0xffffffff))) { /* Large entry. */ + + if (zip->entry_flags & ZIP_ENTRY_FLAG_ENCRYPTED) { + switch (zip->entry_encryption) { + case ENCRYPTION_TRADITIONAL: + additional_size = TRAD_HEADER_SIZE; + version_needed = 20; + break; + case ENCRYPTION_WINZIP_AES128: + additional_size = WINZIP_AES128_HEADER_SIZE + + AUTH_CODE_SIZE; + version_needed = 20; + break; + case ENCRYPTION_WINZIP_AES256: + additional_size = WINZIP_AES256_HEADER_SIZE + + AUTH_CODE_SIZE; + version_needed = 20; + break; + default: + break; + } + if (zip->entry_compression == COMPRESSION_STORE) + zip->entry_compressed_size += additional_size; + } + + /* + * Set Zip64 extension in any of the following cases + * (this was suggested by discussion on info-zip-dev + * mailing list): + * = Zip64 is being forced by user + * = File is over 4GiB uncompressed + * (including encryption header, if any) + * = File is close to 4GiB and is being compressed + * (compression might make file larger) + */ + if ((zip->flags & ZIP_FLAG_FORCE_ZIP64) + || (zip->entry_uncompressed_size + additional_size > + ARCHIVE_LITERAL_LL(0xffffffff)) + || (zip->entry_uncompressed_size > + ARCHIVE_LITERAL_LL(0xff000000) + && zip->entry_compression != COMPRESSION_STORE)) { zip->entry_uses_zip64 = 1; version_needed = 45; } @@ -569,9 +739,11 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) /* We may know the size, but never the CRC. */ zip->entry_flags |= ZIP_ENTRY_FLAG_LENGTH_AT_END; } else { - /* Prefer deflate if it's available, because deflate - * has a clear end-of-data marker that makes - * length-at-end more reliable. */ + /* We don't know the size. In this case, we prefer + * deflate (it has a clear end-of-data marker which + * makes length-at-end more reliable) and will + * enable Zip64 extensions unless we're told not to. + */ zip->entry_compression = COMPRESSION_DEFAULT; zip->entry_flags |= ZIP_ENTRY_FLAG_LENGTH_AT_END; if ((zip->flags & ZIP_FLAG_AVOID_ZIP64) == 0) { @@ -582,6 +754,19 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) } else { version_needed = 20; } + + if (zip->entry_flags & ZIP_ENTRY_FLAG_ENCRYPTED) { + switch (zip->entry_encryption) { + case ENCRYPTION_TRADITIONAL: + case ENCRYPTION_WINZIP_AES128: + case ENCRYPTION_WINZIP_AES256: + if (version_needed < 20) + version_needed = 20; + break; + default: + break; + } + } } /* Format the local header. */ @@ -589,8 +774,13 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) memcpy(local_header, "PK\003\004", 4); archive_le16enc(local_header + 4, version_needed); archive_le16enc(local_header + 6, zip->entry_flags); - archive_le16enc(local_header + 8, zip->entry_compression); - archive_le32enc(local_header + 10, dos_time(archive_entry_mtime(zip->entry))); + if (zip->entry_encryption == ENCRYPTION_WINZIP_AES128 + || zip->entry_encryption == ENCRYPTION_WINZIP_AES256) + archive_le16enc(local_header + 8, WINZIP_AES_ENCRYPTION); + else + archive_le16enc(local_header + 8, zip->entry_compression); + archive_le32enc(local_header + 10, + dos_time(archive_entry_mtime(zip->entry))); archive_le32enc(local_header + 14, zip->entry_crc32); if (zip->entry_uses_zip64) { /* Zip64 data in the local header "must" include both @@ -601,10 +791,17 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) archive_le32enc(local_header + 18, ARCHIVE_LITERAL_LL(0xffffffff)); archive_le32enc(local_header + 22, ARCHIVE_LITERAL_LL(0xffffffff)); } else { - archive_le32enc(local_header + 18, zip->entry_compressed_size); - archive_le32enc(local_header + 22, zip->entry_uncompressed_size); + archive_le32enc(local_header + 18, (uint32_t)zip->entry_compressed_size); + archive_le32enc(local_header + 22, (uint32_t)zip->entry_uncompressed_size); + } + archive_le16enc(local_header + 26, (uint16_t)filename_length); + + if (zip->entry_encryption == ENCRYPTION_TRADITIONAL) { + if (zip->entry_flags & ZIP_ENTRY_FLAG_LENGTH_AT_END) + zip->trad_chkdat = local_header[11]; + else + zip->trad_chkdat = local_header[17]; } - archive_le16enc(local_header + 26, filename_length); /* Format as much of central directory file header as we can: */ zip->file_header = cd_alloc(zip, 46); @@ -616,9 +813,14 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) archive_le16enc(zip->file_header + 4, 3 * 256 + version_needed); archive_le16enc(zip->file_header + 6, version_needed); archive_le16enc(zip->file_header + 8, zip->entry_flags); - archive_le16enc(zip->file_header + 10, zip->entry_compression); - archive_le32enc(zip->file_header + 12, dos_time(archive_entry_mtime(zip->entry))); - archive_le16enc(zip->file_header + 28, filename_length); + if (zip->entry_encryption == ENCRYPTION_WINZIP_AES128 + || zip->entry_encryption == ENCRYPTION_WINZIP_AES256) + archive_le16enc(zip->file_header + 10, WINZIP_AES_ENCRYPTION); + else + archive_le16enc(zip->file_header + 10, zip->entry_compression); + archive_le32enc(zip->file_header + 12, + dos_time(archive_entry_mtime(zip->entry))); + archive_le16enc(zip->file_header + 28, (uint16_t)filename_length); /* Following Info-Zip, store mode in the "external attributes" field. */ archive_le32enc(zip->file_header + 38, ((uint32_t)archive_entry_mode(zip->entry)) << 16); @@ -670,7 +872,33 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) archive_le32enc(e, (uint32_t)archive_entry_gid(entry)); e += 4; - /* Copy UT and ux into central directory as well. */ + /* AES extra data field: WinZIP AES information, ID=0x9901 */ + if ((zip->entry_flags & ZIP_ENTRY_FLAG_ENCRYPTED) + && (zip->entry_encryption == ENCRYPTION_WINZIP_AES128 + || zip->entry_encryption == ENCRYPTION_WINZIP_AES256)) { + + memcpy(e, "\001\231\007\000\001\000AE", 8); + /* AES vendoer version AE-2 does not store a CRC. + * WinZip 11 uses AE-1, which does store the CRC, + * but it does not store the CRC when the file size + * is less than 20 bytes. So we simulate what + * WinZip 11 does. + * NOTE: WinZip 9.0 and 10.0 uses AE-2 by default. */ + if (archive_entry_size_is_set(zip->entry) + && archive_entry_size(zip->entry) < 20) { + archive_le16enc(e+4, AES_VENDOR_AE_2); + zip->aes_vendor = AES_VENDOR_AE_2;/* no CRC. */ + } else + zip->aes_vendor = AES_VENDOR_AE_1; + e += 8; + /* AES encryption strength. */ + *e++ = (zip->entry_encryption == ENCRYPTION_WINZIP_AES128)?1:3; + /* Actual compression method. */ + archive_le16enc(e, zip->entry_compression); + e += 2; + } + + /* Copy UT ,ux, and AES-extra into central directory as well. */ zip->file_header_extra_offset = zip->central_directory_bytes; cd_extra = cd_alloc(zip, e - local_extra); memcpy(cd_extra, local_extra, e - local_extra); @@ -692,14 +920,14 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) e += 8; archive_le64enc(e, zip->entry_compressed_size); e += 8; - archive_le16enc(zip64_start + 2, e - (zip64_start + 4)); + archive_le16enc(zip64_start + 2, (uint16_t)(e - (zip64_start + 4))); } - if (zip->flags & ZIP_FLAG_EXPERIMENT_EL) { - /* Experimental 'el' extension to improve streaming. */ + if (zip->flags & ZIP_FLAG_EXPERIMENT_xl) { + /* Experimental 'xl' extension to improve streaming. */ unsigned char *external_info = e; int included = 7; - memcpy(e, "el\000\000", 4); // 0x6c65 + 2-byte length + memcpy(e, "xl\000\000", 4); // 0x6c65 + 2-byte length e += 4; e[0] = included; /* bitmap of included fields */ e += 1; @@ -720,11 +948,11 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) if (included & 8) { // Libarchive does not currently support file comments. } - archive_le16enc(external_info + 2, e - (external_info + 4)); + archive_le16enc(external_info + 2, (uint16_t)(e - (external_info + 4))); } /* Update local header with size of extra data and write it all out: */ - archive_le16enc(local_header + 28, e - local_extra); + archive_le16enc(local_header + 28, (uint16_t)(e - local_extra)); ret = __archive_write_output(a, local_header, 30); if (ret != ARCHIVE_OK) @@ -758,7 +986,7 @@ archive_write_zip_header(struct archive_write *a, struct archive_entry *entry) zip->stream.opaque = Z_NULL; zip->stream.next_out = zip->buf; zip->stream.avail_out = (uInt)zip->len_buf; - if (deflateInit2(&zip->stream, Z_DEFAULT_COMPRESSION, + if (deflateInit2(&zip->stream, zip->deflate_compression_level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK) { archive_set_error(&a->archive, ENOMEM, "Can't init deflate compressor"); @@ -782,13 +1010,72 @@ archive_write_zip_data(struct archive_write *a, const void *buff, size_t s) if (s == 0) return 0; + if (zip->entry_flags & ZIP_ENTRY_FLAG_ENCRYPTED) { + switch (zip->entry_encryption) { + case ENCRYPTION_TRADITIONAL: + /* Initialize traditoinal PKWARE encryption context. */ + if (!zip->tctx_valid) { + ret = init_traditional_pkware_encryption(a); + if (ret != ARCHIVE_OK) + return (ret); + zip->tctx_valid = 1; + } + break; + case ENCRYPTION_WINZIP_AES128: + case ENCRYPTION_WINZIP_AES256: + if (!zip->cctx_valid) { + ret = init_winzip_aes_encryption(a); + if (ret != ARCHIVE_OK) + return (ret); + zip->cctx_valid = zip->hctx_valid = 1; + } + break; + default: + break; + } + } + switch (zip->entry_compression) { case COMPRESSION_STORE: - ret = __archive_write_output(a, buff, s); - if (ret != ARCHIVE_OK) - return (ret); - zip->written_bytes += s; - zip->entry_compressed_written += s; + if (zip->tctx_valid || zip->cctx_valid) { + const uint8_t *rb = (const uint8_t *)buff; + const uint8_t * const re = rb + s; + + while (rb < re) { + size_t l; + + if (zip->tctx_valid) { + l = trad_enc_encrypt_update(&zip->tctx, + rb, re - rb, + zip->buf, zip->len_buf); + } else { + l = zip->len_buf; + ret = archive_encrypto_aes_ctr_update( + &zip->cctx, + rb, re - rb, zip->buf, &l); + if (ret < 0) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_MISC, + "Failed to encrypt file"); + return (ARCHIVE_FAILED); + } + archive_hmac_sha1_update(&zip->hctx, + zip->buf, l); + } + ret = __archive_write_output(a, zip->buf, l); + if (ret != ARCHIVE_OK) + return (ret); + zip->entry_compressed_written += l; + zip->written_bytes += l; + rb += l; + } + } else { + ret = __archive_write_output(a, buff, s); + if (ret != ARCHIVE_OK) + return (ret); + zip->written_bytes += s; + zip->entry_compressed_written += s; + } break; #if HAVE_ZLIB_H case COMPRESSION_DEFLATE: @@ -799,6 +1086,25 @@ archive_write_zip_data(struct archive_write *a, const void *buff, size_t s) if (ret == Z_STREAM_ERROR) return (ARCHIVE_FATAL); if (zip->stream.avail_out == 0) { + if (zip->tctx_valid) { + trad_enc_encrypt_update(&zip->tctx, + zip->buf, zip->len_buf, + zip->buf, zip->len_buf); + } else if (zip->cctx_valid) { + size_t outl = zip->len_buf; + ret = archive_encrypto_aes_ctr_update( + &zip->cctx, + zip->buf, zip->len_buf, + zip->buf, &outl); + if (ret < 0) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_MISC, + "Failed to encrypt file"); + return (ARCHIVE_FAILED); + } + archive_hmac_sha1_update(&zip->hctx, + zip->buf, zip->len_buf); + } ret = __archive_write_output(a, zip->buf, zip->len_buf); if (ret != ARCHIVE_OK) @@ -819,7 +1125,9 @@ archive_write_zip_data(struct archive_write *a, const void *buff, size_t s) } zip->entry_uncompressed_limit -= s; - zip->entry_crc32 = zip->crc32func(zip->entry_crc32, buff, (unsigned)s); + if (!zip->cctx_valid || zip->aes_vendor != AES_VENDOR_AE_2) + zip->entry_crc32 = + zip->crc32func(zip->entry_crc32, buff, (unsigned)s); return (s); } @@ -834,10 +1142,28 @@ archive_write_zip_finish_entry(struct archive_write *a) if (zip->entry_compression == COMPRESSION_DEFLATE) { for (;;) { size_t remainder; + ret = deflate(&zip->stream, Z_FINISH); if (ret == Z_STREAM_ERROR) return (ARCHIVE_FATAL); remainder = zip->len_buf - zip->stream.avail_out; + if (zip->tctx_valid) { + trad_enc_encrypt_update(&zip->tctx, + zip->buf, remainder, zip->buf, remainder); + } else if (zip->cctx_valid) { + size_t outl = remainder; + ret = archive_encrypto_aes_ctr_update( + &zip->cctx, zip->buf, remainder, + zip->buf, &outl); + if (ret < 0) { + archive_set_error(&a->archive, + ARCHIVE_ERRNO_MISC, + "Failed to encrypt file"); + return (ARCHIVE_FAILED); + } + archive_hmac_sha1_update(&zip->hctx, + zip->buf, remainder); + } ret = __archive_write_output(a, zip->buf, remainder); if (ret != ARCHIVE_OK) return (ret); @@ -851,20 +1177,38 @@ archive_write_zip_finish_entry(struct archive_write *a) deflateEnd(&zip->stream); } #endif + if (zip->hctx_valid) { + uint8_t hmac[20]; + size_t hmac_len = 20; + + archive_hmac_sha1_final(&zip->hctx, hmac, &hmac_len); + ret = __archive_write_output(a, hmac, AUTH_CODE_SIZE); + if (ret != ARCHIVE_OK) + return (ret); + zip->entry_compressed_written += AUTH_CODE_SIZE; + zip->written_bytes += AUTH_CODE_SIZE; + } /* Write trailing data descriptor. */ if ((zip->entry_flags & ZIP_ENTRY_FLAG_LENGTH_AT_END) != 0) { char d[24]; memcpy(d, "PK\007\010", 4); - archive_le32enc(d + 4, zip->entry_crc32); + if (zip->cctx_valid && zip->aes_vendor == AES_VENDOR_AE_2) + archive_le32enc(d + 4, 0);/* no CRC.*/ + else + archive_le32enc(d + 4, zip->entry_crc32); if (zip->entry_uses_zip64) { - archive_le64enc(d + 8, (uint64_t)zip->entry_compressed_written); - archive_le64enc(d + 16, (uint64_t)zip->entry_uncompressed_written); + archive_le64enc(d + 8, + (uint64_t)zip->entry_compressed_written); + archive_le64enc(d + 16, + (uint64_t)zip->entry_uncompressed_written); ret = __archive_write_output(a, d, 24); zip->written_bytes += 24; } else { - archive_le32enc(d + 8, (uint32_t)zip->entry_compressed_written); - archive_le32enc(d + 12, (uint32_t)zip->entry_uncompressed_written); + archive_le32enc(d + 8, + (uint32_t)zip->entry_compressed_written); + archive_le32enc(d + 12, + (uint32_t)zip->entry_uncompressed_written); ret = __archive_write_output(a, d, 16); zip->written_bytes += 16; } @@ -892,7 +1236,7 @@ archive_write_zip_finish_entry(struct archive_write *a) archive_le64enc(z, zip->entry_offset); z += 8; } - archive_le16enc(zip64 + 2, z - (zip64 + 4)); + archive_le16enc(zip64 + 2, (uint16_t)(z - (zip64 + 4))); zd = cd_alloc(zip, z - zip64); if (zd == NULL) { archive_set_error(&a->archive, ENOMEM, @@ -906,15 +1250,21 @@ archive_write_zip_finish_entry(struct archive_write *a) } /* Fix up central directory file header. */ - archive_le32enc(zip->file_header + 16, zip->entry_crc32); + if (zip->cctx_valid && zip->aes_vendor == AES_VENDOR_AE_2) + archive_le32enc(zip->file_header + 16, 0);/* no CRC.*/ + else + archive_le32enc(zip->file_header + 16, zip->entry_crc32); archive_le32enc(zip->file_header + 20, - zipmin(zip->entry_compressed_written, ARCHIVE_LITERAL_LL(0xffffffff))); + (uint32_t)zipmin(zip->entry_compressed_written, + ARCHIVE_LITERAL_LL(0xffffffff))); archive_le32enc(zip->file_header + 24, - zipmin(zip->entry_uncompressed_written, ARCHIVE_LITERAL_LL(0xffffffff))); + (uint32_t)zipmin(zip->entry_uncompressed_written, + ARCHIVE_LITERAL_LL(0xffffffff))); archive_le16enc(zip->file_header + 30, - zip->central_directory_bytes - zip->file_header_extra_offset); + (uint16_t)(zip->central_directory_bytes - zip->file_header_extra_offset)); archive_le32enc(zip->file_header + 42, - zipmin(zip->entry_offset, ARCHIVE_LITERAL_LL(0xffffffff))); + (uint32_t)zipmin(zip->entry_offset, + ARCHIVE_LITERAL_LL(0xffffffff))); return (ARCHIVE_OK); } @@ -977,10 +1327,16 @@ archive_write_zip_close(struct archive_write *a) /* Format and write end of central directory. */ memset(buff, 0, sizeof(buff)); memcpy(buff, "PK\005\006", 4); - archive_le16enc(buff + 8, zipmin(0xffffU, zip->central_directory_entries)); - archive_le16enc(buff + 10, zipmin(0xffffU, zip->central_directory_entries)); - archive_le32enc(buff + 12, (uint32_t)zipmin(ARCHIVE_LITERAL_LL(0xffffffff), (offset_end - offset_start))); - archive_le32enc(buff + 16, (uint32_t)zipmin(ARCHIVE_LITERAL_LL(0xffffffff), offset_start)); + archive_le16enc(buff + 8, (uint16_t)zipmin(0xffffU, + zip->central_directory_entries)); + archive_le16enc(buff + 10, (uint16_t)zipmin(0xffffU, + zip->central_directory_entries)); + archive_le32enc(buff + 12, + (uint32_t)zipmin(ARCHIVE_LITERAL_LL(0xffffffff), + (offset_end - offset_start))); + archive_le32enc(buff + 16, + (uint32_t)zipmin(ARCHIVE_LITERAL_LL(0xffffffff), + offset_start)); ret = __archive_write_output(a, buff, 22); if (ret != ARCHIVE_OK) return (ARCHIVE_FATAL); @@ -1001,10 +1357,12 @@ archive_write_zip_free(struct archive_write *a) free(segment->buff); free(segment); } -#ifdef HAVE_ZLIB_H free(zip->buf); -#endif archive_entry_free(zip->entry); + if (zip->cctx_valid) + archive_encrypto_aes_ctr_release(&zip->cctx); + if (zip->hctx_valid) + archive_hmac_sha1_cleanup(&zip->hctx); /* TODO: Free opt_sconv, sconv_default */ free(zip); @@ -1124,3 +1482,199 @@ get_sconv(struct archive_write *a, struct zip *zip) } return (zip->sconv_default); } + +/* + Traditional PKWARE Decryption functions. + */ + +static void +trad_enc_update_keys(struct trad_enc_ctx *ctx, uint8_t c) +{ + uint8_t t; +#define CRC32(c, b) (crc32(c ^ 0xffffffffUL, &b, 1) ^ 0xffffffffUL) + + ctx->keys[0] = CRC32(ctx->keys[0], c); + ctx->keys[1] = (ctx->keys[1] + (ctx->keys[0] & 0xff)) * 134775813L + 1; + t = (ctx->keys[1] >> 24) & 0xff; + ctx->keys[2] = CRC32(ctx->keys[2], t); +#undef CRC32 +} + +static uint8_t +trad_enc_decypt_byte(struct trad_enc_ctx *ctx) +{ + unsigned temp = ctx->keys[2] | 2; + return (uint8_t)((temp * (temp ^ 1)) >> 8) & 0xff; +} + +static unsigned +trad_enc_encrypt_update(struct trad_enc_ctx *ctx, const uint8_t *in, + size_t in_len, uint8_t *out, size_t out_len) +{ + unsigned i, max; + + max = (unsigned)((in_len < out_len)? in_len: out_len); + + for (i = 0; i < max; i++) { + uint8_t t = in[i]; + out[i] = t ^ trad_enc_decypt_byte(ctx); + trad_enc_update_keys(ctx, t); + } + return i; +} + +static int +trad_enc_init(struct trad_enc_ctx *ctx, const char *pw, size_t pw_len) +{ + + ctx->keys[0] = 305419896L; + ctx->keys[1] = 591751049L; + ctx->keys[2] = 878082192L; + + for (;pw_len; --pw_len) + trad_enc_update_keys(ctx, *pw++); + return 0; +} + +static int +is_traditional_pkware_encryption_supported(void) +{ + uint8_t key[TRAD_HEADER_SIZE]; + + if (archive_random(key, sizeof(key)-1) != ARCHIVE_OK) + return (0); + return (1); +} + +static int +init_traditional_pkware_encryption(struct archive_write *a) +{ + struct zip *zip = a->format_data; + const char *passphrase; + uint8_t key[TRAD_HEADER_SIZE]; + uint8_t key_encrypted[TRAD_HEADER_SIZE]; + int ret; + + passphrase = __archive_write_get_passphrase(a); + if (passphrase == NULL) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Encryption needs passphrase"); + return ARCHIVE_FAILED; + } + if (archive_random(key, sizeof(key)-1) != ARCHIVE_OK) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Can't generate random number for encryption"); + return ARCHIVE_FATAL; + } + trad_enc_init(&zip->tctx, passphrase, strlen(passphrase)); + /* Set the last key code which will be used as a check code + * for verifying passphrase in decryption. */ + key[TRAD_HEADER_SIZE-1] = zip->trad_chkdat; + trad_enc_encrypt_update(&zip->tctx, key, TRAD_HEADER_SIZE, + key_encrypted, TRAD_HEADER_SIZE); + /* Write encrypted keys in the top of the file content. */ + ret = __archive_write_output(a, key_encrypted, TRAD_HEADER_SIZE); + if (ret != ARCHIVE_OK) + return (ret); + zip->written_bytes += TRAD_HEADER_SIZE; + zip->entry_compressed_written += TRAD_HEADER_SIZE; + return (ret); +} + +static int +init_winzip_aes_encryption(struct archive_write *a) +{ + struct zip *zip = a->format_data; + const char *passphrase; + size_t key_len, salt_len; + uint8_t salt[16 + 2]; + uint8_t derived_key[MAX_DERIVED_KEY_BUF_SIZE]; + int ret; + + passphrase = __archive_write_get_passphrase(a); + if (passphrase == NULL) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Encryption needs passphrase"); + return (ARCHIVE_FAILED); + } + if (zip->entry_encryption == ENCRYPTION_WINZIP_AES128) { + salt_len = 8; + key_len = 16; + } else { + /* AES 256 */ + salt_len = 16; + key_len = 32; + } + if (archive_random(salt, salt_len) != ARCHIVE_OK) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Can't generate random number for encryption"); + return (ARCHIVE_FATAL); + } + archive_pbkdf2_sha1(passphrase, strlen(passphrase), + salt, salt_len, 1000, derived_key, key_len * 2 + 2); + + ret = archive_encrypto_aes_ctr_init(&zip->cctx, derived_key, key_len); + if (ret != 0) { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Decryption is unsupported due to lack of crypto library"); + return (ARCHIVE_FAILED); + } + ret = archive_hmac_sha1_init(&zip->hctx, derived_key + key_len, + key_len); + if (ret != 0) { + archive_encrypto_aes_ctr_release(&zip->cctx); + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Failed to initialize HMAC-SHA1"); + return (ARCHIVE_FAILED); + } + + /* Set a passowrd verification value after the 'salt'. */ + salt[salt_len] = derived_key[key_len * 2]; + salt[salt_len + 1] = derived_key[key_len * 2 + 1]; + + /* Write encrypted keys in the top of the file content. */ + ret = __archive_write_output(a, salt, salt_len + 2); + if (ret != ARCHIVE_OK) + return (ret); + zip->written_bytes += salt_len + 2; + zip->entry_compressed_written += salt_len + 2; + + return (ARCHIVE_OK); +} + +static int +is_winzip_aes_encryption_supported(int encryption) +{ + size_t key_len, salt_len; + uint8_t salt[16 + 2]; + uint8_t derived_key[MAX_DERIVED_KEY_BUF_SIZE]; + archive_crypto_ctx cctx; + archive_hmac_sha1_ctx hctx; + int ret; + + if (encryption == ENCRYPTION_WINZIP_AES128) { + salt_len = 8; + key_len = 16; + } else { + /* AES 256 */ + salt_len = 16; + key_len = 32; + } + if (archive_random(salt, salt_len) != ARCHIVE_OK) + return (0); + ret = archive_pbkdf2_sha1("p", 1, salt, salt_len, 1000, + derived_key, key_len * 2 + 2); + if (ret != 0) + return (0); + + ret = archive_encrypto_aes_ctr_init(&cctx, derived_key, key_len); + if (ret != 0) + return (0); + ret = archive_hmac_sha1_init(&hctx, derived_key + key_len, + key_len); + archive_encrypto_aes_ctr_release(&cctx); + if (ret != 0) + return (0); + archive_hmac_sha1_cleanup(&hctx); + return (1); +} diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_options.3 b/Utilities/cmlibarchive/libarchive/archive_write_set_options.3 index f8fb039..a2f4b57 100644 --- a/Utilities/cmlibarchive/libarchive/archive_write_set_options.3 +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_options.3 @@ -397,6 +397,48 @@ Specifies a filename that should not be compressed when using This option can be provided multiple times to suppress compression on many files. .El +.It Format zip +.Bl -tag -compact -width indent +.It Cm compression +The value is either +.Dq store +or +.Dq deflate +to indicate how the following entries should be compressed. +Note that this setting is ignored for directories, symbolic links, +and other special entries. +.It Cm experimental +This boolean option enables or disables experimental Zip features +that may not be compatible with other Zip implementations. +.It Cm fakecrc32 +This boolean option disables CRC calculations. +All CRC fields are set to zero. +It should not be used except for testing purposes. +.It Cm hdrcharset +This sets the character set used for filenames. +.It Cm zip64 +Zip64 extensions provide additional file size information +for entries larger than 4 GiB. +They also provide extended file offset and archive size information +when archives exceed 4 GiB. +By default, the Zip writer selectively enables these extensions only as needed. +In particular, if the file size is unknown, the Zip writer will +include Zip64 extensions to guard against the possibility that the +file might be larger than 4 GiB. +.Pp +Setting this boolean option will force the writer to use Zip64 extensions +even for small files that would not otherwise require them. +This is primarily useful for testing. +.Pp +Disabling this option with +.Cm !zip64 +will force the Zip writer to avoid Zip64 extensions: +It will reject files with size greater than 4 GiB, +it will reject any new entries once the total archive size reaches 4 GiB, +and it will not use Zip64 extensions for files with unknown size. +In particular, this can improve compatibility when generating archives +where the entry sizes are not known in advance. +.El .El .Sh EXAMPLES The following example creates an archive write handle to diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_passphrase.3 b/Utilities/cmlibarchive/libarchive/archive_write_set_passphrase.3 new file mode 100644 index 0000000..2585595 --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_passphrase.3 @@ -0,0 +1,74 @@ +.\" Copyright (c) 2014 Michihiro NAKAJIMA +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd September 21, 2014 +.Dt ARCHIVE_WRITE_SET_PASSPHRASE 3 +.Os +.Sh NAME +.Nm archive_write_set_passphrase , +.Nm archive_write_set_passphrase_callback +.Nd functions for writing encrypted archives +.Sh LIBRARY +Streaming Archive Library (libarchive, -larchive) +.Sh SYNOPSIS +.In archive.h +.Ft int +.Fo archive_write_set_passphrase +.Fa "struct archive *" +.Fa "const char *passphrase" +.Fc +.Ft int +.Fo archive_write_set_passphrase_callback +.Fa "struct archive *" +.Fa "void *client_data" +.Fa "archive_passphrase_callback *" +.Fc +.Sh DESCRIPTION +.Bl -tag -width indent +.It Fn archive_write_set_passphrase +Set a passphrase for writing an encryption archive. +If +.Ar passphrase +is +.Dv NULL +or empty, this function will do nothing and +.Cm ARCHIVE_FAILED +will be returned. +Otherwise, +.Cm ARCHIVE_OK +will be returned. +.It Fn archive_write_set_passphrase_callback +Register callback function that will be invoked to get a passphrase +for encrption if the passphrase was not set by the +.Fn archive_write_set_passphrase +function. +.El +.\" .Sh ERRORS +.Sh SEE ALSO +.Xr tar 1 , +.Xr libarchive 3 , +.Xr archive_write 3 , +.Xr archive_write_set_options 3 diff --git a/Utilities/cmlibarchive/libarchive/archive_write_set_passphrase.c b/Utilities/cmlibarchive/libarchive/archive_write_set_passphrase.c new file mode 100644 index 0000000..710ecba --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_write_set_passphrase.c @@ -0,0 +1,95 @@ +/*- + * Copyright (c) 2014 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "archive_platform.h" +__FBSDID("$FreeBSD$"); + +#ifdef HAVE_ERRNO_H +#include <errno.h> +#endif +#include "archive_write_private.h" + +int +archive_write_set_passphrase(struct archive *_a, const char *p) +{ + struct archive_write *a = (struct archive_write *)_a; + + archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, ARCHIVE_STATE_NEW, + "archive_write_set_passphrase"); + + if (p == NULL || p[0] == '\0') { + archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC, + "Empty passphrase is unacceptable"); + return (ARCHIVE_FAILED); + } + free(a->passphrase); + a->passphrase = strdup(p); + if (a->passphrase == NULL) { + archive_set_error(&a->archive, ENOMEM, + "Can't allocate data for passphrase"); + return (ARCHIVE_FATAL); + } + return (ARCHIVE_OK); +} + + +int +archive_write_set_passphrase_callback(struct archive *_a, void *client_data, + archive_passphrase_callback *cb) +{ + struct archive_write *a = (struct archive_write *)_a; + + archive_check_magic(_a, ARCHIVE_WRITE_MAGIC, ARCHIVE_STATE_NEW, + "archive_write_set_passphrase_callback"); + + a->passphrase_callback = cb; + a->passphrase_client_data = client_data; + return (ARCHIVE_OK); +} + + +const char * +__archive_write_get_passphrase(struct archive_write *a) +{ + + if (a->passphrase != NULL) + return (a->passphrase); + + if (a->passphrase_callback != NULL) { + const char *p; + p = a->passphrase_callback(&a->archive, + a->passphrase_client_data); + if (p != NULL) { + a->passphrase = strdup(p); + if (a->passphrase == NULL) { + archive_set_error(&a->archive, ENOMEM, + "Can't allocate data for passphrase"); + return (NULL); + } + return (a->passphrase); + } + } + return (NULL); +} diff --git a/Utilities/cmlibarchive/libarchive/archive_xxhash.h b/Utilities/cmlibarchive/libarchive/archive_xxhash.h new file mode 100644 index 0000000..4272416 --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/archive_xxhash.h @@ -0,0 +1,47 @@ +/*- + * Copyright (c) 2014 Michihiro NAKAJIMA + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef __LIBARCHIVE_BUILD +#error This header is only to be used internally to libarchive. +#endif + +#ifndef ARCHIVE_XXHASH_H +#define ARCHIVE_XXHASH_H + +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + +struct archive_xxhash { + unsigned int (*XXH32)(const void* input, unsigned int len, + unsigned int seed); + void* (*XXH32_init)(unsigned int seed); + XXH_errorcode (*XXH32_update)(void* state, const void* input, + unsigned int len); + unsigned int (*XXH32_digest)(void* state); +}; + +extern const struct archive_xxhash __archive_xxhash; + +#endif diff --git a/Utilities/cmlibarchive/libarchive/libarchive.3 b/Utilities/cmlibarchive/libarchive/libarchive.3 index 3a9a841..c6894d2 100644 --- a/Utilities/cmlibarchive/libarchive/libarchive.3 +++ b/Utilities/cmlibarchive/libarchive/libarchive.3 @@ -146,11 +146,11 @@ pages for each API or utility function. .\" .Sh READING AN ARCHIVE See -.Xr libarchive_read 3 . +.Xr archive_read 3 . .\" .Sh WRITING AN ARCHIVE See -.Xr libarchive_write 3 . +.Xr archive_write 3 . .\" .Sh WRITING ENTRIES TO DISK The diff --git a/Utilities/cmlibarchive/libarchive/libarchive_internals.3 b/Utilities/cmlibarchive/libarchive/libarchive_internals.3 index 4aa09f9..8275d66 100644 --- a/Utilities/cmlibarchive/libarchive/libarchive_internals.3 +++ b/Utilities/cmlibarchive/libarchive/libarchive_internals.3 @@ -347,11 +347,11 @@ Fortunately, such archives are very rare, and libarchive can read most ZIP archives, though it cannot always extract as much information as a dedicated ZIP program. .Sh SEE ALSO -.Xr archive 3 , .Xr archive_entry 3 , .Xr archive_read 3 , .Xr archive_write 3 , .Xr archive_write_disk 3 +.Xr libarchive 3 , .Sh HISTORY The .Nm libarchive diff --git a/Utilities/cmlibarchive/libarchive/mtree.5 b/Utilities/cmlibarchive/libarchive/mtree.5 index 8c45a7d..16c8abe 100644 --- a/Utilities/cmlibarchive/libarchive/mtree.5 +++ b/Utilities/cmlibarchive/libarchive/mtree.5 @@ -56,14 +56,6 @@ corresponding character. .Pp Each line is interpreted independently as one of the following types: .Bl -tag -width Cm -.It Signature -The first line of any mtree file must begin with -.Dq #mtree . -If a file contains any full path entries, the first line should -begin with -.Dq #mtree v2.0 , -otherwise, the first line should begin with -.Dq #mtree v1.0 . .It Blank Blank lines are ignored. .It Comment @@ -302,16 +294,6 @@ The file owner as a symbolic name. .Xr find 1 , .Xr mtree 8 .Sh BUGS -The -.Fx -implementation of mtree does not currently support -the -.Nm -2.0 -format. -The requirement for a -.Dq #mtree -signature line is new and not yet widely implemented. .Sh HISTORY The .Nm diff --git a/Utilities/cmlibarchive/libarchive/tar.5 b/Utilities/cmlibarchive/libarchive/tar.5 index 688bb92..6e6f0c0 100644 --- a/Utilities/cmlibarchive/libarchive/tar.5 +++ b/Utilities/cmlibarchive/libarchive/tar.5 @@ -935,7 +935,7 @@ and formed the basis of (circa 1988). Joerg Shilling's .Nm star -archiver is another open-source (GPL) archiver (originally developed +archiver is another open-source (CDDL) archiver (originally developed circa 1985) which features complete support for pax interchange format. .Pp diff --git a/Utilities/cmlibarchive/libarchive/xxhash.c b/Utilities/cmlibarchive/libarchive/xxhash.c new file mode 100644 index 0000000..f7647a5 --- /dev/null +++ b/Utilities/cmlibarchive/libarchive/xxhash.c @@ -0,0 +1,500 @@ +/* +xxHash - Fast Hash algorithm +Copyright (C) 2012-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash source repository : http://code.google.com/p/xxhash/ +*/ + +#include "archive_platform.h" +#ifdef HAVE_LIBLZ4 + +/*************************************** +** Tuning parameters +****************************************/ +/* Unaligned memory access is automatically enabled for "common" CPU, such as x86. +** For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. +** If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. +** You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). +*/ +#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_USE_UNALIGNED_ACCESS 1 +#endif + +/* XXH_ACCEPT_NULL_INPUT_POINTER : +** If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. +** When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. +** This option has a very small performance cost (only measurable on small inputs). +** By default, this option is disabled. To enable it, uncomment below define : +** #define XXH_ACCEPT_NULL_INPUT_POINTER 1 + +** XXH_FORCE_NATIVE_FORMAT : +** By default, xxHash library provides endian-independant Hash values, based on little-endian convention. +** Results are therefore identical for little-endian and big-endian CPU. +** This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. +** Should endian-independance be of no importance for your application, you may set the #define below to 1. +** It will improve speed for Big-endian CPU. +** This option has no impact on Little_Endian CPU. +*/ +#define XXH_FORCE_NATIVE_FORMAT 0 + +/*************************************** +** Compiler Specific Options +****************************************/ +/* Disable some Visual warning messages */ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE __forceinline +#else +# ifdef __GNUC__ +# define FORCE_INLINE inline __attribute__((always_inline)) +# else +# define FORCE_INLINE inline +# endif +#endif + +/*************************************** +** Includes & Memory related functions +****************************************/ +#include "archive_xxhash.h" +#include <stdlib.h> +#define XXH_malloc malloc +#define XXH_free free +#include <string.h> +#define XXH_memcpy memcpy + + +static unsigned int XXH32 (const void*, unsigned int, unsigned int); +static void* XXH32_init (unsigned int); +static XXH_errorcode XXH32_update (void*, const void*, unsigned int); +static unsigned int XXH32_digest (void*); +/*static int XXH32_sizeofState(void);*/ +static XXH_errorcode XXH32_resetState(void*, unsigned int); +#define XXH32_SIZEOFSTATE 48 +typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t; +static unsigned int XXH32_intermediateDigest (void*); + +/*************************************** +** Basic Types +****************************************/ +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# include <stdint.h> + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + +#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) +# define _PACKED __attribute__ ((packed)) +#else +# define _PACKED +#endif + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# ifdef __IBMC__ +# pragma pack(1) +# else +# pragma pack(push, 1) +# endif +#endif + +typedef struct _U32_S { U32 v; } _PACKED U32_S; + +#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) +# pragma pack(pop) +#endif + +#define A32(x) (((const U32_S *)(x))->v) + + +/**************************************** +** Compiler-specific Functions and Macros +*****************************************/ +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static inline U32 XXH_swap32 (U32 x) { + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff );} +#endif + + +/*************************************** +** Constants +****************************************/ +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + + +/*************************************** +** Architecture Macros +****************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +#ifndef XXH_CPU_LITTLE_ENDIAN /* It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch */ + static const int one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one)) +#endif + + +/*************************************** +** Macros +****************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ + + +/***************************** +** Memory reads +******************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +static +FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); + else + return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); +} + +static +FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } + + +/***************************** +** Simple Hash Functions +******************************/ +static +FORCE_INLINE U32 XXH32_endian_align(const void* input, unsigned int len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align((const U32*)p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { len=0; bEnd=p=(const BYTE*)(size_t)16; } +#endif + + if (len>=16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do + { + v1 += XXH_get32bits(p) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; + v2 += XXH_get32bits(p) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; + v3 += XXH_get32bits(p) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; + v4 += XXH_get32bits(p) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; + } while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } + else + { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p<=bEnd-4) + { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p<bEnd) + { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1 ; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32(const void* input, unsigned int len, U32 seed) +{ +#if 0 + // Simple version, good for code maintenance, but unfortunately slow for small inputs + void* state = XXH32_init(seed); + XXH32_update(state, input, len); + return XXH32_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 3) == 0) /* Input is aligned, let's leverage the speed advantage */ + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +/***************************** +** Advanced Hash Functions +******************************/ + +struct XXH_state32_t +{ + U64 total_len; + U32 seed; + U32 v1; + U32 v2; + U32 v3; + U32 v4; + int memsize; + char memory[16]; +}; + +#if 0 +static +int XXH32_sizeofState(void) +{ + XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); /* A compilation error here means XXH32_SIZEOFSTATE is not large enough */ + return sizeof(struct XXH_state32_t); +} +#endif + +static +XXH_errorcode XXH32_resetState(void* state_in, U32 seed) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + state->seed = seed; + state->v1 = seed + PRIME32_1 + PRIME32_2; + state->v2 = seed + PRIME32_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME32_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + +static +void* XXH32_init (U32 seed) +{ + void* state = XXH_malloc (sizeof(struct XXH_state32_t)); + XXH32_resetState(state, seed); + return state; +} + +static +FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 16) /* fill in tmp buffer */ + { + XXH_memcpy(state->memory + state->memsize, input, len); + state->memsize += len; + return XXH_OK; + } + + if (state->memsize) /* some data left from previous update */ + { + XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); + { + const U32* p32 = (const U32*)state->memory; + state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++; + state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++; + state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++; + state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do + { + v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; + v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; + v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; + v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->memory, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +static +XXH_errorcode XXH32_update (void* state_in, const void* input, unsigned int len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +static +FORCE_INLINE U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian) +{ + struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; + const BYTE * p = (const BYTE*)state->memory; + BYTE* bEnd = (BYTE*)state->memory + state->memsize; + U32 h32; + + if (state->total_len >= 16) + { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } + else + { + h32 = state->seed + PRIME32_5; + } + + h32 += (U32) state->total_len; + + while (p<=bEnd-4) + { + h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p<bEnd) + { + h32 += (*p) * PRIME32_5; + h32 = XXH_rotl32(h32, 11) * PRIME32_1; + p++; + } + + h32 ^= h32 >> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + +static +U32 XXH32_intermediateDigest (void* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian); + else + return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian); +} + +static +U32 XXH32_digest (void* state_in) +{ + U32 h32 = XXH32_intermediateDigest(state_in); + + XXH_free(state_in); + + return h32; +} + +const +struct archive_xxhash __archive_xxhash = { + XXH32, + XXH32_init, + XXH32_update, + XXH32_digest +}; +#endif /* HAVE_LIBLZ4 */ |