diff options
Diffstat (limited to 'thirdparty')
144 files changed, 5130 insertions, 5700 deletions
diff --git a/thirdparty/README.md b/thirdparty/README.md index 2a8c6e1bb3..c99c2165f6 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -4,7 +4,7 @@ ## assimp - Upstream: http://github.com/assimp/assimp -- Version: git (1d565b0aab5a2ee00462f18c5b8a81f6a5454a48) +- Version: git (308db73d0b3c2d1870cd3e465eaa283692a4cf23) - License: BSD-3-Clause @@ -157,12 +157,12 @@ Files extracted from upstream source: ## libogg - Upstream: https://www.xiph.org/ogg -- Version: 1.3.3 +- Version: git (c8fca6b, 2019) - License: BSD-3-Clause Files extracted from upstream source: -- `src/*.c` +- `src/*.{c,h}` - `include/ogg/*.h` in ogg/ - COPYING @@ -376,7 +376,7 @@ Collection of single-file libraries used in Godot components. * License: zlib - `stb_vorbis.c` * Upstream: https://github.com/nothings/stb - * Version: 1.16 + * Version: 1.17 * License: Public Domain (Unlicense) or MIT @@ -536,7 +536,7 @@ Files extracted from upstream source: ## zstd - Upstream: https://github.com/facebook/zstd -- Version: 1.4.1 +- Version: 1.4.4 - License: BSD-3-Clause Files extracted from upstream source: diff --git a/thirdparty/assimp/code/Common/BaseImporter.cpp b/thirdparty/assimp/code/Common/BaseImporter.cpp index de5018a250..5c1e605549 100644 --- a/thirdparty/assimp/code/Common/BaseImporter.cpp +++ b/thirdparty/assimp/code/Common/BaseImporter.cpp @@ -67,7 +67,20 @@ using namespace Assimp; // Constructor to be privately used by Importer BaseImporter::BaseImporter() AI_NO_EXCEPT : m_progress() { - // nothing to do here + /** + * Assimp Importer + * unit conversions available + * if you need another measurment unit add it below. + * it's currently defined in assimp that we prefer meters. + * + * NOTE: Initialised here rather than in the header file + * to workaround a VS2013 bug with brace initialisers + * */ + importerUnits[ImporterUnits::M] = 1.0; + importerUnits[ImporterUnits::CM] = 0.01; + importerUnits[ImporterUnits::MM] = 0.001; + importerUnits[ImporterUnits::INCHES] = 0.0254; + importerUnits[ImporterUnits::FEET] = 0.3048; } // ------------------------------------------------------------------------------------------------ @@ -85,7 +98,7 @@ void BaseImporter::UpdateImporterScale( Importer* pImp ) double activeScale = importerScale * fileScale; // Set active scaling - pImp->SetPropertyFloat( AI_CONFIG_APP_SCALE_KEY, activeScale); + pImp->SetPropertyFloat( AI_CONFIG_APP_SCALE_KEY, static_cast<float>( activeScale) ); ASSIMP_LOG_DEBUG_F("UpdateImporterScale scale set: %f", activeScale ); } diff --git a/thirdparty/assimp/code/Common/DefaultIOSystem.cpp b/thirdparty/assimp/code/Common/DefaultIOSystem.cpp index d40b67de32..6fdc24dd80 100644 --- a/thirdparty/assimp/code/Common/DefaultIOSystem.cpp +++ b/thirdparty/assimp/code/Common/DefaultIOSystem.cpp @@ -61,83 +61,66 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. using namespace Assimp; -// maximum path length -// XXX http://insanecoding.blogspot.com/2007/11/pathmax-simply-isnt.html -#ifdef PATH_MAX -# define PATHLIMIT PATH_MAX -#else -# define PATHLIMIT 4096 +#ifdef _WIN32 +static std::wstring Utf8ToWide(const char* in) +{ + int size = MultiByteToWideChar(CP_UTF8, 0, in, -1, nullptr, 0); + // size includes terminating null; std::wstring adds null automatically + std::wstring out(static_cast<size_t>(size) - 1, L'\0'); + MultiByteToWideChar(CP_UTF8, 0, in, -1, &out[0], size); + return out; +} + +static std::string WideToUtf8(const wchar_t* in) +{ + int size = WideCharToMultiByte(CP_UTF8, 0, in, -1, nullptr, 0, nullptr, nullptr); + // size includes terminating null; std::string adds null automatically + std::string out(static_cast<size_t>(size) - 1, '\0'); + WideCharToMultiByte(CP_UTF8, 0, in, -1, &out[0], size, nullptr, nullptr); + return out; +} #endif // ------------------------------------------------------------------------------------------------ // Tests for the existence of a file at the given path. -bool DefaultIOSystem::Exists( const char* pFile) const +bool DefaultIOSystem::Exists(const char* pFile) const { #ifdef _WIN32 - wchar_t fileName16[PATHLIMIT]; - -#ifndef WindowsStore - bool isUnicode = IsTextUnicode(pFile, static_cast<int>(strlen(pFile)), NULL) != 0; - if (isUnicode) { - - MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, pFile, -1, fileName16, PATHLIMIT); - struct __stat64 filestat; - if (0 != _wstat64(fileName16, &filestat)) { - return false; - } - } else { -#endif - FILE* file = ::fopen(pFile, "rb"); - if (!file) - return false; - - ::fclose(file); -#ifndef WindowsStore + struct __stat64 filestat; + if (_wstat64(Utf8ToWide(pFile).c_str(), &filestat) != 0) { + return false; } -#endif #else - FILE* file = ::fopen( pFile, "rb"); - if( !file) + FILE* file = ::fopen(pFile, "rb"); + if (!file) return false; - ::fclose( file); + ::fclose(file); #endif return true; } // ------------------------------------------------------------------------------------------------ // Open a new file with a given path. -IOStream* DefaultIOSystem::Open( const char* strFile, const char* strMode) +IOStream* DefaultIOSystem::Open(const char* strFile, const char* strMode) { - ai_assert(NULL != strFile); - ai_assert(NULL != strMode); + ai_assert(strFile != nullptr); + ai_assert(strMode != nullptr); FILE* file; #ifdef _WIN32 - wchar_t fileName16[PATHLIMIT]; -#ifndef WindowsStore - bool isUnicode = IsTextUnicode(strFile, static_cast<int>(strlen(strFile)), NULL) != 0; - if (isUnicode) { - MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, strFile, -1, fileName16, PATHLIMIT); - std::string mode8(strMode); - file = ::_wfopen(fileName16, std::wstring(mode8.begin(), mode8.end()).c_str()); - } else { -#endif - file = ::fopen(strFile, strMode); -#ifndef WindowsStore - } -#endif + file = ::_wfopen(Utf8ToWide(strFile).c_str(), Utf8ToWide(strMode).c_str()); #else file = ::fopen(strFile, strMode); #endif - if (nullptr == file) + if (!file) return nullptr; - return new DefaultIOStream(file, (std::string) strFile); + return new DefaultIOStream(file, strFile); } // ------------------------------------------------------------------------------------------------ // Closes the given file and releases all resources associated with it. -void DefaultIOSystem::Close( IOStream* pFile) +void DefaultIOSystem::Close(IOStream* pFile) { delete pFile; } @@ -155,78 +138,56 @@ char DefaultIOSystem::getOsSeparator() const // ------------------------------------------------------------------------------------------------ // IOSystem default implementation (ComparePaths isn't a pure virtual function) -bool IOSystem::ComparePaths (const char* one, const char* second) const +bool IOSystem::ComparePaths(const char* one, const char* second) const { - return !ASSIMP_stricmp(one,second); + return !ASSIMP_stricmp(one, second); } // ------------------------------------------------------------------------------------------------ // Convert a relative path into an absolute path -inline static void MakeAbsolutePath (const char* in, char* _out) +inline static std::string MakeAbsolutePath(const char* in) { - ai_assert(in && _out); -#if defined( _MSC_VER ) || defined( __MINGW32__ ) -#ifndef WindowsStore - bool isUnicode = IsTextUnicode(in, static_cast<int>(strlen(in)), NULL) != 0; - if (isUnicode) { - wchar_t out16[PATHLIMIT]; - wchar_t in16[PATHLIMIT]; - MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED, in, -1, out16, PATHLIMIT); - wchar_t* ret = ::_wfullpath(out16, in16, PATHLIMIT); - if (ret) { - WideCharToMultiByte(CP_UTF8, MB_PRECOMPOSED, out16, -1, _out, PATHLIMIT, nullptr, nullptr); - } - if (!ret) { - // preserve the input path, maybe someone else is able to fix - // the path before it is accessed (e.g. our file system filter) - ASSIMP_LOG_WARN_F("Invalid path: ", std::string(in)); - strcpy(_out, in); - } - - } else { -#endif - char* ret = :: _fullpath(_out, in, PATHLIMIT); - if (!ret) { - // preserve the input path, maybe someone else is able to fix - // the path before it is accessed (e.g. our file system filter) - ASSIMP_LOG_WARN_F("Invalid path: ", std::string(in)); - strcpy(_out, in); - } -#ifndef WindowsStore + ai_assert(in); + std::string out; +#ifdef _WIN32 + wchar_t* ret = ::_wfullpath(nullptr, Utf8ToWide(in).c_str(), 0); + if (ret) { + out = WideToUtf8(ret); + free(ret); } -#endif #else - // use realpath - char* ret = realpath(in, _out); - if(!ret) { + char* ret = realpath(in, nullptr); + if (ret) { + out = ret; + free(ret); + } +#endif + if (!ret) { // preserve the input path, maybe someone else is able to fix // the path before it is accessed (e.g. our file system filter) ASSIMP_LOG_WARN_F("Invalid path: ", std::string(in)); - strcpy(_out,in); + out = in; } -#endif + return out; } // ------------------------------------------------------------------------------------------------ // DefaultIOSystem's more specialized implementation -bool DefaultIOSystem::ComparePaths (const char* one, const char* second) const +bool DefaultIOSystem::ComparePaths(const char* one, const char* second) const { // chances are quite good both paths are formatted identically, // so we can hopefully return here already - if( !ASSIMP_stricmp(one,second) ) + if (!ASSIMP_stricmp(one, second)) return true; - char temp1[PATHLIMIT]; - char temp2[PATHLIMIT]; - - MakeAbsolutePath (one, temp1); - MakeAbsolutePath (second, temp2); + std::string temp1 = MakeAbsolutePath(one); + std::string temp2 = MakeAbsolutePath(second); - return !ASSIMP_stricmp(temp1,temp2); + return !ASSIMP_stricmp(temp1, temp2); } // ------------------------------------------------------------------------------------------------ -std::string DefaultIOSystem::fileName( const std::string &path ) +std::string DefaultIOSystem::fileName(const std::string& path) { std::string ret = path; std::size_t last = ret.find_last_of("\\/"); @@ -235,16 +196,16 @@ std::string DefaultIOSystem::fileName( const std::string &path ) } // ------------------------------------------------------------------------------------------------ -std::string DefaultIOSystem::completeBaseName( const std::string &path ) +std::string DefaultIOSystem::completeBaseName(const std::string& path) { std::string ret = fileName(path); std::size_t pos = ret.find_last_of('.'); - if(pos != ret.npos) ret = ret.substr(0, pos); + if (pos != std::string::npos) ret = ret.substr(0, pos); return ret; } // ------------------------------------------------------------------------------------------------ -std::string DefaultIOSystem::absolutePath( const std::string &path ) +std::string DefaultIOSystem::absolutePath(const std::string& path) { std::string ret = path; std::size_t last = ret.find_last_of("\\/"); @@ -253,5 +214,3 @@ std::string DefaultIOSystem::absolutePath( const std::string &path ) } // ------------------------------------------------------------------------------------------------ - -#undef PATHLIMIT diff --git a/thirdparty/assimp/code/Common/Exporter.cpp b/thirdparty/assimp/code/Common/Exporter.cpp index 090b561ae0..4ce1a2bd80 100644 --- a/thirdparty/assimp/code/Common/Exporter.cpp +++ b/thirdparty/assimp/code/Common/Exporter.cpp @@ -102,6 +102,8 @@ void ExportSceneX3D(const char*, IOSystem*, const aiScene*, const ExportProperti void ExportSceneFBX(const char*, IOSystem*, const aiScene*, const ExportProperties*); void ExportSceneFBXA(const char*, IOSystem*, const aiScene*, const ExportProperties*); void ExportScene3MF( const char*, IOSystem*, const aiScene*, const ExportProperties* ); +void ExportSceneM3D(const char*, IOSystem*, const aiScene*, const ExportProperties*); +void ExportSceneA3D(const char*, IOSystem*, const aiScene*, const ExportProperties*); void ExportAssimp2Json(const char* , IOSystem*, const aiScene* , const Assimp::ExportProperties*); // ------------------------------------------------------------------------------------------------ @@ -179,6 +181,11 @@ Exporter::ExportFormatEntry gExporters[] = Exporter::ExportFormatEntry( "fbxa", "Autodesk FBX (ascii)", "fbx", &ExportSceneFBXA, 0 ), #endif +#ifndef ASSIMP_BUILD_NO_M3D_EXPORTER + Exporter::ExportFormatEntry( "m3d", "Model 3D (binary)", "m3d", &ExportSceneM3D, 0 ), + Exporter::ExportFormatEntry( "a3d", "Model 3D (ascii)", "m3d", &ExportSceneA3D, 0 ), +#endif + #ifndef ASSIMP_BUILD_NO_3MF_EXPORTER Exporter::ExportFormatEntry( "3mf", "The 3MF-File-Format", "3mf", &ExportScene3MF, 0 ), #endif @@ -316,34 +323,6 @@ const aiExportDataBlob* Exporter::ExportToBlob( const aiScene* pScene, const cha } // ------------------------------------------------------------------------------------------------ -bool IsVerboseFormat(const aiMesh* mesh) { - // avoid slow vector<bool> specialization - std::vector<unsigned int> seen(mesh->mNumVertices,0); - for(unsigned int i = 0; i < mesh->mNumFaces; ++i) { - const aiFace& f = mesh->mFaces[i]; - for(unsigned int j = 0; j < f.mNumIndices; ++j) { - if(++seen[f.mIndices[j]] == 2) { - // found a duplicate index - return false; - } - } - } - - return true; -} - -// ------------------------------------------------------------------------------------------------ -bool IsVerboseFormat(const aiScene* pScene) { - for(unsigned int i = 0; i < pScene->mNumMeshes; ++i) { - if(!IsVerboseFormat(pScene->mMeshes[i])) { - return false; - } - } - - return true; -} - -// ------------------------------------------------------------------------------------------------ aiReturn Exporter::Export( const aiScene* pScene, const char* pFormatId, const char* pPath, unsigned int pPreprocessing, const ExportProperties* pProperties) { ASSIMP_BEGIN_EXCEPTION_REGION(); @@ -352,7 +331,7 @@ aiReturn Exporter::Export( const aiScene* pScene, const char* pFormatId, const c // format. They will likely not be aware that there is a flag in the scene to indicate // this, however. To avoid surprises and bug reports, we check for duplicates in // meshes upfront. - const bool is_verbose_format = !(pScene->mFlags & AI_SCENE_FLAGS_NON_VERBOSE_FORMAT) || IsVerboseFormat(pScene); + const bool is_verbose_format = !(pScene->mFlags & AI_SCENE_FLAGS_NON_VERBOSE_FORMAT) || MakeVerboseFormatProcess::IsVerboseFormat(pScene); pimpl->mProgressHandler->UpdateFileWrite(0, 4); @@ -472,7 +451,10 @@ aiReturn Exporter::Export( const aiScene* pScene, const char* pFormatId, const c } ExportProperties emptyProperties; // Never pass NULL ExportProperties so Exporters don't have to worry. - exp.mExportFunction(pPath,pimpl->mIOSystem.get(),scenecopy.get(), pProperties ? pProperties : &emptyProperties); + ExportProperties* pProp = pProperties ? (ExportProperties*)pProperties : &emptyProperties; + pProp->SetPropertyBool("bJoinIdenticalVertices", must_join_again); + exp.mExportFunction(pPath,pimpl->mIOSystem.get(),scenecopy.get(), pProp); + exp.mExportFunction(pPath,pimpl->mIOSystem.get(),scenecopy.get(), pProp); pimpl->mProgressHandler->UpdateFileWrite(4, 4); } catch (DeadlyExportError& err) { diff --git a/thirdparty/assimp/code/Common/ImporterRegistry.cpp b/thirdparty/assimp/code/Common/ImporterRegistry.cpp index 32ac3b4168..b9f28f0356 100644 --- a/thirdparty/assimp/code/Common/ImporterRegistry.cpp +++ b/thirdparty/assimp/code/Common/ImporterRegistry.cpp @@ -197,6 +197,9 @@ corresponding preprocessor flag to selectively disable formats. #ifndef ASSIMP_BUILD_NO_MMD_IMPORTER # include "MMD/MMDImporter.h" #endif +#ifndef ASSIMP_BUILD_NO_M3D_IMPORTER +# include "M3D/M3DImporter.h" +#endif #ifndef ASSIMP_BUILD_NO_STEP_IMPORTER # include "Importer/StepFile/StepFileImporter.h" #endif @@ -223,6 +226,9 @@ void GetImporterInstanceList(std::vector< BaseImporter* >& out) #if (!defined ASSIMP_BUILD_NO_3DS_IMPORTER) out.push_back( new Discreet3DSImporter()); #endif +#if (!defined ASSIMP_BUILD_NO_M3D_IMPORTER) + out.push_back( new M3DImporter()); +#endif #if (!defined ASSIMP_BUILD_NO_MD3_IMPORTER) out.push_back( new MD3Importer()); #endif diff --git a/thirdparty/assimp/code/Common/PostStepRegistry.cpp b/thirdparty/assimp/code/Common/PostStepRegistry.cpp index ef58f8ddfd..8ff4af0400 100644 --- a/thirdparty/assimp/code/Common/PostStepRegistry.cpp +++ b/thirdparty/assimp/code/Common/PostStepRegistry.cpp @@ -131,11 +131,15 @@ corresponding preprocessor flag to selectively disable steps. #if (!defined ASSIMP_BUILD_NO_GLOBALSCALE_PROCESS) # include "PostProcessing/ScaleProcess.h" #endif +#if (!defined ASSIMP_BUILD_NO_ARMATUREPOPULATE_PROCESS) +# include "PostProcessing/ArmaturePopulate.h" +#endif #if (!defined ASSIMP_BUILD_NO_GENBOUNDINGBOXES_PROCESS) # include "PostProcessing/GenBoundingBoxesProcess.h" #endif + namespace Assimp { // ------------------------------------------------------------------------------------------------ @@ -180,6 +184,9 @@ void GetPostProcessingStepInstanceList(std::vector< BaseProcess* >& out) #if (!defined ASSIMP_BUILD_NO_GLOBALSCALE_PROCESS) out.push_back( new ScaleProcess()); #endif +#if (!defined ASSIMP_BUILD_NO_ARMATUREPOPULATE_PROCESS) + out.push_back( new ArmaturePopulate()); +#endif #if (!defined ASSIMP_BUILD_NO_PRETRANSFORMVERTICES_PROCESS) out.push_back( new PretransformVertices()); #endif diff --git a/thirdparty/assimp/code/Common/SceneCombiner.cpp b/thirdparty/assimp/code/Common/SceneCombiner.cpp index e445bd7434..f7b13cc951 100644 --- a/thirdparty/assimp/code/Common/SceneCombiner.cpp +++ b/thirdparty/assimp/code/Common/SceneCombiner.cpp @@ -1091,6 +1091,35 @@ void SceneCombiner::Copy( aiMesh** _dest, const aiMesh* src ) { aiFace& f = dest->mFaces[i]; GetArrayCopy(f.mIndices,f.mNumIndices); } + + // make a deep copy of all blend shapes + CopyPtrArray(dest->mAnimMeshes, dest->mAnimMeshes, dest->mNumAnimMeshes); +} + +// ------------------------------------------------------------------------------------------------ +void SceneCombiner::Copy(aiAnimMesh** _dest, const aiAnimMesh* src) { + if (nullptr == _dest || nullptr == src) { + return; + } + + aiAnimMesh* dest = *_dest = new aiAnimMesh(); + + // get a flat copy + ::memcpy(dest, src, sizeof(aiAnimMesh)); + + // and reallocate all arrays + GetArrayCopy(dest->mVertices, dest->mNumVertices); + GetArrayCopy(dest->mNormals, dest->mNumVertices); + GetArrayCopy(dest->mTangents, dest->mNumVertices); + GetArrayCopy(dest->mBitangents, dest->mNumVertices); + + unsigned int n = 0; + while (dest->HasTextureCoords(n)) + GetArrayCopy(dest->mTextureCoords[n++], dest->mNumVertices); + + n = 0; + while (dest->HasVertexColors(n)) + GetArrayCopy(dest->mColors[n++], dest->mNumVertices); } // ------------------------------------------------------------------------------------------------ @@ -1167,6 +1196,7 @@ void SceneCombiner::Copy( aiAnimation** _dest, const aiAnimation* src ) { // and reallocate all arrays CopyPtrArray( dest->mChannels, src->mChannels, dest->mNumChannels ); + CopyPtrArray( dest->mMorphMeshChannels, src->mMorphMeshChannels, dest->mNumMorphMeshChannels ); } // ------------------------------------------------------------------------------------------------ @@ -1186,6 +1216,26 @@ void SceneCombiner::Copy(aiNodeAnim** _dest, const aiNodeAnim* src) { GetArrayCopy( dest->mRotationKeys, dest->mNumRotationKeys ); } +void SceneCombiner::Copy(aiMeshMorphAnim** _dest, const aiMeshMorphAnim* src) { + if ( nullptr == _dest || nullptr == src ) { + return; + } + + aiMeshMorphAnim* dest = *_dest = new aiMeshMorphAnim(); + + // get a flat copy + ::memcpy(dest,src,sizeof(aiMeshMorphAnim)); + + // and reallocate all arrays + GetArrayCopy( dest->mKeys, dest->mNumKeys ); + for (ai_uint i = 0; i < dest->mNumKeys;++i) { + dest->mKeys[i].mValues = new unsigned int[dest->mKeys[i].mNumValuesAndWeights]; + dest->mKeys[i].mWeights = new double[dest->mKeys[i].mNumValuesAndWeights]; + ::memcpy(dest->mKeys[i].mValues, src->mKeys[i].mValues, dest->mKeys[i].mNumValuesAndWeights * sizeof(unsigned int)); + ::memcpy(dest->mKeys[i].mWeights, src->mKeys[i].mWeights, dest->mKeys[i].mNumValuesAndWeights * sizeof(double)); + } +} + // ------------------------------------------------------------------------------------------------ void SceneCombiner::Copy( aiCamera** _dest,const aiCamera* src) { if ( nullptr == _dest || nullptr == src ) { diff --git a/thirdparty/assimp/code/Common/Version.cpp b/thirdparty/assimp/code/Common/Version.cpp index cc94340ac8..cf1da7d5ba 100644 --- a/thirdparty/assimp/code/Common/Version.cpp +++ b/thirdparty/assimp/code/Common/Version.cpp @@ -46,8 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <assimp/scene.h> #include "ScenePrivate.h" -static const unsigned int MajorVersion = 4; -static const unsigned int MinorVersion = 1; +#include "revision.h" // -------------------------------------------------------------------------------- // Legal information string - don't remove this. @@ -56,9 +55,9 @@ static const char* LEGAL_INFORMATION = "Open Asset Import Library (Assimp).\n" "A free C/C++ library to import various 3D file formats into applications\n\n" -"(c) 2008-2017, assimp team\n" +"(c) 2006-2019, assimp team\n" "License under the terms and conditions of the 3-clause BSD license\n" -"http://assimp.sourceforge.net\n" +"http://assimp.org\n" ; // ------------------------------------------------------------------------------------------------ @@ -70,13 +69,13 @@ ASSIMP_API const char* aiGetLegalString () { // ------------------------------------------------------------------------------------------------ // Get Assimp minor version ASSIMP_API unsigned int aiGetVersionMinor () { - return MinorVersion; + return VER_MINOR; } // ------------------------------------------------------------------------------------------------ // Get Assimp major version ASSIMP_API unsigned int aiGetVersionMajor () { - return MajorVersion; + return VER_MAJOR; } // ------------------------------------------------------------------------------------------------ @@ -104,9 +103,6 @@ ASSIMP_API unsigned int aiGetCompileFlags () { return flags; } -// include current build revision, which is even updated from time to time -- :-) -#include "revision.h" - // ------------------------------------------------------------------------------------------------ ASSIMP_API unsigned int aiGetVersionRevision() { return GitVersion; diff --git a/thirdparty/assimp/code/Common/scene.cpp b/thirdparty/assimp/code/Common/scene.cpp index 2acb348d81..d15619acff 100644 --- a/thirdparty/assimp/code/Common/scene.cpp +++ b/thirdparty/assimp/code/Common/scene.cpp @@ -44,23 +44,23 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. aiNode::aiNode() : mName("") -, mParent(NULL) +, mParent(nullptr) , mNumChildren(0) -, mChildren(NULL) +, mChildren(nullptr) , mNumMeshes(0) -, mMeshes(NULL) -, mMetaData(NULL) { +, mMeshes(nullptr) +, mMetaData(nullptr) { // empty } aiNode::aiNode(const std::string& name) : mName(name) -, mParent(NULL) +, mParent(nullptr) , mNumChildren(0) -, mChildren(NULL) +, mChildren(nullptr) , mNumMeshes(0) -, mMeshes(NULL) -, mMetaData(NULL) { +, mMeshes(nullptr) +, mMetaData(nullptr) { // empty } @@ -68,7 +68,7 @@ aiNode::aiNode(const std::string& name) aiNode::~aiNode() { // delete all children recursively // to make sure we won't crash if the data is invalid ... - if (mChildren && mNumChildren) + if (mNumChildren && mChildren) { for (unsigned int a = 0; a < mNumChildren; a++) delete mChildren[a]; diff --git a/thirdparty/assimp/code/FBX/FBXCompileConfig.h b/thirdparty/assimp/code/FBX/FBXCompileConfig.h index 3a3841fa5b..03536a1823 100644 --- a/thirdparty/assimp/code/FBX/FBXCompileConfig.h +++ b/thirdparty/assimp/code/FBX/FBXCompileConfig.h @@ -47,6 +47,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define INCLUDED_AI_FBX_COMPILECONFIG_H #include <map> +#include <set> // #if _MSC_VER > 1500 || (defined __GNUC___) @@ -54,16 +55,23 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # else # define fbx_unordered_map map # define fbx_unordered_multimap multimap +# define fbx_unordered_set set +# define fbx_unordered_multiset multiset #endif #ifdef ASSIMP_FBX_USE_UNORDERED_MULTIMAP # include <unordered_map> +# include <unordered_set> # if _MSC_VER > 1600 # define fbx_unordered_map unordered_map # define fbx_unordered_multimap unordered_multimap +# define fbx_unordered_set unordered_set +# define fbx_unordered_multiset unordered_multiset # else # define fbx_unordered_map tr1::unordered_map # define fbx_unordered_multimap tr1::unordered_multimap +# define fbx_unordered_set tr1::unordered_set +# define fbx_unordered_multiset tr1::unordered_multiset # endif #endif diff --git a/thirdparty/assimp/code/FBX/FBXConverter.cpp b/thirdparty/assimp/code/FBX/FBXConverter.cpp index 3f64016ea4..d8a22d9f74 100644 --- a/thirdparty/assimp/code/FBX/FBXConverter.cpp +++ b/thirdparty/assimp/code/FBX/FBXConverter.cpp @@ -55,6 +55,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "FBXImporter.h" #include <assimp/StringComparison.h> +#include <assimp/MathFunctions.h> #include <assimp/scene.h> @@ -67,7 +68,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <sstream> #include <iomanip> #include <cstdint> - +#include <iostream> +#include <stdlib.h> namespace Assimp { namespace FBX { @@ -76,7 +78,7 @@ namespace Assimp { #define MAGIC_NODE_TAG "_$AssimpFbx$" -#define CONVERT_FBX_TIME(time) static_cast<double>(time) / 46186158000L +#define CONVERT_FBX_TIME(time) static_cast<double>(time) / 46186158000LL FBXConverter::FBXConverter(aiScene* out, const Document& doc, bool removeEmptyBones ) : defaultMaterialIndex() @@ -95,6 +97,14 @@ namespace Assimp { // populate the node_anim_chain_bits map, which is needed // to determine which nodes need to be generated. ConvertAnimations(); + // Embedded textures in FBX could be connected to nothing but to itself, + // for instance Texture -> Video connection only but not to the main graph, + // The idea here is to traverse all objects to find these Textures and convert them, + // so later during material conversion it will find converted texture in the textures_converted array. + if (doc.Settings().readTextures) + { + ConvertOrphantEmbeddedTextures(); + } ConvertRootNode(); if (doc.Settings().readAllMaterials) { @@ -144,7 +154,7 @@ namespace Assimp { out->mRootNode->mName.Set(unique_name); // root has ID 0 - ConvertNodes(0L, *out->mRootNode); + ConvertNodes(0L, out->mRootNode, out->mRootNode); } static std::string getAncestorBaseName(const aiNode* node) @@ -178,8 +188,11 @@ namespace Assimp { GetUniqueName(original_name, unique_name); return unique_name; } - - void FBXConverter::ConvertNodes(uint64_t id, aiNode& parent, const aiMatrix4x4& parent_transform) { + /// todo: pre-build node hierarchy + /// todo: get bone from stack + /// todo: make map of aiBone* to aiNode* + /// then update convert clusters to the new format + void FBXConverter::ConvertNodes(uint64_t id, aiNode *parent, aiNode *root_node) { const std::vector<const Connection*>& conns = doc.GetConnectionsByDestinationSequenced(id, "Model"); std::vector<aiNode*> nodes; @@ -190,62 +203,69 @@ namespace Assimp { try { for (const Connection* con : conns) { - // ignore object-property links if (con->PropertyName().length()) { - continue; + // really important we document why this is ignored. + FBXImporter::LogInfo("ignoring property link - no docs on why this is ignored"); + continue; //? } + // convert connection source object into Object base class const Object* const object = con->SourceObject(); if (nullptr == object) { - FBXImporter::LogWarn("failed to convert source object for Model link"); + FBXImporter::LogError("failed to convert source object for Model link"); continue; } + // FBX Model::Cube, Model::Bone001, etc elements + // This detects if we can cast the object into this model structure. const Model* const model = dynamic_cast<const Model*>(object); if (nullptr != model) { nodes_chain.clear(); post_nodes_chain.clear(); - aiMatrix4x4 new_abs_transform = parent_transform; - - std::string unique_name = MakeUniqueNodeName(model, parent); - + aiMatrix4x4 new_abs_transform = parent->mTransformation; + std::string node_name = FixNodeName(model->Name()); // even though there is only a single input node, the design of // assimp (or rather: the complicated transformation chain that // is employed by fbx) means that we may need multiple aiNode's // to represent a fbx node's transformation. - const bool need_additional_node = GenerateTransformationNodeChain(*model, unique_name, nodes_chain, post_nodes_chain); + + // generate node transforms - this includes pivot data + // if need_additional_node is true then you t + const bool need_additional_node = GenerateTransformationNodeChain(*model, node_name, nodes_chain, post_nodes_chain); + + // assert that for the current node we must have at least a single transform ai_assert(nodes_chain.size()); if (need_additional_node) { - nodes_chain.push_back(new aiNode(unique_name)); + nodes_chain.push_back(new aiNode(node_name)); } //setup metadata on newest node SetupNodeMetadata(*model, *nodes_chain.back()); // link all nodes in a row - aiNode* last_parent = &parent; - for (aiNode* prenode : nodes_chain) { - ai_assert(prenode); + aiNode* last_parent = parent; + for (aiNode* child : nodes_chain) { + ai_assert(child); - if (last_parent != &parent) { + if (last_parent != parent) { last_parent->mNumChildren = 1; last_parent->mChildren = new aiNode*[1]; - last_parent->mChildren[0] = prenode; + last_parent->mChildren[0] = child; } - prenode->mParent = last_parent; - last_parent = prenode; + child->mParent = last_parent; + last_parent = child; - new_abs_transform *= prenode->mTransformation; + new_abs_transform *= child->mTransformation; } // attach geometry - ConvertModel(*model, *nodes_chain.back(), new_abs_transform); + ConvertModel(*model, nodes_chain.back(), root_node, new_abs_transform); // check if there will be any child nodes const std::vector<const Connection*>& child_conns @@ -257,7 +277,7 @@ namespace Assimp { for (aiNode* postnode : post_nodes_chain) { ai_assert(postnode); - if (last_parent != &parent) { + if (last_parent != parent) { last_parent->mNumChildren = 1; last_parent->mChildren = new aiNode*[1]; last_parent->mChildren[0] = postnode; @@ -279,15 +299,15 @@ namespace Assimp { ); } - // attach sub-nodes (if any) - ConvertNodes(model->ID(), *last_parent, new_abs_transform); + // recursion call - child nodes + ConvertNodes(model->ID(), last_parent, root_node); if (doc.Settings().readLights) { - ConvertLights(*model, unique_name); + ConvertLights(*model, node_name); } if (doc.Settings().readCameras) { - ConvertCameras(*model, unique_name); + ConvertCameras(*model, node_name); } nodes.push_back(nodes_chain.front()); @@ -296,11 +316,17 @@ namespace Assimp { } if (nodes.size()) { - parent.mChildren = new aiNode*[nodes.size()](); - parent.mNumChildren = static_cast<unsigned int>(nodes.size()); + parent->mChildren = new aiNode*[nodes.size()](); + parent->mNumChildren = static_cast<unsigned int>(nodes.size()); - std::swap_ranges(nodes.begin(), nodes.end(), parent.mChildren); + std::swap_ranges(nodes.begin(), nodes.end(), parent->mChildren); } + else + { + parent->mNumChildren = 0; + parent->mChildren = nullptr; + } + } catch (std::exception&) { Util::delete_fun<aiNode> deleter; @@ -553,7 +579,7 @@ namespace Assimp { return; } - const float angle_epsilon = 1e-6f; + const float angle_epsilon = Math::getEpsilon<float>(); out = aiMatrix4x4(); @@ -694,7 +720,7 @@ namespace Assimp { std::fill_n(chain, static_cast<unsigned int>(TransformationComp_MAXIMUM), aiMatrix4x4()); // generate transformation matrices for all the different transformation components - const float zero_epsilon = 1e-6f; + const float zero_epsilon = Math::getEpsilon<float>(); const aiVector3D all_ones(1.0f, 1.0f, 1.0f); const aiVector3D& PreRotation = PropertyGet<aiVector3D>(props, "PreRotation", ok); @@ -802,7 +828,7 @@ namespace Assimp { // is_complex needs to be consistent with NeedsComplexTransformationChain() // or the interplay between this code and the animation converter would // not be guaranteed. - ai_assert(NeedsComplexTransformationChain(model) == ((chainBits & chainMaskComplex) != 0)); + //ai_assert(NeedsComplexTransformationChain(model) == ((chainBits & chainMaskComplex) != 0)); // now, if we have more than just Translation, Scaling and Rotation, // we need to generate a full node chain to accommodate for assimp's @@ -904,7 +930,8 @@ namespace Assimp { } } - void FBXConverter::ConvertModel(const Model& model, aiNode& nd, const aiMatrix4x4& node_global_transform) + void FBXConverter::ConvertModel(const Model &model, aiNode *parent, aiNode *root_node, + const aiMatrix4x4 &absolute_transform) { const std::vector<const Geometry*>& geos = model.GetGeometry(); @@ -916,11 +943,12 @@ namespace Assimp { const MeshGeometry* const mesh = dynamic_cast<const MeshGeometry*>(geo); const LineGeometry* const line = dynamic_cast<const LineGeometry*>(geo); if (mesh) { - const std::vector<unsigned int>& indices = ConvertMesh(*mesh, model, node_global_transform, nd); + const std::vector<unsigned int>& indices = ConvertMesh(*mesh, model, parent, root_node, + absolute_transform); std::copy(indices.begin(), indices.end(), std::back_inserter(meshes)); } else if (line) { - const std::vector<unsigned int>& indices = ConvertLine(*line, model, node_global_transform, nd); + const std::vector<unsigned int>& indices = ConvertLine(*line, model, parent, root_node); std::copy(indices.begin(), indices.end(), std::back_inserter(meshes)); } else { @@ -929,15 +957,16 @@ namespace Assimp { } if (meshes.size()) { - nd.mMeshes = new unsigned int[meshes.size()](); - nd.mNumMeshes = static_cast<unsigned int>(meshes.size()); + parent->mMeshes = new unsigned int[meshes.size()](); + parent->mNumMeshes = static_cast<unsigned int>(meshes.size()); - std::swap_ranges(meshes.begin(), meshes.end(), nd.mMeshes); + std::swap_ranges(meshes.begin(), meshes.end(), parent->mMeshes); } } - std::vector<unsigned int> FBXConverter::ConvertMesh(const MeshGeometry& mesh, const Model& model, - const aiMatrix4x4& node_global_transform, aiNode& nd) + std::vector<unsigned int> + FBXConverter::ConvertMesh(const MeshGeometry &mesh, const Model &model, aiNode *parent, aiNode *root_node, + const aiMatrix4x4 &absolute_transform) { std::vector<unsigned int> temp; @@ -961,18 +990,18 @@ namespace Assimp { const MatIndexArray::value_type base = mindices[0]; for (MatIndexArray::value_type index : mindices) { if (index != base) { - return ConvertMeshMultiMaterial(mesh, model, node_global_transform, nd); + return ConvertMeshMultiMaterial(mesh, model, parent, root_node, absolute_transform); } } } // faster code-path, just copy the data - temp.push_back(ConvertMeshSingleMaterial(mesh, model, node_global_transform, nd)); + temp.push_back(ConvertMeshSingleMaterial(mesh, model, absolute_transform, parent, root_node)); return temp; } std::vector<unsigned int> FBXConverter::ConvertLine(const LineGeometry& line, const Model& model, - const aiMatrix4x4& node_global_transform, aiNode& nd) + aiNode *parent, aiNode *root_node) { std::vector<unsigned int> temp; @@ -983,7 +1012,7 @@ namespace Assimp { return temp; } - aiMesh* const out_mesh = SetupEmptyMesh(line, nd); + aiMesh* const out_mesh = SetupEmptyMesh(line, root_node); out_mesh->mPrimitiveTypes |= aiPrimitiveType_LINE; // copy vertices @@ -1018,7 +1047,7 @@ namespace Assimp { return temp; } - aiMesh* FBXConverter::SetupEmptyMesh(const Geometry& mesh, aiNode& nd) + aiMesh* FBXConverter::SetupEmptyMesh(const Geometry& mesh, aiNode *parent) { aiMesh* const out_mesh = new aiMesh(); meshes.push_back(out_mesh); @@ -1035,17 +1064,18 @@ namespace Assimp { } else { - out_mesh->mName = nd.mName; + out_mesh->mName = parent->mName; } return out_mesh; } - unsigned int FBXConverter::ConvertMeshSingleMaterial(const MeshGeometry& mesh, const Model& model, - const aiMatrix4x4& node_global_transform, aiNode& nd) + unsigned int FBXConverter::ConvertMeshSingleMaterial(const MeshGeometry &mesh, const Model &model, + const aiMatrix4x4 &absolute_transform, aiNode *parent, + aiNode *root_node) { const MatIndexArray& mindices = mesh.GetMaterialIndices(); - aiMesh* const out_mesh = SetupEmptyMesh(mesh, nd); + aiMesh* const out_mesh = SetupEmptyMesh(mesh, parent); const std::vector<aiVector3D>& vertices = mesh.GetVertices(); const std::vector<unsigned int>& faces = mesh.GetFaceIndexCounts(); @@ -1112,7 +1142,7 @@ namespace Assimp { binormals = &tempBinormals; } else { - binormals = NULL; + binormals = nullptr; } } @@ -1162,8 +1192,9 @@ namespace Assimp { ConvertMaterialForMesh(out_mesh, model, mesh, mindices[0]); } - if (doc.Settings().readWeights && mesh.DeformerSkin() != NULL) { - ConvertWeights(out_mesh, model, mesh, node_global_transform, NO_MATERIAL_SEPARATION); + if (doc.Settings().readWeights && mesh.DeformerSkin() != nullptr) { + ConvertWeights(out_mesh, model, mesh, absolute_transform, parent, root_node, NO_MATERIAL_SEPARATION, + nullptr); } std::vector<aiAnimMesh*> animMeshes; @@ -1208,8 +1239,10 @@ namespace Assimp { return static_cast<unsigned int>(meshes.size() - 1); } - std::vector<unsigned int> FBXConverter::ConvertMeshMultiMaterial(const MeshGeometry& mesh, const Model& model, - const aiMatrix4x4& node_global_transform, aiNode& nd) + std::vector<unsigned int> + FBXConverter::ConvertMeshMultiMaterial(const MeshGeometry &mesh, const Model &model, aiNode *parent, + aiNode *root_node, + const aiMatrix4x4 &absolute_transform) { const MatIndexArray& mindices = mesh.GetMaterialIndices(); ai_assert(mindices.size()); @@ -1220,7 +1253,7 @@ namespace Assimp { for (MatIndexArray::value_type index : mindices) { if (had.find(index) == had.end()) { - indices.push_back(ConvertMeshMultiMaterial(mesh, model, index, node_global_transform, nd)); + indices.push_back(ConvertMeshMultiMaterial(mesh, model, index, parent, root_node, absolute_transform)); had.insert(index); } } @@ -1228,18 +1261,18 @@ namespace Assimp { return indices; } - unsigned int FBXConverter::ConvertMeshMultiMaterial(const MeshGeometry& mesh, const Model& model, - MatIndexArray::value_type index, - const aiMatrix4x4& node_global_transform, - aiNode& nd) + unsigned int FBXConverter::ConvertMeshMultiMaterial(const MeshGeometry &mesh, const Model &model, + MatIndexArray::value_type index, + aiNode *parent, aiNode *root_node, + const aiMatrix4x4 &absolute_transform) { - aiMesh* const out_mesh = SetupEmptyMesh(mesh, nd); + aiMesh* const out_mesh = SetupEmptyMesh(mesh, parent); const MatIndexArray& mindices = mesh.GetMaterialIndices(); const std::vector<aiVector3D>& vertices = mesh.GetVertices(); const std::vector<unsigned int>& faces = mesh.GetFaceIndexCounts(); - const bool process_weights = doc.Settings().readWeights && mesh.DeformerSkin() != NULL; + const bool process_weights = doc.Settings().readWeights && mesh.DeformerSkin() != nullptr; unsigned int count_faces = 0; unsigned int count_vertices = 0; @@ -1299,7 +1332,7 @@ namespace Assimp { binormals = &tempBinormals; } else { - binormals = NULL; + binormals = nullptr; } } @@ -1398,7 +1431,7 @@ namespace Assimp { ConvertMaterialForMesh(out_mesh, model, mesh, index); if (process_weights) { - ConvertWeights(out_mesh, model, mesh, node_global_transform, index, &reverseMapping); + ConvertWeights(out_mesh, model, mesh, absolute_transform, parent, root_node, index, &reverseMapping); } std::vector<aiAnimMesh*> animMeshes; @@ -1448,10 +1481,10 @@ namespace Assimp { return static_cast<unsigned int>(meshes.size() - 1); } - void FBXConverter::ConvertWeights(aiMesh* out, const Model& model, const MeshGeometry& geo, - const aiMatrix4x4& node_global_transform, - unsigned int materialIndex, - std::vector<unsigned int>* outputVertStartIndices) + void FBXConverter::ConvertWeights(aiMesh *out, const Model &model, const MeshGeometry &geo, + const aiMatrix4x4 &absolute_transform, + aiNode *parent, aiNode *root_node, unsigned int materialIndex, + std::vector<unsigned int> *outputVertStartIndices) { ai_assert(geo.DeformerSkin()); @@ -1462,13 +1495,12 @@ namespace Assimp { const Skin& sk = *geo.DeformerSkin(); std::vector<aiBone*> bones; - bones.reserve(sk.Clusters().size()); const bool no_mat_check = materialIndex == NO_MATERIAL_SEPARATION; ai_assert(no_mat_check || outputVertStartIndices); try { - + // iterate over the sub deformers for (const Cluster* cluster : sk.Clusters()) { ai_assert(cluster); @@ -1482,15 +1514,16 @@ namespace Assimp { index_out_indices.clear(); out_indices.clear(); + // now check if *any* of these weights is contained in the output mesh, // taking notes so we don't need to do it twice. for (WeightIndexArray::value_type index : indices) { unsigned int count = 0; const unsigned int* const out_idx = geo.ToOutputVertexIndex(index, count); - // ToOutputVertexIndex only returns NULL if index is out of bounds + // ToOutputVertexIndex only returns nullptr if index is out of bounds // which should never happen - ai_assert(out_idx != NULL); + ai_assert(out_idx != nullptr); index_out_indices.push_back(no_index_sentinel); count_out_indices.push_back(0); @@ -1519,68 +1552,107 @@ namespace Assimp { } } } - + // if we found at least one, generate the output bones // XXX this could be heavily simplified by collecting the bone // data in a single step. - ConvertCluster(bones, model, *cluster, out_indices, index_out_indices, - count_out_indices, node_global_transform); + ConvertCluster(bones, cluster, out_indices, index_out_indices, + count_out_indices, absolute_transform, parent, root_node); } + + bone_map.clear(); } - catch (std::exception&) { + catch (std::exception&e) { std::for_each(bones.begin(), bones.end(), Util::delete_fun<aiBone>()); throw; } if (bones.empty()) { + out->mBones = nullptr; + out->mNumBones = 0; return; - } - - out->mBones = new aiBone*[bones.size()](); - out->mNumBones = static_cast<unsigned int>(bones.size()); + } else { + out->mBones = new aiBone *[bones.size()](); + out->mNumBones = static_cast<unsigned int>(bones.size()); - std::swap_ranges(bones.begin(), bones.end(), out->mBones); + std::swap_ranges(bones.begin(), bones.end(), out->mBones); + } } - void FBXConverter::ConvertCluster(std::vector<aiBone*>& bones, const Model& /*model*/, const Cluster& cl, - std::vector<size_t>& out_indices, - std::vector<size_t>& index_out_indices, - std::vector<size_t>& count_out_indices, - const aiMatrix4x4& node_global_transform) + const aiNode* FBXConverter::GetNodeByName( const aiString& name, aiNode *current_node ) { + aiNode * iter = current_node; + //printf("Child count: %d", iter->mNumChildren); + return iter; + } - aiBone* const bone = new aiBone(); - bones.push_back(bone); + void FBXConverter::ConvertCluster(std::vector<aiBone *> &local_mesh_bones, const Cluster *cl, + std::vector<size_t> &out_indices, std::vector<size_t> &index_out_indices, + std::vector<size_t> &count_out_indices, const aiMatrix4x4 &absolute_transform, + aiNode *parent, aiNode *root_node) { + ai_assert(cl); // make sure cluster valid + std::string deformer_name = cl->TargetNode()->Name(); + aiString bone_name = aiString(FixNodeName(deformer_name)); - bone->mName = FixNodeName(cl.TargetNode()->Name()); + aiBone *bone = nullptr; - bone->mOffsetMatrix = cl.TransformLink(); - bone->mOffsetMatrix.Inverse(); + if (bone_map.count(deformer_name)) { + std::cout << "retrieved bone from lookup " << bone_name.C_Str() << ". Deformer: " << deformer_name + << std::endl; + bone = bone_map[deformer_name]; + } else { + std::cout << "created new bone " << bone_name.C_Str() << ". Deformer: " << deformer_name << std::endl; + bone = new aiBone(); + bone->mName = bone_name; - bone->mOffsetMatrix = bone->mOffsetMatrix * node_global_transform; + // store local transform link for post processing + bone->mOffsetMatrix = cl->TransformLink(); + bone->mOffsetMatrix.Inverse(); - bone->mNumWeights = static_cast<unsigned int>(out_indices.size()); - aiVertexWeight* cursor = bone->mWeights = new aiVertexWeight[out_indices.size()]; + aiMatrix4x4 matrix = (aiMatrix4x4)absolute_transform; - const size_t no_index_sentinel = std::numeric_limits<size_t>::max(); - const WeightArray& weights = cl.GetWeights(); + bone->mOffsetMatrix = bone->mOffsetMatrix * matrix; // * mesh_offset - const size_t c = index_out_indices.size(); - for (size_t i = 0; i < c; ++i) { - const size_t index_index = index_out_indices[i]; - if (index_index == no_index_sentinel) { - continue; - } + // + // Now calculate the aiVertexWeights + // + + aiVertexWeight *cursor = nullptr; + + bone->mNumWeights = static_cast<unsigned int>(out_indices.size()); + cursor = bone->mWeights = new aiVertexWeight[out_indices.size()]; - const size_t cc = count_out_indices[i]; - for (size_t j = 0; j < cc; ++j) { - aiVertexWeight& out_weight = *cursor++; + const size_t no_index_sentinel = std::numeric_limits<size_t>::max(); + const WeightArray& weights = cl->GetWeights(); - out_weight.mVertexId = static_cast<unsigned int>(out_indices[index_index + j]); - out_weight.mWeight = weights[i]; + const size_t c = index_out_indices.size(); + for (size_t i = 0; i < c; ++i) { + const size_t index_index = index_out_indices[i]; + + if (index_index == no_index_sentinel) { + continue; + } + + const size_t cc = count_out_indices[i]; + for (size_t j = 0; j < cc; ++j) { + // cursor runs from first element relative to the start + // or relative to the start of the next indexes. + aiVertexWeight& out_weight = *cursor++; + + out_weight.mVertexId = static_cast<unsigned int>(out_indices[index_index + j]); + out_weight.mWeight = weights[i]; + } } + + bone_map.insert(std::pair<const std::string, aiBone *>(deformer_name, bone)); } + + std::cout << "bone research: Indicies size: " << out_indices.size() << std::endl; + + // lookup must be populated in case something goes wrong + // this also allocates bones to mesh instance outside + local_mesh_bones.push_back(bone); } void FBXConverter::ConvertMaterialForMesh(aiMesh* out, const Model& model, const MeshGeometry& geo, @@ -1710,7 +1782,7 @@ namespace Assimp { bool textureReady = false; //tells if our texture is ready (if it was loaded or if it was found) unsigned int index; - VideoMap::const_iterator it = textures_converted.find(media); + VideoMap::const_iterator it = textures_converted.find(*media); if (it != textures_converted.end()) { index = (*it).second; textureReady = true; @@ -1718,7 +1790,7 @@ namespace Assimp { else { if (media->ContentLength() > 0) { index = ConvertVideo(*media); - textures_converted[media] = index; + textures_converted[*media] = index; textureReady = true; } } @@ -2242,13 +2314,13 @@ void FBXConverter::SetShadingPropertiesRaw(aiMaterial* out_mat, const PropertyTa if (media != nullptr && media->ContentLength() > 0) { unsigned int index; - VideoMap::const_iterator it = textures_converted.find(media); + VideoMap::const_iterator it = textures_converted.find(*media); if (it != textures_converted.end()) { index = (*it).second; } else { index = ConvertVideo(*media); - textures_converted[media] = index; + textures_converted[*media] = index; } // setup texture reference string (copied from ColladaLoader::FindFilenameForEffectTexture) @@ -2676,7 +2748,7 @@ void FBXConverter::SetShadingPropertiesRaw(aiMaterial* out_mat, const PropertyTa // sanity check whether the input is ok static void validateAnimCurveNodes(const std::vector<const AnimationCurveNode*>& curves, bool strictMode) { - const Object* target(NULL); + const Object* target(nullptr); for (const AnimationCurveNode* node : curves) { if (!target) { target = node->Target(); @@ -2707,7 +2779,7 @@ void FBXConverter::SetShadingPropertiesRaw(aiMaterial* out_mat, const PropertyTa #ifdef ASSIMP_BUILD_DEBUG validateAnimCurveNodes(curves, doc.Settings().strictMode); #endif - const AnimationCurveNode* curve_node = NULL; + const AnimationCurveNode* curve_node = nullptr; for (const AnimationCurveNode* node : curves) { ai_assert(node); @@ -2967,7 +3039,7 @@ void FBXConverter::SetShadingPropertiesRaw(aiMaterial* out_mat, const PropertyTa TransformationCompDefaultValue(comp) ); - const float epsilon = 1e-6f; + const float epsilon = Math::getEpsilon<float>(); return (dyn_val - static_val).SquareLength() < epsilon; } @@ -3555,7 +3627,7 @@ void FBXConverter::SetShadingPropertiesRaw(aiMaterial* out_mat, const PropertyTa ai_assert(!out->mMeshes); ai_assert(!out->mNumMeshes); - // note: the trailing () ensures initialization with NULL - not + // note: the trailing () ensures initialization with nullptr - not // many C++ users seem to know this, so pointing it out to avoid // confusion why this code works. @@ -3602,6 +3674,47 @@ void FBXConverter::SetShadingPropertiesRaw(aiMaterial* out_mat, const PropertyTa } } + void FBXConverter::ConvertOrphantEmbeddedTextures() + { + // in C++14 it could be: + // for (auto&& [id, object] : objects) + for (auto&& id_and_object : doc.Objects()) + { + auto&& id = std::get<0>(id_and_object); + auto&& object = std::get<1>(id_and_object); + // If an object doesn't have parent + if (doc.ConnectionsBySource().count(id) == 0) + { + const Texture* realTexture = nullptr; + try + { + const auto& element = object->GetElement(); + const Token& key = element.KeyToken(); + const char* obtype = key.begin(); + const size_t length = static_cast<size_t>(key.end() - key.begin()); + if (strncmp(obtype, "Texture", length) == 0) + { + const Texture* texture = static_cast<const Texture*>(object->Get()); + if (texture->Media() && texture->Media()->ContentLength() > 0) + { + realTexture = texture; + } + } + } + catch (...) + { + // do nothing + } + if (realTexture) + { + const Video* media = realTexture->Media(); + unsigned int index = ConvertVideo(*media); + textures_converted[*media] = index; + } + } + } + } + // ------------------------------------------------------------------------------------------------ void ConvertToAssimpScene(aiScene* out, const Document& doc, bool removeEmptyBones) { diff --git a/thirdparty/assimp/code/FBX/FBXConverter.h b/thirdparty/assimp/code/FBX/FBXConverter.h index ab610058a4..46693bdca6 100644 --- a/thirdparty/assimp/code/FBX/FBXConverter.h +++ b/thirdparty/assimp/code/FBX/FBXConverter.h @@ -76,16 +76,6 @@ namespace Assimp { namespace FBX { class Document; - -enum class FbxUnit { - cm = 0, - m, - km, - NumUnits, - - Undefined -}; - /** * Convert a FBX #Document to #aiScene * @param out Empty scene to be populated @@ -133,7 +123,7 @@ private: // ------------------------------------------------------------------------------------------------ // collect and assign child nodes - void ConvertNodes(uint64_t id, aiNode& parent, const aiMatrix4x4& parent_transform = aiMatrix4x4()); + void ConvertNodes(uint64_t id, aiNode *parent, aiNode *root_node); // ------------------------------------------------------------------------------------------------ void ConvertLights(const Model& model, const std::string &orig_name ); @@ -189,32 +179,35 @@ private: void SetupNodeMetadata(const Model& model, aiNode& nd); // ------------------------------------------------------------------------------------------------ - void ConvertModel(const Model& model, aiNode& nd, const aiMatrix4x4& node_global_transform); + void ConvertModel(const Model &model, aiNode *parent, aiNode *root_node, + const aiMatrix4x4 &absolute_transform); // ------------------------------------------------------------------------------------------------ // MeshGeometry -> aiMesh, return mesh index + 1 or 0 if the conversion failed - std::vector<unsigned int> ConvertMesh(const MeshGeometry& mesh, const Model& model, - const aiMatrix4x4& node_global_transform, aiNode& nd); + std::vector<unsigned int> + ConvertMesh(const MeshGeometry &mesh, const Model &model, aiNode *parent, aiNode *root_node, + const aiMatrix4x4 &absolute_transform); // ------------------------------------------------------------------------------------------------ std::vector<unsigned int> ConvertLine(const LineGeometry& line, const Model& model, - const aiMatrix4x4& node_global_transform, aiNode& nd); + aiNode *parent, aiNode *root_node); // ------------------------------------------------------------------------------------------------ - aiMesh* SetupEmptyMesh(const Geometry& mesh, aiNode& nd); + aiMesh* SetupEmptyMesh(const Geometry& mesh, aiNode *parent); // ------------------------------------------------------------------------------------------------ - unsigned int ConvertMeshSingleMaterial(const MeshGeometry& mesh, const Model& model, - const aiMatrix4x4& node_global_transform, aiNode& nd); + unsigned int ConvertMeshSingleMaterial(const MeshGeometry &mesh, const Model &model, + const aiMatrix4x4 &absolute_transform, aiNode *parent, + aiNode *root_node); // ------------------------------------------------------------------------------------------------ - std::vector<unsigned int> ConvertMeshMultiMaterial(const MeshGeometry& mesh, const Model& model, - const aiMatrix4x4& node_global_transform, aiNode& nd); + std::vector<unsigned int> + ConvertMeshMultiMaterial(const MeshGeometry &mesh, const Model &model, aiNode *parent, aiNode *root_node, + const aiMatrix4x4 &absolute_transform); // ------------------------------------------------------------------------------------------------ - unsigned int ConvertMeshMultiMaterial(const MeshGeometry& mesh, const Model& model, - MatIndexArray::value_type index, - const aiMatrix4x4& node_global_transform, aiNode& nd); + unsigned int ConvertMeshMultiMaterial(const MeshGeometry &mesh, const Model &model, MatIndexArray::value_type index, + aiNode *parent, aiNode *root_node, const aiMatrix4x4 &absolute_transform); // ------------------------------------------------------------------------------------------------ static const unsigned int NO_MATERIAL_SEPARATION = /* std::numeric_limits<unsigned int>::max() */ @@ -227,17 +220,17 @@ private: * - outputVertStartIndices is only used when a material index is specified, it gives for * each output vertex the DOM index it maps to. */ - void ConvertWeights(aiMesh* out, const Model& model, const MeshGeometry& geo, - const aiMatrix4x4& node_global_transform = aiMatrix4x4(), - unsigned int materialIndex = NO_MATERIAL_SEPARATION, - std::vector<unsigned int>* outputVertStartIndices = NULL); - + void ConvertWeights(aiMesh *out, const Model &model, const MeshGeometry &geo, const aiMatrix4x4 &absolute_transform, + aiNode *parent = NULL, aiNode *root_node = NULL, + unsigned int materialIndex = NO_MATERIAL_SEPARATION, + std::vector<unsigned int> *outputVertStartIndices = NULL); + // lookup + static const aiNode* GetNodeByName( const aiString& name, aiNode *current_node ); // ------------------------------------------------------------------------------------------------ - void ConvertCluster(std::vector<aiBone*>& bones, const Model& /*model*/, const Cluster& cl, - std::vector<size_t>& out_indices, - std::vector<size_t>& index_out_indices, - std::vector<size_t>& count_out_indices, - const aiMatrix4x4& node_global_transform); + void ConvertCluster(std::vector<aiBone *> &local_mesh_bones, const Cluster *cl, + std::vector<size_t> &out_indices, std::vector<size_t> &index_out_indices, + std::vector<size_t> &count_out_indices, const aiMatrix4x4 &absolute_transform, + aiNode *parent, aiNode *root_node); // ------------------------------------------------------------------------------------------------ void ConvertMaterialForMesh(aiMesh* out, const Model& model, const MeshGeometry& geo, @@ -434,6 +427,10 @@ private: // copy generated meshes, animations, lights, cameras and textures to the output scene void TransferDataToScene(); + // ------------------------------------------------------------------------------------------------ + // FBX file could have embedded textures not connected to anything + void ConvertOrphantEmbeddedTextures(); + private: // 0: not assigned yet, others: index is value - 1 unsigned int defaultMaterialIndex; @@ -445,28 +442,47 @@ private: std::vector<aiCamera*> cameras; std::vector<aiTexture*> textures; - using MaterialMap = std::map<const Material*, unsigned int>; + using MaterialMap = std::fbx_unordered_map<const Material*, unsigned int>; MaterialMap materials_converted; - using VideoMap = std::map<const Video*, unsigned int>; + using VideoMap = std::fbx_unordered_map<const Video, unsigned int>; VideoMap textures_converted; - using MeshMap = std::map<const Geometry*, std::vector<unsigned int> >; + using MeshMap = std::fbx_unordered_map<const Geometry*, std::vector<unsigned int> >; MeshMap meshes_converted; // fixed node name -> which trafo chain components have animations? - using NodeAnimBitMap = std::map<std::string, unsigned int> ; + using NodeAnimBitMap = std::fbx_unordered_map<std::string, unsigned int> ; NodeAnimBitMap node_anim_chain_bits; // number of nodes with the same name - using NodeNameCache = std::unordered_map<std::string, unsigned int>; + using NodeNameCache = std::fbx_unordered_map<std::string, unsigned int>; NodeNameCache mNodeNames; + // Deformer name is not the same as a bone name - it does contain the bone name though :) + // Deformer names in FBX are always unique in an FBX file. + std::map<const std::string, aiBone *> bone_map; + double anim_fps; aiScene* const out; const FBX::Document& doc; - FbxUnit mCurrentUnit; + + static void BuildBoneList(aiNode *current_node, const aiNode *root_node, const aiScene *scene, + std::vector<aiBone*>& bones); + + void BuildBoneStack(aiNode *current_node, const aiNode *root_node, const aiScene *scene, + const std::vector<aiBone *> &bones, + std::map<aiBone *, aiNode *> &bone_stack, + std::vector<aiNode*> &node_stack ); + + static void BuildNodeList(aiNode *current_node, std::vector<aiNode *> &nodes); + + static aiNode *GetNodeFromStack(const aiString &node_name, std::vector<aiNode *> &nodes); + + static aiNode *GetArmatureRoot(aiNode *bone_node, std::vector<aiBone*> &bone_list); + + static bool IsBoneNode(const aiString &bone_name, std::vector<aiBone *> &bones); }; } diff --git a/thirdparty/assimp/code/FBX/FBXDocument.h b/thirdparty/assimp/code/FBX/FBXDocument.h index 18e5c38f13..a60d7d9efa 100644 --- a/thirdparty/assimp/code/FBX/FBXDocument.h +++ b/thirdparty/assimp/code/FBX/FBXDocument.h @@ -637,6 +637,20 @@ public: return ptr; } + bool operator==(const Video& other) const + { + return ( + type == other.type + && relativeFileName == other.relativeFileName + && fileName == other.fileName + ); + } + + bool operator<(const Video& other) const + { + return std::tie(type, relativeFileName, fileName) < std::tie(other.type, other.relativeFileName, other.fileName); + } + private: std::string type; std::string relativeFileName; @@ -1005,10 +1019,10 @@ public: // during their entire lifetime (Document). FBX files have // up to many thousands of objects (most of which we never use), // so the memory overhead for them should be kept at a minimum. -typedef std::map<uint64_t, LazyObject*> ObjectMap; +typedef std::fbx_unordered_map<uint64_t, LazyObject*> ObjectMap; typedef std::fbx_unordered_map<std::string, std::shared_ptr<const PropertyTable> > PropertyTemplateMap; -typedef std::multimap<uint64_t, const Connection*> ConnectionMap; +typedef std::fbx_unordered_multimap<uint64_t, const Connection*> ConnectionMap; /** DOM class for global document settings, a single instance per document can * be accessed via Document.Globals(). */ @@ -1177,4 +1191,25 @@ private: } // Namespace FBX } // Namespace Assimp +namespace std +{ + template <> + struct hash<const Assimp::FBX::Video> + { + std::size_t operator()(const Assimp::FBX::Video& video) const + { + using std::size_t; + using std::hash; + using std::string; + + size_t res = 17; + res = res * 31 + hash<string>()(video.Name()); + res = res * 31 + hash<string>()(video.RelativeFilename()); + res = res * 31 + hash<string>()(video.Type()); + + return res; + } + }; +} + #endif // INCLUDED_AI_FBX_DOCUMENT_H diff --git a/thirdparty/assimp/code/FBX/FBXExportProperty.cpp b/thirdparty/assimp/code/FBX/FBXExportProperty.cpp index f8593e6295..f2a63b72b9 100644 --- a/thirdparty/assimp/code/FBX/FBXExportProperty.cpp +++ b/thirdparty/assimp/code/FBX/FBXExportProperty.cpp @@ -59,11 +59,7 @@ namespace FBX { FBXExportProperty::FBXExportProperty(bool v) : type('C') -, data(1) { - data = { - uint8_t(v) - }; -} +, data(1, uint8_t(v)) {} FBXExportProperty::FBXExportProperty(int16_t v) : type('Y') diff --git a/thirdparty/assimp/code/FBX/FBXExporter.cpp b/thirdparty/assimp/code/FBX/FBXExporter.cpp index 8ebc8555a2..9316dc4f02 100644 --- a/thirdparty/assimp/code/FBX/FBXExporter.cpp +++ b/thirdparty/assimp/code/FBX/FBXExporter.cpp @@ -67,6 +67,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <vector> #include <array> #include <unordered_set> +#include <numeric> // RESOURCES: // https://code.blender.org/2013/08/fbx-binary-file-format-specification/ @@ -1005,6 +1006,9 @@ void FBXExporter::WriteObjects () object_node.EndProperties(outstream, binary, indent); object_node.BeginChildren(outstream, binary, indent); + bool bJoinIdenticalVertices = mProperties->GetPropertyBool("bJoinIdenticalVertices", true); + std::vector<std::vector<int32_t>> vVertexIndice;//save vertex_indices as it is needed later + // geometry (aiMesh) mesh_uids.clear(); indent = 1; @@ -1031,21 +1035,35 @@ void FBXExporter::WriteObjects () std::vector<int32_t> vertex_indices; // map of vertex value to its index in the data vector std::map<aiVector3D,size_t> index_by_vertex_value; - int32_t index = 0; - for (size_t vi = 0; vi < m->mNumVertices; ++vi) { - aiVector3D vtx = m->mVertices[vi]; - auto elem = index_by_vertex_value.find(vtx); - if (elem == index_by_vertex_value.end()) { - vertex_indices.push_back(index); - index_by_vertex_value[vtx] = index; - flattened_vertices.push_back(vtx[0]); - flattened_vertices.push_back(vtx[1]); - flattened_vertices.push_back(vtx[2]); - ++index; - } else { - vertex_indices.push_back(int32_t(elem->second)); + if(bJoinIdenticalVertices){ + int32_t index = 0; + for (size_t vi = 0; vi < m->mNumVertices; ++vi) { + aiVector3D vtx = m->mVertices[vi]; + auto elem = index_by_vertex_value.find(vtx); + if (elem == index_by_vertex_value.end()) { + vertex_indices.push_back(index); + index_by_vertex_value[vtx] = index; + flattened_vertices.push_back(vtx[0]); + flattened_vertices.push_back(vtx[1]); + flattened_vertices.push_back(vtx[2]); + ++index; + } else { + vertex_indices.push_back(int32_t(elem->second)); + } + } + } + else { // do not join vertex, respect the export flag + vertex_indices.resize(m->mNumVertices); + std::iota(vertex_indices.begin(), vertex_indices.end(), 0); + for(unsigned int v = 0; v < m->mNumVertices; ++ v) { + aiVector3D vtx = m->mVertices[v]; + flattened_vertices.push_back(vtx.x); + flattened_vertices.push_back(vtx.y); + flattened_vertices.push_back(vtx.z); } } + vVertexIndice.push_back(vertex_indices); + FBX::Node::WritePropertyNode( "Vertices", flattened_vertices, outstream, binary, indent ); @@ -1116,6 +1134,51 @@ void FBXExporter::WriteObjects () normals.End(outstream, binary, indent, true); } + // colors, if any + // TODO only one color channel currently + const int32_t colorChannelIndex = 0; + if (m->HasVertexColors(colorChannelIndex)) { + FBX::Node vertexcolors("LayerElementColor", int32_t(colorChannelIndex)); + vertexcolors.Begin(outstream, binary, indent); + vertexcolors.DumpProperties(outstream, binary, indent); + vertexcolors.EndProperties(outstream, binary, indent); + vertexcolors.BeginChildren(outstream, binary, indent); + indent = 3; + FBX::Node::WritePropertyNode( + "Version", int32_t(101), outstream, binary, indent + ); + char layerName[8]; + sprintf(layerName, "COLOR_%d", colorChannelIndex); + FBX::Node::WritePropertyNode( + "Name", (const char*)layerName, outstream, binary, indent + ); + FBX::Node::WritePropertyNode( + "MappingInformationType", "ByPolygonVertex", + outstream, binary, indent + ); + FBX::Node::WritePropertyNode( + "ReferenceInformationType", "Direct", + outstream, binary, indent + ); + std::vector<double> color_data; + color_data.reserve(4 * polygon_data.size()); + for (size_t fi = 0; fi < m->mNumFaces; ++fi) { + const aiFace &f = m->mFaces[fi]; + for (size_t pvi = 0; pvi < f.mNumIndices; ++pvi) { + const aiColor4D &c = m->mColors[colorChannelIndex][f.mIndices[pvi]]; + color_data.push_back(c.r); + color_data.push_back(c.g); + color_data.push_back(c.b); + color_data.push_back(c.a); + } + } + FBX::Node::WritePropertyNode( + "Colors", color_data, outstream, binary, indent + ); + indent = 2; + vertexcolors.End(outstream, binary, indent, true); + } + // uvs, if any for (size_t uvi = 0; uvi < m->GetNumUVChannels(); ++uvi) { if (m->mNumUVComponents[uvi] > 2) { @@ -1209,6 +1272,11 @@ void FBXExporter::WriteObjects () le.AddChild("Type", "LayerElementNormal"); le.AddChild("TypedIndex", int32_t(0)); layer.AddChild(le); + // TODO only 1 color channel currently + le = FBX::Node("LayerElement"); + le.AddChild("Type", "LayerElementColor"); + le.AddChild("TypedIndex", int32_t(0)); + layer.AddChild(le); le = FBX::Node("LayerElement"); le.AddChild("Type", "LayerElementMaterial"); le.AddChild("TypedIndex", int32_t(0)); @@ -1221,7 +1289,7 @@ void FBXExporter::WriteObjects () for(unsigned int lr = 1; lr < m->GetNumUVChannels(); ++ lr) { - FBX::Node layerExtra("Layer", int32_t(1)); + FBX::Node layerExtra("Layer", int32_t(lr)); layerExtra.AddChild("Version", int32_t(100)); FBX::Node leExtra("LayerElement"); leExtra.AddChild("Type", "LayerElementUV"); @@ -1748,28 +1816,8 @@ void FBXExporter::WriteObjects () // connect it connections.emplace_back("C", "OO", deformer_uid, mesh_uids[mi]); - // we will be indexing by vertex... - // but there might be a different number of "vertices" - // between assimp and our output FBX. - // this code is cut-and-pasted from the geometry section above... - // ideally this should not be so. - // --- - // index of original vertex in vertex data vector - std::vector<int32_t> vertex_indices; - // map of vertex value to its index in the data vector - std::map<aiVector3D,size_t> index_by_vertex_value; - int32_t index = 0; - for (size_t vi = 0; vi < m->mNumVertices; ++vi) { - aiVector3D vtx = m->mVertices[vi]; - auto elem = index_by_vertex_value.find(vtx); - if (elem == index_by_vertex_value.end()) { - vertex_indices.push_back(index); - index_by_vertex_value[vtx] = index; - ++index; - } else { - vertex_indices.push_back(int32_t(elem->second)); - } - } + //computed before + std::vector<int32_t>& vertex_indices = vVertexIndice[mi]; // TODO, FIXME: this won't work if anything is not in the bind pose. // for now if such a situation is detected, we throw an exception. @@ -2435,7 +2483,7 @@ void FBXExporter::WriteModelNodes( void FBXExporter::WriteAnimationCurveNode( StreamWriterLE& outstream, int64_t uid, - std::string name, // "T", "R", or "S" + const std::string& name, // "T", "R", or "S" aiVector3D default_value, std::string property_name, // "Lcl Translation" etc int64_t layer_uid, diff --git a/thirdparty/assimp/code/FBX/FBXExporter.h b/thirdparty/assimp/code/FBX/FBXExporter.h index 71fb55c57f..1ae727eda9 100644 --- a/thirdparty/assimp/code/FBX/FBXExporter.h +++ b/thirdparty/assimp/code/FBX/FBXExporter.h @@ -156,7 +156,7 @@ namespace Assimp void WriteAnimationCurveNode( StreamWriterLE& outstream, int64_t uid, - std::string name, // "T", "R", or "S" + const std::string& name, // "T", "R", or "S" aiVector3D default_value, std::string property_name, // "Lcl Translation" etc int64_t animation_layer_uid, diff --git a/thirdparty/assimp/code/FBX/FBXImporter.cpp b/thirdparty/assimp/code/FBX/FBXImporter.cpp index 271935a568..afcc1ddc78 100644 --- a/thirdparty/assimp/code/FBX/FBXImporter.cpp +++ b/thirdparty/assimp/code/FBX/FBXImporter.cpp @@ -48,26 +48,26 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "FBXImporter.h" -#include "FBXTokenizer.h" +#include "FBXConverter.h" +#include "FBXDocument.h" #include "FBXParser.h" +#include "FBXTokenizer.h" #include "FBXUtil.h" -#include "FBXDocument.h" -#include "FBXConverter.h" -#include <assimp/StreamReader.h> #include <assimp/MemoryIOWrapper.h> -#include <assimp/Importer.hpp> +#include <assimp/StreamReader.h> #include <assimp/importerdesc.h> +#include <assimp/Importer.hpp> namespace Assimp { -template<> -const char* LogFunctions<FBXImporter>::Prefix() { - static auto prefix = "FBX: "; - return prefix; +template <> +const char *LogFunctions<FBXImporter>::Prefix() { + static auto prefix = "FBX: "; + return prefix; } -} +} // namespace Assimp using namespace Assimp; using namespace Assimp::Formatter; @@ -76,136 +76,123 @@ using namespace Assimp::FBX; namespace { static const aiImporterDesc desc = { - "Autodesk FBX Importer", - "", - "", - "", - aiImporterFlags_SupportTextFlavour, - 0, - 0, - 0, - 0, - "fbx" + "Autodesk FBX Importer", + "", + "", + "", + aiImporterFlags_SupportTextFlavour, + 0, + 0, + 0, + 0, + "fbx" }; } // ------------------------------------------------------------------------------------------------ // Constructor to be privately used by #Importer -FBXImporter::FBXImporter() -{ +FBXImporter::FBXImporter() { } // ------------------------------------------------------------------------------------------------ // Destructor, private as well -FBXImporter::~FBXImporter() -{ +FBXImporter::~FBXImporter() { } // ------------------------------------------------------------------------------------------------ // Returns whether the class can handle the format of the given file. -bool FBXImporter::CanRead( const std::string& pFile, IOSystem* pIOHandler, bool checkSig) const -{ - const std::string& extension = GetExtension(pFile); - if (extension == std::string( desc.mFileExtensions ) ) { - return true; - } - - else if ((!extension.length() || checkSig) && pIOHandler) { - // at least ASCII-FBX files usually have a 'FBX' somewhere in their head - const char* tokens[] = {"fbx"}; - return SearchFileHeaderForToken(pIOHandler,pFile,tokens,1); - } - return false; +bool FBXImporter::CanRead(const std::string &pFile, IOSystem *pIOHandler, bool checkSig) const { + const std::string &extension = GetExtension(pFile); + if (extension == std::string(desc.mFileExtensions)) { + return true; + } + + else if ((!extension.length() || checkSig) && pIOHandler) { + // at least ASCII-FBX files usually have a 'FBX' somewhere in their head + const char *tokens[] = { "fbx" }; + return SearchFileHeaderForToken(pIOHandler, pFile, tokens, 1); + } + return false; } // ------------------------------------------------------------------------------------------------ // List all extensions handled by this loader -const aiImporterDesc* FBXImporter::GetInfo () const -{ - return &desc; +const aiImporterDesc *FBXImporter::GetInfo() const { + return &desc; } // ------------------------------------------------------------------------------------------------ // Setup configuration properties for the loader -void FBXImporter::SetupProperties(const Importer* pImp) -{ - settings.readAllLayers = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_ALL_GEOMETRY_LAYERS, true); - settings.readAllMaterials = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_ALL_MATERIALS, false); - settings.readMaterials = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_MATERIALS, true); - settings.readTextures = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_TEXTURES, true); - settings.readCameras = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_CAMERAS, true); - settings.readLights = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_LIGHTS, true); - settings.readAnimations = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_ANIMATIONS, true); - settings.strictMode = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_STRICT_MODE, false); - settings.preservePivots = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_PRESERVE_PIVOTS, true); - settings.optimizeEmptyAnimationCurves = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_OPTIMIZE_EMPTY_ANIMATION_CURVES, true); - settings.useLegacyEmbeddedTextureNaming = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_EMBEDDED_TEXTURES_LEGACY_NAMING, false); - settings.removeEmptyBones = pImp->GetPropertyBool(AI_CONFIG_IMPORT_REMOVE_EMPTY_BONES, true); - settings.convertToMeters = pImp->GetPropertyBool(AI_CONFIG_FBX_CONVERT_TO_M, false); +void FBXImporter::SetupProperties(const Importer *pImp) { + settings.readAllLayers = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_ALL_GEOMETRY_LAYERS, true); + settings.readAllMaterials = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_ALL_MATERIALS, false); + settings.readMaterials = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_MATERIALS, true); + settings.readTextures = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_TEXTURES, true); + settings.readCameras = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_CAMERAS, true); + settings.readLights = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_LIGHTS, true); + settings.readAnimations = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_READ_ANIMATIONS, true); + settings.strictMode = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_STRICT_MODE, false); + settings.preservePivots = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_PRESERVE_PIVOTS, true); + settings.optimizeEmptyAnimationCurves = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_OPTIMIZE_EMPTY_ANIMATION_CURVES, true); + settings.useLegacyEmbeddedTextureNaming = pImp->GetPropertyBool(AI_CONFIG_IMPORT_FBX_EMBEDDED_TEXTURES_LEGACY_NAMING, false); + settings.removeEmptyBones = pImp->GetPropertyBool(AI_CONFIG_IMPORT_REMOVE_EMPTY_BONES, true); + settings.convertToMeters = pImp->GetPropertyBool(AI_CONFIG_FBX_CONVERT_TO_M, false); } // ------------------------------------------------------------------------------------------------ // Imports the given file into the given scene structure. -void FBXImporter::InternReadFile( const std::string& pFile, aiScene* pScene, IOSystem* pIOHandler) -{ - std::unique_ptr<IOStream> stream(pIOHandler->Open(pFile,"rb")); - if (!stream) { - ThrowException("Could not open file for reading"); - } - - // read entire file into memory - no streaming for this, fbx - // files can grow large, but the assimp output data structure - // then becomes very large, too. Assimp doesn't support - // streaming for its output data structures so the net win with - // streaming input data would be very low. - std::vector<char> contents; - contents.resize(stream->FileSize()+1); - stream->Read( &*contents.begin(), 1, contents.size()-1 ); - contents[ contents.size() - 1 ] = 0; - const char* const begin = &*contents.begin(); - - // broadphase tokenizing pass in which we identify the core - // syntax elements of FBX (brackets, commas, key:value mappings) - TokenList tokens; - try { - - bool is_binary = false; - if (!strncmp(begin,"Kaydara FBX Binary",18)) { - is_binary = true; - TokenizeBinary(tokens,begin,contents.size()); - } - else { - Tokenize(tokens,begin); - } - - // use this information to construct a very rudimentary - // parse-tree representing the FBX scope structure - Parser parser(tokens, is_binary); - - // take the raw parse-tree and convert it to a FBX DOM - Document doc(parser,settings); - - FbxUnit unit(FbxUnit::cm); - if (settings.convertToMeters) { - unit = FbxUnit::m; - } - - // convert the FBX DOM to aiScene - ConvertToAssimpScene(pScene, doc, settings.removeEmptyBones); - - // size relative to cm - float size_relative_to_cm = doc.GlobalSettings().UnitScaleFactor(); - - // Set FBX file scale is relative to CM must be converted to M for - // assimp universal format (M) - SetFileScale( size_relative_to_cm * 0.01f); - - std::for_each(tokens.begin(),tokens.end(),Util::delete_fun<Token>()); - } - catch(std::exception&) { - std::for_each(tokens.begin(),tokens.end(),Util::delete_fun<Token>()); - throw; - } +void FBXImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSystem *pIOHandler) { + std::unique_ptr<IOStream> stream(pIOHandler->Open(pFile, "rb")); + if (!stream) { + ThrowException("Could not open file for reading"); + } + + // read entire file into memory - no streaming for this, fbx + // files can grow large, but the assimp output data structure + // then becomes very large, too. Assimp doesn't support + // streaming for its output data structures so the net win with + // streaming input data would be very low. + std::vector<char> contents; + contents.resize(stream->FileSize() + 1); + stream->Read(&*contents.begin(), 1, contents.size() - 1); + contents[contents.size() - 1] = 0; + const char *const begin = &*contents.begin(); + + // broadphase tokenizing pass in which we identify the core + // syntax elements of FBX (brackets, commas, key:value mappings) + TokenList tokens; + try { + + bool is_binary = false; + if (!strncmp(begin, "Kaydara FBX Binary", 18)) { + is_binary = true; + TokenizeBinary(tokens, begin, contents.size()); + } else { + Tokenize(tokens, begin); + } + + // use this information to construct a very rudimentary + // parse-tree representing the FBX scope structure + Parser parser(tokens, is_binary); + + // take the raw parse-tree and convert it to a FBX DOM + Document doc(parser, settings); + + // convert the FBX DOM to aiScene + ConvertToAssimpScene(pScene, doc, settings.removeEmptyBones); + + // size relative to cm + float size_relative_to_cm = doc.GlobalSettings().UnitScaleFactor(); + + // Set FBX file scale is relative to CM must be converted to M for + // assimp universal format (M) + SetFileScale(size_relative_to_cm * 0.01f); + + std::for_each(tokens.begin(), tokens.end(), Util::delete_fun<Token>()); + } catch (std::exception &) { + std::for_each(tokens.begin(), tokens.end(), Util::delete_fun<Token>()); + throw; + } } #endif // !ASSIMP_BUILD_NO_FBX_IMPORTER diff --git a/thirdparty/assimp/code/FBX/FBXMeshGeometry.cpp b/thirdparty/assimp/code/FBX/FBXMeshGeometry.cpp index 5c9a0e309d..1386e2383c 100644 --- a/thirdparty/assimp/code/FBX/FBXMeshGeometry.cpp +++ b/thirdparty/assimp/code/FBX/FBXMeshGeometry.cpp @@ -610,11 +610,11 @@ void MeshGeometry::ReadVertexDataMaterials(std::vector<int>& materials_out, cons const std::string& ReferenceInformationType) { const size_t face_count = m_faces.size(); - if(face_count <= 0) + if( 0 == face_count ) { return; } - + // materials are handled separately. First of all, they are assigned per-face // and not per polyvert. Secondly, ReferenceInformationType=IndexToDirect // has a slightly different meaning for materials. @@ -625,16 +625,14 @@ void MeshGeometry::ReadVertexDataMaterials(std::vector<int>& materials_out, cons if (materials_out.empty()) { FBXImporter::LogError(Formatter::format("expected material index, ignoring")); return; - } - else if (materials_out.size() > 1) { + } else if (materials_out.size() > 1) { FBXImporter::LogWarn(Formatter::format("expected only a single material index, ignoring all except the first one")); materials_out.clear(); } materials_out.resize(m_vertices.size()); std::fill(materials_out.begin(), materials_out.end(), materials_out.at(0)); - } - else if (MappingInformationType == "ByPolygon" && ReferenceInformationType == "IndexToDirect") { + } else if (MappingInformationType == "ByPolygon" && ReferenceInformationType == "IndexToDirect") { materials_out.resize(face_count); if(materials_out.size() != face_count) { @@ -643,18 +641,16 @@ void MeshGeometry::ReadVertexDataMaterials(std::vector<int>& materials_out, cons ); return; } - } - else { + } else { FBXImporter::LogError(Formatter::format("ignoring material assignments, access type not implemented: ") << MappingInformationType << "," << ReferenceInformationType); } } // ------------------------------------------------------------------------------------------------ ShapeGeometry::ShapeGeometry(uint64_t id, const Element& element, const std::string& name, const Document& doc) - : Geometry(id, element, name, doc) -{ - const Scope* sc = element.Compound(); - if (!sc) { +: Geometry(id, element, name, doc) { + const Scope *sc = element.Compound(); + if (nullptr == sc) { DOMError("failed to read Geometry object (class: Shape), no data scope found"); } const Element& Indexes = GetRequiredElement(*sc, "Indexes", &element); diff --git a/thirdparty/assimp/code/MMD/MMDCpp14.h b/thirdparty/assimp/code/MMD/MMDCpp14.h deleted file mode 100644 index 638b0bfd2f..0000000000 --- a/thirdparty/assimp/code/MMD/MMDCpp14.h +++ /dev/null @@ -1,83 +0,0 @@ -/* -Open Asset Import Library (assimp) ----------------------------------------------------------------------- - -Copyright (c) 2006-2019, assimp team - - -All rights reserved. - -Redistribution and use of this software in source and binary forms, -with or without modification, are permitted provided that the -following conditions are met: - -* Redistributions of source code must retain the above -copyright notice, this list of conditions and the -following disclaimer. - -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the -following disclaimer in the documentation and/or other -materials provided with the distribution. - -* Neither the name of the assimp team, nor the names of its -contributors may be used to endorse or promote products -derived from this software without specific prior -written permission of the assimp team. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- -*/ -#pragma once - -#ifndef MMD_CPP14_H -#define MMD_CPP14_H - -#include <cstddef> -#include <memory> -#include <type_traits> -#include <utility> - -namespace mmd { - template<class T> struct _Unique_if { - typedef std::unique_ptr<T> _Single_object; - }; - - template<class T> struct _Unique_if<T[]> { - typedef std::unique_ptr<T[]> _Unknown_bound; - }; - - template<class T, size_t N> struct _Unique_if<T[N]> { - typedef void _Known_bound; - }; - - template<class T, class... Args> - typename _Unique_if<T>::_Single_object - make_unique(Args&&... args) { - return std::unique_ptr<T>(new T(std::forward<Args>(args)...)); - } - - template<class T> - typename _Unique_if<T>::_Unknown_bound - make_unique(size_t n) { - typedef typename std::remove_extent<T>::type U; - return std::unique_ptr<T>(new U[n]()); - } - - template<class T, class... Args> - typename _Unique_if<T>::_Known_bound - make_unique(Args&&...) = delete; -} - -#endif diff --git a/thirdparty/assimp/code/MMD/MMDImporter.cpp b/thirdparty/assimp/code/MMD/MMDImporter.cpp deleted file mode 100644 index e7744e4cd0..0000000000 --- a/thirdparty/assimp/code/MMD/MMDImporter.cpp +++ /dev/null @@ -1,372 +0,0 @@ -/* ---------------------------------------------------------------------------- -Open Asset Import Library (assimp) ---------------------------------------------------------------------------- - -Copyright (c) 2006-2016, assimp team - -All rights reserved. - -Redistribution and use of this software in source and binary forms, -with or without modification, are permitted provided that the following -conditions are met: - -* Redistributions of source code must retain the above - copyright notice, this list of conditions and the - following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the - following disclaimer in the documentation and/or other - materials provided with the distribution. - -* Neither the name of the assimp team, nor the names of its - contributors may be used to endorse or promote products - derived from this software without specific prior - written permission of the assimp team. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------------- -*/ - -#ifndef ASSIMP_BUILD_NO_MMD_IMPORTER - -#include "MMD/MMDImporter.h" -#include "MMD/MMDPmdParser.h" -#include "MMD/MMDPmxParser.h" -#include "MMD/MMDVmdParser.h" -#include "PostProcessing/ConvertToLHProcess.h" - -#include <assimp/DefaultIOSystem.h> -#include <assimp/Importer.hpp> -#include <assimp/ai_assert.h> -#include <assimp/scene.h> - -#include <fstream> -#include <iomanip> -#include <memory> - -static const aiImporterDesc desc = {"MMD Importer", - "", - "", - "surfaces supported?", - aiImporterFlags_SupportTextFlavour, - 0, - 0, - 0, - 0, - "pmx"}; - -namespace Assimp { - -using namespace std; - -// ------------------------------------------------------------------------------------------------ -// Default constructor -MMDImporter::MMDImporter() -: m_Buffer() -, m_strAbsPath("") { - DefaultIOSystem io; - m_strAbsPath = io.getOsSeparator(); -} - -// ------------------------------------------------------------------------------------------------ -// Destructor. -MMDImporter::~MMDImporter() { - // empty -} - -// ------------------------------------------------------------------------------------------------ -// Returns true, if file is an pmx file. -bool MMDImporter::CanRead(const std::string &pFile, IOSystem *pIOHandler, - bool checkSig) const { - if (!checkSig) // Check File Extension - { - return SimpleExtensionCheck(pFile, "pmx"); - } else // Check file Header - { - static const char *pTokens[] = {"PMX "}; - return BaseImporter::SearchFileHeaderForToken(pIOHandler, pFile, pTokens, 1); - } -} - -// ------------------------------------------------------------------------------------------------ -const aiImporterDesc *MMDImporter::GetInfo() const { return &desc; } - -// ------------------------------------------------------------------------------------------------ -// MMD import implementation -void MMDImporter::InternReadFile(const std::string &file, aiScene *pScene, - IOSystem * /*pIOHandler*/) { - // Read file by istream - std::filebuf fb; - if (!fb.open(file, std::ios::in | std::ios::binary)) { - throw DeadlyImportError("Failed to open file " + file + "."); - } - - std::istream fileStream(&fb); - - // Get the file-size and validate it, throwing an exception when fails - fileStream.seekg(0, fileStream.end); - size_t fileSize = static_cast<size_t>(fileStream.tellg()); - fileStream.seekg(0, fileStream.beg); - - if (fileSize < sizeof(pmx::PmxModel)) { - throw DeadlyImportError(file + " is too small."); - } - - pmx::PmxModel model; - model.Read(&fileStream); - - CreateDataFromImport(&model, pScene); -} - -// ------------------------------------------------------------------------------------------------ -void MMDImporter::CreateDataFromImport(const pmx::PmxModel *pModel, - aiScene *pScene) { - if (pModel == NULL) { - return; - } - - aiNode *pNode = new aiNode; - if (!pModel->model_name.empty()) { - pNode->mName.Set(pModel->model_name); - } - - pScene->mRootNode = pNode; - - pNode = new aiNode; - pScene->mRootNode->addChildren(1, &pNode); - pNode->mName.Set(string(pModel->model_name) + string("_mesh")); - - // split mesh by materials - pNode->mNumMeshes = pModel->material_count; - pNode->mMeshes = new unsigned int[pNode->mNumMeshes]; - for (unsigned int index = 0; index < pNode->mNumMeshes; index++) { - pNode->mMeshes[index] = index; - } - - pScene->mNumMeshes = pModel->material_count; - pScene->mMeshes = new aiMesh *[pScene->mNumMeshes]; - for (unsigned int i = 0, indexStart = 0; i < pScene->mNumMeshes; i++) { - const int indexCount = pModel->materials[i].index_count; - - pScene->mMeshes[i] = CreateMesh(pModel, indexStart, indexCount); - pScene->mMeshes[i]->mName = pModel->materials[i].material_name; - pScene->mMeshes[i]->mMaterialIndex = i; - indexStart += indexCount; - } - - // create node hierarchy for bone position - std::unique_ptr<aiNode *[]> ppNode(new aiNode *[pModel->bone_count]); - for (auto i = 0; i < pModel->bone_count; i++) { - ppNode[i] = new aiNode(pModel->bones[i].bone_name); - } - - for (auto i = 0; i < pModel->bone_count; i++) { - const pmx::PmxBone &bone = pModel->bones[i]; - - if (bone.parent_index < 0) { - pScene->mRootNode->addChildren(1, ppNode.get() + i); - } else { - ppNode[bone.parent_index]->addChildren(1, ppNode.get() + i); - - aiVector3D v3 = aiVector3D( - bone.position[0] - pModel->bones[bone.parent_index].position[0], - bone.position[1] - pModel->bones[bone.parent_index].position[1], - bone.position[2] - pModel->bones[bone.parent_index].position[2]); - aiMatrix4x4::Translation(v3, ppNode[i]->mTransformation); - } - } - - // create materials - pScene->mNumMaterials = pModel->material_count; - pScene->mMaterials = new aiMaterial *[pScene->mNumMaterials]; - for (unsigned int i = 0; i < pScene->mNumMaterials; i++) { - pScene->mMaterials[i] = CreateMaterial(&pModel->materials[i], pModel); - } - - // Convert everything to OpenGL space - MakeLeftHandedProcess convertProcess; - convertProcess.Execute(pScene); - - FlipUVsProcess uvFlipper; - uvFlipper.Execute(pScene); - - FlipWindingOrderProcess windingFlipper; - windingFlipper.Execute(pScene); -} - -// ------------------------------------------------------------------------------------------------ -aiMesh *MMDImporter::CreateMesh(const pmx::PmxModel *pModel, - const int indexStart, const int indexCount) { - aiMesh *pMesh = new aiMesh; - - pMesh->mNumVertices = indexCount; - - pMesh->mNumFaces = indexCount / 3; - pMesh->mFaces = new aiFace[pMesh->mNumFaces]; - - const int numIndices = 3; // triangular face - for (unsigned int index = 0; index < pMesh->mNumFaces; index++) { - pMesh->mFaces[index].mNumIndices = numIndices; - unsigned int *indices = new unsigned int[numIndices]; - indices[0] = numIndices * index; - indices[1] = numIndices * index + 1; - indices[2] = numIndices * index + 2; - pMesh->mFaces[index].mIndices = indices; - } - - pMesh->mVertices = new aiVector3D[pMesh->mNumVertices]; - pMesh->mNormals = new aiVector3D[pMesh->mNumVertices]; - pMesh->mTextureCoords[0] = new aiVector3D[pMesh->mNumVertices]; - pMesh->mNumUVComponents[0] = 2; - - // additional UVs - for (int i = 1; i <= pModel->setting.uv; i++) { - pMesh->mTextureCoords[i] = new aiVector3D[pMesh->mNumVertices]; - pMesh->mNumUVComponents[i] = 4; - } - - map<int, vector<aiVertexWeight>> bone_vertex_map; - - // fill in contents and create bones - for (int index = 0; index < indexCount; index++) { - const pmx::PmxVertex *v = - &pModel->vertices[pModel->indices[indexStart + index]]; - const float *position = v->position; - pMesh->mVertices[index].Set(position[0], position[1], position[2]); - const float *normal = v->normal; - - pMesh->mNormals[index].Set(normal[0], normal[1], normal[2]); - pMesh->mTextureCoords[0][index].x = v->uv[0]; - pMesh->mTextureCoords[0][index].y = v->uv[1]; - - for (int i = 1; i <= pModel->setting.uv; i++) { - // TODO: wrong here? use quaternion transform? - pMesh->mTextureCoords[i][index].x = v->uva[i][0]; - pMesh->mTextureCoords[i][index].y = v->uva[i][1]; - } - - // handle bone map - const auto vsBDEF1_ptr = - dynamic_cast<pmx::PmxVertexSkinningBDEF1 *>(v->skinning.get()); - const auto vsBDEF2_ptr = - dynamic_cast<pmx::PmxVertexSkinningBDEF2 *>(v->skinning.get()); - const auto vsBDEF4_ptr = - dynamic_cast<pmx::PmxVertexSkinningBDEF4 *>(v->skinning.get()); - const auto vsSDEF_ptr = - dynamic_cast<pmx::PmxVertexSkinningSDEF *>(v->skinning.get()); - switch (v->skinning_type) { - case pmx::PmxVertexSkinningType::BDEF1: - bone_vertex_map[vsBDEF1_ptr->bone_index].push_back( - aiVertexWeight(index, 1.0)); - break; - case pmx::PmxVertexSkinningType::BDEF2: - bone_vertex_map[vsBDEF2_ptr->bone_index1].push_back( - aiVertexWeight(index, vsBDEF2_ptr->bone_weight)); - bone_vertex_map[vsBDEF2_ptr->bone_index2].push_back( - aiVertexWeight(index, 1.0f - vsBDEF2_ptr->bone_weight)); - break; - case pmx::PmxVertexSkinningType::BDEF4: - bone_vertex_map[vsBDEF4_ptr->bone_index1].push_back( - aiVertexWeight(index, vsBDEF4_ptr->bone_weight1)); - bone_vertex_map[vsBDEF4_ptr->bone_index2].push_back( - aiVertexWeight(index, vsBDEF4_ptr->bone_weight2)); - bone_vertex_map[vsBDEF4_ptr->bone_index3].push_back( - aiVertexWeight(index, vsBDEF4_ptr->bone_weight3)); - bone_vertex_map[vsBDEF4_ptr->bone_index4].push_back( - aiVertexWeight(index, vsBDEF4_ptr->bone_weight4)); - break; - case pmx::PmxVertexSkinningType::SDEF: // TODO: how to use sdef_c, sdef_r0, - // sdef_r1? - bone_vertex_map[vsSDEF_ptr->bone_index1].push_back( - aiVertexWeight(index, vsSDEF_ptr->bone_weight)); - bone_vertex_map[vsSDEF_ptr->bone_index2].push_back( - aiVertexWeight(index, 1.0f - vsSDEF_ptr->bone_weight)); - break; - case pmx::PmxVertexSkinningType::QDEF: - const auto vsQDEF_ptr = - dynamic_cast<pmx::PmxVertexSkinningQDEF *>(v->skinning.get()); - bone_vertex_map[vsQDEF_ptr->bone_index1].push_back( - aiVertexWeight(index, vsQDEF_ptr->bone_weight1)); - bone_vertex_map[vsQDEF_ptr->bone_index2].push_back( - aiVertexWeight(index, vsQDEF_ptr->bone_weight2)); - bone_vertex_map[vsQDEF_ptr->bone_index3].push_back( - aiVertexWeight(index, vsQDEF_ptr->bone_weight3)); - bone_vertex_map[vsQDEF_ptr->bone_index4].push_back( - aiVertexWeight(index, vsQDEF_ptr->bone_weight4)); - break; - } - } - - // make all bones for each mesh - // assign bone weights to skinned bones (otherwise just initialize) - auto bone_ptr_ptr = new aiBone *[pModel->bone_count]; - pMesh->mNumBones = pModel->bone_count; - pMesh->mBones = bone_ptr_ptr; - for (auto ii = 0; ii < pModel->bone_count; ++ii) { - auto pBone = new aiBone; - const auto &pmxBone = pModel->bones[ii]; - pBone->mName = pmxBone.bone_name; - aiVector3D pos(pmxBone.position[0], pmxBone.position[1], pmxBone.position[2]); - aiMatrix4x4::Translation(-pos, pBone->mOffsetMatrix); - auto it = bone_vertex_map.find(ii); - if (it != bone_vertex_map.end()) { - pBone->mNumWeights = static_cast<unsigned int>(it->second.size()); - pBone->mWeights = new aiVertexWeight[pBone->mNumWeights]; - for (unsigned int j = 0; j < pBone->mNumWeights; j++) { - pBone->mWeights[j] = it->second[j]; - } - } - bone_ptr_ptr[ii] = pBone; - } - - return pMesh; -} - -// ------------------------------------------------------------------------------------------------ -aiMaterial *MMDImporter::CreateMaterial(const pmx::PmxMaterial *pMat, - const pmx::PmxModel *pModel) { - aiMaterial *mat = new aiMaterial(); - aiString name(pMat->material_english_name); - mat->AddProperty(&name, AI_MATKEY_NAME); - - aiColor3D diffuse(pMat->diffuse[0], pMat->diffuse[1], pMat->diffuse[2]); - mat->AddProperty(&diffuse, 1, AI_MATKEY_COLOR_DIFFUSE); - aiColor3D specular(pMat->specular[0], pMat->specular[1], pMat->specular[2]); - mat->AddProperty(&specular, 1, AI_MATKEY_COLOR_SPECULAR); - aiColor3D ambient(pMat->ambient[0], pMat->ambient[1], pMat->ambient[2]); - mat->AddProperty(&ambient, 1, AI_MATKEY_COLOR_AMBIENT); - - float opacity = pMat->diffuse[3]; - mat->AddProperty(&opacity, 1, AI_MATKEY_OPACITY); - float shininess = pMat->specularlity; - mat->AddProperty(&shininess, 1, AI_MATKEY_SHININESS_STRENGTH); - - if(pMat->diffuse_texture_index >= 0) { - aiString texture_path(pModel->textures[pMat->diffuse_texture_index]); - mat->AddProperty(&texture_path, AI_MATKEY_TEXTURE(aiTextureType_DIFFUSE, 0)); - } - - int mapping_uvwsrc = 0; - mat->AddProperty(&mapping_uvwsrc, 1, - AI_MATKEY_UVWSRC(aiTextureType_DIFFUSE, 0)); - - return mat; -} - -// ------------------------------------------------------------------------------------------------ - -} // Namespace Assimp - -#endif // !! ASSIMP_BUILD_NO_MMD_IMPORTER diff --git a/thirdparty/assimp/code/MMD/MMDImporter.h b/thirdparty/assimp/code/MMD/MMDImporter.h deleted file mode 100644 index 4ee94eeb00..0000000000 --- a/thirdparty/assimp/code/MMD/MMDImporter.h +++ /dev/null @@ -1,96 +0,0 @@ -/* -Open Asset Import Library (assimp) ----------------------------------------------------------------------- - -Copyright (c) 2006-2016, assimp team -All rights reserved. - -Redistribution and use of this software in source and binary forms, -with or without modification, are permitted provided that the -following conditions are met: - -* Redistributions of source code must retain the above - copyright notice, this list of conditions and the - following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the - following disclaimer in the documentation and/or other - materials provided with the distribution. - -* Neither the name of the assimp team, nor the names of its - contributors may be used to endorse or promote products - derived from this software without specific prior - written permission of the assimp team. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- -*/ -#ifndef MMD_FILE_IMPORTER_H_INC -#define MMD_FILE_IMPORTER_H_INC - -#include <assimp/BaseImporter.h> -#include "MMDPmxParser.h" -#include <assimp/material.h> -#include <vector> - -struct aiMesh; - -namespace Assimp { - -// ------------------------------------------------------------------------------------------------ -/// \class MMDImporter -/// \brief Imports MMD a pmx/pmd/vmd file -// ------------------------------------------------------------------------------------------------ -class MMDImporter : public BaseImporter { -public: - /// \brief Default constructor - MMDImporter(); - - /// \brief Destructor - ~MMDImporter(); - -public: - /// \brief Returns whether the class can handle the format of the given file. - /// \remark See BaseImporter::CanRead() for details. - bool CanRead( const std::string& pFile, IOSystem* pIOHandler, bool checkSig) const; - -private: - //! \brief Appends the supported extension. - const aiImporterDesc* GetInfo () const; - - //! \brief File import implementation. - void InternReadFile(const std::string& pFile, aiScene* pScene, IOSystem* pIOHandler); - - //! \brief Create the data from imported content. - void CreateDataFromImport(const pmx::PmxModel* pModel, aiScene* pScene); - - //! \brief Create the mesh - aiMesh* CreateMesh(const pmx::PmxModel* pModel, const int indexStart, const int indexCount); - - //! \brief Create the material - aiMaterial* CreateMaterial(const pmx::PmxMaterial* pMat, const pmx::PmxModel* pModel); - -private: - //! Data buffer - std::vector<char> m_Buffer; - //! Absolute pathname of model in file system - std::string m_strAbsPath; -}; - -// ------------------------------------------------------------------------------------------------ - -} // Namespace Assimp - -#endif
\ No newline at end of file diff --git a/thirdparty/assimp/code/MMD/MMDPmdParser.h b/thirdparty/assimp/code/MMD/MMDPmdParser.h deleted file mode 100644 index d2f2224aa1..0000000000 --- a/thirdparty/assimp/code/MMD/MMDPmdParser.h +++ /dev/null @@ -1,597 +0,0 @@ -/* -Open Asset Import Library (assimp) ----------------------------------------------------------------------- - -Copyright (c) 2006-2019, assimp team - - -All rights reserved. - -Redistribution and use of this software in source and binary forms, -with or without modification, are permitted provided that the -following conditions are met: - -* Redistributions of source code must retain the above -copyright notice, this list of conditions and the -following disclaimer. - -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the -following disclaimer in the documentation and/or other -materials provided with the distribution. - -* Neither the name of the assimp team, nor the names of its -contributors may be used to endorse or promote products -derived from this software without specific prior -written permission of the assimp team. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- -*/ -#pragma once - -#include <vector> -#include <string> -#include <memory> -#include <iostream> -#include <fstream> -#include "MMDCpp14.h" - -namespace pmd -{ - class PmdHeader - { - public: - std::string name; - std::string name_english; - std::string comment; - std::string comment_english; - - bool Read(std::ifstream* stream) - { - char buffer[256]; - stream->read(buffer, 20); - name = std::string(buffer); - stream->read(buffer, 256); - comment = std::string(buffer); - return true; - } - - bool ReadExtension(std::ifstream* stream) - { - char buffer[256]; - stream->read(buffer, 20); - name_english = std::string(buffer); - stream->read(buffer, 256); - comment_english = std::string(buffer); - return true; - } - }; - - class PmdVertex - { - public: - float position[3]; - - float normal[3]; - - float uv[2]; - - uint16_t bone_index[2]; - - uint8_t bone_weight; - - bool edge_invisible; - - bool Read(std::ifstream* stream) - { - stream->read((char*) position, sizeof(float) * 3); - stream->read((char*) normal, sizeof(float) * 3); - stream->read((char*) uv, sizeof(float) * 2); - stream->read((char*) bone_index, sizeof(uint16_t) * 2); - stream->read((char*) &bone_weight, sizeof(uint8_t)); - stream->read((char*) &edge_invisible, sizeof(uint8_t)); - return true; - } - }; - - class PmdMaterial - { - public: - float diffuse[4]; - float power; - float specular[3]; - float ambient[3]; - uint8_t toon_index; - uint8_t edge_flag; - uint32_t index_count; - std::string texture_filename; - std::string sphere_filename; - - bool Read(std::ifstream* stream) - { - char buffer[20]; - stream->read((char*) &diffuse, sizeof(float) * 4); - stream->read((char*) &power, sizeof(float)); - stream->read((char*) &specular, sizeof(float) * 3); - stream->read((char*) &ambient, sizeof(float) * 3); - stream->read((char*) &toon_index, sizeof(uint8_t)); - stream->read((char*) &edge_flag, sizeof(uint8_t)); - stream->read((char*) &index_count, sizeof(uint32_t)); - stream->read((char*) &buffer, sizeof(char) * 20); - char* pstar = strchr(buffer, '*'); - if (NULL == pstar) - { - texture_filename = std::string(buffer); - sphere_filename.clear(); - } - else { - *pstar = 0; - texture_filename = std::string(buffer); - sphere_filename = std::string(pstar+1); - } - return true; - } - }; - - enum class BoneType : uint8_t - { - Rotation, - RotationAndMove, - IkEffector, - Unknown, - IkEffectable, - RotationEffectable, - IkTarget, - Invisible, - Twist, - RotationMovement - }; - - class PmdBone - { - public: - std::string name; - std::string name_english; - uint16_t parent_bone_index; - uint16_t tail_pos_bone_index; - BoneType bone_type; - uint16_t ik_parent_bone_index; - float bone_head_pos[3]; - - void Read(std::istream *stream) - { - char buffer[20]; - stream->read(buffer, 20); - name = std::string(buffer); - stream->read((char*) &parent_bone_index, sizeof(uint16_t)); - stream->read((char*) &tail_pos_bone_index, sizeof(uint16_t)); - stream->read((char*) &bone_type, sizeof(uint8_t)); - stream->read((char*) &ik_parent_bone_index, sizeof(uint16_t)); - stream->read((char*) &bone_head_pos, sizeof(float) * 3); - } - - void ReadExpantion(std::istream *stream) - { - char buffer[20]; - stream->read(buffer, 20); - name_english = std::string(buffer); - } - }; - - class PmdIk - { - public: - uint16_t ik_bone_index; - uint16_t target_bone_index; - uint16_t interations; - float angle_limit; - std::vector<uint16_t> ik_child_bone_index; - - void Read(std::istream *stream) - { - stream->read((char *) &ik_bone_index, sizeof(uint16_t)); - stream->read((char *) &target_bone_index, sizeof(uint16_t)); - uint8_t ik_chain_length; - stream->read((char*) &ik_chain_length, sizeof(uint8_t)); - stream->read((char *) &interations, sizeof(uint16_t)); - stream->read((char *) &angle_limit, sizeof(float)); - ik_child_bone_index.resize(ik_chain_length); - for (int i = 0; i < ik_chain_length; i++) - { - stream->read((char *) &ik_child_bone_index[i], sizeof(uint16_t)); - } - } - }; - - class PmdFaceVertex - { - public: - int vertex_index; - float position[3]; - - void Read(std::istream *stream) - { - stream->read((char *) &vertex_index, sizeof(int)); - stream->read((char *) position, sizeof(float) * 3); - } - }; - - enum class FaceCategory : uint8_t - { - Base, - Eyebrow, - Eye, - Mouth, - Other - }; - - class PmdFace - { - public: - std::string name; - FaceCategory type; - std::vector<PmdFaceVertex> vertices; - std::string name_english; - - void Read(std::istream *stream) - { - char buffer[20]; - stream->read(buffer, 20); - name = std::string(buffer); - int vertex_count; - stream->read((char*) &vertex_count, sizeof(int)); - stream->read((char*) &type, sizeof(uint8_t)); - vertices.resize(vertex_count); - for (int i = 0; i < vertex_count; i++) - { - vertices[i].Read(stream); - } - } - - void ReadExpantion(std::istream *stream) - { - char buffer[20]; - stream->read(buffer, 20); - name_english = std::string(buffer); - } - }; - - class PmdBoneDispName - { - public: - std::string bone_disp_name; - std::string bone_disp_name_english; - - void Read(std::istream *stream) - { - char buffer[50]; - stream->read(buffer, 50); - bone_disp_name = std::string(buffer); - bone_disp_name_english.clear(); - } - void ReadExpantion(std::istream *stream) - { - char buffer[50]; - stream->read(buffer, 50); - bone_disp_name_english = std::string(buffer); - } - }; - - class PmdBoneDisp - { - public: - uint16_t bone_index; - uint8_t bone_disp_index; - - void Read(std::istream *stream) - { - stream->read((char*) &bone_index, sizeof(uint16_t)); - stream->read((char*) &bone_disp_index, sizeof(uint8_t)); - } - }; - - enum class RigidBodyShape : uint8_t - { - Sphere = 0, - Box = 1, - Cpusel = 2 - }; - - enum class RigidBodyType : uint8_t - { - BoneConnected = 0, - Physics = 1, - ConnectedPhysics = 2 - }; - - class PmdRigidBody - { - public: - std::string name; - uint16_t related_bone_index; - uint8_t group_index; - uint16_t mask; - RigidBodyShape shape; - float size[3]; - float position[3]; - float orientation[3]; - float weight; - float linear_damping; - float anglar_damping; - float restitution; - float friction; - RigidBodyType rigid_type; - - void Read(std::istream *stream) - { - char buffer[20]; - stream->read(buffer, sizeof(char) * 20); - name = (std::string(buffer)); - stream->read((char*) &related_bone_index, sizeof(uint16_t)); - stream->read((char*) &group_index, sizeof(uint8_t)); - stream->read((char*) &mask, sizeof(uint16_t)); - stream->read((char*) &shape, sizeof(uint8_t)); - stream->read((char*) size, sizeof(float) * 3); - stream->read((char*) position, sizeof(float) * 3); - stream->read((char*) orientation, sizeof(float) * 3); - stream->read((char*) &weight, sizeof(float)); - stream->read((char*) &linear_damping, sizeof(float)); - stream->read((char*) &anglar_damping, sizeof(float)); - stream->read((char*) &restitution, sizeof(float)); - stream->read((char*) &friction, sizeof(float)); - stream->read((char*) &rigid_type, sizeof(char)); - } - }; - - class PmdConstraint - { - public: - std::string name; - uint32_t rigid_body_index_a; - uint32_t rigid_body_index_b; - float position[3]; - float orientation[3]; - float linear_lower_limit[3]; - float linear_upper_limit[3]; - float angular_lower_limit[3]; - float angular_upper_limit[3]; - float linear_stiffness[3]; - float angular_stiffness[3]; - - void Read(std::istream *stream) - { - char buffer[20]; - stream->read(buffer, 20); - name = std::string(buffer); - stream->read((char *) &rigid_body_index_a, sizeof(uint32_t)); - stream->read((char *) &rigid_body_index_b, sizeof(uint32_t)); - stream->read((char *) position, sizeof(float) * 3); - stream->read((char *) orientation, sizeof(float) * 3); - stream->read((char *) linear_lower_limit, sizeof(float) * 3); - stream->read((char *) linear_upper_limit, sizeof(float) * 3); - stream->read((char *) angular_lower_limit, sizeof(float) * 3); - stream->read((char *) angular_upper_limit, sizeof(float) * 3); - stream->read((char *) linear_stiffness, sizeof(float) * 3); - stream->read((char *) angular_stiffness, sizeof(float) * 3); - } - }; - - class PmdModel - { - public: - float version; - PmdHeader header; - std::vector<PmdVertex> vertices; - std::vector<uint16_t> indices; - std::vector<PmdMaterial> materials; - std::vector<PmdBone> bones; - std::vector<PmdIk> iks; - std::vector<PmdFace> faces; - std::vector<uint16_t> faces_indices; - std::vector<PmdBoneDispName> bone_disp_name; - std::vector<PmdBoneDisp> bone_disp; - std::vector<std::string> toon_filenames; - std::vector<PmdRigidBody> rigid_bodies; - std::vector<PmdConstraint> constraints; - - static std::unique_ptr<PmdModel> LoadFromFile(const char *filename) - { - std::ifstream stream(filename, std::ios::binary); - if (stream.fail()) - { - std::cerr << "could not open \"" << filename << "\"" << std::endl; - return nullptr; - } - auto result = LoadFromStream(&stream); - stream.close(); - return result; - } - - static std::unique_ptr<PmdModel> LoadFromStream(std::ifstream *stream) - { - auto result = mmd::make_unique<PmdModel>(); - char buffer[100]; - - // magic - char magic[3]; - stream->read(magic, 3); - if (magic[0] != 'P' || magic[1] != 'm' || magic[2] != 'd') - { - std::cerr << "invalid file" << std::endl; - return nullptr; - } - - // version - stream->read((char*) &(result->version), sizeof(float)); - if (result ->version != 1.0f) - { - std::cerr << "invalid version" << std::endl; - return nullptr; - } - - // header - result->header.Read(stream); - - // vertices - uint32_t vertex_num; - stream->read((char*) &vertex_num, sizeof(uint32_t)); - result->vertices.resize(vertex_num); - for (uint32_t i = 0; i < vertex_num; i++) - { - result->vertices[i].Read(stream); - } - - // indices - uint32_t index_num; - stream->read((char*) &index_num, sizeof(uint32_t)); - result->indices.resize(index_num); - for (uint32_t i = 0; i < index_num; i++) - { - stream->read((char*) &result->indices[i], sizeof(uint16_t)); - } - - // materials - uint32_t material_num; - stream->read((char*) &material_num, sizeof(uint32_t)); - result->materials.resize(material_num); - for (uint32_t i = 0; i < material_num; i++) - { - result->materials[i].Read(stream); - } - - // bones - uint16_t bone_num; - stream->read((char*) &bone_num, sizeof(uint16_t)); - result->bones.resize(bone_num); - for (uint32_t i = 0; i < bone_num; i++) - { - result->bones[i].Read(stream); - } - - // iks - uint16_t ik_num; - stream->read((char*) &ik_num, sizeof(uint16_t)); - result->iks.resize(ik_num); - for (uint32_t i = 0; i < ik_num; i++) - { - result->iks[i].Read(stream); - } - - // faces - uint16_t face_num; - stream->read((char*) &face_num, sizeof(uint16_t)); - result->faces.resize(face_num); - for (uint32_t i = 0; i < face_num; i++) - { - result->faces[i].Read(stream); - } - - // face frames - uint8_t face_frame_num; - stream->read((char*) &face_frame_num, sizeof(uint8_t)); - result->faces_indices.resize(face_frame_num); - for (uint32_t i = 0; i < face_frame_num; i++) - { - stream->read((char*) &result->faces_indices[i], sizeof(uint16_t)); - } - - // bone names - uint8_t bone_disp_num; - stream->read((char*) &bone_disp_num, sizeof(uint8_t)); - result->bone_disp_name.resize(bone_disp_num); - for (uint32_t i = 0; i < bone_disp_num; i++) - { - result->bone_disp_name[i].Read(stream); - } - - // bone frame - uint32_t bone_frame_num; - stream->read((char*) &bone_frame_num, sizeof(uint32_t)); - result->bone_disp.resize(bone_frame_num); - for (uint32_t i = 0; i < bone_frame_num; i++) - { - result->bone_disp[i].Read(stream); - } - - // english name - bool english; - stream->read((char*) &english, sizeof(char)); - if (english) - { - result->header.ReadExtension(stream); - for (uint32_t i = 0; i < bone_num; i++) - { - result->bones[i].ReadExpantion(stream); - } - for (uint32_t i = 0; i < face_num; i++) - { - if (result->faces[i].type == pmd::FaceCategory::Base) - { - continue; - } - result->faces[i].ReadExpantion(stream); - } - for (uint32_t i = 0; i < result->bone_disp_name.size(); i++) - { - result->bone_disp_name[i].ReadExpantion(stream); - } - } - - // toon textures - if (stream->peek() == std::ios::traits_type::eof()) - { - result->toon_filenames.clear(); - } - else { - result->toon_filenames.resize(10); - for (uint32_t i = 0; i < 10; i++) - { - stream->read(buffer, 100); - result->toon_filenames[i] = std::string(buffer); - } - } - - // physics - if (stream->peek() == std::ios::traits_type::eof()) - { - result->rigid_bodies.clear(); - result->constraints.clear(); - } - else { - uint32_t rigid_body_num; - stream->read((char*) &rigid_body_num, sizeof(uint32_t)); - result->rigid_bodies.resize(rigid_body_num); - for (uint32_t i = 0; i < rigid_body_num; i++) - { - result->rigid_bodies[i].Read(stream); - } - uint32_t constraint_num; - stream->read((char*) &constraint_num, sizeof(uint32_t)); - result->constraints.resize(constraint_num); - for (uint32_t i = 0; i < constraint_num; i++) - { - result->constraints[i].Read(stream); - } - } - - if (stream->peek() != std::ios::traits_type::eof()) - { - std::cerr << "there is unknown data" << std::endl; - } - - return result; - } - }; -} diff --git a/thirdparty/assimp/code/MMD/MMDPmxParser.cpp b/thirdparty/assimp/code/MMD/MMDPmxParser.cpp deleted file mode 100644 index 80f0986dd7..0000000000 --- a/thirdparty/assimp/code/MMD/MMDPmxParser.cpp +++ /dev/null @@ -1,608 +0,0 @@ -/* -Open Asset Import Library (assimp) ----------------------------------------------------------------------- - -Copyright (c) 2006-2019, assimp team - - -All rights reserved. - -Redistribution and use of this software in source and binary forms, -with or without modification, are permitted provided that the -following conditions are met: - -* Redistributions of source code must retain the above -copyright notice, this list of conditions and the -following disclaimer. - -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the -following disclaimer in the documentation and/or other -materials provided with the distribution. - -* Neither the name of the assimp team, nor the names of its -contributors may be used to endorse or promote products -derived from this software without specific prior -written permission of the assimp team. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- -*/ -#include <utility> -#include "MMDPmxParser.h" -#include <assimp/StringUtils.h> -#ifdef ASSIMP_USE_HUNTER -# include <utf8/utf8.h> -#else -# include "../contrib/utf8cpp/source/utf8.h" -#endif -#include <assimp/Exceptional.h> - -namespace pmx -{ - int ReadIndex(std::istream *stream, int size) - { - switch (size) - { - case 1: - uint8_t tmp8; - stream->read((char*) &tmp8, sizeof(uint8_t)); - if (255 == tmp8) - { - return -1; - } - else { - return (int) tmp8; - } - case 2: - uint16_t tmp16; - stream->read((char*) &tmp16, sizeof(uint16_t)); - if (65535 == tmp16) - { - return -1; - } - else { - return (int) tmp16; - } - case 4: - int tmp32; - stream->read((char*) &tmp32, sizeof(int)); - return tmp32; - default: - return -1; - } - } - - std::string ReadString(std::istream *stream, uint8_t encoding) - { - int size; - stream->read((char*) &size, sizeof(int)); - std::vector<char> buffer; - if (size == 0) - { - return std::string(""); - } - buffer.reserve(size); - stream->read((char*) buffer.data(), size); - if (encoding == 0) - { - // UTF16 to UTF8 - const uint16_t* sourceStart = (uint16_t*)buffer.data(); - const unsigned int targetSize = size * 3; // enough to encode - char *targetStart = new char[targetSize]; - std::memset(targetStart, 0, targetSize * sizeof(char)); - - utf8::utf16to8( sourceStart, sourceStart + size/2, targetStart ); - - std::string result(targetStart); - delete [] targetStart; - return result; - } - else - { - // the name is already UTF8 - return std::string((const char*)buffer.data(), size); - } - } - - void PmxSetting::Read(std::istream *stream) - { - uint8_t count; - stream->read((char*) &count, sizeof(uint8_t)); - if (count < 8) - { - throw DeadlyImportError("MMD: invalid size"); - } - stream->read((char*) &encoding, sizeof(uint8_t)); - stream->read((char*) &uv, sizeof(uint8_t)); - stream->read((char*) &vertex_index_size, sizeof(uint8_t)); - stream->read((char*) &texture_index_size, sizeof(uint8_t)); - stream->read((char*) &material_index_size, sizeof(uint8_t)); - stream->read((char*) &bone_index_size, sizeof(uint8_t)); - stream->read((char*) &morph_index_size, sizeof(uint8_t)); - stream->read((char*) &rigidbody_index_size, sizeof(uint8_t)); - uint8_t temp; - for (int i = 8; i < count; i++) - { - stream->read((char*)&temp, sizeof(uint8_t)); - } - } - - void PmxVertexSkinningBDEF1::Read(std::istream *stream, PmxSetting *setting) - { - this->bone_index = ReadIndex(stream, setting->bone_index_size); - } - - void PmxVertexSkinningBDEF2::Read(std::istream *stream, PmxSetting *setting) - { - this->bone_index1 = ReadIndex(stream, setting->bone_index_size); - this->bone_index2 = ReadIndex(stream, setting->bone_index_size); - stream->read((char*) &this->bone_weight, sizeof(float)); - } - - void PmxVertexSkinningBDEF4::Read(std::istream *stream, PmxSetting *setting) - { - this->bone_index1 = ReadIndex(stream, setting->bone_index_size); - this->bone_index2 = ReadIndex(stream, setting->bone_index_size); - this->bone_index3 = ReadIndex(stream, setting->bone_index_size); - this->bone_index4 = ReadIndex(stream, setting->bone_index_size); - stream->read((char*) &this->bone_weight1, sizeof(float)); - stream->read((char*) &this->bone_weight2, sizeof(float)); - stream->read((char*) &this->bone_weight3, sizeof(float)); - stream->read((char*) &this->bone_weight4, sizeof(float)); - } - - void PmxVertexSkinningSDEF::Read(std::istream *stream, PmxSetting *setting) - { - this->bone_index1 = ReadIndex(stream, setting->bone_index_size); - this->bone_index2 = ReadIndex(stream, setting->bone_index_size); - stream->read((char*) &this->bone_weight, sizeof(float)); - stream->read((char*) this->sdef_c, sizeof(float) * 3); - stream->read((char*) this->sdef_r0, sizeof(float) * 3); - stream->read((char*) this->sdef_r1, sizeof(float) * 3); - } - - void PmxVertexSkinningQDEF::Read(std::istream *stream, PmxSetting *setting) - { - this->bone_index1 = ReadIndex(stream, setting->bone_index_size); - this->bone_index2 = ReadIndex(stream, setting->bone_index_size); - this->bone_index3 = ReadIndex(stream, setting->bone_index_size); - this->bone_index4 = ReadIndex(stream, setting->bone_index_size); - stream->read((char*) &this->bone_weight1, sizeof(float)); - stream->read((char*) &this->bone_weight2, sizeof(float)); - stream->read((char*) &this->bone_weight3, sizeof(float)); - stream->read((char*) &this->bone_weight4, sizeof(float)); - } - - void PmxVertex::Read(std::istream *stream, PmxSetting *setting) - { - stream->read((char*) this->position, sizeof(float) * 3); - stream->read((char*) this->normal, sizeof(float) * 3); - stream->read((char*) this->uv, sizeof(float) * 2); - for (int i = 0; i < setting->uv; ++i) - { - stream->read((char*) this->uva[i], sizeof(float) * 4); - } - stream->read((char*) &this->skinning_type, sizeof(PmxVertexSkinningType)); - switch (this->skinning_type) - { - case PmxVertexSkinningType::BDEF1: - this->skinning = mmd::make_unique<PmxVertexSkinningBDEF1>(); - break; - case PmxVertexSkinningType::BDEF2: - this->skinning = mmd::make_unique<PmxVertexSkinningBDEF2>(); - break; - case PmxVertexSkinningType::BDEF4: - this->skinning = mmd::make_unique<PmxVertexSkinningBDEF4>(); - break; - case PmxVertexSkinningType::SDEF: - this->skinning = mmd::make_unique<PmxVertexSkinningSDEF>(); - break; - case PmxVertexSkinningType::QDEF: - this->skinning = mmd::make_unique<PmxVertexSkinningQDEF>(); - break; - default: - throw "invalid skinning type"; - } - this->skinning->Read(stream, setting); - stream->read((char*) &this->edge, sizeof(float)); - } - - void PmxMaterial::Read(std::istream *stream, PmxSetting *setting) - { - this->material_name = ReadString(stream, setting->encoding); - this->material_english_name = ReadString(stream, setting->encoding); - stream->read((char*) this->diffuse, sizeof(float) * 4); - stream->read((char*) this->specular, sizeof(float) * 3); - stream->read((char*) &this->specularlity, sizeof(float)); - stream->read((char*) this->ambient, sizeof(float) * 3); - stream->read((char*) &this->flag, sizeof(uint8_t)); - stream->read((char*) this->edge_color, sizeof(float) * 4); - stream->read((char*) &this->edge_size, sizeof(float)); - this->diffuse_texture_index = ReadIndex(stream, setting->texture_index_size); - this->sphere_texture_index = ReadIndex(stream, setting->texture_index_size); - stream->read((char*) &this->sphere_op_mode, sizeof(uint8_t)); - stream->read((char*) &this->common_toon_flag, sizeof(uint8_t)); - if (this->common_toon_flag) - { - stream->read((char*) &this->toon_texture_index, sizeof(uint8_t)); - } - else { - this->toon_texture_index = ReadIndex(stream, setting->texture_index_size); - } - this->memo = ReadString(stream, setting->encoding); - stream->read((char*) &this->index_count, sizeof(int)); - } - - void PmxIkLink::Read(std::istream *stream, PmxSetting *setting) - { - this->link_target = ReadIndex(stream, setting->bone_index_size); - stream->read((char*) &this->angle_lock, sizeof(uint8_t)); - if (angle_lock == 1) - { - stream->read((char*) this->max_radian, sizeof(float) * 3); - stream->read((char*) this->min_radian, sizeof(float) * 3); - } - } - - void PmxBone::Read(std::istream *stream, PmxSetting *setting) - { - this->bone_name = ReadString(stream, setting->encoding); - this->bone_english_name = ReadString(stream, setting->encoding); - stream->read((char*) this->position, sizeof(float) * 3); - this->parent_index = ReadIndex(stream, setting->bone_index_size); - stream->read((char*) &this->level, sizeof(int)); - stream->read((char*) &this->bone_flag, sizeof(uint16_t)); - if (this->bone_flag & 0x0001) { - this->target_index = ReadIndex(stream, setting->bone_index_size); - } - else { - stream->read((char*)this->offset, sizeof(float) * 3); - } - if (this->bone_flag & (0x0100 | 0x0200)) { - this->grant_parent_index = ReadIndex(stream, setting->bone_index_size); - stream->read((char*) &this->grant_weight, sizeof(float)); - } - if (this->bone_flag & 0x0400) { - stream->read((char*)this->lock_axis_orientation, sizeof(float) * 3); - } - if (this->bone_flag & 0x0800) { - stream->read((char*)this->local_axis_x_orientation, sizeof(float) * 3); - stream->read((char*)this->local_axis_y_orientation, sizeof(float) * 3); - } - if (this->bone_flag & 0x2000) { - stream->read((char*) &this->key, sizeof(int)); - } - if (this->bone_flag & 0x0020) { - this->ik_target_bone_index = ReadIndex(stream, setting->bone_index_size); - stream->read((char*) &ik_loop, sizeof(int)); - stream->read((char*) &ik_loop_angle_limit, sizeof(float)); - stream->read((char*) &ik_link_count, sizeof(int)); - this->ik_links = mmd::make_unique<PmxIkLink []>(ik_link_count); - for (int i = 0; i < ik_link_count; i++) { - ik_links[i].Read(stream, setting); - } - } - } - - void PmxMorphVertexOffset::Read(std::istream *stream, PmxSetting *setting) - { - this->vertex_index = ReadIndex(stream, setting->vertex_index_size); - stream->read((char*)this->position_offset, sizeof(float) * 3); - } - - void PmxMorphUVOffset::Read(std::istream *stream, PmxSetting *setting) - { - this->vertex_index = ReadIndex(stream, setting->vertex_index_size); - stream->read((char*)this->uv_offset, sizeof(float) * 4); - } - - void PmxMorphBoneOffset::Read(std::istream *stream, PmxSetting *setting) - { - this->bone_index = ReadIndex(stream, setting->bone_index_size); - stream->read((char*)this->translation, sizeof(float) * 3); - stream->read((char*)this->rotation, sizeof(float) * 4); - } - - void PmxMorphMaterialOffset::Read(std::istream *stream, PmxSetting *setting) - { - this->material_index = ReadIndex(stream, setting->material_index_size); - stream->read((char*) &this->offset_operation, sizeof(uint8_t)); - stream->read((char*)this->diffuse, sizeof(float) * 4); - stream->read((char*)this->specular, sizeof(float) * 3); - stream->read((char*) &this->specularity, sizeof(float)); - stream->read((char*)this->ambient, sizeof(float) * 3); - stream->read((char*)this->edge_color, sizeof(float) * 4); - stream->read((char*) &this->edge_size, sizeof(float)); - stream->read((char*)this->texture_argb, sizeof(float) * 4); - stream->read((char*)this->sphere_texture_argb, sizeof(float) * 4); - stream->read((char*)this->toon_texture_argb, sizeof(float) * 4); - } - - void PmxMorphGroupOffset::Read(std::istream *stream, PmxSetting *setting) - { - this->morph_index = ReadIndex(stream, setting->morph_index_size); - stream->read((char*) &this->morph_weight, sizeof(float)); - } - - void PmxMorphFlipOffset::Read(std::istream *stream, PmxSetting *setting) - { - this->morph_index = ReadIndex(stream, setting->morph_index_size); - stream->read((char*) &this->morph_value, sizeof(float)); - } - - void PmxMorphImplusOffset::Read(std::istream *stream, PmxSetting *setting) - { - this->rigid_body_index = ReadIndex(stream, setting->rigidbody_index_size); - stream->read((char*) &this->is_local, sizeof(uint8_t)); - stream->read((char*)this->velocity, sizeof(float) * 3); - stream->read((char*)this->angular_torque, sizeof(float) * 3); - } - - void PmxMorph::Read(std::istream *stream, PmxSetting *setting) - { - this->morph_name = ReadString(stream, setting->encoding); - this->morph_english_name = ReadString(stream, setting->encoding); - stream->read((char*) &category, sizeof(MorphCategory)); - stream->read((char*) &morph_type, sizeof(MorphType)); - stream->read((char*) &this->offset_count, sizeof(int)); - switch (this->morph_type) - { - case MorphType::Group: - group_offsets = mmd::make_unique<PmxMorphGroupOffset []>(this->offset_count); - for (int i = 0; i < offset_count; i++) - { - group_offsets[i].Read(stream, setting); - } - break; - case MorphType::Vertex: - vertex_offsets = mmd::make_unique<PmxMorphVertexOffset []>(this->offset_count); - for (int i = 0; i < offset_count; i++) - { - vertex_offsets[i].Read(stream, setting); - } - break; - case MorphType::Bone: - bone_offsets = mmd::make_unique<PmxMorphBoneOffset []>(this->offset_count); - for (int i = 0; i < offset_count; i++) - { - bone_offsets[i].Read(stream, setting); - } - break; - case MorphType::Matrial: - material_offsets = mmd::make_unique<PmxMorphMaterialOffset []>(this->offset_count); - for (int i = 0; i < offset_count; i++) - { - material_offsets[i].Read(stream, setting); - } - break; - case MorphType::UV: - case MorphType::AdditionalUV1: - case MorphType::AdditionalUV2: - case MorphType::AdditionalUV3: - case MorphType::AdditionalUV4: - uv_offsets = mmd::make_unique<PmxMorphUVOffset []>(this->offset_count); - for (int i = 0; i < offset_count; i++) - { - uv_offsets[i].Read(stream, setting); - } - break; - default: - throw DeadlyImportError("MMD: unknown morth type"); - } - } - - void PmxFrameElement::Read(std::istream *stream, PmxSetting *setting) - { - stream->read((char*) &this->element_target, sizeof(uint8_t)); - if (this->element_target == 0x00) - { - this->index = ReadIndex(stream, setting->bone_index_size); - } - else { - this->index = ReadIndex(stream, setting->morph_index_size); - } - } - - void PmxFrame::Read(std::istream *stream, PmxSetting *setting) - { - this->frame_name = ReadString(stream, setting->encoding); - this->frame_english_name = ReadString(stream, setting->encoding); - stream->read((char*) &this->frame_flag, sizeof(uint8_t)); - stream->read((char*) &this->element_count, sizeof(int)); - this->elements = mmd::make_unique<PmxFrameElement []>(this->element_count); - for (int i = 0; i < this->element_count; i++) - { - this->elements[i].Read(stream, setting); - } - } - - void PmxRigidBody::Read(std::istream *stream, PmxSetting *setting) - { - this->girid_body_name = ReadString(stream, setting->encoding); - this->girid_body_english_name = ReadString(stream, setting->encoding); - this->target_bone = ReadIndex(stream, setting->bone_index_size); - stream->read((char*) &this->group, sizeof(uint8_t)); - stream->read((char*) &this->mask, sizeof(uint16_t)); - stream->read((char*) &this->shape, sizeof(uint8_t)); - stream->read((char*) this->size, sizeof(float) * 3); - stream->read((char*) this->position, sizeof(float) * 3); - stream->read((char*) this->orientation, sizeof(float) * 3); - stream->read((char*) &this->mass, sizeof(float)); - stream->read((char*) &this->move_attenuation, sizeof(float)); - stream->read((char*) &this->rotation_attenuation, sizeof(float)); - stream->read((char*) &this->repulsion, sizeof(float)); - stream->read((char*) &this->friction, sizeof(float)); - stream->read((char*) &this->physics_calc_type, sizeof(uint8_t)); - } - - void PmxJointParam::Read(std::istream *stream, PmxSetting *setting) - { - this->rigid_body1 = ReadIndex(stream, setting->rigidbody_index_size); - this->rigid_body2 = ReadIndex(stream, setting->rigidbody_index_size); - stream->read((char*) this->position, sizeof(float) * 3); - stream->read((char*) this->orientaiton, sizeof(float) * 3); - stream->read((char*) this->move_limitation_min, sizeof(float) * 3); - stream->read((char*) this->move_limitation_max, sizeof(float) * 3); - stream->read((char*) this->rotation_limitation_min, sizeof(float) * 3); - stream->read((char*) this->rotation_limitation_max, sizeof(float) * 3); - stream->read((char*) this->spring_move_coefficient, sizeof(float) * 3); - stream->read((char*) this->spring_rotation_coefficient, sizeof(float) * 3); - } - - void PmxJoint::Read(std::istream *stream, PmxSetting *setting) - { - this->joint_name = ReadString(stream, setting->encoding); - this->joint_english_name = ReadString(stream, setting->encoding); - stream->read((char*) &this->joint_type, sizeof(uint8_t)); - this->param.Read(stream, setting); - } - - void PmxAncherRigidBody::Read(std::istream *stream, PmxSetting *setting) - { - this->related_rigid_body = ReadIndex(stream, setting->rigidbody_index_size); - this->related_vertex = ReadIndex(stream, setting->vertex_index_size); - stream->read((char*) &this->is_near, sizeof(uint8_t)); - } - - void PmxSoftBody::Read(std::istream * /*stream*/, PmxSetting * /*setting*/) - { - std::cerr << "Not Implemented Exception" << std::endl; - throw DeadlyImportError("MMD: Not Implemented Exception"); - } - - void PmxModel::Init() - { - this->version = 0.0f; - this->model_name.clear(); - this->model_english_name.clear(); - this->model_comment.clear(); - this->model_english_comment.clear(); - this->vertex_count = 0; - this->vertices = nullptr; - this->index_count = 0; - this->indices = nullptr; - this->texture_count = 0; - this->textures = nullptr; - this->material_count = 0; - this->materials = nullptr; - this->bone_count = 0; - this->bones = nullptr; - this->morph_count = 0; - this->morphs = nullptr; - this->frame_count = 0; - this->frames = nullptr; - this->rigid_body_count = 0; - this->rigid_bodies = nullptr; - this->joint_count = 0; - this->joints = nullptr; - this->soft_body_count = 0; - this->soft_bodies = nullptr; - } - - void PmxModel::Read(std::istream *stream) - { - char magic[4]; - stream->read((char*) magic, sizeof(char) * 4); - if (magic[0] != 0x50 || magic[1] != 0x4d || magic[2] != 0x58 || magic[3] != 0x20) - { - std::cerr << "invalid magic number." << std::endl; - throw DeadlyImportError("MMD: invalid magic number."); - } - stream->read((char*) &version, sizeof(float)); - if (version != 2.0f && version != 2.1f) - { - std::cerr << "this is not ver2.0 or ver2.1 but " << version << "." << std::endl; - throw DeadlyImportError("MMD: this is not ver2.0 or ver2.1 but " + to_string(version)); - } - this->setting.Read(stream); - - this->model_name = ReadString(stream, setting.encoding); - this->model_english_name = ReadString(stream, setting.encoding); - this->model_comment = ReadString(stream, setting.encoding); - this->model_english_comment = ReadString(stream, setting.encoding); - - // read vertices - stream->read((char*) &vertex_count, sizeof(int)); - this->vertices = mmd::make_unique<PmxVertex []>(vertex_count); - for (int i = 0; i < vertex_count; i++) - { - vertices[i].Read(stream, &setting); - } - - // read indices - stream->read((char*) &index_count, sizeof(int)); - this->indices = mmd::make_unique<int []>(index_count); - for (int i = 0; i < index_count; i++) - { - this->indices[i] = ReadIndex(stream, setting.vertex_index_size); - } - - // read texture names - stream->read((char*) &texture_count, sizeof(int)); - this->textures = mmd::make_unique<std::string []>(texture_count); - for (int i = 0; i < texture_count; i++) - { - this->textures[i] = ReadString(stream, setting.encoding); - } - - // read materials - stream->read((char*) &material_count, sizeof(int)); - this->materials = mmd::make_unique<PmxMaterial []>(material_count); - for (int i = 0; i < material_count; i++) - { - this->materials[i].Read(stream, &setting); - } - - // read bones - stream->read((char*) &this->bone_count, sizeof(int)); - this->bones = mmd::make_unique<PmxBone []>(this->bone_count); - for (int i = 0; i < this->bone_count; i++) - { - this->bones[i].Read(stream, &setting); - } - - // read morphs - stream->read((char*) &this->morph_count, sizeof(int)); - this->morphs = mmd::make_unique<PmxMorph []>(this->morph_count); - for (int i = 0; i < this->morph_count; i++) - { - this->morphs[i].Read(stream, &setting); - } - - // read display frames - stream->read((char*) &this->frame_count, sizeof(int)); - this->frames = mmd::make_unique<PmxFrame []>(this->frame_count); - for (int i = 0; i < this->frame_count; i++) - { - this->frames[i].Read(stream, &setting); - } - - // read rigid bodies - stream->read((char*) &this->rigid_body_count, sizeof(int)); - this->rigid_bodies = mmd::make_unique<PmxRigidBody []>(this->rigid_body_count); - for (int i = 0; i < this->rigid_body_count; i++) - { - this->rigid_bodies[i].Read(stream, &setting); - } - - // read joints - stream->read((char*) &this->joint_count, sizeof(int)); - this->joints = mmd::make_unique<PmxJoint []>(this->joint_count); - for (int i = 0; i < this->joint_count; i++) - { - this->joints[i].Read(stream, &setting); - } - } -} diff --git a/thirdparty/assimp/code/MMD/MMDPmxParser.h b/thirdparty/assimp/code/MMD/MMDPmxParser.h deleted file mode 100644 index cf523a1298..0000000000 --- a/thirdparty/assimp/code/MMD/MMDPmxParser.h +++ /dev/null @@ -1,782 +0,0 @@ -/* -Open Asset Import Library (assimp) ----------------------------------------------------------------------- - -Copyright (c) 2006-2019, assimp team - - -All rights reserved. - -Redistribution and use of this software in source and binary forms, -with or without modification, are permitted provided that the -following conditions are met: - -* Redistributions of source code must retain the above -copyright notice, this list of conditions and the -following disclaimer. - -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the -following disclaimer in the documentation and/or other -materials provided with the distribution. - -* Neither the name of the assimp team, nor the names of its -contributors may be used to endorse or promote products -derived from this software without specific prior -written permission of the assimp team. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- -*/ -#pragma once - -#include <vector> -#include <string> -#include <iostream> -#include <fstream> -#include <memory> -#include "MMDCpp14.h" - -namespace pmx -{ - class PmxSetting - { - public: - PmxSetting() - : encoding(0) - , uv(0) - , vertex_index_size(0) - , texture_index_size(0) - , material_index_size(0) - , bone_index_size(0) - , morph_index_size(0) - , rigidbody_index_size(0) - {} - - uint8_t encoding; - uint8_t uv; - uint8_t vertex_index_size; - uint8_t texture_index_size; - uint8_t material_index_size; - uint8_t bone_index_size; - uint8_t morph_index_size; - uint8_t rigidbody_index_size; - void Read(std::istream *stream); - }; - - enum class PmxVertexSkinningType : uint8_t - { - BDEF1 = 0, - BDEF2 = 1, - BDEF4 = 2, - SDEF = 3, - QDEF = 4, - }; - - class PmxVertexSkinning - { - public: - virtual void Read(std::istream *stream, PmxSetting *setting) = 0; - virtual ~PmxVertexSkinning() {} - }; - - class PmxVertexSkinningBDEF1 : public PmxVertexSkinning - { - public: - PmxVertexSkinningBDEF1() - : bone_index(0) - {} - - int bone_index; - void Read(std::istream *stresam, PmxSetting *setting); - }; - - class PmxVertexSkinningBDEF2 : public PmxVertexSkinning - { - public: - PmxVertexSkinningBDEF2() - : bone_index1(0) - , bone_index2(0) - , bone_weight(0.0f) - {} - - int bone_index1; - int bone_index2; - float bone_weight; - void Read(std::istream *stresam, PmxSetting *setting); - }; - - class PmxVertexSkinningBDEF4 : public PmxVertexSkinning - { - public: - PmxVertexSkinningBDEF4() - : bone_index1(0) - , bone_index2(0) - , bone_index3(0) - , bone_index4(0) - , bone_weight1(0.0f) - , bone_weight2(0.0f) - , bone_weight3(0.0f) - , bone_weight4(0.0f) - {} - - int bone_index1; - int bone_index2; - int bone_index3; - int bone_index4; - float bone_weight1; - float bone_weight2; - float bone_weight3; - float bone_weight4; - void Read(std::istream *stresam, PmxSetting *setting); - }; - - class PmxVertexSkinningSDEF : public PmxVertexSkinning - { - public: - PmxVertexSkinningSDEF() - : bone_index1(0) - , bone_index2(0) - , bone_weight(0.0f) - { - for (int i = 0; i < 3; ++i) { - sdef_c[i] = 0.0f; - sdef_r0[i] = 0.0f; - sdef_r1[i] = 0.0f; - } - } - - int bone_index1; - int bone_index2; - float bone_weight; - float sdef_c[3]; - float sdef_r0[3]; - float sdef_r1[3]; - void Read(std::istream *stresam, PmxSetting *setting); - }; - - class PmxVertexSkinningQDEF : public PmxVertexSkinning - { - public: - PmxVertexSkinningQDEF() - : bone_index1(0) - , bone_index2(0) - , bone_index3(0) - , bone_index4(0) - , bone_weight1(0.0f) - , bone_weight2(0.0f) - , bone_weight3(0.0f) - , bone_weight4(0.0f) - {} - - int bone_index1; - int bone_index2; - int bone_index3; - int bone_index4; - float bone_weight1; - float bone_weight2; - float bone_weight3; - float bone_weight4; - void Read(std::istream *stresam, PmxSetting *setting); - }; - - class PmxVertex - { - public: - PmxVertex() - : edge(0.0f) - { - uv[0] = uv[1] = 0.0f; - for (int i = 0; i < 3; ++i) { - position[i] = 0.0f; - normal[i] = 0.0f; - } - for (int i = 0; i < 4; ++i) { - for (int k = 0; k < 4; ++k) { - uva[i][k] = 0.0f; - } - } - } - - float position[3]; - float normal[3]; - float uv[2]; - float uva[4][4]; - PmxVertexSkinningType skinning_type; - std::unique_ptr<PmxVertexSkinning> skinning; - float edge; - void Read(std::istream *stream, PmxSetting *setting); - }; - - class PmxMaterial - { - public: - PmxMaterial() - : specularlity(0.0f) - , flag(0) - , edge_size(0.0f) - , diffuse_texture_index(0) - , sphere_texture_index(0) - , sphere_op_mode(0) - , common_toon_flag(0) - , toon_texture_index(0) - , index_count(0) - { - for (int i = 0; i < 3; ++i) { - specular[i] = 0.0f; - ambient[i] = 0.0f; - edge_color[i] = 0.0f; - } - for (int i = 0; i < 4; ++i) { - diffuse[i] = 0.0f; - } - } - - std::string material_name; - std::string material_english_name; - float diffuse[4]; - float specular[3]; - float specularlity; - float ambient[3]; - uint8_t flag; - float edge_color[4]; - float edge_size; - int diffuse_texture_index; - int sphere_texture_index; - uint8_t sphere_op_mode; - uint8_t common_toon_flag; - int toon_texture_index; - std::string memo; - int index_count; - void Read(std::istream *stream, PmxSetting *setting); - }; - - class PmxIkLink - { - public: - PmxIkLink() - : link_target(0) - , angle_lock(0) - { - for (int i = 0; i < 3; ++i) { - max_radian[i] = 0.0f; - min_radian[i] = 0.0f; - } - } - - int link_target; - uint8_t angle_lock; - float max_radian[3]; - float min_radian[3]; - void Read(std::istream *stream, PmxSetting *settingn); - }; - - class PmxBone - { - public: - PmxBone() - : parent_index(0) - , level(0) - , bone_flag(0) - , target_index(0) - , grant_parent_index(0) - , grant_weight(0.0f) - , key(0) - , ik_target_bone_index(0) - , ik_loop(0) - , ik_loop_angle_limit(0.0f) - , ik_link_count(0) - { - for (int i = 0; i < 3; ++i) { - position[i] = 0.0f; - offset[i] = 0.0f; - lock_axis_orientation[i] = 0.0f; - local_axis_x_orientation[i] = 0.0f; - local_axis_y_orientation[i] = 0.0f; - } - } - - std::string bone_name; - std::string bone_english_name; - float position[3]; - int parent_index; - int level; - uint16_t bone_flag; - float offset[3]; - int target_index; - int grant_parent_index; - float grant_weight; - float lock_axis_orientation[3]; - float local_axis_x_orientation[3]; - float local_axis_y_orientation[3]; - int key; - int ik_target_bone_index; - int ik_loop; - float ik_loop_angle_limit; - int ik_link_count; - std::unique_ptr<PmxIkLink []> ik_links; - void Read(std::istream *stream, PmxSetting *setting); - }; - - enum class MorphType : uint8_t - { - Group = 0, - Vertex = 1, - Bone = 2, - UV = 3, - AdditionalUV1 = 4, - AdditionalUV2 = 5, - AdditionalUV3 = 6, - AdditionalUV4 = 7, - Matrial = 8, - Flip = 9, - Implus = 10, - }; - - enum class MorphCategory : uint8_t - { - ReservedCategory = 0, - Eyebrow = 1, - Eye = 2, - Mouth = 3, - Other = 4, - }; - - class PmxMorphOffset - { - public: - void virtual Read(std::istream *stream, PmxSetting *setting) = 0; - }; - - class PmxMorphVertexOffset : public PmxMorphOffset - { - public: - PmxMorphVertexOffset() - : vertex_index(0) - { - for (int i = 0; i < 3; ++i) { - position_offset[i] = 0.0f; - } - } - int vertex_index; - float position_offset[3]; - void Read(std::istream *stream, PmxSetting *setting); //override; - }; - - class PmxMorphUVOffset : public PmxMorphOffset - { - public: - PmxMorphUVOffset() - : vertex_index(0) - { - for (int i = 0; i < 4; ++i) { - uv_offset[i] = 0.0f; - } - } - int vertex_index; - float uv_offset[4]; - void Read(std::istream *stream, PmxSetting *setting); //override; - }; - - class PmxMorphBoneOffset : public PmxMorphOffset - { - public: - PmxMorphBoneOffset() - : bone_index(0) - { - for (int i = 0; i < 3; ++i) { - translation[i] = 0.0f; - } - for (int i = 0; i < 4; ++i) { - rotation[i] = 0.0f; - } - } - int bone_index; - float translation[3]; - float rotation[4]; - void Read(std::istream *stream, PmxSetting *setting); //override; - }; - - class PmxMorphMaterialOffset : public PmxMorphOffset - { - public: - PmxMorphMaterialOffset() - : specularity(0.0f) - , edge_size(0.0f) - { - for (int i = 0; i < 3; ++i) { - specular[i] = 0.0f; - ambient[i] = 0.0f; - } - for (int i = 0; i < 4; ++i) { - diffuse[i] = 0.0f; - edge_color[i] = 0.0f; - texture_argb[i] = 0.0f; - sphere_texture_argb[i] = 0.0f; - toon_texture_argb[i] = 0.0f; - } - } - int material_index; - uint8_t offset_operation; - float diffuse[4]; - float specular[3]; - float specularity; - float ambient[3]; - float edge_color[4]; - float edge_size; - float texture_argb[4]; - float sphere_texture_argb[4]; - float toon_texture_argb[4]; - void Read(std::istream *stream, PmxSetting *setting); //override; - }; - - class PmxMorphGroupOffset : public PmxMorphOffset - { - public: - PmxMorphGroupOffset() - : morph_index(0) - , morph_weight(0.0f) - {} - int morph_index; - float morph_weight; - void Read(std::istream *stream, PmxSetting *setting); //override; - }; - - class PmxMorphFlipOffset : public PmxMorphOffset - { - public: - PmxMorphFlipOffset() - : morph_index(0) - , morph_value(0.0f) - {} - int morph_index; - float morph_value; - void Read(std::istream *stream, PmxSetting *setting); //override; - }; - - class PmxMorphImplusOffset : public PmxMorphOffset - { - public: - PmxMorphImplusOffset() - : rigid_body_index(0) - , is_local(0) - { - for (int i = 0; i < 3; ++i) { - velocity[i] = 0.0f; - angular_torque[i] = 0.0f; - } - } - int rigid_body_index; - uint8_t is_local; - float velocity[3]; - float angular_torque[3]; - void Read(std::istream *stream, PmxSetting *setting); //override; - }; - - class PmxMorph - { - public: - PmxMorph() - : offset_count(0) - { - } - std::string morph_name; - std::string morph_english_name; - MorphCategory category; - MorphType morph_type; - int offset_count; - std::unique_ptr<PmxMorphVertexOffset []> vertex_offsets; - std::unique_ptr<PmxMorphUVOffset []> uv_offsets; - std::unique_ptr<PmxMorphBoneOffset []> bone_offsets; - std::unique_ptr<PmxMorphMaterialOffset []> material_offsets; - std::unique_ptr<PmxMorphGroupOffset []> group_offsets; - std::unique_ptr<PmxMorphFlipOffset []> flip_offsets; - std::unique_ptr<PmxMorphImplusOffset []> implus_offsets; - void Read(std::istream *stream, PmxSetting *setting); - }; - - class PmxFrameElement - { - public: - PmxFrameElement() - : element_target(0) - , index(0) - { - } - uint8_t element_target; - int index; - void Read(std::istream *stream, PmxSetting *setting); - }; - - class PmxFrame - { - public: - PmxFrame() - : frame_flag(0) - , element_count(0) - { - } - std::string frame_name; - std::string frame_english_name; - uint8_t frame_flag; - int element_count; - std::unique_ptr<PmxFrameElement []> elements; - void Read(std::istream *stream, PmxSetting *setting); - }; - - class PmxRigidBody - { - public: - PmxRigidBody() - : target_bone(0) - , group(0) - , mask(0) - , shape(0) - , mass(0.0f) - , move_attenuation(0.0f) - , rotation_attenuation(0.0f) - , repulsion(0.0f) - , friction(0.0f) - , physics_calc_type(0) - { - for (int i = 0; i < 3; ++i) { - size[i] = 0.0f; - position[i] = 0.0f; - orientation[i] = 0.0f; - } - } - std::string girid_body_name; - std::string girid_body_english_name; - int target_bone; - uint8_t group; - uint16_t mask; - uint8_t shape; - float size[3]; - float position[3]; - float orientation[3]; - float mass; - float move_attenuation; - float rotation_attenuation; - float repulsion; - float friction; - uint8_t physics_calc_type; - void Read(std::istream *stream, PmxSetting *setting); - }; - - enum class PmxJointType : uint8_t - { - Generic6DofSpring = 0, - Generic6Dof = 1, - Point2Point = 2, - ConeTwist = 3, - Slider = 5, - Hinge = 6 - }; - - class PmxJointParam - { - public: - PmxJointParam() - : rigid_body1(0) - , rigid_body2(0) - { - for (int i = 0; i < 3; ++i) { - position[i] = 0.0f; - orientaiton[i] = 0.0f; - move_limitation_min[i] = 0.0f; - move_limitation_max[i] = 0.0f; - rotation_limitation_min[i] = 0.0f; - rotation_limitation_max[i] = 0.0f; - spring_move_coefficient[i] = 0.0f; - spring_rotation_coefficient[i] = 0.0f; - } - } - int rigid_body1; - int rigid_body2; - float position[3]; - float orientaiton[3]; - float move_limitation_min[3]; - float move_limitation_max[3]; - float rotation_limitation_min[3]; - float rotation_limitation_max[3]; - float spring_move_coefficient[3]; - float spring_rotation_coefficient[3]; - void Read(std::istream *stream, PmxSetting *setting); - }; - - class PmxJoint - { - public: - std::string joint_name; - std::string joint_english_name; - PmxJointType joint_type; - PmxJointParam param; - void Read(std::istream *stream, PmxSetting *setting); - }; - - enum PmxSoftBodyFlag : uint8_t - { - BLink = 0x01, - Cluster = 0x02, - Link = 0x04 - }; - - class PmxAncherRigidBody - { - public: - PmxAncherRigidBody() - : related_rigid_body(0) - , related_vertex(0) - , is_near(false) - {} - int related_rigid_body; - int related_vertex; - bool is_near; - void Read(std::istream *stream, PmxSetting *setting); - }; - - class PmxSoftBody - { - public: - PmxSoftBody() - : shape(0) - , target_material(0) - , group(0) - , mask(0) - , blink_distance(0) - , cluster_count(0) - , mass(0.0) - , collisioni_margin(0.0) - , aero_model(0) - , VCF(0.0f) - , DP(0.0f) - , DG(0.0f) - , LF(0.0f) - , PR(0.0f) - , VC(0.0f) - , DF(0.0f) - , MT(0.0f) - , CHR(0.0f) - , KHR(0.0f) - , SHR(0.0f) - , AHR(0.0f) - , SRHR_CL(0.0f) - , SKHR_CL(0.0f) - , SSHR_CL(0.0f) - , SR_SPLT_CL(0.0f) - , SK_SPLT_CL(0.0f) - , SS_SPLT_CL(0.0f) - , V_IT(0) - , P_IT(0) - , D_IT(0) - , C_IT(0) - , LST(0.0f) - , AST(0.0f) - , VST(0.0f) - , anchor_count(0) - , pin_vertex_count(0) - {} - std::string soft_body_name; - std::string soft_body_english_name; - uint8_t shape; - int target_material; - uint8_t group; - uint16_t mask; - PmxSoftBodyFlag flag; - int blink_distance; - int cluster_count; - float mass; - float collisioni_margin; - int aero_model; - float VCF; - float DP; - float DG; - float LF; - float PR; - float VC; - float DF; - float MT; - float CHR; - float KHR; - float SHR; - float AHR; - float SRHR_CL; - float SKHR_CL; - float SSHR_CL; - float SR_SPLT_CL; - float SK_SPLT_CL; - float SS_SPLT_CL; - int V_IT; - int P_IT; - int D_IT; - int C_IT; - float LST; - float AST; - float VST; - int anchor_count; - std::unique_ptr<PmxAncherRigidBody []> anchers; - int pin_vertex_count; - std::unique_ptr<int []> pin_vertices; - void Read(std::istream *stream, PmxSetting *setting); - }; - - class PmxModel - { - public: - PmxModel() - : version(0.0f) - , vertex_count(0) - , index_count(0) - , texture_count(0) - , material_count(0) - , bone_count(0) - , morph_count(0) - , frame_count(0) - , rigid_body_count(0) - , joint_count(0) - , soft_body_count(0) - {} - - float version; - PmxSetting setting; - std::string model_name; - std::string model_english_name; - std::string model_comment; - std::string model_english_comment; - int vertex_count; - std::unique_ptr<PmxVertex []> vertices; - int index_count; - std::unique_ptr<int []> indices; - int texture_count; - std::unique_ptr< std::string []> textures; - int material_count; - std::unique_ptr<PmxMaterial []> materials; - int bone_count; - std::unique_ptr<PmxBone []> bones; - int morph_count; - std::unique_ptr<PmxMorph []> morphs; - int frame_count; - std::unique_ptr<PmxFrame [] > frames; - int rigid_body_count; - std::unique_ptr<PmxRigidBody []> rigid_bodies; - int joint_count; - std::unique_ptr<PmxJoint []> joints; - int soft_body_count; - std::unique_ptr<PmxSoftBody []> soft_bodies; - void Init(); - void Read(std::istream *stream); - //static std::unique_ptr<PmxModel> ReadFromFile(const char *filename); - //static std::unique_ptr<PmxModel> ReadFromStream(std::istream *stream); - }; -} diff --git a/thirdparty/assimp/code/MMD/MMDVmdParser.h b/thirdparty/assimp/code/MMD/MMDVmdParser.h deleted file mode 100644 index 947c3a2422..0000000000 --- a/thirdparty/assimp/code/MMD/MMDVmdParser.h +++ /dev/null @@ -1,376 +0,0 @@ -/* -Open Asset Import Library (assimp) ----------------------------------------------------------------------- - -Copyright (c) 2006-2019, assimp team - - -All rights reserved. - -Redistribution and use of this software in source and binary forms, -with or without modification, are permitted provided that the -following conditions are met: - -* Redistributions of source code must retain the above -copyright notice, this list of conditions and the -following disclaimer. - -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the -following disclaimer in the documentation and/or other -materials provided with the distribution. - -* Neither the name of the assimp team, nor the names of its -contributors may be used to endorse or promote products -derived from this software without specific prior -written permission of the assimp team. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - ----------------------------------------------------------------------- -*/ -#pragma once - -#include <vector> -#include <string> -#include <memory> -#include <iostream> -#include <fstream> -#include <ostream> -#include "MMDCpp14.h" - -namespace vmd -{ - class VmdBoneFrame - { - public: - std::string name; - int frame; - float position[3]; - float orientation[4]; - char interpolation[4][4][4]; - - void Read(std::istream* stream) - { - char buffer[15]; - stream->read((char*) buffer, sizeof(char)*15); - name = std::string(buffer); - stream->read((char*) &frame, sizeof(int)); - stream->read((char*) position, sizeof(float)*3); - stream->read((char*) orientation, sizeof(float)*4); - stream->read((char*) interpolation, sizeof(char) * 4 * 4 * 4); - } - - void Write(std::ostream* stream) - { - stream->write((char*)name.c_str(), sizeof(char) * 15); - stream->write((char*)&frame, sizeof(int)); - stream->write((char*)position, sizeof(float) * 3); - stream->write((char*)orientation, sizeof(float) * 4); - stream->write((char*)interpolation, sizeof(char) * 4 * 4 * 4); - } - }; - - class VmdFaceFrame - { - public: - std::string face_name; - float weight; - uint32_t frame; - - void Read(std::istream* stream) - { - char buffer[15]; - stream->read((char*) &buffer, sizeof(char) * 15); - face_name = std::string(buffer); - stream->read((char*) &frame, sizeof(int)); - stream->read((char*) &weight, sizeof(float)); - } - - void Write(std::ostream* stream) - { - stream->write((char*)face_name.c_str(), sizeof(char) * 15); - stream->write((char*)&frame, sizeof(int)); - stream->write((char*)&weight, sizeof(float)); - } - }; - - class VmdCameraFrame - { - public: - int frame; - float distance; - float position[3]; - float orientation[3]; - char interpolation[6][4]; - float angle; - char unknown[3]; - - void Read(std::istream *stream) - { - stream->read((char*) &frame, sizeof(int)); - stream->read((char*) &distance, sizeof(float)); - stream->read((char*) position, sizeof(float) * 3); - stream->read((char*) orientation, sizeof(float) * 3); - stream->read((char*) interpolation, sizeof(char) * 24); - stream->read((char*) &angle, sizeof(float)); - stream->read((char*) unknown, sizeof(char) * 3); - } - - void Write(std::ostream *stream) - { - stream->write((char*)&frame, sizeof(int)); - stream->write((char*)&distance, sizeof(float)); - stream->write((char*)position, sizeof(float) * 3); - stream->write((char*)orientation, sizeof(float) * 3); - stream->write((char*)interpolation, sizeof(char) * 24); - stream->write((char*)&angle, sizeof(float)); - stream->write((char*)unknown, sizeof(char) * 3); - } - }; - - class VmdLightFrame - { - public: - int frame; - float color[3]; - float position[3]; - - void Read(std::istream* stream) - { - stream->read((char*) &frame, sizeof(int)); - stream->read((char*) color, sizeof(float) * 3); - stream->read((char*) position, sizeof(float) * 3); - } - - void Write(std::ostream* stream) - { - stream->write((char*)&frame, sizeof(int)); - stream->write((char*)color, sizeof(float) * 3); - stream->write((char*)position, sizeof(float) * 3); - } - }; - - class VmdIkEnable - { - public: - std::string ik_name; - bool enable; - }; - - class VmdIkFrame - { - public: - int frame; - bool display; - std::vector<VmdIkEnable> ik_enable; - - void Read(std::istream *stream) - { - char buffer[20]; - stream->read((char*) &frame, sizeof(int)); - stream->read((char*) &display, sizeof(uint8_t)); - int ik_count; - stream->read((char*) &ik_count, sizeof(int)); - ik_enable.resize(ik_count); - for (int i = 0; i < ik_count; i++) - { - stream->read(buffer, 20); - ik_enable[i].ik_name = std::string(buffer); - stream->read((char*) &ik_enable[i].enable, sizeof(uint8_t)); - } - } - - void Write(std::ostream *stream) - { - stream->write((char*)&frame, sizeof(int)); - stream->write((char*)&display, sizeof(uint8_t)); - int ik_count = static_cast<int>(ik_enable.size()); - stream->write((char*)&ik_count, sizeof(int)); - for (int i = 0; i < ik_count; i++) - { - const VmdIkEnable& ik_enable = this->ik_enable.at(i); - stream->write(ik_enable.ik_name.c_str(), 20); - stream->write((char*)&ik_enable.enable, sizeof(uint8_t)); - } - } - }; - - class VmdMotion - { - public: - std::string model_name; - int version; - std::vector<VmdBoneFrame> bone_frames; - std::vector<VmdFaceFrame> face_frames; - std::vector<VmdCameraFrame> camera_frames; - std::vector<VmdLightFrame> light_frames; - std::vector<VmdIkFrame> ik_frames; - - static std::unique_ptr<VmdMotion> LoadFromFile(char const *filename) - { - std::ifstream stream(filename, std::ios::binary); - auto result = LoadFromStream(&stream); - stream.close(); - return result; - } - - static std::unique_ptr<VmdMotion> LoadFromStream(std::ifstream *stream) - { - - char buffer[30]; - auto result = mmd::make_unique<VmdMotion>(); - - // magic and version - stream->read((char*) buffer, 30); - if (strncmp(buffer, "Vocaloid Motion Data", 20)) - { - std::cerr << "invalid vmd file." << std::endl; - return nullptr; - } - result->version = std::atoi(buffer + 20); - - // name - stream->read(buffer, 20); - result->model_name = std::string(buffer); - - // bone frames - int bone_frame_num; - stream->read((char*) &bone_frame_num, sizeof(int)); - result->bone_frames.resize(bone_frame_num); - for (int i = 0; i < bone_frame_num; i++) - { - result->bone_frames[i].Read(stream); - } - - // face frames - int face_frame_num; - stream->read((char*) &face_frame_num, sizeof(int)); - result->face_frames.resize(face_frame_num); - for (int i = 0; i < face_frame_num; i++) - { - result->face_frames[i].Read(stream); - } - - // camera frames - int camera_frame_num; - stream->read((char*) &camera_frame_num, sizeof(int)); - result->camera_frames.resize(camera_frame_num); - for (int i = 0; i < camera_frame_num; i++) - { - result->camera_frames[i].Read(stream); - } - - // light frames - int light_frame_num; - stream->read((char*) &light_frame_num, sizeof(int)); - result->light_frames.resize(light_frame_num); - for (int i = 0; i < light_frame_num; i++) - { - result->light_frames[i].Read(stream); - } - - // unknown2 - stream->read(buffer, 4); - - // ik frames - if (stream->peek() != std::ios::traits_type::eof()) - { - int ik_num; - stream->read((char*) &ik_num, sizeof(int)); - result->ik_frames.resize(ik_num); - for (int i = 0; i < ik_num; i++) - { - result->ik_frames[i].Read(stream); - } - } - - if (stream->peek() != std::ios::traits_type::eof()) - { - std::cerr << "vmd stream has unknown data." << std::endl; - } - - return result; - } - - bool SaveToFile(const std::u16string& /*filename*/) - { - // TODO: How to adapt u16string to string? - /* - std::ofstream stream(filename.c_str(), std::ios::binary); - auto result = SaveToStream(&stream); - stream.close(); - return result; - */ - return false; - } - - bool SaveToStream(std::ofstream *stream) - { - std::string magic = "Vocaloid Motion Data 0002\0"; - magic.resize(30); - - // magic and version - stream->write(magic.c_str(), 30); - - // name - stream->write(model_name.c_str(), 20); - - // bone frames - const int bone_frame_num = static_cast<int>(bone_frames.size()); - stream->write(reinterpret_cast<const char*>(&bone_frame_num), sizeof(int)); - for (int i = 0; i < bone_frame_num; i++) - { - bone_frames[i].Write(stream); - } - - // face frames - const int face_frame_num = static_cast<int>(face_frames.size()); - stream->write(reinterpret_cast<const char*>(&face_frame_num), sizeof(int)); - for (int i = 0; i < face_frame_num; i++) - { - face_frames[i].Write(stream); - } - - // camera frames - const int camera_frame_num = static_cast<int>(camera_frames.size()); - stream->write(reinterpret_cast<const char*>(&camera_frame_num), sizeof(int)); - for (int i = 0; i < camera_frame_num; i++) - { - camera_frames[i].Write(stream); - } - - // light frames - const int light_frame_num = static_cast<int>(light_frames.size()); - stream->write(reinterpret_cast<const char*>(&light_frame_num), sizeof(int)); - for (int i = 0; i < light_frame_num; i++) - { - light_frames[i].Write(stream); - } - - // self shadow datas - const int self_shadow_num = 0; - stream->write(reinterpret_cast<const char*>(&self_shadow_num), sizeof(int)); - - // ik frames - const int ik_num = static_cast<int>(ik_frames.size()); - stream->write(reinterpret_cast<const char*>(&ik_num), sizeof(int)); - for (int i = 0; i < ik_num; i++) - { - ik_frames[i].Write(stream); - } - - return true; - } - }; -} diff --git a/thirdparty/assimp/code/Material/MaterialSystem.cpp b/thirdparty/assimp/code/Material/MaterialSystem.cpp index d0b39093b6..0be6e9f7bb 100644 --- a/thirdparty/assimp/code/Material/MaterialSystem.cpp +++ b/thirdparty/assimp/code/Material/MaterialSystem.cpp @@ -51,7 +51,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <assimp/types.h> #include <assimp/material.h> #include <assimp/DefaultLogger.hpp> -#include <assimp/Macros.h> using namespace Assimp; @@ -545,23 +544,7 @@ aiReturn aiMaterial::AddProperty (const aiString* pInput, unsigned int type, unsigned int index) { - // We don't want to add the whole buffer .. write a 32 bit length - // prefix followed by the zero-terminated UTF8 string. - // (HACK) I don't want to break the ABI now, but we definitely - // ought to change aiString::mLength to uint32_t one day. - if (sizeof(size_t) == 8) { - aiString copy = *pInput; - uint32_t* s = reinterpret_cast<uint32_t*>(©.length); - s[1] = static_cast<uint32_t>(pInput->length); - - return AddBinaryProperty(s+1, - static_cast<unsigned int>(pInput->length+1+4), - pKey, - type, - index, - aiPTI_String); - } - ai_assert(sizeof(size_t)==4); + ai_assert(sizeof(ai_uint32)==4); return AddBinaryProperty(pInput, static_cast<unsigned int>(pInput->length+1+4), pKey, diff --git a/thirdparty/assimp/code/PostProcessing/ArmaturePopulate.cpp b/thirdparty/assimp/code/PostProcessing/ArmaturePopulate.cpp new file mode 100644 index 0000000000..75daeb6b59 --- /dev/null +++ b/thirdparty/assimp/code/PostProcessing/ArmaturePopulate.cpp @@ -0,0 +1,268 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2019, assimp team + + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ +#include "ArmaturePopulate.h" + +#include <assimp/BaseImporter.h> +#include <assimp/DefaultLogger.hpp> +#include <assimp/postprocess.h> +#include <assimp/scene.h> +#include <iostream> + +namespace Assimp { + +/// The default class constructor. +ArmaturePopulate::ArmaturePopulate() : BaseProcess() +{} + +/// The class destructor. +ArmaturePopulate::~ArmaturePopulate() +{} + +bool ArmaturePopulate::IsActive(unsigned int pFlags) const { + return (pFlags & aiProcess_PopulateArmatureData) != 0; +} + +void ArmaturePopulate::SetupProperties(const Importer *pImp) { + // do nothing +} + +void ArmaturePopulate::Execute(aiScene *out) { + + // Now convert all bone positions to the correct mOffsetMatrix + std::vector<aiBone *> bones; + std::vector<aiNode *> nodes; + std::map<aiBone *, aiNode *> bone_stack; + BuildBoneList(out->mRootNode, out->mRootNode, out, bones); + BuildNodeList(out->mRootNode, nodes); + + BuildBoneStack(out->mRootNode, out->mRootNode, out, bones, bone_stack, nodes); + + ASSIMP_LOG_DEBUG_F("Bone stack size: ", bone_stack.size()); + + for (std::pair<aiBone *, aiNode *> kvp : bone_stack) { + aiBone *bone = kvp.first; + aiNode *bone_node = kvp.second; + ASSIMP_LOG_DEBUG_F("active node lookup: ", bone->mName.C_Str()); + // lcl transform grab - done in generate_nodes :) + + // bone->mOffsetMatrix = bone_node->mTransformation; + aiNode *armature = GetArmatureRoot(bone_node, bones); + + ai_assert(armature); + + // set up bone armature id + bone->mArmature = armature; + + // set this bone node to be referenced properly + ai_assert(bone_node); + bone->mNode = bone_node; + } +} + + +/* Reprocess all nodes to calculate bone transforms properly based on the REAL + * mOffsetMatrix not the local. */ +/* Before this would use mesh transforms which is wrong for bone transforms */ +/* Before this would work for simple character skeletons but not complex meshes + * with multiple origins */ +/* Source: sketch fab log cutter fbx */ +void ArmaturePopulate::BuildBoneList(aiNode *current_node, + const aiNode *root_node, + const aiScene *scene, + std::vector<aiBone *> &bones) { + ai_assert(scene); + for (unsigned int nodeId = 0; nodeId < current_node->mNumChildren; ++nodeId) { + aiNode *child = current_node->mChildren[nodeId]; + ai_assert(child); + + // check for bones + for (unsigned int meshId = 0; meshId < child->mNumMeshes; ++meshId) { + ai_assert(child->mMeshes); + unsigned int mesh_index = child->mMeshes[meshId]; + aiMesh *mesh = scene->mMeshes[mesh_index]; + ai_assert(mesh); + + for (unsigned int boneId = 0; boneId < mesh->mNumBones; ++boneId) { + aiBone *bone = mesh->mBones[boneId]; + ai_assert(bone); + + // duplicate meshes exist with the same bones sometimes :) + // so this must be detected + if (std::find(bones.begin(), bones.end(), bone) == bones.end()) { + // add the element once + bones.push_back(bone); + } + } + + // find mesh and get bones + // then do recursive lookup for bones in root node hierarchy + } + + BuildBoneList(child, root_node, scene, bones); + } +} + +/* Prepare flat node list which can be used for non recursive lookups later */ +void ArmaturePopulate::BuildNodeList(const aiNode *current_node, + std::vector<aiNode *> &nodes) { + ai_assert(current_node); + + for (unsigned int nodeId = 0; nodeId < current_node->mNumChildren; ++nodeId) { + aiNode *child = current_node->mChildren[nodeId]; + ai_assert(child); + + nodes.push_back(child); + + BuildNodeList(child, nodes); + } +} + +/* A bone stack allows us to have multiple armatures, with the same bone names + * A bone stack allows us also to retrieve bones true transform even with + * duplicate names :) + */ +void ArmaturePopulate::BuildBoneStack(aiNode *current_node, + const aiNode *root_node, + const aiScene *scene, + const std::vector<aiBone *> &bones, + std::map<aiBone *, aiNode *> &bone_stack, + std::vector<aiNode *> &node_stack) { + ai_assert(scene); + ai_assert(root_node); + ai_assert(!node_stack.empty()); + + for (aiBone *bone : bones) { + ai_assert(bone); + aiNode *node = GetNodeFromStack(bone->mName, node_stack); + if (node == nullptr) { + node_stack.clear(); + BuildNodeList(root_node, node_stack); + ASSIMP_LOG_DEBUG_F("Resetting bone stack: nullptr element ", bone->mName.C_Str()); + + node = GetNodeFromStack(bone->mName, node_stack); + + if (!node) { + ASSIMP_LOG_ERROR("serious import issue node for bone was not detected"); + continue; + } + } + + ASSIMP_LOG_DEBUG_F("Successfully added bone[", bone->mName.C_Str(), "] to stack and bone node is: ", node->mName.C_Str()); + + bone_stack.insert(std::pair<aiBone *, aiNode *>(bone, node)); + } +} + + +/* Returns the armature root node */ +/* This is required to be detected for a bone initially, it will recurse up + * until it cannot find another bone and return the node No known failure + * points. (yet) + */ +aiNode *ArmaturePopulate::GetArmatureRoot(aiNode *bone_node, + std::vector<aiBone *> &bone_list) { + while (bone_node) { + if (!IsBoneNode(bone_node->mName, bone_list)) { + ASSIMP_LOG_DEBUG_F("GetArmatureRoot() Found valid armature: ", bone_node->mName.C_Str()); + return bone_node; + } + + bone_node = bone_node->mParent; + } + + ASSIMP_LOG_ERROR("GetArmatureRoot() can't find armature!"); + + return nullptr; +} + + + +/* Simple IsBoneNode check if this could be a bone */ +bool ArmaturePopulate::IsBoneNode(const aiString &bone_name, + std::vector<aiBone *> &bones) { + for (aiBone *bone : bones) { + if (bone->mName == bone_name) { + return true; + } + } + + return false; +} + +/* Pop this node by name from the stack if found */ +/* Used in multiple armature situations with duplicate node / bone names */ +/* Known flaw: cannot have nodes with bone names, will be fixed in later release + */ +/* (serious to be fixed) Known flaw: nodes which have more than one bone could + * be prematurely dropped from stack */ +aiNode *ArmaturePopulate::GetNodeFromStack(const aiString &node_name, + std::vector<aiNode *> &nodes) { + std::vector<aiNode *>::iterator iter; + aiNode *found = nullptr; + for (iter = nodes.begin(); iter < nodes.end(); ++iter) { + aiNode *element = *iter; + ai_assert(element); + // node valid and node name matches + if (element->mName == node_name) { + found = element; + break; + } + } + + if (found != nullptr) { + ASSIMP_LOG_INFO_F("Removed node from stack: ", found->mName.C_Str()); + // now pop the element from the node list + nodes.erase(iter); + + return found; + } + + // unique names can cause this problem + ASSIMP_LOG_ERROR("[Serious] GetNodeFromStack() can't find node from stack!"); + + return nullptr; +} + + + + +} // Namespace Assimp diff --git a/thirdparty/assimp/code/PostProcessing/ArmaturePopulate.h b/thirdparty/assimp/code/PostProcessing/ArmaturePopulate.h new file mode 100644 index 0000000000..aa1ad7c80c --- /dev/null +++ b/thirdparty/assimp/code/PostProcessing/ArmaturePopulate.h @@ -0,0 +1,112 @@ +/* +Open Asset Import Library (assimp) +---------------------------------------------------------------------- + +Copyright (c) 2006-2019, assimp team + + +All rights reserved. + +Redistribution and use of this software in source and binary forms, +with or without modification, are permitted provided that the +following conditions are met: + +* Redistributions of source code must retain the above +copyright notice, this list of conditions and the +following disclaimer. + +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the +following disclaimer in the documentation and/or other +materials provided with the distribution. + +* Neither the name of the assimp team, nor the names of its +contributors may be used to endorse or promote products +derived from this software without specific prior +written permission of the assimp team. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +*/ +#ifndef ARMATURE_POPULATE_H_ +#define ARMATURE_POPULATE_H_ + +#include "Common/BaseProcess.h" +#include <assimp/BaseImporter.h> +#include <vector> +#include <map> + + +struct aiNode; +struct aiBone; + +namespace Assimp { + +// --------------------------------------------------------------------------- +/** Armature Populate: This is a post process designed + * To save you time when importing models into your game engines + * This was originally designed only for fbx but will work with other formats + * it is intended to auto populate aiBone data with armature and the aiNode + * This is very useful when dealing with skinned meshes + * or when dealing with many different skeletons + * It's off by default but recommend that you try it and use it + * It should reduce down any glue code you have in your + * importers + * You can contact RevoluPowered <gordon@gordonite.tech> + * For more info about this +*/ +class ASSIMP_API ArmaturePopulate : public BaseProcess { +public: + /// The default class constructor. + ArmaturePopulate(); + + /// The class destructor. + virtual ~ArmaturePopulate(); + + /// Overwritten, @see BaseProcess + virtual bool IsActive( unsigned int pFlags ) const; + + /// Overwritten, @see BaseProcess + virtual void SetupProperties( const Importer* pImp ); + + /// Overwritten, @see BaseProcess + virtual void Execute( aiScene* pScene ); + + static aiNode *GetArmatureRoot(aiNode *bone_node, + std::vector<aiBone *> &bone_list); + + static bool IsBoneNode(const aiString &bone_name, + std::vector<aiBone *> &bones); + + static aiNode *GetNodeFromStack(const aiString &node_name, + std::vector<aiNode *> &nodes); + + static void BuildNodeList(const aiNode *current_node, + std::vector<aiNode *> &nodes); + + static void BuildBoneList(aiNode *current_node, const aiNode *root_node, + const aiScene *scene, + std::vector<aiBone *> &bones); + + static void BuildBoneStack(aiNode *current_node, const aiNode *root_node, + const aiScene *scene, + const std::vector<aiBone *> &bones, + std::map<aiBone *, aiNode *> &bone_stack, + std::vector<aiNode *> &node_stack); +}; + +} // Namespace Assimp + + +#endif // SCALE_PROCESS_H_
\ No newline at end of file diff --git a/thirdparty/assimp/code/PostProcessing/ComputeUVMappingProcess.cpp b/thirdparty/assimp/code/PostProcessing/ComputeUVMappingProcess.cpp index bb571a551b..df4d44337d 100644 --- a/thirdparty/assimp/code/PostProcessing/ComputeUVMappingProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/ComputeUVMappingProcess.cpp @@ -354,12 +354,12 @@ void ComputeUVMappingProcess::ComputePlaneMapping(aiMesh* mesh,const aiVector3D& } else if (axis * base_axis_z >= angle_epsilon) { FindMeshCenter(mesh, center, min, max); - diffu = max.y - min.y; - diffv = max.z - min.z; + diffu = max.x - min.x; + diffv = max.y - min.y; for (unsigned int pnt = 0; pnt < mesh->mNumVertices;++pnt) { const aiVector3D& pos = mesh->mVertices[pnt]; - out[pnt].Set((pos.y - min.y) / diffu,(pos.x - min.x) / diffv,0.0); + out[pnt].Set((pos.x - min.x) / diffu,(pos.y - min.y) / diffv,0.0); } } // slower code path in case the mapping axis is not one of the coordinate system axes diff --git a/thirdparty/assimp/code/PostProcessing/FindInvalidDataProcess.cpp b/thirdparty/assimp/code/PostProcessing/FindInvalidDataProcess.cpp index 433f042448..016884c6e7 100644 --- a/thirdparty/assimp/code/PostProcessing/FindInvalidDataProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/FindInvalidDataProcess.cpp @@ -52,7 +52,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "FindInvalidDataProcess.h" #include "ProcessHelper.h" -#include <assimp/Macros.h> #include <assimp/Exceptional.h> #include <assimp/qnan.h> diff --git a/thirdparty/assimp/code/PostProcessing/JoinVerticesProcess.cpp b/thirdparty/assimp/code/PostProcessing/JoinVerticesProcess.cpp index 914ec05b46..f121fc60d3 100644 --- a/thirdparty/assimp/code/PostProcessing/JoinVerticesProcess.cpp +++ b/thirdparty/assimp/code/PostProcessing/JoinVerticesProcess.cpp @@ -431,31 +431,6 @@ int JoinVerticesProcess::ProcessMesh( aiMesh* pMesh, unsigned int meshIndex) bone->mWeights = new aiVertexWeight[bone->mNumWeights]; memcpy( bone->mWeights, &newWeights[0], bone->mNumWeights * sizeof( aiVertexWeight)); } - else { - - /* NOTE: - * - * In the algorithm above we're assuming that there are no vertices - * with a different bone weight setup at the same position. That wouldn't - * make sense, but it is not absolutely impossible. SkeletonMeshBuilder - * for example generates such input data if two skeleton points - * share the same position. Again this doesn't make sense but is - * reality for some model formats (MD5 for example uses these special - * nodes as attachment tags for its weapons). - * - * Then it is possible that a bone has no weights anymore .... as a quick - * workaround, we're just removing these bones. If they're animated, - * model geometry might be modified but at least there's no risk of a crash. - */ - delete bone; - --pMesh->mNumBones; - for (unsigned int n = a; n < pMesh->mNumBones; ++n) { - pMesh->mBones[n] = pMesh->mBones[n+1]; - } - - --a; - ASSIMP_LOG_WARN("Removing bone -> no weights remaining"); - } } return pMesh->mNumVertices; } diff --git a/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.cpp b/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.cpp index 50ff5ed93d..41f50a5ba5 100644 --- a/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.cpp +++ b/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.cpp @@ -224,3 +224,32 @@ bool MakeVerboseFormatProcess::MakeVerboseFormat(aiMesh* pcMesh) } return (pcMesh->mNumVertices != iOldNumVertices); } + + +// ------------------------------------------------------------------------------------------------ +bool IsMeshInVerboseFormat(const aiMesh* mesh) { + // avoid slow vector<bool> specialization + std::vector<unsigned int> seen(mesh->mNumVertices,0); + for(unsigned int i = 0; i < mesh->mNumFaces; ++i) { + const aiFace& f = mesh->mFaces[i]; + for(unsigned int j = 0; j < f.mNumIndices; ++j) { + if(++seen[f.mIndices[j]] == 2) { + // found a duplicate index + return false; + } + } + } + + return true; +} + +// ------------------------------------------------------------------------------------------------ +bool MakeVerboseFormatProcess::IsVerboseFormat(const aiScene* pScene) { + for(unsigned int i = 0; i < pScene->mNumMeshes; ++i) { + if(!IsMeshInVerboseFormat(pScene->mMeshes[i])) { + return false; + } + } + + return true; +} diff --git a/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.h b/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.h index 1adf8e2f69..8565d5933a 100644 --- a/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.h +++ b/thirdparty/assimp/code/PostProcessing/MakeVerboseFormat.h @@ -94,6 +94,13 @@ public: * @param pScene The imported data to work at. */ void Execute( aiScene* pScene); +public: + + // ------------------------------------------------------------------- + /** Checks whether the scene is already in verbose format. + * @param pScene The data to check. + * @return true if the scene is already in verbose format. */ + static bool IsVerboseFormat(const aiScene* pScene); private: diff --git a/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.cpp b/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.cpp index 712fd6943d..75d1b6ef78 100644 --- a/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.cpp +++ b/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.cpp @@ -538,13 +538,17 @@ void ValidateDSProcess::Validate( const aiAnimation* pAnimation) { Validate(&pAnimation->mName); - // validate all materials - if (pAnimation->mNumChannels) + // validate all animations + if (pAnimation->mNumChannels || pAnimation->mNumMorphMeshChannels) { - if (!pAnimation->mChannels) { + if (!pAnimation->mChannels && pAnimation->mNumChannels) { ReportError("aiAnimation::mChannels is NULL (aiAnimation::mNumChannels is %i)", pAnimation->mNumChannels); } + if (!pAnimation->mMorphMeshChannels && pAnimation->mNumMorphMeshChannels) { + ReportError("aiAnimation::mMorphMeshChannels is NULL (aiAnimation::mNumMorphMeshChannels is %i)", + pAnimation->mNumMorphMeshChannels); + } for (unsigned int i = 0; i < pAnimation->mNumChannels;++i) { if (!pAnimation->mChannels[i]) @@ -554,6 +558,15 @@ void ValidateDSProcess::Validate( const aiAnimation* pAnimation) } Validate(pAnimation, pAnimation->mChannels[i]); } + for (unsigned int i = 0; i < pAnimation->mNumMorphMeshChannels;++i) + { + if (!pAnimation->mMorphMeshChannels[i]) + { + ReportError("aiAnimation::mMorphMeshChannels[%i] is NULL (aiAnimation::mNumMorphMeshChannels is %i)", + i, pAnimation->mNumMorphMeshChannels); + } + Validate(pAnimation, pAnimation->mMorphMeshChannels[i]); + } } else { ReportError("aiAnimation::mNumChannels is 0. At least one node animation channel must be there."); @@ -903,6 +916,48 @@ void ValidateDSProcess::Validate( const aiAnimation* pAnimation, } } +void ValidateDSProcess::Validate( const aiAnimation* pAnimation, + const aiMeshMorphAnim* pMeshMorphAnim) +{ + Validate(&pMeshMorphAnim->mName); + + if (!pMeshMorphAnim->mNumKeys) { + ReportError("Empty mesh morph animation channel"); + } + + // otherwise check whether one of the keys exceeds the total duration of the animation + if (pMeshMorphAnim->mNumKeys) + { + if (!pMeshMorphAnim->mKeys) + { + ReportError("aiMeshMorphAnim::mKeys is NULL (aiMeshMorphAnim::mNumKeys is %i)", + pMeshMorphAnim->mNumKeys); + } + double dLast = -10e10; + for (unsigned int i = 0; i < pMeshMorphAnim->mNumKeys;++i) + { + // ScenePreprocessor will compute the duration if still the default value + // (Aramis) Add small epsilon, comparison tended to fail if max_time == duration, + // seems to be due the compilers register usage/width. + if (pAnimation->mDuration > 0. && pMeshMorphAnim->mKeys[i].mTime > pAnimation->mDuration+0.001) + { + ReportError("aiMeshMorphAnim::mKeys[%i].mTime (%.5f) is larger " + "than aiAnimation::mDuration (which is %.5f)",i, + (float)pMeshMorphAnim->mKeys[i].mTime, + (float)pAnimation->mDuration); + } + if (i && pMeshMorphAnim->mKeys[i].mTime <= dLast) + { + ReportWarning("aiMeshMorphAnim::mKeys[%i].mTime (%.5f) is smaller " + "than aiMeshMorphAnim::mKeys[%i] (which is %.5f)",i, + (float)pMeshMorphAnim->mKeys[i].mTime, + i-1, (float)dLast); + } + dLast = pMeshMorphAnim->mKeys[i].mTime; + } + } +} + // ------------------------------------------------------------------------------------------------ void ValidateDSProcess::Validate( const aiNode* pNode) { @@ -958,7 +1013,7 @@ void ValidateDSProcess::Validate( const aiString* pString) { if (pString->length > MAXLEN) { - ReportError("aiString::length is too large (%lu, maximum is %lu)", + ReportError("aiString::length is too large (%u, maximum is %lu)", pString->length,MAXLEN); } const char* sz = pString->data; diff --git a/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.h b/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.h index 0b891ef414..7b309c9251 100644 --- a/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.h +++ b/thirdparty/assimp/code/PostProcessing/ValidateDataStructure.h @@ -55,6 +55,7 @@ struct aiBone; struct aiMesh; struct aiAnimation; struct aiNodeAnim; +struct aiMeshMorphAnim; struct aiTexture; struct aiMaterial; struct aiNode; @@ -150,6 +151,13 @@ protected: void Validate( const aiAnimation* pAnimation, const aiNodeAnim* pBoneAnim); + /** Validates a mesh morph animation channel. + * @param pAnimation Input animation. + * @param pMeshMorphAnim Mesh morph animation channel. + * */ + void Validate( const aiAnimation* pAnimation, + const aiMeshMorphAnim* pMeshMorphAnim); + // ------------------------------------------------------------------- /** Validates a node and all of its subnodes * @param Node Input node*/ diff --git a/thirdparty/assimp/code/revision.h b/thirdparty/assimp/code/revision.h index 88872aef22..66eb875303 100644 --- a/thirdparty/assimp/code/revision.h +++ b/thirdparty/assimp/code/revision.h @@ -1,7 +1,28 @@ #ifndef ASSIMP_REVISION_H_INC #define ASSIMP_REVISION_H_INC -#define GitVersion 0x00000000 +#define GitVersion 0x308db73d #define GitBranch "master" +#define VER_MAJOR 5 +#define VER_MINOR 0 +#define VER_PATCH 0 +#define VER_BUILD 0 + +#define STR_HELP(x) #x +#define STR(x) STR_HELP(x) + +#define VER_FILEVERSION VER_MAJOR,VER_MINOR,VER_PATCH,VER_BUILD +#if (GitVersion == 0) +#define VER_FILEVERSION_STR STR(VER_MAJOR) "." STR(VER_MINOR) "." STR(VER_PATCH) "." STR(VER_BUILD) +#else +#define VER_FILEVERSION_STR STR(VER_MAJOR) "." STR(VER_MINOR) "." STR(VER_PATCH) "." STR(VER_BUILD) " (Commit 308db73d)" +#endif + +#ifdef NDEBUG +#define VER_ORIGINAL_FILENAME_STR "assimp.dll" +#else +#define VER_ORIGINAL_FILENAME_STR "assimp.dll" +#endif // NDEBUG + #endif // ASSIMP_REVISION_H_INC diff --git a/thirdparty/assimp/include/assimp/.editorconfig b/thirdparty/assimp/include/assimp/.editorconfig new file mode 100644 index 0000000000..9ea66423ad --- /dev/null +++ b/thirdparty/assimp/include/assimp/.editorconfig @@ -0,0 +1,8 @@ +# See <http://EditorConfig.org> for details + +[*.{h,hpp,inl}] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_size = 4 +indent_style = space diff --git a/thirdparty/assimp/include/assimp/BaseImporter.h b/thirdparty/assimp/include/assimp/BaseImporter.h index 55f7fe3754..ad8a3dafd8 100644 --- a/thirdparty/assimp/include/assimp/BaseImporter.h +++ b/thirdparty/assimp/include/assimp/BaseImporter.h @@ -41,9 +41,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** @file Definition of the base class for all importer worker classes. */ +#pragma once #ifndef INCLUDED_AI_BASEIMPORTER_H #define INCLUDED_AI_BASEIMPORTER_H +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include "Exceptional.h" #include <vector> @@ -191,16 +196,13 @@ public: /** * Assimp Importer * unit conversions available - * if you need another measurment unit add it below. - * it's currently defined in assimp that we prefer meters. + * NOTE: Valid options are initialised in the + * constructor in the implementation file to + * work around a VS2013 compiler bug if support + * for that compiler is dropped in the future + * initialisation can be moved back here * */ - std::map<ImporterUnits, double> importerUnits = { - {ImporterUnits::M, 1}, - {ImporterUnits::CM, 0.01}, - {ImporterUnits::MM, 0.001}, - {ImporterUnits::INCHES, 0.0254}, - {ImporterUnits::FEET, 0.3048} - }; + std::map<ImporterUnits, double> importerUnits; virtual void SetApplicationUnits( const ImporterUnits& unit ) { diff --git a/thirdparty/assimp/include/assimp/Bitmap.h b/thirdparty/assimp/include/assimp/Bitmap.h index e6b5fb1327..4c3f5a437b 100644 --- a/thirdparty/assimp/include/assimp/Bitmap.h +++ b/thirdparty/assimp/include/assimp/Bitmap.h @@ -46,10 +46,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Used for file formats which embed their textures into the model file. */ - +#pragma once #ifndef AI_BITMAP_H_INC #define AI_BITMAP_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include "defs.h" #include <stdint.h> #include <cstddef> diff --git a/thirdparty/assimp/include/assimp/ByteSwapper.h b/thirdparty/assimp/include/assimp/ByteSwapper.h index 20a2463fb8..3f14c471a8 100644 --- a/thirdparty/assimp/include/assimp/ByteSwapper.h +++ b/thirdparty/assimp/include/assimp/ByteSwapper.h @@ -42,9 +42,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file Helper class tp perform various byte oder swappings (e.g. little to big endian) */ +#pragma once #ifndef AI_BYTESWAPPER_H_INC #define AI_BYTESWAPPER_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/ai_assert.h> #include <assimp/types.h> #include <stdint.h> diff --git a/thirdparty/assimp/include/assimp/CreateAnimMesh.h b/thirdparty/assimp/include/assimp/CreateAnimMesh.h index a60173588f..1266d1de11 100644 --- a/thirdparty/assimp/include/assimp/CreateAnimMesh.h +++ b/thirdparty/assimp/include/assimp/CreateAnimMesh.h @@ -43,16 +43,26 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file CreateAnimMesh.h * Create AnimMesh from Mesh */ +#pragma once #ifndef INCLUDED_AI_CREATE_ANIM_MESH_H #define INCLUDED_AI_CREATE_ANIM_MESH_H +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/mesh.h> -namespace Assimp { +namespace Assimp { -/** Create aiAnimMesh from aiMesh. */ +/** + * Create aiAnimMesh from aiMesh. + * @param mesh The input mesh to create an animated mesh from. + * @return The new created animated mesh. + */ ASSIMP_API aiAnimMesh *aiCreateAnimMesh(const aiMesh *mesh); } // end of namespace Assimp + #endif // INCLUDED_AI_CREATE_ANIM_MESH_H diff --git a/thirdparty/assimp/include/assimp/DefaultIOStream.h b/thirdparty/assimp/include/assimp/DefaultIOStream.h index 994d728ff5..c6d382c1b5 100644 --- a/thirdparty/assimp/include/assimp/DefaultIOStream.h +++ b/thirdparty/assimp/include/assimp/DefaultIOStream.h @@ -41,15 +41,20 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** @file Default file I/O using fXXX()-family of functions */ +#pragma once #ifndef AI_DEFAULTIOSTREAM_H_INC #define AI_DEFAULTIOSTREAM_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <stdio.h> #include <assimp/IOStream.hpp> #include <assimp/importerdesc.h> #include <assimp/Defines.h> -namespace Assimp { +namespace Assimp { // ---------------------------------------------------------------------------------- //! @class DefaultIOStream @@ -57,8 +62,7 @@ namespace Assimp { //! @note An instance of this class can exist without a valid file handle //! attached to it. All calls fail, but the instance can nevertheless be //! used with no restrictions. -class ASSIMP_API DefaultIOStream : public IOStream -{ +class ASSIMP_API DefaultIOStream : public IOStream { friend class DefaultIOSystem; #if __ANDROID__ # if __ANDROID_API__ > 9 @@ -82,7 +86,6 @@ public: size_t pSize, size_t pCount); - // ------------------------------------------------------------------- /// Write to stream size_t Write(const void* pvBuffer, @@ -107,16 +110,13 @@ public: void Flush(); private: - // File data-structure, using clib FILE* mFile; - // Filename std::string mFilename; - // Cached file size mutable size_t mCachedSize; }; // ---------------------------------------------------------------------------------- -inline +AI_FORCE_INLINE DefaultIOStream::DefaultIOStream() AI_NO_EXCEPT : mFile(nullptr) , mFilename("") @@ -125,7 +125,7 @@ DefaultIOStream::DefaultIOStream() AI_NO_EXCEPT } // ---------------------------------------------------------------------------------- -inline +AI_FORCE_INLINE DefaultIOStream::DefaultIOStream (FILE* pFile, const std::string &strFilename) : mFile(pFile) , mFilename(strFilename) @@ -137,4 +137,3 @@ DefaultIOStream::DefaultIOStream (FILE* pFile, const std::string &strFilename) } // ns assimp #endif //!!AI_DEFAULTIOSTREAM_H_INC - diff --git a/thirdparty/assimp/include/assimp/DefaultIOSystem.h b/thirdparty/assimp/include/assimp/DefaultIOSystem.h index 2dd5c801b5..46f6d447c5 100644 --- a/thirdparty/assimp/include/assimp/DefaultIOSystem.h +++ b/thirdparty/assimp/include/assimp/DefaultIOSystem.h @@ -41,9 +41,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** @file Default implementation of IOSystem using the standard C file functions */ +#pragma once #ifndef AI_DEFAULTIOSYSTEM_H_INC #define AI_DEFAULTIOSYSTEM_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/IOSystem.hpp> namespace Assimp { diff --git a/thirdparty/assimp/include/assimp/Defines.h b/thirdparty/assimp/include/assimp/Defines.h index 15e1d83c26..be3e2fafd6 100644 --- a/thirdparty/assimp/include/assimp/Defines.h +++ b/thirdparty/assimp/include/assimp/Defines.h @@ -38,6 +38,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------- */ +#pragma once +#ifndef AI_DEFINES_H_INC +#define AI_DEFINES_H_INC + +#ifdef __GNUC__ +# pragma GCC system_header +#endif + // We need those constants, workaround for any platforms where nobody defined them yet #if (!defined SIZE_MAX) # define SIZE_MAX (~((size_t)0)) @@ -47,3 +55,4 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # define UINT_MAX (~((unsigned int)0)) #endif +#endif // AI_DEINES_H_INC diff --git a/thirdparty/assimp/include/assimp/Exceptional.h b/thirdparty/assimp/include/assimp/Exceptional.h index 5109b8f077..6bb6ce1e39 100644 --- a/thirdparty/assimp/include/assimp/Exceptional.h +++ b/thirdparty/assimp/include/assimp/Exceptional.h @@ -38,11 +38,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------- */ -#ifndef INCLUDED_EXCEPTIONAL_H -#define INCLUDED_EXCEPTIONAL_H +#pragma once +#ifndef AI_INCLUDED_EXCEPTIONAL_H +#define AI_INCLUDED_EXCEPTIONAL_H + +#ifdef __GNUC__ +# pragma GCC system_header +#endif #include <stdexcept> #include <assimp/DefaultIOStream.h> + using std::runtime_error; #ifdef _MSC_VER @@ -53,17 +59,14 @@ using std::runtime_error; /** FOR IMPORTER PLUGINS ONLY: Simple exception class to be thrown if an * unrecoverable error occurs while importing. Loading APIs return * NULL instead of a valid aiScene then. */ -class DeadlyImportError - : public runtime_error -{ +class DeadlyImportError : public runtime_error { public: /** Constructor with arguments */ explicit DeadlyImportError( const std::string& errorText) - : runtime_error(errorText) - { + : runtime_error(errorText) { + // empty } -private: }; typedef DeadlyImportError DeadlyExportError; @@ -84,7 +87,7 @@ struct ExceptionSwallower { template <typename T> struct ExceptionSwallower<T*> { T* operator ()() const { - return NULL; + return nullptr; } }; @@ -122,4 +125,4 @@ struct ExceptionSwallower<void> { }\ } -#endif // INCLUDED_EXCEPTIONAL_H +#endif // AI_INCLUDED_EXCEPTIONAL_H diff --git a/thirdparty/assimp/include/assimp/Exporter.hpp b/thirdparty/assimp/include/assimp/Exporter.hpp index ea0303e804..2612e1f9d2 100644 --- a/thirdparty/assimp/include/assimp/Exporter.hpp +++ b/thirdparty/assimp/include/assimp/Exporter.hpp @@ -48,6 +48,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_EXPORT_HPP_INC #define AI_EXPORT_HPP_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #ifndef ASSIMP_BUILD_NO_EXPORT #include "cexport.h" diff --git a/thirdparty/assimp/include/assimp/GenericProperty.h b/thirdparty/assimp/include/assimp/GenericProperty.h index 183ecd5197..7796d595b8 100644 --- a/thirdparty/assimp/include/assimp/GenericProperty.h +++ b/thirdparty/assimp/include/assimp/GenericProperty.h @@ -40,12 +40,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------- */ +#pragma once #ifndef AI_GENERIC_PROPERTY_H_INCLUDED #define AI_GENERIC_PROPERTY_H_INCLUDED +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/Importer.hpp> #include <assimp/ai_assert.h> -#include "Hash.h" +#include <assimp/Hash.h> #include <map> diff --git a/thirdparty/assimp/include/assimp/Hash.h b/thirdparty/assimp/include/assimp/Hash.h index 30657be198..9056440789 100644 --- a/thirdparty/assimp/include/assimp/Hash.h +++ b/thirdparty/assimp/include/assimp/Hash.h @@ -39,10 +39,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------- */ - +#pragma once #ifndef AI_HASH_H_INCLUDED #define AI_HASH_H_INCLUDED +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <stdint.h> #include <string.h> diff --git a/thirdparty/assimp/include/assimp/IOStream.hpp b/thirdparty/assimp/include/assimp/IOStream.hpp index 0623d0f70b..39932cd949 100644 --- a/thirdparty/assimp/include/assimp/IOStream.hpp +++ b/thirdparty/assimp/include/assimp/IOStream.hpp @@ -48,14 +48,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_IOSTREAM_H_INC #define AI_IOSTREAM_H_INC -#include "types.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/types.h> #ifndef __cplusplus # error This header requires C++ to be used. aiFileIO.h is the \ corresponding C interface. #endif -namespace Assimp { +namespace Assimp { // ---------------------------------------------------------------------------------- /** @brief CPP-API: Class to handle file I/O for C++ @@ -125,13 +129,13 @@ public: }; //! class IOStream // ---------------------------------------------------------------------------------- -inline +AI_FORCE_INLINE IOStream::IOStream() AI_NO_EXCEPT { // empty } // ---------------------------------------------------------------------------------- -inline +AI_FORCE_INLINE IOStream::~IOStream() { // empty } diff --git a/thirdparty/assimp/include/assimp/IOStreamBuffer.h b/thirdparty/assimp/include/assimp/IOStreamBuffer.h index 58abd97a02..97c84b23e2 100644 --- a/thirdparty/assimp/include/assimp/IOStreamBuffer.h +++ b/thirdparty/assimp/include/assimp/IOStreamBuffer.h @@ -1,5 +1,3 @@ -#pragma once - /* Open Asset Import Library (assimp) ---------------------------------------------------------------------- @@ -42,10 +40,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------- */ +#pragma once +#ifndef AI_IOSTREAMBUFFER_H_INC +#define AI_IOSTREAMBUFFER_H_INC + +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/types.h> #include <assimp/IOStream.hpp> - -#include "ParsingUtils.h" +#include <assimp/ParsingUtils.h> #include <vector> @@ -124,7 +129,7 @@ private: }; template<class T> -inline +AI_FORCE_INLINE IOStreamBuffer<T>::IOStreamBuffer( size_t cache ) : m_stream( nullptr ) , m_filesize( 0 ) @@ -138,13 +143,13 @@ IOStreamBuffer<T>::IOStreamBuffer( size_t cache ) } template<class T> -inline +AI_FORCE_INLINE IOStreamBuffer<T>::~IOStreamBuffer() { // empty } template<class T> -inline +AI_FORCE_INLINE bool IOStreamBuffer<T>::open( IOStream *stream ) { // file still opened! if ( nullptr != m_stream ) { @@ -174,7 +179,7 @@ bool IOStreamBuffer<T>::open( IOStream *stream ) { } template<class T> -inline +AI_FORCE_INLINE bool IOStreamBuffer<T>::close() { if ( nullptr == m_stream ) { return false; @@ -192,19 +197,19 @@ bool IOStreamBuffer<T>::close() { } template<class T> -inline +AI_FORCE_INLINE size_t IOStreamBuffer<T>::size() const { return m_filesize; } template<class T> -inline +AI_FORCE_INLINE size_t IOStreamBuffer<T>::cacheSize() const { return m_cacheSize; } template<class T> -inline +AI_FORCE_INLINE bool IOStreamBuffer<T>::readNextBlock() { m_stream->Seek( m_filePos, aiOrigin_SET ); size_t readLen = m_stream->Read( &m_cache[ 0 ], sizeof( T ), m_cacheSize ); @@ -222,25 +227,25 @@ bool IOStreamBuffer<T>::readNextBlock() { } template<class T> -inline +AI_FORCE_INLINE size_t IOStreamBuffer<T>::getNumBlocks() const { return m_numBlocks; } template<class T> -inline +AI_FORCE_INLINE size_t IOStreamBuffer<T>::getCurrentBlockIndex() const { return m_blockIdx; } template<class T> -inline +AI_FORCE_INLINE size_t IOStreamBuffer<T>::getFilePos() const { return m_filePos; } template<class T> -inline +AI_FORCE_INLINE bool IOStreamBuffer<T>::getNextDataLine( std::vector<T> &buffer, T continuationToken ) { buffer.resize( m_cacheSize ); if ( m_cachePos >= m_cacheSize || 0 == m_filePos ) { @@ -289,13 +294,13 @@ bool IOStreamBuffer<T>::getNextDataLine( std::vector<T> &buffer, T continuationT return true; } -static inline +static AI_FORCE_INLINE bool isEndOfCache( size_t pos, size_t cacheSize ) { return ( pos == cacheSize ); } template<class T> -inline +AI_FORCE_INLINE bool IOStreamBuffer<T>::getNextLine(std::vector<T> &buffer) { buffer.resize(m_cacheSize); if ( isEndOfCache( m_cachePos, m_cacheSize ) || 0 == m_filePos) { @@ -335,7 +340,7 @@ bool IOStreamBuffer<T>::getNextLine(std::vector<T> &buffer) { } template<class T> -inline +AI_FORCE_INLINE bool IOStreamBuffer<T>::getNextBlock( std::vector<T> &buffer) { // Return the last block-value if getNextLine was used before if ( 0 != m_cachePos ) { @@ -353,3 +358,5 @@ bool IOStreamBuffer<T>::getNextBlock( std::vector<T> &buffer) { } } // !ns Assimp + +#endif // AI_IOSTREAMBUFFER_H_INC diff --git a/thirdparty/assimp/include/assimp/IOSystem.hpp b/thirdparty/assimp/include/assimp/IOSystem.hpp index 78139c2839..f1fb3b0c27 100644 --- a/thirdparty/assimp/include/assimp/IOSystem.hpp +++ b/thirdparty/assimp/include/assimp/IOSystem.hpp @@ -50,6 +50,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_IOSYSTEM_H_INC #define AI_IOSYSTEM_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #ifndef __cplusplus # error This header requires C++ to be used. aiFileIO.h is the \ corresponding C interface. diff --git a/thirdparty/assimp/include/assimp/Importer.hpp b/thirdparty/assimp/include/assimp/Importer.hpp index 4941df4122..bf449a9a25 100644 --- a/thirdparty/assimp/include/assimp/Importer.hpp +++ b/thirdparty/assimp/include/assimp/Importer.hpp @@ -48,6 +48,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_ASSIMP_HPP_INC #define AI_ASSIMP_HPP_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #ifndef __cplusplus # error This header requires C++ to be used. Use assimp.h for plain C. #endif // __cplusplus diff --git a/thirdparty/assimp/include/assimp/LineSplitter.h b/thirdparty/assimp/include/assimp/LineSplitter.h index 4afe45b92a..6c1097bb6d 100644 --- a/thirdparty/assimp/include/assimp/LineSplitter.h +++ b/thirdparty/assimp/include/assimp/LineSplitter.h @@ -48,9 +48,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef INCLUDED_LINE_SPLITTER_H #define INCLUDED_LINE_SPLITTER_H +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <stdexcept> -#include "StreamReader.h" -#include "ParsingUtils.h" +#include <assimp/StreamReader.h> +#include <assimp/ParsingUtils.h> namespace Assimp { @@ -140,7 +144,7 @@ private: bool mSwallow, mSkip_empty_lines, mTrim; }; -inline +AI_FORCE_INLINE LineSplitter::LineSplitter(StreamReaderLE& stream, bool skip_empty_lines, bool trim ) : mIdx(0) , mCur() @@ -153,12 +157,12 @@ LineSplitter::LineSplitter(StreamReaderLE& stream, bool skip_empty_lines, bool t mIdx = 0; } -inline +AI_FORCE_INLINE LineSplitter::~LineSplitter() { // empty } -inline +AI_FORCE_INLINE LineSplitter& LineSplitter::operator++() { if (mSwallow) { mSwallow = false; @@ -199,12 +203,12 @@ LineSplitter& LineSplitter::operator++() { return *this; } -inline +AI_FORCE_INLINE LineSplitter &LineSplitter::operator++(int) { return ++(*this); } -inline +AI_FORCE_INLINE const char *LineSplitter::operator[] (size_t idx) const { const char* s = operator->()->c_str(); @@ -222,7 +226,7 @@ const char *LineSplitter::operator[] (size_t idx) const { } template <size_t N> -inline +AI_FORCE_INLINE void LineSplitter::get_tokens(const char* (&tokens)[N]) const { const char* s = operator->()->c_str(); @@ -238,44 +242,44 @@ void LineSplitter::get_tokens(const char* (&tokens)[N]) const { } } -inline +AI_FORCE_INLINE const std::string* LineSplitter::operator -> () const { return &mCur; } -inline +AI_FORCE_INLINE std::string LineSplitter::operator* () const { return mCur; } -inline +AI_FORCE_INLINE LineSplitter::operator bool() const { return mStream.GetRemainingSize() > 0; } -inline +AI_FORCE_INLINE LineSplitter::operator line_idx() const { return mIdx; } -inline +AI_FORCE_INLINE LineSplitter::line_idx LineSplitter::get_index() const { return mIdx; } -inline +AI_FORCE_INLINE StreamReaderLE &LineSplitter::get_stream() { return mStream; } -inline +AI_FORCE_INLINE bool LineSplitter::match_start(const char* check) { const size_t len = ::strlen(check); return len <= mCur.length() && std::equal(check, check + len, mCur.begin()); } -inline +AI_FORCE_INLINE void LineSplitter::swallow_next_increment() { mSwallow = true; } diff --git a/thirdparty/assimp/include/assimp/LogAux.h b/thirdparty/assimp/include/assimp/LogAux.h index 558485272e..bcead78dd3 100644 --- a/thirdparty/assimp/include/assimp/LogAux.h +++ b/thirdparty/assimp/include/assimp/LogAux.h @@ -43,9 +43,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file LogAux.h * @brief Common logging usage patterns for importer implementations */ +#pragma once #ifndef INCLUDED_AI_LOGAUX_H #define INCLUDED_AI_LOGAUX_H +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/TinyFormatter.h> #include <assimp/Exceptional.h> #include <assimp/DefaultLogger.hpp> diff --git a/thirdparty/assimp/include/assimp/Macros.h b/thirdparty/assimp/include/assimp/Macros.h deleted file mode 100644 index 6515303372..0000000000 --- a/thirdparty/assimp/include/assimp/Macros.h +++ /dev/null @@ -1,49 +0,0 @@ -/* ---------------------------------------------------------------------------- -Open Asset Import Library (assimp) ---------------------------------------------------------------------------- - -Copyright (c) 2006-2019, assimp team - -All rights reserved. - -Redistribution and use of this software in source and binary forms, -with or without modification, are permitted provided that the following -conditions are met: - -* Redistributions of source code must retain the above - copyright notice, this list of conditions and the - following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the - following disclaimer in the documentation and/or other - materials provided with the distribution. - -* Neither the name of the assimp team, nor the names of its - contributors may be used to endorse or promote products - derived from this software without specific prior - written permission of the assimp team. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------------- -*/ - -/* Helper macro to set a pointer to NULL in debug builds - */ -#if (defined ASSIMP_BUILD_DEBUG) -# define AI_DEBUG_INVALIDATE_PTR(x) x = NULL; -#else -# define AI_DEBUG_INVALIDATE_PTR(x) -#endif - diff --git a/thirdparty/assimp/include/assimp/MathFunctions.h b/thirdparty/assimp/include/assimp/MathFunctions.h index cb3b696076..b6c5872a72 100644 --- a/thirdparty/assimp/include/assimp/MathFunctions.h +++ b/thirdparty/assimp/include/assimp/MathFunctions.h @@ -39,22 +39,28 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------- */ +#pragma once + +#ifdef __GNUC__ +# pragma GCC system_header +#endif + /** @file MathFunctions.h - * @brief Implementation of the math functions (gcd and lcm) +* @brief Implementation of math utility functions. * - * Copied from BoostWorkaround/math - */ +*/ + +#include <limits> namespace Assimp { namespace Math { // TODO: use binary GCD for unsigned integers .... template < typename IntegerType > -IntegerType gcd( IntegerType a, IntegerType b ) -{ +inline +IntegerType gcd( IntegerType a, IntegerType b ) { const IntegerType zero = (IntegerType)0; - while ( true ) - { + while ( true ) { if ( a == zero ) return b; b %= a; @@ -66,12 +72,19 @@ IntegerType gcd( IntegerType a, IntegerType b ) } template < typename IntegerType > -IntegerType lcm( IntegerType a, IntegerType b ) -{ +inline +IntegerType lcm( IntegerType a, IntegerType b ) { const IntegerType t = gcd (a,b); - if (!t)return t; + if (!t) + return t; return a / t * b; } +template<class T> +inline +T getEpsilon() { + return std::numeric_limits<T>::epsilon(); +} + } } diff --git a/thirdparty/assimp/include/assimp/MemoryIOWrapper.h b/thirdparty/assimp/include/assimp/MemoryIOWrapper.h index c522787184..5598d4fc5f 100644 --- a/thirdparty/assimp/include/assimp/MemoryIOWrapper.h +++ b/thirdparty/assimp/include/assimp/MemoryIOWrapper.h @@ -42,12 +42,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file MemoryIOWrapper.h * Handy IOStream/IOSystem implemetation to read directly from a memory buffer */ +#pragma once #ifndef AI_MEMORYIOSTREAM_H_INC #define AI_MEMORYIOSTREAM_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/IOStream.hpp> #include <assimp/IOSystem.hpp> #include <assimp/ai_assert.h> + #include <stdint.h> namespace Assimp { diff --git a/thirdparty/assimp/include/assimp/ParsingUtils.h b/thirdparty/assimp/include/assimp/ParsingUtils.h index 6b9574fc67..3025601246 100644 --- a/thirdparty/assimp/include/assimp/ParsingUtils.h +++ b/thirdparty/assimp/include/assimp/ParsingUtils.h @@ -44,11 +44,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file ParsingUtils.h * @brief Defines helper functions for text parsing */ +#pragma once #ifndef AI_PARSING_UTILS_H_INC #define AI_PARSING_UTILS_H_INC -#include "StringComparison.h" -#include "StringUtils.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/StringComparison.h> +#include <assimp/StringUtils.h> #include <assimp/defs.h> namespace Assimp { diff --git a/thirdparty/assimp/include/assimp/Profiler.h b/thirdparty/assimp/include/assimp/Profiler.h index 6ff9d41c0a..624029be99 100644 --- a/thirdparty/assimp/include/assimp/Profiler.h +++ b/thirdparty/assimp/include/assimp/Profiler.h @@ -43,12 +43,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file Profiler.h * @brief Utility to measure the respective runtime of each import step */ -#ifndef INCLUDED_PROFILER_H -#define INCLUDED_PROFILER_H +#pragma once +#ifndef AI_INCLUDED_PROFILER_H +#define AI_INCLUDED_PROFILER_H + +#ifdef __GNUC__ +# pragma GCC system_header +#endif #include <chrono> #include <assimp/DefaultLogger.hpp> -#include "TinyFormatter.h" +#include <assimp/TinyFormatter.h> #include <map> @@ -67,7 +72,6 @@ public: // empty } -public: /** Start a named timer */ void BeginRegion(const std::string& region) { @@ -95,5 +99,5 @@ private: } } -#endif +#endif // AI_INCLUDED_PROFILER_H diff --git a/thirdparty/assimp/include/assimp/ProgressHandler.hpp b/thirdparty/assimp/include/assimp/ProgressHandler.hpp index 4e47f1d0a6..8991a64618 100644 --- a/thirdparty/assimp/include/assimp/ProgressHandler.hpp +++ b/thirdparty/assimp/include/assimp/ProgressHandler.hpp @@ -47,9 +47,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_PROGRESSHANDLER_H_INC #define AI_PROGRESSHANDLER_H_INC -#include "types.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/types.h> -namespace Assimp { +namespace Assimp { // ------------------------------------------------------------------------------------ /** @brief CPP-API: Abstract interface for custom progress report receivers. diff --git a/thirdparty/assimp/include/assimp/RemoveComments.h b/thirdparty/assimp/include/assimp/RemoveComments.h index 404b496719..f129420535 100644 --- a/thirdparty/assimp/include/assimp/RemoveComments.h +++ b/thirdparty/assimp/include/assimp/RemoveComments.h @@ -4,7 +4,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2019, assimp team - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -43,9 +42,13 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file Declares a helper class, "CommentRemover", which can be * used to remove comments (single and multi line) from a text file. */ +#pragma once #ifndef AI_REMOVE_COMMENTS_H_INC #define AI_REMOVE_COMMENTS_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif #include <assimp/defs.h> @@ -58,8 +61,7 @@ namespace Assimp { * to those in C or C++ so this code has been moved to a separate * module. */ -class ASSIMP_API CommentRemover -{ +class ASSIMP_API CommentRemover { // class cannot be instanced CommentRemover() {} diff --git a/thirdparty/assimp/include/assimp/SGSpatialSort.h b/thirdparty/assimp/include/assimp/SGSpatialSort.h index 5b4f3f41f2..fdb5ce8174 100644 --- a/thirdparty/assimp/include/assimp/SGSpatialSort.h +++ b/thirdparty/assimp/include/assimp/SGSpatialSort.h @@ -42,9 +42,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** Small helper classes to optimize finding vertices close to a given location */ +#pragma once #ifndef AI_D3DSSPATIALSORT_H_INC #define AI_D3DSSPATIALSORT_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/types.h> #include <vector> #include <stdint.h> diff --git a/thirdparty/assimp/include/assimp/SceneCombiner.h b/thirdparty/assimp/include/assimp/SceneCombiner.h index 679a2acea4..0683c1e052 100644 --- a/thirdparty/assimp/include/assimp/SceneCombiner.h +++ b/thirdparty/assimp/include/assimp/SceneCombiner.h @@ -43,17 +43,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file Declares a helper class, "SceneCombiner" providing various * utilities to merge scenes. */ +#pragma once #ifndef AI_SCENE_COMBINER_H_INC #define AI_SCENE_COMBINER_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/ai_assert.h> #include <assimp/types.h> #include <assimp/Defines.h> + #include <stddef.h> #include <set> #include <list> #include <stdint.h> - #include <vector> struct aiScene; @@ -65,8 +70,10 @@ struct aiLight; struct aiMetadata; struct aiBone; struct aiMesh; +struct aiAnimMesh; struct aiAnimation; struct aiNodeAnim; +struct aiMeshMorphAnim; namespace Assimp { @@ -363,6 +370,7 @@ public: static void Copy (aiMesh** dest, const aiMesh* src); // similar to Copy(): + static void Copy (aiAnimMesh** dest, const aiAnimMesh* src); static void Copy (aiMaterial** dest, const aiMaterial* src); static void Copy (aiTexture** dest, const aiTexture* src); static void Copy (aiAnimation** dest, const aiAnimation* src); @@ -370,6 +378,7 @@ public: static void Copy (aiBone** dest, const aiBone* src); static void Copy (aiLight** dest, const aiLight* src); static void Copy (aiNodeAnim** dest, const aiNodeAnim* src); + static void Copy (aiMeshMorphAnim** dest, const aiMeshMorphAnim* src); static void Copy (aiMetadata** dest, const aiMetadata* src); // recursive, of course diff --git a/thirdparty/assimp/include/assimp/SkeletonMeshBuilder.h b/thirdparty/assimp/include/assimp/SkeletonMeshBuilder.h index f9b8d9f55c..ad979a33fa 100644 --- a/thirdparty/assimp/include/assimp/SkeletonMeshBuilder.h +++ b/thirdparty/assimp/include/assimp/SkeletonMeshBuilder.h @@ -47,9 +47,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * for animation skeletons. */ +#pragma once #ifndef AI_SKELETONMESHBUILDER_H_INC #define AI_SKELETONMESHBUILDER_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <vector> #include <assimp/mesh.h> diff --git a/thirdparty/assimp/include/assimp/SmoothingGroups.h b/thirdparty/assimp/include/assimp/SmoothingGroups.h index 92d65cea02..c1a93947f1 100644 --- a/thirdparty/assimp/include/assimp/SmoothingGroups.h +++ b/thirdparty/assimp/include/assimp/SmoothingGroups.h @@ -43,10 +43,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file Defines the helper data structures for importing 3DS files. http://www.jalix.org/ressources/graphics/3DS/_unofficials/3ds-unofficial.txt */ +#pragma once #ifndef AI_SMOOTHINGGROUPS_H_INC #define AI_SMOOTHINGGROUPS_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/vector3.h> + #include <stdint.h> #include <vector> diff --git a/thirdparty/assimp/include/assimp/SmoothingGroups.inl b/thirdparty/assimp/include/assimp/SmoothingGroups.inl index 84ea4a1b00..37ea083dbe 100644 --- a/thirdparty/assimp/include/assimp/SmoothingGroups.inl +++ b/thirdparty/assimp/include/assimp/SmoothingGroups.inl @@ -41,13 +41,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file Generation of normal vectors basing on smoothing groups */ +#pragma once #ifndef AI_SMOOTHINGGROUPS_INL_INCLUDED #define AI_SMOOTHINGGROUPS_INL_INCLUDED -// internal headers +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/SGSpatialSort.h> -// CRT header #include <algorithm> using namespace Assimp; diff --git a/thirdparty/assimp/include/assimp/SpatialSort.h b/thirdparty/assimp/include/assimp/SpatialSort.h index 61b345bcbf..9f93543150 100644 --- a/thirdparty/assimp/include/assimp/SpatialSort.h +++ b/thirdparty/assimp/include/assimp/SpatialSort.h @@ -41,9 +41,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** Small helper classes to optimise finding vertizes close to a given location */ +#pragma once #ifndef AI_SPATIALSORT_H_INC #define AI_SPATIALSORT_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <vector> #include <assimp/types.h> diff --git a/thirdparty/assimp/include/assimp/StandardShapes.h b/thirdparty/assimp/include/assimp/StandardShapes.h index 3791569b83..c594cb63f4 100644 --- a/thirdparty/assimp/include/assimp/StandardShapes.h +++ b/thirdparty/assimp/include/assimp/StandardShapes.h @@ -41,11 +41,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** @file Declares a helper class, "StandardShapes" which generates - * vertices for standard shapes, such as cylnders, cones, spheres .. + * vertices for standard shapes, such as cylinders, cones, spheres .. */ +#pragma once #ifndef AI_STANDARD_SHAPES_H_INC #define AI_STANDARD_SHAPES_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/vector3.h> #include <vector> diff --git a/thirdparty/assimp/include/assimp/StreamReader.h b/thirdparty/assimp/include/assimp/StreamReader.h index 9116c14261..cb24f1595b 100644 --- a/thirdparty/assimp/include/assimp/StreamReader.h +++ b/thirdparty/assimp/include/assimp/StreamReader.h @@ -44,15 +44,19 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file Defines the StreamReader class which reads data from * a binary stream with a well-defined endianness. */ - +#pragma once #ifndef AI_STREAMREADER_H_INCLUDED #define AI_STREAMREADER_H_INCLUDED +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/IOStream.hpp> #include <assimp/Defines.h> +#include <assimp/ByteSwapper.h> +#include <assimp/Exceptional.h> -#include "ByteSwapper.h" -#include "Exceptional.h" #include <memory> namespace Assimp { diff --git a/thirdparty/assimp/include/assimp/StreamWriter.h b/thirdparty/assimp/include/assimp/StreamWriter.h index c7cf6c0d74..489e8adfe3 100644 --- a/thirdparty/assimp/include/assimp/StreamWriter.h +++ b/thirdparty/assimp/include/assimp/StreamWriter.h @@ -43,11 +43,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file Defines the StreamWriter class which writes data to * a binary stream with a well-defined endianness. */ - +#pragma once #ifndef AI_STREAMWRITER_H_INCLUDED #define AI_STREAMWRITER_H_INCLUDED -#include "ByteSwapper.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/ByteSwapper.h> #include <assimp/IOStream.hpp> #include <memory> diff --git a/thirdparty/assimp/include/assimp/StringComparison.h b/thirdparty/assimp/include/assimp/StringComparison.h index 8acef277b9..d3ca3e9714 100644 --- a/thirdparty/assimp/include/assimp/StringComparison.h +++ b/thirdparty/assimp/include/assimp/StringComparison.h @@ -49,12 +49,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. These functions are not consistently available on all platforms, or the provided implementations behave too differently. */ +#pragma once #ifndef INCLUDED_AI_STRING_WORKERS_H #define INCLUDED_AI_STRING_WORKERS_H +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/ai_assert.h> #include <assimp/defs.h> -#include "StringComparison.h" +#include <assimp/StringComparison.h> #include <string.h> #include <stdint.h> diff --git a/thirdparty/assimp/include/assimp/StringUtils.h b/thirdparty/assimp/include/assimp/StringUtils.h index d68b7fa479..af481f819e 100644 --- a/thirdparty/assimp/include/assimp/StringUtils.h +++ b/thirdparty/assimp/include/assimp/StringUtils.h @@ -39,9 +39,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------- */ +#pragma once #ifndef INCLUDED_AI_STRINGUTILS_H #define INCLUDED_AI_STRINGUTILS_H +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/defs.h> #include <sstream> diff --git a/thirdparty/assimp/include/assimp/Subdivision.h b/thirdparty/assimp/include/assimp/Subdivision.h index 43feb73b30..e9450267ec 100644 --- a/thirdparty/assimp/include/assimp/Subdivision.h +++ b/thirdparty/assimp/include/assimp/Subdivision.h @@ -45,7 +45,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_SUBDISIVION_H_INC #define AI_SUBDISIVION_H_INC -#include <cstddef> +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/types.h> struct aiMesh; diff --git a/thirdparty/assimp/include/assimp/TinyFormatter.h b/thirdparty/assimp/include/assimp/TinyFormatter.h index 1226b482e6..6227e42c52 100644 --- a/thirdparty/assimp/include/assimp/TinyFormatter.h +++ b/thirdparty/assimp/include/assimp/TinyFormatter.h @@ -45,9 +45,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * to get rid of the boost::format dependency. Much slinker, * basically just extends stringstream. */ +#pragma once #ifndef INCLUDED_TINY_FORMATTER_H #define INCLUDED_TINY_FORMATTER_H +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <sstream> namespace Assimp { @@ -65,24 +70,15 @@ namespace Formatter { * @endcode */ template < typename T, typename CharTraits = std::char_traits<T>, - typename Allocator = std::allocator<T> -> -class basic_formatter -{ - -public: - - typedef class std::basic_string< - T,CharTraits,Allocator - > string; - - typedef class std::basic_ostringstream< - T,CharTraits,Allocator - > stringstream; - + typename Allocator = std::allocator<T> > +class basic_formatter { public: + typedef class std::basic_string<T,CharTraits,Allocator> string; + typedef class std::basic_ostringstream<T,CharTraits,Allocator> stringstream; - basic_formatter() {} + basic_formatter() { + // empty + } /* Allow basic_formatter<T>'s to be used almost interchangeably * with std::(w)string or const (w)char* arguments because the @@ -104,14 +100,10 @@ public: } #endif - -public: - operator string () const { return underlying.str(); } - /* note - this is declared const because binding temporaries does only * work for const references, so many function prototypes will * include const basic_formatter<T>& s but might still want to diff --git a/thirdparty/assimp/include/assimp/Vertex.h b/thirdparty/assimp/include/assimp/Vertex.h index 2a7f0256ad..5e63db5fe4 100644 --- a/thirdparty/assimp/include/assimp/Vertex.h +++ b/thirdparty/assimp/include/assimp/Vertex.h @@ -47,12 +47,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. that are not currently well-defined (and would cause compile errors due to missing operators in the math library), are commented. */ +#pragma once #ifndef AI_VERTEX_H_INC #define AI_VERTEX_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/vector3.h> #include <assimp/mesh.h> #include <assimp/ai_assert.h> + #include <functional> namespace Assimp { @@ -91,23 +97,14 @@ namespace Assimp { * to *all* vertex components equally. This is useful for stuff like interpolation * or subdivision, but won't work if special handling is required for some vertex components. */ // ------------------------------------------------------------------------------------------------ -class Vertex -{ +class Vertex { friend Vertex operator + (const Vertex&,const Vertex&); friend Vertex operator - (const Vertex&,const Vertex&); - -// friend Vertex operator + (const Vertex&,ai_real); -// friend Vertex operator - (const Vertex&,ai_real); friend Vertex operator * (const Vertex&,ai_real); friend Vertex operator / (const Vertex&,ai_real); - -// friend Vertex operator + (ai_real, const Vertex&); -// friend Vertex operator - (ai_real, const Vertex&); friend Vertex operator * (ai_real, const Vertex&); -// friend Vertex operator / (ai_real, const Vertex&); public: - Vertex() {} // ---------------------------------------------------------------------------- @@ -158,8 +155,6 @@ public: } } -public: - Vertex& operator += (const Vertex& v) { *this = *this+v; return *this; @@ -170,18 +165,6 @@ public: return *this; } - -/* - Vertex& operator += (ai_real v) { - *this = *this+v; - return *this; - } - - Vertex& operator -= (ai_real v) { - *this = *this-v; - return *this; - } -*/ Vertex& operator *= (ai_real v) { *this = *this*v; return *this; @@ -192,12 +175,9 @@ public: return *this; } -public: - // ---------------------------------------------------------------------------- /** Convert back to non-interleaved storage */ void SortBack(aiMesh* out, unsigned int idx) const { - ai_assert(idx<out->mNumVertices); out->mVertices[idx] = position; @@ -291,8 +271,6 @@ public: aiColor4D colors[AI_MAX_NUMBER_OF_COLOR_SETS]; }; - - // ------------------------------------------------------------------------------------------------ AI_FORCE_INLINE Vertex operator + (const Vertex& v0,const Vertex& v1) { return Vertex::BinaryOp<std::plus>(v0,v1); @@ -302,19 +280,6 @@ AI_FORCE_INLINE Vertex operator - (const Vertex& v0,const Vertex& v1) { return Vertex::BinaryOp<std::minus>(v0,v1); } - -// ------------------------------------------------------------------------------------------------ -/* -AI_FORCE_INLINE Vertex operator + (const Vertex& v0,ai_real f) { - return Vertex::BinaryOp<Intern::plus>(v0,f); -} - -AI_FORCE_INLINE Vertex operator - (const Vertex& v0,ai_real f) { - return Vertex::BinaryOp<Intern::minus>(v0,f); -} - -*/ - AI_FORCE_INLINE Vertex operator * (const Vertex& v0,ai_real f) { return Vertex::BinaryOp<Intern::multiplies>(v0,f); } @@ -323,26 +288,10 @@ AI_FORCE_INLINE Vertex operator / (const Vertex& v0,ai_real f) { return Vertex::BinaryOp<Intern::multiplies>(v0,1.f/f); } -// ------------------------------------------------------------------------------------------------ -/* -AI_FORCE_INLINE Vertex operator + (ai_real f,const Vertex& v0) { - return Vertex::BinaryOp<Intern::plus>(f,v0); -} - -AI_FORCE_INLINE Vertex operator - (ai_real f,const Vertex& v0) { - return Vertex::BinaryOp<Intern::minus>(f,v0); -} -*/ - AI_FORCE_INLINE Vertex operator * (ai_real f,const Vertex& v0) { return Vertex::BinaryOp<Intern::multiplies>(f,v0); } -/* -AI_FORCE_INLINE Vertex operator / (ai_real f,const Vertex& v0) { - return Vertex::BinaryOp<Intern::divides>(f,v0); } -*/ -} -#endif +#endif // AI_VERTEX_H_INC diff --git a/thirdparty/assimp/include/assimp/XMLTools.h b/thirdparty/assimp/include/assimp/XMLTools.h index b0d3276873..95f12cdebf 100644 --- a/thirdparty/assimp/include/assimp/XMLTools.h +++ b/thirdparty/assimp/include/assimp/XMLTools.h @@ -40,9 +40,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------------- */ +#pragma once #ifndef INCLUDED_ASSIMP_XML_TOOLS_H #define INCLUDED_ASSIMP_XML_TOOLS_H +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <string> namespace Assimp { diff --git a/thirdparty/assimp/include/assimp/aabb.h b/thirdparty/assimp/include/assimp/aabb.h index a20f317424..83bb62256b 100644 --- a/thirdparty/assimp/include/assimp/aabb.h +++ b/thirdparty/assimp/include/assimp/aabb.h @@ -5,8 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2019, assimp team - - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -45,6 +43,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_AABB_H_INC #define AI_AABB_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/vector3.h> struct aiAABB { @@ -69,8 +71,9 @@ struct aiAABB { // empty } -#endif +#endif // __cplusplus + }; -#endif +#endif // AI_AABB_H_INC diff --git a/thirdparty/assimp/include/assimp/ai_assert.h b/thirdparty/assimp/include/assimp/ai_assert.h index e5de5d3f36..2b32b01d3e 100644 --- a/thirdparty/assimp/include/assimp/ai_assert.h +++ b/thirdparty/assimp/include/assimp/ai_assert.h @@ -44,6 +44,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_ASSERT_H_INC #define AI_ASSERT_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #ifdef ASSIMP_BUILD_DEBUG # include <assert.h> # define ai_assert(expression) assert( expression ) diff --git a/thirdparty/assimp/include/assimp/anim.h b/thirdparty/assimp/include/assimp/anim.h index 02e92739ec..e208b11adb 100644 --- a/thirdparty/assimp/include/assimp/anim.h +++ b/thirdparty/assimp/include/assimp/anim.h @@ -50,6 +50,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_ANIM_H_INC #define AI_ANIM_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/types.h> #include <assimp/quaternion.h> diff --git a/thirdparty/assimp/include/assimp/camera.h b/thirdparty/assimp/include/assimp/camera.h index e573eea5d1..adb749ff59 100644 --- a/thirdparty/assimp/include/assimp/camera.h +++ b/thirdparty/assimp/include/assimp/camera.h @@ -47,6 +47,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_CAMERA_H_INC #define AI_CAMERA_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include "types.h" #ifdef __cplusplus @@ -113,7 +117,6 @@ struct aiCamera */ C_STRUCT aiVector3D mPosition; - /** 'Up' - vector of the camera coordinate system relative to * the coordinate space defined by the corresponding node. * @@ -134,7 +137,6 @@ struct aiCamera */ C_STRUCT aiVector3D mLookAt; - /** Half horizontal field of view angle, in radians. * * The field of view angle is the angle between the center diff --git a/thirdparty/assimp/include/assimp/cexport.h b/thirdparty/assimp/include/assimp/cexport.h index 1d62dc26b3..cbc0253d50 100644 --- a/thirdparty/assimp/include/assimp/cexport.h +++ b/thirdparty/assimp/include/assimp/cexport.h @@ -3,7 +3,7 @@ Open Asset Import Library (assimp) --------------------------------------------------------------------------- -Copyright (c) 2006-2011, assimp team +Copyright (c) 2006-2019, assimp team All rights reserved. @@ -46,6 +46,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_EXPORT_H_INC #define AI_EXPORT_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #ifndef ASSIMP_BUILD_NO_EXPORT // Public ASSIMP data structures diff --git a/thirdparty/assimp/include/assimp/cfileio.h b/thirdparty/assimp/include/assimp/cfileio.h index 8f7ca45469..be90999d87 100644 --- a/thirdparty/assimp/include/assimp/cfileio.h +++ b/thirdparty/assimp/include/assimp/cfileio.h @@ -48,10 +48,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_FILEIO_H_INC #define AI_FILEIO_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/types.h> + #ifdef __cplusplus extern "C" { #endif + struct aiFileIO; struct aiFile; diff --git a/thirdparty/assimp/include/assimp/cimport.h b/thirdparty/assimp/include/assimp/cimport.h index dbd10f1370..66b1c9a174 100644 --- a/thirdparty/assimp/include/assimp/cimport.h +++ b/thirdparty/assimp/include/assimp/cimport.h @@ -48,8 +48,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_ASSIMP_H_INC #define AI_ASSIMP_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/types.h> -#include "importerdesc.h" +#include <assimp/importerdesc.h> #ifdef __cplusplus extern "C" { diff --git a/thirdparty/assimp/include/assimp/color4.h b/thirdparty/assimp/include/assimp/color4.h index 3c97c8eda2..fa86128f4f 100644 --- a/thirdparty/assimp/include/assimp/color4.h +++ b/thirdparty/assimp/include/assimp/color4.h @@ -47,7 +47,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_COLOR4D_H_INC #define AI_COLOR4D_H_INC -#include "defs.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/defs.h> #ifdef __cplusplus @@ -56,8 +60,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * alpha component. Color values range from 0 to 1. */ // ---------------------------------------------------------------------------------- template <typename TReal> -class aiColor4t -{ +class aiColor4t { public: aiColor4t() AI_NO_EXCEPT : r(), g(), b(), a() {} aiColor4t (TReal _r, TReal _g, TReal _b, TReal _a) @@ -65,14 +68,12 @@ public: explicit aiColor4t (TReal _r) : r(_r), g(_r), b(_r), a(_r) {} aiColor4t (const aiColor4t& o) = default; -public: // combined operators const aiColor4t& operator += (const aiColor4t& o); const aiColor4t& operator -= (const aiColor4t& o); const aiColor4t& operator *= (TReal f); const aiColor4t& operator /= (TReal f); -public: // comparison bool operator == (const aiColor4t& other) const; bool operator != (const aiColor4t& other) const; @@ -85,8 +86,6 @@ public: /** check whether a color is (close to) black */ inline bool IsBlack() const; -public: - // Red, green, blue and alpha color values TReal r, g, b, a; }; // !struct aiColor4D diff --git a/thirdparty/assimp/include/assimp/color4.inl b/thirdparty/assimp/include/assimp/color4.inl index afa53dcb5b..d4a2a98109 100644 --- a/thirdparty/assimp/include/assimp/color4.inl +++ b/thirdparty/assimp/include/assimp/color4.inl @@ -48,36 +48,61 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_COLOR4D_INL_INC #define AI_COLOR4D_INL_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #ifdef __cplusplus -#include "color4.h" +#include <assimp/color4.h> // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE const aiColor4t<TReal>& aiColor4t<TReal>::operator += (const aiColor4t<TReal>& o) { - r += o.r; g += o.g; b += o.b; a += o.a; +AI_FORCE_INLINE +const aiColor4t<TReal>& aiColor4t<TReal>::operator += (const aiColor4t<TReal>& o) { + r += o.r; + g += o.g; + b += o.b; + a += o.a; + return *this; } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE const aiColor4t<TReal>& aiColor4t<TReal>::operator -= (const aiColor4t<TReal>& o) { - r -= o.r; g -= o.g; b -= o.b; a -= o.a; +AI_FORCE_INLINE +const aiColor4t<TReal>& aiColor4t<TReal>::operator -= (const aiColor4t<TReal>& o) { + r -= o.r; + g -= o.g; + b -= o.b; + a -= o.a; + return *this; } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE const aiColor4t<TReal>& aiColor4t<TReal>::operator *= (TReal f) { - r *= f; g *= f; b *= f; a *= f; +AI_FORCE_INLINE +const aiColor4t<TReal>& aiColor4t<TReal>::operator *= (TReal f) { + r *= f; + g *= f; + b *= f; + a *= f; + return *this; } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE const aiColor4t<TReal>& aiColor4t<TReal>::operator /= (TReal f) { - r /= f; g /= f; b /= f; a /= f; +AI_FORCE_INLINE +const aiColor4t<TReal>& aiColor4t<TReal>::operator /= (TReal f) { + r /= f; + g /= f; + b /= f; + a /= f; + return *this; } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE TReal aiColor4t<TReal>::operator[](unsigned int i) const { +AI_FORCE_INLINE +TReal aiColor4t<TReal>::operator[](unsigned int i) const { switch ( i ) { case 0: return r; @@ -94,7 +119,8 @@ AI_FORCE_INLINE TReal aiColor4t<TReal>::operator[](unsigned int i) const { } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE TReal& aiColor4t<TReal>::operator[](unsigned int i) { +AI_FORCE_INLINE +TReal& aiColor4t<TReal>::operator[](unsigned int i) { switch ( i ) { case 0: return r; @@ -111,17 +137,20 @@ AI_FORCE_INLINE TReal& aiColor4t<TReal>::operator[](unsigned int i) { } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE bool aiColor4t<TReal>::operator== (const aiColor4t<TReal>& other) const { +AI_FORCE_INLINE +bool aiColor4t<TReal>::operator== (const aiColor4t<TReal>& other) const { return r == other.r && g == other.g && b == other.b && a == other.a; } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE bool aiColor4t<TReal>::operator!= (const aiColor4t<TReal>& other) const { +AI_FORCE_INLINE +bool aiColor4t<TReal>::operator!= (const aiColor4t<TReal>& other) const { return r != other.r || g != other.g || b != other.b || a != other.a; } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE bool aiColor4t<TReal>::operator< (const aiColor4t<TReal>& other) const { +AI_FORCE_INLINE +bool aiColor4t<TReal>::operator< (const aiColor4t<TReal>& other) const { return r < other.r || ( r == other.r && ( g < other.g || ( @@ -136,14 +165,17 @@ AI_FORCE_INLINE bool aiColor4t<TReal>::operator< (const aiColor4t<TReal>& other) ) ); } + // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE aiColor4t<TReal> operator + (const aiColor4t<TReal>& v1, const aiColor4t<TReal>& v2) { +AI_FORCE_INLINE +aiColor4t<TReal> operator + (const aiColor4t<TReal>& v1, const aiColor4t<TReal>& v2) { return aiColor4t<TReal>( v1.r + v2.r, v1.g + v2.g, v1.b + v2.b, v1.a + v2.a); } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE aiColor4t<TReal> operator - (const aiColor4t<TReal>& v1, const aiColor4t<TReal>& v2) { +AI_FORCE_INLINE +aiColor4t<TReal> operator - (const aiColor4t<TReal>& v1, const aiColor4t<TReal>& v2) { return aiColor4t<TReal>( v1.r - v2.r, v1.g - v2.g, v1.b - v2.b, v1.a - v2.a); } // ------------------------------------------------------------------------------------------------ @@ -153,53 +185,63 @@ AI_FORCE_INLINE aiColor4t<TReal> operator * (const aiColor4t<TReal>& v1, const a } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE aiColor4t<TReal> operator / (const aiColor4t<TReal>& v1, const aiColor4t<TReal>& v2) { +AI_FORCE_INLINE +aiColor4t<TReal> operator / (const aiColor4t<TReal>& v1, const aiColor4t<TReal>& v2) { return aiColor4t<TReal>( v1.r / v2.r, v1.g / v2.g, v1.b / v2.b, v1.a / v2.a); } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE aiColor4t<TReal> operator * ( TReal f, const aiColor4t<TReal>& v) { +AI_FORCE_INLINE +aiColor4t<TReal> operator * ( TReal f, const aiColor4t<TReal>& v) { return aiColor4t<TReal>( f*v.r, f*v.g, f*v.b, f*v.a); } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE aiColor4t<TReal> operator * ( const aiColor4t<TReal>& v, TReal f) { +AI_FORCE_INLINE +aiColor4t<TReal> operator * ( const aiColor4t<TReal>& v, TReal f) { return aiColor4t<TReal>( f*v.r, f*v.g, f*v.b, f*v.a); } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE aiColor4t<TReal> operator / ( const aiColor4t<TReal>& v, TReal f) { +AI_FORCE_INLINE +aiColor4t<TReal> operator / ( const aiColor4t<TReal>& v, TReal f) { return v * (1/f); } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE aiColor4t<TReal> operator / ( TReal f,const aiColor4t<TReal>& v) { +AI_FORCE_INLINE +aiColor4t<TReal> operator / ( TReal f,const aiColor4t<TReal>& v) { return aiColor4t<TReal>(f,f,f,f)/v; } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE aiColor4t<TReal> operator + ( const aiColor4t<TReal>& v, TReal f) { +AI_FORCE_INLINE +aiColor4t<TReal> operator + ( const aiColor4t<TReal>& v, TReal f) { return aiColor4t<TReal>( f+v.r, f+v.g, f+v.b, f+v.a); } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE aiColor4t<TReal> operator - ( const aiColor4t<TReal>& v, TReal f) { +AI_FORCE_INLINE +aiColor4t<TReal> operator - ( const aiColor4t<TReal>& v, TReal f) { return aiColor4t<TReal>( v.r-f, v.g-f, v.b-f, v.a-f); } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE aiColor4t<TReal> operator + ( TReal f, const aiColor4t<TReal>& v) { +AI_FORCE_INLINE +aiColor4t<TReal> operator + ( TReal f, const aiColor4t<TReal>& v) { return aiColor4t<TReal>( f+v.r, f+v.g, f+v.b, f+v.a); } // ------------------------------------------------------------------------------------------------ template <typename TReal> -AI_FORCE_INLINE aiColor4t<TReal> operator - ( TReal f, const aiColor4t<TReal>& v) { +AI_FORCE_INLINE +aiColor4t<TReal> operator - ( TReal f, const aiColor4t<TReal>& v) { return aiColor4t<TReal>( f-v.r, f-v.g, f-v.b, f-v.a); } // ------------------------------------------------------------------------------------------------ template <typename TReal> -inline bool aiColor4t<TReal> :: IsBlack() const { +AI_FORCE_INLINE +bool aiColor4t<TReal>::IsBlack() const { // The alpha component doesn't care here. black is black. static const TReal epsilon = 10e-3f; return std::fabs( r ) < epsilon && std::fabs( g ) < epsilon && std::fabs( b ) < epsilon; diff --git a/thirdparty/assimp/include/assimp/defs.h b/thirdparty/assimp/include/assimp/defs.h index 05a5e3fd4b..6f2f8ae88b 100644 --- a/thirdparty/assimp/include/assimp/defs.h +++ b/thirdparty/assimp/include/assimp/defs.h @@ -5,8 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2019, assimp team - - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -50,6 +48,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_DEFINES_H_INC #define AI_DEFINES_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/config.h> ////////////////////////////////////////////////////////////////////////// @@ -126,16 +128,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * GENBOUNDINGBOXES */ ////////////////////////////////////////////////////////////////////////// -#ifdef _MSC_VER +#ifdef _WIN32 # undef ASSIMP_API - ////////////////////////////////////////////////////////////////////////// /* Define 'ASSIMP_BUILD_DLL_EXPORT' to build a DLL of the library */ ////////////////////////////////////////////////////////////////////////// # ifdef ASSIMP_BUILD_DLL_EXPORT # define ASSIMP_API __declspec(dllexport) # define ASSIMP_API_WINONLY __declspec(dllexport) -# pragma warning (disable : 4251) ////////////////////////////////////////////////////////////////////////// /* Define 'ASSIMP_DLL' before including Assimp to link to ASSIMP in @@ -148,7 +148,19 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # define ASSIMP_API # define ASSIMP_API_WINONLY # endif +#elif defined(SWIG) + + /* Do nothing, the relevant defines are all in AssimpSwigPort.i */ +#else +# define ASSIMP_API __attribute__ ((visibility("default"))) +# define ASSIMP_API_WINONLY +#endif + +#ifdef _MSC_VER +# ifdef ASSIMP_BUILD_DLL_EXPORT +# pragma warning (disable : 4251) +# endif /* Force the compiler to inline a function, if possible */ # define AI_FORCE_INLINE __forceinline @@ -156,17 +168,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* Tells the compiler that a function never returns. Used in code analysis * to skip dead paths (e.g. after an assertion evaluated to false). */ # define AI_WONT_RETURN __declspec(noreturn) - #elif defined(SWIG) /* Do nothing, the relevant defines are all in AssimpSwigPort.i */ #else - # define AI_WONT_RETURN - -# define ASSIMP_API __attribute__ ((visibility("default"))) -# define ASSIMP_API_WINONLY # define AI_FORCE_INLINE inline #endif // (defined _MSC_VER) @@ -291,9 +298,10 @@ static const ai_real ai_epsilon = (ai_real) 0.00001; #endif -/* To avoid running out of memory - * This can be adjusted for specific use cases - * It's NOT a total limit, just a limit for individual allocations +/** + * To avoid running out of memory + * This can be adjusted for specific use cases + * It's NOT a total limit, just a limit for individual allocations */ #define AI_MAX_ALLOC(type) ((256U * 1024 * 1024) / sizeof(type)) @@ -307,4 +315,13 @@ static const ai_real ai_epsilon = (ai_real) 0.00001; # endif #endif // _MSC_VER +/** + * Helper macro to set a pointer to NULL in debug builds + */ +#if (defined ASSIMP_BUILD_DEBUG) +# define AI_DEBUG_INVALIDATE_PTR(x) x = NULL; +#else +# define AI_DEBUG_INVALIDATE_PTR(x) +#endif + #endif // !! AI_DEFINES_H_INC diff --git a/thirdparty/assimp/include/assimp/fast_atof.h b/thirdparty/assimp/include/assimp/fast_atof.h index 62ea969e97..6e9a1bba7a 100644 --- a/thirdparty/assimp/include/assimp/fast_atof.h +++ b/thirdparty/assimp/include/assimp/fast_atof.h @@ -13,10 +13,14 @@ // to ensure long numbers are handled correctly // ------------------------------------------------------------------------------------ - +#pragma once #ifndef FAST_A_TO_F_H_INCLUDED #define FAST_A_TO_F_H_INCLUDED +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <cmath> #include <limits> #include <stdint.h> diff --git a/thirdparty/assimp/include/assimp/importerdesc.h b/thirdparty/assimp/include/assimp/importerdesc.h index 36e387f011..0a6919c1ae 100644 --- a/thirdparty/assimp/include/assimp/importerdesc.h +++ b/thirdparty/assimp/include/assimp/importerdesc.h @@ -48,11 +48,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_IMPORTER_DESC_H_INC #define AI_IMPORTER_DESC_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + /** Mixed set of flags for #aiImporterDesc, indicating some features * common to many importers*/ -enum aiImporterFlags -{ +enum aiImporterFlags { /** Indicates that there is a textual encoding of the * file format; and that it is supported.*/ aiImporterFlags_SupportTextFlavour = 0x1, @@ -87,8 +90,7 @@ enum aiImporterFlags * as importers/exporters are added to Assimp, so it might be useful * to have a common mechanism to query some rough importer * characteristics. */ -struct aiImporterDesc -{ +struct aiImporterDesc { /** Full name of the importer (i.e. Blender3D importer)*/ const char* mName; diff --git a/thirdparty/assimp/include/assimp/light.h b/thirdparty/assimp/include/assimp/light.h index 1667cfb8c1..bdb2368c4f 100644 --- a/thirdparty/assimp/include/assimp/light.h +++ b/thirdparty/assimp/include/assimp/light.h @@ -49,7 +49,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_LIGHT_H_INC #define AI_LIGHT_H_INC -#include "types.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/types.h> #ifdef __cplusplus extern "C" { diff --git a/thirdparty/assimp/include/assimp/material.h b/thirdparty/assimp/include/assimp/material.h index 206eb2a2b0..19a7c69709 100644 --- a/thirdparty/assimp/include/assimp/material.h +++ b/thirdparty/assimp/include/assimp/material.h @@ -48,7 +48,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_MATERIAL_H_INC #define AI_MATERIAL_H_INC -#include "types.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/types.h> #ifdef __cplusplus extern "C" { diff --git a/thirdparty/assimp/include/assimp/material.inl b/thirdparty/assimp/include/assimp/material.inl index b05d6af6c3..8ae6b88d3e 100644 --- a/thirdparty/assimp/include/assimp/material.inl +++ b/thirdparty/assimp/include/assimp/material.inl @@ -49,14 +49,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_MATERIAL_INL_INC #define AI_MATERIAL_INL_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + // --------------------------------------------------------------------------- -inline aiPropertyTypeInfo ai_real_to_property_type_info(float) -{ +AI_FORCE_INLINE +aiPropertyTypeInfo ai_real_to_property_type_info(float) { return aiPTI_Float; } -inline aiPropertyTypeInfo ai_real_to_property_type_info(double) -{ +AI_FORCE_INLINE +aiPropertyTypeInfo ai_real_to_property_type_info(double) { return aiPTI_Double; } // --------------------------------------------------------------------------- @@ -64,30 +68,30 @@ inline aiPropertyTypeInfo ai_real_to_property_type_info(double) //! @cond never // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::GetTexture( aiTextureType type, - unsigned int index, - C_STRUCT aiString* path, - aiTextureMapping* mapping /*= NULL*/, - unsigned int* uvindex /*= NULL*/, - ai_real* blend /*= NULL*/, - aiTextureOp* op /*= NULL*/, - aiTextureMapMode* mapmode /*= NULL*/) const -{ +AI_FORCE_INLINE +aiReturn aiMaterial::GetTexture( aiTextureType type, + unsigned int index, + C_STRUCT aiString* path, + aiTextureMapping* mapping /*= NULL*/, + unsigned int* uvindex /*= NULL*/, + ai_real* blend /*= NULL*/, + aiTextureOp* op /*= NULL*/, + aiTextureMapMode* mapmode /*= NULL*/) const { return ::aiGetMaterialTexture(this,type,index,path,mapping,uvindex,blend,op,mapmode); } // --------------------------------------------------------------------------- -inline unsigned int aiMaterial::GetTextureCount(aiTextureType type) const -{ +AI_FORCE_INLINE +unsigned int aiMaterial::GetTextureCount(aiTextureType type) const { return ::aiGetMaterialTextureCount(this,type); } // --------------------------------------------------------------------------- template <typename Type> -inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, - unsigned int idx, Type* pOut, - unsigned int* pMax) const -{ +AI_FORCE_INLINE +aiReturn aiMaterial::Get(const char* pKey,unsigned int type, + unsigned int idx, Type* pOut, + unsigned int* pMax) const { unsigned int iNum = pMax ? *pMax : 1; const aiMaterialProperty* prop; @@ -114,9 +118,9 @@ inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, // --------------------------------------------------------------------------- template <typename Type> -inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, - unsigned int idx,Type& pOut) const -{ +AI_FORCE_INLINE +aiReturn aiMaterial::Get(const char* pKey,unsigned int type, + unsigned int idx,Type& pOut) const { const aiMaterialProperty* prop; const aiReturn ret = ::aiGetMaterialProperty(this,pKey,type,idx, (const aiMaterialProperty**)&prop); @@ -136,60 +140,56 @@ inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, - unsigned int idx,ai_real* pOut, - unsigned int* pMax) const -{ +AI_FORCE_INLINE +aiReturn aiMaterial::Get(const char* pKey,unsigned int type, + unsigned int idx,ai_real* pOut, + unsigned int* pMax) const { return ::aiGetMaterialFloatArray(this,pKey,type,idx,pOut,pMax); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, - unsigned int idx,int* pOut, - unsigned int* pMax) const -{ +AI_FORCE_INLINE +aiReturn aiMaterial::Get(const char* pKey,unsigned int type, + unsigned int idx,int* pOut, + unsigned int* pMax) const { return ::aiGetMaterialIntegerArray(this,pKey,type,idx,pOut,pMax); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, - unsigned int idx,ai_real& pOut) const -{ +AI_FORCE_INLINE +aiReturn aiMaterial::Get(const char* pKey,unsigned int type, + unsigned int idx,ai_real& pOut) const { return aiGetMaterialFloat(this,pKey,type,idx,&pOut); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, - unsigned int idx,int& pOut) const -{ +AI_FORCE_INLINE +aiReturn aiMaterial::Get(const char* pKey,unsigned int type, + unsigned int idx,int& pOut) const { return aiGetMaterialInteger(this,pKey,type,idx,&pOut); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, - unsigned int idx,aiColor4D& pOut) const -{ +AI_FORCE_INLINE +aiReturn aiMaterial::Get(const char* pKey,unsigned int type, + unsigned int idx,aiColor4D& pOut) const { return aiGetMaterialColor(this,pKey,type,idx,&pOut); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, - unsigned int idx,aiColor3D& pOut) const -{ +AI_FORCE_INLINE aiReturn aiMaterial::Get(const char* pKey,unsigned int type, + unsigned int idx,aiColor3D& pOut) const { aiColor4D c; const aiReturn ret = aiGetMaterialColor(this,pKey,type,idx,&c); pOut = aiColor3D(c.r,c.g,c.b); return ret; } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, - unsigned int idx,aiString& pOut) const -{ +AI_FORCE_INLINE aiReturn aiMaterial::Get(const char* pKey,unsigned int type, + unsigned int idx,aiString& pOut) const { return aiGetMaterialString(this,pKey,type,idx,&pOut); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::Get(const char* pKey,unsigned int type, - unsigned int idx,aiUVTransform& pOut) const -{ +AI_FORCE_INLINE aiReturn aiMaterial::Get(const char* pKey,unsigned int type, + unsigned int idx,aiUVTransform& pOut) const { return aiGetMaterialUVTransform(this,pKey,type,idx,&pOut); } - // --------------------------------------------------------------------------- template<class TYPE> aiReturn aiMaterial::AddProperty (const TYPE* pInput, @@ -204,84 +204,83 @@ aiReturn aiMaterial::AddProperty (const TYPE* pInput, } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::AddProperty(const float* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE aiReturn aiMaterial::AddProperty(const float* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(float), pKey,type,index,aiPTI_Float); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::AddProperty(const double* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty(const double* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(double), pKey,type,index,aiPTI_Double); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::AddProperty(const aiUVTransform* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty(const aiUVTransform* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(aiUVTransform), pKey,type,index,ai_real_to_property_type_info(pInput->mRotation)); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::AddProperty(const aiColor4D* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty(const aiColor4D* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(aiColor4D), pKey,type,index,ai_real_to_property_type_info(pInput->a)); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::AddProperty(const aiColor3D* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty(const aiColor3D* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(aiColor3D), pKey,type,index,ai_real_to_property_type_info(pInput->b)); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::AddProperty(const aiVector3D* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty(const aiVector3D* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(aiVector3D), pKey,type,index,ai_real_to_property_type_info(pInput->x)); } // --------------------------------------------------------------------------- -inline aiReturn aiMaterial::AddProperty(const int* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty(const int* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(int), pKey,type,index,aiPTI_Integer); @@ -296,12 +295,12 @@ inline aiReturn aiMaterial::AddProperty(const int* pInput, // --------------------------------------------------------------------------- template<> -inline aiReturn aiMaterial::AddProperty<float>(const float* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty<float>(const float* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(float), pKey,type,index,aiPTI_Float); @@ -309,12 +308,12 @@ inline aiReturn aiMaterial::AddProperty<float>(const float* pInput, // --------------------------------------------------------------------------- template<> -inline aiReturn aiMaterial::AddProperty<double>(const double* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty<double>(const double* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(double), pKey,type,index,aiPTI_Double); @@ -322,12 +321,12 @@ inline aiReturn aiMaterial::AddProperty<double>(const double* pInput, // --------------------------------------------------------------------------- template<> -inline aiReturn aiMaterial::AddProperty<aiUVTransform>(const aiUVTransform* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty<aiUVTransform>(const aiUVTransform* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(aiUVTransform), pKey,type,index,aiPTI_Float); @@ -335,12 +334,12 @@ inline aiReturn aiMaterial::AddProperty<aiUVTransform>(const aiUVTransform* pInp // --------------------------------------------------------------------------- template<> -inline aiReturn aiMaterial::AddProperty<aiColor4D>(const aiColor4D* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty<aiColor4D>(const aiColor4D* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(aiColor4D), pKey,type,index,aiPTI_Float); @@ -348,12 +347,12 @@ inline aiReturn aiMaterial::AddProperty<aiColor4D>(const aiColor4D* pInput, // --------------------------------------------------------------------------- template<> -inline aiReturn aiMaterial::AddProperty<aiColor3D>(const aiColor3D* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty<aiColor3D>(const aiColor3D* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(aiColor3D), pKey,type,index,aiPTI_Float); @@ -361,12 +360,12 @@ inline aiReturn aiMaterial::AddProperty<aiColor3D>(const aiColor3D* pInput, // --------------------------------------------------------------------------- template<> -inline aiReturn aiMaterial::AddProperty<aiVector3D>(const aiVector3D* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty<aiVector3D>(const aiVector3D* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(aiVector3D), pKey,type,index,aiPTI_Float); @@ -374,12 +373,12 @@ inline aiReturn aiMaterial::AddProperty<aiVector3D>(const aiVector3D* pInput, // --------------------------------------------------------------------------- template<> -inline aiReturn aiMaterial::AddProperty<int>(const int* pInput, - const unsigned int pNumValues, - const char* pKey, - unsigned int type, - unsigned int index) -{ +AI_FORCE_INLINE +aiReturn aiMaterial::AddProperty<int>(const int* pInput, + const unsigned int pNumValues, + const char* pKey, + unsigned int type, + unsigned int index) { return AddBinaryProperty((const void*)pInput, pNumValues * sizeof(int), pKey,type,index,aiPTI_Integer); diff --git a/thirdparty/assimp/include/assimp/matrix3x3.h b/thirdparty/assimp/include/assimp/matrix3x3.h index 22b69561ff..2c26cf92bb 100644 --- a/thirdparty/assimp/include/assimp/matrix3x3.h +++ b/thirdparty/assimp/include/assimp/matrix3x3.h @@ -48,7 +48,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_MATRIX3X3_H_INC #define AI_MATRIX3X3_H_INC -#include "defs.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/defs.h> #ifdef __cplusplus @@ -65,10 +69,8 @@ template <typename T> class aiVector2t; * defined thereby. */ template <typename TReal> -class aiMatrix3x3t -{ +class aiMatrix3x3t { public: - aiMatrix3x3t() AI_NO_EXCEPT : a1(static_cast<TReal>(1.0f)), a2(), a3(), b1(), b2(static_cast<TReal>(1.0f)), b3(), @@ -82,8 +84,6 @@ public: c1(_c1), c2(_c2), c3(_c3) {} -public: - // matrix multiplication. aiMatrix3x3t& operator *= (const aiMatrix3x3t& m); aiMatrix3x3t operator * (const aiMatrix3x3t& m) const; @@ -101,8 +101,6 @@ public: template <typename TOther> operator aiMatrix3x3t<TOther> () const; -public: - // ------------------------------------------------------------------- /** @brief Construction from a 4x4 matrix. The remaining parts * of the matrix are ignored. @@ -122,7 +120,6 @@ public: aiMatrix3x3t& Inverse(); TReal Determinant() const; -public: // ------------------------------------------------------------------- /** @brief Returns a rotation matrix for a rotation around z * @param a Rotation angle, in radians diff --git a/thirdparty/assimp/include/assimp/matrix3x3.inl b/thirdparty/assimp/include/assimp/matrix3x3.inl index d9d45a3e92..1ce8c9691c 100644 --- a/thirdparty/assimp/include/assimp/matrix3x3.inl +++ b/thirdparty/assimp/include/assimp/matrix3x3.inl @@ -48,10 +48,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_MATRIX3X3_INL_INC #define AI_MATRIX3X3_INL_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #ifdef __cplusplus -#include "matrix3x3.h" +#include <assimp/matrix3x3.h> +#include <assimp/matrix4x4.h> -#include "matrix4x4.h" #include <algorithm> #include <cmath> #include <limits> @@ -59,8 +63,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // ------------------------------------------------------------------------------------------------ // Construction from a 4x4 matrix. The remaining parts of the matrix are ignored. template <typename TReal> -inline aiMatrix3x3t<TReal>::aiMatrix3x3t( const aiMatrix4x4t<TReal>& pMatrix) -{ +AI_FORCE_INLINE +aiMatrix3x3t<TReal>::aiMatrix3x3t( const aiMatrix4x4t<TReal>& pMatrix) { a1 = pMatrix.a1; a2 = pMatrix.a2; a3 = pMatrix.a3; b1 = pMatrix.b1; b2 = pMatrix.b2; b3 = pMatrix.b3; c1 = pMatrix.c1; c2 = pMatrix.c2; c3 = pMatrix.c3; @@ -68,8 +72,8 @@ inline aiMatrix3x3t<TReal>::aiMatrix3x3t( const aiMatrix4x4t<TReal>& pMatrix) // ------------------------------------------------------------------------------------------------ template <typename TReal> -inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::operator *= (const aiMatrix3x3t<TReal>& m) -{ +AI_FORCE_INLINE +aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::operator *= (const aiMatrix3x3t<TReal>& m) { *this = aiMatrix3x3t<TReal>(m.a1 * a1 + m.b1 * a2 + m.c1 * a3, m.a2 * a1 + m.b2 * a2 + m.c2 * a3, m.a3 * a1 + m.b3 * a2 + m.c3 * a3, @@ -85,8 +89,7 @@ inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::operator *= (const aiMatrix3x3t // ------------------------------------------------------------------------------------------------ template <typename TReal> template <typename TOther> -aiMatrix3x3t<TReal>::operator aiMatrix3x3t<TOther> () const -{ +aiMatrix3x3t<TReal>::operator aiMatrix3x3t<TOther> () const { return aiMatrix3x3t<TOther>(static_cast<TOther>(a1),static_cast<TOther>(a2),static_cast<TOther>(a3), static_cast<TOther>(b1),static_cast<TOther>(b2),static_cast<TOther>(b3), static_cast<TOther>(c1),static_cast<TOther>(c2),static_cast<TOther>(c3)); @@ -94,8 +97,8 @@ aiMatrix3x3t<TReal>::operator aiMatrix3x3t<TOther> () const // ------------------------------------------------------------------------------------------------ template <typename TReal> -inline aiMatrix3x3t<TReal> aiMatrix3x3t<TReal>::operator* (const aiMatrix3x3t<TReal>& m) const -{ +AI_FORCE_INLINE +aiMatrix3x3t<TReal> aiMatrix3x3t<TReal>::operator* (const aiMatrix3x3t<TReal>& m) const { aiMatrix3x3t<TReal> temp( *this); temp *= m; return temp; @@ -103,7 +106,8 @@ inline aiMatrix3x3t<TReal> aiMatrix3x3t<TReal>::operator* (const aiMatrix3x3t<TR // ------------------------------------------------------------------------------------------------ template <typename TReal> -inline TReal* aiMatrix3x3t<TReal>::operator[] (unsigned int p_iIndex) { +AI_FORCE_INLINE +TReal* aiMatrix3x3t<TReal>::operator[] (unsigned int p_iIndex) { switch ( p_iIndex ) { case 0: return &a1; @@ -119,7 +123,8 @@ inline TReal* aiMatrix3x3t<TReal>::operator[] (unsigned int p_iIndex) { // ------------------------------------------------------------------------------------------------ template <typename TReal> -inline const TReal* aiMatrix3x3t<TReal>::operator[] (unsigned int p_iIndex) const { +AI_FORCE_INLINE +const TReal* aiMatrix3x3t<TReal>::operator[] (unsigned int p_iIndex) const { switch ( p_iIndex ) { case 0: return &a1; @@ -135,8 +140,8 @@ inline const TReal* aiMatrix3x3t<TReal>::operator[] (unsigned int p_iIndex) cons // ------------------------------------------------------------------------------------------------ template <typename TReal> -inline bool aiMatrix3x3t<TReal>::operator== (const aiMatrix4x4t<TReal>& m) const -{ +AI_FORCE_INLINE +bool aiMatrix3x3t<TReal>::operator== (const aiMatrix4x4t<TReal>& m) const { return a1 == m.a1 && a2 == m.a2 && a3 == m.a3 && b1 == m.b1 && b2 == m.b2 && b3 == m.b3 && c1 == m.c1 && c2 == m.c2 && c3 == m.c3; @@ -144,14 +149,15 @@ inline bool aiMatrix3x3t<TReal>::operator== (const aiMatrix4x4t<TReal>& m) const // ------------------------------------------------------------------------------------------------ template <typename TReal> -inline bool aiMatrix3x3t<TReal>::operator!= (const aiMatrix4x4t<TReal>& m) const -{ +AI_FORCE_INLINE +bool aiMatrix3x3t<TReal>::operator!= (const aiMatrix4x4t<TReal>& m) const { return !(*this == m); } // --------------------------------------------------------------------------- template<typename TReal> -inline bool aiMatrix3x3t<TReal>::Equal(const aiMatrix4x4t<TReal>& m, TReal epsilon) const { +AI_FORCE_INLINE +bool aiMatrix3x3t<TReal>::Equal(const aiMatrix4x4t<TReal>& m, TReal epsilon) const { return std::abs(a1 - m.a1) <= epsilon && std::abs(a2 - m.a2) <= epsilon && @@ -166,8 +172,8 @@ inline bool aiMatrix3x3t<TReal>::Equal(const aiMatrix4x4t<TReal>& m, TReal epsil // ------------------------------------------------------------------------------------------------ template <typename TReal> -inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Transpose() -{ +AI_FORCE_INLINE +aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Transpose() { // (TReal&) don't remove, GCC complains cause of packed fields std::swap( (TReal&)a2, (TReal&)b1); std::swap( (TReal&)a3, (TReal&)c1); @@ -177,15 +183,15 @@ inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Transpose() // ---------------------------------------------------------------------------------------- template <typename TReal> -inline TReal aiMatrix3x3t<TReal>::Determinant() const -{ +AI_FORCE_INLINE +TReal aiMatrix3x3t<TReal>::Determinant() const { return a1*b2*c3 - a1*b3*c2 + a2*b3*c1 - a2*b1*c3 + a3*b1*c2 - a3*b2*c1; } // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Inverse() -{ +AI_FORCE_INLINE +aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Inverse() { // Compute the reciprocal determinant TReal det = Determinant(); if(det == static_cast<TReal>(0.0)) @@ -219,8 +225,8 @@ inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Inverse() // ------------------------------------------------------------------------------------------------ template <typename TReal> -inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::RotationZ(TReal a, aiMatrix3x3t<TReal>& out) -{ +AI_FORCE_INLINE +aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::RotationZ(TReal a, aiMatrix3x3t<TReal>& out) { out.a1 = out.b2 = std::cos(a); out.b1 = std::sin(a); out.a2 = - out.b1; @@ -234,8 +240,8 @@ inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::RotationZ(TReal a, aiMatrix3x3t // ------------------------------------------------------------------------------------------------ // Returns a rotation matrix for a rotation around an arbitrary axis. template <typename TReal> -inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Rotation( TReal a, const aiVector3t<TReal>& axis, aiMatrix3x3t<TReal>& out) -{ +AI_FORCE_INLINE +aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Rotation( TReal a, const aiVector3t<TReal>& axis, aiMatrix3x3t<TReal>& out) { TReal c = std::cos( a), s = std::sin( a), t = 1 - c; TReal x = axis.x, y = axis.y, z = axis.z; @@ -249,8 +255,8 @@ inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Rotation( TReal a, const aiVect // ------------------------------------------------------------------------------------------------ template <typename TReal> -inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Translation( const aiVector2t<TReal>& v, aiMatrix3x3t<TReal>& out) -{ +AI_FORCE_INLINE +aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Translation( const aiVector2t<TReal>& v, aiMatrix3x3t<TReal>& out) { out = aiMatrix3x3t<TReal>(); out.a3 = v.x; out.b3 = v.y; @@ -268,9 +274,8 @@ inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::Translation( const aiVector2t<T */ // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::FromToMatrix(const aiVector3t<TReal>& from, - const aiVector3t<TReal>& to, aiMatrix3x3t<TReal>& mtx) -{ +AI_FORCE_INLINE aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::FromToMatrix(const aiVector3t<TReal>& from, + const aiVector3t<TReal>& to, aiMatrix3x3t<TReal>& mtx) { const TReal e = from * to; const TReal f = (e < 0)? -e:e; @@ -352,6 +357,5 @@ inline aiMatrix3x3t<TReal>& aiMatrix3x3t<TReal>::FromToMatrix(const aiVector3t<T return mtx; } - #endif // __cplusplus #endif // AI_MATRIX3X3_INL_INC diff --git a/thirdparty/assimp/include/assimp/matrix4x4.h b/thirdparty/assimp/include/assimp/matrix4x4.h index 046bb535f2..8fc216f669 100644 --- a/thirdparty/assimp/include/assimp/matrix4x4.h +++ b/thirdparty/assimp/include/assimp/matrix4x4.h @@ -47,8 +47,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_MATRIX4X4_H_INC #define AI_MATRIX4X4_H_INC -#include "vector3.h" -#include "defs.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/vector3.h> +#include <assimp/defs.h> #ifdef __cplusplus @@ -66,8 +70,7 @@ template<typename TReal> class aiQuaterniont; * defined thereby. */ template<typename TReal> -class aiMatrix4x4t -{ +class aiMatrix4x4t { public: /** set to identity */ @@ -91,8 +94,6 @@ public: aiMatrix4x4t(const aiVector3t<TReal>& scaling, const aiQuaterniont<TReal>& rotation, const aiVector3t<TReal>& position); -public: - // array access operators /** @fn TReal* operator[] (unsigned int p_iIndex) * @param [in] p_iIndex - index of the row. @@ -120,8 +121,6 @@ public: template <typename TOther> operator aiMatrix4x4t<TOther> () const; -public: - // ------------------------------------------------------------------- /** @brief Transpose the matrix */ aiMatrix4x4t& Transpose(); @@ -182,7 +181,6 @@ public: void DecomposeNoScaling (aiQuaterniont<TReal>& rotation, aiVector3t<TReal>& position) const; - // ------------------------------------------------------------------- /** @brief Creates a trafo matrix from a set of euler angles * @param x Rotation angle for the x-axis, in radians @@ -192,7 +190,6 @@ public: aiMatrix4x4t& FromEulerAnglesXYZ(TReal x, TReal y, TReal z); aiMatrix4x4t& FromEulerAnglesXYZ(const aiVector3t<TReal>& blubb); -public: // ------------------------------------------------------------------- /** @brief Returns a rotation matrix for a rotation around the x axis * @param a Rotation angle, in radians @@ -256,7 +253,6 @@ public: static aiMatrix4x4t& FromToMatrix(const aiVector3t<TReal>& from, const aiVector3t<TReal>& to, aiMatrix4x4t& out); -public: TReal a1, a2, a3, a4; TReal b1, b2, b3, b4; TReal c1, c2, c3, c4; diff --git a/thirdparty/assimp/include/assimp/matrix4x4.inl b/thirdparty/assimp/include/assimp/matrix4x4.inl index ebc67a06ec..84079974f7 100644 --- a/thirdparty/assimp/include/assimp/matrix4x4.inl +++ b/thirdparty/assimp/include/assimp/matrix4x4.inl @@ -5,8 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2019, assimp team - - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -53,6 +51,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "matrix4x4.h" #include "matrix3x3.h" #include "quaternion.h" +#include "MathFunctions.h" #include <algorithm> #include <limits> @@ -61,12 +60,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // ---------------------------------------------------------------------------------------- template <typename TReal> aiMatrix4x4t<TReal>::aiMatrix4x4t() AI_NO_EXCEPT : - a1(1.0f), a2(), a3(), a4(), - b1(), b2(1.0f), b3(), b4(), - c1(), c2(), c3(1.0f), c4(), - d1(), d2(), d3(), d4(1.0f) -{ - + a1(1.0f), a2(), a3(), a4(), + b1(), b2(1.0f), b3(), b4(), + c1(), c2(), c3(1.0f), c4(), + d1(), d2(), d3(), d4(1.0f) { + // empty } // ---------------------------------------------------------------------------------------- @@ -75,19 +73,17 @@ aiMatrix4x4t<TReal>::aiMatrix4x4t (TReal _a1, TReal _a2, TReal _a3, TReal _a4, TReal _b1, TReal _b2, TReal _b3, TReal _b4, TReal _c1, TReal _c2, TReal _c3, TReal _c4, TReal _d1, TReal _d2, TReal _d3, TReal _d4) : - a1(_a1), a2(_a2), a3(_a3), a4(_a4), - b1(_b1), b2(_b2), b3(_b3), b4(_b4), - c1(_c1), c2(_c2), c3(_c3), c4(_c4), - d1(_d1), d2(_d2), d3(_d3), d4(_d4) -{ - + a1(_a1), a2(_a2), a3(_a3), a4(_a4), + b1(_b1), b2(_b2), b3(_b3), b4(_b4), + c1(_c1), c2(_c2), c3(_c3), c4(_c4), + d1(_d1), d2(_d2), d3(_d3), d4(_d4) { + // empty } // ------------------------------------------------------------------------------------------------ template <typename TReal> template <typename TOther> -aiMatrix4x4t<TReal>::operator aiMatrix4x4t<TOther> () const -{ +aiMatrix4x4t<TReal>::operator aiMatrix4x4t<TOther> () const { return aiMatrix4x4t<TOther>(static_cast<TOther>(a1),static_cast<TOther>(a2),static_cast<TOther>(a3),static_cast<TOther>(a4), static_cast<TOther>(b1),static_cast<TOther>(b2),static_cast<TOther>(b3),static_cast<TOther>(b4), static_cast<TOther>(c1),static_cast<TOther>(c2),static_cast<TOther>(c3),static_cast<TOther>(c4), @@ -97,8 +93,8 @@ aiMatrix4x4t<TReal>::operator aiMatrix4x4t<TOther> () const // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>::aiMatrix4x4t (const aiMatrix3x3t<TReal>& m) -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal>::aiMatrix4x4t (const aiMatrix3x3t<TReal>& m) { a1 = m.a1; a2 = m.a2; a3 = m.a3; a4 = static_cast<TReal>(0.0); b1 = m.b1; b2 = m.b2; b3 = m.b3; b4 = static_cast<TReal>(0.0); c1 = m.c1; c2 = m.c2; c3 = m.c3; c4 = static_cast<TReal>(0.0); @@ -107,8 +103,8 @@ inline aiMatrix4x4t<TReal>::aiMatrix4x4t (const aiMatrix3x3t<TReal>& m) // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>::aiMatrix4x4t (const aiVector3t<TReal>& scaling, const aiQuaterniont<TReal>& rotation, const aiVector3t<TReal>& position) -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal>::aiMatrix4x4t (const aiVector3t<TReal>& scaling, const aiQuaterniont<TReal>& rotation, const aiVector3t<TReal>& position) { // build a 3x3 rotation matrix aiMatrix3x3t<TReal> m = rotation.GetMatrix(); @@ -135,8 +131,8 @@ inline aiMatrix4x4t<TReal>::aiMatrix4x4t (const aiVector3t<TReal>& scaling, cons // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::operator *= (const aiMatrix4x4t<TReal>& m) -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::operator *= (const aiMatrix4x4t<TReal>& m) { *this = aiMatrix4x4t<TReal>( m.a1 * a1 + m.b1 * a2 + m.c1 * a3 + m.d1 * a4, m.a2 * a1 + m.b2 * a2 + m.c2 * a3 + m.d2 * a4, @@ -159,8 +155,7 @@ inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::operator *= (const aiMatrix4x4t // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal> aiMatrix4x4t<TReal>::operator* (const TReal& aFloat) const -{ +AI_FORCE_INLINE aiMatrix4x4t<TReal> aiMatrix4x4t<TReal>::operator* (const TReal& aFloat) const { aiMatrix4x4t<TReal> temp( a1 * aFloat, a2 * aFloat, @@ -183,8 +178,8 @@ inline aiMatrix4x4t<TReal> aiMatrix4x4t<TReal>::operator* (const TReal& aFloat) // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal> aiMatrix4x4t<TReal>::operator+ (const aiMatrix4x4t<TReal>& m) const -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal> aiMatrix4x4t<TReal>::operator+ (const aiMatrix4x4t<TReal>& m) const { aiMatrix4x4t<TReal> temp( m.a1 + a1, m.a2 + a2, @@ -207,18 +202,16 @@ inline aiMatrix4x4t<TReal> aiMatrix4x4t<TReal>::operator+ (const aiMatrix4x4t<TR // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal> aiMatrix4x4t<TReal>::operator* (const aiMatrix4x4t<TReal>& m) const -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal> aiMatrix4x4t<TReal>::operator* (const aiMatrix4x4t<TReal>& m) const { aiMatrix4x4t<TReal> temp( *this); temp *= m; return temp; } - // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Transpose() -{ +AI_FORCE_INLINE aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Transpose() { // (TReal&) don't remove, GCC complains cause of packed fields std::swap( (TReal&)b1, (TReal&)a2); std::swap( (TReal&)c1, (TReal&)a3); @@ -229,11 +222,10 @@ inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Transpose() return *this; } - // ---------------------------------------------------------------------------------------- template <typename TReal> -inline TReal aiMatrix4x4t<TReal>::Determinant() const -{ +AI_FORCE_INLINE +TReal aiMatrix4x4t<TReal>::Determinant() const { return a1*b2*c3*d4 - a1*b2*c4*d3 + a1*b3*c4*d2 - a1*b3*c2*d4 + a1*b4*c2*d3 - a1*b4*c3*d2 - a2*b3*c4*d1 + a2*b3*c1*d4 - a2*b4*c1*d3 + a2*b4*c3*d1 - a2*b1*c3*d4 + a2*b1*c4*d3 @@ -244,8 +236,8 @@ inline TReal aiMatrix4x4t<TReal>::Determinant() const // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Inverse() -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Inverse() { // Compute the reciprocal determinant const TReal det = Determinant(); if(det == static_cast<TReal>(0.0)) @@ -289,9 +281,10 @@ inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Inverse() // ---------------------------------------------------------------------------------------- template <typename TReal> -inline TReal* aiMatrix4x4t<TReal>::operator[](unsigned int p_iIndex) { +AI_FORCE_INLINE +TReal* aiMatrix4x4t<TReal>::operator[](unsigned int p_iIndex) { if (p_iIndex > 3) { - return NULL; + return nullptr; } switch ( p_iIndex ) { case 0: @@ -310,9 +303,10 @@ inline TReal* aiMatrix4x4t<TReal>::operator[](unsigned int p_iIndex) { // ---------------------------------------------------------------------------------------- template <typename TReal> -inline const TReal* aiMatrix4x4t<TReal>::operator[](unsigned int p_iIndex) const { +AI_FORCE_INLINE +const TReal* aiMatrix4x4t<TReal>::operator[](unsigned int p_iIndex) const { if (p_iIndex > 3) { - return NULL; + return nullptr; } switch ( p_iIndex ) { @@ -332,8 +326,8 @@ inline const TReal* aiMatrix4x4t<TReal>::operator[](unsigned int p_iIndex) const // ---------------------------------------------------------------------------------------- template <typename TReal> -inline bool aiMatrix4x4t<TReal>::operator== (const aiMatrix4x4t<TReal>& m) const -{ +AI_FORCE_INLINE +bool aiMatrix4x4t<TReal>::operator== (const aiMatrix4x4t<TReal>& m) const { return (a1 == m.a1 && a2 == m.a2 && a3 == m.a3 && a4 == m.a4 && b1 == m.b1 && b2 == m.b2 && b3 == m.b3 && b4 == m.b4 && c1 == m.c1 && c2 == m.c2 && c3 == m.c3 && c4 == m.c4 && @@ -342,14 +336,15 @@ inline bool aiMatrix4x4t<TReal>::operator== (const aiMatrix4x4t<TReal>& m) const // ---------------------------------------------------------------------------------------- template <typename TReal> -inline bool aiMatrix4x4t<TReal>::operator!= (const aiMatrix4x4t<TReal>& m) const -{ +AI_FORCE_INLINE +bool aiMatrix4x4t<TReal>::operator!= (const aiMatrix4x4t<TReal>& m) const { return !(*this == m); } // --------------------------------------------------------------------------- template<typename TReal> -inline bool aiMatrix4x4t<TReal>::Equal(const aiMatrix4x4t<TReal>& m, TReal epsilon) const { +AI_FORCE_INLINE +bool aiMatrix4x4t<TReal>::Equal(const aiMatrix4x4t<TReal>& m, TReal epsilon) const { return std::abs(a1 - m.a1) <= epsilon && std::abs(a2 - m.a2) <= epsilon && @@ -401,13 +396,10 @@ inline bool aiMatrix4x4t<TReal>::Equal(const aiMatrix4x4t<TReal>& m, TReal epsil \ do {} while(false) - - - template <typename TReal> -inline void aiMatrix4x4t<TReal>::Decompose (aiVector3t<TReal>& pScaling, aiQuaterniont<TReal>& pRotation, - aiVector3t<TReal>& pPosition) const -{ +AI_FORCE_INLINE +void aiMatrix4x4t<TReal>::Decompose (aiVector3t<TReal>& pScaling, aiQuaterniont<TReal>& pRotation, + aiVector3t<TReal>& pPosition) const { ASSIMP_MATRIX4_4_DECOMPOSE_PART; // build a 3x3 rotation matrix @@ -420,8 +412,8 @@ inline void aiMatrix4x4t<TReal>::Decompose (aiVector3t<TReal>& pScaling, aiQuate } template <typename TReal> -inline void aiMatrix4x4t<TReal>::Decompose(aiVector3t<TReal>& pScaling, aiVector3t<TReal>& pRotation, aiVector3t<TReal>& pPosition) const -{ +AI_FORCE_INLINE +void aiMatrix4x4t<TReal>::Decompose(aiVector3t<TReal>& pScaling, aiVector3t<TReal>& pRotation, aiVector3t<TReal>& pPosition) const { ASSIMP_MATRIX4_4_DECOMPOSE_PART; /* @@ -442,7 +434,7 @@ inline void aiMatrix4x4t<TReal>::Decompose(aiVector3t<TReal>& pScaling, aiVector */ // Use a small epsilon to solve floating-point inaccuracies - const TReal epsilon = 10e-3f; + const TReal epsilon = Assimp::Math::getEpsilon<TReal>(); pRotation.y = std::asin(-vCols[0].z);// D. Angle around oY. @@ -475,10 +467,10 @@ inline void aiMatrix4x4t<TReal>::Decompose(aiVector3t<TReal>& pScaling, aiVector #undef ASSIMP_MATRIX4_4_DECOMPOSE_PART template <typename TReal> -inline void aiMatrix4x4t<TReal>::Decompose(aiVector3t<TReal>& pScaling, aiVector3t<TReal>& pRotationAxis, TReal& pRotationAngle, - aiVector3t<TReal>& pPosition) const -{ -aiQuaterniont<TReal> pRotation; +AI_FORCE_INLINE +void aiMatrix4x4t<TReal>::Decompose(aiVector3t<TReal>& pScaling, aiVector3t<TReal>& pRotationAxis, TReal& pRotationAngle, + aiVector3t<TReal>& pPosition) const { + aiQuaterniont<TReal> pRotation; Decompose(pScaling, pRotation, pPosition); pRotation.Normalize(); @@ -500,9 +492,9 @@ aiQuaterniont<TReal> pRotation; // ---------------------------------------------------------------------------------------- template <typename TReal> -inline void aiMatrix4x4t<TReal>::DecomposeNoScaling (aiQuaterniont<TReal>& rotation, - aiVector3t<TReal>& position) const -{ +AI_FORCE_INLINE +void aiMatrix4x4t<TReal>::DecomposeNoScaling (aiQuaterniont<TReal>& rotation, + aiVector3t<TReal>& position) const { const aiMatrix4x4t<TReal>& _this = *this; // extract translation @@ -516,15 +508,15 @@ inline void aiMatrix4x4t<TReal>::DecomposeNoScaling (aiQuaterniont<TReal>& rotat // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::FromEulerAnglesXYZ(const aiVector3t<TReal>& blubb) -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::FromEulerAnglesXYZ(const aiVector3t<TReal>& blubb) { return FromEulerAnglesXYZ(blubb.x,blubb.y,blubb.z); } // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::FromEulerAnglesXYZ(TReal x, TReal y, TReal z) -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::FromEulerAnglesXYZ(TReal x, TReal y, TReal z) { aiMatrix4x4t<TReal>& _this = *this; TReal cx = std::cos(x); @@ -552,8 +544,8 @@ inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::FromEulerAnglesXYZ(TReal x, TRe // ---------------------------------------------------------------------------------------- template <typename TReal> -inline bool aiMatrix4x4t<TReal>::IsIdentity() const -{ +AI_FORCE_INLINE +bool aiMatrix4x4t<TReal>::IsIdentity() const { // Use a small epsilon to solve floating-point inaccuracies const static TReal epsilon = 10e-3f; @@ -577,8 +569,8 @@ inline bool aiMatrix4x4t<TReal>::IsIdentity() const // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::RotationX(TReal a, aiMatrix4x4t<TReal>& out) -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::RotationX(TReal a, aiMatrix4x4t<TReal>& out) { /* | 1 0 0 0 | M = | 0 cos(A) -sin(A) 0 | @@ -592,8 +584,8 @@ inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::RotationX(TReal a, aiMatrix4x4t // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::RotationY(TReal a, aiMatrix4x4t<TReal>& out) -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::RotationY(TReal a, aiMatrix4x4t<TReal>& out) { /* | cos(A) 0 sin(A) 0 | M = | 0 1 0 0 | @@ -608,8 +600,8 @@ inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::RotationY(TReal a, aiMatrix4x4t // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::RotationZ(TReal a, aiMatrix4x4t<TReal>& out) -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::RotationZ(TReal a, aiMatrix4x4t<TReal>& out) { /* | cos(A) -sin(A) 0 0 | M = | sin(A) cos(A) 0 0 | @@ -624,26 +616,25 @@ inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::RotationZ(TReal a, aiMatrix4x4t // ---------------------------------------------------------------------------------------- // Returns a rotation matrix for a rotation around an arbitrary axis. template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Rotation( TReal a, const aiVector3t<TReal>& axis, aiMatrix4x4t<TReal>& out) -{ - TReal c = std::cos( a), s = std::sin( a), t = 1 - c; - TReal x = axis.x, y = axis.y, z = axis.z; - - // Many thanks to MathWorld and Wikipedia - out.a1 = t*x*x + c; out.a2 = t*x*y - s*z; out.a3 = t*x*z + s*y; - out.b1 = t*x*y + s*z; out.b2 = t*y*y + c; out.b3 = t*y*z - s*x; - out.c1 = t*x*z - s*y; out.c2 = t*y*z + s*x; out.c3 = t*z*z + c; - out.a4 = out.b4 = out.c4 = static_cast<TReal>(0.0); - out.d1 = out.d2 = out.d3 = static_cast<TReal>(0.0); - out.d4 = static_cast<TReal>(1.0); - - return out; +AI_FORCE_INLINE +aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Rotation( TReal a, const aiVector3t<TReal>& axis, aiMatrix4x4t<TReal>& out) { + TReal c = std::cos( a), s = std::sin( a), t = 1 - c; + TReal x = axis.x, y = axis.y, z = axis.z; + + // Many thanks to MathWorld and Wikipedia + out.a1 = t*x*x + c; out.a2 = t*x*y - s*z; out.a3 = t*x*z + s*y; + out.b1 = t*x*y + s*z; out.b2 = t*y*y + c; out.b3 = t*y*z - s*x; + out.c1 = t*x*z - s*y; out.c2 = t*y*z + s*x; out.c3 = t*z*z + c; + out.a4 = out.b4 = out.c4 = static_cast<TReal>(0.0); + out.d1 = out.d2 = out.d3 = static_cast<TReal>(0.0); + out.d4 = static_cast<TReal>(1.0); + + return out; } // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Translation( const aiVector3t<TReal>& v, aiMatrix4x4t<TReal>& out) -{ +AI_FORCE_INLINE aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Translation( const aiVector3t<TReal>& v, aiMatrix4x4t<TReal>& out) { out = aiMatrix4x4t<TReal>(); out.a4 = v.x; out.b4 = v.y; @@ -653,8 +644,8 @@ inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Translation( const aiVector3t<T // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Scaling( const aiVector3t<TReal>& v, aiMatrix4x4t<TReal>& out) -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Scaling( const aiVector3t<TReal>& v, aiMatrix4x4t<TReal>& out) { out = aiMatrix4x4t<TReal>(); out.a1 = v.x; out.b2 = v.y; @@ -673,9 +664,9 @@ inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::Scaling( const aiVector3t<TReal */ // ---------------------------------------------------------------------------------------- template <typename TReal> -inline aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::FromToMatrix(const aiVector3t<TReal>& from, - const aiVector3t<TReal>& to, aiMatrix4x4t<TReal>& mtx) -{ +AI_FORCE_INLINE +aiMatrix4x4t<TReal>& aiMatrix4x4t<TReal>::FromToMatrix(const aiVector3t<TReal>& from, + const aiVector3t<TReal>& to, aiMatrix4x4t<TReal>& mtx) { aiMatrix3x3t<TReal> m3; aiMatrix3x3t<TReal>::FromToMatrix(from,to,m3); mtx = aiMatrix4x4t<TReal>(m3); diff --git a/thirdparty/assimp/include/assimp/mesh.h b/thirdparty/assimp/include/assimp/mesh.h index f1628f1f54..fbf2a857ad 100644 --- a/thirdparty/assimp/include/assimp/mesh.h +++ b/thirdparty/assimp/include/assimp/mesh.h @@ -48,6 +48,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_MESH_H_INC #define AI_MESH_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/types.h> #include <assimp/aabb.h> @@ -248,6 +252,9 @@ struct aiVertexWeight { }; +// Forward declare aiNode (pointer use only) +struct aiNode; + // --------------------------------------------------------------------------- /** @brief A single bone of a mesh. * @@ -264,6 +271,16 @@ struct aiBone { //! The maximum value for this member is #AI_MAX_BONE_WEIGHTS. unsigned int mNumWeights; +#ifndef ASSIMP_BUILD_NO_ARMATUREPOPULATE_PROCESS + // The bone armature node - used for skeleton conversion + // you must enable aiProcess_PopulateArmatureData to populate this + C_STRUCT aiNode* mArmature; + + // The bone node in the scene - used for skeleton conversion + // you must enable aiProcess_PopulateArmatureData to populate this + C_STRUCT aiNode* mNode; + +#endif //! The influence weights of this bone, by vertex index. C_STRUCT aiVertexWeight* mWeights; @@ -418,11 +435,11 @@ struct aiAnimMesh /**Anim Mesh name */ C_STRUCT aiString mName; - /** Replacement for aiMesh::mVertices. If this array is non-NULL, + /** Replacement for aiMesh::mVertices. If this array is non-nullptr, * it *must* contain mNumVertices entries. The corresponding - * array in the host mesh must be non-NULL as well - animation + * array in the host mesh must be non-nullptr as well - animation * meshes may neither add or nor remove vertex components (if - * a replacement array is NULL and the corresponding source + * a replacement array is nullptr and the corresponding source * array is not, the source data is taken instead)*/ C_STRUCT aiVector3D* mVertices; @@ -596,7 +613,7 @@ struct aiMesh C_STRUCT aiVector3D* mVertices; /** Vertex normals. - * The array contains normalized vectors, NULL if not present. + * The array contains normalized vectors, nullptr if not present. * The array is mNumVertices in size. Normals are undefined for * point and line primitives. A mesh consisting of points and * lines only may not have normal vectors. Meshes with mixed @@ -619,7 +636,7 @@ struct aiMesh /** Vertex tangents. * The tangent of a vertex points in the direction of the positive - * X texture axis. The array contains normalized vectors, NULL if + * X texture axis. The array contains normalized vectors, nullptr if * not present. The array is mNumVertices in size. A mesh consisting * of points and lines only may not have normal vectors. Meshes with * mixed primitive types (i.e. lines and triangles) may have @@ -633,7 +650,7 @@ struct aiMesh /** Vertex bitangents. * The bitangent of a vertex points in the direction of the positive - * Y texture axis. The array contains normalized vectors, NULL if not + * Y texture axis. The array contains normalized vectors, nullptr if not * present. The array is mNumVertices in size. * @note If the mesh contains tangents, it automatically also contains * bitangents. @@ -642,14 +659,14 @@ struct aiMesh /** Vertex color sets. * A mesh may contain 0 to #AI_MAX_NUMBER_OF_COLOR_SETS vertex - * colors per vertex. NULL if not present. Each array is + * colors per vertex. nullptr if not present. Each array is * mNumVertices in size if present. */ C_STRUCT aiColor4D* mColors[AI_MAX_NUMBER_OF_COLOR_SETS]; /** Vertex texture coords, also known as UV channels. * A mesh may contain 0 to AI_MAX_NUMBER_OF_TEXTURECOORDS per - * vertex. NULL if not present. The array is mNumVertices in size. + * vertex. nullptr if not present. The array is mNumVertices in size. */ C_STRUCT aiVector3D* mTextureCoords[AI_MAX_NUMBER_OF_TEXTURECOORDS]; @@ -671,7 +688,7 @@ struct aiMesh C_STRUCT aiFace* mFaces; /** The number of bones this mesh contains. - * Can be 0, in which case the mBones array is NULL. + * Can be 0, in which case the mBones array is nullptr. */ unsigned int mNumBones; @@ -769,7 +786,10 @@ struct aiMesh // DO NOT REMOVE THIS ADDITIONAL CHECK if (mNumBones && mBones) { for( unsigned int a = 0; a < mNumBones; a++) { - delete mBones[a]; + if(mBones[a]) + { + delete mBones[a]; + } } delete [] mBones; } diff --git a/thirdparty/assimp/include/assimp/metadata.h b/thirdparty/assimp/include/assimp/metadata.h index 3a1dd1442a..849d90f485 100644 --- a/thirdparty/assimp/include/assimp/metadata.h +++ b/thirdparty/assimp/include/assimp/metadata.h @@ -48,6 +48,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_METADATA_H_INC #define AI_METADATA_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #if defined(_MSC_VER) && (_MSC_VER <= 1500) # include "Compiler/pstdint.h" #else diff --git a/thirdparty/assimp/include/assimp/pbrmaterial.h b/thirdparty/assimp/include/assimp/pbrmaterial.h index ce5f822173..892a6347f7 100644 --- a/thirdparty/assimp/include/assimp/pbrmaterial.h +++ b/thirdparty/assimp/include/assimp/pbrmaterial.h @@ -5,8 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2019, assimp team - - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -44,9 +42,14 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /** @file pbrmaterial.h * @brief Defines the material system of the library */ +#pragma once #ifndef AI_PBRMATERIAL_H_INC #define AI_PBRMATERIAL_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_BASE_COLOR_FACTOR "$mat.gltf.pbrMetallicRoughness.baseColorFactor", 0, 0 #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_METALLIC_FACTOR "$mat.gltf.pbrMetallicRoughness.metallicFactor", 0, 0 #define AI_MATKEY_GLTF_PBRMETALLICROUGHNESS_ROUGHNESS_FACTOR "$mat.gltf.pbrMetallicRoughness.roughnessFactor", 0, 0 diff --git a/thirdparty/assimp/include/assimp/postprocess.h b/thirdparty/assimp/include/assimp/postprocess.h index 2a74414216..4b6732e80a 100644 --- a/thirdparty/assimp/include/assimp/postprocess.h +++ b/thirdparty/assimp/include/assimp/postprocess.h @@ -47,7 +47,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_POSTPROCESS_H_INC #define AI_POSTPROCESS_H_INC -#include "types.h" +#include <assimp/types.h> + +#ifdef __GNUC__ +# pragma GCC system_header +#endif #ifdef __cplusplus extern "C" { @@ -316,6 +320,19 @@ enum aiPostProcessSteps */ aiProcess_FixInfacingNormals = 0x2000, + + + // ------------------------------------------------------------------------- + /** + * This step generically populates aiBone->mArmature and aiBone->mNode generically + * The point of these is it saves you later having to calculate these elements + * This is useful when handling rest information or skin information + * If you have multiple armatures on your models we strongly recommend enabling this + * Instead of writing your own multi-root, multi-armature lookups we have done the + * hard work for you :) + */ + aiProcess_PopulateArmatureData = 0x4000, + // ------------------------------------------------------------------------- /** <hr>This step splits meshes with more than one primitive type in * homogeneous sub-meshes. @@ -533,6 +550,8 @@ enum aiPostProcessSteps */ aiProcess_Debone = 0x4000000, + + // ------------------------------------------------------------------------- /** <hr>This step will perform a global scale of the model. * diff --git a/thirdparty/assimp/include/assimp/qnan.h b/thirdparty/assimp/include/assimp/qnan.h index 0918bde5e7..06780da5b8 100644 --- a/thirdparty/assimp/include/assimp/qnan.h +++ b/thirdparty/assimp/include/assimp/qnan.h @@ -50,19 +50,23 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * but last time I checked compiler coverage was so bad that I decided * to reinvent the wheel. */ - +#pragma once #ifndef AI_QNAN_H_INCLUDED #define AI_QNAN_H_INCLUDED +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #include <assimp/defs.h> + #include <limits> #include <stdint.h> // --------------------------------------------------------------------------- /** Data structure to represent the bit pattern of a 32 Bit * IEEE 754 floating-point number. */ -union _IEEESingle -{ +union _IEEESingle { float Float; struct { @@ -75,8 +79,7 @@ union _IEEESingle // --------------------------------------------------------------------------- /** Data structure to represent the bit pattern of a 64 Bit * IEEE 754 floating-point number. */ -union _IEEEDouble -{ +union _IEEEDouble { double Double; struct { @@ -89,8 +92,7 @@ union _IEEEDouble // --------------------------------------------------------------------------- /** Check whether a given float is qNaN. * @param in Input value */ -AI_FORCE_INLINE bool is_qnan(float in) -{ +AI_FORCE_INLINE bool is_qnan(float in) { // the straightforward solution does not work: // return (in != in); // compiler generates code like this @@ -107,8 +109,7 @@ AI_FORCE_INLINE bool is_qnan(float in) // --------------------------------------------------------------------------- /** Check whether a given double is qNaN. * @param in Input value */ -AI_FORCE_INLINE bool is_qnan(double in) -{ +AI_FORCE_INLINE bool is_qnan(double in) { // the straightforward solution does not work: // return (in != in); // compiler generates code like this @@ -127,8 +128,7 @@ AI_FORCE_INLINE bool is_qnan(double in) * * Denorms return false, they're treated like normal values. * @param in Input value */ -AI_FORCE_INLINE bool is_special_float(float in) -{ +AI_FORCE_INLINE bool is_special_float(float in) { _IEEESingle temp; memcpy(&temp, &in, sizeof(float)); return (temp.IEEE.Exp == (1u << 8)-1); @@ -139,8 +139,7 @@ AI_FORCE_INLINE bool is_special_float(float in) * * Denorms return false, they're treated like normal values. * @param in Input value */ -AI_FORCE_INLINE bool is_special_float(double in) -{ +AI_FORCE_INLINE bool is_special_float(double in) { _IEEESingle temp; memcpy(&temp, &in, sizeof(float)); return (temp.IEEE.Exp == (1u << 11)-1); @@ -150,15 +149,13 @@ AI_FORCE_INLINE bool is_special_float(double in) /** Check whether a float is NOT qNaN. * @param in Input value */ template<class TReal> -AI_FORCE_INLINE bool is_not_qnan(TReal in) -{ +AI_FORCE_INLINE bool is_not_qnan(TReal in) { return !is_qnan(in); } // --------------------------------------------------------------------------- /** @brief Get a fresh qnan. */ -AI_FORCE_INLINE ai_real get_qnan() -{ +AI_FORCE_INLINE ai_real get_qnan() { return std::numeric_limits<ai_real>::quiet_NaN(); } diff --git a/thirdparty/assimp/include/assimp/quaternion.h b/thirdparty/assimp/include/assimp/quaternion.h index 96574d24b9..ae45959b41 100644 --- a/thirdparty/assimp/include/assimp/quaternion.h +++ b/thirdparty/assimp/include/assimp/quaternion.h @@ -49,7 +49,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifdef __cplusplus -#include "defs.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/defs.h> template <typename TReal> class aiVector3t; template <typename TReal> class aiMatrix3x3t; diff --git a/thirdparty/assimp/include/assimp/quaternion.inl b/thirdparty/assimp/include/assimp/quaternion.inl index c26648215e..3ce514d1bb 100644 --- a/thirdparty/assimp/include/assimp/quaternion.inl +++ b/thirdparty/assimp/include/assimp/quaternion.inl @@ -48,8 +48,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_QUATERNION_INL_INC #define AI_QUATERNION_INL_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #ifdef __cplusplus -#include "quaternion.h" +#include <assimp/quaternion.h> #include <cmath> diff --git a/thirdparty/assimp/include/assimp/scene.h b/thirdparty/assimp/include/assimp/scene.h index 2667db85b3..b76709eb15 100644 --- a/thirdparty/assimp/include/assimp/scene.h +++ b/thirdparty/assimp/include/assimp/scene.h @@ -48,14 +48,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_SCENE_H_INC #define AI_SCENE_H_INC -#include "types.h" -#include "texture.h" -#include "mesh.h" -#include "light.h" -#include "camera.h" -#include "material.h" -#include "anim.h" -#include "metadata.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/types.h> +#include <assimp/texture.h> +#include <assimp/mesh.h> +#include <assimp/light.h> +#include <assimp/camera.h> +#include <assimp/material.h> +#include <assimp/anim.h> +#include <assimp/metadata.h> #ifdef __cplusplus # include <cstdlib> @@ -106,13 +110,13 @@ struct ASSIMP_API aiNode /** The transformation relative to the node's parent. */ C_STRUCT aiMatrix4x4 mTransformation; - /** Parent node. NULL if this node is the root node. */ + /** Parent node. nullptr if this node is the root node. */ C_STRUCT aiNode* mParent; /** The number of child nodes of this node. */ unsigned int mNumChildren; - /** The child nodes of this node. NULL if mNumChildren is 0. */ + /** The child nodes of this node. nullptr if mNumChildren is 0. */ C_STRUCT aiNode** mChildren; /** The number of meshes of this node. */ @@ -123,7 +127,7 @@ struct ASSIMP_API aiNode */ unsigned int* mMeshes; - /** Metadata associated with this node or NULL if there is no metadata. + /** Metadata associated with this node or nullptr if there is no metadata. * Whether any metadata is generated depends on the source file format. See the * @link importer_notes @endlink page for more information on every source file * format. Importers that don't document any metadata don't write any. @@ -145,7 +149,7 @@ struct ASSIMP_API aiNode * of the scene. * * @param name Name to search for - * @return NULL or a valid Node if the search was successful. + * @return nullptr or a valid Node if the search was successful. */ inline const aiNode* FindNode(const aiString& name) const { @@ -340,7 +344,7 @@ struct aiScene #ifdef __cplusplus - //! Default constructor - set everything to 0/NULL + //! Default constructor - set everything to 0/nullptr ASSIMP_API aiScene(); //! Destructor @@ -349,33 +353,33 @@ struct aiScene //! Check whether the scene contains meshes //! Unless no special scene flags are set this will always be true. inline bool HasMeshes() const { - return mMeshes != NULL && mNumMeshes > 0; + return mMeshes != nullptr && mNumMeshes > 0; } //! Check whether the scene contains materials //! Unless no special scene flags are set this will always be true. inline bool HasMaterials() const { - return mMaterials != NULL && mNumMaterials > 0; + return mMaterials != nullptr && mNumMaterials > 0; } //! Check whether the scene contains lights inline bool HasLights() const { - return mLights != NULL && mNumLights > 0; + return mLights != nullptr && mNumLights > 0; } //! Check whether the scene contains textures inline bool HasTextures() const { - return mTextures != NULL && mNumTextures > 0; + return mTextures != nullptr && mNumTextures > 0; } //! Check whether the scene contains cameras inline bool HasCameras() const { - return mCameras != NULL && mNumCameras > 0; + return mCameras != nullptr && mNumCameras > 0; } //! Check whether the scene contains animations inline bool HasAnimations() const { - return mAnimations != NULL && mNumAnimations > 0; + return mAnimations != nullptr && mNumAnimations > 0; } //! Returns a short filename from a full path diff --git a/thirdparty/assimp/include/assimp/texture.h b/thirdparty/assimp/include/assimp/texture.h index dc6cbef65c..0867659f46 100644 --- a/thirdparty/assimp/include/assimp/texture.h +++ b/thirdparty/assimp/include/assimp/texture.h @@ -5,8 +5,6 @@ Open Asset Import Library (assimp) Copyright (c) 2006-2019, assimp team - - All rights reserved. Redistribution and use of this software in source and binary forms, @@ -53,13 +51,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_TEXTURE_H_INC #define AI_TEXTURE_H_INC -#include "types.h" +#ifdef __GNUC__ +# pragma GCC system_header +#endif + +#include <assimp/types.h> #ifdef __cplusplus extern "C" { #endif - // -------------------------------------------------------------------------------- /** \def AI_EMBEDDED_TEXNAME_PREFIX @@ -79,7 +80,6 @@ extern "C" { # define AI_MAKE_EMBEDDED_TEXNAME(_n_) AI_EMBEDDED_TEXNAME_PREFIX # _n_ #endif - #include "./Compiler/pushpack1.h" // -------------------------------------------------------------------------------- @@ -87,8 +87,7 @@ extern "C" { * * Used by aiTexture. */ -struct aiTexel -{ +struct aiTexel { unsigned char b,g,r,a; #ifdef __cplusplus diff --git a/thirdparty/assimp/include/assimp/types.h b/thirdparty/assimp/include/assimp/types.h index 331b8cd03f..e32cae331c 100644 --- a/thirdparty/assimp/include/assimp/types.h +++ b/thirdparty/assimp/include/assimp/types.h @@ -48,22 +48,30 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_TYPES_H_INC #define AI_TYPES_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + // Some runtime headers #include <sys/types.h> #include <stddef.h> #include <string.h> #include <limits.h> +#include <stdint.h> // Our compile configuration -#include "defs.h" +#include <assimp/defs.h> // Some types moved to separate header due to size of operators -#include "vector3.h" -#include "vector2.h" -#include "color4.h" -#include "matrix3x3.h" -#include "matrix4x4.h" -#include "quaternion.h" +#include <assimp/vector3.h> +#include <assimp/vector2.h> +#include <assimp/color4.h> +#include <assimp/matrix3x3.h> +#include <assimp/matrix4x4.h> +#include <assimp/quaternion.h> + +typedef int32_t ai_int32; +typedef uint32_t ai_uint32 ; #ifdef __cplusplus #include <cstring> @@ -71,7 +79,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <string> // for aiString::Set(const std::string&) namespace Assimp { - //! @cond never +//! @cond never namespace Intern { // -------------------------------------------------------------------- /** @brief Internal helper class to utilize our internal new/delete @@ -269,8 +277,8 @@ struct aiString } /** Copy constructor */ - aiString(const aiString& rOther) : - length(rOther.length) + aiString(const aiString& rOther) + : length(rOther.length) { // Crop the string to the maximum length length = length>=MAXLEN?MAXLEN-1:length; @@ -280,7 +288,7 @@ struct aiString /** Constructor from std::string */ explicit aiString(const std::string& pString) : - length(pString.length()) + length( (ai_uint32) pString.length()) { length = length>=MAXLEN?MAXLEN-1:length; memcpy( data, pString.c_str(), length); @@ -292,15 +300,15 @@ struct aiString if( pString.length() > MAXLEN - 1) { return; } - length = pString.length(); + length = (ai_uint32)pString.length(); memcpy( data, pString.c_str(), length); data[length] = 0; } /** Copy a const char* to the aiString */ void Set( const char* sz) { - const size_t len = ::strlen(sz); - if( len > MAXLEN - 1) { + const ai_int32 len = (ai_uint32) ::strlen(sz); + if( len > (ai_int32)MAXLEN - 1) { return; } length = len; @@ -346,7 +354,7 @@ struct aiString /** Append a string to the string */ void Append (const char* app) { - const size_t len = ::strlen(app); + const ai_uint32 len = (ai_uint32) ::strlen(app); if (!len) { return; } @@ -379,7 +387,7 @@ struct aiString /** Binary length of the string excluding the terminal 0. This is NOT the * logical length of strings containing UTF-8 multi-byte sequences! It's * the number of bytes from the beginning of the string to its end.*/ - size_t length; + ai_uint32 length; /** String buffer. Size limit is MAXLEN */ char data[MAXLEN]; diff --git a/thirdparty/assimp/include/assimp/vector2.h b/thirdparty/assimp/include/assimp/vector2.h index d5ef001542..c8b1ebbbc2 100644 --- a/thirdparty/assimp/include/assimp/vector2.h +++ b/thirdparty/assimp/include/assimp/vector2.h @@ -47,6 +47,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_VECTOR2D_H_INC #define AI_VECTOR2D_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #ifdef __cplusplus # include <cmath> #else diff --git a/thirdparty/assimp/include/assimp/vector2.inl b/thirdparty/assimp/include/assimp/vector2.inl index 3b7a7beabb..4bbf432ff8 100644 --- a/thirdparty/assimp/include/assimp/vector2.inl +++ b/thirdparty/assimp/include/assimp/vector2.inl @@ -48,8 +48,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_VECTOR2D_INL_INC #define AI_VECTOR2D_INL_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #ifdef __cplusplus -#include "vector2.h" +#include <assimp/vector2.h> #include <cmath> diff --git a/thirdparty/assimp/include/assimp/vector3.h b/thirdparty/assimp/include/assimp/vector3.h index 7ff25cf0ad..fffeb12ad7 100644 --- a/thirdparty/assimp/include/assimp/vector3.h +++ b/thirdparty/assimp/include/assimp/vector3.h @@ -47,13 +47,17 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_VECTOR3D_H_INC #define AI_VECTOR3D_H_INC +#ifdef __GNUC__ +# pragma GCC system_header +#endif + #ifdef __cplusplus # include <cmath> #else # include <math.h> #endif -#include "defs.h" +#include <assimp/defs.h> #ifdef __cplusplus @@ -63,16 +67,13 @@ template<typename TReal> class aiMatrix4x4t; // --------------------------------------------------------------------------- /** Represents a three-dimensional vector. */ template <typename TReal> -class aiVector3t -{ +class aiVector3t { public: aiVector3t() AI_NO_EXCEPT : x(), y(), z() {} aiVector3t(TReal _x, TReal _y, TReal _z) : x(_x), y(_y), z(_z) {} explicit aiVector3t (TReal _xyz ) : x(_xyz), y(_xyz), z(_xyz) {} aiVector3t( const aiVector3t& o ) = default; -public: - // combined operators const aiVector3t& operator += (const aiVector3t& o); const aiVector3t& operator -= (const aiVector3t& o); @@ -97,7 +98,6 @@ public: template <typename TOther> operator aiVector3t<TOther> () const; -public: /** @brief Set the components of a vector * @param pX X component * @param pY Y component diff --git a/thirdparty/assimp/include/assimp/vector3.inl b/thirdparty/assimp/include/assimp/vector3.inl index 2fce6eddee..6682d3b32c 100644 --- a/thirdparty/assimp/include/assimp/vector3.inl +++ b/thirdparty/assimp/include/assimp/vector3.inl @@ -49,7 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define AI_VECTOR3D_INL_INC #ifdef __cplusplus -#include "vector3.h" +#include <assimp/vector3.h> #include <cmath> diff --git a/thirdparty/assimp/include/assimp/version.h b/thirdparty/assimp/include/assimp/version.h index c62a40e118..2fdd37a43c 100644 --- a/thirdparty/assimp/include/assimp/version.h +++ b/thirdparty/assimp/include/assimp/version.h @@ -49,7 +49,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef AI_VERSION_H_INC #define AI_VERSION_H_INC -#include "defs.h" +#include <assimp/defs.h> #ifdef __cplusplus extern "C" { diff --git a/thirdparty/libogg/bitwise.c b/thirdparty/libogg/bitwise.c index fa2b572029..f5ef79122e 100644 --- a/thirdparty/libogg/bitwise.c +++ b/thirdparty/libogg/bitwise.c @@ -11,7 +11,6 @@ ******************************************************************** function: packing variable sized words into an octet stream - last mod: $Id$ ********************************************************************/ @@ -890,7 +889,7 @@ int main(void){ for(i=0;i<test2size;i++){ if(oggpack_look(&r,32)==-1)report("out of data. failed!"); if(oggpack_look(&r,32)!=large[i]){ - fprintf(stderr,"%ld != %ld (%lx!=%lx):",oggpack_look(&r,32),large[i], + fprintf(stderr,"%ld != %lu (%lx!=%lx):",oggpack_look(&r,32),large[i], oggpack_look(&r,32),large[i]); report("read incorrect value!\n"); } @@ -1000,7 +999,7 @@ int main(void){ for(i=0;i<test2size;i++){ if(oggpackB_look(&r,32)==-1)report("out of data. failed!"); if(oggpackB_look(&r,32)!=large[i]){ - fprintf(stderr,"%ld != %ld (%lx!=%lx):",oggpackB_look(&r,32),large[i], + fprintf(stderr,"%ld != %lu (%lx!=%lx):",oggpackB_look(&r,32),large[i], oggpackB_look(&r,32),large[i]); report("read incorrect value!\n"); } diff --git a/thirdparty/libogg/crctable.h b/thirdparty/libogg/crctable.h new file mode 100644 index 0000000000..dcc378b309 --- /dev/null +++ b/thirdparty/libogg/crctable.h @@ -0,0 +1,278 @@ +/******************************************************************** + * * + * THIS FILE IS PART OF THE Ogg CONTAINER SOURCE CODE. * + * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS * + * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE * + * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. * + * * + * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2018 * + * by the Xiph.Org Foundation http://www.xiph.org/ * + * * + ********************************************************************/ + +#include <ogg/os_types.h> + +static const ogg_uint32_t crc_lookup[8][256]={ +{0x00000000,0x04c11db7,0x09823b6e,0x0d4326d9,0x130476dc,0x17c56b6b,0x1a864db2,0x1e475005, + 0x2608edb8,0x22c9f00f,0x2f8ad6d6,0x2b4bcb61,0x350c9b64,0x31cd86d3,0x3c8ea00a,0x384fbdbd, + 0x4c11db70,0x48d0c6c7,0x4593e01e,0x4152fda9,0x5f15adac,0x5bd4b01b,0x569796c2,0x52568b75, + 0x6a1936c8,0x6ed82b7f,0x639b0da6,0x675a1011,0x791d4014,0x7ddc5da3,0x709f7b7a,0x745e66cd, + 0x9823b6e0,0x9ce2ab57,0x91a18d8e,0x95609039,0x8b27c03c,0x8fe6dd8b,0x82a5fb52,0x8664e6e5, + 0xbe2b5b58,0xbaea46ef,0xb7a96036,0xb3687d81,0xad2f2d84,0xa9ee3033,0xa4ad16ea,0xa06c0b5d, + 0xd4326d90,0xd0f37027,0xddb056fe,0xd9714b49,0xc7361b4c,0xc3f706fb,0xceb42022,0xca753d95, + 0xf23a8028,0xf6fb9d9f,0xfbb8bb46,0xff79a6f1,0xe13ef6f4,0xe5ffeb43,0xe8bccd9a,0xec7dd02d, + 0x34867077,0x30476dc0,0x3d044b19,0x39c556ae,0x278206ab,0x23431b1c,0x2e003dc5,0x2ac12072, + 0x128e9dcf,0x164f8078,0x1b0ca6a1,0x1fcdbb16,0x018aeb13,0x054bf6a4,0x0808d07d,0x0cc9cdca, + 0x7897ab07,0x7c56b6b0,0x71159069,0x75d48dde,0x6b93dddb,0x6f52c06c,0x6211e6b5,0x66d0fb02, + 0x5e9f46bf,0x5a5e5b08,0x571d7dd1,0x53dc6066,0x4d9b3063,0x495a2dd4,0x44190b0d,0x40d816ba, + 0xaca5c697,0xa864db20,0xa527fdf9,0xa1e6e04e,0xbfa1b04b,0xbb60adfc,0xb6238b25,0xb2e29692, + 0x8aad2b2f,0x8e6c3698,0x832f1041,0x87ee0df6,0x99a95df3,0x9d684044,0x902b669d,0x94ea7b2a, + 0xe0b41de7,0xe4750050,0xe9362689,0xedf73b3e,0xf3b06b3b,0xf771768c,0xfa325055,0xfef34de2, + 0xc6bcf05f,0xc27dede8,0xcf3ecb31,0xcbffd686,0xd5b88683,0xd1799b34,0xdc3abded,0xd8fba05a, + 0x690ce0ee,0x6dcdfd59,0x608edb80,0x644fc637,0x7a089632,0x7ec98b85,0x738aad5c,0x774bb0eb, + 0x4f040d56,0x4bc510e1,0x46863638,0x42472b8f,0x5c007b8a,0x58c1663d,0x558240e4,0x51435d53, + 0x251d3b9e,0x21dc2629,0x2c9f00f0,0x285e1d47,0x36194d42,0x32d850f5,0x3f9b762c,0x3b5a6b9b, + 0x0315d626,0x07d4cb91,0x0a97ed48,0x0e56f0ff,0x1011a0fa,0x14d0bd4d,0x19939b94,0x1d528623, + 0xf12f560e,0xf5ee4bb9,0xf8ad6d60,0xfc6c70d7,0xe22b20d2,0xe6ea3d65,0xeba91bbc,0xef68060b, + 0xd727bbb6,0xd3e6a601,0xdea580d8,0xda649d6f,0xc423cd6a,0xc0e2d0dd,0xcda1f604,0xc960ebb3, + 0xbd3e8d7e,0xb9ff90c9,0xb4bcb610,0xb07daba7,0xae3afba2,0xaafbe615,0xa7b8c0cc,0xa379dd7b, + 0x9b3660c6,0x9ff77d71,0x92b45ba8,0x9675461f,0x8832161a,0x8cf30bad,0x81b02d74,0x857130c3, + 0x5d8a9099,0x594b8d2e,0x5408abf7,0x50c9b640,0x4e8ee645,0x4a4ffbf2,0x470cdd2b,0x43cdc09c, + 0x7b827d21,0x7f436096,0x7200464f,0x76c15bf8,0x68860bfd,0x6c47164a,0x61043093,0x65c52d24, + 0x119b4be9,0x155a565e,0x18197087,0x1cd86d30,0x029f3d35,0x065e2082,0x0b1d065b,0x0fdc1bec, + 0x3793a651,0x3352bbe6,0x3e119d3f,0x3ad08088,0x2497d08d,0x2056cd3a,0x2d15ebe3,0x29d4f654, + 0xc5a92679,0xc1683bce,0xcc2b1d17,0xc8ea00a0,0xd6ad50a5,0xd26c4d12,0xdf2f6bcb,0xdbee767c, + 0xe3a1cbc1,0xe760d676,0xea23f0af,0xeee2ed18,0xf0a5bd1d,0xf464a0aa,0xf9278673,0xfde69bc4, + 0x89b8fd09,0x8d79e0be,0x803ac667,0x84fbdbd0,0x9abc8bd5,0x9e7d9662,0x933eb0bb,0x97ffad0c, + 0xafb010b1,0xab710d06,0xa6322bdf,0xa2f33668,0xbcb4666d,0xb8757bda,0xb5365d03,0xb1f740b4}, + +{0x00000000,0xd219c1dc,0xa0f29e0f,0x72eb5fd3,0x452421a9,0x973de075,0xe5d6bfa6,0x37cf7e7a, + 0x8a484352,0x5851828e,0x2abadd5d,0xf8a31c81,0xcf6c62fb,0x1d75a327,0x6f9efcf4,0xbd873d28, + 0x10519b13,0xc2485acf,0xb0a3051c,0x62bac4c0,0x5575baba,0x876c7b66,0xf58724b5,0x279ee569, + 0x9a19d841,0x4800199d,0x3aeb464e,0xe8f28792,0xdf3df9e8,0x0d243834,0x7fcf67e7,0xadd6a63b, + 0x20a33626,0xf2baf7fa,0x8051a829,0x524869f5,0x6587178f,0xb79ed653,0xc5758980,0x176c485c, + 0xaaeb7574,0x78f2b4a8,0x0a19eb7b,0xd8002aa7,0xefcf54dd,0x3dd69501,0x4f3dcad2,0x9d240b0e, + 0x30f2ad35,0xe2eb6ce9,0x9000333a,0x4219f2e6,0x75d68c9c,0xa7cf4d40,0xd5241293,0x073dd34f, + 0xbabaee67,0x68a32fbb,0x1a487068,0xc851b1b4,0xff9ecfce,0x2d870e12,0x5f6c51c1,0x8d75901d, + 0x41466c4c,0x935fad90,0xe1b4f243,0x33ad339f,0x04624de5,0xd67b8c39,0xa490d3ea,0x76891236, + 0xcb0e2f1e,0x1917eec2,0x6bfcb111,0xb9e570cd,0x8e2a0eb7,0x5c33cf6b,0x2ed890b8,0xfcc15164, + 0x5117f75f,0x830e3683,0xf1e56950,0x23fca88c,0x1433d6f6,0xc62a172a,0xb4c148f9,0x66d88925, + 0xdb5fb40d,0x094675d1,0x7bad2a02,0xa9b4ebde,0x9e7b95a4,0x4c625478,0x3e890bab,0xec90ca77, + 0x61e55a6a,0xb3fc9bb6,0xc117c465,0x130e05b9,0x24c17bc3,0xf6d8ba1f,0x8433e5cc,0x562a2410, + 0xebad1938,0x39b4d8e4,0x4b5f8737,0x994646eb,0xae893891,0x7c90f94d,0x0e7ba69e,0xdc626742, + 0x71b4c179,0xa3ad00a5,0xd1465f76,0x035f9eaa,0x3490e0d0,0xe689210c,0x94627edf,0x467bbf03, + 0xfbfc822b,0x29e543f7,0x5b0e1c24,0x8917ddf8,0xbed8a382,0x6cc1625e,0x1e2a3d8d,0xcc33fc51, + 0x828cd898,0x50951944,0x227e4697,0xf067874b,0xc7a8f931,0x15b138ed,0x675a673e,0xb543a6e2, + 0x08c49bca,0xdadd5a16,0xa83605c5,0x7a2fc419,0x4de0ba63,0x9ff97bbf,0xed12246c,0x3f0be5b0, + 0x92dd438b,0x40c48257,0x322fdd84,0xe0361c58,0xd7f96222,0x05e0a3fe,0x770bfc2d,0xa5123df1, + 0x189500d9,0xca8cc105,0xb8679ed6,0x6a7e5f0a,0x5db12170,0x8fa8e0ac,0xfd43bf7f,0x2f5a7ea3, + 0xa22feebe,0x70362f62,0x02dd70b1,0xd0c4b16d,0xe70bcf17,0x35120ecb,0x47f95118,0x95e090c4, + 0x2867adec,0xfa7e6c30,0x889533e3,0x5a8cf23f,0x6d438c45,0xbf5a4d99,0xcdb1124a,0x1fa8d396, + 0xb27e75ad,0x6067b471,0x128ceba2,0xc0952a7e,0xf75a5404,0x254395d8,0x57a8ca0b,0x85b10bd7, + 0x383636ff,0xea2ff723,0x98c4a8f0,0x4add692c,0x7d121756,0xaf0bd68a,0xdde08959,0x0ff94885, + 0xc3cab4d4,0x11d37508,0x63382adb,0xb121eb07,0x86ee957d,0x54f754a1,0x261c0b72,0xf405caae, + 0x4982f786,0x9b9b365a,0xe9706989,0x3b69a855,0x0ca6d62f,0xdebf17f3,0xac544820,0x7e4d89fc, + 0xd39b2fc7,0x0182ee1b,0x7369b1c8,0xa1707014,0x96bf0e6e,0x44a6cfb2,0x364d9061,0xe45451bd, + 0x59d36c95,0x8bcaad49,0xf921f29a,0x2b383346,0x1cf74d3c,0xceee8ce0,0xbc05d333,0x6e1c12ef, + 0xe36982f2,0x3170432e,0x439b1cfd,0x9182dd21,0xa64da35b,0x74546287,0x06bf3d54,0xd4a6fc88, + 0x6921c1a0,0xbb38007c,0xc9d35faf,0x1bca9e73,0x2c05e009,0xfe1c21d5,0x8cf77e06,0x5eeebfda, + 0xf33819e1,0x2121d83d,0x53ca87ee,0x81d34632,0xb61c3848,0x6405f994,0x16eea647,0xc4f7679b, + 0x79705ab3,0xab699b6f,0xd982c4bc,0x0b9b0560,0x3c547b1a,0xee4dbac6,0x9ca6e515,0x4ebf24c9}, + +{0x00000000,0x01d8ac87,0x03b1590e,0x0269f589,0x0762b21c,0x06ba1e9b,0x04d3eb12,0x050b4795, + 0x0ec56438,0x0f1dc8bf,0x0d743d36,0x0cac91b1,0x09a7d624,0x087f7aa3,0x0a168f2a,0x0bce23ad, + 0x1d8ac870,0x1c5264f7,0x1e3b917e,0x1fe33df9,0x1ae87a6c,0x1b30d6eb,0x19592362,0x18818fe5, + 0x134fac48,0x129700cf,0x10fef546,0x112659c1,0x142d1e54,0x15f5b2d3,0x179c475a,0x1644ebdd, + 0x3b1590e0,0x3acd3c67,0x38a4c9ee,0x397c6569,0x3c7722fc,0x3daf8e7b,0x3fc67bf2,0x3e1ed775, + 0x35d0f4d8,0x3408585f,0x3661add6,0x37b90151,0x32b246c4,0x336aea43,0x31031fca,0x30dbb34d, + 0x269f5890,0x2747f417,0x252e019e,0x24f6ad19,0x21fdea8c,0x2025460b,0x224cb382,0x23941f05, + 0x285a3ca8,0x2982902f,0x2beb65a6,0x2a33c921,0x2f388eb4,0x2ee02233,0x2c89d7ba,0x2d517b3d, + 0x762b21c0,0x77f38d47,0x759a78ce,0x7442d449,0x714993dc,0x70913f5b,0x72f8cad2,0x73206655, + 0x78ee45f8,0x7936e97f,0x7b5f1cf6,0x7a87b071,0x7f8cf7e4,0x7e545b63,0x7c3daeea,0x7de5026d, + 0x6ba1e9b0,0x6a794537,0x6810b0be,0x69c81c39,0x6cc35bac,0x6d1bf72b,0x6f7202a2,0x6eaaae25, + 0x65648d88,0x64bc210f,0x66d5d486,0x670d7801,0x62063f94,0x63de9313,0x61b7669a,0x606fca1d, + 0x4d3eb120,0x4ce61da7,0x4e8fe82e,0x4f5744a9,0x4a5c033c,0x4b84afbb,0x49ed5a32,0x4835f6b5, + 0x43fbd518,0x4223799f,0x404a8c16,0x41922091,0x44996704,0x4541cb83,0x47283e0a,0x46f0928d, + 0x50b47950,0x516cd5d7,0x5305205e,0x52dd8cd9,0x57d6cb4c,0x560e67cb,0x54679242,0x55bf3ec5, + 0x5e711d68,0x5fa9b1ef,0x5dc04466,0x5c18e8e1,0x5913af74,0x58cb03f3,0x5aa2f67a,0x5b7a5afd, + 0xec564380,0xed8eef07,0xefe71a8e,0xee3fb609,0xeb34f19c,0xeaec5d1b,0xe885a892,0xe95d0415, + 0xe29327b8,0xe34b8b3f,0xe1227eb6,0xe0fad231,0xe5f195a4,0xe4293923,0xe640ccaa,0xe798602d, + 0xf1dc8bf0,0xf0042777,0xf26dd2fe,0xf3b57e79,0xf6be39ec,0xf766956b,0xf50f60e2,0xf4d7cc65, + 0xff19efc8,0xfec1434f,0xfca8b6c6,0xfd701a41,0xf87b5dd4,0xf9a3f153,0xfbca04da,0xfa12a85d, + 0xd743d360,0xd69b7fe7,0xd4f28a6e,0xd52a26e9,0xd021617c,0xd1f9cdfb,0xd3903872,0xd24894f5, + 0xd986b758,0xd85e1bdf,0xda37ee56,0xdbef42d1,0xdee40544,0xdf3ca9c3,0xdd555c4a,0xdc8df0cd, + 0xcac91b10,0xcb11b797,0xc978421e,0xc8a0ee99,0xcdaba90c,0xcc73058b,0xce1af002,0xcfc25c85, + 0xc40c7f28,0xc5d4d3af,0xc7bd2626,0xc6658aa1,0xc36ecd34,0xc2b661b3,0xc0df943a,0xc10738bd, + 0x9a7d6240,0x9ba5cec7,0x99cc3b4e,0x981497c9,0x9d1fd05c,0x9cc77cdb,0x9eae8952,0x9f7625d5, + 0x94b80678,0x9560aaff,0x97095f76,0x96d1f3f1,0x93dab464,0x920218e3,0x906bed6a,0x91b341ed, + 0x87f7aa30,0x862f06b7,0x8446f33e,0x859e5fb9,0x8095182c,0x814db4ab,0x83244122,0x82fceda5, + 0x8932ce08,0x88ea628f,0x8a839706,0x8b5b3b81,0x8e507c14,0x8f88d093,0x8de1251a,0x8c39899d, + 0xa168f2a0,0xa0b05e27,0xa2d9abae,0xa3010729,0xa60a40bc,0xa7d2ec3b,0xa5bb19b2,0xa463b535, + 0xafad9698,0xae753a1f,0xac1ccf96,0xadc46311,0xa8cf2484,0xa9178803,0xab7e7d8a,0xaaa6d10d, + 0xbce23ad0,0xbd3a9657,0xbf5363de,0xbe8bcf59,0xbb8088cc,0xba58244b,0xb831d1c2,0xb9e97d45, + 0xb2275ee8,0xb3fff26f,0xb19607e6,0xb04eab61,0xb545ecf4,0xb49d4073,0xb6f4b5fa,0xb72c197d}, + +{0x00000000,0xdc6d9ab7,0xbc1a28d9,0x6077b26e,0x7cf54c05,0xa098d6b2,0xc0ef64dc,0x1c82fe6b, + 0xf9ea980a,0x258702bd,0x45f0b0d3,0x999d2a64,0x851fd40f,0x59724eb8,0x3905fcd6,0xe5686661, + 0xf7142da3,0x2b79b714,0x4b0e057a,0x97639fcd,0x8be161a6,0x578cfb11,0x37fb497f,0xeb96d3c8, + 0x0efeb5a9,0xd2932f1e,0xb2e49d70,0x6e8907c7,0x720bf9ac,0xae66631b,0xce11d175,0x127c4bc2, + 0xeae946f1,0x3684dc46,0x56f36e28,0x8a9ef49f,0x961c0af4,0x4a719043,0x2a06222d,0xf66bb89a, + 0x1303defb,0xcf6e444c,0xaf19f622,0x73746c95,0x6ff692fe,0xb39b0849,0xd3ecba27,0x0f812090, + 0x1dfd6b52,0xc190f1e5,0xa1e7438b,0x7d8ad93c,0x61082757,0xbd65bde0,0xdd120f8e,0x017f9539, + 0xe417f358,0x387a69ef,0x580ddb81,0x84604136,0x98e2bf5d,0x448f25ea,0x24f89784,0xf8950d33, + 0xd1139055,0x0d7e0ae2,0x6d09b88c,0xb164223b,0xade6dc50,0x718b46e7,0x11fcf489,0xcd916e3e, + 0x28f9085f,0xf49492e8,0x94e32086,0x488eba31,0x540c445a,0x8861deed,0xe8166c83,0x347bf634, + 0x2607bdf6,0xfa6a2741,0x9a1d952f,0x46700f98,0x5af2f1f3,0x869f6b44,0xe6e8d92a,0x3a85439d, + 0xdfed25fc,0x0380bf4b,0x63f70d25,0xbf9a9792,0xa31869f9,0x7f75f34e,0x1f024120,0xc36fdb97, + 0x3bfad6a4,0xe7974c13,0x87e0fe7d,0x5b8d64ca,0x470f9aa1,0x9b620016,0xfb15b278,0x277828cf, + 0xc2104eae,0x1e7dd419,0x7e0a6677,0xa267fcc0,0xbee502ab,0x6288981c,0x02ff2a72,0xde92b0c5, + 0xcceefb07,0x108361b0,0x70f4d3de,0xac994969,0xb01bb702,0x6c762db5,0x0c019fdb,0xd06c056c, + 0x3504630d,0xe969f9ba,0x891e4bd4,0x5573d163,0x49f12f08,0x959cb5bf,0xf5eb07d1,0x29869d66, + 0xa6e63d1d,0x7a8ba7aa,0x1afc15c4,0xc6918f73,0xda137118,0x067eebaf,0x660959c1,0xba64c376, + 0x5f0ca517,0x83613fa0,0xe3168dce,0x3f7b1779,0x23f9e912,0xff9473a5,0x9fe3c1cb,0x438e5b7c, + 0x51f210be,0x8d9f8a09,0xede83867,0x3185a2d0,0x2d075cbb,0xf16ac60c,0x911d7462,0x4d70eed5, + 0xa81888b4,0x74751203,0x1402a06d,0xc86f3ada,0xd4edc4b1,0x08805e06,0x68f7ec68,0xb49a76df, + 0x4c0f7bec,0x9062e15b,0xf0155335,0x2c78c982,0x30fa37e9,0xec97ad5e,0x8ce01f30,0x508d8587, + 0xb5e5e3e6,0x69887951,0x09ffcb3f,0xd5925188,0xc910afe3,0x157d3554,0x750a873a,0xa9671d8d, + 0xbb1b564f,0x6776ccf8,0x07017e96,0xdb6ce421,0xc7ee1a4a,0x1b8380fd,0x7bf43293,0xa799a824, + 0x42f1ce45,0x9e9c54f2,0xfeebe69c,0x22867c2b,0x3e048240,0xe26918f7,0x821eaa99,0x5e73302e, + 0x77f5ad48,0xab9837ff,0xcbef8591,0x17821f26,0x0b00e14d,0xd76d7bfa,0xb71ac994,0x6b775323, + 0x8e1f3542,0x5272aff5,0x32051d9b,0xee68872c,0xf2ea7947,0x2e87e3f0,0x4ef0519e,0x929dcb29, + 0x80e180eb,0x5c8c1a5c,0x3cfba832,0xe0963285,0xfc14ccee,0x20795659,0x400ee437,0x9c637e80, + 0x790b18e1,0xa5668256,0xc5113038,0x197caa8f,0x05fe54e4,0xd993ce53,0xb9e47c3d,0x6589e68a, + 0x9d1cebb9,0x4171710e,0x2106c360,0xfd6b59d7,0xe1e9a7bc,0x3d843d0b,0x5df38f65,0x819e15d2, + 0x64f673b3,0xb89be904,0xd8ec5b6a,0x0481c1dd,0x18033fb6,0xc46ea501,0xa419176f,0x78748dd8, + 0x6a08c61a,0xb6655cad,0xd612eec3,0x0a7f7474,0x16fd8a1f,0xca9010a8,0xaae7a2c6,0x768a3871, + 0x93e25e10,0x4f8fc4a7,0x2ff876c9,0xf395ec7e,0xef171215,0x337a88a2,0x530d3acc,0x8f60a07b}, + +{0x00000000,0x490d678d,0x921acf1a,0xdb17a897,0x20f48383,0x69f9e40e,0xb2ee4c99,0xfbe32b14, + 0x41e90706,0x08e4608b,0xd3f3c81c,0x9afeaf91,0x611d8485,0x2810e308,0xf3074b9f,0xba0a2c12, + 0x83d20e0c,0xcadf6981,0x11c8c116,0x58c5a69b,0xa3268d8f,0xea2bea02,0x313c4295,0x78312518, + 0xc23b090a,0x8b366e87,0x5021c610,0x192ca19d,0xe2cf8a89,0xabc2ed04,0x70d54593,0x39d8221e, + 0x036501af,0x4a686622,0x917fceb5,0xd872a938,0x2391822c,0x6a9ce5a1,0xb18b4d36,0xf8862abb, + 0x428c06a9,0x0b816124,0xd096c9b3,0x999bae3e,0x6278852a,0x2b75e2a7,0xf0624a30,0xb96f2dbd, + 0x80b70fa3,0xc9ba682e,0x12adc0b9,0x5ba0a734,0xa0438c20,0xe94eebad,0x3259433a,0x7b5424b7, + 0xc15e08a5,0x88536f28,0x5344c7bf,0x1a49a032,0xe1aa8b26,0xa8a7ecab,0x73b0443c,0x3abd23b1, + 0x06ca035e,0x4fc764d3,0x94d0cc44,0xddddabc9,0x263e80dd,0x6f33e750,0xb4244fc7,0xfd29284a, + 0x47230458,0x0e2e63d5,0xd539cb42,0x9c34accf,0x67d787db,0x2edae056,0xf5cd48c1,0xbcc02f4c, + 0x85180d52,0xcc156adf,0x1702c248,0x5e0fa5c5,0xa5ec8ed1,0xece1e95c,0x37f641cb,0x7efb2646, + 0xc4f10a54,0x8dfc6dd9,0x56ebc54e,0x1fe6a2c3,0xe40589d7,0xad08ee5a,0x761f46cd,0x3f122140, + 0x05af02f1,0x4ca2657c,0x97b5cdeb,0xdeb8aa66,0x255b8172,0x6c56e6ff,0xb7414e68,0xfe4c29e5, + 0x444605f7,0x0d4b627a,0xd65ccaed,0x9f51ad60,0x64b28674,0x2dbfe1f9,0xf6a8496e,0xbfa52ee3, + 0x867d0cfd,0xcf706b70,0x1467c3e7,0x5d6aa46a,0xa6898f7e,0xef84e8f3,0x34934064,0x7d9e27e9, + 0xc7940bfb,0x8e996c76,0x558ec4e1,0x1c83a36c,0xe7608878,0xae6deff5,0x757a4762,0x3c7720ef, + 0x0d9406bc,0x44996131,0x9f8ec9a6,0xd683ae2b,0x2d60853f,0x646de2b2,0xbf7a4a25,0xf6772da8, + 0x4c7d01ba,0x05706637,0xde67cea0,0x976aa92d,0x6c898239,0x2584e5b4,0xfe934d23,0xb79e2aae, + 0x8e4608b0,0xc74b6f3d,0x1c5cc7aa,0x5551a027,0xaeb28b33,0xe7bfecbe,0x3ca84429,0x75a523a4, + 0xcfaf0fb6,0x86a2683b,0x5db5c0ac,0x14b8a721,0xef5b8c35,0xa656ebb8,0x7d41432f,0x344c24a2, + 0x0ef10713,0x47fc609e,0x9cebc809,0xd5e6af84,0x2e058490,0x6708e31d,0xbc1f4b8a,0xf5122c07, + 0x4f180015,0x06156798,0xdd02cf0f,0x940fa882,0x6fec8396,0x26e1e41b,0xfdf64c8c,0xb4fb2b01, + 0x8d23091f,0xc42e6e92,0x1f39c605,0x5634a188,0xadd78a9c,0xe4daed11,0x3fcd4586,0x76c0220b, + 0xccca0e19,0x85c76994,0x5ed0c103,0x17dda68e,0xec3e8d9a,0xa533ea17,0x7e244280,0x3729250d, + 0x0b5e05e2,0x4253626f,0x9944caf8,0xd049ad75,0x2baa8661,0x62a7e1ec,0xb9b0497b,0xf0bd2ef6, + 0x4ab702e4,0x03ba6569,0xd8adcdfe,0x91a0aa73,0x6a438167,0x234ee6ea,0xf8594e7d,0xb15429f0, + 0x888c0bee,0xc1816c63,0x1a96c4f4,0x539ba379,0xa878886d,0xe175efe0,0x3a624777,0x736f20fa, + 0xc9650ce8,0x80686b65,0x5b7fc3f2,0x1272a47f,0xe9918f6b,0xa09ce8e6,0x7b8b4071,0x328627fc, + 0x083b044d,0x413663c0,0x9a21cb57,0xd32cacda,0x28cf87ce,0x61c2e043,0xbad548d4,0xf3d82f59, + 0x49d2034b,0x00df64c6,0xdbc8cc51,0x92c5abdc,0x692680c8,0x202be745,0xfb3c4fd2,0xb231285f, + 0x8be90a41,0xc2e46dcc,0x19f3c55b,0x50fea2d6,0xab1d89c2,0xe210ee4f,0x390746d8,0x700a2155, + 0xca000d47,0x830d6aca,0x581ac25d,0x1117a5d0,0xeaf48ec4,0xa3f9e949,0x78ee41de,0x31e32653}, + +{0x00000000,0x1b280d78,0x36501af0,0x2d781788,0x6ca035e0,0x77883898,0x5af02f10,0x41d82268, + 0xd9406bc0,0xc26866b8,0xef107130,0xf4387c48,0xb5e05e20,0xaec85358,0x83b044d0,0x989849a8, + 0xb641ca37,0xad69c74f,0x8011d0c7,0x9b39ddbf,0xdae1ffd7,0xc1c9f2af,0xecb1e527,0xf799e85f, + 0x6f01a1f7,0x7429ac8f,0x5951bb07,0x4279b67f,0x03a19417,0x1889996f,0x35f18ee7,0x2ed9839f, + 0x684289d9,0x736a84a1,0x5e129329,0x453a9e51,0x04e2bc39,0x1fcab141,0x32b2a6c9,0x299aabb1, + 0xb102e219,0xaa2aef61,0x8752f8e9,0x9c7af591,0xdda2d7f9,0xc68ada81,0xebf2cd09,0xf0dac071, + 0xde0343ee,0xc52b4e96,0xe853591e,0xf37b5466,0xb2a3760e,0xa98b7b76,0x84f36cfe,0x9fdb6186, + 0x0743282e,0x1c6b2556,0x311332de,0x2a3b3fa6,0x6be31dce,0x70cb10b6,0x5db3073e,0x469b0a46, + 0xd08513b2,0xcbad1eca,0xe6d50942,0xfdfd043a,0xbc252652,0xa70d2b2a,0x8a753ca2,0x915d31da, + 0x09c57872,0x12ed750a,0x3f956282,0x24bd6ffa,0x65654d92,0x7e4d40ea,0x53355762,0x481d5a1a, + 0x66c4d985,0x7decd4fd,0x5094c375,0x4bbcce0d,0x0a64ec65,0x114ce11d,0x3c34f695,0x271cfbed, + 0xbf84b245,0xa4acbf3d,0x89d4a8b5,0x92fca5cd,0xd32487a5,0xc80c8add,0xe5749d55,0xfe5c902d, + 0xb8c79a6b,0xa3ef9713,0x8e97809b,0x95bf8de3,0xd467af8b,0xcf4fa2f3,0xe237b57b,0xf91fb803, + 0x6187f1ab,0x7aaffcd3,0x57d7eb5b,0x4cffe623,0x0d27c44b,0x160fc933,0x3b77debb,0x205fd3c3, + 0x0e86505c,0x15ae5d24,0x38d64aac,0x23fe47d4,0x622665bc,0x790e68c4,0x54767f4c,0x4f5e7234, + 0xd7c63b9c,0xccee36e4,0xe196216c,0xfabe2c14,0xbb660e7c,0xa04e0304,0x8d36148c,0x961e19f4, + 0xa5cb3ad3,0xbee337ab,0x939b2023,0x88b32d5b,0xc96b0f33,0xd243024b,0xff3b15c3,0xe41318bb, + 0x7c8b5113,0x67a35c6b,0x4adb4be3,0x51f3469b,0x102b64f3,0x0b03698b,0x267b7e03,0x3d53737b, + 0x138af0e4,0x08a2fd9c,0x25daea14,0x3ef2e76c,0x7f2ac504,0x6402c87c,0x497adff4,0x5252d28c, + 0xcaca9b24,0xd1e2965c,0xfc9a81d4,0xe7b28cac,0xa66aaec4,0xbd42a3bc,0x903ab434,0x8b12b94c, + 0xcd89b30a,0xd6a1be72,0xfbd9a9fa,0xe0f1a482,0xa12986ea,0xba018b92,0x97799c1a,0x8c519162, + 0x14c9d8ca,0x0fe1d5b2,0x2299c23a,0x39b1cf42,0x7869ed2a,0x6341e052,0x4e39f7da,0x5511faa2, + 0x7bc8793d,0x60e07445,0x4d9863cd,0x56b06eb5,0x17684cdd,0x0c4041a5,0x2138562d,0x3a105b55, + 0xa28812fd,0xb9a01f85,0x94d8080d,0x8ff00575,0xce28271d,0xd5002a65,0xf8783ded,0xe3503095, + 0x754e2961,0x6e662419,0x431e3391,0x58363ee9,0x19ee1c81,0x02c611f9,0x2fbe0671,0x34960b09, + 0xac0e42a1,0xb7264fd9,0x9a5e5851,0x81765529,0xc0ae7741,0xdb867a39,0xf6fe6db1,0xedd660c9, + 0xc30fe356,0xd827ee2e,0xf55ff9a6,0xee77f4de,0xafafd6b6,0xb487dbce,0x99ffcc46,0x82d7c13e, + 0x1a4f8896,0x016785ee,0x2c1f9266,0x37379f1e,0x76efbd76,0x6dc7b00e,0x40bfa786,0x5b97aafe, + 0x1d0ca0b8,0x0624adc0,0x2b5cba48,0x3074b730,0x71ac9558,0x6a849820,0x47fc8fa8,0x5cd482d0, + 0xc44ccb78,0xdf64c600,0xf21cd188,0xe934dcf0,0xa8ecfe98,0xb3c4f3e0,0x9ebce468,0x8594e910, + 0xab4d6a8f,0xb06567f7,0x9d1d707f,0x86357d07,0xc7ed5f6f,0xdcc55217,0xf1bd459f,0xea9548e7, + 0x720d014f,0x69250c37,0x445d1bbf,0x5f7516c7,0x1ead34af,0x058539d7,0x28fd2e5f,0x33d52327}, + +{0x00000000,0x4f576811,0x9eaed022,0xd1f9b833,0x399cbdf3,0x76cbd5e2,0xa7326dd1,0xe86505c0, + 0x73397be6,0x3c6e13f7,0xed97abc4,0xa2c0c3d5,0x4aa5c615,0x05f2ae04,0xd40b1637,0x9b5c7e26, + 0xe672f7cc,0xa9259fdd,0x78dc27ee,0x378b4fff,0xdfee4a3f,0x90b9222e,0x41409a1d,0x0e17f20c, + 0x954b8c2a,0xda1ce43b,0x0be55c08,0x44b23419,0xacd731d9,0xe38059c8,0x3279e1fb,0x7d2e89ea, + 0xc824f22f,0x87739a3e,0x568a220d,0x19dd4a1c,0xf1b84fdc,0xbeef27cd,0x6f169ffe,0x2041f7ef, + 0xbb1d89c9,0xf44ae1d8,0x25b359eb,0x6ae431fa,0x8281343a,0xcdd65c2b,0x1c2fe418,0x53788c09, + 0x2e5605e3,0x61016df2,0xb0f8d5c1,0xffafbdd0,0x17cab810,0x589dd001,0x89646832,0xc6330023, + 0x5d6f7e05,0x12381614,0xc3c1ae27,0x8c96c636,0x64f3c3f6,0x2ba4abe7,0xfa5d13d4,0xb50a7bc5, + 0x9488f9e9,0xdbdf91f8,0x0a2629cb,0x457141da,0xad14441a,0xe2432c0b,0x33ba9438,0x7cedfc29, + 0xe7b1820f,0xa8e6ea1e,0x791f522d,0x36483a3c,0xde2d3ffc,0x917a57ed,0x4083efde,0x0fd487cf, + 0x72fa0e25,0x3dad6634,0xec54de07,0xa303b616,0x4b66b3d6,0x0431dbc7,0xd5c863f4,0x9a9f0be5, + 0x01c375c3,0x4e941dd2,0x9f6da5e1,0xd03acdf0,0x385fc830,0x7708a021,0xa6f11812,0xe9a67003, + 0x5cac0bc6,0x13fb63d7,0xc202dbe4,0x8d55b3f5,0x6530b635,0x2a67de24,0xfb9e6617,0xb4c90e06, + 0x2f957020,0x60c21831,0xb13ba002,0xfe6cc813,0x1609cdd3,0x595ea5c2,0x88a71df1,0xc7f075e0, + 0xbadefc0a,0xf589941b,0x24702c28,0x6b274439,0x834241f9,0xcc1529e8,0x1dec91db,0x52bbf9ca, + 0xc9e787ec,0x86b0effd,0x574957ce,0x181e3fdf,0xf07b3a1f,0xbf2c520e,0x6ed5ea3d,0x2182822c, + 0x2dd0ee65,0x62878674,0xb37e3e47,0xfc295656,0x144c5396,0x5b1b3b87,0x8ae283b4,0xc5b5eba5, + 0x5ee99583,0x11befd92,0xc04745a1,0x8f102db0,0x67752870,0x28224061,0xf9dbf852,0xb68c9043, + 0xcba219a9,0x84f571b8,0x550cc98b,0x1a5ba19a,0xf23ea45a,0xbd69cc4b,0x6c907478,0x23c71c69, + 0xb89b624f,0xf7cc0a5e,0x2635b26d,0x6962da7c,0x8107dfbc,0xce50b7ad,0x1fa90f9e,0x50fe678f, + 0xe5f41c4a,0xaaa3745b,0x7b5acc68,0x340da479,0xdc68a1b9,0x933fc9a8,0x42c6719b,0x0d91198a, + 0x96cd67ac,0xd99a0fbd,0x0863b78e,0x4734df9f,0xaf51da5f,0xe006b24e,0x31ff0a7d,0x7ea8626c, + 0x0386eb86,0x4cd18397,0x9d283ba4,0xd27f53b5,0x3a1a5675,0x754d3e64,0xa4b48657,0xebe3ee46, + 0x70bf9060,0x3fe8f871,0xee114042,0xa1462853,0x49232d93,0x06744582,0xd78dfdb1,0x98da95a0, + 0xb958178c,0xf60f7f9d,0x27f6c7ae,0x68a1afbf,0x80c4aa7f,0xcf93c26e,0x1e6a7a5d,0x513d124c, + 0xca616c6a,0x8536047b,0x54cfbc48,0x1b98d459,0xf3fdd199,0xbcaab988,0x6d5301bb,0x220469aa, + 0x5f2ae040,0x107d8851,0xc1843062,0x8ed35873,0x66b65db3,0x29e135a2,0xf8188d91,0xb74fe580, + 0x2c139ba6,0x6344f3b7,0xb2bd4b84,0xfdea2395,0x158f2655,0x5ad84e44,0x8b21f677,0xc4769e66, + 0x717ce5a3,0x3e2b8db2,0xefd23581,0xa0855d90,0x48e05850,0x07b73041,0xd64e8872,0x9919e063, + 0x02459e45,0x4d12f654,0x9ceb4e67,0xd3bc2676,0x3bd923b6,0x748e4ba7,0xa577f394,0xea209b85, + 0x970e126f,0xd8597a7e,0x09a0c24d,0x46f7aa5c,0xae92af9c,0xe1c5c78d,0x303c7fbe,0x7f6b17af, + 0xe4376989,0xab600198,0x7a99b9ab,0x35ced1ba,0xddabd47a,0x92fcbc6b,0x43050458,0x0c526c49}, + +{0x00000000,0x5ba1dcca,0xb743b994,0xece2655e,0x6a466e9f,0x31e7b255,0xdd05d70b,0x86a40bc1, + 0xd48cdd3e,0x8f2d01f4,0x63cf64aa,0x386eb860,0xbecab3a1,0xe56b6f6b,0x09890a35,0x5228d6ff, + 0xadd8a7cb,0xf6797b01,0x1a9b1e5f,0x413ac295,0xc79ec954,0x9c3f159e,0x70dd70c0,0x2b7cac0a, + 0x79547af5,0x22f5a63f,0xce17c361,0x95b61fab,0x1312146a,0x48b3c8a0,0xa451adfe,0xfff07134, + 0x5f705221,0x04d18eeb,0xe833ebb5,0xb392377f,0x35363cbe,0x6e97e074,0x8275852a,0xd9d459e0, + 0x8bfc8f1f,0xd05d53d5,0x3cbf368b,0x671eea41,0xe1bae180,0xba1b3d4a,0x56f95814,0x0d5884de, + 0xf2a8f5ea,0xa9092920,0x45eb4c7e,0x1e4a90b4,0x98ee9b75,0xc34f47bf,0x2fad22e1,0x740cfe2b, + 0x262428d4,0x7d85f41e,0x91679140,0xcac64d8a,0x4c62464b,0x17c39a81,0xfb21ffdf,0xa0802315, + 0xbee0a442,0xe5417888,0x09a31dd6,0x5202c11c,0xd4a6cadd,0x8f071617,0x63e57349,0x3844af83, + 0x6a6c797c,0x31cda5b6,0xdd2fc0e8,0x868e1c22,0x002a17e3,0x5b8bcb29,0xb769ae77,0xecc872bd, + 0x13380389,0x4899df43,0xa47bba1d,0xffda66d7,0x797e6d16,0x22dfb1dc,0xce3dd482,0x959c0848, + 0xc7b4deb7,0x9c15027d,0x70f76723,0x2b56bbe9,0xadf2b028,0xf6536ce2,0x1ab109bc,0x4110d576, + 0xe190f663,0xba312aa9,0x56d34ff7,0x0d72933d,0x8bd698fc,0xd0774436,0x3c952168,0x6734fda2, + 0x351c2b5d,0x6ebdf797,0x825f92c9,0xd9fe4e03,0x5f5a45c2,0x04fb9908,0xe819fc56,0xb3b8209c, + 0x4c4851a8,0x17e98d62,0xfb0be83c,0xa0aa34f6,0x260e3f37,0x7dafe3fd,0x914d86a3,0xcaec5a69, + 0x98c48c96,0xc365505c,0x2f873502,0x7426e9c8,0xf282e209,0xa9233ec3,0x45c15b9d,0x1e608757, + 0x79005533,0x22a189f9,0xce43eca7,0x95e2306d,0x13463bac,0x48e7e766,0xa4058238,0xffa45ef2, + 0xad8c880d,0xf62d54c7,0x1acf3199,0x416eed53,0xc7cae692,0x9c6b3a58,0x70895f06,0x2b2883cc, + 0xd4d8f2f8,0x8f792e32,0x639b4b6c,0x383a97a6,0xbe9e9c67,0xe53f40ad,0x09dd25f3,0x527cf939, + 0x00542fc6,0x5bf5f30c,0xb7179652,0xecb64a98,0x6a124159,0x31b39d93,0xdd51f8cd,0x86f02407, + 0x26700712,0x7dd1dbd8,0x9133be86,0xca92624c,0x4c36698d,0x1797b547,0xfb75d019,0xa0d40cd3, + 0xf2fcda2c,0xa95d06e6,0x45bf63b8,0x1e1ebf72,0x98bab4b3,0xc31b6879,0x2ff90d27,0x7458d1ed, + 0x8ba8a0d9,0xd0097c13,0x3ceb194d,0x674ac587,0xe1eece46,0xba4f128c,0x56ad77d2,0x0d0cab18, + 0x5f247de7,0x0485a12d,0xe867c473,0xb3c618b9,0x35621378,0x6ec3cfb2,0x8221aaec,0xd9807626, + 0xc7e0f171,0x9c412dbb,0x70a348e5,0x2b02942f,0xada69fee,0xf6074324,0x1ae5267a,0x4144fab0, + 0x136c2c4f,0x48cdf085,0xa42f95db,0xff8e4911,0x792a42d0,0x228b9e1a,0xce69fb44,0x95c8278e, + 0x6a3856ba,0x31998a70,0xdd7bef2e,0x86da33e4,0x007e3825,0x5bdfe4ef,0xb73d81b1,0xec9c5d7b, + 0xbeb48b84,0xe515574e,0x09f73210,0x5256eeda,0xd4f2e51b,0x8f5339d1,0x63b15c8f,0x38108045, + 0x9890a350,0xc3317f9a,0x2fd31ac4,0x7472c60e,0xf2d6cdcf,0xa9771105,0x4595745b,0x1e34a891, + 0x4c1c7e6e,0x17bda2a4,0xfb5fc7fa,0xa0fe1b30,0x265a10f1,0x7dfbcc3b,0x9119a965,0xcab875af, + 0x3548049b,0x6ee9d851,0x820bbd0f,0xd9aa61c5,0x5f0e6a04,0x04afb6ce,0xe84dd390,0xb3ec0f5a, + 0xe1c4d9a5,0xba65056f,0x56876031,0x0d26bcfb,0x8b82b73a,0xd0236bf0,0x3cc10eae,0x6760d264}}; diff --git a/thirdparty/libogg/framing.c b/thirdparty/libogg/framing.c index 79fc715c8c..83601199ad 100644 --- a/thirdparty/libogg/framing.c +++ b/thirdparty/libogg/framing.c @@ -5,14 +5,13 @@ * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE * * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. * * * - * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2010 * + * THE OggVorbis SOURCE CODE IS (C) COPYRIGHT 1994-2018 * * by the Xiph.Org Foundation http://www.xiph.org/ * * * ******************************************************************** function: code raw packets into framed OggSquish stream and decode Ogg streams back into raw packets - last mod: $Id$ note: The CRC code is directly derived from public domain code by Ross Williams (ross@guest.adelaide.edu.au). See docs/framing.html @@ -20,6 +19,10 @@ ********************************************************************/ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + #include <stdlib.h> #include <limits.h> #include <string.h> @@ -45,7 +48,7 @@ int ogg_page_eos(const ogg_page *og){ ogg_int64_t ogg_page_granulepos(const ogg_page *og){ unsigned char *page=og->header; - ogg_int64_t granulepos=page[13]&(0xff); + ogg_uint64_t granulepos=page[13]&(0xff); granulepos= (granulepos<<8)|(page[12]&0xff); granulepos= (granulepos<<8)|(page[11]&0xff); granulepos= (granulepos<<8)|(page[10]&0xff); @@ -53,21 +56,21 @@ ogg_int64_t ogg_page_granulepos(const ogg_page *og){ granulepos= (granulepos<<8)|(page[8]&0xff); granulepos= (granulepos<<8)|(page[7]&0xff); granulepos= (granulepos<<8)|(page[6]&0xff); - return(granulepos); + return((ogg_int64_t)granulepos); } int ogg_page_serialno(const ogg_page *og){ - return(og->header[14] | - (og->header[15]<<8) | - (og->header[16]<<16) | - (og->header[17]<<24)); + return((int)((ogg_uint32_t)og->header[14]) | + ((ogg_uint32_t)og->header[15]<<8) | + ((ogg_uint32_t)og->header[16]<<16) | + ((ogg_uint32_t)og->header[17]<<24)); } long ogg_page_pageno(const ogg_page *og){ - return(og->header[18] | - (og->header[19]<<8) | - (og->header[20]<<16) | - (og->header[21]<<24)); + return((long)((ogg_uint32_t)og->header[18]) | + ((ogg_uint32_t)og->header[19]<<8) | + ((ogg_uint32_t)og->header[20]<<16) | + ((ogg_uint32_t)og->header[21]<<24)); } @@ -99,90 +102,31 @@ int ogg_page_packets(const ogg_page *og){ #if 0 /* helper to initialize lookup for direct-table CRC (illustrative; we - use the static init below) */ - -static ogg_uint32_t _ogg_crc_entry(unsigned long index){ - int i; - unsigned long r; - - r = index << 24; - for (i=0; i<8; i++) - if (r & 0x80000000UL) - r = (r << 1) ^ 0x04c11db7; /* The same as the ethernet generator - polynomial, although we use an - unreflected alg and an init/final - of 0, not 0xffffffff */ - else - r<<=1; - return (r & 0xffffffffUL); + use the static init in crctable.h) */ + +static void _ogg_crc_init(){ + int i, j; + ogg_uint32_t polynomial, crc; + polynomial = 0x04c11db7; /* The same as the ethernet generator + polynomial, although we use an + unreflected alg and an init/final + of 0, not 0xffffffff */ + for (i = 0; i <= 0xFF; i++){ + crc = i << 24; + + for (j = 0; j < 8; j++) + crc = (crc << 1) ^ (crc & (1 << 31) ? polynomial : 0); + + crc_lookup[0][i] = crc; + } + + for (i = 0; i <= 0xFF; i++) + for (j = 1; j < 8; j++) + crc_lookup[j][i] = crc_lookup[0][(crc_lookup[j - 1][i] >> 24) & 0xFF] ^ (crc_lookup[j - 1][i] << 8); } #endif -static const ogg_uint32_t crc_lookup[256]={ - 0x00000000,0x04c11db7,0x09823b6e,0x0d4326d9, - 0x130476dc,0x17c56b6b,0x1a864db2,0x1e475005, - 0x2608edb8,0x22c9f00f,0x2f8ad6d6,0x2b4bcb61, - 0x350c9b64,0x31cd86d3,0x3c8ea00a,0x384fbdbd, - 0x4c11db70,0x48d0c6c7,0x4593e01e,0x4152fda9, - 0x5f15adac,0x5bd4b01b,0x569796c2,0x52568b75, - 0x6a1936c8,0x6ed82b7f,0x639b0da6,0x675a1011, - 0x791d4014,0x7ddc5da3,0x709f7b7a,0x745e66cd, - 0x9823b6e0,0x9ce2ab57,0x91a18d8e,0x95609039, - 0x8b27c03c,0x8fe6dd8b,0x82a5fb52,0x8664e6e5, - 0xbe2b5b58,0xbaea46ef,0xb7a96036,0xb3687d81, - 0xad2f2d84,0xa9ee3033,0xa4ad16ea,0xa06c0b5d, - 0xd4326d90,0xd0f37027,0xddb056fe,0xd9714b49, - 0xc7361b4c,0xc3f706fb,0xceb42022,0xca753d95, - 0xf23a8028,0xf6fb9d9f,0xfbb8bb46,0xff79a6f1, - 0xe13ef6f4,0xe5ffeb43,0xe8bccd9a,0xec7dd02d, - 0x34867077,0x30476dc0,0x3d044b19,0x39c556ae, - 0x278206ab,0x23431b1c,0x2e003dc5,0x2ac12072, - 0x128e9dcf,0x164f8078,0x1b0ca6a1,0x1fcdbb16, - 0x018aeb13,0x054bf6a4,0x0808d07d,0x0cc9cdca, - 0x7897ab07,0x7c56b6b0,0x71159069,0x75d48dde, - 0x6b93dddb,0x6f52c06c,0x6211e6b5,0x66d0fb02, - 0x5e9f46bf,0x5a5e5b08,0x571d7dd1,0x53dc6066, - 0x4d9b3063,0x495a2dd4,0x44190b0d,0x40d816ba, - 0xaca5c697,0xa864db20,0xa527fdf9,0xa1e6e04e, - 0xbfa1b04b,0xbb60adfc,0xb6238b25,0xb2e29692, - 0x8aad2b2f,0x8e6c3698,0x832f1041,0x87ee0df6, - 0x99a95df3,0x9d684044,0x902b669d,0x94ea7b2a, - 0xe0b41de7,0xe4750050,0xe9362689,0xedf73b3e, - 0xf3b06b3b,0xf771768c,0xfa325055,0xfef34de2, - 0xc6bcf05f,0xc27dede8,0xcf3ecb31,0xcbffd686, - 0xd5b88683,0xd1799b34,0xdc3abded,0xd8fba05a, - 0x690ce0ee,0x6dcdfd59,0x608edb80,0x644fc637, - 0x7a089632,0x7ec98b85,0x738aad5c,0x774bb0eb, - 0x4f040d56,0x4bc510e1,0x46863638,0x42472b8f, - 0x5c007b8a,0x58c1663d,0x558240e4,0x51435d53, - 0x251d3b9e,0x21dc2629,0x2c9f00f0,0x285e1d47, - 0x36194d42,0x32d850f5,0x3f9b762c,0x3b5a6b9b, - 0x0315d626,0x07d4cb91,0x0a97ed48,0x0e56f0ff, - 0x1011a0fa,0x14d0bd4d,0x19939b94,0x1d528623, - 0xf12f560e,0xf5ee4bb9,0xf8ad6d60,0xfc6c70d7, - 0xe22b20d2,0xe6ea3d65,0xeba91bbc,0xef68060b, - 0xd727bbb6,0xd3e6a601,0xdea580d8,0xda649d6f, - 0xc423cd6a,0xc0e2d0dd,0xcda1f604,0xc960ebb3, - 0xbd3e8d7e,0xb9ff90c9,0xb4bcb610,0xb07daba7, - 0xae3afba2,0xaafbe615,0xa7b8c0cc,0xa379dd7b, - 0x9b3660c6,0x9ff77d71,0x92b45ba8,0x9675461f, - 0x8832161a,0x8cf30bad,0x81b02d74,0x857130c3, - 0x5d8a9099,0x594b8d2e,0x5408abf7,0x50c9b640, - 0x4e8ee645,0x4a4ffbf2,0x470cdd2b,0x43cdc09c, - 0x7b827d21,0x7f436096,0x7200464f,0x76c15bf8, - 0x68860bfd,0x6c47164a,0x61043093,0x65c52d24, - 0x119b4be9,0x155a565e,0x18197087,0x1cd86d30, - 0x029f3d35,0x065e2082,0x0b1d065b,0x0fdc1bec, - 0x3793a651,0x3352bbe6,0x3e119d3f,0x3ad08088, - 0x2497d08d,0x2056cd3a,0x2d15ebe3,0x29d4f654, - 0xc5a92679,0xc1683bce,0xcc2b1d17,0xc8ea00a0, - 0xd6ad50a5,0xd26c4d12,0xdf2f6bcb,0xdbee767c, - 0xe3a1cbc1,0xe760d676,0xea23f0af,0xeee2ed18, - 0xf0a5bd1d,0xf464a0aa,0xf9278673,0xfde69bc4, - 0x89b8fd09,0x8d79e0be,0x803ac667,0x84fbdbd0, - 0x9abc8bd5,0x9e7d9662,0x933eb0bb,0x97ffad0c, - 0xafb010b1,0xab710d06,0xa6322bdf,0xa2f33668, - 0xbcb4666d,0xb8757bda,0xb5365d03,0xb1f740b4}; +#include "crctable.h" /* init the encode/decode logical stream state */ @@ -290,10 +234,27 @@ static int _os_lacing_expand(ogg_stream_state *os,long needed){ /* Direct table CRC; note that this will be faster in the future if we perform the checksum simultaneously with other copies */ +static ogg_uint32_t _os_update_crc(ogg_uint32_t crc, unsigned char *buffer, int size){ + while (size>=8){ + crc^=((ogg_uint32_t)buffer[0]<<24)|((ogg_uint32_t)buffer[1]<<16)|((ogg_uint32_t)buffer[2]<<8)|((ogg_uint32_t)buffer[3]); + + crc=crc_lookup[7][ crc>>24 ]^crc_lookup[6][(crc>>16)&0xFF]^ + crc_lookup[5][(crc>> 8)&0xFF]^crc_lookup[4][ crc &0xFF]^ + crc_lookup[3][buffer[4] ]^crc_lookup[2][buffer[5] ]^ + crc_lookup[1][buffer[6] ]^crc_lookup[0][buffer[7] ]; + + buffer+=8; + size-=8; + } + + while (size--) + crc=(crc<<8)^crc_lookup[0][((crc >> 24)&0xff)^*buffer++]; + return crc; +} + void ogg_page_checksum_set(ogg_page *og){ if(og){ ogg_uint32_t crc_reg=0; - int i; /* safety; needed for API behavior, but not framing code */ og->header[22]=0; @@ -301,10 +262,8 @@ void ogg_page_checksum_set(ogg_page *og){ og->header[24]=0; og->header[25]=0; - for(i=0;i<og->header_len;i++) - crc_reg=(crc_reg<<8)^crc_lookup[((crc_reg >> 24)&0xff)^og->header[i]]; - for(i=0;i<og->body_len;i++) - crc_reg=(crc_reg<<8)^crc_lookup[((crc_reg >> 24)&0xff)^og->body[i]]; + crc_reg=_os_update_crc(crc_reg,og->header,og->header_len); + crc_reg=_os_update_crc(crc_reg,og->body,og->body_len); og->header[22]=(unsigned char)(crc_reg&0xff); og->header[23]=(unsigned char)((crc_reg>>8)&0xff); @@ -414,9 +373,9 @@ static int ogg_stream_flush_i(ogg_stream_state *os,ogg_page *og, int force, int }else{ /* The extra packets_done, packet_just_done logic here attempts to do two things: - 1) Don't unneccessarily span pages. + 1) Don't unnecessarily span pages. 2) Unless necessary, don't flush pages if there are less than four packets on - them; this expands page size to reduce unneccessary overhead if incoming packets + them; this expands page size to reduce unnecessary overhead if incoming packets are large. These are not necessary behaviors, just 'always better than naive flushing' without requiring an application to explicitly request a specific optimized @@ -723,16 +682,15 @@ long ogg_sync_pageseek(ogg_sync_state *oy,ogg_page *og){ /* replace the computed checksum with the one actually read in */ memcpy(page+22,chksum,4); +#ifndef DISABLE_CRC /* Bad checksum. Lose sync */ goto sync_fail; +#endif } } /* yes, have a whole page all ready to go */ { - unsigned char *page=oy->data+oy->returned; - long bytes; - if(og){ og->header=page; og->header_len=oy->headerbytes; @@ -1814,6 +1772,7 @@ int main(void){ test_pack(packets,headret,0,0,0); } +#ifndef DISABLE_CRC { /* test for the libogg 1.1.1 resync in large continuation bug found by Josh Coalson) */ @@ -1823,6 +1782,9 @@ int main(void){ fprintf(stderr,"testing continuation resync in very large packets... "); test_pack(packets,headret,100,2,3); } +#else + fprintf(stderr,"Skipping continuation resync test due to --disable-crc\n"); +#endif { /* term only page. why not? */ @@ -2084,6 +2046,7 @@ int main(void){ fprintf(stderr,"ok.\n"); } +#ifndef DISABLE_CRC /* Test recapture: page + garbage + page */ { ogg_page og_de; @@ -2125,6 +2088,9 @@ int main(void){ fprintf(stderr,"ok.\n"); } +#else + fprintf(stderr,"Skipping recapture test due to --disable-crc\n"); +#endif /* Free page data that was previously copied */ { @@ -2133,6 +2099,9 @@ int main(void){ } } } + ogg_sync_clear(&oy); + ogg_stream_clear(&os_en); + ogg_stream_clear(&os_de); return(0); } diff --git a/thirdparty/libogg/ogg/config_types.h b/thirdparty/libogg/ogg/config_types.h index e630657547..3574a8ad44 100644 --- a/thirdparty/libogg/ogg/config_types.h +++ b/thirdparty/libogg/ogg/config_types.h @@ -8,5 +8,6 @@ typedef uint16_t ogg_uint16_t; typedef int32_t ogg_int32_t; typedef uint32_t ogg_uint32_t; typedef int64_t ogg_int64_t; +typedef uint64_t ogg_uint64_t; #endif diff --git a/thirdparty/libogg/ogg/ogg.h b/thirdparty/libogg/ogg/ogg.h index 7609fc24d6..c4325aa76d 100644 --- a/thirdparty/libogg/ogg/ogg.h +++ b/thirdparty/libogg/ogg/ogg.h @@ -11,7 +11,6 @@ ******************************************************************** function: toplevel libogg include - last mod: $Id$ ********************************************************************/ #ifndef _OGG_H diff --git a/thirdparty/libogg/ogg/os_types.h b/thirdparty/libogg/ogg/os_types.h index b8f56308b5..e655a1d628 100644 --- a/thirdparty/libogg/ogg/os_types.h +++ b/thirdparty/libogg/ogg/os_types.h @@ -10,8 +10,7 @@ * * ******************************************************************** - function: #ifdef jail to whip a few platforms into the UNIX ideal. - last mod: $Id$ + function: Define a consistent set of types on each platform. ********************************************************************/ #ifndef _OS_TYPES_H @@ -44,6 +43,7 @@ typedef unsigned long long ogg_uint64_t; # elif defined(__MWERKS__) typedef long long ogg_int64_t; + typedef unsigned long long ogg_uint64_t; typedef int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef short ogg_int16_t; @@ -62,6 +62,7 @@ typedef __int64 ogg_int64_t; typedef __int32 ogg_int32_t; typedef unsigned __int32 ogg_uint32_t; + typedef unsigned __int64 ogg_uint64_t; typedef __int16 ogg_int16_t; typedef unsigned __int16 ogg_uint16_t; # endif @@ -69,12 +70,13 @@ #elif (defined(__APPLE__) && defined(__MACH__)) /* MacOS X Framework build */ -# include <inttypes.h> +# include <sys/types.h> typedef int16_t ogg_int16_t; - typedef uint16_t ogg_uint16_t; + typedef u_int16_t ogg_uint16_t; typedef int32_t ogg_int32_t; - typedef uint32_t ogg_uint32_t; + typedef u_int32_t ogg_uint32_t; typedef int64_t ogg_int64_t; + typedef u_int64_t ogg_uint64_t; #elif defined(__HAIKU__) @@ -85,6 +87,7 @@ typedef int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef long long ogg_int64_t; + typedef unsigned long long ogg_uint64_t; #elif defined(__BEOS__) @@ -95,6 +98,7 @@ typedef int32_t ogg_int32_t; typedef uint32_t ogg_uint32_t; typedef int64_t ogg_int64_t; + typedef uint64_t ogg_uint64_t; #elif defined (__EMX__) @@ -104,6 +108,8 @@ typedef int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef long long ogg_int64_t; + typedef unsigned long long ogg_uint64_t; + #elif defined (DJGPP) @@ -112,11 +118,13 @@ typedef int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef long long ogg_int64_t; + typedef unsigned long long ogg_uint64_t; #elif defined(R5900) /* PS2 EE */ typedef long ogg_int64_t; + typedef unsigned long ogg_uint64_t; typedef int ogg_int32_t; typedef unsigned ogg_uint32_t; typedef short ogg_int16_t; @@ -129,6 +137,7 @@ typedef signed int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef long long int ogg_int64_t; + typedef unsigned long long int ogg_uint64_t; #elif defined(__TMS320C6X__) @@ -138,6 +147,7 @@ typedef signed int ogg_int32_t; typedef unsigned int ogg_uint32_t; typedef long long int ogg_int64_t; + typedef unsigned long long int ogg_uint64_t; #else diff --git a/thirdparty/misc/stb_vorbis.c b/thirdparty/misc/stb_vorbis.c index 71af404dae..4ab8880d5d 100644 --- a/thirdparty/misc/stb_vorbis.c +++ b/thirdparty/misc/stb_vorbis.c @@ -1,4 +1,4 @@ -// Ogg Vorbis audio decoder - v1.16 - public domain +// Ogg Vorbis audio decoder - v1.17 - public domain // http://nothings.org/stb_vorbis/ // // Original version written by Sean Barrett in 2007. @@ -30,9 +30,10 @@ // Tom Beaumont Ingo Leitgeb Nicolas Guillemot // Phillip Bennefall Rohit Thiago Goulart // manxorist@github saga musix github:infatum -// Timur Gagiev +// Timur Gagiev Maxwell Koo // // Partial history: +// 1.17 - 2019-07-08 - fix CVE-2019-13217..CVE-2019-13223 (by ForAllSecure) // 1.16 - 2019-03-04 - fix warnings // 1.15 - 2019-02-07 - explicit failure if Ogg Skeleton data is found // 1.14 - 2018-02-11 - delete bogus dealloca usage @@ -1202,8 +1203,10 @@ static int lookup1_values(int entries, int dim) int r = (int) floor(exp((float) log((float) entries) / dim)); if ((int) floor(pow((float) r+1, dim)) <= entries) // (int) cast for MinGW warning; ++r; // floor() to avoid _ftol() when non-CRT - assert(pow((float) r+1, dim) > entries); - assert((int) floor(pow((float) r, dim)) <= entries); // (int),floor() as above + if (pow((float) r+1, dim) <= entries) + return -1; + if ((int) floor(pow((float) r, dim)) > entries) + return -1; return r; } @@ -2013,7 +2016,7 @@ static __forceinline void draw_line(float *output, int x0, int y0, int x1, int y ady -= abs(base) * adx; if (x1 > n) x1 = n; if (x < x1) { - LINE_OP(output[x], inverse_db_table[y]); + LINE_OP(output[x], inverse_db_table[y&255]); for (++x; x < x1; ++x) { err += ady; if (err >= adx) { @@ -2021,7 +2024,7 @@ static __forceinline void draw_line(float *output, int x0, int y0, int x1, int y y += sy; } else y += base; - LINE_OP(output[x], inverse_db_table[y]); + LINE_OP(output[x], inverse_db_table[y&255]); } } } @@ -3048,7 +3051,6 @@ static float *get_window(vorb *f, int len) len <<= 1; if (len == f->blocksize_0) return f->window[0]; if (len == f->blocksize_1) return f->window[1]; - assert(0); return NULL; } @@ -3454,6 +3456,7 @@ static int vorbis_finish_frame(stb_vorbis *f, int len, int left, int right) if (f->previous_length) { int i,j, n = f->previous_length; float *w = get_window(f, n); + if (w == NULL) return 0; for (i=0; i < f->channels; ++i) { for (j=0; j < n; ++j) f->channel_buffers[i][left+j] = @@ -3695,6 +3698,7 @@ static int start_decoder(vorb *f) while (current_entry < c->entries) { int limit = c->entries - current_entry; int n = get_bits(f, ilog(limit)); + if (current_length >= 32) return error(f, VORBIS_invalid_setup); if (current_entry + n > (int) c->entries) { return error(f, VORBIS_invalid_setup); } memset(lengths + current_entry, current_length, n); current_entry += n; @@ -3798,7 +3802,9 @@ static int start_decoder(vorb *f) c->value_bits = get_bits(f, 4)+1; c->sequence_p = get_bits(f,1); if (c->lookup_type == 1) { - c->lookup_values = lookup1_values(c->entries, c->dimensions); + int values = lookup1_values(c->entries, c->dimensions); + if (values < 0) return error(f, VORBIS_invalid_setup); + c->lookup_values = (uint32) values; } else { c->lookup_values = c->entries * c->dimensions; } @@ -3934,6 +3940,9 @@ static int start_decoder(vorb *f) p[j].id = j; } qsort(p, g->values, sizeof(p[0]), point_compare); + for (j=0; j < g->values-1; ++j) + if (p[j].x == p[j+1].x) + return error(f, VORBIS_invalid_setup); for (j=0; j < g->values; ++j) g->sorted_order[j] = (uint8) p[j].id; // precompute the neighbors @@ -4020,6 +4029,7 @@ static int start_decoder(vorb *f) max_submaps = m->submaps; if (get_bits(f,1)) { m->coupling_steps = get_bits(f,8)+1; + if (m->coupling_steps > f->channels) return error(f, VORBIS_invalid_setup); for (k=0; k < m->coupling_steps; ++k) { m->chan[k].magnitude = get_bits(f, ilog(f->channels-1)); m->chan[k].angle = get_bits(f, ilog(f->channels-1)); @@ -5386,6 +5396,12 @@ int stb_vorbis_get_samples_float(stb_vorbis *f, int channels, float **buffer, in #endif // STB_VORBIS_NO_PULLDATA_API /* Version history + 1.17 - 2019-07-08 - fix CVE-2019-13217, -13218, -13219, -13220, -13221, -13222, -13223 + found with Mayhem by ForAllSecure + 1.16 - 2019-03-04 - fix warnings + 1.15 - 2019-02-07 - explicit failure if Ogg Skeleton data is found + 1.14 - 2018-02-11 - delete bogus dealloca usage + 1.13 - 2018-01-29 - fix truncation of last frame (hopefully) 1.12 - 2017-11-21 - limit residue begin/end to blocksize/2 to avoid large temp allocs in bad/corrupt files 1.11 - 2017-07-23 - fix MinGW compilation 1.10 - 2017-03-03 - more robust seeking; fix negative ilog(); clear error in open_memory diff --git a/thirdparty/zstd/common/bitstream.h b/thirdparty/zstd/common/bitstream.h index d955bd677b..1c294b80d1 100644 --- a/thirdparty/zstd/common/bitstream.h +++ b/thirdparty/zstd/common/bitstream.h @@ -57,6 +57,8 @@ extern "C" { =========================================*/ #if defined(__BMI__) && defined(__GNUC__) # include <immintrin.h> /* support for bextr (experimental) */ +#elif defined(__ICCARM__) +# include <intrinsics.h> #endif #define STREAM_ACCUMULATOR_MIN_32 25 @@ -162,7 +164,9 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val) _BitScanReverse ( &r, val ); return (unsigned) r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return 31 - __builtin_clz (val); + return __builtin_clz (val) ^ 31; +# elif defined(__ICCARM__) /* IAR Intrinsic */ + return 31 - __CLZ(val); # else /* Software version */ static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, @@ -240,9 +244,9 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) { size_t const nbBytes = bitC->bitPos >> 3; assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); MEM_writeLEST(bitC->ptr, bitC->bitContainer); bitC->ptr += nbBytes; - assert(bitC->ptr <= bitC->endPtr); bitC->bitPos &= 7; bitC->bitContainer >>= nbBytes*8; } @@ -256,6 +260,7 @@ MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) { size_t const nbBytes = bitC->bitPos >> 3; assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); MEM_writeLEST(bitC->ptr, bitC->bitContainer); bitC->ptr += nbBytes; if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; diff --git a/thirdparty/zstd/common/compiler.h b/thirdparty/zstd/common/compiler.h index 87bf51ae8c..1877a0c1d9 100644 --- a/thirdparty/zstd/common/compiler.h +++ b/thirdparty/zstd/common/compiler.h @@ -23,7 +23,7 @@ # define INLINE_KEYWORD #endif -#if defined(__GNUC__) +#if defined(__GNUC__) || defined(__ICCARM__) # define FORCE_INLINE_ATTR __attribute__((always_inline)) #elif defined(_MSC_VER) # define FORCE_INLINE_ATTR __forceinline @@ -61,11 +61,18 @@ # define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR #endif +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + /* force no inlining */ #ifdef _MSC_VER # define FORCE_NOINLINE static __declspec(noinline) #else -# ifdef __GNUC__ +# if defined(__GNUC__) || defined(__ICCARM__) # define FORCE_NOINLINE static __attribute__((__noinline__)) # else # define FORCE_NOINLINE static @@ -76,7 +83,7 @@ #ifndef __has_attribute #define __has_attribute(x) 0 /* Compatibility with non-clang compilers. */ #endif -#if defined(__GNUC__) +#if defined(__GNUC__) || defined(__ICCARM__) # define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) #else # define TARGET_ATTRIBUTE(target) @@ -127,9 +134,14 @@ } \ } -/* vectorization */ +/* vectorization + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */ #if !defined(__clang__) && defined(__GNUC__) -# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# else +# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")") +# endif #else # define DONT_VECTORIZE #endif diff --git a/thirdparty/zstd/common/fse.h b/thirdparty/zstd/common/fse.h index 811c670bdd..a7553e3721 100644 --- a/thirdparty/zstd/common/fse.h +++ b/thirdparty/zstd/common/fse.h @@ -308,7 +308,7 @@ If there is an error, the function will return an error code, which can be teste *******************************************/ /* FSE buffer bounds */ #define FSE_NCOUNTBOUND 512 -#define FSE_BLOCKBOUND(size) (size + (size>>7)) +#define FSE_BLOCKBOUND(size) (size + (size>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) #define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ /* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ diff --git a/thirdparty/zstd/common/fse_decompress.c b/thirdparty/zstd/common/fse_decompress.c index 72bbead5be..4f07378982 100644 --- a/thirdparty/zstd/common/fse_decompress.c +++ b/thirdparty/zstd/common/fse_decompress.c @@ -52,7 +52,9 @@ #define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ /* check and forward error code */ +#ifndef CHECK_F #define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } +#endif /* ************************************************************** diff --git a/thirdparty/zstd/common/mem.h b/thirdparty/zstd/common/mem.h index 5da248756f..530d30c8f7 100644 --- a/thirdparty/zstd/common/mem.h +++ b/thirdparty/zstd/common/mem.h @@ -47,6 +47,79 @@ extern "C" { #define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; } MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } +/* detects whether we are being compiled under msan */ +#if defined (__has_feature) +# if __has_feature(memory_sanitizer) +# define MEMORY_SANITIZER 1 +# endif +#endif + +#if defined (MEMORY_SANITIZER) +/* Not all platforms that support msan provide sanitizers/msan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ + +#include <stdint.h> /* intptr_t */ + +/* Make memory region fully initialized (without changing its contents). */ +void __msan_unpoison(const volatile void *a, size_t size); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void __msan_poison(const volatile void *a, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t __msan_test_shadow(const volatile void *x, size_t size); +#endif + +/* detects whether we are being compiled under asan */ +#if defined (__has_feature) +# if __has_feature(address_sanitizer) +# define ADDRESS_SANITIZER 1 +# endif +#elif defined(__SANITIZE_ADDRESS__) +# define ADDRESS_SANITIZER 1 +#endif + +#if defined (ADDRESS_SANITIZER) +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ + +/** + * Marks a memory region (<c>[addr, addr+size)</c>) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of <c>[addr, addr+size)</c> due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region (<c>[addr, addr+size)</c>) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of <c>[addr, addr+size)</c> due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +#endif + /*-************************************************************** * Basic Types @@ -102,7 +175,7 @@ MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (size #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define MEM_FORCE_MEMORY_ACCESS 2 -# elif defined(__INTEL_COMPILER) || defined(__GNUC__) +# elif defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) # define MEM_FORCE_MEMORY_ACCESS 1 # endif #endif diff --git a/thirdparty/zstd/common/pool.c b/thirdparty/zstd/common/pool.c index 7a82945432..f575935076 100644 --- a/thirdparty/zstd/common/pool.c +++ b/thirdparty/zstd/common/pool.c @@ -127,9 +127,13 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, ctx->queueTail = 0; ctx->numThreadsBusy = 0; ctx->queueEmpty = 1; - (void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); - (void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); - (void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + { + int error = 0; + error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + if (error) { POOL_free(ctx); return NULL; } + } ctx->shutdown = 0; /* Allocate space for the thread handles */ ctx->threads = (ZSTD_pthread_t*)ZSTD_malloc(numThreads * sizeof(ZSTD_pthread_t), customMem); diff --git a/thirdparty/zstd/common/threading.c b/thirdparty/zstd/common/threading.c index f3d4fa8418..482664bd9a 100644 --- a/thirdparty/zstd/common/threading.c +++ b/thirdparty/zstd/common/threading.c @@ -14,6 +14,8 @@ * This file will hold wrapper for systems, which do not support pthreads */ +#include "threading.h" + /* create fake symbol to avoid empty translation unit warning */ int g_ZSTD_threading_useless_symbol; @@ -28,7 +30,6 @@ int g_ZSTD_threading_useless_symbol; /* === Dependencies === */ #include <process.h> #include <errno.h> -#include "threading.h" /* === Implementation === */ @@ -73,3 +74,47 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr) } #endif /* ZSTD_MULTITHREAD */ + +#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32) + +#include <stdlib.h> + +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr) +{ + *mutex = (pthread_mutex_t*)malloc(sizeof(pthread_mutex_t)); + if (!*mutex) + return 1; + return pthread_mutex_init(*mutex, attr); +} + +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) +{ + if (!*mutex) + return 0; + { + int const ret = pthread_mutex_destroy(*mutex); + free(*mutex); + return ret; + } +} + +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr) +{ + *cond = (pthread_cond_t*)malloc(sizeof(pthread_cond_t)); + if (!*cond) + return 1; + return pthread_cond_init(*cond, attr); +} + +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) +{ + if (!*cond) + return 0; + { + int const ret = pthread_cond_destroy(*cond); + free(*cond); + return ret; + } +} + +#endif diff --git a/thirdparty/zstd/common/threading.h b/thirdparty/zstd/common/threading.h index d806c89d01..3193ca7db8 100644 --- a/thirdparty/zstd/common/threading.h +++ b/thirdparty/zstd/common/threading.h @@ -13,6 +13,8 @@ #ifndef THREADING_H_938743 #define THREADING_H_938743 +#include "debug.h" + #if defined (__cplusplus) extern "C" { #endif @@ -75,10 +77,12 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); */ -#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ +#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ /* === POSIX Systems === */ # include <pthread.h> +#if DEBUGLEVEL < 1 + #define ZSTD_pthread_mutex_t pthread_mutex_t #define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) #define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) @@ -96,6 +100,33 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) #define ZSTD_pthread_join(a, b) pthread_join((a),(b)) +#else /* DEBUGLEVEL >= 1 */ + +/* Debug implementation of threading. + * In this implementation we use pointers for mutexes and condition variables. + * This way, if we forget to init/destroy them the program will crash or ASAN + * will report leaks. + */ + +#define ZSTD_pthread_mutex_t pthread_mutex_t* +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr); +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex); +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a)) + +#define ZSTD_pthread_cond_t pthread_cond_t* +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr); +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond); +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) + +#endif + #else /* ZSTD_MULTITHREAD not defined */ /* No multithreading support */ diff --git a/thirdparty/zstd/common/xxhash.c b/thirdparty/zstd/common/xxhash.c index 30599aaae4..99d2459621 100644 --- a/thirdparty/zstd/common/xxhash.c +++ b/thirdparty/zstd/common/xxhash.c @@ -53,7 +53,8 @@ # if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) # define XXH_FORCE_MEMORY_ACCESS 2 # elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ - (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) || \ + defined(__ICCARM__) # define XXH_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -120,7 +121,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcp # define INLINE_KEYWORD #endif -#if defined(__GNUC__) +#if defined(__GNUC__) || defined(__ICCARM__) # define FORCE_INLINE_ATTR __attribute__((always_inline)) #elif defined(_MSC_VER) # define FORCE_INLINE_ATTR __forceinline @@ -206,7 +207,12 @@ static U64 XXH_read64(const void* memPtr) # define XXH_rotl32(x,r) _rotl(x,r) # define XXH_rotl64(x,r) _rotl64(x,r) #else +#if defined(__ICCARM__) +# include <intrinsics.h> +# define XXH_rotl32(x,r) __ROR(x,(32 - r)) +#else # define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +#endif # define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) #endif diff --git a/thirdparty/zstd/common/zstd_internal.h b/thirdparty/zstd/common/zstd_internal.h index 81b16eac2e..dcdcbdb81c 100644 --- a/thirdparty/zstd/common/zstd_internal.h +++ b/thirdparty/zstd/common/zstd_internal.h @@ -56,9 +56,9 @@ extern "C" { /** * Return the specified error if the condition evaluates to true. * - * In debug modes, prints additional information. In order to do that - * (particularly, printing the conditional that failed), this can't just wrap - * RETURN_ERROR(). + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). */ #define RETURN_ERROR_IF(cond, err, ...) \ if (cond) { \ @@ -197,79 +197,56 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } #define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } -#define WILDCOPY_OVERLENGTH 8 -#define VECLEN 16 +#define WILDCOPY_OVERLENGTH 32 +#define WILDCOPY_VECLEN 16 typedef enum { ZSTD_no_overlap, - ZSTD_overlap_src_before_dst, + ZSTD_overlap_src_before_dst /* ZSTD_overlap_dst_before_src, */ } ZSTD_overlap_e; /*! ZSTD_wildcopy() : - * custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */ + * Custom version of memcpy(), can over read/write up to WILDCOPY_OVERLENGTH bytes (if length==0) + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap, but they MUST be at least 8 bytes apart. + * The src buffer must be before the dst buffer. + */ MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE -void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) +void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e const ovtype) { ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; BYTE* const oend = op + length; - assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); - if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) { - do - COPY8(op, ip) - while (op < oend); - } - else { - if ((length & 8) == 0) - COPY8(op, ip); - do { + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff <= -WILDCOPY_VECLEN)); + + if (ovtype == ZSTD_overlap_src_before_dst && diff < WILDCOPY_VECLEN) { + /* Handle short offset copies. */ + do { + COPY8(op, ip) + } while (op < oend); + } else { + assert(diff >= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first two COPY16() calls because the copy length is + * almost certain to be short, so the branches have different + * probabilities. + * On gcc-9 unrolling once is +1.6%, twice is +2%, thrice is +1.8%. + * On clang-8 unrolling once is +1.4%, twice is +3.3%, thrice is +3%. + */ COPY16(op, ip); - } - while (op < oend); - } -} - -/*! ZSTD_wildcopy_16min() : - * same semantics as ZSTD_wilcopy() except guaranteed to be able to copy 16 bytes at the start */ -MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE -void ZSTD_wildcopy_16min(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) -{ - ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - BYTE* const oend = op + length; - - assert(length >= 8); - assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); - - if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) { - do - COPY8(op, ip) - while (op < oend); - } - else { - if ((length & 8) == 0) - COPY8(op, ip); - do { COPY16(op, ip); - } - while (op < oend); + if (op >= oend) return; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); } } -MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ -{ - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; - BYTE* const oend = (BYTE*)dstEnd; - do - COPY8(op, ip) - while (op < oend); -} - /*-******************************************* * Private declarations @@ -323,7 +300,9 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus _BitScanReverse(&r, val); return (unsigned)r; # elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return 31 - __builtin_clz(val); + return __builtin_clz (val) ^ 31; +# elif defined(__ICCARM__) /* IAR Intrinsic */ + return 31 - __CLZ(val); # else /* Software version */ static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; U32 v = val; diff --git a/thirdparty/zstd/compress/zstd_compress.c b/thirdparty/zstd/compress/zstd_compress.c index 1476512580..35346b92cb 100644 --- a/thirdparty/zstd/compress/zstd_compress.c +++ b/thirdparty/zstd/compress/zstd_compress.c @@ -21,6 +21,8 @@ #define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_compress_internal.h" +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" #include "zstd_fast.h" #include "zstd_double_fast.h" #include "zstd_lazy.h" @@ -40,15 +42,15 @@ size_t ZSTD_compressBound(size_t srcSize) { * Context memory management ***************************************/ struct ZSTD_CDict_s { - void* dictBuffer; const void* dictContent; size_t dictContentSize; - void* workspace; - size_t workspaceSize; + U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ + ZSTD_cwksp workspace; ZSTD_matchState_t matchState; ZSTD_compressedBlockState_t cBlockState; ZSTD_customMem customMem; U32 dictID; + int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ }; /* typedef'd to ZSTD_CDict within "zstd.h" */ ZSTD_CCtx* ZSTD_createCCtx(void) @@ -82,23 +84,26 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) ZSTD_CCtx* ZSTD_initStaticCCtx(void *workspace, size_t workspaceSize) { - ZSTD_CCtx* const cctx = (ZSTD_CCtx*) workspace; + ZSTD_cwksp ws; + ZSTD_CCtx* cctx; if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ - memset(workspace, 0, workspaceSize); /* may be a bit generous, could memset be smaller ? */ + ZSTD_cwksp_init(&ws, workspace, workspaceSize); + + cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); + if (cctx == NULL) { + return NULL; + } + memset(cctx, 0, sizeof(ZSTD_CCtx)); + ZSTD_cwksp_move(&cctx->workspace, &ws); cctx->staticSize = workspaceSize; - cctx->workSpace = (void*)(cctx+1); - cctx->workSpaceSize = workspaceSize - sizeof(ZSTD_CCtx); /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ - if (cctx->workSpaceSize < HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t)) return NULL; - assert(((size_t)cctx->workSpace & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ - cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)cctx->workSpace; - cctx->blockState.nextCBlock = cctx->blockState.prevCBlock + 1; - { - void* const ptr = cctx->blockState.nextCBlock + 1; - cctx->entropyWorkspace = (U32*)ptr; - } + if (!ZSTD_cwksp_check_available(&cctx->workspace, HUF_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object( + &cctx->workspace, HUF_WORKSPACE_SIZE); cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); return cctx; } @@ -126,11 +131,11 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) { assert(cctx != NULL); assert(cctx->staticSize == 0); - ZSTD_free(cctx->workSpace, cctx->customMem); cctx->workSpace = NULL; ZSTD_clearAllDicts(cctx); #ifdef ZSTD_MULTITHREAD ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; #endif + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); } size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) @@ -138,8 +143,13 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) if (cctx==NULL) return 0; /* support free on NULL */ RETURN_ERROR_IF(cctx->staticSize, memory_allocation, "not compatible with static CCtx"); - ZSTD_freeCCtxContent(cctx); - ZSTD_free(cctx, cctx->customMem); + { + int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + ZSTD_freeCCtxContent(cctx); + if (!cctxInWorkspace) { + ZSTD_free(cctx, cctx->customMem); + } + } return 0; } @@ -158,7 +168,9 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { if (cctx==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*cctx) + cctx->workSpaceSize + /* cctx may be in the workspace */ + return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) + + ZSTD_cwksp_sizeof(&cctx->workspace) + ZSTD_sizeof_localDict(cctx->localDict) + ZSTD_sizeof_mtctx(cctx); } @@ -227,23 +239,23 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete RETURN_ERROR_IF(!cctxParams, GENERIC); FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); memset(cctxParams, 0, sizeof(*cctxParams)); + assert(!ZSTD_checkCParams(params.cParams)); cctxParams->cParams = params.cParams; cctxParams->fParams = params.fParams; cctxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ - assert(!ZSTD_checkCParams(params.cParams)); return 0; } /* ZSTD_assignParamsToCCtxParams() : * params is presumed valid at this stage */ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams( - ZSTD_CCtx_params cctxParams, ZSTD_parameters params) + const ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) { - ZSTD_CCtx_params ret = cctxParams; + ZSTD_CCtx_params ret = *cctxParams; + assert(!ZSTD_checkCParams(params.cParams)); ret.cParams = params.cParams; ret.fParams = params.fParams; ret.compressionLevel = ZSTD_CLEVEL_DEFAULT; /* should not matter, as all cParams are presumed properly defined */ - assert(!ZSTD_checkCParams(params.cParams)); return ret; } @@ -376,7 +388,7 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) case ZSTD_c_forceAttachDict: ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy); bounds.lowerBound = ZSTD_dictDefaultAttach; - bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */ + bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ return bounds; case ZSTD_c_literalCompressionMode: @@ -390,6 +402,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; return bounds; + case ZSTD_c_srcSizeHint: + bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN; + bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; + return bounds; + default: { ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 }; return boundError; @@ -397,18 +414,6 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) } } -/* ZSTD_cParam_withinBounds: - * @return 1 if value is within cParam bounds, - * 0 otherwise */ -static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) -{ - ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); - if (ZSTD_isError(bounds.error)) return 0; - if (value < bounds.lowerBound) return 0; - if (value > bounds.upperBound) return 0; - return 1; -} - /* ZSTD_cParam_clampBounds: * Clamps the value into the bounded range. */ @@ -458,6 +463,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_forceAttachDict: case ZSTD_c_literalCompressionMode: case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: default: return 0; } @@ -504,6 +510,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_ldmMinMatch: case ZSTD_c_ldmBucketSizeLog: case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: break; default: RETURN_ERROR(parameter_unsupported); @@ -527,33 +534,33 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, if (value) { /* 0 : does not change current level */ CCtxParams->compressionLevel = value; } - if (CCtxParams->compressionLevel >= 0) return CCtxParams->compressionLevel; + if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; return 0; /* return type (size_t) cannot represent negative values */ } case ZSTD_c_windowLog : if (value!=0) /* 0 => use default */ BOUNDCHECK(ZSTD_c_windowLog, value); - CCtxParams->cParams.windowLog = value; + CCtxParams->cParams.windowLog = (U32)value; return CCtxParams->cParams.windowLog; case ZSTD_c_hashLog : if (value!=0) /* 0 => use default */ BOUNDCHECK(ZSTD_c_hashLog, value); - CCtxParams->cParams.hashLog = value; + CCtxParams->cParams.hashLog = (U32)value; return CCtxParams->cParams.hashLog; case ZSTD_c_chainLog : if (value!=0) /* 0 => use default */ BOUNDCHECK(ZSTD_c_chainLog, value); - CCtxParams->cParams.chainLog = value; + CCtxParams->cParams.chainLog = (U32)value; return CCtxParams->cParams.chainLog; case ZSTD_c_searchLog : if (value!=0) /* 0 => use default */ BOUNDCHECK(ZSTD_c_searchLog, value); - CCtxParams->cParams.searchLog = value; - return value; + CCtxParams->cParams.searchLog = (U32)value; + return (size_t)value; case ZSTD_c_minMatch : if (value!=0) /* 0 => use default */ @@ -684,6 +691,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, CCtxParams->targetCBlockSize = value; return CCtxParams->targetCBlockSize; + case ZSTD_c_srcSizeHint : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_srcSizeHint, value); + CCtxParams->srcSizeHint = value; + return CCtxParams->srcSizeHint; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } @@ -789,6 +802,9 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_targetCBlockSize : *value = (int)CCtxParams->targetCBlockSize; break; + case ZSTD_c_srcSizeHint : + *value = (int)CCtxParams->srcSizeHint; + break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; @@ -1039,7 +1055,11 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar, ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize) { - ZSTD_compressionParameters cParams = ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize); + ZSTD_compressionParameters cParams; + if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { + srcSizeHint = CCtxParams->srcSizeHint; + } + cParams = ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize); if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; if (CCtxParams->cParams.windowLog) cParams.windowLog = CCtxParams->cParams.windowLog; if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog; @@ -1059,10 +1079,19 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; - size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits)) * sizeof(U32) - + (ZSTD_OPT_NUM+1) * (sizeof(ZSTD_match_t)+sizeof(ZSTD_optimal_t)); + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't + * surrounded by redzones in ASAN. */ + size_t const tableSpace = chainSize * sizeof(U32) + + hSize * sizeof(U32) + + h3Size * sizeof(U32); + size_t const optPotentialSpace = + ZSTD_cwksp_alloc_size((MaxML+1) * sizeof(U32)) + + ZSTD_cwksp_alloc_size((MaxLL+1) * sizeof(U32)) + + ZSTD_cwksp_alloc_size((MaxOff+1) * sizeof(U32)) + + ZSTD_cwksp_alloc_size((1<<Litbits) * sizeof(U32)) + + ZSTD_cwksp_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)) + + ZSTD_cwksp_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) ? optPotentialSpace : 0; @@ -1079,20 +1108,23 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); U32 const divider = (cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; - size_t const entropySpace = HUF_WORKSPACE_SIZE; - size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); + size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 1); size_t const ldmSpace = ZSTD_ldm_getTableSize(params->ldmParams); - size_t const ldmSeqSpace = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq); + size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize) * sizeof(rawSeq)); size_t const neededSpace = entropySpace + blockStateSpace + tokenSpace + matchStateSize + ldmSpace + ldmSeqSpace; + size_t const cctxSpace = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)); - DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)sizeof(ZSTD_CCtx)); - DEBUGLOG(5, "estimate workSpace : %u", (U32)neededSpace); - return sizeof(ZSTD_CCtx) + neededSpace; + DEBUGLOG(5, "sizeof(ZSTD_CCtx) : %u", (U32)cctxSpace); + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); + return cctxSpace + neededSpace; } } @@ -1128,7 +1160,8 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); size_t const inBuffSize = ((size_t)1 << cParams.windowLog) + blockSize; size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; - size_t const streamingSize = inBuffSize + outBuffSize; + size_t const streamingSize = ZSTD_cwksp_alloc_size(inBuffSize) + + ZSTD_cwksp_alloc_size(outBuffSize); return CCtxSize + streamingSize; } @@ -1196,17 +1229,6 @@ size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ } - - -static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1, - ZSTD_compressionParameters cParams2) -{ - return (cParams1.hashLog == cParams2.hashLog) - & (cParams1.chainLog == cParams2.chainLog) - & (cParams1.strategy == cParams2.strategy) /* opt parser space */ - & ((cParams1.minMatch==3) == (cParams2.minMatch==3)); /* hashlog3 space */ -} - static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, ZSTD_compressionParameters cParams2) { @@ -1221,71 +1243,6 @@ static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, assert(cParams1.strategy == cParams2.strategy); } -/** The parameters are equivalent if ldm is not enabled in both sets or - * all the parameters are equivalent. */ -static U32 ZSTD_equivalentLdmParams(ldmParams_t ldmParams1, - ldmParams_t ldmParams2) -{ - return (!ldmParams1.enableLdm && !ldmParams2.enableLdm) || - (ldmParams1.enableLdm == ldmParams2.enableLdm && - ldmParams1.hashLog == ldmParams2.hashLog && - ldmParams1.bucketSizeLog == ldmParams2.bucketSizeLog && - ldmParams1.minMatchLength == ldmParams2.minMatchLength && - ldmParams1.hashRateLog == ldmParams2.hashRateLog); -} - -typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e; - -/* ZSTD_sufficientBuff() : - * check internal buffers exist for streaming if buffPol == ZSTDb_buffered . - * Note : they are assumed to be correctly sized if ZSTD_equivalentCParams()==1 */ -static U32 ZSTD_sufficientBuff(size_t bufferSize1, size_t maxNbSeq1, - size_t maxNbLit1, - ZSTD_buffered_policy_e buffPol2, - ZSTD_compressionParameters cParams2, - U64 pledgedSrcSize) -{ - size_t const windowSize2 = MAX(1, (size_t)MIN(((U64)1 << cParams2.windowLog), pledgedSrcSize)); - size_t const blockSize2 = MIN(ZSTD_BLOCKSIZE_MAX, windowSize2); - size_t const maxNbSeq2 = blockSize2 / ((cParams2.minMatch == 3) ? 3 : 4); - size_t const maxNbLit2 = blockSize2; - size_t const neededBufferSize2 = (buffPol2==ZSTDb_buffered) ? windowSize2 + blockSize2 : 0; - DEBUGLOG(4, "ZSTD_sufficientBuff: is neededBufferSize2=%u <= bufferSize1=%u", - (U32)neededBufferSize2, (U32)bufferSize1); - DEBUGLOG(4, "ZSTD_sufficientBuff: is maxNbSeq2=%u <= maxNbSeq1=%u", - (U32)maxNbSeq2, (U32)maxNbSeq1); - DEBUGLOG(4, "ZSTD_sufficientBuff: is maxNbLit2=%u <= maxNbLit1=%u", - (U32)maxNbLit2, (U32)maxNbLit1); - return (maxNbLit2 <= maxNbLit1) - & (maxNbSeq2 <= maxNbSeq1) - & (neededBufferSize2 <= bufferSize1); -} - -/** Equivalence for resetCCtx purposes */ -static U32 ZSTD_equivalentParams(ZSTD_CCtx_params params1, - ZSTD_CCtx_params params2, - size_t buffSize1, - size_t maxNbSeq1, size_t maxNbLit1, - ZSTD_buffered_policy_e buffPol2, - U64 pledgedSrcSize) -{ - DEBUGLOG(4, "ZSTD_equivalentParams: pledgedSrcSize=%u", (U32)pledgedSrcSize); - if (!ZSTD_equivalentCParams(params1.cParams, params2.cParams)) { - DEBUGLOG(4, "ZSTD_equivalentCParams() == 0"); - return 0; - } - if (!ZSTD_equivalentLdmParams(params1.ldmParams, params2.ldmParams)) { - DEBUGLOG(4, "ZSTD_equivalentLdmParams() == 0"); - return 0; - } - if (!ZSTD_sufficientBuff(buffSize1, maxNbSeq1, maxNbLit1, buffPol2, - params2.cParams, pledgedSrcSize)) { - DEBUGLOG(4, "ZSTD_sufficientBuff() == 0"); - return 0; - } - return 1; -} - static void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) { int i; @@ -1311,87 +1268,104 @@ static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) ms->dictMatchState = NULL; } -/*! ZSTD_continueCCtx() : - * reuse CCtx without reset (note : requires no dictionary) */ -static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_CCtx_params params, U64 pledgedSrcSize) -{ - size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - DEBUGLOG(4, "ZSTD_continueCCtx: re-use context in place"); +/** + * Indicates whether this compression proceeds directly from user-provided + * source buffer to user-provided destination buffer (ZSTDb_not_buffered), or + * whether the context needs to buffer the input/output (ZSTDb_buffered). + */ +typedef enum { + ZSTDb_not_buffered, + ZSTDb_buffered +} ZSTD_buffered_policy_e; - cctx->blockSize = blockSize; /* previous block size could be different even for same windowLog, due to pledgedSrcSize */ - cctx->appliedParams = params; - cctx->blockState.matchState.cParams = params.cParams; - cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; - cctx->consumedSrcSize = 0; - cctx->producedCSize = 0; - if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) - cctx->appliedParams.fParams.contentSizeFlag = 0; - DEBUGLOG(4, "pledged content size : %u ; flag : %u", - (U32)pledgedSrcSize, cctx->appliedParams.fParams.contentSizeFlag); - cctx->stage = ZSTDcs_init; - cctx->dictID = 0; - if (params.ldmParams.enableLdm) - ZSTD_window_clear(&cctx->ldmState.window); - ZSTD_referenceExternalSequences(cctx, NULL, 0); - ZSTD_invalidateMatchState(&cctx->blockState.matchState); - ZSTD_reset_compressedBlockState(cctx->blockState.prevCBlock); - XXH64_reset(&cctx->xxhState, 0); - return 0; -} +/** + * Controls, for this matchState reset, whether the tables need to be cleared / + * prepared for the coming compression (ZSTDcrp_makeClean), or whether the + * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a + * subsequent operation will overwrite the table space anyways (e.g., copying + * the matchState contents in from a CDict). + */ +typedef enum { + ZSTDcrp_makeClean, + ZSTDcrp_leaveDirty +} ZSTD_compResetPolicy_e; -typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset } ZSTD_compResetPolicy_e; +/** + * Controls, for this matchState reset, whether indexing can continue where it + * left off (ZSTDirp_continue), or whether it needs to be restarted from zero + * (ZSTDirp_reset). + */ +typedef enum { + ZSTDirp_continue, + ZSTDirp_reset +} ZSTD_indexResetPolicy_e; -typedef enum { ZSTD_resetTarget_CDict, ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; +typedef enum { + ZSTD_resetTarget_CDict, + ZSTD_resetTarget_CCtx +} ZSTD_resetTarget_e; -static void* +static size_t ZSTD_reset_matchState(ZSTD_matchState_t* ms, - void* ptr, + ZSTD_cwksp* ws, const ZSTD_compressionParameters* cParams, - ZSTD_compResetPolicy_e const crp, ZSTD_resetTarget_e const forWho) + const ZSTD_compResetPolicy_e crp, + const ZSTD_indexResetPolicy_e forceResetIndex, + const ZSTD_resetTarget_e forWho) { size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; - size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - - assert(((size_t)ptr & 3) == 0); + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + + DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); + if (forceResetIndex == ZSTDirp_reset) { + memset(&ms->window, 0, sizeof(ms->window)); + ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ + ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ + ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ + ZSTD_cwksp_mark_tables_dirty(ws); + } ms->hashLog3 = hashLog3; - memset(&ms->window, 0, sizeof(ms->window)); - ms->window.dictLimit = 1; /* start from 1, so that 1st position is valid */ - ms->window.lowLimit = 1; /* it ensures first and later CCtx usages compress the same */ - ms->window.nextSrc = ms->window.base + 1; /* see issue #1241 */ + ZSTD_invalidateMatchState(ms); + assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ + + ZSTD_cwksp_clear_tables(ws); + + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); + if (crp!=ZSTDcrp_leaveDirty) { + /* reset tables only */ + ZSTD_cwksp_clean_tables(ws); + } + /* opt parser space */ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); - ms->opt.litFreq = (unsigned*)ptr; - ms->opt.litLengthFreq = ms->opt.litFreq + (1<<Litbits); - ms->opt.matchLengthFreq = ms->opt.litLengthFreq + (MaxLL+1); - ms->opt.offCodeFreq = ms->opt.matchLengthFreq + (MaxML+1); - ptr = ms->opt.offCodeFreq + (MaxOff+1); - ms->opt.matchTable = (ZSTD_match_t*)ptr; - ptr = ms->opt.matchTable + ZSTD_OPT_NUM+1; - ms->opt.priceTable = (ZSTD_optimal_t*)ptr; - ptr = ms->opt.priceTable + ZSTD_OPT_NUM+1; + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<<Litbits) * sizeof(unsigned)); + ms->opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); } - /* table Space */ - DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_noMemset); - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - if (crp!=ZSTDcrp_noMemset) memset(ptr, 0, tableSpace); /* reset tables only */ - ms->hashTable = (U32*)(ptr); - ms->chainTable = ms->hashTable + hSize; - ms->hashTable3 = ms->chainTable + chainSize; - ptr = ms->hashTable3 + h3Size; - ms->cParams = *cParams; - assert(((size_t)ptr & 3) == 0); - return ptr; + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + return 0; } /* ZSTD_indexTooCloseToMax() : @@ -1407,13 +1381,6 @@ static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); } -#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 /* define "workspace is too large" as this number of times larger than needed */ -#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 /* when workspace is continuously too large - * during at least this number of times, - * context's memory usage is considered wasteful, - * because it's sized to handle a worst case scenario which rarely happens. - * In which case, resize it down to free some memory */ - /*! ZSTD_resetCCtx_internal() : note : `params` are assumed fully validated at this stage */ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, @@ -1422,30 +1389,12 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_compResetPolicy_e const crp, ZSTD_buffered_policy_e const zbuff) { + ZSTD_cwksp* const ws = &zc->workspace; DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u", (U32)pledgedSrcSize, params.cParams.windowLog); assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - if (crp == ZSTDcrp_continue) { - if (ZSTD_equivalentParams(zc->appliedParams, params, - zc->inBuffSize, - zc->seqStore.maxNbSeq, zc->seqStore.maxNbLit, - zbuff, pledgedSrcSize) ) { - DEBUGLOG(4, "ZSTD_equivalentParams()==1 -> consider continue mode"); - zc->workSpaceOversizedDuration += (zc->workSpaceOversizedDuration > 0); /* if it was too large, it still is */ - if (zc->workSpaceOversizedDuration <= ZSTD_WORKSPACETOOLARGE_MAXDURATION) { - DEBUGLOG(4, "continue mode confirmed (wLog1=%u, blockSize1=%zu)", - zc->appliedParams.cParams.windowLog, zc->blockSize); - if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { - /* prefer a reset, faster than a rescale */ - ZSTD_reset_matchState(&zc->blockState.matchState, - zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, - ¶ms.cParams, - crp, ZSTD_resetTarget_CCtx); - } - return ZSTD_continueCCtx(zc, params, pledgedSrcSize); - } } } - DEBUGLOG(4, "ZSTD_equivalentParams()==0 -> reset CCtx"); + zc->isFirstBlock = 1; if (params.ldmParams.enableLdm) { /* Adjust long distance matching parameters */ @@ -1459,58 +1408,74 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; size_t const buffInSize = (zbuff==ZSTDb_buffered) ? windowSize + blockSize : 0; size_t const matchStateSize = ZSTD_sizeof_matchState(¶ms.cParams, /* forCCtx */ 1); size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params.ldmParams, blockSize); - void* ptr; /* used to partition workSpace */ - /* Check if workSpace is large enough, alloc a new one if needed */ - { size_t const entropySpace = HUF_WORKSPACE_SIZE; - size_t const blockStateSpace = 2 * sizeof(ZSTD_compressedBlockState_t); - size_t const bufferSpace = buffInSize + buffOutSize; + ZSTD_indexResetPolicy_e needsIndexReset = ZSTDirp_continue; + + if (ZSTD_indexTooCloseToMax(zc->blockState.matchState.window)) { + needsIndexReset = ZSTDirp_reset; + } + + ZSTD_cwksp_bump_oversized_duration(ws, 0); + + /* Check if workspace is large enough, alloc a new one if needed */ + { size_t const cctxSpace = zc->staticSize ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + size_t const entropySpace = ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); + size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + ZSTD_cwksp_alloc_size(buffOutSize); size_t const ldmSpace = ZSTD_ldm_getTableSize(params.ldmParams); - size_t const ldmSeqSpace = maxNbLdmSeq * sizeof(rawSeq); + size_t const ldmSeqSpace = ZSTD_cwksp_alloc_size(maxNbLdmSeq * sizeof(rawSeq)); - size_t const neededSpace = entropySpace + blockStateSpace + ldmSpace + - ldmSeqSpace + matchStateSize + tokenSpace + - bufferSpace; + size_t const neededSpace = + cctxSpace + + entropySpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace; - int const workSpaceTooSmall = zc->workSpaceSize < neededSpace; - int const workSpaceTooLarge = zc->workSpaceSize > ZSTD_WORKSPACETOOLARGE_FACTOR * neededSpace; - int const workSpaceWasteful = workSpaceTooLarge && (zc->workSpaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION); - zc->workSpaceOversizedDuration = workSpaceTooLarge ? zc->workSpaceOversizedDuration+1 : 0; + int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); DEBUGLOG(4, "Need %zuKB workspace, including %zuKB for match state, and %zuKB for buffers", neededSpace>>10, matchStateSize>>10, bufferSpace>>10); DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); - if (workSpaceTooSmall || workSpaceWasteful) { - DEBUGLOG(4, "Resize workSpaceSize from %zuKB to %zuKB", - zc->workSpaceSize >> 10, + if (workspaceTooSmall || workspaceWasteful) { + DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", + ZSTD_cwksp_sizeof(ws) >> 10, neededSpace >> 10); RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); - zc->workSpaceSize = 0; - ZSTD_free(zc->workSpace, zc->customMem); - zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); - RETURN_ERROR_IF(zc->workSpace == NULL, memory_allocation); - zc->workSpaceSize = neededSpace; - zc->workSpaceOversizedDuration = 0; + needsIndexReset = ZSTDirp_reset; + + ZSTD_cwksp_free(ws, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem)); + DEBUGLOG(5, "reserving object space"); /* Statically sized space. * entropyWorkspace never moves, * though prev/next block swap places */ - assert(((size_t)zc->workSpace & 3) == 0); /* ensure correct alignment */ - assert(zc->workSpaceSize >= 2 * sizeof(ZSTD_compressedBlockState_t)); - zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)zc->workSpace; - zc->blockState.nextCBlock = zc->blockState.prevCBlock + 1; - ptr = zc->blockState.nextCBlock + 1; - zc->entropyWorkspace = (U32*)ptr; + assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, HUF_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); } } + ZSTD_cwksp_clear(ws); + /* init params */ zc->appliedParams = params; zc->blockState.matchState.cParams = params.cParams; @@ -1529,58 +1494,58 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); - ptr = ZSTD_reset_matchState(&zc->blockState.matchState, - zc->entropyWorkspace + HUF_WORKSPACE_SIZE_U32, - ¶ms.cParams, - crp, ZSTD_resetTarget_CCtx); - - /* ldm hash table */ - /* initialize bucketOffsets table later for pointer alignment */ - if (params.ldmParams.enableLdm) { - size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; - memset(ptr, 0, ldmHSize * sizeof(ldmEntry_t)); - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - zc->ldmState.hashTable = (ldmEntry_t*)ptr; - ptr = zc->ldmState.hashTable + ldmHSize; - zc->ldmSequences = (rawSeq*)ptr; - ptr = zc->ldmSequences + maxNbLdmSeq; - zc->maxNbLdmSequences = maxNbLdmSeq; - - memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); - } - assert(((size_t)ptr & 3) == 0); /* ensure ptr is properly aligned */ - - /* sequences storage */ - zc->seqStore.maxNbSeq = maxNbSeq; - zc->seqStore.sequencesStart = (seqDef*)ptr; - ptr = zc->seqStore.sequencesStart + maxNbSeq; - zc->seqStore.llCode = (BYTE*) ptr; - zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; - zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; - zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; /* ZSTD_wildcopy() is used to copy into the literals buffer, * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. */ + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); zc->seqStore.maxNbLit = blockSize; - ptr = zc->seqStore.litStart + blockSize + WILDCOPY_OVERLENGTH; + + /* buffers */ + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); + zc->outBuffSize = buffOutSize; + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); /* ldm bucketOffsets table */ if (params.ldmParams.enableLdm) { + /* TODO: avoid memset? */ size_t const ldmBucketSize = ((size_t)1) << (params.ldmParams.hashLog - params.ldmParams.bucketSizeLog); - memset(ptr, 0, ldmBucketSize); - zc->ldmState.bucketOffsets = (BYTE*)ptr; - ptr = zc->ldmState.bucketOffsets + ldmBucketSize; - ZSTD_window_clear(&zc->ldmState.window); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, ldmBucketSize); + memset(zc->ldmState.bucketOffsets, 0, ldmBucketSize); } + + /* sequences storage */ ZSTD_referenceExternalSequences(zc, NULL, 0); + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); + + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + ws, + ¶ms.cParams, + crp, + needsIndexReset, + ZSTD_resetTarget_CCtx)); - /* buffers */ - zc->inBuffSize = buffInSize; - zc->inBuff = (char*)ptr; - zc->outBuffSize = buffOutSize; - zc->outBuff = zc->inBuff + buffInSize; + /* ldm hash table */ + if (params.ldmParams.enableLdm) { + /* TODO: avoid memset? */ + size_t const ldmHSize = ((size_t)1) << params.ldmParams.hashLog; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); + memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); + zc->maxNbLdmSequences = maxNbLdmSeq; + + memset(&zc->ldmState.window, 0, sizeof(zc->ldmState.window)); + ZSTD_window_clear(&zc->ldmState.window); + } + + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); return 0; } @@ -1614,15 +1579,15 @@ static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { }; static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize) { size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; return ( pledgedSrcSize <= cutoff || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN - || params.attachDictPref == ZSTD_dictForceAttach ) - && params.attachDictPref != ZSTD_dictForceCopy - && !params.forceWindow; /* dictMatchState isn't correctly + || params->attachDictPref == ZSTD_dictForceAttach ) + && params->attachDictPref != ZSTD_dictForceCopy + && !params->forceWindow; /* dictMatchState isn't correctly * handled in _enforceMaxDist */ } @@ -1640,8 +1605,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, * has its own tables. */ params.cParams = ZSTD_adjustCParams_internal(*cdict_cParams, pledgedSrcSize, 0); params.cParams.windowLog = windowLog; - ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_continue, zbuff); + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_makeClean, zbuff)); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); } @@ -1689,30 +1654,36 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, /* Copy only compression parameters related to tables. */ params.cParams = *cdict_cParams; params.cParams.windowLog = windowLog; - ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_noMemset, zbuff); + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + ZSTDcrp_leaveDirty, zbuff)); assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); } + ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); + /* copy tables */ { size_t const chainSize = (cdict_cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cdict_cParams->chainLog); size_t const hSize = (size_t)1 << cdict_cParams->hashLog; - size_t const tableSpace = (chainSize + hSize) * sizeof(U32); - assert((U32*)cctx->blockState.matchState.chainTable == (U32*)cctx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)cctx->blockState.matchState.hashTable3 == (U32*)cctx->blockState.matchState.chainTable + chainSize); - assert((U32*)cdict->matchState.chainTable == (U32*)cdict->matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)cdict->matchState.hashTable3 == (U32*)cdict->matchState.chainTable + chainSize); - memcpy(cctx->blockState.matchState.hashTable, cdict->matchState.hashTable, tableSpace); /* presumes all tables follow each other */ + + memcpy(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize * sizeof(U32)); + memcpy(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize * sizeof(U32)); } /* Zero the hashTable3, since the cdict never fills it */ - { size_t const h3Size = (size_t)1 << cctx->blockState.matchState.hashLog3; + { int const h3log = cctx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; assert(cdict->matchState.hashLog3 == 0); memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); } + ZSTD_cwksp_mark_tables_clean(&cctx->workspace); + /* copy dictionary offsets */ { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; @@ -1734,7 +1705,7 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, * in-place. We decide here which strategy to use. */ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { @@ -1744,10 +1715,10 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { return ZSTD_resetCCtx_byAttachingCDict( - cctx, cdict, params, pledgedSrcSize, zbuff); + cctx, cdict, *params, pledgedSrcSize, zbuff); } else { return ZSTD_resetCCtx_byCopyingCDict( - cctx, cdict, params, pledgedSrcSize, zbuff); + cctx, cdict, *params, pledgedSrcSize, zbuff); } } @@ -1773,7 +1744,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, params.cParams = srcCCtx->appliedParams.cParams; params.fParams = fParams; ZSTD_resetCCtx_internal(dstCCtx, params, pledgedSrcSize, - ZSTDcrp_noMemset, zbuff); + ZSTDcrp_leaveDirty, zbuff); assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); @@ -1781,16 +1752,27 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); } + ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); + /* copy tables */ { size_t const chainSize = (srcCCtx->appliedParams.cParams.strategy == ZSTD_fast) ? 0 : ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog); size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; - size_t const h3Size = (size_t)1 << srcCCtx->blockState.matchState.hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - assert((U32*)dstCCtx->blockState.matchState.chainTable == (U32*)dstCCtx->blockState.matchState.hashTable + hSize); /* chainTable must follow hashTable */ - assert((U32*)dstCCtx->blockState.matchState.hashTable3 == (U32*)dstCCtx->blockState.matchState.chainTable + chainSize); - memcpy(dstCCtx->blockState.matchState.hashTable, srcCCtx->blockState.matchState.hashTable, tableSpace); /* presumes all tables follow each other */ + int const h3log = srcCCtx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + + memcpy(dstCCtx->blockState.matchState.hashTable, + srcCCtx->blockState.matchState.hashTable, + hSize * sizeof(U32)); + memcpy(dstCCtx->blockState.matchState.chainTable, + srcCCtx->blockState.matchState.chainTable, + chainSize * sizeof(U32)); + memcpy(dstCCtx->blockState.matchState.hashTable3, + srcCCtx->blockState.matchState.hashTable3, + h3Size * sizeof(U32)); } + ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); + /* copy dictionary offsets */ { const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; @@ -1841,6 +1823,20 @@ ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerVa int rowNb; assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ assert(size < (1U<<31)); /* can be casted to int */ + +#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table re-use logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * + * This function however is intended to operate on those dirty tables and + * re-clean them. So when this function is used correctly, we can unpoison + * the memory it operated on. This introduces a blind spot though, since + * if we now try to operate on __actually__ poisoned memory, we will not + * detect that. */ + __msan_unpoison(table, size * sizeof(U32)); +#endif + for (rowNb=0 ; rowNb < nbRows ; rowNb++) { int column; for (column=0; column<ZSTD_ROWSIZE; column++) { @@ -1903,155 +1899,6 @@ static size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* s return ZSTD_blockHeaderSize + srcSize; } -static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - BYTE* const ostart = (BYTE* const)dst; - U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); - - RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall); - - switch(flSize) - { - case 1: /* 2 - 1 - 5 */ - ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); - break; - case 2: /* 2 - 2 - 12 */ - MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); - break; - case 3: /* 2 - 2 - 20 */ - MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); - break; - default: /* not necessary : flSize is {1,2,3} */ - assert(0); - } - - memcpy(ostart + flSize, src, srcSize); - return srcSize + flSize; -} - -static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - BYTE* const ostart = (BYTE* const)dst; - U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); - - (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ - - switch(flSize) - { - case 1: /* 2 - 1 - 5 */ - ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); - break; - case 2: /* 2 - 2 - 12 */ - MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); - break; - case 3: /* 2 - 2 - 20 */ - MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); - break; - default: /* not necessary : flSize is {1,2,3} */ - assert(0); - } - - ostart[flSize] = *(const BYTE*)src; - return flSize+1; -} - - -/* ZSTD_minGain() : - * minimum compression required - * to generate a compress block or a compressed literals section. - * note : use same formula for both situations */ -static size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) -{ - U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; - ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); - assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); - return (srcSize >> minlog) + 2; -} - -static size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_strategy strategy, int disableLiteralCompression, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - void* workspace, size_t wkspSize, - const int bmi2) -{ - size_t const minGain = ZSTD_minGain(srcSize, strategy); - size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); - BYTE* const ostart = (BYTE*)dst; - U32 singleStream = srcSize < 256; - symbolEncodingType_e hType = set_compressed; - size_t cLitSize; - - DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i)", - disableLiteralCompression); - - /* Prepare nextEntropy assuming reusing the existing table */ - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - - if (disableLiteralCompression) - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - - /* small ? don't even attempt compression (speed opt) */ -# define COMPRESS_LITERALS_SIZE_MIN 63 - { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; - if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } - - RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); - { HUF_repeat repeat = prevHuf->repeatMode; - int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; - if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; - cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) - : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); - if (repeat != HUF_repeat_none) { - /* reused the existing table */ - hType = set_repeat; - } - } - - if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) { - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } - if (cLitSize==1) { - memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); - } - - if (hType == set_compressed) { - /* using a newly constructed table */ - nextHuf->repeatMode = HUF_repeat_check; - } - - /* Build header */ - switch(lhSize) - { - case 3: /* 2 - 2 - 10 - 10 */ - { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); - MEM_writeLE24(ostart, lhc); - break; - } - case 4: /* 2 - 2 - 14 - 14 */ - { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); - MEM_writeLE32(ostart, lhc); - break; - } - case 5: /* 2 - 2 - 18 - 18 */ - { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); - MEM_writeLE32(ostart, lhc); - ostart[4] = (BYTE)(cLitSize >> 10); - break; - } - default: /* not possible : lhSize is {3,4,5} */ - assert(0); - } - return lhSize+cLitSize; -} - - void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) { const seqDef* const sequences = seqStorePtr->sequencesStart; @@ -2074,418 +1921,6 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) mlCodeTable[seqStorePtr->longLengthPos] = MaxML; } - -/** - * -log2(x / 256) lookup table for x in [0, 256). - * If x == 0: Return 0 - * Else: Return floor(-log2(x / 256) * 256) - */ -static unsigned const kInverseProbabilityLog256[256] = { - 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, - 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, - 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, - 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, - 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, - 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, - 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, - 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, - 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, - 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, - 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, - 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, - 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, - 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, - 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, - 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, - 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, - 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, - 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, - 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, - 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, - 5, 4, 2, 1, -}; - - -/** - * Returns the cost in bits of encoding the distribution described by count - * using the entropy bound. - */ -static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) -{ - unsigned cost = 0; - unsigned s; - for (s = 0; s <= max; ++s) { - unsigned norm = (unsigned)((256 * count[s]) / total); - if (count[s] != 0 && norm == 0) - norm = 1; - assert(count[s] < total); - cost += count[s] * kInverseProbabilityLog256[norm]; - } - return cost >> 8; -} - - -/** - * Returns the cost in bits of encoding the distribution in count using the - * table described by norm. The max symbol support by norm is assumed >= max. - * norm must be valid for every symbol with non-zero probability in count. - */ -static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, - unsigned const* count, unsigned const max) -{ - unsigned const shift = 8 - accuracyLog; - size_t cost = 0; - unsigned s; - assert(accuracyLog <= 8); - for (s = 0; s <= max; ++s) { - unsigned const normAcc = norm[s] != -1 ? norm[s] : 1; - unsigned const norm256 = normAcc << shift; - assert(norm256 > 0); - assert(norm256 < 256); - cost += count[s] * kInverseProbabilityLog256[norm256]; - } - return cost >> 8; -} - - -static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { - void const* ptr = ctable; - U16 const* u16ptr = (U16 const*)ptr; - U32 const maxSymbolValue = MEM_read16(u16ptr + 1); - return maxSymbolValue; -} - - -/** - * Returns the cost in bits of encoding the distribution in count using ctable. - * Returns an error if ctable cannot represent all the symbols in count. - */ -static size_t ZSTD_fseBitCost( - FSE_CTable const* ctable, - unsigned const* count, - unsigned const max) -{ - unsigned const kAccuracyLog = 8; - size_t cost = 0; - unsigned s; - FSE_CState_t cstate; - FSE_initCState(&cstate, ctable); - RETURN_ERROR_IF(ZSTD_getFSEMaxSymbolValue(ctable) < max, GENERIC, - "Repeat FSE_CTable has maxSymbolValue %u < %u", - ZSTD_getFSEMaxSymbolValue(ctable), max); - for (s = 0; s <= max; ++s) { - unsigned const tableLog = cstate.stateLog; - unsigned const badCost = (tableLog + 1) << kAccuracyLog; - unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); - if (count[s] == 0) - continue; - RETURN_ERROR_IF(bitCost >= badCost, GENERIC, - "Repeat FSE_CTable has Prob[%u] == 0", s); - cost += count[s] * bitCost; - } - return cost >> kAccuracyLog; -} - -/** - * Returns the cost in bytes of encoding the normalized count header. - * Returns an error if any of the helper functions return an error. - */ -static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, - size_t const nbSeq, unsigned const FSELog) -{ - BYTE wksp[FSE_NCOUNTBOUND]; - S16 norm[MaxSeq + 1]; - const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max)); - return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); -} - - -typedef enum { - ZSTD_defaultDisallowed = 0, - ZSTD_defaultAllowed = 1 -} ZSTD_defaultPolicy_e; - -MEM_STATIC symbolEncodingType_e -ZSTD_selectEncodingType( - FSE_repeat* repeatMode, unsigned const* count, unsigned const max, - size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, - FSE_CTable const* prevCTable, - short const* defaultNorm, U32 defaultNormLog, - ZSTD_defaultPolicy_e const isDefaultAllowed, - ZSTD_strategy const strategy) -{ - ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); - if (mostFrequent == nbSeq) { - *repeatMode = FSE_repeat_none; - if (isDefaultAllowed && nbSeq <= 2) { - /* Prefer set_basic over set_rle when there are 2 or less symbols, - * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. - * If basic encoding isn't possible, always choose RLE. - */ - DEBUGLOG(5, "Selected set_basic"); - return set_basic; - } - DEBUGLOG(5, "Selected set_rle"); - return set_rle; - } - if (strategy < ZSTD_lazy) { - if (isDefaultAllowed) { - size_t const staticFse_nbSeq_max = 1000; - size_t const mult = 10 - strategy; - size_t const baseLog = 3; - size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ - assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ - assert(mult <= 9 && mult >= 7); - if ( (*repeatMode == FSE_repeat_valid) - && (nbSeq < staticFse_nbSeq_max) ) { - DEBUGLOG(5, "Selected set_repeat"); - return set_repeat; - } - if ( (nbSeq < dynamicFse_nbSeq_min) - || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { - DEBUGLOG(5, "Selected set_basic"); - /* The format allows default tables to be repeated, but it isn't useful. - * When using simple heuristics to select encoding type, we don't want - * to confuse these tables with dictionaries. When running more careful - * analysis, we don't need to waste time checking both repeating tables - * and default tables. - */ - *repeatMode = FSE_repeat_none; - return set_basic; - } - } - } else { - size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); - size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); - size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); - size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); - - if (isDefaultAllowed) { - assert(!ZSTD_isError(basicCost)); - assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); - } - assert(!ZSTD_isError(NCountCost)); - assert(compressedCost < ERROR(maxCode)); - DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", - (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); - if (basicCost <= repeatCost && basicCost <= compressedCost) { - DEBUGLOG(5, "Selected set_basic"); - assert(isDefaultAllowed); - *repeatMode = FSE_repeat_none; - return set_basic; - } - if (repeatCost <= compressedCost) { - DEBUGLOG(5, "Selected set_repeat"); - assert(!ZSTD_isError(repeatCost)); - return set_repeat; - } - assert(compressedCost < basicCost && compressedCost < repeatCost); - } - DEBUGLOG(5, "Selected set_compressed"); - *repeatMode = FSE_repeat_check; - return set_compressed; -} - -MEM_STATIC size_t -ZSTD_buildCTable(void* dst, size_t dstCapacity, - FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, - unsigned* count, U32 max, - const BYTE* codeTable, size_t nbSeq, - const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, - const FSE_CTable* prevCTable, size_t prevCTableSize, - void* workspace, size_t workspaceSize) -{ - BYTE* op = (BYTE*)dst; - const BYTE* const oend = op + dstCapacity; - DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); - - switch (type) { - case set_rle: - FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max)); - RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall); - *op = codeTable[0]; - return 1; - case set_repeat: - memcpy(nextCTable, prevCTable, prevCTableSize); - return 0; - case set_basic: - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, workspace, workspaceSize)); /* note : could be pre-calculated */ - return 0; - case set_compressed: { - S16 norm[MaxSeq + 1]; - size_t nbSeq_1 = nbSeq; - const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); - if (count[codeTable[nbSeq-1]] > 1) { - count[codeTable[nbSeq-1]]--; - nbSeq_1--; - } - assert(nbSeq_1 > 1); - FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); - { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ - FORWARD_IF_ERROR(NCountSize); - FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, workspace, workspaceSize)); - return NCountSize; - } - } - default: assert(0); RETURN_ERROR(GENERIC); - } -} - -FORCE_INLINE_TEMPLATE size_t -ZSTD_encodeSequences_body( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) -{ - BIT_CStream_t blockStream; - FSE_CState_t stateMatchLength; - FSE_CState_t stateOffsetBits; - FSE_CState_t stateLitLength; - - RETURN_ERROR_IF( - ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), - dstSize_tooSmall, "not enough space remaining"); - DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", - (int)(blockStream.endPtr - blockStream.startPtr), - (unsigned)dstCapacity); - - /* first symbols */ - FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); - FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); - FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); - BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); - if (MEM_32bits()) BIT_flushBits(&blockStream); - if (longOffsets) { - U32 const ofBits = ofCodeTable[nbSeq-1]; - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); - BIT_flushBits(&blockStream); - } - BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, - ofBits - extraBits); - } else { - BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); - } - BIT_flushBits(&blockStream); - - { size_t n; - for (n=nbSeq-2 ; n<nbSeq ; n--) { /* intentional underflow */ - BYTE const llCode = llCodeTable[n]; - BYTE const ofCode = ofCodeTable[n]; - BYTE const mlCode = mlCodeTable[n]; - U32 const llBits = LL_bits[llCode]; - U32 const ofBits = ofCode; - U32 const mlBits = ML_bits[mlCode]; - DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u", - (unsigned)sequences[n].litLength, - (unsigned)sequences[n].matchLength + MINMATCH, - (unsigned)sequences[n].offset); - /* 32b*/ /* 64b*/ - /* (7)*/ /* (7)*/ - FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ - FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ - if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/ - FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */ - if (MEM_32bits() || (ofBits+mlBits+llBits >= 64-7-(LLFSELog+MLFSELog+OffFSELog))) - BIT_flushBits(&blockStream); /* (7)*/ - BIT_addBits(&blockStream, sequences[n].litLength, llBits); - if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); - BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); - if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); - if (longOffsets) { - int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); - if (extraBits) { - BIT_addBits(&blockStream, sequences[n].offset, extraBits); - BIT_flushBits(&blockStream); /* (7)*/ - } - BIT_addBits(&blockStream, sequences[n].offset >> extraBits, - ofBits - extraBits); /* 31 */ - } else { - BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ - } - BIT_flushBits(&blockStream); /* (7)*/ - DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); - } } - - DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); - FSE_flushCState(&blockStream, &stateMatchLength); - DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); - FSE_flushCState(&blockStream, &stateOffsetBits); - DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); - FSE_flushCState(&blockStream, &stateLitLength); - - { size_t const streamSize = BIT_closeCStream(&blockStream); - RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); - return streamSize; - } -} - -static size_t -ZSTD_encodeSequences_default( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) -{ - return ZSTD_encodeSequences_body(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); -} - - -#if DYNAMIC_BMI2 - -static TARGET_ATTRIBUTE("bmi2") size_t -ZSTD_encodeSequences_bmi2( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets) -{ - return ZSTD_encodeSequences_body(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); -} - -#endif - -static size_t ZSTD_encodeSequences( - void* dst, size_t dstCapacity, - FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, - FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, - FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, - seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) -{ - DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); -#if DYNAMIC_BMI2 - if (bmi2) { - return ZSTD_encodeSequences_bmi2(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); - } -#endif - (void)bmi2; - return ZSTD_encodeSequences_default(dst, dstCapacity, - CTable_MatchLength, mlCodeTable, - CTable_OffsetBits, ofCodeTable, - CTable_LitLength, llCodeTable, - sequences, nbSeq, longOffsets); -} - static int ZSTD_disableLiteralsCompression(const ZSTD_CCtx_params* cctxParams) { switch (cctxParams->literalCompressionMode) { @@ -2509,7 +1944,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_entropyCTables_t* nextEntropy, const ZSTD_CCtx_params* cctxParams, void* dst, size_t dstCapacity, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWkspSize, const int bmi2) { const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; @@ -2526,23 +1961,23 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, BYTE* const ostart = (BYTE*)dst; BYTE* const oend = ostart + dstCapacity; BYTE* op = ostart; - size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); BYTE* seqHead; BYTE* lastNCount = NULL; + DEBUGLOG(5, "ZSTD_compressSequences_internal (nbSeq=%zu)", nbSeq); ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); - DEBUGLOG(5, "ZSTD_compressSequences_internal"); /* Compress literals */ { const BYTE* const literals = seqStorePtr->litStart; - size_t const litSize = seqStorePtr->lit - literals; + size_t const litSize = (size_t)(seqStorePtr->lit - literals); size_t const cSize = ZSTD_compressLiterals( &prevEntropy->huf, &nextEntropy->huf, cctxParams->cParams.strategy, ZSTD_disableLiteralsCompression(cctxParams), op, dstCapacity, literals, litSize, - workspace, wkspSize, + entropyWorkspace, entropyWkspSize, bmi2); FORWARD_IF_ERROR(cSize); assert(cSize <= dstCapacity); @@ -2552,17 +1987,22 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, /* Sequences Header */ RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, dstSize_tooSmall); - if (nbSeq < 0x7F) + if (nbSeq < 128) { *op++ = (BYTE)nbSeq; - else if (nbSeq < LONGNBSEQ) - op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; - else - op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + } else if (nbSeq < LONGNBSEQ) { + op[0] = (BYTE)((nbSeq>>8) + 0x80); + op[1] = (BYTE)nbSeq; + op+=2; + } else { + op[0]=0xFF; + MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); + op+=3; + } assert(op <= oend); if (nbSeq==0) { /* Copy the old tables over as if we repeated them */ memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); - return op - ostart; + return (size_t)(op - ostart); } /* seqHead : flags for FSE encoding type */ @@ -2573,7 +2013,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_seqToCodes(seqStorePtr); /* build CTable for Literal Lengths */ { unsigned max = MaxLL; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ DEBUGLOG(5, "Building LL table"); nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, @@ -2583,10 +2023,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ZSTD_defaultAllowed, strategy); assert(set_basic < set_compressed && set_rle < set_compressed); assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, - count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, - prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, + count, max, llCodeTable, nbSeq, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->fse.litlengthCTable, + sizeof(prevEntropy->fse.litlengthCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (LLtype == set_compressed) lastNCount = op; @@ -2595,7 +2039,8 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } } /* build CTable for Offsets */ { unsigned max = MaxOff; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp( + count, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; DEBUGLOG(5, "Building OF table"); @@ -2606,10 +2051,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, OF_defaultNorm, OF_defaultNormLog, defaultPolicy, strategy); assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, - count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, + count, max, ofCodeTable, nbSeq, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->fse.offcodeCTable, + sizeof(prevEntropy->fse.offcodeCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (Offtype == set_compressed) lastNCount = op; @@ -2618,7 +2067,8 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } } /* build CTable for MatchLengths */ { unsigned max = MaxML; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + size_t const mostFrequent = HIST_countFast_wksp( + count, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, @@ -2627,10 +2077,14 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, ML_defaultNorm, ML_defaultNormLog, ZSTD_defaultAllowed, strategy); assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ - { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, - count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, - prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), - workspace, wkspSize); + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, + count, max, mlCodeTable, nbSeq, + ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->fse.matchlengthCTable, + sizeof(prevEntropy->fse.matchlengthCTable), + entropyWorkspace, entropyWkspSize); FORWARD_IF_ERROR(countSize); if (MLtype == set_compressed) lastNCount = op; @@ -2641,7 +2095,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); { size_t const bitstreamSize = ZSTD_encodeSequences( - op, oend - op, + op, (size_t)(oend - op), CTable_MatchLength, mlCodeTable, CTable_OffsetBits, ofCodeTable, CTable_LitLength, llCodeTable, @@ -2668,7 +2122,7 @@ ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); - return op - ostart; + return (size_t)(op - ostart); } MEM_STATIC size_t @@ -2678,13 +2132,13 @@ ZSTD_compressSequences(seqStore_t* seqStorePtr, const ZSTD_CCtx_params* cctxParams, void* dst, size_t dstCapacity, size_t srcSize, - void* workspace, size_t wkspSize, + void* entropyWorkspace, size_t entropyWkspSize, int bmi2) { size_t const cSize = ZSTD_compressSequences_internal( seqStorePtr, prevEntropy, nextEntropy, cctxParams, dst, dstCapacity, - workspace, wkspSize, bmi2); + entropyWorkspace, entropyWkspSize, bmi2); if (cSize == 0) return 0; /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. @@ -2835,19 +2289,113 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) return ZSTDbss_compress; } +static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) +{ + const seqStore_t* seqStore = ZSTD_getSeqStore(zc); + const seqDef* seqs = seqStore->sequencesStart; + size_t seqsSize = seqStore->sequences - seqs; + + ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; + size_t i; size_t position; int repIdx; + + assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); + for (i = 0, position = 0; i < seqsSize; ++i) { + outSeqs[i].offset = seqs[i].offset; + outSeqs[i].litLength = seqs[i].litLength; + outSeqs[i].matchLength = seqs[i].matchLength + MINMATCH; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthID == 1) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthID == 2) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (outSeqs[i].offset <= ZSTD_REP_NUM) { + outSeqs[i].rep = outSeqs[i].offset; + repIdx = (unsigned int)i - outSeqs[i].offset; + + if (outSeqs[i].litLength == 0) { + if (outSeqs[i].offset < 3) { + --repIdx; + } else { + repIdx = (unsigned int)i - 1; + } + ++outSeqs[i].rep; + } + assert(repIdx >= -3); + outSeqs[i].offset = repIdx >= 0 ? outSeqs[repIdx].offset : repStartValue[-repIdx - 1]; + if (outSeqs[i].rep == 4) { + --outSeqs[i].offset; + } + } else { + outSeqs[i].offset -= ZSTD_REP_NUM; + } + + position += outSeqs[i].litLength; + outSeqs[i].matchPos = (unsigned int)position; + position += outSeqs[i].matchLength; + } + zc->seqCollector.seqIndex += seqsSize; +} + +size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize) +{ + const size_t dstCapacity = ZSTD_compressBound(srcSize); + void* dst = ZSTD_malloc(dstCapacity, ZSTD_defaultCMem); + SeqCollector seqCollector; + + RETURN_ERROR_IF(dst == NULL, memory_allocation); + + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqIndex = 0; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_free(dst, ZSTD_defaultCMem); + return zc->seqCollector.seqIndex; +} + +/* Returns true if the given block is a RLE block */ +static int ZSTD_isRLE(const BYTE *ip, size_t length) { + size_t i; + if (length < 2) return 1; + for (i = 1; i < length; ++i) { + if (ip[0] != ip[i]) return 0; + } + return 1; +} + static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, - const void* src, size_t srcSize) + const void* src, size_t srcSize, U32 frame) { + /* This the upper bound for the length of an rle block. + * This isn't the actual upper bound. Finding the real threshold + * needs further investigation. + */ + const U32 rleMaxLength = 25; size_t cSize; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", - (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate); + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); FORWARD_IF_ERROR(bss); if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } } + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + return 0; + } + /* encode sequences and literals */ cSize = ZSTD_compressSequences(&zc->seqStore, &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, @@ -2857,8 +2405,21 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */, zc->bmi2); + if (frame && + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + cSize < rleMaxLength && + ZSTD_isRLE(ip, srcSize)) + { + cSize = 1; + op[0] = ip[0]; + } + out: - if (!ZSTD_isError(cSize) && cSize != 0) { + if (!ZSTD_isError(cSize) && cSize > 1) { /* confirm repcodes and entropy tables when emitting a compressed block */ ZSTD_compressedBlockState_t* const tmp = zc->blockState.prevCBlock; zc->blockState.prevCBlock = zc->blockState.nextCBlock; @@ -2875,7 +2436,11 @@ out: } -static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params const* params, void const* ip, void const* iend) +static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + void const* ip, + void const* iend) { if (ZSTD_window_needOverflowCorrection(ms->window, iend)) { U32 const maxDist = (U32)1 << params->cParams.windowLog; @@ -2884,7 +2449,9 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_cwksp_mark_tables_dirty(ws); ZSTD_reduceIndex(ms, params, correction); + ZSTD_cwksp_mark_tables_clean(ws); if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; else ms->nextToUpdate -= correction; /* invalidate dictionaries on overflow correction */ @@ -2893,7 +2460,6 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, ZSTD_CCtx_params } } - /*! ZSTD_compress_frameChunk() : * Compress a chunk of data into one or multiple blocks. * All blocks will be terminated, all input will be consumed. @@ -2927,7 +2493,8 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, "not enough space to store compressed block"); if (remaining < blockSize) blockSize = remaining; - ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, ip, ip + blockSize); + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ @@ -2935,15 +2502,16 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, { size_t cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, - ip, blockSize); + ip, blockSize, 1 /* frame */); FORWARD_IF_ERROR(cSize); - if (cSize == 0) { /* block is not compressible */ cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); FORWARD_IF_ERROR(cSize); } else { - U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); - MEM_writeLE24(op, cBlockHeader24); + const U32 cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); cSize += ZSTD_blockHeaderSize; } @@ -2953,6 +2521,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, op += cSize; assert(dstCapacity >= cSize); dstCapacity -= cSize; + cctx->isFirstBlock = 0; DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", (unsigned)cSize); } } @@ -2963,25 +2532,25 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, - ZSTD_CCtx_params params, U64 pledgedSrcSize, U32 dictID) + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) { BYTE* const op = (BYTE*)dst; U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ - U32 const dictIDSizeCode = params.fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ - U32 const checksumFlag = params.fParams.checksumFlag>0; - U32 const windowSize = (U32)1 << params.cParams.windowLog; - U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); - BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); - U32 const fcsCode = params.fParams.contentSizeFlag ? + U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ + U32 const checksumFlag = params->fParams.checksumFlag>0; + U32 const windowSize = (U32)1 << params->cParams.windowLog; + U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); + BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params->fParams.contentSizeFlag ? (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); size_t pos=0; - assert(!(params.fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); + assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall); DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", - !params.fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); + !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); - if (params.format == ZSTD_f_zstd1) { + if (params->format == ZSTD_f_zstd1) { MEM_writeLE32(dst, ZSTD_MAGICNUMBER); pos = 4; } @@ -3047,7 +2616,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, "missing init (ZSTD_compressBegin)"); if (frame && (cctx->stage==ZSTDcs_init)) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, cctx->pledgedSrcSizePlusOne-1, cctx->dictID); FORWARD_IF_ERROR(fhSize); assert(fhSize <= dstCapacity); @@ -3067,13 +2636,15 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (!frame) { /* overflow check and correction for block mode */ - ZSTD_overflowCorrectIfNeeded(ms, &cctx->appliedParams, src, (BYTE const*)src + srcSize); + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, + src, (BYTE const*)src + srcSize); } DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); { size_t const cSize = frame ? ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : - ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); + ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); FORWARD_IF_ERROR(cSize); cctx->consumedSrcSize += srcSize; cctx->producedCSize += (cSize + fhSize); @@ -3109,8 +2680,9 @@ size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - size_t const blockSizeMax = ZSTD_getBlockSize(cctx); - RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong); + DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); + { size_t const blockSizeMax = ZSTD_getBlockSize(cctx); + RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong); } return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); } @@ -3119,6 +2691,7 @@ size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const * @return : 0, or an error code */ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, const void* src, size_t srcSize, ZSTD_dictTableLoadMethod_e dtlm) @@ -3135,11 +2708,11 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, if (srcSize <= HASH_READ_SIZE) return 0; while (iend - ip > HASH_READ_SIZE) { - size_t const remaining = iend - ip; + size_t const remaining = (size_t)(iend - ip); size_t const chunk = MIN(remaining, ZSTD_CHUNKSIZE_MAX); const BYTE* const ichunk = ip + chunk; - ZSTD_overflowCorrectIfNeeded(ms, params, ip, ichunk); + ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, ichunk); switch(params->cParams.strategy) { @@ -3198,10 +2771,11 @@ static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSym /*! ZSTD_loadZstdDictionary() : * @return : dictID, or an error code * assumptions : magic number supposed already checked - * dictSize supposed > 8 + * dictSize supposed >= 8 */ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, const void* dict, size_t dictSize, ZSTD_dictTableLoadMethod_e dtlm, @@ -3214,7 +2788,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, size_t dictID; ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<<MAX(MLFSELog,LLFSELog))); - assert(dictSize > 8); + assert(dictSize >= 8); assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); dictPtr += 4; /* skip magic number */ @@ -3297,7 +2871,8 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, bs->entropy.fse.offcode_repeatMode = FSE_repeat_valid; bs->entropy.fse.matchlength_repeatMode = FSE_repeat_valid; bs->entropy.fse.litlength_repeatMode = FSE_repeat_valid; - FORWARD_IF_ERROR(ZSTD_loadDictionaryContent(ms, params, dictPtr, dictContentSize, dtlm)); + FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( + ms, ws, params, dictPtr, dictContentSize, dtlm)); return dictID; } } @@ -3307,6 +2882,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, static size_t ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, const ZSTD_CCtx_params* params, const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, @@ -3314,27 +2890,35 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, void* workspace) { DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); - if ((dict==NULL) || (dictSize<=8)) return 0; + if ((dict==NULL) || (dictSize<8)) { + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong); + return 0; + } ZSTD_reset_compressedBlockState(bs); /* dict restricted modes */ if (dictContentType == ZSTD_dct_rawContent) - return ZSTD_loadDictionaryContent(ms, params, dict, dictSize, dtlm); + return ZSTD_loadDictionaryContent(ms, ws, params, dict, dictSize, dtlm); if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { if (dictContentType == ZSTD_dct_auto) { DEBUGLOG(4, "raw content dictionary detected"); - return ZSTD_loadDictionaryContent(ms, params, dict, dictSize, dtlm); + return ZSTD_loadDictionaryContent( + ms, ws, params, dict, dictSize, dtlm); } RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong); assert(0); /* impossible */ } /* dict as full zstd dictionary */ - return ZSTD_loadZstdDictionary(bs, ms, params, dict, dictSize, dtlm, workspace); + return ZSTD_loadZstdDictionary( + bs, ms, ws, params, dict, dictSize, dtlm, workspace); } +#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) +#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6) + /*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, @@ -3342,23 +2926,34 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, U64 pledgedSrcSize, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, ZSTD_buffered_policy_e zbuff) { - DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params.cParams.windowLog); + DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); /* params are supposed to be fully validated at this point */ - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ - - if (cdict && cdict->dictContentSize>0) { + if ( (cdict) + && (cdict->dictContentSize > 0) + && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceLoad) ) { return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); } - FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, - ZSTDcrp_continue, zbuff) ); - { size_t const dictID = ZSTD_compress_insertDictionary( - cctx->blockState.prevCBlock, &cctx->blockState.matchState, - ¶ms, dict, dictSize, dictContentType, dtlm, cctx->entropyWorkspace); + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, *params, pledgedSrcSize, + ZSTDcrp_makeClean, zbuff) ); + { size_t const dictID = cdict ? + ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->workspace, params, cdict->dictContent, cdict->dictContentSize, + dictContentType, dtlm, cctx->entropyWorkspace) + : ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->workspace, params, dict, dictSize, + dictContentType, dtlm, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; @@ -3371,12 +2966,12 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize) { - DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params.cParams.windowLog); + DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); /* compression parameters verification and optimization */ - FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); + FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) ); return ZSTD_compressBegin_internal(cctx, dict, dictSize, dictContentType, dtlm, cdict, @@ -3391,21 +2986,21 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, ZSTD_parameters params, unsigned long long pledgedSrcSize) { ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); return ZSTD_compressBegin_advanced_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL /*cdict*/, - cctxParams, pledgedSrcSize); + &cctxParams, pledgedSrcSize); } size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, - cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); } size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) @@ -3428,7 +3023,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) /* special case : empty frame */ if (cctx->stage == ZSTDcs_init) { - fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->appliedParams, 0, 0); + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); FORWARD_IF_ERROR(fhSize); dstCapacity -= fhSize; op += fhSize; @@ -3489,13 +3084,13 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, ZSTD_parameters params) { ZSTD_CCtx_params const cctxParams = - ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); DEBUGLOG(4, "ZSTD_compress_internal"); return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, - cctxParams); + &cctxParams); } size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, @@ -3519,7 +3114,7 @@ size_t ZSTD_compress_advanced_internal( void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, - ZSTD_CCtx_params params) + const ZSTD_CCtx_params* params) { DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, @@ -3535,9 +3130,9 @@ size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, int compressionLevel) { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize + (!srcSize), dict ? dictSize : 0); - ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); + ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(&cctx->requestedParams, params); assert(params.fParams.contentSizeFlag == 1); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, cctxParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctxParams); } size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, @@ -3572,8 +3167,11 @@ size_t ZSTD_estimateCDictSize_advanced( ZSTD_dictLoadMethod_e dictLoadMethod) { DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); - return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); } size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) @@ -3586,7 +3184,9 @@ size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); - return cdict->workspaceSize + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); + /* cdict may be in the workspace */ + return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) + + ZSTD_cwksp_sizeof(&cdict->workspace); } static size_t ZSTD_initCDict_internal( @@ -3600,28 +3200,29 @@ static size_t ZSTD_initCDict_internal( assert(!ZSTD_checkCParams(cParams)); cdict->matchState.cParams = cParams; if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { - cdict->dictBuffer = NULL; cdict->dictContent = dictBuffer; } else { - void* const internalBuffer = ZSTD_malloc(dictSize, cdict->customMem); - cdict->dictBuffer = internalBuffer; - cdict->dictContent = internalBuffer; + void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); RETURN_ERROR_IF(!internalBuffer, memory_allocation); + cdict->dictContent = internalBuffer; memcpy(internalBuffer, dictBuffer, dictSize); } cdict->dictContentSize = dictSize; + cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); + + /* Reset the state to no dictionary */ ZSTD_reset_compressedBlockState(&cdict->cBlockState); - { void* const end = ZSTD_reset_matchState(&cdict->matchState, - (U32*)cdict->workspace + HUF_WORKSPACE_SIZE_U32, - &cParams, - ZSTDcrp_continue, ZSTD_resetTarget_CDict); - assert(end == (char*)cdict->workspace + cdict->workspaceSize); - (void)end; - } + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &cdict->matchState, + &cdict->workspace, + &cParams, + ZSTDcrp_makeClean, + ZSTDirp_reset, + ZSTD_resetTarget_CDict)); /* (Maybe) load the dictionary - * Skips loading the dictionary if it is <= 8 bytes. + * Skips loading the dictionary if it is < 8 bytes. */ { ZSTD_CCtx_params params; memset(¶ms, 0, sizeof(params)); @@ -3629,9 +3230,9 @@ static size_t ZSTD_initCDict_internal( params.fParams.contentSizeFlag = 1; params.cParams = cParams; { size_t const dictID = ZSTD_compress_insertDictionary( - &cdict->cBlockState, &cdict->matchState, ¶ms, - cdict->dictContent, cdict->dictContentSize, - dictContentType, ZSTD_dtlm_full, cdict->workspace); + &cdict->cBlockState, &cdict->matchState, &cdict->workspace, + ¶ms, cdict->dictContent, cdict->dictContentSize, + dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); FORWARD_IF_ERROR(dictID); assert(dictID <= (size_t)(U32)-1); cdict->dictID = (U32)dictID; @@ -3649,18 +3250,29 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (unsigned)dictContentType); if (!customMem.customAlloc ^ !customMem.customFree) return NULL; - { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); - size_t const workspaceSize = HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); + { size_t const workspaceSize = + ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); void* const workspace = ZSTD_malloc(workspaceSize, customMem); + ZSTD_cwksp ws; + ZSTD_CDict* cdict; - if (!cdict || !workspace) { - ZSTD_free(cdict, customMem); + if (!workspace) { ZSTD_free(workspace, customMem); return NULL; } + + ZSTD_cwksp_init(&ws, workspace, workspaceSize); + + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + assert(cdict != NULL); + ZSTD_cwksp_move(&cdict->workspace, &ws); cdict->customMem = customMem; - cdict->workspace = workspace; - cdict->workspaceSize = workspaceSize; + cdict->compressionLevel = 0; /* signals advanced API usage */ + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, dictLoadMethod, dictContentType, @@ -3676,9 +3288,12 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) { ZSTD_compressionParameters cParams = ZSTD_getCParams(compressionLevel, 0, dictSize); - return ZSTD_createCDict_advanced(dict, dictSize, - ZSTD_dlm_byCopy, ZSTD_dct_auto, - cParams, ZSTD_defaultCMem); + ZSTD_CDict* cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = compressionLevel == 0 ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; } ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) @@ -3693,9 +3308,11 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = cdict->customMem; - ZSTD_free(cdict->workspace, cMem); - ZSTD_free(cdict->dictBuffer, cMem); - ZSTD_free(cdict, cMem); + int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); + ZSTD_cwksp_free(&cdict->workspace, cMem); + if (!cdictInWorkspace) { + ZSTD_free(cdict, cMem); + } return 0; } } @@ -3721,28 +3338,30 @@ const ZSTD_CDict* ZSTD_initStaticCDict( ZSTD_compressionParameters cParams) { size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0); - size_t const neededSize = sizeof(ZSTD_CDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize) - + HUF_WORKSPACE_SIZE + matchStateSize; - ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; - void* ptr; + size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + matchStateSize; + ZSTD_CDict* cdict; + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + + { + ZSTD_cwksp ws; + ZSTD_cwksp_init(&ws, workspace, workspaceSize); + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + if (cdict == NULL) return NULL; + ZSTD_cwksp_move(&cdict->workspace, &ws); + } + DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; - if (dictLoadMethod == ZSTD_dlm_byCopy) { - memcpy(cdict+1, dict, dictSize); - dict = cdict+1; - ptr = (char*)workspace + sizeof(ZSTD_CDict) + dictSize; - } else { - ptr = cdict+1; - } - cdict->workspace = ptr; - cdict->workspaceSize = HUF_WORKSPACE_SIZE + matchStateSize; - if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, - ZSTD_dlm_byRef, dictContentType, + dictLoadMethod, dictContentType, cParams) )) return NULL; @@ -3764,7 +3383,15 @@ size_t ZSTD_compressBegin_usingCDict_advanced( DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_advanced"); RETURN_ERROR_IF(cdict==NULL, dictionary_wrong); { ZSTD_CCtx_params params = cctx->requestedParams; - params.cParams = ZSTD_getCParamsFromCDict(cdict); + params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0 ) + && (params.attachDictPref != ZSTD_dictForceLoad) ? + ZSTD_getCParamsFromCDict(cdict) + : ZSTD_getCParams(cdict->compressionLevel, + pledgedSrcSize, + cdict->dictContentSize); /* Increase window log to fit the entire dictionary and source if the * source size is known. Limit the increase to 19, which is the * window log for compression level 1 with the largest source size. @@ -3778,7 +3405,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced( return ZSTD_compressBegin_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, cdict, - params, pledgedSrcSize, + ¶ms, pledgedSrcSize, ZSTDb_not_buffered); } } @@ -3869,7 +3496,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, dict, dictSize, dictContentType, ZSTD_dtlm_fast, cdict, - params, pledgedSrcSize, + ¶ms, pledgedSrcSize, ZSTDb_buffered) ); cctx->inToCompress = 0; @@ -3903,13 +3530,14 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) * Assumption 2 : either dict, or cdict, is defined, not both */ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, const void* dict, size_t dictSize, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_internal"); FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); - assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); - zcs->requestedParams = params; + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + zcs->requestedParams = *params; assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if (dict) { FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); @@ -3948,7 +3576,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) /* ZSTD_initCStream_advanced() : * pledgedSrcSize must be exact. * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. - * dict is loaded with default parameters ZSTD_dm_auto and ZSTD_dlm_byCopy. */ + * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pss) @@ -3962,7 +3590,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) ); FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) ); FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) ); - zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); + zcs->requestedParams = ZSTD_assignParamsToCCtxParams(&zcs->requestedParams, params); FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) ); return 0; } @@ -4212,7 +3840,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, if (cctx->mtctx == NULL) { DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", params.nbWorkers); - cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbWorkers, cctx->customMem); + cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem); RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation); } /* mt compression */ @@ -4340,8 +3968,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ - { 21, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */ - { 21, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */ + { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ + { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */ { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */ { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */ @@ -4365,8 +3993,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV /* W, C, H, S, L, T, strat */ { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ - { 18, 14, 14, 1, 5, 1, ZSTD_dfast }, /* level 2 */ - { 18, 16, 16, 1, 4, 1, ZSTD_dfast }, /* level 3 */ + { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ + { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ { 18, 16, 17, 2, 5, 2, ZSTD_greedy }, /* level 4.*/ { 18, 18, 18, 3, 5, 2, ZSTD_greedy }, /* level 5.*/ { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ @@ -4392,8 +4020,8 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ - { 17, 15, 16, 2, 5, 1, ZSTD_dfast }, /* level 3 */ - { 17, 17, 17, 2, 4, 1, ZSTD_dfast }, /* level 4 */ + { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ + { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ { 17, 17, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ { 17, 17, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ @@ -4418,7 +4046,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ - { 14, 14, 15, 2, 4, 1, ZSTD_dfast }, /* level 3 */ + { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ diff --git a/thirdparty/zstd/compress/zstd_compress_internal.h b/thirdparty/zstd/compress/zstd_compress_internal.h index 5495899be3..14036f873f 100644 --- a/thirdparty/zstd/compress/zstd_compress_internal.h +++ b/thirdparty/zstd/compress/zstd_compress_internal.h @@ -19,6 +19,7 @@ * Dependencies ***************************************/ #include "zstd_internal.h" +#include "zstd_cwksp.h" #ifdef ZSTD_MULTITHREAD # include "zstdmt_compress.h" #endif @@ -134,9 +135,15 @@ typedef struct { typedef struct ZSTD_matchState_t ZSTD_matchState_t; struct ZSTD_matchState_t { ZSTD_window_t window; /* State for window round buffer management */ - U32 loadedDictEnd; /* index of end of dictionary, within context's referential. When dict referential is copied into active context (i.e. not attached), effectively same value as dictSize, since referential starts from zero */ + U32 loadedDictEnd; /* index of end of dictionary, within context's referential. + * When loadedDictEnd != 0, a dictionary is in use, and still valid. + * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance. + * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity(). + * When dict referential is copied into active context (i.e. not attached), + * loadedDictEnd == dictSize, since referential starts from zero. + */ U32 nextToUpdate; /* index from which to continue table update */ - U32 hashLog3; /* dispatch table : larger == faster, more memory */ + U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ U32* hashTable; U32* hashTable3; U32* chainTable; @@ -186,6 +193,13 @@ typedef struct { size_t capacity; /* The capacity starting from `seq` pointer */ } rawSeqStore_t; +typedef struct { + int collectSequences; + ZSTD_Sequence* seqStart; + size_t seqIndex; + size_t maxSequences; +} SeqCollector; + struct ZSTD_CCtx_params_s { ZSTD_format_e format; ZSTD_compressionParameters cParams; @@ -197,6 +211,9 @@ struct ZSTD_CCtx_params_s { size_t targetCBlockSize; /* Tries to fit compressed block size to be around targetCBlockSize. * No target when targetCBlockSize == 0. * There is no guarantee on compressed block size */ + int srcSizeHint; /* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size */ ZSTD_dictAttachPref_e attachDictPref; ZSTD_literalCompressionMode_e literalCompressionMode; @@ -222,9 +239,7 @@ struct ZSTD_CCtx_s { ZSTD_CCtx_params appliedParams; U32 dictID; - int workSpaceOversizedDuration; - void* workSpace; - size_t workSpaceSize; + ZSTD_cwksp workspace; /* manages buffer for dynamic allocations */ size_t blockSize; unsigned long long pledgedSrcSizePlusOne; /* this way, 0 (default) == unknown */ unsigned long long consumedSrcSize; @@ -232,6 +247,8 @@ struct ZSTD_CCtx_s { XXH64_state_t xxhState; ZSTD_customMem customMem; size_t staticSize; + SeqCollector seqCollector; + int isFirstBlock; seqStore_t seqStore; /* sequences storage ptrs */ ldmState_t ldmState; /* long distance matching state */ @@ -307,26 +324,81 @@ MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; } +/* ZSTD_cParam_withinBounds: + * @return 1 if value is within cParam bounds, + * 0 otherwise */ +MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +/* ZSTD_minGain() : + * minimum compression required + * to generate a compress block or a compressed literals section. + * note : use same formula for both situations */ +MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) +{ + U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; + ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + return (srcSize >> minlog) + 2; +} + +/*! ZSTD_safecopyLiterals() : + * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w. + * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single + * large copies. + */ +static void ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) { + assert(iend > ilimit_w); + if (ip <= ilimit_w) { + ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap); + op += ilimit_w - ip; + ip = ilimit_w; + } + while (ip < iend) *op++ = *ip++; +} + /*! ZSTD_storeSeq() : - * Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. - * `offsetCode` : distance to match + 3 (values 1-3 are repCodes). + * Store a sequence (litlen, litPtr, offCode and mlBase) into seqStore_t. + * `offCode` : distance to match + ZSTD_REP_MOVE (values <= ZSTD_REP_MOVE are repCodes). * `mlBase` : matchLength - MINMATCH + * Allowed to overread literals up to litLimit. */ -MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t mlBase) +HINT_INLINE UNUSED_ATTR +void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, U32 offCode, size_t mlBase) { + BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; + BYTE const* const litEnd = literals + litLength; #if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) static const BYTE* g_start = NULL; if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ { U32 const pos = (U32)((const BYTE*)literals - g_start); DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u", - pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offsetCode); + pos, (U32)litLength, (U32)mlBase+MINMATCH, (U32)offCode); } #endif assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); /* copy Literals */ assert(seqStorePtr->maxNbLit <= 128 KB); assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); - ZSTD_wildcopy(seqStorePtr->lit, literals, litLength, ZSTD_no_overlap); + assert(literals + litLength <= litLimit); + if (litEnd <= litLimit_w) { + /* Common case we can use wildcopy. + * First copy 16 bytes, because literals are likely short. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(seqStorePtr->lit, literals); + if (litLength > 16) { + ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); + } + } else { + ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w); + } seqStorePtr->lit += litLength; /* literal Length */ @@ -338,7 +410,7 @@ MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const v seqStorePtr->sequences[0].litLength = (U16)litLength; /* match offset */ - seqStorePtr->sequences[0].offset = offsetCode + 1; + seqStorePtr->sequences[0].offset = offCode + 1; /* match Length */ if (mlBase>0xFFFF) { @@ -739,24 +811,37 @@ ZSTD_window_enforceMaxDist(ZSTD_window_t* window, /* Similar to ZSTD_window_enforceMaxDist(), * but only invalidates dictionary - * when input progresses beyond window size. */ + * when input progresses beyond window size. + * assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL) + * loadedDictEnd uses same referential as window->base + * maxDist is the window size */ MEM_STATIC void -ZSTD_checkDictValidity(ZSTD_window_t* window, +ZSTD_checkDictValidity(const ZSTD_window_t* window, const void* blockEnd, U32 maxDist, U32* loadedDictEndPtr, const ZSTD_matchState_t** dictMatchStatePtr) { - U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); - U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; - DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", - (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); - - if (loadedDictEnd && (blockEndIdx > maxDist + loadedDictEnd)) { - /* On reaching window size, dictionaries are invalidated */ - if (loadedDictEndPtr) *loadedDictEndPtr = 0; - if (dictMatchStatePtr) *dictMatchStatePtr = NULL; - } + assert(loadedDictEndPtr != NULL); + assert(dictMatchStatePtr != NULL); + { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = *loadedDictEndPtr; + DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + assert(blockEndIdx >= loadedDictEnd); + + if (blockEndIdx > loadedDictEnd + maxDist) { + /* On reaching window size, dictionaries are invalidated. + * For simplification, if window size is reached anywhere within next block, + * the dictionary is invalidated for the full block. + */ + DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)"); + *loadedDictEndPtr = 0; + *dictMatchStatePtr = NULL; + } else { + if (*loadedDictEndPtr != 0) { + DEBUGLOG(6, "dictionary considered valid for current block"); + } } } } /** @@ -798,6 +883,17 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, return contiguous; } +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 current, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.lowLimit; + U32 const withinWindow = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + + /* debug functions */ #if (DEBUGLEVEL>=2) @@ -856,7 +952,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, const void* dict, size_t dictSize, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); void ZSTD_resetSeqStore(seqStore_t* ssPtr); @@ -871,7 +967,7 @@ size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); /* ZSTD_compress_advanced_internal() : @@ -880,7 +976,7 @@ size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict,size_t dictSize, - ZSTD_CCtx_params params); + const ZSTD_CCtx_params* params); /* ZSTD_writeLastEmptyBlock() : diff --git a/thirdparty/zstd/compress/zstd_compress_literals.c b/thirdparty/zstd/compress/zstd_compress_literals.c new file mode 100644 index 0000000000..6c13331182 --- /dev/null +++ b/thirdparty/zstd/compress/zstd_compress_literals.c @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_literals.h" + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + memcpy(ostart + flSize, src, srcSize); + return srcSize + flSize; +} + +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ostart[flSize] = *(const BYTE*)src; + return flSize+1; +} + +size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const int bmi2) +{ + size_t const minGain = ZSTD_minGain(srcSize, strategy); + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + symbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i)", + disableLiteralCompression); + + /* Prepare nextEntropy assuming reusing the existing table */ + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (disableLiteralCompression) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + /* small ? don't even attempt compression (speed opt) */ +# define COMPRESS_LITERALS_SIZE_MIN 63 + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + + RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); + { HUF_repeat repeat = prevHuf->repeatMode; + int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; + if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; + cLitSize = singleStream ? + HUF_compress1X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + 255, 11, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) : + HUF_compress4X_repeat( + ostart+lhSize, dstCapacity-lhSize, src, srcSize, + 255, 11, entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); + if (repeat != HUF_repeat_none) { + /* reused the existing table */ + hType = set_repeat; + } + } + + if ((cLitSize==0) | (cLitSize >= srcSize - minGain) | ERR_isError(cLitSize)) { + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + if (cLitSize==1) { + memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } + + if (hType == set_compressed) { + /* using a newly constructed table */ + nextHuf->repeatMode = HUF_repeat_check; + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + return lhSize+cLitSize; +} diff --git a/thirdparty/zstd/compress/zstd_compress_literals.h b/thirdparty/zstd/compress/zstd_compress_literals.h new file mode 100644 index 0000000000..97273d7cfd --- /dev/null +++ b/thirdparty/zstd/compress/zstd_compress_literals.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_LITERALS_H +#define ZSTD_COMPRESS_LITERALS_H + +#include "zstd_compress_internal.h" /* ZSTD_hufCTables_t, ZSTD_minGain() */ + + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const int bmi2); + +#endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/thirdparty/zstd/compress/zstd_compress_sequences.c b/thirdparty/zstd/compress/zstd_compress_sequences.c new file mode 100644 index 0000000000..0ff7a26823 --- /dev/null +++ b/thirdparty/zstd/compress/zstd_compress_sequences.c @@ -0,0 +1,415 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_sequences.h" + +/** + * -log2(x / 256) lookup table for x in [0, 256). + * If x == 0: Return 0 + * Else: Return floor(-log2(x / 256) * 256) + */ +static unsigned const kInverseProbabilityLog256[256] = { + 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, + 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, + 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, + 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, + 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, + 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, + 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, + 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, + 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, + 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, + 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, + 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, + 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, + 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, + 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, + 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, + 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, + 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, + 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, + 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, + 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, + 5, 4, 2, 1, +}; + +static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { + void const* ptr = ctable; + U16 const* u16ptr = (U16 const*)ptr; + U32 const maxSymbolValue = MEM_read16(u16ptr + 1); + return maxSymbolValue; +} + +/** + * Returns the cost in bytes of encoding the normalized count header. + * Returns an error if any of the helper functions return an error. + */ +static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, + size_t const nbSeq, unsigned const FSELog) +{ + BYTE wksp[FSE_NCOUNTBOUND]; + S16 norm[MaxSeq + 1]; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max)); + return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); +} + +/** + * Returns the cost in bits of encoding the distribution described by count + * using the entropy bound. + */ +static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) +{ + unsigned cost = 0; + unsigned s; + for (s = 0; s <= max; ++s) { + unsigned norm = (unsigned)((256 * count[s]) / total); + if (count[s] != 0 && norm == 0) + norm = 1; + assert(count[s] < total); + cost += count[s] * kInverseProbabilityLog256[norm]; + } + return cost >> 8; +} + +/** + * Returns the cost in bits of encoding the distribution in count using ctable. + * Returns an error if ctable cannot represent all the symbols in count. + */ +static size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max) +{ + unsigned const kAccuracyLog = 8; + size_t cost = 0; + unsigned s; + FSE_CState_t cstate; + FSE_initCState(&cstate, ctable); + RETURN_ERROR_IF(ZSTD_getFSEMaxSymbolValue(ctable) < max, GENERIC, + "Repeat FSE_CTable has maxSymbolValue %u < %u", + ZSTD_getFSEMaxSymbolValue(ctable), max); + for (s = 0; s <= max; ++s) { + unsigned const tableLog = cstate.stateLog; + unsigned const badCost = (tableLog + 1) << kAccuracyLog; + unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); + if (count[s] == 0) + continue; + RETURN_ERROR_IF(bitCost >= badCost, GENERIC, + "Repeat FSE_CTable has Prob[%u] == 0", s); + cost += count[s] * bitCost; + } + return cost >> kAccuracyLog; +} + +/** + * Returns the cost in bits of encoding the distribution in count using the + * table described by norm. The max symbol support by norm is assumed >= max. + * norm must be valid for every symbol with non-zero probability in count. + */ +static size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max) +{ + unsigned const shift = 8 - accuracyLog; + size_t cost = 0; + unsigned s; + assert(accuracyLog <= 8); + for (s = 0; s <= max; ++s) { + unsigned const normAcc = norm[s] != -1 ? norm[s] : 1; + unsigned const norm256 = normAcc << shift; + assert(norm256 > 0); + assert(norm256 < 256); + cost += count[s] * kInverseProbabilityLog256[norm256]; + } + return cost >> 8; +} + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy) +{ + ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); + if (mostFrequent == nbSeq) { + *repeatMode = FSE_repeat_none; + if (isDefaultAllowed && nbSeq <= 2) { + /* Prefer set_basic over set_rle when there are 2 or less symbols, + * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. + * If basic encoding isn't possible, always choose RLE. + */ + DEBUGLOG(5, "Selected set_basic"); + return set_basic; + } + DEBUGLOG(5, "Selected set_rle"); + return set_rle; + } + if (strategy < ZSTD_lazy) { + if (isDefaultAllowed) { + size_t const staticFse_nbSeq_max = 1000; + size_t const mult = 10 - strategy; + size_t const baseLog = 3; + size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ + assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ + assert(mult <= 9 && mult >= 7); + if ( (*repeatMode == FSE_repeat_valid) + && (nbSeq < staticFse_nbSeq_max) ) { + DEBUGLOG(5, "Selected set_repeat"); + return set_repeat; + } + if ( (nbSeq < dynamicFse_nbSeq_min) + || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { + DEBUGLOG(5, "Selected set_basic"); + /* The format allows default tables to be repeated, but it isn't useful. + * When using simple heuristics to select encoding type, we don't want + * to confuse these tables with dictionaries. When running more careful + * analysis, we don't need to waste time checking both repeating tables + * and default tables. + */ + *repeatMode = FSE_repeat_none; + return set_basic; + } + } + } else { + size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); + size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); + size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); + size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); + + if (isDefaultAllowed) { + assert(!ZSTD_isError(basicCost)); + assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); + } + assert(!ZSTD_isError(NCountCost)); + assert(compressedCost < ERROR(maxCode)); + DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", + (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); + if (basicCost <= repeatCost && basicCost <= compressedCost) { + DEBUGLOG(5, "Selected set_basic"); + assert(isDefaultAllowed); + *repeatMode = FSE_repeat_none; + return set_basic; + } + if (repeatCost <= compressedCost) { + DEBUGLOG(5, "Selected set_repeat"); + assert(!ZSTD_isError(repeatCost)); + return set_repeat; + } + assert(compressedCost < basicCost && compressedCost < repeatCost); + } + DEBUGLOG(5, "Selected set_compressed"); + *repeatMode = FSE_repeat_check; + return set_compressed; +} + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize) +{ + BYTE* op = (BYTE*)dst; + const BYTE* const oend = op + dstCapacity; + DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); + + switch (type) { + case set_rle: + FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max)); + RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall); + *op = codeTable[0]; + return 1; + case set_repeat: + memcpy(nextCTable, prevCTable, prevCTableSize); + return 0; + case set_basic: + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize)); /* note : could be pre-calculated */ + return 0; + case set_compressed: { + S16 norm[MaxSeq + 1]; + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + if (count[codeTable[nbSeq-1]] > 1) { + count[codeTable[nbSeq-1]]--; + nbSeq_1--; + } + assert(nbSeq_1 > 1); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max)); + { size_t const NCountSize = FSE_writeNCount(op, oend - op, norm, max, tableLog); /* overflow protected */ + FORWARD_IF_ERROR(NCountSize); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, norm, max, tableLog, entropyWorkspace, entropyWorkspaceSize)); + return NCountSize; + } + } + default: assert(0); RETURN_ERROR(GENERIC); + } +} + +FORCE_INLINE_TEMPLATE size_t +ZSTD_encodeSequences_body( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + RETURN_ERROR_IF( + ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), + dstSize_tooSmall, "not enough space remaining"); + DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", + (int)(blockStream.endPtr - blockStream.startPtr), + (unsigned)dstCapacity); + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + if (longOffsets) { + U32 const ofBits = ofCodeTable[nbSeq-1]; + int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, extraBits); + BIT_flushBits(&blockStream); + } + BIT_addBits(&blockStream, sequences[nbSeq-1].offset >> extraBits, + ofBits - extraBits); + } else { + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); + } + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n<nbSeq ; n--) { /* intentional underflow */ + BYTE const llCode = llCodeTable[n]; + BYTE const ofCode = ofCodeTable[n]; + BYTE const mlCode = mlCodeTable[n]; + U32 const llBits = LL_bits[llCode]; + U32 const ofBits = ofCode; + U32 const mlBits = ML_bits[mlCode]; + DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u", + (unsigned)sequences[n].litLength, + (unsigned)sequences[n].matchLength + MINMATCH, + (unsigned)sequences[n].offset); + /* 32b*/ /* 64b*/ + /* (7)*/ /* (7)*/ + FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ + FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ + if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/ + FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */ + if (MEM_32bits() || (ofBits+mlBits+llBits >= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); + if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); + if (longOffsets) { + int const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[n].offset, extraBits); + BIT_flushBits(&blockStream); /* (7)*/ + } + BIT_addBits(&blockStream, sequences[n].offset >> extraBits, + ofBits - extraBits); /* 31 */ + } else { + BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ + } + BIT_flushBits(&blockStream); /* (7)*/ + DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); + } } + + DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); + FSE_flushCState(&blockStream, &stateMatchLength); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); + FSE_flushCState(&blockStream, &stateOffsetBits); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); + return streamSize; + } +} + +static size_t +ZSTD_encodeSequences_default( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + + +#if DYNAMIC_BMI2 + +static TARGET_ATTRIBUTE("bmi2") size_t +ZSTD_encodeSequences_bmi2( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + +#endif + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) +{ + DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); +#if DYNAMIC_BMI2 + if (bmi2) { + return ZSTD_encodeSequences_bmi2(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); + } +#endif + (void)bmi2; + return ZSTD_encodeSequences_default(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} diff --git a/thirdparty/zstd/compress/zstd_compress_sequences.h b/thirdparty/zstd/compress/zstd_compress_sequences.h new file mode 100644 index 0000000000..57e8e367b0 --- /dev/null +++ b/thirdparty/zstd/compress/zstd_compress_sequences.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_SEQUENCES_H +#define ZSTD_COMPRESS_SEQUENCES_H + +#include "fse.h" /* FSE_repeat, FSE_CTable */ +#include "zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */ + +typedef enum { + ZSTD_defaultDisallowed = 0, + ZSTD_defaultAllowed = 1 +} ZSTD_defaultPolicy_e; + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy); + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize); + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); + +#endif /* ZSTD_COMPRESS_SEQUENCES_H */ diff --git a/thirdparty/zstd/compress/zstd_cwksp.h b/thirdparty/zstd/compress/zstd_cwksp.h new file mode 100644 index 0000000000..fc9765bd3f --- /dev/null +++ b/thirdparty/zstd/compress/zstd_cwksp.h @@ -0,0 +1,535 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CWKSP_H +#define ZSTD_CWKSP_H + +/*-************************************* +* Dependencies +***************************************/ +#include "zstd_internal.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Since the workspace is effectively its own little malloc implementation / + * arena, when we run under ASAN, we should similarly insert redzones between + * each internal element of the workspace, so ASAN will catch overruns that + * reach outside an object but that stay inside the workspace. + * + * This defines the size of that redzone. + */ +#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 +#endif + +/*-************************************* +* Structures +***************************************/ +typedef enum { + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_buffers, + ZSTD_cwksp_alloc_aligned +} ZSTD_cwksp_alloc_phase_e; + +/** + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each internal datastructure: + * + * - These different internal datastructures have different setup requirements: + * + * - The static objects need to be cleared once and can then be trivially + * reused for each compression. + * + * - Various buffers don't need to be initialized at all--they are always + * written into before they're read. + * + * - The matchstate tables have a unique requirement that they don't need + * their memory to be totally cleared, but they do need the memory to have + * some bound, i.e., a guarantee that all values in the memory they've been + * allocated is less than some maximum value (which is the starting value + * for the indices that they will then use for compression). When this + * guarantee is provided to them, they can use the memory without any setup + * work. When it can't, they have to clear the area. + * + * - These buffers also have different alignment requirements. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp + * abstraction was created. It works as follows: + * + * Workspace Layout: + * + * [ ... workspace ... ] + * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] + * + * The various objects that live in the workspace are divided into the + * following categories, and are allocated separately: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_free{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. + * + * - Aligned: these buffers are used for various purposes that require 4 byte + * alignment, but don't require any initialization before they're used. + * + * - Buffers: these buffers are used for various purposes that don't require + * any alignment or initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Buffers + * 3. Aligned + * 4. Tables + * + * Attempts to reserve objects of different types out of order will fail. + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + void* tableEnd; + void* tableValidEnd; + void* allocStart; + + int allocFailed; + int workspaceOversizedDuration; + ZSTD_cwksp_alloc_phase_e phase; +} ZSTD_cwksp; + +/*-************************************* +* Functions +***************************************/ + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); + +MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { + (void)ws; + assert(ws->workspace <= ws->objectEnd); + assert(ws->objectEnd <= ws->tableEnd); + assert(ws->objectEnd <= ws->tableValidEnd); + assert(ws->tableEnd <= ws->allocStart); + assert(ws->tableValidEnd <= ws->allocStart); + assert(ws->allocStart <= ws->workspaceEnd); +} + +/** + * Align must be a power of 2. + */ +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; +} + +/** + * Use this to determine how much space in the workspace we will consume to + * allocate this object. (Normally it should be exactly the size of the object, + * but under special conditions, like ASAN, where we pad each object, it might + * be larger.) + * + * Since tables aren't currently redzoned, you don't need to call through this + * to figure out how much space you need for the matchState tables. Everything + * else is though. + */ +MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#else + return size; +#endif +} + +MEM_STATIC void ZSTD_cwksp_internal_advance_phase( + ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) { + assert(phase >= ws->phase); + if (phase > ws->phase) { + if (ws->phase < ZSTD_cwksp_alloc_buffers && + phase >= ZSTD_cwksp_alloc_buffers) { + ws->tableValidEnd = ws->objectEnd; + } + if (ws->phase < ZSTD_cwksp_alloc_aligned && + phase >= ZSTD_cwksp_alloc_aligned) { + /* If unaligned allocations down from a too-large top have left us + * unaligned, we need to realign our alloc ptr. Technically, this + * can consume space that is unaccounted for in the neededSpace + * calculation. However, I believe this can only happen when the + * workspace is too large, and specifically when it is too large + * by a larger margin than the space that will be consumed. */ + /* TODO: cleaner, compiler warning friendly way to do this??? */ + ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1)); + if (ws->allocStart < ws->tableValidEnd) { + ws->tableValidEnd = ws->allocStart; + } + } + ws->phase = phase; + } +} + +/** + * Returns whether this object/buffer/etc was allocated in this workspace. + */ +MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) { + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd); +} + +/** + * Internal function. Do not use directly. + */ +MEM_STATIC void* ZSTD_cwksp_reserve_internal( + ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) { + void* alloc; + void* bottom = ws->tableEnd; + ZSTD_cwksp_internal_advance_phase(ws, phase); + alloc = (BYTE *)ws->allocStart - bytes; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + alloc = (BYTE *)alloc - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(alloc >= bottom); + if (alloc < bottom) { + DEBUGLOG(4, "cwksp: alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + if (alloc < ws->tableValidEnd) { + ws->tableValidEnd = alloc; + } + ws->allocStart = alloc; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + __asan_unpoison_memory_region(alloc, bytes); +#endif + + return alloc; +} + +/** + * Reserves and returns unaligned memory. + */ +MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) { + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} + +/** + * Reserves and returns memory sized on and aligned on sizeof(unsigned). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) { + assert((bytes & (sizeof(U32)-1)) == 0); + return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned); +} + +/** + * Aligned on sizeof(unsigned). These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; + void* alloc = ws->tableEnd; + void* end = (BYTE *)alloc + bytes; + void* top = ws->allocStart; + + DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); + ZSTD_cwksp_internal_advance_phase(ws, phase); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(end <= top); + if (end > top) { + DEBUGLOG(4, "cwksp: table alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + __asan_unpoison_memory_region(alloc, bytes); +#endif + + return alloc; +} + +/** + * Aligned on sizeof(void*). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) { + size_t roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* alloc = ws->objectEnd; + void* end = (BYTE*)alloc + roundedBytes; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(5, + "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", + alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert(((size_t)alloc & (sizeof(void*)-1)) == 0); + assert((bytes & (sizeof(void*)-1)) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + /* we must be in the first phase, no advance is possible */ + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(4, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + ws->tableValidEnd = end; + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + __asan_unpoison_memory_region(alloc, bytes); +#endif + + return alloc; +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); + +#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table re-use logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. */ + { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + assert(__msan_test_shadow(ws->objectEnd, size) == -1); + __msan_poison(ws->objectEnd, size); + } +#endif + + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + ws->tableValidEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Zero the part of the allocated tables not already marked clean. + */ +MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd); + } + ZSTD_cwksp_mark_tables_clean(ws); +} + +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing tables!"); + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing!"); + +#if defined (MEMORY_SANITIZER) && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the context re-use logic is sound, and that we don't + * access stuff that this compression hasn't initialized, we re-"poison" + * the workspace (or at least the non-static, non-table parts of it) + * every time we start a new compression. */ + { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd; + __msan_poison(ws->tableValidEnd, size); + } +#endif + +#if defined (ADDRESS_SANITIZER) && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ws->allocStart = ws->workspaceEnd; + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_buffers) { + ws->phase = ZSTD_cwksp_alloc_buffers; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * The provided workspace takes ownership of the buffer [start, start+size). + * Any existing values in the workspace are ignored (the previously managed + * buffer, if present, must be separately freed). + */ +MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) { + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->tableValidEnd = ws->objectEnd; + ws->phase = ZSTD_cwksp_alloc_objects; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_malloc(size, customMem); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation); + ZSTD_cwksp_init(ws, workspace, size); + return 0; +} + +MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + void *ptr = ws->workspace; + DEBUGLOG(4, "cwksp: freeing workspace"); + memset(ws, 0, sizeof(ZSTD_cwksp)); + ZSTD_free(ptr, customMem); +} + +/** + * Moves the management of a workspace from one cwksp to another. The src cwksp + * is left in an invalid state (src must be re-init()'ed before its used again). + */ +MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + memset(src, 0, sizeof(ZSTD_cwksp)); +} + +MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); +} + +MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} + +/*-************************************* +* Functions Checking Free Space +***************************************/ + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + +MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; +} + +MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} + +MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CWKSP_H */ diff --git a/thirdparty/zstd/compress/zstd_double_fast.c b/thirdparty/zstd/compress/zstd_double_fast.c index 5957255d90..a661a48534 100644 --- a/thirdparty/zstd/compress/zstd_double_fast.c +++ b/thirdparty/zstd/compress/zstd_double_fast.c @@ -65,6 +65,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); const U32 lowestValid = ms->window.dictLimit; const U32 maxDistance = 1U << cParams->windowLog; + /* presumes that, if there is a dictionary, it must be using Attach mode */ const U32 prefixLowestIndex = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; const BYTE* const prefixLowest = base + prefixLowestIndex; const BYTE* const iend = istart + srcSize; @@ -147,7 +148,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); goto _match_stored; } @@ -156,7 +157,7 @@ size_t ZSTD_compressBlock_doubleFast_generic( && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); goto _match_stored; } @@ -246,7 +247,7 @@ _match_found: offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); _match_stored: /* match found */ @@ -277,7 +278,7 @@ _match_stored: const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; ip += repLength2; @@ -296,7 +297,7 @@ _match_stored: U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); - ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, rLength-MINMATCH); ip += rLength; anchor = ip; continue; /* faster when present ... (?) */ @@ -369,9 +370,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const ilimit = iend - 8; const BYTE* const base = ms->window.base; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - const U32 maxDistance = 1U << cParams->windowLog; - const U32 lowestValid = ms->window.lowLimit; - const U32 lowLimit = (endIndex - lowestValid > maxDistance) ? endIndex - maxDistance : lowestValid; + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); const U32 dictStartIndex = lowLimit; const U32 dictLimit = ms->window.dictLimit; const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; @@ -412,7 +411,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); } else { if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; @@ -423,7 +422,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); @@ -448,7 +447,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( } offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } else { ip += ((ip-anchor) >> kSearchStrength) + 1; @@ -480,7 +479,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; ip += repLength2; diff --git a/thirdparty/zstd/compress/zstd_fast.c b/thirdparty/zstd/compress/zstd_fast.c index a05b8a47f1..6dbefee6b7 100644 --- a/thirdparty/zstd/compress/zstd_fast.c +++ b/thirdparty/zstd/compress/zstd_fast.c @@ -8,7 +8,7 @@ * You may select, at your option, one of the above-listed licenses. */ -#include "zstd_compress_internal.h" +#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */ #include "zstd_fast.h" @@ -43,8 +43,8 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, } -FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_fast_generic( +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_fast_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize, U32 const mls) @@ -71,10 +71,10 @@ size_t ZSTD_compressBlock_fast_generic( U32 offsetSaved = 0; /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); ip0 += (ip0 == prefixStart); ip1 = ip0 + 1; - { - U32 const maxRep = (U32)(ip0 - prefixStart); + { U32 const maxRep = (U32)(ip0 - prefixStart); if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; } @@ -117,8 +117,7 @@ size_t ZSTD_compressBlock_fast_generic( match0 = match1; goto _offset; } - { - size_t const step = ((ip0-anchor) >> (kSearchStrength - 1)) + stepSize; + { size_t const step = ((size_t)(ip0-anchor) >> (kSearchStrength - 1)) + stepSize; assert(step >= 2); ip0 += step; ip1 += step; @@ -137,7 +136,7 @@ _offset: /* Requires: ip0, match0 */ _match: /* Requires: ip0, match0, offcode */ /* Count the forward length */ mLength += ZSTD_count(ip0+mLength+4, match0+mLength+4, iend) + 4; - ZSTD_storeSeq(seqStore, ip0-anchor, anchor, offcode, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip0-anchor), anchor, iend, offcode, mLength-MINMATCH); /* match found */ ip0 += mLength; anchor = ip0; @@ -149,16 +148,15 @@ _match: /* Requires: ip0, match0, offcode */ hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); - while ( (ip0 <= ilimit) - && ( (offset_2>0) - & (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) )) { + while ( ((ip0 <= ilimit) & (offset_2>0)) /* offset_2==0 means offset_2 is invalidated */ + && (MEM_read32(ip0) == MEM_read32(ip0 - offset_2)) ) { /* store sequence */ size_t const rLength = ZSTD_count(ip0+4, ip0+4-offset_2, iend) + 4; - U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); ip0 += rLength; ip1 = ip0 + 1; - ZSTD_storeSeq(seqStore, 0, anchor, 0, rLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, 0 /*offCode*/, rLength-MINMATCH); anchor = ip0; continue; /* faster when present (confirmed on gcc-8) ... (?) */ } @@ -178,8 +176,7 @@ size_t ZSTD_compressBlock_fast( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->minMatch; + U32 const mls = ms->cParams.minMatch; assert(ms->dictMatchState == NULL); switch(mls) { @@ -239,6 +236,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); ip += (dictAndPrefixLength == 0); /* dictMatchState repCode checks don't currently handle repCode == 0 * disabling. */ @@ -263,7 +261,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, mLength-MINMATCH); } else if ( (matchIndex <= prefixStartIndex) ) { size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); U32 const dictMatchIndex = dictHashTable[dictHash]; @@ -283,7 +281,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } } else if (MEM_read32(match) != MEM_read32(ip)) { /* it's not a match, and we're not going to check the dictionary */ @@ -298,7 +296,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); } /* match found */ @@ -323,7 +321,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, repLength2-MINMATCH); hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; ip += repLength2; anchor = ip; @@ -346,8 +344,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->minMatch; + U32 const mls = ms->cParams.minMatch; assert(ms->dictMatchState != NULL); switch(mls) { @@ -379,9 +376,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* ip = istart; const BYTE* anchor = istart; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); - const U32 maxDistance = 1U << cParams->windowLog; - const U32 validLow = ms->window.lowLimit; - const U32 lowLimit = (endIndex - validLow > maxDistance) ? endIndex - maxDistance : validLow; + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); const U32 dictStartIndex = lowLimit; const BYTE* const dictStart = dictBase + dictStartIndex; const U32 dictLimit = ms->window.dictLimit; @@ -392,6 +387,8 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const ilimit = iend - 8; U32 offset_1=rep[0], offset_2=rep[1]; + DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic"); + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ if (prefixStartIndex == dictStartIndex) return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); @@ -406,16 +403,17 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const U32 repIndex = current + 1 - offset_1; const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - size_t mLength; hashTable[h] = current; /* update hash table */ assert(offset_1 <= current +1); /* check repIndex */ if ( (((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > dictStartIndex)) && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, 0, mLength-MINMATCH); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; } else { if ( (matchIndex < dictStartIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { @@ -423,21 +421,17 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( ip += ((ip-anchor) >> kSearchStrength) + stepSize; continue; } - { const BYTE* matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; - const BYTE* lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; - U32 offset; - mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; + { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; + U32 const offset = current - matchIndex; + size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - offset = current - matchIndex; - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + offset_2 = offset_1; offset_1 = offset; /* update offset history */ + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + ip += mLength; + anchor = ip; } } - /* found a match : store it */ - ip += mLength; - anchor = ip; - if (ip <= ilimit) { /* Fill Table */ hashTable[ZSTD_hashPtr(base+current+2, hlog, mls)] = current+2; @@ -446,13 +440,13 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( while (ip <= ilimit) { U32 const current2 = (U32)(ip-base); U32 const repIndex2 = current2 - offset_2; - const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (repIndex2 > dictStartIndex)) /* intentional overflow */ && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; - U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, repLength2-MINMATCH); + { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, 0 /*offcode*/, repLength2-MINMATCH); hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; ip += repLength2; anchor = ip; @@ -474,8 +468,7 @@ size_t ZSTD_compressBlock_fast_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->minMatch; + U32 const mls = ms->cParams.minMatch; switch(mls) { default: /* includes case 3 */ diff --git a/thirdparty/zstd/compress/zstd_lazy.c b/thirdparty/zstd/compress/zstd_lazy.c index 94d906c01f..9ad7e03b54 100644 --- a/thirdparty/zstd/compress/zstd_lazy.c +++ b/thirdparty/zstd/compress/zstd_lazy.c @@ -242,9 +242,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); - U32 const maxDistance = 1U << cParams->windowLog; - U32 const windowValid = ms->window.lowLimit; - U32 const windowLow = (current - windowValid > maxDistance) ? current - maxDistance : windowValid; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, current, cParams->windowLog); U32* const bt = ms->chainTable; U32 const btLog = cParams->chainLog - 1; @@ -497,8 +495,10 @@ size_t ZSTD_HcFindBestMatch_generic ( const BYTE* const dictEnd = dictBase + dictLimit; const U32 current = (U32)(ip-base); const U32 maxDistance = 1U << cParams->windowLog; - const U32 lowValid = ms->window.lowLimit; - const U32 lowLimit = (current - lowValid > maxDistance) ? current - maxDistance : lowValid; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (current - lowestValid > maxDistance) ? current - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; const U32 minChain = current > chainSize ? current - chainSize : 0; U32 nbAttempts = 1U << cParams->searchLog; size_t ml=4-1; @@ -619,12 +619,14 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( /* ******************************* * Common parser - lazy strategy *********************************/ -FORCE_INLINE_TEMPLATE -size_t ZSTD_compressBlock_lazy_generic( +typedef enum { search_hashChain, search_binaryTree } searchMethod_e; + +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_lazy_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, - const U32 searchMethod, const U32 depth, + const searchMethod_e searchMethod, const U32 depth, ZSTD_dictMode_e const dictMode) { const BYTE* const istart = (const BYTE*)src; @@ -640,8 +642,10 @@ size_t ZSTD_compressBlock_lazy_generic( ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); searchMax_f const searchMax = dictMode == ZSTD_dictMatchState ? - (searchMethod ? ZSTD_BtFindBestMatch_dictMatchState_selectMLS : ZSTD_HcFindBestMatch_dictMatchState_selectMLS) : - (searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS); + (searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_dictMatchState_selectMLS + : ZSTD_HcFindBestMatch_dictMatchState_selectMLS) : + (searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_selectMLS + : ZSTD_HcFindBestMatch_selectMLS); U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; const ZSTD_matchState_t* const dms = ms->dictMatchState; @@ -806,7 +810,7 @@ size_t ZSTD_compressBlock_lazy_generic( /* store sequence */ _storeSequence: { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); anchor = ip = start + matchLength; } @@ -824,7 +828,7 @@ _storeSequence: const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; @@ -839,7 +843,7 @@ _storeSequence: /* store sequence */ matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ @@ -850,7 +854,7 @@ _storeSequence: rep[1] = offset_2 ? offset_2 : savedOffset; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -858,56 +862,56 @@ size_t ZSTD_compressBlock_btlazy2( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 1, 2, ZSTD_noDict); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); } size_t ZSTD_compressBlock_lazy2( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 2, ZSTD_noDict); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); } size_t ZSTD_compressBlock_lazy( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 1, ZSTD_noDict); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); } size_t ZSTD_compressBlock_greedy( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 0, ZSTD_noDict); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict); } size_t ZSTD_compressBlock_btlazy2_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 1, 2, ZSTD_dictMatchState); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState); } size_t ZSTD_compressBlock_lazy2_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 2, ZSTD_dictMatchState); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); } size_t ZSTD_compressBlock_lazy_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 1, ZSTD_dictMatchState); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); } size_t ZSTD_compressBlock_greedy_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, 0, 0, ZSTD_dictMatchState); + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState); } @@ -916,7 +920,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], const void* src, size_t srcSize, - const U32 searchMethod, const U32 depth) + const searchMethod_e searchMethod, const U32 depth) { const BYTE* const istart = (const BYTE*)src; const BYTE* ip = istart; @@ -934,7 +938,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( typedef size_t (*searchMax_f)( ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; + searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS; U32 offset_1 = rep[0], offset_2 = rep[1]; @@ -1047,7 +1051,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( /* store sequence */ _storeSequence: { size_t const litLength = start - anchor; - ZSTD_storeSeq(seqStore, litLength, anchor, (U32)offset, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offset, matchLength-MINMATCH); anchor = ip = start + matchLength; } @@ -1062,7 +1066,7 @@ _storeSequence: const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ - ZSTD_storeSeq(seqStore, 0, anchor, 0, matchLength-MINMATCH); + ZSTD_storeSeq(seqStore, 0, anchor, iend, 0, matchLength-MINMATCH); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ @@ -1075,7 +1079,7 @@ _storeSequence: rep[1] = offset_2; /* Return the last literals size */ - return iend - anchor; + return (size_t)(iend - anchor); } @@ -1083,7 +1087,7 @@ size_t ZSTD_compressBlock_greedy_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 0); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0); } size_t ZSTD_compressBlock_lazy_extDict( @@ -1091,7 +1095,7 @@ size_t ZSTD_compressBlock_lazy_extDict( void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 1); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1); } size_t ZSTD_compressBlock_lazy2_extDict( @@ -1099,7 +1103,7 @@ size_t ZSTD_compressBlock_lazy2_extDict( void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 0, 2); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2); } size_t ZSTD_compressBlock_btlazy2_extDict( @@ -1107,5 +1111,5 @@ size_t ZSTD_compressBlock_btlazy2_extDict( void const* src, size_t srcSize) { - return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, 1, 2); + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2); } diff --git a/thirdparty/zstd/compress/zstd_ldm.c b/thirdparty/zstd/compress/zstd_ldm.c index 3dcf86e6e8..c3312ad3e3 100644 --- a/thirdparty/zstd/compress/zstd_ldm.c +++ b/thirdparty/zstd/compress/zstd_ldm.c @@ -49,9 +49,9 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params) { size_t const ldmHSize = ((size_t)1) << params.hashLog; size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); - size_t const ldmBucketSize = - ((size_t)1) << (params.hashLog - ldmBucketSizeLog); - size_t const totalSize = ldmBucketSize + ldmHSize * sizeof(ldmEntry_t); + size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); + size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) + + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); return params.enableLdm ? totalSize : 0; } @@ -583,7 +583,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, rep[i] = rep[i-1]; rep[0] = sequence.offset; /* Store the sequence */ - ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, + ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, sequence.offset + ZSTD_REP_MOVE, sequence.matchLength - MINMATCH); ip += sequence.matchLength; diff --git a/thirdparty/zstd/compress/zstd_opt.c b/thirdparty/zstd/compress/zstd_opt.c index e32e542e02..2e50fca6ff 100644 --- a/thirdparty/zstd/compress/zstd_opt.c +++ b/thirdparty/zstd/compress/zstd_opt.c @@ -552,7 +552,6 @@ U32 ZSTD_insertBtAndGetAllMatches ( { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); - U32 const maxDistance = 1U << cParams->windowLog; const BYTE* const base = ms->window.base; U32 const current = (U32)(ip-base); U32 const hashLog = cParams->hashLog; @@ -569,8 +568,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; U32 const btLow = (btMask >= current) ? 0 : current - btMask; - U32 const windowValid = ms->window.lowLimit; - U32 const windowLow = ((current - windowValid) > maxDistance) ? current - maxDistance : windowValid; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, current, cParams->windowLog); U32 const matchLow = windowLow ? windowLow : 1; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; @@ -674,19 +672,21 @@ U32 ZSTD_insertBtAndGetAllMatches ( while (nbCompares-- && (matchIndex >= matchLow)) { U32* const nextPtr = bt + 2*(matchIndex & btMask); - size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ const BYTE* match; + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ assert(current > matchIndex); if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ match = base + matchIndex; + if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); } else { match = dictBase + matchIndex; + assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* prepare for match[matchLength] */ + match = base + matchIndex; /* prepare for match[matchLength] read */ } if (matchLength > bestLength) { @@ -1098,7 +1098,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ assert(anchor + llen <= iend); ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen); - ZSTD_storeSeq(seqStore, llen, anchor, offCode, mlen-MINMATCH); + ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen-MINMATCH); anchor += advance; ip = anchor; } } diff --git a/thirdparty/zstd/compress/zstdmt_compress.c b/thirdparty/zstd/compress/zstdmt_compress.c index 9e537b8848..bc3062b530 100644 --- a/thirdparty/zstd/compress/zstdmt_compress.c +++ b/thirdparty/zstd/compress/zstdmt_compress.c @@ -668,7 +668,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) /* init */ if (job->cdict) { - size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, jobParams, job->fullFrameSize); + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize); assert(job->firstJob); /* only allowed for first job */ if (ZSTD_isError(initError)) JOB_ERROR(initError); } else { /* srcStart points at reloaded section */ @@ -680,7 +680,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ ZSTD_dtlm_fast, NULL, /*cdict*/ - jobParams, pledgedSrcSize); + &jobParams, pledgedSrcSize); if (ZSTD_isError(initError)) JOB_ERROR(initError); } } @@ -927,12 +927,18 @@ static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) unsigned jobID; DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) { + /* Copy the mutex/cond out */ + ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex; + ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond; + DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start); ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); - mtctx->jobs[jobID].dstBuff = g_nullBuffer; - mtctx->jobs[jobID].cSize = 0; + + /* Clear the job description, but keep the mutex/cond */ + memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID])); + mtctx->jobs[jobID].job_mutex = mutex; + mtctx->jobs[jobID].job_cond = cond; } - memset(mtctx->jobs, 0, (mtctx->jobIDMask+1)*sizeof(ZSTDMT_jobDescription)); mtctx->inBuff.buffer = g_nullBuffer; mtctx->inBuff.filled = 0; mtctx->allJobsCompleted = 1; @@ -1028,9 +1034,9 @@ size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, /* Sets parameters relevant to the compression job, * initializing others to default values. */ -static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(ZSTD_CCtx_params const params) +static ZSTD_CCtx_params ZSTDMT_initJobCCtxParams(const ZSTD_CCtx_params* params) { - ZSTD_CCtx_params jobParams = params; + ZSTD_CCtx_params jobParams = *params; /* Clear parameters related to multithreading */ jobParams.forceWindow = 0; jobParams.nbWorkers = 0; @@ -1151,16 +1157,16 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) /* ===== Multi-threaded compression ===== */ /* ------------------------------------------ */ -static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params) +static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params) { unsigned jobLog; - if (params.ldmParams.enableLdm) { + if (params->ldmParams.enableLdm) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize * based on chainLog instead. */ - jobLog = MAX(21, params.cParams.chainLog + 4); + jobLog = MAX(21, params->cParams.chainLog + 4); } else { - jobLog = MAX(20, params.cParams.windowLog + 2); + jobLog = MAX(20, params->cParams.windowLog + 2); } return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); } @@ -1193,27 +1199,27 @@ static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat) return ovlog; } -static size_t ZSTDMT_computeOverlapSize(ZSTD_CCtx_params const params) +static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) { - int const overlapRLog = 9 - ZSTDMT_overlapLog(params.overlapLog, params.cParams.strategy); - int ovLog = (overlapRLog >= 8) ? 0 : (params.cParams.windowLog - overlapRLog); + int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy); + int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog); assert(0 <= overlapRLog && overlapRLog <= 8); - if (params.ldmParams.enableLdm) { + if (params->ldmParams.enableLdm) { /* In Long Range Mode, the windowLog is typically oversized. * In which case, it's preferable to determine the jobSize * based on chainLog instead. * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */ - ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) + ovLog = MIN(params->cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog; } assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); - DEBUGLOG(4, "overlapLog : %i", params.overlapLog); + DEBUGLOG(4, "overlapLog : %i", params->overlapLog); DEBUGLOG(4, "overlap size : %i", 1 << ovLog); return (ovLog==0) ? 0 : (size_t)1 << ovLog; } static unsigned -ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers) +ZSTDMT_computeNbJobs(const ZSTD_CCtx_params* params, size_t srcSize, unsigned nbWorkers) { assert(nbWorkers>0); { size_t const jobSizeTarget = (size_t)1 << ZSTDMT_computeTargetJobLog(params); @@ -1236,9 +1242,9 @@ static size_t ZSTDMT_compress_advanced_internal( const ZSTD_CDict* cdict, ZSTD_CCtx_params params) { - ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(params); - size_t const overlapSize = ZSTDMT_computeOverlapSize(params); - unsigned const nbJobs = ZSTDMT_computeNbJobs(params, srcSize, params.nbWorkers); + ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(¶ms); + size_t const overlapSize = ZSTDMT_computeOverlapSize(¶ms); + unsigned const nbJobs = ZSTDMT_computeNbJobs(¶ms, srcSize, params.nbWorkers); size_t const proposedJobSize = (srcSize + (nbJobs-1)) / nbJobs; size_t const avgJobSize = (((proposedJobSize-1) & 0x1FFFF) < 0x7FFF) ? proposedJobSize + 0xFFFF : proposedJobSize; /* avoid too small last block */ const char* const srcStart = (const char*)src; @@ -1256,7 +1262,7 @@ static size_t ZSTDMT_compress_advanced_internal( ZSTD_CCtx* const cctx = mtctx->cctxPool->cctx[0]; DEBUGLOG(4, "ZSTDMT_compress_advanced_internal: fallback to single-thread mode"); if (cdict) return ZSTD_compress_usingCDict_advanced(cctx, dst, dstCapacity, src, srcSize, cdict, jobParams.fParams); - return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, NULL, 0, jobParams); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, NULL, 0, &jobParams); } assert(avgJobSize >= 256 KB); /* condition for ZSTD_compressBound(A) + ZSTD_compressBound(B) <= ZSTD_compressBound(A+B), required to compress directly into Dst (no additional buffer) */ @@ -1404,12 +1410,12 @@ size_t ZSTDMT_initCStream_internal( mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ if (mtctx->singleBlockingThread) { - ZSTD_CCtx_params const singleThreadParams = ZSTDMT_initJobCCtxParams(params); + ZSTD_CCtx_params const singleThreadParams = ZSTDMT_initJobCCtxParams(¶ms); DEBUGLOG(5, "ZSTDMT_initCStream_internal: switch to single blocking thread mode"); assert(singleThreadParams.nbWorkers == 0); return ZSTD_initCStream_internal(mtctx->cctxPool->cctx[0], dict, dictSize, cdict, - singleThreadParams, pledgedSrcSize); + &singleThreadParams, pledgedSrcSize); } DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); @@ -1435,11 +1441,11 @@ size_t ZSTDMT_initCStream_internal( mtctx->cdict = cdict; } - mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(params); + mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms); DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); mtctx->targetSectionSize = params.jobSize; if (mtctx->targetSectionSize == 0) { - mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params); + mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(¶ms); } assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); diff --git a/thirdparty/zstd/decompress/huf_decompress.c b/thirdparty/zstd/decompress/huf_decompress.c index 3f8bd29732..bb2d0a96bc 100644 --- a/thirdparty/zstd/decompress/huf_decompress.c +++ b/thirdparty/zstd/decompress/huf_decompress.c @@ -61,7 +61,9 @@ * Error Management ****************************************************************/ #define HUF_isError ERR_isError +#ifndef CHECK_F #define CHECK_F(f) { size_t const err_ = (f); if (HUF_isError(err_)) return err_; } +#endif /* ************************************************************** diff --git a/thirdparty/zstd/decompress/zstd_decompress.c b/thirdparty/zstd/decompress/zstd_decompress.c index e42872ad96..dd4591b7be 100644 --- a/thirdparty/zstd/decompress/zstd_decompress.c +++ b/thirdparty/zstd/decompress/zstd_decompress.c @@ -88,10 +88,7 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } static size_t ZSTD_startingInputLength(ZSTD_format_e format) { - size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ? - ZSTD_FRAMEHEADERSIZE_PREFIX - ZSTD_FRAMEIDSIZE : - ZSTD_FRAMEHEADERSIZE_PREFIX; - ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE); + size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); return startingInputLength; @@ -376,7 +373,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) { unsigned long long totalDstSize = 0; - while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) { + while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) { U32 const magicNumber = MEM_readLE32(src); if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { @@ -574,9 +571,10 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) } /** ZSTD_insertBlock() : - insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ + * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) { + DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize); ZSTD_checkContinuity(dctx, blockStart); dctx->previousDstEnd = (const char*)blockStart + blockSize; return blockSize; @@ -628,11 +626,12 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, /* check */ RETURN_ERROR_IF( - remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize, + remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, srcSize_wrong); /* Frame Header */ - { size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_FRAMEHEADERSIZE_PREFIX); + { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( + ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, srcSize_wrong); @@ -713,7 +712,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, dictSize = ZSTD_DDict_dictSize(ddict); } - while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) { + while (srcSize >= ZSTD_startingInputLength(dctx->format)) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (ZSTD_isLegacy(src, srcSize)) { @@ -909,6 +908,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c { blockProperties_t bp; size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); if (ZSTD_isError(cBlockSize)) return cBlockSize; + RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); dctx->expected = cBlockSize; dctx->bType = bp.blockType; dctx->rleSize = bp.origSize; @@ -953,6 +953,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c RETURN_ERROR(corruption_detected); } if (ZSTD_isError(rSize)) return rSize; + RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); dctx->decodedSize += rSize; if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); @@ -1095,7 +1096,7 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); for (i=0; i<3; i++) { U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; - RETURN_ERROR_IF(rep==0 || rep >= dictContentSize, + RETURN_ERROR_IF(rep==0 || rep > dictContentSize, dictionary_corrupted); entropy->rep[i] = rep; } } @@ -1264,7 +1265,7 @@ size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, { RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong); ZSTD_clearDict(dctx); - if (dict && dictSize >= 8) { + if (dict && dictSize != 0) { dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation); dctx->ddict = dctx->ddictLocal; @@ -1297,14 +1298,14 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz /* ZSTD_initDStream_usingDict() : - * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX. + * return : expected size, aka ZSTD_startingInputLength(). * this function cannot fail */ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) { DEBUGLOG(4, "ZSTD_initDStream_usingDict"); FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) ); FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) ); - return ZSTD_FRAMEHEADERSIZE_PREFIX; + return ZSTD_startingInputLength(zds->format); } /* note : this variant can't fail */ @@ -1321,16 +1322,16 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) { FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) ); FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) ); - return ZSTD_FRAMEHEADERSIZE_PREFIX; + return ZSTD_startingInputLength(dctx->format); } /* ZSTD_resetDStream() : - * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX. + * return : expected size, aka ZSTD_startingInputLength(). * this function cannot fail */ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) { FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only)); - return ZSTD_FRAMEHEADERSIZE_PREFIX; + return ZSTD_startingInputLength(dctx->format); } @@ -1561,7 +1562,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->lhSize += remainingInput; } input->pos = input->size; - return (MAX(ZSTD_FRAMEHEADERSIZE_MIN, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } assert(ip != NULL); memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; diff --git a/thirdparty/zstd/decompress/zstd_decompress_block.c b/thirdparty/zstd/decompress/zstd_decompress_block.c index 24f4859c56..767e5f9a0b 100644 --- a/thirdparty/zstd/decompress/zstd_decompress_block.c +++ b/thirdparty/zstd/decompress/zstd_decompress_block.c @@ -79,6 +79,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ { + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected); { const BYTE* const istart = (const BYTE*) src; @@ -87,6 +88,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, switch(litEncType) { case set_repeat: + DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted); /* fall-through */ @@ -116,7 +118,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, /* 2 - 2 - 18 - 18 */ lhSize = 5; litSize = (lhc >> 4) & 0x3FFFF; - litCSize = (lhc >> 22) + (istart[4] << 10); + litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); break; } RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected); @@ -391,7 +393,8 @@ ZSTD_buildFSETable(ZSTD_seqSymbol* dt, symbolNext[s] = 1; } else { if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; + assert(normalizedCounter[s]>=0); + symbolNext[s] = (U16)normalizedCounter[s]; } } } memcpy(dt, &DTableH, sizeof(DTableH)); } @@ -570,38 +573,118 @@ typedef struct { size_t pos; } seqState_t; +/*! ZSTD_overlapCopy8() : + * Copies 8 bytes from ip to op and updates op and ip where ip <= op. + * If the offset is < 8 then the offset is spread to at least 8 bytes. + * + * Precondition: *ip <= *op + * Postcondition: *op - *op >= 8 + */ +static void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { + assert(*ip <= *op); + if (offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[offset]; + (*op)[0] = (*ip)[0]; + (*op)[1] = (*ip)[1]; + (*op)[2] = (*ip)[2]; + (*op)[3] = (*ip)[3]; + *ip += dec32table[offset]; + ZSTD_copy4(*op+4, *ip); + *ip -= sub2; + } else { + ZSTD_copy8(*op, *ip); + } + *ip += 8; + *op += 8; + assert(*op - *ip >= 8); +} + +/*! ZSTD_safecopy() : + * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer + * and write up to 16 bytes past oend_w (op >= oend_w is allowed). + * This function is only called in the uncommon case where the sequence is near the end of the block. It + * should be fast for a single long sequence, but can be slow for several short sequences. + * + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. + * The src buffer must be before the dst buffer. + */ +static void ZSTD_safecopy(BYTE* op, BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; -/* ZSTD_execSequenceLast7(): - * exceptional case : decompress a match starting within last 7 bytes of output buffer. - * requires more careful checks, to ensure there is no overflow. - * performance does not matter though. - * note : this case is supposed to be never generated "naturally" by reference encoder, - * since in most cases it needs at least 8 bytes to look for a match. - * but it's allowed by the specification. */ + assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || + (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); + + if (length < 8) { + /* Handle short lengths. */ + while (op < oend) *op++ = *ip++; + return; + } + if (ovtype == ZSTD_overlap_src_before_dst) { + /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ + assert(length >= 8); + ZSTD_overlapCopy8(&op, &ip, diff); + assert(op - ip >= 8); + assert(op <= oend); + } + + if (oend <= oend_w) { + /* No risk of overwrite. */ + ZSTD_wildcopy(op, ip, length, ovtype); + return; + } + if (op <= oend_w) { + /* Wildcopy until we get close to the end. */ + assert(oend > oend_w); + ZSTD_wildcopy(op, ip, oend_w - op, ovtype); + ip += oend_w - op; + op = oend_w; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_execSequenceEnd(): + * This version handles cases that are near the end of the output buffer. It requires + * more careful checks to make sure there is no overflow. By separating out these hard + * and unlikely cases, we can speed up the common cases. + * + * NOTE: This function needs to be fast for a single long sequence, but doesn't need + * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). + */ FORCE_NOINLINE -size_t ZSTD_execSequenceLast7(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +size_t ZSTD_execSequenceEnd(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) { BYTE* const oLitEnd = op + sequence.litLength; size_t const sequenceLength = sequence.litLength + sequence.matchLength; BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - /* check */ - RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must fit within dstBuffer"); + /* bounds checks */ + assert(oLitEnd < oMatchEnd); + RETURN_ERROR_IF(oMatchEnd > oend, dstSize_tooSmall, "last match must fit within dstBuffer"); RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "try to read beyond literal buffer"); /* copy literals */ - while (op < oLitEnd) *op++ = *(*litPtr)++; + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - vBase),corruption_detected); - match = dictEnd - (base-match); + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected); + match = dictEnd - (prefixStart-match); if (match + sequence.matchLength <= dictEnd) { memmove(oLitEnd, match, sequence.matchLength); return sequenceLength; @@ -611,13 +694,12 @@ size_t ZSTD_execSequenceLast7(BYTE* op, memmove(oLitEnd, match, length1); op = oLitEnd + length1; sequence.matchLength -= length1; - match = base; + match = prefixStart; } } - while (op < oMatchEnd) *op++ = *match++; + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); return sequenceLength; } - HINT_INLINE size_t ZSTD_execSequence(BYTE* op, BYTE* const oend, seq_t sequence, @@ -631,20 +713,29 @@ size_t ZSTD_execSequence(BYTE* op, const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; - /* check */ - RETURN_ERROR_IF(oMatchEnd>oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend"); - RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer"); - if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); - - /* copy Literals */ - if (sequence.litLength > 8) - ZSTD_wildcopy_16min(op, (*litPtr), sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - else - ZSTD_copy8(op, *litPtr); + /* Errors and uncommon cases handled here. */ + assert(oLitEnd < oMatchEnd); + if (iLitEnd > litLimit || oMatchEnd > oend_w) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (sequence.litLength > 16) { + ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + } op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ - /* copy Match */ + /* Copy Match */ if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { /* offset beyond prefix -> go into extDict */ RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected); @@ -659,123 +750,33 @@ size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = prefixStart; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; - return sequenceLength; - } } } - /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op+4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); - } - op += 8; match += 8; - - if (oMatchEnd > oend-(16-MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (sequence.offset >= WILDCOPY_VECLEN) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; } - return sequenceLength; -} - - -HINT_INLINE -size_t ZSTD_execSequenceLong(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const dictStart, const BYTE* const dictEnd) -{ - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = sequence.match; + assert(sequence.offset < WILDCOPY_VECLEN); - /* check */ - RETURN_ERROR_IF(oMatchEnd > oend, dstSize_tooSmall, "last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend"); - RETURN_ERROR_IF(iLitEnd > litLimit, corruption_detected, "over-read beyond lit buffer"); - if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd); - - /* copy Literals */ - if (sequence.litLength > 8) - ZSTD_wildcopy_16min(op, *litPtr, sequence.litLength, ZSTD_no_overlap); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - else - ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { - /* offset beyond prefix */ - RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - dictStart), corruption_detected); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; - return sequenceLength; - } - } } - assert(op <= oend_w); - assert(sequence.matchLength >= MINMATCH); - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op+4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); - } - op += 8; match += 8; - - if (oMatchEnd > oend-(16-MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op, ZSTD_overlap_src_before_dst); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); /* works even if matchLength < 8 */ + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); } return sequenceLength; } @@ -1095,7 +1096,7 @@ ZSTD_decompressSequencesLong_body( /* decode and decompress */ for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && (seqNb<nbSeq) ; seqNb++) { seq_t const sequence = ZSTD_decodeSequenceLong(&seqState, isLongOffset); - size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb-ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); if (ZSTD_isError(oneSeqSize)) return oneSeqSize; PREFETCH_L1(sequence.match); PREFETCH_L1(sequence.match + sequence.matchLength - 1); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ sequences[seqNb & STORED_SEQS_MASK] = sequence; @@ -1106,7 +1107,7 @@ ZSTD_decompressSequencesLong_body( /* finish queue */ seqNb -= seqAdvance; for ( ; seqNb<nbSeq ; seqNb++) { - size_t const oneSeqSize = ZSTD_execSequenceLong(op, oend, sequences[seqNb&STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[seqNb&STORED_SEQS_MASK], &litPtr, litEnd, prefixStart, dictStart, dictEnd); if (ZSTD_isError(oneSeqSize)) return oneSeqSize; op += oneSeqSize; } diff --git a/thirdparty/zstd/zstd.h b/thirdparty/zstd/zstd.h index a1910ee223..72080ea87e 100644 --- a/thirdparty/zstd/zstd.h +++ b/thirdparty/zstd/zstd.h @@ -15,6 +15,7 @@ extern "C" { #define ZSTD_H_235446 /* ====== Dependency ======*/ +#include <limits.h> /* INT_MAX */ #include <stddef.h> /* size_t */ @@ -71,7 +72,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 4 -#define ZSTD_VERSION_RELEASE 1 +#define ZSTD_VERSION_RELEASE 4 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ @@ -196,9 +197,13 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /*! ZSTD_compressCCtx() : - * Same as ZSTD_compress(), using an explicit ZSTD_CCtx - * The function will compress at requested compression level, - * ignoring any other parameter */ + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to behave similarly to `ZSTD_compress()`, + * this function compresses at requested compression level, + * __ignoring any other parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only `compressionLevel` remains. + */ ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -233,7 +238,7 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, * using ZSTD_CCtx_set*() functions. * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! - * They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx() + * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . * * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). * @@ -261,18 +266,26 @@ typedef enum { /* compression parameters * Note: When compressing with a ZSTD_CDict these parameters are superseded - * by the parameters used to construct the ZSTD_CDict. See ZSTD_CCtx_refCDict() - * for more info (superseded-by-cdict). */ - ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). * Default level is ZSTD_CLEVEL_DEFAULT==3. * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. * Note 1 : it's possible to pass a negative compression level. - * Note 2 : setting a level sets all default values of other compression parameters */ + * Note 2 : setting a level resets all other compression parameters to default */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. * Special: value 0 means "use default windowLog". * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT - * requires explicitly allowing such window size at decompression stage if using streaming. */ + * requires explicitly allowing such size at streaming decompression stage. */ ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. * Resulting memory usage is (1 << (hashLog+2)). * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. @@ -283,13 +296,13 @@ typedef enum { * Resulting memory usage is (1 << (chainLog+2)). * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. * Larger tables result in better and slower compression. - * This parameter is useless when using "fast" strategy. + * This parameter is useless for "fast" strategy. * It's still useful when using "dfast" strategy, * in which case it defines a secondary probe table. * Special: value 0 means "use default chainLog". */ ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. * More attempts result in better and slower compression. - * This parameter is useless when using "fast" and "dFast" strategies. + * This parameter is useless for "fast" and "dFast" strategies. * Special: value 0 means "use default searchLog". */ ZSTD_c_minMatch=105, /* Minimum size of searched matches. * Note that Zstandard can still find matches of smaller size, @@ -344,7 +357,7 @@ typedef enum { ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) * Content size must be known at the beginning of compression. * This is automatically the case when using ZSTD_compress2(), - * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ @@ -363,7 +376,7 @@ typedef enum { * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. * 0 means default, which is dynamically determined based on compression parameters. * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. - * The minimum size is automatically and transparently enforced */ + * The minimum size is automatically and transparently enforced. */ ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. * It helps preserve compression ratio, while each job is compressed in parallel. @@ -386,6 +399,7 @@ typedef enum { * ZSTD_c_forceAttachDict * ZSTD_c_literalCompressionMode * ZSTD_c_targetCBlockSize + * ZSTD_c_srcSizeHint * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -396,6 +410,7 @@ typedef enum { ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, ZSTD_c_experimentalParam6=1003, + ZSTD_c_experimentalParam7=1004 } ZSTD_cParameter; typedef struct { @@ -793,12 +808,17 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, typedef struct ZSTD_CDict_s ZSTD_CDict; /*! ZSTD_createCDict() : - * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once. - * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost. + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict. - * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content. - * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */ + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel); @@ -925,7 +945,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. * It's a CPU consuming operation, with non-negligible impact on latency. * If there is a need to use the same prefix multiple times, consider loadDictionary instead. - * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); @@ -969,7 +989,7 @@ ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. * Prefix buffer must remain unmodified up to the end of frame, * reached when ZSTD_decompressStream() returns 0. - * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. * A full dictionary is more costly, as it requires building tables. @@ -1014,8 +1034,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * Some of them might be removed in the future (especially when redundant with existing stable functions) * ***************************************************************************************/ -#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */ -#define ZSTD_FRAMEHEADERSIZE_MIN 6 +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ #define ZSTD_SKIPPABLEHEADERSIZE 8 @@ -1063,6 +1083,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); /* Advanced parameter bounds */ #define ZSTD_TARGETCBLOCKSIZE_MIN 64 #define ZSTD_TARGETCBLOCKSIZE_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_SRCSIZEHINT_MIN 0 +#define ZSTD_SRCSIZEHINT_MAX INT_MAX /* internal */ #define ZSTD_HASHLOG3_MAX 17 @@ -1073,6 +1095,24 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); typedef struct ZSTD_CCtx_params_s ZSTD_CCtx_params; typedef struct { + unsigned int matchPos; /* Match pos in dst */ + /* If seqDef.offset > 3, then this is seqDef.offset - 3 + * If seqDef.offset < 3, then this is the corresponding repeat offset + * But if seqDef.offset < 3 and litLength == 0, this is the + * repeat offset before the corresponding repeat offset + * And if seqDef.offset == 3 and litLength == 0, this is the + * most recent repeat offset - 1 + */ + unsigned int offset; + unsigned int litLength; /* Literal length */ + unsigned int matchLength; /* Match length */ + /* 0 when seq not rep and seqDef.offset otherwise + * when litLength == 0 this will be <= 4, otherwise <= 3 like normal + */ + unsigned int rep; +} ZSTD_Sequence; + +typedef struct { unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ unsigned hashLog; /**< dispatch table : larger == faster, more memory */ @@ -1101,21 +1141,12 @@ typedef enum { typedef enum { ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ - ZSTD_dlm_byRef = 1, /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ } ZSTD_dictLoadMethod_e; typedef enum { - /* Opened question : should we have a format ZSTD_f_auto ? - * Today, it would mean exactly the same as ZSTD_f_zstd1. - * But, in the future, should several formats become supported, - * on the compression side, it would mean "default format". - * On the decompression side, it would mean "automatic format detection", - * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames". - * Since meaning is a little different, another option could be to define different enums for compression and decompression. - * This question could be kept for later, when there are actually multiple formats to support, - * but there is also the question of pinning enum values, and pinning value `0` is especially important */ ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ - ZSTD_f_zstd1_magicless = 1, /* Variant of zstd frame format, without initial 4-bytes magic number. + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. * Useful to save 4 bytes per generated frame. * Decoder cannot recognise automatically this format, requiring this instruction. */ } ZSTD_format_e; @@ -1126,7 +1157,7 @@ typedef enum { * to evolve and should be considered only in the context of extremely * advanced performance tuning. * - * Zstd currently supports the use of a CDict in two ways: + * Zstd currently supports the use of a CDict in three ways: * * - The contents of the CDict can be copied into the working context. This * means that the compression can search both the dictionary and input @@ -1142,6 +1173,12 @@ typedef enum { * working context's tables can be reused). For small inputs, this can be * faster than copying the CDict's tables. * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * * Zstd has a simple internal heuristic that selects which strategy to use * at the beginning of a compression. However, if experimentation shows that * Zstd is making poor choices, it is possible to override that choice with @@ -1150,6 +1187,7 @@ typedef enum { ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ } ZSTD_dictAttachPref_e; typedef enum { @@ -1158,7 +1196,7 @@ typedef enum { * levels will be compressed. */ ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be * emitted if Huffman compression is not profitable. */ - ZSTD_lcm_uncompressed = 2, /**< Always emit uncompressed literals. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ } ZSTD_literalCompressionMode_e; @@ -1210,20 +1248,38 @@ ZSTDLIB_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcS * or an error code (if srcSize is too small) */ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); +/*! ZSTD_getSequences() : + * Extract sequences from the sequence store + * zc can be used to insert custom compression params. + * This function invokes ZSTD_compress2 + * @return : number of sequences extracted + */ +ZSTDLIB_API size_t ZSTD_getSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize); + /*************************************** * Memory management ***************************************/ /*! ZSTD_estimate*() : - * These functions make it possible to estimate memory usage - * of a future {D,C}Ctx, before its creation. - * ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one. - * It will also consider src size to be arbitrarily "large", which is worst case. - * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. - * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. - * Note : CCtx size estimation is only correct for single-threaded compression. */ + * These functions make it possible to estimate memory usage of a future + * {D,C}Ctx, before its creation. + * + * ZSTD_estimateCCtxSize() will provide a budget large enough for any + * compression level up to selected one. Unlike ZSTD_estimateCStreamSize*(), + * this estimate does not include space for a window buffer, so this estimate + * is guaranteed to be enough for single-shot compressions, but not streaming + * compressions. It will however assume the input may be arbitrarily large, + * which is the worst case. If srcSize is known to always be small, + * ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with + * ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with + * ZSTD_CCtxParams_setParameter(). + * + * Note: only single-threaded compression is supported. This function will + * return an error code if ZSTD_c_nbWorkers is >= 1. */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); @@ -1334,7 +1390,8 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS * Create a digested dictionary for compression * Dictionary content is just referenced, not duplicated. * As a consequence, `dictBuffer` **must** outlive CDict, - * and its content must remain unmodified throughout the lifetime of CDict. */ + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); /*! ZSTD_getCParams() : @@ -1361,7 +1418,9 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); /*! ZSTD_compress_advanced() : - * Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */ + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will be marked as deprecated and generate compilation warning on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -1369,7 +1428,9 @@ ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, ZSTD_parameters params); /*! ZSTD_compress_usingCDict_advanced() : - * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */ + * Note : this function is now REDUNDANT. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will be marked as deprecated and generate compilation warning in some future version */ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -1441,6 +1502,12 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* pre * There is no guarantee on compressed block size (default:0) */ #define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. @@ -1613,8 +1680,13 @@ ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( * pledgedSrcSize must be correct. If it is not known at init time, use * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, * "0" also disables frame content size field. It may be enabled in the future. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t +ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + /**! ZSTD_initCStream_usingDict() : * This function is deprecated, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); @@ -1623,42 +1695,66 @@ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLe * * Creates of an internal CDict (incompatible with static CCtx), except if * dict == NULL or dictSize < 8, in which case no dict is used. - * Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t +ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + /**! ZSTD_initCStream_advanced() : * This function is deprecated, and is approximately equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setZstdParams(zcs, params); // Set the zstd params and leave the rest as-is + * // Pseudocode: Set each zstd parameter and leave the rest as-is. + * for ((param, value) : params) { + * ZSTD_CCtx_setParameter(zcs, param, value); + * } * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); * - * pledgedSrcSize must be correct. If srcSize is not known at init time, use - * value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t +ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + /**! ZSTD_initCStream_usingCDict() : * This function is deprecated, and equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, cdict); * * note : cdict will just be referenced, and must outlive compression session + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + /**! ZSTD_initCStream_usingCDict_advanced() : - * This function is deprecated, and is approximately equivalent to: + * This function is DEPRECATED, and is approximately equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * ZSTD_CCtx_setZstdFrameParams(zcs, fParams); // Set the zstd frame params and leave the rest as-is + * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. + * for ((fParam, value) : fParams) { + * ZSTD_CCtx_setParameter(zcs, fParam, value); + * } * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_refCDict(zcs, cdict); * * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. * pledgedSrcSize must be correct. If srcSize is not known at init time, use * value ZSTD_CONTENTSIZE_UNKNOWN. + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ -ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t +ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); /*! ZSTD_resetCStream() : * This function is deprecated, and is equivalent to: @@ -1673,6 +1769,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); @@ -1718,8 +1815,10 @@ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); * * note: no dictionary will be used if dict == NULL or dictSize < 8 + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + /** * This function is deprecated, and is equivalent to: * @@ -1727,14 +1826,17 @@ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dic * ZSTD_DCtx_refDDict(zds, ddict); * * note : ddict is referenced, it must outlive decompression session + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + /** * This function is deprecated, and is equivalent to: * * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); * * re-use decompression parameters from previous init; saves dictionary loading + * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); @@ -1908,8 +2010,8 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); /*! Block functions produce and decode raw zstd blocks, without frame metadata. - Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). - User will have to take in charge required information to regenerate data, such as compressed and content sizes. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. A few rules to respect : - Compressing and decompressing require a context structure @@ -1920,12 +2022,14 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + copyCCtx() and copyDCtx() can be used too - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + If input is larger than a block size, it's necessary to split input data into multiple blocks - + For inputs larger than a single block, really consider using regular ZSTD_compress() instead. - Frame metadata is not that costly, and quickly becomes negligible as source size grows larger. - - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. - In which case, nothing is produced into `dst` ! - + User must test for such outcome and deal directly with uncompressed data - + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + In case of multiple successive blocks, should some of them be uncompressed, decoder must be informed of their existence in order to follow proper history. Use ZSTD_insertBlock() for such a case. |