From 5b1d3c6bce47c9cca1528e3f4e2f4e4131c84838 Mon Sep 17 00:00:00 2001 From: d1str4ught <> Date: Sun, 21 Sep 2025 05:28:55 +0200 Subject: [PATCH] new pack system --- src/AudioLib/CMakeLists.txt | 3 +- src/AudioLib/SoundEngine.cpp | 11 +- src/CMakeLists.txt | 2 +- src/EffectLib/CMakeLists.txt | 3 +- src/EffectLib/EffectMesh.cpp | 16 +- src/EterBase/CMakeLists.txt | 1 + src/EterBase/MappedFile.cpp | 259 --- src/EterBase/MappedFile.h | 59 - src/EterGrnLib/CMakeLists.txt | 3 +- src/EterImageLib/CMakeLists.txt | 3 +- src/EterImageLib/TGAImage.cpp | 364 ---- src/EterImageLib/TGAImage.h | 37 - src/EterLib/CMakeLists.txt | 2 +- src/EterLib/FileLoaderThread.cpp | 15 +- src/EterLib/FileLoaderThread.h | 7 +- src/EterLib/GrpImageTexture.cpp | 11 +- src/EterLib/Resource.cpp | 18 +- src/EterLib/ResourceManager.cpp | 7 +- src/EterLib/TargaResource.cpp | 55 - src/EterLib/TargaResource.h | 32 - src/EterLib/TextFileLoader.cpp | 15 +- src/EterLib/TextFileLoader.h | 1 - src/EterLib/Util.cpp | 17 +- src/EterLocale/CMakeLists.txt | 6 +- src/EterPack/CMakeLists.txt | 10 - src/EterPack/EterPack.cpp | 1613 --------------- src/EterPack/EterPack.h | 245 --- src/EterPack/EterPackCursor.cpp | 59 - src/EterPack/EterPackCursor.h | 25 - src/EterPack/EterPackManager.cpp | 621 ------ src/EterPack/EterPackManager.h | 96 - src/EterPack/EterPackPolicy_CSHybridCrypt.cpp | 503 ----- src/EterPack/EterPackPolicy_CSHybridCrypt.h | 94 - src/EterPack/Inline.h | 34 - src/EterPack/StdAfx.cpp | 2 - src/EterPack/StdAfx.h | 7 - src/EterPack/md5.c | 1033 ---------- src/EterPack/md5.h | 128 -- src/EterPythonLib/CMakeLists.txt | 2 +- src/GameLib/AreaTerrain.cpp | 11 +- src/GameLib/CMakeLists.txt | 2 + src/GameLib/ItemManager.cpp | 41 +- src/GameLib/MapManager.cpp | 11 +- src/GameLib/MapOutdoorLoad.cpp | 12 +- src/GameLib/Property.cpp | 48 - src/GameLib/Property.h | 2 - src/GameLib/PropertyManager.cpp | 137 +- src/GameLib/PropertyManager.h | 17 +- src/GameLib/RaceManager.cpp | 9 +- src/PRTerrainLib/CMakeLists.txt | 2 +- src/PRTerrainLib/Terrain.cpp | 35 +- src/PRTerrainLib/Terrain.h | 4 - src/PackLib/CMakeLists.txt | 11 + src/PackLib/Pack.cpp | 71 + src/PackLib/Pack.h | 21 + src/PackLib/PackManager.cpp | 40 + src/PackLib/PackManager.h | 22 + src/PackLib/config.h | 43 + src/PackMaker/main.cpp | 50 +- src/ScriptLib/CMakeLists.txt | 2 +- src/ScriptLib/PythonLauncher.cpp | 28 +- src/SpeedTreeLib/CMakeLists.txt | 2 +- src/SpeedTreeLib/SpeedTreeForest.cpp | 13 +- src/UserInterface/AccountConnector.cpp | 6 +- src/UserInterface/CMakeLists.txt | 3 +- src/UserInterface/Locale.cpp | 1 - src/UserInterface/MarkImage.cpp | 1 + src/UserInterface/PythonApplicationModule.cpp | 12 +- src/UserInterface/PythonBackground.cpp | 16 +- src/UserInterface/PythonEventManager.cpp | 12 +- src/UserInterface/PythonMiniMap.cpp | 4 +- .../PythonNetworkStreamPhaseGame.cpp | 1 - .../PythonNetworkStreamPhaseHandShake.cpp | 4 +- .../PythonNetworkStreamPhaseLoading.cpp | 20 +- src/UserInterface/PythonNonPlayer.cpp | 17 +- src/UserInterface/PythonPackModule.cpp | 12 +- src/UserInterface/PythonSkill.cpp | 16 +- src/UserInterface/UserInterface.cpp | 184 +- vendor/CMakeLists.txt | 1 + vendor/mio/.gitignore | 5 + vendor/mio/.travis.yml | 50 + vendor/mio/CMakeLists.txt | 223 +++ vendor/mio/LICENSE | 21 + vendor/mio/README.md | 339 ++++ vendor/mio/cmake/CTestCustom.cmake | 3 + vendor/mio/cmake/mio-config.cmake.in | 13 + vendor/mio/include/mio/CMakeLists.txt | 11 + vendor/mio/include/mio/detail/CMakeLists.txt | 3 + vendor/mio/include/mio/detail/mmap.ipp | 533 +++++ vendor/mio/include/mio/detail/string_util.hpp | 170 ++ vendor/mio/include/mio/mmap.hpp | 492 +++++ vendor/mio/include/mio/page.hpp | 78 + vendor/mio/include/mio/shared_mmap.hpp | 406 ++++ vendor/mio/single_include/mio/mio.hpp | 1763 +++++++++++++++++ vendor/mio/test/CMakeLists.txt | 25 + vendor/mio/test/example.cpp | 75 + vendor/mio/test/test.cpp | 182 ++ vendor/mio/third_party/LICENSE.md | 28 + vendor/mio/third_party/amalgamate.py | 300 +++ vendor/mio/third_party/config.json | 11 + 100 files changed, 5269 insertions(+), 5825 deletions(-) delete mode 100644 src/EterBase/MappedFile.cpp delete mode 100644 src/EterBase/MappedFile.h delete mode 100644 src/EterImageLib/TGAImage.cpp delete mode 100644 src/EterImageLib/TGAImage.h delete mode 100644 src/EterLib/TargaResource.cpp delete mode 100644 src/EterLib/TargaResource.h delete mode 100644 src/EterPack/CMakeLists.txt delete mode 100644 src/EterPack/EterPack.cpp delete mode 100644 src/EterPack/EterPack.h delete mode 100644 src/EterPack/EterPackCursor.cpp delete mode 100644 src/EterPack/EterPackCursor.h delete mode 100644 src/EterPack/EterPackManager.cpp delete mode 100644 src/EterPack/EterPackManager.h delete mode 100644 src/EterPack/EterPackPolicy_CSHybridCrypt.cpp delete mode 100644 src/EterPack/EterPackPolicy_CSHybridCrypt.h delete mode 100644 src/EterPack/Inline.h delete mode 100644 src/EterPack/StdAfx.cpp delete mode 100644 src/EterPack/StdAfx.h delete mode 100644 src/EterPack/md5.c delete mode 100644 src/EterPack/md5.h create mode 100644 src/PackLib/CMakeLists.txt create mode 100644 src/PackLib/Pack.cpp create mode 100644 src/PackLib/Pack.h create mode 100644 src/PackLib/PackManager.cpp create mode 100644 src/PackLib/PackManager.h create mode 100644 src/PackLib/config.h create mode 100644 vendor/mio/.gitignore create mode 100644 vendor/mio/.travis.yml create mode 100644 vendor/mio/CMakeLists.txt create mode 100644 vendor/mio/LICENSE create mode 100644 vendor/mio/README.md create mode 100644 vendor/mio/cmake/CTestCustom.cmake create mode 100644 vendor/mio/cmake/mio-config.cmake.in create mode 100644 vendor/mio/include/mio/CMakeLists.txt create mode 100644 vendor/mio/include/mio/detail/CMakeLists.txt create mode 100644 vendor/mio/include/mio/detail/mmap.ipp create mode 100644 vendor/mio/include/mio/detail/string_util.hpp create mode 100644 vendor/mio/include/mio/mmap.hpp create mode 100644 vendor/mio/include/mio/page.hpp create mode 100644 vendor/mio/include/mio/shared_mmap.hpp create mode 100644 vendor/mio/single_include/mio/mio.hpp create mode 100644 vendor/mio/test/CMakeLists.txt create mode 100644 vendor/mio/test/example.cpp create mode 100644 vendor/mio/test/test.cpp create mode 100644 vendor/mio/third_party/LICENSE.md create mode 100644 vendor/mio/third_party/amalgamate.py create mode 100644 vendor/mio/third_party/config.json diff --git a/src/AudioLib/CMakeLists.txt b/src/AudioLib/CMakeLists.txt index 2493e62..efc5312 100644 --- a/src/AudioLib/CMakeLists.txt +++ b/src/AudioLib/CMakeLists.txt @@ -3,7 +3,8 @@ file(GLOB_RECURSE FILE_SOURCES "*.h" "*.c" "*.cpp") add_library(AudioLib STATIC ${FILE_SOURCES}) target_link_libraries(AudioLib - lzo2 + cryptopp-static + mio ) GroupSourcesByFolder(AudioLib) diff --git a/src/AudioLib/SoundEngine.cpp b/src/AudioLib/SoundEngine.cpp index 87af183..580b812 100644 --- a/src/AudioLib/SoundEngine.cpp +++ b/src/AudioLib/SoundEngine.cpp @@ -3,7 +3,7 @@ #include "EterBase/Random.h" #include "EterBase/Timer.h" -#include "Eterpack/EterPackManager.h" +#include "PackLib/PackManager.h" SoundEngine::SoundEngine() { @@ -246,17 +246,16 @@ bool SoundEngine::Internal_LoadSoundFromPack(const std::string& name) { if (m_Files.find(name) == m_Files.end()) { - LPCVOID soundData; - CMappedFile soundFile; - if (!CEterPackManager::Instance().Get(soundFile, name.c_str(), &soundData)) + TPackFile soundFile; + if (!CPackManager::Instance().GetFile(name, soundFile)) { TraceError("Internal_LoadSoundFromPack: SoundEngine: Failed to register file '%s' - not found.", name.c_str()); return false; } auto& buffer = m_Files[name]; - buffer.resize(soundFile.Size()); - memcpy(buffer.data(), soundData, soundFile.Size()); + buffer.resize(soundFile.size()); + memcpy(buffer.data(), soundFile.data(), soundFile.size()); } return true; } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ae8f271..33b5778 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,7 +6,6 @@ add_subdirectory(EterGrnLib) add_subdirectory(EterImageLib) add_subdirectory(EterLib) add_subdirectory(EterLocale) -add_subdirectory(EterPack) add_subdirectory(EterPythonLib) add_subdirectory(GameLib) add_subdirectory(PRTerrainLib) @@ -15,3 +14,4 @@ add_subdirectory(SpeedTreeLib) add_subdirectory(SphereLib) add_subdirectory(UserInterface) add_subdirectory(PackMaker) +add_subdirectory(PackLib) diff --git a/src/EffectLib/CMakeLists.txt b/src/EffectLib/CMakeLists.txt index 245f3cc..43dd133 100644 --- a/src/EffectLib/CMakeLists.txt +++ b/src/EffectLib/CMakeLists.txt @@ -3,7 +3,8 @@ add_library(EffectLib STATIC ${FILE_SOURCES}) target_link_libraries(EffectLib - lzo2 + cryptopp-static + mio ) GroupSourcesByFolder(EffectLib) diff --git a/src/EffectLib/EffectMesh.cpp b/src/EffectLib/EffectMesh.cpp index e522ac0..b6224d3 100644 --- a/src/EffectLib/EffectMesh.cpp +++ b/src/EffectLib/EffectMesh.cpp @@ -1,7 +1,7 @@ #include "StdAfx.h" #include "Eterlib/StateManager.h" #include "Eterlib/ResourceManager.h" -#include "Eterpack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "EffectMesh.h" CDynamicPool CEffectMesh::SEffectMeshData::ms_kPool; @@ -192,15 +192,14 @@ BOOL CEffectMesh::__LoadData_Ver002(int iSize, const BYTE * c_pbBuf) if (0 == strExtension.compare("ifl")) { - LPCVOID pMotionData; - CMappedFile File; + TPackFile File; - if (CEterPackManager::Instance().Get(File, pMeshData->szDiffuseMapFileName, &pMotionData)) + if (CPackManager::Instance().GetFile(pMeshData->szDiffuseMapFileName, File)) { CMemoryTextFileLoader textFileLoader; std::vector stTokenVector; - textFileLoader.Bind(File.Size(), pMotionData); + textFileLoader.Bind(File.size(), File.data()); std::string strPathName; GetOnlyPathName(pMeshData->szDiffuseMapFileName, strPathName); @@ -337,15 +336,14 @@ BOOL CEffectMesh::__LoadData_Ver001(int iSize, const BYTE * c_pbBuf) if (0 == strExtension.compare("ifl")) { - LPCVOID pMotionData; - CMappedFile File; + TPackFile File; - if (CEterPackManager::Instance().Get(File, pMeshData->szDiffuseMapFileName, &pMotionData)) + if (CPackManager::Instance().GetFile(pMeshData->szDiffuseMapFileName, File)) { CMemoryTextFileLoader textFileLoader; std::vector stTokenVector; - textFileLoader.Bind(File.Size(), pMotionData); + textFileLoader.Bind(File.size(), File.data()); std::string strPathName; GetOnlyPathName(pMeshData->szDiffuseMapFileName, strPathName); diff --git a/src/EterBase/CMakeLists.txt b/src/EterBase/CMakeLists.txt index 9f28f3b..5b77398 100644 --- a/src/EterBase/CMakeLists.txt +++ b/src/EterBase/CMakeLists.txt @@ -5,6 +5,7 @@ add_library(EterBase STATIC ${FILE_SOURCES}) target_link_libraries(EterBase lzo2 cryptopp-static + mio ) GroupSourcesByFolder(EterBase) diff --git a/src/EterBase/MappedFile.cpp b/src/EterBase/MappedFile.cpp deleted file mode 100644 index 904ba6f..0000000 --- a/src/EterBase/MappedFile.cpp +++ /dev/null @@ -1,259 +0,0 @@ -#include "StdAfx.h" -#include "MappedFile.h" -#include "Debug.h" - -CMappedFile::CMappedFile() : -m_hFM(NULL), -m_lpMapData(NULL), -m_dataOffset(0), -m_mapSize(0), -m_seekPosition(0), -m_pLZObj(NULL), -m_pbBufLinkData(NULL), -m_dwBufLinkSize(0), -m_pbAppendResultDataBlock(NULL), -m_dwAppendResultDataSize(0) -{ -} - -CMappedFile::~CMappedFile() -{ - Destroy(); -} - -BOOL CMappedFile::Create(const char * filename) -{ - Destroy(); - return CFileBase::Create(filename, FILEMODE_READ); -} - -BOOL CMappedFile::Create(const char * filename, const void** dest, int offset, int size) -{ - if (!CMappedFile::Create(filename)) - return NULL; - - int ret = Map(dest, offset, size); - return (ret) > 0; -} - -LPCVOID CMappedFile::Get() -{ - return m_lpData; -} - -void CMappedFile::Link(DWORD dwBufSize, const void* c_pvBufData) -{ - m_dwBufLinkSize=dwBufSize; - m_pbBufLinkData=(BYTE*)c_pvBufData; -} - -void CMappedFile::BindLZObject(CLZObject * pLZObj) -{ - assert(m_pLZObj == NULL); - m_pLZObj = pLZObj; - - Link(m_pLZObj->GetSize(), m_pLZObj->GetBuffer()); -} - -void CMappedFile::BindLZObjectWithBufferedSize(CLZObject * pLZObj) -{ - assert(m_pLZObj == NULL); - m_pLZObj = pLZObj; - - Link(m_pLZObj->GetBufferSize(), m_pLZObj->GetBuffer()); -} - -BYTE* CMappedFile::AppendDataBlock( const void* pBlock, DWORD dwBlockSize ) -{ - if( m_pbAppendResultDataBlock ) - { - delete []m_pbAppendResultDataBlock; - } - - //realloc - m_dwAppendResultDataSize = m_dwBufLinkSize+dwBlockSize; - m_pbAppendResultDataBlock = new BYTE[m_dwAppendResultDataSize]; - - memcpy(m_pbAppendResultDataBlock, m_pbBufLinkData, m_dwBufLinkSize ); - memcpy(m_pbAppendResultDataBlock + m_dwBufLinkSize, pBlock, dwBlockSize ); - - //redirect - Link(m_dwAppendResultDataSize, m_pbAppendResultDataBlock); - - return m_pbAppendResultDataBlock; -} - -void CMappedFile::Destroy() -{ - if (m_pLZObj) // 압축된 데이터가 이 포인터로 연결 된다 - { - delete m_pLZObj; - m_pLZObj = NULL; - } - - if (NULL != m_lpMapData) - { - Unmap(m_lpMapData); - m_lpMapData = NULL; - } - - if (NULL != m_hFM) - { - CloseHandle(m_hFM); - m_hFM = NULL; - } - - if( m_pbAppendResultDataBlock ) - { - delete []m_pbAppendResultDataBlock; - m_pbAppendResultDataBlock = NULL; - } - - m_dwAppendResultDataSize = 0; - - m_pbBufLinkData = NULL; - m_dwBufLinkSize = 0; - - m_seekPosition = 0; - m_dataOffset = 0; - m_mapSize = 0; - - CFileBase::Destroy(); -} - -int CMappedFile::Seek(DWORD offset, int iSeekType) -{ - switch (iSeekType) - { - case SEEK_TYPE_BEGIN: - if (offset > m_dwSize) - offset = m_dwSize; - - m_seekPosition = offset; - break; - - case SEEK_TYPE_CURRENT: - m_seekPosition = std::min(m_seekPosition + offset, Size()); - break; - - case SEEK_TYPE_END: - m_seekPosition = std::max(0ul, Size() - offset); - break; - } - - return m_seekPosition; -} - -// 2004.09.16.myevan.MemoryMappedFile 98/ME 개수 제한 문제 체크 -//DWORD g_dwCount=0; - -int CMappedFile::Map(const void **dest, int offset, int size) -{ - m_dataOffset = offset; - - if (size == 0) - m_mapSize = m_dwSize; - else - m_mapSize = size; - - if (m_dataOffset + m_mapSize > m_dwSize) - return NULL; - - SYSTEM_INFO SysInfo; - GetSystemInfo(&SysInfo); - DWORD dwSysGran = SysInfo.dwAllocationGranularity; - DWORD dwFileMapStart = (m_dataOffset / dwSysGran) * dwSysGran; - DWORD dwMapViewSize = (m_dataOffset % dwSysGran) + m_mapSize; - INT iViewDelta = m_dataOffset - dwFileMapStart; - - - m_hFM = CreateFileMapping(m_hFile, // handle - NULL, // security - PAGE_READONLY, // flProtect - 0, // high - m_dataOffset + m_mapSize, // low - NULL); // name - - if (!m_hFM) - { - OutputDebugString("CMappedFile::Map !m_hFM\n"); - return NULL; - } - - m_lpMapData = MapViewOfFile(m_hFM, - FILE_MAP_READ, - 0, - dwFileMapStart, - dwMapViewSize); - - if (!m_lpMapData) // Success - { - TraceError("CMappedFile::Map !m_lpMapData %lu", GetLastError()); - return 0; - } - - // 2004.09.16.myevan.MemoryMappedFile 98/ME 개수 제한 문제 체크 - //g_dwCount++; - //Tracenf("MAPFILE %d", g_dwCount); - - m_lpData = (char*) m_lpMapData + iViewDelta; - *dest = (char*) m_lpData; - m_seekPosition = 0; - - Link(m_mapSize, m_lpData); - - return (m_mapSize); -} - -BYTE * CMappedFile::GetCurrentSeekPoint() -{ - return m_pbBufLinkData+m_seekPosition; - //return m_pLZObj ? m_pLZObj->GetBuffer() + m_seekPosition : (BYTE *) m_lpData + m_seekPosition; -} - - -DWORD CMappedFile::Size() -{ - return m_dwBufLinkSize; - /* - if (m_pLZObj) - return m_pLZObj->GetSize(); - - return (m_mapSize); - */ -} - -DWORD CMappedFile::GetPosition() -{ - return m_dataOffset; -} - -BOOL CMappedFile::Read(void * dest, int bytes) -{ - if (m_seekPosition + bytes > Size()) - return FALSE; - - memcpy(dest, GetCurrentSeekPoint(), bytes); - m_seekPosition += bytes; - return TRUE; -} - -DWORD CMappedFile::GetSeekPosition(void) -{ - return m_seekPosition; -} - -void CMappedFile::Unmap(LPCVOID data) -{ - if (UnmapViewOfFile(data)) - { - // 2004.09.16.myevan.MemoryMappedFile 98/ME 개수 제한 문제 체크 - //g_dwCount--; - //Tracenf("UNMAPFILE %d", g_dwCount); - } - else - { - TraceError("CMappedFile::Unmap - Error"); - } - m_lpData = NULL; -} diff --git a/src/EterBase/MappedFile.h b/src/EterBase/MappedFile.h deleted file mode 100644 index 6913099..0000000 --- a/src/EterBase/MappedFile.h +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef __INC_MAPPEDFILE_H__ -#define __INC_MAPPEDFILE_H__ - -#include "lzo.h" -#include "FileBase.h" - -class CMappedFile : public CFileBase -{ -public: - enum ESeekType - { - SEEK_TYPE_BEGIN, - SEEK_TYPE_CURRENT, - SEEK_TYPE_END - }; - -public: - CMappedFile(); - virtual ~CMappedFile(); - - void Link(DWORD dwBufSize, const void* c_pvBufData); - - BOOL Create(const char* filename); - BOOL Create(const char* filename, const void** dest, int offset, int size); - LPCVOID Get(); - void Destroy(); - int Seek(DWORD offset, int iSeekType = SEEK_TYPE_BEGIN); - int Map(const void **dest, int offset=0, int size=0); - DWORD Size(); - DWORD GetPosition(); - BOOL Read(void* dest, int bytes); - DWORD GetSeekPosition(); - void BindLZObject(CLZObject * pLZObj); - void BindLZObjectWithBufferedSize(CLZObject * pLZObj); - BYTE* AppendDataBlock( const void* pBlock, DWORD dwBlockSize ); - - BYTE * GetCurrentSeekPoint(); - -private: - void Unmap(LPCVOID data); - -private: - BYTE* m_pbBufLinkData; - DWORD m_dwBufLinkSize; - - BYTE* m_pbAppendResultDataBlock; - DWORD m_dwAppendResultDataSize; - - DWORD m_seekPosition; - HANDLE m_hFM; - DWORD m_dataOffset; - DWORD m_mapSize; - LPVOID m_lpMapData; - LPVOID m_lpData; - - CLZObject * m_pLZObj; -}; - -#endif diff --git a/src/EterGrnLib/CMakeLists.txt b/src/EterGrnLib/CMakeLists.txt index 63ab63a..0dd3c7b 100644 --- a/src/EterGrnLib/CMakeLists.txt +++ b/src/EterGrnLib/CMakeLists.txt @@ -3,7 +3,8 @@ add_library(EterGrnLib STATIC ${FILE_SOURCES}) target_link_libraries(EterGrnLib - lzo2 + cryptopp-static + mio ) GroupSourcesByFolder(EterGrnLib) diff --git a/src/EterImageLib/CMakeLists.txt b/src/EterImageLib/CMakeLists.txt index 369c697..791fa9b 100644 --- a/src/EterImageLib/CMakeLists.txt +++ b/src/EterImageLib/CMakeLists.txt @@ -3,7 +3,8 @@ add_library(EterImageLib STATIC ${FILE_SOURCES}) target_link_libraries(EterImageLib - lzo2 + cryptopp-static + mio ) GroupSourcesByFolder(EterImageLib) diff --git a/src/EterImageLib/TGAImage.cpp b/src/EterImageLib/TGAImage.cpp deleted file mode 100644 index 18ca0b3..0000000 --- a/src/EterImageLib/TGAImage.cpp +++ /dev/null @@ -1,364 +0,0 @@ -#include "StdAfx.h" - -#include - -#include "EterBase/MappedFile.h" -#include "TGAImage.h" - -CTGAImage::CTGAImage() : m_dwFlag(0) -{ -} - -CTGAImage::~CTGAImage() -{ -} - -CTGAImage::CTGAImage(CImage &image) : m_dwFlag(0) -{ - int w = image.GetWidth(); - int h = image.GetHeight(); - - Create(w, h); - - DWORD * pdwDest = GetBasePointer(); - memcpy(pdwDest, image.GetBasePointer(), w * h * sizeof(DWORD)); - FlipTopToBottom(); -} - -void CTGAImage::Create(int width, int height) -{ - memset(&m_Header, 0, sizeof(m_Header)); - - m_Header.imgType = 2; - m_Header.width = (short) width; - m_Header.height = (short) height; - m_Header.colorBits = 32; - m_Header.desc = 0x08; // alpha channel 있음 - - CImage::Create(width, height); -} - -bool CTGAImage::LoadFromMemory(int iSize, const BYTE * c_pbMem) -{ - memcpy(&m_Header, c_pbMem, 18); - c_pbMem += 18; - iSize -= 18; - - CImage::Create(m_Header.width, m_Header.height); - - UINT hxw = m_Header.width * m_Header.height; - BYTE r, g, b, a; - DWORD i; - - DWORD * pdwDest = GetBasePointer(); - - switch (m_Header.imgType) - { - case 3: // 알파만 있는 것 (1bytes per pixel, 거의 안쓰임) - { - for (i = 0; i < hxw; ++i) - { - a = (char) *(c_pbMem++); - pdwDest[i] = (a << 24) | (a << 16) | (a << 8) | a; - } - } - break; - - case 2: // 압축 안된 TGA - { - if (m_Header.colorBits == 16) // 16bit - { - for (i = 0; i < hxw; ++i) - { - WORD w; - - memcpy(&w, c_pbMem, sizeof(WORD)); - c_pbMem += sizeof(WORD); - iSize -= sizeof(WORD); - - b = (BYTE) (w & 0x1F); - g = (BYTE) ((w >> 5) & 0x1F); - r = (BYTE) ((w >> 10) & 0x1F); - - b <<= 3; - g <<= 3; - r <<= 3; - a = 0xff; - - pdwDest[i] = (a << 24) | (r << 16) | (g << 8) | b; - } - } - else if (m_Header.colorBits == 24) // 24bit - { - for (i = 0; i < hxw; ++i) - { - r = (BYTE) *(c_pbMem++); --iSize; - g = (BYTE) *(c_pbMem++); --iSize; - b = (BYTE) *(c_pbMem++); --iSize; - a = 0xff; - - pdwDest[i] = (a << 24) | (r << 16) | (g << 8) | b; - } - } - else if (m_Header.colorBits == 32) // 32bit - { - int size = GetWidth(); - size *= GetHeight() * 4; - - memcpy(pdwDest, c_pbMem, size); - c_pbMem += size; - iSize -= size; - } - } - break; - - case 10: // 압축 된 TGA (RLE) - { - BYTE rle; - - if (m_Header.colorBits == 24) - { - i = 0; - while (i < hxw) - { - rle = (BYTE) *(c_pbMem++); --iSize; - - if (rle < 0x80) // 압축 안된 곳 - { - rle++; - - while (rle) - { - b = (BYTE) *(c_pbMem++); --iSize; - g = (BYTE) *(c_pbMem++); --iSize; - r = (BYTE) *(c_pbMem++); --iSize; - a = 0xff; - pdwDest[i++] = (a << 24) | (r << 16) | (g << 8) | b; - - if (i > hxw) - { - assert(!"RLE overflow"); - printf("RLE overflow"); - return false; - } - --rle; - } - } - else - { - // 압축 된 곳 - rle -= 127; - - b = (BYTE) *(c_pbMem++); --iSize; - g = (BYTE) *(c_pbMem++); --iSize; - r = (BYTE) *(c_pbMem++); --iSize; - a = 0xff; - - while (rle) - { - pdwDest[i++] = (a << 24) | (r << 16) | (g << 8) | b; - - if (i > hxw) - { - assert(!"RLE overflow"); - printf("RLE overflow"); - return false; - } - --rle; - } - } - } - } - else if (m_Header.colorBits == 32) - { - i = 0; - while (i < hxw) - { - rle = (BYTE) *(c_pbMem++); --iSize; - - if (rle < 0x80) - { - rle++; - - while (rle) - { - b = (BYTE) *(c_pbMem++); --iSize; - g = (BYTE) *(c_pbMem++); --iSize; - r = (BYTE) *(c_pbMem++); --iSize; - a = (BYTE) *(c_pbMem++); --iSize; - pdwDest[i++] = (a << 24) | (r << 16) | (g << 8) | b; - - if (i > hxw) - { - assert(!"RLE overflow"); - printf("RLE overflow"); - return false; - } - --rle; - } - } - else - { - rle -= 127; - - b = (BYTE) *(c_pbMem++); --iSize; - g = (BYTE) *(c_pbMem++); --iSize; - r = (BYTE) *(c_pbMem++); --iSize; - a = (BYTE) *(c_pbMem++); --iSize; - - while (rle) - { - pdwDest[i++] = (a << 24) | (r << 16) | (g << 8) | b; - - if (i > hxw) - { - assert(!"RLE overflow"); - printf("RLE overflow"); - return false; - } - - --rle; - } - } - } - } - } - break; - } - - if (!(m_Header.desc & 0x20)) - { - FlipTopToBottom(); - } - - return true; -} - -bool CTGAImage::LoadFromDiskFile(const char * c_szFileName) -{ - CMappedFile file; - - const BYTE * c_pbMap; - - if (!file.Create(c_szFileName, (const void **) &c_pbMap, 0, 0)) - return false; - - return LoadFromMemory(file.Size(), c_pbMap); -} - -int CTGAImage::GetRLEPixelCount(const DWORD * data) -{ - int r = 0; - DWORD pixel; - - r = 1; - - if (data >= m_pdwEndPtr) - return 0; - - pixel = *data; - - while ((r < 127) && (data < m_pdwEndPtr)) - { - if (pixel != *(++data)) - return r; - - r++; - } - - return r; -} - -int CTGAImage::GetRawPixelCount(const DWORD * data) -{ - int i = 0; - - if (data >= m_pdwEndPtr) - return 0; - - while ((data < m_pdwEndPtr) && (i < 127)) - { - int rle = GetRLEPixelCount(data); - - if (rle >= 4) - break; - - data++; - i++; - } - - return i; -} - -void CTGAImage::SetCompressed(bool isCompress) -{ - if (isCompress) - m_Header.imgType = 10; - else - m_Header.imgType = 2; -} - -void CTGAImage::SetAlphaChannel(bool isExist) -{ - if (isExist) - m_Header.desc |= 0x08; - else - m_Header.desc &= ~0x08; -} - -bool CTGAImage::SaveToDiskFile(const char* c_szFileName) -{ - FILE * fp = fopen(c_szFileName, "wb"); - - if (!fp) - return false; - - fwrite(&m_Header, 18, 1, fp); - - if (m_Header.imgType == 10) // RLE 압축으로 저장 - { - DWORD * data = GetBasePointer(); - - while (data < m_pdwEndPtr) - { - int rle = GetRLEPixelCount(data); - - if (rle < 4) - { - int raw = GetRawPixelCount(data); - - if (raw == 0) - break; - - fputc(raw - 1, fp); - - while (raw) - { - fwrite(data, sizeof(DWORD), 1, fp); - data++; - raw--; - } - } - else - { - fputc((rle - 1) | 0x80, fp); - fwrite(data, sizeof(DWORD), 1, fp); - data += rle; - } - } - } - else - { - int size = GetWidth(); - size *= GetHeight() * 4; - fwrite(GetBasePointer(), size, 1, fp); - } - - fclose(fp); - return true; -} - -TGA_HEADER & CTGAImage::GetHeader() -{ - return m_Header; -} diff --git a/src/EterImageLib/TGAImage.h b/src/EterImageLib/TGAImage.h deleted file mode 100644 index 9b1e58c..0000000 --- a/src/EterImageLib/TGAImage.h +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef __INC_ETERIMAGELIB_TGAIMAGE_H__ -#define __INC_ETERIMAGELIB_TGAIMAGE_H__ - -#include "Image.h" - -class CTGAImage : public CImage -{ - public: - enum ETGAImageFlags - { - FLAG_RLE_COMPRESS = (1 << 0) - }; - - CTGAImage(); - CTGAImage(CImage &image); - virtual ~CTGAImage(); - - virtual void Create(int width, int height); - virtual bool LoadFromMemory(int iSize, const BYTE * c_pbMem); - virtual bool LoadFromDiskFile(const char * c_szFileName); - virtual bool SaveToDiskFile(const char* c_szFileName); - - void SetCompressed(bool isCompress = true); - void SetAlphaChannel(bool isExist = true); - - TGA_HEADER & GetHeader(); - protected: - int GetRawPixelCount(const DWORD * data); - int GetRLEPixelCount(const DWORD * data); - - protected: - TGA_HEADER m_Header; - DWORD m_dwFlag; - DWORD * m_pdwEndPtr; -}; - -#endif diff --git a/src/EterLib/CMakeLists.txt b/src/EterLib/CMakeLists.txt index 5bfc36a..cac00b3 100644 --- a/src/EterLib/CMakeLists.txt +++ b/src/EterLib/CMakeLists.txt @@ -3,8 +3,8 @@ add_library(EterLib STATIC ${FILE_SOURCES}) target_link_libraries(EterLib - lzo2 cryptopp-static + mio ) GroupSourcesByFolder(EterLib) diff --git a/src/EterLib/FileLoaderThread.cpp b/src/EterLib/FileLoaderThread.cpp index dd41160..46eb531 100644 --- a/src/EterLib/FileLoaderThread.cpp +++ b/src/EterLib/FileLoaderThread.cpp @@ -1,5 +1,6 @@ #include "StdAfx.h" -#include "EterPack/EterPackManager.h" +#include "EterBase/Stl.h" +#include "PackLib/PackManager.h" #include "FileLoaderThread.h" #include "ResourceManager.h" @@ -115,8 +116,7 @@ void CFileLoaderThread::Request(std::string & c_rstFileName) // called in main t { TData * pData = new TData; - pData->dwSize = 0; - pData->pvBuf = NULL; + pData->File.clear(); pData->stFileName = c_rstFileName; m_RequestMutex.Lock(); @@ -163,14 +163,7 @@ void CFileLoaderThread::Process() // called in loader thread m_RequestMutex.Unlock(); - LPCVOID pvBuf; - - if (CEterPackManager::Instance().Get(pData->File, pData->stFileName.c_str(), &pvBuf)) - { - pData->dwSize = pData->File.Size(); - pData->pvBuf = new char [pData->dwSize]; - memcpy(pData->pvBuf, pvBuf, pData->dwSize); - } + CPackManager::instance().GetFile(pData->stFileName, pData->File); m_CompleteMutex.Lock(); m_pCompleteDeque.push_back(pData); diff --git a/src/EterLib/FileLoaderThread.h b/src/EterLib/FileLoaderThread.h index 1709a95..5fc0f65 100644 --- a/src/EterLib/FileLoaderThread.h +++ b/src/EterLib/FileLoaderThread.h @@ -4,7 +4,7 @@ #include #include "Thread.h" #include "Mutex.h" -#include "EterBase/MappedFile.h" +#include "PackLib/PackManager.h" class CFileLoaderThread { @@ -12,10 +12,7 @@ class CFileLoaderThread typedef struct SData { std::string stFileName; - - CMappedFile File; - LPVOID pvBuf; - DWORD dwSize; + TPackFile File; } TData; public: diff --git a/src/EterLib/GrpImageTexture.cpp b/src/EterLib/GrpImageTexture.cpp index 9ad60a3..05582eb 100644 --- a/src/EterLib/GrpImageTexture.cpp +++ b/src/EterLib/GrpImageTexture.cpp @@ -1,6 +1,5 @@ #include "StdAfx.h" -#include "EterBase/MappedFile.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "GrpImageTexture.h" #include "EterImageLib/DDSTextureLoader9.h" @@ -53,13 +52,11 @@ bool CGraphicImageTexture::CreateDeviceObjects() } else { - CMappedFile mappedFile; - LPCVOID c_pvMap; - - if (!CEterPackManager::Instance().Get(mappedFile, m_stFileName.c_str(), &c_pvMap)) + TPackFile mappedFile; + if (!CPackManager::Instance().GetFile(m_stFileName, mappedFile)) return false; - return CreateFromMemoryFile(mappedFile.Size(), c_pvMap, m_d3dFmt, m_dwFilter); + return CreateFromMemoryFile(mappedFile.size(), mappedFile.data(), m_d3dFmt, m_dwFilter); } m_bEmpty = false; diff --git a/src/EterLib/Resource.cpp b/src/EterLib/Resource.cpp index 02b0cfc..d21c0b0 100644 --- a/src/EterLib/Resource.cpp +++ b/src/EterLib/Resource.cpp @@ -1,5 +1,6 @@ #include "StdAfx.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" +#include "EterBase/Stl.h" #include "EterBase/CRC32.h" #include "EterBase/Timer.h" @@ -43,17 +44,16 @@ void CResource::Load() const char * c_szFileName = GetFileName(); DWORD dwStart = ELTimer_GetMSec(); - CMappedFile file; - LPCVOID fileData; + TPackFile file; //Tracenf("Load %s", c_szFileName); - if (CEterPackManager::Instance().Get(file, c_szFileName, &fileData)) + if (CPackManager::Instance().GetFile(c_szFileName, file)) { m_dwLoadCostMiliiSecond = ELTimer_GetMSec() - dwStart; //Tracef("CResource::Load %s (%d bytes) in %d ms\n", c_szFileName, file.Size(), m_dwLoadCostMiliiSecond); - if (OnLoad(file.Size(), fileData)) + if (OnLoad(file.size(), file.data())) { me_state = STATE_EXIST; } @@ -81,12 +81,10 @@ void CResource::Reload() Clear(); Tracef("CResource::Reload %s\n", GetFileName()); - CMappedFile file; - LPCVOID fileData; - - if (CEterPackManager::Instance().Get(file, GetFileName(), &fileData)) + TPackFile file; + if (CPackManager::Instance().GetFile(GetFileName(), file)) { - if (OnLoad(file.Size(), fileData)) + if (OnLoad(file.size(), file.data())) { me_state = STATE_EXIST; } diff --git a/src/EterLib/ResourceManager.cpp b/src/EterLib/ResourceManager.cpp index 5807788..b25355a 100644 --- a/src/EterLib/ResourceManager.cpp +++ b/src/EterLib/ResourceManager.cpp @@ -3,7 +3,7 @@ #include "EterBase/CRC32.h" #include "EterBase/Timer.h" #include "EterBase/Stl.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "ResourceManager.h" #include "GrpImage.h" @@ -71,7 +71,7 @@ void CResourceManager::ProcessBackgroundLoading() { if (pResource->IsEmpty()) { - pResource->OnLoad(pData->dwSize, pData->pvBuf); + pResource->OnLoad(pData->File.size(), pData->File.data()); pResource->AddReferenceOnly(); // 여기서 올라간 레퍼런스 카운트를 일정 시간이 지난 뒤에 풀어주기 위하여 @@ -81,7 +81,6 @@ void CResourceManager::ProcessBackgroundLoading() m_WaitingMap.erase(GetCRC32(pData->stFileName.c_str(), pData->stFileName.size())); - delete [] ((char *) pData->pvBuf); delete pData; } @@ -472,7 +471,7 @@ void CResourceManager::DumpFileListToTextFile(const char* c_szFileName) bool CResourceManager::IsFileExist(const char * c_szFileName) { - return CEterPackManager::Instance().isExist(c_szFileName); + return CPackManager::Instance().IsExist(c_szFileName); } void CResourceManager::Update() diff --git a/src/EterLib/TargaResource.cpp b/src/EterLib/TargaResource.cpp deleted file mode 100644 index bbe978c..0000000 --- a/src/EterLib/TargaResource.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#include "StdAfx.h" -#include "TargaResource.h" - -CTargaResource::CTargaResource(const char * c_pszFileName) : CResource(c_pszFileName) -{ -} - -CTargaResource::~CTargaResource() -{ -} - -CTargaResource::TType CTargaResource::Type() -{ - static TType s_type = StringToType("CTargaResource"); - return s_type; -} - -bool CTargaResource::OnIsType(TType type) -{ - if (CTargaResource::Type() == type) - return true; - - return CResource::OnIsType(type); -} - -bool CTargaResource::OnLoad(int iSize, const void * c_pvBuf) -{ - return image.LoadFromMemory(iSize, static_cast(c_pvBuf)); -} - -void CTargaResource::OnClear() -{ - image.Clear(); -} - -bool CTargaResource::OnIsEmpty() const -{ - return image.IsEmpty(); -} - -void CTargaResource::GetRect(DWORD & w, DWORD & h) -{ - w = image.GetWidth(); - h = image.GetHeight(); -} - -DWORD * CTargaResource::GetMemPtr() -{ - return image.GetBasePointer(); -} - -TGA_HEADER & CTargaResource::GetTgaHeader() -{ - return image.GetHeader(); -} diff --git a/src/EterLib/TargaResource.h b/src/EterLib/TargaResource.h deleted file mode 100644 index 750ae1e..0000000 --- a/src/EterLib/TargaResource.h +++ /dev/null @@ -1,32 +0,0 @@ -#pragma once - -#include "Ref.h" -#include "Resource.h" -#include "EterImageLib/TGAImage.h" - -class CTargaResource : public CResource -{ - public: - typedef CRef TRef; - - public: - static TType Type(); - - public: - CTargaResource(const char * c_pszFileName); - virtual ~CTargaResource(); - - DWORD * GetMemPtr(); - void GetRect(DWORD & w, DWORD & h); - - TGA_HEADER & GetTgaHeader(); - - protected: - virtual bool OnLoad(int iSize, const void * c_pvBuf); - virtual void OnClear(); - virtual bool OnIsEmpty() const; - virtual bool OnIsType(TType type); - - protected: - CTGAImage image; -}; diff --git a/src/EterLib/TextFileLoader.cpp b/src/EterLib/TextFileLoader.cpp index cb76922..5a2de62 100644 --- a/src/EterLib/TextFileLoader.cpp +++ b/src/EterLib/TextFileLoader.cpp @@ -1,7 +1,7 @@ #include "StdAfx.h" #include "EterBase/CRC32.h" #include -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "Pool.h" #include "TextFileLoader.h" @@ -179,14 +179,13 @@ bool CTextFileLoader::Load(const char * c_szFileName) { m_strFileName = ""; - const VOID* pvData; - CMappedFile kFile; - if (!CEterPackManager::Instance().Get(kFile, c_szFileName, &pvData)) + TPackFile kFile; + if (!CPackManager::Instance().GetFile( c_szFileName, kFile)) return false; - if (m_dwBufCapacity -#include -#include -#include - -#include "EterPack.h" -#include "Inline.h" -#include "EterPackPolicy_CSHybridCrypt.h" - -#pragma warning(push, 3) -#include -#include -#include -#include -#include -#include -#include -#include -#pragma warning(pop) - -#include "EterBase/utils.h" -#include "EterBase/Debug.h" -#include "EterBase/CRC32.h" - -#ifdef __THEMIDA__ -#include -#endif - -#include -#include - -void CMakePackLog::SetFileName(const char* c_szFileName) -{ - m_stFileName=c_szFileName; - m_stFileName +=".log"; - m_stErrorFileName = c_szFileName; - m_stErrorFileName += ".err"; -} - -CMakePackLog& CMakePackLog::GetSingleton() -{ - static CMakePackLog s_kMakePackLog; - return s_kMakePackLog; -} - -CMakePackLog::CMakePackLog() -{ - m_fp=NULL; - m_fp_err = NULL; -} - -CMakePackLog::~CMakePackLog() -{ - if (NULL!=m_fp) - { - fclose(m_fp); - m_fp=NULL; - } - if (NULL != m_fp_err) - { - fclose(m_fp_err); - m_fp_err = NULL; - } -} - -bool CMakePackLog::__IsLogMode() -{ - if (m_stFileName.empty()) - return false; - - return true; -} - -void CMakePackLog::Writef(const char* c_szFormat, ...) -{ - if (!__IsLogMode()) - return; - - va_list args; - va_start(args, c_szFormat); - - char szBuf[1024]; - int nBufLen = _vsnprintf(szBuf, sizeof(szBuf), c_szFormat, args); - szBuf[nBufLen++] = '\0'; - __Write(szBuf, nBufLen); -} - -void CMakePackLog::Writenf(const char* c_szFormat, ...) -{ - if (!__IsLogMode()) - return; - - va_list args; - va_start(args, c_szFormat); - - char szBuf[1024+1]; - int nBufLen = _vsnprintf(szBuf, sizeof(szBuf)-1, c_szFormat, args); - if (nBufLen > 0) - { - szBuf[nBufLen++] = '\n'; - szBuf[nBufLen++] = '\0'; - } - __Write(szBuf, nBufLen); -} - -void CMakePackLog::Write(const char* c_szBuf) -{ - if (!__IsLogMode()) - return; - - __Write(c_szBuf, strlen(c_szBuf)+1); -} - -void CMakePackLog::__Write(const char* c_szBuf, int nBufLen) -{ - if (!__IsLogMode()) - return; - - if (NULL==m_fp) - m_fp=fopen(m_stFileName.c_str(), "w"); - - fwrite(c_szBuf, nBufLen, 1, m_fp); - - printf("%s", c_szBuf); -} - - - -void CMakePackLog::WriteErrorf(const char* c_szFormat, ...) -{ - if (!__IsLogMode()) - return; - - va_list args; - va_start(args, c_szFormat); - - char szBuf[1024]; - int nBufLen = _vsnprintf(szBuf, sizeof(szBuf), c_szFormat, args); - szBuf[nBufLen++] = '\0'; - __WriteError(szBuf, nBufLen); -} - -void CMakePackLog::WriteErrornf(const char* c_szFormat, ...) -{ - if (!__IsLogMode()) - return; - - va_list args; - va_start(args, c_szFormat); - - char szBuf[1024+1]; - int nBufLen = _vsnprintf(szBuf, sizeof(szBuf)-1, c_szFormat, args); - if (nBufLen > 0) - { - szBuf[nBufLen++] = '\n'; - szBuf[nBufLen++] = '\0'; - } - __WriteError(szBuf, nBufLen); -} - -void CMakePackLog::WriteError(const char* c_szBuf) -{ - if (!__IsLogMode()) - return; - - __WriteError(c_szBuf, strlen(c_szBuf)+1); -} - -void CMakePackLog::__WriteError(const char* c_szBuf, int nBufLen) -{ - if (!__IsLogMode()) - return; - - if (NULL==m_fp_err) - m_fp_err=fopen(m_stErrorFileName.c_str(), "w"); - - fwrite(c_szBuf, nBufLen, 1, m_fp_err); - - printf("Error: %s", c_szBuf); -} - -void CMakePackLog::FlushError() -{ - std::wifstream iFile(m_stErrorFileName.c_str()); - std::istream_iterator iit(iFile); - std::istream_iterator eos; - - std::vector vText; - - std::copy (iit, eos, std::back_inserter(vText)); - - std::ostream_iterator > oit(std::wcout); - - std::sort (vText.begin(), vText.end()); - std::copy (vText.begin(), vText.end(), oit); -} -#ifdef __MAKE_PACK__ -FILE * CEterPack::ms_PackLogFile = NULL; -#endif -/////////////////////////////////////////////////////////////////////////////// -CEterPack::CEterPack() : m_indexCount(0), m_indexData(NULL), m_FragmentSize(0), m_bEncrypted(false), m_bReadOnly(false), m_bDecrypedIV(false) -{ - m_pCSHybridCryptPolicy = new EterPackPolicy_CSHybridCrypt; - -} - -CEterPack::~CEterPack() -{ - Destroy(); - - delete m_pCSHybridCryptPolicy; - m_pCSHybridCryptPolicy = NULL; -} - -void CEterPack::Destroy() -{ - m_bReadOnly = false; - m_bEncrypted = false; - m_indexCount = 0; - m_DataPositionMap.clear(); - - for (int i = 0; i < FREE_INDEX_MAX_SIZE + 1; ++i) - m_FreeIndexList[i].clear(); - - SAFE_DELETE_ARRAY(m_indexData); - - m_FragmentSize = 0; - - memset(m_dbName, 0, sizeof(m_dbName)); - memset(m_indexFileName, 0, sizeof(m_indexFileName)); -} - -const std::string& CEterPack::GetPathName() -{ - return m_stPathName; -} - -bool CEterPack::Create(CEterFileDict& rkFileDict, const char * dbname, const char* pathName, bool bReadOnly, const BYTE* iv) -{ - if (iv) - { - m_stIV_Panama.assign((const char*) iv, 32); - m_bDecrypedIV = false; - } - - m_stPathName = pathName; - - strncpy(m_dbName, dbname, DBNAME_MAX_LEN); - - strncpy(m_indexFileName, dbname, MAX_PATH); - strcat(m_indexFileName, ".eix"); - - m_stDataFileName = dbname; - m_stDataFileName += ".epk"; - - m_bReadOnly = bReadOnly; - - // bReadOnly 모드가 아니고 데이터 베이스가 열린다면 생성 실패 - if (!CreateIndexFile()) - return false; - - if (!CreateDataFile()) - return false; - - bool bOverwrite = (iv != NULL); - __BuildIndex(rkFileDict, bOverwrite); - - if (m_bReadOnly) - { - //m_bIsDataLoaded = true; - //if (!m_file.Create(m_stDataFileName.c_str(), (const void**)&m_file_data, 0, 0)) - // return false; - } - else - { - DecryptIndexFile(); - } - - return true; -} - -bool CEterPack::DecryptIV(DWORD dwPanamaKey) -{ - if (m_stIV_Panama.length() != 32) - return false; - - if (m_bDecrypedIV) // 이미 암호화가 풀렸으면 다시 처리 안함 - return true; - - DWORD* ivs = (DWORD*)&m_stIV_Panama[0]; - for (int i = 0; i != m_stIV_Panama.length() / sizeof(DWORD); ++i) - { - ivs[i] ^= dwPanamaKey + i * 16777619; - } - - m_bDecrypedIV = true; - return true; -} - -bool CEterPack::DecryptIndexFile() -{ - if (!m_bEncrypted) - return true; - - CFileBase file; - - if (!file.Create(m_indexFileName, CFileBase::FILEMODE_WRITE)) - return false; - - file.Write(&eterpack::c_IndexCC, sizeof(DWORD)); - file.Write(&eterpack::c_Version, sizeof(DWORD)); - file.Write(&m_indexCount, sizeof(long)); - file.Write(m_indexData, sizeof(TEterPackIndex) * m_indexCount); - - file.Close(); - - m_bEncrypted = false; - return true; -} - -static DWORD s_adwEterPackKey[] = -{ - 45129401, - 92367215, - 681285731, - 1710201, -}; - -static DWORD s_adwEterPackSecurityKey[] = -{ - 78952482, - 527348324, - 1632942, - 486274726, -}; - -bool CEterPack::EncryptIndexFile() -{ - CMappedFile file; - LPCVOID pvData; - - if (NULL == file.Create(m_indexFileName, &pvData, 0, 0)) - { - TraceError("EncryptIndex: Cannot open pack index file! %s", m_dbName); - return false; - } - - BYTE * pbData = new BYTE[file.Size()]; - memcpy(pbData, pvData, file.Size()); - - CLZObject zObj; - - if (!CLZO::Instance().CompressEncryptedMemory(zObj, pbData, file.Size(), s_adwEterPackKey)) - { - TraceError("EncryptIndex: Cannot encrypt! %s", m_dbName); - SAFE_DELETE_ARRAY(pbData); - return false; - } - - file.Destroy(); - - while (!DeleteFile(m_indexFileName)); - - FILE * fp; - - fp = fopen(m_indexFileName, "wb"); - - if (!fp) - { - TraceError("EncryptIndex: Cannot open file for writing! %s", m_dbName); - SAFE_DELETE_ARRAY(pbData); - return false; - } - - if (1 != fwrite(zObj.GetBuffer(), zObj.GetSize(), 1, fp)) - { - TraceError("Encryptindex: Cannot write to file! %s", m_indexFileName); - SAFE_DELETE_ARRAY(pbData); - fclose(fp); - return false; - } - - fclose(fp); - - m_bEncrypted = true; - delete [] pbData; - return true; -} - -bool CEterPack::__BuildIndex(CEterFileDict& rkFileDict, bool bOverwrite) -{ - //DWORD dwBeginTime = ELTimer_GetMSec(); - CMappedFile file; - LPCVOID pvData; - CLZObject zObj; - - if (NULL == file.Create(m_indexFileName, &pvData, 0, 0)) - { - TraceError("Cannot open pack index file! %s", m_dbName); - return false; - } - - if (file.Size() < eterpack::c_HeaderSize) - { - TraceError("Pack index file header error! %s", m_dbName); - return false; - } - - DWORD fourcc = *(DWORD *) pvData; - - BYTE * pbData; - UINT uiFileSize; - - if (fourcc == eterpack::c_IndexCC) - { - pbData = (BYTE *) pvData; - uiFileSize = file.Size(); - } - else if (fourcc == CLZObject::ms_dwFourCC) - { - m_bEncrypted = true; - - if (!CLZO::Instance().Decompress(zObj, (const BYTE *) pvData, s_adwEterPackKey)) - return false; - - if (zObj.GetSize() < eterpack::c_HeaderSize) - return false; - - pbData = zObj.GetBuffer(); - uiFileSize = zObj.GetSize(); - } - else - { - TraceError("Pack index file fourcc error! %s", m_dbName); - return false; - } - - pbData += sizeof(DWORD); - - DWORD ver = *(DWORD *) pbData; - pbData += sizeof(DWORD); - - if (ver != eterpack::c_Version) - { - TraceError("Pack index file version error! %s", m_dbName); - return false; - } - - m_indexCount = *(long *) pbData; - pbData += sizeof(long); - - if (uiFileSize < eterpack::c_HeaderSize + sizeof(TEterPackIndex) * m_indexCount) - { - TraceError("Pack index file size error! %s, indexCount %d", m_dbName, m_indexCount); - return false; - } - - //Tracef("Loading Pack file %s elements: %d ... ", m_dbName, m_indexCount); - - m_indexData = new TEterPackIndex[m_indexCount]; - memcpy(m_indexData, pbData, sizeof(TEterPackIndex) * m_indexCount); - - TEterPackIndex * index = m_indexData; - - for (int i = 0; i < m_indexCount; ++i, ++index) - { - if (!index->filename_crc) - { - PushFreeIndex(index); - } - else - { - if (index->real_data_size > index->data_size) - m_FragmentSize += index->real_data_size - index->data_size; - - m_DataPositionMap.insert(TDataPositionMap::value_type(index->filename_crc, index)); - - if (bOverwrite) // 서버 연동 패킹 파일은 나중에 들어오지만 최상위로 등록해야한다 - rkFileDict.UpdateItem(this, index); - else - rkFileDict.InsertItem(this, index); - } - } - - //Tracef("Done. (spent %dms)\n", ELTimer_GetMSec() - dwBeginTime); - return true; -} -// -//void CEterPack::UpdateLastAccessTime() -//{ -// m_tLastAccessTime = time(NULL); -//} -// -//void CEterPack::ClearDataMemoryMap() -//{ -// // m_file이 data file이다... -// m_file.Destroy(); -// m_tLastAccessTime = 0; -// m_bIsDataLoaded = false; -//} - -bool CEterPack::Get(CMappedFile& out_file, const char * filename, LPCVOID * data) -{ - TEterPackIndex * index = FindIndex(filename); - - if (!index) - { - return false; - } - - //UpdateLastAccessTime(); - //if (!m_bIsDataLoaded) - //{ - // if (!m_file.Create(m_stDataFileName.c_str(), (const void**)&m_file_data, 0, 0)) - // return false; - // - // m_bIsDataLoaded = true; - //} - - // 기존에는 CEterPack에서 epk를 memory map에 올려놓고, 요청이 오면 그 부분을 링크해서 넘겨 줬었는데, - // 이제는 요청이 오면, 필요한 부분만 memory map에 올리고, 요청이 끝나면 해제하게 함. - out_file.Create(m_stDataFileName.c_str(), data, index->data_position, index->data_size); - - bool bIsSecurityCheckRequired = ( index->compressed_type == COMPRESSED_TYPE_SECURITY || - index->compressed_type == COMPRESSED_TYPE_PANAMA ); - - if( bIsSecurityCheckRequired ) - { -#ifdef CHECKSUM_CHECK_MD5 - MD5_CTX context; - GenerateMD5Hash( (BYTE*)(*data), index->data_size, context ); - - if( memcmp( index->MD5Digest, context.digest, 16 ) != 0 ) - { - return false; - } -#else - DWORD dwCrc32 = GetCRC32((const char*)(*data), index->data_size); - - if( index->data_crc != dwCrc32 ) - { - return false; - } -#endif - } - - - if (COMPRESSED_TYPE_COMPRESS == index->compressed_type) - { - CLZObject * zObj = new CLZObject; - - if (!CLZO::Instance().Decompress(*zObj, static_cast(*data))) - { - TraceError("Failed to decompress : %s", filename); - delete zObj; - return false; - } - - out_file.BindLZObject(zObj); - *data = zObj->GetBuffer(); - } - else if (COMPRESSED_TYPE_SECURITY == index->compressed_type) - { - CLZObject * zObj = new CLZObject; - - if (!CLZO::Instance().Decompress(*zObj, static_cast(*data), s_adwEterPackSecurityKey)) - { - TraceError("Failed to encrypt : %s", filename); - delete zObj; - return false; - } - - out_file.BindLZObject(zObj); - *data = zObj->GetBuffer(); - } - else if (COMPRESSED_TYPE_PANAMA == index->compressed_type) - { - CLZObject* zObj = new CLZObject; - if (!__Decrypt_Panama(filename, static_cast(*data), index->data_size, *zObj)) - { - delete zObj; - return false; - } - - out_file.BindLZObjectWithBufferedSize(zObj); - *data = zObj->GetBuffer(); - } - else if (COMPRESSED_TYPE_HYBRIDCRYPT == index->compressed_type || COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) - { -#ifdef __THEMIDA__ - VM_START -#endif - - CLZObject * zObj = new CLZObject; - auto ssvv = std::string(filename); - if (!m_pCSHybridCryptPolicy->DecryptMemory(ssvv, static_cast(*data), index->data_size, *zObj)) - { - delete zObj; - return false; - } - - out_file.BindLZObjectWithBufferedSize(zObj); - if( COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) - { - BYTE* pSDBData; - int iSDBSize; - - if (!m_pCSHybridCryptPolicy->GetSupplementaryDataBlock(ssvv, pSDBData, iSDBSize)) - { - delete zObj; - return false; - } - - *data = out_file.AppendDataBlock( pSDBData, iSDBSize ); - } - else - { - *data = zObj->GetBuffer(); - } -#ifdef __THEMIDA__ - VM_END -#endif - } - return true; -} - -bool CEterPack::Get2(CMappedFile& out_file, const char * filename, TEterPackIndex * index, LPCVOID * data) -{ - if (!index) - { - return false; - } - - //UpdateLastAccessTime(); - //if (!m_bIsDataLoaded) - //{ - // if (!m_file.Create(m_stDataFileName.c_str(), (const void**)&m_file_data, 0, 0)) - // return false; - // - // m_bIsDataLoaded = true; - //} - out_file.Create(m_stDataFileName.c_str(), data, index->data_position, index->data_size); - - bool bIsSecurityCheckRequired = ( index->compressed_type == COMPRESSED_TYPE_SECURITY || - index->compressed_type == COMPRESSED_TYPE_PANAMA ); - - if( bIsSecurityCheckRequired ) - { -#ifdef CHECKSUM_CHECK_MD5 - MD5_CTX context; - GenerateMD5Hash( (BYTE*)(*data), index->data_size, context ); - - if( memcmp( index->MD5Digest, context.digest, 16 ) != 0 ) - { - return false; - } -#else - DWORD dwCrc32 = GetCRC32((const char*)(*data), index->data_size); - - if( index->data_crc != dwCrc32 ) - { - return false; - } -#endif - } - - - if (COMPRESSED_TYPE_COMPRESS == index->compressed_type) - { - CLZObject * zObj = new CLZObject; - - if (!CLZO::Instance().Decompress(*zObj, static_cast(*data))) - { - TraceError("Failed to decompress : %s", filename); - delete zObj; - return false; - } - - out_file.BindLZObject(zObj); - *data = zObj->GetBuffer(); - } - else if (COMPRESSED_TYPE_SECURITY == index->compressed_type) - { - CLZObject * zObj = new CLZObject; - - if (!CLZO::Instance().Decompress(*zObj, static_cast(*data), s_adwEterPackSecurityKey)) - { - TraceError("Failed to encrypt : %s", filename); - delete zObj; - return false; - } - - out_file.BindLZObject(zObj); - *data = zObj->GetBuffer(); - } - else if (COMPRESSED_TYPE_PANAMA == index->compressed_type) - { - CLZObject * zObj = new CLZObject; - __Decrypt_Panama(filename, static_cast(*data), index->data_size, *zObj); - out_file.BindLZObjectWithBufferedSize(zObj); - *data = zObj->GetBuffer(); - } - else if (COMPRESSED_TYPE_HYBRIDCRYPT == index->compressed_type || COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) - { -#ifdef __THEMIDA__ - VM_START -#endif - - CLZObject * zObj = new CLZObject; - auto ssvv = std::string(filename); - if( !m_pCSHybridCryptPolicy->DecryptMemory(ssvv, static_cast(*data), index->data_size, *zObj) ) - { - delete zObj; - return false; - } - - out_file.BindLZObjectWithBufferedSize(zObj); - - if( COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) - { - BYTE* pSDBData; - int iSDBSize; - - if( !m_pCSHybridCryptPolicy->GetSupplementaryDataBlock(ssvv, pSDBData, iSDBSize) ) - { - return false; - } - - *data = out_file.AppendDataBlock( pSDBData, iSDBSize ); - } - else - { - *data = zObj->GetBuffer(); - } -#ifdef __THEMIDA__ - VM_END -#endif - } - - return true; -} - - -bool CEterPack::Delete(TEterPackIndex * pIndex) -{ - CFileBase fileIndex; - - if (!fileIndex.Create(m_indexFileName, CFileBase::FILEMODE_WRITE)) - return false; - - PushFreeIndex(pIndex); - WriteIndex(fileIndex, pIndex); - return true; -} - -bool CEterPack::Delete(const char * filename) -{ - TEterPackIndex * pIndex = FindIndex(filename); - - if (!pIndex) - return false; - - return Delete(pIndex); -} - -bool CEterPack::Extract() -{ - CMappedFile dataMapFile; - LPCVOID data; - - if (!dataMapFile.Create(m_stDataFileName.c_str(), &data, 0, 0)) - return false; - - CLZObject zObj; - - for (TDataPositionMap::iterator i = m_DataPositionMap.begin(); - i != m_DataPositionMap.end(); - ++i) - { - TEterPackIndex* index = i->second; - CFileBase writeFile; - - inlinePathCreate(index->filename); - printf("%s\n", index->filename); - - writeFile.Create(index->filename, CFileBase::FILEMODE_WRITE); - - if (COMPRESSED_TYPE_COMPRESS == index->compressed_type) - { - if (!CLZO::Instance().Decompress(zObj, (const BYTE *) data + index->data_position)) - { - printf("cannot decompress"); - return false; - } - - writeFile.Write(zObj.GetBuffer(), zObj.GetSize()); - zObj.Clear(); - } - else if (COMPRESSED_TYPE_SECURITY == index->compressed_type) - { - if (!CLZO::Instance().Decompress(zObj, (const BYTE *) data + index->data_position, s_adwEterPackSecurityKey)) - { - printf("cannot decompress"); - return false; - } - - writeFile.Write(zObj.GetBuffer(), zObj.GetSize()); - zObj.Clear(); - } - else if (COMPRESSED_TYPE_PANAMA == index->compressed_type) - { - __Decrypt_Panama(index->filename, (const BYTE *) data + index->data_position, index->data_size, zObj); - writeFile.Write(zObj.GetBuffer(), zObj.GetBufferSize()); - zObj.Clear(); - } - else if (COMPRESSED_TYPE_HYBRIDCRYPT == index->compressed_type || COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) - { -#ifdef __THEMIDA__ - VM_START -#endif - auto ssvv = std::string(index->filename); - if( !m_pCSHybridCryptPolicy->DecryptMemory(ssvv, (const BYTE *) data + index->data_position, index->data_size, zObj) ) - return false; - - if( COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB == index->compressed_type) - { - dataMapFile.BindLZObjectWithBufferedSize(&zObj); - - BYTE* pSDBData; - int iSDBSize; - - if( !m_pCSHybridCryptPolicy->GetSupplementaryDataBlock(ssvv, pSDBData, iSDBSize) ) - return false; - - dataMapFile.AppendDataBlock( pSDBData, iSDBSize ); - writeFile.Write(dataMapFile.AppendDataBlock( pSDBData, iSDBSize ),dataMapFile.Size()); - } - else - { - writeFile.Write(zObj.GetBuffer(), zObj.GetBufferSize()); - } - zObj.Clear(); -#ifdef __THEMIDA__ - VM_END -#endif - } - else if (COMPRESSED_TYPE_NONE == index->compressed_type) - writeFile.Write((const char *) data + index->data_position, index->data_size); - - writeFile.Destroy(); - } - return true; -} - -bool CEterPack::GetNames(std::vector* retNames) -{ - CMappedFile dataMapFile; - LPCVOID data; - - if (!dataMapFile.Create(m_stDataFileName.c_str(), &data, 0, 0)) - return false; - - CLZObject zObj; - - for (TDataPositionMap::iterator i = m_DataPositionMap.begin(); - i != m_DataPositionMap.end(); - ++i) - { - TEterPackIndex* index = i->second; - - inlinePathCreate(index->filename); - - retNames->push_back(index->filename); - } - return true; -} - -bool CEterPack::Put(const char * filename, const char * sourceFilename, BYTE packType, const std::string& strRelateMapName ) -{ - CMappedFile mapFile; - LPCVOID data; - - if (sourceFilename) - { - if (!mapFile.Create(sourceFilename, &data, 0, 0)) - { - return false; - } - } - else if (!mapFile.Create(filename, &data, 0, 0)) - { - return false; - } - - BYTE* pMappedData = (BYTE*)data; - int iMappedDataSize = mapFile.Size(); - - if( packType == COMPRESSED_TYPE_HYBRIDCRYPT || packType == COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB ) - { -#ifdef __THEMIDA__ - VM_START -#endif - auto ssvvv = std::string(filename); - m_pCSHybridCryptPolicy->GenerateCryptKey(ssvvv); - - if( packType == COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB ) - { - if( !m_pCSHybridCryptPolicy->GenerateSupplementaryDataBlock(ssvvv, strRelateMapName, (const BYTE*)data, mapFile.Size(), pMappedData, iMappedDataSize )) - { - return false; - } - } -#ifdef __THEMIDA__ - VM_END -#endif - } - - return Put(filename, pMappedData, iMappedDataSize, packType); -} - -#ifdef CHECKSUM_CHECK_MD5 -void CEterPack::GenerateMD5Hash( BYTE* pData, int nLength, IN OUT MD5_CTX& mdContext ) -{ - MD5Init (&mdContext); - - const int nBlockSize = 1024; - - int nLoopCnt = nLength / nBlockSize; - int nRemainder = nLength % nBlockSize; - int i; - - for(i = 0; i < nLoopCnt; ++i ) - { - MD5Update (&mdContext, reinterpret_cast(pData + i*nBlockSize), nBlockSize); - } - - if( nRemainder > 0 ) - MD5Update (&mdContext, reinterpret_cast(pData + i*nBlockSize), nRemainder); - - MD5Final (&mdContext); -} -#endif -bool CEterPack::Put(const char * filename, LPCVOID data, long len, BYTE packType) -{ - if (m_bEncrypted) - { - TraceError("EterPack::Put : Cannot put to encrypted pack (filename: %s, DB: %s)", filename, m_dbName); - return false; - } - - CFileBase fileIndex; - - if (!fileIndex.Create(m_indexFileName, CFileBase::FILEMODE_WRITE)) - { - return false; - } - - CFileBase fileData; - - if (!fileData.Create(m_stDataFileName.c_str(), CFileBase::FILEMODE_WRITE)) - { - return false; - } - - TEterPackIndex * pIndex; - pIndex = FindIndex(filename); - - CLZObject zObj; - std::string encryptStr; - - if (packType == COMPRESSED_TYPE_SECURITY || - packType == COMPRESSED_TYPE_COMPRESS) - { - if (packType == COMPRESSED_TYPE_SECURITY) - { - if (!CLZO::Instance().CompressEncryptedMemory(zObj, data, len, s_adwEterPackSecurityKey)) - { - return false; - } - } - else - { - if (!CLZO::Instance().CompressMemory(zObj, data, len)) - { - return false; - } - } - - data = zObj.GetBuffer(); - len = zObj.GetSize(); - } - else if (packType == COMPRESSED_TYPE_PANAMA) - { - if (!__Encrypt_Panama(filename, (const BYTE *) data, len, zObj)) - { - return false; - } - - data = zObj.GetBuffer(); - len = zObj.GetBufferSize(); - } - else if (packType == COMPRESSED_TYPE_HYBRIDCRYPT || packType == COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB) - { -#ifdef __THEMIDA__ - VM_START -#endif - - auto ssvv = std::string(filename); - if (!m_pCSHybridCryptPolicy->EncryptMemory(ssvv, (const BYTE*)data, len, zObj)) - { - return false; - } - - data = zObj.GetBuffer(); - len = zObj.GetBufferSize(); -#ifdef __THEMIDA__ - VM_END -#endif - - } - - -#ifdef CHECKSUM_CHECK_MD5 - MD5_CTX context; - GenerateMD5Hash( (BYTE*)(data), len, context ); -#else - DWORD data_crc; - data_crc = GetCRC32((const char *) data, len); -#endif - - // 기존 데이터가 있으면.. - if (pIndex) - { - // 기존 data 크기가 넣을 데이터 크기를 수용할 수 있다면 - if (pIndex->real_data_size >= len) - { - ++m_map_indexRefCount[pIndex->id]; - - // 길이가 틀리거나, checksum이 틀릴 때만 저장 한다. - if ( (pIndex->data_size != len) || -#ifdef CHECKSUM_CHECK_MD5 - (memcmp( pIndex->MD5Digest, context.digest, 16 ) != 0) ) -#else - (pIndex->data_crc != data_crc) ) -#endif - { -#ifdef __MAKE_PACK__ - if (ms_PackLogFile) - fprintf(ms_PackLogFile, "Overwrite[%d/%d] %s\n", bCompress, bSecurity, pIndex->filename); - printf("Overwrite[%d/%d] %s\n", bCompress, bSecurity, pIndex->filename); -#endif - pIndex->data_size = len; -#ifdef CHECKSUM_CHECK_MD5 - memcpy( pIndex->MD5Digest, context.digest, 16 ); - -#else - pIndex->data_crc = data_crc; -#endif - - pIndex->compressed_type = packType; - - CMakePackLog::GetSingleton().Writef("Overwrite[type:%u] %s\n", pIndex->compressed_type, pIndex->filename); - - WriteIndex(fileIndex, pIndex); - WriteData(fileData, pIndex, data); - } - - return true; - } - - // 기존 데이터 크기가 새로 들어갈 것 보다 적다면, 새로 인덱스를 할당해 - // 넣어야 한다. 원래 있던 인덱스는 비워 둔다. - PushFreeIndex(pIndex); - WriteIndex(fileIndex, pIndex); - } - - // 새 데이터 - pIndex = NewIndex(fileIndex, filename, len); - pIndex->data_size = len; - -#ifdef CHECKSUM_CHECK_MD5 - memcpy( pIndex->MD5Digest, context.digest, 16 ); -#else - pIndex->data_crc = data_crc; -#endif - - - pIndex->data_position = GetNewDataPosition(fileData); - pIndex->compressed_type = packType; - - WriteIndex(fileIndex, pIndex); - WriteNewData(fileData, pIndex, data); - - ++m_map_indexRefCount[pIndex->id]; - -#ifdef __MAKE_PACK__ - if (ms_PackLogFile) - fprintf(ms_PackLogFile, "Write[%d/%d] %s\n", bCompress, bSecurity, pIndex->filename); - printf("Write[%d/%d] %s, position %ld realsize %ld size %ld\n", - bCompress, bSecurity, pIndex->filename, pIndex->data_position, pIndex->real_data_size, pIndex->data_size); -#endif - CMakePackLog::GetSingleton().Writef("Write[type:%u] %s\n", pIndex->compressed_type, pIndex->filename); - - return true; -} - -long CEterPack::GetFragmentSize() -{ - return m_FragmentSize; -} - -// Private methods -bool CEterPack::CreateIndexFile() -{ - FILE * fp; - - if (NULL != (fp = fopen(m_indexFileName, "rb"))) - { - fclose(fp); - return true; - } - else if (m_bReadOnly) - return false; - - // - // 파일이 없으므로 새로 만든다. - // - fp = fopen(m_indexFileName, "wb"); - - if (!fp) - return false; - - fwrite(&eterpack::c_IndexCC, sizeof(DWORD), 1, fp); - fwrite(&eterpack::c_Version, sizeof(DWORD), 1, fp); - fwrite(&m_indexCount, sizeof(long), 1, fp); - - fclose(fp); - return true; -} - - -void CEterPack::WriteIndex(CFileBase & file, TEterPackIndex * index) -{ - file.Seek(sizeof(DWORD) + sizeof(DWORD)); - file.Write(&m_indexCount, sizeof(long)); - file.Seek(eterpack::c_HeaderSize + (index->id * sizeof(TEterPackIndex))); - - if (!file.Write(index, sizeof(TEterPackIndex))) - { - assert(!"WriteIndex: fwrite failed"); - return; - } -} - -/* - * Free Block 이란 데이터에서 지워진 부분을 말한다. - * Free Block 들은 각각 FREE_INDEX_BLOCK_SIZE (32768) 단위로 나누어져 - * 리스트로 관리된다. - * - * 예를 들어 128k 의 데이터는 - * 128 * 1024 / FREE_INDEX_BLOCK_SIZE = 4 이므로 - * 최종 적으로는 m_FreeIndexList[4] 에 들어간다. - * - * FREE_INDEX_BLOCK_SIZE 의 최대 값은 FREE_INDEX_MAX_SIZE(512) 이다. - * 따라서 16MB 이상의 데이터는 무조건 배열의 512 위치에 들어간다. - */ -int CEterPack::GetFreeBlockIndex(long size) -{ - return std::min((long)FREE_INDEX_MAX_SIZE, size / FREE_INDEX_BLOCK_SIZE); -} - -void CEterPack::PushFreeIndex(TEterPackIndex* index) -{ - if (index->filename_crc != 0) - { - TDataPositionMap::iterator i = m_DataPositionMap.find(index->filename_crc); - - if (i != m_DataPositionMap.end()) - m_DataPositionMap.erase(i); - - index->filename_crc = 0; - memset(index->filename, 0, sizeof(index->filename)); - } - - int blockidx = GetFreeBlockIndex(index->real_data_size); - m_FreeIndexList[blockidx].push_back(index); - m_FragmentSize += index->real_data_size; - //printf("FreeIndex: size %d: blockidx: %d\n", index->real_data_size, blockidx); -} - -long CEterPack::GetNewIndexPosition(CFileBase & file) -{ - long pos = (file.Size() - eterpack::c_HeaderSize) / sizeof(TEterPackIndex); - ++m_indexCount; - return (pos); -} - -TEterPackIndex* CEterPack::NewIndex(CFileBase& file, const char* filename, long size) -{ - TEterPackIndex* index = NULL; - int block_size = size + (DATA_BLOCK_SIZE - (size % DATA_BLOCK_SIZE)); -// if ((index = FindIndex(filename))) // 이미 인덱스가 존재하는지 확인 -// return index; - - int blockidx = GetFreeBlockIndex(block_size); - - for (TFreeIndexList::iterator i = m_FreeIndexList[blockidx].begin(); - i != m_FreeIndexList[blockidx].end(); - ++i) - { - if ((*i)->real_data_size >= size) - { - index = *i; - m_FreeIndexList[blockidx].erase(i); - - assert(index->filename_crc == 0); - break; - } - } - - if (!index) - { - index = new TEterPackIndex; - index->real_data_size = block_size; - index->id = GetNewIndexPosition(file); - } - - strncpy(index->filename, filename, FILENAME_MAX_LEN); - index->filename[FILENAME_MAX_LEN] = '\0'; - inlineConvertPackFilename(index->filename); - - index->filename_crc = GetCRC32(index->filename, strlen(index->filename)); - - m_DataPositionMap.insert(TDataPositionMap::value_type(index->filename_crc, index)); - return index; -} - -TEterPackIndex* CEterPack::FindIndex(const char * filename) -{ - static char tmpFilename[MAX_PATH + 1]; - strncpy(tmpFilename, filename, MAX_PATH); - inlineConvertPackFilename(tmpFilename); - - DWORD filename_crc = GetCRC32(tmpFilename, strlen(tmpFilename)); - TDataPositionMap::iterator i = m_DataPositionMap.find(filename_crc); - - if (i == m_DataPositionMap.end()) - return NULL; - - return (i->second); -} - -bool CEterPack::IsExist(const char * filename) -{ - return FindIndex(filename) ? true : false; -} - -bool CEterPack::CreateDataFile() -{ - FILE * fp; - - if (NULL != (fp = fopen(m_stDataFileName.c_str(), "rb"))) - { - fclose(fp); - return true; - } - else if (m_bReadOnly) - return false; - - fp = fopen(m_stDataFileName.c_str(), "wb"); - - if (!fp) - return false; - - fclose(fp); - return true; -} - -long CEterPack::GetNewDataPosition(CFileBase& file) -{ - return file.Size(); -} - -bool CEterPack::ReadData(CFileBase & file, TEterPackIndex* index, LPVOID data, long maxsize) -{ - if (index->data_size > maxsize) - return false; - - file.Seek(index->data_position); - file.Read(data, index->data_size); - return true; -} - -bool CEterPack::WriteData(CFileBase & file, TEterPackIndex* index, LPCVOID data) -{ - file.Seek(index->data_position); - - if (!file.Write(data, index->data_size)) - { - assert(!"WriteData: fwrite data failed"); - return false; - } - - return true; -} - -bool CEterPack::WriteNewData(CFileBase& file, TEterPackIndex* index, LPCVOID data) -{ - file.Seek(index->data_position); - - if (!file.Write(data, index->data_size)) - { - assert(!"WriteData: fwrite data failed"); - return false; - } - - int empty_size = index->real_data_size - index->data_size; - - if (empty_size < 0) - { - printf("SYSERR: WriteNewData(): CRITICAL ERROR: empty_size lower than 0!\n"); - exit(1); - } - - if (empty_size == 0) - return true; - - char * empty_buf = (char *) calloc(empty_size, sizeof(char)); - - if (!file.Write(empty_buf, empty_size)) - { - assert(!"WriteData: fwrite empty data failed"); - return false; - } - - free(empty_buf); - return true; -} - -TDataPositionMap & CEterPack::GetIndexMap() -{ - return m_DataPositionMap; -} - -DWORD CEterPack::DeleteUnreferencedData() -{ - TDataPositionMap::iterator i = m_DataPositionMap.begin(); - DWORD dwCount = 0; - - while (i != m_DataPositionMap.end()) - { - TEterPackIndex * pIndex = (i++)->second; - - if (0 == m_map_indexRefCount[pIndex->id]) - { - printf("Unref File %s\n", pIndex->filename); - Delete(pIndex); - ++dwCount; - } - } - - return dwCount; -} - -const char * CEterPack::GetDBName() -{ - return m_dbName; -} - -void CEterPack::__CreateFileNameKey_Panama(const char * filename, BYTE * key, unsigned int keySize) -{ - // 키 암호화 - if (keySize != 32) - return; - - std::string SrcStringForKey(filename); - unsigned int idx = GetCRC32(SrcStringForKey.c_str(), SrcStringForKey.length()) & 3; - - CryptoPP::HashTransformation* hm1 = NULL; - CryptoPP::HashTransformation* hm2 = NULL; - - static CryptoPP::Tiger tiger; - static CryptoPP::SHA1 sha1; - static CryptoPP::RIPEMD128 ripemd128; - static CryptoPP::Whirlpool whirlpool; - - switch (idx & 3) - { - case 0: - hm1 = &whirlpool; - break; - - case 1: - hm1 = &tiger; - break; - - case 2: - hm1 = &sha1; - break; - - case 3: - hm1 = &ripemd128; - break; - } - - CryptoPP::StringSource(SrcStringForKey, true, - new CryptoPP::HashFilter(*hm1, - //new CryptoPP::HexEncoder( - new CryptoPP::ArraySink(key, 16) - //) // HexEncoder - ) // HashFilter - ); // StringSource - - // 만들어진 키의 첫번째 4바이트로 다음 16바이트 키 생성 알고리즘 선택 - unsigned int idx2 = *(unsigned int*) key; - - switch (idx2 & 3) - { - case 0: - hm2 = &sha1; - break; - - case 1: - hm2 = &ripemd128; - break; - - case 2: - hm2 = &whirlpool; - break; - - case 3: - hm2 = &tiger; - break; - } - - CryptoPP::StringSource(SrcStringForKey, true, - new CryptoPP::HashFilter(*hm2, - //new CryptoPP::HexEncoder( - new CryptoPP::ArraySink(key + 16, 16) - //) // HexEncoder - ) // HashFilter - ); // StringSource - // 키 생성 완료 -} - -bool CEterPack::__Encrypt_Panama(const char* filename, const BYTE* data, SIZE_T dataSize, CLZObject& zObj) -{ - if (32 != m_stIV_Panama.length()) - { - // 해커가 이 메세지를 보면 힌트를 얻을까봐 디버그에서만 출력 -#ifdef _DEBUG - TraceError("IV not set (filename: %s)", filename); -#endif - return false; - } - - CryptoPP::PanamaCipher::Encryption Encryptor; - - if (dataSize < Encryptor.MandatoryBlockSize()) - { -#ifdef _DEBUG - TraceError("Type 3 pack file must be bigger than %u bytes (filename: %s)", Encryptor.MandatoryBlockSize(), filename); -#endif - return false; - } - - BYTE key[32]; - - __CreateFileNameKey_Panama(filename, key, sizeof(key)); - Encryptor.SetKeyWithIV(key, sizeof(key), (const BYTE*) m_stIV_Panama.c_str(), 32); - - // MandatoryBlockSize에 나누어 떨어지게 만들고 최대 2048 바이트만 - DWORD cryptSize = dataSize - (dataSize % Encryptor.MandatoryBlockSize()); - cryptSize = cryptSize > 2048 ? 2048 : cryptSize; - - std::string tmp; - - tmp.reserve(cryptSize); - - CryptoPP::ArraySource(data, cryptSize, true, - new CryptoPP::StreamTransformationFilter(Encryptor, - new CryptoPP::StringSink(tmp) - ) - ); - - if (tmp.length() != cryptSize) - { -#ifdef _DEBUG - TraceError("Type 3 pack crypt buffer size error (out %u should be %u)", tmp.length(), cryptSize); -#endif - return false; - } - - zObj.AllocBuffer(dataSize); - memcpy(zObj.GetBuffer(), tmp.c_str(), cryptSize); - - if (dataSize - cryptSize > 0) - memcpy(zObj.GetBuffer() + cryptSize, data + cryptSize, dataSize - cryptSize); - - return true; -} - -bool CEterPack::__Decrypt_Panama(const char* filename, const BYTE* data, SIZE_T dataSize, CLZObject& zObj) -{ - if (32 != m_stIV_Panama.length()) - { - // 해커가 이 메세지를 보면 힌트를 얻을까봐 디버그에서만 출력 -#ifdef _DEBUG - TraceError("IV not set (filename: %s)", filename); -#endif - return false; - } - - CryptoPP::PanamaCipher::Decryption Decryptor; - - BYTE key[32]; - - __CreateFileNameKey_Panama(filename, key, sizeof(key)); - Decryptor.SetKeyWithIV(key, sizeof(key), (const BYTE*) m_stIV_Panama.c_str(), 32); - - // MandatoryBlockSize에 나누어 떨어지게 만들고 최대 2048 바이트만 - DWORD cryptSize = dataSize - (dataSize % Decryptor.MandatoryBlockSize()); - cryptSize = cryptSize > 2048 ? 2048 : cryptSize; - - std::string tmp; - - tmp.reserve(cryptSize); - - CryptoPP::ArraySource(data, cryptSize, true, - new CryptoPP::StreamTransformationFilter(Decryptor, - new CryptoPP::StringSink(tmp) - ) - ); - - if (tmp.length() != cryptSize) - { -#ifdef _DEBUG - TraceError("Type 3 pack crypt buffer size error (out %u should be %u)", tmp.length(), cryptSize); -#endif - return false; - } - - zObj.AllocBuffer(dataSize); - memcpy(zObj.GetBuffer(), tmp.c_str(), cryptSize); - - if (dataSize - cryptSize > 0) - memcpy(zObj.GetBuffer() + cryptSize, data + cryptSize, dataSize - cryptSize); - - return true; -} - -EterPackPolicy_CSHybridCrypt* CEterPack::GetPackPolicy_HybridCrypt() const -{ - return m_pCSHybridCryptPolicy; -} - - -///////////////////////// - -void CEterFileDict::InsertItem(CEterPack* pkPack, TEterPackIndex* pkInfo) -{ - Item item; - - item.pkPack = pkPack; - item.pkInfo = pkInfo; - - m_dict.insert(TDict::value_type(pkInfo->filename_crc, item)); -} - -void CEterFileDict::UpdateItem(CEterPack* pkPack, TEterPackIndex* pkInfo) -{ - Item item; - - item.pkPack = pkPack; - item.pkInfo = pkInfo; - - TDict::iterator f = m_dict.find(pkInfo->filename_crc); - if (f == m_dict.end()) - m_dict.insert(TDict::value_type(pkInfo->filename_crc, item)); - else - { - if (strcmp(f->second.pkInfo->filename, item.pkInfo->filename) == 0) - { - f->second = item; - } - else - { - TraceError("NAME_COLLISION: OLD: %s NEW: %s", f->second.pkInfo->filename, item.pkInfo->filename); - } - - } -} - -CEterFileDict::Item* CEterFileDict::GetItem(DWORD dwFileNameHash, const char * c_pszFileName) -{ - std::pair iter_pair = m_dict.equal_range(dwFileNameHash); - - TDict::iterator iter = iter_pair.first; - - while (iter != iter_pair.second) - { - Item& item = iter->second; - - if (0 == strcmp(c_pszFileName, item.pkInfo->filename)) - return &item; - - ++iter; - } - - return NULL; -} diff --git a/src/EterPack/EterPack.h b/src/EterPack/EterPack.h deleted file mode 100644 index 343fadb..0000000 --- a/src/EterPack/EterPack.h +++ /dev/null @@ -1,245 +0,0 @@ -#ifndef __INC_ETERPACKLIB_ETERPACK_H__ -#define __INC_ETERPACKLIB_ETERPACK_H__ - -#ifndef MAKEFOURCC -#include -#endif - -#include -#include - -#include "EterBase/MappedFile.h" - -//#define CHECKSUM_CHECK_MD5 -#include "md5.h" - - -namespace eterpack -{ - const DWORD c_PackCC = MAKEFOURCC('E', 'P', 'K', 'D'); - const DWORD c_IndexCC = MAKEFOURCC('E', 'P', 'K', 'D'); - const DWORD c_Version = 2; - // FourCC + Version + m_indexCount - const DWORD c_HeaderSize = sizeof(DWORD) + sizeof(DWORD) + sizeof(long); -}; - -enum EEterPackTypes -{ - DBNAME_MAX_LEN = 255, - FILENAME_MAX_LEN = 160, - FREE_INDEX_BLOCK_SIZE = 32768, - FREE_INDEX_MAX_SIZE = 512, - DATA_BLOCK_SIZE = 256, - - COMPRESSED_TYPE_NONE = 0, - COMPRESSED_TYPE_COMPRESS = 1, - COMPRESSED_TYPE_SECURITY = 2, - COMPRESSED_TYPE_PANAMA = 3, - COMPRESSED_TYPE_HYBRIDCRYPT = 4, - COMPRESSED_TYPE_HYBRIDCRYPT_WITHSDB = 5, - COMPRESSED_TYPE_COUNT = 6, -}; - -#pragma pack(push, 4) -typedef struct SEterPackIndex -{ - long id; - char filename[FILENAME_MAX_LEN + 1]; - DWORD filename_crc; - long real_data_size; - long data_size; -#ifdef CHECKSUM_CHECK_MD5 - BYTE MD5Digest[16]; -#else - DWORD data_crc; -#endif - long data_position; - char compressed_type; -} TEterPackIndex; -#pragma pack(pop) - -typedef std::unordered_map TDataPositionMap; -typedef std::list TFreeIndexList; - - -class CEterPack; - -class CEterFileDict -{ -public: - struct Item - { - Item() : pkPack(NULL), pkInfo(NULL) {} - - CEterPack* pkPack; - TEterPackIndex* pkInfo; - }; - - enum - { - BUCKET_SIZE = 16, - }; - - typedef std::unordered_multimap TDict; - -public: - void InsertItem(CEterPack* pkPack, TEterPackIndex* pkInfo); - void UpdateItem(CEterPack* pkPack, TEterPackIndex* pkInfo); - - Item* GetItem(DWORD dwFileNameHash, const char* c_pszFileName); - - const TDict& GetDict() const - { - return m_dict; - } - -private: - TDict m_dict; -}; - -class EterPackPolicy_CSHybridCrypt; - -class CEterPack -{ - public: - CEterPack(); - virtual ~CEterPack(); - - void Destroy(); - bool Create(CEterFileDict& rkFileDict, const char * dbname, const char * pathName, bool bReadOnly = true, const BYTE* iv = NULL); - bool DecryptIV(DWORD dwPanamaKey); - - const std::string& GetPathName(); - const char * GetDBName(); - - //THEMIDA - bool Get(CMappedFile & mappedFile, const char * filename, LPCVOID * data); - //THEMIDA - bool Get2(CMappedFile & mappedFile, const char * filename, TEterPackIndex* index, LPCVOID * data); - - - //THEMIDA - bool Put(const char * filename, const char * sourceFilename, BYTE packType, const std::string& strRelateMapName); - //THEMIDA - bool Put(const char * filename, LPCVOID data, long len, BYTE packType); - - bool Delete(const char * filename); - - bool Extract(); - - long GetFragmentSize(); - - bool IsExist(const char * filename); - - TDataPositionMap & GetIndexMap(); - - bool EncryptIndexFile(); - bool DecryptIndexFile(); - - DWORD DeleteUnreferencedData(); // 몇개가 삭제 되었는지 리턴 한다. - - bool GetNames(std::vector* retNames); - - EterPackPolicy_CSHybridCrypt* GetPackPolicy_HybridCrypt() const; - - private: - bool __BuildIndex(CEterFileDict& rkFileDict, bool bOverwirte=false); - - bool CreateIndexFile(); - TEterPackIndex * FindIndex(const char * filename); - long GetNewIndexPosition(CFileBase& file); - TEterPackIndex * NewIndex(CFileBase& file, const char * filename, long size); - void WriteIndex(CFileBase& file, TEterPackIndex * index); - int GetFreeBlockIndex(long size); - void PushFreeIndex(TEterPackIndex * index); - - bool CreateDataFile(); - long GetNewDataPosition(CFileBase& file); - bool ReadData(CFileBase& file, TEterPackIndex * index, LPVOID data, long maxsize); - bool WriteData(CFileBase& file, TEterPackIndex * index, LPCVOID data); - bool WriteNewData(CFileBase& file, TEterPackIndex * index, LPCVOID data); - - bool Delete(TEterPackIndex * pIndex); - - protected: - CMappedFile m_file; - - char* m_file_data; - unsigned m_file_size; - - long m_indexCount; - bool m_bEncrypted; - - char m_dbName[DBNAME_MAX_LEN+1]; - char m_indexFileName[MAX_PATH+1]; - TEterPackIndex * m_indexData; - long m_FragmentSize; - bool m_bReadOnly; - bool m_bDecrypedIV; - - std::unordered_map m_map_indexRefCount; - TDataPositionMap m_DataPositionMap; - TFreeIndexList m_FreeIndexList[FREE_INDEX_MAX_SIZE + 1]; // MAX 도 억세스 하므로 + 1 크기만큼 만든다. - - std::string m_stDataFileName; - std::string m_stPathName; - - - EterPackPolicy_CSHybridCrypt* m_pCSHybridCryptPolicy; - - private: - void __CreateFileNameKey_Panama(const char * filename, BYTE * key, unsigned int keySize); - bool __Decrypt_Panama(const char* filename, const BYTE* data, SIZE_T dataSize, CLZObject& zObj); - bool __Encrypt_Panama(const char* filename, const BYTE* data, SIZE_T dataSize, CLZObject& zObj); - std::string m_stIV_Panama; - - //private: - // bool m_bIsDataLoaded; - // // 그냥 time_t를 쓰면, 32bit time_t를 사용하는 소스에서는, - // // CEterPack의 size를 실제 size - 4로 인식하기 때문에 문제가 발생할 수 있다. - // __time64_t m_tLastAccessTime; - //public: - // __time64_t GetLastAccessTime() { return m_tLastAccessTime; } - // void UpdateLastAccessTime(); - // void ClearDataMemoryMap(); - -#ifdef CHECKSUM_CHECK_MD5 - void GenerateMD5Hash( BYTE* pData, int nLength, IN OUT MD5_CTX& context ); -#endif -}; - -class CMakePackLog -{ - public: - static CMakePackLog& GetSingleton(); - - public: - CMakePackLog(); - ~CMakePackLog(); - - void SetFileName(const char* c_szFileName); - - void Writef(const char* c_szFormat, ...); - void Writenf(const char* c_szFormat, ...); - void Write(const char* c_szBuf); - - void WriteErrorf(const char* c_szFormat, ...); - void WriteErrornf(const char* c_szFormat, ...); - void WriteError(const char* c_szBuf); - - void FlushError(); - - private: - void __Write(const char* c_szBuf, int nBufLen); - void __WriteError(const char* c_szBuf, int nBufLen); - bool __IsLogMode(); - - private: - FILE* m_fp; - FILE* m_fp_err; - - std::string m_stFileName; - std::string m_stErrorFileName; -}; - -#endif diff --git a/src/EterPack/EterPackCursor.cpp b/src/EterPack/EterPackCursor.cpp deleted file mode 100644 index 2d58c4d..0000000 --- a/src/EterPack/EterPackCursor.cpp +++ /dev/null @@ -1,59 +0,0 @@ -#include "StdAfx.h" -#include "EterPackCursor.h" -#include "Inline.h" - -CEterPackCursor::CEterPackCursor(CEterPack* pack) : m_pPack(pack), m_pData(NULL), m_ReadPoint(0) -{ -} - -CEterPackCursor::~CEterPackCursor() -{ - Close(); -} - -bool CEterPackCursor::Open(const char* filename) -{ - assert(m_pPack != NULL); - - char tmpFilename[MAX_PATH + 1]; - strncpy(tmpFilename, filename, MAX_PATH); - inlineConvertPackFilename(tmpFilename); - - if (!m_pPack->Get(m_file, tmpFilename, &m_pData)) - return false; - - return true; -} - -void CEterPackCursor::Close() -{ - m_file.Destroy(); - m_pData = NULL; - m_ReadPoint = 0; -} - -void CEterPackCursor::Seek(long offset) -{ - m_ReadPoint = std::max(0l, std::min(Size(), offset)); -} - -bool CEterPackCursor::Read(LPVOID data, long size) -{ - if (m_file.IsNull()) - return false; - - if (m_ReadPoint + size > Size()) - return false; - - memcpy(data, (char*) m_pData + m_ReadPoint, size); - m_ReadPoint += size; - return true; -} - -long CEterPackCursor::Size() -{ - if (m_file.IsNull()) - return 0; - - return m_file.Size(); -} diff --git a/src/EterPack/EterPackCursor.h b/src/EterPack/EterPackCursor.h deleted file mode 100644 index 5ba802a..0000000 --- a/src/EterPack/EterPackCursor.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef __INC_ETERPACKCURSOR_H__ -#define __INC_ETERPACKCURSOR_H__ - -#include "EterPack.h" - -class CEterPackCursor -{ - public: - CEterPackCursor(CEterPack * pack); - ~CEterPackCursor(); - - bool Open(const char* filename); - void Close(); - void Seek(long offset); - bool Read(LPVOID data, long size); - long Size(); - - private: - CEterPack * m_pPack; - CMappedFile m_file; - LPCVOID m_pData; - long m_ReadPoint; -}; - -#endif diff --git a/src/EterPack/EterPackManager.cpp b/src/EterPack/EterPackManager.cpp deleted file mode 100644 index abcebe5..0000000 --- a/src/EterPack/EterPackManager.cpp +++ /dev/null @@ -1,621 +0,0 @@ -#include "StdAfx.h" - -#include -#include - -#include "EterPackManager.h" -#include "EterPackPolicy_CSHybridCrypt.h" - -#include "EterBase/Debug.h" -#include "EterBase/CRC32.h" - -#define PATH_ABSOLUTE_YMIRWORK1 "d:/ymir work/" -#define PATH_ABSOLUTE_YMIRWORK2 "d:\\ymir work\\" - -#ifdef __THEMIDA__ -#include -#endif - - -CEterPack* CEterPackManager::FindPack(const char* c_szPathName) -{ - std::string strFileName; - - if (0 == ConvertFileName(c_szPathName, strFileName)) - { - return &m_RootPack; - } - else - { - for (TEterPackMap::iterator itor = m_DirPackMap.begin(); itor != m_DirPackMap.end(); ++itor) - { - const std::string & c_rstrName = itor->first; - CEterPack * pEterPack = itor->second; - - if (CompareName(c_rstrName.c_str(), c_rstrName.length(), strFileName.c_str())) - { - return pEterPack; - } - } - } - - return NULL; -} - -void CEterPackManager::SetCacheMode() -{ - m_isCacheMode=true; -} - -void CEterPackManager::SetRelativePathMode() -{ - m_bTryRelativePath = true; -} - - -// StringPath std::string 버전 -int CEterPackManager::ConvertFileName(const char * c_szFileName, std::string & rstrFileName) -{ - rstrFileName = c_szFileName; - stl_lowers(rstrFileName); - - int iCount = 0; - - for (DWORD i = 0; i < rstrFileName.length(); ++i) - { - if (rstrFileName[i] == '/') - ++iCount; - else if (rstrFileName[i] == '\\') - { - rstrFileName[i] = '/'; - ++iCount; - } - } - - return iCount; -} - -bool CEterPackManager::CompareName(const char * c_szDirectoryName, DWORD /*dwLength*/, const char * c_szFileName) -{ - const char * c_pszSrc = c_szDirectoryName; - const char * c_pszCmp = c_szFileName; - - while (*c_pszSrc) - { - if (*(c_pszSrc++) != *(c_pszCmp++)) - return false; - - if (!*c_pszCmp) - return false; - } - - return true; -} - -void CEterPackManager::LoadStaticCache(const char* c_szFileName) -{ - if (!m_isCacheMode) - return; - - std::string strFileName; - if (0 == ConvertFileName(c_szFileName, strFileName)) - { - return; - } - - DWORD dwFileNameHash = GetCRC32(strFileName.c_str(), strFileName.length()); - - std::unordered_map::iterator f = m_kMap_dwNameKey_kCache.find(dwFileNameHash); - if (m_kMap_dwNameKey_kCache.end() != f) - return; - - CMappedFile kMapFile; - const void* c_pvData; - if (!Get(kMapFile, c_szFileName, &c_pvData)) - return; - - SCache kNewCache; - kNewCache.m_dwBufSize = kMapFile.Size(); - kNewCache.m_abBufData = new BYTE[kNewCache.m_dwBufSize]; - memcpy(kNewCache.m_abBufData, c_pvData, kNewCache.m_dwBufSize); - m_kMap_dwNameKey_kCache.insert(std::unordered_map::value_type(dwFileNameHash, kNewCache)); -} - -CEterPackManager::SCache* CEterPackManager::__FindCache(DWORD dwFileNameHash) -{ - std::unordered_map::iterator f=m_kMap_dwNameKey_kCache.find(dwFileNameHash); - if (m_kMap_dwNameKey_kCache.end()==f) - return NULL; - - return &f->second; -} - -void CEterPackManager::__ClearCacheMap() -{ - std::unordered_map::iterator i; - - for (i = m_kMap_dwNameKey_kCache.begin(); i != m_kMap_dwNameKey_kCache.end(); ++i) - delete [] i->second.m_abBufData; - - m_kMap_dwNameKey_kCache.clear(); -} - -struct TimeChecker -{ - TimeChecker(const char* name) : name(name) - { - baseTime = timeGetTime(); - } - ~TimeChecker() - { - printf("load %s (%d)\n", name, timeGetTime() - baseTime); - } - - const char* name; - DWORD baseTime; -}; - -bool CEterPackManager::Get(CMappedFile & rMappedFile, const char * c_szFileName, LPCVOID * pData) -{ - //TimeChecker timeChecker(c_szFileName); - //Logf(1, "Load %s\n", c_szFileName); - - if (m_iSearchMode == SEARCH_FILE_FIRST) - { - if (GetFromFile(rMappedFile, c_szFileName, pData)) - { - return true; - } - - return GetFromPack(rMappedFile, c_szFileName, pData); - } - - if (GetFromPack(rMappedFile, c_szFileName, pData)) - { - return true; - } - - return GetFromFile(rMappedFile, c_szFileName, pData); -} - -struct FinderLock -{ - FinderLock(CRITICAL_SECTION& cs) : p_cs(&cs) - { - EnterCriticalSection(p_cs); - } - - ~FinderLock() - { - LeaveCriticalSection(p_cs); - } - - CRITICAL_SECTION* p_cs; -}; - -bool CEterPackManager::GetFromPack(CMappedFile & rMappedFile, const char * c_szFileName, LPCVOID * pData) -{ - FinderLock lock(m_csFinder); - - static std::string strFileName; - - if (0 == ConvertFileName(c_szFileName, strFileName)) - { - return m_RootPack.Get(rMappedFile, strFileName.c_str(), pData); - } - else - { - DWORD dwFileNameHash = GetCRC32(strFileName.c_str(), strFileName.length()); - SCache* pkCache = __FindCache(dwFileNameHash); - - if (pkCache) - { - rMappedFile.Link(pkCache->m_dwBufSize, pkCache->m_abBufData); - return true; - } - - CEterFileDict::Item* pkFileItem = m_FileDict.GetItem(dwFileNameHash, strFileName.c_str()); - - if (pkFileItem) - if (pkFileItem->pkPack) - { - bool r = pkFileItem->pkPack->Get2(rMappedFile, strFileName.c_str(), pkFileItem->pkInfo, pData); - //pkFileItem->pkPack->ClearDataMemoryMap(); - return r; - } - } -#ifdef _DEBUG - TraceError("CANNOT_FIND_PACK_FILE [%s]", strFileName.c_str()); -#endif - - return false; -} - -const time_t g_tCachingInterval = 10; // 10초 -void CEterPackManager::ArrangeMemoryMappedPack() -{ - //time_t curTime = time(NULL); - //CEterFileDict::TDict dict = m_FileDict.GetDict(); - //for (CEterFileDict::TDict::iterator it = dict.begin(); it != dict.end(); ++it) - //{ - // CEterFileDict::Item &rFileItem = it->second; - // CEterPack* pkPack = rFileItem.pkPack; - // if (pkPack) - // { - // if (curTime - pkPack->GetLastAccessTime() > g_tCachingInterval) - // { - // pkPack->ClearDataMemoryMap(); - // } - // } - //} -} - -bool CEterPackManager::GetFromFile(CMappedFile & rMappedFile, const char * c_szFileName, LPCVOID * pData) -{ -#ifndef _DEBUG - //const char *pcExt = strchr(c_szFileName, '.'); - //if (pcExt && - // _strnicmp(pcExt, ".py", 3) == 0 && // python 스크립트 중 - // stricmp(c_szFileName, "logininfo.py") != 0 && // 로그인 정보 파일이 아니고 - // strnicmp(c_szFileName, "locale", 6) != 0 - // ) - //{ - // return false; - //} -#endif - - //if(m_bTryRelativePath) { - // if (strnicmp(c_szFileName, PATH_ABSOLUTE_YMIRWORK1, strlen(PATH_ABSOLUTE_YMIRWORK1)) == 0 || strnicmp(c_szFileName, PATH_ABSOLUTE_YMIRWORK2, strlen(PATH_ABSOLUTE_YMIRWORK2)) == 0) { - // if(rMappedFile.Create(c_szFileName+strlen(PATH_ABSOLUTE_YMIRWORK1), pData, 0, 0)) - // { - // return true; - // } - // } - //} - - return rMappedFile.Create(c_szFileName, pData, 0, 0) ? true : false; -} - -bool CEterPackManager::isExistInPack(const char * c_szFileName) -{ - std::string strFileName; - - if (0 == ConvertFileName(c_szFileName, strFileName)) - { - return m_RootPack.IsExist(strFileName.c_str()); - } - else - { - DWORD dwFileNameHash = GetCRC32(strFileName.c_str(), strFileName.length()); - CEterFileDict::Item* pkFileItem = m_FileDict.GetItem(dwFileNameHash, strFileName.c_str()); - - if (pkFileItem) - if (pkFileItem->pkPack) - return pkFileItem->pkPack->IsExist(strFileName.c_str()); - } - - // NOTE : 매치 되는 팩이 없다면 false - [levites] - return false; -} - -bool CEterPackManager::isExist(const char * c_szFileName) -{ - if (m_iSearchMode == SEARCH_PACK_FIRST) - { - if (isExistInPack(c_szFileName)) - return true; - - return _access(c_szFileName, 0) == 0 ? true : false; - } - - //if(m_bTryRelativePath) { - // if (strnicmp(c_szFileName, PATH_ABSOLUTE_YMIRWORK1, strlen(PATH_ABSOLUTE_YMIRWORK1)) == 0 || strnicmp(c_szFileName, PATH_ABSOLUTE_YMIRWORK2, strlen(PATH_ABSOLUTE_YMIRWORK2)) == 0) { - // if(access(c_szFileName+strlen(PATH_ABSOLUTE_YMIRWORK1), 0) == 0) - // return true; - // } - //} - - if (_access(c_szFileName, 0) == 0) - return true; - - return isExistInPack(c_szFileName); -} - - -void CEterPackManager::RegisterRootPack(const char * c_szName) -{ - if (!m_RootPack.Create(m_FileDict, c_szName, "")) - { - TraceError("%s: Pack file does not exist", c_szName); - } -} - -const char * CEterPackManager::GetRootPackFileName() -{ - return m_RootPack.GetDBName(); -} - -bool CEterPackManager::DecryptPackIV(DWORD dwPanamaKey) -{ - TEterPackMap::iterator itor = m_PackMap.begin(); - while (itor != m_PackMap.end()) - { - itor->second->DecryptIV(dwPanamaKey); - itor++; - } - return true; -} - -bool CEterPackManager::RegisterPackWhenPackMaking(const char * c_szName, const char * c_szDirectory, CEterPack* pPack) -{ - m_PackMap.insert(TEterPackMap::value_type(c_szName, pPack)); - m_PackList.push_front(pPack); - - m_DirPackMap.insert(TEterPackMap::value_type(c_szDirectory, pPack)); - return true; -} - - -bool CEterPackManager::RegisterPack(const char * c_szName, const char * c_szDirectory, const BYTE* c_pbIV) -{ - CEterPack * pEterPack = NULL; - { - TEterPackMap::iterator itor = m_PackMap.find(c_szName); - - if (m_PackMap.end() == itor) - { - bool bReadOnly = true; - - pEterPack = new CEterPack; - if (pEterPack->Create(m_FileDict, c_szName, c_szDirectory, bReadOnly, c_pbIV)) - { - m_PackMap.insert(TEterPackMap::value_type(c_szName, pEterPack)); - } - else - { -#ifdef _DEBUG - Tracef("The eterpack doesn't exist [%s]\n", c_szName); -#endif - delete pEterPack; - pEterPack = NULL; - return false; - } - } - else - { - pEterPack = itor->second; - } - } - - if (c_szDirectory && c_szDirectory[0] != '*') - { - TEterPackMap::iterator itor = m_DirPackMap.find(c_szDirectory); - if (m_DirPackMap.end() == itor) - { - m_PackList.push_front(pEterPack); - m_DirPackMap.insert(TEterPackMap::value_type(c_szDirectory, pEterPack)); - } - } - - return true; -} - -void CEterPackManager::SetSearchMode(bool bPackFirst) -{ - m_iSearchMode = bPackFirst ? SEARCH_PACK_FIRST : SEARCH_FILE_FIRST; -} - -int CEterPackManager::GetSearchMode() -{ - return m_iSearchMode; -} - -CEterPackManager::CEterPackManager() : m_bTryRelativePath(false), m_iSearchMode(SEARCH_FILE_FIRST), m_isCacheMode(false) -{ - InitializeCriticalSection(&m_csFinder); -} - -CEterPackManager::~CEterPackManager() -{ - __ClearCacheMap(); - - TEterPackMap::iterator i = m_PackMap.begin(); - TEterPackMap::iterator e = m_PackMap.end(); - while (i != e) - { - delete i->second; - i++; - } - DeleteCriticalSection(&m_csFinder); -} - -void CEterPackManager::RetrieveHybridCryptPackKeys(const BYTE *pStream) -{ - ////dump file format - //total packagecnt (4byte) - // for packagecntpackage - // db name hash ( stl.h stringhash ) - // extension cnt( 4byte) - // for extension cnt - // ext hash ( stl.h stringhash ) - // key-16byte - // iv-16byte - int iMemOffset = 0; - - int iPackageCnt; - DWORD dwPackageNameHash; - - memcpy( &iPackageCnt, pStream + iMemOffset, sizeof(int) ); - iMemOffset += sizeof(iPackageCnt); - - for( int i = 0; i < iPackageCnt; ++i ) - { - int iRecvedCryptKeySize = 0; - memcpy( &iRecvedCryptKeySize, pStream + iMemOffset, sizeof(iRecvedCryptKeySize) ); - iRecvedCryptKeySize -= sizeof(dwPackageNameHash); // 서버에서 받은 key stream에는 filename hash가 포함되어 있으므로, hash 사이즈 만큼 배줌. - iMemOffset += sizeof(iRecvedCryptKeySize); - - memcpy( &dwPackageNameHash, pStream + iMemOffset, sizeof(dwPackageNameHash) ); - iMemOffset += sizeof(dwPackageNameHash); - - TEterPackMap::const_iterator cit; - for( cit = m_PackMap.begin(); cit != m_PackMap.end(); ++cit ) - { - auto ssvv = std::string(cit->first); - std::string noPathName = CFileNameHelper::NoPath(ssvv); - if( dwPackageNameHash == stringhash().GetHash(noPathName) ) - { - EterPackPolicy_CSHybridCrypt* pCryptPolicy = cit->second->GetPackPolicy_HybridCrypt(); - int iHavedCryptKeySize = pCryptPolicy->ReadCryptKeyInfoFromStream( pStream + iMemOffset ); - if (iRecvedCryptKeySize != iHavedCryptKeySize) - { - TraceError("CEterPackManager::RetrieveHybridCryptPackKeys cryptokey length of file(%s) is not matched. received(%d) != haved(%d)", noPathName.c_str(), iRecvedCryptKeySize, iHavedCryptKeySize); - } - break; - } - } - iMemOffset += iRecvedCryptKeySize; - } -} - -void CEterPackManager::RetrieveHybridCryptPackSDB( const BYTE* pStream ) -{ - //cnt - //for cnt - //DWORD dwPackageIdentifier; - //DWORD dwFileIdentifier; - //std::vector vecSDBStream; - int iReadOffset = 0; - int iSDBInfoCount = 0; - - memcpy( &iSDBInfoCount, pStream+iReadOffset, sizeof(int) ); - iReadOffset += sizeof(int); - - for( int i = 0; i < iSDBInfoCount; ++i ) - { - DWORD dwPackgeIdentifier; - memcpy( &dwPackgeIdentifier, pStream+iReadOffset, sizeof(DWORD) ); - iReadOffset += sizeof(DWORD); - - TEterPackMap::const_iterator cit; - for( cit = m_PackMap.begin(); cit != m_PackMap.end(); ++cit ) - { - auto ssvv = std::string(cit->first); - std::string noPathName = CFileNameHelper::NoPath(ssvv); - if( dwPackgeIdentifier == stringhash().GetHash(noPathName) ) - { - EterPackPolicy_CSHybridCrypt* pCryptPolicy = cit->second->GetPackPolicy_HybridCrypt(); - iReadOffset += pCryptPolicy->ReadSupplementatyDataBlockFromStream( pStream+iReadOffset ); - break; - } - } - } -} - - -void CEterPackManager::WriteHybridCryptPackInfo(const char* pFileName) -{ - //NOTE : this file format contains a little bit of redundant data. - //however it`s better for seperating cryptkey & supplementary data block. - - //dump file format - - //SDB data offset(4) - - // about cryptkey - //total packagecnt (4byte) - // for packagecnt - // db name hash 4byte( stl.h stringhash ) - // extension cnt( 4byte) - // for extension cnt - // ext hash ( stl.h stringhash ) - // key-16byte - // iv-16byte - - //about SDB data - //total packagecnt (4byte) - // for packagecnt - // db name hash 4byte( stl.h stringhash ) +child node size(4byte) - // sdb file cnt( 4byte ) - // for sdb file cnt - // filename hash ( stl.h stringhash ) - // related map name size(4), relate map name - // sdb block size( 1byte ) - // sdb blocks - - CFileBase keyFile; - - if( !keyFile.Create( pFileName, CFileBase::FILEMODE_WRITE) ) - { - //TODO : write log - return; - } - - int iKeyPackageCount = 0; - - //write later ( SDB Offset & PackageCnt for Key ) - keyFile.SeekCur(2*sizeof(int)); - - TEterPackMap::const_iterator cit; - for( cit = m_PackMap.begin(); cit != m_PackMap.end(); ++cit ) - { - EterPackPolicy_CSHybridCrypt* pPolicy = cit->second->GetPackPolicy_HybridCrypt(); - if( !pPolicy || !pPolicy->IsContainingCryptKey() ) - continue; - - iKeyPackageCount++; - auto ssvv = std::string(cit->first); - std::string noPathName = CFileNameHelper::NoPath(ssvv); - - DWORD dwPackNamehash = stringhash().GetHash(noPathName); - - CMakePackLog::GetSingleton().Writef("CEterPackManager::WriteHybridCryptPackInfo PackName : %s, Hash : %x", noPathName.c_str(), dwPackNamehash); - keyFile.Write( &dwPackNamehash, sizeof(DWORD) ); - - pPolicy->WriteCryptKeyToFile( keyFile ); - } - - //Write SDB Data - int iSDBDataOffset = keyFile.GetPosition(); - int iSDBPackageCnt = 0; - - //Write SDB PackageCnt Later - keyFile.SeekCur(sizeof(int)); - for( cit = m_PackMap.begin(); cit != m_PackMap.end(); ++cit ) - { - EterPackPolicy_CSHybridCrypt* pPolicy = cit->second->GetPackPolicy_HybridCrypt(); - if( !pPolicy || !pPolicy->IsContainingSDBFile() ) - continue; - - iSDBPackageCnt++; - - auto ssvv1 = std::string(cit->first); - auto noPathName = CFileNameHelper::NoPath(ssvv1); - - DWORD dwPackNamehash = stringhash().GetHash(noPathName); - keyFile.Write( &dwPackNamehash, sizeof(DWORD) ); - - int iSDBSizeWriteOffset = keyFile.GetPosition(); - keyFile.SeekCur(sizeof(int)); - - pPolicy->WriteSupplementaryDataBlockToFile( keyFile ); - int iSDBSizeAfterWrite = keyFile.GetPosition(); - - keyFile.Seek(iSDBSizeWriteOffset); - - int iSDBSize = iSDBSizeAfterWrite-(iSDBSizeWriteOffset+4); - keyFile.Write( &iSDBSize, sizeof(int) ); - - keyFile.Seek(iSDBSizeAfterWrite); - } - - //write sdb data start offset & package cnt - keyFile.Seek(0); - keyFile.Write( &iSDBDataOffset, sizeof(int)); - keyFile.Write( &iKeyPackageCount, sizeof(int)); - - keyFile.Seek(iSDBDataOffset); - keyFile.Write( &iSDBPackageCnt, sizeof(int)); - - keyFile.Close(); -} diff --git a/src/EterPack/EterPackManager.h b/src/EterPack/EterPackManager.h deleted file mode 100644 index 248b3c4..0000000 --- a/src/EterPack/EterPackManager.h +++ /dev/null @@ -1,96 +0,0 @@ -#pragma once - -#include -#include -#include "EterBase/Singleton.h" -#include "EterBase/Stl.h" - -#include "EterPack.h" - -class CEterPackManager : public CSingleton -{ - public: - struct SCache - { - BYTE* m_abBufData; - DWORD m_dwBufSize; - }; - public: - enum ESearchModes - { - SEARCH_FILE_FIRST, - SEARCH_PACK_FIRST - }; - - typedef std::list TEterPackList; - typedef std::unordered_map TEterPackMap; - - public: - CEterPackManager(); - virtual ~CEterPackManager(); - - void SetCacheMode(); - void SetRelativePathMode(); - - void LoadStaticCache(const char* c_szFileName); - - void SetSearchMode(bool bPackFirst); - int GetSearchMode(); - - //THEMIDA - bool Get(CMappedFile & rMappedFile, const char * c_szFileName, LPCVOID * pData); - - //THEMIDA - bool GetFromPack(CMappedFile & rMappedFile, const char * c_szFileName, LPCVOID * pData); - - //THEMIDA - bool GetFromFile(CMappedFile & rMappedFile, const char * c_szFileName, LPCVOID * pData); - bool isExist(const char * c_szFileName); - bool isExistInPack(const char * c_szFileName); - - bool RegisterPack(const char * c_szName, const char * c_szDirectory, const BYTE* c_pbIV = NULL); - void RegisterRootPack(const char * c_szName); - bool RegisterPackWhenPackMaking(const char * c_szName, const char * c_szDirectory, CEterPack* pPack); - - - bool DecryptPackIV(DWORD key); - - const char * GetRootPackFileName(); - - //for hybridcrypt - //THEMIDA - void WriteHybridCryptPackInfo(const char* pFileName); - - //THEMIDA - void RetrieveHybridCryptPackKeys( const BYTE* pStream ); - //THEMIDA - void RetrieveHybridCryptPackSDB( const BYTE* pStream ); - - // 메모리에 매핑된 팩들 가운데, 정리해야할 것들 정리. - public: - void ArrangeMemoryMappedPack(); - - protected: - int ConvertFileName(const char * c_szFileName, std::string & rstrFileName); // StringPath std::string 버전 - bool CompareName(const char * c_szDirectoryName, DWORD iLength, const char * c_szFileName); - - CEterPack* FindPack(const char* c_szPathName); - - SCache* __FindCache(DWORD dwFileNameHash); - void __ClearCacheMap(); - - protected: - bool m_bTryRelativePath; - bool m_isCacheMode; - int m_iSearchMode; - - CEterFileDict m_FileDict; - CEterPack m_RootPack; - TEterPackList m_PackList; - TEterPackMap m_PackMap; - TEterPackMap m_DirPackMap; - - std::unordered_map m_kMap_dwNameKey_kCache; - - CRITICAL_SECTION m_csFinder; -}; diff --git a/src/EterPack/EterPackPolicy_CSHybridCrypt.cpp b/src/EterPack/EterPackPolicy_CSHybridCrypt.cpp deleted file mode 100644 index 4df786f..0000000 --- a/src/EterPack/EterPackPolicy_CSHybridCrypt.cpp +++ /dev/null @@ -1,503 +0,0 @@ -#include "Stdafx.h" -#include "EterPackPolicy_CSHybridCrypt.h" -#include "EterBase/Stl.h" -#include "EterBase/FileName.h" -#include "EterBase/FileBase.h" -#include "EterBase/Crc32.h" -#include "EterBase/lzo.h" -#include "EterBase/Random.h" -#include -#include - -using namespace CryptoPP; - -#define CIPHER_MODE CTR_Mode - -#ifdef __THEMIDA__ -#include -#endif -//Cipher -//Block Size -//Key Length -// -//Default Minimum Maximum -//AES(Rijndael) 16 16 16 32 -//Blowfish 8 16 0 56 -//Camellia 16 16 16 32 -//CAST-128 8 16 5 16 -//CAST-256 16 16 16 32 -//DES 8 8 8 8 -//DES-EDE2 8 16 16 16 -//DES-EDE3 8 24 24 24 -//DES-XEX3 8 24 24 24 -//GOST 8 32 32 32 -//IDEA 8 16 16 16 -//MARS 16 16 16 56 -//RC2 8 16 1 128 -//RC5 8 16 0 255 -//RC6 16 16 0 255 -//SAFER-K 8 16 8 16 -//SAFER-SK 8 16 8 16 -//Serpent 16 16 1 32 -//SHACAL-2 32 16 1 64 -//SHARK-E 8 16 1 16 -//SKIPJACK 8 10 1 10 -//3-Way 12 12 1 12 -//Twofish 16 16 0 32 -//XTEA 8 16 1 16 - -inline std::string GetFileExt( std::string& rfileName ) -{ - stl_lowers(rfileName); - return CFileNameHelper::GetExtension(rfileName); -} - -EterPackPolicy_CSHybridCrypt::~EterPackPolicy_CSHybridCrypt() -{ - m_mapHybridCryptKey.clear(); - m_mapSDBMap.clear(); -} - -bool EterPackPolicy_CSHybridCrypt::IsContainingCryptKey() const -{ - return (m_mapHybridCryptKey.size() > 0) ? true : false; -} - - -bool EterPackPolicy_CSHybridCrypt::GenerateCryptKey( std::string& rfileName ) -{ -#ifdef __THEMIDA__ - VM_START -#endif - - //make lower & extract ext - std::string extName = GetFileExt(rfileName); - stl_lowers(extName); - - DWORD dwExtHash = stringhash().GetHash(extName); - - TCSHybridCryptKeyMap::const_iterator cit = m_mapHybridCryptKey.find( dwExtHash ); - - if( cit != m_mapHybridCryptKey.end() ) - { - //TODO : log already registered - return false; - } - - static AutoSeededRandomPool rnd; - - TCSHybridCryptKey info; - { - rnd.GenerateBlock( &(info.uEncryptKey.key[0]), sizeof(info.uEncryptKey) ); - rnd.GenerateBlock( &(info.uEncryptIV.iv[0]), sizeof(info.uEncryptIV) ); - - //for test - /* memset( &info.uEncryptKey.key, 0x10, sizeof(info.uEncryptKey) ); - memset( &info.uEncryptIV.iv, 0x10, sizeof(info.uEncryptIV) ); */ - } - m_mapHybridCryptKey[dwExtHash] = info; - -#ifdef __THEMIDA__ - VM_END -#endif - - return true; -} - -bool EterPackPolicy_CSHybridCrypt::GetPerFileCryptKey( std::string& rfileName, eHybridCipherAlgorithm& eAlgorithm, TEncryptKey& key, TEncryptIV& iv ) -{ -#ifdef __THEMIDA__ - VM_START -#endif - - std::string fileNamelower = rfileName; - stl_lowers(fileNamelower); - - std::string extName = GetFileExt(fileNamelower); - - TCSHybridCryptKeyMap::const_iterator cit = m_mapHybridCryptKey.find( stringhash().GetHash(extName)); - - if( cit == m_mapHybridCryptKey.end() ) - { - //TODO : log no file ext info - return false; - } - - DWORD dwfileNameCrc = GetCRC32(fileNamelower.c_str(), fileNamelower.size()); - - //make file specific algorithm & key & iv - eAlgorithm = (eHybridCipherAlgorithm)(dwfileNameCrc % Num_Of_Ciphers); - - ::memcpy(key.key, cit->second.uEncryptKey.key, sizeof(key) ); - ::memcpy(iv.iv, cit->second.uEncryptIV.iv, sizeof(iv) ); - - - //Themida Warning - for( int i = 0; i < (sizeof(key)/sizeof(dwfileNameCrc)); ++i) - { - *((DWORD*)key.key + i) ^= dwfileNameCrc; - } - for( int i = 0; i < (sizeof(iv)/sizeof(dwfileNameCrc)); ++i) - { - *((DWORD*)iv.iv + i) ^= dwfileNameCrc; - } -#ifdef __THEMIDA__ - VM_END -#endif - - return true; -} - - -bool EterPackPolicy_CSHybridCrypt::EncryptMemory( std::string& rfileName, IN const BYTE* pSrcData, IN int iSrcLen, OUT CLZObject& zObj ) -{ -#ifdef __THEMIDA__ - VM_START -#endif - - eHybridCipherAlgorithm eAlgorithm; - TEncryptKey key; - TEncryptIV iv; - - if( !GetPerFileCryptKey( rfileName, eAlgorithm, key, iv ) ) - { - return false; - } - - // start cipher - std::string strCipher; - //NOTE : ciphered stream size could be different from original size if you choose diffrent cipher mode & algorithm - //( i.e ECB or CBC mode ) - strCipher.reserve(iSrcLen); - - if( eAlgorithm == e_Cipher_Camellia ) - { - // Encryptor - CIPHER_MODE::Encryption Encryptor; - Encryptor.SetKeyWithIV(key.keyCamellia, sizeof(key.keyCamellia), iv.ivCamellia, sizeof(iv.ivCamellia)); - - ArraySource(pSrcData, iSrcLen, true, - new StreamTransformationFilter(Encryptor, new CryptoPP::StringSink(strCipher))); - } - else if( eAlgorithm == e_Cipher_Twofish ) - { - // Encryptor - CIPHER_MODE::Encryption Encryptor; - Encryptor.SetKeyWithIV(key.keyTwofish, sizeof(key.keyTwofish), iv.ivTwofish, sizeof(iv.ivTwofish)); - - ArraySource(pSrcData, iSrcLen, true, - new StreamTransformationFilter(Encryptor, new CryptoPP::StringSink(strCipher))); - } - else if( eAlgorithm == e_Cipher_XTEA ) - { - // Encryptor - CIPHER_MODE::Encryption Encryptor; - Encryptor.SetKeyWithIV(key.keyXTEA, sizeof(key.keyXTEA), iv.ivXTEA, sizeof(iv.ivXTEA)); - - ArraySource(pSrcData, iSrcLen, true, - new StreamTransformationFilter(Encryptor, new CryptoPP::StringSink(strCipher))); - } - - if (strCipher.length() != iSrcLen) - { - //TODO: size error log - return false; - } - - zObj.AllocBuffer(iSrcLen); - memcpy(zObj.GetBuffer(), strCipher.c_str(), strCipher.length() ); - -#ifdef __THEMIDA__ - VM_END -#endif - - return true; -} - -bool EterPackPolicy_CSHybridCrypt::DecryptMemory( std::string& rfilename, IN const BYTE* pEncryptedData, IN int iEncryptedLen, OUT CLZObject& zObj ) -{ -#ifdef __THEMIDA__ - VM_START -#endif - - eHybridCipherAlgorithm eAlgorithm; - TEncryptKey key; - TEncryptIV iv; - - if( !GetPerFileCryptKey( rfilename, eAlgorithm, key, iv ) ) - { - return false; - } - - // start decipher - std::string strDecipher; - //NOTE : ciphered stream size could be different from original size if you choose diffrent cipher mode & algorithm - //( i.e ECB or CBC mode ) - strDecipher.reserve(iEncryptedLen); - - if( eAlgorithm == e_Cipher_Camellia ) - { - // Decryptor - CIPHER_MODE::Decryption Decryptor; - Decryptor.SetKeyWithIV(key.keyCamellia, sizeof(key.keyCamellia), iv.ivCamellia, sizeof(iv.ivCamellia)); - - ArraySource(pEncryptedData, iEncryptedLen, true, - new StreamTransformationFilter(Decryptor, new CryptoPP::StringSink(strDecipher))); - } - else if( eAlgorithm == e_Cipher_Twofish ) - { - // Decryptor - CIPHER_MODE::Decryption Decryptor; - Decryptor.SetKeyWithIV(key.keyTwofish, sizeof(key.keyTwofish), iv.ivTwofish, sizeof(iv.ivTwofish)); - - ArraySource(pEncryptedData, iEncryptedLen, true, - new StreamTransformationFilter(Decryptor, new CryptoPP::StringSink(strDecipher))); - } - else if( eAlgorithm == e_Cipher_XTEA ) - { - // Decryptor - CIPHER_MODE::Decryption Decryptor; - Decryptor.SetKeyWithIV(key.keyXTEA, sizeof(key.keyXTEA), iv.ivXTEA, sizeof(iv.ivXTEA)); - - ArraySource(pEncryptedData, iEncryptedLen, true, - new StreamTransformationFilter(Decryptor, new CryptoPP::StringSink(strDecipher))); - } - - - if (strDecipher.length() != iEncryptedLen) - { - //TODO: size error log - return false; - } - - zObj.AllocBuffer(iEncryptedLen); - memcpy(zObj.GetBuffer(), strDecipher.c_str(), strDecipher.length() ); - -#ifdef __THEMIDA__ - VM_END -#endif - - return true; -} - -void EterPackPolicy_CSHybridCrypt::WriteCryptKeyToFile( CFileBase& rFile ) -{ - // ext cnt 4byte - // for ext hash ( crc32 ) - // key-16byte - // iv-16byte - - DWORD dwCryptKeySize = m_mapHybridCryptKey.size(); - rFile.Write( &dwCryptKeySize, sizeof(DWORD) ); - - TCSHybridCryptKeyMap::const_iterator cit; - for( cit = m_mapHybridCryptKey.begin(); cit != m_mapHybridCryptKey.end(); ++cit ) - { - DWORD extNamehash = cit->first; - const TCSHybridCryptKey& CryptKey = cit->second; - - rFile.Write( &extNamehash, sizeof(DWORD) ); - rFile.Write( CryptKey.uEncryptKey.key, sizeof(TEncryptKey) ); - rFile.Write( CryptKey.uEncryptIV.iv, sizeof(TEncryptIV) ); - } -} - -int EterPackPolicy_CSHybridCrypt::ReadCryptKeyInfoFromStream( IN const BYTE* pStream ) -{ - int iStreamOffset = 0; - - DWORD dwCryptoInfoSize; - memcpy(&dwCryptoInfoSize, pStream, sizeof(DWORD) ); - iStreamOffset += sizeof(DWORD); - - DWORD dwExtHash; - - m_mapHybridCryptKey.clear(); - - for( int i = 0; i < dwCryptoInfoSize; ++i ) - { - memcpy(&dwExtHash, pStream + iStreamOffset, sizeof(DWORD) ); - iStreamOffset += sizeof(DWORD); - - TCSHybridCryptKey info; - { - memcpy(info.uEncryptKey.key, pStream + iStreamOffset, sizeof(TEncryptKey) ); - iStreamOffset += sizeof(TEncryptKey); - - memcpy(info.uEncryptIV.iv, pStream + iStreamOffset, sizeof(TEncryptIV) ); - iStreamOffset += sizeof(TEncryptIV); - } - - m_mapHybridCryptKey[dwExtHash] = info; - } - - return iStreamOffset; -} - -bool EterPackPolicy_CSHybridCrypt::GenerateSupplementaryDataBlock(std::string& rfilename, const std::string& strMapName, IN const BYTE* pSrcData, IN int iSrcLen, OUT LPBYTE& pDestData, OUT int& iDestLen ) -{ -#ifdef __THEMIDA__ - VM_START -#endif - - std::string fileNamelower = rfilename; - stl_lowers( fileNamelower ); - - DWORD dwFileNameHash = stringhash().GetHash(fileNamelower); - TSupplementaryDataBlockMap::const_iterator cit = m_mapSDBMap.find( dwFileNameHash ); - - if( cit != m_mapSDBMap.end() ) - { - //TODO : log already registered - return false; - } - - //TODO : Find Better Method for deciding SDB Postion & Size - - //prevent stream copy duplication - TSupplementaryDataBlockInfo info; - m_mapSDBMap[dwFileNameHash] = info; - - std::string& strRelatedMapName = m_mapSDBMap[dwFileNameHash].strRelatedMapName; - std::vector& sdbVector = m_mapSDBMap[dwFileNameHash].vecStream; - - - //fill the data!! - { - strRelatedMapName = strMapName; - - int iSDBSize = random_range( 64, 128 ); - - if( iSrcLen < iSDBSize ) - { - iSDBSize = iSrcLen - 1; - if( iSDBSize <= 0 ) - { - //TODO : is there 1byte file exist??? - return false; - } - } - - sdbVector.resize( iSDBSize ); - - iDestLen = iSrcLen - iSDBSize; - pDestData = (LPBYTE)pSrcData; - - memcpy( &sdbVector[0], pDestData + iDestLen, iSDBSize ); - } -#ifdef __THEMIDA__ - VM_END -#endif - return true; -} - - -bool EterPackPolicy_CSHybridCrypt::GetSupplementaryDataBlock( std::string& rfilename, OUT LPBYTE& pSDB, OUT int& iSDBSize ) -{ -#ifdef __THEMIDA__ - VM_START -#endif - - std::string fileNamelower = rfilename; - stl_lowers( fileNamelower ); - - DWORD dwFileNameHash = stringhash().GetHash(fileNamelower); - TSupplementaryDataBlockMap::const_iterator cit = m_mapSDBMap.find( dwFileNameHash ); - - if( cit == m_mapSDBMap.end() ) - { - //TODO : log already registered - return false; - } - - const std::vector& vecSDB = cit->second.vecStream; - - iSDBSize = vecSDB.size(); - - if(iSDBSize <= 0) - { - pSDB = NULL; - return false; - } - - pSDB = (BYTE*)&vecSDB[0]; -#ifdef __THEMIDA__ - VM_END -#endif - - return true; -} - -bool EterPackPolicy_CSHybridCrypt::IsContainingSDBFile() const -{ - return m_mapSDBMap.size() > 0 ? true : false; -} - -void EterPackPolicy_CSHybridCrypt::WriteSupplementaryDataBlockToFile( CFileBase& rFile ) -{ - //about SDB data - // sdb file cnt( 4byte ) - // for sdb file cnt - // filename hash ( stl.h stringhash ) - // related map name size(4), relate map name - // sdb block size( 1byte ) - // sdb blocks - - DWORD dwSDBMapSize = m_mapSDBMap.size(); - rFile.Write( &dwSDBMapSize, sizeof(DWORD) ); - - TSupplementaryDataBlockMap::const_iterator cit; - for( cit = m_mapSDBMap.begin(); cit != m_mapSDBMap.end(); ++cit ) - { - DWORD dwFileNamehash = cit->first; - rFile.Write( &dwFileNamehash, sizeof(DWORD) ); - - const std::string strRelatedMapName = cit->second.strRelatedMapName; - DWORD dwMapNameSize = strRelatedMapName.size(); - rFile.Write( &dwMapNameSize, sizeof(DWORD) ); - rFile.Write( strRelatedMapName.c_str(), dwMapNameSize ); - - const std::vector& sdbVector = cit->second.vecStream; - BYTE bSDBSize = (BYTE)(sdbVector.size()); - - rFile.Write( &bSDBSize, sizeof(bSDBSize) ); - if( bSDBSize > 0 ) - rFile.Write( &sdbVector[0], bSDBSize ); - } -} - -int EterPackPolicy_CSHybridCrypt::ReadSupplementatyDataBlockFromStream( IN const BYTE* pStream ) -{ -#ifdef __THEMIDA__ - VM_START -#endif - - //DWORD dwFileIdentifier; - //std::vector vecSDBStream; - - DWORD dwFileNameHash; - BYTE bSDBSize; - int iStreamOffset = 0; - - memcpy(&dwFileNameHash, pStream + iStreamOffset, sizeof(DWORD) ); - iStreamOffset += sizeof(DWORD); - - memcpy(&bSDBSize, pStream + iStreamOffset, sizeof(BYTE) ); - iStreamOffset += sizeof(BYTE); - - // NOTE : related map name isn`t required in client. so we don`t recv it from stream to reduce packet size. - TSupplementaryDataBlockInfo info; - { - info.vecStream.resize( bSDBSize ); - memcpy(&info.vecStream[0], pStream + iStreamOffset, bSDBSize ); - iStreamOffset += bSDBSize; - - m_mapSDBMap[dwFileNameHash] = info; - } - -#ifdef __THEMIDA__ - VM_END -#endif - - return iStreamOffset; -} diff --git a/src/EterPack/EterPackPolicy_CSHybridCrypt.h b/src/EterPack/EterPackPolicy_CSHybridCrypt.h deleted file mode 100644 index 0b1a5fc..0000000 --- a/src/EterPack/EterPackPolicy_CSHybridCrypt.h +++ /dev/null @@ -1,94 +0,0 @@ -#ifndef __INC_ETERPACKLIB_ETERPACKPOLICY_CSHYBRIDCRYPT_H__ -#define __INC_ETERPACKLIB_ETERPACKPOLICY_CSHYBRIDCRYPT_H__ - -#include -#include -#include -#include -#include - -enum eHybridCipherAlgorithm -{ - e_Cipher_Camellia, - e_Cipher_Twofish, - e_Cipher_XTEA, - Num_Of_Ciphers -}; - - -class CFileBase; -class CLZObject; - -//THEMIDA -class EterPackPolicy_CSHybridCrypt -{ -public: - ~EterPackPolicy_CSHybridCrypt(); - - bool GenerateCryptKey( std::string& rfileName ); - bool EncryptMemory( std::string& rfilename, IN const BYTE* pSrcData, IN int iSrcLen, OUT CLZObject& zObj ); - bool DecryptMemory( std::string& rfilename, IN const BYTE* pSrcData, IN int iSrcLen, OUT CLZObject& zObj ); - bool IsContainingCryptKey() const; - - //Supplementary Data Block (SDB) - bool GenerateSupplementaryDataBlock(std::string& rfilename, const std::string& strMapName, IN const BYTE* pSrcData, IN int iSrcLen, OUT LPBYTE& pDestData, OUT int& iDestLen ); - bool GetSupplementaryDataBlock( std::string& rfilename, OUT LPBYTE& pSDB, OUT int& iSDBSize ); - bool IsContainingSDBFile() const; - - // Read/Write IO - void WriteCryptKeyToFile( CFileBase& rFile ); - int ReadCryptKeyInfoFromStream( IN const BYTE* pStream ); - - void WriteSupplementaryDataBlockToFile( CFileBase& rFile ); - int ReadSupplementatyDataBlockFromStream( IN const BYTE* pStream ); - -protected: - - typedef union UEncryptKey - { - BYTE key[16]; - BYTE keyCamellia[ CryptoPP::Camellia::DEFAULT_KEYLENGTH]; - BYTE keyTwofish [ CryptoPP::Twofish::DEFAULT_KEYLENGTH]; - BYTE keyXTEA [ CryptoPP::XTEA::DEFAULT_KEYLENGTH]; - - } TEncryptKey; - - typedef union UEncryptIV - { - BYTE iv[16]; - BYTE ivCamellia [ CryptoPP::Camellia::BLOCKSIZE]; - BYTE ivTwofish [ CryptoPP::Twofish::BLOCKSIZE]; - BYTE ivXTEA [ CryptoPP::XTEA::BLOCKSIZE]; - - } TEncryptIV; - - - typedef struct SCSHybridCryptKey - { - TEncryptKey uEncryptKey; - TEncryptIV uEncryptIV; - - } TCSHybridCryptKey; - - typedef std::unordered_map TCSHybridCryptKeyMap; - TCSHybridCryptKeyMap m_mapHybridCryptKey; - - typedef struct SSupplementaryDataBlockInfo - { - std::string strRelatedMapName; - std::vector vecStream; - - } TSupplementaryDataBlockInfo; - - typedef std::unordered_map TSupplementaryDataBlockMap; //key filename hash - TSupplementaryDataBlockMap m_mapSDBMap; - -private: - bool GetPerFileCryptKey( std::string& rfileName, eHybridCipherAlgorithm& eAlgorithm, TEncryptKey& key, TEncryptIV& iv ); - - -}; - - - -#endif // __INC_ETERPACKLIB_ETERPACKPOLICY_CSHYBRIDCRYPT_H__ diff --git a/src/EterPack/Inline.h b/src/EterPack/Inline.h deleted file mode 100644 index 92dbb49..0000000 --- a/src/EterPack/Inline.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef __INC_ETERPACK_INLINE_H__ -#define __INC_ETERPACK_INLINE_H__ - -inline void inlinePathCreate(const char* path) -{ - char dir[64]; - const char* p, *k; - - p = path + 3; - - while (NULL != (k = strchr(p, '/'))) - { - memset(dir, 0, sizeof(dir)); - strncpy(dir, path, k - path); - CreateDirectory(dir, NULL); - p = k + 1; - } -} - -inline void inlineConvertPackFilename(char* name) -{ - char * p = name; - - while (*p) - { - if (*p == '\\') - *p = '/'; - else - *p = (int) tolower(*p); - p++; - } -} - -#endif diff --git a/src/EterPack/StdAfx.cpp b/src/EterPack/StdAfx.cpp deleted file mode 100644 index 804d088..0000000 --- a/src/EterPack/StdAfx.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#include "stdafx.h" - diff --git a/src/EterPack/StdAfx.h b/src/EterPack/StdAfx.h deleted file mode 100644 index 04faaad..0000000 --- a/src/EterPack/StdAfx.h +++ /dev/null @@ -1,7 +0,0 @@ -#pragma once - -//#include -#include -#include - -#include "EterBase/StdAfx.h" \ No newline at end of file diff --git a/src/EterPack/md5.c b/src/EterPack/md5.c deleted file mode 100644 index 2fe2f4e..0000000 --- a/src/EterPack/md5.c +++ /dev/null @@ -1,1033 +0,0 @@ -/* - - *********************************************************************** - - ** Copyright (C) 1990, RSA Data Security, Inc. All rights reserved. ** - - ** ** - - ** License to copy and use this software is granted provided that ** - - ** it is identified as the "RSA Data Security, Inc. MD5 Message- ** - - ** Digest Algorithm" in all material mentioning or referencing this ** - - ** software or this function. ** - - ** ** - - ** License is also granted to make and use derivative works ** - - ** provided that such works are identified as "derived from the RSA ** - - ** Data Security, Inc. MD5 Message-Digest Algorithm" in all ** - - ** material mentioning or referencing the derived work. ** - - ** ** - - ** RSA Data Security, Inc. makes no representations concerning ** - - ** either the merchantability of this software or the suitability ** - - ** of this software for any particular purpose. It is provided "as ** - - ** is" without express or implied warranty of any kind. ** - - ** ** - - ** These notices must be retained in any copies of any part of this ** - - ** documentation and/or software. ** - - *********************************************************************** - - */ - - -#include "md5.h" - - - -/* - - *********************************************************************** - - ** Message-digest routines: ** - - ** To form the message digest for a message M ** - - ** (1) Initialize a context buffer mdContext using MD5Init ** - - ** (2) Call MD5Update on mdContext and M ** - - ** (3) Call MD5Final on mdContext ** - - ** The message digest is now in mdContext->digest[0...15] ** - - *********************************************************************** - - */ - - - -static unsigned char PADDING[64] = { - - 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 - -}; - - - -#ifndef CPU386 /* Alternate defs exist for 386 assembler version */ - -/* F, G, H and I are basic MD5 functions */ - -#define F(x, y, z) (((x) & (y)) | ((~x) & (z))) - -#define G(x, y, z) (((x) & (z)) | ((y) & (~z))) - -#define H(x, y, z) ((x) ^ (y) ^ (z)) - -#define I(x, y, z) ((y) ^ ((x) | (~z))) - - - -/* ROTATE_LEFT rotates x left n bits */ - -#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n)))) - - - -/* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4 */ - -/* Rotation is separate from addition to prevent recomputation */ - -#define FF(a, b, c, d, x, s, ac) \ - (a) += F ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - -#define GG(a, b, c, d, x, s, ac) \ - (a) += G ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - -#define HH(a, b, c, d, x, s, ac) \ - (a) += H ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - -#define II(a, b, c, d, x, s, ac) \ - (a) += I ((b), (c), (d)) + (x) + (UINT4)(ac); \ - (a) = ROTATE_LEFT ((a), (s)); \ - (a) += (b); \ - -#endif /* CPU386 */ - - - -/* The routine MD5Init initializes the message-digest context - - mdContext. All fields are set to zero. - - */ - -void MD5Init (MD5_CTX *mdContext) -{ - - mdContext->i[0] = mdContext->i[1] = (UINT4)0; - - - - /* Load magic initialization constants. - - */ - - mdContext->buf[0] = (UINT4)0x67452301; - - mdContext->buf[1] = (UINT4)0xefcdab89; - - mdContext->buf[2] = (UINT4)0x98badcfe; - - mdContext->buf[3] = (UINT4)0x10325476; - -} - - - -/* The routine MD5Update updates the message-digest context to - - account for the presence of each of the characters inBuf[0..inLen-1] - - in the message whose digest is being computed. - - */ - -void MD5Update (MD5_CTX *mdContext, unsigned char *inBuf, unsigned int inLen) -{ - - UINT4 in[16]; - - int mdi; - - unsigned int i, ii; - - - - /* compute number of bytes mod 64 */ - - mdi = (int)((mdContext->i[0] >> 3) & 0x3F); - - - - /* update number of bits */ - - if ((mdContext->i[0] + ((UINT4)inLen << 3)) < mdContext->i[0]) - - mdContext->i[1]++; - - mdContext->i[0] += ((UINT4)inLen << 3); - - mdContext->i[1] += ((UINT4)inLen >> 29); - - - -#ifdef LITTLE_ENDIAN - - /* Speedup for little-endian machines suggested in MD5 report --P Karn */ - - if(mdi == 0 && ((int)inBuf & 3) == 0){ - - while(inLen >= 64){ - - MD5Transform(mdContext->buf,(UINT4 *)inBuf); - - inLen -= 64; - - inBuf += 64; - - } - - } - -#endif /* LITTLE_ENDIAN */ - - while (inLen--) { - - /* add new character to buffer, increment mdi */ - - mdContext->in[mdi++] = *inBuf++; - - - - /* transform if necessary */ - - if (mdi == 0x40) { - - for (i = 0, ii = 0; i < 16; i++, ii += 4) - - in[i] = (((UINT4)mdContext->in[ii+3]) << 24) | - - (((UINT4)mdContext->in[ii+2]) << 16) | - - (((UINT4)mdContext->in[ii+1]) << 8) | - - ((UINT4)mdContext->in[ii]); - - MD5Transform (mdContext->buf, in); - - mdi = 0; - - } - - } - -} - - - -/* The routine MD5Final terminates the message-digest computation and - - ends with the desired message digest in mdContext->digest[0...15]. - - */ - -void MD5Final (MD5_CTX *mdContext) -{ - - UINT4 in[16]; - - int mdi; - - unsigned int i, ii; - - unsigned int padLen; - - - - /* save number of bits */ - - in[14] = mdContext->i[0]; - - in[15] = mdContext->i[1]; - - - - /* compute number of bytes mod 64 */ - - mdi = (int)((mdContext->i[0] >> 3) & 0x3F); - - - - /* pad out to 56 mod 64 */ - - padLen = (mdi < 56) ? (56 - mdi) : (120 - mdi); - - MD5Update (mdContext, PADDING, padLen); - - - - /* append length in bits and transform */ - - for (i = 0, ii = 0; i < 14; i++, ii += 4) - - in[i] = (((UINT4)mdContext->in[ii+3]) << 24) | - - (((UINT4)mdContext->in[ii+2]) << 16) | - - (((UINT4)mdContext->in[ii+1]) << 8) | - - ((UINT4)mdContext->in[ii]); - - MD5Transform (mdContext->buf, in); - - - - /* store buffer in digest */ - - for (i = 0, ii = 0; i < 4; i++, ii += 4) { - - mdContext->digest[ii] = (unsigned char)(mdContext->buf[i] & 0xFF); - - mdContext->digest[ii+1] = - - (unsigned char)((mdContext->buf[i] >> 8) & 0xFF); - - mdContext->digest[ii+2] = - - (unsigned char)((mdContext->buf[i] >> 16) & 0xFF); - - mdContext->digest[ii+3] = - - (unsigned char)((mdContext->buf[i] >> 24) & 0xFF); - - } - -} - - - -#ifndef CPU386 /* Fast assembler version exists for 386/486 */ - - - -/* Basic MD5 step. Transforms buf based on in. - - */ - -void MD5Transform (buf, in) - -UINT4 *buf; - -UINT4 *in; - -{ - - UINT4 a = buf[0], b = buf[1], c = buf[2], d = buf[3]; - - - - /* Round 1 */ - -#define S11 7 - -#define S12 12 - -#define S13 17 - -#define S14 22 - - FF ( a, b, c, d, in[ 0], S11, 3614090360); /* 1 */ - - FF ( d, a, b, c, in[ 1], S12, 3905402710); /* 2 */ - - FF ( c, d, a, b, in[ 2], S13, 606105819); /* 3 */ - - FF ( b, c, d, a, in[ 3], S14, 3250441966); /* 4 */ - - FF ( a, b, c, d, in[ 4], S11, 4118548399); /* 5 */ - - FF ( d, a, b, c, in[ 5], S12, 1200080426); /* 6 */ - - FF ( c, d, a, b, in[ 6], S13, 2821735955); /* 7 */ - - FF ( b, c, d, a, in[ 7], S14, 4249261313); /* 8 */ - - FF ( a, b, c, d, in[ 8], S11, 1770035416); /* 9 */ - - FF ( d, a, b, c, in[ 9], S12, 2336552879); /* 10 */ - - FF ( c, d, a, b, in[10], S13, 4294925233); /* 11 */ - - FF ( b, c, d, a, in[11], S14, 2304563134); /* 12 */ - - FF ( a, b, c, d, in[12], S11, 1804603682); /* 13 */ - - FF ( d, a, b, c, in[13], S12, 4254626195); /* 14 */ - - FF ( c, d, a, b, in[14], S13, 2792965006); /* 15 */ - - FF ( b, c, d, a, in[15], S14, 1236535329); /* 16 */ - - - - /* Round 2 */ - -#define S21 5 - -#define S22 9 - -#define S23 14 - -#define S24 20 - - GG ( a, b, c, d, in[ 1], S21, 4129170786); /* 17 */ - - GG ( d, a, b, c, in[ 6], S22, 3225465664); /* 18 */ - - GG ( c, d, a, b, in[11], S23, 643717713); /* 19 */ - - GG ( b, c, d, a, in[ 0], S24, 3921069994); /* 20 */ - - GG ( a, b, c, d, in[ 5], S21, 3593408605); /* 21 */ - - GG ( d, a, b, c, in[10], S22, 38016083); /* 22 */ - - GG ( c, d, a, b, in[15], S23, 3634488961); /* 23 */ - - GG ( b, c, d, a, in[ 4], S24, 3889429448); /* 24 */ - - GG ( a, b, c, d, in[ 9], S21, 568446438); /* 25 */ - - GG ( d, a, b, c, in[14], S22, 3275163606); /* 26 */ - - GG ( c, d, a, b, in[ 3], S23, 4107603335); /* 27 */ - - GG ( b, c, d, a, in[ 8], S24, 1163531501); /* 28 */ - - GG ( a, b, c, d, in[13], S21, 2850285829); /* 29 */ - - GG ( d, a, b, c, in[ 2], S22, 4243563512); /* 30 */ - - GG ( c, d, a, b, in[ 7], S23, 1735328473); /* 31 */ - - GG ( b, c, d, a, in[12], S24, 2368359562); /* 32 */ - - - - /* Round 3 */ - -#define S31 4 - -#define S32 11 - -#define S33 16 - -#define S34 23 - - HH ( a, b, c, d, in[ 5], S31, 4294588738); /* 33 */ - - HH ( d, a, b, c, in[ 8], S32, 2272392833); /* 34 */ - - HH ( c, d, a, b, in[11], S33, 1839030562); /* 35 */ - - HH ( b, c, d, a, in[14], S34, 4259657740); /* 36 */ - - HH ( a, b, c, d, in[ 1], S31, 2763975236); /* 37 */ - - HH ( d, a, b, c, in[ 4], S32, 1272893353); /* 38 */ - - HH ( c, d, a, b, in[ 7], S33, 4139469664); /* 39 */ - - HH ( b, c, d, a, in[10], S34, 3200236656); /* 40 */ - - HH ( a, b, c, d, in[13], S31, 681279174); /* 41 */ - - HH ( d, a, b, c, in[ 0], S32, 3936430074); /* 42 */ - - HH ( c, d, a, b, in[ 3], S33, 3572445317); /* 43 */ - - HH ( b, c, d, a, in[ 6], S34, 76029189); /* 44 */ - - HH ( a, b, c, d, in[ 9], S31, 3654602809); /* 45 */ - - HH ( d, a, b, c, in[12], S32, 3873151461); /* 46 */ - - HH ( c, d, a, b, in[15], S33, 530742520); /* 47 */ - - HH ( b, c, d, a, in[ 2], S34, 3299628645); /* 48 */ - - - - /* Round 4 */ - -#define S41 6 - -#define S42 10 - -#define S43 15 - -#define S44 21 - - II ( a, b, c, d, in[ 0], S41, 4096336452); /* 49 */ - - II ( d, a, b, c, in[ 7], S42, 1126891415); /* 50 */ - - II ( c, d, a, b, in[14], S43, 2878612391); /* 51 */ - - II ( b, c, d, a, in[ 5], S44, 4237533241); /* 52 */ - - II ( a, b, c, d, in[12], S41, 1700485571); /* 53 */ - - II ( d, a, b, c, in[ 3], S42, 2399980690); /* 54 */ - - II ( c, d, a, b, in[10], S43, 4293915773); /* 55 */ - - II ( b, c, d, a, in[ 1], S44, 2240044497); /* 56 */ - - II ( a, b, c, d, in[ 8], S41, 1873313359); /* 57 */ - - II ( d, a, b, c, in[15], S42, 4264355552); /* 58 */ - - II ( c, d, a, b, in[ 6], S43, 2734768916); /* 59 */ - - II ( b, c, d, a, in[13], S44, 1309151649); /* 60 */ - - II ( a, b, c, d, in[ 4], S41, 4149444226); /* 61 */ - - II ( d, a, b, c, in[11], S42, 3174756917); /* 62 */ - - II ( c, d, a, b, in[ 2], S43, 718787259); /* 63 */ - - II ( b, c, d, a, in[ 9], S44, 3951481745); /* 64 */ - - - - buf[0] += a; - - buf[1] += b; - - buf[2] += c; - - buf[3] += d; - -} - -#else /* CPU386 */ - -/* Fast 386 Borland C inline assembler version of the MD5Transform() function - - * from the RSA Data Security, Inc, MD5 Message Digest Algorithm. - - * - - * This version uses native 32 bit registers, so it needs a 386 or 486 CPU. - - * - - * Because this function does *lots* of 32-bit operations, this version is - - * MUCH faster than the reference C version compiled with a garden- - - * variety 16-bit MS-DOS C compiler. - - * - - * Written and placed into the public domain on - - * 22 February 1992 by Phil Karn, KA9Q - - */ - -/* I really shouldn't have to do this explicitly... */ - -#ifdef __COMPACT__ - - asm .MODEL COMPACT - -#elif __HUGE__ - - asm .MODEL HUGE - -#elif __LARGE__ - - asm .MODEL LARGE - -#elif __MEDIUM__ - - asm .MODEL MEDIUM - -#elif __SMALL__ - - asm .MODEL SMALL - -#elif __TINY__ - - asm .MODEL TINY - -#endif - - - -/* Code sequence common to all four rounds. - - * evaluates a = b + (a + edi + x + t) <<< s - - * Assumes a,b are registers, s,t are immediate constants - - * The 'lea' instruction is just a fast way to compute "a = a+t+edi" - - */ - -#define COM(a,b,x,s,t)\ - - asm lea a,t[a+edi];\ - - asm add a,x;\ - - asm rol a,s;\ - - asm add a,b; - - - -/* Round 1 functions */ - -/* edi = F(x,y,z) = (x & y) | (~x & z) */ - -#define F(x,y,z)\ - - asm mov edi,x;\ - - asm and edi,y;\ - - asm mov esi,x;\ - - asm not esi;\ - - asm and esi,z;\ - - asm or edi,esi - - - -/* a = b + ((a + F(x,y,z) + x + t) <<< s); */ - -#define FF(a,b,c,d,x,s,t)\ - - F(b,c,d);\ - - COM(a,b,x,s,t) - - - -/* Round 2 functions */ - -/* edi = G(x,y,z) = F(z,x,y) = (x & z) | (y & ~z) */ - -#define G(x,y,z) F(z,x,y) - - - -/* a = b + ((a + G(b,c,d) + x + t) <<< s) */ - -#define GG(a,b,c,d,x,s,t)\ - - G(b,c,d);\ - - COM(a,b,x,s,t) - - - -/* Round 3 functions */ - -/* edi = H(x,y,z) = x ^ y ^ z */ - -#define H(x,y,z)\ - - asm mov edi,x;\ - - asm xor edi,y;\ - - asm xor edi,z - - - -/* a = b + ((a + H(b,c,d) + x + t) <<< s) */ - -#define HH(a,b,c,d,x,s,t)\ - - H(b,c,d);\ - - COM(a,b,x,s,t) - - - -/* Round 4 functions */ - -/* edi = I(x,y,z) = y ^ (x | ~z) */ - -#define I(x,y,z)\ - - asm mov edi,z;\ - - asm not edi;\ - - asm or edi,x;\ - - asm xor edi,y - - - -/* a = b + ((a + I(b,c,d) + x + t) <<< s) */ - -#define II(a,b,c,d,x,s,t)\ - - I(b,c,d);\ - - COM(a,b,x,s,t) - - - -#define A eax - -#define B ebx - -#define C ecx - -#define D edx - - - -void - -MD5Transform(buf,input) - -UINT4 *buf; - -UINT4 *input; - -{ - - asm .386; /* Allow use of 32-bit general registers */ - - - - /* Save caller's registers */ - - asm push si; - - asm push di; - - asm push es; - - asm if @DataSize NE 0 - - asm push ds; - - asm endif - - - - /* Get buf argument */ - - asm if @DataSize NE 0 - - asm lds si,buf; - - asm else - - asm mov si,buf; - - asm endif - - asm mov A,dword ptr si[0*4]; /* A = buf[0] */ - - asm mov B,dword ptr si[1*4]; /* B = buf[1] */ - - asm mov C,dword ptr si[2*4]; /* C = buf[2] */ - - asm mov D,dword ptr si[3*4]; /* D = buf[3] */ - - - - /* Warning: This makes our other args inaccessible until bp - - * is restored! - - */ - - asm push bp; - - asm les bp,input - - - -/* Round 1. The *4 factors in the subscripts to bp account for the - - * byte offsets of each long element in the input array. - - */ - -#define S11 7 - -#define S12 12 - -#define S13 17 - -#define S14 22 - - FF(A,B,C,D,es:bp[ 0*4],S11,3614090360); /* 1 */ - - FF(D,A,B,C,es:bp[ 1*4],S12,3905402710); /* 2 */ - - FF(C,D,A,B,es:bp[ 2*4],S13, 606105819); /* 3 */ - - FF(B,C,D,A,es:bp[ 3*4],S14,3250441966); /* 4 */ - - FF(A,B,C,D,es:bp[ 4*4],S11,4118548399); /* 5 */ - - FF(D,A,B,C,es:bp[ 5*4],S12,1200080426); /* 6 */ - - FF(C,D,A,B,es:bp[ 6*4],S13,2821735955); /* 7 */ - - FF(B,C,D,A,es:bp[ 7*4],S14,4249261313); /* 8 */ - - FF(A,B,C,D,es:bp[ 8*4],S11,1770035416); /* 9 */ - - FF(D,A,B,C,es:bp[ 9*4],S12,2336552879); /* 10 */ - - FF(C,D,A,B,es:bp[10*4],S13,4294925233); /* 11 */ - - FF(B,C,D,A,es:bp[11*4],S14,2304563134); /* 12 */ - - FF(A,B,C,D,es:bp[12*4],S11,1804603682); /* 13 */ - - FF(D,A,B,C,es:bp[13*4],S12,4254626195); /* 14 */ - - FF(C,D,A,B,es:bp[14*4],S13,2792965006); /* 15 */ - - FF(B,C,D,A,es:bp[15*4],S14,1236535329); /* 16 */ - - - -/* Round 2 */ - -#define S21 5 - -#define S22 9 - -#define S23 14 - -#define S24 20 - - GG(A,B,C,D,es:bp[ 1*4],S21,4129170786); /* 17 */ - - GG(D,A,B,C,es:bp[ 6*4],S22,3225465664); /* 18 */ - - GG(C,D,A,B,es:bp[11*4],S23, 643717713); /* 19 */ - - GG(B,C,D,A,es:bp[ 0*4],S24,3921069994); /* 20 */ - - GG(A,B,C,D,es:bp[ 5*4],S21,3593408605); /* 21 */ - - GG(D,A,B,C,es:bp[10*4],S22, 38016083); /* 22 */ - - GG(C,D,A,B,es:bp[15*4],S23,3634488961); /* 23 */ - - GG(B,C,D,A,es:bp[ 4*4],S24,3889429448); /* 24 */ - - GG(A,B,C,D,es:bp[ 9*4],S21, 568446438); /* 25 */ - - GG(D,A,B,C,es:bp[14*4],S22,3275163606); /* 26 */ - - GG(C,D,A,B,es:bp[ 3*4],S23,4107603335); /* 27 */ - - GG(B,C,D,A,es:bp[ 8*4],S24,1163531501); /* 28 */ - - GG(A,B,C,D,es:bp[13*4],S21,2850285829); /* 29 */ - - GG(D,A,B,C,es:bp[ 2*4],S22,4243563512); /* 30 */ - - GG(C,D,A,B,es:bp[ 7*4],S23,1735328473); /* 31 */ - - GG(B,C,D,A,es:bp[12*4],S24,2368359562); /* 32 */ - - - -/* Round 3 */ - -#define S31 4 - -#define S32 11 - -#define S33 16 - -#define S34 23 - - HH(A,B,C,D,es:bp[ 5*4],S31,4294588738); /* 33 */ - - HH(D,A,B,C,es:bp[ 8*4],S32,2272392833); /* 34 */ - - HH(C,D,A,B,es:bp[11*4],S33,1839030562); /* 35 */ - - HH(B,C,D,A,es:bp[14*4],S34,4259657740); /* 36 */ - - HH(A,B,C,D,es:bp[ 1*4],S31,2763975236); /* 37 */ - - HH(D,A,B,C,es:bp[ 4*4],S32,1272893353); /* 38 */ - - HH(C,D,A,B,es:bp[ 7*4],S33,4139469664); /* 39 */ - - HH(B,C,D,A,es:bp[10*4],S34,3200236656); /* 40 */ - - HH(A,B,C,D,es:bp[13*4],S31, 681279174); /* 41 */ - - HH(D,A,B,C,es:bp[ 0*4],S32,3936430074); /* 42 */ - - HH(C,D,A,B,es:bp[ 3*4],S33,3572445317); /* 43 */ - - HH(B,C,D,A,es:bp[ 6*4],S34, 76029189); /* 44 */ - - HH(A,B,C,D,es:bp[ 9*4],S31,3654602809); /* 45 */ - - HH(D,A,B,C,es:bp[12*4],S32,3873151461); /* 46 */ - - HH(C,D,A,B,es:bp[15*4],S33, 530742520); /* 47 */ - - HH(B,C,D,A,es:bp[ 2*4],S34,3299628645); /* 48 */ - - - -/* Round 4 */ - -#define S41 6 - -#define S42 10 - -#define S43 15 - -#define S44 21 - - II(A,B,C,D,es:bp[ 0*4],S41,4096336452); /* 49 */ - - II(D,A,B,C,es:bp[ 7*4],S42,1126891415); /* 50 */ - - II(C,D,A,B,es:bp[14*4],S43,2878612391); /* 51 */ - - II(B,C,D,A,es:bp[ 5*4],S44,4237533241); /* 52 */ - - II(A,B,C,D,es:bp[12*4],S41,1700485571); /* 53 */ - - II(D,A,B,C,es:bp[ 3*4],S42,2399980690); /* 54 */ - - II(C,D,A,B,es:bp[10*4],S43,4293915773); /* 55 */ - - II(B,C,D,A,es:bp[ 1*4],S44,2240044497); /* 56 */ - - II(A,B,C,D,es:bp[ 8*4],S41,1873313359); /* 57 */ - - II(D,A,B,C,es:bp[15*4],S42,4264355552); /* 58 */ - - II(C,D,A,B,es:bp[ 6*4],S43,2734768916); /* 59 */ - - II(B,C,D,A,es:bp[13*4],S44,1309151649); /* 60 */ - - II(A,B,C,D,es:bp[ 4*4],S41,4149444226); /* 61 */ - - II(D,A,B,C,es:bp[11*4],S42,3174756917); /* 62 */ - - II(C,D,A,B,es:bp[ 2*4],S43, 718787259); /* 63 */ - - II(B,C,D,A,es:bp[ 9*4],S44,3951481745); /* 64 */ - - - - asm pop bp; /* We can address our args again */ - - asm if @DataSize NE 0 - - asm lds si,buf - - asm else - - asm mov si,buf; - - asm endif - - asm add dword ptr si[0*4],A; /* buf[0] += A */ - - asm add dword ptr si[1*4],B; /* buf[1] += B */ - - asm add dword ptr si[2*4],C; /* buf[2] += C */ - - asm add dword ptr si[3*4],D; /* buf[3] += D */ - - - - /* Restore caller's registers */ - - asm if @DataSize NE 0 - - asm pop ds - - asm endif - - - - asm pop es; - - asm pop di; - - asm pop si; - -} - -#endif /* CPU386 */ - - - -/* - - *********************************************************************** - - ** End of md5.c ** - - ******************************** (cut) ******************************** - - */ \ No newline at end of file diff --git a/src/EterPack/md5.h b/src/EterPack/md5.h deleted file mode 100644 index 9ed936f..0000000 --- a/src/EterPack/md5.h +++ /dev/null @@ -1,128 +0,0 @@ -#ifndef __INC_ETERPACKLIB_MD5_H__ -#define __INC_ETERPACKLIB_MD5_H__ - -/* - - *********************************************************************** - - ** md5.h -- header file for implementation of MD5 ** - - ** RSA Data Security, Inc. MD5 Message-Digest Algorithm ** - - ** Created: 2/17/90 RLR ** - - ** Revised: 12/27/90 SRD,AJ,BSK,JT Reference C version ** - - ** Revised (for MD5): RLR 4/27/91 ** - - ** -- G modified to have y&~z instead of y&z ** - - ** -- FF, GG, HH modified to add in last register done ** - - ** -- Access pattern: round 2 works mod 5, round 3 works mod 3 ** - - ** -- distinct additive constant for each step ** - - ** -- round 4 added, working mod 7 ** - - *********************************************************************** - - */ - -/* - - *********************************************************************** - - ** Copyright (C) 1990, RSA Data Security, Inc. All rights reserved. ** - - ** ** - - ** License to copy and use this software is granted provided that ** - - ** it is identified as the "RSA Data Security, Inc. MD5 Message- ** - - ** Digest Algorithm" in all material mentioning or referencing this ** - - ** software or this function. ** - - ** ** - - ** License is also granted to make and use derivative works ** - - ** provided that such works are identified as "derived from the RSA ** - - ** Data Security, Inc. MD5 Message-Digest Algorithm" in all ** - - ** material mentioning or referencing the derived work. ** - - ** ** - - ** RSA Data Security, Inc. makes no representations concerning ** - - ** either the merchantability of this software or the suitability ** - - ** of this software for any particular purpose. It is provided "as ** - - ** is" without express or implied warranty of any kind. ** - - ** ** - - ** These notices must be retained in any copies of any part of this ** - - ** documentation and/or software. ** - - *********************************************************************** - - */ - - - -/* typedef a 32-bit type */ - -typedef unsigned long int UINT4; - - - -/* Data structure for MD5 (Message-Digest) computation */ - -typedef struct { - - UINT4 i[2]; /* number of _bits_ handled mod 2^64 */ - - UINT4 buf[4]; /* scratch buffer */ - - unsigned char in[64]; /* input buffer */ - - unsigned char digest[16]; /* actual digest after MD5Final call */ - -} MD5_CTX; - - -#if defined (__cplusplus) -extern "C" { -#endif - -void MD5Init (MD5_CTX *); - -void MD5Update (MD5_CTX *,unsigned char *,unsigned int); - -void MD5Final (MD5_CTX *); - -void MD5Transform(UINT4 *,UINT4 *); - -#if defined (__cplusplus) -}; -#endif - -/* - - *********************************************************************** - - ** End of md5.h ** - - ******************************** (cut) ******************************** - - */ - - -#endif \ No newline at end of file diff --git a/src/EterPythonLib/CMakeLists.txt b/src/EterPythonLib/CMakeLists.txt index 55944ea..9494a91 100644 --- a/src/EterPythonLib/CMakeLists.txt +++ b/src/EterPythonLib/CMakeLists.txt @@ -3,8 +3,8 @@ add_library(EterPythonLib STATIC ${FILE_SOURCES}) target_link_libraries(EterPythonLib - lzo2 cryptopp-static + mio ) GroupSourcesByFolder(EterPythonLib) diff --git a/src/GameLib/AreaTerrain.cpp b/src/GameLib/AreaTerrain.cpp index cfd7cf6..d5138ab 100644 --- a/src/GameLib/AreaTerrain.cpp +++ b/src/GameLib/AreaTerrain.cpp @@ -3,7 +3,7 @@ #include "EterLib/ResourceManager.h" #include "EterLib/StateManager.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "AreaTerrain.h" #include "MapOutdoor.h" @@ -106,10 +106,9 @@ bool CTerrain::LoadShadowMap(const char * c_pszFileName) DWORD dwStart = ELTimer_GetMSec(); Tracef("LoadShadowMap %s ", c_pszFileName); - CMappedFile file; - LPCVOID c_pvData; + TPackFile file; - if (!CEterPackManager::Instance().Get(file, c_pszFileName, &c_pvData)) + if (!CPackManager::Instance().GetFile(c_pszFileName, file)) { TraceError(" CTerrain::LoadShadowMap - %s OPEN ERROR", c_pszFileName); return false; @@ -117,13 +116,13 @@ bool CTerrain::LoadShadowMap(const char * c_pszFileName) DWORD dwShadowMapSize = sizeof(WORD) * 256 * 256; - if (file.Size() != dwShadowMapSize) + if (file.size() != dwShadowMapSize) { TraceError(" CTerrain::LoadShadowMap - %s SIZE ERROR", c_pszFileName); return false; } - memcpy(m_awShadowMap, c_pvData, dwShadowMapSize); + memcpy(m_awShadowMap, file.data(), dwShadowMapSize); Tracef("%d ms\n", ELTimer_GetMSec() - dwStart); return true; diff --git a/src/GameLib/CMakeLists.txt b/src/GameLib/CMakeLists.txt index b87f2ee..e32514d 100644 --- a/src/GameLib/CMakeLists.txt +++ b/src/GameLib/CMakeLists.txt @@ -4,6 +4,8 @@ add_library(GameLib STATIC ${FILE_SOURCES}) target_link_libraries(GameLib lzo2 + cryptopp-static + mio ) GroupSourcesByFolder(GameLib) diff --git a/src/GameLib/ItemManager.cpp b/src/GameLib/ItemManager.cpp index 828e49d..43f3d35 100644 --- a/src/GameLib/ItemManager.cpp +++ b/src/GameLib/ItemManager.cpp @@ -1,6 +1,7 @@ #include "StdAfx.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "EterLib/ResourceManager.h" +#include "EterBase/lzo.h" #include "ItemManager.h" @@ -95,14 +96,13 @@ CItemData * CItemManager::MakeItemData(DWORD dwIndex) bool CItemManager::LoadItemList(const char * c_szFileName) { - CMappedFile File; - LPCVOID pData; + TPackFile File; - if (!CEterPackManager::Instance().Get(File, c_szFileName, &pData)) + if (!CPackManager::Instance().GetFile(c_szFileName, File)) return false; CMemoryTextFileLoader textFileLoader; - textFileLoader.Bind(File.Size(), pData); + textFileLoader.Bind(File.size(), File.data()); CTokenVector TokenVector; for (DWORD i = 0; i < textFileLoader.GetLineCount(); ++i) @@ -196,16 +196,15 @@ const std::string& __SnapString(const std::string& c_rstSrc, std::string& rstTem bool CItemManager::LoadItemDesc(const char* c_szFileName) { - const VOID* pvData; - CMappedFile kFile; - if (!CEterPackManager::Instance().Get(kFile, c_szFileName, &pvData)) + TPackFile kFile; + if (!CPackManager::Instance().GetFile(c_szFileName, kFile)) { Tracenf("CItemManager::LoadItemDesc(c_szFileName=%s) - Load Error", c_szFileName); return false; } CMemoryTextFileLoader kTextFileLoader; - kTextFileLoader.Bind(kFile.Size(), pvData); + kTextFileLoader.Bind(kFile.size(), kFile.data()); std::string stTemp; @@ -254,22 +253,27 @@ DWORD GetHashCode( const char* pString ) bool CItemManager::LoadItemTable(const char* c_szFileName) { - CMappedFile file; + TPackFile file; LPCVOID pvData; - if (!CEterPackManager::Instance().Get(file, c_szFileName, &pvData)) + if (!CPackManager::Instance().GetFile(c_szFileName, file)) return false; DWORD dwFourCC, dwElements, dwDataSize; DWORD dwVersion=0; DWORD dwStride=0; - file.Read(&dwFourCC, sizeof(DWORD)); + uint8_t* p = file.data(); + memcpy(&dwFourCC, p, sizeof(DWORD)); + p += sizeof(DWORD); if (dwFourCC == MAKEFOURCC('M', 'I', 'P', 'X')) { - file.Read(&dwVersion, sizeof(DWORD)); - file.Read(&dwStride, sizeof(DWORD)); + memcpy(&dwVersion, p, sizeof(DWORD)); + p += sizeof(DWORD); + + memcpy(&dwStride, p, sizeof(DWORD)); + p += sizeof(DWORD); if (dwVersion != 1) { @@ -289,11 +293,14 @@ bool CItemManager::LoadItemTable(const char* c_szFileName) return false; } - file.Read(&dwElements, sizeof(DWORD)); - file.Read(&dwDataSize, sizeof(DWORD)); + memcpy(&dwElements, p, sizeof(DWORD)); + p += sizeof(DWORD); + + memcpy(&dwDataSize, p, sizeof(DWORD)); + p += sizeof(DWORD); BYTE * pbData = new BYTE[dwDataSize]; - file.Read(pbData, dwDataSize); + memcpy(pbData, p, dwDataSize); ///// diff --git a/src/GameLib/MapManager.cpp b/src/GameLib/MapManager.cpp index e42014e..4a4bcd8 100644 --- a/src/GameLib/MapManager.cpp +++ b/src/GameLib/MapManager.cpp @@ -1,6 +1,6 @@ #include "StdAfx.h" #include "EterLib/StateManager.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "MapManager.h" #include "MapOutdoor.h" @@ -597,14 +597,13 @@ void CMapManager::GetBaseXY(DWORD * pdwBaseX, DWORD * pdwBaseY) void CMapManager::__LoadMapInfoVector() { - CMappedFile kFile; - LPCVOID pData; - if (!CEterPackManager::Instance().Get(kFile, m_stAtlasInfoFileName.c_str(), &pData)) - if (!CEterPackManager::Instance().Get(kFile, "AtlasInfo.txt", &pData)) + TPackFile kFile; + if (!CPackManager::Instance().GetFile(m_stAtlasInfoFileName, kFile)) + if (!CPackManager::Instance().GetFile("AtlasInfo.txt", kFile)) return; CMemoryTextFileLoader textFileLoader; - textFileLoader.Bind(kFile.Size(), pData); + textFileLoader.Bind(kFile.size(), kFile.data()); char szMapName[256]; int x, y; diff --git a/src/GameLib/MapOutdoorLoad.cpp b/src/GameLib/MapOutdoorLoad.cpp index a50a000..7faa19f 100644 --- a/src/GameLib/MapOutdoorLoad.cpp +++ b/src/GameLib/MapOutdoorLoad.cpp @@ -3,7 +3,7 @@ #include "AreaTerrain.h" #include "AreaLoaderThread.h" #include "EterLib/ResourceManager.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" //CAreaLoaderThread CMapOutdoor::ms_AreaLoaderThread; @@ -11,7 +11,6 @@ bool CMapOutdoor::Load(float x, float y, float z) { Destroy(); - CEterPackManager& rkPackMgr=CEterPackManager::Instance(); { static std::string s_strOldPathName=""; @@ -42,7 +41,7 @@ bool CMapOutdoor::Load(float x, float y, float z) // LOCAL_ENVIRONMENT_DATA std::string local_envDataName = GetMapDataDirectory() + "\\" + m_settings_envDataName; - if (rkPackMgr.isExist(local_envDataName.c_str())) + if (CPackManager::instance().IsExist(local_envDataName.c_str())) { m_envDataName = local_envDataName; } @@ -440,10 +439,9 @@ bool CMapOutdoor::LoadMonsterAreaInfo() char c_szFileName[256]; sprintf(c_szFileName, "%s\\regen.txt", GetMapDataDirectory().c_str()); - LPCVOID pModelData; - CMappedFile File; + TPackFile File; - if (!CEterPackManager::Instance().Get(File, c_szFileName, &pModelData)) + if (!CPackManager::Instance().GetFile(c_szFileName, File)) { //TraceError(" CMapOutdoorAccessor::LoadMonsterAreaInfo Load File %s ERROR", c_szFileName); return false; @@ -452,7 +450,7 @@ bool CMapOutdoor::LoadMonsterAreaInfo() CMemoryTextFileLoader textFileLoader; CTokenVector stTokenVector; - textFileLoader.Bind(File.Size(), pModelData); + textFileLoader.Bind(File.size(), File.data()); for (DWORD i = 0; i < textFileLoader.GetLineCount(); ++i) { diff --git a/src/GameLib/Property.cpp b/src/GameLib/Property.cpp index 7cfd92f..83229fc 100644 --- a/src/GameLib/Property.cpp +++ b/src/GameLib/Property.cpp @@ -136,54 +136,6 @@ void GetTimeString(char * str, time_t ct) tm.tm_sec); } -bool CProperty::Save(const char * c_pszFileName) -{ - CTempFile file; - - DWORD fourcc = MAKEFOURCC('Y', 'P', 'R', 'T'); - file.Write(&fourcc, sizeof(DWORD)); - file.Write("\r\n", 2); - - if (0 == m_stCRC.length()) - { - char szCRC[MAX_PATH + 16 + 1]; - - GetTimeString(szCRC, time(0)); - strcpy(szCRC + strlen(szCRC), c_pszFileName); - - m_dwCRC = CPropertyManager::Instance().GetUniqueCRC(szCRC); - _snprintf(szCRC, sizeof(szCRC), "%u", m_dwCRC); - - m_stCRC.assign(szCRC); - } - - file.Write(m_stCRC.c_str(), m_stCRC.length()); - file.Write("\r\n", 2); - - CTokenVectorMap::iterator itor = m_stTokenMap.begin(); - char buf[4096 + 1]; - - while (itor != m_stTokenMap.end()) - { - CTokenVector & tokenVector = itor->second; - - int len = _snprintf(buf, sizeof(buf), "%s\t", itor->first.c_str()); - file.Write(buf, len); - - for (DWORD i = 0; i < tokenVector.size(); ++i) - { - len = _snprintf(buf, sizeof(buf), "\t\"%s\"", tokenVector[i].c_str()); - file.Write(buf, len); - } - - file.Write("\r\n", 2); - ++itor; - } - - file.Close(); - return CPropertyManager::Instance().Put(c_pszFileName, file.GetFileName()); -} - bool CProperty::ReadFromMemory(const void * c_pvData, int iLen, const char * c_pszFileName) { const char * pcData = (const char *) c_pvData; diff --git a/src/GameLib/Property.h b/src/GameLib/Property.h index bf20d38..5268f66 100644 --- a/src/GameLib/Property.h +++ b/src/GameLib/Property.h @@ -19,8 +19,6 @@ class CProperty void PutVector(const char * c_pszKey, const CTokenVector & c_rTokenVector); void PutString(const char * c_pszKey, const char * c_pszString); - bool Save(const char * c_pszFileName); - DWORD GetSize(); DWORD GetCRC(); diff --git a/src/GameLib/PropertyManager.cpp b/src/GameLib/PropertyManager.cpp index 975532a..e809030 100644 --- a/src/GameLib/PropertyManager.cpp +++ b/src/GameLib/PropertyManager.cpp @@ -1,5 +1,5 @@ #include "StdAfx.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "PropertyManager.h" #include "Property.h" @@ -17,37 +17,22 @@ bool CPropertyManager::Initialize(const char * c_pszPackFileName) { if (c_pszPackFileName) { - if (!m_pack.Create(m_fileDict, c_pszPackFileName, "", true)) - { + m_pack = std::make_shared(); + if (!m_pack->Open(c_pszPackFileName, m_fileDict)) { LogBoxf("Cannot open property pack file (filename %s)", c_pszPackFileName); return false; } m_isFileMode = false; - TDataPositionMap & indexMap = m_pack.GetIndexMap(); - - TDataPositionMap::iterator itor = indexMap.begin(); - - typedef std::map TDataPositionMap; - - int i = 0; - - while (indexMap.end() != itor) - { - TEterPackIndex * pIndex = itor->second; - ++itor; - - if (!stricmp("property/reserve", pIndex->filename)) - { - LoadReservedCRC(pIndex->filename); - continue; + for (auto it = m_fileDict.begin(); it != m_fileDict.end(); ++it) { + std::string stFileName = it->second.second.file_name; + if (!stricmp("property/reserve", stFileName.c_str())) { + LoadReservedCRC(stFileName.c_str()); + } + else { + Register(stFileName.c_str()); } - - if (!Register(pIndex->filename)) - continue; - - ++i; } } else @@ -60,43 +45,15 @@ bool CPropertyManager::Initialize(const char * c_pszPackFileName) return true; } -bool CPropertyManager::BuildPack() -{ - if (!m_pack.Create(m_fileDict, "property", "")) - return false; - - WIN32_FIND_DATA fdata; - HANDLE hFind = FindFirstFile("property\\*", &fdata); - - if (hFind == INVALID_HANDLE_VALUE) - return false; - - do - { - if (fdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) - continue; - - char szSourceFileName[256 + 1]; - _snprintf(szSourceFileName, sizeof(szSourceFileName), "property\\%s", fdata.cFileName); - - m_pack.Put(fdata.cFileName, szSourceFileName,COMPRESSED_TYPE_NONE,""); - } - while (FindNextFile(hFind, &fdata)); - - FindClose(hFind); - return true; -} - bool CPropertyManager::LoadReservedCRC(const char * c_pszFileName) { - CMappedFile file; - LPCVOID c_pvData; + TPackFile file; - if (!CEterPackManager::Instance().Get(file, c_pszFileName, &c_pvData)) + if (!CPackManager::Instance().GetFile(c_pszFileName, file)) return false; CMemoryTextFileLoader textFileLoader; - textFileLoader.Bind(file.Size(), c_pvData); + textFileLoader.Bind(file.size(), file.data()); for (DWORD i = 0; i < textFileLoader.GetLineCount(); ++i) { @@ -136,15 +93,14 @@ DWORD CPropertyManager::GetUniqueCRC(const char * c_szSeed) bool CPropertyManager::Register(const char * c_pszFileName, CProperty ** ppProperty) { - CMappedFile file; - LPCVOID c_pvData; + TPackFile file; - if (!CEterPackManager::Instance().Get(file, c_pszFileName, &c_pvData)) + if (!CPackManager::Instance().GetFile(c_pszFileName, file)) return false; CProperty * pProperty = new CProperty(c_pszFileName); - if (!pProperty->ReadFromMemory(c_pvData, file.Size(), c_pszFileName)) + if (!pProperty->ReadFromMemory(file.data(), file.size(), c_pszFileName)) { delete pProperty; return false; @@ -188,67 +144,6 @@ bool CPropertyManager::Get(DWORD dwCRC, CProperty ** ppProperty) return true; } -bool CPropertyManager::Put(const char * c_pszFileName, const char * c_pszSourceFileName) -{ - if (!CopyFile(c_pszSourceFileName, c_pszFileName, FALSE)) - return false; - - if (!m_isFileMode) // 팩 파일에도 넣음 - { - if (!m_pack.Put(c_pszFileName, NULL, COMPRESSED_TYPE_NONE,"")) - { - assert(!"CPropertyManager::Put cannot write to pack file"); - return false; - } - } - - Register(c_pszFileName); - return true; -} - -bool CPropertyManager::Erase(DWORD dwCRC) -{ - TPropertyCRCMap::iterator itor = m_PropertyByCRCMap.find(dwCRC); - - if (m_PropertyByCRCMap.end() == itor) - return false; - - CProperty * pProperty = itor->second; - m_PropertyByCRCMap.erase(itor); - - DeleteFile(pProperty->GetFileName()); - ReserveCRC(pProperty->GetCRC()); - - if (!m_isFileMode) // 파일 모드가 아니면 팩에서도 지움 - m_pack.Delete(pProperty->GetFileName()); - - FILE * fp = fopen("property/reserve", "a+"); - - if (!fp) - LogBox("예약 CRC 파일을 열 수 없습니다."); - else - { - char szCRC[64 + 1]; - _snprintf(szCRC, sizeof(szCRC), "%u\r\n", pProperty->GetCRC()); - - fputs(szCRC, fp); - fclose(fp); - } - - delete pProperty; - return true; -} - -bool CPropertyManager::Erase(const char * c_pszFileName) -{ - CProperty * pProperty = NULL; - - if (Get(c_pszFileName, &pProperty)) - return Erase(pProperty->GetCRC()); - - return false; -} - void CPropertyManager::Clear() { stl_wipe_second(m_PropertyByCRCMap); diff --git a/src/GameLib/PropertyManager.h b/src/GameLib/PropertyManager.h index d3e3ac7..e9b4fed 100644 --- a/src/GameLib/PropertyManager.h +++ b/src/GameLib/PropertyManager.h @@ -1,6 +1,6 @@ #pragma once -#include "EterPack/EterPack.h" +#include "PackLib/PackManager.h" class CPropertyManager : public CSingleton { @@ -10,9 +10,6 @@ class CPropertyManager : public CSingleton void Clear(); - void SetPack(CEterPack * pPack); - bool BuildPack(); - bool LoadReservedCRC(const char * c_pszFileName); void ReserveCRC(DWORD dwCRC); DWORD GetUniqueCRC(const char * c_szSeed); @@ -23,14 +20,6 @@ class CPropertyManager : public CSingleton bool Get(DWORD dwCRC, CProperty ** ppProperty); bool Get(const char * c_pszFileName, CProperty ** ppProperty); -// bool Add(const char * c_pszFileName); -// bool Remove(DWORD dwCRC); - - bool Put(const char * c_pszFileName, const char * c_pszSourceFileName); - - bool Erase(DWORD dwCRC); - bool Erase(const char * c_pszFileName); - protected: typedef std::map TPropertyCRCMap; typedef std::set TCRCSet; @@ -38,6 +27,6 @@ class CPropertyManager : public CSingleton bool m_isFileMode; TPropertyCRCMap m_PropertyByCRCMap; TCRCSet m_ReservedCRCSet; - CEterPack m_pack; - CEterFileDict m_fileDict; + std::shared_ptr m_pack; + TPackFileMap m_fileDict; }; diff --git a/src/GameLib/RaceManager.cpp b/src/GameLib/RaceManager.cpp index bcc1438..11d43ee 100644 --- a/src/GameLib/RaceManager.cpp +++ b/src/GameLib/RaceManager.cpp @@ -1,7 +1,7 @@ #include "StdAfx.h" #include "RaceManager.h" #include "RaceMotionData.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" bool __IsGuildRace(unsigned race) { @@ -237,14 +237,13 @@ bool CRaceManager::__LoadRaceMotionList(CRaceData& rkRaceData, const char* pathN s_kMap_stType_dwIndex.insert(std::map::value_type("SKILL5", CRaceMotionData::NAME_SKILL+125)); } - const void* pvData; - CMappedFile kMappedFile; - if (!CEterPackManager::Instance().Get(kMappedFile, motionListFileName, &pvData)) + TPackFile kMappedFile; + if (!CPackManager::Instance().GetFile(motionListFileName, kMappedFile)) return false; CMemoryTextFileLoader kTextFileLoader; - kTextFileLoader.Bind(kMappedFile.Size(), pvData); + kTextFileLoader.Bind(kMappedFile.size(), kMappedFile.data()); rkRaceData.RegisterMotionMode(CRaceMotionData::MODE_GENERAL); diff --git a/src/PRTerrainLib/CMakeLists.txt b/src/PRTerrainLib/CMakeLists.txt index c75a740..5ec03de 100644 --- a/src/PRTerrainLib/CMakeLists.txt +++ b/src/PRTerrainLib/CMakeLists.txt @@ -3,8 +3,8 @@ add_library(PRTerrainLib STATIC ${FILE_SOURCES}) target_link_libraries(PRTerrainLib - lzo2 cryptopp-static + mio ) GroupSourcesByFolder(PRTerrainLib) diff --git a/src/PRTerrainLib/Terrain.cpp b/src/PRTerrainLib/Terrain.cpp index 8f7c3f4..bbddcc2 100644 --- a/src/PRTerrainLib/Terrain.cpp +++ b/src/PRTerrainLib/Terrain.cpp @@ -1,5 +1,5 @@ #include "Stdafx.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "terrain.h" #include @@ -42,7 +42,6 @@ void CTerrainImpl::Initialize() memset(m_lWaterHeight, -1, sizeof(m_lWaterHeight)); m_byNumWater = 0; - memset(&m_HeightMapHeader, 0, sizeof(TGA_HEADER)); memset(&m_awShadowMap, 0xFFFF, sizeof(m_awShadowMap)); memset(&m_lpAlphaTexture, NULL, sizeof(m_lpAlphaTexture)); @@ -76,17 +75,16 @@ bool CTerrainImpl::LoadHeightMap(const char*c_szFileName) { Tracef("LoadRawHeightMapFile %s ", c_szFileName); - CMappedFile kMappedFile; - LPCVOID lpcvFileData; + TPackFile kMappedFile; - if (!CEterPackManager::Instance().Get(kMappedFile, c_szFileName, &lpcvFileData)) + if (!CPackManager::Instance().GetFile( c_szFileName, kMappedFile)) { Tracen("Error"); TraceError("CTerrainImpl::LoadHeightMap - %s OPEN ERROR", c_szFileName); return false; } - memcpy(m_awRawHeightMap, lpcvFileData, sizeof(WORD)*HEIGHTMAP_RAW_XSIZE*HEIGHTMAP_RAW_YSIZE); + memcpy(m_awRawHeightMap, kMappedFile.data(), sizeof(WORD) * HEIGHTMAP_RAW_XSIZE * HEIGHTMAP_RAW_YSIZE); return true; } @@ -96,17 +94,16 @@ bool CTerrainImpl::LoadAttrMap(const char *c_szFileName) DWORD dwStart = ELTimer_GetMSec(); Tracef("LoadAttrMapFile %s ", c_szFileName); - CMappedFile kMappedFile; - LPCVOID lpcvFileData; + TPackFile kMappedFile; - if (!CEterPackManager::Instance().Get(kMappedFile, c_szFileName, &lpcvFileData)) + if (!CPackManager::Instance().GetFile(c_szFileName, kMappedFile)) { TraceError("CTerrainImpl::LoadAttrMap - %s OPEN ERROR", c_szFileName); return false; } - DWORD dwFileSize = kMappedFile.Size(); - BYTE * abFileData = (BYTE *) lpcvFileData; + DWORD dwFileSize = kMappedFile.size(); + BYTE * abFileData = kMappedFile.data(); // LoadAttrMap { @@ -168,17 +165,16 @@ bool CTerrainImpl::RAW_LoadTileMap(const char * c_szFileName) { Tracef("LoadSplatFile %s ", c_szFileName); - CMappedFile kMappedFile; - LPCVOID lpcvFileData; + TPackFile kMappedFile; - if (!CEterPackManager::Instance().Get(kMappedFile, c_szFileName, &lpcvFileData)) + if (!CPackManager::Instance().GetFile(c_szFileName, kMappedFile)) { Tracen("Error"); TraceError("CTerrainImpl::RAW_LoadTileMap - %s OPEN ERROR", c_szFileName); return false; } - memcpy(m_abyTileMap, lpcvFileData, sizeof(BYTE)*(TILEMAP_RAW_XSIZE)*(TILEMAP_RAW_YSIZE)); + memcpy(m_abyTileMap, kMappedFile.data(), sizeof(BYTE) * (TILEMAP_RAW_XSIZE) * (TILEMAP_RAW_YSIZE)); return true; @@ -206,18 +202,17 @@ bool CTerrainImpl::LoadWaterMap(const char * c_szFileName) bool CTerrainImpl::LoadWaterMapFile(const char * c_szFileName) { - CMappedFile kMappedFile; - LPCVOID lpcvFileData; + TPackFile kMappedFile; - if (!CEterPackManager::Instance().Get(kMappedFile, c_szFileName, &lpcvFileData)) + if (!CPackManager::Instance().GetFile(c_szFileName, kMappedFile)) { Tracen("Error"); TraceError("CTerrainImpl::LoadWaterMap - %s OPEN ERROR", c_szFileName); return false; } - DWORD dwFileSize = kMappedFile.Size(); - BYTE* abFileData = (BYTE*)lpcvFileData; + DWORD dwFileSize = kMappedFile.size(); + BYTE* abFileData = kMappedFile.data(); { #pragma pack(push) diff --git a/src/PRTerrainLib/Terrain.h b/src/PRTerrainLib/Terrain.h index 79a1d62..0c0bd63 100644 --- a/src/PRTerrainLib/Terrain.h +++ b/src/PRTerrainLib/Terrain.h @@ -9,8 +9,6 @@ #pragma once #endif // _MSC_VER > 1000 -#include "EterImageLib/TGAImage.h" - #include "TextureSet.h" #include "TerrainType.h" @@ -116,8 +114,6 @@ class CTerrainImpl BYTE m_abyWaterMap[WATERMAP_YSIZE*WATERMAP_XSIZE]; CHAR m_acNormalMap[NORMALMAP_YSIZE*NORMALMAP_XSIZE*3]; - TGA_HEADER m_HeightMapHeader; - WORD m_wTileMapVersion; long m_lViewRadius; diff --git a/src/PackLib/CMakeLists.txt b/src/PackLib/CMakeLists.txt new file mode 100644 index 0000000..b6403fb --- /dev/null +++ b/src/PackLib/CMakeLists.txt @@ -0,0 +1,11 @@ +file(GLOB_RECURSE FILE_SOURCES "*.h" "*.c" "*.cpp") + +add_library(PackLib STATIC ${FILE_SOURCES}) + +target_link_libraries(PackLib + libzstd_static + cryptopp-static + mio +) + +GroupSourcesByFolder(PackLib) diff --git a/src/PackLib/Pack.cpp b/src/PackLib/Pack.cpp new file mode 100644 index 0000000..c87d1d5 --- /dev/null +++ b/src/PackLib/Pack.cpp @@ -0,0 +1,71 @@ +#include "Pack.h" +#include + +bool CPack::Open(const std::string& path, TPackFileMap& entries) +{ + std::error_code ec; + m_file.map(path, ec); + + if (ec) { + return false; + } + + size_t file_size = m_file.size(); + if (file_size < sizeof(TPackFileHeader)) { + return false; + } + + memcpy(&m_header, m_file.data(), sizeof(TPackFileHeader)); + m_decryption.SetKeyWithIV(PACK_KEY.data(), PACK_KEY.size(), m_header.iv, CryptoPP::Camellia::BLOCKSIZE); + + if (file_size < sizeof(TPackFileHeader) + m_header.entry_num * sizeof(TPackFileEntry)) { + return false; + } + + for (size_t i = 0; i < m_header.entry_num; i++) { + TPackFileEntry entry; + memcpy(&entry, m_file.data() + sizeof(TPackFileHeader) + i * sizeof(TPackFileEntry), sizeof(TPackFileEntry)); + m_decryption.ProcessData((CryptoPP::byte*)&entry, (CryptoPP::byte*)&entry, sizeof(TPackFileEntry)); + + entries[entry.file_name] = std::make_pair(shared_from_this(), entry); + + if (file_size < m_header.data_begin + entry.offset + entry.compressed_size) { + return false; + } + } + + return true; +} + +bool CPack::GetFile(const TPackFileEntry& entry, TPackFile& result) +{ + result.resize(entry.file_size); + + size_t offset = m_header.data_begin + entry.offset; + switch (entry.encryption) + { + case 0: { + size_t decompressed_size = ZSTD_decompress(result.data(), result.size(), m_file.data() + offset, entry.compressed_size); + if (decompressed_size != entry.file_size) { + return false; + } + } break; + + case 1: { + std::vector compressed_data(entry.compressed_size); + memcpy(compressed_data.data(), m_file.data() + offset, entry.compressed_size); + + m_decryption.Resynchronize(entry.iv, sizeof(entry.iv)); + m_decryption.ProcessData(compressed_data.data(), compressed_data.data(), entry.compressed_size); + + size_t decompressed_size = ZSTD_decompress(result.data(), result.size(), compressed_data.data(), compressed_data.size()); + if (decompressed_size != entry.file_size) { + return false; + } + } break; + + default: return false; + } + + return true; +} diff --git a/src/PackLib/Pack.h b/src/PackLib/Pack.h new file mode 100644 index 0000000..740807d --- /dev/null +++ b/src/PackLib/Pack.h @@ -0,0 +1,21 @@ +#pragma once +#include +#include + +#include "config.h" + +class CPack : public std::enable_shared_from_this +{ +public: + CPack() = default; + ~CPack() = default; + + bool Open(const std::string& path, TPackFileMap& entries); + bool GetFile(const TPackFileEntry& entry, TPackFile& result); + +private: + TPackFileHeader m_header; + mio::mmap_source m_file; + + CryptoPP::CTR_Mode::Decryption m_decryption; +}; \ No newline at end of file diff --git a/src/PackLib/PackManager.cpp b/src/PackLib/PackManager.cpp new file mode 100644 index 0000000..0fb7e38 --- /dev/null +++ b/src/PackLib/PackManager.cpp @@ -0,0 +1,40 @@ +#include "PackManager.h" + +bool CPackManager::AddPack(const std::string& path) +{ + std::shared_ptr pack = std::make_shared(); + return pack->Open(path, m_entries); +} + +bool CPackManager::GetFile(std::string_view path, TPackFile& result) +{ + thread_local std::string buf; + NormalizePath(path, buf); + + auto it = m_entries.find(buf); + if (it != m_entries.end()) { + return it->second.first->GetFile(it->second.second, result); + } + + return false; +} + +bool CPackManager::IsExist(std::string_view path) const +{ + thread_local std::string buf; + NormalizePath(path, buf); + + auto it = m_entries.find(buf); + return it != m_entries.end(); +} + +void CPackManager::NormalizePath(std::string_view in, std::string& out) const +{ + out.resize(in.size()); + for (std::size_t i = 0; i < out.size(); ++i) { + if (in[i] == '\\') + out[i] = '/'; + else + out[i] = static_cast(std::tolower(in[i])); + } +} diff --git a/src/PackLib/PackManager.h b/src/PackLib/PackManager.h new file mode 100644 index 0000000..07b290b --- /dev/null +++ b/src/PackLib/PackManager.h @@ -0,0 +1,22 @@ +#pragma once +#include + +#include "EterBase/Singleton.h" +#include "Pack.h" + +class CPackManager : public CSingleton +{ +public: + CPackManager() = default; + virtual ~CPackManager() = default; + + bool AddPack(const std::string& path); + bool GetFile(std::string_view path, TPackFile& result); + bool IsExist(std::string_view path) const; + +private: + void NormalizePath(std::string_view in, std::string& out) const; + +private: + TPackFileMap m_entries; +}; diff --git a/src/PackLib/config.h b/src/PackLib/config.h new file mode 100644 index 0000000..26a6e24 --- /dev/null +++ b/src/PackLib/config.h @@ -0,0 +1,43 @@ +#pragma once +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +constexpr std::array PACK_KEY = { + 0x00,0x11,0x22,0x33, 0x44,0x55,0x66,0x77, + 0x88,0x99,0xAA,0xBB, 0xCC,0xDD,0xEE,0xFF, + 0x01,0x23,0x45,0x67, 0x89,0xAB,0xCD,0xEF, + 0xFE,0xDC,0xBA,0x98, 0x76,0x54,0x32,0x10 +}; + +#pragma pack(push, 1) +struct TPackFileHeader +{ + uint64_t entry_num; + uint64_t data_begin; + uint8_t iv[CryptoPP::Camellia::BLOCKSIZE]; +}; +struct TPackFileEntry +{ + char file_name[FILENAME_MAX+1]; + uint64_t offset; + uint64_t file_size; + uint64_t compressed_size; + uint8_t encryption; + uint8_t iv[CryptoPP::Camellia::BLOCKSIZE]; +}; +#pragma pack(pop) + +class CPack; +using TPackFile = std::vector; +using TPackFileMapEntry = std::pair, TPackFileEntry>; +using TPackFileMap = std::unordered_map; \ No newline at end of file diff --git a/src/PackMaker/main.cpp b/src/PackMaker/main.cpp index 4fd53f6..e8adf18 100644 --- a/src/PackMaker/main.cpp +++ b/src/PackMaker/main.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -7,36 +6,7 @@ #include #include -#include -#include -#include -#include -#include - -constexpr std::array PACK_KEY = { - 0x00,0x11,0x22,0x33, 0x44,0x55,0x66,0x77, - 0x88,0x99,0xAA,0xBB, 0xCC,0xDD,0xEE,0xFF, - 0x01,0x23,0x45,0x67, 0x89,0xAB,0xCD,0xEF, - 0xFE,0xDC,0xBA,0x98, 0x76,0x54,0x32,0x10 -}; - -#pragma pack(push, 1) -struct TPackFileHeader -{ - uint64_t entry_num; - uint64_t data_begin; - uint8_t iv[CryptoPP::Camellia::BLOCKSIZE]; -}; -struct TPackFileEntry -{ - char file_name[FILENAME_MAX]; - uint64_t offset; - uint64_t file_size; - uint64_t compressed_size; - uint8_t encryption; - uint8_t iv[CryptoPP::Camellia::BLOCKSIZE]; -}; -#pragma pack(pop) +#include "PackLib/config.h" int main(int argc, char* argv[]) { @@ -110,7 +80,7 @@ int main(int argc, char* argv[]) CryptoPP::CTR_Mode::Encryption encryption; encryption.SetKeyWithIV(PACK_KEY.data(), PACK_KEY.size(), header.iv, CryptoPP::Camellia::BLOCKSIZE); - uint64_t offset = header.data_begin; + uint64_t offset = 0; for (auto& [path, entry] : entries) { std::ifstream ifs(input / path, std::ios::binary); if (!ifs.is_open()) { @@ -151,18 +121,14 @@ int main(int argc, char* argv[]) offset += entry.compressed_size; } - std::vector entry_buffer(sizeof(TPackFileEntry) * entries.size()); - char* ptr = entry_buffer.data(); + ofs.seekp(sizeof(TPackFileHeader), std::ios::beg); + encryption.Resynchronize(header.iv, sizeof(header.iv)); + for (auto& [path, entry] : entries) { - memcpy(ptr, &entry, sizeof(entry)); - ptr += sizeof(entry); + TPackFileEntry tmp = entry; + encryption.ProcessData((uint8_t*)&tmp, (uint8_t*)&tmp, sizeof(TPackFileEntry)); + ofs.write((const char*)&tmp, sizeof(TPackFileEntry)); } - encryption.Resynchronize(header.iv, sizeof(header.iv)); - encryption.ProcessData((uint8_t*)entry_buffer.data(), (uint8_t*)entry_buffer.data(), entry_buffer.size()); - - ofs.seekp(sizeof(TPackFileHeader), std::ios::beg); - ofs.write(entry_buffer.data(), entry_buffer.size()); - return EXIT_SUCCESS; } diff --git a/src/ScriptLib/CMakeLists.txt b/src/ScriptLib/CMakeLists.txt index a0657f4..1bd895a 100644 --- a/src/ScriptLib/CMakeLists.txt +++ b/src/ScriptLib/CMakeLists.txt @@ -3,8 +3,8 @@ add_library(ScriptLib STATIC ${FILE_SOURCES}) target_link_libraries(ScriptLib - lzo2 cryptopp-static + mio ) GroupSourcesByFolder(ScriptLib) diff --git a/src/ScriptLib/PythonLauncher.cpp b/src/ScriptLib/PythonLauncher.cpp index e316cb9..e88e237 100644 --- a/src/ScriptLib/PythonLauncher.cpp +++ b/src/ScriptLib/PythonLauncher.cpp @@ -4,7 +4,7 @@ #undef BYTE #endif -#include "eterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "PythonLauncher.h" @@ -232,29 +232,13 @@ bool CPythonLauncher::RunMemoryTextFile(const char* c_szFileName, UINT uFileSize bool CPythonLauncher::RunFile(const char* c_szFileName) { - char* acBufData=NULL; - DWORD dwBufSize=0; - - { - CMappedFile file; - const VOID* pvData; - CEterPackManager::Instance().Get(file, c_szFileName, &pvData); + TPackFile file; + CPackManager::Instance().GetFile(c_szFileName, file); - dwBufSize=file.Size(); - if (dwBufSize==0) - return false; + if (file.empty()) + return false; - acBufData=new char[dwBufSize]; - memcpy(acBufData, pvData, dwBufSize); - } - - bool ret=false; - - ret=RunMemoryTextFile(c_szFileName, dwBufSize, acBufData); - - delete [] acBufData; - - return ret; + return RunMemoryTextFile(c_szFileName, file.size(), file.data()); } bool CPythonLauncher::RunLine(const char* c_szSrc) diff --git a/src/SpeedTreeLib/CMakeLists.txt b/src/SpeedTreeLib/CMakeLists.txt index 6284815..d2cfc06 100644 --- a/src/SpeedTreeLib/CMakeLists.txt +++ b/src/SpeedTreeLib/CMakeLists.txt @@ -3,8 +3,8 @@ add_library(SpeedTreeLib STATIC ${FILE_SOURCES}) target_link_libraries(SpeedTreeLib - lzo2 cryptopp-static + mio ) GroupSourcesByFolder(SpeedTreeLib) diff --git a/src/SpeedTreeLib/SpeedTreeForest.cpp b/src/SpeedTreeLib/SpeedTreeForest.cpp index 49fdc4a..7b74fe4 100644 --- a/src/SpeedTreeLib/SpeedTreeForest.cpp +++ b/src/SpeedTreeLib/SpeedTreeForest.cpp @@ -7,9 +7,7 @@ #include #include "EterBase/Filename.h" - -#include "EterBase/MappedFile.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "SpeedTreeForest.h" #include "SpeedTreeConfig.h" @@ -84,23 +82,20 @@ BOOL CSpeedTreeForest::GetMainTree(DWORD dwCRC, SpeedTreeWrapperPtr &ppMainTree, pTree = itor->second; else { - CMappedFile file; - LPCVOID c_pvData; + TPackFile file; - if (!CEterPackManager::Instance().Get(file, c_pszFileName, &c_pvData)) + if (!CPackManager::Instance().GetFile(c_pszFileName, file)) return FALSE; pTree = std::make_shared(); - if (!pTree->LoadTree(c_pszFileName, (const BYTE *) c_pvData, file.Size())) + if (!pTree->LoadTree(c_pszFileName, (const BYTE *)file.data(), file.size())) { pTree.reset(); return FALSE; } m_pMainTreeMap.insert(std::map::value_type(dwCRC, pTree)); - - file.Destroy(); } ppMainTree = pTree; diff --git a/src/UserInterface/AccountConnector.cpp b/src/UserInterface/AccountConnector.cpp index 2949bef..cd99474 100644 --- a/src/UserInterface/AccountConnector.cpp +++ b/src/UserInterface/AccountConnector.cpp @@ -3,7 +3,7 @@ #include "Packet.h" #include "PythonNetworkStream.h" #include "EterBase/tea.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" // CHINA_CRYPT_KEY extern DWORD g_adwEncryptKey[4]; @@ -288,7 +288,6 @@ bool CAccountConnector::__AuthState_RecvPanamaPack() if (!Recv(sizeof(TPacketGCPanamaPack), &kPacket)) return false; - CEterPackManager::instance().RegisterPack(kPacket.szPackName, "*", kPacket.abIV); return true; } @@ -304,7 +303,6 @@ bool CAccountConnector::__AuthState_RecvHybridCryptKeys(int iTotalSize) if (!Recv(kPacket.iKeyStreamLen, kPacket.m_pStream)) return false; - CEterPackManager::Instance().RetrieveHybridCryptPackKeys( kPacket.m_pStream ); return true; } @@ -320,7 +318,6 @@ bool CAccountConnector::__AuthState_RecvHybridCryptSDB(int iTotalSize) if (!Recv(kPacket.iSDBStreamLen, kPacket.m_pStream)) return false; - CEterPackManager::Instance().RetrieveHybridCryptPackSDB( kPacket.m_pStream ); return true; } @@ -363,7 +360,6 @@ bool CAccountConnector::__AuthState_RecvAuthSuccess() else { DWORD dwPanamaKey = kAuthSuccessPacket.dwLoginKey ^ g_adwEncryptKey[0] ^ g_adwEncryptKey[1] ^ g_adwEncryptKey[2] ^ g_adwEncryptKey[3]; - CEterPackManager::instance().DecryptPackIV(dwPanamaKey); CPythonNetworkStream & rkNet = CPythonNetworkStream::Instance(); rkNet.SetLoginKey(kAuthSuccessPacket.dwLoginKey); diff --git a/src/UserInterface/CMakeLists.txt b/src/UserInterface/CMakeLists.txt index 91a1986..a09d0e5 100644 --- a/src/UserInterface/CMakeLists.txt +++ b/src/UserInterface/CMakeLists.txt @@ -17,17 +17,18 @@ target_link_libraries(UserInterface EterImageLib EterLib EterLocale - EterPack EterPythonLib GameLib PRTerrainLib ScriptLib SpeedTreeLib SphereLib + PackLib cryptopp-static lzo2 libzstd_static + mio DirectX Granny diff --git a/src/UserInterface/Locale.cpp b/src/UserInterface/Locale.cpp index 16c24fc..c64c8a0 100644 --- a/src/UserInterface/Locale.cpp +++ b/src/UserInterface/Locale.cpp @@ -3,7 +3,6 @@ #include "PythonApplication.h" #include "resource.h" #include "EterBase/CRC32.h" -#include "EterPack/EterPackManager.h" #include "EterLocale/Japanese.h" #include diff --git a/src/UserInterface/MarkImage.cpp b/src/UserInterface/MarkImage.cpp index 5be6482..7c38b55 100644 --- a/src/UserInterface/MarkImage.cpp +++ b/src/UserInterface/MarkImage.cpp @@ -1,5 +1,6 @@ #include "stdafx.h" #include "MarkImage.h" +#include "EterBase/lzo.h" #include #include diff --git a/src/UserInterface/PythonApplicationModule.cpp b/src/UserInterface/PythonApplicationModule.cpp index 55db4ba..96056dd 100644 --- a/src/UserInterface/PythonApplicationModule.cpp +++ b/src/UserInterface/PythonApplicationModule.cpp @@ -2,6 +2,7 @@ #include "Resource.h" #include "PythonApplication.h" #include "EterLib/Camera.h" +#include "PackLib/PackManager.h" #include @@ -339,15 +340,13 @@ PyObject* appGetImageInfo(PyObject* poSelf, PyObject* poArgs) } #endif -#include "EterPack/EterPackManager.h" - PyObject* appIsExistFile(PyObject* poSelf, PyObject* poArgs) { char* szFileName; if (!PyTuple_GetString(poArgs, 0, &szFileName)) return Py_BuildException(); - bool isExist=CEterPackManager::Instance().isExist(szFileName); + bool isExist=CPackManager::Instance().IsExist(szFileName); return Py_BuildValue("i", isExist); } @@ -1021,12 +1020,11 @@ class CTextLineLoader public: CTextLineLoader(const char * c_szFileName) { - const VOID* pvData; - CMappedFile kFile; - if (!CEterPackManager::Instance().Get(kFile, c_szFileName, &pvData)) + TPackFile kFile; + if (!CPackManager::Instance().GetFile(c_szFileName, kFile)) return; - m_kTextFileLoader.Bind(kFile.Size(), pvData); + m_kTextFileLoader.Bind(kFile.size(), kFile.data()); } DWORD GetLineCount() diff --git a/src/UserInterface/PythonBackground.cpp b/src/UserInterface/PythonBackground.cpp index ae6bd13..4ed6f3d 100644 --- a/src/UserInterface/PythonBackground.cpp +++ b/src/UserInterface/PythonBackground.cpp @@ -5,7 +5,7 @@ #include "stdafx.h" #include "EterLib/CullingManager.h" #include "EterLib/Camera.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "GameLib/MapOutDoor.h" #include "GameLib/PropertyLoader.h" @@ -241,19 +241,7 @@ void CPythonBackground::Initialize() void CPythonBackground::__CreateProperty() { - if (CEterPackManager::SEARCH_FILE_FIRST == CEterPackManager::Instance().GetSearchMode() && - _access("property", 0) == 0) - { - m_PropertyManager.Initialize(NULL); - - CPropertyLoader PropertyLoader; - PropertyLoader.SetPropertyManager(&m_PropertyManager); - PropertyLoader.Create("*.*", "Property"); - } - else - { - m_PropertyManager.Initialize("pack/property"); - } + m_PropertyManager.Initialize("pack/property.pck"); } ////////////////////////////////////////////////////////////////////// diff --git a/src/UserInterface/PythonEventManager.cpp b/src/UserInterface/PythonEventManager.cpp index 7839a1b..d55f8dd 100644 --- a/src/UserInterface/PythonEventManager.cpp +++ b/src/UserInterface/PythonEventManager.cpp @@ -2,7 +2,7 @@ #include "PythonEventManager.h" #include "PythonNetworkStream.h" #include "PythonNonPlayer.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "PythonMiniMap.h" #include "AbstractApplication.h" @@ -104,16 +104,14 @@ void CPythonEventManager::__InitEventSet(TEventSet& rEventSet) int CPythonEventManager::RegisterEventSet(const char * c_szFileName) { - CMappedFile File; - LPCVOID pMap; + TPackFile File; - if (!CEterPackManager::Instance().Get(File, c_szFileName, &pMap)) + if (!CPackManager::Instance().GetFile(c_szFileName, File)) return -1; std::string strEventString; - strEventString.resize(File.Size()+1); - - File.Read(&strEventString[0], File.Size()); + strEventString.resize(File.size() + 1); + memcpy(strEventString.data(), File.data(), File.size()); TEventSet * pEventSet = m_EventSetPool.Alloc(); if (!pEventSet) diff --git a/src/UserInterface/PythonMiniMap.cpp b/src/UserInterface/PythonMiniMap.cpp index 4feed7f..2f123a7 100644 --- a/src/UserInterface/PythonMiniMap.cpp +++ b/src/UserInterface/PythonMiniMap.cpp @@ -2,7 +2,7 @@ #include "EterLib/StateManager.h" #include "EterLib/GrpSubImage.h" #include "EterLib/Camera.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" #include "PythonMiniMap.h" #include "PythonBackground.h" @@ -878,7 +878,7 @@ bool CPythonMiniMap::LoadAtlas() char atlasFileName[1024+1]; snprintf(atlasFileName, sizeof(atlasFileName), "%s/atlas.sub", rkMap.GetName().c_str()); - if (!CEterPackManager::Instance().isExist(atlasFileName)) + if (!CPackManager::Instance().IsExist(atlasFileName)) { snprintf(atlasFileName, sizeof(atlasFileName), "d:/ymir work/ui/atlas/%s/atlas.sub", rkMap.GetName().c_str()); } diff --git a/src/UserInterface/PythonNetworkStreamPhaseGame.cpp b/src/UserInterface/PythonNetworkStreamPhaseGame.cpp index f808d87..f7a1c56 100644 --- a/src/UserInterface/PythonNetworkStreamPhaseGame.cpp +++ b/src/UserInterface/PythonNetworkStreamPhaseGame.cpp @@ -17,7 +17,6 @@ #include "PythonMessenger.h" #include "PythonApplication.h" -#include "EterPack/EterPackManager.h" #include "GameLib/ItemManager.h" #include "AbstractApplication.h" diff --git a/src/UserInterface/PythonNetworkStreamPhaseHandShake.cpp b/src/UserInterface/PythonNetworkStreamPhaseHandShake.cpp index aafede3..d445e46 100644 --- a/src/UserInterface/PythonNetworkStreamPhaseHandShake.cpp +++ b/src/UserInterface/PythonNetworkStreamPhaseHandShake.cpp @@ -2,7 +2,7 @@ #include "PythonNetworkStream.h" #include "PythonApplication.h" #include "Packet.h" -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" // HandShake --------------------------------------------------------------------------- void CPythonNetworkStream::HandShakePhase() @@ -173,7 +173,6 @@ bool CPythonNetworkStream::RecvHybridCryptKeyPacket() if (!Recv(kPacket.iKeyStreamLen, kPacket.m_pStream)) return false; - CEterPackManager::Instance().RetrieveHybridCryptPackKeys( kPacket.m_pStream ); return true; } @@ -193,7 +192,6 @@ bool CPythonNetworkStream::RecvHybridCryptSDBPacket() if (!Recv(kPacket.iSDBStreamLen, kPacket.m_pStream)) return false; - CEterPackManager::Instance().RetrieveHybridCryptPackSDB( kPacket.m_pStream ); return true; } diff --git a/src/UserInterface/PythonNetworkStreamPhaseLoading.cpp b/src/UserInterface/PythonNetworkStreamPhaseLoading.cpp index 19348a2..b30dd35 100644 --- a/src/UserInterface/PythonNetworkStreamPhaseLoading.cpp +++ b/src/UserInterface/PythonNetworkStreamPhaseLoading.cpp @@ -3,10 +3,8 @@ #include "Packet.h" #include "PythonApplication.h" #include "NetworkActorManager.h" - #include "AbstractPlayer.h" - -#include "EterPack/EterPackManager.h" +#include "PackLib/PackManager.h" void CPythonNetworkStream::EnableChatInsultFilter(bool isEnable) { @@ -33,13 +31,12 @@ bool CPythonNetworkStream::IsInsultIn(const char* c_szMsg) bool CPythonNetworkStream::LoadInsultList(const char* c_szInsultListFileName) { - CMappedFile file; - const VOID* pvData; - if (!CEterPackManager::Instance().Get(file, c_szInsultListFileName, &pvData)) + TPackFile file; + if (!CPackManager::Instance().GetFile(c_szInsultListFileName, file)) return false; CMemoryTextFileLoader kMemTextFileLoader; - kMemTextFileLoader.Bind(file.Size(), pvData); + kMemTextFileLoader.Bind(file.size(), file.data()); m_kInsultChecker.Clear(); for (DWORD dwLineIndex=0; dwLineIndex=4) return false; - CMappedFile file; - const VOID* pvData; - if (!CEterPackManager::Instance().Get(file, c_szFileName, &pvData)) + TPackFile file; + if (!CPackManager::Instance().GetFile(c_szFileName, file)) return false; DWORD dwEngCount=26; @@ -65,10 +61,10 @@ bool CPythonNetworkStream::LoadConvertTable(DWORD dwEmpireID, const char* c_szFi DWORD dwHanSize=dwHanCount*2; DWORD dwFileSize=dwEngCount*2+dwHanSize; - if (file.Size() #endif -#include "eterPack/EterPackManager.h" #include "eterLib/Util.h" #include "eterBase/CPostIt.h" +#include "EterBase/lzo.h" + +#include "PackLib/PackManager.h" +#include extern "C" { extern int _fltused; @@ -136,73 +139,124 @@ bool PackInitialize(const char * c_pszFolder) if (_access(c_pszFolder, 0) != 0) return false; - std::string stFolder(c_pszFolder); - stFolder += "/"; + std::vector packFiles = { + "patch1", + "season3_eu", + "patch2", + "metin2_patch_snow", + "metin2_patch_snow_dungeon", + "metin2_patch_etc_costume1", + "metin2_patch_pet1", + "metin2_patch_pet2", + "metin2_patch_ramadan_costume", + "metin2_patch_flame", + "metin2_patch_flame_dungeon", + "metin2_patch_w21_etc", + "metin2_patch_w21_mobs", + "metin2_patch_w21_mobs_m", + "metin2_patch_dss_box", + "metin2_patch_costume_soccer", + "metin2_patch_easter1", + "metin2_patch_mineral", + "metin2_patch_w20_sound", + "metin2_patch_ds", + "metin2_patch_5th_armor", + "metin2_patch_w20_etc", + "metin2_patch_dragon_rock", + "metin2_patch_dragon_rock_mobs", + "metin2_patch_etc", + "metin2_patch_xmas", + "metin2_patch_eu3", + "metin2_patch_eu4", + "metin2_patch_mundi", + "metin2_patch_sd", + "metin2_patch_halloween", + "metin2_patch_party", + "metin2_patch_dance", + "pc", + "pc2", + "monster", + "monster2", + "effect", + "zone", + "terrain", + "npc", + "npc2", + "tree", + "guild", + "item", + "textureset", + "property", + "icon", + "season1", + "season2", + "outdoora1", + "outdoora2", + "outdoora3", + "outdoorb1", + "outdoorb3", + "outdoorc1", + "outdoorc3", + "outdoorsnow1", + "outdoordesert1", + "outdoorflame1", + "outdoorfielddungeon1", + "outdoort1", + "outdoort2", + "outdoort3", + "outdoort4", + "outdoorwedding", + "outdoormilgyo1", + "indoorspiderdungeon1", + "indoordeviltower1", + "indoormonkeydungeon1", + "indoormonkeydungeon2", + "indoormonkeydungeon3", + "outdoortrent", + "outdoortrent02", + "outdoorguild1", + "outdoorguild2", + "outdoorguild3", + "outdoorduel", + "outdoorgmguildbuild", + "sound", + "sound_m", + "sound2", + "bgm", + "locale_ca", + "locale_ae", + "locale_de", + "locale_es", + "locale_fr", + "locale_gr", + "locale_it", + "locale_nl", + "locale_pl", + "locale_pt", + "locale_tr", + "locale_uk", + "locale_bg", + "locale_en", + "locale_mx", + "locale_ro", + "locale_ru", + "locale_dk", + "locale_cz", + "locale_hu", + "locale_us", + "locale_pa", + "uiscript", + "ETC", + "uiloading", + }; - std::string stFileName(stFolder); - stFileName += "Index"; + std::filesystem::path folderPath = c_pszFolder; - CMappedFile file; - LPCVOID pvData; - - if (!file.Create(stFileName.c_str(), &pvData, 0, 0)) - { - LogBoxf("FATAL ERROR! File not exist: %s", stFileName.c_str()); - TraceError("FATAL ERROR! File not exist: %s", stFileName.c_str()); - return true; + CPackManager::instance().AddPack((folderPath / "root.pck").generic_string()); + for (const std::string& packFileName : packFiles) { + CPackManager::instance().AddPack((folderPath / (packFileName + ".pck")).generic_string()); } - CMemoryTextFileLoader TextLoader; - TextLoader.Bind(file.Size(), pvData); - - bool bPackFirst = TRUE; - - const std::string& strPackType = TextLoader.GetLineString(0); - - if (strPackType.compare("FILE") && strPackType.compare("PACK")) - { - TraceError("Pack/Index has invalid syntax. First line must be 'PACK' or 'FILE'"); - return false; - } - -#ifdef _DISTRIBUTE - Tracef("알림: 팩 모드입니다.\n"); - - //if (0 == strPackType.compare("FILE")) - //{ - // bPackFirst = FALSE; - // Tracef("알림: 파일 모드입니다.\n"); - //} - //else - //{ - // Tracef("알림: 팩 모드입니다.\n"); - //} -#else - bPackFirst = FALSE; - Tracef("알림: 파일 모드입니다.\n"); -#endif - - CTextFileLoader::SetCacheMode(); -#if defined(USE_RELATIVE_PATH) - CEterPackManager::Instance().SetRelativePathMode(); -#endif - CEterPackManager::Instance().SetCacheMode(); - CEterPackManager::Instance().SetSearchMode(bPackFirst); - - std::string strPackName, strTexCachePackName; - for (DWORD i = 1; i < TextLoader.GetLineCount() - 1; i += 2) - { - const std::string & c_rstFolder = TextLoader.GetLineString(i); - const std::string & c_rstName = TextLoader.GetLineString(i + 1); - - strPackName = stFolder + c_rstName; - strTexCachePackName = strPackName + "_texcache"; - - CEterPackManager::Instance().RegisterPack(strPackName.c_str(), c_rstFolder.c_str()); - CEterPackManager::Instance().RegisterPack(strTexCachePackName.c_str(), c_rstFolder.c_str()); - } - - CEterPackManager::Instance().RegisterRootPack((stFolder + std::string("root")).c_str()); NANOEND return true; } @@ -362,7 +416,7 @@ bool Main(HINSTANCE hInstance, LPSTR lpCmdLine) #endif static CLZO lzo; - static CEterPackManager EterPackManager; + CPackManager packMgr; if (!PackInitialize("pack")) { diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index 6a93749..4348b3e 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -1,5 +1,6 @@ add_subdirectory(cryptopp) add_subdirectory(lzo-2.10) +add_subdirectory(mio) ## zstd is a bit tricky set(ZSTD_BUILD_SHARED OFF CACHE BOOL "BUILD SHARED LIBRARIES" FORCE) diff --git a/vendor/mio/.gitignore b/vendor/mio/.gitignore new file mode 100644 index 0000000..f37d11b --- /dev/null +++ b/vendor/mio/.gitignore @@ -0,0 +1,5 @@ +test/** +!test/test.cpp +!test/example.cpp +!test/CMakeLists.txt +build/ diff --git a/vendor/mio/.travis.yml b/vendor/mio/.travis.yml new file mode 100644 index 0000000..5923fe2 --- /dev/null +++ b/vendor/mio/.travis.yml @@ -0,0 +1,50 @@ +language: cpp + +addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-trusty-5.0 + packages: + - g++-6 + - clang-5.0 + +matrix: + include: + - os: linux + env: CXX_COMPILER=g++-6 BUILD_TYPE=Debug + - os: linux + env: CXX_COMPILER=g++-6 BUILD_TYPE=Release + - os: linux + env: CXX_COMPILER=clang++-5.0 BUILD_TYPE=Debug + - os: linux + env: CXX_COMPILER=clang++-5.0 BUILD_TYPE=Release + - os: osx + osx_image: xcode8.3 + env: CXX_COMPILER=clang++ BUILD_TYPE=Debug + - os: osx + osx_image: xcode8.3 + env: CXX_COMPILER=clang++ BUILD_TYPE=Release + - os: linux + env: CXX_COMPILER=g++-6 BUILD_TYPE=Debug COVERAGE_FLAGS="--coverage" + +script: + - mkdir build + - cd build + - > + cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + -DCMAKE_CXX_FLAGS="$COVERAGE_FLAGS" \ + -DCMAKE_CXX_COMPILER=${CXX_COMPILER} \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCTEST_DROP_SITE_CDASH=TRUE \ + -DDROP_METHOD="http" \ + -DDROP_SITE="my.cdash.org" \ + -DDROP_LOCATION="/submit.php?project=mio" \ + -DBUILDNAME="$(uname -s)-${CXX_COMPILER}-${BUILD_TYPE}" .. + - ctest -VV -M Experimental -T Start \ + && ctest -VV -M Experimental -T Build \ + && ctest -VV -M Experimental -T Test + - if [[ -n "${COVERAGE_FLAGS+x}" ]]; then ctest -VV -M Experimental -T Coverage; fi + - ctest -VV -M Experimental -T Submit + + diff --git a/vendor/mio/CMakeLists.txt b/vendor/mio/CMakeLists.txt new file mode 100644 index 0000000..54f8768 --- /dev/null +++ b/vendor/mio/CMakeLists.txt @@ -0,0 +1,223 @@ +cmake_minimum_required(VERSION 3.8) + +# +# Here we check whether mio is being configured in isolation or as a component +# of a larger project. To do so, we query whether the `PROJECT_NAME` CMake +# variable has been defined. In the case it has, we can conclude mio is a +# subproject. +# +# This convention has been borrowed from the Catch C++ unit testing library. +# +if(DEFINED PROJECT_NAME) + set(subproject ON) + if(NOT DEFINED INSTALL_SUBPROJECTS) + set(INSTALL_SUBPROJECTS ON CACHE BOOL "Install subproject dependencies") + endif() +else() + set(subproject OFF) + set_property(GLOBAL PROPERTY USE_FOLDERS ON) +endif() + +project(mio VERSION 1.1.0 LANGUAGES C CXX) +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") +include(CMakeDependentOption) +include(CMakePackageConfigHelpers) +include(CTest) +include(GNUInstallDirs) + +# Generate 'compile_commands.json' for clang_complete +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# +# mio requires C++ 11 support, at a minimum. The `CMAKE_CXX_STANDARD` variable +# is referenced when a target is created to initialize the CXX_STANDARD target +# property. +# +# ** NOTE ** +# This is a directory scope variable. This has several implicitations. +# +# 1. It DOES NOT propegate to parent scopes (such as parent projects) +# 2. It hides cache variables of the same name for this directory and below +# +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# +# The `mio.testing` options only appear as cmake-gui and ccmake options iff +# mio is the highest level project. In the case that mio is a subproject, these +# options are hidden from the user interface and set to `OFF` +# +# Iff mio is the highest level project, this option is defaulted to the value +# of the traditional course grain testing option `BUILD_TESTING` established by +# the CTest module +# +CMAKE_DEPENDENT_OPTION(mio.tests + "Build the mio tests and integrate with ctest" + ON "BUILD_TESTING; NOT subproject" OFF) + +# +# On Windows, so as to be a "good citizen", mio offers two mechanisms to control +# the imported surface area of the Windows API. The default `mio` target sets +# the necessary flags for a minimal Win API (`WIN32_LEAN_AND_MEAN`, etc.) on +# linking targets. This is, in our view, the conservative behavior. +# +# However, an option is published in the cache allowing client to opt out of +# these compiler definintions. This preference will persist in the installed +# cmake configuration file, but can be modified by downstream users by way of +# the same cmake cache variable. This allows intermediate consumers (e.g. other +# libraries) to defer this decision making to downstream clients. +# +# Beyond the option-based mechanism, two additional targets, +# mio::mio_min_winapi and mio::mio_full_winapi, are specified below for those +# that expressly requiring either the minimal or full windows API, respectively. +# +CMAKE_DEPENDENT_OPTION(mio.windows.full_api + "Configure mio without WIN32_LEAN_AND_MEAN and NOMINMAX definitions" + OFF "WIN32" ON) + +# +# When the end user is consuming mio as a nested subproject, an option +# is provided such that the user may exlude mio from the set of installed +# cmake projects. This accomodates end users building executables or +# compiled libraries which privately link to mio, but wish to only ship their +# artifacts in an installation +# +CMAKE_DEPENDENT_OPTION(mio.installation + "Include mio in the install set" + "${INSTALL_SUBPROJECTS}" "subproject" ON) +mark_as_advanced(mio.installation) + +# +# mio has no compiled components. As such, we declare it as an `INTERFACE` +# library, which denotes a collection of target properties to be applied +# transitively to linking targets. In our case, this amounts to an include +# directory and (potentially) some preprocessor definitions. +# +add_library(mio INTERFACE) +add_library(mio::mio ALIAS mio) + +# +# The include directory for mio can be expected to vary between build +# and installaion. Here we use a CMake generator expression to dispatch +# on how the configuration under which this library is being consumed. +# +# We define the generator expression as a variable, such that the logic +# need not be repeated when populating source file paths. +# +string(CONCAT prefix + "$" + "$") + +target_include_directories(mio INTERFACE ${prefix}) + +if(NOT mio.windows.full_api) + target_compile_definitions(mio INTERFACE + $ + $) +endif() + +if(WIN32) + add_library(mio_full_winapi INTERFACE) + add_library(mio::mio_full_winapi ALIAS mio_full_winapi) + target_include_directories(mio_full_winapi INTERFACE ${prefix}) + + add_library(mio_min_winapi INTERFACE) + add_library(mio::mio_min_winapi ALIAS mio_full_winapi) + target_compile_definitions(mio INTERFACE WIN32_LEAN_AND_MEAN NOMINMAX) + target_include_directories(mio_min_winapi INTERFACE ${prefix}) +endif() + +# +# In order to collect mio's header files in IDE tools such as XCode or Visual +# Studio, there must exist a target adding any such header files as source files. +# +# Given mio is an interface target, source files may only be added with the +# INTERFACE keyword, which consequently propegate to linking targets. This +# behavior isn't desirable to all clients. +# +# To accomodate, a second target is declared which collects the header files, +# which links to the primary mio target. As such, the header files are available +# in IDE tools. +# +add_library(mio-headers INTERFACE) +add_library(mio::mio-headers ALIAS mio-headers) +target_link_libraries(mio-headers INTERFACE mio) + +add_subdirectory(include/mio) + +if(mio.tests) + add_subdirectory(test) +endif() + +if(mio.installation) + # + # Non-testing header files (preserving relative paths) are installed to the + # `include` subdirectory of the `$INSTALL_DIR/${CMAKE_INSTALL_PREFIX}` + # directory. Source file permissions preserved. + # + install(DIRECTORY include/ + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + FILES_MATCHING PATTERN "*.*pp") + + # + # As a header-only library, there are no target components to be installed + # directly (the PUBLIC_HEADER property is not white listed for INTERFACE + # targets for some reason). + # + # However, it is worthwhile export our target description in order to later + # generate a CMake configuration file for consumption by CMake's `find_package` + # intrinsic + # + install(TARGETS mio mio-headers EXPORT mioTargets) + + if(WIN32) + install(TARGETS mio_full_winapi mio_min_winapi EXPORT mioTargets) + endif() + + install(EXPORT mioTargets + FILE mio-targets.cmake + NAMESPACE mio:: + DESTINATION share/cmake/mio) + + write_basic_package_version_file("mio-config-version.cmake" + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion) + + configure_file( + "${PROJECT_SOURCE_DIR}/cmake/mio-config.cmake.in" + "${PROJECT_BINARY_DIR}/mio-config.cmake" + @ONLY) + + install(FILES + "${PROJECT_BINARY_DIR}/mio-config-version.cmake" + "${PROJECT_BINARY_DIR}/mio-config.cmake" + DESTINATION share/cmake/mio) + + # + # Rudimentary CPack support. + # + # CPack provides a mechanism to generate installation packaging for a project, + # e.g., self-extracting shell scripts, compressed tarballs, Debian Package files, + # RPM Package Manager files, Windows NSIS installation wizards, + # Apple Disk Images (.dmg), etc. + # + # A packaged installation can be generated by calling + # + # ```sh + # cpack -G --config CPackConfig.cmake + # ``` + # + # See `cpack --help` or the CPack documentation for more information. + # + if(NOT subproject) + set(CPACK_PACKAGE_VENDOR "mandreyel") + set(CPACK_PACKAGE_DESCRIPTION_SUMMARY + "Cross-platform C++11 header-only library for memory mapped file IO") + set(CMAKE_PROJECT_HOMEPAGE_URL "https://github.com/mandreyel/mio") + set(CPACK_RESOURCE_FILE_LICENSE "${PROJECT_SOURCE_DIR}/LICENSE") + set(CPACK_PACKAGE_VERSION_MAJOR "${PROJECT_VERSION_MAJOR}") + set(CPACK_PACKAGE_VERSION_MINOR "${PROJECT_VERSION_MINOR}") + set(CPACK_PACKAGE_VERSION_PATCH "${PROJECT_VERSION_PATCH}") + include(CPack) + endif() +endif() diff --git a/vendor/mio/LICENSE b/vendor/mio/LICENSE new file mode 100644 index 0000000..3617707 --- /dev/null +++ b/vendor/mio/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 https://github.com/mandreyel/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/mio/README.md b/vendor/mio/README.md new file mode 100644 index 0000000..a9ab409 --- /dev/null +++ b/vendor/mio/README.md @@ -0,0 +1,339 @@ +# mio +An easy to use header-only cross-platform C++11 memory mapping library with an MIT license. + +mio has been created with the goal to be easily includable (i.e. no dependencies) in any C++ project that needs memory mapped file IO without the need to pull in Boost. + +Please feel free to open an issue, I'll try to address any concerns as best I can. + +### Why? +Because memory mapping is the best thing since sliced bread! + +More seriously, the primary motivation for writing this library instead of using Boost.Iostreams, was the lack of support for establishing a memory mapping with an already open file handle/descriptor. This is possible with mio. + +Furthermore, Boost.Iostreams' solution requires that the user pick offsets exactly at page boundaries, which is cumbersome and error prone. mio, on the other hand, manages this internally, accepting any offset and finding the nearest page boundary. + +Albeit a minor nitpick, Boost.Iostreams implements memory mapped file IO with a `std::shared_ptr` to provide shared semantics, even if not needed, and the overhead of the heap allocation may be unnecessary and/or unwanted. +In mio, there are two classes to cover the two use-cases: one that is move-only (basically a zero-cost abstraction over the system specific mmapping functions), and the other that acts just like its Boost.Iostreams counterpart, with shared semantics. + +### How to create a mapping +NOTE: the file must exist before creating a mapping. + +There are three ways to map a file into memory: + +- Using the constructor, which throws a `std::system_error` on failure: +```c++ +mio::mmap_source mmap(path, offset, size_to_map); +``` +or you can omit the `offset` and `size_to_map` arguments, in which case the +entire file is mapped: +```c++ +mio::mmap_source mmap(path); +``` + +- Using the factory function: +```c++ +std::error_code error; +mio::mmap_source mmap = mio::make_mmap_source(path, offset, size_to_map, error); +``` +or: +```c++ +mio::mmap_source mmap = mio::make_mmap_source(path, error); +``` + +- Using the `map` member function: +```c++ +std::error_code error; +mio::mmap_source mmap; +mmap.map(path, offset, size_to_map, error); +``` +or: +```c++ +mmap.map(path, error); +``` +**NOTE:** The constructors **require** exceptions to be enabled. If you prefer +to build your projects with `-fno-exceptions`, you can still use the other ways. + +Moreover, in each case, you can provide either some string type for the file's path, or you can use an existing, valid file handle. +```c++ +#include +#include +#include +#include +// #include if using single header +#include + +int main() +{ + // NOTE: error handling omitted for brevity. + const int fd = open("file.txt", O_RDONLY); + mio::mmap_source mmap(fd, 0, mio::map_entire_file); + // ... +} +``` +However, mio does not check whether the provided file descriptor has the same access permissions as the desired mapping, so the mapping may fail. Such errors are reported via the `std::error_code` out parameter that is passed to the mapping function. + +**WINDOWS USERS**: This library *does* support the use of wide character types +for functions where character strings are expected (e.g. path parameters). + +### Example + +```c++ +#include +// #include if using single header +#include // for std::error_code +#include // for std::printf +#include +#include +#include + +int handle_error(const std::error_code& error); +void allocate_file(const std::string& path, const int size); + +int main() +{ + const auto path = "file.txt"; + + // NOTE: mio does *not* create the file for you if it doesn't exist! You + // must ensure that the file exists before establishing a mapping. It + // must also be non-empty. So for illustrative purposes the file is + // created now. + allocate_file(path, 155); + + // Read-write memory map the whole file by using `map_entire_file` where the + // length of the mapping is otherwise expected, with the factory method. + std::error_code error; + mio::mmap_sink rw_mmap = mio::make_mmap_sink( + path, 0, mio::map_entire_file, error); + if (error) { return handle_error(error); } + + // You can use any iterator based function. + std::fill(rw_mmap.begin(), rw_mmap.end(), 'a'); + + // Or manually iterate through the mapped region just as if it were any other + // container, and change each byte's value (since this is a read-write mapping). + for (auto& b : rw_mmap) { + b += 10; + } + + // Or just change one value with the subscript operator. + const int answer_index = rw_mmap.size() / 2; + rw_mmap[answer_index] = 42; + + // Don't forget to flush changes to disk before unmapping. However, if + // `rw_mmap` were to go out of scope at this point, the destructor would also + // automatically invoke `sync` before `unmap`. + rw_mmap.sync(error); + if (error) { return handle_error(error); } + + // We can then remove the mapping, after which rw_mmap will be in a default + // constructed state, i.e. this and the above call to `sync` have the same + // effect as if the destructor had been invoked. + rw_mmap.unmap(); + + // Now create the same mapping, but in read-only mode. Note that calling the + // overload without the offset and file length parameters maps the entire + // file. + mio::mmap_source ro_mmap; + ro_mmap.map(path, error); + if (error) { return handle_error(error); } + + const int the_answer_to_everything = ro_mmap[answer_index]; + assert(the_answer_to_everything == 42); +} + +int handle_error(const std::error_code& error) +{ + const auto& errmsg = error.message(); + std::printf("error mapping file: %s, exiting...\n", errmsg.c_str()); + return error.value(); +} + +void allocate_file(const std::string& path, const int size) +{ + std::ofstream file(path); + std::string s(size, '0'); + file << s; +} +``` + +`mio::basic_mmap` is move-only, but if multiple copies to the same mapping are needed, use `mio::basic_shared_mmap` which has `std::shared_ptr` semantics and has the same interface as `mio::basic_mmap`. +```c++ +#include + +mio::shared_mmap_source shared_mmap1("path", offset, size_to_map); +mio::shared_mmap_source shared_mmap2(std::move(mmap1)); // or use operator= +mio::shared_mmap_source shared_mmap3(std::make_shared(mmap1)); // or use operator= +mio::shared_mmap_source shared_mmap4; +shared_mmap4.map("path", offset, size_to_map, error); +``` + +It's possible to define the type of a byte (which has to be the same width as `char`), though aliases for the most common ones are provided by default: +```c++ +using mmap_source = basic_mmap_source; +using ummap_source = basic_mmap_source; + +using mmap_sink = basic_mmap_sink; +using ummap_sink = basic_mmap_sink; +``` +But it may be useful to define your own types, say when using the new `std::byte` type in C++17: +```c++ +using mmap_source = mio::basic_mmap_source; +using mmap_sink = mio::basic_mmap_sink; +``` + +Though generally not needed, since mio maps users requested offsets to page boundaries, you can query the underlying system's page allocation granularity by invoking `mio::page_size()`, which is located in `mio/page.hpp`. + +### Single Header File +Mio can be added to your project as a single header file simply by including `\single_include\mio\mio.hpp`. Single header files can be regenerated at any time by running the `amalgamate.py` script within `\third_party`. +``` +python amalgamate.py -c config.json -s ../include +``` + +## CMake +As a header-only library, mio has no compiled components. Nevertheless, a [CMake](https://cmake.org/overview/) build system is provided to allow easy testing, installation, and subproject composition on many platforms and operating systems. + +### Testing +Mio is distributed with a small suite of tests and examples. +When mio is configured as the highest level CMake project, this suite of executables is built by default. +Mio's test executables are integrated with the CMake test driver program, [CTest](https://cmake.org/cmake/help/latest/manual/ctest.1.html). + +CMake supports a number of backends for compilation and linking. + +To use a static configuration build tool, such as GNU Make or Ninja: + +```sh +cd +mkdir build +cd build + +# Configure the build +cmake -D CMAKE_BUILD_TYPE= \ + -G <"Unix Makefiles" | "Ninja"> .. + +# build the tests +< make | ninja | cmake --build . > + +# run the tests +< make test | ninja test | cmake --build . --target test | ctest > +``` + +To use a dynamic configuration build tool, such as Visual Studio or Xcode: + +```sh +cd +mkdir build +cd build + +# Configure the build +cmake -G <"Visual Studio 14 2015 Win64" | "Xcode"> .. + +# build the tests +cmake --build . --config + +# run the tests via ctest... +ctest --build-config + +# ... or via CMake build tool mode... +cmake --build . --config --target test +``` + +Of course the **build** and **test** steps can also be executed via the **all** and **test** targets, respectively, from within the IDE after opening the project file generated during the configuration step. + +Mio's testing is also configured to operate as a client to the [CDash](https://www.cdash.org/) software quality dashboard application. Please see the [Kitware documentation](https://cmake.org/cmake/help/latest/manual/ctest.1.html#dashboard-client) for more information on this mode of operation. + +### Installation + +Mio's build system provides an installation target and support for downstream consumption via CMake's [`find_package`](https://cmake.org/cmake/help/v3.0/command/find_package.html) intrinsic function. +CMake allows installation to an arbitrary location, which may be specified by defining `CMAKE_INSTALL_PREFIX` at configure time. +In the absense of a user specification, CMake will install mio to conventional location based on the platform operating system. + +To use a static configuration build tool, such as GNU Make or Ninja: + +```sh +cd +mkdir build +cd build + +# Configure the build +cmake [-D CMAKE_INSTALL_PREFIX="path/to/installation"] \ + [-D BUILD_TESTING=False] \ + -D CMAKE_BUILD_TYPE=Release \ + -G <"Unix Makefiles" | "Ninja"> .. + +# install mio + +``` + +To use a dynamic configuration build tool, such as Visual Studio or Xcode: + +```sh +cd +mkdir build +cd build + +# Configure the project +cmake [-D CMAKE_INSTALL_PREFIX="path/to/installation"] \ + [-D BUILD_TESTING=False] \ + -G <"Visual Studio 14 2015 Win64" | "Xcode"> .. + +# install mio +cmake --build . --config Release --target install +``` + +Note that the last command of the installation sequence may require administrator privileges (e.g. `sudo`) if the installation root directory lies outside your home directory. + +This installation ++ copies the mio header files to the `include/mio` subdirectory of the installation root ++ generates and copies several CMake configuration files to the `share/cmake/mio` subdirectory of the installation root + +This latter step allows downstream CMake projects to consume mio via `find_package`, e.g. + +```cmake +find_package( mio REQUIRED ) +target_link_libraries( MyTarget PUBLIC mio::mio ) +``` + +**WINDOWS USERS**: The `mio::mio` target `#define`s `WIN32_LEAN_AND_MEAN` and `NOMINMAX`. The former ensures the imported surface area of the Win API is minimal, and the latter disables Windows' `min` and `max` macros so they don't intefere with `std::min` and `std::max`. Because *mio* is a header only library, these defintions will leak into downstream CMake builds. If their presence is causing problems with your build then you can use the alternative `mio::mio_full_winapi` target, which adds none of these defintions. + +If mio was installed to a non-conventional location, it may be necessary for downstream projects to specify the mio installation root directory via either + ++ the `CMAKE_PREFIX_PATH` configuration option, ++ the `CMAKE_PREFIX_PATH` environment variable, or ++ `mio_DIR` environment variable. + +Please see the [Kitware documentation](https://cmake.org/cmake/help/v3.0/command/find_package.html) for more information. + +In addition, mio supports packaged relocatable installations via [CPack](https://cmake.org/cmake/help/latest/manual/cpack.1.html). +Following configuration, from the build directory, invoke cpack as follows to generate a packaged installation: + +```sh +cpack -G -C Release +``` + +The list of supported generators varies from platform to platform. See the output of `cpack --help` for a complete list of supported generators on your platform. + +### Subproject Composition +To use mio as a subproject, copy the mio repository to your project's dependencies/externals folder. +If your project is version controlled using git, a git submodule or git subtree can be used to syncronize with the updstream repository. +The [use](https://services.github.com/on-demand/downloads/submodule-vs-subtree-cheat-sheet/) and [relative advantages](https://andrey.nering.com.br/2016/git-submodules-vs-subtrees/) of these git facilities is beyond the scope of this document, but in brief, each may be established as follows: + +```sh +# via git submodule +cd +git submodule add -b master https://github.com/mandreyel/mio.git + +# via git subtree +cd +git subtree add --prefix /mio \ + https://github.com/mandreyel/mio.git master --squash +``` + +Given a mio subdirectory in a project, simply add the following lines to your project's to add mio include directories to your target's include path. + +```cmake +add_subdirectory( path/to/mio/ ) +target_link_libraries( MyTarget PUBLIC ) +``` + +Note that, as a subproject, mio's tests and examples will not be built and CPack integration is deferred to the host project. + diff --git a/vendor/mio/cmake/CTestCustom.cmake b/vendor/mio/cmake/CTestCustom.cmake new file mode 100644 index 0000000..1e68e7f --- /dev/null +++ b/vendor/mio/cmake/CTestCustom.cmake @@ -0,0 +1,3 @@ +set(CTEST_CUSTOM_COVERAGE_EXCLUDE + ".*test.*" + ".*c[+][+].*") diff --git a/vendor/mio/cmake/mio-config.cmake.in b/vendor/mio/cmake/mio-config.cmake.in new file mode 100644 index 0000000..43c59dd --- /dev/null +++ b/vendor/mio/cmake/mio-config.cmake.in @@ -0,0 +1,13 @@ +include(CMakeDependentOption) + +CMAKE_DEPENDENT_OPTION(mio.windows.full_api + "Configure mio without WIN32_LEAN_AND_MEAN and NOMINMAX definitions" + @mio.windows.full_api@ "WIN32" ON) + +include("${CMAKE_CURRENT_LIST_DIR}/mio-targets.cmake") + +if(NOT mio.windows.full_api) + set_property(TARGET mio::mio APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS + WIN32_LEAN_AND_MEAN + NOMINMAX) +endif() diff --git a/vendor/mio/include/mio/CMakeLists.txt b/vendor/mio/include/mio/CMakeLists.txt new file mode 100644 index 0000000..1da153e --- /dev/null +++ b/vendor/mio/include/mio/CMakeLists.txt @@ -0,0 +1,11 @@ +# +# While not strictly necessary to specify header files as target sources, +# doing so populates these files in the source listing when CMake is used +# to generate XCode and Visual Studios projects +# +target_sources(mio-headers INTERFACE + "${prefix}/mio/mmap.hpp" + "${prefix}/mio/page.hpp" + "${prefix}/mio/shared_mmap.hpp") + +add_subdirectory(detail) diff --git a/vendor/mio/include/mio/detail/CMakeLists.txt b/vendor/mio/include/mio/detail/CMakeLists.txt new file mode 100644 index 0000000..de957ea --- /dev/null +++ b/vendor/mio/include/mio/detail/CMakeLists.txt @@ -0,0 +1,3 @@ +target_sources(mio-headers INTERFACE + "${prefix}/mio/detail/mmap.ipp" + "${prefix}/mio/detail/string_util.hpp") diff --git a/vendor/mio/include/mio/detail/mmap.ipp b/vendor/mio/include/mio/detail/mmap.ipp new file mode 100644 index 0000000..716a171 --- /dev/null +++ b/vendor/mio/include/mio/detail/mmap.ipp @@ -0,0 +1,533 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_BASIC_MMAP_IMPL +#define MIO_BASIC_MMAP_IMPL + +#include "mio/mmap.hpp" +#include "mio/page.hpp" +#include "mio/detail/string_util.hpp" + +#include + +#ifndef _WIN32 +# include +# include +# include +# include +#endif + +namespace mio { +namespace detail { + +#ifdef _WIN32 +namespace win { + +/** Returns the 4 upper bytes of an 8-byte integer. */ +inline DWORD int64_high(int64_t n) noexcept +{ + return n >> 32; +} + +/** Returns the 4 lower bytes of an 8-byte integer. */ +inline DWORD int64_low(int64_t n) noexcept +{ + return n & 0xffffffff; +} + +inline std::wstring s_2_ws(const std::string& s) +{ + std::wstring ret; + if (!s.empty()) + { + ret.resize(s.size()); + int wide_char_count = MultiByteToWideChar(CP_UTF8, 0, s.c_str(), + static_cast(s.size()), &ret[0], static_cast(s.size())); + ret.resize(wide_char_count); + } + return ret; +} + +template< + typename String, + typename = typename std::enable_if< + std::is_same::type, char>::value + >::type +> file_handle_type open_file_helper(const String& path, const access_mode mode) +{ + return ::CreateFileW(s_2_ws(path).c_str(), + mode == access_mode::read ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + 0, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + 0); +} + +template +typename std::enable_if< + std::is_same::type, wchar_t>::value, + file_handle_type +>::type open_file_helper(const String& path, const access_mode mode) +{ + return ::CreateFileW(c_str(path), + mode == access_mode::read ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + 0, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + 0); +} + +} // win +#endif // _WIN32 + +/** + * Returns the last platform specific system error (errno on POSIX and + * GetLastError on Win) as a `std::error_code`. + */ +inline std::error_code last_error() noexcept +{ + std::error_code error; +#ifdef _WIN32 + error.assign(GetLastError(), std::system_category()); +#else + error.assign(errno, std::system_category()); +#endif + return error; +} + +template +file_handle_type open_file(const String& path, const access_mode mode, + std::error_code& error) +{ + error.clear(); + if(detail::empty(path)) + { + error = std::make_error_code(std::errc::invalid_argument); + return invalid_handle; + } +#ifdef _WIN32 + const auto handle = win::open_file_helper(path, mode); +#else // POSIX + const auto handle = ::open(c_str(path), + mode == access_mode::read ? O_RDONLY : O_RDWR); +#endif + if(handle == invalid_handle) + { + error = detail::last_error(); + } + return handle; +} + +inline size_t query_file_size(file_handle_type handle, std::error_code& error) +{ + error.clear(); +#ifdef _WIN32 + LARGE_INTEGER file_size; + if(::GetFileSizeEx(handle, &file_size) == 0) + { + error = detail::last_error(); + return 0; + } + return static_cast(file_size.QuadPart); +#else // POSIX + struct stat sbuf; + if(::fstat(handle, &sbuf) == -1) + { + error = detail::last_error(); + return 0; + } + return sbuf.st_size; +#endif +} + +struct mmap_context +{ + char* data; + int64_t length; + int64_t mapped_length; +#ifdef _WIN32 + file_handle_type file_mapping_handle; +#endif +}; + +inline mmap_context memory_map(const file_handle_type file_handle, const int64_t offset, + const int64_t length, const access_mode mode, std::error_code& error) +{ + const int64_t aligned_offset = make_offset_page_aligned(offset); + const int64_t length_to_map = offset - aligned_offset + length; +#ifdef _WIN32 + const int64_t max_file_size = offset + length; + const auto file_mapping_handle = ::CreateFileMapping( + file_handle, + 0, + mode == access_mode::read ? PAGE_READONLY : PAGE_READWRITE, + win::int64_high(max_file_size), + win::int64_low(max_file_size), + 0); + if(file_mapping_handle == invalid_handle) + { + error = detail::last_error(); + return {}; + } + char* mapping_start = static_cast(::MapViewOfFile( + file_mapping_handle, + mode == access_mode::read ? FILE_MAP_READ : FILE_MAP_WRITE, + win::int64_high(aligned_offset), + win::int64_low(aligned_offset), + length_to_map)); + if(mapping_start == nullptr) + { + // Close file handle if mapping it failed. + ::CloseHandle(file_mapping_handle); + error = detail::last_error(); + return {}; + } +#else // POSIX + char* mapping_start = static_cast(::mmap( + 0, // Don't give hint as to where to map. + length_to_map, + mode == access_mode::read ? PROT_READ : PROT_WRITE, + MAP_SHARED, + file_handle, + aligned_offset)); + if(mapping_start == MAP_FAILED) + { + error = detail::last_error(); + return {}; + } +#endif + mmap_context ctx; + ctx.data = mapping_start + offset - aligned_offset; + ctx.length = length; + ctx.mapped_length = length_to_map; +#ifdef _WIN32 + ctx.file_mapping_handle = file_mapping_handle; +#endif + return ctx; +} + +} // namespace detail + +// -- basic_mmap -- + +template +basic_mmap::~basic_mmap() +{ + conditional_sync(); + unmap(); +} + +template +basic_mmap::basic_mmap(basic_mmap&& other) + : data_(std::move(other.data_)) + , length_(std::move(other.length_)) + , mapped_length_(std::move(other.mapped_length_)) + , file_handle_(std::move(other.file_handle_)) +#ifdef _WIN32 + , file_mapping_handle_(std::move(other.file_mapping_handle_)) +#endif + , is_handle_internal_(std::move(other.is_handle_internal_)) +{ + other.data_ = nullptr; + other.length_ = other.mapped_length_ = 0; + other.file_handle_ = invalid_handle; +#ifdef _WIN32 + other.file_mapping_handle_ = invalid_handle; +#endif +} + +template +basic_mmap& +basic_mmap::operator=(basic_mmap&& other) +{ + if(this != &other) + { + // First the existing mapping needs to be removed. + unmap(); + data_ = std::move(other.data_); + length_ = std::move(other.length_); + mapped_length_ = std::move(other.mapped_length_); + file_handle_ = std::move(other.file_handle_); +#ifdef _WIN32 + file_mapping_handle_ = std::move(other.file_mapping_handle_); +#endif + is_handle_internal_ = std::move(other.is_handle_internal_); + + // The moved from basic_mmap's fields need to be reset, because + // otherwise other's destructor will unmap the same mapping that was + // just moved into this. + other.data_ = nullptr; + other.length_ = other.mapped_length_ = 0; + other.file_handle_ = invalid_handle; +#ifdef _WIN32 + other.file_mapping_handle_ = invalid_handle; +#endif + other.is_handle_internal_ = false; + } + return *this; +} + +template +typename basic_mmap::handle_type +basic_mmap::mapping_handle() const noexcept +{ +#ifdef _WIN32 + return file_mapping_handle_; +#else + return file_handle_; +#endif +} + +template +template +void basic_mmap::map(const String& path, const size_type offset, + const size_type length, std::error_code& error) +{ + error.clear(); + if(detail::empty(path)) + { + error = std::make_error_code(std::errc::invalid_argument); + return; + } + const auto handle = detail::open_file(path, AccessMode, error); + if(error) + { + return; + } + + map(handle, offset, length, error); + // This MUST be after the call to map, as that sets this to true. + if(!error) + { + is_handle_internal_ = true; + } +} + +template +void basic_mmap::map(const handle_type handle, + const size_type offset, const size_type length, std::error_code& error) +{ + error.clear(); + if(handle == invalid_handle) + { + error = std::make_error_code(std::errc::bad_file_descriptor); + return; + } + + const auto file_size = detail::query_file_size(handle, error); + if(error) + { + return; + } + + if(offset + length > file_size) + { + error = std::make_error_code(std::errc::invalid_argument); + return; + } + + const auto ctx = detail::memory_map(handle, offset, + length == map_entire_file ? (file_size - offset) : length, + AccessMode, error); + if(!error) + { + // We must unmap the previous mapping that may have existed prior to this call. + // Note that this must only be invoked after a new mapping has been created in + // order to provide the strong guarantee that, should the new mapping fail, the + // `map` function leaves this instance in a state as though the function had + // never been invoked. + unmap(); + file_handle_ = handle; + is_handle_internal_ = false; + data_ = reinterpret_cast(ctx.data); + length_ = ctx.length; + mapped_length_ = ctx.mapped_length; +#ifdef _WIN32 + file_mapping_handle_ = ctx.file_mapping_handle; +#endif + } +} + +template +template +typename std::enable_if::type +basic_mmap::sync(std::error_code& error) +{ + error.clear(); + if(!is_open()) + { + error = std::make_error_code(std::errc::bad_file_descriptor); + return; + } + + if(data()) + { +#ifdef _WIN32 + if(::FlushViewOfFile(get_mapping_start(), mapped_length_) == 0 + || ::FlushFileBuffers(file_handle_) == 0) +#else // POSIX + if(::msync(get_mapping_start(), mapped_length_, MS_SYNC) != 0) +#endif + { + error = detail::last_error(); + return; + } + } +#ifdef _WIN32 + if(::FlushFileBuffers(file_handle_) == 0) + { + error = detail::last_error(); + } +#endif +} + +template +void basic_mmap::unmap() +{ + if(!is_open()) { return; } + // TODO do we care about errors here? +#ifdef _WIN32 + if(is_mapped()) + { + ::UnmapViewOfFile(get_mapping_start()); + ::CloseHandle(file_mapping_handle_); + } +#else // POSIX + if(data_) { ::munmap(const_cast(get_mapping_start()), mapped_length_); } +#endif + + // If `file_handle_` was obtained by our opening it (when map is called with + // a path, rather than an existing file handle), we need to close it, + // otherwise it must not be closed as it may still be used outside this + // instance. + if(is_handle_internal_) + { +#ifdef _WIN32 + ::CloseHandle(file_handle_); +#else // POSIX + ::close(file_handle_); +#endif + } + + // Reset fields to their default values. + data_ = nullptr; + length_ = mapped_length_ = 0; + file_handle_ = invalid_handle; +#ifdef _WIN32 + file_mapping_handle_ = invalid_handle; +#endif +} + +template +bool basic_mmap::is_mapped() const noexcept +{ +#ifdef _WIN32 + return file_mapping_handle_ != invalid_handle; +#else // POSIX + return is_open(); +#endif +} + +template +void basic_mmap::swap(basic_mmap& other) +{ + if(this != &other) + { + using std::swap; + swap(data_, other.data_); + swap(file_handle_, other.file_handle_); +#ifdef _WIN32 + swap(file_mapping_handle_, other.file_mapping_handle_); +#endif + swap(length_, other.length_); + swap(mapped_length_, other.mapped_length_); + swap(is_handle_internal_, other.is_handle_internal_); + } +} + +template +template +typename std::enable_if::type +basic_mmap::conditional_sync() +{ + // This is invoked from the destructor, so not much we can do about + // failures here. + std::error_code ec; + sync(ec); +} + +template +template +typename std::enable_if::type +basic_mmap::conditional_sync() +{ + // noop +} + +template +bool operator==(const basic_mmap& a, + const basic_mmap& b) +{ + return a.data() == b.data() + && a.size() == b.size(); +} + +template +bool operator!=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a == b); +} + +template +bool operator<(const basic_mmap& a, + const basic_mmap& b) +{ + if(a.data() == b.data()) { return a.size() < b.size(); } + return a.data() < b.data(); +} + +template +bool operator<=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a > b); +} + +template +bool operator>(const basic_mmap& a, + const basic_mmap& b) +{ + if(a.data() == b.data()) { return a.size() > b.size(); } + return a.data() > b.data(); +} + +template +bool operator>=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a < b); +} + +} // namespace mio + +#endif // MIO_BASIC_MMAP_IMPL diff --git a/vendor/mio/include/mio/detail/string_util.hpp b/vendor/mio/include/mio/detail/string_util.hpp new file mode 100644 index 0000000..2f375aa --- /dev/null +++ b/vendor/mio/include/mio/detail/string_util.hpp @@ -0,0 +1,170 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_STRING_UTIL_HEADER +#define MIO_STRING_UTIL_HEADER + +#include + +namespace mio { +namespace detail { + +template< + typename S, + typename C = typename std::decay::type, + typename = decltype(std::declval().data()), + typename = typename std::enable_if< + std::is_same::value +#ifdef _WIN32 + || std::is_same::value +#endif + >::type +> struct char_type_helper { + using type = typename C::value_type; +}; + +template +struct char_type { + using type = typename char_type_helper::type; +}; + +// TODO: can we avoid this brute force approach? +template<> +struct char_type { + using type = char; +}; + +template<> +struct char_type { + using type = char; +}; + +template +struct char_type { + using type = char; +}; + +template +struct char_type { + using type = char; +}; + +#ifdef _WIN32 +template<> +struct char_type { + using type = wchar_t; +}; + +template<> +struct char_type { + using type = wchar_t; +}; + +template +struct char_type { + using type = wchar_t; +}; + +template +struct char_type { + using type = wchar_t; +}; +#endif // _WIN32 + +template +struct is_c_str_helper +{ + static constexpr bool value = std::is_same< + CharT*, + // TODO: I'm so sorry for this... Can this be made cleaner? + typename std::add_pointer< + typename std::remove_cv< + typename std::remove_pointer< + typename std::decay< + S + >::type + >::type + >::type + >::type + >::value; +}; + +template +struct is_c_str +{ + static constexpr bool value = is_c_str_helper::value; +}; + +#ifdef _WIN32 +template +struct is_c_wstr +{ + static constexpr bool value = is_c_str_helper::value; +}; +#endif // _WIN32 + +template +struct is_c_str_or_c_wstr +{ + static constexpr bool value = is_c_str::value +#ifdef _WIN32 + || is_c_wstr::value +#endif + ; +}; + +template< + typename String, + typename = decltype(std::declval().data()), + typename = typename std::enable_if::value>::type +> const typename char_type::type* c_str(const String& path) +{ + return path.data(); +} + +template< + typename String, + typename = decltype(std::declval().empty()), + typename = typename std::enable_if::value>::type +> bool empty(const String& path) +{ + return path.empty(); +} + +template< + typename String, + typename = typename std::enable_if::value>::type +> const typename char_type::type* c_str(String path) +{ + return path; +} + +template< + typename String, + typename = typename std::enable_if::value>::type +> bool empty(String path) +{ + return !path || (*path == 0); +} + +} // namespace detail +} // namespace mio + +#endif // MIO_STRING_UTIL_HEADER diff --git a/vendor/mio/include/mio/mmap.hpp b/vendor/mio/include/mio/mmap.hpp new file mode 100644 index 0000000..def559a --- /dev/null +++ b/vendor/mio/include/mio/mmap.hpp @@ -0,0 +1,492 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_MMAP_HEADER +#define MIO_MMAP_HEADER + +#include "mio/page.hpp" + +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif // WIN32_LEAN_AND_MEAN +# include +#else // ifdef _WIN32 +# define INVALID_HANDLE_VALUE -1 +#endif // ifdef _WIN32 + +namespace mio { + +// This value may be provided as the `length` parameter to the constructor or +// `map`, in which case a memory mapping of the entire file is created. +enum { map_entire_file = 0 }; + +#ifdef _WIN32 +using file_handle_type = HANDLE; +#else +using file_handle_type = int; +#endif + +// This value represents an invalid file handle type. This can be used to +// determine whether `basic_mmap::file_handle` is valid, for example. +const static file_handle_type invalid_handle = INVALID_HANDLE_VALUE; + +template +struct basic_mmap +{ + using value_type = ByteT; + using size_type = size_t; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using difference_type = std::ptrdiff_t; + using iterator = pointer; + using const_iterator = const_pointer; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using iterator_category = std::random_access_iterator_tag; + using handle_type = file_handle_type; + + static_assert(sizeof(ByteT) == sizeof(char), "ByteT must be the same size as char."); + +private: + // Points to the first requested byte, and not to the actual start of the mapping. + pointer data_ = nullptr; + + // Length--in bytes--requested by user (which may not be the length of the + // full mapping) and the length of the full mapping. + size_type length_ = 0; + size_type mapped_length_ = 0; + + // Letting user map a file using both an existing file handle and a path + // introcudes some complexity (see `is_handle_internal_`). + // On POSIX, we only need a file handle to create a mapping, while on + // Windows systems the file handle is necessary to retrieve a file mapping + // handle, but any subsequent operations on the mapped region must be done + // through the latter. + handle_type file_handle_ = INVALID_HANDLE_VALUE; +#ifdef _WIN32 + handle_type file_mapping_handle_ = INVALID_HANDLE_VALUE; +#endif + + // Letting user map a file using both an existing file handle and a path + // introcudes some complexity in that we must not close the file handle if + // user provided it, but we must close it if we obtained it using the + // provided path. For this reason, this flag is used to determine when to + // close `file_handle_`. + bool is_handle_internal_; + +public: + /** + * The default constructed mmap object is in a non-mapped state, that is, + * any operation that attempts to access nonexistent underlying data will + * result in undefined behaviour/segmentation faults. + */ + basic_mmap() = default; + +#ifdef __cpp_exceptions + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + template + basic_mmap(const String& path, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(path, offset, length, error); + if(error) { throw std::system_error(error); } + } + + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + basic_mmap(const handle_type handle, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(handle, offset, length, error); + if(error) { throw std::system_error(error); } + } +#endif // __cpp_exceptions + + /** + * `basic_mmap` has single-ownership semantics, so transferring ownership + * may only be accomplished by moving the object. + */ + basic_mmap(const basic_mmap&) = delete; + basic_mmap(basic_mmap&&); + basic_mmap& operator=(const basic_mmap&) = delete; + basic_mmap& operator=(basic_mmap&&); + + /** + * If this is a read-write mapping, the destructor invokes sync. Regardless + * of the access mode, unmap is invoked as a final step. + */ + ~basic_mmap(); + + /** + * On UNIX systems 'file_handle' and 'mapping_handle' are the same. On Windows, + * however, a mapped region of a file gets its own handle, which is returned by + * 'mapping_handle'. + */ + handle_type file_handle() const noexcept { return file_handle_; } + handle_type mapping_handle() const noexcept; + + /** Returns whether a valid memory mapping has been created. */ + bool is_open() const noexcept { return file_handle_ != invalid_handle; } + + /** + * Returns true if no mapping was established, that is, conceptually the + * same as though the length that was mapped was 0. This function is + * provided so that this class has Container semantics. + */ + bool empty() const noexcept { return length() == 0; } + + /** Returns true if a mapping was established. */ + bool is_mapped() const noexcept; + + /** + * `size` and `length` both return the logical length, i.e. the number of bytes + * user requested to be mapped, while `mapped_length` returns the actual number of + * bytes that were mapped which is a multiple of the underlying operating system's + * page allocation granularity. + */ + size_type size() const noexcept { return length(); } + size_type length() const noexcept { return length_; } + size_type mapped_length() const noexcept { return mapped_length_; } + + /** Returns the offset relative to the start of the mapping. */ + size_type mapping_offset() const noexcept + { + return mapped_length_ - length_; + } + + /** + * Returns a pointer to the first requested byte, or `nullptr` if no memory mapping + * exists. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer data() noexcept { return data_; } + const_pointer data() const noexcept { return data_; } + + /** + * Returns an iterator to the first requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator begin() noexcept { return data(); } + const_iterator begin() const noexcept { return data(); } + const_iterator cbegin() const noexcept { return data(); } + + /** + * Returns an iterator one past the last requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator end() noexcept { return data() + length(); } + const_iterator end() const noexcept { return data() + length(); } + const_iterator cend() const noexcept { return data() + length(); } + + /** + * Returns a reverse iterator to the last memory mapped byte, if a valid + * memory mapping exists, otherwise this function call is undefined + * behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rbegin() noexcept { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const noexcept + { return const_reverse_iterator(end()); } + const_reverse_iterator crbegin() const noexcept + { return const_reverse_iterator(end()); } + + /** + * Returns a reverse iterator past the first mapped byte, if a valid memory + * mapping exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rend() noexcept { return reverse_iterator(begin()); } + const_reverse_iterator rend() const noexcept + { return const_reverse_iterator(begin()); } + const_reverse_iterator crend() const noexcept + { return const_reverse_iterator(begin()); } + + /** + * Returns a reference to the `i`th byte from the first requested byte (as returned + * by `data`). If this is invoked when no valid memory mapping has been created + * prior to this call, undefined behaviour ensues. + */ + reference operator[](const size_type i) noexcept { return data_[i]; } + const_reference operator[](const size_type i) const noexcept { return data_[i]; } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + template + void map(const String& path, const size_type offset, + const size_type length, std::error_code& error); + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * The entire file is mapped. + */ + template + void map(const String& path, std::error_code& error) + { + map(path, 0, map_entire_file, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is + * unsuccesful, the reason is reported via `error` and the object remains in + * a state as if this function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + void map(const handle_type handle, const size_type offset, + const size_type length, std::error_code& error); + + /** + * Establishes a memory mapping with AccessMode. If the mapping is + * unsuccesful, the reason is reported via `error` and the object remains in + * a state as if this function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * The entire file is mapped. + */ + void map(const handle_type handle, std::error_code& error) + { + map(handle, 0, map_entire_file, error); + } + + /** + * If a valid memory mapping has been created prior to this call, this call + * instructs the kernel to unmap the memory region and disassociate this object + * from the file. + * + * The file handle associated with the file that is mapped is only closed if the + * mapping was created using a file path. If, on the other hand, an existing + * file handle was used to create the mapping, the file handle is not closed. + */ + void unmap(); + + void swap(basic_mmap& other); + + /** Flushes the memory mapped page to disk. Errors are reported via `error`. */ + template + typename std::enable_if::type + sync(std::error_code& error); + + /** + * All operators compare the address of the first byte and size of the two mapped + * regions. + */ + +private: + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer get_mapping_start() noexcept + { + return !data() ? nullptr : data() - mapping_offset(); + } + + const_pointer get_mapping_start() const noexcept + { + return !data() ? nullptr : data() - mapping_offset(); + } + + /** + * The destructor syncs changes to disk if `AccessMode` is `write`, but not + * if it's `read`, but since the destructor cannot be templated, we need to + * do SFINAE in a dedicated function, where one syncs and the other is a noop. + */ + template + typename std::enable_if::type + conditional_sync(); + template + typename std::enable_if::type conditional_sync(); +}; + +template +bool operator==(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator!=(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator<(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator<=(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator>(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator>=(const basic_mmap& a, + const basic_mmap& b); + +/** + * This is the basis for all read-only mmap objects and should be preferred over + * directly using `basic_mmap`. + */ +template +using basic_mmap_source = basic_mmap; + +/** + * This is the basis for all read-write mmap objects and should be preferred over + * directly using `basic_mmap`. + */ +template +using basic_mmap_sink = basic_mmap; + +/** + * These aliases cover the most common use cases, both representing a raw byte stream + * (either with a char or an unsigned char/uint8_t). + */ +using mmap_source = basic_mmap_source; +using ummap_source = basic_mmap_source; + +using mmap_sink = basic_mmap_sink; +using ummap_sink = basic_mmap_sink; + +/** + * Convenience factory method that constructs a mapping for any `basic_mmap` or + * `basic_mmap` type. + */ +template< + typename MMap, + typename MappingToken +> MMap make_mmap(const MappingToken& token, + int64_t offset, int64_t length, std::error_code& error) +{ + MMap mmap; + mmap.map(token, offset, length, error); + return mmap; +} + +/** + * Convenience factory method. + * + * MappingToken may be a String (`std::string`, `std::string_view`, `const char*`, + * `std::filesystem::path`, `std::vector`, or similar), or a + * `mmap_source::handle_type`. + */ +template +mmap_source make_mmap_source(const MappingToken& token, mmap_source::size_type offset, + mmap_source::size_type length, std::error_code& error) +{ + return make_mmap(token, offset, length, error); +} + +template +mmap_source make_mmap_source(const MappingToken& token, std::error_code& error) +{ + return make_mmap_source(token, 0, map_entire_file, error); +} + +/** + * Convenience factory method. + * + * MappingToken may be a String (`std::string`, `std::string_view`, `const char*`, + * `std::filesystem::path`, `std::vector`, or similar), or a + * `mmap_sink::handle_type`. + */ +template +mmap_sink make_mmap_sink(const MappingToken& token, mmap_sink::size_type offset, + mmap_sink::size_type length, std::error_code& error) +{ + return make_mmap(token, offset, length, error); +} + +template +mmap_sink make_mmap_sink(const MappingToken& token, std::error_code& error) +{ + return make_mmap_sink(token, 0, map_entire_file, error); +} + +} // namespace mio + +#include "detail/mmap.ipp" + +#endif // MIO_MMAP_HEADER diff --git a/vendor/mio/include/mio/page.hpp b/vendor/mio/include/mio/page.hpp new file mode 100644 index 0000000..cae7377 --- /dev/null +++ b/vendor/mio/include/mio/page.hpp @@ -0,0 +1,78 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_PAGE_HEADER +#define MIO_PAGE_HEADER + +#ifdef _WIN32 +# include +#else +# include +#endif + +namespace mio { + +/** + * This is used by `basic_mmap` to determine whether to create a read-only or + * a read-write memory mapping. + */ +enum class access_mode +{ + read, + write +}; + +/** + * Determines the operating system's page allocation granularity. + * + * On the first call to this function, it invokes the operating system specific syscall + * to determine the page size, caches the value, and returns it. Any subsequent call to + * this function serves the cached value, so no further syscalls are made. + */ +inline size_t page_size() +{ + static const size_t page_size = [] + { +#ifdef _WIN32 + SYSTEM_INFO SystemInfo; + GetSystemInfo(&SystemInfo); + return SystemInfo.dwAllocationGranularity; +#else + return sysconf(_SC_PAGE_SIZE); +#endif + }(); + return page_size; +} + +/** + * Alligns `offset` to the operating's system page size such that it subtracts the + * difference until the nearest page boundary before `offset`, or does nothing if + * `offset` is already page aligned. + */ +inline size_t make_offset_page_aligned(size_t offset) noexcept +{ + const size_t page_size_ = page_size(); + // Use integer division to round down to the nearest page alignment. + return offset / page_size_ * page_size_; +} + +} // namespace mio + +#endif // MIO_PAGE_HEADER diff --git a/vendor/mio/include/mio/shared_mmap.hpp b/vendor/mio/include/mio/shared_mmap.hpp new file mode 100644 index 0000000..f125a59 --- /dev/null +++ b/vendor/mio/include/mio/shared_mmap.hpp @@ -0,0 +1,406 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_SHARED_MMAP_HEADER +#define MIO_SHARED_MMAP_HEADER + +#include "mio/mmap.hpp" + +#include // std::error_code +#include // std::shared_ptr + +namespace mio { + +/** + * Exposes (nearly) the same interface as `basic_mmap`, but endowes it with + * `std::shared_ptr` semantics. + * + * This is not the default behaviour of `basic_mmap` to avoid allocating on the heap if + * shared semantics are not required. + */ +template< + access_mode AccessMode, + typename ByteT +> class basic_shared_mmap +{ + using impl_type = basic_mmap; + std::shared_ptr pimpl_; + +public: + using value_type = typename impl_type::value_type; + using size_type = typename impl_type::size_type; + using reference = typename impl_type::reference; + using const_reference = typename impl_type::const_reference; + using pointer = typename impl_type::pointer; + using const_pointer = typename impl_type::const_pointer; + using difference_type = typename impl_type::difference_type; + using iterator = typename impl_type::iterator; + using const_iterator = typename impl_type::const_iterator; + using reverse_iterator = typename impl_type::reverse_iterator; + using const_reverse_iterator = typename impl_type::const_reverse_iterator; + using iterator_category = typename impl_type::iterator_category; + using handle_type = typename impl_type::handle_type; + using mmap_type = impl_type; + + basic_shared_mmap() = default; + basic_shared_mmap(const basic_shared_mmap&) = default; + basic_shared_mmap& operator=(const basic_shared_mmap&) = default; + basic_shared_mmap(basic_shared_mmap&&) = default; + basic_shared_mmap& operator=(basic_shared_mmap&&) = default; + + /** Takes ownership of an existing mmap object. */ + basic_shared_mmap(mmap_type&& mmap) + : pimpl_(std::make_shared(std::move(mmap))) + {} + + /** Takes ownership of an existing mmap object. */ + basic_shared_mmap& operator=(mmap_type&& mmap) + { + pimpl_ = std::make_shared(std::move(mmap)); + return *this; + } + + /** Initializes this object with an already established shared mmap. */ + basic_shared_mmap(std::shared_ptr mmap) : pimpl_(std::move(mmap)) {} + + /** Initializes this object with an already established shared mmap. */ + basic_shared_mmap& operator=(std::shared_ptr mmap) + { + pimpl_ = std::move(mmap); + return *this; + } + +#ifdef __cpp_exceptions + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + template + basic_shared_mmap(const String& path, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(path, offset, length, error); + if(error) { throw std::system_error(error); } + } + + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + basic_shared_mmap(const handle_type handle, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(handle, offset, length, error); + if(error) { throw std::system_error(error); } + } +#endif // __cpp_exceptions + + /** + * If this is a read-write mapping and the last reference to the mapping, + * the destructor invokes sync. Regardless of the access mode, unmap is + * invoked as a final step. + */ + ~basic_shared_mmap() = default; + + /** Returns the underlying `std::shared_ptr` instance that holds the mmap. */ + std::shared_ptr get_shared_ptr() { return pimpl_; } + + /** + * On UNIX systems 'file_handle' and 'mapping_handle' are the same. On Windows, + * however, a mapped region of a file gets its own handle, which is returned by + * 'mapping_handle'. + */ + handle_type file_handle() const noexcept + { + return pimpl_ ? pimpl_->file_handle() : invalid_handle; + } + + handle_type mapping_handle() const noexcept + { + return pimpl_ ? pimpl_->mapping_handle() : invalid_handle; + } + + /** Returns whether a valid memory mapping has been created. */ + bool is_open() const noexcept { return pimpl_ && pimpl_->is_open(); } + + /** + * Returns true if no mapping was established, that is, conceptually the + * same as though the length that was mapped was 0. This function is + * provided so that this class has Container semantics. + */ + bool empty() const noexcept { return !pimpl_ || pimpl_->empty(); } + + /** + * `size` and `length` both return the logical length, i.e. the number of bytes + * user requested to be mapped, while `mapped_length` returns the actual number of + * bytes that were mapped which is a multiple of the underlying operating system's + * page allocation granularity. + */ + size_type size() const noexcept { return pimpl_ ? pimpl_->length() : 0; } + size_type length() const noexcept { return pimpl_ ? pimpl_->length() : 0; } + size_type mapped_length() const noexcept + { + return pimpl_ ? pimpl_->mapped_length() : 0; + } + + /** + * Returns a pointer to the first requested byte, or `nullptr` if no memory mapping + * exists. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer data() noexcept { return pimpl_->data(); } + const_pointer data() const noexcept { return pimpl_ ? pimpl_->data() : nullptr; } + + /** + * Returns an iterator to the first requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + iterator begin() noexcept { return pimpl_->begin(); } + const_iterator begin() const noexcept { return pimpl_->begin(); } + const_iterator cbegin() const noexcept { return pimpl_->cbegin(); } + + /** + * Returns an iterator one past the last requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator end() noexcept { return pimpl_->end(); } + const_iterator end() const noexcept { return pimpl_->end(); } + const_iterator cend() const noexcept { return pimpl_->cend(); } + + /** + * Returns a reverse iterator to the last memory mapped byte, if a valid + * memory mapping exists, otherwise this function call is undefined + * behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rbegin() noexcept { return pimpl_->rbegin(); } + const_reverse_iterator rbegin() const noexcept { return pimpl_->rbegin(); } + const_reverse_iterator crbegin() const noexcept { return pimpl_->crbegin(); } + + /** + * Returns a reverse iterator past the first mapped byte, if a valid memory + * mapping exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rend() noexcept { return pimpl_->rend(); } + const_reverse_iterator rend() const noexcept { return pimpl_->rend(); } + const_reverse_iterator crend() const noexcept { return pimpl_->crend(); } + + /** + * Returns a reference to the `i`th byte from the first requested byte (as returned + * by `data`). If this is invoked when no valid memory mapping has been created + * prior to this call, undefined behaviour ensues. + */ + reference operator[](const size_type i) noexcept { return (*pimpl_)[i]; } + const_reference operator[](const size_type i) const noexcept { return (*pimpl_)[i]; } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + template + void map(const String& path, const size_type offset, + const size_type length, std::error_code& error) + { + map_impl(path, offset, length, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * The entire file is mapped. + */ + template + void map(const String& path, std::error_code& error) + { + map_impl(path, 0, map_entire_file, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + void map(const handle_type handle, const size_type offset, + const size_type length, std::error_code& error) + { + map_impl(handle, offset, length, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * The entire file is mapped. + */ + void map(const handle_type handle, std::error_code& error) + { + map_impl(handle, 0, map_entire_file, error); + } + + /** + * If a valid memory mapping has been created prior to this call, this call + * instructs the kernel to unmap the memory region and disassociate this object + * from the file. + * + * The file handle associated with the file that is mapped is only closed if the + * mapping was created using a file path. If, on the other hand, an existing + * file handle was used to create the mapping, the file handle is not closed. + */ + void unmap() { if(pimpl_) pimpl_->unmap(); } + + void swap(basic_shared_mmap& other) { pimpl_.swap(other.pimpl_); } + + /** Flushes the memory mapped page to disk. Errors are reported via `error`. */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > void sync(std::error_code& error) { if(pimpl_) pimpl_->sync(error); } + + /** All operators compare the underlying `basic_mmap`'s addresses. */ + + friend bool operator==(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ == b.pimpl_; + } + + friend bool operator!=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return !(a == b); + } + + friend bool operator<(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ < b.pimpl_; + } + + friend bool operator<=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ <= b.pimpl_; + } + + friend bool operator>(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ > b.pimpl_; + } + + friend bool operator>=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ >= b.pimpl_; + } + +private: + template + void map_impl(const MappingToken& token, const size_type offset, + const size_type length, std::error_code& error) + { + if(!pimpl_) + { + mmap_type mmap = make_mmap(token, offset, length, error); + if(error) { return; } + pimpl_ = std::make_shared(std::move(mmap)); + } + else + { + pimpl_->map(token, offset, length, error); + } + } +}; + +/** + * This is the basis for all read-only mmap objects and should be preferred over + * directly using basic_shared_mmap. + */ +template +using basic_shared_mmap_source = basic_shared_mmap; + +/** + * This is the basis for all read-write mmap objects and should be preferred over + * directly using basic_shared_mmap. + */ +template +using basic_shared_mmap_sink = basic_shared_mmap; + +/** + * These aliases cover the most common use cases, both representing a raw byte stream + * (either with a char or an unsigned char/uint8_t). + */ +using shared_mmap_source = basic_shared_mmap_source; +using shared_ummap_source = basic_shared_mmap_source; + +using shared_mmap_sink = basic_shared_mmap_sink; +using shared_ummap_sink = basic_shared_mmap_sink; + +} // namespace mio + +#endif // MIO_SHARED_MMAP_HEADER diff --git a/vendor/mio/single_include/mio/mio.hpp b/vendor/mio/single_include/mio/mio.hpp new file mode 100644 index 0000000..6ad6a74 --- /dev/null +++ b/vendor/mio/single_include/mio/mio.hpp @@ -0,0 +1,1763 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_MMAP_HEADER +#define MIO_MMAP_HEADER + +// #include "mio/page.hpp" +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_PAGE_HEADER +#define MIO_PAGE_HEADER + +#ifdef _WIN32 +# include +#else +# include +#endif + +namespace mio { + +/** + * This is used by `basic_mmap` to determine whether to create a read-only or + * a read-write memory mapping. + */ +enum class access_mode +{ + read, + write +}; + +/** + * Determines the operating system's page allocation granularity. + * + * On the first call to this function, it invokes the operating system specific syscall + * to determine the page size, caches the value, and returns it. Any subsequent call to + * this function serves the cached value, so no further syscalls are made. + */ +inline size_t page_size() +{ + static const size_t page_size = [] + { +#ifdef _WIN32 + SYSTEM_INFO SystemInfo; + GetSystemInfo(&SystemInfo); + return SystemInfo.dwAllocationGranularity; +#else + return sysconf(_SC_PAGE_SIZE); +#endif + }(); + return page_size; +} + +/** + * Alligns `offset` to the operating's system page size such that it subtracts the + * difference until the nearest page boundary before `offset`, or does nothing if + * `offset` is already page aligned. + */ +inline size_t make_offset_page_aligned(size_t offset) noexcept +{ + const size_t page_size_ = page_size(); + // Use integer division to round down to the nearest page alignment. + return offset / page_size_ * page_size_; +} + +} // namespace mio + +#endif // MIO_PAGE_HEADER + + +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif // WIN32_LEAN_AND_MEAN +# include +#else // ifdef _WIN32 +# define INVALID_HANDLE_VALUE -1 +#endif // ifdef _WIN32 + +namespace mio { + +// This value may be provided as the `length` parameter to the constructor or +// `map`, in which case a memory mapping of the entire file is created. +enum { map_entire_file = 0 }; + +#ifdef _WIN32 +using file_handle_type = HANDLE; +#else +using file_handle_type = int; +#endif + +// This value represents an invalid file handle type. This can be used to +// determine whether `basic_mmap::file_handle` is valid, for example. +const static file_handle_type invalid_handle = INVALID_HANDLE_VALUE; + +template +struct basic_mmap +{ + using value_type = ByteT; + using size_type = size_t; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using difference_type = std::ptrdiff_t; + using iterator = pointer; + using const_iterator = const_pointer; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using iterator_category = std::random_access_iterator_tag; + using handle_type = file_handle_type; + + static_assert(sizeof(ByteT) == sizeof(char), "ByteT must be the same size as char."); + +private: + // Points to the first requested byte, and not to the actual start of the mapping. + pointer data_ = nullptr; + + // Length--in bytes--requested by user (which may not be the length of the + // full mapping) and the length of the full mapping. + size_type length_ = 0; + size_type mapped_length_ = 0; + + // Letting user map a file using both an existing file handle and a path + // introcudes some complexity (see `is_handle_internal_`). + // On POSIX, we only need a file handle to create a mapping, while on + // Windows systems the file handle is necessary to retrieve a file mapping + // handle, but any subsequent operations on the mapped region must be done + // through the latter. + handle_type file_handle_ = INVALID_HANDLE_VALUE; +#ifdef _WIN32 + handle_type file_mapping_handle_ = INVALID_HANDLE_VALUE; +#endif + + // Letting user map a file using both an existing file handle and a path + // introcudes some complexity in that we must not close the file handle if + // user provided it, but we must close it if we obtained it using the + // provided path. For this reason, this flag is used to determine when to + // close `file_handle_`. + bool is_handle_internal_; + +public: + /** + * The default constructed mmap object is in a non-mapped state, that is, + * any operation that attempts to access nonexistent underlying data will + * result in undefined behaviour/segmentation faults. + */ + basic_mmap() = default; + +#ifdef __cpp_exceptions + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + template + basic_mmap(const String& path, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(path, offset, length, error); + if(error) { throw std::system_error(error); } + } + + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + basic_mmap(const handle_type handle, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(handle, offset, length, error); + if(error) { throw std::system_error(error); } + } +#endif // __cpp_exceptions + + /** + * `basic_mmap` has single-ownership semantics, so transferring ownership + * may only be accomplished by moving the object. + */ + basic_mmap(const basic_mmap&) = delete; + basic_mmap(basic_mmap&&); + basic_mmap& operator=(const basic_mmap&) = delete; + basic_mmap& operator=(basic_mmap&&); + + /** + * If this is a read-write mapping, the destructor invokes sync. Regardless + * of the access mode, unmap is invoked as a final step. + */ + ~basic_mmap(); + + /** + * On UNIX systems 'file_handle' and 'mapping_handle' are the same. On Windows, + * however, a mapped region of a file gets its own handle, which is returned by + * 'mapping_handle'. + */ + handle_type file_handle() const noexcept { return file_handle_; } + handle_type mapping_handle() const noexcept; + + /** Returns whether a valid memory mapping has been created. */ + bool is_open() const noexcept { return file_handle_ != invalid_handle; } + + /** + * Returns true if no mapping was established, that is, conceptually the + * same as though the length that was mapped was 0. This function is + * provided so that this class has Container semantics. + */ + bool empty() const noexcept { return length() == 0; } + + /** Returns true if a mapping was established. */ + bool is_mapped() const noexcept; + + /** + * `size` and `length` both return the logical length, i.e. the number of bytes + * user requested to be mapped, while `mapped_length` returns the actual number of + * bytes that were mapped which is a multiple of the underlying operating system's + * page allocation granularity. + */ + size_type size() const noexcept { return length(); } + size_type length() const noexcept { return length_; } + size_type mapped_length() const noexcept { return mapped_length_; } + + /** Returns the offset relative to the start of the mapping. */ + size_type mapping_offset() const noexcept + { + return mapped_length_ - length_; + } + + /** + * Returns a pointer to the first requested byte, or `nullptr` if no memory mapping + * exists. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer data() noexcept { return data_; } + const_pointer data() const noexcept { return data_; } + + /** + * Returns an iterator to the first requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator begin() noexcept { return data(); } + const_iterator begin() const noexcept { return data(); } + const_iterator cbegin() const noexcept { return data(); } + + /** + * Returns an iterator one past the last requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator end() noexcept { return data() + length(); } + const_iterator end() const noexcept { return data() + length(); } + const_iterator cend() const noexcept { return data() + length(); } + + /** + * Returns a reverse iterator to the last memory mapped byte, if a valid + * memory mapping exists, otherwise this function call is undefined + * behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rbegin() noexcept { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const noexcept + { return const_reverse_iterator(end()); } + const_reverse_iterator crbegin() const noexcept + { return const_reverse_iterator(end()); } + + /** + * Returns a reverse iterator past the first mapped byte, if a valid memory + * mapping exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rend() noexcept { return reverse_iterator(begin()); } + const_reverse_iterator rend() const noexcept + { return const_reverse_iterator(begin()); } + const_reverse_iterator crend() const noexcept + { return const_reverse_iterator(begin()); } + + /** + * Returns a reference to the `i`th byte from the first requested byte (as returned + * by `data`). If this is invoked when no valid memory mapping has been created + * prior to this call, undefined behaviour ensues. + */ + reference operator[](const size_type i) noexcept { return data_[i]; } + const_reference operator[](const size_type i) const noexcept { return data_[i]; } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + template + void map(const String& path, const size_type offset, + const size_type length, std::error_code& error); + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * The entire file is mapped. + */ + template + void map(const String& path, std::error_code& error) + { + map(path, 0, map_entire_file, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is + * unsuccesful, the reason is reported via `error` and the object remains in + * a state as if this function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + void map(const handle_type handle, const size_type offset, + const size_type length, std::error_code& error); + + /** + * Establishes a memory mapping with AccessMode. If the mapping is + * unsuccesful, the reason is reported via `error` and the object remains in + * a state as if this function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * The entire file is mapped. + */ + void map(const handle_type handle, std::error_code& error) + { + map(handle, 0, map_entire_file, error); + } + + /** + * If a valid memory mapping has been created prior to this call, this call + * instructs the kernel to unmap the memory region and disassociate this object + * from the file. + * + * The file handle associated with the file that is mapped is only closed if the + * mapping was created using a file path. If, on the other hand, an existing + * file handle was used to create the mapping, the file handle is not closed. + */ + void unmap(); + + void swap(basic_mmap& other); + + /** Flushes the memory mapped page to disk. Errors are reported via `error`. */ + template + typename std::enable_if::type + sync(std::error_code& error); + + /** + * All operators compare the address of the first byte and size of the two mapped + * regions. + */ + +private: + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer get_mapping_start() noexcept + { + return !data() ? nullptr : data() - mapping_offset(); + } + + const_pointer get_mapping_start() const noexcept + { + return !data() ? nullptr : data() - mapping_offset(); + } + + /** + * The destructor syncs changes to disk if `AccessMode` is `write`, but not + * if it's `read`, but since the destructor cannot be templated, we need to + * do SFINAE in a dedicated function, where one syncs and the other is a noop. + */ + template + typename std::enable_if::type + conditional_sync(); + template + typename std::enable_if::type conditional_sync(); +}; + +template +bool operator==(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator!=(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator<(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator<=(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator>(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator>=(const basic_mmap& a, + const basic_mmap& b); + +/** + * This is the basis for all read-only mmap objects and should be preferred over + * directly using `basic_mmap`. + */ +template +using basic_mmap_source = basic_mmap; + +/** + * This is the basis for all read-write mmap objects and should be preferred over + * directly using `basic_mmap`. + */ +template +using basic_mmap_sink = basic_mmap; + +/** + * These aliases cover the most common use cases, both representing a raw byte stream + * (either with a char or an unsigned char/uint8_t). + */ +using mmap_source = basic_mmap_source; +using ummap_source = basic_mmap_source; + +using mmap_sink = basic_mmap_sink; +using ummap_sink = basic_mmap_sink; + +/** + * Convenience factory method that constructs a mapping for any `basic_mmap` or + * `basic_mmap` type. + */ +template< + typename MMap, + typename MappingToken +> MMap make_mmap(const MappingToken& token, + int64_t offset, int64_t length, std::error_code& error) +{ + MMap mmap; + mmap.map(token, offset, length, error); + return mmap; +} + +/** + * Convenience factory method. + * + * MappingToken may be a String (`std::string`, `std::string_view`, `const char*`, + * `std::filesystem::path`, `std::vector`, or similar), or a + * `mmap_source::handle_type`. + */ +template +mmap_source make_mmap_source(const MappingToken& token, mmap_source::size_type offset, + mmap_source::size_type length, std::error_code& error) +{ + return make_mmap(token, offset, length, error); +} + +template +mmap_source make_mmap_source(const MappingToken& token, std::error_code& error) +{ + return make_mmap_source(token, 0, map_entire_file, error); +} + +/** + * Convenience factory method. + * + * MappingToken may be a String (`std::string`, `std::string_view`, `const char*`, + * `std::filesystem::path`, `std::vector`, or similar), or a + * `mmap_sink::handle_type`. + */ +template +mmap_sink make_mmap_sink(const MappingToken& token, mmap_sink::size_type offset, + mmap_sink::size_type length, std::error_code& error) +{ + return make_mmap(token, offset, length, error); +} + +template +mmap_sink make_mmap_sink(const MappingToken& token, std::error_code& error) +{ + return make_mmap_sink(token, 0, map_entire_file, error); +} + +} // namespace mio + +// #include "detail/mmap.ipp" +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_BASIC_MMAP_IMPL +#define MIO_BASIC_MMAP_IMPL + +// #include "mio/mmap.hpp" + +// #include "mio/page.hpp" + +// #include "mio/detail/string_util.hpp" +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_STRING_UTIL_HEADER +#define MIO_STRING_UTIL_HEADER + +#include + +namespace mio { +namespace detail { + +template< + typename S, + typename C = typename std::decay::type, + typename = decltype(std::declval().data()), + typename = typename std::enable_if< + std::is_same::value +#ifdef _WIN32 + || std::is_same::value +#endif + >::type +> struct char_type_helper { + using type = typename C::value_type; +}; + +template +struct char_type { + using type = typename char_type_helper::type; +}; + +// TODO: can we avoid this brute force approach? +template<> +struct char_type { + using type = char; +}; + +template<> +struct char_type { + using type = char; +}; + +template +struct char_type { + using type = char; +}; + +template +struct char_type { + using type = char; +}; + +#ifdef _WIN32 +template<> +struct char_type { + using type = wchar_t; +}; + +template<> +struct char_type { + using type = wchar_t; +}; + +template +struct char_type { + using type = wchar_t; +}; + +template +struct char_type { + using type = wchar_t; +}; +#endif // _WIN32 + +template +struct is_c_str_helper +{ + static constexpr bool value = std::is_same< + CharT*, + // TODO: I'm so sorry for this... Can this be made cleaner? + typename std::add_pointer< + typename std::remove_cv< + typename std::remove_pointer< + typename std::decay< + S + >::type + >::type + >::type + >::type + >::value; +}; + +template +struct is_c_str +{ + static constexpr bool value = is_c_str_helper::value; +}; + +#ifdef _WIN32 +template +struct is_c_wstr +{ + static constexpr bool value = is_c_str_helper::value; +}; +#endif // _WIN32 + +template +struct is_c_str_or_c_wstr +{ + static constexpr bool value = is_c_str::value +#ifdef _WIN32 + || is_c_wstr::value +#endif + ; +}; + +template< + typename String, + typename = decltype(std::declval().data()), + typename = typename std::enable_if::value>::type +> const typename char_type::type* c_str(const String& path) +{ + return path.data(); +} + +template< + typename String, + typename = decltype(std::declval().empty()), + typename = typename std::enable_if::value>::type +> bool empty(const String& path) +{ + return path.empty(); +} + +template< + typename String, + typename = typename std::enable_if::value>::type +> const typename char_type::type* c_str(String path) +{ + return path; +} + +template< + typename String, + typename = typename std::enable_if::value>::type +> bool empty(String path) +{ + return !path || (*path == 0); +} + +} // namespace detail +} // namespace mio + +#endif // MIO_STRING_UTIL_HEADER + + +#include + +#ifndef _WIN32 +# include +# include +# include +# include +#endif + +namespace mio { +namespace detail { + +#ifdef _WIN32 +namespace win { + +/** Returns the 4 upper bytes of an 8-byte integer. */ +inline DWORD int64_high(int64_t n) noexcept +{ + return n >> 32; +} + +/** Returns the 4 lower bytes of an 8-byte integer. */ +inline DWORD int64_low(int64_t n) noexcept +{ + return n & 0xffffffff; +} + +inline std::wstring s_2_ws(const std::string& s) +{ + std::wstring ret; + if (!s.empty()) + { + ret.resize(s.size()); + int wide_char_count = MultiByteToWideChar(CP_UTF8, 0, s.c_str(), + static_cast(s.size()), &ret[0], static_cast(s.size())); + ret.resize(wide_char_count); + } + return ret; +} + +template< + typename String, + typename = typename std::enable_if< + std::is_same::type, char>::value + >::type +> file_handle_type open_file_helper(const String& path, const access_mode mode) +{ + return ::CreateFileW(s_2_ws(path).c_str(), + mode == access_mode::read ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + 0, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + 0); +} + +template +typename std::enable_if< + std::is_same::type, wchar_t>::value, + file_handle_type +>::type open_file_helper(const String& path, const access_mode mode) +{ + return ::CreateFileW(c_str(path), + mode == access_mode::read ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + 0, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + 0); +} + +} // win +#endif // _WIN32 + +/** + * Returns the last platform specific system error (errno on POSIX and + * GetLastError on Win) as a `std::error_code`. + */ +inline std::error_code last_error() noexcept +{ + std::error_code error; +#ifdef _WIN32 + error.assign(GetLastError(), std::system_category()); +#else + error.assign(errno, std::system_category()); +#endif + return error; +} + +template +file_handle_type open_file(const String& path, const access_mode mode, + std::error_code& error) +{ + error.clear(); + if(detail::empty(path)) + { + error = std::make_error_code(std::errc::invalid_argument); + return invalid_handle; + } +#ifdef _WIN32 + const auto handle = win::open_file_helper(path, mode); +#else // POSIX + const auto handle = ::open(c_str(path), + mode == access_mode::read ? O_RDONLY : O_RDWR); +#endif + if(handle == invalid_handle) + { + error = detail::last_error(); + } + return handle; +} + +inline size_t query_file_size(file_handle_type handle, std::error_code& error) +{ + error.clear(); +#ifdef _WIN32 + LARGE_INTEGER file_size; + if(::GetFileSizeEx(handle, &file_size) == 0) + { + error = detail::last_error(); + return 0; + } + return static_cast(file_size.QuadPart); +#else // POSIX + struct stat sbuf; + if(::fstat(handle, &sbuf) == -1) + { + error = detail::last_error(); + return 0; + } + return sbuf.st_size; +#endif +} + +struct mmap_context +{ + char* data; + int64_t length; + int64_t mapped_length; +#ifdef _WIN32 + file_handle_type file_mapping_handle; +#endif +}; + +inline mmap_context memory_map(const file_handle_type file_handle, const int64_t offset, + const int64_t length, const access_mode mode, std::error_code& error) +{ + const int64_t aligned_offset = make_offset_page_aligned(offset); + const int64_t length_to_map = offset - aligned_offset + length; +#ifdef _WIN32 + const int64_t max_file_size = offset + length; + const auto file_mapping_handle = ::CreateFileMapping( + file_handle, + 0, + mode == access_mode::read ? PAGE_READONLY : PAGE_READWRITE, + win::int64_high(max_file_size), + win::int64_low(max_file_size), + 0); + if(file_mapping_handle == invalid_handle) + { + error = detail::last_error(); + return {}; + } + char* mapping_start = static_cast(::MapViewOfFile( + file_mapping_handle, + mode == access_mode::read ? FILE_MAP_READ : FILE_MAP_WRITE, + win::int64_high(aligned_offset), + win::int64_low(aligned_offset), + length_to_map)); + if(mapping_start == nullptr) + { + // Close file handle if mapping it failed. + ::CloseHandle(file_mapping_handle); + error = detail::last_error(); + return {}; + } +#else // POSIX + char* mapping_start = static_cast(::mmap( + 0, // Don't give hint as to where to map. + length_to_map, + mode == access_mode::read ? PROT_READ : PROT_WRITE, + MAP_SHARED, + file_handle, + aligned_offset)); + if(mapping_start == MAP_FAILED) + { + error = detail::last_error(); + return {}; + } +#endif + mmap_context ctx; + ctx.data = mapping_start + offset - aligned_offset; + ctx.length = length; + ctx.mapped_length = length_to_map; +#ifdef _WIN32 + ctx.file_mapping_handle = file_mapping_handle; +#endif + return ctx; +} + +} // namespace detail + +// -- basic_mmap -- + +template +basic_mmap::~basic_mmap() +{ + conditional_sync(); + unmap(); +} + +template +basic_mmap::basic_mmap(basic_mmap&& other) + : data_(std::move(other.data_)) + , length_(std::move(other.length_)) + , mapped_length_(std::move(other.mapped_length_)) + , file_handle_(std::move(other.file_handle_)) +#ifdef _WIN32 + , file_mapping_handle_(std::move(other.file_mapping_handle_)) +#endif + , is_handle_internal_(std::move(other.is_handle_internal_)) +{ + other.data_ = nullptr; + other.length_ = other.mapped_length_ = 0; + other.file_handle_ = invalid_handle; +#ifdef _WIN32 + other.file_mapping_handle_ = invalid_handle; +#endif +} + +template +basic_mmap& +basic_mmap::operator=(basic_mmap&& other) +{ + if(this != &other) + { + // First the existing mapping needs to be removed. + unmap(); + data_ = std::move(other.data_); + length_ = std::move(other.length_); + mapped_length_ = std::move(other.mapped_length_); + file_handle_ = std::move(other.file_handle_); +#ifdef _WIN32 + file_mapping_handle_ = std::move(other.file_mapping_handle_); +#endif + is_handle_internal_ = std::move(other.is_handle_internal_); + + // The moved from basic_mmap's fields need to be reset, because + // otherwise other's destructor will unmap the same mapping that was + // just moved into this. + other.data_ = nullptr; + other.length_ = other.mapped_length_ = 0; + other.file_handle_ = invalid_handle; +#ifdef _WIN32 + other.file_mapping_handle_ = invalid_handle; +#endif + other.is_handle_internal_ = false; + } + return *this; +} + +template +typename basic_mmap::handle_type +basic_mmap::mapping_handle() const noexcept +{ +#ifdef _WIN32 + return file_mapping_handle_; +#else + return file_handle_; +#endif +} + +template +template +void basic_mmap::map(const String& path, const size_type offset, + const size_type length, std::error_code& error) +{ + error.clear(); + if(detail::empty(path)) + { + error = std::make_error_code(std::errc::invalid_argument); + return; + } + const auto handle = detail::open_file(path, AccessMode, error); + if(error) + { + return; + } + + map(handle, offset, length, error); + // This MUST be after the call to map, as that sets this to true. + if(!error) + { + is_handle_internal_ = true; + } +} + +template +void basic_mmap::map(const handle_type handle, + const size_type offset, const size_type length, std::error_code& error) +{ + error.clear(); + if(handle == invalid_handle) + { + error = std::make_error_code(std::errc::bad_file_descriptor); + return; + } + + const auto file_size = detail::query_file_size(handle, error); + if(error) + { + return; + } + + if(offset + length > file_size) + { + error = std::make_error_code(std::errc::invalid_argument); + return; + } + + const auto ctx = detail::memory_map(handle, offset, + length == map_entire_file ? (file_size - offset) : length, + AccessMode, error); + if(!error) + { + // We must unmap the previous mapping that may have existed prior to this call. + // Note that this must only be invoked after a new mapping has been created in + // order to provide the strong guarantee that, should the new mapping fail, the + // `map` function leaves this instance in a state as though the function had + // never been invoked. + unmap(); + file_handle_ = handle; + is_handle_internal_ = false; + data_ = reinterpret_cast(ctx.data); + length_ = ctx.length; + mapped_length_ = ctx.mapped_length; +#ifdef _WIN32 + file_mapping_handle_ = ctx.file_mapping_handle; +#endif + } +} + +template +template +typename std::enable_if::type +basic_mmap::sync(std::error_code& error) +{ + error.clear(); + if(!is_open()) + { + error = std::make_error_code(std::errc::bad_file_descriptor); + return; + } + + if(data()) + { +#ifdef _WIN32 + if(::FlushViewOfFile(get_mapping_start(), mapped_length_) == 0 + || ::FlushFileBuffers(file_handle_) == 0) +#else // POSIX + if(::msync(get_mapping_start(), mapped_length_, MS_SYNC) != 0) +#endif + { + error = detail::last_error(); + return; + } + } +#ifdef _WIN32 + if(::FlushFileBuffers(file_handle_) == 0) + { + error = detail::last_error(); + } +#endif +} + +template +void basic_mmap::unmap() +{ + if(!is_open()) { return; } + // TODO do we care about errors here? +#ifdef _WIN32 + if(is_mapped()) + { + ::UnmapViewOfFile(get_mapping_start()); + ::CloseHandle(file_mapping_handle_); + } +#else // POSIX + if(data_) { ::munmap(const_cast(get_mapping_start()), mapped_length_); } +#endif + + // If `file_handle_` was obtained by our opening it (when map is called with + // a path, rather than an existing file handle), we need to close it, + // otherwise it must not be closed as it may still be used outside this + // instance. + if(is_handle_internal_) + { +#ifdef _WIN32 + ::CloseHandle(file_handle_); +#else // POSIX + ::close(file_handle_); +#endif + } + + // Reset fields to their default values. + data_ = nullptr; + length_ = mapped_length_ = 0; + file_handle_ = invalid_handle; +#ifdef _WIN32 + file_mapping_handle_ = invalid_handle; +#endif +} + +template +bool basic_mmap::is_mapped() const noexcept +{ +#ifdef _WIN32 + return file_mapping_handle_ != invalid_handle; +#else // POSIX + return is_open(); +#endif +} + +template +void basic_mmap::swap(basic_mmap& other) +{ + if(this != &other) + { + using std::swap; + swap(data_, other.data_); + swap(file_handle_, other.file_handle_); +#ifdef _WIN32 + swap(file_mapping_handle_, other.file_mapping_handle_); +#endif + swap(length_, other.length_); + swap(mapped_length_, other.mapped_length_); + swap(is_handle_internal_, other.is_handle_internal_); + } +} + +template +template +typename std::enable_if::type +basic_mmap::conditional_sync() +{ + // This is invoked from the destructor, so not much we can do about + // failures here. + std::error_code ec; + sync(ec); +} + +template +template +typename std::enable_if::type +basic_mmap::conditional_sync() +{ + // noop +} + +template +bool operator==(const basic_mmap& a, + const basic_mmap& b) +{ + return a.data() == b.data() + && a.size() == b.size(); +} + +template +bool operator!=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a == b); +} + +template +bool operator<(const basic_mmap& a, + const basic_mmap& b) +{ + if(a.data() == b.data()) { return a.size() < b.size(); } + return a.data() < b.data(); +} + +template +bool operator<=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a > b); +} + +template +bool operator>(const basic_mmap& a, + const basic_mmap& b) +{ + if(a.data() == b.data()) { return a.size() > b.size(); } + return a.data() > b.data(); +} + +template +bool operator>=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a < b); +} + +} // namespace mio + +#endif // MIO_BASIC_MMAP_IMPL + + +#endif // MIO_MMAP_HEADER +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_PAGE_HEADER +#define MIO_PAGE_HEADER + +#ifdef _WIN32 +# include +#else +# include +#endif + +namespace mio { + +/** + * This is used by `basic_mmap` to determine whether to create a read-only or + * a read-write memory mapping. + */ +enum class access_mode +{ + read, + write +}; + +/** + * Determines the operating system's page allocation granularity. + * + * On the first call to this function, it invokes the operating system specific syscall + * to determine the page size, caches the value, and returns it. Any subsequent call to + * this function serves the cached value, so no further syscalls are made. + */ +inline size_t page_size() +{ + static const size_t page_size = [] + { +#ifdef _WIN32 + SYSTEM_INFO SystemInfo; + GetSystemInfo(&SystemInfo); + return SystemInfo.dwAllocationGranularity; +#else + return sysconf(_SC_PAGE_SIZE); +#endif + }(); + return page_size; +} + +/** + * Alligns `offset` to the operating's system page size such that it subtracts the + * difference until the nearest page boundary before `offset`, or does nothing if + * `offset` is already page aligned. + */ +inline size_t make_offset_page_aligned(size_t offset) noexcept +{ + const size_t page_size_ = page_size(); + // Use integer division to round down to the nearest page alignment. + return offset / page_size_ * page_size_; +} + +} // namespace mio + +#endif // MIO_PAGE_HEADER +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_SHARED_MMAP_HEADER +#define MIO_SHARED_MMAP_HEADER + +// #include "mio/mmap.hpp" + + +#include // std::error_code +#include // std::shared_ptr + +namespace mio { + +/** + * Exposes (nearly) the same interface as `basic_mmap`, but endowes it with + * `std::shared_ptr` semantics. + * + * This is not the default behaviour of `basic_mmap` to avoid allocating on the heap if + * shared semantics are not required. + */ +template< + access_mode AccessMode, + typename ByteT +> class basic_shared_mmap +{ + using impl_type = basic_mmap; + std::shared_ptr pimpl_; + +public: + using value_type = typename impl_type::value_type; + using size_type = typename impl_type::size_type; + using reference = typename impl_type::reference; + using const_reference = typename impl_type::const_reference; + using pointer = typename impl_type::pointer; + using const_pointer = typename impl_type::const_pointer; + using difference_type = typename impl_type::difference_type; + using iterator = typename impl_type::iterator; + using const_iterator = typename impl_type::const_iterator; + using reverse_iterator = typename impl_type::reverse_iterator; + using const_reverse_iterator = typename impl_type::const_reverse_iterator; + using iterator_category = typename impl_type::iterator_category; + using handle_type = typename impl_type::handle_type; + using mmap_type = impl_type; + + basic_shared_mmap() = default; + basic_shared_mmap(const basic_shared_mmap&) = default; + basic_shared_mmap& operator=(const basic_shared_mmap&) = default; + basic_shared_mmap(basic_shared_mmap&&) = default; + basic_shared_mmap& operator=(basic_shared_mmap&&) = default; + + /** Takes ownership of an existing mmap object. */ + basic_shared_mmap(mmap_type&& mmap) + : pimpl_(std::make_shared(std::move(mmap))) + {} + + /** Takes ownership of an existing mmap object. */ + basic_shared_mmap& operator=(mmap_type&& mmap) + { + pimpl_ = std::make_shared(std::move(mmap)); + return *this; + } + + /** Initializes this object with an already established shared mmap. */ + basic_shared_mmap(std::shared_ptr mmap) : pimpl_(std::move(mmap)) {} + + /** Initializes this object with an already established shared mmap. */ + basic_shared_mmap& operator=(std::shared_ptr mmap) + { + pimpl_ = std::move(mmap); + return *this; + } + +#ifdef __cpp_exceptions + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + template + basic_shared_mmap(const String& path, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(path, offset, length, error); + if(error) { throw std::system_error(error); } + } + + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + basic_shared_mmap(const handle_type handle, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(handle, offset, length, error); + if(error) { throw std::system_error(error); } + } +#endif // __cpp_exceptions + + /** + * If this is a read-write mapping and the last reference to the mapping, + * the destructor invokes sync. Regardless of the access mode, unmap is + * invoked as a final step. + */ + ~basic_shared_mmap() = default; + + /** Returns the underlying `std::shared_ptr` instance that holds the mmap. */ + std::shared_ptr get_shared_ptr() { return pimpl_; } + + /** + * On UNIX systems 'file_handle' and 'mapping_handle' are the same. On Windows, + * however, a mapped region of a file gets its own handle, which is returned by + * 'mapping_handle'. + */ + handle_type file_handle() const noexcept + { + return pimpl_ ? pimpl_->file_handle() : invalid_handle; + } + + handle_type mapping_handle() const noexcept + { + return pimpl_ ? pimpl_->mapping_handle() : invalid_handle; + } + + /** Returns whether a valid memory mapping has been created. */ + bool is_open() const noexcept { return pimpl_ && pimpl_->is_open(); } + + /** + * Returns true if no mapping was established, that is, conceptually the + * same as though the length that was mapped was 0. This function is + * provided so that this class has Container semantics. + */ + bool empty() const noexcept { return !pimpl_ || pimpl_->empty(); } + + /** + * `size` and `length` both return the logical length, i.e. the number of bytes + * user requested to be mapped, while `mapped_length` returns the actual number of + * bytes that were mapped which is a multiple of the underlying operating system's + * page allocation granularity. + */ + size_type size() const noexcept { return pimpl_ ? pimpl_->length() : 0; } + size_type length() const noexcept { return pimpl_ ? pimpl_->length() : 0; } + size_type mapped_length() const noexcept + { + return pimpl_ ? pimpl_->mapped_length() : 0; + } + + /** + * Returns a pointer to the first requested byte, or `nullptr` if no memory mapping + * exists. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer data() noexcept { return pimpl_->data(); } + const_pointer data() const noexcept { return pimpl_ ? pimpl_->data() : nullptr; } + + /** + * Returns an iterator to the first requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + iterator begin() noexcept { return pimpl_->begin(); } + const_iterator begin() const noexcept { return pimpl_->begin(); } + const_iterator cbegin() const noexcept { return pimpl_->cbegin(); } + + /** + * Returns an iterator one past the last requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator end() noexcept { return pimpl_->end(); } + const_iterator end() const noexcept { return pimpl_->end(); } + const_iterator cend() const noexcept { return pimpl_->cend(); } + + /** + * Returns a reverse iterator to the last memory mapped byte, if a valid + * memory mapping exists, otherwise this function call is undefined + * behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rbegin() noexcept { return pimpl_->rbegin(); } + const_reverse_iterator rbegin() const noexcept { return pimpl_->rbegin(); } + const_reverse_iterator crbegin() const noexcept { return pimpl_->crbegin(); } + + /** + * Returns a reverse iterator past the first mapped byte, if a valid memory + * mapping exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rend() noexcept { return pimpl_->rend(); } + const_reverse_iterator rend() const noexcept { return pimpl_->rend(); } + const_reverse_iterator crend() const noexcept { return pimpl_->crend(); } + + /** + * Returns a reference to the `i`th byte from the first requested byte (as returned + * by `data`). If this is invoked when no valid memory mapping has been created + * prior to this call, undefined behaviour ensues. + */ + reference operator[](const size_type i) noexcept { return (*pimpl_)[i]; } + const_reference operator[](const size_type i) const noexcept { return (*pimpl_)[i]; } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + template + void map(const String& path, const size_type offset, + const size_type length, std::error_code& error) + { + map_impl(path, offset, length, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * The entire file is mapped. + */ + template + void map(const String& path, std::error_code& error) + { + map_impl(path, 0, map_entire_file, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + void map(const handle_type handle, const size_type offset, + const size_type length, std::error_code& error) + { + map_impl(handle, offset, length, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * The entire file is mapped. + */ + void map(const handle_type handle, std::error_code& error) + { + map_impl(handle, 0, map_entire_file, error); + } + + /** + * If a valid memory mapping has been created prior to this call, this call + * instructs the kernel to unmap the memory region and disassociate this object + * from the file. + * + * The file handle associated with the file that is mapped is only closed if the + * mapping was created using a file path. If, on the other hand, an existing + * file handle was used to create the mapping, the file handle is not closed. + */ + void unmap() { if(pimpl_) pimpl_->unmap(); } + + void swap(basic_shared_mmap& other) { pimpl_.swap(other.pimpl_); } + + /** Flushes the memory mapped page to disk. Errors are reported via `error`. */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > void sync(std::error_code& error) { if(pimpl_) pimpl_->sync(error); } + + /** All operators compare the underlying `basic_mmap`'s addresses. */ + + friend bool operator==(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ == b.pimpl_; + } + + friend bool operator!=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return !(a == b); + } + + friend bool operator<(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ < b.pimpl_; + } + + friend bool operator<=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ <= b.pimpl_; + } + + friend bool operator>(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ > b.pimpl_; + } + + friend bool operator>=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ >= b.pimpl_; + } + +private: + template + void map_impl(const MappingToken& token, const size_type offset, + const size_type length, std::error_code& error) + { + if(!pimpl_) + { + mmap_type mmap = make_mmap(token, offset, length, error); + if(error) { return; } + pimpl_ = std::make_shared(std::move(mmap)); + } + else + { + pimpl_->map(token, offset, length, error); + } + } +}; + +/** + * This is the basis for all read-only mmap objects and should be preferred over + * directly using basic_shared_mmap. + */ +template +using basic_shared_mmap_source = basic_shared_mmap; + +/** + * This is the basis for all read-write mmap objects and should be preferred over + * directly using basic_shared_mmap. + */ +template +using basic_shared_mmap_sink = basic_shared_mmap; + +/** + * These aliases cover the most common use cases, both representing a raw byte stream + * (either with a char or an unsigned char/uint8_t). + */ +using shared_mmap_source = basic_shared_mmap_source; +using shared_ummap_source = basic_shared_mmap_source; + +using shared_mmap_sink = basic_shared_mmap_sink; +using shared_ummap_sink = basic_shared_mmap_sink; + +} // namespace mio + +#endif // MIO_SHARED_MMAP_HEADER diff --git a/vendor/mio/test/CMakeLists.txt b/vendor/mio/test/CMakeLists.txt new file mode 100644 index 0000000..652ede4 --- /dev/null +++ b/vendor/mio/test/CMakeLists.txt @@ -0,0 +1,25 @@ +configure_file( + "${PROJECT_SOURCE_DIR}/cmake/CTestCustom.cmake" + "${PROJECT_BINARY_DIR}/CTestCustom.cmake" + COPYONLY) + +add_executable(mio.test test.cpp) +target_link_libraries(mio.test PRIVATE mio::mio) +add_test(NAME mio.test COMMAND mio.test) + +if(WIN32) + add_executable(mio.unicode.test test.cpp) + target_link_libraries(mio.unicode.test PRIVATE mio::mio) + target_compile_definitions(mio.unicode.test PRIVATE UNICODE) + add_test(NAME mio.unicode.test COMMAND mio.test) + + add_executable(mio.fullwinapi.test test.cpp) + target_link_libraries(mio.fullwinapi.test + PRIVATE mio::mio_full_winapi) + add_test(NAME mio.fullwinapi.test COMMAND mio.fullwinapi.test) + + add_executable(mio.minwinapi.test test.cpp) + target_link_libraries(mio.minwinapi.test + PRIVATE mio::mio_min_winapi) + add_test(NAME mio.minwinapi.test COMMAND mio.minwinapi.test) +endif() diff --git a/vendor/mio/test/example.cpp b/vendor/mio/test/example.cpp new file mode 100644 index 0000000..841a57f --- /dev/null +++ b/vendor/mio/test/example.cpp @@ -0,0 +1,75 @@ +#include +#include // for std::error_code +#include // for std::printf +#include +#include +#include + +int handle_error(const std::error_code& error); +void allocate_file(const std::string& path, const int size); + +int main() +{ + const auto path = "file.txt"; + + // NOTE: mio does *not* create the file for you if it doesn't exist! You + // must ensure that the file exists before establishing a mapping. It + // must also be non-empty. So for illustrative purposes the file is + // created now. + allocate_file(path, 155); + + // Read-write memory map the whole file by using `map_entire_file` where the + // length of the mapping is otherwise expected, with the factory method. + std::error_code error; + mio::mmap_sink rw_mmap = mio::make_mmap_sink( + path, 0, mio::map_entire_file, error); + if (error) { return handle_error(error); } + + // You can use any iterator based function. + std::fill(rw_mmap.begin(), rw_mmap.end(), 'a'); + + // Or manually iterate through the mapped region just as if it were any other + // container, and change each byte's value (since this is a read-write mapping). + for (auto& b : rw_mmap) { + b += 10; + } + + // Or just change one value with the subscript operator. + const int answer_index = rw_mmap.size() / 2; + rw_mmap[answer_index] = 42; + + // Don't forget to flush changes to disk before unmapping. However, if + // `rw_mmap` were to go out of scope at this point, the destructor would also + // automatically invoke `sync` before `unmap`. + rw_mmap.sync(error); + if (error) { return handle_error(error); } + + // We can then remove the mapping, after which rw_mmap will be in a default + // constructed state, i.e. this and the above call to `sync` have the same + // effect as if the destructor had been invoked. + rw_mmap.unmap(); + + // Now create the same mapping, but in read-only mode. Note that calling the + // overload without the offset and file length parameters maps the entire + // file. + mio::mmap_source ro_mmap; + ro_mmap.map(path, error); + if (error) { return handle_error(error); } + + const int the_answer_to_everything = ro_mmap[answer_index]; + assert(the_answer_to_everything == 42); +} + +int handle_error(const std::error_code& error) +{ + const auto& errmsg = error.message(); + std::printf("error mapping file: %s, exiting...\n", errmsg.c_str()); + return error.value(); +} + +void allocate_file(const std::string& path, const int size) +{ + std::ofstream file(path); + std::string s(size, '0'); + file << s; +} diff --git a/vendor/mio/test/test.cpp b/vendor/mio/test/test.cpp new file mode 100644 index 0000000..82827f9 --- /dev/null +++ b/vendor/mio/test/test.cpp @@ -0,0 +1,182 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#include +#include +#endif + +// Just make sure this compiles. +#ifdef CXX17 +# include +using mmap_source = mio::basic_mmap_source; +#endif + +template +void test_at_offset(const MMap& file_view, const std::string& buffer, + const size_t offset); +void test_at_offset(const std::string& buffer, const char* path, + const size_t offset, std::error_code& error); +int handle_error(const std::error_code& error); + +int main() +{ + std::error_code error; + + // Make sure mio compiles with non-const char* strings too. + const char _path[] = "test-file"; + const int path_len = sizeof(_path); + char* path = new char[path_len]; + std::copy(_path, _path + path_len, path); + + const auto page_size = mio::page_size(); + // Fill buffer, then write it to file. + const int file_size = 4 * page_size - 250; // 16134, if page size is 4KiB + std::string buffer(file_size, 0); + // Start at first printable ASCII character. + char v = 33; + for (auto& b : buffer) { + b = v; + ++v; + // Limit to last printable ASCII character. + v %= 126; + if(v == 0) { + v = 33; + } + } + + std::ofstream file(path); + file << buffer; + file.close(); + + // Test whole file mapping. + test_at_offset(buffer, path, 0, error); + if (error) { return handle_error(error); } + + // Test starting from below the page size. + test_at_offset(buffer, path, page_size - 3, error); + if (error) { return handle_error(error); } + + // Test starting from above the page size. + test_at_offset(buffer, path, page_size + 3, error); + if (error) { return handle_error(error); } + + // Test starting from above the page size. + test_at_offset(buffer, path, 2 * page_size + 3, error); + if (error) { return handle_error(error); } + + { +#define CHECK_INVALID_MMAP(m) do { \ + assert(error); \ + assert(m.empty()); \ + assert(!m.is_open()); \ + error.clear(); } while(0) + + mio::mmap_source m; + + // See if mapping an invalid file results in an error. + m = mio::make_mmap_source("garbage-that-hopefully-doesnt-exist", 0, 0, error); + CHECK_INVALID_MMAP(m); + + // Empty path? + m = mio::make_mmap_source(static_cast(0), 0, 0, error); + CHECK_INVALID_MMAP(m); + m = mio::make_mmap_source(std::string(), 0, 0, error); + CHECK_INVALID_MMAP(m); + + // Invalid handle? + m = mio::make_mmap_source(mio::invalid_handle, 0, 0, error); + CHECK_INVALID_MMAP(m); + + // Invalid offset? + m = mio::make_mmap_source(path, 100 * buffer.size(), buffer.size(), error); + CHECK_INVALID_MMAP(m); + } + + // Make sure these compile. + { + mio::ummap_source _1; + mio::shared_ummap_source _2; + // Make sure shared_mmap mapping compiles as all testing was done on + // normal mmaps. + mio::shared_mmap_source _3(path, 0, mio::map_entire_file); + auto _4 = mio::make_mmap_source(path, error); + auto _5 = mio::make_mmap(path, 0, mio::map_entire_file, error); +#ifdef _WIN32 + const wchar_t* wpath1 = L"dasfsf"; + auto _6 = mio::make_mmap_source(wpath1, error); + mio::mmap_source _7; + _7.map(wpath1, error); + const std::wstring wpath2 = wpath1; + auto _8 = mio::make_mmap_source(wpath2, error); + mio::mmap_source _9; + _9.map(wpath1, error); +#else + const int fd = open(path, O_RDONLY); + mio::mmap_source _fdmmap(fd, 0, mio::map_entire_file); + _fdmmap.unmap(); + _fdmmap.map(fd, error); +#endif + } + + std::printf("all tests passed!\n"); +} + +void test_at_offset(const std::string& buffer, const char* path, + const size_t offset, std::error_code& error) +{ + // Sanity check. + assert(offset < buffer.size()); + + // Map the region of the file to which buffer was written. + mio::mmap_source file_view = mio::make_mmap_source( + path, offset, mio::map_entire_file, error); + if(error) { return; } + + assert(file_view.is_open()); + const size_t mapped_size = buffer.size() - offset; + assert(file_view.size() == mapped_size); + + test_at_offset(file_view, buffer, offset); + + // Turn file_view into a shared mmap. + mio::shared_mmap_source shared_file_view(std::move(file_view)); + assert(!file_view.is_open()); + assert(shared_file_view.is_open()); + assert(shared_file_view.size() == mapped_size); + + //test_at_offset(shared_file_view, buffer, offset); +} + +template +void test_at_offset(const MMap& file_view, const std::string& buffer, + const size_t offset) +{ + // Then verify that mmap's bytes correspond to that of buffer. + for(size_t buf_idx = offset, view_idx = 0; + buf_idx < buffer.size() && view_idx < file_view.size(); + ++buf_idx, ++view_idx) { + if(file_view[view_idx] != buffer[buf_idx]) { + std::printf("%luth byte mismatch: expected(%d) <> actual(%d)", + buf_idx, buffer[buf_idx], file_view[view_idx]); + std::cout << std::flush; + assert(0); + } + } +} + +int handle_error(const std::error_code& error) +{ + const auto& errmsg = error.message(); + std::printf("Error mapping file: %s, exiting...\n", errmsg.c_str()); + return error.value(); +} diff --git a/vendor/mio/third_party/LICENSE.md b/vendor/mio/third_party/LICENSE.md new file mode 100644 index 0000000..23edca5 --- /dev/null +++ b/vendor/mio/third_party/LICENSE.md @@ -0,0 +1,28 @@ +amalgamate.py - Amalgamate C source and header files +Copyright (c) 2012, Erik Edlund + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Erik Edlund, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/vendor/mio/third_party/amalgamate.py b/vendor/mio/third_party/amalgamate.py new file mode 100644 index 0000000..174e1c5 --- /dev/null +++ b/vendor/mio/third_party/amalgamate.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python +# coding=utf-8 + +# amalgamate.py - Amalgamate C source and header files. +# Copyright (c) 2012, Erik Edlund +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of Erik Edlund, nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import datetime +import json +import os +import re + + +class Amalgamation(object): + + # Prepends self.source_path to file_path if needed. + def actual_path(self, file_path): + if not os.path.isabs(file_path): + file_path = os.path.join(self.source_path, file_path) + return file_path + + # Search included file_path in self.include_paths and + # in source_dir if specified. + def find_included_file(self, file_path, source_dir): + search_dirs = self.include_paths[:] + if source_dir: + search_dirs.insert(0, source_dir) + + for search_dir in search_dirs: + search_path = os.path.join(search_dir, file_path) + if os.path.isfile(self.actual_path(search_path)): + return search_path + return None + + def __init__(self, args): + with open(args.config, 'r') as f: + config = json.loads(f.read()) + for key in config: + setattr(self, key, config[key]) + + self.verbose = args.verbose == "yes" + self.prologue = args.prologue + self.source_path = args.source_path + self.included_files = [] + + # Generate the amalgamation and write it to the target file. + def generate(self): + amalgamation = "" + + if self.prologue: + with open(self.prologue, 'r') as f: + amalgamation += datetime.datetime.now().strftime(f.read()) + + if self.verbose: + print("Config:") + print(" target = {0}".format(self.target)) + print(" working_dir = {0}".format(os.getcwd())) + print(" include_paths = {0}".format(self.include_paths)) + print("Creating amalgamation:") + for file_path in self.sources: + # Do not check the include paths while processing the source + # list, all given source paths must be correct. + # actual_path = self.actual_path(file_path) + print(" - processing \"{0}\"".format(file_path)) + t = TranslationUnit(file_path, self, True) + amalgamation += t.content + + with open(self.target, 'w') as f: + f.write(amalgamation) + + print("...done!\n") + if self.verbose: + print("Files processed: {0}".format(self.sources)) + print("Files included: {0}".format(self.included_files)) + print("") + + +def _is_within(match, matches): + for m in matches: + if match.start() > m.start() and \ + match.end() < m.end(): + return True + return False + + +class TranslationUnit(object): + # // C++ comment. + cpp_comment_pattern = re.compile(r"//.*?\n") + + # /* C comment. */ + c_comment_pattern = re.compile(r"/\*.*?\*/", re.S) + + # "complex \"stri\\\ng\" value". + string_pattern = re.compile("[^']" r'".*?(?<=[^\\])"', re.S) + + # Handle simple include directives. Support for advanced + # directives where macros and defines needs to expanded is + # not a concern right now. + include_pattern = re.compile( + r'#\s*include\s+(<|")(?P.*?)("|>)', re.S) + + # #pragma once + pragma_once_pattern = re.compile(r'#\s*pragma\s+once', re.S) + + # Search for pattern in self.content, add the match to + # contexts if found and update the index accordingly. + def _search_content(self, index, pattern, contexts): + match = pattern.search(self.content, index) + if match: + contexts.append(match) + return match.end() + return index + 2 + + # Return all the skippable contexts, i.e., comments and strings + def _find_skippable_contexts(self): + # Find contexts in the content in which a found include + # directive should not be processed. + skippable_contexts = [] + + # Walk through the content char by char, and try to grab + # skippable contexts using regular expressions when found. + i = 1 + content_len = len(self.content) + while i < content_len: + j = i - 1 + current = self.content[i] + previous = self.content[j] + + if current == '"': + # String value. + i = self._search_content(j, self.string_pattern, + skippable_contexts) + elif current == '*' and previous == '/': + # C style comment. + i = self._search_content(j, self.c_comment_pattern, + skippable_contexts) + elif current == '/' and previous == '/': + # C++ style comment. + i = self._search_content(j, self.cpp_comment_pattern, + skippable_contexts) + else: + # Skip to the next char. + i += 1 + + return skippable_contexts + + # Returns True if the match is within list of other matches + + # Removes pragma once from content + def _process_pragma_once(self): + content_len = len(self.content) + if content_len < len("#include "): + return 0 + + # Find contexts in the content in which a found include + # directive should not be processed. + skippable_contexts = self._find_skippable_contexts() + + pragmas = [] + pragma_once_match = self.pragma_once_pattern.search(self.content) + while pragma_once_match: + if not _is_within(pragma_once_match, skippable_contexts): + pragmas.append(pragma_once_match) + + pragma_once_match = self.pragma_once_pattern.search(self.content, + pragma_once_match.end()) + + # Handle all collected pragma once directives. + prev_end = 0 + tmp_content = '' + for pragma_match in pragmas: + tmp_content += self.content[prev_end:pragma_match.start()] + prev_end = pragma_match.end() + tmp_content += self.content[prev_end:] + self.content = tmp_content + + # Include all trivial #include directives into self.content. + def _process_includes(self): + content_len = len(self.content) + if content_len < len("#include "): + return 0 + + # Find contexts in the content in which a found include + # directive should not be processed. + skippable_contexts = self._find_skippable_contexts() + + # Search for include directives in the content, collect those + # which should be included into the content. + includes = [] + include_match = self.include_pattern.search(self.content) + while include_match: + if not _is_within(include_match, skippable_contexts): + include_path = include_match.group("path") + search_same_dir = include_match.group(1) == '"' + found_included_path = self.amalgamation.find_included_file( + include_path, self.file_dir if search_same_dir else None) + if found_included_path: + includes.append((include_match, found_included_path)) + + include_match = self.include_pattern.search(self.content, + include_match.end()) + + # Handle all collected include directives. + prev_end = 0 + tmp_content = '' + for include in includes: + include_match, found_included_path = include + tmp_content += self.content[prev_end:include_match.start()] + tmp_content += "// {0}\n".format(include_match.group(0)) + if found_included_path not in self.amalgamation.included_files: + t = TranslationUnit(found_included_path, self.amalgamation, False) + tmp_content += t.content + prev_end = include_match.end() + tmp_content += self.content[prev_end:] + self.content = tmp_content + + return len(includes) + + # Make all content processing + def _process(self): + if not self.is_root: + self._process_pragma_once() + self._process_includes() + + def __init__(self, file_path, amalgamation, is_root): + self.file_path = file_path + self.file_dir = os.path.dirname(file_path) + self.amalgamation = amalgamation + self.is_root = is_root + + self.amalgamation.included_files.append(self.file_path) + + actual_path = self.amalgamation.actual_path(file_path) + if not os.path.isfile(actual_path): + raise IOError("File not found: \"{0}\"".format(file_path)) + with open(actual_path, 'r') as f: + self.content = f.read() + self._process() + + +def main(): + description = "Amalgamate C source and header files." + usage = " ".join([ + "amalgamate.py", + "[-v]", + "-c path/to/config.json", + "-s path/to/source/dir", + "[-p path/to/prologue.(c|h)]" + ]) + argsparser = argparse.ArgumentParser( + description=description, usage=usage) + + argsparser.add_argument("-v", "--verbose", dest="verbose", + choices=["yes", "no"], metavar="", help="be verbose") + + argsparser.add_argument("-c", "--config", dest="config", + required=True, metavar="", help="path to a JSON config file") + + argsparser.add_argument("-s", "--source", dest="source_path", + required=True, metavar="", help="source code path") + + argsparser.add_argument("-p", "--prologue", dest="prologue", + required=False, metavar="", help="path to a C prologue file") + + amalgamation = Amalgamation(argsparser.parse_args()) + amalgamation.generate() + + +if __name__ == "__main__": + main() + diff --git a/vendor/mio/third_party/config.json b/vendor/mio/third_party/config.json new file mode 100644 index 0000000..fa2e71a --- /dev/null +++ b/vendor/mio/third_party/config.json @@ -0,0 +1,11 @@ +{ + "project": "Cross-platform C++11 header-only library for memory mapped file IO", + "target": "../single_include/mio/mio.hpp", + "sources": [ + "../include/mio/mmap.hpp", + "../include/mio/page.hpp", + "../include/mio/shared_mmap.hpp" + ], + "include_paths": ["../include"] +} +