build: option to use OpenSSL for MD5

- Drop thread priority by 1
- Reduce signed/unsigned conversions

.gitignore

@@ -1,3 +1,4 @@
 build/
 .cache/
 cmake-build-*
+.vscode/

CMakeLists.txt

@@ -1,19 +1,27 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.14)
 
 project(kgg-dec VERSION 0.6.0 LANGUAGES CXX)
 
+option(USE_SYSTEM_SQLITE3 "Use system SQLite3 (if not using WinSQLite3 and available)" ON)
+option(USE_OPENSSL "Use OpenSSL API (if not using WinCrypto API and available)" ON)
+
+if(WIN32)
     option(USE_WIN_SQLITE3 "Use Windows SQLite3 (MSVC Only)" ${MSVC})
     option(USE_WIN_CRYPTO "Use Windows Crypto API" ${WIN32})
+else()
+    set(USE_WIN_SQLITE3 OFF)
+    set(USE_WIN_CRYPTO OFF)
+endif()
 
+# Setup CryptoAPI
+if (NOT USE_WIN_CRYPTO AND USE_OPENSSL)
+    find_package(OpenSSL REQUIRED)
+endif()
+
+include(cmake/SetupSQLite3.cmake)
 add_subdirectory(third-party/aes)
 add_subdirectory(third-party/md5)
 
-include(cmake/FindWinSQLite3.cmake)
-if (NOT WinSQLite3_Found)
-    message("including sqlite3 to the build")
-    add_subdirectory(third-party/sqlite3)
-endif ()
-
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
@@ -35,19 +43,29 @@ target_include_directories(kgg-dec
         src/tc_tea
 )
 
+# Base crypto implementations
 target_link_libraries(kgg-dec PRIVATE libaes libmd5)
+if (USE_WIN_CRYPTO)
+    target_compile_definitions(kgg-dec PRIVATE USE_WIN_CRYPTO=1)
+elseif(USE_OPENSSL)
+    target_compile_definitions(kgg-dec PRIVATE USE_OPENSSL=1)
+endif ()
+
+# Win32 specific
 if (WIN32)
     target_link_libraries(kgg-dec PRIVATE shell32 ole32)
+    target_compile_definitions(kgg-dec PRIVATE NOMINMAX)
 endif ()
+
+# SQLite3
 if (WinSQLite3_Found)
     target_link_libraries(kgg-dec PRIVATE WinSQLite3)
     target_include_directories(kgg-dec PRIVATE ${WindowsKitInclude})
+elseif(SQLite3_FOUND)
+    target_link_libraries(kgg-dec PRIVATE SQLite::SQLite3)
 else ()
     target_link_libraries(kgg-dec PRIVATE sqlite3)
 endif ()
-target_compile_definitions(kgg-dec PRIVATE NOMINMAX)
-target_compile_definitions(kgg-dec PRIVATE KGGDEC_PROJECT_VERSION="${PROJECT_VERSION}")
 
-if (USE_WIN_CRYPTO)
-    target_compile_definitions(kgg-dec PRIVATE USE_WIN_CRYPTO=1)
-endif ()
+# Extra definitions
+target_compile_definitions(kgg-dec PRIVATE KGGDEC_PROJECT_VERSION="${PROJECT_VERSION}")

cmake/SetupSQLite3.cmake

@@ -12,3 +12,13 @@ if (MSVC AND USE_WIN_SQLITE3)
         message("Using WinSQLite3 from Windows SDK (${WindowsKitVersion}).")
     endif ()
 endif ()
+
+if (NOT WinSQLite3_Found)
+    if (USE_SYSTEM_SQLITE3)
+        message("Using existing SQLite3.")
+        find_package(SQLite3 REQUIRED)
+    else()
+        message("including sqlite3 to the build")
+        add_subdirectory(third-party/sqlite3)
+    endif()
+endif()

src/jobs.hpp

@@ -1,12 +1,16 @@
 #pragma once
 
+#include <algorithm>
 #include "infra/infra.h"
 #include "qmc2/qmc2.h"
 
 #ifdef _WIN32
 #include <windows.h>
+#else
+#include <pthread.h>
 #endif
 
+#include <endian_helper.h>
 #include <str_helper.h>
 #include <array>
 #include <condition_variable>
@@ -14,7 +18,6 @@
 #include <fstream>
 #include <mutex>
 #include <queue>
-#include <string_view>
 #include <vector>
 
 class KggTask {
@@ -34,16 +37,15 @@ class KggTask {
                                                               0xA8, 0xAF, 0xA6, 0x8E, 0x0F, 0xFF, 0x99, 0x14};
 
         std::ifstream kgg_stream_in(kgg_path_, std::ios::binary);
-        char header[0x100]{};
-        kgg_stream_in.read(header, sizeof(header));
-        if (std::equal(kMagicHeader.cbegin(), kMagicHeader.cend(), header)) {
-            warning(LSR_STR("invalid kgg header"));
+        std::array<uint8_t, 0x400> header{};
+        kgg_stream_in.read(reinterpret_cast<char*>(header.data()), header.size());
+        if (!std::equal(kMagicHeader.cbegin(), kMagicHeader.cend(), header.cbegin())) {
+            warning(LSR_STR("invalid kgg header (not a kgg file)"));
             return;
         }
-        const uint32_t offset_to_audio = *reinterpret_cast<uint32_t*>(&header[0x10]);
-        const uint32_t encrypt_mode = *reinterpret_cast<uint32_t*>(&header[0x14]);
-        if (encrypt_mode != 5) {
-            lsr_eprintf("[WARN] unsupported enc_version (expect=0x05, got 0x%02x) (%s)\n", encrypt_mode,
+        const auto offset_to_audio = Endian::le_read<uint32_t>(&header[0x10]);
+        if (const auto mode = Endian::le_read<uint32_t>(&header[0x14]); mode != 5) {
+            lsr_eprintf("[WARN] unsupported enc_version (expect=0x05, got 0x%02x) (%s)\n", mode,
                         kgg_path_.filename().c_str());
             return;
         }
@@ -69,20 +71,22 @@ class KggTask {
             return;
         }
 
-        std::string magic(4, 0);
+        std::array<uint8_t, 4> magic{};
         kgg_stream_in.seekg(offset_to_audio, std::ios::beg);
-        kgg_stream_in.read(magic.data(), 4);
-        qmc2->Decrypt(std::span(reinterpret_cast<uint8_t*>(magic.data()), 4), 0);
+        kgg_stream_in.read(reinterpret_cast<char*>(magic.data()), 4);
+        qmc2->Decrypt(magic, 0);
         auto real_ext = DetectRealExt(magic);
 
         lsr::string new_name = kgg_path_.stem().native() + lsr::string(suffix) + LSR_STR(".") + real_ext;
         auto out_path = out_dir_ / new_name;
 
         if (exists(out_path)) {
-            warning(lsr::string(LSR_STR("output file already exists: ")) + out_path.filename().native());
+            warning(lsr::string(LSR_STR("output file already exists: ")) + new_name);
             return;
         }
 
+        kgg_stream_in.seekg(0, std::ios::end);
+        const auto file_size = static_cast<size_t>(kgg_stream_in.tellg());
         kgg_stream_in.seekg(offset_to_audio, std::ios::beg);
         std::ofstream ofs_decrypted(out_path, std::ios::binary);
         if (!ofs_decrypted.is_open()) {
@@ -92,29 +96,32 @@ class KggTask {
 
         size_t offset{0};
         thread_local std::vector<uint8_t> temp_buffer(1024 * 1024, 0);
-        auto buf_signed = std::span(reinterpret_cast<char*>(temp_buffer.data()), temp_buffer.size());
-        auto buf_unsigned = std::span(temp_buffer);
+        auto read_page_len = static_cast<std::streamsize>(temp_buffer.size());
 
         while (!kgg_stream_in.eof()) {
-            kgg_stream_in.read(buf_signed.data(), buf_signed.size());
-            const auto n = static_cast<size_t>(kgg_stream_in.gcount());
-            qmc2->Decrypt(buf_unsigned.subspan(0, n), offset);
-            ofs_decrypted.write(buf_signed.data(), n);
+            kgg_stream_in.read(reinterpret_cast<char*>(temp_buffer.data()), read_page_len);
+            const auto n = kgg_stream_in.gcount();
+            qmc2->Decrypt(std::span(temp_buffer.begin(), temp_buffer.begin() + n), offset);
+            ofs_decrypted.write(reinterpret_cast<char*>(temp_buffer.data()), n);
             offset += n;
         }
 
+        if (offset + offset_to_audio != file_size) {
+            warning(LSR_STR("OK (size mismatch)"));
+        } else {
             info(lsr::string(LSR_STR("** OK **  -> ")) + out_path.filename().native());
         }
+    }
 
    private:
     std::filesystem::path kgg_path_;
     std::filesystem::path out_dir_;
 
-    static const lsr::character* DetectRealExt(const std::string_view magic) {
-        if (magic == "fLaC") {
+    static const lsr::character* DetectRealExt(const std::span<uint8_t> magic) {
+        if (std::equal(magic.begin(), magic.end(), "fLaC")) {
             return LSR_STR("flac");
         }
-        if (magic == "OggS") {
+        if (std::equal(magic.begin(), magic.end(), "OggS")) {
             return LSR_STR("ogg");
         }
         return LSR_STR("mp3");
@@ -169,16 +176,32 @@ class KggTaskQueue {
     Infra::kgm_ekey_db_t ekey_db_;
     lsr::string suffix_;
     void WorkerThreadBody() {
-#ifdef _WIN32
-        SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL);
-#endif
-
+        ReduceThreadPriority();
         std::unique_ptr<KggTask> task{nullptr};
         while ((task = Pop())) {
             task->Execute(ekey_db_, suffix_);
         }
     }
 
+    static void ReduceThreadPriority() {
+#ifdef _WIN32
+        SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL);
+#else
+        pthread_t this_thread = pthread_self();
+        sched_param params;
+        int policy;
+
+        if (pthread_getschedparam(this_thread, &policy, &params) != 0) {
+            perror("pthread_getschedparam");
+            return;
+        }
+        params.sched_priority = std::max(params.sched_priority, sched_get_priority_min(policy));
+        if (pthread_setschedparam(this_thread, policy, &params) != 0) {
+            perror("pthread_setschedparam");
+        }
+#endif
+    }
+
     std::mutex mutex_{};
     std::condition_variable signal_;
     std::queue<std::unique_ptr<KggTask>> tasks_{};

third-party/aes/CMakeLists.txt

@@ -7,10 +7,16 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set(SOURCES)
 if (USE_WIN_CRYPTO)
+    message("Using Windows Crypto API for AES-CBC-128")
     list(APPEND SOURCES aes_win32.cpp)
+elseif (USE_OPENSSL)
+    message("Using OpenSSL API for AES-CBC-128")
+    find_package(OpenSSL REQUIRED)    
+    list(APPEND SOURCES aes_openssl.cpp)
 else ()
     # Tiny AES in C (https://github.com/kokke/tiny-AES-c/)
     #   is licensed under the Unlicense license.
+    message("Using included AES-CBC-128 implementation")
     list(APPEND SOURCES aes.cpp)
 endif ()
 
@@ -23,4 +29,7 @@ target_include_directories(libaes
 if (USE_WIN_CRYPTO)
     target_link_libraries(libaes PRIVATE bcrypt)
     target_compile_definitions(libaes PRIVATE USE_WIN_CRYPTO=1)
+elseif (USE_OPENSSL)
+    target_link_libraries(libaes PRIVATE OpenSSL::Crypto)
+    target_compile_definitions(libaes PRIVATE USE_OPENSSL=1)
 endif ()

third-party/aes/aes.h

@@ -7,6 +7,8 @@
 #include <windows.h>
 
 #include <bcrypt.h>
+#elif USE_OPENSSL
+#include <openssl/evp.h>
 #endif
 
 namespace AES {
@@ -20,6 +22,8 @@ struct AES_ctx {
     BCRYPT_ALG_HANDLE hAlg;
     BCRYPT_KEY_HANDLE hKey;
     uint8_t iv[0x10];
+#elif USE_OPENSSL
+    EVP_CIPHER_CTX* cipher_ctx;
 #else
     uint8_t RoundKey[kKeyExpansionSize];
     uint8_t Iv[16];
@@ -35,7 +39,7 @@ bool AES_init_ctx_iv(AES_ctx* ctx, const uint8_t* key, const uint8_t* iv);
 size_t AES_CBC_encrypt_buffer(AES_ctx* ctx, uint8_t* buf, size_t length);
 size_t AES_CBC_decrypt_buffer(AES_ctx* ctx, uint8_t* buf, size_t length);
 
-#if USE_WIN_CRYPTO
+#if USE_WIN_CRYPTO || USE_OPENSSL
 bool AES_cleanup(AES_ctx* ctx);
 #else
 inline bool AES_cleanup(AES_ctx* ctx) {

third-party/aes/aes_openssl.cpp

@@ -0,0 +1,43 @@
+#include <cassert>
+#include <cstring>
+
+#include "aes.h"
+
+namespace AES {
+
+bool AES_init_ctx_iv(AES_ctx* ctx, const uint8_t* key, const uint8_t* iv) {
+    ctx->cipher_ctx = EVP_CIPHER_CTX_new();
+    if (!ctx->cipher_ctx) {
+        return false;
+    }
+
+    if (EVP_DecryptInit_ex(ctx->cipher_ctx, EVP_aes_128_cbc(), nullptr, key, iv) != 1) {
+        AES_cleanup(ctx);
+        return false;
+    }
+
+    EVP_CIPHER_CTX_set_padding(ctx->cipher_ctx, 0);
+    return true;
+}
+
+size_t AES_CBC_encrypt_buffer(struct AES_ctx* ctx, uint8_t* buf, size_t length) {
+    // not implemented
+    return 0;
+}
+
+size_t AES_CBC_decrypt_buffer(struct AES_ctx* ctx, uint8_t* buf, size_t length) {
+    auto buf_len = static_cast<int>(length);
+    EVP_DecryptUpdate(ctx->cipher_ctx, buf, &buf_len, buf, buf_len);
+    assert(buf_len == length && "AES_CBC_decrypt_buffer: buffer length mismatch");
+    return buf_len;
+}
+
+bool AES_cleanup(AES_ctx* ctx) {
+    if (ctx->cipher_ctx) {
+        EVP_CIPHER_CTX_free(ctx->cipher_ctx);
+        ctx->cipher_ctx = nullptr;
+    }
+    return true;
+}
+
+}  // namespace AES

third-party/md5/CMakeLists.txt

@@ -7,10 +7,16 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set(SOURCES)
 if (USE_WIN_CRYPTO)
+    message("Using Windows Crypto API for MD5")
     list(APPEND SOURCES md5_win32.cpp)
+elseif (USE_OPENSSL)
+    message("Using OpenSSL API for MD5")
+    find_package(OpenSSL REQUIRED)
+    list(APPEND SOURCES md5_openssl.cpp)
 else ()
     # Derived from the "RSA Data Security, Inc. MD5 Message-Digest Algorithm":
     # https://github.com/freebsd/freebsd-src/blob/release/14.2.0/sys/kern/md5c.c
+    message("Using included MD5 implementation")
     list(APPEND SOURCES md5.cpp)
 endif ()
 
@@ -24,4 +30,7 @@ target_include_directories(libmd5
 if (USE_WIN_CRYPTO)
     target_link_libraries(libmd5 PRIVATE crypt32)
     target_compile_definitions(libmd5 PRIVATE USE_WIN_CRYPTO=1)
+elseif (USE_OPENSSL)
+    target_link_libraries(libmd5 PRIVATE OpenSSL::Crypto)
+    target_compile_definitions(libmd5 PRIVATE USE_OPENSSL=1)
 endif ()

third-party/md5/md5.h

@@ -1,21 +1,23 @@
 #pragma once
 
+#include <cstdint>
+
 #if USE_WIN_CRYPTO
 #include <windows.h>
 
 #include <wincrypt.h>
+#elif USE_OPENSSL
+#include <openssl/evp.h>
 #endif
 
-#include <cstdint>
-
-#define MD5_BLOCK_LENGTH 64
 #define MD5_DIGEST_LENGTH 16
-#define MD5_DIGEST_STRING_LENGTH (MD5_DIGEST_LENGTH * 2 + 1)
 
 struct MD5_CTX {
 #if USE_WIN_CRYPTO
     HCRYPTPROV hProv;
     HCRYPTHASH hHash;
+#elif USE_OPENSSL
+    EVP_MD_CTX* md_ctx;
 #else
     uint64_t count;           /* number of bits, modulo 2^64 (lsb first) */
     uint32_t state[4];        /* state (ABCD) */
@@ -23,8 +25,8 @@ struct MD5_CTX {
 #endif
 };
 
-#if USE_WIN_CRYPTO
-bool md5_init(MD5_CTX* context);
+#if USE_WIN_CRYPTO || USE_OPENSSL
+bool md5_init(MD5_CTX* ctx);
 bool md5_cleanup(MD5_CTX* ctx);
 #else
 /* MD5 initialization. Begins an MD5 operation, writing a new context. */

third-party/md5/md5_openssl.cpp

@@ -0,0 +1,32 @@
+#include <openssl/evp.h>
+#include <cstring>
+#include "md5.h"
+
+bool md5_init(MD5_CTX* ctx) {
+    memset(ctx, 0, sizeof(*ctx));
+
+    ctx->md_ctx = EVP_MD_CTX_new();
+    if (!ctx->md_ctx) {
+        return false;
+    }
+
+    return EVP_DigestInit_ex(ctx->md_ctx, EVP_md5(), nullptr) == 1;
+}
+
+bool md5_cleanup(MD5_CTX* ctx) {
+    if (ctx->md_ctx != nullptr) {
+        EVP_MD_CTX_free(ctx->md_ctx);
+    }
+    memset(ctx, 0xcc, sizeof(*ctx));
+    return true;
+}
+
+void md5_update(MD5_CTX* ctx, const uint8_t* in, size_t len) {
+    EVP_DigestUpdate(ctx->md_ctx, in, len);
+}
+
+void md5_final(MD5_CTX* ctx, uint8_t* digest) {
+    unsigned int len{MD5_DIGEST_LENGTH};
+    EVP_DigestFinal_ex(ctx->md_ctx, digest, &len);
+    EVP_MD_CTX_reset(ctx->md_ctx);
+}