feat: footer parser

Cargo.lock

@@ -69,6 +69,12 @@ version = "3.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
 
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
 [[package]]
 name = "cc"
 version = "1.1.15"
@@ -312,6 +318,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "base64",
+ "byteorder",
  "itertools",
  "thiserror",
 ]

um_crypto/qmc/Cargo.toml

@@ -8,3 +8,4 @@ base64 = "0.22.1"
 itertools = "0.13.0"
 anyhow = "1.0.86"
 thiserror = "1.0.63"
+byteorder = "1.5.0"

um_crypto/qmc/src/footer/android_qtag.rs

@@ -0,0 +1,51 @@
+use crate::footer::utils::is_base64;
+use crate::footer::{Data, FooterParseError, Metadata, MetadataParser};
+use byteorder::{ByteOrder, BE};
+use itertools::Itertools;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct QTagMetadata {
+    /// The old, numeric id of the resource.
+    pub resource_id: u64,
+}
+
+impl MetadataParser for QTagMetadata {
+    fn from_byte_slice(buffer: &[u8]) -> anyhow::Result<Option<Metadata>> {
+        if buffer.len() < 8 {
+            Err(FooterParseError::BufferTooSmall(8))?;
+        }
+
+        if let Some(footer) = buffer.strip_suffix(b"QTag") {
+            let (payload, payload_len) = footer.split_at(footer.len() - 4);
+            let actual_payload_len = BE::read_u32(payload_len) as usize;
+            if payload.len() < actual_payload_len {
+                Err(FooterParseError::BufferTooSmall(actual_payload_len + 8))?;
+            }
+
+            // CSV: ekey,resource_id,version
+            let payload = String::from_utf8_lossy(&payload[payload.len() - actual_payload_len..]);
+            if let Some((ekey, resource_id, version)) = payload.split(',').collect_tuple() {
+                if version != "2" {
+                    Err(FooterParseError::QTagInvalidVersion(version.to_string()))?;
+                }
+                if !resource_id.as_bytes().iter().all(|&b| b.is_ascii_digit()) {
+                    Err(FooterParseError::QTagInvalidId(resource_id.to_string()))?;
+                }
+                if !is_base64(ekey.as_bytes()) {
+                    Err(FooterParseError::QTagInvalidEKey(ekey.to_string()))?;
+                }
+
+                return Ok(Some(Metadata {
+                    ekey: Some(ekey.into()),
+                    size: actual_payload_len + 8,
+                    data: Data::AndroidQTag(QTagMetadata {
+                        resource_id: resource_id.parse()?,
+                    }),
+                }));
+            }
+
+            Err(FooterParseError::STagInvalidCSV(payload.to_string()))?;
+        }
+        Ok(None)
+    }
+}

um_crypto/qmc/src/footer/android_stag.rs

@@ -0,0 +1,50 @@
+use crate::footer::{Data, FooterParseError, Metadata, MetadataParser};
+use byteorder::{ByteOrder, BE};
+use itertools::Itertools;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct STagMetadata {
+    /// Resource identifier (aka. `file.media_mid`).
+    pub media_mid: String,
+    
+    /// Resource id (numeric)
+    pub resource_id: u64,
+}
+
+impl MetadataParser for STagMetadata {
+    fn from_byte_slice(buffer: &[u8]) -> anyhow::Result<Option<Metadata>> {
+        if buffer.len() < 8 {
+            Err(FooterParseError::BufferTooSmall(8))?;
+        }
+
+        if let Some(footer) = buffer.strip_suffix(b"STag") {
+            let (payload, payload_len) = footer.split_at(footer.len() - 4);
+            let actual_payload_len = BE::read_u32(payload_len) as usize;
+            if payload.len() < actual_payload_len {
+                Err(FooterParseError::BufferTooSmall(actual_payload_len + 8))?;
+            }
+
+            let payload = String::from_utf8_lossy(&payload[payload.len() - actual_payload_len..]);
+            if let Some((id, version, media_mid)) = payload.split(',').collect_tuple() {
+                if version != "2" {
+                    Err(FooterParseError::STagInvalidVersion(version.to_string()))?;
+                }
+                if !id.as_bytes().iter().all(|&b| b.is_ascii_digit()) {
+                    Err(FooterParseError::STagInvalidId(id.to_string()))?;
+                }
+
+                return Ok(Some(Metadata {
+                    ekey: None,
+                    size: actual_payload_len + 8,
+                    data: Data::AndroidSTag(STagMetadata {
+                        resource_id: id.parse()?,
+                        media_mid: media_mid.to_string(),
+                    }),
+                }));
+            }
+
+            Err(FooterParseError::STagInvalidCSV(payload.to_string()))?;
+        }
+        Ok(None)
+    }
+}

um_crypto/qmc/src/footer/mod.rs

@@ -0,0 +1,165 @@
+pub mod android_qtag;
+pub mod android_stag;
+mod musicex_v1;
+pub mod pc_v1_legacy;
+pub mod pc_v2_musicex;
+mod utils;
+
+use crate::footer::{
+    android_qtag::QTagMetadata, android_stag::STagMetadata, pc_v1_legacy::PcV1Legacy,
+    pc_v2_musicex::PcV2MusicEx,
+};
+use anyhow::Result;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum FooterParseError {
+    #[error("Footer: Buffer too small, require at least {0} bytes")]
+    BufferTooSmall(usize),
+    #[error("PCv1/EKey: Buffer too large, might not be valid EKey (len={0})")]
+    PCv1EKeyTooLarge(usize),
+    #[error("PCv1/EKey: Found invalid EKey char")]
+    PCv1EKeyInvalid,
+
+    #[error("PCv2/MusicEx: Invalid metadata version {0}")]
+    PCv2InvalidVersion(u32),
+    #[error("PCv2/MusicEx: Invalid `MusicEx` size: {0}")]
+    PCv2MusicExUnsupportedPayloadSize(usize),
+
+    #[error("Android/STag: Invalid ID field: {0}")]
+    STagInvalidId(String),
+    #[error("Android/STag: Invalid Version: {0}")]
+    STagInvalidVersion(String),
+    #[error("Android/STag: Invalid CSV metadata: {0}")]
+    STagInvalidCSV(String),
+
+    #[error("Android/QTag: Invalid ID field: {0}")]
+    QTagInvalidId(String),
+    #[error("Android/QTag: Invalid Version: {0}")]
+    QTagInvalidVersion(String),
+    #[error("Android/QTag: Invalid EKey field: {0}")]
+    QTagInvalidEKey(String),
+}
+
+/// Footer type
+#[derive(Debug, Clone, PartialEq)]
+pub enum Data {
+    /// No extra metadata.
+    PCv1Legacy(pc_v1_legacy::PcV1Legacy),
+    /// "MusicEx" footer.
+    PCv2MusicEx(pc_v2_musicex::PcV2MusicEx),
+
+    /// Android "QTag", with ekey.
+    AndroidQTag(android_qtag::QTagMetadata),
+    /// Android "STag", metadata only.
+    AndroidSTag(android_stag::STagMetadata),
+}
+
+/// File Footer metadata
+#[derive(Debug, Clone, PartialEq)]
+pub struct Metadata {
+    /// Footer size to trim off.
+    pub size: usize,
+
+    /// Embedded key (not decrypted).
+    pub ekey: Option<String>,
+
+    /// data/type
+    pub data: Data,
+}
+
+pub trait MetadataParser {
+    fn from_byte_slice(buffer: &[u8]) -> Result<Option<Metadata>>;
+}
+
+pub fn from_byte_slice(buffer: &[u8]) -> Result<Option<Metadata>> {
+    if let Some(metadata) = STagMetadata::from_byte_slice(buffer)? {
+        return Ok(Some(metadata));
+    }
+    if let Some(metadata) = QTagMetadata::from_byte_slice(buffer)? {
+        return Ok(Some(metadata));
+    }
+    if let Some(metadata) = PcV2MusicEx::from_byte_slice(buffer)? {
+        return Ok(Some(metadata));
+    }
+    if let Some(metadata) = PcV1Legacy::from_byte_slice(buffer)? {
+        return Ok(Some(metadata));
+    }
+
+    Ok(None)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::footer::android_qtag::QTagMetadata;
+    use crate::footer::android_stag::STagMetadata;
+    use crate::footer::pc_v1_legacy::PcV1Legacy;
+
+    #[test]
+    fn test_qtag() {
+        let payload = include_bytes!("fixtures/ekey_android_qtag.bin");
+        let payload = from_byte_slice(payload)
+            .expect("Should not fail")
+            .expect("should parse to qtag");
+
+        assert_eq!(payload.ekey, Some("00112233aBcD+/=".into()));
+        assert_eq!(payload.size, 0x23);
+        assert_eq!(
+            payload.data,
+            Data::AndroidQTag(QTagMetadata {
+                resource_id: 326454301
+            })
+        )
+    }
+
+    #[test]
+    fn test_stag() {
+        let payload = include_bytes!("fixtures/ekey_android_stag.bin");
+        let payload = from_byte_slice(payload)
+            .expect("Should not fail")
+            .expect("should parse to stag");
+
+        assert_eq!(payload.ekey, None);
+        assert_eq!(payload.size, 0x20);
+        assert_eq!(
+            payload.data,
+            Data::AndroidSTag(STagMetadata {
+                media_mid: "001y7CaR29k6YP".into(),
+                resource_id: 5177785,
+            })
+        )
+    }
+
+    #[test]
+    fn test_pc_enc_v1() {
+        let payload = include_bytes!("fixtures/ekey_pc_enc_v1.bin");
+        let payload = from_byte_slice(payload)
+            .expect("Should not fail")
+            .expect("should parse pc v1");
+
+        let ekey = payload.ekey.expect("ekey should be present");
+        assert!(ekey.starts_with("NUZ6b0la"));
+
+        assert_eq!(payload.size, 0x2C4);
+        assert_eq!(payload.data, Data::PCv1Legacy(PcV1Legacy))
+    }
+
+    #[test]
+    fn test_pc_enc_v2() {
+        let payload = include_bytes!("fixtures/ekey_pc_enc_v2.bin");
+        let payload = from_byte_slice(payload)
+            .expect("Should not fail")
+            .expect("should parse pc v2");
+
+        assert_eq!(payload.ekey, None);
+        assert_eq!(payload.size, 0xC0);
+        assert_eq!(
+            payload.data,
+            Data::PCv2MusicEx(PcV2MusicEx {
+                mid: "AaBbCcDdEeFfGg".into(),
+                media_filename: "F0M000112233445566.mflac".into()
+            })
+        )
+    }
+}

um_crypto/qmc/src/footer/musicex_v1.rs

@@ -0,0 +1,77 @@
+use std::io::{Cursor, Read};
+use byteorder::{ByteOrder,ReadBytesExt, LE};
+use crate::footer::{Data, FooterParseError, Metadata};
+use crate::footer::pc_v2_musicex::PcV2MusicEx;
+use crate::footer::utils::from_ascii_utf16;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct MusicExV1 {
+    /// unused & unknown
+    unknown_0: u32,
+    /// unused & unknown
+    unknown_1: u32,
+    /// unused & unknown
+    unknown_2: u32,
+
+    /// Media ID
+    mid: [u8; 30 * 2],
+    /// Media file name
+    media_filename: [u8; 50 * 2],
+
+    /// unused; uninitialized memory?
+    unknown_3: u32,
+}
+
+impl Default for MusicExV1 {
+    fn default() -> Self {
+        MusicExV1 {
+            unknown_0: 0,
+            unknown_1: 0,
+            unknown_2: 0,
+            mid: [0; 30 * 2],
+            media_filename: [0; 50 * 2],
+            unknown_3: 0,
+        }
+    }
+}
+
+impl MusicExV1 {
+    pub fn from_bytes(buffer: &[u8]) -> anyhow::Result<MusicExV1> {
+        assert_eq!(buffer.len(), 0xC0 - 0x10);
+
+        let mut cursor = Cursor::new(&buffer);
+        let mut result = MusicExV1::default();
+        result.unknown_0 = cursor.read_u32::<LE>()?;
+        result.unknown_1 = cursor.read_u32::<LE>()?;
+        result.unknown_2 = cursor.read_u32::<LE>()?;
+        cursor.read(&mut result.mid)?;
+        cursor.read(&mut result.media_filename)?;
+        result.unknown_3 = cursor.read_u32::<LE>()?;
+
+        Ok(result)
+    }
+}
+
+pub fn parse_v1(footer: &[u8]) -> anyhow::Result<Option<Metadata>> {
+    let (payload, payload_len) = footer.split_at(footer.len() - 4);
+    let payload_len = LE::read_u32(&payload_len) as usize;
+    if payload_len != 0xC0 {
+        Err(FooterParseError::PCv2MusicExUnsupportedPayloadSize(
+            payload_len,
+        ))?;
+    }
+    
+    let payload = &payload[payload.len() - (payload_len - 0x10)..];
+    let payload = MusicExV1::from_bytes(payload)?;
+    let mid = from_ascii_utf16(&payload.mid);
+    let media_filename = from_ascii_utf16(&payload.media_filename);
+
+    Ok(Some(Metadata {
+        ekey: None,
+        size: payload_len,
+        data: Data::PCv2MusicEx(PcV2MusicEx {
+            mid,
+            media_filename,
+        }),
+    }))
+}

um_crypto/qmc/src/footer/pc_v1_legacy.rs

@@ -0,0 +1,44 @@
+use crate::footer::utils::is_base64;
+use crate::footer::{Data, FooterParseError, Metadata, MetadataParser};
+use byteorder::{ByteOrder, LE};
+
+pub const MAX_ALLOWED_EKEY_LEN: usize = 0x500;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct PcV1Legacy;
+
+impl MetadataParser for PcV1Legacy {
+    fn from_byte_slice(buffer: &[u8]) -> anyhow::Result<Option<Metadata>> {
+        if buffer.len() < 8 {
+            Err(FooterParseError::BufferTooSmall(8))?;
+        }
+
+        let (payload, payload_len) = buffer.split_at(buffer.len() - 4);
+        let payload_len = LE::read_u32(payload_len) as usize;
+
+        // EKey payload is too large, probably not a valid V1 footer.
+        if payload_len > MAX_ALLOWED_EKEY_LEN {
+            Err(FooterParseError::PCv1EKeyTooLarge(payload_len))?;
+        }
+        if payload.len() < payload_len {
+            Err(FooterParseError::BufferTooSmall(payload_len + 4))?;
+        }
+
+        let payload = &payload[payload.len() - payload_len..];
+        let ekey = payload
+            .iter()
+            .take_while(|&&b| b != 0)
+            .map(|&b| b)
+            .collect::<Vec<_>>();
+        let ekey = String::from_utf8_lossy(ekey.as_slice());
+        if !is_base64(ekey.as_bytes()) {
+            Err(FooterParseError::PCv1EKeyInvalid)?;
+        }
+
+        Ok(Some(Metadata {
+            ekey: Some(ekey.into()),
+            size: payload_len + 4,
+            data: Data::PCv1Legacy(PcV1Legacy),
+        }))
+    }
+}

um_crypto/qmc/src/footer/pc_v2_musicex.rs

@@ -0,0 +1,32 @@
+use crate::footer::{musicex_v1, FooterParseError, Metadata, MetadataParser};
+use anyhow::Result;
+use byteorder::{ByteOrder, LE};
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct PcV2MusicEx {
+    /// Resource identifier (`.mid`)
+    pub mid: String,
+
+    /// The actual file name used for `ekey` lookup (`.file.media_mid` + extension).
+    pub media_filename: String,
+}
+
+impl MetadataParser for PcV2MusicEx {
+    fn from_byte_slice(payload: &[u8]) -> Result<Option<Metadata>> {
+        if payload.len() < 16 {
+            Err(FooterParseError::BufferTooSmall(16))?;
+        }
+
+        if let Some(payload) = payload.strip_suffix(b"musicex\x00") {
+            let (payload, version) = payload.split_at(payload.len() - 4);
+            let version = LE::read_u32(version);
+
+            return match version {
+                1 => musicex_v1::parse_v1(payload),
+                _ => Err(FooterParseError::PCv2InvalidVersion(version))?,
+            };
+        }
+
+        Ok(None)
+    }
+}

um_crypto/qmc/src/footer/utils.rs

@@ -0,0 +1,17 @@
+fn is_base64_chr(chr: u8) -> bool {
+    chr.is_ascii_alphanumeric() || (chr == b'+') || (chr == b'/') || (chr == b'=')
+}
+
+pub fn is_base64(s: &[u8]) -> bool {
+    s.iter().all(|&c| is_base64_chr(c))
+}
+
+/// Convert UTF-16 LE string (within ASCII char range) to UTF-8
+pub fn from_ascii_utf16(data: &[u8]) -> String {
+    let data = data
+        .chunks_exact(2)
+        .take_while(|chunk| chunk[0] != 0 && chunk[0].is_ascii() && chunk[1] == 0)
+        .map(|chunk| chunk[0])
+        .collect::<Vec<_>>();
+    String::from_utf8_lossy(&data).to_string()
+}

um_crypto/qmc/src/lib.rs

@@ -1,5 +1,6 @@
 use thiserror::Error;
 
+pub mod footer;
 pub mod v1;
 pub mod v2_map;
 pub mod v2_rc4;