[xmly] feat #5: implement xmly decipher

.idea/lib_um_crypto.iml

@@ -12,6 +12,7 @@
       <sourceFolder url="file://$MODULE_DIR$/um_audio/src" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/um_crypto/kgm/src" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/um_crypto/joox/src" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/um_crypto/xmly/src" isTestSource="false" />
       <excludeFolder url="file://$MODULE_DIR$/target" />
       <excludeFolder url="file://$MODULE_DIR$/um_wasm_loader/dist" />
       <excludeFolder url="file://$MODULE_DIR$/um_wasm_loader/pkg" />

Cargo.lock

@@ -104,6 +104,15 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
+[[package]]
+name = "cbc"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
+dependencies = [
+ "cipher",
+]
+
 [[package]]
 name = "cc"
 version = "1.1.15"
@@ -271,6 +280,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
 [[package]]
 name = "hmac"
 version = "0.12.1"
@@ -513,6 +528,7 @@ dependencies = [
  "umc_ncm",
  "umc_qmc",
  "umc_utils",
+ "umc_xmly",
 ]
 
 [[package]]
@@ -606,6 +622,20 @@ dependencies = [
  "md-5",
 ]
 
+[[package]]
+name = "umc_xmly"
+version = "0.1.0"
+dependencies = [
+ "aes",
+ "byteorder",
+ "cbc",
+ "cipher",
+ "hex",
+ "lazy_static",
+ "thiserror",
+ "umc_utils",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.12"

um_cli/Cargo.toml

@@ -11,4 +11,5 @@ umc_kgm = { path = "../um_crypto/kgm" }
 umc_kuwo = { path = "../um_crypto/kuwo" }
 umc_ncm = { path = "../um_crypto/ncm" }
 umc_qmc = { path = "../um_crypto/qmc" }
+umc_xmly = { path = "../um_crypto/xmly" }
 umc_utils = { path = "../um_crypto/utils" }

um_cli/src/cmd/mod.rs

@@ -5,6 +5,7 @@ pub mod kgm;
 pub mod ncm;
 pub mod qmc1;
 pub mod qmc2;
+pub mod xmly;
 
 #[derive(Subcommand)]
 pub enum Commands {
@@ -18,4 +19,6 @@ pub enum Commands {
     KGM(kgm::ArgsKGM),
     #[command(name = "joox")]
     JOOX(joox::ArgsJoox),
+    #[command(name = "xmly")]
+    XMLY(xmly::ArgsXimalaya),
 }

um_cli/src/cmd/xmly.rs

@@ -0,0 +1,94 @@
+use crate::Cli;
+use anyhow::{bail, Result};
+use clap::Args;
+use std::ffi::OsStr;
+use std::fs::File;
+use std::io;
+use std::io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write};
+use std::path::PathBuf;
+
+#[derive(Debug, Clone, clap::ValueEnum)]
+enum XimalayaType {
+    /// Android: *.x2m
+    X2M,
+    /// Android: *.x3m
+    X3M,
+    /// PC: *.xm
+    XM,
+}
+
+/// Decrypt a X2M/X3M/XM file (Ximalaya)
+#[derive(Args)]
+pub struct ArgsXimalaya {
+    /// Path to output file, e.g. /export/Music/song.flac
+    #[clap(short, long)]
+    output: PathBuf,
+
+    /// Path to input file, e.g. /export/Music/song.x3m
+    #[arg(name = "input")]
+    input: PathBuf,
+
+    #[clap(short = 't', long = "type")]
+    file_type: Option<XimalayaType>,
+}
+
+impl ArgsXimalaya {
+    pub fn run(&self, cli: &Cli) -> Result<i32> {
+        let file_type = match &self.file_type {
+            Some(x) => x.clone(),
+            None => match self.input.extension().and_then(|ext: &OsStr| ext.to_str()) {
+                Some("x2m") => XimalayaType::X2M,
+                Some("x3m") => XimalayaType::X3M,
+                Some("xm") => XimalayaType::XM,
+                Some(ext) => bail!("invalid ext: {ext}"),
+                _ => bail!("ext not found"),
+            },
+        };
+        let mut read_stream = BufReader::with_capacity(cli.buffer_size, File::open(&self.input)?);
+        let mut write_stream =
+            BufWriter::with_capacity(cli.buffer_size, File::create(&self.output)?);
+
+        match file_type {
+            XimalayaType::X2M | XimalayaType::X3M => {
+                let android_type = match file_type {
+                    XimalayaType::X2M => umc_xmly::android::FileType::X2M,
+                    XimalayaType::X3M => umc_xmly::android::FileType::X3M,
+                    _ => bail!("this should not happen"),
+                };
+                let mut header = [0u8; 0x400];
+                read_stream.read_exact(&mut header)?;
+                umc_xmly::android::decrypt_android(android_type, &mut header);
+                write_stream.write_all(&header)?;
+                io::copy(&mut read_stream, &mut write_stream)?;
+            }
+            XimalayaType::XM => {
+                let mut header = vec![0u8; 1024];
+                read_stream.read_exact(&mut header)?;
+                let xm_file = match umc_xmly::pc::Header::from_buffer(&header) {
+                    Ok(hdr) => hdr,
+                    Err(umc_xmly::XmlyError::MetadataTooSmall(n)) => {
+                        let old_size = header.len();
+                        header.resize(n, 0);
+                        read_stream.read_exact(&mut header[old_size..])?;
+                        umc_xmly::pc::Header::from_buffer(&header)?
+                    }
+                    Err(err) => bail!("failed to parse file: {err}"),
+                };
+
+                // Copy header
+                write_stream.write_all(xm_file.copy_m4a_header().as_slice())?;
+
+                // Process encrypted data
+                read_stream.seek(SeekFrom::Start(xm_file.data_start_offset as u64))?;
+                let mut header = vec![0u8; xm_file.encrypted_header_size];
+                read_stream.read_exact(&mut header[..])?;
+                write_stream.write_all(xm_file.decrypt(&mut header[..])?)?;
+
+                // Copy rest of the file
+                io::copy(&mut read_stream, &mut write_stream)?;
+            }
+        }
+
+        Ok(0)
+    }
+}

um_cli/src/main.rs

@@ -32,6 +32,7 @@ fn run_command(cli: &Cli) -> Result<i32> {
         Some(Commands::NCM(cmd)) => cmd.run(&cli),
         Some(Commands::KGM(cmd)) => cmd.run(&cli),
         Some(Commands::JOOX(cmd)) => cmd.run(&cli),
+        Some(Commands::XMLY(cmd)) => cmd.run(&cli),
         None => {
             // https://github.com/clap-rs/clap/issues/3857#issuecomment-1161796261
             todo!("implement a sensible default command, similar to um/cli");

um_crypto/utils/src/base64.rs

@@ -22,3 +22,14 @@ where
 {
     ENGINE.decode(data)
 }
+
+pub fn decode_overwrite<T>(data: &mut T) -> Result<&[u8], DecodeError>
+where
+    T: AsMut<[u8]> + ?Sized,
+{
+    let data = data.as_mut();
+    let decoded = decode(&mut data[..])?;
+    let len = decoded.len();
+    data[..len].copy_from_slice(&decoded);
+    Ok(&data[..len])
+}

um_crypto/xmly/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "umc_xmly"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+aes = "0.8.4"
+byteorder = "1.5.0"
+cbc = "0.1.2"
+cipher = "0.4.4"
+hex = "0.4.3"
+lazy_static = "1.5.0"
+thiserror = "1.0.63"
+umc_utils = { path = "../utils" }

um_crypto/xmly/src/__fixture__/.gitignore

@@ -0,0 +1,2 @@
+/sample.xm
+*.m4a

um_crypto/xmly/src/android.rs

@@ -0,0 +1,75 @@
+use lazy_static::lazy_static;
+
+pub fn derive_table<const N: usize>(init: f64, step: f64) -> [usize; N] {
+    debug_assert!(step > 0.0);
+    debug_assert!(init > 0.0);
+
+    let mut result = [0usize; N];
+
+    let mut temp = init;
+    let mut data = [0f64; N];
+    for datum in data.iter_mut() {
+        *datum = temp;
+        temp = temp * step * (1.0 - temp);
+    }
+
+    let mut sorted = data;
+    sorted.sort_unstable_by(|a, b| a.total_cmp(b));
+
+    for (item, needle) in result.iter_mut().zip(data) {
+        let idx = sorted
+            .iter()
+            .position(|&x| x == needle)
+            .expect("could not find item");
+        *item = idx;
+        sorted[idx] = -f64::NAN; // values can not be negative, so...
+    }
+
+    result
+}
+
+lazy_static! {
+    static ref TABLE_X2M: [usize; 0x400] = derive_table(0.615243, 3.837465);
+    static ref TABLE_X3M: [usize; 0x400] = derive_table(0.726354, 3.948576);
+}
+
+pub enum FileType {
+    X2M,
+    X3M,
+}
+
+/// Decrypt the first 0x400 bytes.
+pub fn decrypt_android(version: FileType, buffer: &mut [u8; 0x400]) {
+    let (content_key, scramble_table) = match version {
+        FileType::X2M => (*b"xmlyxmlyxmlyxmlyxmlyxmlyxmlyxmly", &*TABLE_X2M),
+        FileType::X3M => (*b"3989d111aad5613940f4fc44b639b292", &*TABLE_X3M),
+    };
+
+    let src = *buffer;
+    for (i, hdr) in buffer.iter_mut().enumerate() {
+        *hdr = src[scramble_table[i % scramble_table.len()]] ^ content_key[i % content_key.len()];
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_x2m() {
+        let mut buffer = *include_bytes!("__fixture__/x2m_hdr.bin");
+        let expected = *include_bytes!("__fixture__/x2m_hdr_plain.bin");
+        let buffer_slice: &mut [u8] = &mut buffer;
+        decrypt_android(FileType::X2M, buffer_slice.try_into().unwrap());
+        assert_eq!(expected, buffer);
+    }
+
+    #[test]
+    fn test_x3m() {
+        let mut buffer = *include_bytes!("__fixture__/x3m_hdr.bin");
+        let expected = *include_bytes!("__fixture__/x3m_hdr_plain.bin");
+        let buffer_slice: &mut [u8] = &mut buffer;
+        decrypt_android(FileType::X3M, buffer_slice.try_into().unwrap());
+        assert_eq!(expected, buffer);
+    }
+}

um_crypto/xmly/src/lib.rs

@@ -0,0 +1,43 @@
+use cipher::block_padding::UnpadError;
+use cipher::InvalidLength;
+use thiserror::Error;
+use umc_utils::base64::DecodeError;
+
+pub mod android;
+pub mod pc;
+
+#[derive(Error, Debug)]
+pub enum XmlyError {
+    #[error("Expected ID3 metadata")]
+    MetadataMissing,
+
+    #[error("ID3 Metadata too small (require {0} bytes)")]
+    MetadataTooSmall(usize),
+
+    #[error("Failed to extract encrypted audio segment size")]
+    ParseHeaderSizeError,
+
+    #[error("Failed to extract Stage 1 IV data")]
+    ParseStage1IVError,
+
+    #[error("Failed to extract Stage 2 decryption key")]
+    ParseStage2KeyError,
+
+    #[error("Failed to extract audio header")]
+    ParseAudioHeaderError,
+
+    #[error("Decryption stage 1 failed (padding)")]
+    DecryptStage1Error(UnpadError),
+
+    #[error("Decryption stage 1 failed (b64 decode)")]
+    DecryptStage1B64Error(DecodeError),
+
+    #[error("Decryption stage 2 failed (init)")]
+    InitStage2Error(InvalidLength),
+
+    #[error("Decryption stage 2 failed (padding)")]
+    DecryptStage2Error(UnpadError),
+
+    #[error("Decryption stage 2 failed (b64 decode)")]
+    DecryptStage2B64Error(DecodeError),
+}

um_crypto/xmly/src/pc.rs

@@ -0,0 +1,196 @@
+use crate::XmlyError;
+use aes::cipher::generic_array::GenericArray;
+use aes::cipher::BlockDecryptMut;
+use aes::{Aes192Dec, Aes256Dec};
+use byteorder::{ByteOrder, BE};
+use cbc::cipher::KeyIvInit;
+use cbc::Decryptor;
+use cipher::block_padding::Pkcs7;
+use umc_utils::base64;
+
+type Aes256CbcDec = Decryptor<Aes256Dec>;
+type Aes192CbcDec = Decryptor<Aes192Dec>;
+
+fn parse_safe_sync_int(v: u32) -> u32 {
+    ((v & 0x7f00_0000) >> 3)
+        | ((v & 0x007f_0000) >> 2)
+        | ((v & 0x0000_7f00) >> 1)
+        | (v & 0x0000_007f)
+}
+
+fn from_unicode(buf: &[u8]) -> String {
+    let data = buf
+        .iter()
+        .step_by(2)
+        .map_while(|&b| match b {
+            0 => None,
+            b => Some(b),
+        })
+        .collect::<Vec<u8>>();
+    String::from_utf8_lossy(&data[..]).to_string()
+}
+
+pub struct Header {
+    pub data_start_offset: usize,
+    pub encrypted_header_size: usize,
+    stage1_iv: [u8; 0x10],
+    stage2_key: [u8; 0x18],
+    m4a_header: Vec<u8>,
+}
+
+impl Header {
+    pub fn from_buffer(buffer: &[u8]) -> Result<Self, XmlyError> {
+        if buffer.len() < 10 {
+            Err(XmlyError::MetadataTooSmall(10))?;
+        }
+        if &buffer[0..3] != b"ID3" {
+            Err(XmlyError::MetadataMissing)?;
+        }
+
+        let header_size = parse_safe_sync_int(BE::read_u32(&buffer[6..10]));
+        let data_start_offset = 10 + header_size as usize;
+
+        if buffer.len() < data_start_offset {
+            Err(XmlyError::MetadataTooSmall(data_start_offset))?;
+        }
+
+        let mut encrypted_header_size = None;
+        let mut stage1_iv = None;
+        let mut stage2_key = None;
+        let mut m4a_header = None;
+
+        let mut offset = 10;
+        while offset < data_start_offset {
+            // Safety check
+            if offset + 10 > buffer.len() {
+                break;
+            }
+
+            let tag_name: &[u8; 4] = &buffer[offset..offset + 4].try_into().unwrap();
+            offset += 4;
+
+            let tag_value_size = BE::read_u32(&buffer[offset..offset + 4]) as usize;
+            offset += 4;
+
+            // flags, ignore
+            offset += 2;
+
+            if offset + tag_value_size > buffer.len() {
+                Err(XmlyError::MetadataTooSmall(offset + tag_value_size))?;
+            }
+            let data = &buffer[offset + 3..offset + tag_value_size];
+            offset += tag_value_size;
+
+            match tag_name {
+                b"TSIZ" => {
+                    let data = from_unicode(data)
+                        .parse::<usize>()
+                        .map_err(|_| XmlyError::ParseHeaderSizeError)?;
+                    encrypted_header_size = Some(data);
+                }
+                b"TSRC" | b"TENC" => {
+                    let data = hex::decode(from_unicode(data))
+                        .map_err(|_| XmlyError::ParseStage1IVError)?
+                        .try_into()
+                        .map_err(|_| XmlyError::ParseStage1IVError)?;
+                    stage1_iv = Some(data)
+                }
+                b"TSSE" => {
+                    let data = base64::decode(from_unicode(data))
+                        .map_err(|_| XmlyError::ParseAudioHeaderError)?;
+                    m4a_header = Some(data);
+                }
+                b"TRCK" => {
+                    let data = from_unicode(data);
+                    let tmp = data.as_bytes();
+
+                    let mut key = *b"123456781234567812345678";
+                    key[24 - tmp.len()..].copy_from_slice(tmp);
+                    stage2_key = Some(key);
+                }
+                // ignore unknown tags
+                _ => {}
+            }
+        }
+
+        Ok(Self {
+            data_start_offset,
+            encrypted_header_size: encrypted_header_size.ok_or(XmlyError::ParseHeaderSizeError)?,
+            stage1_iv: stage1_iv.ok_or(XmlyError::ParseStage1IVError)?,
+            stage2_key: stage2_key.ok_or(XmlyError::ParseStage2KeyError)?,
+            m4a_header: m4a_header.ok_or(XmlyError::ParseAudioHeaderError)?,
+        })
+    }
+
+    const STAGE1_KEY: [u8; 32] = *b"ximalayaximalayaximalayaximalaya";
+    fn decrypt_stage1<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a [u8], XmlyError> {
+        let key = GenericArray::from(Self::STAGE1_KEY);
+        let iv = GenericArray::from(self.stage1_iv);
+        let aes = Aes256CbcDec::new(&key, &iv);
+
+        let len = aes
+            .decrypt_padded_mut::<Pkcs7>(buffer)
+            .map_err(XmlyError::DecryptStage1Error)?
+            .len();
+        base64::decode_overwrite(&mut buffer[..len]).map_err(XmlyError::DecryptStage1B64Error)
+    }
+
+    fn decrypt_stage2<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a [u8], XmlyError> {
+        let key = &self.stage2_key[..24];
+        let iv = &self.stage2_key[..16];
+        let aes = Aes192CbcDec::new_from_slices(key, iv).map_err(XmlyError::InitStage2Error)?;
+
+        let len = aes
+            .decrypt_padded_mut::<Pkcs7>(buffer)
+            .map_err(XmlyError::DecryptStage2Error)?
+            .len();
+        base64::decode_overwrite(&mut buffer[..len]).map_err(XmlyError::DecryptStage2B64Error)
+    }
+
+    pub fn decrypt<'a>(&self, buffer: &'a mut [u8]) -> Result<&'a [u8], XmlyError> {
+        let len = self.decrypt_stage1(&mut buffer[..])?.len();
+        self.decrypt_stage2(&mut buffer[..len])
+    }
+
+    pub fn copy_m4a_header(&self) -> Vec<u8> {
+        self.m4a_header.clone()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::pc::Header;
+    use crate::XmlyError;
+    use std::env;
+    use std::fs;
+    use std::fs::File;
+    use std::io::Write;
+
+    #[test]
+    fn test_sample_xm() -> Result<(), XmlyError> {
+        let manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not set");
+        let sample_path = format!("{}/src/__fixture__/sample.xm", manifest_dir);
+        let sample_out_path = format!("{}/src/__fixture__/sample.m4a", manifest_dir);
+
+        println!("decrypt {} -> {}", sample_path, sample_out_path);
+
+        if let Ok(mut xm) = fs::read(sample_path) {
+            let file = match Header::from_buffer(&xm[..1024]) {
+                Err(XmlyError::MetadataTooSmall(n)) => Header::from_buffer(&xm[..n])?,
+                Err(err) => Err(err)?,
+                Ok(x) => x,
+            };
+
+            let (_hdr, buffer) = xm.split_at_mut(file.data_start_offset);
+            let (buffer, plain) = buffer.split_at_mut(file.encrypted_header_size);
+
+            let decrypted = file.decrypt(buffer)?;
+            let mut f_out = File::create(sample_out_path).expect("can't open test output file");
+            f_out.write_all(&file.copy_m4a_header()).expect("header");
+            f_out.write_all(decrypted).expect("decrypted part");
+            f_out.write_all(plain).expect("plain part");
+        }
+
+        Ok(())
+    }
+}