From 825784e60510c5360fdab17c8c36bfdb71453f6c Mon Sep 17 00:00:00 2001 From: pemako Date: Sat, 3 Sep 2022 00:35:56 +0800 Subject: [PATCH] feat: imporve code --- algo/common/dispatch.go | 1 + algo/common/sniff.go | 4 +++ algo/kgm/kgm.go | 6 +++++ algo/kgm/mask.go | 5 ++-- algo/kwm/kwm.go | 9 +++++-- algo/ncm/meta.go | 9 ++++--- algo/ncm/ncm.go | 19 ++++++++++--- algo/qmc/cipher_map_test.go | 1 + algo/qmc/cipher_rc4.go | 5 +++- algo/qmc/cipher_static.go | 1 + algo/qmc/key_dec.go | 11 ++++++-- algo/qmc/key_dec_test.go | 3 +++ algo/qmc/qmc.go | 7 +++++ algo/qmc/qmc_test.go | 3 +-- algo/tm/tm.go | 2 +- algo/xm/xm.go | 3 +++ cmd/um/main.go | 53 ++++++++++++++++++++++--------------- internal/logging/logging.go | 2 -- 18 files changed, 104 insertions(+), 40 deletions(-) diff --git a/algo/common/dispatch.go b/algo/common/dispatch.go index 71ce298..95b1425 100644 --- a/algo/common/dispatch.go +++ b/algo/common/dispatch.go @@ -18,6 +18,7 @@ func RegisterDecoder(ext string, noop bool, dispatchFunc NewDecoderFunc) { DecoderRegistry[ext] = append(DecoderRegistry[ext], decoderItem{noop: noop, decoder: dispatchFunc}) } + func GetDecoder(filename string, skipNoop bool) (rs []NewDecoderFunc) { ext := strings.ToLower(strings.TrimLeft(filepath.Ext(filename), ".")) for _, dec := range DecoderRegistry[ext] { diff --git a/algo/common/sniff.go b/algo/common/sniff.go index 3370305..58ea43a 100644 --- a/algo/common/sniff.go +++ b/algo/common/sniff.go @@ -35,15 +35,19 @@ func SnifferOGG(header []byte) bool { func SnifferFLAC(header []byte) bool { return bytes.HasPrefix(header, []byte("fLaC")) } + func SnifferMP3(header []byte) bool { return bytes.HasPrefix(header, []byte("ID3")) } + func SnifferWAV(header []byte) bool { return bytes.HasPrefix(header, []byte("RIFF")) } + func SnifferWMA(header []byte) bool { return bytes.HasPrefix(header, []byte("\x30\x26\xb2\x75\x8e\x66\xcf\x11\xa6\xd9\x00\xaa\x00\x62\xce\x6c")) } + func SnifferAAC(header []byte) bool { return bytes.HasPrefix(header, []byte{0xFF, 0xF1}) } diff --git a/algo/kgm/kgm.go b/algo/kgm/kgm.go index 37595c0..afebc41 100644 --- a/algo/kgm/kgm.go +++ b/algo/kgm/kgm.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "errors" + "github.com/unlock-music/cli/algo/common" "github.com/unlock-music/cli/internal/logging" ) @@ -59,6 +60,7 @@ func (d *Decoder) Validate() error { d.key = d.file[0x1c:0x2c] d.key = append(d.key, 0x00) _ = d.file[0x2c:0x3c] //todo: key2 + return nil } @@ -67,6 +69,7 @@ func (d *Decoder) Decode() error { dataEncrypted := d.file[headerLen:] lenData := len(dataEncrypted) initMask() + if fullMaskLen < lenData { logging.Log().Warn("The file is too large and the processed audio is incomplete, " + "please report to us about this file at https://github.com/unlock-music/cli/issues") @@ -78,13 +81,16 @@ func (d *Decoder) Decode() error { med8 := dataEncrypted[i] ^ d.key[i%17] ^ maskV2PreDef[i%(16*17)] ^ maskV2[i>>4] d.audio[i] = med8 ^ (med8&0xf)<<4 } + if d.isVpr { for i := 0; i < lenData; i++ { d.audio[i] ^= maskDiffVpr[i%17] } } + return nil } + func init() { // Kugou common.RegisterDecoder("kgm", false, NewDecoder) diff --git a/algo/kgm/mask.go b/algo/kgm/mask.go index dc9fa62..45309dc 100644 --- a/algo/kgm/mask.go +++ b/algo/kgm/mask.go @@ -3,10 +3,11 @@ package kgm import ( "bytes" _ "embed" + "io" + "github.com/ulikunitz/xz" "github.com/unlock-music/cli/internal/logging" "go.uber.org/zap" - "io" ) var maskDiffVpr = []byte{ @@ -41,7 +42,7 @@ var maskV2 []byte var fullMaskLen int var initMaskOK = false -//todo: decompress mask on demand +// TODO: decompress mask on demand func initMask() { if initMaskOK { return diff --git a/algo/kwm/kwm.go b/algo/kwm/kwm.go index a9fa2d6..8103803 100644 --- a/algo/kwm/kwm.go +++ b/algo/kwm/kwm.go @@ -4,10 +4,11 @@ import ( "bytes" "encoding/binary" "errors" - "github.com/unlock-music/cli/algo/common" "strconv" "strings" "unicode" + + "github.com/unlock-music/cli/algo/common" ) var ( @@ -57,6 +58,7 @@ func (d *Decoder) Validate() error { if lenData < 1024 { return ErrKwFileSize } + if !bytes.Equal(magicHeader, d.file[:16]) { return ErrKwMagicHeader } @@ -69,9 +71,11 @@ func generateMask(key []byte) []byte { keyStr := strconv.FormatUint(keyInt, 10) keyStrTrim := padOrTruncate(keyStr, 32) mask := make([]byte, 32) + for i := 0; i < 32; i++ { mask[i] = keyPreDefined[i] ^ keyStrTrim[i] } + return mask } @@ -83,13 +87,13 @@ func (d *Decoder) parseBitrateAndType() { break } } + var err error d.bitrate, err = strconv.Atoi(bitType[:charPos]) if err != nil { d.bitrate = 0 } d.outputExt = strings.ToLower(bitType[charPos:]) - } func (d *Decoder) Decode() error { @@ -102,6 +106,7 @@ func (d *Decoder) Decode() error { for i := 0; i < dataLen; i++ { d.audio[i] ^= d.mask[i&0x1F] //equals: [i % 32] } + return nil } diff --git a/algo/ncm/meta.go b/algo/ncm/meta.go index 58060b3..adacd9a 100644 --- a/algo/ncm/meta.go +++ b/algo/ncm/meta.go @@ -1,8 +1,9 @@ package ncm import ( - "github.com/unlock-music/cli/algo/common" "strings" + + "github.com/unlock-music/cli/algo/common" ) type RawMeta interface { @@ -10,6 +11,7 @@ type RawMeta interface { GetFormat() string GetAlbumImageURL() string } + type RawMetaMusic struct { Format string `json:"format"` MusicID int `json:"musicId"` @@ -30,11 +32,11 @@ type RawMetaMusic struct { func (m RawMetaMusic) GetAlbumImageURL() string { return m.AlbumPic } + func (m RawMetaMusic) GetArtists() (artists []string) { for _, artist := range m.Artist { for _, item := range artist { - name, ok := item.(string) - if ok { + if name, ok := item.(string); ok { artists = append(artists, name) } } @@ -49,6 +51,7 @@ func (m RawMetaMusic) GetTitle() string { func (m RawMetaMusic) GetAlbum() string { return m.Album } + func (m RawMetaMusic) GetFormat() string { return m.Format } diff --git a/algo/ncm/ncm.go b/algo/ncm/ncm.go index 80da8b3..882d56b 100644 --- a/algo/ncm/ncm.go +++ b/algo/ncm/ncm.go @@ -6,13 +6,14 @@ import ( "encoding/binary" "encoding/json" "errors" + "io/ioutil" + "net/http" + "strings" + "github.com/unlock-music/cli/algo/common" "github.com/unlock-music/cli/internal/logging" "github.com/unlock-music/cli/internal/utils" "go.uber.org/zap" - "io/ioutil" - "net/http" - "strings" ) var ( @@ -65,6 +66,7 @@ func (d *Decoder) readKeyData() error { if d.offsetKey == 0 || d.offsetKey+4 > d.fileLen { return errors.New("invalid cover file offset") } + bKeyLen := d.file[d.offsetKey : d.offsetKey+4] iKeyLen := binary.LittleEndian.Uint32(bKeyLen) d.offsetMeta = d.offsetKey + 4 + iKeyLen @@ -82,9 +84,11 @@ func (d *Decoder) readMetaData() error { if d.offsetMeta == 0 || d.offsetMeta+4 > d.fileLen { return errors.New("invalid meta file offset") } + bMetaLen := d.file[d.offsetMeta : d.offsetMeta+4] iMetaLen := binary.LittleEndian.Uint32(bMetaLen) d.offsetCover = d.offsetMeta + 4 + iMetaLen + if iMetaLen == 0 { return errors.New("no any meta file found") } @@ -159,7 +163,9 @@ func (d *Decoder) readCoverData() error { if iCoverLen == 0 { return errors.New("no any cover file found") } + d.cover = d.file[coverLenStart+4 : 4+coverLenStart+iCoverLen] + return nil } @@ -167,6 +173,7 @@ func (d *Decoder) readAudioData() error { if d.offsetAudio == 0 || d.offsetAudio > d.fileLen { return errors.New("invalid audio offset") } + audioRaw := d.file[d.offsetAudio:] audioLen := len(audioRaw) d.audio = make([]byte, audioLen) @@ -186,6 +193,7 @@ func (d *Decoder) Decode() error { if err == nil { err = d.parseMeta() } + if err != nil { logging.Log().Warn("parse ncm meta file failed", zap.Error(err)) } @@ -215,27 +223,32 @@ func (d Decoder) GetCoverImage() []byte { if d.cover != nil { return d.cover } + { imgURL := d.meta.GetAlbumImageURL() if d.meta != nil && !strings.HasPrefix(imgURL, "http") { return nil } + resp, err := http.Get(imgURL) if err != nil { logging.Log().Warn("download image failed", zap.Error(err), zap.String("url", imgURL)) return nil } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { logging.Log().Warn("download image failed", zap.String("http", resp.Status), zap.String("url", imgURL)) return nil } + data, err := ioutil.ReadAll(resp.Body) if err != nil { logging.Log().Warn("download image failed", zap.Error(err), zap.String("url", imgURL)) return nil } + return data } } diff --git a/algo/qmc/cipher_map_test.go b/algo/qmc/cipher_map_test.go index 400ef94..d61856b 100644 --- a/algo/qmc/cipher_map_test.go +++ b/algo/qmc/cipher_map_test.go @@ -22,6 +22,7 @@ func loadTestDataMapCipher(name string) ([]byte, []byte, []byte, error) { } return key, raw, target, nil } + func Test_mapCipher_Decrypt(t *testing.T) { tests := []struct { diff --git a/algo/qmc/cipher_rc4.go b/algo/qmc/cipher_rc4.go index 791d6da..9a37bab 100644 --- a/algo/qmc/cipher_rc4.go +++ b/algo/qmc/cipher_rc4.go @@ -4,7 +4,7 @@ import ( "errors" ) -// A rc4Cipher is an instance of RC4 using a particular key. +// rc4Cipher is an instance of RC4 using a particular key. type rc4Cipher struct { box []byte key []byte @@ -87,6 +87,7 @@ func (c *rc4Cipher) Decrypt(src []byte, offset int) { return } } + for toProcess > rc4SegmentSize { c.encASegment(src[processed:processed+rc4SegmentSize], offset) markProcess(rc4SegmentSize) @@ -96,6 +97,7 @@ func (c *rc4Cipher) Decrypt(src []byte, offset int) { c.encASegment(src[processed:], offset) } } + func (c *rc4Cipher) encFirstSegment(buf []byte, offset int) { for i := 0; i < len(buf); i++ { buf[i] ^= c.key[c.getSegmentSkip(offset+i)] @@ -117,6 +119,7 @@ func (c *rc4Cipher) encASegment(buf []byte, offset int) { } } } + func (c *rc4Cipher) getSegmentSkip(id int) int { seed := int(c.key[id%c.n]) idx := int64(float64(c.hash) / float64((id+1)*seed) * 100.0) diff --git a/algo/qmc/cipher_static.go b/algo/qmc/cipher_static.go index f6db30b..09377a6 100644 --- a/algo/qmc/cipher_static.go +++ b/algo/qmc/cipher_static.go @@ -13,6 +13,7 @@ func (c *staticCipher) Decrypt(buf []byte, offset int) { buf[i] ^= c.getMask(offset + i) } } + func (c *staticCipher) getMask(offset int) byte { if offset > 0x7FFF { offset %= 0x7FFF diff --git a/algo/qmc/key_dec.go b/algo/qmc/key_dec.go index 9f7ddf8..eae01c1 100644 --- a/algo/qmc/key_dec.go +++ b/algo/qmc/key_dec.go @@ -16,17 +16,19 @@ func simpleMakeKey(salt byte, length int) []byte { } return keyBuf } + func DecryptKey(rawKey []byte) ([]byte, error) { rawKeyDec := make([]byte, base64.StdEncoding.DecodedLen(len(rawKey))) n, err := base64.StdEncoding.Decode(rawKeyDec, rawKey) if err != nil { return nil, err } + if n < 16 { return nil, errors.New("key length is too short") } - rawKeyDec = rawKeyDec[:n] + rawKeyDec = rawKeyDec[:n] simpleKey := simpleMakeKey(106, 8) teaKey := make([]byte, 16) for i := 0; i < 8; i++ { @@ -38,14 +40,17 @@ func DecryptKey(rawKey []byte) ([]byte, error) { if err != nil { return nil, err } + return append(rawKeyDec[:8], rs...), nil } + func decryptTencentTea(inBuf []byte, key []byte) ([]byte, error) { const saltLen = 2 const zeroLen = 7 if len(inBuf)%8 != 0 { return nil, errors.New("inBuf size not a multiple of the block size") } + if len(inBuf) < 16 { return nil, errors.New("inBuf size too small") } @@ -62,8 +67,8 @@ func decryptTencentTea(inBuf []byte, key []byte) ([]byte, error) { if padLen+saltLen != 8 { return nil, errors.New("invalid pad len") } - out := make([]byte, outLen) + out := make([]byte, outLen) ivPrev := make([]byte, 8) ivCur := inBuf[:8] @@ -80,6 +85,7 @@ func decryptTencentTea(inBuf []byte, key []byte) ([]byte, error) { inBufPos += 8 destIdx = 0 } + for i := 1; i <= saltLen; { if destIdx < 8 { destIdx++ @@ -108,6 +114,7 @@ func decryptTencentTea(inBuf []byte, key []byte) ([]byte, error) { return out, nil } + func xor8Bytes(dst, a, b []byte) { for i := 0; i < 8; i++ { dst[i] = a[i] ^ b[i] diff --git a/algo/qmc/key_dec_test.go b/algo/qmc/key_dec_test.go index 4ebf8fd..919de20 100644 --- a/algo/qmc/key_dec_test.go +++ b/algo/qmc/key_dec_test.go @@ -15,6 +15,7 @@ func TestSimpleMakeKey(t *testing.T) { } }) } + func loadDecryptKeyData(name string) ([]byte, []byte, error) { keyRaw, err := os.ReadFile(fmt.Sprintf("./testdata/%s_key_raw.bin", name)) if err != nil { @@ -26,6 +27,7 @@ func loadDecryptKeyData(name string) ([]byte, []byte, error) { } return keyRaw, keyDec, nil } + func TestDecryptKey(t *testing.T) { tests := []struct { name string @@ -37,6 +39,7 @@ func TestDecryptKey(t *testing.T) { {"mflac_rc4(256)", "mflac_rc4", false}, {"mgg_map(256)", "mgg_map", false}, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { raw, want, err := loadDecryptKeyData(tt.filename) diff --git a/algo/qmc/qmc.go b/algo/qmc/qmc.go index de7b06e..a7082ec 100644 --- a/algo/qmc/qmc.go +++ b/algo/qmc/qmc.go @@ -100,10 +100,12 @@ func (d *Decoder) searchKey() error { if err != nil { return err } + buf, err := io.ReadAll(io.LimitReader(d.r, 4)) if err != nil { return err } + if string(buf) == "QTag" { if err := d.readRawMetaQTag(); err != nil { return err @@ -118,6 +120,7 @@ func (d *Decoder) searchKey() error { return nil } } + return nil } @@ -146,10 +149,12 @@ func (d *Decoder) readRawMetaQTag() error { if _, err := d.r.Seek(-8, io.SeekEnd); err != nil { return err } + buf, err := io.ReadAll(io.LimitReader(d.r, 4)) if err != nil { return err } + rawMetaLen := int64(binary.BigEndian.Uint32(buf)) // read raw meta data @@ -157,6 +162,7 @@ func (d *Decoder) readRawMetaQTag() error { if err != nil { return err } + d.audioLen = int(audioLen) rawMetaData, err := io.ReadAll(io.LimitReader(d.r, rawMetaLen)) if err != nil { @@ -177,6 +183,7 @@ func (d *Decoder) readRawMetaQTag() error { if err != nil { return err } + d.rawMetaExtra2, err = strconv.Atoi(items[2]) if err != nil { return err diff --git a/algo/qmc/qmc_test.go b/algo/qmc/qmc_test.go index 11bbbaf..0574e72 100644 --- a/algo/qmc/qmc_test.go +++ b/algo/qmc/qmc_test.go @@ -24,8 +24,8 @@ func loadTestDataQmcDecoder(filename string) ([]byte, []byte, error) { return nil, nil, err } return bytes.Join([][]byte{encBody, encSuffix}, nil), target, nil - } + func TestMflac0Decoder_Read(t *testing.T) { tests := []struct { name string @@ -60,7 +60,6 @@ func TestMflac0Decoder_Read(t *testing.T) { } }) } - } func TestMflac0Decoder_Validate(t *testing.T) { diff --git a/algo/tm/tm.go b/algo/tm/tm.go index d1ccc18..b672c91 100644 --- a/algo/tm/tm.go +++ b/algo/tm/tm.go @@ -3,6 +3,7 @@ package tm import ( "bytes" "errors" + "github.com/unlock-music/cli/algo/common" ) @@ -75,5 +76,4 @@ func init() { // QQ Music IOS Mp3 common.RegisterDecoder("tm0", false, common.NewRawDecoder) common.RegisterDecoder("tm3", false, common.NewRawDecoder) - } diff --git a/algo/xm/xm.go b/algo/xm/xm.go index 42a88ba..7d5d70d 100644 --- a/algo/xm/xm.go +++ b/algo/xm/xm.go @@ -3,6 +3,7 @@ package xm import ( "bytes" "errors" + "github.com/unlock-music/cli/algo/common" "github.com/unlock-music/cli/internal/logging" "go.uber.org/zap" @@ -58,6 +59,7 @@ func (d *Decoder) Validate() error { if lenData < 16 { return ErrFileSize } + if !bytes.Equal(magicHeader, d.file[:4]) || !bytes.Equal(magicHeader2, d.file[8:12]) { return ErrMagicHeader @@ -76,6 +78,7 @@ func (d *Decoder) Validate() error { if d.headerLen+16 > uint32(lenData) { return ErrFileSize } + return nil } diff --git a/cmd/um/main.go b/cmd/um/main.go index 2382861..91f6cd4 100644 --- a/cmd/um/main.go +++ b/cmd/um/main.go @@ -9,17 +9,17 @@ import ( "sort" "strings" + "github.com/unlock-music/cli/algo/common" + "github.com/unlock-music/cli/internal/logging" "github.com/urfave/cli/v2" "go.uber.org/zap" - "github.com/unlock-music/cli/algo/common" _ "github.com/unlock-music/cli/algo/kgm" _ "github.com/unlock-music/cli/algo/kwm" _ "github.com/unlock-music/cli/algo/ncm" _ "github.com/unlock-music/cli/algo/qmc" _ "github.com/unlock-music/cli/algo/tm" _ "github.com/unlock-music/cli/algo/xm" - "github.com/unlock-music/cli/internal/logging" ) var AppVersion = "v0.0.6" @@ -43,26 +43,30 @@ func main() { HideHelpCommand: true, UsageText: "um [-o /path/to/output/dir] [--extra-flags] [-i] /path/to/input", } - err := app.Run(os.Args) - if err != nil { + + if err := app.Run(os.Args); err != nil { logging.Log().Fatal("run app failed", zap.Error(err)) } } + func printSupportedExtensions() { exts := []string{} for ext := range common.DecoderRegistry { exts = append(exts, ext) } + sort.Strings(exts) for _, ext := range exts { fmt.Printf("%s: %d\n", ext, len(common.DecoderRegistry[ext])) } } + func appMain(c *cli.Context) (err error) { if c.Bool("supported-ext") { printSupportedExtensions() return nil } + input := c.String("input") if input == "" { switch c.Args().Len() { @@ -90,9 +94,6 @@ func appMain(c *cli.Context) (err error) { } } - skipNoop := c.Bool("skip-noop") - removeSource := c.Bool("remove-source") - inputStat, err := os.Stat(input) if err != nil { return err @@ -110,26 +111,32 @@ func appMain(c *cli.Context) (err error) { return errors.New("output should be a writable directory") } + skipNoop := c.Bool("skip-noop") + removeSource := c.Bool("remove-source") + if inputStat.IsDir() { return dealDirectory(input, output, skipNoop, removeSource) - } else { - allDec := common.GetDecoder(inputStat.Name(), skipNoop) - if len(allDec) == 0 { - logging.Log().Fatal("skipping while no suitable decoder") - } - return tryDecFile(input, output, allDec, removeSource) } + allDec := common.GetDecoder(inputStat.Name(), skipNoop) + if len(allDec) == 0 { + logging.Log().Fatal("skipping while no suitable decoder") + } + + return tryDecFile(input, output, allDec, removeSource) } + func dealDirectory(inputDir string, outputDir string, skipNoop bool, removeSource bool) error { items, err := os.ReadDir(inputDir) if err != nil { return err } + for _, item := range items { if item.IsDir() { continue } + allDec := common.GetDecoder(item.Name(), skipNoop) if len(allDec) == 0 { logging.Log().Info("skipping while no suitable decoder", zap.String("file", item.Name())) @@ -141,6 +148,7 @@ func dealDirectory(inputDir string, outputDir string, skipNoop bool, removeSourc logging.Log().Error("conversion failed", zap.String("source", item.Name())) } } + return nil } @@ -159,9 +167,11 @@ func tryDecFile(inputFile string, outputDir string, allDec []common.NewDecoderFu logging.Log().Warn("try decode failed", zap.Error(err)) dec = nil } + if dec == nil { return errors.New("no any decoder can resolve the file") } + if err := dec.Decode(); err != nil { return errors.New("failed while decoding: " + err.Error()) } @@ -178,21 +188,20 @@ func tryDecFile(inputFile string, outputDir string, allDec []common.NewDecoderFu filenameOnly := strings.TrimSuffix(filepath.Base(inputFile), filepath.Ext(inputFile)) outPath := filepath.Join(outputDir, filenameOnly+outExt) - err = os.WriteFile(outPath, outData, 0644) - if err != nil { + if err = os.WriteFile(outPath, outData, 0644); err != nil { return err } // if source file need to be removed - if removeSource { - err := os.RemoveAll(inputFile) - if err != nil { - return err - } - logging.Log().Info("successfully converted, and source file is removed", zap.String("source", inputFile), zap.String("destination", outPath)) - } else { + if !removeSource { logging.Log().Info("successfully converted", zap.String("source", inputFile), zap.String("destination", outPath)) + return nil } + if err := os.RemoveAll(inputFile); err != nil { + return err + } + logging.Log().Info("successfully converted, and source file is removed", zap.String("source", inputFile), zap.String("destination", outPath)) + return nil } diff --git a/internal/logging/logging.go b/internal/logging/logging.go index b14012d..6ca885f 100644 --- a/internal/logging/logging.go +++ b/internal/logging/logging.go @@ -4,9 +4,7 @@ import ( "os" "sync" "time" -) -import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" )