diff --git a/pokedex/__init__.py b/pokedex/__init__.py index 7067fd3..2be0f80 100644 --- a/pokedex/__init__.py +++ b/pokedex/__init__.py @@ -3,6 +3,7 @@ from optparse import OptionParser import os import sys +# XXX importing pokedex.whatever should not import all these import pokedex.db import pokedex.db.load import pokedex.db.tables diff --git a/pokedex/savefile.py b/pokedex/struct/__init__.py similarity index 94% rename from pokedex/savefile.py rename to pokedex/struct/__init__.py index a3c4090..d1142d0 100644 --- a/pokedex/savefile.py +++ b/pokedex/struct/__init__.py @@ -9,7 +9,9 @@ derived. """ import struct + from pokedex.util import permutations +from pokedex.struct._pokemon_struct import pokemon_struct def pokemon_prng(seed): u"""Creates a generator that simulates the main Pokémon PRNG.""" @@ -19,14 +21,11 @@ def pokemon_prng(seed): yield seed >> 16 -class PokemonSave(object): +class SaveFilePokemon(object): u"""Represents an individual Pokémon, from the game's point of view. Handles translating between the on-disk encrypted form, the in-RAM blob (also used by pokesav), and something vaguely intelligible. - - XXX: Okay, well, right now it's just encryption and decryption. But, you - know. """ def __init__(self, blob, encrypted=False): @@ -37,9 +36,6 @@ class PokemonSave(object): is left alone. """ - # XXX Sometime this should have an abstract internal representation. - # For now, just store the decrypted version - if encrypted: # Decrypt it. # Interpret as one word (pid), followed by a bunch of shorts @@ -56,6 +52,8 @@ class PokemonSave(object): # Already decrypted self.blob = blob + self.structure = pokemon_struct.parse(self.blob) + @property def as_struct(self): diff --git a/pokedex/struct/_pokemon_struct.py b/pokedex/struct/_pokemon_struct.py new file mode 100644 index 0000000..d863718 --- /dev/null +++ b/pokedex/struct/_pokemon_struct.py @@ -0,0 +1,761 @@ +# encoding: utf8 +u"""Defines a construct `pokemon_struct`, containing the structure of a single +Pokémon saved within a game -- often seen as a .pkm file. This is the same +format sent back and forth over the GTS. +""" + +import datetime + +from construct import * + +# TODO: +# - strings should be validated, going both in and out +# - strings need to pad themselves when being re-encoded +# - strings sometimes need specific padding christ +# - date_met is not optional +# - some way to be more lenient with junk data, or at least +# - higher-level validation; see XXXes below +# - personality indirectly influences IVs due to PRNG use + +# The entire gen 4 character table: +character_table = { + 0x0002: u'ぁ', + 0x0003: u'あ', + 0x0004: u'ぃ', + 0x0005: u'い', + 0x0006: u'ぅ', + 0x0007: u'う', + 0x0008: u'ぇ', + 0x0009: u'え', + 0x000a: u'ぉ', + 0x000b: u'お', + 0x000c: u'か', + 0x000d: u'が', + 0x000e: u'き', + 0x000f: u'ぎ', + 0x0010: u'く', + 0x0011: u'ぐ', + 0x0012: u'け', + 0x0013: u'げ', + 0x0014: u'こ', + 0x0015: u'ご', + 0x0016: u'さ', + 0x0017: u'ざ', + 0x0018: u'し', + 0x0019: u'じ', + 0x001a: u'す', + 0x001b: u'ず', + 0x001c: u'せ', + 0x001d: u'ぜ', + 0x001e: u'そ', + 0x001f: u'ぞ', + 0x0020: u'た', + 0x0021: u'だ', + 0x0022: u'ち', + 0x0023: u'ぢ', + 0x0024: u'っ', + 0x0025: u'つ', + 0x0026: u'づ', + 0x0027: u'て', + 0x0028: u'で', + 0x0029: u'と', + 0x002a: u'ど', + 0x002b: u'な', + 0x002c: u'に', + 0x002d: u'ぬ', + 0x002e: u'ね', + 0x002f: u'の', + 0x0030: u'は', + 0x0031: u'ば', + 0x0032: u'ぱ', + 0x0033: u'ひ', + 0x0034: u'び', + 0x0035: u'ぴ', + 0x0036: u'ふ', + 0x0037: u'ぶ', + 0x0038: u'ぷ', + 0x0039: u'へ', + 0x003a: u'べ', + 0x003b: u'ぺ', + 0x003c: u'ほ', + 0x003d: u'ぼ', + 0x003e: u'ぽ', + 0x003f: u'ま', + 0x0040: u'み', + 0x0041: u'む', + 0x0042: u'め', + 0x0043: u'も', + 0x0044: u'ゃ', + 0x0045: u'や', + 0x0046: u'ゅ', + 0x0047: u'ゆ', + 0x0048: u'ょ', + 0x0049: u'よ', + 0x004a: u'ら', + 0x004b: u'り', + 0x004c: u'る', + 0x004d: u'れ', + 0x004e: u'ろ', + 0x004f: u'わ', + 0x0050: u'を', + 0x0051: u'ん', + 0x0052: u'ァ', + 0x0053: u'ア', + 0x0054: u'ィ', + 0x0055: u'イ', + 0x0056: u'ゥ', + 0x0057: u'ウ', + 0x0058: u'ェ', + 0x0059: u'エ', + 0x005a: u'ォ', + 0x005b: u'オ', + 0x005c: u'カ', + 0x005d: u'ガ', + 0x005e: u'キ', + 0x005f: u'ギ', + 0x0060: u'ク', + 0x0061: u'グ', + 0x0062: u'ケ', + 0x0063: u'ゲ', + 0x0064: u'コ', + 0x0065: u'ゴ', + 0x0066: u'サ', + 0x0067: u'ザ', + 0x0068: u'シ', + 0x0069: u'ジ', + 0x006a: u'ス', + 0x006b: u'ズ', + 0x006c: u'セ', + 0x006d: u'ゼ', + 0x006e: u'ソ', + 0x006f: u'ゾ', + 0x0070: u'タ', + 0x0071: u'ダ', + 0x0072: u'チ', + 0x0073: u'ヂ', + 0x0074: u'ッ', + 0x0075: u'ツ', + 0x0076: u'ヅ', + 0x0077: u'テ', + 0x0078: u'デ', + 0x0079: u'ト', + 0x007a: u'ド', + 0x007b: u'ナ', + 0x007c: u'ニ', + 0x007d: u'ヌ', + 0x007e: u'ネ', + 0x007f: u'ノ', + 0x0080: u'ハ', + 0x0081: u'バ', + 0x0082: u'パ', + 0x0083: u'ヒ', + 0x0084: u'ビ', + 0x0085: u'ピ', + 0x0086: u'フ', + 0x0087: u'ブ', + 0x0088: u'プ', + 0x0089: u'ヘ', + 0x008a: u'ベ', + 0x008b: u'ペ', + 0x008c: u'ホ', + 0x008d: u'ボ', + 0x008e: u'ポ', + 0x008f: u'マ', + 0x0090: u'ミ', + 0x0091: u'ム', + 0x0092: u'メ', + 0x0093: u'モ', + 0x0094: u'ャ', + 0x0095: u'ヤ', + 0x0096: u'ュ', + 0x0097: u'ユ', + 0x0098: u'ョ', + 0x0099: u'ヨ', + 0x009a: u'ラ', + 0x009b: u'リ', + 0x009c: u'ル', + 0x009d: u'レ', + 0x009e: u'ロ', + 0x009f: u'ワ', + 0x00a0: u'ヲ', + 0x00a1: u'ン', + 0x00a2: u'0', + 0x00a3: u'1', + 0x00a4: u'2', + 0x00a5: u'3', + 0x00a6: u'4', + 0x00a7: u'5', + 0x00a8: u'6', + 0x00a9: u'7', + 0x00aa: u'8', + 0x00ab: u'9', + 0x00ac: u'A', + 0x00ad: u'B', + 0x00ae: u'C', + 0x00af: u'D', + 0x00b0: u'E', + 0x00b1: u'F', + 0x00b2: u'G', + 0x00b3: u'H', + 0x00b4: u'I', + 0x00b5: u'J', + 0x00b6: u'K', + 0x00b7: u'L', + 0x00b8: u'M', + 0x00b9: u'N', + 0x00ba: u'O', + 0x00bb: u'P', + 0x00bc: u'Q', + 0x00bd: u'R', + 0x00be: u'S', + 0x00bf: u'T', + 0x00c0: u'U', + 0x00c1: u'V', + 0x00c2: u'W', + 0x00c3: u'X', + 0x00c4: u'Y', + 0x00c5: u'Z', + 0x00c6: u'a', + 0x00c7: u'b', + 0x00c8: u'c', + 0x00c9: u'd', + 0x00ca: u'e', + 0x00cb: u'f', + 0x00cc: u'g', + 0x00cd: u'h', + 0x00ce: u'i', + 0x00cf: u'j', + 0x00d0: u'k', + 0x00d1: u'l', + 0x00d2: u'm', + 0x00d3: u'n', + 0x00d4: u'o', + 0x00d5: u'p', + 0x00d6: u'q', + 0x00d7: u'r', + 0x00d8: u's', + 0x00d9: u't', + 0x00da: u'u', + 0x00db: u'v', + 0x00dc: u'w', + 0x00dd: u'x', + 0x00de: u'y', + 0x00df: u'z', + 0x00e0: u'à', + 0x00e1: u'!', + 0x00e2: u'?', + 0x00e3: u'、', + 0x00e4: u'。', + 0x00e5: u'…', + 0x00e6: u'・', + 0x00e7: u'/', + 0x00e8: u'「', + 0x00e9: u'」', + 0x00ea: u'『', + 0x00eb: u'』', + 0x00ec: u'(', + 0x00ed: u')', + 0x00ee: u'♂', + 0x00ef: u'♀', + 0x00f0: u'+', + 0x00f1: u'ー', + 0x00f2: u'×', + 0x00f3: u'÷', + 0x00f4: u'=', + 0x00f5: u'~', + 0x00f6: u':', + 0x00f7: u';', + 0x00f8: u'.', + 0x00f9: u',', + 0x00fa: u'♠', + 0x00fb: u'♣', + 0x00fc: u'♥', + 0x00fd: u'♦', + 0x00fe: u'★', + 0x00ff: u'◎', + 0x0100: u'○', + 0x0101: u'□', + 0x0102: u'△', + 0x0103: u'◇', + 0x0104: u'@', + 0x0105: u'♪', + 0x0106: u'%', + 0x0107: u'☀', + 0x0108: u'☁', + 0x0109: u'☂', + 0x010a: u'☃', + 0x010f: u'⤴', + 0x0110: u'⤵', + 0x0112: u'円', + 0x0116: u'✉', + 0x011b: u'←', + 0x011c: u'↑', + 0x011d: u'↓', + 0x011e: u'→', + 0x0120: u'&', + 0x0121: u'0', + 0x0122: u'1', + 0x0123: u'2', + 0x0124: u'3', + 0x0125: u'4', + 0x0126: u'5', + 0x0127: u'6', + 0x0128: u'7', + 0x0129: u'8', + 0x012a: u'9', + 0x012b: u'A', + 0x012c: u'B', + 0x012d: u'C', + 0x012e: u'D', + 0x012f: u'E', + 0x0130: u'F', + 0x0131: u'G', + 0x0132: u'H', + 0x0133: u'I', + 0x0134: u'J', + 0x0135: u'K', + 0x0136: u'L', + 0x0137: u'M', + 0x0138: u'N', + 0x0139: u'O', + 0x013a: u'P', + 0x013b: u'Q', + 0x013c: u'R', + 0x013d: u'S', + 0x013e: u'T', + 0x013f: u'U', + 0x0140: u'V', + 0x0141: u'W', + 0x0142: u'X', + 0x0143: u'Y', + 0x0144: u'Z', + 0x0145: u'a', + 0x0146: u'b', + 0x0147: u'c', + 0x0148: u'd', + 0x0149: u'e', + 0x014a: u'f', + 0x014b: u'g', + 0x014c: u'h', + 0x014d: u'i', + 0x014e: u'j', + 0x014f: u'k', + 0x0150: u'l', + 0x0151: u'm', + 0x0152: u'n', + 0x0153: u'o', + 0x0154: u'p', + 0x0155: u'q', + 0x0156: u'r', + 0x0157: u's', + 0x0158: u't', + 0x0159: u'u', + 0x015a: u'v', + 0x015b: u'w', + 0x015c: u'x', + 0x015d: u'y', + 0x015e: u'z', + 0x015f: u'À', + 0x0160: u'Á', + 0x0161: u'Â', + 0x0163: u'Ä', + 0x0166: u'Ç', + 0x0167: u'È', + 0x0168: u'É', + 0x0169: u'Ê', + 0x016a: u'Ë', + 0x016b: u'Ì', + 0x016c: u'Í', + 0x016d: u'Î', + 0x016e: u'Ï', + 0x0170: u'Ñ', + 0x0171: u'Ò', + 0x0172: u'Ó', + 0x0173: u'Ô', + 0x0175: u'Ö', + 0x0176: u'×', + 0x0178: u'Ù', + 0x0179: u'Ú', + 0x017a: u'Û', + 0x017b: u'Ü', + 0x017e: u'ß', + 0x017f: u'à', + 0x0180: u'á', + 0x0181: u'â', + 0x0183: u'ä', + 0x0186: u'ç', + 0x0187: u'è', + 0x0188: u'é', + 0x0189: u'ê', + 0x018a: u'ë', + 0x018b: u'ì', + 0x018c: u'í', + 0x018d: u'î', + 0x018e: u'ï', + 0x0190: u'ñ', + 0x0191: u'ò', + 0x0192: u'ó', + 0x0193: u'ô', + 0x0195: u'ö', + 0x0196: u'÷', + 0x0198: u'ù', + 0x0199: u'ú', + 0x019a: u'û', + 0x019b: u'ü', + 0x019f: u'Œ', + 0x01a0: u'œ', + 0x01a3: u'ª', + 0x01a4: u'º', + 0x01a5: u'þ', + 0x01a6: u'Þ', + 0x01a7: u'ʳ', + 0x01a8: u'¥', + 0x01a9: u'¡', + 0x01aa: u'¿', + 0x01ab: u'!', + 0x01ac: u'?', + 0x01ad: u',', + 0x01ae: u'.', + 0x01af: u'…', + 0x01b0: u'·', + 0x01b1: u'/', + 0x01b2: u'‘', + 0x01b3: u'\'', + 0x01b3: u'’', + 0x01b4: u'“', + 0x01b5: u'”', + 0x01b6: u'„', + 0x01b7: u'«', + 0x01b8: u'»', + 0x01b9: u'(', + 0x01ba: u')', + 0x01bb: u'♂', + 0x01bc: u'♀', + 0x01bd: u'+', + 0x01be: u'-', + 0x01bf: u'*', + 0x01c0: u'#', + 0x01c1: u'=', + 0x01c2: u'&', + 0x01c3: u'~', + 0x01c4: u':', + 0x01c5: u';', + 0x01c6: u'♠', + 0x01c7: u'♣', + 0x01c8: u'♥', + 0x01c9: u'♦', + 0x01ca: u'★', + 0x01cb: u'◎', + 0x01cc: u'○', + 0x01cd: u'□', + 0x01ce: u'△', + 0x01cf: u'◇', + 0x01d0: u'@', + 0x01d1: u'♪', + 0x01d2: u'%', + 0x01d3: u'☀', + 0x01d4: u'☁', + 0x01d5: u'☂', + 0x01d6: u'☃', + 0x01db: u'⤴', + 0x01dc: u'⤵', + 0x01de: u' ', + 0xe000: u'\n', + 0x25bc: u'\f', + 0x25bd: u'\r', +} + +# And the reverse dict, used with str.translate() +inverse_character_table = dict() +for in_, out in character_table.iteritems(): + inverse_character_table[ord(out)] = in_ + + +def LittleEndianBitStruct(*args): + """Construct's bit structs read a byte at a time in the order they appear, + reading each bit from most to least significant. Alas, this doesn't work + at all for a 32-bit bit field, because the bytes are 'backwards' in + little-endian files. + + So this acts as a bit struct, but reverses the order of bytes before + reading/writing, so ALL the bits are read from most to least significant. + """ + return Buffered( + BitStruct(*args), + encoder=lambda s: s[::-1], + decoder=lambda s: s[::-1], + resizer=lambda _: _, + ) + +class PokemonStringAdapter(Adapter): + u"""Adapter that encodes/decodes Pokémon-formatted text stored in a regular + String struct. + """ + def _decode(self, obj, context): + decoded_text = obj.decode('utf16') + + # Real string ends at the \uffff character + if u'\uffff' in decoded_text: + decoded_text = decoded_text[0:decoded_text.index(u'\uffff')] + # XXX save "trash bytes" somewhere..? + + return decoded_text.translate(character_table) + + def _encode(self, obj, context): + #padded_text = (obj + u'\xffff' + '\x00' * 12) + padded_text = obj + decoded_text = padded_text.translate(inverse_character_table) + return decoded_text.encode('utf16') + +class DateAdapter(Adapter): + """Converts between a three-byte string and a Python date. + + Only dates in 2000 or later will work! + """ + def _decode(self, obj, context): + if obj == '\x00\x00\x00': + return None + + y, m, d = (ord(byte) for byte in obj) + y += 2000 + return datetime.date(y, m, d) + + def _encode(self, obj, context): + if obj is None: + return '\x00\x00\x00' + + y, m, d = obj.year - 2000, obj.month, obj.day + return ''.join(chr(n) for n in (y, m, d)) + +# And here we go. +# Docs: http://projectpokemon.org/wiki/Pokemon_NDS_Structure +pokemon_struct = Struct('pokemon_struct', + # Header + ULInt32('personality'), # XXX aughgh http://bulbapedia.bulbagarden.net/wiki/Personality + Padding(2), + ULInt16('checksum'), # XXX should be checked or calculated + + # Block A + ULInt16('national_id'), + ULInt16('held_item_id'), + ULInt16('original_trainer_id'), + ULInt16('original_trainer_secret_id'), + ULInt32('exp'), + ULInt8('happiness'), + ULInt8('ability_id'), # XXX needs to match personality + species + BitStruct('markings', + Padding(2), + Flag('diamond'), + Flag('star'), + Flag('heart'), + Flag('square'), + Flag('triangle'), + Flag('circle'), + ), + Enum( + ULInt8('original_country'), + jp=1, + us=2, + fr=3, + it=4, + de=5, + es=7, + kr=8, + ), + + # XXX sum cannot surpass 510 + ULInt8('effort_hp'), + ULInt8('effort_attack'), + ULInt8('effort_defense'), + ULInt8('effort_speed'), + ULInt8('effort_special_attack'), + ULInt8('effort_special_defense'), + + ULInt8('contest_cool'), + ULInt8('contest_beauty'), + ULInt8('contest_cute'), + ULInt8('contest_smart'), + ULInt8('contest_tough'), + ULInt8('contest_sheen'), + + LittleEndianBitStruct('sinnoh_ribbons', + Padding(4), + Flag('premier_ribbon'), + Flag('classic_ribbon'), + Flag('carnival_ribbon'), + Flag('festival_ribbon'), + Flag('blue_ribbon'), + Flag('green_ribbon'), + Flag('red_ribbon'), + Flag('legend_ribbon'), + Flag('history_ribbon'), + Flag('record_ribbon'), + Flag('footprint_ribbon'), + Flag('gorgeous_royal_ribbon'), + Flag('royal_ribbon'), + Flag('gorgeous_ribbon'), + Flag('smile_ribbon'), + Flag('snooze_ribbon'), + Flag('relax_ribbon'), + Flag('careless_ribbon'), + Flag('downcast_ribbon'), + Flag('shock_ribbon'), + Flag('alert_ribbon'), + Flag('world_ability_ribbon'), + Flag('pair_ability_ribbon'), + Flag('multi_ability_ribbon'), + Flag('double_ability_ribbon'), + Flag('great_ability_ribbon'), + Flag('ability_ribbon'), + Flag('sinnoh_champ_ribbon'), + ), + + # Block B + ULInt16('move1_id'), + ULInt16('move2_id'), + ULInt16('move3_id'), + ULInt16('move4_id'), + ULInt8('move1_pp'), + ULInt8('move2_pp'), + ULInt8('move3_pp'), + ULInt8('move4_pp'), + ULInt8('move1_pp_ups'), + ULInt8('move2_pp_ups'), + ULInt8('move3_pp_ups'), + ULInt8('move4_pp_ups'), + + LittleEndianBitStruct('ivs', + Flag('is_nicknamed'), + Flag('is_egg'), + BitField('iv_special_defense', 5), + BitField('iv_special_attack', 5), + BitField('iv_speed', 5), + BitField('iv_defense', 5), + BitField('iv_attack', 5), + BitField('iv_hp', 5), + ), + LittleEndianBitStruct('hoenn_ribbons', + Flag('world_ribbon'), + Flag('earth_ribbon'), + Flag('national_ribbon'), + Flag('country_ribbon'), + Flag('sky_ribbon'), + Flag('land_ribbon'), + Flag('marine_ribbon'), + Flag('effort_ribbon'), + Flag('artist_ribbon'), + Flag('victory_ribbon'), + Flag('winning_ribbon'), + Flag('champion_ribbon'), + Flag('tough_ribbon_master'), + Flag('tough_ribbon_hyper'), + Flag('tough_ribbon_super'), + Flag('tough_ribbon'), + Flag('smart_ribbon_master'), + Flag('smart_ribbon_hyper'), + Flag('smart_ribbon_super'), + Flag('smart_ribbon'), + Flag('cute_ribbon_master'), + Flag('cute_ribbon_hyper'), + Flag('cute_ribbon_super'), + Flag('cute_ribbon'), + Flag('beauty_ribbon_master'), + Flag('beauty_ribbon_hyper'), + Flag('beauty_ribbon_super'), + Flag('beauty_ribbon'), + Flag('cool_ribbon_master'), + Flag('cool_ribbon_hyper'), + Flag('cool_ribbon_super'), + Flag('cool_ribbon'), + ), + EmbeddedBitStruct( + BitField('alternate_form', 5), + Enum(BitField('gender', 2), + genderless = 2, + male = 0, + female = 1, + ), + Flag('fateful_encounter'), + ), + BitStruct('shining_leaves', + Padding(2), + Flag('crown'), + Flag('leaf5'), + Flag('leaf4'), + Flag('leaf3'), + Flag('leaf2'), + Flag('leaf1'), + ), + Padding(2), + ULInt16('pt_egg_location_id'), + ULInt16('pt_met_location_id'), + + # Block C + PokemonStringAdapter(String('nickname', 22)), + Padding(1), + Enum(ULInt8('original_version'), + sapphire = 1, + ruby = 2, + emerald = 3, + firered = 4, + leafgreen = 5, + heartgold = 7, + soulsilver = 8, + diamond = 10, + pearl = 11, + platinum = 12, + orre = 15, + ), + LittleEndianBitStruct('sinnoh_contest_ribbons', + Padding(12), + Flag('tough_ribbon_master'), + Flag('tough_ribbon_ultra'), + Flag('tough_ribbon_great'), + Flag('tough_ribbon'), + Flag('smart_ribbon_master'), + Flag('smart_ribbon_ultra'), + Flag('smart_ribbon_great'), + Flag('smart_ribbon'), + Flag('cute_ribbon_master'), + Flag('cute_ribbon_ultra'), + Flag('cute_ribbon_great'), + Flag('cute_ribbon'), + Flag('beauty_ribbon_master'), + Flag('beauty_ribbon_ultra'), + Flag('beauty_ribbon_great'), + Flag('beauty_ribbon'), + Flag('cool_ribbon_master'), + Flag('cool_ribbon_ultra'), + Flag('cool_ribbon_great'), + Flag('cool_ribbon'), + ), + Padding(4), + + # Block D + PokemonStringAdapter(String('original_trainer_name', 16)), + DateAdapter(String('date_egg_received', 3)), + DateAdapter(String('date_met', 3)), + ULInt16('dp_egg_location_id'), + ULInt16('dp_met_location_id'), + ULInt8('pokerus'), + ULInt8('dppt_pokeball'), + EmbeddedBitStruct( + Enum(Flag('original_trainer_gender'), + male = False, + female = True, + ), + BitField('met_at_level', 7), + ), + Enum(ULInt8('encounter_type'), + special = 0, + grass = 2, + dialga_palkia = 4, + cave = 5, # or hall of origin + water = 7, + building = 9, + safari_zone = 10, + gift = 12, + ), + ULInt8('hgss_pokeball'), + Padding(1), +) diff --git a/setup.py b/setup.py index 52e19da..d66e74c 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ setup( 'SQLAlchemy>=0.6', 'whoosh>=0.3.0b24', 'markdown', + 'construct', ], entry_points = {