veniasilente/veekun_pokedex
1
0
Fork 0
mirror of https://github.com/veekun/pokedex.git synced 2024-08-20 18:16:34 +00:00
Code Issues Projects Releases Wiki Activity

Compat with Python 3.3+

Browse source
This commit is contained in:
Eevee (Alex Munroe) 2015-10-05 08:11:08 -07:00
parent d0e8f503b8
commit b76b74e7a6
15 changed files with 110 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
pokedex/compatibility.py
View file

@ -2,6 +2,7 @@
Currently these are functions missing from Python 2.5.
"""
from __future__ import print_function
try:
from itertools import permutations
@ -127,14 +128,15 @@ except ImportError:
for i, name in enumerate(field_names):
template += ' %s = _property(_itemgetter(%d))\n' % (name, i)
if verbose:
print template
print(template)
# Execute the template string in a temporary namespace
namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename,
_property=property, _tuple=tuple)
try:
exec template in namespace
except SyntaxError, e:
exec ("exec template in namespace")
except SyntaxError:
e = _sys.exc_info()[1]
raise SyntaxError(e.message + ':\n' + template)
result = namespace[typename]

6
pokedex/db/load.py
View file

@ -1,4 +1,6 @@
"""CSV to database or vice versa."""
from __future__ import print_function
import csv
import fnmatch
import os.path
@ -58,7 +60,7 @@ def _get_verbose_prints(verbose):
# Also, space-pad to keep the cursor in a known column
num_spaces = 66 - len(truncated_thing)
print "%s...%s" % (truncated_thing, ' ' * num_spaces),
print("%s...%s" % (truncated_thing, ' ' * num_spaces), end='')
sys.stdout.flush()
if sys.stdout.isatty():
@ -91,7 +93,7 @@ def _get_verbose_prints(verbose):
pass
def print_done(msg='ok'):
print msg
print(msg)
return print_start, print_status, print_done

11
pokedex/db/markdown.py
View file

@ -14,6 +14,7 @@ from __future__ import absolute_import
import re
import markdown
import six
from sqlalchemy.orm.session import object_session
try:
# Markdown 2.1+
@ -22,6 +23,7 @@ except ImportError:
# Old Markdown
from markdown import etree, AtomicString
@six.python_2_unicode_compatible
class MarkdownString(object):
"""Wraps a Markdown string.
@ -44,11 +46,8 @@ class MarkdownString(object):
self.session = session
self.language = language
def __unicode__(self):
return self.as_text()
def __str__(self):
return self.as_text().encode()
return self.as_text()
def __html__(self):
return self.as_html()
@ -98,7 +97,7 @@ def _markdownify_effect_text(move, effect_text, language=None):
return effect_text
effect_text = effect_text.replace(
u'$effect_chance',
unicode(move.effect_chance),
str(move.effect_chance),
)
# "The target" vs "each target"; for Conquest, but hopefully main series
@ -165,7 +164,7 @@ class PokedexLinkPattern(markdown.inlinepatterns.Pattern):
Handles matches using factory
"""
regex = ur'(?x) \[ ([^]]*) \] \{ ([-a-z0-9]+) : ([-a-z0-9 ]+) \}'
regex = u'(?x) \\[ ([^]]*) \\] \\{ ([-a-z0-9]+) : ([-a-z0-9 ]+) \\}'
def __init__(self, factory, session, string_language=None, game_language=None):
markdown.inlinepatterns.Pattern.__init__(self, self.regex)

2
pokedex/db/multilang.py
View file

@ -136,7 +136,7 @@ def create_translation_table(_table_name, foreign_class, relation_name,
# Add ye columns
# Column objects have a _creation_order attribute in ascending order; use
# this to get the (unordered) kwargs sorted correctly
kwitems = kwargs.items()
kwitems = list(kwargs.items())
kwitems.sort(key=lambda kv: kv[1]._creation_order)
for name, column in kwitems:
column.name = name

67
pokedex/db/translations.py
View file

@ -20,18 +20,17 @@ put it in flavor_summary tables.
Routes names and other repetitive numeric things are replaced by e.g.
"Route {num}" so translators only have to work on each set once.
"""
from __future__ import print_function
import binascii
import csv
import heapq
import itertools
import os
import re
import sys
from collections import defaultdict
from six.moves import zip
from pokedex.db import tables
from pokedex.defaults import get_default_csv_dir
@ -253,7 +252,7 @@ class Translations(object):
)
if len(warning) > 79:
warning = warning[:76] + u'...'
print warning.encode('utf-8')
print(warning)
def reader_for_class(self, cls, reader_class=csv.reader):
tablename = cls.__table__.name
@ -366,7 +365,7 @@ def group_by_object(stream):
Yields ((class name, object ID), (list of messages)) pairs.
"""
stream = iter(stream)
current = stream.next()
current = next(stream)
current_key = current.cls, current.id
group = [current]
for message in stream:
@ -395,27 +394,37 @@ class Merge(object):
def add_iterator(self, iterator):
iterator = iter(iterator)
try:
value = iterator.next()
value = next(iterator)
except StopIteration:
return
else:
heapq.heappush(self.next_values, (value, iterator))
self.next_values.append((value, iterator))
def __iter__(self):
return self
def next(self):
if self.next_values:
value, iterator = heapq.heappop(self.next_values)
self.add_iterator(iterator)
return value
else:
def __next__(self):
if not self.next_values:
raise StopIteration
min_idx = min(range(len(self.next_values)), key=lambda i: self.next_values[i][0])
value, iterator = self.next_values[min_idx]
try:
next_value = next(iterator)
except StopIteration:
del self.next_values[min_idx]
else:
self.next_values[min_idx] = next_value, iterator
return value
next = __next__
def merge_adjacent(gen):
"""Merge adjacent messages that compare equal"""
gen = iter(gen)
last = gen.next()
last = next(gen)
for this in gen:
if this.merge_key == last.merge_key:
last.merge(this)
@ -441,16 +450,16 @@ def leftjoin(left_stream, right_stream, key=lambda x: x, unused=None):
left_stream = iter(left_stream)
right_stream = iter(right_stream)
try:
right = right_stream.next()
right = next(right_stream)
for left in left_stream:
while right and key(left) > key(right):
if unused is not None:
unused(right)
right = right_stream.next()
right = next(right_stream)
if key(left) == key(right):
yield left, right
del left
right = right_stream.next()
right = next(right_stream)
else:
yield left, None
except StopIteration:
@ -478,7 +487,7 @@ def yield_source_csv_messages(cls, foreign_cls, csvreader, force_column=None):
"""Yield all messages from one source CSV file.
"""
columns = list(cls.__table__.c)
column_names = csvreader.next()
column_names = next(csvreader)
# Assumptions: rows are in lexicographic order
# (taking numeric values as numbers of course)
# Assumptions about the order of columns:
@ -503,11 +512,13 @@ def _yield_csv_messages(foreign_cls, columns, first_string_index, csvreader, ori
id = int(values[0])
messages = []
for string, column in zip(values[first_string_index:], string_columns):
if isinstance(string, bytes):
string = string.decode('utf-8')
message = Message(
foreign_cls.__name__,
id,
column.name,
string.decode('utf-8'),
string,
column.type.length,
pot=pot_for_column(cls, column, force_column is not None),
origin=origin,
@ -524,7 +535,7 @@ def yield_guessed_csv_messages(file):
"""Yield messages from a CSV file, using the header to figure out what the data means.
"""
csvreader = csv.reader(file, lineterminator='\n')
column_names = csvreader.next()
column_names = next(csvreader)
if column_names == 'language_id,table,id,column,source_crc,string'.split(','):
# A translation CSV
return yield_translation_csv_messages(file, True)
@ -553,14 +564,16 @@ def yield_translation_csv_messages(file, no_header=False):
"""
csvreader = csv.reader(file, lineterminator='\n')
if not no_header:
columns = csvreader.next()
columns = next(csvreader)
assert columns == 'language_id,table,id,column,source_crc,string'.split(',')
for language_id, table, id, column, source_crc, string in csvreader:
if isinstance(string, bytes):
string = string.decode('utf-8')
yield Message(
table,
int(id),
column,
string.decode('utf-8'),
string,
origin='target CSV',
source_crc=source_crc,
language_id=int(language_id),
@ -591,7 +604,7 @@ def pot_for_column(cls, column, summary=False):
def number_replace(source, string):
numbers_iter = iter(number_re.findall(source))
next_number = lambda match: numbers_iter.next()
next_number = lambda match: next(numbers_iter)
return re.sub(r'\{num\}', next_number, string)
def match_to_source(source, *translations):
@ -617,7 +630,7 @@ def match_to_source(source, *translations):
current_source = number_replace(source.string, translation.source)
current_crc = crc(current_source)
elif '{num}' in translation.string:
print (u'Warning: {num} appears in %s, but not marked for number replacement. Discarding!' % translation).encode('utf-8')
print(u'Warning: {num} appears in %s, but not marked for number replacement. Discarding!' % translation)
continue
else:
current_string = translation.string
@ -655,5 +668,5 @@ def merge_translations(source_stream, *translation_streams, **kwargs):
synchronize(source, t, key=lambda m: m.merge_key, unused=kwargs.get('unused'))
for t in translation_streams
]
for messages in itertools.izip(source, *streams):
for messages in zip(source, *streams):
yield match_to_source(*messages)

17
pokedex/lookup.py
View file

@ -4,6 +4,7 @@ import random
import re
import unicodedata
from six import text_type
import whoosh
import whoosh.index
import whoosh.query
@ -196,8 +197,8 @@ class PokedexLookup(object):
q = self.session.query(cls).order_by(cls.id)
for row in q:
row_key = dict(table=unicode(cls.__tablename__),
row_id=unicode(row.id))
row_key = dict(table=text_type(cls.__tablename__),
row_id=text_type(row.id))
def add(name, language, iso639, iso3166):
normalized_name = self.normalize_name(name)
@ -242,7 +243,7 @@ class PokedexLookup(object):
# decompose! But the results are considered letters, not combining
# characters, so testing for Mn works well, and combining them again
# makes them look right.
nkfd_form = unicodedata.normalize('NFKD', unicode(name))
nkfd_form = unicodedata.normalize('NFKD', text_type(name))
name = u"".join(c for c in nkfd_form
if unicodedata.category(c) != 'Mn')
name = unicodedata.normalize('NFC', name)
@ -292,8 +293,8 @@ class PokedexLookup(object):
# And, just to complicate matters: "type" and language need to be
# considered separately.
def merge_requirements(func):
user = filter(func, user_valid_types)
system = filter(func, valid_types)
user = list(filter(func, user_valid_types))
system = list(filter(func, valid_types))
if user and system:
merged = list(set(user) & set(system))
@ -463,7 +464,7 @@ class PokedexLookup(object):
elif name_as_number is not None:
# Don't spell-check numbers!
exact_only = True
query = whoosh.query.Term(u'row_id', unicode(name_as_number))
query = whoosh.query.Term(u'row_id', text_type(name_as_number))
else:
# Not an integer
query = whoosh.query.Term(u'name', name)
@ -547,7 +548,7 @@ class PokedexLookup(object):
# n.b.: It's possible we got a list of valid_types and none of them
# were valid, but this function is guaranteed to return
# *something*, so it politely selects from the entire index instead
table_names = self.indexed_tables.keys()
table_names = list(self.indexed_tables)
table_names.remove('pokemon_forms')
# Pick a random table, then pick a random item from it. Small tables
@ -561,7 +562,7 @@ class PokedexLookup(object):
.offset(random.randint(0, count - 1)) \
.first()
return self.lookup(unicode(id), valid_types=[table_name])
return self.lookup(text_type(id), valid_types=[table_name])
def prefix_lookup(self, prefix, valid_types=[]):
"""Returns terms starting with the given exact prefix.

68
pokedex/main.py
View file

@ -1,4 +1,6 @@
# encoding: utf8
from __future__ import print_function
from optparse import OptionParser
import os
import sys
@ -58,8 +60,8 @@ def get_session(options):
session = pokedex.db.connect(engine_uri)
if options.verbose:
print "Connected to database %(engine)s (from %(got_from)s)" \
% dict(engine=session.bind.url, got_from=got_from)
print("Connected to database %(engine)s (from %(got_from)s)"
% dict(engine=session.bind.url, got_from=got_from))
return session
@ -78,8 +80,8 @@ def get_lookup(options, session=None, recreate=False):
index_dir, got_from = defaults.get_default_index_dir_with_origin()
if options.verbose:
print "Opened lookup index %(index_dir)s (from %(got_from)s)" \
% dict(index_dir=index_dir, got_from=got_from)
print("Opened lookup index %(index_dir)s (from %(got_from)s)"
% dict(index_dir=index_dir, got_from=got_from))
lookup = pokedex.lookup.PokedexLookup(index_dir, session=session)
@ -100,8 +102,8 @@ def get_csv_directory(options):
if csvdir is None:
csvdir, got_from = defaults.get_default_csv_dir_with_origin()
print "Using CSV directory %(csvdir)s (from %(got_from)s)" \
% dict(csvdir=csvdir, got_from=got_from)
print("Using CSV directory %(csvdir)s (from %(got_from)s)"
% dict(csvdir=csvdir, got_from=got_from))
return csvdir
@ -142,11 +144,11 @@ def command_load(*args):
options, tables = parser.parse_args(list(args))
if not options.engine_uri:
print "WARNING: You're reloading the default database, but not the lookup index. They"
print " might get out of sync, and pokedex commands may not work correctly!"
print "To fix this, run `pokedex reindex` when this command finishes. Or, just use"
print "`pokedex setup` to do both at once."
print
print("WARNING: You're reloading the default database, but not the lookup index. They")
print(" might get out of sync, and pokedex commands may not work correctly!")
print("To fix this, run `pokedex reindex` when this command finishes. Or, just use")
print("`pokedex setup` to do both at once.")
print()
if options.langs == 'none':
langs = []
@ -173,7 +175,7 @@ def command_reindex(*args):
session = get_session(options)
lookup = get_lookup(options, session=session, recreate=True)
print "Recreated lookup index."
print("Recreated lookup index.")
def command_setup(*args):
@ -190,7 +192,7 @@ def command_setup(*args):
lookup = get_lookup(options, session=session, recreate=True)
print "Recreated lookup index."
print("Recreated lookup index.")
def command_status(*args):
@ -201,37 +203,37 @@ def command_status(*args):
# Database, and a lame check for whether it's been inited at least once
session = get_session(options)
print " - OK! Connected successfully."
print(" - OK! Connected successfully.")
if pokedex.db.tables.Pokemon.__table__.exists(session.bind):
print " - OK! Database seems to contain some data."
print(" - OK! Database seems to contain some data.")
else:
print " - WARNING: Database appears to be empty."
print(" - WARNING: Database appears to be empty.")
# CSV; simple checks that the dir exists
csvdir = get_csv_directory(options)
if not os.path.exists(csvdir):
print " - ERROR: No such directory!"
print(" - ERROR: No such directory!")
elif not os.path.isdir(csvdir):
print " - ERROR: Not a directory!"
print(" - ERROR: Not a directory!")
else:
print " - OK! Directory exists."
print(" - OK! Directory exists.")
if os.access(csvdir, os.R_OK):
print " - OK! Can read from directory."
print(" - OK! Can read from directory.")
else:
print " - ERROR: Can't read from directory!"
print(" - ERROR: Can't read from directory!")
if os.access(csvdir, os.W_OK):
print " - OK! Can write to directory."
print(" - OK! Can write to directory.")
else:
print " - WARNING: Can't write to directory! " \
"`dump` will not work. You may need to sudo."
print(" - WARNING: Can't write to directory! "
"`dump` will not work. You may need to sudo.")
# Index; the PokedexLookup constructor covers most tests and will
# cheerfully bomb if they fail
lookup = get_lookup(options, recreate=False)
print " - OK! Opened successfully."
print(" - OK! Opened successfully.")
### User-facing commands
@ -247,11 +249,11 @@ def command_lookup(*args):
results = lookup.lookup(name)
if not results:
print "No matches."
print("No matches.")
elif results[0].exact:
print "Matched:"
print("Matched:")
else:
print "Fuzzy-matched:"
print("Fuzzy-matched:")
for result in results:
if hasattr(result.object, 'full_name'):
@ -259,15 +261,15 @@ def command_lookup(*args):
else:
name = result.object.name
print "%s: %s" % (result.object.__tablename__, name),
print("%s: %s" % (result.object.__tablename__, name), end='')
if result.language:
print "(%s in %s)" % (result.name, result.language)
print("(%s in %s)" % (result.name, result.language))
else:
print
print()
def command_help():
print u"""pokedex -- a command-line Pokédex interface
print(u"""pokedex -- a command-line Pokédex interface
usage: pokedex {command} [options...]
Run `pokedex setup` first, or nothing will work!
See https://github.com/veekun/pokedex/wiki/CLI for more documentation.
@ -319,7 +321,7 @@ Dump options:
Additionally, load and dump accept a list of table names (possibly with
wildcards) and/or csv fileames as an argument list.
""".encode(sys.getdefaultencoding(), 'replace')
""".encode(sys.getdefaultencoding(), 'replace'))
sys.exit(0)

2
pokedex/roomaji.py
View file

@ -128,7 +128,7 @@ class Romanizer(object):
if last_kana == 'sokuon':
raise ValueError("Sokuon cannot be the last character.")
return unicode(''.join(characters))
return u''.join(characters)
romanizers = dict()

2
pokedex/struct/_pokemon_struct.py
View file

@ -467,7 +467,7 @@ character_table = {
# And the reverse dict, used with str.translate()
inverse_character_table = dict()
for in_, out in character_table.iteritems():
for in_, out in character_table.items():
inverse_character_table[ord(out)] = in_

1
pokedex/tests/test_database_sanity.py
View file

@ -53,7 +53,6 @@ def test_unique_form_order(session):
query = query.options(joinedload('pokemon.species'))
for form in query:
print form.name
try:
previous_species = species_by_form_order[form.order]
except KeyError:

12
pokedex/tests/test_media.py
View file

@ -4,6 +4,7 @@ If run directly from the command line, also tests the accessors and the names
of all the media by getting just about everything in a naive brute-force way.
This, of course, takes a lot of time to run.
"""
from __future__ import print_function
import pytest
@ -49,7 +50,6 @@ def test_strict_castform(session, media_root):
with pytest.raises(ValueError):
castform = session.query(tables.PokemonSpecies).filter_by(identifier=u'castform').first()
rainy_castform = [f for f in castform.forms if f.form_identifier == 'rainy'][0]
print rainy_castform
rainy_castform = media.PokemonFormMedia(media_root, rainy_castform)
rainy_castform.overworld('up', strict=True)
@ -81,13 +81,11 @@ def hit(filenames, method, *args, **kwargs):
"""
try:
medium = method(*args, **kwargs)
#print 'Hit', medium.relative_path
assert medium.exists
except ValueError, e:
#print 'DNF', e
except ValueError:
return False
except:
print 'Error while processing', method, args, kwargs
print('Error while processing', method, args, kwargs)
raise
try:
filenames.remove(medium.path)
@ -242,9 +240,9 @@ def test_get_everything(session, media_root):
unaccessed_filenames.remove(filename)
if unaccessed_filenames:
print 'Unaccessed files:'
print('Unaccessed files:')
for filename in unaccessed_filenames:
print filename
print(filename)
assert unaccessed_filenames == set()

1
pokedex/tests/test_schema.py
View file

@ -38,7 +38,6 @@ def test_class_order():
class_names = [table.__name__ for table in tables.mapped_classes]
def key(name):
return name != 'Language', name
print [(a,b) for (a,b) in zip(class_names, sorted(class_names, key=key)) if a!=b]
assert class_names == sorted(class_names, key=key)
def test_i18n_table_creation():

6
pokedex/tests/test_strings.py
View file

@ -88,15 +88,15 @@ def test_markdown(session):
assert '10%' in move.effect_map[language].as_text()
assert '10%' in move.effect.as_html()
assert '10%' in move.effect_map[language].as_html()
assert '10%' in unicode(move.effect)
assert '10%' in unicode(move.effect_map[language])
assert '10%' in str(move.effect)
assert '10%' in str(move.effect_map[language])
assert '10%' in move.effect.__html__()
assert '10%' in move.effect_map[language].__html__()
def test_markdown_string(session):
en = util.get(session, tables.Language, 'en')
md = markdown.MarkdownString('[]{move:thunderbolt} [paralyzes]{mechanic:paralysis} []{form:sky shaymin}. []{pokemon:mewthree} does not exist.', session, en)
assert unicode(md) == 'Thunderbolt paralyzes Sky Shaymin. mewthree does not exist.'
assert str(md) == 'Thunderbolt paralyzes Sky Shaymin. mewthree does not exist.'
assert md.as_html() == '<p><span>Thunderbolt</span> <span>paralyzes</span> <span>Sky Shaymin</span>. <span>mewthree</span> does not exist.</p>'
class ObjectTestExtension(markdown.PokedexLinkExtension):

2
pokedex/tests/test_translations.py
View file

@ -112,7 +112,6 @@ def test_merge_translations():
for result, expected in zip(result_stream, expected_list):
res_src, res_crc, res_str, res_match = result
exp_src, exp_match, exp_str = expected
print result, expected
assert res_src.string == exp_src
assert res_str == exp_str, (res_str, exp_str)
if exp_match is None:
@ -122,7 +121,6 @@ def test_merge_translations():
elif exp_match is False:
assert res_crc == translations.crc('----')
assert res_match == exp_match
print 'unused:', unused
for message in unused:
assert message.string == 'unused'
assert message.id == 100

1
setup.py
View file

@ -12,6 +12,7 @@ setup(
'whoosh>=2.5,<2.7',
'markdown',
'construct',
'six>=1.9.0',
],
entry_points = {