From 72f31bc9d3b96bfdcb87683c861d62dbab5650e9 Mon Sep 17 00:00:00 2001 From: Andrew Ekstedt Date: Fri, 19 Mar 2021 20:25:39 -0700 Subject: [PATCH 1/8] workflows: Add github actions file Should be basically the same as the Travis CI file except more verbose (ugh) and a different version matrix. - 2.7 3.4 3.5 3.6 3.7 pypy pypy3 + 2.7 3.5 3.7 3.9 pypy3 Kept 2.7 because we still run it. Dropped pypy2 because who cares? Dropped 3.4 because it's way old at this point. Added 3.9 because it's the latest release. Kept 3.5 and 3.7 and dropped the even releases because it seems silly to test five different releases - the oldest and newest we support should be adequate (plus 3.7 because that's what i have.) --- .github/workflows/build.yml | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..f540a67 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,38 @@ +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Build and test + +on: + push: + branches: '**' + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [2.7, 3.5, 3.7, 3.9, pypy3] + # don't cancel every other job if one fails + fail-fast: false + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest wheel + pip install -e . + - name: Set up pokedex + run: pokedex setup -v + - name: Test with pytest + run: pytest + - name: Dump database and check for differences + run: | + pokedex dump + git --no-pager diff --exit-code pokedex/data/csv/ From 84f88bacc176ba9ad58d651a568cb6ccc73d71dd Mon Sep 17 00:00:00 2001 From: Andrew Ekstedt Date: Fri, 19 Mar 2021 21:04:01 -0700 Subject: [PATCH 2/8] Bump our maximum Markdown version up to 2.6.11 The next commit will fix the warnings, and we need a newer version for Python 3.9 support because 2.4.1 doesn't work with it. Markdown 2.6.11 was the last release before 3.0. Updates #257 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0dfc0f7..089d573 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( install_requires=[ 'SQLAlchemy>=1.0,<1.4', 'whoosh>=2.5,<2.7', - 'markdown==2.4.1', + 'markdown>=2.4.1,<=2.6.11', 'construct==2.5.3', 'six>=1.9.0', ], From 68ee98ffdd0ece821cab953a8087cdc1d67b4e0c Mon Sep 17 00:00:00 2001 From: Andrew Ekstedt Date: Fri, 19 Mar 2021 21:45:14 -0700 Subject: [PATCH 3/8] Fix some Markdown warnings Fixes the following warnings when run with Markdown 2.6.11: DeprecationWarning: "safe_mode" is deprecated in Python-Markdown. Use an HTML sanitizer (like Bleach https://bleach.readthedocs.io/) if you are parsing untrusted markdown text. See the 2.6 release notes for more info DeprecationWarning: Using short names for Markdown's builtin extensions is deprecated. Use the full path to the extension with Python's dot notation (eg: "markdown.extensions.extra" instead of "extra"). The current behavior will raise an error in version 2.7. See the Release Notes for Python-Markdown version 2.6 for more info. Also get rid of some compatability code for Markdown 2.0, which we no longer support. Updates #257 --- pokedex/db/markdown.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pokedex/db/markdown.py b/pokedex/db/markdown.py index 115faa4..f030ba7 100644 --- a/pokedex/db/markdown.py +++ b/pokedex/db/markdown.py @@ -16,12 +16,8 @@ import re import markdown import six from sqlalchemy.orm.session import object_session -try: - # Markdown 2.1+ - from markdown.util import etree, AtomicString -except ImportError: - # Old Markdown - from markdown import etree, AtomicString +from markdown.util import etree, AtomicString + @six.python_2_unicode_compatible class MarkdownString(object): @@ -66,8 +62,7 @@ class MarkdownString(object): extension = self.session.markdown_extension md = markdown.Markdown( - extensions=['extra', extension], - safe_mode='escape', + extensions=['markdown.extensions.extra', EscapeHtml(), extension], output_format='xhtml1', ) @@ -228,6 +223,17 @@ class PokedexLinkPattern(markdown.inlinepatterns.Pattern): el.text = AtomicString(label or name) return el +class EscapeHtml(markdown.Extension): + u"""Markdown extension which escapes raw html elements. + + This is the recommended replacement for safe_mode='escape', + which was deprecated in Markdown 2.5. + See https://python-markdown.github.io/change_log/release-2.5/ + """ + def extendMarkdown(self, md, md_globals): + del md.preprocessors['html_block'] + del md.inlinePatterns['html'] + class PokedexLinkExtension(markdown.Extension): u"""Markdown extension that translates the syntax used in effect text: From 348742fe2a34a34a1af3b1280a37cfdfc48c87a9 Mon Sep 17 00:00:00 2001 From: Andrew Ekstedt Date: Fri, 19 Mar 2021 22:38:32 -0700 Subject: [PATCH 4/8] Close csv files after dumping PyPy doesn't collect garbage as quickly as CPython, so it was possible for the process to exit before all the files had been flushed. --- pokedex/db/load.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pokedex/db/load.py b/pokedex/db/load.py index b787672..631fc9b 100644 --- a/pokedex/db/load.py +++ b/pokedex/db/load.py @@ -441,10 +441,12 @@ def dump(session, tables=[], directory=None, verbose=False, langs=None): # CSV module only works with bytes on 2 and only works with text on 3! if six.PY3: - writer = csv.writer(open(filename, 'w', newline='', encoding="utf8"), lineterminator='\n') + csvfile = open(filename, 'w', newline='', encoding="utf8") + writer = csv.writer(csvfile, lineterminator='\n') columns = [col.name for col in table.columns] else: - writer = csv.writer(open(filename, 'wb'), lineterminator='\n') + csvfile = open(filename, 'wb') + writer = csv.writer(csvfile, lineterminator='\n') columns = [col.name.encode('utf8') for col in table.columns] # For name tables, always dump rows for official languages, as well as @@ -491,4 +493,5 @@ def dump(session, tables=[], directory=None, verbose=False, langs=None): writer.writerow(csvs) + csvfile.close() print_done() From 2faa3ed6af4ad9363a1d20d55dd870a66dd76f9c Mon Sep 17 00:00:00 2001 From: Andrew Ekstedt Date: Sat, 20 Mar 2021 00:19:12 -0700 Subject: [PATCH 5/8] Avoid a warning about regex flags in Python 3.6 Python 3.6 deprecated using regex flag syntax (?x) to set flags in the middle of a pattern. Now you can only use it at the start of a pattern. Fortunately that same release added a new scoped-flag syntax, (?x:). (Wait, you say, it looks like our code *does* set flags at the start of the pattern? That's true, but the markdown module includes our regex in the middle of a larger one, so it's not actually at the start.) Fixes this warning (and a couple similar ones): DeprecationWarning: Flags not at the start of the expression '^(.?)(?x) \[ ([^]]' (truncated) self.compiled_re = re.compile(r"^(.?)%s(.)$" % pattern --- pokedex/db/markdown.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pokedex/db/markdown.py b/pokedex/db/markdown.py index f030ba7..4873118 100644 --- a/pokedex/db/markdown.py +++ b/pokedex/db/markdown.py @@ -11,6 +11,7 @@ which case it is replaced by the name of the thing linked to. """ from __future__ import absolute_import +import sys import re import markdown @@ -159,7 +160,10 @@ class PokedexLinkPattern(markdown.inlinepatterns.Pattern): Handles matches using factory """ - regex = u'(?x) \\[ ([^]]*) \\] \\{ ([-a-z0-9]+) : ([-a-z0-9 ]+) \\}' + if sys.version_info >= (3, 6): + regex = u'(?x: \\[ ([^]]*) \\] \\{ ([-a-z0-9]+) : ([-a-z0-9 ]+) \\} )' + else: + regex = u'(?x) \\[ ([^]]*) \\] \\{ ([-a-z0-9]+) : ([-a-z0-9 ]+) \\}' def __init__(self, factory, session, string_language=None, game_language=None): markdown.inlinepatterns.Pattern.__init__(self, self.regex) From f244234248ff01bb62e8ab66f94be0be07973910 Mon Sep 17 00:00:00 2001 From: Andrew Ekstedt Date: Sat, 20 Mar 2021 00:54:47 -0700 Subject: [PATCH 6/8] Register pytest.mark.slow marker Pytest started warning about unknown markers at some point. Not sure when. My local version doesn't do it but the CI versions do. > PytestUnknownMarkWarning: Unknown pytest.mark.slow - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/mark.html --- conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conftest.py b/conftest.py index a1d0c76..86e4bcd 100644 --- a/conftest.py +++ b/conftest.py @@ -21,6 +21,9 @@ def pytest_runtest_setup(item): if 'slow' in item.keywords and not item.config.getvalue('all'): pytest.skip("skipping slow tests") +def pytest_configure(config): + config.addinivalue_line("markers", "slow: marks tests as slow (require --all to run)") + @pytest.fixture(scope="module") def session(request): import pokedex.db From 3eb213aab41d7fe5dd76600c869e5684049255de Mon Sep 17 00:00:00 2001 From: Andrew Ekstedt Date: Sat, 20 Mar 2021 01:16:04 -0700 Subject: [PATCH 7/8] Update list of supported python versions --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 089d573..16ac667 100644 --- a/setup.py +++ b/setup.py @@ -22,8 +22,10 @@ setup( }, classifiers=[ "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", ] ) From 7630d2c7ba2f2013204557af440e3f1c94740bef Mon Sep 17 00:00:00 2001 From: Andrew Ekstedt Date: Sun, 21 Mar 2021 15:45:54 -0700 Subject: [PATCH 8/8] Update link to github actions docs --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f540a67..98af427 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# For more information see: https://docs.github.com/en/actions/guides/building-and-testing-python name: Build and test