From e974f30517261b2bc95cfb2017a8688f367c8bf3 Mon Sep 17 00:00:00 2001 From: Hugo Date: Wed, 1 Aug 2018 15:31:17 +0300 Subject: [PATCH] Drop support for Python 2 Co-Authored-By: Dustin Ingram Co-Authored-By: Berker Peksag --- .pylintrc | 4 - .travis.yml | 4 - README.rst | 2 +- appveyor.yml | 3 - docs/README.rst | 2 +- docs/sitemap_gen.py | 17 - docs/source/conf.py | 16 +- docs/source/index.rst | 2 +- docs/source/install.rst | 2 +- examples/example_config.py | 2 +- examples/read_django_settings.py | 2 - examples/standalone_app.py | 10 +- gunicorn/_compat.py | 154 +- gunicorn/app/base.py | 2 - gunicorn/app/pasterapp.py | 1 - gunicorn/arbiter.py | 2 - gunicorn/argparse_compat.py | 2362 ------------------------------ gunicorn/config.py | 85 +- gunicorn/glogging.py | 37 +- gunicorn/http/_sendfile.py | 67 - gunicorn/http/body.py | 42 +- gunicorn/http/message.py | 13 +- gunicorn/http/unreader.py | 15 +- gunicorn/http/wsgi.py | 29 +- gunicorn/instrument/statsd.py | 5 +- gunicorn/six.py | 762 ---------- gunicorn/sock.py | 3 +- gunicorn/util.py | 72 +- gunicorn/workers/__init__.py | 8 +- gunicorn/workers/base.py | 33 +- gunicorn/workers/base_async.py | 11 +- gunicorn/workers/gaiohttp.py | 33 +- gunicorn/workers/geventlet.py | 9 +- gunicorn/workers/ggevent.py | 16 +- gunicorn/workers/gthread.py | 34 +- gunicorn/workers/sync.py | 5 +- gunicorn/workers/workertmp.py | 9 +- requirements_test.txt | 2 +- scripts/update_thanks.py | 1 - setup.py | 14 +- tests/t.py | 4 +- tests/test_http.py | 20 +- tests/test_pidfile.py | 8 +- tests/test_ssl.py | 8 +- tests/test_statsd.py | 13 +- tests/test_util.py | 2 +- tests/treq.py | 10 +- tox.ini | 4 +- 48 files changed, 260 insertions(+), 3701 deletions(-) delete mode 100644 gunicorn/argparse_compat.py delete mode 100644 gunicorn/http/_sendfile.py delete mode 100644 gunicorn/six.py diff --git a/.pylintrc b/.pylintrc index 9d6c393b..89b82ab1 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,8 +6,6 @@ ignore= examples, scripts, _compat.py, - argparse_compat.py, - six.py, _gaiohttp.py, [MESSAGES CONTROL] @@ -53,5 +51,3 @@ disable= useless-import-alias, comparison-with-callable, try-except-raise, - # TODO: use dict comprehensions once we drop Python 2.6 support. - consider-using-dict-comprehension, diff --git a/.travis.yml b/.travis.yml index d22ec966..1e6b6808 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,6 @@ sudo: false language: python matrix: include: - - python: 2.6 - env: TOXENV=py26 - - python: 2.7 - env: TOXENV=py27 - python: 3.4 env: TOXENV=py34 - python: 3.5 diff --git a/README.rst b/README.rst index 4bb8fac1..f88697ea 100644 --- a/README.rst +++ b/README.rst @@ -28,7 +28,7 @@ The documentation is hosted at http://docs.gunicorn.org. Installation ------------ -Gunicorn requires **Python 2.x >= 2.6** or **Python 3.x >= 3.4**. +Gunicorn requires **Python **Python 3.x >= 3.4**. Install from PyPI:: diff --git a/appveyor.yml b/appveyor.yml index d7824b8e..02300861 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,15 +3,12 @@ environment: matrix: - TOXENV: lint PYTHON: "C:\\Python35-x64" - - TOXENV: py27 - PYTHON: "C:\\Python27-x64" - TOXENV: py35 PYTHON: "C:\\Python35-x64" - TOXENV: py36 PYTHON: "C:\\Python36-x64" matrix: allow_failures: - - TOXENV: py27 - TOXENV: py35 - TOXENV: py36 init: SET "PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" diff --git a/docs/README.rst b/docs/README.rst index 0bb01c42..c9fe2775 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -6,7 +6,7 @@ Requirements To generate documentation you need to install: - - Python >= 2.5 + - Python >= 3.4 - Sphinx (http://sphinx-doc.org/) diff --git a/docs/sitemap_gen.py b/docs/sitemap_gen.py index f1596b70..1cfbbae1 100755 --- a/docs/sitemap_gen.py +++ b/docs/sitemap_gen.py @@ -50,15 +50,6 @@ Usage: python sitemap_gen.py --config=config.xml [--help] [--testing] --testing, specified when user is experimenting """ -# Please be careful that all syntax used in this file can be parsed on -# Python 1.5 -- this version check is not evaluated until after the -# entire file has been parsed. -import sys -if sys.hexversion < 0x02020000: - print 'This script requires Python 2.2 or later.' - print 'Currently run with version: %s' % sys.version - sys.exit(1) - import fnmatch import glob import gzip @@ -72,14 +63,6 @@ import urllib import urlparse import xml.sax -# True and False were introduced in Python2.2.2 -try: - testTrue=True - del testTrue -except NameError: - True=1 - False=0 - # Text encodings ENC_ASCII = 'ASCII' ENC_UTF8 = 'UTF-8' diff --git a/docs/source/conf.py b/docs/source/conf.py index 57da1082..8c94eb13 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -21,8 +21,8 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'Gunicorn' -copyright = u'2009-%s, Benoit Chesneau' % time.strftime('%Y') +project = 'Gunicorn' +copyright = '2009-%s, Benoit Chesneau' % time.strftime('%Y') # gunicorn version import gunicorn release = version = gunicorn.__version__ @@ -55,19 +55,19 @@ latex_elements = { } latex_documents = [ - ('index', 'Gunicorn.tex', u'Gunicorn Documentation', - u'Benoit Chesneau', 'manual'), + ('index', 'Gunicorn.tex', 'Gunicorn Documentation', + 'Benoit Chesneau', 'manual'), ] # -- Options for manual page output -------------------------------------------- man_pages = [ - ('index', 'gunicorn', u'Gunicorn Documentation', - [u'Benoit Chesneau'], 1) + ('index', 'gunicorn', 'Gunicorn Documentation', + ['Benoit Chesneau'], 1) ] texinfo_documents = [ - ('index', 'Gunicorn', u'Gunicorn Documentation', - u'Benoit Chesneau', 'Gunicorn', 'One line description of project.', + ('index', 'Gunicorn', 'Gunicorn Documentation', + 'Benoit Chesneau', 'Gunicorn', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/source/index.rst b/docs/source/index.rst index 0c9303c5..074a1117 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -23,7 +23,7 @@ Features * Simple Python configuration * Multiple worker configurations * Various server hooks for extensibility -* Compatible with Python 2.x >= 2.6 or 3.x >= 3.4 +* Compatible with Python 3.x >= 3.4 Contents diff --git a/docs/source/install.rst b/docs/source/install.rst index f997a1ac..4a9b0a8c 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -4,7 +4,7 @@ Installation .. highlight:: bash -:Requirements: **Python 2.x >= 2.6** or **Python 3.x >= 3.4** +:Requirements: **Python 3.x >= 3.4** To install the latest released version of Gunicorn:: diff --git a/examples/example_config.py b/examples/example_config.py index ddd41b7b..f8f3c1df 100644 --- a/examples/example_config.py +++ b/examples/example_config.py @@ -200,7 +200,7 @@ def worker_int(worker): ## get traceback info import threading, sys, traceback - id2name = dict([(th.ident, th.name) for th in threading.enumerate()]) + id2name = {th.ident: th.name for th in threading.enumerate()} code = [] for threadId, stack in sys._current_frames().items(): code.append("\n# Thread: %s(%d)" % (id2name.get(threadId,""), diff --git a/examples/read_django_settings.py b/examples/read_django_settings.py index 045fa242..2e97368e 100644 --- a/examples/read_django_settings.py +++ b/examples/read_django_settings.py @@ -2,8 +2,6 @@ Use this config file in your script like this: $ gunicorn project_name.wsgi:application -c read_django_settings.py - -You need to replace the exec() call if you want it to work on Python 2. """ settings_dict = {} diff --git a/examples/standalone_app.py b/examples/standalone_app.py index 07c98956..9abda283 100644 --- a/examples/standalone_app.py +++ b/examples/standalone_app.py @@ -8,14 +8,10 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -from __future__ import unicode_literals - import multiprocessing import gunicorn.app.base -from gunicorn.six import iteritems - def number_of_workers(): return (multiprocessing.cpu_count() * 2) + 1 @@ -42,9 +38,9 @@ class StandaloneApplication(gunicorn.app.base.BaseApplication): super(StandaloneApplication, self).__init__() def load_config(self): - config = dict([(key, value) for key, value in iteritems(self.options) - if key in self.cfg.settings and value is not None]) - for key, value in iteritems(config): + config = {key: value for key, value in self.options.items() + if key in self.cfg.settings and value is not None} + for key, value in config.items(): self.cfg.set(key.lower(), value) def load(self): diff --git a/gunicorn/_compat.py b/gunicorn/_compat.py index 2416b3e3..364ec370 100644 --- a/gunicorn/_compat.py +++ b/gunicorn/_compat.py @@ -1,10 +1,3 @@ -import sys - -from gunicorn import six - -PY26 = (sys.version_info[:2] == (2, 6)) - - def _check_if_pyc(fname): """Return True if the extension is .pyc, False if .py and None if otherwise""" @@ -62,147 +55,12 @@ def _get_codeobj(pyfile): # Return code object return code_obj -if six.PY3: - def execfile_(fname, *args): - if fname.endswith(".pyc"): - code = _get_codeobj(fname) - else: - code = compile(open(fname, 'rb').read(), fname, 'exec') - return six.exec_(code, *args) - def bytes_to_str(b): - if isinstance(b, six.text_type): - return b - return str(b, 'latin1') - - import urllib.parse - - def unquote_to_wsgi_str(string): - return _unquote_to_bytes(string).decode('latin-1') - - _unquote_to_bytes = urllib.parse.unquote_to_bytes - -else: - def execfile_(fname, *args): - """ Overriding PY2 execfile() implementation to support .pyc files """ - if fname.endswith(".pyc"): - return six.exec_(_get_codeobj(fname), *args) - return execfile(fname, *args) - - def bytes_to_str(s): - if isinstance(s, unicode): - return s.encode('utf-8') - return s - - import urllib - unquote_to_wsgi_str = urllib.unquote +def execfile_(fname, *args): + if fname.endswith(".pyc"): + code = _get_codeobj(fname) + else: + code = compile(open(fname, 'rb').read(), fname, 'exec') + return exec(code, *args) -# The following code adapted from trollius.py33_exceptions -def _wrap_error(exc, mapping, key): - if key not in mapping: - return - new_err_cls = mapping[key] - new_err = new_err_cls(*exc.args) - - # raise a new exception with the original traceback - six.reraise(new_err_cls, new_err, - exc.__traceback__ if hasattr(exc, '__traceback__') else sys.exc_info()[2]) - -if PY26: - from urlparse import ( - _parse_cache, MAX_CACHE_SIZE, clear_cache, _splitnetloc, SplitResult, - scheme_chars, - ) - - def urlsplit(url, scheme='', allow_fragments=True): - """Parse a URL into 5 components: - :///?# - Return a 5-tuple: (scheme, netloc, path, query, fragment). - Note that we don't break the components up in smaller bits - (e.g. netloc is a single string) and we don't expand % escapes.""" - allow_fragments = bool(allow_fragments) - key = url, scheme, allow_fragments, type(url), type(scheme) - cached = _parse_cache.get(key, None) - if cached: - return cached - if len(_parse_cache) >= MAX_CACHE_SIZE: # avoid runaway growth - clear_cache() - netloc = query = fragment = '' - i = url.find(':') - if i > 0: - if url[:i] == 'http': # optimize the common case - scheme = url[:i].lower() - url = url[i+1:] - if url[:2] == '//': - netloc, url = _splitnetloc(url, 2) - if (('[' in netloc and ']' not in netloc) or - (']' in netloc and '[' not in netloc)): - raise ValueError("Invalid IPv6 URL") - if allow_fragments and '#' in url: - url, fragment = url.split('#', 1) - if '?' in url: - url, query = url.split('?', 1) - v = SplitResult(scheme, netloc, url, query, fragment) - _parse_cache[key] = v - return v - for c in url[:i]: - if c not in scheme_chars: - break - else: - # make sure "url" is not actually a port number (in which case - # "scheme" is really part of the path) - rest = url[i+1:] - if not rest or any(c not in '0123456789' for c in rest): - # not a port number - scheme, url = url[:i].lower(), rest - - if url[:2] == '//': - netloc, url = _splitnetloc(url, 2) - if (('[' in netloc and ']' not in netloc) or - (']' in netloc and '[' not in netloc)): - raise ValueError("Invalid IPv6 URL") - if allow_fragments and '#' in url: - url, fragment = url.split('#', 1) - if '?' in url: - url, query = url.split('?', 1) - v = SplitResult(scheme, netloc, url, query, fragment) - _parse_cache[key] = v - return v - -else: - from gunicorn.six.moves.urllib.parse import urlsplit - - -import inspect - -if hasattr(inspect, 'signature'): - positionals = ( - inspect.Parameter.POSITIONAL_ONLY, - inspect.Parameter.POSITIONAL_OR_KEYWORD, - ) - - def get_arity(f): - sig = inspect.signature(f) - arity = 0 - - for param in sig.parameters.values(): - if param.kind in positionals: - arity += 1 - - return arity -else: - def get_arity(f): - return len(inspect.getargspec(f)[0]) - - -try: - import html - - def html_escape(s): - return html.escape(s) -except ImportError: - import cgi - - def html_escape(s): - return cgi.escape(s, quote=True) diff --git a/gunicorn/app/base.py b/gunicorn/app/base.py index e468c956..1d6b3897 100644 --- a/gunicorn/app/base.py +++ b/gunicorn/app/base.py @@ -2,8 +2,6 @@ # # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -from __future__ import print_function - import os import sys import traceback diff --git a/gunicorn/app/pasterapp.py b/gunicorn/app/pasterapp.py index dbcd339d..0f9de435 100644 --- a/gunicorn/app/pasterapp.py +++ b/gunicorn/app/pasterapp.py @@ -2,7 +2,6 @@ # # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -from __future__ import print_function # pylint: skip-file diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py index 083ee6a0..47ba2b66 100644 --- a/gunicorn/arbiter.py +++ b/gunicorn/arbiter.py @@ -2,8 +2,6 @@ # # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -from __future__ import print_function - import errno import os import random diff --git a/gunicorn/argparse_compat.py b/gunicorn/argparse_compat.py deleted file mode 100644 index 32d948c0..00000000 --- a/gunicorn/argparse_compat.py +++ /dev/null @@ -1,2362 +0,0 @@ -# Author: Steven J. Bethard . - -"""Command-line parsing library - -This module is an optparse-inspired command-line parsing library that: - - - handles both optional and positional arguments - - produces highly informative usage messages - - supports parsers that dispatch to sub-parsers - -The following is a simple usage example that sums integers from the -command-line and writes the result to a file:: - - parser = argparse.ArgumentParser( - description='sum the integers at the command line') - parser.add_argument( - 'integers', metavar='int', nargs='+', type=int, - help='an integer to be summed') - parser.add_argument( - '--log', default=sys.stdout, type=argparse.FileType('w'), - help='the file where the sum should be written') - args = parser.parse_args() - args.log.write('%s' % sum(args.integers)) - args.log.close() - -The module contains the following public classes: - - - ArgumentParser -- The main entry point for command-line parsing. As the - example above shows, the add_argument() method is used to populate - the parser with actions for optional and positional arguments. Then - the parse_args() method is invoked to convert the args at the - command-line into an object with attributes. - - - ArgumentError -- The exception raised by ArgumentParser objects when - there are errors with the parser's actions. Errors raised while - parsing the command-line are caught by ArgumentParser and emitted - as command-line messages. - - - FileType -- A factory for defining types of files to be created. As the - example above shows, instances of FileType are typically passed as - the type= argument of add_argument() calls. - - - Action -- The base class for parser actions. Typically actions are - selected by passing strings like 'store_true' or 'append_const' to - the action= argument of add_argument(). However, for greater - customization of ArgumentParser actions, subclasses of Action may - be defined and passed as the action= argument. - - - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, - ArgumentDefaultsHelpFormatter -- Formatter classes which - may be passed as the formatter_class= argument to the - ArgumentParser constructor. HelpFormatter is the default, - RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser - not to change the formatting for help text, and - ArgumentDefaultsHelpFormatter adds information about argument defaults - to the help. - -All other classes in this module are considered implementation details. -(Also note that HelpFormatter and RawDescriptionHelpFormatter are only -considered public as object names -- the API of the formatter objects is -still considered an implementation detail.) -""" - -__version__ = '1.2.1' -__all__ = [ - 'ArgumentParser', - 'ArgumentError', - 'ArgumentTypeError', - 'FileType', - 'HelpFormatter', - 'ArgumentDefaultsHelpFormatter', - 'RawDescriptionHelpFormatter', - 'RawTextHelpFormatter', - 'Namespace', - 'Action', - 'ONE_OR_MORE', - 'OPTIONAL', - 'PARSER', - 'REMAINDER', - 'SUPPRESS', - 'ZERO_OR_MORE', -] - - -import copy as _copy -import os as _os -import re as _re -import sys as _sys -import textwrap as _textwrap - -from gettext import gettext as _ - -try: - set -except NameError: - # for python < 2.4 compatibility (sets module is there since 2.3): - from sets import Set as set - -try: - basestring -except NameError: - basestring = str - -try: - sorted -except NameError: - # for python < 2.4 compatibility: - def sorted(iterable, reverse=False): - result = list(iterable) - result.sort() - if reverse: - result.reverse() - return result - - -def _callable(obj): - return hasattr(obj, '__call__') or hasattr(obj, '__bases__') - - -SUPPRESS = '==SUPPRESS==' - -OPTIONAL = '?' -ZERO_OR_MORE = '*' -ONE_OR_MORE = '+' -PARSER = 'A...' -REMAINDER = '...' -_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args' - -# ============================= -# Utility functions and classes -# ============================= - -class _AttributeHolder(object): - """Abstract base class that provides __repr__. - - The __repr__ method returns a string in the format:: - ClassName(attr=name, attr=name, ...) - The attributes are determined either by a class-level attribute, - '_kwarg_names', or by inspecting the instance __dict__. - """ - - def __repr__(self): - type_name = type(self).__name__ - arg_strings = [] - for arg in self._get_args(): - arg_strings.append(repr(arg)) - for name, value in self._get_kwargs(): - arg_strings.append('%s=%r' % (name, value)) - return '%s(%s)' % (type_name, ', '.join(arg_strings)) - - def _get_kwargs(self): - return sorted(self.__dict__.items()) - - def _get_args(self): - return [] - - -def _ensure_value(namespace, name, value): - if getattr(namespace, name, None) is None: - setattr(namespace, name, value) - return getattr(namespace, name) - - -# =============== -# Formatting Help -# =============== - -class HelpFormatter(object): - """Formatter for generating usage messages and argument help strings. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def __init__(self, - prog, - indent_increment=2, - max_help_position=24, - width=None): - - # default setting for width - if width is None: - try: - width = int(_os.environ['COLUMNS']) - except (KeyError, ValueError): - width = 80 - width -= 2 - - self._prog = prog - self._indent_increment = indent_increment - self._max_help_position = max_help_position - self._width = width - - self._current_indent = 0 - self._level = 0 - self._action_max_length = 0 - - self._root_section = self._Section(self, None) - self._current_section = self._root_section - - self._whitespace_matcher = _re.compile(r'\s+') - self._long_break_matcher = _re.compile(r'\n\n\n+') - - # =============================== - # Section and indentation methods - # =============================== - def _indent(self): - self._current_indent += self._indent_increment - self._level += 1 - - def _dedent(self): - self._current_indent -= self._indent_increment - assert self._current_indent >= 0, 'Indent decreased below 0.' - self._level -= 1 - - class _Section(object): - - def __init__(self, formatter, parent, heading=None): - self.formatter = formatter - self.parent = parent - self.heading = heading - self.items = [] - - def format_help(self): - # format the indented section - if self.parent is not None: - self.formatter._indent() - join = self.formatter._join_parts - for func, args in self.items: - func(*args) - item_help = join([func(*args) for func, args in self.items]) - if self.parent is not None: - self.formatter._dedent() - - # return nothing if the section was empty - if not item_help: - return '' - - # add the heading if the section was non-empty - if self.heading is not SUPPRESS and self.heading is not None: - current_indent = self.formatter._current_indent - heading = '%*s%s:\n' % (current_indent, '', self.heading) - else: - heading = '' - - # join the section-initial newline, the heading and the help - return join(['\n', heading, item_help, '\n']) - - def _add_item(self, func, args): - self._current_section.items.append((func, args)) - - # ======================== - # Message building methods - # ======================== - def start_section(self, heading): - self._indent() - section = self._Section(self, self._current_section, heading) - self._add_item(section.format_help, []) - self._current_section = section - - def end_section(self): - self._current_section = self._current_section.parent - self._dedent() - - def add_text(self, text): - if text is not SUPPRESS and text is not None: - self._add_item(self._format_text, [text]) - - def add_usage(self, usage, actions, groups, prefix=None): - if usage is not SUPPRESS: - args = usage, actions, groups, prefix - self._add_item(self._format_usage, args) - - def add_argument(self, action): - if action.help is not SUPPRESS: - - # find all invocations - get_invocation = self._format_action_invocation - invocations = [get_invocation(action)] - for subaction in self._iter_indented_subactions(action): - invocations.append(get_invocation(subaction)) - - # update the maximum item length - invocation_length = max([len(s) for s in invocations]) - action_length = invocation_length + self._current_indent - self._action_max_length = max(self._action_max_length, - action_length) - - # add the item to the list - self._add_item(self._format_action, [action]) - - def add_arguments(self, actions): - for action in actions: - self.add_argument(action) - - # ======================= - # Help-formatting methods - # ======================= - def format_help(self): - help = self._root_section.format_help() - if help: - help = self._long_break_matcher.sub('\n\n', help) - help = help.strip('\n') + '\n' - return help - - def _join_parts(self, part_strings): - return ''.join([part - for part in part_strings - if part and part is not SUPPRESS]) - - def _format_usage(self, usage, actions, groups, prefix): - if prefix is None: - prefix = _('usage: ') - - # if usage is specified, use that - if usage is not None: - usage = usage % dict(prog=self._prog) - - # if no optionals or positionals are available, usage is just prog - elif usage is None and not actions: - usage = '%(prog)s' % dict(prog=self._prog) - - # if optionals and positionals are available, calculate usage - elif usage is None: - prog = '%(prog)s' % dict(prog=self._prog) - - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - - # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) - - # wrap the usage parts if it's too long - text_width = self._width - self._current_indent - if len(prefix) + len(usage) > text_width: - - # break usage into wrappable parts - part_regexp = r'\(.*?\)+|\[.*?\]+|\S+' - opt_usage = format(optionals, groups) - pos_usage = format(positionals, groups) - opt_parts = _re.findall(part_regexp, opt_usage) - pos_parts = _re.findall(part_regexp, pos_usage) - assert ' '.join(opt_parts) == opt_usage - assert ' '.join(pos_parts) == pos_usage - - # helper for wrapping lines - def get_lines(parts, indent, prefix=None): - lines = [] - line = [] - if prefix is not None: - line_len = len(prefix) - 1 - else: - line_len = len(indent) - 1 - for part in parts: - if line_len + 1 + len(part) > text_width: - lines.append(indent + ' '.join(line)) - line = [] - line_len = len(indent) - 1 - line.append(part) - line_len += len(part) + 1 - if line: - lines.append(indent + ' '.join(line)) - if prefix is not None: - lines[0] = lines[0][len(indent):] - return lines - - # if prog is short, follow it with optionals or positionals - if len(prefix) + len(prog) <= 0.75 * text_width: - indent = ' ' * (len(prefix) + len(prog) + 1) - if opt_parts: - lines = get_lines([prog] + opt_parts, indent, prefix) - lines.extend(get_lines(pos_parts, indent)) - elif pos_parts: - lines = get_lines([prog] + pos_parts, indent, prefix) - else: - lines = [prog] - - # if prog is long, put it on its own line - else: - indent = ' ' * len(prefix) - parts = opt_parts + pos_parts - lines = get_lines(parts, indent) - if len(lines) > 1: - lines = [] - lines.extend(get_lines(opt_parts, indent)) - lines.extend(get_lines(pos_parts, indent)) - lines = [prog] + lines - - # join lines into usage - usage = '\n'.join(lines) - - # prefix with 'usage:' - return '%s%s\n\n' % (prefix, usage) - - def _format_actions_usage(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} - for group in groups: - try: - start = actions.index(group._group_actions[0]) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if actions[start:end] == group._group_actions: - for action in group._group_actions: - group_actions.add(action) - if not group.required: - if start in inserts: - inserts[start] += ' [' - else: - inserts[start] = '[' - inserts[end] = ']' - else: - if start in inserts: - inserts[start] += ' (' - else: - inserts[start] = '(' - inserts[end] = ')' - for i in range(start + 1, end): - inserts[i] = '|' - - # collect all actions format strings - parts = [] - for i, action in enumerate(actions): - - # suppressed arguments are marked with None - # remove | separators for suppressed arguments - if action.help is SUPPRESS: - parts.append(None) - if inserts.get(i) == '|': - inserts.pop(i) - elif inserts.get(i + 1) == '|': - inserts.pop(i + 1) - - # produce all arg strings - elif not action.option_strings: - part = self._format_args(action, action.dest) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # add the action string to the list - parts.append(part) - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = '%s' % option_string - - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = action.dest.upper() - args_string = self._format_args(action, default) - part = '%s %s' % (option_string, args_string) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part - - # add the action string to the list - parts.append(part) - - # insert things at the necessary indices - for i in sorted(inserts, reverse=True): - parts[i:i] = [inserts[i]] - - # join all the action items with spaces - text = ' '.join([item for item in parts if item is not None]) - - # clean up separators for mutually exclusive groups - open = r'[\[(]' - close = r'[\])]' - text = _re.sub(r'(%s) ' % open, r'\1', text) - text = _re.sub(r' (%s)' % close, r'\1', text) - text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = _re.sub(r'\(([^|]*)\)', r'\1', text) - text = text.strip() - - # return the text - return text - - def _format_text(self, text): - if '%(prog)' in text: - text = text % dict(prog=self._prog) - text_width = self._width - self._current_indent - indent = ' ' * self._current_indent - return self._fill_text(text, text_width, indent) + '\n\n' - - def _format_action(self, action): - # determine the required width and the entry label - help_position = min(self._action_max_length + 2, - self._max_help_position) - help_width = self._width - help_position - action_width = help_position - self._current_indent - 2 - action_header = self._format_action_invocation(action) - - # ho nelp; start on same line and add a final newline - if not action.help: - tup = self._current_indent, '', action_header - action_header = '%*s%s\n' % tup - - # short action name; start on the same line and pad two spaces - elif len(action_header) <= action_width: - tup = self._current_indent, '', action_width, action_header - action_header = '%*s%-*s ' % tup - indent_first = 0 - - # long action name; start on the next line - else: - tup = self._current_indent, '', action_header - action_header = '%*s%s\n' % tup - indent_first = help_position - - # collect the pieces of the action help - parts = [action_header] - - # if there was help for the action, add lines of help text - if action.help: - help_text = self._expand_help(action) - help_lines = self._split_lines(help_text, help_width) - parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) - for line in help_lines[1:]: - parts.append('%*s%s\n' % (help_position, '', line)) - - # or add a newline if the description doesn't end with one - elif not action_header.endswith('\n'): - parts.append('\n') - - # if there are any sub-actions, add their help as well - for subaction in self._iter_indented_subactions(action): - parts.append(self._format_action(subaction)) - - # return a single string - return self._join_parts(parts) - - def _format_action_invocation(self, action): - if not action.option_strings: - metavar, = self._metavar_formatter(action, action.dest)(1) - return metavar - - else: - parts = [] - - # if the Optional doesn't take a value, format is: - # -s, --long - if action.nargs == 0: - parts.extend(action.option_strings) - - # if the Optional takes a value, format is: - # -s ARGS, --long ARGS - else: - default = action.dest.upper() - args_string = self._format_args(action, default) - for option_string in action.option_strings: - parts.append('%s %s' % (option_string, args_string)) - - return ', '.join(parts) - - def _metavar_formatter(self, action, default_metavar): - if action.metavar is not None: - result = action.metavar - elif action.choices is not None: - choice_strs = [str(choice) for choice in action.choices] - result = '{%s}' % ','.join(choice_strs) - else: - result = default_metavar - - def format(tuple_size): - if isinstance(result, tuple): - return result - else: - return (result, ) * tuple_size - return format - - def _format_args(self, action, default_metavar): - get_metavar = self._metavar_formatter(action, default_metavar) - if action.nargs is None: - result = '%s' % get_metavar(1) - elif action.nargs == OPTIONAL: - result = '[%s]' % get_metavar(1) - elif action.nargs == ZERO_OR_MORE: - result = '[%s [%s ...]]' % get_metavar(2) - elif action.nargs == ONE_OR_MORE: - result = '%s [%s ...]' % get_metavar(2) - elif action.nargs == REMAINDER: - result = '...' - elif action.nargs == PARSER: - result = '%s ...' % get_metavar(1) - else: - formats = ['%s' for _ in range(action.nargs)] - result = ' '.join(formats) % get_metavar(action.nargs) - return result - - def _expand_help(self, action): - params = dict(vars(action), prog=self._prog) - for name in list(params): - if params[name] is SUPPRESS: - del params[name] - for name in list(params): - if hasattr(params[name], '__name__'): - params[name] = params[name].__name__ - if params.get('choices') is not None: - choices_str = ', '.join([str(c) for c in params['choices']]) - params['choices'] = choices_str - return self._get_help_string(action) % params - - def _iter_indented_subactions(self, action): - try: - get_subactions = action._get_subactions - except AttributeError: - pass - else: - self._indent() - for subaction in get_subactions(): - yield subaction - self._dedent() - - def _split_lines(self, text, width): - text = self._whitespace_matcher.sub(' ', text).strip() - return _textwrap.wrap(text, width) - - def _fill_text(self, text, width, indent): - text = self._whitespace_matcher.sub(' ', text).strip() - return _textwrap.fill(text, width, initial_indent=indent, - subsequent_indent=indent) - - def _get_help_string(self, action): - return action.help - - -class RawDescriptionHelpFormatter(HelpFormatter): - """Help message formatter which retains any formatting in descriptions. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _fill_text(self, text, width, indent): - return ''.join([indent + line for line in text.splitlines(True)]) - - -class RawTextHelpFormatter(RawDescriptionHelpFormatter): - """Help message formatter which retains formatting of all help text. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _split_lines(self, text, width): - return text.splitlines() - - -class ArgumentDefaultsHelpFormatter(HelpFormatter): - """Help message formatter which adds default values to argument help. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _get_help_string(self, action): - help = action.help - if '%(default)' not in action.help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += ' (default: %(default)s)' - return help - - -# ===================== -# Options and Arguments -# ===================== - -def _get_action_name(argument): - if argument is None: - return None - elif argument.option_strings: - return '/'.join(argument.option_strings) - elif argument.metavar not in (None, SUPPRESS): - return argument.metavar - elif argument.dest not in (None, SUPPRESS): - return argument.dest - else: - return None - - -class ArgumentError(Exception): - """An error from creating or using an argument (optional or positional). - - The string value of this exception is the message, augmented with - information about the argument that caused it. - """ - - def __init__(self, argument, message): - self.argument_name = _get_action_name(argument) - self.message = message - - def __str__(self): - if self.argument_name is None: - format = '%(message)s' - else: - format = 'argument %(argument_name)s: %(message)s' - return format % dict(message=self.message, - argument_name=self.argument_name) - - -class ArgumentTypeError(Exception): - """An error from trying to convert a command line string to a type.""" - pass - - -# ============== -# Action classes -# ============== - -class Action(_AttributeHolder): - """Information about how to convert command line strings to Python objects. - - Action objects are used by an ArgumentParser to represent the information - needed to parse a single argument from one or more strings from the - command line. The keyword arguments to the Action constructor are also - all attributes of Action instances. - - Keyword Arguments: - - - option_strings -- A list of command-line option strings which - should be associated with this action. - - - dest -- The name of the attribute to hold the created object(s) - - - nargs -- The number of command-line arguments that should be - consumed. By default, one argument will be consumed and a single - value will be produced. Other values include: - - N (an integer) consumes N arguments (and produces a list) - - '?' consumes zero or one arguments - - '*' consumes zero or more arguments (and produces a list) - - '+' consumes one or more arguments (and produces a list) - Note that the difference between the default and nargs=1 is that - with the default, a single value will be produced, while with - nargs=1, a list containing a single value will be produced. - - - const -- The value to be produced if the option is specified and the - option uses an action that takes no values. - - - default -- The value to be produced if the option is not specified. - - - type -- The type which the command-line arguments should be converted - to, should be one of 'string', 'int', 'float', 'complex' or a - callable object that accepts a single string argument. If None, - 'string' is assumed. - - - choices -- A container of values that should be allowed. If not None, - after a command-line argument has been converted to the appropriate - type, an exception will be raised if it is not a member of this - collection. - - - required -- True if the action must always be specified at the - command line. This is only meaningful for optional command-line - arguments. - - - help -- The help string describing the argument. - - - metavar -- The name to be used for the option's argument with the - help string. If None, the 'dest' value will be used as the name. - """ - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - self.option_strings = option_strings - self.dest = dest - self.nargs = nargs - self.const = const - self.default = default - self.type = type - self.choices = choices - self.required = required - self.help = help - self.metavar = metavar - - def _get_kwargs(self): - names = [ - 'option_strings', - 'dest', - 'nargs', - 'const', - 'default', - 'type', - 'choices', - 'help', - 'metavar', - ] - return [(name, getattr(self, name)) for name in names] - - def __call__(self, parser, namespace, values, option_string=None): - raise NotImplementedError(_('.__call__() not defined')) - - -class _StoreAction(Action): - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - if nargs == 0: - raise ValueError('nargs for store actions must be > 0; if you ' - 'have nothing to store, actions such as store ' - 'true or store const may be more appropriate') - if const is not None and nargs != OPTIONAL: - raise ValueError('nargs must be %r to supply const' % OPTIONAL) - super(_StoreAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, values) - - -class _StoreConstAction(Action): - - def __init__(self, - option_strings, - dest, - const, - default=None, - required=False, - help=None, - metavar=None): - super(_StoreConstAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - const=const, - default=default, - required=required, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, self.const) - - -class _StoreTrueAction(_StoreConstAction): - - def __init__(self, - option_strings, - dest, - default=False, - required=False, - help=None): - super(_StoreTrueAction, self).__init__( - option_strings=option_strings, - dest=dest, - const=True, - default=default, - required=required, - help=help) - - -class _StoreFalseAction(_StoreConstAction): - - def __init__(self, - option_strings, - dest, - default=True, - required=False, - help=None): - super(_StoreFalseAction, self).__init__( - option_strings=option_strings, - dest=dest, - const=False, - default=default, - required=required, - help=help) - - -class _AppendAction(Action): - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - if nargs == 0: - raise ValueError('nargs for append actions must be > 0; if arg ' - 'strings are not supplying the value to append, ' - 'the append const action may be more appropriate') - if const is not None and nargs != OPTIONAL: - raise ValueError('nargs must be %r to supply const' % OPTIONAL) - super(_AppendAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - items = _copy.copy(_ensure_value(namespace, self.dest, [])) - items.append(values) - setattr(namespace, self.dest, items) - - -class _AppendConstAction(Action): - - def __init__(self, - option_strings, - dest, - const, - default=None, - required=False, - help=None, - metavar=None): - super(_AppendConstAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - const=const, - default=default, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - items = _copy.copy(_ensure_value(namespace, self.dest, [])) - items.append(self.const) - setattr(namespace, self.dest, items) - - -class _CountAction(Action): - - def __init__(self, - option_strings, - dest, - default=None, - required=False, - help=None): - super(_CountAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - default=default, - required=required, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - new_count = _ensure_value(namespace, self.dest, 0) + 1 - setattr(namespace, self.dest, new_count) - - -class _HelpAction(Action): - - def __init__(self, - option_strings, - dest=SUPPRESS, - default=SUPPRESS, - help=None): - super(_HelpAction, self).__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - parser.print_help() - parser.exit() - - -class _VersionAction(Action): - - def __init__(self, - option_strings, - version=None, - dest=SUPPRESS, - default=SUPPRESS, - help="show program's version number and exit"): - super(_VersionAction, self).__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) - self.version = version - - def __call__(self, parser, namespace, values, option_string=None): - version = self.version - if version is None: - version = parser.version - formatter = parser._get_formatter() - formatter.add_text(version) - parser.exit(message=formatter.format_help()) - - -class _SubParsersAction(Action): - - class _ChoicesPseudoAction(Action): - - def __init__(self, name, help): - sup = super(_SubParsersAction._ChoicesPseudoAction, self) - sup.__init__(option_strings=[], dest=name, help=help) - - def __init__(self, - option_strings, - prog, - parser_class, - dest=SUPPRESS, - help=None, - metavar=None): - - self._prog_prefix = prog - self._parser_class = parser_class - self._name_parser_map = {} - self._choices_actions = [] - - super(_SubParsersAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=PARSER, - choices=self._name_parser_map, - help=help, - metavar=metavar) - - def add_parser(self, name, **kwargs): - # set prog from the existing prefix - if kwargs.get('prog') is None: - kwargs['prog'] = '%s %s' % (self._prog_prefix, name) - - # create a pseudo-action to hold the choice help - if 'help' in kwargs: - help = kwargs.pop('help') - choice_action = self._ChoicesPseudoAction(name, help) - self._choices_actions.append(choice_action) - - # create the parser and add it to the map - parser = self._parser_class(**kwargs) - self._name_parser_map[name] = parser - return parser - - def _get_subactions(self): - return self._choices_actions - - def __call__(self, parser, namespace, values, option_string=None): - parser_name = values[0] - arg_strings = values[1:] - - # set the parser name if requested - if self.dest is not SUPPRESS: - setattr(namespace, self.dest, parser_name) - - # select the parser - try: - parser = self._name_parser_map[parser_name] - except KeyError: - tup = parser_name, ', '.join(self._name_parser_map) - msg = _('unknown parser %r (choices: %s)' % tup) - raise ArgumentError(self, msg) - - # parse all the remaining options into the namespace - # store any unrecognized options on the object, so that the top - # level parser can decide what to do with them - namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) - if arg_strings: - vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) - getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) - - -# ============== -# Type classes -# ============== - -class FileType(object): - """Factory for creating file object types - - Instances of FileType are typically passed as type= arguments to the - ArgumentParser add_argument() method. - - Keyword Arguments: - - mode -- A string indicating how the file is to be opened. Accepts the - same values as the builtin open() function. - - bufsize -- The file's desired buffer size. Accepts the same values as - the builtin open() function. - """ - - def __init__(self, mode='r', bufsize=None): - self._mode = mode - self._bufsize = bufsize - - def __call__(self, string): - # the special argument "-" means sys.std{in,out} - if string == '-': - if 'r' in self._mode: - return _sys.stdin - elif 'w' in self._mode: - return _sys.stdout - else: - msg = _('argument "-" with mode %r' % self._mode) - raise ValueError(msg) - - # all other arguments are used as file names - if self._bufsize: - return open(string, self._mode, self._bufsize) - else: - return open(string, self._mode) - - def __repr__(self): - args = [self._mode, self._bufsize] - args_str = ', '.join([repr(arg) for arg in args if arg is not None]) - return '%s(%s)' % (type(self).__name__, args_str) - -# =========================== -# Optional and Positional Parsing -# =========================== - -class Namespace(_AttributeHolder): - """Simple object for storing attributes. - - Implements equality by attribute names and values, and provides a simple - string representation. - """ - - def __init__(self, **kwargs): - for name in kwargs: - setattr(self, name, kwargs[name]) - - __hash__ = None - - def __eq__(self, other): - return vars(self) == vars(other) - - def __ne__(self, other): - return not (self == other) - - def __contains__(self, key): - return key in self.__dict__ - - -class _ActionsContainer(object): - - def __init__(self, - description, - prefix_chars, - argument_default, - conflict_handler): - super(_ActionsContainer, self).__init__() - - self.description = description - self.argument_default = argument_default - self.prefix_chars = prefix_chars - self.conflict_handler = conflict_handler - - # set up registries - self._registries = {} - - # register actions - self.register('action', None, _StoreAction) - self.register('action', 'store', _StoreAction) - self.register('action', 'store_const', _StoreConstAction) - self.register('action', 'store_true', _StoreTrueAction) - self.register('action', 'store_false', _StoreFalseAction) - self.register('action', 'append', _AppendAction) - self.register('action', 'append_const', _AppendConstAction) - self.register('action', 'count', _CountAction) - self.register('action', 'help', _HelpAction) - self.register('action', 'version', _VersionAction) - self.register('action', 'parsers', _SubParsersAction) - - # raise an exception if the conflict handler is invalid - self._get_handler() - - # action storage - self._actions = [] - self._option_string_actions = {} - - # groups - self._action_groups = [] - self._mutually_exclusive_groups = [] - - # defaults storage - self._defaults = {} - - # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') - - # whether or not there are any optionals that look like negative - # numbers -- uses a list so it can be shared and edited - self._has_negative_number_optionals = [] - - # ==================== - # Registration methods - # ==================== - def register(self, registry_name, value, object): - registry = self._registries.setdefault(registry_name, {}) - registry[value] = object - - def _registry_get(self, registry_name, value, default=None): - return self._registries[registry_name].get(value, default) - - # ================================== - # Namespace default accessor methods - # ================================== - def set_defaults(self, **kwargs): - self._defaults.update(kwargs) - - # if these defaults match any existing arguments, replace - # the previous default on the object with the new one - for action in self._actions: - if action.dest in kwargs: - action.default = kwargs[action.dest] - - def get_default(self, dest): - for action in self._actions: - if action.dest == dest and action.default is not None: - return action.default - return self._defaults.get(dest, None) - - - # ======================= - # Adding argument actions - # ======================= - def add_argument(self, *args, **kwargs): - """ - add_argument(dest, ..., name=value, ...) - add_argument(option_string, option_string, ..., name=value, ...) - """ - - # if no positional args are supplied or only one is supplied and - # it doesn't look like an option string, parse a positional - # argument - chars = self.prefix_chars - if not args or len(args) == 1 and args[0][0] not in chars: - if args and 'dest' in kwargs: - raise ValueError('dest supplied twice for positional argument') - kwargs = self._get_positional_kwargs(*args, **kwargs) - - # otherwise, we're adding an optional argument - else: - kwargs = self._get_optional_kwargs(*args, **kwargs) - - # if no default was supplied, use the parser-level default - if 'default' not in kwargs: - dest = kwargs['dest'] - if dest in self._defaults: - kwargs['default'] = self._defaults[dest] - elif self.argument_default is not None: - kwargs['default'] = self.argument_default - - # create the action object, and add it to the parser - action_class = self._pop_action_class(kwargs) - if not _callable(action_class): - raise ValueError('unknown action "%s"' % action_class) - action = action_class(**kwargs) - - # raise an error if the action type is not callable - type_func = self._registry_get('type', action.type, action.type) - if not _callable(type_func): - raise ValueError('%r is not callable' % type_func) - - return self._add_action(action) - - def add_argument_group(self, *args, **kwargs): - group = _ArgumentGroup(self, *args, **kwargs) - self._action_groups.append(group) - return group - - def add_mutually_exclusive_group(self, **kwargs): - group = _MutuallyExclusiveGroup(self, **kwargs) - self._mutually_exclusive_groups.append(group) - return group - - def _add_action(self, action): - # resolve any conflicts - self._check_conflict(action) - - # add to actions list - self._actions.append(action) - action.container = self - - # index the action by any option strings it has - for option_string in action.option_strings: - self._option_string_actions[option_string] = action - - # set the flag if any option strings look like negative numbers - for option_string in action.option_strings: - if self._negative_number_matcher.match(option_string): - if not self._has_negative_number_optionals: - self._has_negative_number_optionals.append(True) - - # return the created action - return action - - def _remove_action(self, action): - self._actions.remove(action) - - def _add_container_actions(self, container): - # collect groups by titles - title_group_map = {} - for group in self._action_groups: - if group.title in title_group_map: - msg = _('cannot merge actions - two groups are named %r') - raise ValueError(msg % (group.title)) - title_group_map[group.title] = group - - # map each action to its group - group_map = {} - for group in container._action_groups: - - # if a group with the title exists, use that, otherwise - # create a new group matching the container's group - if group.title not in title_group_map: - title_group_map[group.title] = self.add_argument_group( - title=group.title, - description=group.description, - conflict_handler=group.conflict_handler) - - # map the actions to their new group - for action in group._group_actions: - group_map[action] = title_group_map[group.title] - - # add container's mutually exclusive groups - # NOTE: if add_mutually_exclusive_group ever gains title= and - # description= then this code will need to be expanded as above - for group in container._mutually_exclusive_groups: - mutex_group = self.add_mutually_exclusive_group( - required=group.required) - - # map the actions to their new mutex group - for action in group._group_actions: - group_map[action] = mutex_group - - # add all actions to this container or their group - for action in container._actions: - group_map.get(action, self)._add_action(action) - - def _get_positional_kwargs(self, dest, **kwargs): - # make sure required is not specified - if 'required' in kwargs: - msg = _("'required' is an invalid argument for positionals") - raise TypeError(msg) - - # mark positional arguments as required if at least one is - # always required - if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: - kwargs['required'] = True - if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: - kwargs['required'] = True - - # return the keyword arguments with no option strings - return dict(kwargs, dest=dest, option_strings=[]) - - def _get_optional_kwargs(self, *args, **kwargs): - # determine short and long option strings - option_strings = [] - long_option_strings = [] - for option_string in args: - # error on strings that don't start with an appropriate prefix - if not option_string[0] in self.prefix_chars: - msg = _('invalid option string %r: ' - 'must start with a character %r') - tup = option_string, self.prefix_chars - raise ValueError(msg % tup) - - # strings starting with two prefix characters are long options - option_strings.append(option_string) - if option_string[0] in self.prefix_chars: - if len(option_string) > 1: - if option_string[1] in self.prefix_chars: - long_option_strings.append(option_string) - - # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' - dest = kwargs.pop('dest', None) - if dest is None: - if long_option_strings: - dest_option_string = long_option_strings[0] - else: - dest_option_string = option_strings[0] - dest = dest_option_string.lstrip(self.prefix_chars) - if not dest: - msg = _('dest= is required for options like %r') - raise ValueError(msg % option_string) - dest = dest.replace('-', '_') - - # return the updated keyword arguments - return dict(kwargs, dest=dest, option_strings=option_strings) - - def _pop_action_class(self, kwargs, default=None): - action = kwargs.pop('action', default) - return self._registry_get('action', action, action) - - def _get_handler(self): - # determine function from conflict handler string - handler_func_name = '_handle_conflict_%s' % self.conflict_handler - try: - return getattr(self, handler_func_name) - except AttributeError: - msg = _('invalid conflict_resolution value: %r') - raise ValueError(msg % self.conflict_handler) - - def _check_conflict(self, action): - - # find all options that conflict with this option - confl_optionals = [] - for option_string in action.option_strings: - if option_string in self._option_string_actions: - confl_optional = self._option_string_actions[option_string] - confl_optionals.append((option_string, confl_optional)) - - # resolve any conflicts - if confl_optionals: - conflict_handler = self._get_handler() - conflict_handler(action, confl_optionals) - - def _handle_conflict_error(self, action, conflicting_actions): - message = _('conflicting option string(s): %s') - conflict_string = ', '.join([option_string - for option_string, action - in conflicting_actions]) - raise ArgumentError(action, message % conflict_string) - - def _handle_conflict_resolve(self, action, conflicting_actions): - - # remove all conflicting options - for option_string, action in conflicting_actions: - - # remove the conflicting option - action.option_strings.remove(option_string) - self._option_string_actions.pop(option_string, None) - - # if the option now has no option string, remove it from the - # container holding it - if not action.option_strings: - action.container._remove_action(action) - - -class _ArgumentGroup(_ActionsContainer): - - def __init__(self, container, title=None, description=None, **kwargs): - # add any missing keyword arguments by checking the container - update = kwargs.setdefault - update('conflict_handler', container.conflict_handler) - update('prefix_chars', container.prefix_chars) - update('argument_default', container.argument_default) - super_init = super(_ArgumentGroup, self).__init__ - super_init(description=description, **kwargs) - - # group attributes - self.title = title - self._group_actions = [] - - # share most attributes with the container - self._registries = container._registries - self._actions = container._actions - self._option_string_actions = container._option_string_actions - self._defaults = container._defaults - self._has_negative_number_optionals = \ - container._has_negative_number_optionals - - def _add_action(self, action): - action = super(_ArgumentGroup, self)._add_action(action) - self._group_actions.append(action) - return action - - def _remove_action(self, action): - super(_ArgumentGroup, self)._remove_action(action) - self._group_actions.remove(action) - - -class _MutuallyExclusiveGroup(_ArgumentGroup): - - def __init__(self, container, required=False): - super(_MutuallyExclusiveGroup, self).__init__(container) - self.required = required - self._container = container - - def _add_action(self, action): - if action.required: - msg = _('mutually exclusive arguments must be optional') - raise ValueError(msg) - action = self._container._add_action(action) - self._group_actions.append(action) - return action - - def _remove_action(self, action): - self._container._remove_action(action) - self._group_actions.remove(action) - - -class ArgumentParser(_AttributeHolder, _ActionsContainer): - """Object for parsing command line strings into Python objects. - - Keyword Arguments: - - prog -- The name of the program (default: sys.argv[0]) - - usage -- A usage message (default: auto-generated from arguments) - - description -- A description of what the program does - - epilog -- Text following the argument descriptions - - parents -- Parsers whose arguments should be copied into this one - - formatter_class -- HelpFormatter class for printing help messages - - prefix_chars -- Characters that prefix optional arguments - - fromfile_prefix_chars -- Characters that prefix files containing - additional arguments - - argument_default -- The default value for all arguments - - conflict_handler -- String indicating how to handle conflicts - - add_help -- Add a -h/-help option - """ - - def __init__(self, - prog=None, - usage=None, - description=None, - epilog=None, - version=None, - parents=[], - formatter_class=HelpFormatter, - prefix_chars='-', - fromfile_prefix_chars=None, - argument_default=None, - conflict_handler='error', - add_help=True): - - if version is not None: - import warnings - warnings.warn( - """The "version" argument to ArgumentParser is deprecated. """ - """Please use """ - """"add_argument(..., action='version', version="N", ...)" """ - """instead""", DeprecationWarning) - - superinit = super(ArgumentParser, self).__init__ - superinit(description=description, - prefix_chars=prefix_chars, - argument_default=argument_default, - conflict_handler=conflict_handler) - - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog - self.usage = usage - self.epilog = epilog - self.version = version - self.formatter_class = formatter_class - self.fromfile_prefix_chars = fromfile_prefix_chars - self.add_help = add_help - - add_group = self.add_argument_group - self._positionals = add_group(_('positional arguments')) - self._optionals = add_group(_('optional arguments')) - self._subparsers = None - - # register types - def identity(string): - return string - self.register('type', None, identity) - - # add help and version arguments if necessary - # (using explicit default to override global argument_default) - if '-' in prefix_chars: - default_prefix = '-' - else: - default_prefix = prefix_chars[0] - if self.add_help: - self.add_argument( - default_prefix+'h', default_prefix*2+'help', - action='help', default=SUPPRESS, - help=_('show this help message and exit')) - if self.version: - self.add_argument( - default_prefix+'v', default_prefix*2+'version', - action='version', default=SUPPRESS, - version=self.version, - help=_("show program's version number and exit")) - - # add parent arguments and defaults - for parent in parents: - self._add_container_actions(parent) - try: - defaults = parent._defaults - except AttributeError: - pass - else: - self._defaults.update(defaults) - - # ======================= - # Pretty __repr__ methods - # ======================= - def _get_kwargs(self): - names = [ - 'prog', - 'usage', - 'description', - 'version', - 'formatter_class', - 'conflict_handler', - 'add_help', - ] - return [(name, getattr(self, name)) for name in names] - - # ================================== - # Optional/Positional adding methods - # ================================== - def add_subparsers(self, **kwargs): - if self._subparsers is not None: - self.error(_('cannot have multiple subparser arguments')) - - # add the parser class to the arguments if it's not present - kwargs.setdefault('parser_class', type(self)) - - if 'title' in kwargs or 'description' in kwargs: - title = _(kwargs.pop('title', 'subcommands')) - description = _(kwargs.pop('description', None)) - self._subparsers = self.add_argument_group(title, description) - else: - self._subparsers = self._positionals - - # prog defaults to the usage message of this parser, skipping - # optional arguments and with no "usage:" prefix - if kwargs.get('prog') is None: - formatter = self._get_formatter() - positionals = self._get_positional_actions() - groups = self._mutually_exclusive_groups - formatter.add_usage(self.usage, positionals, groups, '') - kwargs['prog'] = formatter.format_help().strip() - - # create the parsers action and add it to the positionals list - parsers_class = self._pop_action_class(kwargs, 'parsers') - action = parsers_class(option_strings=[], **kwargs) - self._subparsers._add_action(action) - - # return the created parsers action - return action - - def _add_action(self, action): - if action.option_strings: - self._optionals._add_action(action) - else: - self._positionals._add_action(action) - return action - - def _get_optional_actions(self): - return [action - for action in self._actions - if action.option_strings] - - def _get_positional_actions(self): - return [action - for action in self._actions - if not action.option_strings] - - # ===================================== - # Command line argument parsing methods - # ===================================== - def parse_args(self, args=None, namespace=None): - args, argv = self.parse_known_args(args, namespace) - if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) - return args - - def parse_known_args(self, args=None, namespace=None): - # args default to the system args - if args is None: - args = _sys.argv[1:] - - # default Namespace built from parser defaults - if namespace is None: - namespace = Namespace() - - # add any action defaults that aren't present - for action in self._actions: - if action.dest is not SUPPRESS: - if not hasattr(namespace, action.dest): - if action.default is not SUPPRESS: - default = action.default - if isinstance(action.default, basestring): - default = self._get_value(action, default) - setattr(namespace, action.dest, default) - - # add any parser defaults that aren't present - for dest in self._defaults: - if not hasattr(namespace, dest): - setattr(namespace, dest, self._defaults[dest]) - - # parse the arguments and exit if there are any errors - try: - namespace, args = self._parse_known_args(args, namespace) - if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): - args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) - delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) - return namespace, args - except ArgumentError: - err = _sys.exc_info()[1] - self.error(str(err)) - - def _parse_known_args(self, arg_strings, namespace): - # replace arg strings that are file references - if self.fromfile_prefix_chars is not None: - arg_strings = self._read_args_from_files(arg_strings) - - # map all mutually exclusive arguments to the other arguments - # they can't occur with - action_conflicts = {} - for mutex_group in self._mutually_exclusive_groups: - group_actions = mutex_group._group_actions - for i, mutex_action in enumerate(mutex_group._group_actions): - conflicts = action_conflicts.setdefault(mutex_action, []) - conflicts.extend(group_actions[:i]) - conflicts.extend(group_actions[i + 1:]) - - # find all option indices, and determine the arg_string_pattern - # which has an 'O' if there is an option at an index, - # an 'A' if there is an argument, or a '-' if there is a '--' - option_string_indices = {} - arg_string_pattern_parts = [] - arg_strings_iter = iter(arg_strings) - for i, arg_string in enumerate(arg_strings_iter): - - # all args after -- are non-options - if arg_string == '--': - arg_string_pattern_parts.append('-') - for arg_string in arg_strings_iter: - arg_string_pattern_parts.append('A') - - # otherwise, add the arg to the arg strings - # and note the index if it was an option - else: - option_tuple = self._parse_optional(arg_string) - if option_tuple is None: - pattern = 'A' - else: - option_string_indices[i] = option_tuple - pattern = 'O' - arg_string_pattern_parts.append(pattern) - - # join the pieces together to form the pattern - arg_strings_pattern = ''.join(arg_string_pattern_parts) - - # converts arg strings to the appropriate and then takes the action - seen_actions = set() - seen_non_default_actions = set() - - def take_action(action, argument_strings, option_string=None): - seen_actions.add(action) - argument_values = self._get_values(action, argument_strings) - - # error if this argument is not allowed with other previously - # seen arguments, assuming that actions that use the default - # value don't really count as "present" - if argument_values is not action.default: - seen_non_default_actions.add(action) - for conflict_action in action_conflicts.get(action, []): - if conflict_action in seen_non_default_actions: - msg = _('not allowed with argument %s') - action_name = _get_action_name(conflict_action) - raise ArgumentError(action, msg % action_name) - - # take the action if we didn't receive a SUPPRESS value - # (e.g. from a default) - if argument_values is not SUPPRESS: - action(self, namespace, argument_values, option_string) - - # function to convert arg_strings into an optional action - def consume_optional(start_index): - - # get the optional identified at this index - option_tuple = option_string_indices[start_index] - action, option_string, explicit_arg = option_tuple - - # identify additional optionals in the same arg string - # (e.g. -xyz is the same as -x -y -z if no args are required) - match_argument = self._match_argument - action_tuples = [] - while True: - - # if we found no optional action, skip it - if action is None: - extras.append(arg_strings[start_index]) - return start_index + 1 - - # if there is an explicit argument, try to match the - # optional's string arguments to only this - if explicit_arg is not None: - arg_count = match_argument(action, 'A') - - # if the action is a single-dash option and takes no - # arguments, try to parse more single-dash options out - # of the tail of the option string - chars = self.prefix_chars - if arg_count == 0 and option_string[1] not in chars: - action_tuples.append((action, [], option_string)) - char = option_string[0] - option_string = char + explicit_arg[0] - new_explicit_arg = explicit_arg[1:] or None - optionals_map = self._option_string_actions - if option_string in optionals_map: - action = optionals_map[option_string] - explicit_arg = new_explicit_arg - else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - - # if the action expect exactly one argument, we've - # successfully matched the option; exit the loop - elif arg_count == 1: - stop = start_index + 1 - args = [explicit_arg] - action_tuples.append((action, args, option_string)) - break - - # error if a double-dash option did not use the - # explicit argument - else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - - # if there is no explicit argument, try to match the - # optional's string arguments with the following strings - # if successful, exit the loop - else: - start = start_index + 1 - selected_patterns = arg_strings_pattern[start:] - arg_count = match_argument(action, selected_patterns) - stop = start + arg_count - args = arg_strings[start:stop] - action_tuples.append((action, args, option_string)) - break - - # add the Optional to the list and return the index at which - # the Optional's string args stopped - assert action_tuples - for action, args, option_string in action_tuples: - take_action(action, args, option_string) - return stop - - # the list of Positionals left to be parsed; this is modified - # by consume_positionals() - positionals = self._get_positional_actions() - - # function to convert arg_strings into positional actions - def consume_positionals(start_index): - # match as many Positionals as possible - match_partial = self._match_arguments_partial - selected_pattern = arg_strings_pattern[start_index:] - arg_counts = match_partial(positionals, selected_pattern) - - # slice off the appropriate arg strings for each Positional - # and add the Positional and its args to the list - for action, arg_count in zip(positionals, arg_counts): - args = arg_strings[start_index: start_index + arg_count] - start_index += arg_count - take_action(action, args) - - # slice off the Positionals that we just parsed and return the - # index at which the Positionals' string args stopped - positionals[:] = positionals[len(arg_counts):] - return start_index - - # consume Positionals and Optionals alternately, until we have - # passed the last option string - extras = [] - start_index = 0 - if option_string_indices: - max_option_string_index = max(option_string_indices) - else: - max_option_string_index = -1 - while start_index <= max_option_string_index: - - # consume any Positionals preceding the next option - next_option_string_index = min([ - index - for index in option_string_indices - if index >= start_index]) - if start_index != next_option_string_index: - positionals_end_index = consume_positionals(start_index) - - # only try to parse the next optional if we didn't consume - # the option string during the positionals parsing - if positionals_end_index > start_index: - start_index = positionals_end_index - continue - else: - start_index = positionals_end_index - - # if we consumed all the positionals we could and we're not - # at the index of an option string, there were extra arguments - if start_index not in option_string_indices: - strings = arg_strings[start_index:next_option_string_index] - extras.extend(strings) - start_index = next_option_string_index - - # consume the next optional and any arguments for it - start_index = consume_optional(start_index) - - # consume any positionals following the last Optional - stop_index = consume_positionals(start_index) - - # if we didn't consume all the argument strings, there were extras - extras.extend(arg_strings[stop_index:]) - - # if we didn't use all the Positional objects, there were too few - # arg strings supplied. - if positionals: - self.error(_('too few arguments')) - - # make sure all required actions were present - for action in self._actions: - if action.required: - if action not in seen_actions: - name = _get_action_name(action) - self.error(_('argument %s is required') % name) - - # make sure all required groups had one option present - for group in self._mutually_exclusive_groups: - if group.required: - for action in group._group_actions: - if action in seen_non_default_actions: - break - - # if no actions were used, report the error - else: - names = [_get_action_name(action) - for action in group._group_actions - if action.help is not SUPPRESS] - msg = _('one of the arguments %s is required') - self.error(msg % ' '.join(names)) - - # return the updated namespace and the extra arguments - return namespace, extras - - def _read_args_from_files(self, arg_strings): - # expand arguments referencing files - new_arg_strings = [] - for arg_string in arg_strings: - - # for regular arguments, just add them back into the list - if arg_string[0] not in self.fromfile_prefix_chars: - new_arg_strings.append(arg_string) - - # replace arguments referencing files with the file content - else: - try: - args_file = open(arg_string[1:]) - try: - arg_strings = [] - for arg_line in args_file.read().splitlines(): - for arg in self.convert_arg_line_to_args(arg_line): - arg_strings.append(arg) - arg_strings = self._read_args_from_files(arg_strings) - new_arg_strings.extend(arg_strings) - finally: - args_file.close() - except IOError: - err = _sys.exc_info()[1] - self.error(str(err)) - - # return the modified argument list - return new_arg_strings - - def convert_arg_line_to_args(self, arg_line): - return [arg_line] - - def _match_argument(self, action, arg_strings_pattern): - # match the pattern for this action to the arg strings - nargs_pattern = self._get_nargs_pattern(action) - match = _re.match(nargs_pattern, arg_strings_pattern) - - # raise an exception if we weren't able to find a match - if match is None: - nargs_errors = { - None: _('expected one argument'), - OPTIONAL: _('expected at most one argument'), - ONE_OR_MORE: _('expected at least one argument'), - } - default = _('expected %s argument(s)') % action.nargs - msg = nargs_errors.get(action.nargs, default) - raise ArgumentError(action, msg) - - # return the number of arguments matched - return len(match.group(1)) - - def _match_arguments_partial(self, actions, arg_strings_pattern): - # progressively shorten the actions list by slicing off the - # final actions until we find a match - result = [] - for i in range(len(actions), 0, -1): - actions_slice = actions[:i] - pattern = ''.join([self._get_nargs_pattern(action) - for action in actions_slice]) - match = _re.match(pattern, arg_strings_pattern) - if match is not None: - result.extend([len(string) for string in match.groups()]) - break - - # return the list of arg string counts - return result - - def _parse_optional(self, arg_string): - # if it's an empty string, it was meant to be a positional - if not arg_string: - return None - - # if it doesn't start with a prefix, it was meant to be positional - if not arg_string[0] in self.prefix_chars: - return None - - # if the option string is present in the parser, return the action - if arg_string in self._option_string_actions: - action = self._option_string_actions[arg_string] - return action, arg_string, None - - # if it's just a single character, it was meant to be positional - if len(arg_string) == 1: - return None - - # if the option string before the "=" is present, return the action - if '=' in arg_string: - option_string, explicit_arg = arg_string.split('=', 1) - if option_string in self._option_string_actions: - action = self._option_string_actions[option_string] - return action, option_string, explicit_arg - - # search through all possible prefixes of the option string - # and all actions in the parser for possible interpretations - option_tuples = self._get_option_tuples(arg_string) - - # if multiple actions match, the option string was ambiguous - if len(option_tuples) > 1: - options = ', '.join([option_string - for action, option_string, explicit_arg in option_tuples]) - tup = arg_string, options - self.error(_('ambiguous option: %s could match %s') % tup) - - # if exactly one action matched, this segmentation is good, - # so return the parsed action - elif len(option_tuples) == 1: - option_tuple, = option_tuples - return option_tuple - - # if it was not found as an option, but it looks like a negative - # number, it was meant to be positional - # unless there are negative-number-like options - if self._negative_number_matcher.match(arg_string): - if not self._has_negative_number_optionals: - return None - - # if it contains a space, it was meant to be a positional - if ' ' in arg_string: - return None - - # it was meant to be an optional but there is no such option - # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None - - def _get_option_tuples(self, option_string): - result = [] - - # option strings starting with two prefix characters are only - # split at the '=' - chars = self.prefix_chars - if option_string[0] in chars and option_string[1] in chars: - if '=' in option_string: - option_prefix, explicit_arg = option_string.split('=', 1) - else: - option_prefix = option_string - explicit_arg = None - for option_string in self._option_string_actions: - if option_string.startswith(option_prefix): - action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg - result.append(tup) - - # single character options can be concatenated with their arguments - # but multiple character options always have to have their argument - # separate - elif option_string[0] in chars and option_string[1] not in chars: - option_prefix = option_string - explicit_arg = None - short_option_prefix = option_string[:2] - short_explicit_arg = option_string[2:] - - for option_string in self._option_string_actions: - if option_string == short_option_prefix: - action = self._option_string_actions[option_string] - tup = action, option_string, short_explicit_arg - result.append(tup) - elif option_string.startswith(option_prefix): - action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg - result.append(tup) - - # shouldn't ever get here - else: - self.error(_('unexpected option string: %s') % option_string) - - # return the collected option tuples - return result - - def _get_nargs_pattern(self, action): - # in all examples below, we have to allow for '--' args - # which are represented as '-' in the pattern - nargs = action.nargs - - # the default (None) is assumed to be a single argument - if nargs is None: - nargs_pattern = '(-*A-*)' - - # allow zero or one arguments - elif nargs == OPTIONAL: - nargs_pattern = '(-*A?-*)' - - # allow zero or more arguments - elif nargs == ZERO_OR_MORE: - nargs_pattern = '(-*[A-]*)' - - # allow one or more arguments - elif nargs == ONE_OR_MORE: - nargs_pattern = '(-*A[A-]*)' - - # allow any number of options or arguments - elif nargs == REMAINDER: - nargs_pattern = '([-AO]*)' - - # allow one argument followed by any number of options or arguments - elif nargs == PARSER: - nargs_pattern = '(-*A[-AO]*)' - - # all others should be integers - else: - nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) - - # if this is an optional action, -- is not allowed - if action.option_strings: - nargs_pattern = nargs_pattern.replace('-*', '') - nargs_pattern = nargs_pattern.replace('-', '') - - # return the pattern - return nargs_pattern - - # ======================== - # Value conversion methods - # ======================== - def _get_values(self, action, arg_strings): - # for everything but PARSER args, strip out '--' - if action.nargs not in [PARSER, REMAINDER]: - arg_strings = [s for s in arg_strings if s != '--'] - - # optional argument produces a default when not present - if not arg_strings and action.nargs == OPTIONAL: - if action.option_strings: - value = action.const - else: - value = action.default - if isinstance(value, basestring): - value = self._get_value(action, value) - self._check_value(action, value) - - # when nargs='*' on a positional, if there were no command-line - # args, use the default if it is anything other than None - elif (not arg_strings and action.nargs == ZERO_OR_MORE and - not action.option_strings): - if action.default is not None: - value = action.default - else: - value = arg_strings - self._check_value(action, value) - - # single argument or optional argument produces a single value - elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: - arg_string, = arg_strings - value = self._get_value(action, arg_string) - self._check_value(action, value) - - # REMAINDER arguments convert all values, checking none - elif action.nargs == REMAINDER: - value = [self._get_value(action, v) for v in arg_strings] - - # PARSER arguments convert all values, but check only the first - elif action.nargs == PARSER: - value = [self._get_value(action, v) for v in arg_strings] - self._check_value(action, value[0]) - - # all other types of nargs produce a list - else: - value = [self._get_value(action, v) for v in arg_strings] - for v in value: - self._check_value(action, v) - - # return the converted value - return value - - def _get_value(self, action, arg_string): - type_func = self._registry_get('type', action.type, action.type) - if not _callable(type_func): - msg = _('%r is not callable') - raise ArgumentError(action, msg % type_func) - - # convert the value to the appropriate type - try: - result = type_func(arg_string) - - # ArgumentTypeErrors indicate errors - except ArgumentTypeError: - name = getattr(action.type, '__name__', repr(action.type)) - msg = str(_sys.exc_info()[1]) - raise ArgumentError(action, msg) - - # TypeErrors or ValueErrors also indicate errors - except (TypeError, ValueError): - name = getattr(action.type, '__name__', repr(action.type)) - msg = _('invalid %s value: %r') - raise ArgumentError(action, msg % (name, arg_string)) - - # return the converted value - return result - - def _check_value(self, action, value): - # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - tup = value, ', '.join(map(repr, action.choices)) - msg = _('invalid choice: %r (choose from %s)') % tup - raise ArgumentError(action, msg) - - # ======================= - # Help-formatting methods - # ======================= - def format_usage(self): - formatter = self._get_formatter() - formatter.add_usage(self.usage, self._actions, - self._mutually_exclusive_groups) - return formatter.format_help() - - def format_help(self): - formatter = self._get_formatter() - - # usage - formatter.add_usage(self.usage, self._actions, - self._mutually_exclusive_groups) - - # description - formatter.add_text(self.description) - - # positionals, optionals and user-defined groups - for action_group in self._action_groups: - formatter.start_section(action_group.title) - formatter.add_text(action_group.description) - formatter.add_arguments(action_group._group_actions) - formatter.end_section() - - # epilog - formatter.add_text(self.epilog) - - # determine help from format above - return formatter.format_help() - - def format_version(self): - import warnings - warnings.warn( - 'The format_version method is deprecated -- the "version" ' - 'argument to ArgumentParser is no longer supported.', - DeprecationWarning) - formatter = self._get_formatter() - formatter.add_text(self.version) - return formatter.format_help() - - def _get_formatter(self): - return self.formatter_class(prog=self.prog) - - # ===================== - # Help-printing methods - # ===================== - def print_usage(self, file=None): - if file is None: - file = _sys.stdout - self._print_message(self.format_usage(), file) - - def print_help(self, file=None): - if file is None: - file = _sys.stdout - self._print_message(self.format_help(), file) - - def print_version(self, file=None): - import warnings - warnings.warn( - 'The print_version method is deprecated -- the "version" ' - 'argument to ArgumentParser is no longer supported.', - DeprecationWarning) - self._print_message(self.format_version(), file) - - def _print_message(self, message, file=None): - if message: - if file is None: - file = _sys.stderr - file.write(message) - - # =============== - # Exiting methods - # =============== - def exit(self, status=0, message=None): - if message: - self._print_message(message, _sys.stderr) - _sys.exit(status) - - def error(self, message): - """error(message: string) - - Prints a usage message incorporating the message to stderr and - exits. - - If you override this in a subclass, it should not return -- it - should either exit or raise an exception. - """ - self.print_usage(_sys.stderr) - self.exit(2, _('%s: error: %s\n') % (self.prog, message)) diff --git a/gunicorn/config.py b/gunicorn/config.py index aa978949..22c7f485 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -5,27 +5,21 @@ # Please remember to run "make -C docs html" after update "desc" attributes. +import argparse import copy import grp import inspect -try: - import argparse -except ImportError: # python 2.6 - from . import argparse_compat as argparse import os import pwd import re +import shlex import ssl import sys import textwrap -import shlex -from gunicorn import __version__ -from gunicorn import _compat +from gunicorn import __version__, util from gunicorn.errors import ConfigError from gunicorn.reloader import reloader_engines -from gunicorn import six -from gunicorn import util KNOWN_SETTINGS = [] PLATFORM = sys.platform @@ -122,7 +116,7 @@ class Config(object): @property def address(self): s = self.settings['bind'].get() - return [util.parse_address(_compat.bytes_to_str(bind)) for bind in s] + return [util.parse_address(util.bytes_to_str(bind)) for bind in s] @property def uid(self): @@ -183,7 +177,7 @@ class Config(object): return env for e in raw_env: - s = _compat.bytes_to_str(e) + s = util.bytes_to_str(e) try: k, v = s.split('=', 1) except ValueError: @@ -216,7 +210,7 @@ class Config(object): global_conf = {} for e in raw_global_conf: - s = _compat.bytes_to_str(e) + s = util.bytes_to_str(e) try: k, v = re.split(r'(?= (2, 7): - class Ciphers(Setting): - name = "ciphers" - section = "SSL" - cli = ["--ciphers"] - validator = validate_string - default = 'TLSv1' - desc = """\ - Ciphers to use (see stdlib ssl module's) - """ +class Ciphers(Setting): + name = "ciphers" + section = "SSL" + cli = ["--ciphers"] + validator = validate_string + default = 'TLSv1' + desc = """\ + Ciphers to use (see stdlib ssl module's) + """ class PasteGlobalConf(Setting): diff --git a/gunicorn/glogging.py b/gunicorn/glogging.py index 041a74d5..3f266977 100644 --- a/gunicorn/glogging.py +++ b/gunicorn/glogging.py @@ -8,12 +8,8 @@ import binascii import time import logging logging.Logger.manager.emittedNoHandlerWarning = 1 +from logging.config import dictConfig from logging.config import fileConfig -try: - from logging.config import dictConfig -except ImportError: - # python 2.6 - dictConfig = None import os import socket import sys @@ -21,7 +17,6 @@ import threading import traceback from gunicorn import util -from gunicorn.six import PY3, string_types # syslog facility codes @@ -104,7 +99,7 @@ class SafeAtoms(dict): def __init__(self, atoms): dict.__init__(self) for key, value in atoms.items(): - if isinstance(value, string_types): + if isinstance(value, str): self[key] = value.replace('"', '\\"') else: self[key] = value @@ -231,11 +226,7 @@ class Logger(object): self.access_log, cfg, self.syslog_fmt, "access" ) - if dictConfig is None and cfg.logconfig_dict: - util.warn("Dictionary-based log configuration requires " - "Python 2.7 or above.") - - if dictConfig and cfg.logconfig_dict: + if cfg.logconfig_dict: config = CONFIG_DEFAULTS.copy() config.update(cfg.logconfig_dict) try: @@ -277,7 +268,7 @@ class Logger(object): self.error_log.exception(msg, *args, **kwargs) def log(self, lvl, msg, *args, **kwargs): - if isinstance(lvl, string_types): + if isinstance(lvl, str): lvl = self.LOG_LEVELS.get(lvl.lower(), logging.INFO) self.error_log.log(lvl, msg, *args, **kwargs) @@ -318,18 +309,18 @@ class Logger(object): if hasattr(req_headers, "items"): req_headers = req_headers.items() - atoms.update(dict([("{%s}i" % k.lower(), v) for k, v in req_headers])) + atoms.update({"{%s}i" % k.lower(): v for k, v in req_headers}) resp_headers = resp.headers if hasattr(resp_headers, "items"): resp_headers = resp_headers.items() # add response headers - atoms.update(dict([("{%s}o" % k.lower(), v) for k, v in resp_headers])) + atoms.update({"{%s}o" % k.lower(): v for k, v in resp_headers}) # add environ variables environ_variables = environ.items() - atoms.update(dict([("{%s}e" % k.lower(), v) for k, v in environ_variables])) + atoms.update({"{%s}e" % k.lower(): v for k, v in environ_variables}) return atoms @@ -444,14 +435,8 @@ class Logger(object): socktype, addr = parse_syslog_address(cfg.syslog_addr) # finally setup the syslog handler - if sys.version_info >= (2, 7): - h = logging.handlers.SysLogHandler(address=addr, - facility=facility, socktype=socktype) - else: - # socktype is only supported in 2.7 and sup - # fix issue #541 - h = logging.handlers.SysLogHandler(address=addr, - facility=facility) + h = logging.handlers.SysLogHandler(address=addr, + facility=facility, socktype=socktype) h.setFormatter(fmt) h._gunicorn = True @@ -467,8 +452,8 @@ class Logger(object): # b64decode doesn't accept unicode in Python < 3.3 # so we need to convert it to a byte string auth = base64.b64decode(auth[1].strip().encode('utf-8')) - if PY3: # b64decode returns a byte string in Python 3 - auth = auth.decode('utf-8') + # b64decode returns a byte string + auth = auth.decode('utf-8') auth = auth.split(":", 1) except (TypeError, binascii.Error, UnicodeDecodeError) as exc: self.debug("Couldn't get username: %s", exc) diff --git a/gunicorn/http/_sendfile.py b/gunicorn/http/_sendfile.py deleted file mode 100644 index 1764cb32..00000000 --- a/gunicorn/http/_sendfile.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 - -# -# This file is part of gunicorn released under the MIT license. -# See the NOTICE for more information. - -import errno -import os -import sys - -try: - import ctypes - import ctypes.util -except MemoryError: - # selinux execmem denial - # https://bugzilla.redhat.com/show_bug.cgi?id=488396 - raise ImportError - -SUPPORTED_PLATFORMS = ( - 'darwin', - 'freebsd', - 'dragonfly', - 'linux2') - -if sys.platform not in SUPPORTED_PLATFORMS: - raise ImportError("sendfile isn't supported on this platform") - -_libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) -_sendfile = _libc.sendfile - - -def sendfile(fdout, fdin, offset, nbytes): - if sys.platform == 'darwin': - _sendfile.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64, - ctypes.POINTER(ctypes.c_uint64), ctypes.c_voidp, - ctypes.c_int] - _nbytes = ctypes.c_uint64(nbytes) - result = _sendfile(fdin, fdout, offset, _nbytes, None, 0) - - if result == -1: - e = ctypes.get_errno() - if e == errno.EAGAIN and _nbytes.value is not None: - return _nbytes.value - raise OSError(e, os.strerror(e)) - return _nbytes.value - elif sys.platform in ('freebsd', 'dragonfly',): - _sendfile.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64, - ctypes.c_uint64, ctypes.c_voidp, - ctypes.POINTER(ctypes.c_uint64), ctypes.c_int] - _sbytes = ctypes.c_uint64() - result = _sendfile(fdin, fdout, offset, nbytes, None, _sbytes, 0) - if result == -1: - e = ctypes.get_errno() - if e == errno.EAGAIN and _sbytes.value is not None: - return _sbytes.value - raise OSError(e, os.strerror(e)) - return _sbytes.value - - else: - _sendfile.argtypes = [ctypes.c_int, ctypes.c_int, - ctypes.POINTER(ctypes.c_uint64), ctypes.c_size_t] - - _offset = ctypes.c_uint64(offset) - sent = _sendfile(fdout, fdin, _offset, nbytes) - if sent == -1: - e = ctypes.get_errno() - raise OSError(e, os.strerror(e)) - return sent diff --git a/gunicorn/http/body.py b/gunicorn/http/body.py index fb8633ed..e75d72de 100644 --- a/gunicorn/http/body.py +++ b/gunicorn/http/body.py @@ -3,19 +3,21 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. +import io +import sys + from gunicorn.http.errors import (NoMoreData, ChunkMissingTerminator, InvalidChunkSize) -from gunicorn import six class ChunkedReader(object): def __init__(self, req, unreader): self.req = req self.parser = self.parse_chunked(unreader) - self.buf = six.BytesIO() + self.buf = io.BytesIO() def read(self, size): - if not isinstance(size, six.integer_types): + if not isinstance(size, int): raise TypeError("size must be an integral type") if size < 0: raise ValueError("Size must be positive.") @@ -25,19 +27,19 @@ class ChunkedReader(object): if self.parser: while self.buf.tell() < size: try: - self.buf.write(six.next(self.parser)) + self.buf.write(next(self.parser)) except StopIteration: self.parser = None break data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf = six.BytesIO() + self.buf = io.BytesIO() self.buf.write(rest) return ret def parse_trailers(self, unreader, data): - buf = six.BytesIO() + buf = io.BytesIO() buf.write(data) idx = buf.getvalue().find(b"\r\n\r\n") @@ -71,7 +73,7 @@ class ChunkedReader(object): (size, rest) = self.parse_chunk_size(unreader, data=rest[2:]) def parse_chunk_size(self, unreader, data=None): - buf = six.BytesIO() + buf = io.BytesIO() if data is not None: buf.write(data) @@ -110,7 +112,7 @@ class LengthReader(object): self.length = length def read(self, size): - if not isinstance(size, six.integer_types): + if not isinstance(size, int): raise TypeError("size must be an integral type") size = min(self.length, size) @@ -119,7 +121,7 @@ class LengthReader(object): if size == 0: return b"" - buf = six.BytesIO() + buf = io.BytesIO() data = self.unreader.read() while data: buf.write(data) @@ -137,11 +139,11 @@ class LengthReader(object): class EOFReader(object): def __init__(self, unreader): self.unreader = unreader - self.buf = six.BytesIO() + self.buf = io.BytesIO() self.finished = False def read(self, size): - if not isinstance(size, six.integer_types): + if not isinstance(size, int): raise TypeError("size must be an integral type") if size < 0: raise ValueError("Size must be positive.") @@ -151,7 +153,7 @@ class EOFReader(object): if self.finished: data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf = six.BytesIO() + self.buf = io.BytesIO() self.buf.write(rest) return ret @@ -167,7 +169,7 @@ class EOFReader(object): data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf = six.BytesIO() + self.buf = io.BytesIO() self.buf.write(rest) return ret @@ -175,7 +177,7 @@ class EOFReader(object): class Body(object): def __init__(self, reader): self.reader = reader - self.buf = six.BytesIO() + self.buf = io.BytesIO() def __iter__(self): return self @@ -189,11 +191,11 @@ class Body(object): def getsize(self, size): if size is None: - return six.MAXSIZE - elif not isinstance(size, six.integer_types): + return sys.maxsize + elif not isinstance(size, int): raise TypeError("size must be an integral type") elif size < 0: - return six.MAXSIZE + return sys.maxsize return size def read(self, size=None): @@ -204,7 +206,7 @@ class Body(object): if size < self.buf.tell(): data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf = six.BytesIO() + self.buf = io.BytesIO() self.buf.write(rest) return ret @@ -216,7 +218,7 @@ class Body(object): data = self.buf.getvalue() ret, rest = data[:size], data[size:] - self.buf = six.BytesIO() + self.buf = io.BytesIO() self.buf.write(rest) return ret @@ -226,7 +228,7 @@ class Body(object): return b"" data = self.buf.getvalue() - self.buf = six.BytesIO() + self.buf = io.BytesIO() ret = [] while 1: diff --git a/gunicorn/http/message.py b/gunicorn/http/message.py index 2700b321..0c6bc053 100644 --- a/gunicorn/http/message.py +++ b/gunicorn/http/message.py @@ -3,11 +3,11 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. +import io import re import socket from errno import ENOTCONN -from gunicorn._compat import bytes_to_str from gunicorn.http.unreader import SocketUnreader from gunicorn.http.body import ChunkedReader, LengthReader, EOFReader, Body from gunicorn.http.errors import (InvalidHeader, InvalidHeaderName, NoMoreData, @@ -15,8 +15,7 @@ from gunicorn.http.errors import (InvalidHeader, InvalidHeaderName, NoMoreData, LimitRequestLine, LimitRequestHeaders) from gunicorn.http.errors import InvalidProxyLine, ForbiddenProxyRequest from gunicorn.http.errors import InvalidSchemeHeaders -from gunicorn.six import BytesIO, string_types -from gunicorn.util import split_request_uri +from gunicorn.util import bytes_to_str, split_request_uri MAX_REQUEST_LINE = 8190 MAX_HEADERS = 32768 @@ -76,7 +75,7 @@ class Message(object): remote_host = remote_addr[0] if remote_host in cfg.forwarded_allow_ips: secure_scheme_headers = cfg.secure_scheme_headers - elif isinstance(remote_addr, string_types): + elif isinstance(remote_addr, str): secure_scheme_headers = cfg.secure_scheme_headers # Parse headers into key/value pairs paying attention @@ -189,7 +188,7 @@ class Request(Message): buf.write(data) def parse(self, unreader): - buf = BytesIO() + buf = io.BytesIO() self.get_data(unreader, buf, stop=True) # get request line @@ -198,12 +197,12 @@ class Request(Message): # proxy protocol if self.proxy_protocol(bytes_to_str(line)): # get next request line - buf = BytesIO() + buf = io.BytesIO() buf.write(rbuf) line, rbuf = self.read_line(unreader, buf, self.limit_request_line) self.parse_request_line(line) - buf = BytesIO() + buf = io.BytesIO() buf.write(rbuf) # Headers diff --git a/gunicorn/http/unreader.py b/gunicorn/http/unreader.py index 9f312a80..be07e7ac 100644 --- a/gunicorn/http/unreader.py +++ b/gunicorn/http/unreader.py @@ -3,23 +3,22 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. +import io import os -from gunicorn import six - # Classes that can undo reading data from # a given type of data source. class Unreader(object): def __init__(self): - self.buf = six.BytesIO() + self.buf = io.BytesIO() def chunk(self): raise NotImplementedError() def read(self, size=None): - if size is not None and not isinstance(size, six.integer_types): + if size is not None and not isinstance(size, int): raise TypeError("size parameter must be an int or long.") if size is not None: @@ -32,7 +31,7 @@ class Unreader(object): if size is None and self.buf.tell(): ret = self.buf.getvalue() - self.buf = six.BytesIO() + self.buf = io.BytesIO() return ret if size is None: d = self.chunk() @@ -42,11 +41,11 @@ class Unreader(object): chunk = self.chunk() if not chunk: ret = self.buf.getvalue() - self.buf = six.BytesIO() + self.buf = io.BytesIO() return ret self.buf.write(chunk) data = self.buf.getvalue() - self.buf = six.BytesIO() + self.buf = io.BytesIO() self.buf.write(data[size:]) return data[:size] @@ -74,7 +73,7 @@ class IterUnreader(Unreader): if not self.iter: return b"" try: - return six.next(self.iter) + return next(self.iter) except StopIteration: self.iter = None return b"" diff --git a/gunicorn/http/wsgi.py b/gunicorn/http/wsgi.py index ff75974d..593c8f24 100644 --- a/gunicorn/http/wsgi.py +++ b/gunicorn/http/wsgi.py @@ -9,22 +9,11 @@ import os import re import sys -from gunicorn._compat import unquote_to_wsgi_str from gunicorn.http.message import HEADER_RE from gunicorn.http.errors import InvalidHeader, InvalidHeaderName -from gunicorn.six import string_types, binary_type, reraise from gunicorn import SERVER_SOFTWARE import gunicorn.util as util -try: - # Python 3.3 has os.sendfile(). - from os import sendfile -except ImportError: - try: - from ._sendfile import sendfile - except ImportError: - sendfile = None - # Send files in at most 1GB blocks as some operating systems can have problems # with sending files in blocks over 2GB. BLKSIZE = 0x3FFFFFFF @@ -155,9 +144,9 @@ def create(req, sock, client, server, cfg): # authors should be aware that REMOTE_HOST and REMOTE_ADDR # may not qualify the remote addr: # http://www.ietf.org/rfc/rfc3875 - if isinstance(client, string_types): + if isinstance(client, str): environ['REMOTE_ADDR'] = client - elif isinstance(client, binary_type): + elif isinstance(client, bytes): environ['REMOTE_ADDR'] = client.decode() else: environ['REMOTE_ADDR'] = client[0] @@ -167,7 +156,7 @@ def create(req, sock, client, server, cfg): # Normally only the application should use the Host header but since the # WSGI spec doesn't support unix sockets, we are using it to create # viable SERVER_* if possible. - if isinstance(server, string_types): + if isinstance(server, str): server = server.split(":") if len(server) == 1: # unix socket @@ -191,7 +180,7 @@ def create(req, sock, client, server, cfg): path_info = req.path if script_name: path_info = path_info.split(script_name, 1)[1] - environ['PATH_INFO'] = unquote_to_wsgi_str(path_info) + environ['PATH_INFO'] = util.unquote_to_wsgi_str(path_info) environ['SCRIPT_NAME'] = script_name # override the environ with the correct remote and server address if @@ -234,7 +223,7 @@ class Response(object): if exc_info: try: if self.status and self.headers_sent: - reraise(exc_info[0], exc_info[1], exc_info[2]) + util.reraise(exc_info[0], exc_info[1], exc_info[2]) finally: exc_info = None elif self.status is not None: @@ -256,7 +245,7 @@ class Response(object): def process_headers(self, headers): for name, value in headers: - if not isinstance(name, string_types): + if not isinstance(name, str): raise TypeError('%r is not a string' % name) if HEADER_RE.search(name): @@ -331,7 +320,7 @@ class Response(object): def write(self, arg): self.send_headers() - if not isinstance(arg, binary_type): + if not isinstance(arg, bytes): raise TypeError('%r is not a byte' % arg) arglen = len(arg) tosend = arglen @@ -353,7 +342,7 @@ class Response(object): util.write(self.sock, arg, self.chunked) def can_sendfile(self): - return self.cfg.sendfile is not False and sendfile is not None + return self.cfg.sendfile is not False def sendfile(self, respiter): if self.cfg.is_ssl or not self.can_sendfile(): @@ -390,7 +379,7 @@ class Response(object): while sent != nbytes: count = min(nbytes - sent, BLKSIZE) - sent += sendfile(sockno, fileno, offset + sent, count) + sent += os.sendfile(sockno, fileno, offset + sent, count) if self.is_chunked(): self.sock.sendall(b"\r\n") diff --git a/gunicorn/instrument/statsd.py b/gunicorn/instrument/statsd.py index 4bbcb20a..12b6de4e 100644 --- a/gunicorn/instrument/statsd.py +++ b/gunicorn/instrument/statsd.py @@ -5,12 +5,11 @@ "Bare-bones implementation of statsD's protocol, client-side" -import socket import logging +import socket from re import sub from gunicorn.glogging import Logger -from gunicorn import six # Instrumentation constants METRIC_VAR = "metric" @@ -115,7 +114,7 @@ class Statsd(Logger): def _sock_send(self, msg): try: - if isinstance(msg, six.text_type): + if isinstance(msg, str): msg = msg.encode("ascii") if self.sock: self.sock.send(msg) diff --git a/gunicorn/six.py b/gunicorn/six.py deleted file mode 100644 index 21b0e803..00000000 --- a/gunicorn/six.py +++ /dev/null @@ -1,762 +0,0 @@ -"""Utilities for writing code that runs on Python 2 and 3""" - -# Copyright (c) 2010-2014 Benjamin Peterson -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from __future__ import absolute_import - -import functools -import operator -import sys -import types - -__author__ = "Benjamin Peterson " -__version__ = "1.8.0" - - -# Useful for very coarse version differentiation. -PY2 = sys.version_info[0] == 2 -PY3 = sys.version_info[0] == 3 - -if PY3: - string_types = str, - integer_types = int, - class_types = type, - text_type = str - binary_type = bytes - - MAXSIZE = sys.maxsize -else: - string_types = basestring, - integer_types = (int, long) - class_types = (type, types.ClassType) - text_type = unicode - binary_type = str - - if sys.platform.startswith("java"): - # Jython always uses 32 bits. - MAXSIZE = int((1 << 31) - 1) - else: - # It's possible to have sizeof(long) != sizeof(Py_ssize_t). - class X(object): - def __len__(self): - return 1 << 31 - try: - len(X()) - except OverflowError: - # 32-bit - MAXSIZE = int((1 << 31) - 1) - else: - # 64-bit - MAXSIZE = int((1 << 63) - 1) - del X - - -def _add_doc(func, doc): - """Add documentation to a function.""" - func.__doc__ = doc - - -def _import_module(name): - """Import module, returning the module after the last dot.""" - __import__(name) - return sys.modules[name] - - -class _LazyDescr(object): - - def __init__(self, name): - self.name = name - - def __get__(self, obj, tp): - result = self._resolve() - setattr(obj, self.name, result) # Invokes __set__. - # This is a bit ugly, but it avoids running this again. - delattr(obj.__class__, self.name) - return result - - -class MovedModule(_LazyDescr): - - def __init__(self, name, old, new=None): - super(MovedModule, self).__init__(name) - if PY3: - if new is None: - new = name - self.mod = new - else: - self.mod = old - - def _resolve(self): - return _import_module(self.mod) - - def __getattr__(self, attr): - _module = self._resolve() - value = getattr(_module, attr) - setattr(self, attr, value) - return value - - -class _LazyModule(types.ModuleType): - - def __init__(self, name): - super(_LazyModule, self).__init__(name) - self.__doc__ = self.__class__.__doc__ - - def __dir__(self): - attrs = ["__doc__", "__name__"] - attrs += [attr.name for attr in self._moved_attributes] - return attrs - - # Subclasses should override this - _moved_attributes = [] - - -class MovedAttribute(_LazyDescr): - - def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): - super(MovedAttribute, self).__init__(name) - if PY3: - if new_mod is None: - new_mod = name - self.mod = new_mod - if new_attr is None: - if old_attr is None: - new_attr = name - else: - new_attr = old_attr - self.attr = new_attr - else: - self.mod = old_mod - if old_attr is None: - old_attr = name - self.attr = old_attr - - def _resolve(self): - module = _import_module(self.mod) - return getattr(module, self.attr) - - -class _SixMetaPathImporter(object): - """ - A meta path importer to import six.moves and its submodules. - - This class implements a PEP302 finder and loader. It should be compatible - with Python 2.5 and all existing versions of Python3 - """ - def __init__(self, six_module_name): - self.name = six_module_name - self.known_modules = {} - - def _add_module(self, mod, *fullnames): - for fullname in fullnames: - self.known_modules[self.name + "." + fullname] = mod - - def _get_module(self, fullname): - return self.known_modules[self.name + "." + fullname] - - def find_module(self, fullname, path=None): - if fullname in self.known_modules: - return self - return None - - def __get_module(self, fullname): - try: - return self.known_modules[fullname] - except KeyError: - raise ImportError("This loader does not know module " + fullname) - - def load_module(self, fullname): - try: - # in case of a reload - return sys.modules[fullname] - except KeyError: - pass - mod = self.__get_module(fullname) - if isinstance(mod, MovedModule): - mod = mod._resolve() - else: - mod.__loader__ = self - sys.modules[fullname] = mod - return mod - - def is_package(self, fullname): - """ - Return true, if the named module is a package. - - We need this method to get correct spec objects with - Python 3.4 (see PEP451) - """ - return hasattr(self.__get_module(fullname), "__path__") - - def get_code(self, fullname): - """Return None - - Required, if is_package is implemented""" - self.__get_module(fullname) # eventually raises ImportError - return None - get_source = get_code # same as get_code - -_importer = _SixMetaPathImporter(__name__) - - -class _MovedItems(_LazyModule): - """Lazy loading of moved objects""" - __path__ = [] # mark as package - - -_moved_attributes = [ - MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), - MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), - MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), - MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), - MovedAttribute("intern", "__builtin__", "sys"), - MovedAttribute("map", "itertools", "builtins", "imap", "map"), - MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("reload_module", "__builtin__", "imp", "reload"), - MovedAttribute("reduce", "__builtin__", "functools"), - MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), - MovedAttribute("StringIO", "StringIO", "io"), - MovedAttribute("UserDict", "UserDict", "collections"), - MovedAttribute("UserList", "UserList", "collections"), - MovedAttribute("UserString", "UserString", "collections"), - MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), - - MovedModule("builtins", "__builtin__"), - MovedModule("configparser", "ConfigParser"), - MovedModule("copyreg", "copy_reg"), - MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), - MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), - MovedModule("http_cookies", "Cookie", "http.cookies"), - MovedModule("html_entities", "htmlentitydefs", "html.entities"), - MovedModule("html_parser", "HTMLParser", "html.parser"), - MovedModule("http_client", "httplib", "http.client"), - MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), - MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), - MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), - MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), - MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), - MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), - MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), - MovedModule("cPickle", "cPickle", "pickle"), - MovedModule("queue", "Queue"), - MovedModule("reprlib", "repr"), - MovedModule("socketserver", "SocketServer"), - MovedModule("_thread", "thread", "_thread"), - MovedModule("tkinter", "Tkinter"), - MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), - MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), - MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), - MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), - MovedModule("tkinter_tix", "Tix", "tkinter.tix"), - MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), - MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), - MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), - MovedModule("tkinter_colorchooser", "tkColorChooser", - "tkinter.colorchooser"), - MovedModule("tkinter_commondialog", "tkCommonDialog", - "tkinter.commondialog"), - MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), - MovedModule("tkinter_font", "tkFont", "tkinter.font"), - MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), - MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", - "tkinter.simpledialog"), - MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), - MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), - MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), - MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), - MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), - MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), - MovedModule("winreg", "_winreg"), -] -for attr in _moved_attributes: - setattr(_MovedItems, attr.name, attr) - if isinstance(attr, MovedModule): - _importer._add_module(attr, "moves." + attr.name) -del attr - -_MovedItems._moved_attributes = _moved_attributes - -moves = _MovedItems(__name__ + ".moves") -_importer._add_module(moves, "moves") - - -class Module_six_moves_urllib_parse(_LazyModule): - """Lazy loading of moved objects in six.moves.urllib_parse""" - - -_urllib_parse_moved_attributes = [ - MovedAttribute("ParseResult", "urlparse", "urllib.parse"), - MovedAttribute("SplitResult", "urlparse", "urllib.parse"), - MovedAttribute("parse_qs", "urlparse", "urllib.parse"), - MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), - MovedAttribute("urldefrag", "urlparse", "urllib.parse"), - MovedAttribute("urljoin", "urlparse", "urllib.parse"), - MovedAttribute("urlparse", "urlparse", "urllib.parse"), - MovedAttribute("urlsplit", "urlparse", "urllib.parse"), - MovedAttribute("urlunparse", "urlparse", "urllib.parse"), - MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), - MovedAttribute("quote", "urllib", "urllib.parse"), - MovedAttribute("quote_plus", "urllib", "urllib.parse"), - MovedAttribute("unquote", "urllib", "urllib.parse"), - MovedAttribute("unquote_plus", "urllib", "urllib.parse"), - MovedAttribute("urlencode", "urllib", "urllib.parse"), - MovedAttribute("splitquery", "urllib", "urllib.parse"), - MovedAttribute("splittag", "urllib", "urllib.parse"), - MovedAttribute("splituser", "urllib", "urllib.parse"), - MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), - MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), - MovedAttribute("uses_params", "urlparse", "urllib.parse"), - MovedAttribute("uses_query", "urlparse", "urllib.parse"), - MovedAttribute("uses_relative", "urlparse", "urllib.parse"), -] -for attr in _urllib_parse_moved_attributes: - setattr(Module_six_moves_urllib_parse, attr.name, attr) -del attr - -Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes - -_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), - "moves.urllib_parse", "moves.urllib.parse") - - -class Module_six_moves_urllib_error(_LazyModule): - """Lazy loading of moved objects in six.moves.urllib_error""" - - -_urllib_error_moved_attributes = [ - MovedAttribute("URLError", "urllib2", "urllib.error"), - MovedAttribute("HTTPError", "urllib2", "urllib.error"), - MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), -] -for attr in _urllib_error_moved_attributes: - setattr(Module_six_moves_urllib_error, attr.name, attr) -del attr - -Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes - -_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), - "moves.urllib_error", "moves.urllib.error") - - -class Module_six_moves_urllib_request(_LazyModule): - """Lazy loading of moved objects in six.moves.urllib_request""" - - -_urllib_request_moved_attributes = [ - MovedAttribute("urlopen", "urllib2", "urllib.request"), - MovedAttribute("install_opener", "urllib2", "urllib.request"), - MovedAttribute("build_opener", "urllib2", "urllib.request"), - MovedAttribute("pathname2url", "urllib", "urllib.request"), - MovedAttribute("url2pathname", "urllib", "urllib.request"), - MovedAttribute("getproxies", "urllib", "urllib.request"), - MovedAttribute("Request", "urllib2", "urllib.request"), - MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), - MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), - MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), - MovedAttribute("BaseHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), - MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), - MovedAttribute("FileHandler", "urllib2", "urllib.request"), - MovedAttribute("FTPHandler", "urllib2", "urllib.request"), - MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), - MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), - MovedAttribute("urlretrieve", "urllib", "urllib.request"), - MovedAttribute("urlcleanup", "urllib", "urllib.request"), - MovedAttribute("URLopener", "urllib", "urllib.request"), - MovedAttribute("FancyURLopener", "urllib", "urllib.request"), - MovedAttribute("proxy_bypass", "urllib", "urllib.request"), -] -for attr in _urllib_request_moved_attributes: - setattr(Module_six_moves_urllib_request, attr.name, attr) -del attr - -Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes - -_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), - "moves.urllib_request", "moves.urllib.request") - - -class Module_six_moves_urllib_response(_LazyModule): - """Lazy loading of moved objects in six.moves.urllib_response""" - - -_urllib_response_moved_attributes = [ - MovedAttribute("addbase", "urllib", "urllib.response"), - MovedAttribute("addclosehook", "urllib", "urllib.response"), - MovedAttribute("addinfo", "urllib", "urllib.response"), - MovedAttribute("addinfourl", "urllib", "urllib.response"), -] -for attr in _urllib_response_moved_attributes: - setattr(Module_six_moves_urllib_response, attr.name, attr) -del attr - -Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes - -_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), - "moves.urllib_response", "moves.urllib.response") - - -class Module_six_moves_urllib_robotparser(_LazyModule): - """Lazy loading of moved objects in six.moves.urllib_robotparser""" - - -_urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), -] -for attr in _urllib_robotparser_moved_attributes: - setattr(Module_six_moves_urllib_robotparser, attr.name, attr) -del attr - -Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes - -_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), - "moves.urllib_robotparser", "moves.urllib.robotparser") - - -class Module_six_moves_urllib(types.ModuleType): - """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" - __path__ = [] # mark as package - parse = _importer._get_module("moves.urllib_parse") - error = _importer._get_module("moves.urllib_error") - request = _importer._get_module("moves.urllib_request") - response = _importer._get_module("moves.urllib_response") - robotparser = _importer._get_module("moves.urllib_robotparser") - - def __dir__(self): - return ['parse', 'error', 'request', 'response', 'robotparser'] - -_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), - "moves.urllib") - - -def add_move(move): - """Add an item to six.moves.""" - setattr(_MovedItems, move.name, move) - - -def remove_move(name): - """Remove item from six.moves.""" - try: - delattr(_MovedItems, name) - except AttributeError: - try: - del moves.__dict__[name] - except KeyError: - raise AttributeError("no such move, %r" % (name,)) - - -if PY3: - _meth_func = "__func__" - _meth_self = "__self__" - - _func_closure = "__closure__" - _func_code = "__code__" - _func_defaults = "__defaults__" - _func_globals = "__globals__" -else: - _meth_func = "im_func" - _meth_self = "im_self" - - _func_closure = "func_closure" - _func_code = "func_code" - _func_defaults = "func_defaults" - _func_globals = "func_globals" - - -try: - advance_iterator = next -except NameError: - def advance_iterator(it): - return it.next() -next = advance_iterator - - -try: - callable = callable -except NameError: - def callable(obj): - return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) - - -if PY3: - def get_unbound_function(unbound): - return unbound - - create_bound_method = types.MethodType - - Iterator = object -else: - def get_unbound_function(unbound): - return unbound.im_func - - def create_bound_method(func, obj): - return types.MethodType(func, obj, obj.__class__) - - class Iterator(object): - - def next(self): - return type(self).__next__(self) - - callable = callable -_add_doc(get_unbound_function, - """Get the function out of a possibly unbound function""") - - -get_method_function = operator.attrgetter(_meth_func) -get_method_self = operator.attrgetter(_meth_self) -get_function_closure = operator.attrgetter(_func_closure) -get_function_code = operator.attrgetter(_func_code) -get_function_defaults = operator.attrgetter(_func_defaults) -get_function_globals = operator.attrgetter(_func_globals) - - -if PY3: - def iterkeys(d, **kw): - return iter(d.keys(**kw)) - - def itervalues(d, **kw): - return iter(d.values(**kw)) - - def iteritems(d, **kw): - return iter(d.items(**kw)) - - def iterlists(d, **kw): - return iter(d.lists(**kw)) -else: - def iterkeys(d, **kw): - return iter(d.iterkeys(**kw)) - - def itervalues(d, **kw): - return iter(d.itervalues(**kw)) - - def iteritems(d, **kw): - return iter(d.iteritems(**kw)) - - def iterlists(d, **kw): - return iter(d.iterlists(**kw)) - -_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") -_add_doc(itervalues, "Return an iterator over the values of a dictionary.") -_add_doc(iteritems, - "Return an iterator over the (key, value) pairs of a dictionary.") -_add_doc(iterlists, - "Return an iterator over the (key, [values]) pairs of a dictionary.") - - -if PY3: - def b(s): - return s.encode("latin-1") - def u(s): - return s - unichr = chr - if sys.version_info[1] <= 1: - def int2byte(i): - return bytes((i,)) - else: - # This is about 2x faster than the implementation above on 3.2+ - int2byte = operator.methodcaller("to_bytes", 1, "big") - byte2int = operator.itemgetter(0) - indexbytes = operator.getitem - iterbytes = iter - import io - StringIO = io.StringIO - BytesIO = io.BytesIO -else: - def b(s): - return s - # Workaround for standalone backslash - def u(s): - return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") - unichr = unichr - int2byte = chr - def byte2int(bs): - return ord(bs[0]) - def indexbytes(buf, i): - return ord(buf[i]) - def iterbytes(buf): - return (ord(byte) for byte in buf) - import StringIO - StringIO = BytesIO = StringIO.StringIO -_add_doc(b, """Byte literal""") -_add_doc(u, """Text literal""") - - -if PY3: - exec_ = getattr(moves.builtins, "exec") - - - def reraise(tp, value, tb=None): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value - -else: - def exec_(_code_, _globs_=None, _locs_=None): - """Execute code in a namespace.""" - if _globs_ is None: - frame = sys._getframe(1) - _globs_ = frame.f_globals - if _locs_ is None: - _locs_ = frame.f_locals - del frame - elif _locs_ is None: - _locs_ = _globs_ - exec("""exec _code_ in _globs_, _locs_""") - - - exec_("""def reraise(tp, value, tb=None): - raise tp, value, tb -""") - - -print_ = getattr(moves.builtins, "print", None) -if print_ is None: - def print_(*args, **kwargs): - """The new-style print function for Python 2.4 and 2.5.""" - fp = kwargs.pop("file", sys.stdout) - if fp is None: - return - def write(data): - if not isinstance(data, basestring): - data = str(data) - # If the file has an encoding, encode unicode with it. - if (isinstance(fp, file) and - isinstance(data, unicode) and - fp.encoding is not None): - errors = getattr(fp, "errors", None) - if errors is None: - errors = "strict" - data = data.encode(fp.encoding, errors) - fp.write(data) - want_unicode = False - sep = kwargs.pop("sep", None) - if sep is not None: - if isinstance(sep, unicode): - want_unicode = True - elif not isinstance(sep, str): - raise TypeError("sep must be None or a string") - end = kwargs.pop("end", None) - if end is not None: - if isinstance(end, unicode): - want_unicode = True - elif not isinstance(end, str): - raise TypeError("end must be None or a string") - if kwargs: - raise TypeError("invalid keyword arguments to print()") - if not want_unicode: - for arg in args: - if isinstance(arg, unicode): - want_unicode = True - break - if want_unicode: - newline = unicode("\n") - space = unicode(" ") - else: - newline = "\n" - space = " " - if sep is None: - sep = space - if end is None: - end = newline - for i, arg in enumerate(args): - if i: - write(sep) - write(arg) - write(end) - -_add_doc(reraise, """Reraise an exception.""") - -if sys.version_info[0:2] < (3, 4): - def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES): - def wrapper(f): - f = functools.wraps(wrapped)(f) - f.__wrapped__ = wrapped - return f - return wrapper -else: - wraps = functools.wraps - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass.""" - # This requires a bit of explanation: the basic idea is to make a dummy - # metaclass for one level of class instantiation that replaces itself with - # the actual metaclass. - class metaclass(meta): - def __new__(cls, name, this_bases, d): - return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) - - -def add_metaclass(metaclass): - """Class decorator for creating a class with a metaclass.""" - def wrapper(cls): - orig_vars = cls.__dict__.copy() - slots = orig_vars.get('__slots__') - if slots is not None: - if isinstance(slots, str): - slots = [slots] - for slots_var in slots: - orig_vars.pop(slots_var) - orig_vars.pop('__dict__', None) - orig_vars.pop('__weakref__', None) - return metaclass(cls.__name__, cls.__bases__, orig_vars) - return wrapper - -# Complete the moves implementation. -# This code is at the end of this module to speed up module loading. -# Turn this module into a package. -__path__ = [] # required for PEP 302 and PEP 451 -__package__ = __name__ # see PEP 366 @ReservedAssignment -if globals().get("__spec__") is not None: - __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable -# Remove other six meta path importers, since they cause problems. This can -# happen if six is removed from sys.modules and then reloaded. (Setuptools does -# this for some reason.) -if sys.meta_path: - for i, importer in enumerate(sys.meta_path): - # Here's some real nastiness: Another "instance" of the six module might - # be floating around. Therefore, we can't use isinstance() to check for - # the six meta path importer, since the other six instance will have - # inserted an importer with different class. - if (type(importer).__name__ == "_SixMetaPathImporter" and - importer.name == __name__): - del sys.meta_path[i] - break - del i, importer -# Finally, add the importer to the meta path import hook. -sys.meta_path.append(_importer) diff --git a/gunicorn/sock.py b/gunicorn/sock.py index 8870936a..08ede89e 100644 --- a/gunicorn/sock.py +++ b/gunicorn/sock.py @@ -11,7 +11,6 @@ import sys import time from gunicorn import util -from gunicorn.six import string_types class BaseSocket(object): @@ -133,7 +132,7 @@ def _sock_type(addr): sock_type = TCP6Socket else: sock_type = TCPSocket - elif isinstance(addr, string_types): + elif isinstance(addr, str): sock_type = UnixSocket else: raise TypeError("Unable to create socket from: %r" % addr) diff --git a/gunicorn/util.py b/gunicorn/util.py index 1e10ae2f..973d7ed3 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -3,30 +3,29 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -from __future__ import print_function - import email.utils +import errno import fcntl +import html +import inspect import io +import logging import os -import pkg_resources import pwd import random +import re import socket import sys import textwrap import time import traceback -import inspect -import errno import warnings -import logging -import re -from gunicorn import _compat +import pkg_resources + from gunicorn.errors import AppImportError -from gunicorn.six import text_type from gunicorn.workers import SUPPORTED_WORKERS +import urllib.parse REDIRECT_TO = getattr(os, 'devnull', '/dev/null') @@ -140,6 +139,23 @@ def load_class(uri, default="gunicorn.workers.sync.SyncWorker", return getattr(mod, klass) +positionals = ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, +) + + +def get_arity(f): + sig = inspect.signature(f) + arity = 0 + + for param in sig.parameters.values(): + if param.kind in positionals: + arity += 1 + + return arity + + def get_username(uid): """ get the username for a user id""" return pwd.getpwuid(uid).pw_name @@ -169,7 +185,6 @@ def set_owner_process(uid, gid, initgroups=False): def chown(path, uid, gid): - gid = abs(gid) & 0x7FFFFFFF # see note above. os.chown(path, uid, gid) @@ -291,7 +306,7 @@ except ImportError: def write_chunk(sock, data): - if isinstance(data, text_type): + if isinstance(data, str): data = data.encode('utf-8') chunk_size = "%X\r\n" % len(data) chunk = b"".join([chunk_size.encode('utf-8'), data, b"\r\n"]) @@ -317,7 +332,7 @@ def write_nonblock(sock, data, chunked=False): def write_error(sock, status_int, reason, mesg): - html = textwrap.dedent("""\ + html_error = textwrap.dedent("""\ %(reason)s @@ -327,7 +342,7 @@ def write_error(sock, status_int, reason, mesg): %(mesg)s - """) % {"reason": reason, "mesg": _compat.html_escape(mesg)} + """) % {"reason": reason, "mesg": html.escape(mesg)} http = textwrap.dedent("""\ HTTP/1.1 %s %s\r @@ -335,7 +350,7 @@ def write_error(sock, status_int, reason, mesg): Content-Type: text/html\r Content-Length: %d\r \r - %s""") % (str(status_int), reason, len(html), html) + %s""") % (str(status_int), reason, len(html_error), html_error) write_nonblock(sock, http.encode('latin1')) @@ -501,7 +516,7 @@ def to_bytestring(value, encoding="utf8"): """Converts a string argument to a byte string""" if isinstance(value, bytes): return value - if not isinstance(value, text_type): + if not isinstance(value, str): raise TypeError('%r is not a string' % value) return value.encode(encoding) @@ -551,7 +566,30 @@ def split_request_uri(uri): # relative uri while the RFC says we should consider it as abs_path # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html#sec5.1.2 # We use temporary dot prefix to workaround this behaviour - parts = _compat.urlsplit("." + uri) + parts = urllib.parse.urlsplit("." + uri) return parts._replace(path=parts.path[1:]) - return _compat.urlsplit(uri) + return urllib.parse.urlsplit(uri) + + +# From six.reraise +def reraise(tp, value, tb=None): + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None + + +def bytes_to_str(b): + if isinstance(b, str): + return b + return str(b, 'latin1') + + +def unquote_to_wsgi_str(string): + return urllib.parse.unquote_to_bytes(string).decode('latin-1') diff --git a/gunicorn/workers/__init__.py b/gunicorn/workers/__init__.py index 074e0012..29c04c2a 100644 --- a/gunicorn/workers/__init__.py +++ b/gunicorn/workers/__init__.py @@ -3,20 +3,14 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -import sys - # supported gunicorn workers. SUPPORTED_WORKERS = { "sync": "gunicorn.workers.sync.SyncWorker", "eventlet": "gunicorn.workers.geventlet.EventletWorker", + "gaiohttp": "gunicorn.workers.gaiohttp.AiohttpWorker", "gevent": "gunicorn.workers.ggevent.GeventWorker", "gevent_wsgi": "gunicorn.workers.ggevent.GeventPyWSGIWorker", "gevent_pywsgi": "gunicorn.workers.ggevent.GeventPyWSGIWorker", "tornado": "gunicorn.workers.gtornado.TornadoWorker", "gthread": "gunicorn.workers.gthread.ThreadWorker", } - - -if sys.version_info >= (3, 4): - # gaiohttp worker can be used with Python 3.4+ only. - SUPPORTED_WORKERS["gaiohttp"] = "gunicorn.workers.gaiohttp.AiohttpWorker" diff --git a/gunicorn/workers/base.py b/gunicorn/workers/base.py index 881efa0f..934ad723 100644 --- a/gunicorn/workers/base.py +++ b/gunicorn/workers/base.py @@ -3,27 +3,27 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -from datetime import datetime +import io import os -from random import randint import signal -from ssl import SSLError import sys import time import traceback +from datetime import datetime +from random import randint +from ssl import SSLError -from gunicorn import six from gunicorn import util -from gunicorn.workers.workertmp import WorkerTmp -from gunicorn.reloader import reloader_engines from gunicorn.http.errors import ( - InvalidHeader, InvalidHeaderName, InvalidRequestLine, InvalidRequestMethod, - InvalidHTTPVersion, LimitRequestLine, LimitRequestHeaders, + ForbiddenProxyRequest, InvalidHeader, + InvalidHeaderName, InvalidHTTPVersion, + InvalidProxyLine, InvalidRequestLine, + InvalidRequestMethod, InvalidSchemeHeaders, + LimitRequestHeaders, LimitRequestLine, ) -from gunicorn.http.errors import InvalidProxyLine, ForbiddenProxyRequest -from gunicorn.http.errors import InvalidSchemeHeaders -from gunicorn.http.wsgi import default_environ, Response -from gunicorn.six import MAXSIZE +from gunicorn.http.wsgi import Response, default_environ +from gunicorn.reloader import reloader_engines +from gunicorn.workers.workertmp import WorkerTmp class Worker(object): @@ -52,7 +52,7 @@ class Worker(object): self.nr = 0 jitter = randint(0, cfg.max_requests_jitter) - self.max_requests = cfg.max_requests + jitter or MAXSIZE + self.max_requests = cfg.max_requests + jitter or sys.maxsize self.alive = True self.log = log self.tmp = WorkerTmp(cfg) @@ -150,7 +150,7 @@ class Worker(object): _, exc_val, exc_tb = sys.exc_info() self.reloader.add_extra_file(exc_val.filename) - tb_string = six.StringIO() + tb_string = io.StringIO() traceback.print_tb(exc_tb, file=tb_string) self.wsgi = util.make_fail_app(tb_string.getvalue()) finally: @@ -170,9 +170,8 @@ class Worker(object): # Don't let SIGTERM and SIGUSR1 disturb active requests # by interrupting system calls - if hasattr(signal, 'siginterrupt'): # python >= 2.6 - signal.siginterrupt(signal.SIGTERM, False) - signal.siginterrupt(signal.SIGUSR1, False) + signal.siginterrupt(signal.SIGTERM, False) + signal.siginterrupt(signal.SIGUSR1, False) if hasattr(signal, 'set_wakeup_fd'): signal.set_wakeup_fd(self.PIPE[1]) diff --git a/gunicorn/workers/base_async.py b/gunicorn/workers/base_async.py index a3a0f912..05f4799a 100644 --- a/gunicorn/workers/base_async.py +++ b/gunicorn/workers/base_async.py @@ -13,7 +13,6 @@ import gunicorn.http as http import gunicorn.http.wsgi as wsgi import gunicorn.util as util import gunicorn.workers.base as base -from gunicorn import six ALREADY_HANDLED = object() @@ -38,7 +37,7 @@ class AsyncWorker(base.Worker): try: listener_name = listener.getsockname() if not self.cfg.keepalive: - req = six.next(parser) + req = next(parser) self.handle_request(listener_name, req, client, addr) else: # keepalive loop @@ -46,7 +45,7 @@ class AsyncWorker(base.Worker): while True: req = None with self.timeout_ctx(): - req = six.next(parser) + req = next(parser) if not req: break if req.proxy_protocol_info: @@ -60,10 +59,10 @@ class AsyncWorker(base.Worker): self.log.debug("Closing connection. %s", e) except ssl.SSLError: # pass to next try-except level - six.reraise(*sys.exc_info()) + util.reraise(*sys.exc_info()) except EnvironmentError: # pass to next try-except level - six.reraise(*sys.exc_info()) + util.reraise(*sys.exc_info()) except Exception as e: self.handle_error(req, client, addr, e) except ssl.SSLError as e: @@ -126,7 +125,7 @@ class AsyncWorker(base.Worker): except EnvironmentError: # If the original exception was a socket.error we delegate # handling it to the caller (where handle() might ignore it) - six.reraise(*sys.exc_info()) + util.reraise(*sys.exc_info()) except Exception: if resp and resp.headers_sent: # If the requests have already been sent, we should close the diff --git a/gunicorn/workers/gaiohttp.py b/gunicorn/workers/gaiohttp.py index bef6b495..b8248259 100644 --- a/gunicorn/workers/gaiohttp.py +++ b/gunicorn/workers/gaiohttp.py @@ -3,25 +3,20 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -import sys - from gunicorn import util -if sys.version_info >= (3, 4): - try: - import aiohttp # pylint: disable=unused-import - except ImportError: - raise RuntimeError("You need aiohttp installed to use this worker.") - else: - try: - from aiohttp.worker import GunicornWebWorker as AiohttpWorker - except ImportError: - from gunicorn.workers._gaiohttp import AiohttpWorker - - util.warn( - "The 'gaiohttp' worker is deprecated. See --worker-class " - "documentation for more information." - ) - __all__ = ['AiohttpWorker'] +try: + import aiohttp # pylint: disable=unused-import +except ImportError: + raise RuntimeError("You need aiohttp installed to use this worker.") else: - raise RuntimeError("You need Python >= 3.4 to use the gaiohttp worker") + try: + from aiohttp.worker import GunicornWebWorker as AiohttpWorker + except ImportError: + from gunicorn.workers._gaiohttp import AiohttpWorker + + util.warn( + "The 'gaiohttp' worker is deprecated. See --worker-class " + "documentation for more information." + ) + __all__ = ['AiohttpWorker'] diff --git a/gunicorn/workers/geventlet.py b/gunicorn/workers/geventlet.py index 189062c8..b0cf4b83 100644 --- a/gunicorn/workers/geventlet.py +++ b/gunicorn/workers/geventlet.py @@ -5,6 +5,7 @@ from functools import partial import errno +import os import sys try: @@ -23,13 +24,12 @@ from eventlet.hubs import trampoline from eventlet.wsgi import ALREADY_HANDLED as EVENTLET_ALREADY_HANDLED import greenlet -from gunicorn.http.wsgi import sendfile as o_sendfile from gunicorn.workers.base_async import AsyncWorker def _eventlet_sendfile(fdout, fdin, offset, nbytes): while True: try: - return o_sendfile(fdout, fdin, offset, nbytes) + return os.sendfile(fdout, fdin, offset, nbytes) except OSError as e: if e.args[0] == errno.EAGAIN: trampoline(fdout, write=True) @@ -79,10 +79,7 @@ def _eventlet_stop(client, server, conn): def patch_sendfile(): - from gunicorn.http import wsgi - - if o_sendfile is not None: - setattr(wsgi, "sendfile", _eventlet_sendfile) + setattr(os, "sendfile", _eventlet_sendfile) class EventletWorker(AsyncWorker): diff --git a/gunicorn/workers/ggevent.py b/gunicorn/workers/ggevent.py index fb9d9194..16dea592 100644 --- a/gunicorn/workers/ggevent.py +++ b/gunicorn/workers/ggevent.py @@ -28,14 +28,13 @@ from gevent import pywsgi import gunicorn from gunicorn.http.wsgi import base_environ from gunicorn.workers.base_async import AsyncWorker -from gunicorn.http.wsgi import sendfile as o_sendfile VERSION = "gevent/%s gunicorn/%s" % (gevent.__version__, gunicorn.__version__) def _gevent_sendfile(fdout, fdin, offset, nbytes): while True: try: - return o_sendfile(fdout, fdin, offset, nbytes) + return os.sendfile(fdout, fdin, offset, nbytes) except OSError as e: if e.args[0] == errno.EAGAIN: wait_write(fdout) @@ -43,10 +42,7 @@ def _gevent_sendfile(fdout, fdin, offset, nbytes): raise def patch_sendfile(): - from gunicorn.http import wsgi - - if o_sendfile is not None: - setattr(wsgi, "sendfile", _gevent_sendfile) + setattr(os, "sendfile", _gevent_sendfile) class GeventWorker(AsyncWorker): @@ -70,12 +66,8 @@ class GeventWorker(AsyncWorker): # patch sockets sockets = [] for s in self.sockets: - if sys.version_info[0] == 3: - sockets.append(socket(s.FAMILY, _socket.SOCK_STREAM, - fileno=s.sock.fileno())) - else: - sockets.append(socket(s.FAMILY, _socket.SOCK_STREAM, - _sock=s)) + sockets.append(socket(s.FAMILY, _socket.SOCK_STREAM, + fileno=s.sock.fileno())) self.sockets = sockets def notify(self): diff --git a/gunicorn/workers/gthread.py b/gunicorn/workers/gthread.py index 43712c45..257499e6 100644 --- a/gunicorn/workers/gthread.py +++ b/gunicorn/workers/gthread.py @@ -10,23 +10,22 @@ # If no event happen after the keep alive timeout, the connectoin is # closed. -from collections import deque -from datetime import datetime import errno -from functools import partial import os +import selectors import socket import ssl import sys -from threading import RLock import time +from collections import deque +from datetime import datetime +from functools import partial +from threading import RLock -from .. import http -from ..http import wsgi -from .. import util from . import base -from .. import six - +from .. import http +from .. import util +from ..http import wsgi try: import concurrent.futures as futures @@ -36,19 +35,6 @@ except ImportError: Python version. """) -try: - # Python 3.4+ - import selectors -except ImportError: - # Python 2 - try: - import selectors34 as selectors - except ImportError: - raise RuntimeError( - "You need to install the 'selectors34' package to use this worker " - "with this Python version." - ) - class TConn(object): def __init__(self, cfg, sock, client, server): @@ -278,7 +264,7 @@ class ThreadWorker(base.Worker): keepalive = False req = None try: - req = six.next(conn.parser) + req = next(conn.parser) if not req: return (False, conn) @@ -352,7 +338,7 @@ class ThreadWorker(base.Worker): return False except EnvironmentError: # pass to next try-except level - six.reraise(*sys.exc_info()) + util.reraise(*sys.exc_info()) except Exception: if resp and resp.headers_sent: # If the requests have already been sent, we should close the diff --git a/gunicorn/workers/sync.py b/gunicorn/workers/sync.py index 1d2ce2f6..7efa354d 100644 --- a/gunicorn/workers/sync.py +++ b/gunicorn/workers/sync.py @@ -16,7 +16,6 @@ import gunicorn.http as http import gunicorn.http.wsgi as wsgi import gunicorn.util as util import gunicorn.workers.base as base -from gunicorn import six class StopWaiting(Exception): """ exception raised to stop waiting for a connnection """ @@ -131,7 +130,7 @@ class SyncWorker(base.Worker): **self.cfg.ssl_options) parser = http.RequestParser(self.cfg, client) - req = six.next(parser) + req = next(parser) self.handle_request(listener, req, client, addr) except http.errors.NoMoreData as e: self.log.debug("Ignored premature client disconnection. %s", e) @@ -188,7 +187,7 @@ class SyncWorker(base.Worker): respiter.close() except EnvironmentError: # pass to next try-except level - six.reraise(*sys.exc_info()) + util.reraise(*sys.exc_info()) except Exception: if resp and resp.headers_sent: # If the requests have already been sent, we should close the diff --git a/gunicorn/workers/workertmp.py b/gunicorn/workers/workertmp.py index 36bc97a6..22aaef34 100644 --- a/gunicorn/workers/workertmp.py +++ b/gunicorn/workers/workertmp.py @@ -38,13 +38,8 @@ class WorkerTmp(object): self.spinner = 0 def notify(self): - try: - self.spinner = (self.spinner + 1) % 2 - os.fchmod(self._tmp.fileno(), self.spinner) - except AttributeError: - # python < 2.6 - self._tmp.truncate(0) - os.write(self._tmp.fileno(), b"X") + self.spinner = (self.spinner + 1) % 2 + os.fchmod(self._tmp.fileno(), self.spinner) def last_update(self): return os.fstat(self._tmp.fileno()).st_ctime diff --git a/requirements_test.txt b/requirements_test.txt index a8729b16..424c4ac1 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,3 +1,3 @@ coverage>=4.0,<4.4 # TODO: https://github.com/benoitc/gunicorn/issues/1548 -pytest==3.2.5 # TODO: upgrade to latest version requires drop support to Python 2.6 +pytest pytest-cov==2.5.1 diff --git a/scripts/update_thanks.py b/scripts/update_thanks.py index 8c355848..b999365f 100644 --- a/scripts/update_thanks.py +++ b/scripts/update_thanks.py @@ -6,7 +6,6 @@ # ======= # pip install validate_email pyDNS # -from __future__ import print_function import sys from validate_email import validate_email diff --git a/setup.py b/setup.py index 639435c5..f7e5d526 100644 --- a/setup.py +++ b/setup.py @@ -20,13 +20,12 @@ CLASSIFIERS = [ 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX', 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3 :: Only', 'Topic :: Internet', 'Topic :: Utilities', 'Topic :: Software Development :: Libraries :: Python Modules', @@ -44,11 +43,6 @@ fname = os.path.join(os.path.dirname(__file__), 'requirements_test.txt') with open(fname) as f: tests_require = [l.strip() for l in f.readlines()] -if sys.version_info[:2] < (3, 3): - tests_require.append('mock') -if sys.version_info[:2] < (2, 7): - tests_require.append('unittest2') - class PyTestCommand(TestCommand): user_options = [ ("cov", None, "measure coverage") @@ -77,8 +71,6 @@ extra_require = { 'tornado': ['tornado>=0.2'], 'gthread': [], } -if sys.version_info[0] < 3: - extra_require['gthread'] = ['futures'] setup( name='gunicorn', @@ -91,7 +83,7 @@ setup( license='MIT', url='http://gunicorn.org', - python_requires='>=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + python_requires='>=3.4', classifiers=CLASSIFIERS, zip_safe=False, packages=find_packages(exclude=['examples', 'tests']), diff --git a/tests/t.py b/tests/t.py index 62b3f71a..539df27e 100644 --- a/tests/t.py +++ b/tests/t.py @@ -4,17 +4,17 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. +import io import os import tempfile dirname = os.path.dirname(__file__) from gunicorn.http.parser import RequestParser -from gunicorn.six import BytesIO def data_source(fname): - buf = BytesIO() + buf = io.BytesIO() with open(fname) as handle: for line in handle: line = line.rstrip("\n").replace("\\r\\n", "\r\n") diff --git a/tests/test_http.py b/tests/test_http.py index 628039ce..a91f4794 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -1,5 +1,6 @@ # -*- encoding: utf-8 -*- +import io import t import pytest @@ -7,7 +8,6 @@ from gunicorn import util from gunicorn.http.body import Body, LengthReader, EOFReader from gunicorn.http.wsgi import Response from gunicorn.http.unreader import Unreader, IterUnreader, SocketUnreader -from gunicorn.six import BytesIO from gunicorn.http.errors import InvalidHeader, InvalidHeaderName try: @@ -17,7 +17,7 @@ except ImportError: def assert_readline(payload, size, expected): - body = Body(BytesIO(payload)) + body = Body(io.BytesIO(payload)) assert body.readline(size) == expected @@ -32,28 +32,28 @@ def test_readline_zero_size(): def test_readline_new_line_before_size(): - body = Body(BytesIO(b"abc\ndef")) + body = Body(io.BytesIO(b"abc\ndef")) assert body.readline(4) == b"abc\n" assert body.readline() == b"def" def test_readline_new_line_after_size(): - body = Body(BytesIO(b"abc\ndef")) + body = Body(io.BytesIO(b"abc\ndef")) assert body.readline(2) == b"ab" assert body.readline() == b"c\n" def test_readline_no_new_line(): - body = Body(BytesIO(b"abcdef")) + body = Body(io.BytesIO(b"abcdef")) assert body.readline() == b"abcdef" - body = Body(BytesIO(b"abcdef")) + body = Body(io.BytesIO(b"abcdef")) assert body.readline(2) == b"ab" assert body.readline(2) == b"cd" assert body.readline(2) == b"ef" def test_readline_buffer_loaded(): - reader = BytesIO(b"abc\ndef") + reader = io.BytesIO(b"abc\ndef") body = Body(reader) body.read(1) # load internal buffer reader.write(b"g\nhi") @@ -64,7 +64,7 @@ def test_readline_buffer_loaded(): def test_readline_buffer_loaded_with_size(): - body = Body(BytesIO(b"abc\ndef")) + body = Body(io.BytesIO(b"abc\ndef")) body.read(1) # load internal buffer assert body.readline(2) == b"bc" assert body.readline(2) == b"\n" @@ -82,7 +82,7 @@ def test_http_header_encoding(): response = Response(mocked_request, mocked_socket, None) # set umlaut header - response.headers.append(('foo', u'häder')) + response.headers.append(('foo', 'häder')) with pytest.raises(UnicodeEncodeError): response.send_headers() @@ -169,7 +169,7 @@ def test_iter_unreader_chunk(): def test_socket_unreader_chunk(): - fake_sock = t.FakeSocket(BytesIO(b'Lorem ipsum dolor')) + fake_sock = t.FakeSocket(io.BytesIO(b'Lorem ipsum dolor')) sock_unreader = SocketUnreader(fake_sock, max_chunk=5) assert sock_unreader.chunk() == b'Lorem' diff --git a/tests/test_pidfile.py b/tests/test_pidfile.py index 0c83ac11..e8c07567 100644 --- a/tests/test_pidfile.py +++ b/tests/test_pidfile.py @@ -4,7 +4,6 @@ # See the NOTICE for more information. import errno -import sys try: import unittest.mock as mock @@ -15,12 +14,7 @@ import gunicorn.pidfile def builtin(name): - if sys.version_info >= (3, 0): - module = 'builtins' - else: - module = '__builtin__' - - return '{0}.{1}'.format(module, name) + return 'builtins.{}'.format(name) @mock.patch(builtin('open'), new_callable=mock.mock_open) diff --git a/tests/test_ssl.py b/tests/test_ssl.py index 52c8caec..6c16212a 100644 --- a/tests/test_ssl.py +++ b/tests/test_ssl.py @@ -5,13 +5,11 @@ # This file is part of gunicorn released under the MIT license. # See the NOTICE for more information. -import sys - import pytest from gunicorn.config import ( KeyFile, CertFile, SSLVersion, CACerts, SuppressRaggedEOFs, - DoHandshakeOnConnect, Setting, + DoHandshakeOnConnect, Setting, Ciphers, ) ssl = pytest.importorskip('ssl') @@ -69,11 +67,7 @@ def test_do_handshake_on_connect(): assert DoHandshakeOnConnect.default is False -@pytest.mark.skipif(sys.version_info < (2, 7), - reason="requires Python 2.7+") def test_ciphers(): - from gunicorn.config import Ciphers - assert issubclass(Ciphers, Setting) assert Ciphers.name == 'ciphers' assert Ciphers.section == 'SSL' diff --git a/tests/test_statsd.py b/tests/test_statsd.py index ed55fa2a..b75057c8 100644 --- a/tests/test_statsd.py +++ b/tests/test_statsd.py @@ -1,14 +1,13 @@ -from datetime import timedelta -import socket +import io import logging -import tempfile -import shutil import os +import shutil +import socket +import tempfile +from datetime import timedelta from gunicorn.config import Config from gunicorn.instrument.statsd import Statsd -from gunicorn.six import StringIO - from support import SimpleNamespace @@ -63,7 +62,7 @@ def test_statsd_fail(): def test_instrument(): logger = Statsd(Config()) # Capture logged messages - sio = StringIO() + sio = io.StringIO() logger.error_log.addHandler(logging.StreamHandler(sio)) logger.sock = MockSocket(False) diff --git a/tests/test_util.py b/tests/test_util.py index 11d74898..4d977981 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -7,7 +7,7 @@ import pytest from gunicorn import util from gunicorn.errors import AppImportError -from gunicorn.six.moves.urllib.parse import SplitResult # pylint: disable=no-name-in-module +from urllib.parse import SplitResult @pytest.mark.parametrize('test_input, expected', [ diff --git a/tests/treq.py b/tests/treq.py index 46f2d5c2..d82d036f 100644 --- a/tests/treq.py +++ b/tests/treq.py @@ -11,7 +11,6 @@ from gunicorn._compat import execfile_ from gunicorn.config import Config from gunicorn.http.parser import RequestParser from gunicorn.util import split_request_uri -from gunicorn import six dirname = os.path.dirname(__file__) random.seed() @@ -71,10 +70,7 @@ class request(object): def send_bytes(self): for d in self.data: - if six.PY3: - yield bytes([d]) - else: - yield d + yield bytes([d]) def send_random(self): maxs = round(len(self.data) / 10) @@ -205,7 +201,7 @@ class request(object): if body: raise AssertionError("Failed to read entire body: %r" % body) try: - data = six.next(iter(req.body)) + data = next(iter(req.body)) raise AssertionError("Read data after body finished: %r" % data) except StopIteration: pass @@ -284,4 +280,4 @@ class badrequest(object): def check(self, cfg): p = RequestParser(cfg, self.send()) - six.next(p) + next(p) diff --git a/tox.ini b/tox.ini index 89f576d4..5a44b7f4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py34, py35, py36, py36-dev, py37, pypy, lint +envlist = py34, py35, py36, py36-dev, py37, pypy, lint skipsdist = True [testenv] @@ -7,8 +7,6 @@ usedevelop = True commands = py.test {posargs} deps = -rrequirements_test.txt - py26: unittest2 - py{26,27},pypy: mock py{34,35,36,36-dev,37}: aiohttp [testenv:lint]