From cc43f89ef56673347ae13e845bcafcfe13cf8de1 Mon Sep 17 00:00:00 2001 From: benoitc Date: Sun, 19 Feb 2012 21:54:17 +0100 Subject: [PATCH] compatibility with django 1.4 and more related fixes. - handle new way to launch applications in django 1.4 - simplify the way we discover the project path and settings - add --pythonpath & --settings options for django applications - still compatible with older versions (>=1.1) -handle DJANGO_SETTINGS_MODULE env. close #283, #275, #274, #241 --- examples/frameworks/djangotest/settings.py | 4 +- gunicorn/app/django_wsgi.py | 112 ++++++ gunicorn/app/djangoapp.py | 351 +++++-------------- gunicorn/config.py | 29 ++ gunicorn/management/commands/run_gunicorn.py | 3 + gunicorn/util.py | 59 ++++ 6 files changed, 287 insertions(+), 271 deletions(-) create mode 100644 gunicorn/app/django_wsgi.py diff --git a/examples/frameworks/djangotest/settings.py b/examples/frameworks/djangotest/settings.py index 1c709c21..0c6613c0 100755 --- a/examples/frameworks/djangotest/settings.py +++ b/examples/frameworks/djangotest/settings.py @@ -14,8 +14,8 @@ MANAGERS = ADMINS DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'test.db', + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': 'test.db', } } diff --git a/gunicorn/app/django_wsgi.py b/gunicorn/app/django_wsgi.py new file mode 100644 index 00000000..dbf2f331 --- /dev/null +++ b/gunicorn/app/django_wsgi.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 - +# +# This file is part of gunicorn released under the MIT license. +# See the NOTICE for more information. + +""" module used to build the django wsgi application """ + +import logging +import os +import re +import sys +import time +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +from django.conf import settings +from django.core.management.base import CommandError +from django.core.management.validation import get_validation_errors +from django.utils import translation +from django.core.servers.basehttp import AdminMediaHandler, WSGIServerException +try: + from django.core.servers.basehttp import get_internal_wsgi_application + django14 = True +except ImportError: + from django.core.handlers.wsgi import WSGIHandler + django14 = False + +from gunicorn import util + +def make_wsgi_application(): + # validate models + s = StringIO() + if get_validation_errors(s): + s.seek(0) + error = s.read() + sys.stderr.write("One or more models did not validate:\n%s" % error) + sys.stderr.flush() + + sys.exit(1) + + translation.activate(settings.LANGUAGE_CODE) + if django14: + return get_internal_wsgi_application() + return WSGIHandler() + +def reload_django_settings(): + mod = util.import_module(os.environ['DJANGO_SETTINGS_MODULE']) + + # reload module + reload(mod) + + # reload settings. + # USe code from django.settings.Settings module. + + # Settings that should be converted into tuples if they're mistakenly entered + # as strings. + tuple_settings = ("INSTALLED_APPS", "TEMPLATE_DIRS") + + for setting in dir(mod): + if setting == setting.upper(): + setting_value = getattr(mod, setting) + if setting in tuple_settings and type(setting_value) == str: + setting_value = (setting_value,) # In case the user forgot the comma. + setattr(settings, setting, setting_value) + + # Expand entries in INSTALLED_APPS like "django.contrib.*" to a list + # of all those apps. + new_installed_apps = [] + for app in settings.INSTALLED_APPS: + if app.endswith('.*'): + app_mod = util.import_module(app[:-2]) + appdir = os.path.dirname(app_mod.__file__) + app_subdirs = os.listdir(appdir) + app_subdirs.sort() + name_pattern = re.compile(r'[a-zA-Z]\w*') + for d in app_subdirs: + if name_pattern.match(d) and os.path.isdir(os.path.join(appdir, d)): + new_installed_apps.append('%s.%s' % (app[:-2], d)) + else: + new_installed_apps.append(app) + setattr(settings, "INSTALLED_APPS", new_installed_apps) + + if hasattr(time, 'tzset') and settings.TIME_ZONE: + # When we can, attempt to validate the timezone. If we can't find + # this file, no check happens and it's harmless. + zoneinfo_root = '/usr/share/zoneinfo' + if (os.path.exists(zoneinfo_root) and not + os.path.exists(os.path.join(zoneinfo_root, + *(settings.TIME_ZONE.split('/'))))): + raise ValueError("Incorrect timezone setting: %s" % + settings.TIME_ZONE) + # Move the time zone info into os.environ. See ticket #2315 for why + # we don't do this unconditionally (breaks Windows). + os.environ['TZ'] = settings.TIME_ZONE + time.tzset() + + # Settings are configured, so we can set up the logger if required + if getattr(settings, 'LOGGING_CONFIG', False): + # First find the logging configuration function ... + logging_config_path, logging_config_func_name = settings.LOGGING_CONFIG.rsplit('.', 1) + logging_config_module = util.import_module(logging_config_path) + logging_config_func = getattr(logging_config_module, logging_config_func_name) + + # ... then invoke it with the logging settings + logging_config_func(settings.LOGGING) + + +def make_command_wsgi_application(admin_mediapath): + reload_django_settings() + return AdminMediaHandler(make_wsgi_application(), admin_mediapath) diff --git a/gunicorn/app/djangoapp.py b/gunicorn/app/djangoapp.py index 09b60fb1..d10de596 100644 --- a/gunicorn/app/djangoapp.py +++ b/gunicorn/app/djangoapp.py @@ -4,132 +4,93 @@ # See the NOTICE for more information. import imp -import logging import os import sys -import time -import traceback -import re from gunicorn.config import Config from gunicorn.app.base import Application +from gunicorn import util + +def find_settings_module(path): + path = os.path.abspath(path) + project_path = None + settings_name = "settings" + + if os.path.isdir(path): + project_path = None + lvl = 0 + for root, dirs, files in os.walk(path): + if "settings.py" in files: + project_path = root + + lvl += 1 + if lvl > 2: + break + elif os.path.isfile(path): + project_path = os.path.dirname(settings_path) + settings_name, _ = os.path.splitext(os.path.basename(settings_path)) + + return project_path, settings_name + + +def make_default_env(cfg): + if cfg.django_settings: + os.environ['DJANGO_SETTINGS_MODULE'] = cfg.django_settings + + if cfg.pythonpath and cfg.pythonpath is not None: + pythonpath = os.path.abspath(cfg.pythonpath) + if pythonpath not in sys.path: + sys.path.insert(0, pythonpath) + + try: + settings_modname = os.environ['DJANGO_SETTINGS_MODULE'] + except KeyError: + # not settings env set, try to build one. + project_path, settings_name = find_settings_module(os.getcwd()) + + if not project_path: + raise RunTimeError("django project not found") + + pythonpath, project_name = os.path.split(project_path) + os.environ['DJANGO_SETTINGS_MODULE'] = "%s.settings" % project_name + if pythonpath not in sys.path: + sys.path.insert(0, pythonpath) + + if project_path not in sys.path: + print project_path + sys.path.insert(0, project_path) -ENVIRONMENT_VARIABLE = 'DJANGO_SETTINGS_MODULE' class DjangoApplication(Application): def init(self, parser, opts, args): - self.global_settings_path = None - self.project_path = None if args: - self.global_settings_path = args[0] - if not os.path.exists(os.path.abspath(args[0])): - self.no_settings(args[0]) + if "." in args[0]: + self.cfg.set("django_settings", args[0]) + else: + # not settings env set, try to build one. + project_path, settings_name = find_settings_module( + os.path.abspath(args[0])) - def get_settings_modname(self): - from django.conf import ENVIRONMENT_VARIABLE + if not project_path: + raise RunTimeError("django project not found") - # get settings module - settings_modname = None - if not self.global_settings_path: - project_path = os.getcwd() - try: - settings_modname = os.environ[ENVIRONMENT_VARIABLE] - except KeyError: - settings_path = os.path.join(project_path, "settings.py") - if not os.path.exists(settings_path): - return self.no_settings(settings_path) - else: - settings_path = os.path.abspath(self.global_settings_path) - if not os.path.exists(settings_path): - return self.no_settings(settings_path) - project_path = os.path.dirname(settings_path) + pythonpath, project_name = os.path.split(project_path) + self.cfg.set("django_settings", "%s.%s" % (project_name, + settings_name)) + self.cfg.set("pythonpath", pythonpath) - if not settings_modname: - project_name = os.path.split(project_path)[-1] - settings_name, ext = os.path.splitext( - os.path.basename(settings_path)) - settings_modname = "%s.%s" % (project_name, settings_name) - os.environ[ENVIRONMENT_VARIABLE] = settings_modname - - self.cfg.set("default_proc_name", settings_modname) - - # add the project path to sys.path - if not project_path in sys.path: - # remove old project path from sys.path - if self.project_path is not None: - idx = sys.path.find(self.project_path) - if idx >= 0: - del sys.path[idx] - self.project_path = project_path - - sys.path.insert(0, project_path) - sys.path.append(os.path.normpath(os.path.join(project_path, - os.pardir))) - - return settings_modname - - def setup_environ(self, settings_modname): - from django.core.management import setup_environ - - # setup environ - try: - parts = settings_modname.split(".") - settings_mod = __import__(settings_modname) - if len(parts) > 1: - settings_mod = __import__(parts[0]) - path = os.path.dirname(os.path.abspath( - os.path.normpath(settings_mod.__file__))) - sys.path.append(path) - for part in parts[1:]: - settings_mod = getattr(settings_mod, part) - setup_environ(settings_mod) - except ImportError: - return self.no_settings(settings_modname, import_error=True) - - def no_settings(self, path, import_error=False): - if import_error: - error = "Error: Can't find '%s' in your PYTHONPATH.\n" % path - else: - error = "Settings file '%s' not found in current folder.\n" % path - sys.stderr.write(error) - sys.stderr.flush() - sys.exit(1) - - def activate_translation(self): - from django.conf import settings - from django.utils import translation - translation.activate(settings.LANGUAGE_CODE) - - def validate(self): - """ Validate models. This also ensures that all models are - imported in case of import-time side effects.""" - from django.core.management.base import CommandError - from django.core.management.validation import get_validation_errors - try: - from cStringIO import StringIO - except ImportError: - from StringIO import StringIO - - s = StringIO() - if get_validation_errors(s): - s.seek(0) - error = s.read() - sys.stderr.write("One or more models did not validate:\n%s" % error) - sys.stderr.flush() - - sys.exit(1) - def load(self): - from django.core.handlers.wsgi import WSGIHandler + # set settings + make_default_env(self.cfg) - self.setup_environ(self.get_settings_modname()) - self.validate() - self.activate_translation() - return WSGIHandler() + # load wsgi application and return it. + mod = util.import_module("gunicorn.app.django_wsgi") + return mod.make_wsgi_application() -class DjangoApplicationCommand(DjangoApplication): + +class DjangoApplicationCommand(Application): def __init__(self, options, admin_media_path): self.usage = None @@ -142,172 +103,24 @@ class DjangoApplicationCommand(DjangoApplication): self.do_load_config() + + def init(self, *args): + if 'settings' in self.options: + self.options['django_settings'] = self.options.pop('settings') + + cfg = {} for k, v in self.options.items(): if k.lower() in self.cfg.settings and v is not None: - self.cfg.set(k.lower(), v) - - def load_config(self): - self.cfg = Config() - - if self.config_file and os.path.exists(self.config_file): - cfg = { - "__builtins__": __builtins__, - "__name__": "__config__", - "__file__": self.config_file, - "__doc__": None, - "__package__": None - } - - try: - execfile(self.config_file, cfg, cfg) - except Exception: - print "Failed to read config file: %s" % self.config_file - traceback.print_exc() - sys.exit(1) - - for k, v in cfg.items(): - # Ignore unknown names - if k not in self.cfg.settings: - continue - try: - self.cfg.set(k.lower(), v) - except: - sys.stderr.write("Invalid value for %s: %s\n\n" % (k, v)) - raise - - for k, v in self.options.items(): - if k.lower() in self.cfg.settings and v is not None: - self.cfg.set(k.lower(), v) - - - def get_settings_modname(self): - from django.conf import ENVIRONMENT_VARIABLE - - settings_modname = None - project_path = os.getcwd() - try: - settings_modname = os.environ[ENVIRONMENT_VARIABLE] - except KeyError: - settings_path = os.path.join(project_path, "settings.py") - if not os.path.exists(settings_path): - return self.no_settings(settings_path) - - if not settings_modname: - project_name = os.path.split(project_path)[-1] - settings_name, ext = os.path.splitext( - os.path.basename(settings_path)) - settings_modname = "%s.%s" % (project_name, settings_name) - os.environ[ENVIRONMENT_VARIABLE] = settings_modname - - self.cfg.set("default_proc_name", settings_modname) - # add the project path to sys.path - if not project_path in sys.path: - # remove old project path from sys.path - if self.project_path is not None: - idx = sys.path.find(self.project_path) - if idx >= 0: - del sys.path[idx] - self.project_path = project_path - sys.path.insert(0, project_path) - sys.path.append(os.path.normpath(os.path.join(project_path, - os.pardir))) - - # reload django settings - self.reload_django_settings(settings_modname) - - return settings_modname - - def reload_django_settings(self, settings_modname): - from django.conf import settings - from django.utils import importlib - - mod = importlib.import_module(settings_modname) - - # reload module - reload(mod) - - # reload settings. - # USe code from django.settings.Settings module. - - # Settings that should be converted into tuples if they're mistakenly entered - # as strings. - tuple_settings = ("INSTALLED_APPS", "TEMPLATE_DIRS") - - for setting in dir(mod): - if setting == setting.upper(): - setting_value = getattr(mod, setting) - if setting in tuple_settings and type(setting_value) == str: - setting_value = (setting_value,) # In case the user forgot the comma. - setattr(settings, setting, setting_value) - - # Expand entries in INSTALLED_APPS like "django.contrib.*" to a list - # of all those apps. - new_installed_apps = [] - for app in settings.INSTALLED_APPS: - if app.endswith('.*'): - app_mod = importlib.import_module(app[:-2]) - appdir = os.path.dirname(app_mod.__file__) - app_subdirs = os.listdir(appdir) - app_subdirs.sort() - name_pattern = re.compile(r'[a-zA-Z]\w*') - for d in app_subdirs: - if name_pattern.match(d) and os.path.isdir(os.path.join(appdir, d)): - new_installed_apps.append('%s.%s' % (app[:-2], d)) - else: - new_installed_apps.append(app) - setattr(settings, "INSTALLED_APPS", new_installed_apps) - - if hasattr(time, 'tzset') and settings.TIME_ZONE: - # When we can, attempt to validate the timezone. If we can't find - # this file, no check happens and it's harmless. - zoneinfo_root = '/usr/share/zoneinfo' - if (os.path.exists(zoneinfo_root) and not - os.path.exists(os.path.join(zoneinfo_root, - *(settings.TIME_ZONE.split('/'))))): - raise ValueError("Incorrect timezone setting: %s" % - settings.TIME_ZONE) - # Move the time zone info into os.environ. See ticket #2315 for why - # we don't do this unconditionally (breaks Windows). - os.environ['TZ'] = settings.TIME_ZONE - time.tzset() - - # Settings are configured, so we can set up the logger if required - if getattr(settings, 'LOGGING_CONFIG', False): - # First find the logging configuration function ... - logging_config_path, logging_config_func_name = settings.LOGGING_CONFIG.rsplit('.', 1) - logging_config_module = importlib.import_module(logging_config_path) - logging_config_func = getattr(logging_config_module, logging_config_func_name) - - # ... then invoke it with the logging settings - logging_config_func(settings.LOGGING) + cfg[k.lower()] = v + return cfg def load(self): - from django.core.handlers.wsgi import WSGIHandler + # set settings + make_default_env(self.cfg) - # reload django settings and setup environ - self.setup_environ(self.get_settings_modname()) - - # validate models and activate translation - self.validate() - self.activate_translation() - - from django.core.servers.basehttp import AdminMediaHandler, WSGIServerException - - try: - return AdminMediaHandler(WSGIHandler(), self.admin_media_path) - except WSGIServerException, e: - # Use helpful error messages instead of ugly tracebacks. - ERRORS = { - 13: "You don't have permission to access that port.", - 98: "That port is already in use.", - 99: "That IP address can't be assigned-to.", - } - try: - error_text = ERRORS[e.args[0].args[0]] - except (AttributeError, KeyError): - error_text = str(e) - sys.stderr.write(self.style.ERROR("Error: %s" % error_text) + '\n') - sys.exit(1) + # load wsgi application and return it. + mod = util.import_module("gunicorn.app.django_wsgi") + return mod.make_command_wsgi_application(self.admin_media_path) def run(): """\ diff --git a/gunicorn/config.py b/gunicorn/config.py index 9b717cd0..3c627934 100644 --- a/gunicorn/config.py +++ b/gunicorn/config.py @@ -727,6 +727,35 @@ class DefaultProcName(Setting): Internal setting that is adjusted for each type of application. """ + +class DjangoSettings(Setting): + name = "django_settings" + section = "Django" + cli = ["--settings"] + meta = "STRING" + validator = validate_string + default = None + desc = """\ + The Python path to a Django settings module. + + e.g. 'myproject.settings.main'. If this isn't provided, the + DJANGO_SETTINGS_MODULE environment variable will be used. + """ + +class DjangoPythonPath(Setting): + name = "pythonpath" + section = "Django" + cli = ["--pythonpath"] + meta = "STRING" + validator = validate_string + default = None + desc = """\ + A directory to add to the Python path for Django. + + e.g. + '/home/djangoprojects/myproject'. + """ + class OnStarting(Setting): name = "on_starting" section = "Server Hooks" diff --git a/gunicorn/management/commands/run_gunicorn.py b/gunicorn/management/commands/run_gunicorn.py index b57ef317..b1773238 100644 --- a/gunicorn/management/commands/run_gunicorn.py +++ b/gunicorn/management/commands/run_gunicorn.py @@ -28,6 +28,9 @@ def make_options(): ] for k in keys: + if k in ('pythonpath', 'django_settings',): + continue + setting = g_settings[k] if not setting.cli: continue diff --git a/gunicorn/util.py b/gunicorn/util.py index 5bf45dab..7aa93b55 100644 --- a/gunicorn/util.py +++ b/gunicorn/util.py @@ -64,6 +64,44 @@ except ImportError: def _setproctitle(title): return + +try: + from importlib import import_module +except ImportError: + def _resolve_name(name, package, level): + """Return the absolute name of the module to be imported.""" + if not hasattr(package, 'rindex'): + raise ValueError("'package' not set to a string") + dot = len(package) + for x in xrange(level, 1, -1): + try: + dot = package.rindex('.', 0, dot) + except ValueError: + raise ValueError("attempted relative import beyond top-level " + "package") + return "%s.%s" % (package[:dot], name) + + + def import_module(name, package=None): + """Import a module. + +The 'package' argument is required when performing a relative import. It +specifies the package to use as the anchor point from which to resolve the +relative import to an absolute import. + +""" + if name.startswith('.'): + if not package: + raise TypeError("relative imports require the 'package' argument") + level = 0 + for character in name: + if character != '.': + break + level += 1 + name = _resolve_name(name[level:], package, level) + __import__(name) + return sys.modules[name] + def load_class(uri, default="sync", section="gunicorn.workers"): if uri.startswith("egg:"): # uses entry points @@ -311,3 +349,24 @@ def check_is_writeable(path): except IOError, e: raise RuntimeError("Error: '%s' isn't writable [%r]" % (path, e)) f.close() + + +if not hasattr(os.path, 'relpath'): + def relpath(path, start=os.path.curdir): + """Return a relative version of a path""" + + if not path: + raise ValueError("no path specified") + + start_list = os.path.abspath(start).split(os.path.sep) + path_list = os.path.abspath(path).split(os.path.sep) + + # Work out how much of the filepath is shared by start and path. + i = len(os.path.commonprefix([start_list, path_list])) + + rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:] + if not rel_list: + return os.path.curdir + return os.path.join(*rel_list) +else: + relpath = os.path.relpath