From 8b49de4ad264d629a8ef6e267fe5588172338dd1 Mon Sep 17 00:00:00 2001 From: Jeryn Mathew Date: Thu, 6 Feb 2014 13:07:22 +0530 Subject: [PATCH 1/4] Fix for issue #693 - Made changes to read and fetch code object from .pyc files. - Error message "Non-ASCII character '\xf3' in file" is resolved. --- gunicorn/six.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/gunicorn/six.py b/gunicorn/six.py index e05e25f6..62c47be7 100644 --- a/gunicorn/six.py +++ b/gunicorn/six.py @@ -286,6 +286,50 @@ _add_doc(b, """Byte literal""") _add_doc(u, """Text literal""") +def _check_if_pyc(fname): + """ Returns True if the extension is .pyc, False if .py and None if otherwise """ + # Make common lambda + test = lambda ext: True if fname.rfind(ext) == len(fname) - len(ext) else False + # Run test + if test(".pyc"): + # TODO: Actually check if file content is .pyc compliant + return True + elif test(".py"): + # TODO: Actually check if file is .py by trying to import + return False + else: + return None + + +def _get_codeobj(pyfile): + """ Returns the code object, given a python file """ + result = _check_if_pyc(pyfile) + if result is True: + # This is a .pyc file. Treat accordingly. + with open(pyfile, "rb") as pycfile: + data = pycfile.read() + + # .pyc format is as follows: + # 0 - 4 bytes: Magic number, which changes with each create of .pyc file. + # First 2 bytes change with each marshal of .pyc file. Last 2 bytes is "\r\n". + # 4 - 8 bytes: Datetime value, when the .py was last changed. + # 8 - EOF: Marshalled code object data. + # So to get code object, just read the 8th byte onwards till EOF, and UN-marshal it. + import marshal + code_obj = marshal.loads(data[8:]) + + elif result is False: + # This is a .py file. + code_obj = compile(open(pyfile, 'rb').read(), pyfile, 'exec') + + else: + # Dunno what this is... + raise Exception("Input file is unknown format: {0}".format(pyfile)) + + # Return code object + return code_obj + + if PY3: import builtins From 2e84d68edbabbefa4be4d9b3812239b6050ef5d9 Mon Sep 17 00:00:00 2001 From: Jeryn Mathew Date: Thu, 6 Feb 2014 13:17:46 +0530 Subject: [PATCH 2/4] Fix for issue #693 - Override older behavior of execfile_ with one with pyc patch --- gunicorn/six.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gunicorn/six.py b/gunicorn/six.py index 62c47be7..fc638455 100644 --- a/gunicorn/six.py +++ b/gunicorn/six.py @@ -344,7 +344,7 @@ if PY3: print_ = getattr(builtins, "print") def execfile_(fname, *args): - return exec_(compile(open(fname, 'rb').read(), fname, 'exec'), *args) + return exec_(_get_codeobj(fname), *args) del builtins @@ -367,7 +367,10 @@ else: raise tp, value, tb """) - execfile_ = execfile + def execfile_(fname, *args): + """ Overriding PY2 execfile() implementation to support .pyc files """ + return exec_(_get_codeobj(fname), *args) + def print_(*args, **kwargs): """The new-style print function.""" From e1e5d3638fa25be1eba013783e4940f35846abe2 Mon Sep 17 00:00:00 2001 From: Jeryn Mathew Date: Sat, 15 Mar 2014 12:16:18 +0530 Subject: [PATCH 3/4] Fix for issue #693 - Remodeled the logic to use imp module to validate the python gunicorn config file --- gunicorn/six.py | 50 ++++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/gunicorn/six.py b/gunicorn/six.py index fc638455..e47a289a 100644 --- a/gunicorn/six.py +++ b/gunicorn/six.py @@ -288,26 +288,37 @@ _add_doc(u, """Text literal""") def _check_if_pyc(fname): """ Returns True if the extension is .pyc, False if .py and None if otherwise """ - # Make common lambda - test = lambda ext: True if fname.rfind(ext) == len(fname) - len(ext) else False - # Run test - if test(".pyc"): - # TODO: Actually check if file content is .pyc compliant - return True - elif test(".py"): - # TODO: Actually check if file is .py by trying to import - return False - else: - return None + from imp import find_module + from os.path import realpath, dirname, basename, splitext + + # Normalize the file-path for the find_module() + filepath = realpath(fname) + dirpath = dirname(filepath) + module_name = splitext(basename(filepath))[0] + + # Validate and fetch + try: + fileobj, fullpath, (_, _, pytype) = find_module(module_name, [ dirpath ]) + + except ImportError: + raise IOError("Cannot find config file. Path maybe incorrect! : {0}".format(filepath)) + + return (pytype, fileobj, fullpath) def _get_codeobj(pyfile): """ Returns the code object, given a python file """ - result = _check_if_pyc(pyfile) - if result is True: + from imp import PY_COMPILED, PY_SOURCE + + result, fileobj, fullpath = _check_if_pyc(pyfile) + + # WARNING: + # fp.read() can blowup if the module is extremely large file. + # Lookout for overflow errors. + if result is PY_COMPILED: # This is a .pyc file. Treat accordingly. - with open(pyfile, "rb") as pycfile: - data = pycfile.read() + data = fileobj.read() + fileobj.close() # .pyc format is as follows: # 0 - 4 bytes: Magic number, which changes with each create of .pyc file. @@ -318,13 +329,14 @@ def _get_codeobj(pyfile): import marshal code_obj = marshal.loads(data[8:]) - elif result is False: + elif result is PY_SOURCE: # This is a .py file. - code_obj = compile(open(pyfile, 'rb').read(), pyfile, 'exec') + code_obj = compile(fileobj.read(), fullpath, 'exec') + fileobj.close() else: - # Dunno what this is... - raise Exception("Input file is unknown format: {0}".format(pyfile)) + # Unsupported extension + raise Exception("Input file is unknown format: {0}".format(fullpath)) # Return code object return code_obj From b6f3cb8e7cb7fd24fbe91d87fdd38cf060b8947a Mon Sep 17 00:00:00 2001 From: Jeryn Mathew Date: Mon, 17 Mar 2014 10:26:53 +0530 Subject: [PATCH 4/4] Fix for issue #693 - Added changes to file read, as per review comments --- gunicorn/six.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gunicorn/six.py b/gunicorn/six.py index e47a289a..f4fb9b86 100644 --- a/gunicorn/six.py +++ b/gunicorn/six.py @@ -315,11 +315,13 @@ def _get_codeobj(pyfile): # WARNING: # fp.read() can blowup if the module is extremely large file. # Lookout for overflow errors. - if result is PY_COMPILED: - # This is a .pyc file. Treat accordingly. + try: data = fileobj.read() + finally: fileobj.close() + # This is a .pyc file. Treat accordingly. + if result is PY_COMPILED: # .pyc format is as follows: # 0 - 4 bytes: Magic number, which changes with each create of .pyc file. # First 2 bytes change with each marshal of .pyc file. Last 2 bytes is "\r\n". @@ -331,8 +333,7 @@ def _get_codeobj(pyfile): elif result is PY_SOURCE: # This is a .py file. - code_obj = compile(fileobj.read(), fullpath, 'exec') - fileobj.close() + code_obj = compile(data, fullpath, 'exec') else: # Unsupported extension