diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..00533c9 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,32 @@ +image: python:2.7 + +stages: + - test + - deploy + +test: + stage: test + script: + - pip install -U nox-automation + - nox + +deploy_production: + stage: deploy + only: + - master + before_script: + - echo "deb http://packages.cloud.google.com/apt cloud-sdk-jessie main" | tee /etc/apt/sources.list.d/google-cloud-sdk.list + - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + - apt-get update && apt-get install -y google-cloud-sdk + script: + - pip install -U -t libs -r requirements-app.txt + - echo $DEPLOY_KEY_FILE_PRODUCTION > /tmp/$CI_PIPELINE_ID.json + - gcloud auth activate-service-account --key-file /tmp/$CI_PIPELINE_ID.json + - echo $ML2GROW_HTTP_AUTH > config.json + - cat config.json + - gcloud --verbosity=info --quiet --project $PROJECT_ID_PRODUCTION app deploy app.yaml + after_script: + - rm /tmp/$CI_PIPELINE_ID.json + + + diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..2948b16 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,2 @@ +# Release 0.0.1 +Initial release of GAEPyPI \ No newline at end of file diff --git a/README.md b/README.md index 9610552..4279e21 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,6 @@ Before deploying, install the required dependencies to the libs folder: pip install -t libs -r requirements-app.txt ``` Next, have a look at config.json, you can set up multiple accounts for the http auth. The passwords are assumed to be hashed with sha1, you can obtain the hash as follows: - ```python from webapp2_extras import security security.hash_password('password', method='sha1') @@ -33,7 +32,6 @@ echo -n "password" | openssl sha1 # shasum echo -n "password" | shasum ``` - Now, you are ready to deploy to google app engine. Depending on how you set up your projects, this may require adjusting app.yaml, or setting the right project for gcloud. Then: ``` gcloud app deploy app.yaml diff --git a/app.yaml b/app.yaml index 7d582c7..791a6de 100644 --- a/app.yaml +++ b/app.yaml @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -runtime: python27 +runtime: python37 api_version: 1 threadsafe: yes @@ -27,8 +27,6 @@ handlers: script: gaepypi.app libraries: -- name: webapp2 - version: latest - name: jinja2 version: latest - name: six diff --git a/gaepypi/_decorators.py b/gaepypi/_decorators.py index c6e8210..b06102e 100644 --- a/gaepypi/_decorators.py +++ b/gaepypi/_decorators.py @@ -17,7 +17,7 @@ import base64 import json from functools import wraps -from webapp2_extras import security +from hashlib import sha1 def basic_auth(required_roles=None): @@ -25,48 +25,49 @@ def basic_auth(required_roles=None): Decorator to require HTTP auth for request handler """ def decorator_basic_auth(func): + @wraps(func) def callf(handler, *args, **kwargs): auth_header = handler.request.headers.get('Authorization') if auth_header is None: - __basic_login(handler) - else: - parts = base64.b64decode(auth_header.split(' ')[1]).split(':') - username = parts[0] - password = ':'.join(parts[1:]) - account = __basic_lookup(username) + return __basic_login(handler) + if not auth_header.startswith('Basic '): + return __basic_login(handler) - # Return 401 Unauthorized if user did not specify credentials or password is mismatched - if not account or account["password"] != __basic_hash(password): - return __basic_login(handler) + (username, password) = base64.b64decode(auth_header.split(' ')[1]).split(':') + account = __basic_lookup(username) - # Return 403 Forbidden if user's account does not have any of the required access roles - user_roles = account['roles'] if 'roles' in account else [] - if required_roles and any([(required_role not in user_roles) for required_role in required_roles]): - return __basic_forbidden(handler) + # Return 401 Unauthorized if user did not specify credentials or password is mismatched + if not account or account["password"] != __basic_hash(password): + return __basic_login(handler) - else: - return func(handler, *args, **kwargs) + # Return 403 Forbidden if user's account does not have any of the required access roles + user_roles = account['roles'] if 'roles' in account else [] + if required_roles and any([(required_role not in user_roles) for required_role in required_roles]): + return __basic_forbidden(handler) + return func(handler, *args, **kwargs) return callf return decorator_basic_auth def __basic_login(handler): - handler.response.set_status(401, message="Authorization Required") - handler.response.headers['WWW-Authenticate'] = 'Basic realm="Secure Area"' + handler.set_header('WWW-Authenticate', 'Basic realm="Secure Area"') + handler.set_status(401) + handler.finish() + return False -def __basic_forbidden(handler): - handler.response.set_status(403, message="Forbidden") +def __basic_hash(password): + return sha1(password.encode('utf-8')).hexdigest() def __basic_lookup(username): with open('config.json') as data_file: config = json.load(data_file) - for account in config["accounts"]: - if account['username'] == username: - return account + return config["accounts"].get(username, None) + + +def __basic_forbidden(handler): + handler.set_status(403, message="Forbidden") -def __basic_hash(password): - return security.hash_password(password, method='sha1') diff --git a/gaepypi/_handlers.py b/gaepypi/_handlers.py index 3787a17..7d3a251 100644 --- a/gaepypi/_handlers.py +++ b/gaepypi/_handlers.py @@ -19,13 +19,13 @@ from .exceptions import GAEPyPIError import os -import webapp2 +from tornado import web from cloudstorage import NotFoundError -from _decorators import basic_auth +from ._decorators import basic_auth from google.appengine.api import app_identity -class BaseHandler(webapp2.RequestHandler): +class BaseHandler(web.RequestHandler): """ Basic handler class for our GAE webapp2 application """ @@ -34,7 +34,7 @@ def write_page(self, body): self.response.write('{}'.format(body)) def write404(self): - self.response.set_status(404) + self.set_status(404) self.write_page('

Not found

') def get_storage(self): @@ -47,19 +47,17 @@ class IndexHandler(BaseHandler): """ Handles / """ - @basic_auth() def get(self): self.write_page('packages') @basic_auth(required_roles=['write']) def post(self): - name = self.request.get('name', default_value=None) - version = self.request.get('version', default_value=None) - action = self.request.get(':action', default_value=None) - upload = self.request.POST.getall('content')[0] - content = upload.file.read() - filename = upload.filename + name = self.get_argument('name') + version = self.get_argument('version') + action = self.get_argument(':action') + content = self.request.files['content'][0]['body'] + filename = self.request.files['content'][0]['filename'] if name and version and content and action == 'file_upload': try: @@ -145,14 +143,16 @@ class PackageDownload(BaseHandler): Handles /packages/package/version/filename """ + def prepare(self): + self.set_header("Content-Type", 'application/octet-stream') + @basic_auth() def get(self, name, version, filename): try: package = Package(self.get_storage(), name, version) with package.get_file(filename) as gcs_file: - self.response.content_type = 'application/octet-stream' - self.response.headers.add('Content-Disposition', 'attachment; filename={0}'.format(filename)) - self.response.write(gcs_file.read()) + self.set_header('Content-Disposition', 'attachment; filename={0}'.format(filename)) + self.write(gcs_file.read()) except NotFoundError: self.write404() diff --git a/gaepypi/_version.py b/gaepypi/_version.py index c07f054..429aa30 100644 --- a/gaepypi/_version.py +++ b/gaepypi/_version.py @@ -14,4 +14,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -__version__ = '0.0.1' \ No newline at end of file +__version__ = '0.0.2' \ No newline at end of file diff --git a/gaepypi/package.py b/gaepypi/package.py index 6db1179..237805f 100644 --- a/gaepypi/package.py +++ b/gaepypi/package.py @@ -18,12 +18,10 @@ from .templates import __templates__ from .renderable import Renderable -import six from contextlib import contextmanager -from abc import ABCMeta, abstractmethod +from abc import abstractmethod -@six.add_metaclass(ABCMeta) class BucketObject(Renderable): def __init__(self, storage, *args, **kwargs): super(BucketObject, self).__init__(*args, **kwargs) diff --git a/gaepypi/renderable.py b/gaepypi/renderable.py index 0170559..f69e262 100644 --- a/gaepypi/renderable.py +++ b/gaepypi/renderable.py @@ -14,8 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from abc import ABCMeta, abstractmethod - +from abc import abstractmethod, ABCMeta import six diff --git a/gaepypi/storage.py b/gaepypi/storage.py index 1424f89..d31d371 100644 --- a/gaepypi/storage.py +++ b/gaepypi/storage.py @@ -18,8 +18,7 @@ from .templates import __templates__ from .renderable import Renderable -import six -from abc import ABCMeta, abstractmethod +from abc import abstractmethod import cloudstorage as gcs my_default_retry_params = gcs.RetryParams(initial_delay=0.2, @@ -29,7 +28,6 @@ gcs.set_default_retry_params(my_default_retry_params) -@six.add_metaclass(ABCMeta) class Storage(Renderable): """ Storage abstract class, describing the interface assumed by the Package/PackageIndex classes @@ -107,8 +105,8 @@ def to_html(self, full_index=True): """ package_indices = PackageIndex.get_all(self) template_file = 'storage-index.html.j2' if not full_index else 'package-index.html.j2' - template = __templates__.get_template(template_file) - return template.render({'indices': package_indices}) + template = __templates__.load(template_file) + return template.generate({'indices': package_indices}) class GCStorage(Storage): diff --git a/gaepypi/templates.py b/gaepypi/templates.py index a4db636..3793523 100644 --- a/gaepypi/templates.py +++ b/gaepypi/templates.py @@ -15,9 +15,7 @@ # along with this program. If not, see . import os -import jinja2 +from tornado import template -__templates__ = jinja2.Environment(loader=jinja2.FileSystemLoader( - os.path.join(os.path.dirname(__file__), 'templates')), - extensions=['jinja2.ext.autoescape'], - autoescape=True) \ No newline at end of file + +__templates__ = template.Loader(os.path.join(os.path.dirname(__file__), 'templates')) diff --git a/gaepypi/wsgi.py b/gaepypi/wsgi.py index b0a5ac1..6640d15 100644 --- a/gaepypi/wsgi.py +++ b/gaepypi/wsgi.py @@ -16,16 +16,31 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import webapp2 -from _handlers import * - -app = webapp2.WSGIApplication([ - ('/', IndexHandler), - ('/pypi/', PypiHandler), - ('/pypi/([^/]+)/', PypiPackageHandler), - ('/pypi/([^/]+)/([^/]+)', PackageVersionHandler), - ('/packages', PackageBase), - ('/packages/([^/]+)', PackageList), - ('/packages/([^/]+)/([^/]+)', PackageVersionHandler), - ('/packages/([^/]+)/([^/]+)/(.+)', PackageDownload) -], debug=True) +import os +from tornado import web +from tornado import wsgi +from ._handlers import * +from .templates import __templates__ + + +settings = { + 'template_loader': __templates__ +} + + +class Application(web.Application): + def __init__(self): + routes = [ + ('/', IndexHandler), + ('/pypi/', PypiHandler), + ('/pypi/([^/]+)/', PypiPackageHandler), + ('/pypi/([^/]+)/([^/]+)', PackageVersionHandler), + ('/packages', PackageBase), + ('/packages/([^/]+)', PackageList), + ('/packages/([^/]+)/([^/]+)', PackageVersionHandler), + ('/packages/([^/]+)/([^/]+)/(.+)', PackageDownload) + ] + super(Application, self).__init__(routes, **settings) + + +app = wsgi.WSGIAdapter(Application()) \ No newline at end of file diff --git a/nox.py b/nox.py deleted file mode 100644 index 98f2ca7..0000000 --- a/nox.py +++ /dev/null @@ -1,14 +0,0 @@ -import nox - - -@nox.session -@nox.parametrize('python_version', ['2.7']) -def default(session, python_version): - session.interpreter = 'python' + python_version - session.install('mock', 'pytest', 'gae_installer', 'pyaml', 'webob', 'GoogleAppEngineCloudStorageClient', 'jinja2', - 'webapp2', 'six') - - session.run( - 'pytest', - 'tests/unit' - ) \ No newline at end of file diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000..23e46b6 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,11 @@ +import nox + + +@nox.session(python="2.7") +def default(session): + session.install('mock', 'pytest', 'gae_installer', 'pyaml', 'webob', 'GoogleAppEngineCloudStorageClient', 'tornado', 'six') + + session.run( + 'pytest', + 'tests/unit' + ) \ No newline at end of file diff --git a/requirements-app.txt b/requirements-app.txt index 6e247aa..86e145a 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -1 +1,3 @@ -GoogleAppEngineCloudStorageClient==1.9.22.1 \ No newline at end of file +GoogleAppEngineCloudStorageClient==1.9.22.1 +tornado==6.0.3 +six \ No newline at end of file diff --git a/tests/unit/test_gcstorage.py b/tests/unit/test_gcstorage.py index 64d927f..50ba2c7 100644 --- a/tests/unit/test_gcstorage.py +++ b/tests/unit/test_gcstorage.py @@ -19,7 +19,6 @@ def tearDown(self): def test_packages_path(self): assert self.s.get_packages_path() == '/mybucket/packages' - def test_package_path(self): assert self.s.get_package_path('dummy') == '/mybucket/packages/dummy' assert self.s.get_package_path('dummy', '0.0.1') == '/mybucket/packages/dummy/0.0.1' @@ -38,15 +37,16 @@ def test_split_path(self): def test_file_exists(self, mock): m = Mock() type(m).is_dir = PropertyMock(return_value=False) + type(m).filename = PropertyMock(return_value='/mybucket/file.txt') mock.return_value = [m] assert self.s.file_exists('/mybucket/file.txt') - mock.assert_called_with('/mybucket/file.txt', delimiter='/') + mock.assert_called_with('/mybucket/file.txt') @patch('gaepypi.storage.gcs.listbucket') def test_file_not_exists(self, mock): mock.return_value = [] assert not self.s.file_exists('/mybucket/file.txt') - mock.assert_called_with('/mybucket/file.txt', delimiter='/') + mock.assert_called_with('/mybucket/file.txt') @patch('gaepypi.storage.gcs.listbucket') def test_file_is_dir(self, mock): @@ -54,12 +54,13 @@ def test_file_is_dir(self, mock): type(m).is_dir = PropertyMock(return_value=True) mock.return_value = [m] assert not self.s.file_exists('/mybucket/file') - mock.assert_called_with('/mybucket/file', delimiter='/') + mock.assert_called_with('/mybucket/file') @patch('gaepypi.storage.gcs.listbucket') def test_path_is_dir(self, mock): m = Mock() type(m).is_dir = PropertyMock(return_value=True) + type(m).filename = PropertyMock(return_value='/mybucket/file') mock.return_value = [m] assert self.s.path_exists('/mybucket/file') mock.assert_called_with('/mybucket/file', delimiter='/') @@ -68,6 +69,7 @@ def test_path_is_dir(self, mock): def test_path_exists(self, mock): m = Mock() type(m).is_dir = PropertyMock(return_value=False) + type(m).filename = PropertyMock(return_value='/mybucket/file.txt') mock.return_value = [m] assert self.s.path_exists('/mybucket/file.txt') mock.assert_called_with('/mybucket/file.txt', delimiter='/')