From 71cdfd8ffa1ce2d5e9e023be0c6d6be0142ebe64 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 17 Feb 2015 14:13:42 -0600 Subject: [PATCH 01/90] added raise_for_status=False when creating blueprint - then you get to actually see the problem in the cli instead of just BAD REQUEST --- stackdio/client/blueprint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/blueprint.py b/stackdio/client/blueprint.py index 263c9dc..beb3b2e 100644 --- a/stackdio/client/blueprint.py +++ b/stackdio/client/blueprint.py @@ -48,7 +48,7 @@ def create_blueprint(self, blueprint, provider="ec2"): self.get_formula(formula_id), component["id"][1]) - return self._post(endpoint, data=json.dumps(blueprint), jsonify=True) + return self._post(endpoint, data=json.dumps(blueprint), jsonify=True, raise_for_status=False) @endpoint("blueprints/") def list_blueprints(self): From 650af89d8d6c84c38869828fb1655a780634c6b6 Mon Sep 17 00:00:00 2001 From: Cory Hughes Date: Mon, 23 Feb 2015 03:18:43 -0600 Subject: [PATCH 02/90] Incrementing client version --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 2bbfa7d..ce01f8f 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -15,7 +15,7 @@ # limitations under the License. # -__version__ = "0.6.0.client.3" +__version__ = "0.6.0.client.4" from functools import wraps import operator From 91ad4a644a2b9072e4d5fc96d9a28c003de6d2ed Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Thu, 5 Mar 2015 16:12:32 -0600 Subject: [PATCH 03/90] Changed profile lookup to be by slug instead of title --- stackdio/client/blueprint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/blueprint.py b/stackdio/client/blueprint.py index beb3b2e..7919c9e 100644 --- a/stackdio/client/blueprint.py +++ b/stackdio/client/blueprint.py @@ -38,7 +38,7 @@ def create_blueprint(self, blueprint, provider="ec2"): host["zone"] = self.get_zone_id(host["zone"], provider) if isinstance(host["cloud_profile"], basestring): - host["cloud_profile"] = self.get_profile_id(host["cloud_profile"], title=True) # noqa + host["cloud_profile"] = self.get_profile_id(host["cloud_profile"]) # noqa for component in host["formula_components"]: if not component.get("sls_path") and isinstance(component["id"], (tuple, list)): From dc49daaf2990df556eabda7d7a6b2a1c58d9b9d3 Mon Sep 17 00:00:00 2001 From: Charlie Penner Date: Thu, 5 Mar 2015 20:54:55 -0600 Subject: [PATCH 04/90] increment client version to .5 --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index ce01f8f..5893583 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -15,7 +15,7 @@ # limitations under the License. # -__version__ = "0.6.0.client.4" +__version__ = "0.6.0.client.5" from functools import wraps import operator From 56003d6de1f11fa4faa0137afe04e4d412ebbcdd Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 17 Mar 2015 11:13:19 -0500 Subject: [PATCH 05/90] Limit requests to < 2.6 --- requirements.txt | 2 +- stackdio/client/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8557dda..ac7d034 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ simplejson -requests>=2.4.0 +requests>=2.4.0,<2.6.0 diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 5893583..286284b 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -15,7 +15,7 @@ # limitations under the License. # -__version__ = "0.6.0.client.5" +__version__ = "0.6.0.client.6" from functools import wraps import operator From 7e3b417cd5a2070a296a6342437c63e404df1164 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 6 Nov 2015 15:47:57 -0600 Subject: [PATCH 06/90] Update for stackdio 0.7 --- requirements.txt | 2 - setup.py | 50 ++++++-------- stackdio/client/__init__.py | 8 +-- stackdio/client/{provider.py => account.py} | 74 +++++++++------------ stackdio/client/blueprint.py | 38 ++++++----- stackdio/client/{profile.py => image.py} | 44 ++++++------ stackdio/client/version.py | 16 +++-- 7 files changed, 112 insertions(+), 120 deletions(-) delete mode 100644 requirements.txt rename stackdio/client/{provider.py => account.py} (65%) rename stackdio/client/{profile.py => image.py} (62%) diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index ac7d034..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -simplejson -requests>=2.4.0,<2.6.0 diff --git a/setup.py b/setup.py index 22795ca..92b72ef 100644 --- a/setup.py +++ b/setup.py @@ -15,54 +15,45 @@ # limitations under the License. # -import os import sys from setuptools import setup, find_packages -from pip.req import parse_requirements -from pip.download import PipSession def test_python_version(): - if float("%d.%d" % sys.version_info[:2]) < 2.6: - print('Your Python version {0}.{1}.{2} is not supported.'.format( - *sys.version_info[:3])) - print('stackdio requires Python 2.6 or newer.') + major = sys.version_info[0] + minor = sys.version_info[1] + micro = sys.version_info[2] + if float('%d.%d' % (major, minor)) < 2.6: + err_msg = ('Your Python version {0}.{1}.{2} is not supported.\n' + 'stackdio-server requires Python 2.6 or newer.\n'.format(major, minor, micro)) + sys.stderr.write(err_msg) sys.exit(1) - -def load_pip_requirements(fp): - reqs, deps = [], [] - for r in parse_requirements(fp, session=PipSession()): - if r.url is not None: - deps.append(str(r.url)) - reqs.append(str(r.req)) - return reqs, deps - # Set version __version__ = '0.0.0' # Explicit default -execfile("stackdio/client/version.py") +execfile('stackdio/client/version.py') SHORT_DESCRIPTION = ('A cloud deployment, automation, and orchestration ' 'platform for everyone.') -LONG_DESCRIPTION = SHORT_DESCRIPTION -# If we have a README.md file, use its contents as the long description -if os.path.isfile('README.md'): - with open('README.md') as f: - LONG_DESCRIPTION = f.read() +# Use the README.md as the long description +with open('README.md') as f: + LONG_DESCRIPTION = f.read() +requirements = [ + 'simplejson', + 'requests>=2.4.0,<2.6.0', +] if __name__ == "__main__": - # build our list of requirements and dependency links based on our - # requirements.txt file - reqs, deps = load_pip_requirements('requirements.txt') + test_python_version() # Call the setup method from setuptools that does all the heavy lifting - # of packaging stackdio + # of packaging stackdio-client setup( - name='stackdio', + name='stackdio-client', version=__version__, url='http://stackd.io', author='Digital Reasoning Systems, Inc.', @@ -73,8 +64,8 @@ def load_pip_requirements(fp): include_package_data=True, packages=find_packages(), zip_safe=False, - install_requires=reqs, - dependency_links=deps, + install_requires=requirements, + dependency_links=[], classifiers=[ 'Development Status :: 3 - Alpha', 'Environment :: Web Environment', @@ -84,6 +75,7 @@ def load_pip_requirements(fp): 'Intended Audience :: System Administrators', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Topic :: System :: Clustering', diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index df3fcec..c7a0d55 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -23,8 +23,8 @@ from .blueprint import BlueprintMixin from .formula import FormulaMixin -from .profile import ProfileMixin -from .provider import ProviderMixin +from .account import AccountMixin +from .image import ImageMixin from .region import RegionMixin from .settings import SettingsMixin from .stack import StackMixin @@ -34,8 +34,8 @@ logger = logging.getLogger(__name__) -class StackdIO(BlueprintMixin, FormulaMixin, ProfileMixin, - ProviderMixin, RegionMixin, StackMixin, SettingsMixin): +class StackdIO(BlueprintMixin, FormulaMixin, AccountMixin, + ImageMixin, RegionMixin, StackMixin, SettingsMixin): def __init__(self, protocol="https", host="localhost", port=443, base_url=None, auth=None, auth_admin=None, diff --git a/stackdio/client/provider.py b/stackdio/client/account.py similarity index 65% rename from stackdio/client/provider.py rename to stackdio/client/account.py index fb6de89..75a802a 100644 --- a/stackdio/client/provider.py +++ b/stackdio/client/account.py @@ -22,44 +22,41 @@ from .version import accepted_versions, deprecated -class ProviderMixin(HttpMixin): +class AccountMixin(HttpMixin): - @endpoint("provider_types/") - def list_provider_types(self): + @endpoint("cloud/providers/") + def list_providers(self): """List all providers""" return self._get(endpoint, jsonify=True)['results'] - @accepted_versions(">=0.6.1") - @endpoint("provider_types/") - def search_provider_types(self, provider_id): - """List all provider_types""" + @endpoint("cloud/providers/") + def search_providers(self, provider_id): + """List all providers""" return self._get(endpoint, jsonify=True)['results'] - @deprecated @accepted_versions("<0.7") - @endpoint("provider_types/") - def get_provider_type_id(self, type_name): + @endpoint("cloud/providers/") + def get_provider_id(self, type_name): """Get the id for the provider specified by type_name""" result = self._get(endpoint, jsonify=True) - for provider_type in result['results']: - if provider_type.get("type_name") == type_name: - return provider_type.get("id") + for provider in result['results']: + if provider.get("type_name") == type_name: + return provider.get("id") raise StackException("Provider type %s not found" % type_name) - @use_admin_auth - @endpoint("providers/") - def create_provider(self, **kwargs): - """Create a provider""" + @endpoint("cloud/accounts/") + def create_account(self, **kwargs): + """Create an account""" form_data = { "title": None, "account_id": None, - "provider_type": None, + "provider": None, "access_key_id": None, "secret_access_key": None, "keypair": None, @@ -74,42 +71,37 @@ def create_provider(self, **kwargs): return self._post(endpoint, data=json.dumps(form_data), jsonify=True) - - @endpoint("providers/") - def list_providers(self): - """List all providers""" + @endpoint("accounts/") + def list_accounts(self): + """List all account""" return self._get(endpoint, jsonify=True)['results'] - - @endpoint("providers/{provider_id}/") - def get_provider(self, provider_id, none_on_404=False): - """Return the provider that matches the given id""" + @endpoint("accounts/{account_id}/") + def get_account(self, account_id, none_on_404=False): + """Return the account that matches the given id""" return self._get(endpoint, jsonify=True, none_on_404=none_on_404) - @accepted_versions(">=0.6.1") - @endpoint("providers/") - def search_providers(self, provider_id): - """List all providers""" + @endpoint("accounts/") + def search_accounts(self, account_id): + """List all accounts""" return self._get(endpoint, jsonify=True)['results'] - - @endpoint("providers/{provider_id}/") - def delete_provider(self, provider_id): - """List all providers""" + @endpoint("accounts/{account_id}/") + def delete_account(self, account_id): + """List all accounts""" return self._delete(endpoint, jsonify=True)['results'] - @deprecated @accepted_versions("<0.7") - def get_provider_id(self, slug, title=False): - """Get the id for a provider that matches slug. If title is True will + def get_account_id(self, slug, title=False): + """Get the id for a account that matches slug. If title is True will look at title instead.""" - providers = self.list_providers() + accounts = self.list_accounts() - for provider in providers: - if provider.get("slug" if not title else "title") == slug: - return provider.get("id") + for account in accounts: + if account.get("slug" if not title else "title") == slug: + return account.get("id") raise StackException("Provider %s not found" % slug) diff --git a/stackdio/client/blueprint.py b/stackdio/client/blueprint.py index 7919c9e..d3d68a4 100644 --- a/stackdio/client/blueprint.py +++ b/stackdio/client/blueprint.py @@ -28,25 +28,33 @@ class BlueprintMixin(HttpMixin): def create_blueprint(self, blueprint, provider="ec2"): """Create a blueprint""" - # check the provided blueprint to see if we need to look up any ids - for host in blueprint["hosts"]: - if isinstance(host["size"], basestring): - host["size"] = self.get_instance_id(host["size"], provider) + formula_map = {} + + if 'formula_versions' in blueprint: + all_formulas = self.list_formulas() - # zone isn't required if you provide a subnet_id - if 'zone' in host and isinstance(host["zone"], basestring): - host["zone"] = self.get_zone_id(host["zone"], provider) + used_formulas = set() - if isinstance(host["cloud_profile"], basestring): - host["cloud_profile"] = self.get_profile_id(host["cloud_profile"]) # noqa + for formula_version in blueprint['formula_versions']: + for formula in all_formulas: + if formula['uri'] == formula_version['formula']: + formula['version'] = formula_version['version'] + used_formulas.add(formula) + break - for component in host["formula_components"]: - if not component.get("sls_path") and isinstance(component["id"], (tuple, list)): - formula_id = self.get_formula_id(component["id"][0]) + for formula in used_formulas: + components = self._get( + '{0}?version={1}'.format(formula['components'], formula['version']), + jsonify=True, + )['results'] + for component in components: + formula_map[component['sls_path']] = formula['uri'] - component["id"] = self.get_component_id( - self.get_formula(formula_id), - component["id"][1]) + # check the provided blueprint to see if we need to look up any ids + for host in blueprint['host_definitions']: + for component in host['formula_components']: + if component['sls_path'] in formula_map: + component['formula'] = formula_map[component['sls_path']] return self._post(endpoint, data=json.dumps(blueprint), jsonify=True, raise_for_status=False) diff --git a/stackdio/client/profile.py b/stackdio/client/image.py similarity index 62% rename from stackdio/client/profile.py rename to stackdio/client/image.py index d16cac9..283c126 100644 --- a/stackdio/client/profile.py +++ b/stackdio/client/image.py @@ -22,13 +22,13 @@ from .version import accepted_versions, deprecated -class ProfileMixin(HttpMixin): +class ImageMixin(HttpMixin): @use_admin_auth - @endpoint("profile/") - def create_profile(self, title, image_id, ssh_user, cloud_provider, + @endpoint("image/") + def create_image(self, title, image_id, ssh_user, cloud_provider, default_instance_size=None): - """Create a profile""" + """Create a image""" data = { "title": title, "image_id": image_id, @@ -39,40 +39,40 @@ def create_profile(self, title, image_id, ssh_user, cloud_provider, return self._post(endpoint, data=json.dumps(data), jsonify=True) - @endpoint("profiles/") - def list_profiles(self): - """List all profiles""" + @endpoint("images/") + def list_images(self): + """List all images""" return self._get(endpoint, jsonify=True)['results'] - @endpoint("profiles/{profile_id}/") - def get_profile(self, profile_id, none_on_404=False): - """Return the profile that matches the given id""" + @endpoint("images/{image_id}/") + def get_image(self, image_id, none_on_404=False): + """Return the image that matches the given id""" return self._get(endpoint, jsonify=True, none_on_404=none_on_404) @accepted_versions(">=0.6.1") - @endpoint("profiles/") - def search_profiles(self, profile_id): - """List all profiles""" + @endpoint("images/") + def search_images(self, image_id): + """List all images""" return self._get(endpoint, jsonify=True)['results'] - @endpoint("profiles/{profile_id}/") - def delete_profile(self, profile_id): - """Delete the profile with the given id""" + @endpoint("images/{image_id}/") + def delete_image(self, image_id): + """Delete the image with the given id""" return self._delete(endpoint, jsonify=True)['results'] @deprecated @accepted_versions("<0.7") - def get_profile_id(self, slug, title=False): - """Get the id for a profile that matches slug. If title is True will look + def get_image_id(self, slug, title=False): + """Get the id for a image that matches slug. If title is True will look at title instead.""" - profiles = self.list_profiles() - for profile in profiles: - if profile.get("slug" if not title else "title") == slug: - return profile.get("id") + images = self.list_images() + for image in images: + if image.get("slug" if not title else "title") == slug: + return image.get("id") raise StackException("Profile %s not found" % slug) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 286284b..dd69ac7 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -15,8 +15,6 @@ # limitations under the License. # -__version__ = "0.6.0.client.6" - from functools import wraps import operator import re @@ -24,12 +22,14 @@ # for setup.py try: - from .exceptions import (IncompatibleVersionException, - InvalidVersionStringException) -except: + from .exceptions import IncompatibleVersionException, InvalidVersionStringException +except Exception: pass +__version__ = '0.7.0.dev' + + def _unsupported_function(func, current_version, accepted_versions): raise IncompatibleVersionException("%s: %s is not one of %s" % (func.__name__, @@ -101,9 +101,11 @@ def wrapper(obj, *args, **kwargs): def deprecated(func): - '''This is a decorator which can be used to mark functions + """ + This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted - when the function is used.''' + when the function is used. + """ @wraps(func) def wrapper(*args, **kwargs): From 316d286f771d4180c75da9410f005c19be5cbddb Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 6 Nov 2015 16:26:42 -0600 Subject: [PATCH 07/90] Fixed a couple more things --- stackdio/client/__init__.py | 6 +++--- stackdio/client/blueprint.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index c7a0d55..1a1edfa 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -79,10 +79,10 @@ def create_security_group(self, name, description, cloud_provider, is_default=Tr } return self._post(endpoint, data=json.dumps(data), jsonify=True) - @endpoint("settings/") + @endpoint("user/") def get_public_key(self): - """Get the public key for the logged in uesr""" - return self._get(endpoint, jsonify=True)['public_key'] + """Get the public key for the logged in user""" + return self._get(endpoint, jsonify=True)['settings']['public_key'] @endpoint("settings/") def set_public_key(self, public_key): diff --git a/stackdio/client/blueprint.py b/stackdio/client/blueprint.py index d3d68a4..3ac6eb6 100644 --- a/stackdio/client/blueprint.py +++ b/stackdio/client/blueprint.py @@ -33,13 +33,13 @@ def create_blueprint(self, blueprint, provider="ec2"): if 'formula_versions' in blueprint: all_formulas = self.list_formulas() - used_formulas = set() + used_formulas = [] for formula_version in blueprint['formula_versions']: for formula in all_formulas: if formula['uri'] == formula_version['formula']: formula['version'] = formula_version['version'] - used_formulas.add(formula) + used_formulas.append(formula) break for formula in used_formulas: From 7391b63b712d7ffc70c2ca4bb7abafcbea19d399 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Mon, 9 Nov 2015 11:45:51 -0600 Subject: [PATCH 08/90] Changed name back to stackdio, removed accepted_version things --- setup.py | 4 +- stackdio/client/__init__.py | 6 ++- stackdio/client/account.py | 34 +--------------- stackdio/client/blueprint.py | 14 ------- stackdio/client/exceptions.py | 1 + stackdio/client/formula.py | 21 ---------- stackdio/client/http.py | 2 - stackdio/client/image.py | 34 +++------------- stackdio/client/region.py | 75 +++++------------------------------ stackdio/client/settings.py | 8 ++-- stackdio/client/stack.py | 15 +------ 11 files changed, 32 insertions(+), 182 deletions(-) diff --git a/setup.py b/setup.py index 92b72ef..043363e 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ def test_python_version(): # Call the setup method from setuptools that does all the heavy lifting # of packaging stackdio-client setup( - name='stackdio-client', + name='stackdio', version=__version__, url='http://stackd.io', author='Digital Reasoning Systems, Inc.', @@ -67,7 +67,7 @@ def test_python_version(): install_requires=requirements, dependency_links=[], classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 4 - Beta', 'Environment :: Web Environment', 'Framework :: Django', 'Intended Audience :: Developers', diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index 1a1edfa..d3c6440 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -19,7 +19,7 @@ import logging from .http import use_admin_auth, endpoint -from .exceptions import BlueprintException, StackException +from .exceptions import BlueprintException, StackException, IncompatibleVersionException from .blueprint import BlueprintMixin from .formula import FormulaMixin @@ -57,6 +57,10 @@ def __init__(self, protocol="https", host="localhost", port=443, _, self.version = _parse_version_string(self.get_version()) + if self.version[0] != 0 or self.version[1] != 7: + raise IncompatibleVersionException('Server version {0}.{1}.{2} not ' + 'supported.'.format(**self.version)) + @endpoint("") def get_root(self): """Get the api root""" diff --git a/stackdio/client/account.py b/stackdio/client/account.py index 75a802a..84e5f12 100644 --- a/stackdio/client/account.py +++ b/stackdio/client/account.py @@ -17,9 +17,7 @@ import json -from .exceptions import StackException -from .http import HttpMixin, endpoint, use_admin_auth -from .version import accepted_versions, deprecated +from .http import HttpMixin, endpoint class AccountMixin(HttpMixin): @@ -29,26 +27,11 @@ def list_providers(self): """List all providers""" return self._get(endpoint, jsonify=True)['results'] - @accepted_versions(">=0.6.1") @endpoint("cloud/providers/") def search_providers(self, provider_id): """List all providers""" return self._get(endpoint, jsonify=True)['results'] - @deprecated - @accepted_versions("<0.7") - @endpoint("cloud/providers/") - def get_provider_id(self, type_name): - """Get the id for the provider specified by type_name""" - - result = self._get(endpoint, jsonify=True) - for provider in result['results']: - if provider.get("type_name") == type_name: - return provider.get("id") - - raise StackException("Provider type %s not found" % type_name) - - @use_admin_auth @endpoint("cloud/accounts/") def create_account(self, **kwargs): """Create an account""" @@ -81,7 +64,6 @@ def get_account(self, account_id, none_on_404=False): """Return the account that matches the given id""" return self._get(endpoint, jsonify=True, none_on_404=none_on_404) - @accepted_versions(">=0.6.1") @endpoint("accounts/") def search_accounts(self, account_id): """List all accounts""" @@ -91,17 +73,3 @@ def search_accounts(self, account_id): def delete_account(self, account_id): """List all accounts""" return self._delete(endpoint, jsonify=True)['results'] - - @deprecated - @accepted_versions("<0.7") - def get_account_id(self, slug, title=False): - """Get the id for a account that matches slug. If title is True will - look at title instead.""" - - accounts = self.list_accounts() - - for account in accounts: - if account.get("slug" if not title else "title") == slug: - return account.get("id") - - raise StackException("Provider %s not found" % slug) diff --git a/stackdio/client/blueprint.py b/stackdio/client/blueprint.py index 3ac6eb6..5caf6f4 100644 --- a/stackdio/client/blueprint.py +++ b/stackdio/client/blueprint.py @@ -17,9 +17,7 @@ import json -from .exceptions import StackException from .http import HttpMixin, endpoint -from .version import accepted_versions, deprecated class BlueprintMixin(HttpMixin): @@ -76,15 +74,3 @@ def search_blueprints(self, **kwargs): @endpoint("blueprints/{blueprint_id}") def delete_blueprint(self, blueprint_id): return self._delete(endpoint, jsonify=True) - - @deprecated - @accepted_versions("<0.7") - def get_blueprint_id(self, title): - """Get the id for a blueprint that matches title""" - - blueprints = self.search_blueprints(title=title) - - if not len(blueprints): - raise StackException("Blueprint %s not found" % title) - - return blueprints[0]['id'] diff --git a/stackdio/client/exceptions.py b/stackdio/client/exceptions.py index b831cdb..b386093 100644 --- a/stackdio/client/exceptions.py +++ b/stackdio/client/exceptions.py @@ -15,6 +15,7 @@ # limitations under the License. # + class StackException(Exception): pass diff --git a/stackdio/client/formula.py b/stackdio/client/formula.py index 587214d..d98be7e 100644 --- a/stackdio/client/formula.py +++ b/stackdio/client/formula.py @@ -17,7 +17,6 @@ import json -from .exceptions import StackException from .http import HttpMixin, endpoint @@ -55,23 +54,3 @@ def delete_formula(self, formula_id): def update_formula(self, formula_id): """Delete formula with matching id""" return self._post(endpoint, json.dumps({"action": "update"}), jsonify=True) - - def get_formula_id(self, title): - """Find a stack id""" - - formulas = self.list_formulas() - for formula in formulas: - if formula.get("title") == title: - return formula.get("id") - - raise StackException("Formula %s not found" % title) - - def get_component_id(self, formula, component_title): - """Get the id for a component from formula_id that matches title""" - - for component in formula.get("components"): - if component.get("title") == component_title: - return component.get("id") - - raise StackException("Component %s not found for formula %s" % - (component_title, formula.get("title"))) diff --git a/stackdio/client/http.py b/stackdio/client/http.py index 4584c1c..a5d6afd 100644 --- a/stackdio/client/http.py +++ b/stackdio/client/http.py @@ -122,8 +122,6 @@ def __init__(self, auth=None, verify=True): from requests.packages.urllib3 import disable_warnings disable_warnings() - - def _request(self, verb, url, quiet=False, none_on_404=False, jsonify=False, raise_for_status=True, *args, **kwargs): diff --git a/stackdio/client/image.py b/stackdio/client/image.py index 283c126..52a972a 100644 --- a/stackdio/client/image.py +++ b/stackdio/client/image.py @@ -17,15 +17,12 @@ import json -from .exceptions import StackException -from .http import HttpMixin, endpoint, use_admin_auth -from .version import accepted_versions, deprecated +from .http import HttpMixin, endpoint class ImageMixin(HttpMixin): - @use_admin_auth - @endpoint("image/") + @endpoint("cloud/images/") def create_image(self, title, image_id, ssh_user, cloud_provider, default_instance_size=None): """Create a image""" @@ -38,41 +35,22 @@ def create_image(self, title, image_id, ssh_user, cloud_provider, } return self._post(endpoint, data=json.dumps(data), jsonify=True) - - @endpoint("images/") + @endpoint("cloud/images/") def list_images(self): """List all images""" return self._get(endpoint, jsonify=True)['results'] - - @endpoint("images/{image_id}/") + @endpoint("cloud/images/{image_id}/") def get_image(self, image_id, none_on_404=False): """Return the image that matches the given id""" return self._get(endpoint, jsonify=True, none_on_404=none_on_404) - - @accepted_versions(">=0.6.1") - @endpoint("images/") + @endpoint("cloud/images/") def search_images(self, image_id): """List all images""" return self._get(endpoint, jsonify=True)['results'] - - @endpoint("images/{image_id}/") + @endpoint("cloud/images/{image_id}/") def delete_image(self, image_id): """Delete the image with the given id""" return self._delete(endpoint, jsonify=True)['results'] - - - @deprecated - @accepted_versions("<0.7") - def get_image_id(self, slug, title=False): - """Get the id for a image that matches slug. If title is True will look - at title instead.""" - - images = self.list_images() - for image in images: - if image.get("slug" if not title else "title") == slug: - return image.get("id") - - raise StackException("Profile %s not found" % slug) diff --git a/stackdio/client/region.py b/stackdio/client/region.py index 9c87da6..5485f2c 100644 --- a/stackdio/client/region.py +++ b/stackdio/client/region.py @@ -15,83 +15,30 @@ # limitations under the License. # -from .exceptions import StackException from .http import HttpMixin, endpoint -from .version import accepted_versions, deprecated class RegionMixin(HttpMixin): - - - @accepted_versions(">=0.6.1") - @endpoint("regions/") - def list_regions(self): + @endpoint("cloud/providers/{provider_name}/regions/") + def list_regions(self, provider_name): return self._get(endpoint, jsonify=True)['results'] - - @endpoint("regions/{region_id}") - def get_region(self, region_id, none_on_404=False): + @endpoint("cloud/providers/{provider_name}/regions/{region_id}") + def get_region(self, provider_name, region_id, none_on_404=False): return self._get(endpoint, jsonify=True, none_on_404=none_on_404) - - @accepted_versions(">=0.6.1") - @endpoint("regions/") - def search_regions(self, **kwargs): + @endpoint("cloud/providers/{provider_name}/regions/") + def search_regions(self, provider_name, **kwargs): return self._get(endpoint, params=kwargs, jsonify=True)['results'] - - @deprecated - @accepted_versions(">=0.6", "<0.7") - @endpoint("regions/") - def get_region_id(self, title, type_name="ec2"): - """Get a zone id for title""" - - provider_type = self.get_provider_type(type_name) - params = { - "title": title, - "provider_type": provider_type["id"] - } - result = self._get(endpoint, params=params, jsonify=True) - if len(result['results']) == 1: - return result['results'][0]['id'] - - raise StackException("Zone %s not found for %s" % (title, type_name)) - - - @accepted_versions("!=0.6") - @endpoint("zones/") + @endpoint("cloud/providers/{provider_name}/zones/") def list_zones(self): return self._get(endpoint, jsonify=True)['results'] - @endpoint("zones/{zone_id}") - def get_zone(self, zone_id, none_on_404=False): + @endpoint("cloud/providers/{provider_name}/zones/{zone_id}") + def get_zone(self, provider_name, zone_id, none_on_404=False): return self._get(endpoint, jsonify=True, none_on_404=none_on_404) - @accepted_versions(">=0.6.1") - @endpoint("zones/") - def search_zones(self, **kwargs): + @endpoint("cloud/providers/{provider_name}/zones/") + def search_zones(self, provider_name, **kwargs): return self._get(endpoint, params=kwargs, jsonify=True)['results'] - - - @deprecated - @accepted_versions("!=0.6", "<0.7") - @endpoint("zones/") - def get_zone_id(self, title, type_name="ec2"): - """Get a zone id for title""" - - result = self._get(endpoint, jsonify=True) - - type_id = self.get_provider_type_id(type_name) - for zone in result['results']: - if zone.get("title") == title: - if 'region' in zone: - # For version 0.6.1 - region = self._get(zone['region'], jsonify=True) - if region.get("provider_type") == type_id: - return zone.get("id") - elif 'provider_type' in zone: - # For versions 0.5.* - if zone['provider_type'] == type_id: - return zone.get("id") - - raise StackException("Zone %s not found for %s" % (title, type_name)) diff --git a/stackdio/client/settings.py b/stackdio/client/settings.py index 4e45650..9be2179 100644 --- a/stackdio/client/settings.py +++ b/stackdio/client/settings.py @@ -23,7 +23,7 @@ class SettingsMixin(HttpMixin): - @endpoint("settings/") + @endpoint("user/") def set_public_key(self, public_key): """Upload a public key for our user. public_key can be the actual key, a file handle, or a path to a key file""" @@ -35,6 +35,8 @@ def set_public_key(self, public_key): public_key = open(public_key, "r").read() data = { - "public_key": public_key + "settings": { + "public_key": public_key, + } } - return self._put(endpoint, data=json.dumps(data), jsonify=True) + return self._patch(endpoint, data=json.dumps(data), jsonify=True) diff --git a/stackdio/client/stack.py b/stackdio/client/stack.py index f78e80e..9dd1e64 100644 --- a/stackdio/client/stack.py +++ b/stackdio/client/stack.py @@ -19,7 +19,7 @@ from .exceptions import StackException from .http import HttpMixin, endpoint -from .version import accepted_versions, deprecated +from .version import deprecated class StackMixin(HttpMixin): @@ -81,18 +81,6 @@ def get_stack_history(self, stack_id): else: return result - @deprecated - @accepted_versions("<0.7") - def get_stack_id(self, title): - """Find a stack id""" - - stacks = self.list_stacks() - for stack in stacks: - if stack.get("title") == title: - return stack.get("id") - - raise StackException("Stack %s not found" % title) - @endpoint("stacks/{stack_id}/hosts/") def get_stack_hosts(self, stack_id): """Get a list of all stack hosts""" @@ -144,7 +132,6 @@ def list_access_rules(self, stack_id): return self._get(endpoint, jsonify=True)['results'] @deprecated - @accepted_versions("<0.7") def get_access_rule_id(self, stack_id, title): """Find an access rule id""" From f7113cf9da68c86a1d6d36411b0ffd1401993c6e Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 10 Nov 2015 17:07:16 -0600 Subject: [PATCH 09/90] Initial port of stackdio-tooling into this repo --- .gitignore | 2 + MANIFEST.in | 1 + README.md | 105 +++++++++- Vagrantfile | 13 ++ blueprints/cdh4-3node.json | 191 +++++++++++++++++ blueprints/cdh5-3node.json | 192 +++++++++++++++++ bootstrap.yaml | 8 + setup.cfg | 5 + setup.py | 23 +- stackdio/blueprints/__init__.py | 56 +++++ stackdio/blueprints/generator.py | 305 +++++++++++++++++++++++++++ stackdio/cli/__init__.py | 214 +++++++++++++++++++ stackdio/cli/mixins/__init__.py | 4 + stackdio/cli/mixins/blueprints.py | 270 ++++++++++++++++++++++++ stackdio/cli/mixins/bootstrap.py | 324 +++++++++++++++++++++++++++++ stackdio/cli/mixins/formulas.py | 81 ++++++++ stackdio/cli/mixins/stacks.py | 334 ++++++++++++++++++++++++++++++ stackdio/cli/polling.py | 26 +++ vbox_setup.sh | 12 ++ 19 files changed, 2164 insertions(+), 2 deletions(-) create mode 100644 MANIFEST.in create mode 100644 Vagrantfile create mode 100644 blueprints/cdh4-3node.json create mode 100644 blueprints/cdh5-3node.json create mode 100644 bootstrap.yaml create mode 100644 setup.cfg create mode 100644 stackdio/blueprints/__init__.py create mode 100644 stackdio/blueprints/generator.py create mode 100644 stackdio/cli/__init__.py create mode 100644 stackdio/cli/mixins/__init__.py create mode 100644 stackdio/cli/mixins/blueprints.py create mode 100644 stackdio/cli/mixins/bootstrap.py create mode 100644 stackdio/cli/mixins/formulas.py create mode 100644 stackdio/cli/mixins/stacks.py create mode 100644 stackdio/cli/polling.py create mode 100644 vbox_setup.sh diff --git a/.gitignore b/.gitignore index d1eadd9..d6c499d 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,5 @@ target/ # PyCharm .idea/ + +.vagrant/ \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..bb3ec5f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include README.md diff --git a/README.md b/README.md index 03bfceb..f314a8b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,107 @@ stackdio-python-client ====================== -The canonical Python client for the stackd.io API +The canonical Python client and cli for the stackd.io API + + +## Overview +This is a small set of tools for internal use of stackd.io. After cloning +this repo, you should be able to quickly get up and running with your own +stacks. + +Advanced usage like creating custom blueprints or writing your own formulas is +beyond the scope of this. + +## Installation +We recommend using virtualenv via [virtualenvwrapper] to install this in a +virtualenv. If you consider yourself a knowledgeable Pythonista, feel free to +install this however you'd like, but this document will assume that you are +using virtualenvwrapper. See the full [virtualenvwrapper] docs for details, +but in short you can install it on most systems like: + + pip install virtualenvwrapper + +Once you've got it, installing this tool goes something like: + + mkvirtualenv stackdio-tooling + + # assuming you are in whatever dir you cloned this repo to: + pip install --process-dependency-links . + +** The --process-dependency-links flag is only needed in pip 1.5.6 ** +You'll see a few things scrolling by, but should be set after this. To use +this later, you'll need to re-activate the virtualenv like: + + workon stackdio-tooling + +Whenever it's activated, `stackdio-cli` should be on your path. + +## First Use +The first time that you fire up `stackdio-cli`, you'll need to run the +`initial-setup` command. This will prompt you for your LDAP username and +password, and store them securely in your OS keychain for later use. It will +import some standard formula, and create a few commonly used blueprints. + + $ stackdio-cli + None @ https://stackd.corp.digitalreasoning.com/api/ + > initial_setup + # YOU WILL BE WALKED THROUGH A SIMPLE SET OF QUESTIONS + +## Stack Operations +All of the following assume that you have run `initial-setup` successfully. To +launch the cli, simply type: + + $ stackdio-cli + +You can run `help` at any point to see available commands. For details on a +specific command you can run `help COMMAND`, e.g. `help stacks`. The rest of +these commands assume you have the cli running. + +### Launching Stacks +Stacks are launched from blueprints. To launch the 3 node HBase stack that's +included with this you do: + + > stacks launch cdh450-ipa-3 MYSTACKNAME + +**NOTE:** To avoid DNS namespace collisions, the stack name needs to be unique. +An easy way to ensure this is to include your name in the stack name. + +### Deleting Stacks +When you are done with a stack you can delete it. This is destructive and +cannot be recovered from, so think carefully before deleting your stack! + + > stacks delete STACK_NAME + +Alternatively you can `terminate` a stack which will terminate all instances, +but leave the stack definition in place. + +### Provisioning Stacks +Occassionally something will go wrong when launching your stack, e.g. network +connections may flake out causing some package installations to fail. If this +happens you can manually provision your stack, causing everything to be brought +back up to date: + + > stacks provision STACK_NAME + +### Stack Info +Once you have launched a stack, you can then monitor the status of it like: + + > stacks history STACK_NAME + +This displays the top level information for a stack. You can supply additional +arguments to pull back additional info about a stack. For example, to get a +list of FQDNs (aka hostnames) for a stack: + + > stacks hostnames STACK_NAME + +There are various logs available that you can access with the `stacks logs` +command. + +## What's Next? +For anything not covered by this tool, you'll need to use the [web UI] or +[API] directly. See someone on the [pi team] with specific questions. + +[virtualenvwrapper]: https://pypi.python.org/pypi/virtualenvwrapper +[web UI]: https://stackd.corp.digitalreasoning.com +[API]: https://stackd.corp.digitalreasoning.com/api +[pi team]: mailto:pi@digitalreasoning.com?subject=stackd.io%20questions diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..95d6aff --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,13 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.box = "precise64" + + config.vm.provision "shell", path: "vbox_setup.sh" + + config.vm.synced_folder "~/Workspace/pi/stackdio-blueprints", "/home/vagrant/.stackdio-blueprints", create: true +end diff --git a/blueprints/cdh4-3node.json b/blueprints/cdh4-3node.json new file mode 100644 index 0000000..99a16bd --- /dev/null +++ b/blueprints/cdh4-3node.json @@ -0,0 +1,191 @@ +{ + "public": false, + "properties": { + "java": { + "enable_jce": false, + "oracle": { + "cookies": "gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie", + "staging": "/tmp/.java_staging", + "jdk7": { + "rpm": "jdk-1.7.0_45", + "uri": "http://download.oracle.com/otn-pub/java/jdk/7u45-b18/jdk-7u45-linux-x64.rpm" + } + } + }, + "cdh4": { + "landing_page": true, + "hue": { + "secret_key": "CHANGE_THIS", + "start_service": true + }, + "mapred": { + "system_dir": "/hadoop/system/mapred", + "map_tasks_max": 5, + "reduce_tasks_max": 3, + "local_dir": "/mnt/hadoop/mapred/local", + "child_java_opts": "-Xmx2000m", + "reduce_tasks": 6, + "child_ulimit": 8000000 + }, + "zookeeper": { + "data_dir": "/mnt/zk/data", + "start_service": true + }, + "max_log_index": 1, + "dfs": { + "name_dir": "/mnt/hadoop/hdfs/nn", + "block_size": 268435456, + "data_dir": "/mnt/hadoop/hdfs/data", + "permissions": true + }, + "version": 4, + "datanode": { + "start_service": true + }, + "hive": { + "home": "/usr/lib/hive", + "metastore_password": "CHANGE_THIS", + "start_service": true, + "user": "hive" + }, + "io": { + "sort_mb": 250, + "sort_factor": 25 + }, + "oozie": { + "start_service": true + }, + "namenode": { + "start_service": true + }, + "hbase": { + "region_initial_heap": "1024m", + "region_max_heap": "1024m", + "tmp_dir": "/mnt/hbase/tmp", + "start_service": true, + "master_young_gen": "256m", + "replication": 3, + "master_initial_heap": "1024m", + "manage_zk": true, + "log_dir": "/mnt/hbase/logs", + "region_young_gen": "256m", + "master_max_heap": "1024m", + "jute_maxbuffer": 1000000 + }, + "impala": { + "version": "1.2.3", + "start_service": true + } + } + }, + "hosts": [ + { + "count": 1, + "description": "NameNode and Master", + "zone": "us-east-1d", + "title": "NameNode and Master", + "hostname_template": "{namespace}-nn", + "cloud_profile": "TO_BE_CHANGED", + "formula_components": [ + { + "id": [ + "OpenJDK and Oracle Java", + "Oracle Java 7 JDK" + ], + "order": 0 + }, + { + "id": [ + "CDH4 Salt Formula", + "NameNode" + ], + "order": 1 + }, + { + "id": [ + "CDH4 Salt Formula", + "HBase Master" + ], + "order": 3 + }, + { + "id": [ + "CDH4 Salt Formula", + "Oozie" + ], + "order": 5 + }, + { + "id": [ + "CDH4 Salt Formula", + "Pig" + ], + "order": 6 + }, + { + "id": [ + "CDH4 Salt Formula", + "Hive" + ], + "order": 7 + }, + { + "id": [ + "CDH4 Salt Formula", + "Impala State Store" + ], + "order": 8 + }, + { + "id": [ + "CDH4 Salt Formula", + "Hue" + ], + "order": 10 + } + ], + "size": "m2.2xlarge" + }, + { + "count": 2, + "description": "DataNodes and Regionservers.", + "zone": "us-east-1d", + "title": "DataNodes and RegionServers", + "hostname_template": "{namespace}-dn-{index}", + "cloud_profile": "TO_BE_CHANGED", + "formula_components": [ + { + "id": [ + "OpenJDK and Oracle Java", + "Oracle Java 7 JDK" + ], + "order": 0 + }, + { + "id": [ + "CDH4 Salt Formula", + "DataNode" + ], + "order": 2 + }, + { + "id": [ + "CDH4 Salt Formula", + "HBase RegionServer" + ], + "order": 4 + }, + { + "id": [ + "CDH4 Salt Formula", + "Impala Server" + ], + "order": 9 + } + ], + "size": "m2.2xlarge" + } + ], + "description": "3 node stack with CDH 4", + "title": "cdh4-3node" +} \ No newline at end of file diff --git a/blueprints/cdh5-3node.json b/blueprints/cdh5-3node.json new file mode 100644 index 0000000..597441e --- /dev/null +++ b/blueprints/cdh5-3node.json @@ -0,0 +1,192 @@ +{ + "public": false, + "properties": { + "java": { + "enable_jce": false, + "oracle": { + "cookies": "gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie", + "staging": "\/tmp\/.java_staging", + "jdk7": { + "rpm": "jdk-1.7.0_45", + "uri": "http:\/\/download.oracle.com\/otn-pub\/java\/jdk\/7u45-b18\/jdk-7u45-linux-x64.rpm" + } + } + }, + "cdh5": { + "hue": { + "secret_key": "CHANGE_THIS", + "start_service": true + }, + "dfs": { + "name_dir": "\/mnt\/hadoop\/hdfs\/nn", + "data_dir": "\/mnt\/hadoop\/hdfs\/data", + "permissions": true + }, + "mapred": { + "local_dir": "\/mnt\/yarn", + "system_dir": "\/hadoop\/system\/mapred", + "reduce_tasks": 6 + }, + "zookeeper": { + "data_dir": "\/mnt\/zk\/data", + "start_service": true + }, + "max_log_index": 1, + "hive": { + "home": "\/usr\/lib\/hive", + "metastore_password": "CHANGE_THIS", + "start_service": true, + "user": "hive" + }, + "io": { + "sort_mb": 250, + "sort_factor": 25 + }, + "oozie": { + "start_service": true + }, + "namenode": { + "start_service": true + }, + "hbase": { + "region_initial_heap": "1024m", + "region_max_heap": "1024m", + "tmp_dir": "\/mnt\/hbase\/tmp", + "start_service": true, + "master_young_gen": "256m", + "replication": 3, + "master_initial_heap": "1024m", + "manage_zk": true, + "log_dir": "\/mnt\/hbase\/logs", + "region_young_gen": "256m", + "master_max_heap": "1024m", + "jute_maxbuffer": 1000000 + }, + "landing_page": true, + "yarn": { + "max_container_size_mb": 11264 + }, + "version": 5, + "datanode": { + "start_service": true + }, + "security": { + "enable": false + }, + "impala": { + "version": "1.2.3", + "start_service": true + } + } + }, + "hosts": [ + { + "count": 1, + "description": "NameNode and Master", + "zone": "us-east-1d", + "title": "NameNode and Master", + "hostname_template": "{namespace}-nn", + "cloud_profile": "TO_BE_CHANGED", + "formula_components": [ + { + "id": [ + "OpenJDK and Oracle Java", + "Oracle Java 7 JDK" + ], + "order": 0 + }, + { + "id": [ + "CDH5 Salt Formula", + "NameNode" + ], + "order": 1 + }, + { + "id": [ + "CDH5 Salt Formula", + "HBase Master" + ], + "order": 3 + }, + { + "id": [ + "CDH5 Salt Formula", + "Oozie" + ], + "order": 5 + }, + { + "id": [ + "CDH5 Salt Formula", + "Pig" + ], + "order": 6 + }, + { + "id": [ + "CDH5 Salt Formula", + "Hive" + ], + "order": 7 + }, + { + "id": [ + "CDH5 Salt Formula", + "Impala State Store" + ], + "order": 8 + }, + { + "id": [ + "CDH5 Salt Formula", + "Hue" + ], + "order": 10 + } + ], + "size": "m2.2xlarge" + }, + { + "count": 2, + "description": "DataNodes and Regionservers.", + "zone": "us-east-1d", + "title": "DataNodes and RegionServers", + "hostname_template": "{namespace}-dn-{index}", + "cloud_profile": "TO_BE_CHANGED", + "formula_components": [ + { + "id": [ + "OpenJDK and Oracle Java", + "Oracle Java 7 JDK" + ], + "order": 0 + }, + { + "id": [ + "CDH5 Salt Formula", + "DataNode" + ], + "order": 2 + }, + { + "id": [ + "CDH5 Salt Formula", + "HBase RegionServer" + ], + "order": 4 + }, + { + "id": [ + "CDH5 Salt Formula", + "Impala Server" + ], + "order": 9 + } + ], + "size": "m2.2xlarge" + } + ], + "description": "3 node stack with CDH 5", + "title": "cdh5-3node" +} \ No newline at end of file diff --git a/bootstrap.yaml b/bootstrap.yaml new file mode 100644 index 0000000..16d892b --- /dev/null +++ b/bootstrap.yaml @@ -0,0 +1,8 @@ +blueprints: + cdh4-3node: cdh4-3node.json + cdh5-3node: cdh5-3node.json + +formulas: + java: https://github.com/stackdio-formulas/java-formula.git + cdh4: https://github.com/stackdio-formulas/cdh4-formula.git + cdh5: https://github.com/stackdio-formulas/cdh5-formula.git diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..79bc678 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[bdist_wheel] +# This flag says that the code is written to work on both Python 2 and Python +# 3. If at all possible, it is good practice to do this. If you cannot, you +# will need to generate wheels for each Python version that you support. +universal=1 diff --git a/setup.py b/setup.py index 043363e..fd2599c 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ # limitations under the License. # +import os import sys from setuptools import setup, find_packages @@ -42,9 +43,15 @@ def test_python_version(): with open('README.md') as f: LONG_DESCRIPTION = f.read() +CFG_DIR = os.path.expanduser("~/.stackdio-cli") + requirements = [ - 'simplejson', + 'Jinja2==2.7.3', + 'PyYAML==3.11', + 'cmd2==0.6.7', + 'keyring==3.7', 'requests>=2.4.0,<2.6.0', + 'simplejson==3.4.0', ] if __name__ == "__main__": @@ -63,9 +70,23 @@ def test_python_version(): license='Apache 2.0', include_package_data=True, packages=find_packages(), + data_files=[ + (CFG_DIR, + [ + 'bootstrap.yaml', + ]), + (os.path.join(CFG_DIR, "blueprints"), + ["blueprints/%s" % f for f in os.listdir("blueprints")]), + ], zip_safe=False, install_requires=requirements, dependency_links=[], + entry_points={ + 'console_scripts': [ + 'stackdio-cli=stackdio.cli:main', + 'blueprint-generator=stackdio.blueprints:main', + ], + }, classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: Web Environment', diff --git a/stackdio/blueprints/__init__.py b/stackdio/blueprints/__init__.py new file mode 100644 index 0000000..1e14fbc --- /dev/null +++ b/stackdio/blueprints/__init__.py @@ -0,0 +1,56 @@ +import argparse +import os +import json +import sys + +from stackdio.blueprints.generator import BlueprintException, BlueprintGenerator + + +def main(): + parser = argparse.ArgumentParser( + description='invoke the stackdio blueprint generator') + + parser.add_argument('template_file', + help='The template file to generate from') + + parser.add_argument('var_files', + metavar='var_file', + nargs='*', + help='The variable files with your custom config. They will be loaded ' + 'from left to right, so variables in the rightmost var files will ' + 'override those in var files to the left.') + + parser.add_argument('-p', '--prompt', + action='store_true', + help='Prompt user for missing variables') + + parser.add_argument('-d', '--debug', + action='store_true', + help='Print out json string before parsing the json') + + args = parser.parse_args() + + try: + # Throw all output to stderr + gen = BlueprintGenerator([os.path.curdir, + os.path.join(os.path.curdir, 'templates'), + os.path.dirname(os.path.abspath(args.template_file))], + output_stream=sys.stderr) + + # Generate the blueprint + blueprint = gen.generate(args.template_file, + var_files=args.var_files, + prompt=args.prompt, + debug=args.debug) + except BlueprintException: + sys.exit(1) + + print(json.dumps(blueprint, indent=2)) + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + sys.stderr.write('Aborting...\n') + sys.exit(1) \ No newline at end of file diff --git a/stackdio/blueprints/generator.py b/stackdio/blueprints/generator.py new file mode 100644 index 0000000..5bb2acf --- /dev/null +++ b/stackdio/blueprints/generator.py @@ -0,0 +1,305 @@ +import sys +import os +import json + +import yaml +from jinja2 import Environment, FileSystemLoader, StrictUndefined, meta +from jinja2.exceptions import TemplateNotFound, TemplateSyntaxError, UndefinedError +from jinja2.nodes import Assign, Block, Const, If, Not +from jinja2.filters import do_replace, evalcontextfilter + + +COLORS = { + 'brown': '\033[0;33m', + 'green': '\033[0;32m', + 'red': '\033[01;31m', + 'endc': '\033[0m', +} + + +class BlueprintException(Exception): + pass + + +class BlueprintGenerator(object): + """ + Blueprint generator class. Uses Jinja2 to generate blueprints from JSON + templates, with inheritance. + """ + + def __init__(self, templates_path, output_stream=sys.stdout): + """ + Need to create the jinja2 environment + + :param templates_path: A list of directories in which to look for templates + :return: + """ + self.settings = { + 'template_dir': os.path.expanduser('~/.stackdio-blueprints/templates') + } + + self.sentinel = object() + + self.out_stream = output_stream + + templates_path.append(self.settings['template_dir']) + + self.env = Environment( + loader=FileSystemLoader(templates_path), + undefined=StrictUndefined) + + # Add a filter for json - then we can put lists, etc in our templates + self.env.filters['json'] = lambda value: json.dumps(value) + + # Add a filter for newline escaping. Essentially just a wrapper around the + # replace('\n', '\\n') filter + # Need the evalcontext filter decorator b/c do_replace uses it too and needs a ctx arg + self.env.filters['longstring'] = evalcontextfilter( + lambda ctx, s: do_replace(ctx, s, '\n', '\\n') + ) + + def error_exit(self, message, newlines=1): + """ + Prints an error message in red and exits with an error code + :param message: The error message + :param newlines: the number of newlines to print at the end + :return: None + """ + self.out_stream.write('{0}{1}{2}'.format(COLORS['red'], message, COLORS['endc'])) + for i in range(0, newlines): + self.out_stream.write('\n') + raise BlueprintException() + + def warning(self, message, newlines=1): + """ + Prints a warning message in brown + :param message: The warning message to print + :param newlines: The number of newlines to print at the end + :return: None + """ + self.out_stream.write('{0}{1}{2}'.format(COLORS['brown'], message, COLORS['endc'])) + for i in range(0, newlines): + self.out_stream.write('\n') + + def prompt(self, message): + """ + Prompts the user for an input. Prints the prompt to the configured output stream in green + :param message: the prompt message + :return: the value the user inputted + """ + self.out_stream.write('{0}{1}{2}'.format(COLORS['green'], message, COLORS['endc'])) + raw = sys.stdin.readline().strip() + + # This should work nicely - if yaml can't parse it properly, then it should be fine to just + # return the raw string + try: + yaml_parsed = yaml.safe_load(raw) + + # safe_load returns None if the input is the empty string, so we want to put it back + # to the empty string + if yaml_parsed is None: + return '' + else: + return yaml_parsed + except: + # yaml couldn't parse it + return raw + + def find_set_vars(self, ast): + """ + We need this due to the fact that jinja DOES NOT allow you to override + variables in base templates from derived templates. This pulls out the + assignments in derived templates so that we can set them in the context + later. + + :param ast: the jinja2 parsed abstract syntax tree + :return: the dict of all set variables + :rtype: dict + """ + ret = {} + + # TODO maybe use a set here. I had originally used a dict because set tags in derived + # templates would not get picked up by base templates. The Jinja2 documentation even says + # that this is the case. So I was going to pick out the values from the set statements and + # pass them in under the template context so they would get picked up by base templates. + # + # As soon as I took the time to write this whole function to do the above, I ran the + # generator again, and my set statements in derived templates were getting picked up by the + # base, and I have no clue why. So none of the values in this dict are getting utilized, + # just the keys. + # + # EDIT: Turns out we do need the values. When you do a template include, those values are + # not allowed to be set by the template that did the include. + + for tag in ast.body: + if type(tag) == Assign: + if type(tag.node) == Const: + ret[tag.target.name] = tag.node.value + else: + # This is OK - RHS is an expr instead of a constant. Keep track of these so we + # don't get an error later + ret[tag.target.name] = self.sentinel + + elif type(tag) == If: + # This is a simple naive check to see if we care about the variable being set. + # Basically, if there is a variable inside an "if is not undefined" block, + # It is OK if that variable is not set in a derived template or var file since it + # is an optional configuration value + if type(tag.test) == Not and tag.test.node.name == 'undefined': + ret[tag.test.node.node.name] = None + + elif type(tag) == Block: + # Found a block, could be if statements within the block. Recursive call + # TODO: not sure if this will cause a bug, since set statements are local to blocks + ret.update(self.find_set_vars(tag)) + + return ret + + def validate(self, template_file): + """ + Find all available and overridden vars in a template. Recursively checks all + super templates. + + :param template_file: The name of the template file + :return: the set and unset variables + :rtype: tuple + """ + ### Get all the info for the CURRENT template + # Get the source of the template + template_source = self.env.loader.get_source(self.env, template_file)[0] + # parse it into an abstract syntax tree + ast = self.env.parse(template_source) + + # the UNSET variables in the current template + unset_vars = meta.find_undeclared_variables(ast) + + # the SET variables in the current template + set_vars = self.find_set_vars(ast) + + # validate the super templates + super_templates = meta.find_referenced_templates(ast) + + for template in super_templates: + # Get all the information about the super template recursively + super_unset, super_set = self.validate(template) + + # We do it this way so values in derived templates override those in base templates + super_set.update(set_vars) + set_vars = super_set + + unset_vars = unset_vars.union(super_unset) + + return unset_vars, set_vars + + def generate(self, template_file, var_files=(), variables=None, prompt=False, debug=False): + """ + Generate the rendered blueprint and return it as a python dict + + The var_file is loaded first. Anything in the variables dict *WILL OVERRIDE* the value in + the var_vile. + + :param template_file: The relative location of the template. It must be in one of the + directories you specified when creating the Generator object. + :param var_file: The location of the variable file (relative or absolute) + :param variables: A dict of variables to put in the template. + :param prompt: Option to prompt for missing variables + :param debug: Print the output of the template before trying to parse it as JSON + :return: the generated blueprint object + :rtype: dict + """ + try: + # Validate the template + all_vars, set_vars = self.validate(template_file) + + context = {} + for var_file in var_files: + yaml_parsed = yaml.safe_load(open(var_file, 'r')) + if yaml_parsed: + context.update(yaml_parsed) + + # Add in the variables + if variables: + context.update(variables) + + # Find the null variables in the var file + null_vars = set() + + for name, value in context.items(): + if value is None: + null_vars.add(name) + context[name] = '' + + # the missing vars should be the subset of all the variables + # with the set of set variables and set of context variables taken + # out + missing_vars = all_vars - set(set_vars) - set(context) + + if missing_vars: + if prompt: + # Prompt for missing vars + for var in sorted(missing_vars): + context[var] = self.prompt('{0}: '.format(var)) + else: + # Print an error + error_str = 'Missing variables:\n' + for var in sorted(missing_vars): + error_str += ' {0}\n'.format(var) + self.error_exit(error_str, 0) + + # Find the set of optional variables (ones inside of a 'if is not undefined' + # block). They were set to None in the set_vars dict inside the validate method + optional_vars = set() + + for var, val in set_vars.items(): + if val is None: + optional_vars.add(var) + # Need to get rid of this now so it doesn't cause problems later + del set_vars[var] + elif val == self.sentinel: + # These are valid assignments, but we don't need to throw them in to the context + del set_vars[var] + + # If it is set elsewhere, it's not an issue + optional_vars = optional_vars - set(context) + + if null_vars: + warn_str = '\nWARNING: Null variables (replaced with empty string):\n' + for var in null_vars: + warn_str += ' {0}\n'.format(var) + self.warning(warn_str, 0) + + # Print a warning if there's unset optional variables + if optional_vars: + warn_str = '\nWARNING: Missing optional variables:\n' + for var in sorted(optional_vars): + warn_str += ' {0}\n'.format(var) + self.warning(warn_str, 0) + + # Generate the blueprint + template = self.env.get_template(template_file) + + # Put the set vars into the context + set_vars.update(context) + context = set_vars + + template_json = template.render(**context) + + if debug: + print('\n') + print(template_json) + print('\n') + + # Return a dict object rather than a string + return json.loads(template_json) + + except TemplateNotFound: + self.error_exit('Your template file {0} was not found.'.format(template_file)) + except TemplateSyntaxError, e: + self.error_exit('Invalid template error at line {0}:\n{1}'.format( + e.lineno, + str(e) + )) + except UndefinedError, e: + self.error_exit('Missing variable: {0}'.format(str(e))) + except ValueError: + self.error_exit('Invalid JSON. Check your template file.') diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py new file mode 100644 index 0000000..a6d2512 --- /dev/null +++ b/stackdio/cli/__init__.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python + +__version__ = '0.2.dev' + +import argparse +import json +import os +import sys + +from cmd2 import Cmd +import keyring +from requests import ConnectionError + + +from stackdio.client import StackdIO + +from stackdio.cli import mixins + + +class StackdioShell( + Cmd, + mixins.bootstrap.BootstrapMixin, + mixins.stacks.StackMixin, + mixins.formulas.FormulaMixin, + mixins.blueprints.BlueprintMixin): + + CFG_DIR = os.path.expanduser("~/.stackdio-cli/") + CFG_FILE = os.path.join(CFG_DIR, "config.json") + BOOTSTRAP_FILE = os.path.join(CFG_DIR, "bootstrap.yaml") + KEYRING_SERVICE = "stackdio_cli" + PROMPT = "\n{username} @ {url}\n> " + HELP_CMDS = [ + "account_summary", + "stacks", "blueprints", "formulas", + "initial_setup", "bootstrap", + "help", "exit", "quit", + ] + + Cmd.intro = """ +###################################################################### + s t a c k d . i o +###################################################################### +""" + + def __init__(self): + Cmd.__init__(self) + mixins.bootstrap.BootstrapMixin.__init__(self) + self._load_config() + if 'url' in self.config and self.config['url']: + self._init_stacks() + self._validate_auth() + + def preloop(self): + self._setprompt() + + def precmd(self, line): + self._setprompt() + return line + + def postloop(self): + print("\nGoodbye!") + + def get_names(self): + if self.validated: + return ["do_%s" % c for c in self.HELP_CMDS] + else: + return ["do_initial_setup", "do_help"] + + def _init_stacks(self): + """Instantiate a StackdIO object""" + self.stacks = StackdIO( + base_url=self.config["url"], + auth=( + self.config["username"], + keyring.get_password(self.KEYRING_SERVICE, self.config.get("username") or "") + ), + verify=self.config.get('verify', True) + ) + + def _load_config(self): + """Attempt to load config file, otherwise fallback to DEFAULT_CONFIG""" + + try: + self.config = json.loads(open(self.CFG_FILE).read()) + self.config['blueprint_dir'] = os.path.expanduser(self.config.get('blueprint_dir', '')) + + except ValueError: + print(self.colorize( + "What happened?! The config file is not valid JSON. A " + "re-install is likely the easiest fix.", "red")) + raise + except IOError: + self.config = { + 'url': None, + 'username': None, + } + print(self.colorize( + "It seems like this is your first time using the CLI. Please run " + "'initial_setup' to configure.", "green")) + # print(self.colorize( + # "What happened?! Unable to find a config file. A re-install " + # "is likely the easiest fix.", "red")) + # raise + + self.has_public_key = None + + def _validate_auth(self): + """Verify that we can connect successfully to the api""" + + # If there is no config, just force the user to go through initial setup + if self.config['url'] is None: + return + + try: + self.stacks.get_root() + status_code = 200 + self.validated = (200 <= status_code <= 299) + except ConnectionError: + print(self.colorize( + "Unable to connect to {0}".format(self.config["url"]), + "red")) + raise + + if self.validated: + print(self.colorize( + "Config loaded and validated", "blue")) + self.has_public_key = self.stacks.get_public_key() + else: + print(self.colorize( + "ERROR: Unable to validate config", "red")) + self.has_public_key = None + + def _setprompt(self): + + Cmd.prompt = self.colorize( + self.PROMPT.format(**self.config), + "blue") + + if not self.validated and self.config['url'] is not None: + print(self.colorize(""" +## +## Unable to validate connection - one of several possibilities exist: +## If this is the first time you've fired this up, you need to run +## 'initial_setup' to configure your account details. If you've already +## done that, there could be a network connection issue anywhere between +## your computer and your stackd.io instance, +## or your password may be incorrect, or ... etc. +## + """, + "green")) + + if self.validated and not self.has_public_key: + print(self.colorize( + "## Your account is missing the public key, run 'bootstrap' to fix", + "red")) + + def _print_summary(self, title, components): + num_components = len(components) + print("## {0} {1}{2}".format( + num_components, + title, + "s" if num_components == 0 or num_components > 1 else "")) + + for item in components: + print("- Title: {0}\n Description: {1}".format( + item.get("title"), item.get("description"))) + + if "status_detail" in item: + print(" Status Detail: {0}\n".format( + item.get("status_detail"))) + else: + print("") + + def do_account_summary(self, args=None): + """Get a summary of your account.""" + sys.stdout.write("Polling {0} ... ".format(self.config["url"])) + sys.stdout.flush() + + public_key = self.stacks.get_public_key() + formulas = self.stacks.list_formulas() + blueprints = self.stacks.list_blueprints() + stacks = self.stacks.list_stacks() + + sys.stdout.write("done\n") + + print("## Username: {0}".format(self.config["username"])) + print("## Public Key:\n{0}".format(public_key)) + + self._print_summary("Formula", formulas) + self._print_summary("Blueprint", blueprints) + self._print_summary("Stack", stacks) + + +def main(): + + parser = argparse.ArgumentParser( + description="Invoke the stackdio cli") + parser.add_argument("--debug", action="store_true", help="Enable debugging output") + args = parser.parse_args() + + # an ugly hack to work around the fact that cmd2 is using optparse to parse + # arguments for the commands; not sure what the "right" fix is, but as long + # as we assume that we don't want any of our arguments to get passed into + # the cmdloop this seems ok + sys.argv = sys.argv[0:1] + + shell = StackdioShell() + if args.debug: + shell.debug = True + shell.cmdloop() + + +if __name__ == '__main__': + main() diff --git a/stackdio/cli/mixins/__init__.py b/stackdio/cli/mixins/__init__.py new file mode 100644 index 0000000..ecb06b5 --- /dev/null +++ b/stackdio/cli/mixins/__init__.py @@ -0,0 +1,4 @@ +import stackdio.cli.mixins.blueprints +import stackdio.cli.mixins.bootstrap +import stackdio.cli.mixins.formulas +import stackdio.cli.mixins.stacks diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py new file mode 100644 index 0000000..a419aca --- /dev/null +++ b/stackdio/cli/mixins/blueprints.py @@ -0,0 +1,270 @@ +import json +import os +import argparse +import sys + +import yaml +from cmd2 import Cmd + +from stackdio.client.exceptions import StackException +from stackdio.blueprints.generator import BlueprintGenerator, BlueprintException + + +class BlueprintNotFound(Exception): + pass + + +class BlueprintMixin(Cmd): + BLUEPRINT_COMMANDS = ["list", "list-templates", "create", "create-all", "delete", "delete-all"] + + def do_blueprints(self, arg): + """Entry point to controlling blueprints.""" + + USAGE = "Usage: blueprints COMMAND\nWhere COMMAND is one of: %s" % ( + ", ".join(self.BLUEPRINT_COMMANDS)) + + args = arg.split() + if not args or args[0] not in self.BLUEPRINT_COMMANDS: + print(USAGE) + return + + bp_cmd = args[0] + + # Sneakiness for argparse + saved = sys.argv[0] + sys.argv[0] = 'blueprints {0}'.format(bp_cmd) + + if bp_cmd == "list": + self._list_blueprints() + elif bp_cmd == "list-templates": + self._list_templates() + elif bp_cmd == "create": + self._create_blueprint(args[1:]) + elif bp_cmd == "create-all": + self._create_all(args[1:]) + elif bp_cmd == "delete": + self._delete_blueprint(args[1:]) + elif bp_cmd == "delete-all": + self._delete_all() + + else: + print(USAGE) + + # End sneakiness + sys.argv[0] = saved + + def complete_blueprints(self, text, line, begidx, endidx): + # not using line, begidx, or endidx, thus the following pylint disable + # pylint: disable=W0613 + return [i for i in self.BLUEPRINT_COMMANDS if i.startswith(text)] + + def help_blueprints(self): + print("Manage blueprints.") + print("Sub-commands can be one of:\n\t{0}".format( + ", ".join(self.BLUEPRINT_COMMANDS))) + print("Try 'blueprints COMMAND' to get help on (most) sub-commands") + + def _list_blueprints(self): + """List all blueprints""" + + print("Getting blueprints ... ") + blueprints = self.stacks.list_blueprints() + self._print_summary("Blueprint", blueprints) + + def _recurse_dir(self, dirname, extensions, prefix=''): + for template in os.listdir(dirname): + if os.path.isdir(os.path.join(dirname, template)): + # Recursively look at the subdirectories + self._recurse_dir(os.path.join(dirname, template), + extensions, + prefix + template + os.sep) + elif template.split('.')[-1] in extensions and not template.startswith('_'): + print(' {0}'.format(prefix + template)) + + def _list_templates(self): + if 'blueprint_dir' not in self.config: + print("Missing blueprint directory config") + return + + blueprint_dir = os.path.expanduser(self.config['blueprint_dir']) + + print('Template mappings:') + mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) + if mapping: + for blueprint in mapping: + print(' {0}'.format(blueprint)) + + print('') + + print('Templates:') + self._recurse_dir(os.path.join(blueprint_dir, 'templates'), ['json']) + + print('') + + print('Var files:') + self._recurse_dir(os.path.join(blueprint_dir, 'var_files'), ['yaml', 'yml']) + + def _create_blueprint(self, args, bootstrap=False): + """Create a blueprint""" + + parser = argparse.ArgumentParser() + + parser.add_argument('-m', '--mapping', + help='The entry in the map file to use') + + parser.add_argument('-t', '--template', + help='The template file to use') + + parser.add_argument('-v', '--var-file', + action='append', + help='The variable files to use. You may pass in more than one. They ' + 'will be loaded from left to right, so variables in the rightmost ' + 'var files will override those in var files to the left.') + + parser.add_argument('-n', '--no-prompt', + action='store_false', + help='Don\'t prompt for missing variables in the template') + + args = parser.parse_args(args) + + if not bootstrap: + print(self.colorize( + "Advanced users only - use the web UI if this isn't you!\n", + "green")) + + if not args.template and not args.mapping: + print(self.colorize('You must specify either a template or a mapping\n', 'red')) + parser.print_help() + return + + blueprint_dir = os.path.expanduser(self.config['blueprint_dir']) + + template_file = args.template + # Should always be a list, and the generator can handle that + var_files = args.var_file + if not var_files: + # If -v is never specified, argparse give back None, we need a list + var_files = [] + + if args.mapping: + mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) + if not mapping or args.mapping not in mapping: + print(self.colorize('You gave an invalid mapping.', 'red')) + return + else: + template_file = mapping[args.mapping].get('template') + var_files = mapping[args.mapping].get('var_files', []) + if not template_file: + print(self.colorize('Your mapping must specify a template.', 'red')) + return + + bp_json = self._create_single(template_file, var_files, args.no_prompt) + + if not bp_json: + # There was an error with the blueprint creation, and there should already be an + # error message printed + return + + if not bootstrap: + print("Creating blueprint") + + r = self.stacks.create_blueprint(bp_json) + print(json.dumps(r, indent=2)) + + def _create_single(self, template_file, var_files, no_prompt): + blueprint_dir = os.path.expanduser(self.config['blueprint_dir']) + + gen = BlueprintGenerator([os.path.join(blueprint_dir, 'templates')]) + + if not os.path.exists(os.path.join(blueprint_dir, 'templates', template_file)): + print(self.colorize('You gave an invalid template', 'red')) + return + + if template_file.startswith('_'): + print(self.colorize("WARNING: Templates beginning with '_' are generally not meant to " + "be used directly. Please be sure this is really what you want.\n", + "magenta")) + + final_var_files = [] + + # Build a list with full paths in it instead of relative paths + for var_file in var_files: + var_file = os.path.join(blueprint_dir, 'var_files', var_file) + if os.path.exists(var_file): + final_var_files.append(var_file) + else: + print(self.colorize("WARNING: Variable file {0} was not found. " + "Ignoring.".format(var_file), "magenta")) + + # Generate the JSON for the blueprint + return gen.generate(template_file, + final_var_files, # Pass in a list + prompt=no_prompt) + + def _create_all(self, args): + """Create all the blueprints in the map file""" + parser = argparse.ArgumentParser() + + parser.add_argument('--no-prompt', + action='store_false', + help='Don\'t prompt to create all blueprints') + + args = parser.parse_args(args) + + if args.no_prompt: + really = raw_input("Really create all blueprints (y/n)? ") + if really not in ['Y', 'y']: + return + + blueprint_dir = os.path.expanduser(self.config['blueprint_dir']) + mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) + + for name, vals in mapping.items(): + try: + bp_json = self._create_single(vals['template'], vals['var_file'], False) + self.stacks.create_blueprint(bp_json) + print(self.colorize('Created blueprint {0}'.format(name), 'green')) + except BlueprintException: + print(self.colorize('Blueprint {0} NOT created\n'.format(name), 'magenta')) + + def _delete_blueprint(self, args): + """Delete a blueprint""" + + if len(args) != 1: + print("Usage: blueprint delete BLUEPRINT_NAME") + return + + blueprint_id = self._get_blueprint_id(args[0]) + + really = raw_input("Really delete blueprint {0} (y/n)? ".format(args[0])) + if really not in ["y", "Y"]: + print("Aborting deletion") + return + + print("Deleting {0}".format(args[0])) + self.stacks.delete_blueprint(blueprint_id) + self._list_blueprints() + + def _delete_all(self): + """Delete all blueprints""" + really = raw_input("Really delete all blueprints? This is completely destructive, and you " + "will never get them back. (y/n) ") + if really not in ['Y', 'y']: + return + + for blueprint in self.stacks.list_blueprints(): + self.stacks.delete_blueprint(blueprint['id']) + print(self.colorize('Deleted blueprint {0}'.format(blueprint['title']), 'magenta')) + + def _get_blueprint_id(self, blueprint_name): + """Validate that a blueprint exists""" + + try: + return self.stacks.get_blueprint_id(blueprint_name) + except StackException: + print(self.colorize( + "Blueprint [{0}] does not exist".format(blueprint_name), + "red")) + raise + + diff --git a/stackdio/cli/mixins/bootstrap.py b/stackdio/cli/mixins/bootstrap.py new file mode 100644 index 0000000..3bae0ff --- /dev/null +++ b/stackdio/cli/mixins/bootstrap.py @@ -0,0 +1,324 @@ +import getpass +import json +import os +import sys + +import requests +from requests import ConnectionError +from requests.exceptions import MissingSchema +from cmd2 import Cmd +import keyring +import yaml + +from stackdio.cli.polling import poll_and_wait, TimeoutException + + +class PublicKeyNotFound(Exception): + pass + + +class BootstrapMixin(Cmd): + + def __init__(self): + + # quieting down pylint + self.has_public_key = None + self.validated = False + self.stacks = None + self.config = None + self.bootstrap_data = None + + def do_initial_setup(self, args=None): + """Perform setup for your stackd.io account""" + + print("Performing initial setup") + special_config = raw_input("Do you have your own config.json file you would like to use (y/n)? ") + + config_from_file = special_config in ['Y', 'y'] + + if config_from_file: + # Prompt for the filename where the config is located + config_file = raw_input("Where is the file located? ") + + # Load the config file + if os.path.exists(config_file): + config = json.load(open(config_file, "r")) + elif os.path.exists(os.path.expanduser(config_file)): + config = json.load(open(os.path.expanduser(config_file), "r")) + else: + print("Unable to find the file.") + return + + # Put the config file contents into the config object + for k, v in config.iteritems(): + self.config[k] = v + + # Validate the url, prompt for a new one if invalid + if not self.config.has_key('url') or not self._test_url(self.config['url']): + print("There seems to be an issue with the url you provided.") + self.config['url'] = None + self._get_url() + + else: + # No config file, just prompt individually + self._get_url() + + self._get_user_creds() + self._init_stacks() + self._validate_auth() + if not self.validated: + return + + if not config_from_file and 'profile' in self.config: + keep_profile = raw_input("Would you like to keep your current default profile (y/n)? ") + + if keep_profile in ['N', 'n']: + self._choose_profile() + + # Only prompt for default profile if it's not already there + if not self.config.has_key('profile') \ + or 'provider' not in self.config \ + or 'provider_type' not in self.config: + if 'profile' in self.config: + print("Profile misconfiguration detected.") + self._choose_profile() + + get_dir = not config_from_file + + if not config_from_file and 'blueprint_dir' in self.config: + keep_dir = raw_input("Would you like to keep your current blueprint directory (y/n)? ") + + if keep_dir not in ['N', 'n']: + get_dir = False + + if get_dir: + new_dir = raw_input("Enter the path of your blueprint templates: ") + self.config['blueprint_dir'] = new_dir + + self._save_config() + self._setprompt() + + if self.validated: + bootstrap = raw_input("Bootstrap your account now (y/n)? ") + if bootstrap not in ["y", "Y"]: + return + self.do_bootstrap() + + print(self.colorize(""" +## +## Success! You're ready to start using stackd.io. Try running +## 'help' to see what all is available here. You can also go to +## {0} to use the UI. +## + """.format( + self.config["url"][0:self.config["url"].find("api/")]), + "green")) + + else: + print(self.colorize( + "Unable to bootstrap your account", + "red")) + return + + def do_bootstrap(self, args=None): + """Bootstrap an account with predefined formulas and blueprints""" + + args = args or [] + + if not self.validated: + print(self.colorize( + "You must run 'initial_setup' before you can bootstrap", + "red")) + return + + if not self.config.has_key('profile'): + print(self.colorize("You must have a default profile in order to run bootstrap. Run 'initial_setup'", + "red")) + return + + print("Bootstrapping your account") + + custom_bootstrap = raw_input("Do you have a custom bootstrap yaml file (y/n)? ") + + if custom_bootstrap in ['Y', 'y']: + custom_bootstrap_file = raw_input("Enter the name of the file: ") + + # Load the bootstrap file + if os.path.exists(custom_bootstrap_file): + self.BOOTSTRAP_FILE = custom_bootstrap_file + elif os.path.exists(os.path.expanduser(custom_bootstrap_file)): + self.BOOTSTRAP_FILE = os.path.expanduser(custom_bootstrap_file) + else: + print("Unable to find the file.") + use_default = raw_input("Would you like to use the default bootstrap file instead (y/n)? ") + if use_default not in ['Y', 'y']: + print("Aborting bootstrap") + return + + # Load the bootstrap data. If the BOOTSTRAP_DATA property was not set just now, it will use the default + self.bootstrap_data = yaml.safe_load(open(self.BOOTSTRAP_FILE).read()) + + self._bootstrap_account() + self._bootstrap_formulas() + self._bootstrap_blueprints() + + def _test_url(self, url): + try: + r = requests.get(url, verify=self.config.get('verify', True)) + return (200 <= r.status_code < 300) or r.status_code == 403 + except ConnectionError: + return False + except MissingSchema: + print("You might have forgotten http:// or https://") + return False + + def _get_url(self): + """Prompt user for url""" + + if self.config['url'] is not None: + keep_url = raw_input("Keep existing url (y/n)? ") + if keep_url not in ["n", "N"]: + return + + verify = raw_input("Does your stackd.io server have a self-signed SSL certificate (y/n)? ") + if verify in ('Y', 'y'): + self.config['verify'] = False + else: + self.config['verify'] = True + + self.config['url'] = None + + while self.config['url'] is None: + url = raw_input("What is the URL of your stackd.io server? ") + if url.endswith('api'): + url += '/' + elif url.endswith('api/'): + pass + elif url.endswith('/'): + url += 'api/' + else: + url += '/api/' + if self._test_url(url): + self.config['url'] = url + else: + print("There was an error while attempting to contact that server. Try again.") + + def _get_user_creds(self): + """Prompt user for credentials""" + + self.config["username"] = raw_input("What is your username? ") + + if keyring.get_password(self.KEYRING_SERVICE, self.config["username"]): + print("Password already stored for {0}".format(self.config["username"])) + keep_password = raw_input("Keep existing password (y/n)? ") + else: + keep_password = "n" + + if keep_password in ["n", "N"]: + password = getpass.getpass("What is your password? ") + keyring.set_password(self.KEYRING_SERVICE, + self.config["username"], + password) + + def _choose_profile(self): + """Prompt user for a default provider/profile""" + auth = (self.config['username'], + keyring.get_password(self.KEYRING_SERVICE, self.config['username'])) + profiles = requests.get(self.config['url']+"profiles/", auth=auth, verify=False).json()['results'] + + print("Choose a default profile:") + + idx = 0 + for profile in profiles: + print(str(idx)+':') + print(' '+profile['title']) + print(' '+profile['description']) + idx += 1 + + print + choice = int(raw_input("Enter the number of the profile you would like to choose: ")) + + provider = requests.get( + self.config['url']+"providers/{0}/".format(profiles[choice]['cloud_provider']), + auth=auth, + verify=False).json() + + self.config['profile'] = profiles[choice]['title'] + self.config['provider'] = provider['title'] + self.config['provider_type'] = provider['provider_type_name'] + + def _save_config(self): + with open(self.CFG_FILE, "w") as f: + f.write(json.dumps(self.config)) + + def _bootstrap_account(self): + """Bootstrap the users account with public key""" + + if self.has_public_key: + keep_public_key = raw_input( + "Keep existing public key? (y,n)? ") + else: + keep_public_key = "n" + + if keep_public_key in ["y", "Y"]: + return + + raw_public_key = raw_input( + "What is your public key (either path to or contents of)? ") + + if os.path.exists(raw_public_key): + public_key = open(raw_public_key, "r").read() + elif os.path.exists(os.path.expanduser(raw_public_key)): + public_key = open(os.path.expanduser(raw_public_key), "r").read() + else: + public_key = raw_public_key + + if not public_key or not public_key.startswith("ssh-rsa"): + print(self.colorize("Unable to find valid public key", "red")) + else: + print("Setting public key") + self.stacks.set_public_key(public_key) + self.has_public_key = True + + def _bootstrap_formulas(self): + """Import and wait for formulas to become ready""" + + def _check_formulas(): + formulas = self.stacks.list_formulas() + for formula in formulas: + if formula.get("status") != "complete": + return False + return True + + formulas = self.bootstrap_data.get("formulas", []) + print("Importing {0} formula{1}".format( + len(formulas), + "s" if len(formulas) == 0 or len(formulas) > 1 else "")) + for name, url in formulas.iteritems(): + print(" - {0} // {1}".format(name, url)) + self.stacks.import_formula(url, public=False) + + sys.stdout.write("Waiting for formulas .") + sys.stdout.flush() + try: + poll_and_wait(_check_formulas) + sys.stdout.write(" done!\n") + except TimeoutException: + print(self.colorize( + "\nTIMEOUT - formulas failed to finish importing, monitor with `formulas list`", + "red")) + + def _bootstrap_blueprints(self): + """Create blueprints""" + + blueprints = self.bootstrap_data.get("blueprints", []) + + print("Creating {0} blueprint{1}".format(len(blueprints), + "s" if len(blueprints) == 0 or len(blueprints) > 1 else "")) + for name, blueprint in blueprints.iteritems(): + print(" - {0} // {1}".format(name, blueprint)) + + # Get the blueprints relative to the bootstrap config file + self._create_blueprint([os.path.join(os.path.dirname(self.BOOTSTRAP_FILE), "blueprints", + blueprint)], bootstrap=True) + diff --git a/stackdio/cli/mixins/formulas.py b/stackdio/cli/mixins/formulas.py new file mode 100644 index 0000000..1f0f8be --- /dev/null +++ b/stackdio/cli/mixins/formulas.py @@ -0,0 +1,81 @@ +from cmd2 import Cmd + + +class FormulaMixin(Cmd): + FORMULA_COMMANDS = ["list", "import", "delete"] + + def do_formulas(self, arg): + """Entry point to controlling formulas.""" + + USAGE = "Usage: formulas COMMAND\nWhere COMMAND is one of: %s" % ( + ", ".join(self.FORMULA_COMMANDS)) + + args = arg.split() + if not args or args[0] not in self.FORMULA_COMMANDS: + print(USAGE) + return + + formula_cmd = args[0] + if formula_cmd == "list": + self._list_formulas() + elif formula_cmd == "import": + self._import_formula(args[1:]) + elif formula_cmd == "delete": + self._delete_formula(args[1:]) + + else: + print(USAGE) + + def complete_formulas(self, text, line, begidx, endidx): + # not using line, begidx, or endidx, thus the following pylint disable + # pylint: disable=W0613 + return [i for i in self.FORMULA_COMMANDS if i.startswith(text)] + + def help_formulas(self): + print("Manage formulas.") + print("Sub-commands can be one of:\n\t{0}".format( + ", ".join(self.FORMULA_COMMANDS))) + print("Try 'formulas COMMAND' to get help on (most) sub-commands") + + def _list_formulas(self): + """List all formulas""" + + print("Getting formulas ... ") + formulas = self.stacks.list_formulas() + self._print_summary("Formula", formulas) + + def _import_formula(self, args): + """Import a formula""" + + if len(args) != 1: + print("Usage: formulas import URL") + return + + formula_url = args[0] + print("Importing formula from {0}".format(formula_url)) + formula = self.stacks.import_formula(formula_url, public=False) + + if isinstance(formula, list): + print("Formula imported, try the 'list' command to monitor status") + elif formula.get("detail"): + print("Error importing: {0}".format(formula.get("detail"))) + + def _delete_formula(self, args): + """Delete a formula""" + + args = " ".join(args) + if len(args) == 0: + print("Usage: formulas delete TITLE") + return + + formula_id = self.stacks.get_formula_id(args) + + really = raw_input("Really delete formula {0} (y/n)? ".format(args)) + if really not in ["y", "Y"]: + print("Aborting deletion") + return + + self.stacks.delete_formula(formula_id) + + print("Formula deleted, try the 'list' command to monitor status") + diff --git a/stackdio/cli/mixins/stacks.py b/stackdio/cli/mixins/stacks.py new file mode 100644 index 0000000..e54023e --- /dev/null +++ b/stackdio/cli/mixins/stacks.py @@ -0,0 +1,334 @@ +import json + +from cmd2 import Cmd + +from stackdio.client.exceptions import StackException + + +class StackMixin(Cmd): + + STACK_ACTIONS = ["start", "stop", "launch_existing", "terminate", "provision", "custom"] + STACK_COMMANDS = ["list", "launch_from_blueprint", "history", "hostnames", + "delete", "logs", "access_rules"] + STACK_ACTIONS + + VALID_LOGS = { + "provisioning": "provisioning.log", + "provisioning-error": "provisioning.err", + "global-orchestration": "global_orchestration.log", + "global-orchestration-error": "global_orchestration.err", + "orchestration": "orchestration.log", + "orchestration-error": "orchestration.err", + "launch": "launch.log", + } + + def do_stacks(self, arg): + """Entry point to controlling.""" + + USAGE = "Usage: stacks COMMAND\nWhere COMMAND is one of: %s" % ( + ", ".join(self.STACK_COMMANDS)) + + # We don't want multiline commands, so include anything after a terminator as well + args = arg.parsed.raw.split()[1:] + if not args or args[0] not in self.STACK_COMMANDS: + print(USAGE) + return + + stack_cmd = args[0] + + if stack_cmd == "list": + self._list_stacks() + elif stack_cmd == "launch_from_blueprint": + self._launch_stack(args[1:]) + elif stack_cmd == "history": + self._stack_history(args[1:]) + elif stack_cmd == "hostnames": + self._stack_hostnames(args[1:]) + elif stack_cmd == "delete": + self._stack_delete(args[1:]) + elif stack_cmd == "logs": + self._stack_logs(args[1:]) + elif stack_cmd == "access_rules": + self._stack_access_rules(args[1:]) + elif stack_cmd in self.STACK_ACTIONS: + self._stack_action(args) + + else: + print(USAGE) + + def complete_stacks(self, text, line, begidx, endidx): + # not using line, begidx, or endidx, thus the following pylint disable + # pylint: disable=W0613 + return [i for i in self.STACK_COMMANDS if i.startswith(text)] + + def help_stacks(self): + print("Manage stacks.") + print("Sub-commands can be one of:\n\t{0}".format( + ", ".join(self.STACK_COMMANDS))) + print("Try 'stacks COMMAND' to get help on (most) sub-commands") + + def _list_stacks(self): + """List all running stacks""" + + print("Getting running stacks ... ") + stacks = self.stacks.list_stacks() + self._print_summary("Stack", stacks) + + def _launch_stack(self, args): + """Launch a stack from a blueprint. + Must provide blueprint name and stack name""" + + if len(args) != 2: + print("Usage: stacks launch BLUEPRINT_NAME STACK_NAME") + return + + blueprint_name = args[0] + stack_name = args[1] + + try: + blueprint_id = self.stacks.get_blueprint_id(blueprint_name) + except StackException: + print(self.colorize( + "Blueprint [{0}] does not exist".format(blueprint_name), + "red")) + return + + print("Launching stack [{0}] from blueprint [{1}]".format( + stack_name, blueprint_name)) + + stack_data = { + "blueprint": blueprint_id, + "title": stack_name, + "description": "Launched from blueprint %s" % (blueprint_name), + "namespace": stack_name, + "max_retries": 1, + } + results = self.stacks.create_stack(stack_data) + print("Stack launch results:\n{0}".format(results)) + + def _get_stack_id(self, stack_name): + """Validate that a stack exists""" + + try: + return self.stacks.get_stack_id(stack_name) + except StackException: + print(self.colorize( + "Stack [{0}] does not exist".format(stack_name), + "red")) + raise + + def _stack_action(self, args): + """Perform an action on a stack.""" + + if len(args) == 1: + print("Usage: stacks {0} STACK_NAME".format(args[0])) + return + elif args[0] != "custom" and len(args) != 2: + print("Usage: stacks ACTION STACK_NAME") + print("Where ACTION is one of {0}".format( + ", ".join(self.STACK_ACTIONS))) + return + elif args[0] == "custom" and len(args) < 4: + print("Usage: stacks custom STACK_NAME HOST_TARGET COMMAND") + print("Where command can be arbitrarily long with spaces") + return + + if args[0] not in self.STACK_ACTIONS: + print(self.colorize( + "Invalid action - must be one of {0}".format(self.STACK_ACTIONS), + "red")) + return + + action = "launch" if args[0] == "launch_existing" else args[0] + stack_name = args[1] + + if action == "terminate": + really = raw_input("Really terminate stack {0} (y/n)? ".format(args[0])) + if really not in ["y", "Y"]: + print("Aborting termination") + return + + if action == "custom": + host_target = args[2] + command = '' + for token in args[3:]: + command += token + ' ' + + stack_id = self._get_stack_id(stack_name) + print("Performing [{0}] on [{1}]".format( + action, stack_name)) + + if action == "custom": + results = self.stacks.do_stack_action(stack_id, "custom", host_target, command) + else: + results = self.stacks.do_stack_action(stack_id, action) + print("Stack action results:\n{0}".format(json.dumps(results, indent=3))) + + def _stack_history(self, args): + """Print recent history for a stack""" + # pylint: disable=W0142 + + NUM_EVENTS = 20 + if len(args) < 1: + print("Usage: stacks history STACK_NAME") + return + + stack_id = self._get_stack_id(args[0]) + history = self.stacks.get_stack_history(stack_id).get("results") + for event in history[0:min(NUM_EVENTS, len(history))]: + print("[{created}] {level} // {event} // {status}".format(**event)) + + def _stack_hostnames(self, args): + """Print hostnames for a stack""" + + if len(args) < 1: + print("Usage: stacks hostnames STACK_NAME") + return + + stack_id = self._get_stack_id(args[0]) + try: + fqdns = self.stacks.describe_hosts(stack_id) + except StackException: + print(self.colorize( + "Hostnames not available - stack still launching?", "red")) + raise + + print("Hostnames:") + for host in fqdns: + print(" - {0}".format(host)) + + def _stack_delete(self, args): + """Delete a stack. PERMANENT AND DESTRUCTIVE!!!""" + + if len(args) < 1: + print("Usage: stacks delete STACK_NAME") + return + + stack_id = self._get_stack_id(args[0]) + really = raw_input("Really delete stack {0} (y/n)? ".format(args[0])) + if really not in ["y", "Y"]: + print("Aborting deletion") + return + + results = self.stacks.delete_stack(stack_id) + print("Delete stack results: {0}".format(results)) + print(self.colorize( + "Run 'stacks history {0}' to monitor status of the deletion".format( + args[0]), + "green")) + + def _stack_logs(self, args): + """Get logs for a stack""" + + MAX_LINES = 25 + + if len(args) < 2: + print("Usage: stacks logs STACK_NAME LOG_TYPE [LOG_LENGTH]") + print("LOG_TYPE is one of {0}".format( + ", ".join(self.stacks.VALID_LOGS.keys()))) + print("This defaults to the last {0} lines of the log.". + format(MAX_LINES)) + return + + if len(args) >= 3: + max_lines = args[2] + else: + max_lines = MAX_LINES + + stack_id = self.stacks.get_stack_id(args[0]) + + split_arg = self.VALID_LOGS[args[1]].split('.') + + log_text = self.stacks.get_logs(stack_id, log_type=split_arg[0], level=split_arg[1], + tail=max_lines) + print(log_text) + + def _stack_access_rules(self, args): + """Get access rules for a stack""" + + + COMMANDS = ["list", "add", "delete"] + + if len(args) < 2 or args[0] not in COMMANDS: + print("Usage: stacks access_rules COMMAND STACK_NAME") + print("Where COMMAND is one of: %s" % (", ".join(COMMANDS))) + return + + if args[0] == "list": + stack_id = self.stacks.get_stack_id(args[1]) + groups = self.stacks.list_access_rules(stack_id) + print "##", len(groups), "Access Groups" + for group in groups: + print "- Name:", group['blueprint_host_definition']['title'] + print " Description:", group['blueprint_host_definition']['description'] + print " Rules:" + for rule in group['rules']: + print " ",rule['protocol'], + if rule['from_port'] == rule['to_port']: + print "port", rule['from_port'], "allows", + else: + print "ports", rule['from_port']+"-"+rule['to_port'], "allow", + print rule['rule'] + print + return + + elif args[0] == "add": + if len(args) < 3: + print("Usage: stacks access_rules add STACK_NAME GROUP_NAME") + return + + stack_id = self.stacks.get_stack_id(args[1]) + group_id = self.stacks.get_access_rule_id(stack_id, args[2]) + + protocol = raw_input("Protocol (tcp, udp, or icmp): ") + from_port = raw_input("From port: ") + to_port = raw_input("To port: ") + rule = raw_input("Rule (IP address or group name): ") + + data = { + "action": "authorize", + "protocol": protocol, + "from_port": from_port, + "to_port": to_port, + "rule": rule + } + + self.stacks.edit_access_rule(group_id, data) + + elif args[0] == "delete": + if len(args) < 3: + print("Usage: stacks access_rules delete STACK_NAME GROUP_NAME") + return + + stack_id = self.stacks.get_stack_id(args[1]) + group_id = self.stacks.get_access_rule_id(stack_id, args[2]) + + index = 0 + + rules = self.stacks.list_rules_for_group(group_id) + + print + for rule in rules: + print str(index)+") ", rule['protocol'], + if rule['from_port'] == rule['to_port']: + print "port", rule['from_port'], "allows", + else: + print "ports", rule['from_port']+"-"+rule['to_port'], "allow", + print rule['rule'] + index += 1 + print + delete_index = int(raw_input("Enter the index of the rule to delete: ")) + + data = rules[delete_index] + data['from_port'] = int(data['from_port']) + data['to_port'] = int(data['to_port']) + data['action'] = "revoke" + + self.stacks.edit_access_rule(group_id, data) + + print + + args[0] = "list" + + self._stack_access_rules(args) + + diff --git a/stackdio/cli/polling.py b/stackdio/cli/polling.py new file mode 100644 index 0000000..ae1f02b --- /dev/null +++ b/stackdio/cli/polling.py @@ -0,0 +1,26 @@ + +import sys +import time + +class TimeoutException(Exception): + pass + +def poll_and_wait(func, args=None, sleep_time=2, max_time=120): + """Execute func in increments of sleep_time for no more than max_time. + Raise TimeoutException if we're not successful in max_time""" + #pylint: disable=W0142 + + args = args or [] + current_time = 0 + + success = func(*args) + while not success and current_time < max_time: + sys.stdout.write(".") + sys.stdout.flush() + current_time += sleep_time + time.sleep(sleep_time) + success = func(*args) + + if not success: + raise TimeoutException() + diff --git a/vbox_setup.sh b/vbox_setup.sh new file mode 100644 index 0000000..99f507d --- /dev/null +++ b/vbox_setup.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +apt-get update +apt-get install -y python-setuptools vim +easy_install pip +pip install virtualenvwrapper + +# Build the bash_profile +echo "source ~/.profile" >> /home/vagrant/.bash_profile +echo "source /usr/local/bin/virtualenvwrapper.sh" >> /home/vagrant/.bash_profile + +chown vagrant:vagrant /home/vagrant/.bash_profile From 1647a88eeb87cae2b9ebccb16149f1adbcfffc5d Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 10 Nov 2015 17:13:20 -0600 Subject: [PATCH 10/90] Moved blueprints stuff inside the cli package --- setup.py | 2 +- stackdio/cli/__init__.py | 2 -- stackdio/{ => cli}/blueprints/__init__.py | 0 stackdio/{ => cli}/blueprints/generator.py | 0 stackdio/cli/mixins/blueprints.py | 2 +- stackdio/cli/mixins/bootstrap.py | 1 + 6 files changed, 3 insertions(+), 4 deletions(-) rename stackdio/{ => cli}/blueprints/__init__.py (100%) rename stackdio/{ => cli}/blueprints/generator.py (100%) diff --git a/setup.py b/setup.py index fd2599c..6585001 100644 --- a/setup.py +++ b/setup.py @@ -84,7 +84,7 @@ def test_python_version(): entry_points={ 'console_scripts': [ 'stackdio-cli=stackdio.cli:main', - 'blueprint-generator=stackdio.blueprints:main', + 'blueprint-generator=stackdio.cli.blueprints:main', ], }, classifiers=[ diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index a6d2512..369c8b5 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -__version__ = '0.2.dev' - import argparse import json import os diff --git a/stackdio/blueprints/__init__.py b/stackdio/cli/blueprints/__init__.py similarity index 100% rename from stackdio/blueprints/__init__.py rename to stackdio/cli/blueprints/__init__.py diff --git a/stackdio/blueprints/generator.py b/stackdio/cli/blueprints/generator.py similarity index 100% rename from stackdio/blueprints/generator.py rename to stackdio/cli/blueprints/generator.py diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index a419aca..9fecd80 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -7,7 +7,7 @@ from cmd2 import Cmd from stackdio.client.exceptions import StackException -from stackdio.blueprints.generator import BlueprintGenerator, BlueprintException +from stackdio.cli.blueprints.generator import BlueprintGenerator, BlueprintException class BlueprintNotFound(Exception): diff --git a/stackdio/cli/mixins/bootstrap.py b/stackdio/cli/mixins/bootstrap.py index 3bae0ff..4aa649f 100644 --- a/stackdio/cli/mixins/bootstrap.py +++ b/stackdio/cli/mixins/bootstrap.py @@ -20,6 +20,7 @@ class PublicKeyNotFound(Exception): class BootstrapMixin(Cmd): def __init__(self): + super(BootstrapMixin, self).__init__() # quieting down pylint self.has_public_key = None From bbb3fd23a10d5a562cb15c988e8cc1a11a25ac9c Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 11 Nov 2015 13:49:15 -0600 Subject: [PATCH 11/90] Updated methods for 0.7 --- setup.py | 5 + stackdio/client/__init__.py | 57 ++------ stackdio/client/account.py | 40 +++--- stackdio/client/blueprint.py | 37 +++--- stackdio/client/formula.py | 46 ++++--- stackdio/client/http.py | 251 ++++++++++++++++++++--------------- stackdio/client/image.py | 32 ++--- stackdio/client/region.py | 30 ++--- stackdio/client/settings.py | 22 +-- stackdio/client/stack.py | 126 +++++++----------- 10 files changed, 316 insertions(+), 330 deletions(-) diff --git a/setup.py b/setup.py index 6585001..d8470b6 100644 --- a/setup.py +++ b/setup.py @@ -99,6 +99,11 @@ def test_python_version(): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: System :: Clustering', 'Topic :: System :: Distributed Computing', ] diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index d3c6440..4151431 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -15,10 +15,9 @@ # limitations under the License. # -import json import logging -from .http import use_admin_auth, endpoint +from .http import get, post, patch from .exceptions import BlueprintException, StackException, IncompatibleVersionException from .blueprint import BlueprintMixin @@ -61,59 +60,25 @@ def __init__(self, protocol="https", host="localhost", port=443, raise IncompatibleVersionException('Server version {0}.{1}.{2} not ' 'supported.'.format(**self.version)) - @endpoint("") + @get('') def get_root(self): - """Get the api root""" - return self._get(endpoint, jsonify=True) + pass - @endpoint("version/") + @get('version/') def get_version(self): - return self._get(endpoint, jsonify=True)['version'] + pass - @use_admin_auth - @endpoint("security_groups/") + @get_version.response + def get_version(self, resp): + return resp['version'] + + @post('cloud/security_groups/') def create_security_group(self, name, description, cloud_provider, is_default=True): """Create a security group""" - data = { + return { "name": name, "description": description, "cloud_provider": cloud_provider, "is_default": is_default } - return self._post(endpoint, data=json.dumps(data), jsonify=True) - - @endpoint("user/") - def get_public_key(self): - """Get the public key for the logged in user""" - return self._get(endpoint, jsonify=True)['settings']['public_key'] - - @endpoint("settings/") - def set_public_key(self, public_key): - """Upload a public key for our user. public_key can be the actual key, a - file handle, or a path to a key file""" - - if isinstance(public_key, file): - public_key = public_key.read() - elif isinstance(public_key, str) and os.path.exists(public_key): - public_key = open(public_key, "r").read() - - data = { - "public_key": public_key - } - return self._put(endpoint, data=json.dumps(data), jsonify=True) - - @endpoint("instance_sizes/") - def get_instance_id(self, instance_id, provider_type="ec2"): - """Get the id for an instance_id. The instance_id parameter is the - provider name (e.g. m1.large). The id returned is the stackd.io id - for use in API calls (e.g. 1).""" - - result = self._get(endpoint, jsonify=True) - for instance in result['results']: - if instance.get("instance_id") == instance_id and \ - instance.get("provider_type") == provider_type: - return instance.get("id") - - raise StackException("Instance type %s from provider %s not found" % - (instance_id, provider_type)) diff --git a/stackdio/client/account.py b/stackdio/client/account.py index 84e5f12..928105f 100644 --- a/stackdio/client/account.py +++ b/stackdio/client/account.py @@ -15,24 +15,22 @@ # limitations under the License. # -import json - -from .http import HttpMixin, endpoint +from .http import HttpMixin, get, post, delete class AccountMixin(HttpMixin): - @endpoint("cloud/providers/") + @get('cloud/providers/', paginate=True) def list_providers(self): """List all providers""" - return self._get(endpoint, jsonify=True)['results'] + pass - @endpoint("cloud/providers/") - def search_providers(self, provider_id): - """List all providers""" - return self._get(endpoint, jsonify=True)['results'] + @get('cloud/providers/', paginate=True) + def search_providers(self, **kwargs): + """Search for a provider""" + pass - @endpoint("cloud/accounts/") + @post('cloud/accounts/') def create_account(self, **kwargs): """Create an account""" @@ -52,24 +50,24 @@ def create_account(self, **kwargs): for key in form_data.keys(): form_data[key] = kwargs.get(key) - return self._post(endpoint, data=json.dumps(form_data), jsonify=True) + return form_data - @endpoint("accounts/") + @get('cloud/accounts/', paginate=True) def list_accounts(self): """List all account""" - return self._get(endpoint, jsonify=True)['results'] + pass - @endpoint("accounts/{account_id}/") - def get_account(self, account_id, none_on_404=False): + @get('cloud/accounts/{account_id}/') + def get_account(self, account_id): """Return the account that matches the given id""" - return self._get(endpoint, jsonify=True, none_on_404=none_on_404) + pass - @endpoint("accounts/") - def search_accounts(self, account_id): + @get('cloud/accounts/') + def search_accounts(self, **kwargs): """List all accounts""" - return self._get(endpoint, jsonify=True)['results'] + pass - @endpoint("accounts/{account_id}/") + @delete('cloud/accounts/{account_id}/') def delete_account(self, account_id): """List all accounts""" - return self._delete(endpoint, jsonify=True)['results'] + pass diff --git a/stackdio/client/blueprint.py b/stackdio/client/blueprint.py index 5caf6f4..e65815a 100644 --- a/stackdio/client/blueprint.py +++ b/stackdio/client/blueprint.py @@ -15,17 +15,19 @@ # limitations under the License. # -import json - -from .http import HttpMixin, endpoint +from .exceptions import BlueprintException +from .http import HttpMixin, get, post, delete class BlueprintMixin(HttpMixin): - @endpoint("blueprints/") - def create_blueprint(self, blueprint, provider="ec2"): + @post('blueprints/') + def create_blueprint(self, blueprint): """Create a blueprint""" + if 'host_definitions' not in blueprint: + raise BlueprintException('Blueprints must contain a list of host_definitions') + formula_map = {} if 'formula_versions' in blueprint: @@ -50,27 +52,24 @@ def create_blueprint(self, blueprint, provider="ec2"): # check the provided blueprint to see if we need to look up any ids for host in blueprint['host_definitions']: - for component in host['formula_components']: + for component in host.get('formula_components', []): if component['sls_path'] in formula_map: component['formula'] = formula_map[component['sls_path']] - return self._post(endpoint, data=json.dumps(blueprint), jsonify=True, raise_for_status=False) + return blueprint - @endpoint("blueprints/") + @get('blueprints/', paginate=True) def list_blueprints(self): - """Return info for a specific blueprint_id""" - return self._get(endpoint, jsonify=True)['results'] + pass - @endpoint("blueprints/{blueprint_id}/") - def get_blueprint(self, blueprint_id, none_on_404=False): - """Return info for a specific blueprint_id""" - return self._get(endpoint, jsonify=True, none_on_404=none_on_404) + @get('blueprints/{blueprint_id}/') + def get_blueprint(self, blueprint_id): + pass - @endpoint("blueprints/") + @get('blueprints/', paginate=True) def search_blueprints(self, **kwargs): - """Return info for a specific blueprint_id""" - return self._get(endpoint, params=kwargs, jsonify=True)['results'] + pass - @endpoint("blueprints/{blueprint_id}") + @delete('blueprints/{blueprint_id}') def delete_blueprint(self, blueprint_id): - return self._delete(endpoint, jsonify=True) + pass diff --git a/stackdio/client/formula.py b/stackdio/client/formula.py index d98be7e..fd2d074 100644 --- a/stackdio/client/formula.py +++ b/stackdio/client/formula.py @@ -15,42 +15,48 @@ # limitations under the License. # -import json - -from .http import HttpMixin, endpoint +from .http import HttpMixin, get, post, delete class FormulaMixin(HttpMixin): - @endpoint("formulas/") - def import_formula(self, formula_uri, public=True): + @post('formulas/') + def import_formula(self, formula_uri, git_username=None, git_password=None, access_token=None): """Import a formula""" data = { - "uri": formula_uri, - "public": public, + 'uri': formula_uri, } - return self._post(endpoint, data=json.dumps(data), jsonify=True) - @endpoint("formulas/") + if git_username: + data['git_username'] = git_username + data['git_password'] = git_password + data['access_token'] = False + elif access_token: + data['git_username'] = access_token + data['access_token'] = True + + return data + + @get('formulas/', paginate=True) def list_formulas(self): """Return all formulas""" - return self._get(endpoint, jsonify=True)['results'] + pass - @endpoint("formulas/{formula_id}/") - def get_formula(self, formula_id, none_on_404=False): + @get('formulas/{formula_id}/') + def get_formula(self, formula_id): """Get a formula with matching id""" - return self._get(endpoint, jsonify=True, none_on_404=none_on_404) + pass - @endpoint("formulas/") + @get('formulas/', paginate=True) def search_formulas(self, **kwargs): """Get a formula with matching id""" - return self._get(endpoint, params=kwargs, jsonify=True)['results'] + pass - @endpoint("formulas/{formula_id}/") + @delete('formulas/{formula_id}/') def delete_formula(self, formula_id): """Delete formula with matching id""" - return self._delete(endpoint, jsonify=True) + pass - @endpoint("formulas/{formula_id}/action/") + @post('formulas/{formula_id}/action/') def update_formula(self, formula_id): - """Delete formula with matching id""" - return self._post(endpoint, json.dumps({"action": "update"}), jsonify=True) + """Update the formula""" + return {"action": "update"} diff --git a/stackdio/client/http.py b/stackdio/client/http.py index a5d6afd..e3cf632 100644 --- a/stackdio/client/http.py +++ b/stackdio/client/http.py @@ -17,6 +17,7 @@ from __future__ import print_function +import json import logging import requests @@ -33,71 +34,6 @@ "This has been your single warning."]) -def use_admin_auth(func): - - @wraps(func) - def wrapper(obj, *args, **kwargs): - # Save and set the auth to use the admin auth - auth = obj._http_options['auth'] - try: - obj._http_options['auth'] = obj._http_options['admin'] - except KeyError: - raise NoAdminException("No admin credentials were specified") - - # Call the original function - output = func(*args, **kwargs) - - # Set the auth back to the original - obj._http_options['auth'] = auth - return output - return wrapper - - -def endpoint(path): - """Takes a path extension and appends to the known API base url. - The result of this is then added to the decorated functions global - scope as a variable named 'endpoint""" - def decorator(func): - @wraps(func) - def wrapper(obj, *args, **kwargs): - - # Get what locals() would return directly after calling - # 'func' with the given args and kwargs - future_locals = getcallargs(func, *((obj,) + args), **kwargs) - - # Build the variable we'll inject - url = "{url}{path}".format( - url=obj.url, - path=path.format(**future_locals)) - - # Grab the global context for the passed function - g = func.__globals__ - - # Create a unique default object so we can accurately determine - # if we replaced a value - sentinel = object() - oldvalue = g.get('endpoint', sentinel) - - # Inject our variable into the global scope - g['endpoint'] = url - - # Logging and function call - if oldvalue: - logger.debug("Value %s for 'endpoint' replaced in global scope " - "for function %s" % (oldvalue, func.__name__)) - logger.debug("%s.__globals__['endpoint'] = %s" % (func.__name__, url)) - - result = func(obj, *args, **kwargs) - - # Replace the previous value, if it existed - if oldvalue is not sentinel: - g['endpoint'] = oldvalue - - return result - return wrapper - return decorator - - class HttpMixin(object): """Add HTTP request features to an object""" @@ -107,6 +43,8 @@ class HttpMixin(object): } def __init__(self, auth=None, verify=True): + super(HttpMixin, self).__init__() + self._http_options = { 'auth': auth, 'verify': verify, @@ -122,55 +60,158 @@ def __init__(self, auth=None, verify=True): from requests.packages.urllib3 import disable_warnings disable_warnings() - def _request(self, verb, url, quiet=False, - none_on_404=False, jsonify=False, raise_for_status=True, - *args, **kwargs): - """Generic request method""" - if not quiet: - self._http_log.info("{0}: {1}".format(verb, url)) - headers = kwargs.get('headers', HttpMixin.HEADERS['json']) +def default_response(obj, response): + return response + + +def request(path, method, paginate=False, jsonify=True, **req_kwargs): + + # Define a class here that uses the path / method we want. We need it inside this function + # so we have access to the path / method. + class Request(object): + def __init__(self, dfunc=None, rfunc=None, quiet=False): + super(Request, self).__init__() + + self.data_func = dfunc + self.response_func = rfunc + + if self.response_func is None: + self.response_func = default_response + + self.quiet = quiet + + self.headers = req_kwargs.get('headers', HttpMixin.HEADERS['json']) + + self._http_log = logging.getLogger(__name__) + + def data(self, dfunc): + return type(self)(dfunc, self.response_func, self.quiet) + + def response(self, rfunc): + return type(self)(self.data_func, rfunc, self.quiet) - result = requests.request(verb, url, - auth=self._http_options['auth'], - headers=headers, - verify=self._http_options['verify'], - *args, **kwargs) + # Here's how the request actually happens + def __get__(self, obj, objtype=None): + def do_request(*args, **kwargs): + none_on_404 = kwargs.pop('none_on_404', False) + raise_for_status = kwargs.pop('raise_for_status', True) - # Handle special conditions - if none_on_404 and result.status_code == 404: - return None + # Get what locals() would return directly after calling + # 'func' with the given args and kwargs + future_locals = getcallargs(self.data_func, *((obj,) + args), **kwargs) - elif result.status_code == 204: - return None + # Build the variable we'll inject + url = '{url}{path}'.format( + url=obj.url, + path=path.format(**future_locals) + ) - elif raise_for_status: - try: - result.raise_for_status() - except Exception: - logger.error(result.text) - raise + if not self.quiet: + self._http_log.info("{0}: {1}".format(method, url)) - # return - if jsonify: - return result.json() - else: - return result + data = None + if self.data_func: + data = json.dumps(self.data_func(obj, *args, **kwargs)) - def _head(self, url, *args, **kwargs): - return self._request("HEAD", url, *args, **kwargs) + result = requests.request(method, + url, + data=data, + auth=obj._http_options['auth'], + headers=self.headers, + params=kwargs, + verify=obj._http_options['verify']) - def _get(self, url, *args, **kwargs): - return self._request("GET", url, *args, **kwargs) + # Handle special conditions + if none_on_404 and result.status_code == 404: + return None - def _delete(self, url, *args, **kwargs): - return self._request("DELETE", url, *args, **kwargs) + elif result.status_code == 204: + return None - def _post(self, url, data=None, *args, **kwargs): - return self._request("POST", url, data=data, *args, **kwargs) + elif raise_for_status: + try: + result.raise_for_status() + except Exception: + logger.error(result.text) + raise - def _put(self, url, data=None, *args, **kwargs): - return self._request("PUT", url, data=data, *args, **kwargs) + if jsonify: + response = result.json() + else: + response = result.text - def _patch(self, url, data=None, *args, **kwargs): - return self._request("PATCH", url, data=data, *args, **kwargs) + if method == 'GET' and paginate and jsonify: + res = response['results'] + + next_url = response['next'] + + while next_url: + next_page = requests.request(method, + next_url, + data=data, + auth=obj._http_options['auth'], + headers=self.headers, + params=kwargs, + verify=obj._http_options['verify']).json() + res.extend(next_page['results']) + next_url = next_page['next'] + + response = res + + # now process the result + return self.response_func(obj, response) + + return do_request + + return Request + + +# Define the decorators for all the methods + +def get(path, paginate=False, jsonify=True): + return request(path, 'GET', paginate=paginate, jsonify=jsonify) + + +def head(path): + return request(path, 'HEAD') + + +def options(path): + return request(path, 'OPTIONS') + + +def post(path): + return request(path, 'POST') + + +def put(path): + return request(path, 'PUT') + + +def patch(path): + return request(path, 'PATCH') + + +def delete(path): + return request(path, 'DELETE') + + +def use_admin_auth(func): + + @wraps(func) + def wrapper(obj, *args, **kwargs): + # Save and set the auth to use the admin auth + auth = obj._http_options['auth'] + try: + obj._http_options['auth'] = obj._http_options['admin'] + except KeyError: + raise NoAdminException("No admin credentials were specified") + + # Call the original function + output = func(*args, **kwargs) + + # Set the auth back to the original + obj._http_options['auth'] = auth + return output + return wrapper diff --git a/stackdio/client/image.py b/stackdio/client/image.py index 52a972a..8af4f15 100644 --- a/stackdio/client/image.py +++ b/stackdio/client/image.py @@ -15,42 +15,38 @@ # limitations under the License. # -import json - -from .http import HttpMixin, endpoint +from .http import HttpMixin, get, post, delete class ImageMixin(HttpMixin): - @endpoint("cloud/images/") - def create_image(self, title, image_id, ssh_user, cloud_provider, - default_instance_size=None): + @post('cloud/images/') + def create_image(self, title, image_id, ssh_user, cloud_provider, default_instance_size=None): """Create a image""" - data = { + return { "title": title, "image_id": image_id, "ssh_user": ssh_user, "cloud_provider": cloud_provider, "default_instance_size": default_instance_size } - return self._post(endpoint, data=json.dumps(data), jsonify=True) - @endpoint("cloud/images/") + @get('cloud/images/', paginate=True) def list_images(self): """List all images""" - return self._get(endpoint, jsonify=True)['results'] + pass - @endpoint("cloud/images/{image_id}/") - def get_image(self, image_id, none_on_404=False): + @get('cloud/images/{image_id}/') + def get_image(self, image_id): """Return the image that matches the given id""" - return self._get(endpoint, jsonify=True, none_on_404=none_on_404) + pass - @endpoint("cloud/images/") - def search_images(self, image_id): + @get('cloud/images/', paginate=True) + def search_images(self, **kwargs): """List all images""" - return self._get(endpoint, jsonify=True)['results'] + pass - @endpoint("cloud/images/{image_id}/") + @delete('cloud/images/{image_id}/') def delete_image(self, image_id): """Delete the image with the given id""" - return self._delete(endpoint, jsonify=True)['results'] + pass diff --git a/stackdio/client/region.py b/stackdio/client/region.py index 5485f2c..2689d8b 100644 --- a/stackdio/client/region.py +++ b/stackdio/client/region.py @@ -15,30 +15,30 @@ # limitations under the License. # -from .http import HttpMixin, endpoint +from .http import HttpMixin, get class RegionMixin(HttpMixin): - @endpoint("cloud/providers/{provider_name}/regions/") + @get('cloud/providers/{provider_name}/regions/', paginate=True) def list_regions(self, provider_name): - return self._get(endpoint, jsonify=True)['results'] + pass - @endpoint("cloud/providers/{provider_name}/regions/{region_id}") - def get_region(self, provider_name, region_id, none_on_404=False): - return self._get(endpoint, jsonify=True, none_on_404=none_on_404) + @get('cloud/providers/{provider_name}/regions/{region_id}/') + def get_region(self, provider_name, region_id): + pass - @endpoint("cloud/providers/{provider_name}/regions/") + @get('cloud/providers/{provider_name}/regions/', paginate=True) def search_regions(self, provider_name, **kwargs): - return self._get(endpoint, params=kwargs, jsonify=True)['results'] + pass - @endpoint("cloud/providers/{provider_name}/zones/") + @get('cloud/providers/{provider_name}/zones/', paginate=True) def list_zones(self): - return self._get(endpoint, jsonify=True)['results'] + pass - @endpoint("cloud/providers/{provider_name}/zones/{zone_id}") - def get_zone(self, provider_name, zone_id, none_on_404=False): - return self._get(endpoint, jsonify=True, none_on_404=none_on_404) + @get('cloud/providers/{provider_name}/zones/{zone_id}') + def get_zone(self, provider_name, zone_id): + pass - @endpoint("cloud/providers/{provider_name}/zones/") + @get('cloud/providers/{provider_name}/zones/', paginate=True) def search_zones(self, provider_name, **kwargs): - return self._get(endpoint, params=kwargs, jsonify=True)['results'] + pass diff --git a/stackdio/client/settings.py b/stackdio/client/settings.py index 9be2179..bfc57d7 100644 --- a/stackdio/client/settings.py +++ b/stackdio/client/settings.py @@ -15,28 +15,34 @@ # limitations under the License. # -import json import os -from .http import HttpMixin, endpoint +from .http import HttpMixin, get, patch class SettingsMixin(HttpMixin): - @endpoint("user/") + @get('user/') + def get_public_key(self): + """Get the public key for the logged in user""" + pass + + @get_public_key.response + def get_public_key(self, resp): + return resp['settings']['public_key'] + + @patch('user/') def set_public_key(self, public_key): """Upload a public key for our user. public_key can be the actual key, a file handle, or a path to a key file""" if isinstance(public_key, file): public_key = public_key.read() - elif isinstance(public_key, str) and os.path.exists(public_key): public_key = open(public_key, "r").read() - data = { - "settings": { - "public_key": public_key, + return { + 'settings': { + 'public_key': public_key, } } - return self._patch(endpoint, data=json.dumps(data), jsonify=True) diff --git a/stackdio/client/stack.py b/stackdio/client/stack.py index 9dd1e64..4a7ff39 100644 --- a/stackdio/client/stack.py +++ b/stackdio/client/stack.py @@ -15,121 +15,92 @@ # limitations under the License. # -import json - from .exceptions import StackException -from .http import HttpMixin, endpoint +from .http import HttpMixin, get, post, put, delete from .version import deprecated class StackMixin(HttpMixin): VALID_LOG_TYPES = { - "provisioning": ['log', 'err'], - "global-orchestration": ['log', 'err'], - "orchestration": ['log', 'err'], - "launch": ['log'], + 'provisioning': ['log', 'err'], + 'global-orchestration': ['log', 'err'], + 'orchestration': ['log', 'err'], + 'launch': ['log'], } - @endpoint("stacks/") + @post('stacks/') def create_stack(self, stack_data): """Launch a stack as described by stack_data""" - return self._post(endpoint, data=json.dumps(stack_data), jsonify=True) + return stack_data - @endpoint("stacks/") + @get('stacks/', paginate=True) def list_stacks(self): """Return a list of all stacks""" - return self._get(endpoint, jsonify=True)['results'] + pass - @endpoint("stacks/{stack_id}/") - def get_stack(self, stack_id, none_on_404=False): + @get('stacks/{stack_id}/') + def get_stack(self, stack_id): """Get stack info""" - return self._get(endpoint, jsonify=True, none_on_404=none_on_404) + pass - @endpoint("stacks/") + @get('stacks/', paginate=True) def search_stacks(self, **kwargs): """Search for stacks that match the given criteria""" - return self._get(endpoint, params=kwargs, jsonify=True)['results'] + pass - @endpoint("stacks/{stack_id}/") + @delete('stacks/{stack_id}/') def delete_stack(self, stack_id): """Destructively delete a stack forever.""" - return self._delete(endpoint, jsonify=True) + pass - @endpoint("stacks/{stack_id}/action/") + @get('stacks/{stack_id}/action/') def get_valid_stack_actions(self, stack_id): - return self._get(endpoint, jsonify=True)['available_actions'] + pass + + @get_valid_stack_actions.response + def get_valid_stack_actions(self, resp): + return resp['available_actions'] - @endpoint("stacks/{stack_id}/action/") + @post('stacks/{stack_id}/action/') def do_stack_action(self, stack_id, action): """Execute an action on a stack""" valid_actions = self.get_valid_stack_actions(stack_id) if action not in valid_actions: - raise StackException("Invalid action, must be one of %s" % - ", ".join(valid_actions)) + raise StackException('Invalid action, must be one of %s' % + ', '.join(valid_actions)) - data = {"action": action} + return {'action': action} - return self._post(endpoint, data=json.dumps(data), jsonify=True) - - @endpoint("stacks/{stack_id}/history/") + @get('stacks/{stack_id}/history/', paginate=True) def get_stack_history(self, stack_id): """Get stack info""" - result = self._get(endpoint, none_on_404=True, jsonify=True) - if result is None: - raise StackException("Stack %s not found" % stack_id) - else: - return result + pass - @endpoint("stacks/{stack_id}/hosts/") + @get('stacks/{stack_id}/hosts/', paginate=True) def get_stack_hosts(self, stack_id): """Get a list of all stack hosts""" - return self._get(endpoint, jsonify=True)['results'] - - @endpoint("stacks/{stack_id}/hosts/") - def describe_hosts(self, stack_id, key="fqdn", ec2=False): - """Retrieve a list of info about a stack. Defaults to the id for each - host, but you can specify any available key. Setting ec2=True will - force it to inspect the ec2_metadata field.""" - - EC2 = "ec2_metadata" - result = self._get(endpoint, jsonify=True) - - stack_details = [] - - for host in result['results']: - if not ec2: - host_details = host.get(key) - else: - host_details = host.get(EC2).get(key) + pass - if host_details is not None: - stack_details.append(host_details) - - if stack_details: - return stack_details - - raise StackException("Key %s for stack %s not available" % (key, stack_id)) - - @endpoint("stacks/{stack_id}/logs/{log_type}.{level}.{date}") + @get('stacks/{stack_id}/logs/{log_type}.{level}.{date}', jsonify=False) def get_logs(self, stack_id, log_type, level='log', date='latest', tail=None): """Get logs for a stack""" if log_type and log_type not in self.VALID_LOG_TYPES: - raise StackException("Invalid log type, must be one of %s" % - ", ".join(self.VALID_LOG_TYPES.keys())) + raise StackException('Invalid log type, must be one of %s' % + ', '.join(self.VALID_LOG_TYPES.keys())) if level not in self.VALID_LOG_TYPES[log_type]: - raise StackException("Invalid log level, must be one of %s" % - ", ".join(self.VALID_LOG_TYPES[log_type])) + raise StackException('Invalid log level, must be one of %s' % + ', '.join(self.VALID_LOG_TYPES[log_type])) - return self._get(endpoint, params={'tail': tail}).text - - @endpoint("stacks/{stack_id}/security_groups/") + @get('stacks/{stack_id}/security_groups/', paginate=True) def list_access_rules(self, stack_id): - """Get Access rules for a stack""" - - return self._get(endpoint, jsonify=True)['results'] + """ + Get Access rules for a stack + :rtype: list + """ + pass @deprecated def get_access_rule_id(self, stack_id, title): @@ -139,19 +110,18 @@ def get_access_rule_id(self, stack_id, title): try: for group in rules: - if group.get("blueprint_host_definition").get("title") == title: - return group.get("id") + if group.get('blueprint_host_definition').get('title') == title: + return group.get('id') except TypeError: pass - raise StackException("Access Rule %s not found" % title) + raise StackException('Access Rule %s not found' % title) - @endpoint("security_groups/{group_id}/rules/") + @get('security_groups/{group_id}/rules/', paginate=True) def list_rules_for_group(self, group_id): - return self._get(endpoint, jsonify=True) + pass - @endpoint("security_groups/{group_id}/rules/") + @put('security_groups/{group_id}/rules/') def edit_access_rule(self, group_id, data=None): """Add an access rule to a group""" - - return self._put(endpoint, jsonify=True, data=json.dumps(data)) + return data From 56e5de96f7c260436531a2d5eadeeae99a41489d Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 11 Nov 2015 13:57:44 -0600 Subject: [PATCH 12/90] Make sure cli actually loads --- stackdio/cli/blueprints/__init__.py | 2 +- stackdio/cli/mixins/bootstrap.py | 24 ------------------------ stackdio/cli/polling.py | 2 ++ 3 files changed, 3 insertions(+), 25 deletions(-) diff --git a/stackdio/cli/blueprints/__init__.py b/stackdio/cli/blueprints/__init__.py index 1e14fbc..bb76c0a 100644 --- a/stackdio/cli/blueprints/__init__.py +++ b/stackdio/cli/blueprints/__init__.py @@ -3,7 +3,7 @@ import json import sys -from stackdio.blueprints.generator import BlueprintException, BlueprintGenerator +from stackdio.cli.blueprints.generator import BlueprintException, BlueprintGenerator def main(): diff --git a/stackdio/cli/mixins/bootstrap.py b/stackdio/cli/mixins/bootstrap.py index 4aa649f..9cba14a 100644 --- a/stackdio/cli/mixins/bootstrap.py +++ b/stackdio/cli/mixins/bootstrap.py @@ -20,8 +20,6 @@ class PublicKeyNotFound(Exception): class BootstrapMixin(Cmd): def __init__(self): - super(BootstrapMixin, self).__init__() - # quieting down pylint self.has_public_key = None self.validated = False @@ -99,28 +97,6 @@ def do_initial_setup(self, args=None): self._save_config() self._setprompt() - if self.validated: - bootstrap = raw_input("Bootstrap your account now (y/n)? ") - if bootstrap not in ["y", "Y"]: - return - self.do_bootstrap() - - print(self.colorize(""" -## -## Success! You're ready to start using stackd.io. Try running -## 'help' to see what all is available here. You can also go to -## {0} to use the UI. -## - """.format( - self.config["url"][0:self.config["url"].find("api/")]), - "green")) - - else: - print(self.colorize( - "Unable to bootstrap your account", - "red")) - return - def do_bootstrap(self, args=None): """Bootstrap an account with predefined formulas and blueprints""" diff --git a/stackdio/cli/polling.py b/stackdio/cli/polling.py index ae1f02b..5fbbc79 100644 --- a/stackdio/cli/polling.py +++ b/stackdio/cli/polling.py @@ -2,9 +2,11 @@ import sys import time + class TimeoutException(Exception): pass + def poll_and_wait(func, args=None, sleep_time=2, max_time=120): """Execute func in increments of sleep_time for no more than max_time. Raise TimeoutException if we're not successful in max_time""" From 38b4137bb7018838d4a0c977cb66d82fe8d4356b Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 11 Nov 2015 14:00:43 -0600 Subject: [PATCH 13/90] Updated bootstrap things --- blueprints/.keep | 0 blueprints/cdh4-3node.json | 191 ------------------------------------ blueprints/cdh5-3node.json | 192 ------------------------------------- bootstrap.yaml | 6 +- 4 files changed, 2 insertions(+), 387 deletions(-) create mode 100644 blueprints/.keep delete mode 100644 blueprints/cdh4-3node.json delete mode 100644 blueprints/cdh5-3node.json diff --git a/blueprints/.keep b/blueprints/.keep new file mode 100644 index 0000000..e69de29 diff --git a/blueprints/cdh4-3node.json b/blueprints/cdh4-3node.json deleted file mode 100644 index 99a16bd..0000000 --- a/blueprints/cdh4-3node.json +++ /dev/null @@ -1,191 +0,0 @@ -{ - "public": false, - "properties": { - "java": { - "enable_jce": false, - "oracle": { - "cookies": "gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie", - "staging": "/tmp/.java_staging", - "jdk7": { - "rpm": "jdk-1.7.0_45", - "uri": "http://download.oracle.com/otn-pub/java/jdk/7u45-b18/jdk-7u45-linux-x64.rpm" - } - } - }, - "cdh4": { - "landing_page": true, - "hue": { - "secret_key": "CHANGE_THIS", - "start_service": true - }, - "mapred": { - "system_dir": "/hadoop/system/mapred", - "map_tasks_max": 5, - "reduce_tasks_max": 3, - "local_dir": "/mnt/hadoop/mapred/local", - "child_java_opts": "-Xmx2000m", - "reduce_tasks": 6, - "child_ulimit": 8000000 - }, - "zookeeper": { - "data_dir": "/mnt/zk/data", - "start_service": true - }, - "max_log_index": 1, - "dfs": { - "name_dir": "/mnt/hadoop/hdfs/nn", - "block_size": 268435456, - "data_dir": "/mnt/hadoop/hdfs/data", - "permissions": true - }, - "version": 4, - "datanode": { - "start_service": true - }, - "hive": { - "home": "/usr/lib/hive", - "metastore_password": "CHANGE_THIS", - "start_service": true, - "user": "hive" - }, - "io": { - "sort_mb": 250, - "sort_factor": 25 - }, - "oozie": { - "start_service": true - }, - "namenode": { - "start_service": true - }, - "hbase": { - "region_initial_heap": "1024m", - "region_max_heap": "1024m", - "tmp_dir": "/mnt/hbase/tmp", - "start_service": true, - "master_young_gen": "256m", - "replication": 3, - "master_initial_heap": "1024m", - "manage_zk": true, - "log_dir": "/mnt/hbase/logs", - "region_young_gen": "256m", - "master_max_heap": "1024m", - "jute_maxbuffer": 1000000 - }, - "impala": { - "version": "1.2.3", - "start_service": true - } - } - }, - "hosts": [ - { - "count": 1, - "description": "NameNode and Master", - "zone": "us-east-1d", - "title": "NameNode and Master", - "hostname_template": "{namespace}-nn", - "cloud_profile": "TO_BE_CHANGED", - "formula_components": [ - { - "id": [ - "OpenJDK and Oracle Java", - "Oracle Java 7 JDK" - ], - "order": 0 - }, - { - "id": [ - "CDH4 Salt Formula", - "NameNode" - ], - "order": 1 - }, - { - "id": [ - "CDH4 Salt Formula", - "HBase Master" - ], - "order": 3 - }, - { - "id": [ - "CDH4 Salt Formula", - "Oozie" - ], - "order": 5 - }, - { - "id": [ - "CDH4 Salt Formula", - "Pig" - ], - "order": 6 - }, - { - "id": [ - "CDH4 Salt Formula", - "Hive" - ], - "order": 7 - }, - { - "id": [ - "CDH4 Salt Formula", - "Impala State Store" - ], - "order": 8 - }, - { - "id": [ - "CDH4 Salt Formula", - "Hue" - ], - "order": 10 - } - ], - "size": "m2.2xlarge" - }, - { - "count": 2, - "description": "DataNodes and Regionservers.", - "zone": "us-east-1d", - "title": "DataNodes and RegionServers", - "hostname_template": "{namespace}-dn-{index}", - "cloud_profile": "TO_BE_CHANGED", - "formula_components": [ - { - "id": [ - "OpenJDK and Oracle Java", - "Oracle Java 7 JDK" - ], - "order": 0 - }, - { - "id": [ - "CDH4 Salt Formula", - "DataNode" - ], - "order": 2 - }, - { - "id": [ - "CDH4 Salt Formula", - "HBase RegionServer" - ], - "order": 4 - }, - { - "id": [ - "CDH4 Salt Formula", - "Impala Server" - ], - "order": 9 - } - ], - "size": "m2.2xlarge" - } - ], - "description": "3 node stack with CDH 4", - "title": "cdh4-3node" -} \ No newline at end of file diff --git a/blueprints/cdh5-3node.json b/blueprints/cdh5-3node.json deleted file mode 100644 index 597441e..0000000 --- a/blueprints/cdh5-3node.json +++ /dev/null @@ -1,192 +0,0 @@ -{ - "public": false, - "properties": { - "java": { - "enable_jce": false, - "oracle": { - "cookies": "gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie", - "staging": "\/tmp\/.java_staging", - "jdk7": { - "rpm": "jdk-1.7.0_45", - "uri": "http:\/\/download.oracle.com\/otn-pub\/java\/jdk\/7u45-b18\/jdk-7u45-linux-x64.rpm" - } - } - }, - "cdh5": { - "hue": { - "secret_key": "CHANGE_THIS", - "start_service": true - }, - "dfs": { - "name_dir": "\/mnt\/hadoop\/hdfs\/nn", - "data_dir": "\/mnt\/hadoop\/hdfs\/data", - "permissions": true - }, - "mapred": { - "local_dir": "\/mnt\/yarn", - "system_dir": "\/hadoop\/system\/mapred", - "reduce_tasks": 6 - }, - "zookeeper": { - "data_dir": "\/mnt\/zk\/data", - "start_service": true - }, - "max_log_index": 1, - "hive": { - "home": "\/usr\/lib\/hive", - "metastore_password": "CHANGE_THIS", - "start_service": true, - "user": "hive" - }, - "io": { - "sort_mb": 250, - "sort_factor": 25 - }, - "oozie": { - "start_service": true - }, - "namenode": { - "start_service": true - }, - "hbase": { - "region_initial_heap": "1024m", - "region_max_heap": "1024m", - "tmp_dir": "\/mnt\/hbase\/tmp", - "start_service": true, - "master_young_gen": "256m", - "replication": 3, - "master_initial_heap": "1024m", - "manage_zk": true, - "log_dir": "\/mnt\/hbase\/logs", - "region_young_gen": "256m", - "master_max_heap": "1024m", - "jute_maxbuffer": 1000000 - }, - "landing_page": true, - "yarn": { - "max_container_size_mb": 11264 - }, - "version": 5, - "datanode": { - "start_service": true - }, - "security": { - "enable": false - }, - "impala": { - "version": "1.2.3", - "start_service": true - } - } - }, - "hosts": [ - { - "count": 1, - "description": "NameNode and Master", - "zone": "us-east-1d", - "title": "NameNode and Master", - "hostname_template": "{namespace}-nn", - "cloud_profile": "TO_BE_CHANGED", - "formula_components": [ - { - "id": [ - "OpenJDK and Oracle Java", - "Oracle Java 7 JDK" - ], - "order": 0 - }, - { - "id": [ - "CDH5 Salt Formula", - "NameNode" - ], - "order": 1 - }, - { - "id": [ - "CDH5 Salt Formula", - "HBase Master" - ], - "order": 3 - }, - { - "id": [ - "CDH5 Salt Formula", - "Oozie" - ], - "order": 5 - }, - { - "id": [ - "CDH5 Salt Formula", - "Pig" - ], - "order": 6 - }, - { - "id": [ - "CDH5 Salt Formula", - "Hive" - ], - "order": 7 - }, - { - "id": [ - "CDH5 Salt Formula", - "Impala State Store" - ], - "order": 8 - }, - { - "id": [ - "CDH5 Salt Formula", - "Hue" - ], - "order": 10 - } - ], - "size": "m2.2xlarge" - }, - { - "count": 2, - "description": "DataNodes and Regionservers.", - "zone": "us-east-1d", - "title": "DataNodes and RegionServers", - "hostname_template": "{namespace}-dn-{index}", - "cloud_profile": "TO_BE_CHANGED", - "formula_components": [ - { - "id": [ - "OpenJDK and Oracle Java", - "Oracle Java 7 JDK" - ], - "order": 0 - }, - { - "id": [ - "CDH5 Salt Formula", - "DataNode" - ], - "order": 2 - }, - { - "id": [ - "CDH5 Salt Formula", - "HBase RegionServer" - ], - "order": 4 - }, - { - "id": [ - "CDH5 Salt Formula", - "Impala Server" - ], - "order": 9 - } - ], - "size": "m2.2xlarge" - } - ], - "description": "3 node stack with CDH 5", - "title": "cdh5-3node" -} \ No newline at end of file diff --git a/bootstrap.yaml b/bootstrap.yaml index 16d892b..480cf07 100644 --- a/bootstrap.yaml +++ b/bootstrap.yaml @@ -1,8 +1,6 @@ -blueprints: - cdh4-3node: cdh4-3node.json - cdh5-3node: cdh5-3node.json +blueprints: {} formulas: java: https://github.com/stackdio-formulas/java-formula.git - cdh4: https://github.com/stackdio-formulas/cdh4-formula.git cdh5: https://github.com/stackdio-formulas/cdh5-formula.git + elasticsearch: https://github.com/stackdio-formulas/elasticsearch-formula.git From 7c83a177a91e9a3d37ecb567de6f4778772ee704 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 11 Nov 2015 14:08:52 -0600 Subject: [PATCH 14/90] Cleaned up README --- README.md | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f314a8b..5ef8a58 100644 --- a/README.md +++ b/README.md @@ -23,32 +23,31 @@ but in short you can install it on most systems like: Once you've got it, installing this tool goes something like: - mkvirtualenv stackdio-tooling + mkvirtualenv stackdio-client # assuming you are in whatever dir you cloned this repo to: - pip install --process-dependency-links . + pip install . -** The --process-dependency-links flag is only needed in pip 1.5.6 ** You'll see a few things scrolling by, but should be set after this. To use this later, you'll need to re-activate the virtualenv like: - workon stackdio-tooling + workon stackdio-client Whenever it's activated, `stackdio-cli` should be on your path. ## First Use The first time that you fire up `stackdio-cli`, you'll need to run the -`initial-setup` command. This will prompt you for your LDAP username and +`initial_setup` command. This will prompt you for your LDAP username and password, and store them securely in your OS keychain for later use. It will import some standard formula, and create a few commonly used blueprints. $ stackdio-cli - None @ https://stackd.corp.digitalreasoning.com/api/ + None @ None > initial_setup # YOU WILL BE WALKED THROUGH A SIMPLE SET OF QUESTIONS ## Stack Operations -All of the following assume that you have run `initial-setup` successfully. To +All of the following assume that you have run `initial_setup` successfully. To launch the cli, simply type: $ stackdio-cli @@ -64,7 +63,7 @@ included with this you do: > stacks launch cdh450-ipa-3 MYSTACKNAME **NOTE:** To avoid DNS namespace collisions, the stack name needs to be unique. -An easy way to ensure this is to include your name in the stack name. +An easy way to ensure this is to include your name in the stack name. ### Deleting Stacks When you are done with a stack you can delete it. This is destructive and @@ -98,10 +97,7 @@ There are various logs available that you can access with the `stacks logs` command. ## What's Next? -For anything not covered by this tool, you'll need to use the [web UI] or -[API] directly. See someone on the [pi team] with specific questions. +For anything not covered by this tool, you'll need to use the stackdio-server web UI or +API directly. For more information on that, check out http://docs.stackd.io. [virtualenvwrapper]: https://pypi.python.org/pypi/virtualenvwrapper -[web UI]: https://stackd.corp.digitalreasoning.com -[API]: https://stackd.corp.digitalreasoning.com/api -[pi team]: mailto:pi@digitalreasoning.com?subject=stackd.io%20questions From 0e2735ea34e7a9815eec9dba3e16e369175fbeda Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 13 Nov 2015 11:04:07 -0600 Subject: [PATCH 15/90] Updated how the request method gets called --- stackdio/client/http.py | 154 ++++++++++++++++++++++------------------ 1 file changed, 83 insertions(+), 71 deletions(-) diff --git a/stackdio/client/http.py b/stackdio/client/http.py index e3cf632..fd623cc 100644 --- a/stackdio/client/http.py +++ b/stackdio/client/http.py @@ -73,6 +73,8 @@ class Request(object): def __init__(self, dfunc=None, rfunc=None, quiet=False): super(Request, self).__init__() + self.obj = None + self.data_func = dfunc self.response_func = rfunc @@ -91,78 +93,88 @@ def data(self, dfunc): def response(self, rfunc): return type(self)(self.data_func, rfunc, self.quiet) - # Here's how the request actually happens + def __repr__(self): + if self.obj: + return ''.format(method, path, repr(self.obj)) + else: + return super(Request, self).__repr__() + + # We need this so we can save the client object as an attribute, and then it can be used + # in __call__ def __get__(self, obj, objtype=None): - def do_request(*args, **kwargs): - none_on_404 = kwargs.pop('none_on_404', False) - raise_for_status = kwargs.pop('raise_for_status', True) - - # Get what locals() would return directly after calling - # 'func' with the given args and kwargs - future_locals = getcallargs(self.data_func, *((obj,) + args), **kwargs) - - # Build the variable we'll inject - url = '{url}{path}'.format( - url=obj.url, - path=path.format(**future_locals) - ) - - if not self.quiet: - self._http_log.info("{0}: {1}".format(method, url)) - - data = None - if self.data_func: - data = json.dumps(self.data_func(obj, *args, **kwargs)) - - result = requests.request(method, - url, - data=data, - auth=obj._http_options['auth'], - headers=self.headers, - params=kwargs, - verify=obj._http_options['verify']) - - # Handle special conditions - if none_on_404 and result.status_code == 404: - return None - - elif result.status_code == 204: - return None - - elif raise_for_status: - try: - result.raise_for_status() - except Exception: - logger.error(result.text) - raise - - if jsonify: - response = result.json() - else: - response = result.text - - if method == 'GET' and paginate and jsonify: - res = response['results'] - - next_url = response['next'] - - while next_url: - next_page = requests.request(method, - next_url, - data=data, - auth=obj._http_options['auth'], - headers=self.headers, - params=kwargs, - verify=obj._http_options['verify']).json() - res.extend(next_page['results']) - next_url = next_page['next'] - - response = res - - # now process the result - return self.response_func(obj, response) - - return do_request + self.obj = obj + assert issubclass(objtype, HttpMixin) + return self + + # Here's how the request actually happens + def __call__(self, *args, **kwargs): + none_on_404 = kwargs.pop('none_on_404', False) + raise_for_status = kwargs.pop('raise_for_status', True) + + # Get what locals() would return directly after calling + # 'func' with the given args and kwargs + future_locals = getcallargs(self.data_func, *((self.obj,) + args), **kwargs) + + # Build the variable we'll inject + url = '{url}{path}'.format( + url=self.obj.url, + path=path.format(**future_locals) + ) + + if not self.quiet: + self._http_log.info("{0}: {1}".format(method, url)) + + data = None + if self.data_func: + data = json.dumps(self.data_func(self.obj, *args, **kwargs)) + + result = requests.request(method, + url, + data=data, + auth=self.obj._http_options['auth'], + headers=self.headers, + params=kwargs, + verify=self.obj._http_options['verify']) + + # Handle special conditions + if none_on_404 and result.status_code == 404: + return None + + elif result.status_code == 204: + return None + + elif raise_for_status: + try: + result.raise_for_status() + except Exception: + logger.error(result.text) + raise + + if jsonify: + response = result.json() + else: + response = result.text + + if method == 'GET' and paginate and jsonify: + res = response['results'] + + next_url = response['next'] + + while next_url: + next_page = requests.request(method, + next_url, + data=data, + auth=self.obj._http_options['auth'], + headers=self.headers, + params=kwargs, + verify=self.obj._http_options['verify']).json() + res.extend(next_page['results']) + next_url = next_page['next'] + + response = res + + # now process the result + return self.response_func(self.obj, response) return Request From df08398fe8ef530dadcccf2739939b4aadde4038 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 20 Nov 2015 14:32:25 -0600 Subject: [PATCH 16/90] Fixed an issue with creating blueprints --- stackdio/client/blueprint.py | 5 +---- stackdio/client/formula.py | 4 ++++ stackdio/client/http.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/stackdio/client/blueprint.py b/stackdio/client/blueprint.py index e65815a..051ea6b 100644 --- a/stackdio/client/blueprint.py +++ b/stackdio/client/blueprint.py @@ -43,10 +43,7 @@ def create_blueprint(self, blueprint): break for formula in used_formulas: - components = self._get( - '{0}?version={1}'.format(formula['components'], formula['version']), - jsonify=True, - )['results'] + components = self.list_components_for_version(formula['id'], formula['version']) for component in components: formula_map[component['sls_path']] = formula['uri'] diff --git a/stackdio/client/formula.py b/stackdio/client/formula.py index fd2d074..5e30935 100644 --- a/stackdio/client/formula.py +++ b/stackdio/client/formula.py @@ -46,6 +46,10 @@ def get_formula(self, formula_id): """Get a formula with matching id""" pass + @get('formulas/{formula_id}/components/?version={version}', paginate=True) + def list_components_for_version(self, formula_id, version): + pass + @get('formulas/', paginate=True) def search_formulas(self, **kwargs): """Get a formula with matching id""" diff --git a/stackdio/client/http.py b/stackdio/client/http.py index fd623cc..43efde1 100644 --- a/stackdio/client/http.py +++ b/stackdio/client/http.py @@ -158,7 +158,7 @@ def __call__(self, *args, **kwargs): if method == 'GET' and paginate and jsonify: res = response['results'] - next_url = response['next'] + next_url = response.get('next') while next_url: next_page = requests.request(method, @@ -169,7 +169,7 @@ def __call__(self, *args, **kwargs): params=kwargs, verify=self.obj._http_options['verify']).json() res.extend(next_page['results']) - next_url = next_page['next'] + next_url = next_page.get('next') response = res From 1808ed88f17ee98494fbfabc36a51b1658852fb7 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 20 Nov 2015 16:38:47 -0600 Subject: [PATCH 17/90] Added travis, fixed pep8 / pylint errors --- .travis.yml | 57 ++++++++++ pylintrc | 162 +++++++++++++++++++++++++++ setup.cfg | 5 + setup.py | 9 ++ stackdio/cli/__init__.py | 18 +-- stackdio/cli/blueprints/__init__.py | 4 +- stackdio/cli/blueprints/generator.py | 18 +-- stackdio/cli/mixins/blueprints.py | 4 +- stackdio/cli/mixins/bootstrap.py | 49 ++++---- stackdio/cli/mixins/formulas.py | 3 +- stackdio/cli/mixins/stacks.py | 69 ++++++------ stackdio/cli/polling.py | 2 - stackdio/client/http.py | 2 +- 13 files changed, 322 insertions(+), 80 deletions(-) create mode 100644 .travis.yml create mode 100644 pylintrc diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..61704cc --- /dev/null +++ b/.travis.yml @@ -0,0 +1,57 @@ +language: python + +python: + - "2.7" + - "3.2" + - "3.3" + - "3.4" + - "3.5" + +cache: + directories: + - $HOME/.cache/pip + +# Set up our environment +env: + NOSE_WITH_XUNIT: 1 + NOSE_WITH_COVERAGE: 1 + NOSE_COVER_BRANCHES: 1 + NOSE_COVER_INCLUSIVE: 1 + DJANGO_SETTINGS_MODULE: stackdio.server.settings.testing + +# So that we get a docker container +sudo: false + +## Customize dependencies +install: + - pip install -U pip + - pip install -U wheel + - pip install -U -e .[testing] + +## Customize test commands +before_script: + - pep8 stackdio/ && echo 'Finished PEP-8 Check Cleanly' || echo 'Finished PEP-8 Check With Errors' + - pylint stackdio/ && echo 'Finished Pylint Check Cleanly' || echo 'Finished Pylint Check With Errors' + +# Nothing to do here yet +script: + - date + +# Only build artifacts on success +after_success: + - coveralls + - export STACKDIO_VERSION=`python setup.py --version` + - python setup.py sdist + - python setup.py bdist_wheel + +deploy: + provider: releases + api_key: + secure: dJIj78Kl5nvtE2OpYl2I4ICEw20kLVXyr+eGOcWVV3kbU+PS6zKqOtLM+sGuVYNSbqWviRR9um6zbzjqS3S2wjFOdeStMogo19EKepSc0S97t6BkmbH0KooFuFah/YFOzLu+UBzDa3EETvgd1/988Eoojr0Ea2kZRJcvx/S0vDI= + file: + - dist/stackdio-server-${STACKDIO_VERSION}.tar.gz + - dist/stackdio_server-${STACKDIO_VERSION}-py2.py3-none-any.whl + skip_cleanup: true + on: + tags: true + repo: stackdio/stackdio diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..58e20e3 --- /dev/null +++ b/pylintrc @@ -0,0 +1,162 @@ +[MASTER] + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +init-hook=" + exec 'aW1wb3J0IG9zLCBzeXMKCmlmICdWSVJUVUFMX0VOVicgaW4gb3MuZW52aXJvbjoKCiAgICB2ZV9k \ + aXIgPSBvcy5lbnZpcm9uWydWSVJUVUFMX0VOViddCiAgICB2ZV9kaXIgaW4gc3lzLnBhdGggb3Ig \ + c3lzLnBhdGguaW5zZXJ0KDAsIHZlX2RpcikKICAgIGFjdGl2YXRlX3RoaXMgPSBvcy5wYXRoLmpv \ + aW4ob3MucGF0aC5qb2luKHZlX2RpciwgJ2JpbicpLCAnYWN0aXZhdGVfdGhpcy5weScpCgogICAg \ + IyBGaXggZm9yIHdpbmRvd3MKICAgIGlmIG5vdCBvcy5wYXRoLmV4aXN0cyhhY3RpdmF0ZV90aGlz \ + KToKICAgICAgICBhY3RpdmF0ZV90aGlzID0gb3MucGF0aC5qb2luKG9zLnBhdGguam9pbih2ZV9k \ + aXIsICdTY3JpcHRzJyksICdhY3RpdmF0ZV90aGlzLnB5JykKCiAgICBleGVjZmlsZShhY3RpdmF0 \ + ZV90aGlzLCBkaWN0KF9fZmlsZV9fPWFjdGl2YXRlX3RoaXMpKQo='.decode('base64')" +# Pickle collected data for later comparisons. +persistent=no + +#ignore= + +#load-plugins= + +[MESSAGES CONTROL] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable=E0001,E0100,E0101,E0102,E0103,E0104,E0105,E0106,E0107,E0108,E0202,E0203,E0211,E0213,E0221,E0222,E0601,E0602,E0603,E0604,E0701,E0702,E0710,E0711,E1001,E1002,E1003,E1102,E1111,E1120,E1121,E1122,E1123,E1124,E1200,E1201,E1205,E1206,E1300,E1301,E1302,E1303,E1304,E1305,E1306,E1310,C0112,C0121,C0202,C0321,C0322,C0323,C0324,W0101,W0102,W0104,W0105,W0106,W0107,W0108,W0109,W0120,W0141,W0150,W0199,W0211,W0221,W0222,W0223,W0231,W0232,W0233,W0301,W0311,W0312,W0331,W0332,W0333,W0401,W0402,W0403,W0406,W0410,W0601,W0602,W0604,W0614,W0623,W0701,W0702,W0703,W0710,W0711,W1001,W1111,W1201,W1300,W1301,W1401,W1402,F0202 + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=R, + I0011, + I0013, + E1101, + E1103, + C0102, + C0103, + C0111, + C0203, + C0204, + C0302, + C0330, + C1001, + W0110, + W0122, + W0142, + W0201, + W0212, + W0231, + W0232, + W0404, + W0511, + W0603, + W0612, + W0613, + W0621, + W0622, + W0631, + W0704, + F0220, + E8121, + E8122, + E8123, + E8124, + E8125, + E8126, + E8127, + E8128, + E8129, + E8131, + E8265 + +# Disabled: +# R* [refactoring suggestions & reports] +# I0011 (locally-disabling) +# I0013 (file-ignored) +# E1101 (no-member) [pylint isn't smart enough] +# E1103 (maybe-no-member) +# C0102 (blacklisted-name) [because it activates C0103 too] +# C0103 (invalid-name) +# C0111 (missing-docstring) +# C0203 (bad-mcs-method-argument) +# C0204 (bad-mcs-classmethod-argument) +# C0302 (too-many-lines) +# C0330 (bad-continuation) +# C1001 (old-style-class) [Way to many Meta inner classes that don't need it] +# W0110 (deprecated-lambda) +# W0122 (exec-statement) +# W0142 (star-args) +# W0201 (attribute-defined-outside-init) [done in several places in the codebase] +# W0212 (protected-access) +# W0231 (super-init-not-called) [this doesn't play well with our mixins] +# W0232 (no-init) [classes missing __init__ method - several Meta classes that don't need it] +# W0404 (reimported) [done intentionally for legit reasons] +# W0511 (fixme) [several outstanding instances currently in the codebase] +# W0603 (global-statement) +# W0612 (unused-variable) [unused return values] +# W0613 (unused-argument) +# W0621 (redefined-outer-name) +# W0622 (redefined-builtin) [many parameter names shadow builtins] +# W0631 (undefined-loop-variable) [~3 instances, seem to be okay] +# W0704 (pointless-except) [misnomer; "ignores the exception" rather than "pointless"] +# F0220 (unresolved-interface) +# +# E812* All PEP8 E12* +# E8265 PEP8 E265 - block comment should start with "# " + +[REPORTS] + +# Tells whether to display a full report or only the messages +reports=no + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +# output-format=parseable +msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=apply,input + + +[FORMAT] +# Maximum number of characters on a single line. +max-line-length=120 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +#additional-builtins= + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=BaseException \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 79bc678..f43dfb2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,3 +3,8 @@ # 3. If at all possible, it is good practice to do this. If you cannot, you # will need to generate wheels for each Python version that you support. universal=1 + +[pep8] + +max-line-length=100 +ignore=E501,E12 \ No newline at end of file diff --git a/setup.py b/setup.py index d8470b6..ce7cdce 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,12 @@ def test_python_version(): 'simplejson==3.4.0', ] +testing_requirements = [ + 'coveralls', + 'pep8', + 'pylint<=1.2.0', +] + if __name__ == "__main__": test_python_version() @@ -81,6 +87,9 @@ def test_python_version(): zip_safe=False, install_requires=requirements, dependency_links=[], + extras_require={ + 'testing': testing_requirements, + }, entry_points={ 'console_scripts': [ 'stackdio-cli=stackdio.cli:main', diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index 369c8b5..f7995e9 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -1,5 +1,7 @@ #!/usr/bin/env python +from __future__ import print_function + import argparse import json import os @@ -29,14 +31,14 @@ class StackdioShell( PROMPT = "\n{username} @ {url}\n> " HELP_CMDS = [ "account_summary", - "stacks", "blueprints", "formulas", - "initial_setup", "bootstrap", + "stacks", "blueprints", "formulas", + "initial_setup", "bootstrap", "help", "exit", "quit", ] Cmd.intro = """ ###################################################################### - s t a c k d . i o + s t a c k d . i o ###################################################################### """ @@ -69,7 +71,7 @@ def _init_stacks(self): self.stacks = StackdIO( base_url=self.config["url"], auth=( - self.config["username"], + self.config["username"], keyring.get_password(self.KEYRING_SERVICE, self.config.get("username") or "") ), verify=self.config.get('verify', True) @@ -133,7 +135,7 @@ def _setprompt(self): Cmd.prompt = self.colorize( self.PROMPT.format(**self.config), "blue") - + if not self.validated and self.config['url'] is not None: print(self.colorize(""" ## @@ -142,7 +144,7 @@ def _setprompt(self): ## 'initial_setup' to configure your account details. If you've already ## done that, there could be a network connection issue anywhere between ## your computer and your stackd.io instance, -## or your password may be incorrect, or ... etc. +## or your password may be incorrect, or ... etc. ## """, "green")) @@ -155,7 +157,7 @@ def _setprompt(self): def _print_summary(self, title, components): num_components = len(components) print("## {0} {1}{2}".format( - num_components, + num_components, title, "s" if num_components == 0 or num_components > 1 else "")) @@ -183,7 +185,7 @@ def do_account_summary(self, args=None): print("## Username: {0}".format(self.config["username"])) print("## Public Key:\n{0}".format(public_key)) - + self._print_summary("Formula", formulas) self._print_summary("Blueprint", blueprints) self._print_summary("Stack", stacks) diff --git a/stackdio/cli/blueprints/__init__.py b/stackdio/cli/blueprints/__init__.py index bb76c0a..71ee86e 100644 --- a/stackdio/cli/blueprints/__init__.py +++ b/stackdio/cli/blueprints/__init__.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import argparse import os import json @@ -53,4 +55,4 @@ def main(): main() except KeyboardInterrupt: sys.stderr.write('Aborting...\n') - sys.exit(1) \ No newline at end of file + sys.exit(1) diff --git a/stackdio/cli/blueprints/generator.py b/stackdio/cli/blueprints/generator.py index 5bb2acf..13eeefc 100644 --- a/stackdio/cli/blueprints/generator.py +++ b/stackdio/cli/blueprints/generator.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import sys import os import json @@ -49,7 +51,7 @@ def __init__(self, templates_path, output_stream=sys.stdout): undefined=StrictUndefined) # Add a filter for json - then we can put lists, etc in our templates - self.env.filters['json'] = lambda value: json.dumps(value) + self.env.filters['json'] = lambda value: json.dumps(value) # pylint: disable=unnecessary-lambda # Add a filter for newline escaping. Essentially just a wrapper around the # replace('\n', '\\n') filter @@ -101,7 +103,7 @@ def prompt(self, message): return '' else: return yaml_parsed - except: + except Exception: # yaml couldn't parse it return raw @@ -132,23 +134,23 @@ def find_set_vars(self, ast): # not allowed to be set by the template that did the include. for tag in ast.body: - if type(tag) == Assign: - if type(tag.node) == Const: + if isinstance(tag, Assign): + if isinstance(tag.node, Const): ret[tag.target.name] = tag.node.value else: # This is OK - RHS is an expr instead of a constant. Keep track of these so we # don't get an error later ret[tag.target.name] = self.sentinel - elif type(tag) == If: + elif isinstance(tag, If): # This is a simple naive check to see if we care about the variable being set. # Basically, if there is a variable inside an "if is not undefined" block, # It is OK if that variable is not set in a derived template or var file since it # is an optional configuration value - if type(tag.test) == Not and tag.test.node.name == 'undefined': + if isinstance(tag.test, Not) and tag.test.node.name == 'undefined': ret[tag.test.node.node.name] = None - elif type(tag) == Block: + elif isinstance(tag, Block): # Found a block, could be if statements within the block. Recursive call # TODO: not sure if this will cause a bug, since set statements are local to blocks ret.update(self.find_set_vars(tag)) @@ -164,7 +166,7 @@ def validate(self, template_file): :return: the set and unset variables :rtype: tuple """ - ### Get all the info for the CURRENT template + # Get all the info for the CURRENT template # Get the source of the template template_source = self.env.loader.get_source(self.env, template_file)[0] # parse it into an abstract syntax tree diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index 9fecd80..2e4f4b6 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import json import os import argparse @@ -266,5 +268,3 @@ def _get_blueprint_id(self, blueprint_name): "Blueprint [{0}] does not exist".format(blueprint_name), "red")) raise - - diff --git a/stackdio/cli/mixins/bootstrap.py b/stackdio/cli/mixins/bootstrap.py index 9cba14a..fcd78a6 100644 --- a/stackdio/cli/mixins/bootstrap.py +++ b/stackdio/cli/mixins/bootstrap.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import getpass import json import os @@ -26,7 +28,7 @@ def __init__(self): self.stacks = None self.config = None self.bootstrap_data = None - + def do_initial_setup(self, args=None): """Perform setup for your stackd.io account""" @@ -53,7 +55,7 @@ def do_initial_setup(self, args=None): self.config[k] = v # Validate the url, prompt for a new one if invalid - if not self.config.has_key('url') or not self._test_url(self.config['url']): + if 'url' not in self.config or not self._test_url(self.config['url']): print("There seems to be an issue with the url you provided.") self.config['url'] = None self._get_url() @@ -75,7 +77,7 @@ def do_initial_setup(self, args=None): self._choose_profile() # Only prompt for default profile if it's not already there - if not self.config.has_key('profile') \ + if 'profile' not in self.config \ or 'provider' not in self.config \ or 'provider_type' not in self.config: if 'profile' in self.config: @@ -108,11 +110,11 @@ def do_bootstrap(self, args=None): "red")) return - if not self.config.has_key('profile'): - print(self.colorize("You must have a default profile in order to run bootstrap. Run 'initial_setup'", - "red")) + if 'profile' not in self.config: + print(self.colorize("You must have a default profile in order to run bootstrap. " + "Run 'initial_setup'", "red")) return - + print("Bootstrapping your account") custom_bootstrap = raw_input("Do you have a custom bootstrap yaml file (y/n)? ") @@ -184,11 +186,11 @@ def _get_user_creds(self): """Prompt user for credentials""" self.config["username"] = raw_input("What is your username? ") - + if keyring.get_password(self.KEYRING_SERVICE, self.config["username"]): print("Password already stored for {0}".format(self.config["username"])) keep_password = raw_input("Keep existing password (y/n)? ") - else: + else: keep_password = "n" if keep_password in ["n", "N"]: @@ -201,24 +203,26 @@ def _choose_profile(self): """Prompt user for a default provider/profile""" auth = (self.config['username'], keyring.get_password(self.KEYRING_SERVICE, self.config['username'])) - profiles = requests.get(self.config['url']+"profiles/", auth=auth, verify=False).json()['results'] + profiles = requests.get(self.config['url'] + "profiles/", + auth=auth, verify=False).json()['results'] print("Choose a default profile:") idx = 0 for profile in profiles: - print(str(idx)+':') - print(' '+profile['title']) - print(' '+profile['description']) + print(str(idx) + ':') + print(' ' + profile['title']) + print(' ' + profile['description']) idx += 1 - print + print('') choice = int(raw_input("Enter the number of the profile you would like to choose: ")) provider = requests.get( - self.config['url']+"providers/{0}/".format(profiles[choice]['cloud_provider']), + self.config['url'] + "providers/{0}/".format(profiles[choice]['cloud_provider']), auth=auth, - verify=False).json() + verify=False + ).json() self.config['profile'] = profiles[choice]['title'] self.config['provider'] = provider['title'] @@ -255,7 +259,7 @@ def _bootstrap_account(self): else: print("Setting public key") self.stacks.set_public_key(public_key) - self.has_public_key = True + self.has_public_key = True def _bootstrap_formulas(self): """Import and wait for formulas to become ready""" @@ -266,7 +270,7 @@ def _check_formulas(): if formula.get("status") != "complete": return False return True - + formulas = self.bootstrap_data.get("formulas", []) print("Importing {0} formula{1}".format( len(formulas), @@ -284,7 +288,7 @@ def _check_formulas(): print(self.colorize( "\nTIMEOUT - formulas failed to finish importing, monitor with `formulas list`", "red")) - + def _bootstrap_blueprints(self): """Create blueprints""" @@ -296,6 +300,7 @@ def _bootstrap_blueprints(self): print(" - {0} // {1}".format(name, blueprint)) # Get the blueprints relative to the bootstrap config file - self._create_blueprint([os.path.join(os.path.dirname(self.BOOTSTRAP_FILE), "blueprints", - blueprint)], bootstrap=True) - + self._create_blueprint( + [os.path.join(os.path.dirname(self.BOOTSTRAP_FILE), "blueprints", blueprint)], + bootstrap=True, + ) diff --git a/stackdio/cli/mixins/formulas.py b/stackdio/cli/mixins/formulas.py index 1f0f8be..54664ce 100644 --- a/stackdio/cli/mixins/formulas.py +++ b/stackdio/cli/mixins/formulas.py @@ -1,3 +1,5 @@ +from __future__ import print_function + from cmd2 import Cmd @@ -78,4 +80,3 @@ def _delete_formula(self, args): self.stacks.delete_formula(formula_id) print("Formula deleted, try the 'list' command to monitor status") - diff --git a/stackdio/cli/mixins/stacks.py b/stackdio/cli/mixins/stacks.py index e54023e..8be3dea 100644 --- a/stackdio/cli/mixins/stacks.py +++ b/stackdio/cli/mixins/stacks.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import json from cmd2 import Cmd @@ -8,7 +10,7 @@ class StackMixin(Cmd): STACK_ACTIONS = ["start", "stop", "launch_existing", "terminate", "provision", "custom"] - STACK_COMMANDS = ["list", "launch_from_blueprint", "history", "hostnames", + STACK_COMMANDS = ["list", "launch_from_blueprint", "history", "hostnames", "delete", "logs", "access_rules"] + STACK_ACTIONS VALID_LOGS = { @@ -23,7 +25,7 @@ class StackMixin(Cmd): def do_stacks(self, arg): """Entry point to controlling.""" - + USAGE = "Usage: stacks COMMAND\nWhere COMMAND is one of: %s" % ( ", ".join(self.STACK_COMMANDS)) @@ -54,7 +56,7 @@ def do_stacks(self, arg): else: print(USAGE) - + def complete_stacks(self, text, line, begidx, endidx): # not using line, begidx, or endidx, thus the following pylint disable # pylint: disable=W0613 @@ -74,9 +76,9 @@ def _list_stacks(self): self._print_summary("Stack", stacks) def _launch_stack(self, args): - """Launch a stack from a blueprint. + """Launch a stack from a blueprint. Must provide blueprint name and stack name""" - + if len(args) != 2: print("Usage: stacks launch BLUEPRINT_NAME STACK_NAME") return @@ -94,7 +96,7 @@ def _launch_stack(self, args): print("Launching stack [{0}] from blueprint [{1}]".format( stack_name, blueprint_name)) - + stack_data = { "blueprint": blueprint_id, "title": stack_name, @@ -131,7 +133,7 @@ def _stack_action(self, args): print("Usage: stacks custom STACK_NAME HOST_TARGET COMMAND") print("Where command can be arbitrarily long with spaces") return - + if args[0] not in self.STACK_ACTIONS: print(self.colorize( "Invalid action - must be one of {0}".format(self.STACK_ACTIONS), @@ -165,7 +167,6 @@ def _stack_action(self, args): def _stack_history(self, args): """Print recent history for a stack""" - # pylint: disable=W0142 NUM_EVENTS = 20 if len(args) < 1: @@ -183,7 +184,7 @@ def _stack_hostnames(self, args): if len(args) < 1: print("Usage: stacks hostnames STACK_NAME") return - + stack_id = self._get_stack_id(args[0]) try: fqdns = self.stacks.describe_hosts(stack_id) @@ -208,7 +209,7 @@ def _stack_delete(self, args): if really not in ["y", "Y"]: print("Aborting deletion") return - + results = self.stacks.delete_stack(stack_id) print("Delete stack results: {0}".format(results)) print(self.colorize( @@ -218,7 +219,7 @@ def _stack_delete(self, args): def _stack_logs(self, args): """Get logs for a stack""" - + MAX_LINES = 25 if len(args) < 2: @@ -230,7 +231,7 @@ def _stack_logs(self, args): return if len(args) >= 3: - max_lines = args[2] + max_lines = args[2] else: max_lines = MAX_LINES @@ -245,37 +246,37 @@ def _stack_logs(self, args): def _stack_access_rules(self, args): """Get access rules for a stack""" - COMMANDS = ["list", "add", "delete"] if len(args) < 2 or args[0] not in COMMANDS: print("Usage: stacks access_rules COMMAND STACK_NAME") print("Where COMMAND is one of: %s" % (", ".join(COMMANDS))) return - + if args[0] == "list": stack_id = self.stacks.get_stack_id(args[1]) groups = self.stacks.list_access_rules(stack_id) - print "##", len(groups), "Access Groups" + print("## {0} Access Groups".format(len(groups))) for group in groups: - print "- Name:", group['blueprint_host_definition']['title'] - print " Description:", group['blueprint_host_definition']['description'] - print " Rules:" + print("- Name: {0}".format(group['blueprint_host_definition']['title'])) + print(" Description: {0}".format(group['blueprint_host_definition']['description'])) + print(" Rules:") for rule in group['rules']: - print " ",rule['protocol'], + print(" {0}".format(rule['protocol']), end='') if rule['from_port'] == rule['to_port']: - print "port", rule['from_port'], "allows", + print("port {0} allows".format(rule['from_port']), end='') else: - print "ports", rule['from_port']+"-"+rule['to_port'], "allow", - print rule['rule'] - print + print("ports {0}-{1} allow".format(rule['from_port'], + rule['to_port']), end='') + print(rule['rule']) + print('') return elif args[0] == "add": if len(args) < 3: print("Usage: stacks access_rules add STACK_NAME GROUP_NAME") return - + stack_id = self.stacks.get_stack_id(args[1]) group_id = self.stacks.get_access_rule_id(stack_id, args[2]) @@ -306,16 +307,16 @@ def _stack_access_rules(self, args): rules = self.stacks.list_rules_for_group(group_id) - print + print('') for rule in rules: - print str(index)+") ", rule['protocol'], + print("{0}) {1}".format(index, rule['protocol']), end='') if rule['from_port'] == rule['to_port']: - print "port", rule['from_port'], "allows", + print("port {0} allows".format(rule['from_port']), end='') else: - print "ports", rule['from_port']+"-"+rule['to_port'], "allow", - print rule['rule'] + print("ports {0}-{1} allow".format(rule['from_port'], rule['to_port']), end='') + print(rule['rule']) index += 1 - print + print('') delete_index = int(raw_input("Enter the index of the rule to delete: ")) data = rules[delete_index] @@ -325,10 +326,8 @@ def _stack_access_rules(self, args): self.stacks.edit_access_rule(group_id, data) - print - - args[0] = "list" - - self._stack_access_rules(args) + print('') + args[0] = "list" + self._stack_access_rules(args) diff --git a/stackdio/cli/polling.py b/stackdio/cli/polling.py index 5fbbc79..aa1c5a3 100644 --- a/stackdio/cli/polling.py +++ b/stackdio/cli/polling.py @@ -10,7 +10,6 @@ class TimeoutException(Exception): def poll_and_wait(func, args=None, sleep_time=2, max_time=120): """Execute func in increments of sleep_time for no more than max_time. Raise TimeoutException if we're not successful in max_time""" - #pylint: disable=W0142 args = args or [] current_time = 0 @@ -25,4 +24,3 @@ def poll_and_wait(func, args=None, sleep_time=2, max_time=120): if not success: raise TimeoutException() - diff --git a/stackdio/client/http.py b/stackdio/client/http.py index 43efde1..a82849f 100644 --- a/stackdio/client/http.py +++ b/stackdio/client/http.py @@ -122,7 +122,7 @@ def __call__(self, *args, **kwargs): ) if not self.quiet: - self._http_log.info("{0}: {1}".format(method, url)) + self._http_log.info("%s: %s", method, url) data = None if self.data_func: From 4ee2b2259e61a4c8da72c2989986feee383d21e9 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 20 Nov 2015 16:42:32 -0600 Subject: [PATCH 18/90] Fixed deploy step --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 61704cc..16a8542 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,11 +47,11 @@ after_success: deploy: provider: releases api_key: - secure: dJIj78Kl5nvtE2OpYl2I4ICEw20kLVXyr+eGOcWVV3kbU+PS6zKqOtLM+sGuVYNSbqWviRR9um6zbzjqS3S2wjFOdeStMogo19EKepSc0S97t6BkmbH0KooFuFah/YFOzLu+UBzDa3EETvgd1/988Eoojr0Ea2kZRJcvx/S0vDI= + secure: T4jI1aZQ+wDJBgGxcbdrtLz3zpXA9yZwmrsm8d3GqEGxApMtkKLWq0uqf86C8VkqaY6p4Nm1a/PTApV1isbuSoJbdeMVJA1MlYB/G7QMK7eI8nFqkw7Q4jzuOdEC0D1CPZx7ZWBn0bYxSRTcSeQSnGeGDy2KxekGSZFfIxe4APo= file: - - dist/stackdio-server-${STACKDIO_VERSION}.tar.gz - - dist/stackdio_server-${STACKDIO_VERSION}-py2.py3-none-any.whl + - dist/stackdio-${STACKDIO_VERSION}.tar.gz + - dist/stackdio-${STACKDIO_VERSION}-py2.py3-none-any.whl skip_cleanup: true on: tags: true - repo: stackdio/stackdio + repo: stackdio/stackdio-python-client From c0089b1f95b4f251b98379efcc5e23d85fe6450f Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 20 Nov 2015 16:43:51 -0600 Subject: [PATCH 19/90] Removed extra env var --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 16a8542..0552c7f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,6 @@ env: NOSE_WITH_COVERAGE: 1 NOSE_COVER_BRANCHES: 1 NOSE_COVER_INCLUSIVE: 1 - DJANGO_SETTINGS_MODULE: stackdio.server.settings.testing # So that we get a docker container sudo: false From e7716b35a34c96d02ed6da26bba30ff3bb100723 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 20 Nov 2015 16:46:01 -0600 Subject: [PATCH 20/90] Fix for python3 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ce7cdce..a5fda5e 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,8 @@ def test_python_version(): # Set version __version__ = '0.0.0' # Explicit default -execfile('stackdio/client/version.py') +with open('stackdio/client/version.py') as f: + exec(f.read()) SHORT_DESCRIPTION = ('A cloud deployment, automation, and orchestration ' From 6dfc5969ecedb733469834914a2f2948f3342084 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 20 Nov 2015 16:49:51 -0600 Subject: [PATCH 21/90] Fixed a couple more python3 things --- stackdio/cli/blueprints/generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stackdio/cli/blueprints/generator.py b/stackdio/cli/blueprints/generator.py index 13eeefc..d4c90d3 100644 --- a/stackdio/cli/blueprints/generator.py +++ b/stackdio/cli/blueprints/generator.py @@ -296,12 +296,12 @@ def generate(self, template_file, var_files=(), variables=None, prompt=False, de except TemplateNotFound: self.error_exit('Your template file {0} was not found.'.format(template_file)) - except TemplateSyntaxError, e: + except TemplateSyntaxError as e: self.error_exit('Invalid template error at line {0}:\n{1}'.format( e.lineno, str(e) )) - except UndefinedError, e: + except UndefinedError as e: self.error_exit('Missing variable: {0}'.format(str(e))) except ValueError: self.error_exit('Invalid JSON. Check your template file.') From ce610dfa935a6f129c5f8c5ad9c1178a68946eb1 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 20 Nov 2015 16:52:17 -0600 Subject: [PATCH 22/90] Travis doesn't work with python 3.5 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0552c7f..201e105 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ python: - "3.2" - "3.3" - "3.4" - - "3.5" cache: directories: From 11bfd57c84cbffe6a0d7779cecf1379ad0926ee2 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 20 Nov 2015 16:54:24 -0600 Subject: [PATCH 23/90] Added travis badge to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5ef8a58..9cf19a1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ stackdio-python-client ====================== +[![Build Status](https://travis-ci.org/stackdio/stackdio-python-client.svg?branch=master)](https://travis-ci.org/stackdio/stackdio-python-client) + The canonical Python client and cli for the stackd.io API From 597cd428834064c6eef5f849042571f85d31e742 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 1 Dec 2015 17:39:50 -0600 Subject: [PATCH 24/90] Removed a couple unused pieces --- stackdio/cli/__init__.py | 15 ++++----------- stackdio/client/__init__.py | 14 ++++---------- stackdio/client/exceptions.py | 4 ---- stackdio/client/http.py | 33 ++++++--------------------------- 4 files changed, 14 insertions(+), 52 deletions(-) diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index f7995e9..33eca52 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -7,22 +7,16 @@ import os import sys -from cmd2 import Cmd import keyring +from cmd2 import Cmd from requests import ConnectionError - -from stackdio.client import StackdIO - from stackdio.cli import mixins +from stackdio.client import StackdIO -class StackdioShell( - Cmd, - mixins.bootstrap.BootstrapMixin, - mixins.stacks.StackMixin, - mixins.formulas.FormulaMixin, - mixins.blueprints.BlueprintMixin): +class StackdioShell(Cmd, mixins.bootstrap.BootstrapMixin, mixins.stacks.StackMixin, + mixins.formulas.FormulaMixin, mixins.blueprints.BlueprintMixin): CFG_DIR = os.path.expanduser("~/.stackdio-cli/") CFG_FILE = os.path.join(CFG_DIR, "config.json") @@ -192,7 +186,6 @@ def do_account_summary(self, args=None): def main(): - parser = argparse.ArgumentParser( description="Invoke the stackdio cli") parser.add_argument("--debug", action="store_true", help="Enable debugging output") diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index 4151431..d55bfad 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -17,17 +17,15 @@ import logging -from .http import get, post, patch -from .exceptions import BlueprintException, StackException, IncompatibleVersionException - +from .account import AccountMixin from .blueprint import BlueprintMixin +from .exceptions import BlueprintException, StackException, IncompatibleVersionException from .formula import FormulaMixin -from .account import AccountMixin +from .http import get, post, patch from .image import ImageMixin from .region import RegionMixin from .settings import SettingsMixin from .stack import StackMixin - from .version import _parse_version_string logger = logging.getLogger(__name__) @@ -37,10 +35,7 @@ class StackdIO(BlueprintMixin, FormulaMixin, AccountMixin, ImageMixin, RegionMixin, StackMixin, SettingsMixin): def __init__(self, protocol="https", host="localhost", port=443, - base_url=None, auth=None, auth_admin=None, - verify=True): - """auth_admin is optional, only needed for creating provider, profile, - and base security groups""" + base_url=None, auth=None, verify=True): super(StackdIO, self).__init__(auth=auth, verify=verify) if base_url: @@ -52,7 +47,6 @@ def __init__(self, protocol="https", host="localhost", port=443, port=port) self.auth = auth - self.auth_admin = auth_admin _, self.version = _parse_version_string(self.get_version()) diff --git a/stackdio/client/exceptions.py b/stackdio/client/exceptions.py index b386093..a96cb7d 100644 --- a/stackdio/client/exceptions.py +++ b/stackdio/client/exceptions.py @@ -24,10 +24,6 @@ class BlueprintException(Exception): pass -class NoAdminException(Exception): - pass - - class IncompatibleVersionException(Exception): pass diff --git a/stackdio/client/http.py b/stackdio/client/http.py index a82849f..8a47f9f 100644 --- a/stackdio/client/http.py +++ b/stackdio/client/http.py @@ -21,17 +21,16 @@ import logging import requests -from functools import wraps from inspect import getcallargs -from .exceptions import NoAdminException - logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) HTTP_INSECURE_MESSAGE = "\n".join([ "You have chosen not to verify ssl connections.", "This is insecure, but it's your choice.", - "This has been your single warning."]) + "This has been your single warning." +]) class HttpMixin(object): @@ -49,7 +48,7 @@ def __init__(self, auth=None, verify=True): 'auth': auth, 'verify': verify, } - self._http_log = logging.getLogger(__name__) + self._http_log = logger if not verify: if self._http_log.handlers: @@ -95,7 +94,8 @@ def response(self, rfunc): def __repr__(self): if self.obj: - return ''.format(method, path, repr(self.obj)) + return (''.format(method, path, repr(self.obj))) else: return super(Request, self).__repr__() @@ -180,7 +180,6 @@ def __call__(self, *args, **kwargs): # Define the decorators for all the methods - def get(path, paginate=False, jsonify=True): return request(path, 'GET', paginate=paginate, jsonify=jsonify) @@ -207,23 +206,3 @@ def patch(path): def delete(path): return request(path, 'DELETE') - - -def use_admin_auth(func): - - @wraps(func) - def wrapper(obj, *args, **kwargs): - # Save and set the auth to use the admin auth - auth = obj._http_options['auth'] - try: - obj._http_options['auth'] = obj._http_options['admin'] - except KeyError: - raise NoAdminException("No admin credentials were specified") - - # Call the original function - output = func(*args, **kwargs) - - # Set the auth back to the original - obj._http_options['auth'] = auth - return output - return wrapper From eea4bd2f35f27c0f63d8b680ba4105d212041626 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 2 Dec 2015 10:38:54 -0600 Subject: [PATCH 25/90] First shot at click --- setup.py | 8 +- stackdio/cli/__init__.py | 165 +++++++++++++++++++++++++++++++++------ 2 files changed, 145 insertions(+), 28 deletions(-) diff --git a/setup.py b/setup.py index a5fda5e..38136b8 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,8 @@ # limitations under the License. # +from __future__ import unicode_literals + import os import sys @@ -49,8 +51,9 @@ def test_python_version(): requirements = [ 'Jinja2==2.7.3', 'PyYAML==3.11', - 'cmd2==0.6.7', + 'click>=6.0,<7.0', 'keyring==3.7', + 'readline', 'requests>=2.4.0,<2.6.0', 'simplejson==3.4.0', ] @@ -64,8 +67,6 @@ def test_python_version(): if __name__ == "__main__": test_python_version() - # Call the setup method from setuptools that does all the heavy lifting - # of packaging stackdio-client setup( name='stackdio', version=__version__, @@ -100,7 +101,6 @@ def test_python_version(): classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: Web Environment', - 'Framework :: Django', 'Intended Audience :: Developers', 'Intended Audience :: Information Technology', 'Intended Audience :: System Administrators', diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index 33eca52..d1e36a7 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -2,20 +2,136 @@ from __future__ import print_function -import argparse import json import os import sys +from cmd import Cmd +import click import keyring -from cmd2 import Cmd from requests import ConnectionError from stackdio.cli import mixins from stackdio.client import StackdIO +from stackdio.client.version import __version__ -class StackdioShell(Cmd, mixins.bootstrap.BootstrapMixin, mixins.stacks.StackMixin, +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + +CFG_DIR = os.path.expanduser("~/.stackdio-cli/") +CFG_FILE = os.path.join(CFG_DIR, "config.json") +KEYRING_SERVICE = "stackdio_cli" + + +def get_client(): + if not os.path.isfile(CFG_FILE): + click.echo('It looks like you haven\'t used this CLI before. Please run ' + '`stackdio-cli configure`'.format(sys.argv[0])) + sys.exit(1) + + config = json.load(open(CFG_FILE, 'r')) + config['blueprint_dir'] = os.path.expanduser(config.get('blueprint_dir', '')) + + return StackdIO( + base_url=config["url"], + auth=( + config["username"], + keyring.get_password(KEYRING_SERVICE, config.get("username") or "") + ), + verify=config.get('verify', True) + ) + + +def get_invoke(ctx, command): + def invoke(self, arg): + return ctx.invoke(command) + return invoke + + +def get_help(command): + def help(self): + click.echo(command.help) + return help + + +def get_shell(ctx): + + # Make it a new-style class so we can use super! + class StackdioShell(Cmd, object): + + def __init__(self): + super(StackdioShell, self).__init__() + + prompt = 'stackdio > ' + + def emptyline(self): + pass + + def do_quit(self, arg): + return True + + def do_exit(self, arg): + return True + + def do_EOF(self, arg): + return True + + def get_names(self): + ret = super(StackdioShell, self).get_names() + # We don't want to display + ret.remove('do_EOF') + return ret + + for name, command in ctx.command.commands.items(): + setattr(StackdioShell, 'do_%s' % name.replace('-', '_'), get_invoke(ctx, command)) + + if command.help is not None: + setattr(StackdioShell, 'help_%s' % name.replace('-', '_'), get_help(command)) + + return StackdioShell() + + +@click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True) +@click.version_option(__version__, '-v', '--version') +@click.pass_context +def stackdio(ctx): + ctx.client = get_client() + + if ctx.invoked_subcommand is None: + shell = get_shell(ctx) + shell.cmdloop() + + +@stackdio.group() +def stacks(): + pass + + +@stackdio.group() +def blueprints(): + """ + Foo bar + """ + pass + + +@stackdio.group() +def formulas(): + pass + + +@stackdio.command() +def configure(): + pass + + +@stackdio.command('server-version') +def server_version(): + client = get_client() + click.echo('stackdio-server, version {0}'.format(client.get_version())) + + +class StackdioShell(mixins.bootstrap.BootstrapMixin, mixins.stacks.StackMixin, mixins.formulas.FormulaMixin, mixins.blueprints.BlueprintMixin): CFG_DIR = os.path.expanduser("~/.stackdio-cli/") @@ -30,14 +146,13 @@ class StackdioShell(Cmd, mixins.bootstrap.BootstrapMixin, mixins.stacks.StackMix "help", "exit", "quit", ] - Cmd.intro = """ -###################################################################### - s t a c k d . i o -###################################################################### -""" +# Cmd.intro = """ +# ###################################################################### +# s t a c k d . i o +# ###################################################################### +# """ def __init__(self): - Cmd.__init__(self) mixins.bootstrap.BootstrapMixin.__init__(self) self._load_config() if 'url' in self.config and self.config['url']: @@ -186,21 +301,23 @@ def do_account_summary(self, args=None): def main(): - parser = argparse.ArgumentParser( - description="Invoke the stackdio cli") - parser.add_argument("--debug", action="store_true", help="Enable debugging output") - args = parser.parse_args() - - # an ugly hack to work around the fact that cmd2 is using optparse to parse - # arguments for the commands; not sure what the "right" fix is, but as long - # as we assume that we don't want any of our arguments to get passed into - # the cmdloop this seems ok - sys.argv = sys.argv[0:1] - - shell = StackdioShell() - if args.debug: - shell.debug = True - shell.cmdloop() + stackdio() + + # parser = argparse.ArgumentParser( + # description="Invoke the stackdio cli") + # parser.add_argument("--debug", action="store_true", help="Enable debugging output") + # args = parser.parse_args() + # + # # an ugly hack to work around the fact that cmd2 is using optparse to parse + # # arguments for the commands; not sure what the "right" fix is, but as long + # # as we assume that we don't want any of our arguments to get passed into + # # the cmdloop this seems ok + # sys.argv = sys.argv[0:1] + # + # shell = StackdioShell() + # if args.debug: + # shell.debug = True + # shell.cmdloop() if __name__ == '__main__': From feb2dcb3961da88ad5b3177f5b72ea456411c03a Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 2 Dec 2015 14:11:33 -0600 Subject: [PATCH 26/90] Got autocomplete working in the shell\! --- stackdio/cli/__init__.py | 69 +++++--------------- stackdio/cli/shell.py | 133 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 55 deletions(-) create mode 100644 stackdio/cli/shell.py diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index d1e36a7..f8686f4 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -12,6 +12,7 @@ from requests import ConnectionError from stackdio.cli import mixins +from stackdio.cli.shell import get_shell from stackdio.client import StackdIO from stackdio.client.version import __version__ @@ -42,64 +43,22 @@ def get_client(): ) -def get_invoke(ctx, command): - def invoke(self, arg): - return ctx.invoke(command) - return invoke +class StackdioObj(object): - -def get_help(command): - def help(self): - click.echo(command.help) - return help - - -def get_shell(ctx): - - # Make it a new-style class so we can use super! - class StackdioShell(Cmd, object): - - def __init__(self): - super(StackdioShell, self).__init__() - - prompt = 'stackdio > ' - - def emptyline(self): - pass - - def do_quit(self, arg): - return True - - def do_exit(self, arg): - return True - - def do_EOF(self, arg): - return True - - def get_names(self): - ret = super(StackdioShell, self).get_names() - # We don't want to display - ret.remove('do_EOF') - return ret - - for name, command in ctx.command.commands.items(): - setattr(StackdioShell, 'do_%s' % name.replace('-', '_'), get_invoke(ctx, command)) - - if command.help is not None: - setattr(StackdioShell, 'help_%s' % name.replace('-', '_'), get_help(command)) - - return StackdioShell() + def __init__(self, ctx): + super(StackdioObj, self).__init__() + self.shell = get_shell(ctx) + self.client = get_client() @click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True) @click.version_option(__version__, '-v', '--version') @click.pass_context def stackdio(ctx): - ctx.client = get_client() + ctx.obj = StackdioObj(ctx) if ctx.invoked_subcommand is None: - shell = get_shell(ctx) - shell.cmdloop() + ctx.obj.shell.cmdloop() @stackdio.group() @@ -131,7 +90,7 @@ def server_version(): click.echo('stackdio-server, version {0}'.format(client.get_version())) -class StackdioShell(mixins.bootstrap.BootstrapMixin, mixins.stacks.StackMixin, +class StackdioShell(Cmd, mixins.bootstrap.BootstrapMixin, mixins.stacks.StackMixin, mixins.formulas.FormulaMixin, mixins.blueprints.BlueprintMixin): CFG_DIR = os.path.expanduser("~/.stackdio-cli/") @@ -146,11 +105,11 @@ class StackdioShell(mixins.bootstrap.BootstrapMixin, mixins.stacks.StackMixin, "help", "exit", "quit", ] -# Cmd.intro = """ -# ###################################################################### -# s t a c k d . i o -# ###################################################################### -# """ + intro = """ +###################################################################### + s t a c k d . i o +###################################################################### +""" def __init__(self): mixins.bootstrap.BootstrapMixin.__init__(self) diff --git a/stackdio/cli/shell.py b/stackdio/cli/shell.py new file mode 100644 index 0000000..43d5510 --- /dev/null +++ b/stackdio/cli/shell.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014, Digital Reasoning +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from cmd import Cmd + +import click + +from stackdio.client.version import __version__ + + +try: + import readline +except ImportError: + readline = None + + +HIST_FILE = os.path.join(os.path.expanduser('~'), '.stackdio-cli', 'history') + + +def get_invoke(ctx, command): + def invoke(self, arg): + return ctx.invoke(command) + return invoke + + +def get_help(command): + def help(self): + click.echo(command.help) + return help + + +def get_shell(ctx): + + # Make it a new-style class so we can use super + class StackdioShell(Cmd, object): + + def __init__(self): + super(StackdioShell, self).__init__() + self.old_completer = None + + def preloop(self): + # read our history + if readline: + try: + readline.read_history_file(HIST_FILE) + except IOError: + pass + + def postloop(self): + # Write our history + if readline: + readline.write_history_file(HIST_FILE) + + prompt = 'stackdio > ' + intro = 'stackdio shell v{0}'.format(__version__) + + # We need to override this to fix readline + def cmdloop(self, intro=None): + self.preloop() + if self.use_rawinput and self.completekey and readline: + self.old_completer = readline.get_completer() + readline.set_completer(self.complete) + if 'libedit' in readline.__doc__: + # For mac + readline.parse_and_bind('bind ^I rl_complete') + else: + # for other platforms + readline.parse_and_bind(self.completekey + ': complete') + try: + if intro is not None: + self.intro = intro + if self.intro: + self.stdout.write(str(self.intro)+"\n") + stop = None + while not stop: + if self.cmdqueue: + line = self.cmdqueue.pop(0) + else: + if self.use_rawinput: + try: + line = raw_input(self.prompt) + except EOFError: + # We just want to quit here + self.stdout.write('\n') + break + else: + self.stdout.write(self.prompt) + self.stdout.flush() + line = self.stdin.readline() + if not len(line): + line = 'EOF' + else: + line = line.rstrip('\r\n') + line = self.precmd(line) + stop = self.onecmd(line) + stop = self.postcmd(stop, line) + + finally: + self.postloop() + if self.use_rawinput and self.completekey and readline: + readline.set_completer(self.old_completer) + + def emptyline(self): + pass + + def do_quit(self, arg): + return True + + def do_exit(self, arg): + return True + + for name, command in ctx.command.commands.items(): + setattr(StackdioShell, 'do_%s' % name.replace('-', '_'), get_invoke(ctx, command)) + + if command.help is not None: + setattr(StackdioShell, 'help_%s' % name.replace('-', '_'), get_help(command)) + + return StackdioShell() From 0a3e0c80c7ad041fe020e1f6946e632ccfeb0c31 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Mon, 7 Dec 2015 13:50:03 -0600 Subject: [PATCH 27/90] Make history/autocomplete work in shell --- setup.py | 12 ++-- stackdio/cli/__init__.py | 45 ++++++++----- stackdio/cli/shell.py | 3 +- stackdio/client/compat.py | 23 +++++++ stackdio/client/config.py | 121 ++++++++++++++++++++++++++++++++++ stackdio/client/exceptions.py | 4 ++ 6 files changed, 182 insertions(+), 26 deletions(-) create mode 100644 stackdio/client/compat.py create mode 100644 stackdio/client/config.py diff --git a/setup.py b/setup.py index 38136b8..9543f56 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,6 @@ # limitations under the License. # -from __future__ import unicode_literals - import os import sys @@ -46,12 +44,13 @@ def test_python_version(): with open('README.md') as f: LONG_DESCRIPTION = f.read() -CFG_DIR = os.path.expanduser("~/.stackdio-cli") +CFG_DIR = os.path.join(os.path.expanduser('~'), '.stackdio-cli') requirements = [ 'Jinja2==2.7.3', 'PyYAML==3.11', 'click>=6.0,<7.0', + 'cmd2>=0.6,<0.7', 'keyring==3.7', 'readline', 'requests>=2.4.0,<2.6.0', @@ -64,7 +63,7 @@ def test_python_version(): 'pylint<=1.2.0', ] -if __name__ == "__main__": +if __name__ == '__main__': test_python_version() setup( @@ -83,8 +82,8 @@ def test_python_version(): [ 'bootstrap.yaml', ]), - (os.path.join(CFG_DIR, "blueprints"), - ["blueprints/%s" % f for f in os.listdir("blueprints")]), + (os.path.join(CFG_DIR, 'blueprints'), + ['blueprints/%s' % f for f in os.listdir('blueprints')]), ], zip_safe=False, install_requires=requirements, @@ -113,7 +112,6 @@ def test_python_version(): 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', 'Topic :: System :: Clustering', 'Topic :: System :: Distributed Computing', ] diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index f8686f4..27947bf 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -14,30 +14,35 @@ from stackdio.cli import mixins from stackdio.cli.shell import get_shell from stackdio.client import StackdIO +from stackdio.client.config import StackdioConfig +from stackdio.client.exceptions import MissingConfigException from stackdio.client.version import __version__ CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) -CFG_DIR = os.path.expanduser("~/.stackdio-cli/") -CFG_FILE = os.path.join(CFG_DIR, "config.json") -KEYRING_SERVICE = "stackdio_cli" +KEYRING_SERVICE = 'stackdio_cli' -def get_client(): - if not os.path.isfile(CFG_FILE): - click.echo('It looks like you haven\'t used this CLI before. Please run ' - '`stackdio-cli configure`'.format(sys.argv[0])) - sys.exit(1) - config = json.load(open(CFG_FILE, 'r')) - config['blueprint_dir'] = os.path.expanduser(config.get('blueprint_dir', '')) +def load_config(fail_on_misconfigure, section='stackdio'): + try: + return StackdioConfig(section) + except MissingConfigException: + if fail_on_misconfigure: + click.echo('It looks like you haven\'t used this CLI before. Please run ' + '`stackdio-cli configure`'.format(sys.argv[0])) + sys.exit(1) + else: + return None + +def get_client(config): return StackdIO( - base_url=config["url"], + base_url=config['url'], auth=( - config["username"], - keyring.get_password(KEYRING_SERVICE, config.get("username") or "") + config['username'], + keyring.get_password(KEYRING_SERVICE, config.get('username') or '') ), verify=config.get('verify', True) ) @@ -45,17 +50,19 @@ def get_client(): class StackdioObj(object): - def __init__(self, ctx): + def __init__(self, ctx, fail_on_misconfigure): super(StackdioObj, self).__init__() + self.config = load_config(fail_on_misconfigure) self.shell = get_shell(ctx) - self.client = get_client() + if self.config: + self.client = get_client(self.config) @click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True) @click.version_option(__version__, '-v', '--version') @click.pass_context def stackdio(ctx): - ctx.obj = StackdioObj(ctx) + ctx.obj = StackdioObj(ctx, ctx.invoked_subcommand != 'configure') if ctx.invoked_subcommand is None: ctx.obj.shell.cmdloop() @@ -81,7 +88,11 @@ def formulas(): @stackdio.command() def configure(): - pass + config = StackdioConfig(create=True) + + config.prompt_for_config() + + config.save() @stackdio.command('server-version') diff --git a/stackdio/cli/shell.py b/stackdio/cli/shell.py index 43d5510..31e63ab 100644 --- a/stackdio/cli/shell.py +++ b/stackdio/cli/shell.py @@ -16,13 +16,12 @@ # import os -from cmd import Cmd +from cmd2 import Cmd import click from stackdio.client.version import __version__ - try: import readline except ImportError: diff --git a/stackdio/client/compat.py b/stackdio/client/compat.py new file mode 100644 index 0000000..70a0ff1 --- /dev/null +++ b/stackdio/client/compat.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014, Digital Reasoning +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +try: + # Python 2 + from ConfigParser import ConfigParser, NoOptionError +except ImportError: + # Python 3 + from configparser import ConfigParser, NoOptionError diff --git a/stackdio/client/config.py b/stackdio/client/config.py new file mode 100644 index 0000000..1057b4b --- /dev/null +++ b/stackdio/client/config.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014, Digital Reasoning +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +import click +import requests +from requests.exceptions import ConnectionError, MissingSchema + +from stackdio.client.compat import ConfigParser, NoOptionError +from stackdio.client.exceptions import MissingConfigException + + +CFG_DIR = os.path.join(os.path.expanduser('~'), '.stackdio') +CFG_FILE = os.path.join(CFG_DIR, 'client.cfg') + + +class StackdioConfig(object): + + def __init__(self, section='stackdio', config_file=CFG_FILE, create=False): + super(StackdioConfig, self).__init__() + + self._cfg_file = config_file + + if not create and not os.path.isfile(config_file): + raise MissingConfigException('{0} does not exist'.format(config_file)) + + self._config = ConfigParser() + + if create: + self._config.add_section(section) + else: + self._config.read(config_file) + + # Make the blueprint dir usable + new_blueprint_dir = os.path.expanduser(self.get('blueprint_dir')) + self._config.set(section, 'blueprint_dir', new_blueprint_dir) + + self.section = section + + def save(self): + with open(self._cfg_file, 'w') as f: + self._config.write(f) + + def __getitem__(self, item): + try: + return self._config.get(self.section, item) + except NoOptionError: + raise KeyError(item) + + def __setitem__(self, key, value): + self._config.set(self.section, key, value) + + def get(self, item, default=None): + try: + return self[item] + except KeyError: + return default + + def items(self): + return self._config.items(self.section) + + def prompt_for_config(self): + self.get_url() + + def _test_url(self, url): + try: + r = requests.get(url, verify=self.get('verify', True)) + return (200 <= r.status_code < 300) or r.status_code == 403 + except ConnectionError: + return False + except MissingSchema: + print("You might have forgotten http:// or https://") + return False + + def get_url(self): + + if self.get('url') is not None: + val = click.prompt('Keep existing url', default='y', prompt_suffix=' (y|n)? ') + if val not in ('N', 'n'): + return + + val = click.prompt('Does your stackd.io server have a self-signed SSL certificate', + default='n', prompt_suffix=' (y|n)? ') + + if val in ('Y', 'y'): + self['verify'] = False + else: + self['verify'] = True + + self['url'] = None + + while self['url'] is None: + url = click.prompt('What is the URL of your stackd.io server', prompt_suffix='? ') + if url.endswith('api'): + url += '/' + elif url.endswith('api/'): + pass + elif url.endswith('/'): + url += 'api/' + else: + url += '/api/' + if self._test_url(url): + self['url'] = url + else: + click.echo('There was an error while attempting to contact that server. ' + 'Try again.') diff --git a/stackdio/client/exceptions.py b/stackdio/client/exceptions.py index a96cb7d..7455378 100644 --- a/stackdio/client/exceptions.py +++ b/stackdio/client/exceptions.py @@ -16,6 +16,10 @@ # +class MissingConfigException(Exception): + pass + + class StackException(Exception): pass From 2946d1af6d10161927e4df91c06f77e1043424b2 Mon Sep 17 00:00:00 2001 From: Dan Cain Date: Tue, 8 Dec 2015 10:43:09 -0600 Subject: [PATCH 28/90] Allowing overide of CFG_(DIR|FILE) from ENV to support multiple endpoints via post(de)activate in our virtual environments --- stackdio/cli/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index f7995e9..08508d3 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -24,8 +24,8 @@ class StackdioShell( mixins.formulas.FormulaMixin, mixins.blueprints.BlueprintMixin): - CFG_DIR = os.path.expanduser("~/.stackdio-cli/") - CFG_FILE = os.path.join(CFG_DIR, "config.json") + CFG_DIR = os.path.expanduser(os.getenv('STACKDIO_CFG_DIR',"~/.stackdio-cli/")) + CFG_FILE = os.path.join(CFG_DIR, os.getenv('STACKDIO_CFG_FILE',"config.json")) BOOTSTRAP_FILE = os.path.join(CFG_DIR, "bootstrap.yaml") KEYRING_SERVICE = "stackdio_cli" PROMPT = "\n{username} @ {url}\n> " From f9b0078c74b0d486c9f3a0b5e085e46da71de264 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 16 Dec 2015 15:25:18 -0600 Subject: [PATCH 29/90] Moved a couple things to click-shell --- setup.py | 3 +- stackdio/cli/__init__.py | 26 ++++---- stackdio/cli/shell.py | 132 -------------------------------------- stackdio/client/config.py | 25 ++++++-- 4 files changed, 35 insertions(+), 151 deletions(-) delete mode 100644 stackdio/cli/shell.py diff --git a/setup.py b/setup.py index 9543f56..ffe709a 100644 --- a/setup.py +++ b/setup.py @@ -50,9 +50,8 @@ def test_python_version(): 'Jinja2==2.7.3', 'PyYAML==3.11', 'click>=6.0,<7.0', - 'cmd2>=0.6,<0.7', + 'click-shell==0.1', 'keyring==3.7', - 'readline', 'requests>=2.4.0,<2.6.0', 'simplejson==3.4.0', ] diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index 27947bf..03efe5b 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -8,11 +8,11 @@ from cmd import Cmd import click +import click_shell import keyring from requests import ConnectionError from stackdio.cli import mixins -from stackdio.cli.shell import get_shell from stackdio.client import StackdIO from stackdio.client.config import StackdioConfig from stackdio.client.exceptions import MissingConfigException @@ -21,6 +21,7 @@ CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +HIST_FILE = os.path.join(os.path.expanduser('~'), '.stackdio-cli', 'history') KEYRING_SERVICE = 'stackdio_cli' @@ -53,20 +54,17 @@ class StackdioObj(object): def __init__(self, ctx, fail_on_misconfigure): super(StackdioObj, self).__init__() self.config = load_config(fail_on_misconfigure) - self.shell = get_shell(ctx) if self.config: self.client = get_client(self.config) -@click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True) +@click_shell.shell(prompt='stackdio > ', intro='stackdio shell v{0}'.format(__version__), + hist_file=HIST_FILE, context_settings=CONTEXT_SETTINGS) @click.version_option(__version__, '-v', '--version') @click.pass_context def stackdio(ctx): ctx.obj = StackdioObj(ctx, ctx.invoked_subcommand != 'configure') - if ctx.invoked_subcommand is None: - ctx.obj.shell.cmdloop() - @stackdio.group() def stacks(): @@ -75,12 +73,17 @@ def stacks(): @stackdio.group() def blueprints(): - """ - Foo bar - """ pass +@blueprints.command() +@click.option('--template', '-t') +@click.option('--var-file', '-v', multiple=True) +def create(template, var_file): + click.echo(template) + click.echo(var_file) + + @stackdio.group() def formulas(): pass @@ -96,8 +99,9 @@ def configure(): @stackdio.command('server-version') -def server_version(): - client = get_client() +@click.pass_obj +def server_version(obj): + client = obj.client click.echo('stackdio-server, version {0}'.format(client.get_version())) diff --git a/stackdio/cli/shell.py b/stackdio/cli/shell.py deleted file mode 100644 index 31e63ab..0000000 --- a/stackdio/cli/shell.py +++ /dev/null @@ -1,132 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2014, Digital Reasoning -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -from cmd2 import Cmd - -import click - -from stackdio.client.version import __version__ - -try: - import readline -except ImportError: - readline = None - - -HIST_FILE = os.path.join(os.path.expanduser('~'), '.stackdio-cli', 'history') - - -def get_invoke(ctx, command): - def invoke(self, arg): - return ctx.invoke(command) - return invoke - - -def get_help(command): - def help(self): - click.echo(command.help) - return help - - -def get_shell(ctx): - - # Make it a new-style class so we can use super - class StackdioShell(Cmd, object): - - def __init__(self): - super(StackdioShell, self).__init__() - self.old_completer = None - - def preloop(self): - # read our history - if readline: - try: - readline.read_history_file(HIST_FILE) - except IOError: - pass - - def postloop(self): - # Write our history - if readline: - readline.write_history_file(HIST_FILE) - - prompt = 'stackdio > ' - intro = 'stackdio shell v{0}'.format(__version__) - - # We need to override this to fix readline - def cmdloop(self, intro=None): - self.preloop() - if self.use_rawinput and self.completekey and readline: - self.old_completer = readline.get_completer() - readline.set_completer(self.complete) - if 'libedit' in readline.__doc__: - # For mac - readline.parse_and_bind('bind ^I rl_complete') - else: - # for other platforms - readline.parse_and_bind(self.completekey + ': complete') - try: - if intro is not None: - self.intro = intro - if self.intro: - self.stdout.write(str(self.intro)+"\n") - stop = None - while not stop: - if self.cmdqueue: - line = self.cmdqueue.pop(0) - else: - if self.use_rawinput: - try: - line = raw_input(self.prompt) - except EOFError: - # We just want to quit here - self.stdout.write('\n') - break - else: - self.stdout.write(self.prompt) - self.stdout.flush() - line = self.stdin.readline() - if not len(line): - line = 'EOF' - else: - line = line.rstrip('\r\n') - line = self.precmd(line) - stop = self.onecmd(line) - stop = self.postcmd(stop, line) - - finally: - self.postloop() - if self.use_rawinput and self.completekey and readline: - readline.set_completer(self.old_completer) - - def emptyline(self): - pass - - def do_quit(self, arg): - return True - - def do_exit(self, arg): - return True - - for name, command in ctx.command.commands.items(): - setattr(StackdioShell, 'do_%s' % name.replace('-', '_'), get_invoke(ctx, command)) - - if command.help is not None: - setattr(StackdioShell, 'help_%s' % name.replace('-', '_'), get_help(command)) - - return StackdioShell() diff --git a/stackdio/client/config.py b/stackdio/client/config.py index 1057b4b..5f7ff24 100644 --- a/stackdio/client/config.py +++ b/stackdio/client/config.py @@ -31,9 +31,16 @@ class StackdioConfig(object): + BOOL_MAP = { + str(True): True, + str(False): False, + } + def __init__(self, section='stackdio', config_file=CFG_FILE, create=False): super(StackdioConfig, self).__init__() + self.section = section + self._cfg_file = config_file if not create and not os.path.isfile(config_file): @@ -47,10 +54,10 @@ def __init__(self, section='stackdio', config_file=CFG_FILE, create=False): self._config.read(config_file) # Make the blueprint dir usable - new_blueprint_dir = os.path.expanduser(self.get('blueprint_dir')) - self._config.set(section, 'blueprint_dir', new_blueprint_dir) - - self.section = section + blueprint_dir = self.get('blueprint_dir') + if blueprint_dir: + new_blueprint_dir = os.path.expanduser(blueprint_dir) + self._config.set(section, 'blueprint_dir', new_blueprint_dir) def save(self): with open(self._cfg_file, 'w') as f: @@ -58,11 +65,17 @@ def save(self): def __getitem__(self, item): try: - return self._config.get(self.section, item) + ret = self._config.get(self.section, item) + if ret in self.BOOL_MAP: + return self.BOOL_MAP[ret] + else: + return str(ret) except NoOptionError: raise KeyError(item) def __setitem__(self, key, value): + if isinstance(value, bool): + value = str(value) self._config.set(self.section, key, value) def get(self, item, default=None): @@ -84,7 +97,7 @@ def _test_url(self, url): except ConnectionError: return False except MissingSchema: - print("You might have forgotten http:// or https://") + click.echo('You might have forgotten http:// or https://') return False def get_url(self): From ffdcc3c4ecd863906be80b003cad841be9e17a59 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 18 Dec 2015 15:16:40 -0600 Subject: [PATCH 30/90] Start using config in the client --- setup.py | 2 +- stackdio/cli/__init__.py | 34 +++--------------- stackdio/cli/mixins/__init__.py | 4 --- stackdio/cli/mixins/blueprints.py | 2 +- stackdio/client/__init__.py | 57 ++++++++++++++++++------------- stackdio/client/config.py | 11 +++--- stackdio/client/exceptions.py | 2 +- stackdio/client/http.py | 34 ++++++++++++------ 8 files changed, 72 insertions(+), 74 deletions(-) diff --git a/setup.py b/setup.py index ffe709a..0c61176 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def test_python_version(): requirements = [ 'Jinja2==2.7.3', - 'PyYAML==3.11', + 'PyYAML>=3.10', 'click>=6.0,<7.0', 'click-shell==0.1', 'keyring==3.7', diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index 03efe5b..dc45997 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -13,9 +13,8 @@ from requests import ConnectionError from stackdio.cli import mixins -from stackdio.client import StackdIO +from stackdio.client import StackdioClient from stackdio.client.config import StackdioConfig -from stackdio.client.exceptions import MissingConfigException from stackdio.client.version import __version__ @@ -39,7 +38,7 @@ def load_config(fail_on_misconfigure, section='stackdio'): def get_client(config): - return StackdIO( + return StackdioClient( base_url=config['url'], auth=( config['username'], @@ -66,29 +65,6 @@ def stackdio(ctx): ctx.obj = StackdioObj(ctx, ctx.invoked_subcommand != 'configure') -@stackdio.group() -def stacks(): - pass - - -@stackdio.group() -def blueprints(): - pass - - -@blueprints.command() -@click.option('--template', '-t') -@click.option('--var-file', '-v', multiple=True) -def create(template, var_file): - click.echo(template) - click.echo(var_file) - - -@stackdio.group() -def formulas(): - pass - - @stackdio.command() def configure(): config = StackdioConfig(create=True) @@ -150,8 +126,8 @@ def get_names(self): return ["do_initial_setup", "do_help"] def _init_stacks(self): - """Instantiate a StackdIO object""" - self.stacks = StackdIO( + """Instantiate a StackdioClient object""" + self.stacks = StackdioClient( base_url=self.config["url"], auth=( self.config["username"], @@ -275,7 +251,7 @@ def do_account_summary(self, args=None): def main(): - stackdio() + stackdio(obj={}) # parser = argparse.ArgumentParser( # description="Invoke the stackdio cli") diff --git a/stackdio/cli/mixins/__init__.py b/stackdio/cli/mixins/__init__.py index ecb06b5..e69de29 100644 --- a/stackdio/cli/mixins/__init__.py +++ b/stackdio/cli/mixins/__init__.py @@ -1,4 +0,0 @@ -import stackdio.cli.mixins.blueprints -import stackdio.cli.mixins.bootstrap -import stackdio.cli.mixins.formulas -import stackdio.cli.mixins.stacks diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index 2e4f4b6..5646e69 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -170,7 +170,7 @@ def _create_blueprint(self, args, bootstrap=False): if not bootstrap: print("Creating blueprint") - r = self.stacks.create_blueprint(bp_json) + r = self.stacks.create_blueprint(bp_json, raise_for_status=False) print(json.dumps(r, indent=2)) def _create_single(self, template_file, var_files, no_prompt): diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index d55bfad..90089a0 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -19,9 +19,15 @@ from .account import AccountMixin from .blueprint import BlueprintMixin -from .exceptions import BlueprintException, StackException, IncompatibleVersionException +from .config import StackdioConfig +from .exceptions import ( + BlueprintException, + StackException, + IncompatibleVersionException, + MissingUrlException +) from .formula import FormulaMixin -from .http import get, post, patch +from .http import HttpMixin, get, post, patch from .image import ImageMixin from .region import RegionMixin from .settings import SettingsMixin @@ -29,31 +35,36 @@ from .version import _parse_version_string logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) -class StackdIO(BlueprintMixin, FormulaMixin, AccountMixin, - ImageMixin, RegionMixin, StackMixin, SettingsMixin): +class StackdioClient(BlueprintMixin, FormulaMixin, AccountMixin, ImageMixin, + RegionMixin, StackMixin, SettingsMixin, HttpMixin): - def __init__(self, protocol="https", host="localhost", port=443, - base_url=None, auth=None, verify=True): + def __init__(self, url=None, username=None, password=None, verify=True): + self.config = StackdioConfig() - super(StackdIO, self).__init__(auth=auth, verify=verify) - if base_url: - self.url = base_url if base_url.endswith('/') else "%s/" % base_url - else: - self.url = "{protocol}://{host}:{port}/api/".format( - protocol=protocol, - host=host, - port=port) + if self.config.usable_config: + # Grab stuff from the config + url = self.config['url'] + username = self.config['username'] + verify = self.config['verify'] - self.auth = auth + super(StackdioClient, self).__init__(url=url, auth=(username, password), verify=verify) + self.url = url - _, self.version = _parse_version_string(self.get_version()) + try: + _, self.version = _parse_version_string(self.get_version()) + except MissingUrlException: + self.version = None - if self.version[0] != 0 or self.version[1] != 7: + if self.version and self.version[0] != 0 or self.version[1] != 7: raise IncompatibleVersionException('Server version {0}.{1}.{2} not ' 'supported.'.format(**self.version)) + def usable(self): + return self.config.usable_config + @get('') def get_root(self): pass @@ -67,12 +78,12 @@ def get_version(self, resp): return resp['version'] @post('cloud/security_groups/') - def create_security_group(self, name, description, cloud_provider, is_default=True): - """Create a security group""" + def create_security_group(self, name, description, cloud_account, group_id, is_default=True): return { - "name": name, - "description": description, - "cloud_provider": cloud_provider, - "is_default": is_default + 'name': name, + 'description': description, + 'cloud_account': cloud_account, + 'group_id': group_id, + 'is_default': is_default } diff --git a/stackdio/client/config.py b/stackdio/client/config.py index 5f7ff24..629616a 100644 --- a/stackdio/client/config.py +++ b/stackdio/client/config.py @@ -22,7 +22,6 @@ from requests.exceptions import ConnectionError, MissingSchema from stackdio.client.compat import ConfigParser, NoOptionError -from stackdio.client.exceptions import MissingConfigException CFG_DIR = os.path.join(os.path.expanduser('~'), '.stackdio') @@ -30,25 +29,27 @@ class StackdioConfig(object): + """ + A wrapper around python's ConfigParser class + """ BOOL_MAP = { str(True): True, str(False): False, } - def __init__(self, section='stackdio', config_file=CFG_FILE, create=False): + def __init__(self, config_file=CFG_FILE, section='default'): super(StackdioConfig, self).__init__() self.section = section self._cfg_file = config_file - if not create and not os.path.isfile(config_file): - raise MissingConfigException('{0} does not exist'.format(config_file)) + self.usable_config = os.path.isfile(config_file) self._config = ConfigParser() - if create: + if not self.usable_config: self._config.add_section(section) else: self._config.read(config_file) diff --git a/stackdio/client/exceptions.py b/stackdio/client/exceptions.py index 7455378..b003e39 100644 --- a/stackdio/client/exceptions.py +++ b/stackdio/client/exceptions.py @@ -16,7 +16,7 @@ # -class MissingConfigException(Exception): +class MissingUrlException(Exception): pass diff --git a/stackdio/client/http.py b/stackdio/client/http.py index 8a47f9f..7e03228 100644 --- a/stackdio/client/http.py +++ b/stackdio/client/http.py @@ -23,10 +23,12 @@ from inspect import getcallargs +from .exceptions import MissingUrlException + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -HTTP_INSECURE_MESSAGE = "\n".join([ +HTTP_INSECURE_MESSAGE = '\n'.join([ "You have chosen not to verify ssl connections.", "This is insecure, but it's your choice.", "This has been your single warning." @@ -37,14 +39,15 @@ class HttpMixin(object): """Add HTTP request features to an object""" HEADERS = { - 'json': {"content-type": "application/json"}, - 'xml': {"content-type": "application/xml"} + 'json': {'content-type': 'application/json'}, + 'xml': {'content-type': 'application/xml'}, } - def __init__(self, auth=None, verify=True): + def __init__(self, url, auth=None, verify=True): super(HttpMixin, self).__init__() - self._http_options = { + self.url = url + self.http_options = { 'auth': auth, 'verify': verify, } @@ -59,6 +62,9 @@ def __init__(self, auth=None, verify=True): from requests.packages.urllib3 import disable_warnings disable_warnings() + def usable(self): + raise NotImplementedError() + def default_response(obj, response): return response @@ -102,12 +108,20 @@ def __repr__(self): # We need this so we can save the client object as an attribute, and then it can be used # in __call__ def __get__(self, obj, objtype=None): + if objtype: + assert issubclass(objtype, HttpMixin) + assert isinstance(obj, HttpMixin) + self.obj = obj - assert issubclass(objtype, HttpMixin) return self # Here's how the request actually happens def __call__(self, *args, **kwargs): + assert isinstance(self.obj, HttpMixin) + + if not self.obj.usable(): + raise MissingUrlException('No url is set') + none_on_404 = kwargs.pop('none_on_404', False) raise_for_status = kwargs.pop('raise_for_status', True) @@ -131,10 +145,10 @@ def __call__(self, *args, **kwargs): result = requests.request(method, url, data=data, - auth=self.obj._http_options['auth'], + auth=self.obj.http_options['auth'], headers=self.headers, params=kwargs, - verify=self.obj._http_options['verify']) + verify=self.obj.http_options['verify']) # Handle special conditions if none_on_404 and result.status_code == 404: @@ -164,10 +178,10 @@ def __call__(self, *args, **kwargs): next_page = requests.request(method, next_url, data=data, - auth=self.obj._http_options['auth'], + auth=self.obj.http_options['auth'], headers=self.headers, params=kwargs, - verify=self.obj._http_options['verify']).json() + verify=self.obj.http_options['verify']).json() res.extend(next_page['results']) next_url = next_page.get('next') From ea0dc775ad980743f631e2732861b87ff1d4b604 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 5 Jan 2016 17:43:43 -0600 Subject: [PATCH 31/90] Converted blueprints to click --- setup.py | 4 +- stackdio/cli/__init__.py | 131 ++-------- stackdio/cli/mixins/blueprints.py | 401 ++++++++++++++---------------- stackdio/cli/utils.py | 40 +++ stackdio/client/__init__.py | 30 ++- stackdio/client/config.py | 17 +- 6 files changed, 286 insertions(+), 337 deletions(-) create mode 100644 stackdio/cli/utils.py diff --git a/setup.py b/setup.py index 0c61176..2bdea4d 100644 --- a/setup.py +++ b/setup.py @@ -50,9 +50,9 @@ def test_python_version(): 'Jinja2==2.7.3', 'PyYAML>=3.10', 'click>=6.0,<7.0', - 'click-shell==0.1', + 'colorama>=0.3,<0.4', 'keyring==3.7', - 'requests>=2.4.0,<2.6.0', + 'requests>=2.4.0', 'simplejson==3.4.0', ] diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index dc45997..6078c4d 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -8,13 +8,10 @@ from cmd import Cmd import click -import click_shell -import keyring from requests import ConnectionError -from stackdio.cli import mixins +from stackdio.cli.mixins import blueprints, bootstrap, formulas, stacks from stackdio.client import StackdioClient -from stackdio.client.config import StackdioConfig from stackdio.client.version import __version__ @@ -22,67 +19,40 @@ HIST_FILE = os.path.join(os.path.expanduser('~'), '.stackdio-cli', 'history') -KEYRING_SERVICE = 'stackdio_cli' - -def load_config(fail_on_misconfigure, section='stackdio'): - try: - return StackdioConfig(section) - except MissingConfigException: - if fail_on_misconfigure: - click.echo('It looks like you haven\'t used this CLI before. Please run ' - '`stackdio-cli configure`'.format(sys.argv[0])) - sys.exit(1) - else: - return None - - -def get_client(config): - return StackdioClient( - base_url=config['url'], - auth=( - config['username'], - keyring.get_password(KEYRING_SERVICE, config.get('username') or '') - ), - verify=config.get('verify', True) - ) - - -class StackdioObj(object): - - def __init__(self, ctx, fail_on_misconfigure): - super(StackdioObj, self).__init__() - self.config = load_config(fail_on_misconfigure) - if self.config: - self.client = get_client(self.config) - - -@click_shell.shell(prompt='stackdio > ', intro='stackdio shell v{0}'.format(__version__), - hist_file=HIST_FILE, context_settings=CONTEXT_SETTINGS) +@click.group(context_settings=CONTEXT_SETTINGS) @click.version_option(__version__, '-v', '--version') @click.pass_context def stackdio(ctx): - ctx.obj = StackdioObj(ctx, ctx.invoked_subcommand != 'configure') - + client = StackdioClient() + if ctx.invoked_subcommand not in ('configure', None) and not client.usable(): + raise click.UsageError('It looks like you haven\'t used this CLI before. Please run ' + '`stackdio-cli configure`') -@stackdio.command() -def configure(): - config = StackdioConfig(create=True) + # Put the client in the obj + ctx.obj['client'] = client - config.prompt_for_config() - config.save() +@stackdio.command() +@click.pass_obj +def configure(obj): + client = obj['client'] + print('configuring') @stackdio.command('server-version') @click.pass_obj def server_version(obj): - client = obj.client + client = obj['client'] click.echo('stackdio-server, version {0}'.format(client.get_version())) -class StackdioShell(Cmd, mixins.bootstrap.BootstrapMixin, mixins.stacks.StackMixin, - mixins.formulas.FormulaMixin, mixins.blueprints.BlueprintMixin): +# Add all our other commands +stackdio.add_command(blueprints.blueprints) + + +class StackdioShell(Cmd, bootstrap.BootstrapMixin, stacks.StackMixin, + formulas.FormulaMixin): CFG_DIR = os.path.expanduser("~/.stackdio-cli/") CFG_FILE = os.path.join(CFG_DIR, "config.json") @@ -109,33 +79,6 @@ def __init__(self): self._init_stacks() self._validate_auth() - def preloop(self): - self._setprompt() - - def precmd(self, line): - self._setprompt() - return line - - def postloop(self): - print("\nGoodbye!") - - def get_names(self): - if self.validated: - return ["do_%s" % c for c in self.HELP_CMDS] - else: - return ["do_initial_setup", "do_help"] - - def _init_stacks(self): - """Instantiate a StackdioClient object""" - self.stacks = StackdioClient( - base_url=self.config["url"], - auth=( - self.config["username"], - keyring.get_password(self.KEYRING_SERVICE, self.config.get("username") or "") - ), - verify=self.config.get('verify', True) - ) - def _load_config(self): """Attempt to load config file, otherwise fallback to DEFAULT_CONFIG""" @@ -213,23 +156,6 @@ def _setprompt(self): "## Your account is missing the public key, run 'bootstrap' to fix", "red")) - def _print_summary(self, title, components): - num_components = len(components) - print("## {0} {1}{2}".format( - num_components, - title, - "s" if num_components == 0 or num_components > 1 else "")) - - for item in components: - print("- Title: {0}\n Description: {1}".format( - item.get("title"), item.get("description"))) - - if "status_detail" in item: - print(" Status Detail: {0}\n".format( - item.get("status_detail"))) - else: - print("") - def do_account_summary(self, args=None): """Get a summary of your account.""" sys.stdout.write("Polling {0} ... ".format(self.config["url"])) @@ -251,24 +177,9 @@ def do_account_summary(self, args=None): def main(): + # Just run our CLI tool stackdio(obj={}) - # parser = argparse.ArgumentParser( - # description="Invoke the stackdio cli") - # parser.add_argument("--debug", action="store_true", help="Enable debugging output") - # args = parser.parse_args() - # - # # an ugly hack to work around the fact that cmd2 is using optparse to parse - # # arguments for the commands; not sure what the "right" fix is, but as long - # # as we assume that we don't want any of our arguments to get passed into - # # the cmdloop this seems ok - # sys.argv = sys.argv[0:1] - # - # shell = StackdioShell() - # if args.debug: - # shell.debug = True - # shell.cmdloop() - if __name__ == '__main__': main() diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index 5646e69..2005049 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -5,266 +5,233 @@ import argparse import sys +import click import yaml from cmd2 import Cmd from stackdio.client.exceptions import StackException from stackdio.cli.blueprints.generator import BlueprintGenerator, BlueprintException +from stackdio.cli.utils import print_summary class BlueprintNotFound(Exception): pass -class BlueprintMixin(Cmd): - BLUEPRINT_COMMANDS = ["list", "list-templates", "create", "create-all", "delete", "delete-all"] - - def do_blueprints(self, arg): - """Entry point to controlling blueprints.""" - - USAGE = "Usage: blueprints COMMAND\nWhere COMMAND is one of: %s" % ( - ", ".join(self.BLUEPRINT_COMMANDS)) - - args = arg.split() - if not args or args[0] not in self.BLUEPRINT_COMMANDS: - print(USAGE) - return - - bp_cmd = args[0] - - # Sneakiness for argparse - saved = sys.argv[0] - sys.argv[0] = 'blueprints {0}'.format(bp_cmd) - - if bp_cmd == "list": - self._list_blueprints() - elif bp_cmd == "list-templates": - self._list_templates() - elif bp_cmd == "create": - self._create_blueprint(args[1:]) - elif bp_cmd == "create-all": - self._create_all(args[1:]) - elif bp_cmd == "delete": - self._delete_blueprint(args[1:]) - elif bp_cmd == "delete-all": - self._delete_all() - - else: - print(USAGE) - - # End sneakiness - sys.argv[0] = saved - - def complete_blueprints(self, text, line, begidx, endidx): - # not using line, begidx, or endidx, thus the following pylint disable - # pylint: disable=W0613 - return [i for i in self.BLUEPRINT_COMMANDS if i.startswith(text)] - - def help_blueprints(self): - print("Manage blueprints.") - print("Sub-commands can be one of:\n\t{0}".format( - ", ".join(self.BLUEPRINT_COMMANDS))) - print("Try 'blueprints COMMAND' to get help on (most) sub-commands") - - def _list_blueprints(self): - """List all blueprints""" - - print("Getting blueprints ... ") - blueprints = self.stacks.list_blueprints() - self._print_summary("Blueprint", blueprints) - - def _recurse_dir(self, dirname, extensions, prefix=''): - for template in os.listdir(dirname): - if os.path.isdir(os.path.join(dirname, template)): - # Recursively look at the subdirectories - self._recurse_dir(os.path.join(dirname, template), - extensions, - prefix + template + os.sep) - elif template.split('.')[-1] in extensions and not template.startswith('_'): - print(' {0}'.format(prefix + template)) - - def _list_templates(self): - if 'blueprint_dir' not in self.config: - print("Missing blueprint directory config") - return - - blueprint_dir = os.path.expanduser(self.config['blueprint_dir']) - - print('Template mappings:') - mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) - if mapping: - for blueprint in mapping: - print(' {0}'.format(blueprint)) +@click.group() +def blueprints(): + pass - print('') - print('Templates:') - self._recurse_dir(os.path.join(blueprint_dir, 'templates'), ['json']) +@blueprints.command(name='list') +@click.pass_obj +def list_blueprints(obj): + """ + List all blueprints + """ + client = obj['client'] - print('') + click.echo('Getting blueprints ... ') + print_summary('Blueprint', client.list_blueprints()) - print('Var files:') - self._recurse_dir(os.path.join(blueprint_dir, 'var_files'), ['yaml', 'yml']) - def _create_blueprint(self, args, bootstrap=False): - """Create a blueprint""" +def _recurse_dir(dirname, extensions, prefix=''): + for template in os.listdir(dirname): + if os.path.isdir(os.path.join(dirname, template)): + # Recursively look at the subdirectories + _recurse_dir(os.path.join(dirname, template), + extensions, + prefix + template + os.sep) + elif template.split('.')[-1] in extensions and not template.startswith('_'): + click.echo(' {0}'.format(prefix + template)) - parser = argparse.ArgumentParser() - parser.add_argument('-m', '--mapping', - help='The entry in the map file to use') +@blueprints.command(name='list-templates') +@click.pass_obj +def list_templates(obj): + """ + List all the blueprint templates + """ + client = obj['client'] - parser.add_argument('-t', '--template', - help='The template file to use') + if 'blueprint_dir' not in client.config: + click.echo('Missing blueprint directory config') + return - parser.add_argument('-v', '--var-file', - action='append', - help='The variable files to use. You may pass in more than one. They ' - 'will be loaded from left to right, so variables in the rightmost ' - 'var files will override those in var files to the left.') + blueprint_dir = os.path.expanduser(client.config['blueprint_dir']) - parser.add_argument('-n', '--no-prompt', - action='store_false', - help='Don\'t prompt for missing variables in the template') + click.echo('Template mappings:') + mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) + if mapping: + for blueprint in mapping: + click.echo(' {0}'.format(blueprint)) - args = parser.parse_args(args) + click.echo() - if not bootstrap: - print(self.colorize( - "Advanced users only - use the web UI if this isn't you!\n", - "green")) + click.echo('Templates:') + _recurse_dir(os.path.join(blueprint_dir, 'templates'), ['json']) - if not args.template and not args.mapping: - print(self.colorize('You must specify either a template or a mapping\n', 'red')) - parser.print_help() - return + click.echo() - blueprint_dir = os.path.expanduser(self.config['blueprint_dir']) + click.echo('Var files:') + _recurse_dir(os.path.join(blueprint_dir, 'var_files'), ['yaml', 'yml']) - template_file = args.template - # Should always be a list, and the generator can handle that - var_files = args.var_file - if not var_files: - # If -v is never specified, argparse give back None, we need a list - var_files = [] - if args.mapping: - mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) - if not mapping or args.mapping not in mapping: - print(self.colorize('You gave an invalid mapping.', 'red')) - return - else: - template_file = mapping[args.mapping].get('template') - var_files = mapping[args.mapping].get('var_files', []) - if not template_file: - print(self.colorize('Your mapping must specify a template.', 'red')) - return - - bp_json = self._create_single(template_file, var_files, args.no_prompt) - - if not bp_json: - # There was an error with the blueprint creation, and there should already be an - # error message printed - return +def _create_single_blueprint(config, template_file, var_files, no_prompt): + blueprint_dir = os.path.expanduser(config['blueprint_dir']) - if not bootstrap: - print("Creating blueprint") + gen = BlueprintGenerator([os.path.join(blueprint_dir, 'templates')]) - r = self.stacks.create_blueprint(bp_json, raise_for_status=False) - print(json.dumps(r, indent=2)) + if not os.path.exists(os.path.join(blueprint_dir, 'templates', template_file)): + click.secho('You gave an invalid template', fg='red') + return - def _create_single(self, template_file, var_files, no_prompt): - blueprint_dir = os.path.expanduser(self.config['blueprint_dir']) + if template_file.startswith('_'): + click.secho('WARNING: Templates beginning with \'_\' are generally not meant to ' + 'be used directly. Please be sure this is really what you want.\n', + fg='magenta') - gen = BlueprintGenerator([os.path.join(blueprint_dir, 'templates')]) + final_var_files = [] - if not os.path.exists(os.path.join(blueprint_dir, 'templates', template_file)): - print(self.colorize('You gave an invalid template', 'red')) + # Build a list with full paths in it instead of relative paths + for var_file in var_files: + var_file = os.path.join(blueprint_dir, 'var_files', var_file) + if os.path.exists(var_file): + final_var_files.append(var_file) + else: + click.secho('WARNING: Variable file {0} was not found. Ignoring.'.format(var_file), + fg='magenta') + + # Generate the JSON for the blueprint + return gen.generate(template_file, + final_var_files, # Pass in a list + prompt=no_prompt) + + +@blueprints.command(name='create') +@click.pass_obj +@click.option('-m', '--mapping', + help='The entry in the map file to use') +@click.option('-t', '--template', + help='The template file to use') +@click.option('-v', '--var-file', multiple=True, + help='The variable files to use. You may pass in more than one. They ' + 'will be loaded from left to right, so variables in the rightmost ' + 'var files will override those in var files to the left.') +@click.option('-n', '--no-prompt', is_flag=True, default=True, + help='Don\'t prompt for missing variables in the template') +def create_blueprint(obj, mapping, template, var_file, no_prompt): + """ + Create a blueprint + """ + print(mapping) + print(template) + print(var_file) + print(no_prompt) + + client = obj['client'] + + if not template and not mapping: + raise click.UsageError('You must specify either a template or a mapping.') + + click.secho('Advanced users only - use the web UI if this isn\'t you!\n', fg='green') + + blueprint_dir = client.config['blueprint_dir'] + + if mapping: + mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) + if not mapping or mapping not in mapping: + click.secho('You gave an invalid mapping.', fg='red') return - - if template_file.startswith('_'): - print(self.colorize("WARNING: Templates beginning with '_' are generally not meant to " - "be used directly. Please be sure this is really what you want.\n", - "magenta")) - - final_var_files = [] - - # Build a list with full paths in it instead of relative paths - for var_file in var_files: - var_file = os.path.join(blueprint_dir, 'var_files', var_file) - if os.path.exists(var_file): - final_var_files.append(var_file) - else: - print(self.colorize("WARNING: Variable file {0} was not found. " - "Ignoring.".format(var_file), "magenta")) - - # Generate the JSON for the blueprint - return gen.generate(template_file, - final_var_files, # Pass in a list - prompt=no_prompt) - - def _create_all(self, args): - """Create all the blueprints in the map file""" - parser = argparse.ArgumentParser() - - parser.add_argument('--no-prompt', - action='store_false', - help='Don\'t prompt to create all blueprints') - - args = parser.parse_args(args) - - if args.no_prompt: - really = raw_input("Really create all blueprints (y/n)? ") - if really not in ['Y', 'y']: + else: + template = mapping[mapping].get('template') + var_file = mapping[mapping].get('var_files', []) + if not template: + click.secho('Your mapping must specify a template.', fg='red') return - blueprint_dir = os.path.expanduser(self.config['blueprint_dir']) - mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) + bp_json = _create_single_blueprint(client.config, template, var_file, no_prompt) - for name, vals in mapping.items(): - try: - bp_json = self._create_single(vals['template'], vals['var_file'], False) - self.stacks.create_blueprint(bp_json) - print(self.colorize('Created blueprint {0}'.format(name), 'green')) - except BlueprintException: - print(self.colorize('Blueprint {0} NOT created\n'.format(name), 'magenta')) + if not bp_json: + # There was an error with the blueprint creation, and there should already be an + # error message printed + return - def _delete_blueprint(self, args): - """Delete a blueprint""" + click.echo('Creating blueprint') - if len(args) != 1: - print("Usage: blueprint delete BLUEPRINT_NAME") - return + r = client.create_blueprint(bp_json, raise_for_status=False) + click.echo(json.dumps(r, indent=2)) - blueprint_id = self._get_blueprint_id(args[0]) - really = raw_input("Really delete blueprint {0} (y/n)? ".format(args[0])) - if really not in ["y", "Y"]: - print("Aborting deletion") - return - - print("Deleting {0}".format(args[0])) - self.stacks.delete_blueprint(blueprint_id) - self._list_blueprints() +@blueprints.command(name='create-all') +@click.pass_obj +@click.option('--no-prompt', is_flag=True, default=True, + help='Don\'t prompt to create all blueprints') +def create_all_blueprints(obj, no_prompt): + """ + Create all the blueprints in the map file + """ + client = obj['client'] - def _delete_all(self): - """Delete all blueprints""" - really = raw_input("Really delete all blueprints? This is completely destructive, and you " - "will never get them back. (y/n) ") + if no_prompt: + really = raw_input('Really create all blueprints (y/n)? ') if really not in ['Y', 'y']: return - for blueprint in self.stacks.list_blueprints(): - self.stacks.delete_blueprint(blueprint['id']) - print(self.colorize('Deleted blueprint {0}'.format(blueprint['title']), 'magenta')) - - def _get_blueprint_id(self, blueprint_name): - """Validate that a blueprint exists""" + blueprint_dir = os.path.expanduser(client.config['blueprint_dir']) + mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) + for name, vals in mapping.items(): try: - return self.stacks.get_blueprint_id(blueprint_name) - except StackException: - print(self.colorize( - "Blueprint [{0}] does not exist".format(blueprint_name), - "red")) - raise + bp_json = _create_single_blueprint(client.config, vals['template'], + vals['var_file'], False) + client.create_blueprint(bp_json) + click.secho('Created blueprint {0}'.format(name), fg='green') + except BlueprintException: + click.secho('Blueprint {0} NOT created\n'.format(name), fg='magenta') + + +def _get_blueprint_id(client, blueprint_title): + blueprints = client.search_blueprints(title=blueprint_title) + + if len(blueprints) == 0: + raise click.UsageError('Blueprint [{0}] does not exist'.format(blueprint_title)) + elif len(blueprints) > 1: + raise click.UsageError('Blueprint [{0}] does not exist'.format(blueprint_title)) + else: + return blueprints[0]['id'] + + +@blueprints.command(name='delete') +@click.pass_obj +@click.argument('title') +def delete_blueprint(obj, title): + """ + Delete a blueprint + """ + client = obj['client'] + + blueprint_id = _get_blueprint_id(client, title) + + really = raw_input('Really delete blueprint {0} (y/n)? '.format(title)) + if really not in ['y', 'Y']: + click.echo('Aborting deletion') + return + + click.echo('Deleting {0}'.format(title)) + client.delete_blueprint(blueprint_id) + + +@blueprints.command(name='delete-all') +@click.pass_obj +@click.confirmation_option('-y', '--yes', prompt='Really delete all blueprints? This is ' + 'completely destructive, and you will never get them back.') +def delete_all_blueprints(obj): + """ + Delete all blueprints + """ + client = obj['client'] + + for blueprint in client.list_blueprints(): + client.delete_blueprint(blueprint['id']) + click.secho('Deleted blueprint {0}'.format(blueprint['title']), fg='magenta') diff --git a/stackdio/cli/utils.py b/stackdio/cli/utils.py new file mode 100644 index 0000000..c5d8ac3 --- /dev/null +++ b/stackdio/cli/utils.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014, Digital Reasoning +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import click + + +def print_summary(title, components): + num_components = len(components) + + if num_components != 1: + title += 's' + + click.echo('## {0} {1}'.format(num_components, title)) + + for item in components: + click.echo('- Title: {0}'.format( + item.get('title'))) + + if 'description' in item: + click.echo(' Description: {0}'.format(item['description'])) + + if 'status_detail' in item: + click.echo(' Status Detail: {0}'.format(item['status_detail'])) + + # Print a newline after each entry + click.echo() diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index 90089a0..1ec1f9f 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -44,26 +44,42 @@ class StackdioClient(BlueprintMixin, FormulaMixin, AccountMixin, ImageMixin, def __init__(self, url=None, username=None, password=None, verify=True): self.config = StackdioConfig() + self.url = None + self.username = None + self.password = None + self.verify = None + if self.config.usable_config: # Grab stuff from the config - url = self.config['url'] - username = self.config['username'] - verify = self.config['verify'] + self.url = self.config['url'] + self.username = self.config['username'] + self.password = self.config['password'] + self.verify = self.config['verify'] + + if url is not None: + self.url = url + + if username is not None and password is not None: + self.username = username + self.password = password + + if verify is not None: + self.verify = verify - super(StackdioClient, self).__init__(url=url, auth=(username, password), verify=verify) - self.url = url + super(StackdioClient, self).__init__(url=self.url, auth=(self.username, self.password), + verify=self.verify) try: _, self.version = _parse_version_string(self.get_version()) except MissingUrlException: self.version = None - if self.version and self.version[0] != 0 or self.version[1] != 7: + if self.version and (self.version[0] != 0 or self.version[1] != 7): raise IncompatibleVersionException('Server version {0}.{1}.{2} not ' 'supported.'.format(**self.version)) def usable(self): - return self.config.usable_config + return self.config.usable_config or self.url @get('') def get_root(self): diff --git a/stackdio/client/config.py b/stackdio/client/config.py index 629616a..5ba4fb8 100644 --- a/stackdio/client/config.py +++ b/stackdio/client/config.py @@ -18,6 +18,7 @@ import os import click +import keyring import requests from requests.exceptions import ConnectionError, MissingSchema @@ -33,12 +34,14 @@ class StackdioConfig(object): A wrapper around python's ConfigParser class """ + KEYRING_SERVICE = 'stackdio_cli' + BOOL_MAP = { str(True): True, str(False): False, } - def __init__(self, config_file=CFG_FILE, section='default'): + def __init__(self, config_file=CFG_FILE, section='main'): super(StackdioConfig, self).__init__() self.section = section @@ -54,6 +57,11 @@ def __init__(self, config_file=CFG_FILE, section='default'): else: self._config.read(config_file) + username = self.get('username') + + if username is not None: + self['password'] = keyring.get_password(self.KEYRING_SERVICE, username) + # Make the blueprint dir usable blueprint_dir = self.get('blueprint_dir') if blueprint_dir: @@ -64,6 +72,13 @@ def save(self): with open(self._cfg_file, 'w') as f: self._config.write(f) + def __contains__(self, item): + try: + self._config.get(self.section, item) + return True + except NoOptionError: + return False + def __getitem__(self, item): try: ret = self._config.get(self.section, item) From ac949590ee21df91167692064f4cb48588247ea2 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 6 Jan 2016 17:55:58 -0600 Subject: [PATCH 32/90] Updated generator to use click --- stackdio/cli/blueprints/__init__.py | 53 ++++++++++------------------ stackdio/cli/blueprints/generator.py | 31 ++++++---------- stackdio/cli/mixins/blueprints.py | 40 +++++++-------------- 3 files changed, 42 insertions(+), 82 deletions(-) diff --git a/stackdio/cli/blueprints/__init__.py b/stackdio/cli/blueprints/__init__.py index 71ee86e..5cfa4af 100644 --- a/stackdio/cli/blueprints/__init__.py +++ b/stackdio/cli/blueprints/__init__.py @@ -1,58 +1,43 @@ from __future__ import print_function -import argparse import os import json import sys -from stackdio.cli.blueprints.generator import BlueprintException, BlueprintGenerator - - -def main(): - parser = argparse.ArgumentParser( - description='invoke the stackdio blueprint generator') +import click - parser.add_argument('template_file', - help='The template file to generate from') +from stackdio.cli.blueprints.generator import BlueprintException, BlueprintGenerator - parser.add_argument('var_files', - metavar='var_file', - nargs='*', - help='The variable files with your custom config. They will be loaded ' - 'from left to right, so variables in the rightmost var files will ' - 'override those in var files to the left.') - parser.add_argument('-p', '--prompt', - action='store_true', - help='Prompt user for missing variables') +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) - parser.add_argument('-d', '--debug', - action='store_true', - help='Print out json string before parsing the json') - args = parser.parse_args() +@click.command(context_settings=CONTEXT_SETTINGS) +@click.argument('template_file') +@click.argument('var_files', nargs=-1, type=click.File('r')) +@click.option('-p', '--prompt', is_flag=True, default=False, + help='Prompt user for missing variables') +@click.option('-d', '--debug', is_flag=True, default=False, + help='Print out json string before parsing the json') +def main(template_file, var_files, prompt, debug): try: # Throw all output to stderr gen = BlueprintGenerator([os.path.curdir, os.path.join(os.path.curdir, 'templates'), - os.path.dirname(os.path.abspath(args.template_file))], + os.path.dirname(os.path.abspath(template_file))], output_stream=sys.stderr) # Generate the blueprint - blueprint = gen.generate(args.template_file, - var_files=args.var_files, - prompt=args.prompt, - debug=args.debug) + blueprint = gen.generate(template_file, + var_files=var_files, + prompt=prompt, + debug=debug) except BlueprintException: - sys.exit(1) + raise click.Abort('Error processing blueprint') - print(json.dumps(blueprint, indent=2)) + click.echo(json.dumps(blueprint, indent=2)) if __name__ == '__main__': - try: - main() - except KeyboardInterrupt: - sys.stderr.write('Aborting...\n') - sys.exit(1) + main() diff --git a/stackdio/cli/blueprints/generator.py b/stackdio/cli/blueprints/generator.py index d4c90d3..b70c2ce 100644 --- a/stackdio/cli/blueprints/generator.py +++ b/stackdio/cli/blueprints/generator.py @@ -4,6 +4,7 @@ import os import json +import click import yaml from jinja2 import Environment, FileSystemLoader, StrictUndefined, meta from jinja2.exceptions import TemplateNotFound, TemplateSyntaxError, UndefinedError @@ -11,14 +12,6 @@ from jinja2.filters import do_replace, evalcontextfilter -COLORS = { - 'brown': '\033[0;33m', - 'green': '\033[0;32m', - 'red': '\033[01;31m', - 'endc': '\033[0m', -} - - class BlueprintException(Exception): pass @@ -67,9 +60,8 @@ def error_exit(self, message, newlines=1): :param newlines: the number of newlines to print at the end :return: None """ - self.out_stream.write('{0}{1}{2}'.format(COLORS['red'], message, COLORS['endc'])) - for i in range(0, newlines): - self.out_stream.write('\n') + click.secho(message, file=self.out_stream, nl=False, fg='red') + click.echo('\n' * newlines, nl=False) raise BlueprintException() def warning(self, message, newlines=1): @@ -79,9 +71,8 @@ def warning(self, message, newlines=1): :param newlines: The number of newlines to print at the end :return: None """ - self.out_stream.write('{0}{1}{2}'.format(COLORS['brown'], message, COLORS['endc'])) - for i in range(0, newlines): - self.out_stream.write('\n') + click.secho(message, file=self.out_stream, nl=False, fg='yellow') + click.echo('\n' * newlines, nl=False) def prompt(self, message): """ @@ -89,7 +80,7 @@ def prompt(self, message): :param message: the prompt message :return: the value the user inputted """ - self.out_stream.write('{0}{1}{2}'.format(COLORS['green'], message, COLORS['endc'])) + click.secho(message, file=self.out_stream, nl=False, fg='green') raw = sys.stdin.readline().strip() # This should work nicely - if yaml can't parse it properly, then it should be fine to just @@ -202,7 +193,7 @@ def generate(self, template_file, var_files=(), variables=None, prompt=False, de :param template_file: The relative location of the template. It must be in one of the directories you specified when creating the Generator object. - :param var_file: The location of the variable file (relative or absolute) + :param var_files: The location of the variable file(s) (relative or absolute) :param variables: A dict of variables to put in the template. :param prompt: Option to prompt for missing variables :param debug: Print the output of the template before trying to parse it as JSON @@ -215,7 +206,7 @@ def generate(self, template_file, var_files=(), variables=None, prompt=False, de context = {} for var_file in var_files: - yaml_parsed = yaml.safe_load(open(var_file, 'r')) + yaml_parsed = yaml.safe_load(var_file) if yaml_parsed: context.update(yaml_parsed) @@ -287,9 +278,9 @@ def generate(self, template_file, var_files=(), variables=None, prompt=False, de template_json = template.render(**context) if debug: - print('\n') - print(template_json) - print('\n') + click.echo('\n') + click.echo(template_json) + click.echo('\n') # Return a dict object rather than a string return json.loads(template_json) diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index 2005049..17ba374 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -1,15 +1,10 @@ -from __future__ import print_function import json import os -import argparse -import sys import click import yaml -from cmd2 import Cmd -from stackdio.client.exceptions import StackException from stackdio.cli.blueprints.generator import BlueprintGenerator, BlueprintException from stackdio.cli.utils import print_summary @@ -20,6 +15,9 @@ class BlueprintNotFound(Exception): @click.group() def blueprints(): + """ + Perform actions on blueprints + """ pass @@ -124,11 +122,6 @@ def create_blueprint(obj, mapping, template, var_file, no_prompt): """ Create a blueprint """ - print(mapping) - print(template) - print(var_file) - print(no_prompt) - client = obj['client'] if not template and not mapping: @@ -165,19 +158,13 @@ def create_blueprint(obj, mapping, template, var_file, no_prompt): @blueprints.command(name='create-all') @click.pass_obj -@click.option('--no-prompt', is_flag=True, default=True, - help='Don\'t prompt to create all blueprints') -def create_all_blueprints(obj, no_prompt): +@click.confirmation_option('-y', '--yes', prompt='Really create all blueprints?') +def create_all_blueprints(obj): """ Create all the blueprints in the map file """ client = obj['client'] - if no_prompt: - really = raw_input('Really create all blueprints (y/n)? ') - if really not in ['Y', 'y']: - return - blueprint_dir = os.path.expanduser(client.config['blueprint_dir']) mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) @@ -192,14 +179,14 @@ def create_all_blueprints(obj, no_prompt): def _get_blueprint_id(client, blueprint_title): - blueprints = client.search_blueprints(title=blueprint_title) + found_blueprints = client.search_blueprints(title=blueprint_title) - if len(blueprints) == 0: - raise click.UsageError('Blueprint [{0}] does not exist'.format(blueprint_title)) - elif len(blueprints) > 1: - raise click.UsageError('Blueprint [{0}] does not exist'.format(blueprint_title)) + if len(found_blueprints) == 0: + raise click.Abort('Blueprint "{0}" does not exist'.format(blueprint_title)) + elif len(found_blueprints) > 1: + raise click.Abort('Multiple blueprints matching "{0}" were found'.format(blueprint_title)) else: - return blueprints[0]['id'] + return found_blueprints[0]['id'] @blueprints.command(name='delete') @@ -213,10 +200,7 @@ def delete_blueprint(obj, title): blueprint_id = _get_blueprint_id(client, title) - really = raw_input('Really delete blueprint {0} (y/n)? '.format(title)) - if really not in ['y', 'Y']: - click.echo('Aborting deletion') - return + click.confirm('Really delete blueprint {0}?'.format(title), abort=True) click.echo('Deleting {0}'.format(title)) client.delete_blueprint(blueprint_id) From 5a16b10c6d43e3666e821cb2da1de0802e64056b Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 6 Jan 2016 18:23:47 -0600 Subject: [PATCH 33/90] Beginnings of stacks --- stackdio/cli/__init__.py | 1 + stackdio/cli/mixins/blueprints.py | 4 +- stackdio/cli/mixins/stacks.py | 149 +++++++++++++++--------------- 3 files changed, 78 insertions(+), 76 deletions(-) diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index 6078c4d..a86d9a5 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -49,6 +49,7 @@ def server_version(obj): # Add all our other commands stackdio.add_command(blueprints.blueprints) +stackdio.add_command(stacks.stacks) class StackdioShell(Cmd, bootstrap.BootstrapMixin, stacks.StackMixin, diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index 17ba374..10bf7f6 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -178,7 +178,7 @@ def create_all_blueprints(obj): click.secho('Blueprint {0} NOT created\n'.format(name), fg='magenta') -def _get_blueprint_id(client, blueprint_title): +def get_blueprint_id(client, blueprint_title): found_blueprints = client.search_blueprints(title=blueprint_title) if len(found_blueprints) == 0: @@ -198,7 +198,7 @@ def delete_blueprint(obj, title): """ client = obj['client'] - blueprint_id = _get_blueprint_id(client, title) + blueprint_id = get_blueprint_id(client, title) click.confirm('Really delete blueprint {0}?'.format(title), abort=True) diff --git a/stackdio/cli/mixins/stacks.py b/stackdio/cli/mixins/stacks.py index 8be3dea..bf91848 100644 --- a/stackdio/cli/mixins/stacks.py +++ b/stackdio/cli/mixins/stacks.py @@ -2,11 +2,86 @@ import json +import click from cmd2 import Cmd +from stackdio.cli.mixins.blueprints import get_blueprint_id +from stackdio.cli.utils import print_summary from stackdio.client.exceptions import StackException +@click.group() +def stacks(): + """ + Perform actions on stacks + """ + pass + + +@stacks.command(name='list') +@click.pass_obj +def list_stacks(obj): + """ + List all stacks + """ + client = obj['client'] + + click.echo('Getting stacks ... ') + print_summary('Stack', client.list_stacks()) + + +@stacks.command(name='launch') +@click.pass_obj +@click.argument('blueprint_title') +@click.argument('stack_title') +def launch_stack(obj, blueprint_title, stack_title): + """ + Launch a stack from a blueprint + """ + client = obj['client'] + + blueprint_id = get_blueprint_id(client, blueprint_title) + + click.echo('Launching stack "{0}" from blueprint "{1}"'.format(stack_title, + blueprint_title)) + + stack_data = { + 'blueprint': blueprint_id, + 'title': stack_title, + 'description': 'Launched from blueprint %s' % (blueprint_title), + 'namespace': stack_title, + } + results = client.create_stack(stack_data) + click.echo('Stack launch results:\n{0}'.format(results)) + + +def get_stack_id(client, stack_title): + found_stacks = client.search_stacks(title=stack_title) + + if len(found_stacks) == 0: + raise click.Abort('Stack "{0}" does not exist'.format(stack_title)) + elif len(found_stacks) > 1: + raise click.Abort('Multiple stacks matching "{0}" were found'.format(stack_title)) + else: + return found_stacks[0]['id'] + + +@stacks.command(name='history') +@click.pass_obj +@click.argument('stack_title') +@click.option('-l', '--length', type=click.INT, default=20, help='The number of entries to show') +def stack_history(obj, stack_title, length): + """ + Print recent history for a stack + """ + client = obj['client'] + + stack_id = get_stack_id(client, stack_title) + history = client.get_stack_history(stack_id) + for event in history[0:min(length, len(history))]: + click.echo('[{created}] {level} // {event} // {status}'.format(**event)) + + class StackMixin(Cmd): STACK_ACTIONS = ["start", "stop", "launch_existing", "terminate", "provision", "custom"] @@ -57,67 +132,6 @@ def do_stacks(self, arg): else: print(USAGE) - def complete_stacks(self, text, line, begidx, endidx): - # not using line, begidx, or endidx, thus the following pylint disable - # pylint: disable=W0613 - return [i for i in self.STACK_COMMANDS if i.startswith(text)] - - def help_stacks(self): - print("Manage stacks.") - print("Sub-commands can be one of:\n\t{0}".format( - ", ".join(self.STACK_COMMANDS))) - print("Try 'stacks COMMAND' to get help on (most) sub-commands") - - def _list_stacks(self): - """List all running stacks""" - - print("Getting running stacks ... ") - stacks = self.stacks.list_stacks() - self._print_summary("Stack", stacks) - - def _launch_stack(self, args): - """Launch a stack from a blueprint. - Must provide blueprint name and stack name""" - - if len(args) != 2: - print("Usage: stacks launch BLUEPRINT_NAME STACK_NAME") - return - - blueprint_name = args[0] - stack_name = args[1] - - try: - blueprint_id = self.stacks.get_blueprint_id(blueprint_name) - except StackException: - print(self.colorize( - "Blueprint [{0}] does not exist".format(blueprint_name), - "red")) - return - - print("Launching stack [{0}] from blueprint [{1}]".format( - stack_name, blueprint_name)) - - stack_data = { - "blueprint": blueprint_id, - "title": stack_name, - "description": "Launched from blueprint %s" % (blueprint_name), - "namespace": stack_name, - "max_retries": 1, - } - results = self.stacks.create_stack(stack_data) - print("Stack launch results:\n{0}".format(results)) - - def _get_stack_id(self, stack_name): - """Validate that a stack exists""" - - try: - return self.stacks.get_stack_id(stack_name) - except StackException: - print(self.colorize( - "Stack [{0}] does not exist".format(stack_name), - "red")) - raise - def _stack_action(self, args): """Perform an action on a stack.""" @@ -165,19 +179,6 @@ def _stack_action(self, args): results = self.stacks.do_stack_action(stack_id, action) print("Stack action results:\n{0}".format(json.dumps(results, indent=3))) - def _stack_history(self, args): - """Print recent history for a stack""" - - NUM_EVENTS = 20 - if len(args) < 1: - print("Usage: stacks history STACK_NAME") - return - - stack_id = self._get_stack_id(args[0]) - history = self.stacks.get_stack_history(stack_id).get("results") - for event in history[0:min(NUM_EVENTS, len(history))]: - print("[{created}] {level} // {event} // {status}".format(**event)) - def _stack_hostnames(self, args): """Print hostnames for a stack""" From d598f33269a7618f925dc341b36ff0f73d50e6ae Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Mon, 11 Jan 2016 10:59:51 -0600 Subject: [PATCH 34/90] Finished most of stacks --- stackdio/cli/__init__.py | 135 +------------ stackdio/cli/blueprints/generator.py | 4 +- stackdio/cli/mixins/blueprints.py | 2 +- stackdio/cli/mixins/bootstrap.py | 3 +- stackdio/cli/mixins/stacks.py | 279 ++++++++++++--------------- stackdio/cli/polling.py | 33 ++-- stackdio/client/stack.py | 7 +- 7 files changed, 163 insertions(+), 300 deletions(-) diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index a86d9a5..c9284cb 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -2,13 +2,9 @@ from __future__ import print_function -import json import os -import sys -from cmd import Cmd import click -from requests import ConnectionError from stackdio.cli.mixins import blueprints, bootstrap, formulas, stacks from stackdio.client import StackdioClient @@ -36,6 +32,9 @@ def stackdio(ctx): @stackdio.command() @click.pass_obj def configure(obj): + """ + Configure the client + """ client = obj['client'] print('configuring') @@ -43,6 +42,9 @@ def configure(obj): @stackdio.command('server-version') @click.pass_obj def server_version(obj): + """ + Print the version of the server + """ client = obj['client'] click.echo('stackdio-server, version {0}'.format(client.get_version())) @@ -52,131 +54,6 @@ def server_version(obj): stackdio.add_command(stacks.stacks) -class StackdioShell(Cmd, bootstrap.BootstrapMixin, stacks.StackMixin, - formulas.FormulaMixin): - - CFG_DIR = os.path.expanduser("~/.stackdio-cli/") - CFG_FILE = os.path.join(CFG_DIR, "config.json") - BOOTSTRAP_FILE = os.path.join(CFG_DIR, "bootstrap.yaml") - KEYRING_SERVICE = "stackdio_cli" - PROMPT = "\n{username} @ {url}\n> " - HELP_CMDS = [ - "account_summary", - "stacks", "blueprints", "formulas", - "initial_setup", "bootstrap", - "help", "exit", "quit", - ] - - intro = """ -###################################################################### - s t a c k d . i o -###################################################################### -""" - - def __init__(self): - mixins.bootstrap.BootstrapMixin.__init__(self) - self._load_config() - if 'url' in self.config and self.config['url']: - self._init_stacks() - self._validate_auth() - - def _load_config(self): - """Attempt to load config file, otherwise fallback to DEFAULT_CONFIG""" - - try: - self.config = json.loads(open(self.CFG_FILE).read()) - self.config['blueprint_dir'] = os.path.expanduser(self.config.get('blueprint_dir', '')) - - except ValueError: - print(self.colorize( - "What happened?! The config file is not valid JSON. A " - "re-install is likely the easiest fix.", "red")) - raise - except IOError: - self.config = { - 'url': None, - 'username': None, - } - print(self.colorize( - "It seems like this is your first time using the CLI. Please run " - "'initial_setup' to configure.", "green")) - # print(self.colorize( - # "What happened?! Unable to find a config file. A re-install " - # "is likely the easiest fix.", "red")) - # raise - - self.has_public_key = None - - def _validate_auth(self): - """Verify that we can connect successfully to the api""" - - # If there is no config, just force the user to go through initial setup - if self.config['url'] is None: - return - - try: - self.stacks.get_root() - status_code = 200 - self.validated = (200 <= status_code <= 299) - except ConnectionError: - print(self.colorize( - "Unable to connect to {0}".format(self.config["url"]), - "red")) - raise - - if self.validated: - print(self.colorize( - "Config loaded and validated", "blue")) - self.has_public_key = self.stacks.get_public_key() - else: - print(self.colorize( - "ERROR: Unable to validate config", "red")) - self.has_public_key = None - - def _setprompt(self): - - Cmd.prompt = self.colorize( - self.PROMPT.format(**self.config), - "blue") - - if not self.validated and self.config['url'] is not None: - print(self.colorize(""" -## -## Unable to validate connection - one of several possibilities exist: -## If this is the first time you've fired this up, you need to run -## 'initial_setup' to configure your account details. If you've already -## done that, there could be a network connection issue anywhere between -## your computer and your stackd.io instance, -## or your password may be incorrect, or ... etc. -## - """, - "green")) - - if self.validated and not self.has_public_key: - print(self.colorize( - "## Your account is missing the public key, run 'bootstrap' to fix", - "red")) - - def do_account_summary(self, args=None): - """Get a summary of your account.""" - sys.stdout.write("Polling {0} ... ".format(self.config["url"])) - sys.stdout.flush() - - public_key = self.stacks.get_public_key() - formulas = self.stacks.list_formulas() - blueprints = self.stacks.list_blueprints() - stacks = self.stacks.list_stacks() - - sys.stdout.write("done\n") - - print("## Username: {0}".format(self.config["username"])) - print("## Public Key:\n{0}".format(public_key)) - - self._print_summary("Formula", formulas) - self._print_summary("Blueprint", blueprints) - self._print_summary("Stack", stacks) - - def main(): # Just run our CLI tool stackdio(obj={}) diff --git a/stackdio/cli/blueprints/generator.py b/stackdio/cli/blueprints/generator.py index b70c2ce..94c0503 100644 --- a/stackdio/cli/blueprints/generator.py +++ b/stackdio/cli/blueprints/generator.py @@ -294,5 +294,5 @@ def generate(self, template_file, var_files=(), variables=None, prompt=False, de )) except UndefinedError as e: self.error_exit('Missing variable: {0}'.format(str(e))) - except ValueError: - self.error_exit('Invalid JSON. Check your template file.') + # except ValueError: + # self.error_exit('Invalid JSON. Check your template file.') diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index 10bf7f6..a484c89 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -95,7 +95,7 @@ def _create_single_blueprint(config, template_file, var_files, no_prompt): for var_file in var_files: var_file = os.path.join(blueprint_dir, 'var_files', var_file) if os.path.exists(var_file): - final_var_files.append(var_file) + final_var_files.append(open(var_file, 'r')) else: click.secho('WARNING: Variable file {0} was not found. Ignoring.'.format(var_file), fg='magenta') diff --git a/stackdio/cli/mixins/bootstrap.py b/stackdio/cli/mixins/bootstrap.py index fcd78a6..c5af57e 100644 --- a/stackdio/cli/mixins/bootstrap.py +++ b/stackdio/cli/mixins/bootstrap.py @@ -264,6 +264,7 @@ def _bootstrap_account(self): def _bootstrap_formulas(self): """Import and wait for formulas to become ready""" + @poll_and_wait def _check_formulas(): formulas = self.stacks.list_formulas() for formula in formulas: @@ -282,7 +283,7 @@ def _check_formulas(): sys.stdout.write("Waiting for formulas .") sys.stdout.flush() try: - poll_and_wait(_check_formulas) + _check_formulas() sys.stdout.write(" done!\n") except TimeoutException: print(self.colorize( diff --git a/stackdio/cli/mixins/stacks.py b/stackdio/cli/mixins/stacks.py index bf91848..7577be2 100644 --- a/stackdio/cli/mixins/stacks.py +++ b/stackdio/cli/mixins/stacks.py @@ -1,7 +1,5 @@ from __future__ import print_function -import json - import click from cmd2 import Cmd @@ -10,6 +8,9 @@ from stackdio.client.exceptions import StackException +REQUIRE_ACTION_CONFIRMATION = ['terminate'] + + @click.group() def stacks(): """ @@ -35,24 +36,24 @@ def list_stacks(obj): @click.argument('blueprint_title') @click.argument('stack_title') def launch_stack(obj, blueprint_title, stack_title): - """ - Launch a stack from a blueprint - """ - client = obj['client'] + """ + Launch a stack from a blueprint + """ + client = obj['client'] - blueprint_id = get_blueprint_id(client, blueprint_title) + blueprint_id = get_blueprint_id(client, blueprint_title) - click.echo('Launching stack "{0}" from blueprint "{1}"'.format(stack_title, - blueprint_title)) + click.echo('Launching stack "{0}" from blueprint "{1}"'.format(stack_title, + blueprint_title)) - stack_data = { - 'blueprint': blueprint_id, - 'title': stack_title, - 'description': 'Launched from blueprint %s' % (blueprint_title), - 'namespace': stack_title, - } - results = client.create_stack(stack_data) - click.echo('Stack launch results:\n{0}'.format(results)) + stack_data = { + 'blueprint': blueprint_id, + 'title': stack_title, + 'description': 'Launched from blueprint %s' % (blueprint_title), + 'namespace': stack_title, + } + results = client.create_stack(stack_data) + click.echo('Stack launch results:\n{0}'.format(results)) def get_stack_id(client, stack_title): @@ -82,167 +83,141 @@ def stack_history(obj, stack_title, length): click.echo('[{created}] {level} // {event} // {status}'.format(**event)) -class StackMixin(Cmd): +@stacks.command(name='hostnames') +@click.pass_obj +@click.argument('stack_title') +def stack_hostnames(obj, stack_title): + """ + Print hostnames for a stack + """ + client = obj['client'] - STACK_ACTIONS = ["start", "stop", "launch_existing", "terminate", "provision", "custom"] - STACK_COMMANDS = ["list", "launch_from_blueprint", "history", "hostnames", - "delete", "logs", "access_rules"] + STACK_ACTIONS - - VALID_LOGS = { - "provisioning": "provisioning.log", - "provisioning-error": "provisioning.err", - "global-orchestration": "global_orchestration.log", - "global-orchestration-error": "global_orchestration.err", - "orchestration": "orchestration.log", - "orchestration-error": "orchestration.err", - "launch": "launch.log", - } + stack_id = get_stack_id(client, stack_title) + hosts = client.get_stack_hosts(stack_id) - def do_stacks(self, arg): - """Entry point to controlling.""" + click.echo('Hostnames:') + for host in hosts: + click.echo(' - {0} ({1})'.format(host['fqdn'], host['state'])) - USAGE = "Usage: stacks COMMAND\nWhere COMMAND is one of: %s" % ( - ", ".join(self.STACK_COMMANDS)) - # We don't want multiline commands, so include anything after a terminator as well - args = arg.parsed.raw.split()[1:] - if not args or args[0] not in self.STACK_COMMANDS: - print(USAGE) - return +@stacks.command(name='delete') +@click.pass_obj +@click.argument('stack_title') +def delete_stack(obj, stack_title): + """ + Delete a stack. PERMANENT AND DESTRUCTIVE!!! + """ + client = obj['client'] - stack_cmd = args[0] - - if stack_cmd == "list": - self._list_stacks() - elif stack_cmd == "launch_from_blueprint": - self._launch_stack(args[1:]) - elif stack_cmd == "history": - self._stack_history(args[1:]) - elif stack_cmd == "hostnames": - self._stack_hostnames(args[1:]) - elif stack_cmd == "delete": - self._stack_delete(args[1:]) - elif stack_cmd == "logs": - self._stack_logs(args[1:]) - elif stack_cmd == "access_rules": - self._stack_access_rules(args[1:]) - elif stack_cmd in self.STACK_ACTIONS: - self._stack_action(args) - - else: - print(USAGE) - - def _stack_action(self, args): - """Perform an action on a stack.""" - - if len(args) == 1: - print("Usage: stacks {0} STACK_NAME".format(args[0])) - return - elif args[0] != "custom" and len(args) != 2: - print("Usage: stacks ACTION STACK_NAME") - print("Where ACTION is one of {0}".format( - ", ".join(self.STACK_ACTIONS))) - return - elif args[0] == "custom" and len(args) < 4: - print("Usage: stacks custom STACK_NAME HOST_TARGET COMMAND") - print("Where command can be arbitrarily long with spaces") - return + stack_id = get_stack_id(client, stack_title) - if args[0] not in self.STACK_ACTIONS: - print(self.colorize( - "Invalid action - must be one of {0}".format(self.STACK_ACTIONS), - "red")) - return + click.confirm('Really delete stack {0}?'.format(stack_title), abort=True) - action = "launch" if args[0] == "launch_existing" else args[0] - stack_name = args[1] + results = client.delete_stack(stack_id) + click.echo('Delete stack results: \n{0}'.format(results)) + click.secho('Run "stacks history {0}" to monitor status of the deletion'.format(stack_title), + fg='green') - if action == "terminate": - really = raw_input("Really terminate stack {0} (y/n)? ".format(args[0])) - if really not in ["y", "Y"]: - print("Aborting termination") - return - if action == "custom": - host_target = args[2] - command = '' - for token in args[3:]: - command += token + ' ' +@stacks.command(name='action') +@click.pass_obj +@click.argument('stack_title') +@click.argument('action') +def perform_action(obj, stack_title, action): + """ + Perform an action on a stack + """ + client = obj['client'] - stack_id = self._get_stack_id(stack_name) - print("Performing [{0}] on [{1}]".format( - action, stack_name)) + stack_id = get_stack_id(client, stack_title) - if action == "custom": - results = self.stacks.do_stack_action(stack_id, "custom", host_target, command) - else: - results = self.stacks.do_stack_action(stack_id, action) - print("Stack action results:\n{0}".format(json.dumps(results, indent=3))) + # Prompt for confirmation if need be + if action in REQUIRE_ACTION_CONFIRMATION: + click.confirm('Really {0} stack {1}?'.format(action, stack_title), abort=True) - def _stack_hostnames(self, args): - """Print hostnames for a stack""" + try: + client.do_stack_action(stack_id, action) + except StackException as e: + raise click.UsageError(e.message) - if len(args) < 1: - print("Usage: stacks hostnames STACK_NAME") - return - stack_id = self._get_stack_id(args[0]) - try: - fqdns = self.stacks.describe_hosts(stack_id) - except StackException: - print(self.colorize( - "Hostnames not available - stack still launching?", "red")) - raise +@stacks.command(name='run') +@click.pass_obj +@click.argument('stack_title') +@click.argument('command') +@click.option('-w', '--wait', is_flag=True, help='Wait for the command to finish running') +def run_command(obj, stack_title, command, wait): + """ + Run a command on all hosts in the stack + """ + pass - print("Hostnames:") - for host in fqdns: - print(" - {0}".format(host)) - def _stack_delete(self, args): - """Delete a stack. PERMANENT AND DESTRUCTIVE!!!""" +def print_logs(client, stack_id): + logs = client.list_stack_logs(stack_id) - if len(args) < 1: - print("Usage: stacks delete STACK_NAME") - return + click.echo('Latest:') + for log in logs['latest']: + click.echo(' {0}'.format(log.split('/')[-1])) - stack_id = self._get_stack_id(args[0]) - really = raw_input("Really delete stack {0} (y/n)? ".format(args[0])) - if really not in ["y", "Y"]: - print("Aborting deletion") - return + click.echo() - results = self.stacks.delete_stack(stack_id) - print("Delete stack results: {0}".format(results)) - print(self.colorize( - "Run 'stacks history {0}' to monitor status of the deletion".format( - args[0]), - "green")) + click.echo('Historical:') + for log in logs['historical']: + click.echo(' {0}'.format(log.split('/')[-1])) - def _stack_logs(self, args): - """Get logs for a stack""" - MAX_LINES = 25 +@stacks.command(name='list-logs') +@click.pass_obj +@click.argument('stack_title') +def list_stack_logs(obj, stack_title): + """ + Get a list of stack logs + """ + client = obj['client'] - if len(args) < 2: - print("Usage: stacks logs STACK_NAME LOG_TYPE [LOG_LENGTH]") - print("LOG_TYPE is one of {0}".format( - ", ".join(self.stacks.VALID_LOGS.keys()))) - print("This defaults to the last {0} lines of the log.". - format(MAX_LINES)) - return + stack_id = get_stack_id(client, stack_title) + + print_logs(client, stack_id) - if len(args) >= 3: - max_lines = args[2] - else: - max_lines = MAX_LINES - stack_id = self.stacks.get_stack_id(args[0]) +@stacks.command(name='logs') +@click.pass_obj +@click.argument('stack_title') +@click.argument('log_type') +@click.option('-l', '--lines', type=click.INT, default=25, help='number of lines to tail') +def stack_logs(obj, stack_title, log_type, lines): + """ + Get logs for a stack + """ + client = obj['client'] + + stack_id = get_stack_id(client, stack_title) + + split_arg = log_type.split('.') + + valid_log = True - split_arg = self.VALID_LOGS[args[1]].split('.') + if len(split_arg) != 3: + valid_log = False - log_text = self.stacks.get_logs(stack_id, log_type=split_arg[0], level=split_arg[1], - tail=max_lines) - print(log_text) + if valid_log: + try: + log_text = client.get_logs(stack_id, log_type=split_arg[0], level=split_arg[1], + date=split_arg[2], tail=lines) + click.echo(log_text) + except StackException: + valid_log = True + + if not valid_log: + click.echo('Please use one of these logs:\n') + + print_logs(client, stack_id) + + raise click.UsageError('Invalid log') + + +class StackMixin(Cmd): def _stack_access_rules(self, args): """Get access rules for a stack""" diff --git a/stackdio/cli/polling.py b/stackdio/cli/polling.py index aa1c5a3..e10ca8a 100644 --- a/stackdio/cli/polling.py +++ b/stackdio/cli/polling.py @@ -1,26 +1,31 @@ -import sys import time +import click + class TimeoutException(Exception): pass -def poll_and_wait(func, args=None, sleep_time=2, max_time=120): - """Execute func in increments of sleep_time for no more than max_time. - Raise TimeoutException if we're not successful in max_time""" +def poll_and_wait(func): + """ + Execute func in increments of sleep_time for no more than max_time. + Raise TimeoutException if we're not successful in max_time + """ + def decorator(args=None, sleep_time=2, max_time=120): + args = args or [] - args = args or [] - current_time = 0 + current_time = 0 - success = func(*args) - while not success and current_time < max_time: - sys.stdout.write(".") - sys.stdout.flush() - current_time += sleep_time - time.sleep(sleep_time) success = func(*args) + while not success and current_time < max_time: + click.echo('.', nl=False) + current_time += sleep_time + time.sleep(sleep_time) + success = func(*args) + + if not success: + raise TimeoutException() - if not success: - raise TimeoutException() + return decorator diff --git a/stackdio/client/stack.py b/stackdio/client/stack.py index 4a7ff39..228877b 100644 --- a/stackdio/client/stack.py +++ b/stackdio/client/stack.py @@ -82,7 +82,12 @@ def get_stack_hosts(self, stack_id): """Get a list of all stack hosts""" pass - @get('stacks/{stack_id}/logs/{log_type}.{level}.{date}', jsonify=False) + @get('stacks/{stack_id}/logs/') + def list_stack_logs(self, stack_id): + """Get a list of stack logs""" + pass + + @get('stacks/{stack_id}/logs/{log_type}.{level}.{date}?tail={tail}', jsonify=False) def get_logs(self, stack_id, log_type, level='log', date='latest', tail=None): """Get logs for a stack""" From 3d29e26b06b584482bf371005ba3a93c48a98f0d Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Mon, 11 Jan 2016 17:15:20 -0600 Subject: [PATCH 35/90] Finished the bulk of stack things. Left stack access rules off for now since we don't use them --- stackdio/cli/mixins/stacks.py | 161 ++++++++++++++-------------------- stackdio/cli/polling.py | 4 +- stackdio/cli/utils.py | 3 + stackdio/client/stack.py | 24 +++++ stackdio/client/version.py | 2 + 5 files changed, 96 insertions(+), 98 deletions(-) diff --git a/stackdio/cli/mixins/stacks.py b/stackdio/cli/mixins/stacks.py index 7577be2..ba6bfc3 100644 --- a/stackdio/cli/mixins/stacks.py +++ b/stackdio/cli/mixins/stacks.py @@ -1,9 +1,9 @@ from __future__ import print_function import click -from cmd2 import Cmd from stackdio.cli.mixins.blueprints import get_blueprint_id +from stackdio.cli.polling import poll_and_wait from stackdio.cli.utils import print_summary from stackdio.client.exceptions import StackException @@ -141,16 +141,75 @@ def perform_action(obj, stack_title, action): raise click.UsageError(e.message) +def print_command_output(json_blob): + for host in sorted(json_blob['std_out'], key=lambda x: x['host']): + click.secho('{0}:'.format(host['host']), fg='green') + click.echo(host['output']) + click.echo() + + @stacks.command(name='run') -@click.pass_obj +@click.pass_context @click.argument('stack_title') +@click.argument('host_target') @click.argument('command') -@click.option('-w', '--wait', is_flag=True, help='Wait for the command to finish running') -def run_command(obj, stack_title, command, wait): +@click.option('-w', '--wait', is_flag=True, default=False, + help='Wait for the command to finish running') +@click.option('-t', '--timeout', type=click.INT, default=120, + help='The amount of time to wait for the command in seconds. ' + 'Ignored if used without the -w option.') +def run_command(ctx, stack_title, host_target, command, wait, timeout): """ Run a command on all hosts in the stack """ - pass + client = ctx.obj['client'] + + stack_id = get_stack_id(client, stack_title) + + resp = client.run_command(stack_id, host_target, command) + + if not wait: + # Grab the parent info name + name = ctx.parent.parent.info_name + + click.echo('Command "{0}" running on "{1}" hosts. ' + 'Check the status by running:'.format(command, host_target)) + click.echo() + click.secho(' {0} stacks command-output {1}'.format(name, resp['id']), fg='yellow') + click.echo() + return + + @poll_and_wait + def check_status(): + command_out = client.get_command(resp['id']) + + if command_out['status'] != 'finished': + return False + + click.echo() + print_command_output(command_out) + + return True + + check_status(max_time=timeout) + + +@stacks.command(name='command-output') +@click.pass_obj +@click.argument('command_id') +def get_command_output(obj, command_id): + """ + Get the status and output of a command + """ + client = obj['client'] + + resp = client.get_command(command_id) + + if resp['status'] != 'finished': + click.secho('Status: {0}'.format(resp['status']), fg='yellow') + return + + print_command_output(resp) def print_logs(client, stack_id): @@ -215,95 +274,3 @@ def stack_logs(obj, stack_title, log_type, lines): print_logs(client, stack_id) raise click.UsageError('Invalid log') - - -class StackMixin(Cmd): - - def _stack_access_rules(self, args): - """Get access rules for a stack""" - - COMMANDS = ["list", "add", "delete"] - - if len(args) < 2 or args[0] not in COMMANDS: - print("Usage: stacks access_rules COMMAND STACK_NAME") - print("Where COMMAND is one of: %s" % (", ".join(COMMANDS))) - return - - if args[0] == "list": - stack_id = self.stacks.get_stack_id(args[1]) - groups = self.stacks.list_access_rules(stack_id) - print("## {0} Access Groups".format(len(groups))) - for group in groups: - print("- Name: {0}".format(group['blueprint_host_definition']['title'])) - print(" Description: {0}".format(group['blueprint_host_definition']['description'])) - print(" Rules:") - for rule in group['rules']: - print(" {0}".format(rule['protocol']), end='') - if rule['from_port'] == rule['to_port']: - print("port {0} allows".format(rule['from_port']), end='') - else: - print("ports {0}-{1} allow".format(rule['from_port'], - rule['to_port']), end='') - print(rule['rule']) - print('') - return - - elif args[0] == "add": - if len(args) < 3: - print("Usage: stacks access_rules add STACK_NAME GROUP_NAME") - return - - stack_id = self.stacks.get_stack_id(args[1]) - group_id = self.stacks.get_access_rule_id(stack_id, args[2]) - - protocol = raw_input("Protocol (tcp, udp, or icmp): ") - from_port = raw_input("From port: ") - to_port = raw_input("To port: ") - rule = raw_input("Rule (IP address or group name): ") - - data = { - "action": "authorize", - "protocol": protocol, - "from_port": from_port, - "to_port": to_port, - "rule": rule - } - - self.stacks.edit_access_rule(group_id, data) - - elif args[0] == "delete": - if len(args) < 3: - print("Usage: stacks access_rules delete STACK_NAME GROUP_NAME") - return - - stack_id = self.stacks.get_stack_id(args[1]) - group_id = self.stacks.get_access_rule_id(stack_id, args[2]) - - index = 0 - - rules = self.stacks.list_rules_for_group(group_id) - - print('') - for rule in rules: - print("{0}) {1}".format(index, rule['protocol']), end='') - if rule['from_port'] == rule['to_port']: - print("port {0} allows".format(rule['from_port']), end='') - else: - print("ports {0}-{1} allow".format(rule['from_port'], rule['to_port']), end='') - print(rule['rule']) - index += 1 - print('') - delete_index = int(raw_input("Enter the index of the rule to delete: ")) - - data = rules[delete_index] - data['from_port'] = int(data['from_port']) - data['to_port'] = int(data['to_port']) - data['action'] = "revoke" - - self.stacks.edit_access_rule(group_id, data) - - print('') - - args[0] = "list" - - self._stack_access_rules(args) diff --git a/stackdio/cli/polling.py b/stackdio/cli/polling.py index e10ca8a..29d1a20 100644 --- a/stackdio/cli/polling.py +++ b/stackdio/cli/polling.py @@ -1,5 +1,6 @@ import time +from functools import wraps import click @@ -13,6 +14,7 @@ def poll_and_wait(func): Execute func in increments of sleep_time for no more than max_time. Raise TimeoutException if we're not successful in max_time """ + @wraps(func) def decorator(args=None, sleep_time=2, max_time=120): args = args or [] @@ -20,9 +22,9 @@ def decorator(args=None, sleep_time=2, max_time=120): success = func(*args) while not success and current_time < max_time: - click.echo('.', nl=False) current_time += sleep_time time.sleep(sleep_time) + click.echo('.', nl=False) success = func(*args) if not success: diff --git a/stackdio/cli/utils.py b/stackdio/cli/utils.py index c5d8ac3..811a886 100644 --- a/stackdio/cli/utils.py +++ b/stackdio/cli/utils.py @@ -33,6 +33,9 @@ def print_summary(title, components): if 'description' in item: click.echo(' Description: {0}'.format(item['description'])) + if 'status' in item: + click.echo(' Status: {0}'.format(item['status'])) + if 'status_detail' in item: click.echo(' Status Detail: {0}'.format(item['status_detail'])) diff --git a/stackdio/client/stack.py b/stackdio/client/stack.py index 228877b..927fe97 100644 --- a/stackdio/client/stack.py +++ b/stackdio/client/stack.py @@ -72,6 +72,30 @@ def do_stack_action(self, stack_id, action): return {'action': action} + @post('stacks/{stack_id}/commands/') + def run_command(self, stack_id, host_target, command): + """ + Run a command on all stacks + """ + return { + 'host_target': host_target, + 'command': command, + } + + @get('stacks/{stack_id}/commands/') + def get_stack_commands(self, stack_id): + """ + Get all commands for a stack + """ + pass + + @get('commands/{command_id}/') + def get_command(self, command_id): + """ + Get information about a command + """ + pass + @get('stacks/{stack_id}/history/', paginate=True) def get_stack_history(self, stack_id): """Get stack info""" diff --git a/stackdio/client/version.py b/stackdio/client/version.py index dd69ac7..bab03a8 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -66,6 +66,8 @@ def _parse_version_string(version_string): # String trailing info version_string = re.split("[a-zA-Z]", version_string)[0] + if version_string[-1] == '.': + version_string = version_string[:-1] version = version_string.split(".") # Pad length to 3 From abf131540ea217538be4c013c813d8cd692a972e Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Mon, 11 Jan 2016 17:36:54 -0600 Subject: [PATCH 36/90] Added functionality for formulas --- stackdio/cli/__init__.py | 1 + stackdio/cli/mixins/formulas.py | 111 ++++++++++++++------------------ 2 files changed, 51 insertions(+), 61 deletions(-) diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index c9284cb..9587cc0 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -52,6 +52,7 @@ def server_version(obj): # Add all our other commands stackdio.add_command(blueprints.blueprints) stackdio.add_command(stacks.stacks) +stackdio.add_command(formulas.formulas) def main(): diff --git a/stackdio/cli/mixins/formulas.py b/stackdio/cli/mixins/formulas.py index 54664ce..916a2a1 100644 --- a/stackdio/cli/mixins/formulas.py +++ b/stackdio/cli/mixins/formulas.py @@ -1,82 +1,71 @@ from __future__ import print_function -from cmd2 import Cmd +import click +from stackdio.cli.utils import print_summary -class FormulaMixin(Cmd): - FORMULA_COMMANDS = ["list", "import", "delete"] - def do_formulas(self, arg): - """Entry point to controlling formulas.""" +@click.group() +def formulas(): + """ + Perform actions on formulas + """ + pass - USAGE = "Usage: formulas COMMAND\nWhere COMMAND is one of: %s" % ( - ", ".join(self.FORMULA_COMMANDS)) - args = arg.split() - if not args or args[0] not in self.FORMULA_COMMANDS: - print(USAGE) - return +@formulas.command(name='list') +@click.pass_obj +def list_formulas(obj): + """ + List all formulas + """ + client = obj['client'] - formula_cmd = args[0] - if formula_cmd == "list": - self._list_formulas() - elif formula_cmd == "import": - self._import_formula(args[1:]) - elif formula_cmd == "delete": - self._delete_formula(args[1:]) + click.echo('Getting formulas ... ') + print_summary('Formula', client.list_formulas()) - else: - print(USAGE) - def complete_formulas(self, text, line, begidx, endidx): - # not using line, begidx, or endidx, thus the following pylint disable - # pylint: disable=W0613 - return [i for i in self.FORMULA_COMMANDS if i.startswith(text)] +@formulas.command(name='import') +@click.pass_obj +@click.argument('uri') +@click.option('-u', '--username', type=click.STRING, help='Git username') +@click.option('-p', '--password', type=click.STRING, prompt=True, hide_input=True, + help='Git password') +def import_formula(obj, uri, username, password): + """ + Import a formula + """ + client = obj['client'] - def help_formulas(self): - print("Manage formulas.") - print("Sub-commands can be one of:\n\t{0}".format( - ", ".join(self.FORMULA_COMMANDS))) - print("Try 'formulas COMMAND' to get help on (most) sub-commands") + if username and not password: + raise click.UsageError('You must provide a password when providing a username') - def _list_formulas(self): - """List all formulas""" + click.echo('Importing formula from {0}'.format(uri)) + formula = client.import_formula(uri, git_username=username, git_password=password) - print("Getting formulas ... ") - formulas = self.stacks.list_formulas() - self._print_summary("Formula", formulas) + click.echo('Detail: {0}'.format(formula['status_detail'])) - def _import_formula(self, args): - """Import a formula""" - if len(args) != 1: - print("Usage: formulas import URL") - return +def get_formula_id(client, formula_uri): + found_formulas = client.search_formulas(uri=formula_uri) - formula_url = args[0] - print("Importing formula from {0}".format(formula_url)) - formula = self.stacks.import_formula(formula_url, public=False) + if len(found_formulas) == 0: + raise click.Abort('Formula "{0}" does not exist'.format(formula_uri)) + else: + return found_formulas[0]['id'] - if isinstance(formula, list): - print("Formula imported, try the 'list' command to monitor status") - elif formula.get("detail"): - print("Error importing: {0}".format(formula.get("detail"))) - def _delete_formula(self, args): - """Delete a formula""" +@formulas.command(name='delete') +@click.pass_obj +@click.argument('uri') +def delete_formula(obj, uri): + """ + Delete a formula + """ + client = obj['client'] - args = " ".join(args) - if len(args) == 0: - print("Usage: formulas delete TITLE") - return + formula_id = get_formula_id(client, uri) - formula_id = self.stacks.get_formula_id(args) + click.confirm('Really delete formula {0}?'.format(uri), abort=True) - really = raw_input("Really delete formula {0} (y/n)? ".format(args)) - if really not in ["y", "Y"]: - print("Aborting deletion") - return - - self.stacks.delete_formula(formula_id) - - print("Formula deleted, try the 'list' command to monitor status") + client.delete_formula(formula_id) From c0735428990eef8345b8e2199d4b44d3e4e444de Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 12 Jan 2016 10:43:48 -0600 Subject: [PATCH 37/90] Change from pass_obj to pass_client --- stackdio/cli/__init__.py | 17 +++++------ stackdio/cli/mixins/blueprints.py | 38 ++++++++--------------- stackdio/cli/mixins/bootstrap.py | 2 +- stackdio/cli/mixins/formulas.py | 14 ++++----- stackdio/cli/mixins/stacks.py | 39 ++++++++++++------------ stackdio/cli/polling.py | 33 -------------------- stackdio/cli/utils.py | 50 +++++++++++++++++++++++++++++++ stackdio/client/http.py | 8 +++-- 8 files changed, 104 insertions(+), 97 deletions(-) delete mode 100644 stackdio/cli/polling.py diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index 9587cc0..37adb77 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -7,6 +7,7 @@ import click from stackdio.cli.mixins import blueprints, bootstrap, formulas, stacks +from stackdio.cli.utils import pass_client from stackdio.client import StackdioClient from stackdio.client.version import __version__ @@ -26,26 +27,24 @@ def stackdio(ctx): '`stackdio-cli configure`') # Put the client in the obj - ctx.obj['client'] = client + ctx.obj = client @stackdio.command() -@click.pass_obj -def configure(obj): +@pass_client +def configure(client): """ Configure the client """ - client = obj['client'] - print('configuring') + click.echo('configuring') @stackdio.command('server-version') -@click.pass_obj -def server_version(obj): +@pass_client +def server_version(client): """ Print the version of the server """ - client = obj['client'] click.echo('stackdio-server, version {0}'.format(client.get_version())) @@ -57,7 +56,7 @@ def server_version(obj): def main(): # Just run our CLI tool - stackdio(obj={}) + stackdio() if __name__ == '__main__': diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index a484c89..f3d3469 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -6,7 +6,7 @@ import yaml from stackdio.cli.blueprints.generator import BlueprintGenerator, BlueprintException -from stackdio.cli.utils import print_summary +from stackdio.cli.utils import print_summary, pass_client class BlueprintNotFound(Exception): @@ -22,13 +22,11 @@ def blueprints(): @blueprints.command(name='list') -@click.pass_obj -def list_blueprints(obj): +@pass_client +def list_blueprints(client): """ List all blueprints """ - client = obj['client'] - click.echo('Getting blueprints ... ') print_summary('Blueprint', client.list_blueprints()) @@ -45,13 +43,11 @@ def _recurse_dir(dirname, extensions, prefix=''): @blueprints.command(name='list-templates') -@click.pass_obj -def list_templates(obj): +@pass_client +def list_templates(client): """ List all the blueprint templates """ - client = obj['client'] - if 'blueprint_dir' not in client.config: click.echo('Missing blueprint directory config') return @@ -107,7 +103,7 @@ def _create_single_blueprint(config, template_file, var_files, no_prompt): @blueprints.command(name='create') -@click.pass_obj +@pass_client @click.option('-m', '--mapping', help='The entry in the map file to use') @click.option('-t', '--template', @@ -118,12 +114,10 @@ def _create_single_blueprint(config, template_file, var_files, no_prompt): 'var files will override those in var files to the left.') @click.option('-n', '--no-prompt', is_flag=True, default=True, help='Don\'t prompt for missing variables in the template') -def create_blueprint(obj, mapping, template, var_file, no_prompt): +def create_blueprint(client, mapping, template, var_file, no_prompt): """ Create a blueprint """ - client = obj['client'] - if not template and not mapping: raise click.UsageError('You must specify either a template or a mapping.') @@ -157,14 +151,12 @@ def create_blueprint(obj, mapping, template, var_file, no_prompt): @blueprints.command(name='create-all') -@click.pass_obj +@pass_client @click.confirmation_option('-y', '--yes', prompt='Really create all blueprints?') -def create_all_blueprints(obj): +def create_all_blueprints(client): """ Create all the blueprints in the map file """ - client = obj['client'] - blueprint_dir = os.path.expanduser(client.config['blueprint_dir']) mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) @@ -190,14 +182,12 @@ def get_blueprint_id(client, blueprint_title): @blueprints.command(name='delete') -@click.pass_obj +@pass_client @click.argument('title') -def delete_blueprint(obj, title): +def delete_blueprint(client, title): """ Delete a blueprint """ - client = obj['client'] - blueprint_id = get_blueprint_id(client, title) click.confirm('Really delete blueprint {0}?'.format(title), abort=True) @@ -207,15 +197,13 @@ def delete_blueprint(obj, title): @blueprints.command(name='delete-all') -@click.pass_obj +@pass_client @click.confirmation_option('-y', '--yes', prompt='Really delete all blueprints? This is ' 'completely destructive, and you will never get them back.') -def delete_all_blueprints(obj): +def delete_all_blueprints(client): """ Delete all blueprints """ - client = obj['client'] - for blueprint in client.list_blueprints(): client.delete_blueprint(blueprint['id']) click.secho('Deleted blueprint {0}'.format(blueprint['title']), fg='magenta') diff --git a/stackdio/cli/mixins/bootstrap.py b/stackdio/cli/mixins/bootstrap.py index c5af57e..7cd8d02 100644 --- a/stackdio/cli/mixins/bootstrap.py +++ b/stackdio/cli/mixins/bootstrap.py @@ -12,7 +12,7 @@ import keyring import yaml -from stackdio.cli.polling import poll_and_wait, TimeoutException +from stackdio.cli.utils import TimeoutException, poll_and_wait class PublicKeyNotFound(Exception): diff --git a/stackdio/cli/mixins/formulas.py b/stackdio/cli/mixins/formulas.py index 916a2a1..fd71643 100644 --- a/stackdio/cli/mixins/formulas.py +++ b/stackdio/cli/mixins/formulas.py @@ -2,7 +2,7 @@ import click -from stackdio.cli.utils import print_summary +from stackdio.cli.utils import pass_client, print_summary @click.group() @@ -14,8 +14,8 @@ def formulas(): @formulas.command(name='list') -@click.pass_obj -def list_formulas(obj): +@pass_client +def list_formulas(client): """ List all formulas """ @@ -26,12 +26,12 @@ def list_formulas(obj): @formulas.command(name='import') -@click.pass_obj +@pass_client @click.argument('uri') @click.option('-u', '--username', type=click.STRING, help='Git username') @click.option('-p', '--password', type=click.STRING, prompt=True, hide_input=True, help='Git password') -def import_formula(obj, uri, username, password): +def import_formula(client, uri, username, password): """ Import a formula """ @@ -56,9 +56,9 @@ def get_formula_id(client, formula_uri): @formulas.command(name='delete') -@click.pass_obj +@pass_client @click.argument('uri') -def delete_formula(obj, uri): +def delete_formula(client, uri): """ Delete a formula """ diff --git a/stackdio/cli/mixins/stacks.py b/stackdio/cli/mixins/stacks.py index ba6bfc3..fc0dece 100644 --- a/stackdio/cli/mixins/stacks.py +++ b/stackdio/cli/mixins/stacks.py @@ -3,8 +3,7 @@ import click from stackdio.cli.mixins.blueprints import get_blueprint_id -from stackdio.cli.polling import poll_and_wait -from stackdio.cli.utils import print_summary +from stackdio.cli.utils import pass_client, print_summary, poll_and_wait from stackdio.client.exceptions import StackException @@ -20,8 +19,8 @@ def stacks(): @stacks.command(name='list') -@click.pass_obj -def list_stacks(obj): +@pass_client +def list_stacks(client): """ List all stacks """ @@ -32,10 +31,10 @@ def list_stacks(obj): @stacks.command(name='launch') -@click.pass_obj +@pass_client @click.argument('blueprint_title') @click.argument('stack_title') -def launch_stack(obj, blueprint_title, stack_title): +def launch_stack(client, blueprint_title, stack_title): """ Launch a stack from a blueprint """ @@ -68,10 +67,10 @@ def get_stack_id(client, stack_title): @stacks.command(name='history') -@click.pass_obj +@pass_client @click.argument('stack_title') @click.option('-l', '--length', type=click.INT, default=20, help='The number of entries to show') -def stack_history(obj, stack_title, length): +def stack_history(client, stack_title, length): """ Print recent history for a stack """ @@ -84,9 +83,9 @@ def stack_history(obj, stack_title, length): @stacks.command(name='hostnames') -@click.pass_obj +@pass_client @click.argument('stack_title') -def stack_hostnames(obj, stack_title): +def stack_hostnames(client, stack_title): """ Print hostnames for a stack """ @@ -101,9 +100,9 @@ def stack_hostnames(obj, stack_title): @stacks.command(name='delete') -@click.pass_obj +@pass_client @click.argument('stack_title') -def delete_stack(obj, stack_title): +def delete_stack(client, stack_title): """ Delete a stack. PERMANENT AND DESTRUCTIVE!!! """ @@ -120,10 +119,10 @@ def delete_stack(obj, stack_title): @stacks.command(name='action') -@click.pass_obj +@pass_client @click.argument('stack_title') @click.argument('action') -def perform_action(obj, stack_title, action): +def perform_action(client, stack_title, action): """ Perform an action on a stack """ @@ -195,9 +194,9 @@ def check_status(): @stacks.command(name='command-output') -@click.pass_obj +@pass_client @click.argument('command_id') -def get_command_output(obj, command_id): +def get_command_output(client, command_id): """ Get the status and output of a command """ @@ -227,9 +226,9 @@ def print_logs(client, stack_id): @stacks.command(name='list-logs') -@click.pass_obj +@pass_client @click.argument('stack_title') -def list_stack_logs(obj, stack_title): +def list_stack_logs(client, stack_title): """ Get a list of stack logs """ @@ -241,11 +240,11 @@ def list_stack_logs(obj, stack_title): @stacks.command(name='logs') -@click.pass_obj +@pass_client @click.argument('stack_title') @click.argument('log_type') @click.option('-l', '--lines', type=click.INT, default=25, help='number of lines to tail') -def stack_logs(obj, stack_title, log_type, lines): +def stack_logs(client, stack_title, log_type, lines): """ Get logs for a stack """ diff --git a/stackdio/cli/polling.py b/stackdio/cli/polling.py deleted file mode 100644 index 29d1a20..0000000 --- a/stackdio/cli/polling.py +++ /dev/null @@ -1,33 +0,0 @@ - -import time -from functools import wraps - -import click - - -class TimeoutException(Exception): - pass - - -def poll_and_wait(func): - """ - Execute func in increments of sleep_time for no more than max_time. - Raise TimeoutException if we're not successful in max_time - """ - @wraps(func) - def decorator(args=None, sleep_time=2, max_time=120): - args = args or [] - - current_time = 0 - - success = func(*args) - while not success and current_time < max_time: - current_time += sleep_time - time.sleep(sleep_time) - click.echo('.', nl=False) - success = func(*args) - - if not success: - raise TimeoutException() - - return decorator diff --git a/stackdio/cli/utils.py b/stackdio/cli/utils.py index 811a886..78bb9aa 100644 --- a/stackdio/cli/utils.py +++ b/stackdio/cli/utils.py @@ -14,9 +14,36 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import time +from functools import update_wrapper import click +from stackdio.client import StackdioClient + + +class TimeoutException(Exception): + pass + + +pass_client = click.make_pass_decorator(StackdioClient) + + +# def pass_client(f): +# def new_func(*args, **kwargs): +# obj = click.get_current_context().obj +# +# if not isinstance(obj, dict): +# raise click.Abort('obj is not an instance of `dict`') +# +# client = obj.get('client') +# +# if not client or not isinstance(client, StackdioClient): +# raise click.Abort('No StackdioClient available') +# +# return f(client, *args, **kwargs) +# return update_wrapper(new_func, f) + def print_summary(title, components): num_components = len(components) @@ -41,3 +68,26 @@ def print_summary(title, components): # Print a newline after each entry click.echo() + + +def poll_and_wait(func): + """ + Execute func in increments of sleep_time for no more than max_time. + Raise TimeoutException if we're not successful in max_time + """ + def decorator(args=None, sleep_time=2, max_time=120): + args = args or [] + + current_time = 0 + + success = func(*args) + while not success and current_time < max_time: + current_time += sleep_time + time.sleep(sleep_time) + click.echo('.', nl=False) + success = func(*args) + + if not success: + raise TimeoutException() + + return update_wrapper(decorator, func) diff --git a/stackdio/client/http.py b/stackdio/client/http.py index 7e03228..54a6b30 100644 --- a/stackdio/client/http.py +++ b/stackdio/client/http.py @@ -19,10 +19,11 @@ import json import logging -import requests - +from functools import update_wrapper from inspect import getcallargs +import requests + from .exceptions import MissingUrlException logger = logging.getLogger(__name__) @@ -78,6 +79,9 @@ class Request(object): def __init__(self, dfunc=None, rfunc=None, quiet=False): super(Request, self).__init__() + if dfunc: + update_wrapper(self, dfunc) + self.obj = None self.data_func = dfunc From b43563344adde0b8cf738d1a0bce1e0329d1828e Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 12 Jan 2016 14:57:17 -0600 Subject: [PATCH 38/90] Added configuration --- stackdio/cli/__init__.py | 11 +- stackdio/cli/mixins/bootstrap.py | 307 ------------------------------- stackdio/cli/utils.py | 17 +- stackdio/client/__init__.py | 28 +-- stackdio/client/config.py | 118 +++++++++--- 5 files changed, 117 insertions(+), 364 deletions(-) delete mode 100644 stackdio/cli/mixins/bootstrap.py diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index 37adb77..a1a3454 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -21,25 +21,28 @@ @click.version_option(__version__, '-v', '--version') @click.pass_context def stackdio(ctx): + # Create a client instance client = StackdioClient() + + # Throw an error if we're not configured already if ctx.invoked_subcommand not in ('configure', None) and not client.usable(): raise click.UsageError('It looks like you haven\'t used this CLI before. Please run ' '`stackdio-cli configure`') - # Put the client in the obj + # Put the client in the obj so other commands can pick it up ctx.obj = client -@stackdio.command() +@stackdio.command(name='configure') @pass_client def configure(client): """ Configure the client """ - click.echo('configuring') + client.config.prompt_for_config() -@stackdio.command('server-version') +@stackdio.command(name='server-version') @pass_client def server_version(client): """ diff --git a/stackdio/cli/mixins/bootstrap.py b/stackdio/cli/mixins/bootstrap.py deleted file mode 100644 index 7cd8d02..0000000 --- a/stackdio/cli/mixins/bootstrap.py +++ /dev/null @@ -1,307 +0,0 @@ -from __future__ import print_function - -import getpass -import json -import os -import sys - -import requests -from requests import ConnectionError -from requests.exceptions import MissingSchema -from cmd2 import Cmd -import keyring -import yaml - -from stackdio.cli.utils import TimeoutException, poll_and_wait - - -class PublicKeyNotFound(Exception): - pass - - -class BootstrapMixin(Cmd): - - def __init__(self): - # quieting down pylint - self.has_public_key = None - self.validated = False - self.stacks = None - self.config = None - self.bootstrap_data = None - - def do_initial_setup(self, args=None): - """Perform setup for your stackd.io account""" - - print("Performing initial setup") - special_config = raw_input("Do you have your own config.json file you would like to use (y/n)? ") - - config_from_file = special_config in ['Y', 'y'] - - if config_from_file: - # Prompt for the filename where the config is located - config_file = raw_input("Where is the file located? ") - - # Load the config file - if os.path.exists(config_file): - config = json.load(open(config_file, "r")) - elif os.path.exists(os.path.expanduser(config_file)): - config = json.load(open(os.path.expanduser(config_file), "r")) - else: - print("Unable to find the file.") - return - - # Put the config file contents into the config object - for k, v in config.iteritems(): - self.config[k] = v - - # Validate the url, prompt for a new one if invalid - if 'url' not in self.config or not self._test_url(self.config['url']): - print("There seems to be an issue with the url you provided.") - self.config['url'] = None - self._get_url() - - else: - # No config file, just prompt individually - self._get_url() - - self._get_user_creds() - self._init_stacks() - self._validate_auth() - if not self.validated: - return - - if not config_from_file and 'profile' in self.config: - keep_profile = raw_input("Would you like to keep your current default profile (y/n)? ") - - if keep_profile in ['N', 'n']: - self._choose_profile() - - # Only prompt for default profile if it's not already there - if 'profile' not in self.config \ - or 'provider' not in self.config \ - or 'provider_type' not in self.config: - if 'profile' in self.config: - print("Profile misconfiguration detected.") - self._choose_profile() - - get_dir = not config_from_file - - if not config_from_file and 'blueprint_dir' in self.config: - keep_dir = raw_input("Would you like to keep your current blueprint directory (y/n)? ") - - if keep_dir not in ['N', 'n']: - get_dir = False - - if get_dir: - new_dir = raw_input("Enter the path of your blueprint templates: ") - self.config['blueprint_dir'] = new_dir - - self._save_config() - self._setprompt() - - def do_bootstrap(self, args=None): - """Bootstrap an account with predefined formulas and blueprints""" - - args = args or [] - - if not self.validated: - print(self.colorize( - "You must run 'initial_setup' before you can bootstrap", - "red")) - return - - if 'profile' not in self.config: - print(self.colorize("You must have a default profile in order to run bootstrap. " - "Run 'initial_setup'", "red")) - return - - print("Bootstrapping your account") - - custom_bootstrap = raw_input("Do you have a custom bootstrap yaml file (y/n)? ") - - if custom_bootstrap in ['Y', 'y']: - custom_bootstrap_file = raw_input("Enter the name of the file: ") - - # Load the bootstrap file - if os.path.exists(custom_bootstrap_file): - self.BOOTSTRAP_FILE = custom_bootstrap_file - elif os.path.exists(os.path.expanduser(custom_bootstrap_file)): - self.BOOTSTRAP_FILE = os.path.expanduser(custom_bootstrap_file) - else: - print("Unable to find the file.") - use_default = raw_input("Would you like to use the default bootstrap file instead (y/n)? ") - if use_default not in ['Y', 'y']: - print("Aborting bootstrap") - return - - # Load the bootstrap data. If the BOOTSTRAP_DATA property was not set just now, it will use the default - self.bootstrap_data = yaml.safe_load(open(self.BOOTSTRAP_FILE).read()) - - self._bootstrap_account() - self._bootstrap_formulas() - self._bootstrap_blueprints() - - def _test_url(self, url): - try: - r = requests.get(url, verify=self.config.get('verify', True)) - return (200 <= r.status_code < 300) or r.status_code == 403 - except ConnectionError: - return False - except MissingSchema: - print("You might have forgotten http:// or https://") - return False - - def _get_url(self): - """Prompt user for url""" - - if self.config['url'] is not None: - keep_url = raw_input("Keep existing url (y/n)? ") - if keep_url not in ["n", "N"]: - return - - verify = raw_input("Does your stackd.io server have a self-signed SSL certificate (y/n)? ") - if verify in ('Y', 'y'): - self.config['verify'] = False - else: - self.config['verify'] = True - - self.config['url'] = None - - while self.config['url'] is None: - url = raw_input("What is the URL of your stackd.io server? ") - if url.endswith('api'): - url += '/' - elif url.endswith('api/'): - pass - elif url.endswith('/'): - url += 'api/' - else: - url += '/api/' - if self._test_url(url): - self.config['url'] = url - else: - print("There was an error while attempting to contact that server. Try again.") - - def _get_user_creds(self): - """Prompt user for credentials""" - - self.config["username"] = raw_input("What is your username? ") - - if keyring.get_password(self.KEYRING_SERVICE, self.config["username"]): - print("Password already stored for {0}".format(self.config["username"])) - keep_password = raw_input("Keep existing password (y/n)? ") - else: - keep_password = "n" - - if keep_password in ["n", "N"]: - password = getpass.getpass("What is your password? ") - keyring.set_password(self.KEYRING_SERVICE, - self.config["username"], - password) - - def _choose_profile(self): - """Prompt user for a default provider/profile""" - auth = (self.config['username'], - keyring.get_password(self.KEYRING_SERVICE, self.config['username'])) - profiles = requests.get(self.config['url'] + "profiles/", - auth=auth, verify=False).json()['results'] - - print("Choose a default profile:") - - idx = 0 - for profile in profiles: - print(str(idx) + ':') - print(' ' + profile['title']) - print(' ' + profile['description']) - idx += 1 - - print('') - choice = int(raw_input("Enter the number of the profile you would like to choose: ")) - - provider = requests.get( - self.config['url'] + "providers/{0}/".format(profiles[choice]['cloud_provider']), - auth=auth, - verify=False - ).json() - - self.config['profile'] = profiles[choice]['title'] - self.config['provider'] = provider['title'] - self.config['provider_type'] = provider['provider_type_name'] - - def _save_config(self): - with open(self.CFG_FILE, "w") as f: - f.write(json.dumps(self.config)) - - def _bootstrap_account(self): - """Bootstrap the users account with public key""" - - if self.has_public_key: - keep_public_key = raw_input( - "Keep existing public key? (y,n)? ") - else: - keep_public_key = "n" - - if keep_public_key in ["y", "Y"]: - return - - raw_public_key = raw_input( - "What is your public key (either path to or contents of)? ") - - if os.path.exists(raw_public_key): - public_key = open(raw_public_key, "r").read() - elif os.path.exists(os.path.expanduser(raw_public_key)): - public_key = open(os.path.expanduser(raw_public_key), "r").read() - else: - public_key = raw_public_key - - if not public_key or not public_key.startswith("ssh-rsa"): - print(self.colorize("Unable to find valid public key", "red")) - else: - print("Setting public key") - self.stacks.set_public_key(public_key) - self.has_public_key = True - - def _bootstrap_formulas(self): - """Import and wait for formulas to become ready""" - - @poll_and_wait - def _check_formulas(): - formulas = self.stacks.list_formulas() - for formula in formulas: - if formula.get("status") != "complete": - return False - return True - - formulas = self.bootstrap_data.get("formulas", []) - print("Importing {0} formula{1}".format( - len(formulas), - "s" if len(formulas) == 0 or len(formulas) > 1 else "")) - for name, url in formulas.iteritems(): - print(" - {0} // {1}".format(name, url)) - self.stacks.import_formula(url, public=False) - - sys.stdout.write("Waiting for formulas .") - sys.stdout.flush() - try: - _check_formulas() - sys.stdout.write(" done!\n") - except TimeoutException: - print(self.colorize( - "\nTIMEOUT - formulas failed to finish importing, monitor with `formulas list`", - "red")) - - def _bootstrap_blueprints(self): - """Create blueprints""" - - blueprints = self.bootstrap_data.get("blueprints", []) - - print("Creating {0} blueprint{1}".format(len(blueprints), - "s" if len(blueprints) == 0 or len(blueprints) > 1 else "")) - for name, blueprint in blueprints.iteritems(): - print(" - {0} // {1}".format(name, blueprint)) - - # Get the blueprints relative to the bootstrap config file - self._create_blueprint( - [os.path.join(os.path.dirname(self.BOOTSTRAP_FILE), "blueprints", blueprint)], - bootstrap=True, - ) diff --git a/stackdio/cli/utils.py b/stackdio/cli/utils.py index 78bb9aa..64df326 100644 --- a/stackdio/cli/utils.py +++ b/stackdio/cli/utils.py @@ -26,25 +26,10 @@ class TimeoutException(Exception): pass +# Create our decorator pass_client = click.make_pass_decorator(StackdioClient) -# def pass_client(f): -# def new_func(*args, **kwargs): -# obj = click.get_current_context().obj -# -# if not isinstance(obj, dict): -# raise click.Abort('obj is not an instance of `dict`') -# -# client = obj.get('client') -# -# if not client or not isinstance(client, StackdioClient): -# raise click.Abort('No StackdioClient available') -# -# return f(client, *args, **kwargs) -# return update_wrapper(new_func, f) - - def print_summary(title, components): num_components = len(components) diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index 1ec1f9f..19400d8 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -51,10 +51,10 @@ def __init__(self, url=None, username=None, password=None, verify=True): if self.config.usable_config: # Grab stuff from the config - self.url = self.config['url'] - self.username = self.config['username'] - self.password = self.config['password'] - self.verify = self.config['verify'] + self.url = self.config.get('url') + self.username = self.config.get('username') + self.password = self.config.get_password() + self.verify = self.config.get('verify', True) if url is not None: self.url = url @@ -66,20 +66,22 @@ def __init__(self, url=None, username=None, password=None, verify=True): if verify is not None: self.verify = verify - super(StackdioClient, self).__init__(url=self.url, auth=(self.username, self.password), + super(StackdioClient, self).__init__(url=self.url, + auth=(self.username, self.password), verify=self.verify) - try: - _, self.version = _parse_version_string(self.get_version()) - except MissingUrlException: - self.version = None + if self.usable(): + try: + _, self.version = _parse_version_string(self.get_version(raise_for_status=False)) + except MissingUrlException: + self.version = None - if self.version and (self.version[0] != 0 or self.version[1] != 7): - raise IncompatibleVersionException('Server version {0}.{1}.{2} not ' - 'supported.'.format(**self.version)) + if self.version and (self.version[0] != 0 or self.version[1] != 7): + raise IncompatibleVersionException('Server version {0}.{1}.{2} not ' + 'supported.'.format(**self.version)) def usable(self): - return self.config.usable_config or self.url + return self.url and self.username and self.password @get('') def get_root(self): diff --git a/stackdio/client/config.py b/stackdio/client/config.py index 5ba4fb8..b15fa82 100644 --- a/stackdio/client/config.py +++ b/stackdio/client/config.py @@ -29,6 +29,12 @@ CFG_FILE = os.path.join(CFG_DIR, 'client.cfg') +class UserPath(click.Path): + + def convert(self, value, param, ctx): + return super(UserPath, self).convert(os.path.expanduser(value), param, ctx) + + class StackdioConfig(object): """ A wrapper around python's ConfigParser class @@ -57,21 +63,26 @@ def __init__(self, config_file=CFG_FILE, section='main'): else: self._config.read(config_file) - username = self.get('username') - - if username is not None: - self['password'] = keyring.get_password(self.KEYRING_SERVICE, username) - - # Make the blueprint dir usable - blueprint_dir = self.get('blueprint_dir') - if blueprint_dir: - new_blueprint_dir = os.path.expanduser(blueprint_dir) - self._config.set(section, 'blueprint_dir', new_blueprint_dir) - def save(self): with open(self._cfg_file, 'w') as f: self._config.write(f) + def get_password(self, username=None): + username = username or self.get('username') + + if username is not None: + return keyring.get_password(self.KEYRING_SERVICE, username) + else: + return None + + def set_password(self, new_password): + username = self.get('username') + + if username is None: + raise KeyError('Not username provided') + + keyring.set_password(self.KEYRING_SERVICE, username, new_password) + def __contains__(self, item): try: self._config.get(self.section, item) @@ -105,6 +116,11 @@ def items(self): def prompt_for_config(self): self.get_url() + self.get_username() + self.get_blueprint_dir() + + # Save when we're done + self.save() def _test_url(self, url): try: @@ -116,24 +132,27 @@ def _test_url(self, url): click.echo('You might have forgotten http:// or https://') return False - def get_url(self): + def _test_credentials(self, username, password): + try: + r = requests.get(self['url'], + verify=self.get('verify', True), + auth=(username, password)) + return 200 <= r.status_code < 300 + except (ConnectionError, MissingSchema): + click.echo('There is something wrong with your URL.') + return False + def get_url(self): if self.get('url') is not None: - val = click.prompt('Keep existing url', default='y', prompt_suffix=' (y|n)? ') - if val not in ('N', 'n'): + if click.confirm('Keep existing url ({0})?'.format(self['url']), default=True): return - val = click.prompt('Does your stackd.io server have a self-signed SSL certificate', - default='n', prompt_suffix=' (y|n)? ') + self['verify'] = not click.confirm('Does your stackd.io server have a self-signed ' + 'SSL certificate?') - if val in ('Y', 'y'): - self['verify'] = False - else: - self['verify'] = True + new_url = None - self['url'] = None - - while self['url'] is None: + while new_url is None: url = click.prompt('What is the URL of your stackd.io server', prompt_suffix='? ') if url.endswith('api'): url += '/' @@ -144,7 +163,58 @@ def get_url(self): else: url += '/api/' if self._test_url(url): - self['url'] = url + new_url = url else: click.echo('There was an error while attempting to contact that server. ' 'Try again.') + + self['url'] = new_url + + def get_username(self): + valid_creds = False + + while not valid_creds: + keep_username = False + + username = self.get('username') + + if username is not None: + if click.confirm('Keep existing username ({0})?'.format(username), default=True): + keep_username = True + + if not keep_username: + username = click.prompt('What is your stackd.io username', prompt_suffix='? ') + + password = self.get_password(username) + + keep_password = False + + if password is not None: + if click.confirm('Keep existing password for user {0}?'.format(username), + default=True): + keep_password = True + + if not keep_password: + password = click.prompt('What is the password for {0}'.format(username), + prompt_suffix='? ', hide_input=True) + + if self._test_credentials(username, password): + self['username'] = username + self.set_password(password) + valid_creds = True + else: + click.echo('Invalid credentials. Please try again.') + valid_creds = False + + def get_blueprint_dir(self): + blueprints = self.get('blueprint_dir') + + if blueprints is not None: + if click.confirm('Keep existing blueprints directory ({0})?'.format(blueprints), + default=True): + return + + self['blueprint_dir'] = click.prompt('Where are your blueprints stored', + prompt_suffix='? ', + type=UserPath(exists=True, file_okay=False, + resolve_path=True)) From 716ae0648674114a27c8f1b3ceaab42ab4931ac3 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 12 Jan 2016 17:50:47 -0600 Subject: [PATCH 39/90] Added ability to specify a different config file --- stackdio/cli/__init__.py | 10 +++++++--- stackdio/client/__init__.py | 4 ++-- stackdio/client/config.py | 3 +-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index a1a3454..2c9673a 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -6,9 +6,10 @@ import click -from stackdio.cli.mixins import blueprints, bootstrap, formulas, stacks +from stackdio.cli.mixins import blueprints, formulas, stacks from stackdio.cli.utils import pass_client from stackdio.client import StackdioClient +from stackdio.client.config import CFG_FILE from stackdio.client.version import __version__ @@ -19,10 +20,13 @@ @click.group(context_settings=CONTEXT_SETTINGS) @click.version_option(__version__, '-v', '--version') +@click.option('-c', '--config-file', help='The config file to use.', + type=click.Path(dir_okay=False, file_okay=True), default=CFG_FILE, + envvar='STACKDIO_CLI_CONFIG_FILE') @click.pass_context -def stackdio(ctx): +def stackdio(ctx, config_file): # Create a client instance - client = StackdioClient() + client = StackdioClient(cfg_file=config_file) # Throw an error if we're not configured already if ctx.invoked_subcommand not in ('configure', None) and not client.usable(): diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index 19400d8..b00524d 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -41,8 +41,8 @@ class StackdioClient(BlueprintMixin, FormulaMixin, AccountMixin, ImageMixin, RegionMixin, StackMixin, SettingsMixin, HttpMixin): - def __init__(self, url=None, username=None, password=None, verify=True): - self.config = StackdioConfig() + def __init__(self, url=None, username=None, password=None, verify=True, cfg_file=None): + self.config = StackdioConfig(cfg_file) self.url = None self.username = None diff --git a/stackdio/client/config.py b/stackdio/client/config.py index b15fa82..0657d23 100644 --- a/stackdio/client/config.py +++ b/stackdio/client/config.py @@ -25,8 +25,7 @@ from stackdio.client.compat import ConfigParser, NoOptionError -CFG_DIR = os.path.join(os.path.expanduser('~'), '.stackdio') -CFG_FILE = os.path.join(CFG_DIR, 'client.cfg') +CFG_FILE = os.path.join(os.path.expanduser('~'), '.stackdio', 'client.cfg') class UserPath(click.Path): From 3c957b5005207132d75a42dc3ad058f2cf40bc41 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 12 Jan 2016 18:07:06 -0600 Subject: [PATCH 40/90] Fixed logic error in config object --- stackdio/cli/__init__.py | 2 -- stackdio/client/config.py | 16 ++++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index 2c9673a..e63723d 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -1,7 +1,5 @@ #!/usr/bin/env python -from __future__ import print_function - import os import click diff --git a/stackdio/client/config.py b/stackdio/client/config.py index 0657d23..97cc0a8 100644 --- a/stackdio/client/config.py +++ b/stackdio/client/config.py @@ -46,22 +46,26 @@ class StackdioConfig(object): str(False): False, } - def __init__(self, config_file=CFG_FILE, section='main'): + def __init__(self, config_file=CFG_FILE, section='stackdio'): super(StackdioConfig, self).__init__() self.section = section self._cfg_file = config_file - self.usable_config = os.path.isfile(config_file) - self._config = ConfigParser() - if not self.usable_config: - self._config.add_section(section) - else: + self.usable_file = os.path.isfile(self._cfg_file) + + if self.usable_file: self._config.read(config_file) + self.usable_section = self._config.has_section(self.section) + self.usable_config = self.usable_file and self.usable_section + + if not self.usable_section: + self._config.add_section(section) + def save(self): with open(self._cfg_file, 'w') as f: self._config.write(f) From 53fd42e3e8f7f132a44c1d3811add2b5dc739238 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 13 Jan 2016 16:28:48 -0600 Subject: [PATCH 41/90] Removed the rest of the references to obj, added access rules --- blueprints/.keep | 0 stackdio/cli/mixins/formulas.py | 6 ---- stackdio/cli/mixins/stacks.py | 61 +++++++++++++++++++++------------ stackdio/cli/utils.py | 5 ++- 4 files changed, 44 insertions(+), 28 deletions(-) delete mode 100644 blueprints/.keep diff --git a/blueprints/.keep b/blueprints/.keep deleted file mode 100644 index e69de29..0000000 diff --git a/stackdio/cli/mixins/formulas.py b/stackdio/cli/mixins/formulas.py index fd71643..2fcb748 100644 --- a/stackdio/cli/mixins/formulas.py +++ b/stackdio/cli/mixins/formulas.py @@ -19,8 +19,6 @@ def list_formulas(client): """ List all formulas """ - client = obj['client'] - click.echo('Getting formulas ... ') print_summary('Formula', client.list_formulas()) @@ -35,8 +33,6 @@ def import_formula(client, uri, username, password): """ Import a formula """ - client = obj['client'] - if username and not password: raise click.UsageError('You must provide a password when providing a username') @@ -62,8 +58,6 @@ def delete_formula(client, uri): """ Delete a formula """ - client = obj['client'] - formula_id = get_formula_id(client, uri) click.confirm('Really delete formula {0}?'.format(uri), abort=True) diff --git a/stackdio/cli/mixins/stacks.py b/stackdio/cli/mixins/stacks.py index fc0dece..a1fa3f1 100644 --- a/stackdio/cli/mixins/stacks.py +++ b/stackdio/cli/mixins/stacks.py @@ -24,8 +24,6 @@ def list_stacks(client): """ List all stacks """ - client = obj['client'] - click.echo('Getting stacks ... ') print_summary('Stack', client.list_stacks()) @@ -38,8 +36,6 @@ def launch_stack(client, blueprint_title, stack_title): """ Launch a stack from a blueprint """ - client = obj['client'] - blueprint_id = get_blueprint_id(client, blueprint_title) click.echo('Launching stack "{0}" from blueprint "{1}"'.format(stack_title, @@ -74,8 +70,6 @@ def stack_history(client, stack_title, length): """ Print recent history for a stack """ - client = obj['client'] - stack_id = get_stack_id(client, stack_title) history = client.get_stack_history(stack_id) for event in history[0:min(length, len(history))]: @@ -89,8 +83,6 @@ def stack_hostnames(client, stack_title): """ Print hostnames for a stack """ - client = obj['client'] - stack_id = get_stack_id(client, stack_title) hosts = client.get_stack_hosts(stack_id) @@ -106,8 +98,6 @@ def delete_stack(client, stack_title): """ Delete a stack. PERMANENT AND DESTRUCTIVE!!! """ - client = obj['client'] - stack_id = get_stack_id(client, stack_title) click.confirm('Really delete stack {0}?'.format(stack_title), abort=True) @@ -126,8 +116,6 @@ def perform_action(client, stack_title, action): """ Perform an action on a stack """ - client = obj['client'] - stack_id = get_stack_id(client, stack_title) # Prompt for confirmation if need be @@ -148,6 +136,7 @@ def print_command_output(json_blob): @stacks.command(name='run') +@pass_client @click.pass_context @click.argument('stack_title') @click.argument('host_target') @@ -157,12 +146,10 @@ def print_command_output(json_blob): @click.option('-t', '--timeout', type=click.INT, default=120, help='The amount of time to wait for the command in seconds. ' 'Ignored if used without the -w option.') -def run_command(ctx, stack_title, host_target, command, wait, timeout): +def run_command(ctx, client, stack_title, host_target, command, wait, timeout): """ Run a command on all hosts in the stack """ - client = ctx.obj['client'] - stack_id = get_stack_id(client, stack_title) resp = client.run_command(stack_id, host_target, command) @@ -200,8 +187,6 @@ def get_command_output(client, command_id): """ Get the status and output of a command """ - client = obj['client'] - resp = client.get_command(command_id) if resp['status'] != 'finished': @@ -232,8 +217,6 @@ def list_stack_logs(client, stack_title): """ Get a list of stack logs """ - client = obj['client'] - stack_id = get_stack_id(client, stack_title) print_logs(client, stack_id) @@ -248,8 +231,6 @@ def stack_logs(client, stack_title, log_type, lines): """ Get logs for a stack """ - client = obj['client'] - stack_id = get_stack_id(client, stack_title) split_arg = log_type.split('.') @@ -273,3 +254,41 @@ def stack_logs(client, stack_title, log_type, lines): print_logs(client, stack_id) raise click.UsageError('Invalid log') + + +@stacks.group(name='access-rules') +def stack_access_rules(): + """ + Perform actions on stack access rules + """ + pass + + +def print_access_rules(components): + title = 'Access Rule' + num_components = len(components) + + if num_components != 1: + title += 's' + + click.echo('## {0} {1}'.format(num_components, title)) + + for item in components: + click.echo('- Name: {0}'.format(item.get('name'))) + click.echo(' Description: {0}'.format(item['description'])) + click.echo(' Group ID: {0}'.format(item['group_id'])) + click.echo(' Host Definition: {0}'.format(item['blueprint_host_definition']['title'])) + + # Print a newline after each entry + click.echo() + + +@stack_access_rules.command(name='list') +@pass_client +@click.argument('stack_title') +def list_access_rules(client, stack_title): + stack_id = get_stack_id(client, stack_title) + + rules = client.list_access_rules(stack_id) + + print_access_rules(rules) diff --git a/stackdio/cli/utils.py b/stackdio/cli/utils.py index 64df326..6ace295 100644 --- a/stackdio/cli/utils.py +++ b/stackdio/cli/utils.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +import sys import time from functools import update_wrapper @@ -65,11 +67,12 @@ def decorator(args=None, sleep_time=2, max_time=120): current_time = 0 + click.echo('.', nl=False, file=sys.stderr) success = func(*args) while not success and current_time < max_time: current_time += sleep_time time.sleep(sleep_time) - click.echo('.', nl=False) + click.echo('.', nl=False, file=sys.stderr) success = func(*args) if not success: From 3619890ee3abe7cc91f55f68d34a0602094b5a48 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 13 Jan 2016 16:48:57 -0600 Subject: [PATCH 42/90] Updated setup.py to remove bootstrap things --- bootstrap.yaml | 6 ------ setup.py | 11 +---------- stackdio/cli/__init__.py | 4 +++- 3 files changed, 4 insertions(+), 17 deletions(-) delete mode 100644 bootstrap.yaml diff --git a/bootstrap.yaml b/bootstrap.yaml deleted file mode 100644 index 480cf07..0000000 --- a/bootstrap.yaml +++ /dev/null @@ -1,6 +0,0 @@ -blueprints: {} - -formulas: - java: https://github.com/stackdio-formulas/java-formula.git - cdh5: https://github.com/stackdio-formulas/cdh5-formula.git - elasticsearch: https://github.com/stackdio-formulas/elasticsearch-formula.git diff --git a/setup.py b/setup.py index 2bdea4d..9d6fe7f 100644 --- a/setup.py +++ b/setup.py @@ -44,12 +44,11 @@ def test_python_version(): with open('README.md') as f: LONG_DESCRIPTION = f.read() -CFG_DIR = os.path.join(os.path.expanduser('~'), '.stackdio-cli') - requirements = [ 'Jinja2==2.7.3', 'PyYAML>=3.10', 'click>=6.0,<7.0', + 'click-shell==0.3', 'colorama>=0.3,<0.4', 'keyring==3.7', 'requests>=2.4.0', @@ -76,14 +75,6 @@ def test_python_version(): license='Apache 2.0', include_package_data=True, packages=find_packages(), - data_files=[ - (CFG_DIR, - [ - 'bootstrap.yaml', - ]), - (os.path.join(CFG_DIR, 'blueprints'), - ['blueprints/%s' % f for f in os.listdir('blueprints')]), - ], zip_safe=False, install_requires=requirements, dependency_links=[], diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index e63723d..b3d5e64 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -3,6 +3,7 @@ import os import click +import click_shell from stackdio.cli.mixins import blueprints, formulas, stacks from stackdio.cli.utils import pass_client @@ -16,7 +17,8 @@ HIST_FILE = os.path.join(os.path.expanduser('~'), '.stackdio-cli', 'history') -@click.group(context_settings=CONTEXT_SETTINGS) +@click_shell.shell(context_settings=CONTEXT_SETTINGS, prompt='stackdio > ', + intro='stackdio-cli, v{0}'.format(__version__), hist_file=HIST_FILE) @click.version_option(__version__, '-v', '--version') @click.option('-c', '--config-file', help='The config file to use.', type=click.Path(dir_okay=False, file_okay=True), default=CFG_FILE, From 6bcb57aa1871a8d45b9efd2c719e5242165c5523 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 13 Jan 2016 16:57:29 -0600 Subject: [PATCH 43/90] Fixed a couple issues with instantiating a config object --- stackdio/client/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stackdio/client/config.py b/stackdio/client/config.py index 97cc0a8..42deb09 100644 --- a/stackdio/client/config.py +++ b/stackdio/client/config.py @@ -46,19 +46,19 @@ class StackdioConfig(object): str(False): False, } - def __init__(self, config_file=CFG_FILE, section='stackdio'): + def __init__(self, config_file=None, section='stackdio'): super(StackdioConfig, self).__init__() self.section = section - self._cfg_file = config_file + self._cfg_file = config_file or CFG_FILE self._config = ConfigParser() self.usable_file = os.path.isfile(self._cfg_file) if self.usable_file: - self._config.read(config_file) + self._config.read(self._cfg_file) self.usable_section = self._config.has_section(self.section) self.usable_config = self.usable_file and self.usable_section From fb6269bbcc3d22c47581ce37ca5323bba9447faa Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 13 Jan 2016 17:24:44 -0600 Subject: [PATCH 44/90] Added script to convert config file --- setup.py | 1 + stackdio/client/config.py | 45 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/setup.py b/setup.py index 9d6fe7f..5dad96f 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,7 @@ def test_python_version(): 'console_scripts': [ 'stackdio-cli=stackdio.cli:main', 'blueprint-generator=stackdio.cli.blueprints:main', + 'stackdio-config-convert=stackdio.client.config:main', ], }, classifiers=[ diff --git a/stackdio/client/config.py b/stackdio/client/config.py index 42deb09..f689e67 100644 --- a/stackdio/client/config.py +++ b/stackdio/client/config.py @@ -221,3 +221,48 @@ def get_blueprint_dir(self): prompt_suffix='? ', type=UserPath(exists=True, file_okay=False, resolve_path=True)) + + +# FOR BACKWARDS COMPATIBILITY!!! To be removed at some point. +@click.command(context_settings=dict(help_option_names=['-h', '--help'])) +@click.option('-o', '--old-file', default=os.path.expanduser('~/.stackdio-cli/config.json'), + type=click.Path(exists=True), help='Path to the old JSON config file') +@click.option('-n', '--new-file', default=CFG_FILE, type=click.Path(), + help='Path to the new cfg format file. Must not already exist.') +def migrate_old_config(old_file, new_file): + """ + Used to migrate an old JSON config file to a new one + """ + if os.path.exists(new_file): + raise click.UsageError('The new config file must not already exist') + + old_keys = ['username', 'url', 'verify', 'blueprint_dir'] + + config = StackdioConfig(new_file) + + import json + with open(old_file, 'r') as f: + old_config = json.load(f) + + for key in old_keys: + if key in old_config: + config[key] = old_config[key] + + if 'url' not in config: + config.get_url() + + if 'username' not in config or config.get_password() is None: + config.get_username() + + if 'blueprint_dir' not in config: + config.get_blueprint_dir() + + config.save() + + +def main(): + migrate_old_config() + + +if __name__ == '__main__': + main() From e6f86e2df8624761a5643e6268ee845a7d9d4d0b Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 13 Jan 2016 17:28:49 -0600 Subject: [PATCH 45/90] removed deprecated pieces from client --- stackdio/client/stack.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/stackdio/client/stack.py b/stackdio/client/stack.py index 927fe97..1d6f7a5 100644 --- a/stackdio/client/stack.py +++ b/stackdio/client/stack.py @@ -131,21 +131,6 @@ def list_access_rules(self, stack_id): """ pass - @deprecated - def get_access_rule_id(self, stack_id, title): - """Find an access rule id""" - - rules = self.list_access_rules(stack_id) - - try: - for group in rules: - if group.get('blueprint_host_definition').get('title') == title: - return group.get('id') - except TypeError: - pass - - raise StackException('Access Rule %s not found' % title) - @get('security_groups/{group_id}/rules/', paginate=True) def list_rules_for_group(self, group_id): pass From c89b2acb214555ffda8112b402ad9c90a6980773 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 22 Jan 2016 17:06:10 -0600 Subject: [PATCH 46/90] Fixed a couple issues with a config file not existing --- setup.py | 2 +- stackdio/client/__init__.py | 40 ++++++++++++++++++++----------------- stackdio/client/config.py | 8 +++++++- stackdio/client/http.py | 36 +++++++++++++++++++++------------ 4 files changed, 53 insertions(+), 33 deletions(-) diff --git a/setup.py b/setup.py index 5dad96f..2aad5ee 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def test_python_version(): 'Jinja2==2.7.3', 'PyYAML>=3.10', 'click>=6.0,<7.0', - 'click-shell==0.3', + 'click-shell>=0.4', 'colorama>=0.3,<0.4', 'keyring==3.7', 'requests>=2.4.0', diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index b00524d..1c4bc42 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -44,31 +44,19 @@ class StackdioClient(BlueprintMixin, FormulaMixin, AccountMixin, ImageMixin, def __init__(self, url=None, username=None, password=None, verify=True, cfg_file=None): self.config = StackdioConfig(cfg_file) - self.url = None - self.username = None - self.password = None - self.verify = None - - if self.config.usable_config: - # Grab stuff from the config - self.url = self.config.get('url') - self.username = self.config.get('username') - self.password = self.config.get_password() - self.verify = self.config.get('verify', True) + self._password = self.config.get_password() if url is not None: - self.url = url + self.config['url'] = url if username is not None and password is not None: - self.username = username - self.password = password + self.config['username'] = username + self._password = password if verify is not None: - self.verify = verify + self.config['verify'] = verify - super(StackdioClient, self).__init__(url=self.url, - auth=(self.username, self.password), - verify=self.verify) + super(StackdioClient, self).__init__() if self.usable(): try: @@ -80,6 +68,22 @@ def __init__(self, url=None, username=None, password=None, verify=True, cfg_file raise IncompatibleVersionException('Server version {0}.{1}.{2} not ' 'supported.'.format(**self.version)) + @property + def url(self): + return self.config.get('url') + + @property + def username(self): + return self.config.get('username') + + @property + def password(self): + return self._password or self.config.get_password() + + @property + def verify(self): + return self.config.get('verify', True) + def usable(self): return self.url and self.username and self.password diff --git a/stackdio/client/config.py b/stackdio/client/config.py index f689e67..e5e8a9b 100644 --- a/stackdio/client/config.py +++ b/stackdio/client/config.py @@ -16,6 +16,7 @@ # import os +import shutil import click import keyring @@ -51,7 +52,7 @@ def __init__(self, config_file=None, section='stackdio'): self.section = section - self._cfg_file = config_file or CFG_FILE + self._cfg_file = os.path.abspath(config_file or CFG_FILE) self._config = ConfigParser() @@ -67,6 +68,11 @@ def __init__(self, config_file=None, section='stackdio'): self._config.add_section(section) def save(self): + full_path = os.path.dirname(self._cfg_file) + + if not os.path.isdir(full_path): + os.makedirs(full_path) + with open(self._cfg_file, 'w') as f: self._config.write(f) diff --git a/stackdio/client/http.py b/stackdio/client/http.py index 54a6b30..4eb3d4d 100644 --- a/stackdio/client/http.py +++ b/stackdio/client/http.py @@ -44,17 +44,11 @@ class HttpMixin(object): 'xml': {'content-type': 'application/xml'}, } - def __init__(self, url, auth=None, verify=True): + def __init__(self): super(HttpMixin, self).__init__() - - self.url = url - self.http_options = { - 'auth': auth, - 'verify': verify, - } self._http_log = logger - if not verify: + if not self.verify: if self._http_log.handlers: self._http_log.warn(HTTP_INSECURE_MESSAGE) else: @@ -63,6 +57,22 @@ def __init__(self, url, auth=None, verify=True): from requests.packages.urllib3 import disable_warnings disable_warnings() + @property + def url(self): + raise NotImplementedError() + + @property + def username(self): + raise NotImplementedError() + + @property + def password(self): + raise NotImplementedError() + + @property + def verify(self): + raise NotImplementedError() + def usable(self): raise NotImplementedError() @@ -124,7 +134,7 @@ def __call__(self, *args, **kwargs): assert isinstance(self.obj, HttpMixin) if not self.obj.usable(): - raise MissingUrlException('No url is set') + raise MissingUrlException('No url is set. Please run `configure`.') none_on_404 = kwargs.pop('none_on_404', False) raise_for_status = kwargs.pop('raise_for_status', True) @@ -149,10 +159,10 @@ def __call__(self, *args, **kwargs): result = requests.request(method, url, data=data, - auth=self.obj.http_options['auth'], + auth=(self.obj.username, self.obj.password), headers=self.headers, params=kwargs, - verify=self.obj.http_options['verify']) + verify=self.obj.verify) # Handle special conditions if none_on_404 and result.status_code == 404: @@ -182,10 +192,10 @@ def __call__(self, *args, **kwargs): next_page = requests.request(method, next_url, data=data, - auth=self.obj.http_options['auth'], + auth=(self.obj.username, self.obj.password), headers=self.headers, params=kwargs, - verify=self.obj.http_options['verify']).json() + verify=self.obj.verify).json() res.extend(next_page['results']) next_url = next_page.get('next') From e445f561e8c0089aa347930584086e1d03917158 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 22 Jan 2016 17:09:24 -0600 Subject: [PATCH 47/90] Updated versioning scheme --- stackdio/client/version.py | 71 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index bab03a8..f36a786 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -15,10 +15,13 @@ # limitations under the License. # -from functools import wraps +import datetime import operator +import os import re +import subprocess import warnings +from functools import wraps # for setup.py try: @@ -26,8 +29,72 @@ except Exception: pass +VERSION = (0, 7, 0, 'dev', 0) + + +def get_version(version): + """ + Returns a PEP 440-compliant version number from VERSION. + + Created by modifying django.utils.version.get_version + """ + + # Now build the two parts of the version number: + # major = X.Y[.Z] + # sub = .devN - for development releases + # | {a|b|rc}N - for alpha, beta and rc releases + # | .postN - for post-release releases + + assert len(version) == 5 + + version_parts = version[:3] + + # Build the first part of the version + major = '.'.join(str(x) for x in version_parts) + + # Just return it if this is a final release version + if version[3] == 'final': + return major + + # Add the rest + sub = ''.join(str(x) for x in version[3:5]) + + if version[3] == 'dev': + # Override the sub part. Add in a timestamp + timestamp = get_git_changeset() + sub = 'dev%s' % (timestamp if timestamp else '') + return '%s.%s' % (major, sub) + if version[3] == 'post': + # We need a dot for post + return '%s.%s' % (major, sub) + elif version[3] in ('a', 'b', 'rc'): + # No dot for these + return '%s%s' % (major, sub) + else: + raise ValueError('Invalid version: %s' % str(version)) + + +# Borrowed directly from django +def get_git_changeset(): + """Returns a numeric identifier of the latest git changeset. + + The result is the UTC timestamp of the changeset in YYYYMMDDHHMMSS format. + This value isn't guaranteed to be unique, but collisions are very unlikely, + so it's sufficient for generating the development version numbers. + """ + repo_dir = os.path.dirname(os.path.abspath(__file__)) + git_log = subprocess.Popen('git log --pretty=format:%ct --quiet -1 HEAD', + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True, cwd=repo_dir, universal_newlines=True) + timestamp = git_log.communicate()[0] + try: + timestamp = datetime.datetime.utcfromtimestamp(int(timestamp)) + return timestamp.strftime('%Y%m%d%H%M%S') + except ValueError: + return None + -__version__ = '0.7.0.dev' +__version__ = get_version(VERSION) def _unsupported_function(func, current_version, accepted_versions): From 23aa8fba86d22ad90f70d779add94698558d4313 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 22 Jan 2016 17:13:21 -0600 Subject: [PATCH 48/90] pip doesn't support python 3.2 anymore --- .travis.yml | 1 - setup.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 201e105..6b4d5c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: python python: - "2.7" - - "3.2" - "3.3" - "3.4" diff --git a/setup.py b/setup.py index 2aad5ee..32376ed 100644 --- a/setup.py +++ b/setup.py @@ -97,10 +97,8 @@ def test_python_version(): 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Topic :: System :: Clustering', From 34a7ac07fff6009f6cb0d6dfda9a6fea8ba70be3 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Thu, 28 Jan 2016 12:06:58 -0600 Subject: [PATCH 49/90] Use a config-dir option instead of config-file --- stackdio/cli/__init__.py | 19 ++++++++++--------- stackdio/client/config.py | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/stackdio/cli/__init__.py b/stackdio/cli/__init__.py index b3d5e64..84744b1 100644 --- a/stackdio/cli/__init__.py +++ b/stackdio/cli/__init__.py @@ -8,25 +8,26 @@ from stackdio.cli.mixins import blueprints, formulas, stacks from stackdio.cli.utils import pass_client from stackdio.client import StackdioClient -from stackdio.client.config import CFG_FILE +from stackdio.client.config import CFG_DIR from stackdio.client.version import __version__ CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) -HIST_FILE = os.path.join(os.path.expanduser('~'), '.stackdio-cli', 'history') - @click_shell.shell(context_settings=CONTEXT_SETTINGS, prompt='stackdio > ', - intro='stackdio-cli, v{0}'.format(__version__), hist_file=HIST_FILE) + intro='stackdio-cli, v{0}'.format(__version__)) @click.version_option(__version__, '-v', '--version') -@click.option('-c', '--config-file', help='The config file to use.', - type=click.Path(dir_okay=False, file_okay=True), default=CFG_FILE, - envvar='STACKDIO_CLI_CONFIG_FILE') +@click.option('-c', '--config-dir', help='The config directory to use.', + type=click.Path(dir_okay=True, file_okay=False), default=CFG_DIR, + envvar='STACKDIO_CONFIG_DIR') @click.pass_context -def stackdio(ctx, config_file): +def stackdio(ctx, config_dir): # Create a client instance - client = StackdioClient(cfg_file=config_file) + client = StackdioClient(cfg_file=os.path.join(config_dir, 'client.cfg')) + + # Set this hist file + ctx.command.hist_file = os.path.join(config_dir, 'cli-history') # Throw an error if we're not configured already if ctx.invoked_subcommand not in ('configure', None) and not client.usable(): diff --git a/stackdio/client/config.py b/stackdio/client/config.py index e5e8a9b..5d1ab49 100644 --- a/stackdio/client/config.py +++ b/stackdio/client/config.py @@ -16,7 +16,6 @@ # import os -import shutil import click import keyring @@ -26,7 +25,8 @@ from stackdio.client.compat import ConfigParser, NoOptionError -CFG_FILE = os.path.join(os.path.expanduser('~'), '.stackdio', 'client.cfg') +CFG_DIR = os.path.join(os.path.expanduser('~'), '.stackdio') +CFG_FILE = os.path.join(CFG_DIR, 'client.cfg') class UserPath(click.Path): From 16ad23796f6d7ada4328fb3e31a0c161f5f7d0cf Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Thu, 28 Jan 2016 12:12:37 -0600 Subject: [PATCH 50/90] Give a better error message when blueprint_dir is missing --- stackdio/cli/mixins/blueprints.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index f3d3469..82bdca3 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -52,7 +52,10 @@ def list_templates(client): click.echo('Missing blueprint directory config') return - blueprint_dir = os.path.expanduser(client.config['blueprint_dir']) + try: + blueprint_dir = os.path.expanduser(client.config['blueprint_dir']) + except KeyError: + raise click.UsageError('Missing \'blueprint_dir\' in config. Please run `configure`.') click.echo('Template mappings:') mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) @@ -123,7 +126,10 @@ def create_blueprint(client, mapping, template, var_file, no_prompt): click.secho('Advanced users only - use the web UI if this isn\'t you!\n', fg='green') - blueprint_dir = client.config['blueprint_dir'] + try: + blueprint_dir = client.config['blueprint_dir'] + except KeyError: + raise click.UsageError('Missing \'blueprint_dir\' in config. Please run `configure`.') if mapping: mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) @@ -157,7 +163,10 @@ def create_all_blueprints(client): """ Create all the blueprints in the map file """ - blueprint_dir = os.path.expanduser(client.config['blueprint_dir']) + try: + blueprint_dir = os.path.expanduser(client.config['blueprint_dir']) + except KeyError: + raise click.UsageError('Missing \'blueprint_dir\' in config. Please run `configure`.') mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) for name, vals in mapping.items(): From 1d93bd58dd5557d16986d1bb6777ceef709d8373 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Mon, 8 Feb 2016 16:02:35 -0600 Subject: [PATCH 51/90] Updating version for 0.7.0 --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index f36a786..12d885a 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -29,7 +29,7 @@ except Exception: pass -VERSION = (0, 7, 0, 'dev', 0) +VERSION = (0, 7, 0, 'final', 0) def get_version(version): From d69be912e3a47234846b04d5e385fa75ecbebc5e Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Mon, 8 Feb 2016 16:03:38 -0600 Subject: [PATCH 52/90] Updating version for 0.8 --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index f36a786..2c6a193 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -29,7 +29,7 @@ except Exception: pass -VERSION = (0, 7, 0, 'dev', 0) +VERSION = (0, 8, 0, 'dev', 0) def get_version(version): From 7fde6e9a491f0e9c23ec9ae08a4b3312091fac49 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Mon, 8 Feb 2016 16:09:09 -0600 Subject: [PATCH 53/90] Updating version for 0.7.1 development --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 12d885a..4fc39da 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -29,7 +29,7 @@ except Exception: pass -VERSION = (0, 7, 0, 'final', 0) +VERSION = (0, 7, 1, 'dev', 0) def get_version(version): From c2a0db52ec965afcff3724b1b75e46e0dee54193 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Mon, 8 Feb 2016 16:25:35 -0600 Subject: [PATCH 54/90] Converted README to rst --- README.md => README.rst | 96 ++++++++++++++++++++++++++++++----------- setup.py | 2 +- 2 files changed, 71 insertions(+), 27 deletions(-) rename README.md => README.rst (59%) diff --git a/README.md b/README.rst similarity index 59% rename from README.md rename to README.rst index 9cf19a1..e475fa0 100644 --- a/README.md +++ b/README.rst @@ -1,12 +1,14 @@ stackdio-python-client ====================== -[![Build Status](https://travis-ci.org/stackdio/stackdio-python-client.svg?branch=master)](https://travis-ci.org/stackdio/stackdio-python-client) +|Travis CI| The canonical Python client and cli for the stackd.io API -## Overview +Overview +-------- + This is a small set of tools for internal use of stackd.io. After cloning this repo, you should be able to quickly get up and running with your own stacks. @@ -14,92 +16,134 @@ stacks. Advanced usage like creating custom blueprints or writing your own formulas is beyond the scope of this. -## Installation -We recommend using virtualenv via [virtualenvwrapper] to install this in a +Installation +------------ + +We recommend using virtualenv via `virtualenvwrapper`_ to install this in a virtualenv. If you consider yourself a knowledgeable Pythonista, feel free to install this however you'd like, but this document will assume that you are -using virtualenvwrapper. See the full [virtualenvwrapper] docs for details, +using virtualenvwrapper. See the full `virtualenvwrapper`_ docs for details, but in short you can install it on most systems like: +.. code:: bash + pip install virtualenvwrapper Once you've got it, installing this tool goes something like: +.. code:: bash + mkvirtualenv stackdio-client - # assuming you are in whatever dir you cloned this repo to: - pip install . + pip install stackdio You'll see a few things scrolling by, but should be set after this. To use this later, you'll need to re-activate the virtualenv like: +.. code:: bash + workon stackdio-client -Whenever it's activated, `stackdio-cli` should be on your path. +Whenever it's activated, ``stackdio-cli`` should be on your path. + +First Use +--------- -## First Use -The first time that you fire up `stackdio-cli`, you'll need to run the -`initial_setup` command. This will prompt you for your LDAP username and +The first time that you fire up ``stackdio-cli``, you'll need to run the +``configure`` command. This will prompt you for your LDAP username and password, and store them securely in your OS keychain for later use. It will import some standard formula, and create a few commonly used blueprints. +.. code:: bash + $ stackdio-cli None @ None - > initial_setup + > configure # YOU WILL BE WALKED THROUGH A SIMPLE SET OF QUESTIONS -## Stack Operations -All of the following assume that you have run `initial_setup` successfully. To +Stack Operations +---------------- + +All of the following assume that you have run ``initial_setup`` successfully. To launch the cli, simply type: +.. code:: bash + $ stackdio-cli -You can run `help` at any point to see available commands. For details on a -specific command you can run `help COMMAND`, e.g. `help stacks`. The rest of +You can run ``help`` at any point to see available commands. For details on a +specific command you can run ``help COMMAND``, e.g. ``help stacks``. The rest of these commands assume you have the cli running. -### Launching Stacks +Launching Stacks +~~~~~~~~~~~~~~~~ Stacks are launched from blueprints. To launch the 3 node HBase stack that's included with this you do: +.. code:: bash + > stacks launch cdh450-ipa-3 MYSTACKNAME -**NOTE:** To avoid DNS namespace collisions, the stack name needs to be unique. -An easy way to ensure this is to include your name in the stack name. -### Deleting Stacks +.. note:: + + To avoid DNS namespace collisions, the stack name needs to be unique. + An easy way to ensure this is to include your name in the stack name. + +Deleting Stacks +~~~~~~~~~~~~~~~ + When you are done with a stack you can delete it. This is destructive and cannot be recovered from, so think carefully before deleting your stack! +.. code:: bash + > stacks delete STACK_NAME -Alternatively you can `terminate` a stack which will terminate all instances, +Alternatively you can ``terminate`` a stack which will terminate all instances, but leave the stack definition in place. -### Provisioning Stacks +Provisioning Stacks +~~~~~~~~~~~~~~~~~~~ + Occassionally something will go wrong when launching your stack, e.g. network connections may flake out causing some package installations to fail. If this happens you can manually provision your stack, causing everything to be brought back up to date: +.. code:: bash + > stacks provision STACK_NAME -### Stack Info +Stack Info +~~~~~~~~~~ + Once you have launched a stack, you can then monitor the status of it like: +.. code:: bash + > stacks history STACK_NAME This displays the top level information for a stack. You can supply additional arguments to pull back additional info about a stack. For example, to get a list of FQDNs (aka hostnames) for a stack: +.. code:: bash + > stacks hostnames STACK_NAME -There are various logs available that you can access with the `stacks logs` +There are various logs available that you can access with the ``stacks logs`` command. -## What's Next? +What's Next? +------------ + For anything not covered by this tool, you'll need to use the stackdio-server web UI or API directly. For more information on that, check out http://docs.stackd.io. -[virtualenvwrapper]: https://pypi.python.org/pypi/virtualenvwrapper + +.. |Travis CI| image:: https://travis-ci.org/stackdio/stackdio-python-client.svg?branch=master + :target: https://travis-ci.org/stackdio/stackdio-python-client + :alt: Build Status + +.. _virtualenvwrapper: https://pypi.python.org/pypi/virtualenvwrapper diff --git a/setup.py b/setup.py index 32376ed..7015fe8 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def test_python_version(): 'platform for everyone.') # Use the README.md as the long description -with open('README.md') as f: +with open('README.rst') as f: LONG_DESCRIPTION = f.read() requirements = [ From afa420d2b47a8b5021a482902a55d6251e2544c6 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Mon, 8 Feb 2016 16:25:35 -0600 Subject: [PATCH 55/90] Converted README to rst --- README.md => README.rst | 96 ++++++++++++++++++++++++++++++----------- setup.py | 2 +- 2 files changed, 71 insertions(+), 27 deletions(-) rename README.md => README.rst (59%) diff --git a/README.md b/README.rst similarity index 59% rename from README.md rename to README.rst index 9cf19a1..e475fa0 100644 --- a/README.md +++ b/README.rst @@ -1,12 +1,14 @@ stackdio-python-client ====================== -[![Build Status](https://travis-ci.org/stackdio/stackdio-python-client.svg?branch=master)](https://travis-ci.org/stackdio/stackdio-python-client) +|Travis CI| The canonical Python client and cli for the stackd.io API -## Overview +Overview +-------- + This is a small set of tools for internal use of stackd.io. After cloning this repo, you should be able to quickly get up and running with your own stacks. @@ -14,92 +16,134 @@ stacks. Advanced usage like creating custom blueprints or writing your own formulas is beyond the scope of this. -## Installation -We recommend using virtualenv via [virtualenvwrapper] to install this in a +Installation +------------ + +We recommend using virtualenv via `virtualenvwrapper`_ to install this in a virtualenv. If you consider yourself a knowledgeable Pythonista, feel free to install this however you'd like, but this document will assume that you are -using virtualenvwrapper. See the full [virtualenvwrapper] docs for details, +using virtualenvwrapper. See the full `virtualenvwrapper`_ docs for details, but in short you can install it on most systems like: +.. code:: bash + pip install virtualenvwrapper Once you've got it, installing this tool goes something like: +.. code:: bash + mkvirtualenv stackdio-client - # assuming you are in whatever dir you cloned this repo to: - pip install . + pip install stackdio You'll see a few things scrolling by, but should be set after this. To use this later, you'll need to re-activate the virtualenv like: +.. code:: bash + workon stackdio-client -Whenever it's activated, `stackdio-cli` should be on your path. +Whenever it's activated, ``stackdio-cli`` should be on your path. + +First Use +--------- -## First Use -The first time that you fire up `stackdio-cli`, you'll need to run the -`initial_setup` command. This will prompt you for your LDAP username and +The first time that you fire up ``stackdio-cli``, you'll need to run the +``configure`` command. This will prompt you for your LDAP username and password, and store them securely in your OS keychain for later use. It will import some standard formula, and create a few commonly used blueprints. +.. code:: bash + $ stackdio-cli None @ None - > initial_setup + > configure # YOU WILL BE WALKED THROUGH A SIMPLE SET OF QUESTIONS -## Stack Operations -All of the following assume that you have run `initial_setup` successfully. To +Stack Operations +---------------- + +All of the following assume that you have run ``initial_setup`` successfully. To launch the cli, simply type: +.. code:: bash + $ stackdio-cli -You can run `help` at any point to see available commands. For details on a -specific command you can run `help COMMAND`, e.g. `help stacks`. The rest of +You can run ``help`` at any point to see available commands. For details on a +specific command you can run ``help COMMAND``, e.g. ``help stacks``. The rest of these commands assume you have the cli running. -### Launching Stacks +Launching Stacks +~~~~~~~~~~~~~~~~ Stacks are launched from blueprints. To launch the 3 node HBase stack that's included with this you do: +.. code:: bash + > stacks launch cdh450-ipa-3 MYSTACKNAME -**NOTE:** To avoid DNS namespace collisions, the stack name needs to be unique. -An easy way to ensure this is to include your name in the stack name. -### Deleting Stacks +.. note:: + + To avoid DNS namespace collisions, the stack name needs to be unique. + An easy way to ensure this is to include your name in the stack name. + +Deleting Stacks +~~~~~~~~~~~~~~~ + When you are done with a stack you can delete it. This is destructive and cannot be recovered from, so think carefully before deleting your stack! +.. code:: bash + > stacks delete STACK_NAME -Alternatively you can `terminate` a stack which will terminate all instances, +Alternatively you can ``terminate`` a stack which will terminate all instances, but leave the stack definition in place. -### Provisioning Stacks +Provisioning Stacks +~~~~~~~~~~~~~~~~~~~ + Occassionally something will go wrong when launching your stack, e.g. network connections may flake out causing some package installations to fail. If this happens you can manually provision your stack, causing everything to be brought back up to date: +.. code:: bash + > stacks provision STACK_NAME -### Stack Info +Stack Info +~~~~~~~~~~ + Once you have launched a stack, you can then monitor the status of it like: +.. code:: bash + > stacks history STACK_NAME This displays the top level information for a stack. You can supply additional arguments to pull back additional info about a stack. For example, to get a list of FQDNs (aka hostnames) for a stack: +.. code:: bash + > stacks hostnames STACK_NAME -There are various logs available that you can access with the `stacks logs` +There are various logs available that you can access with the ``stacks logs`` command. -## What's Next? +What's Next? +------------ + For anything not covered by this tool, you'll need to use the stackdio-server web UI or API directly. For more information on that, check out http://docs.stackd.io. -[virtualenvwrapper]: https://pypi.python.org/pypi/virtualenvwrapper + +.. |Travis CI| image:: https://travis-ci.org/stackdio/stackdio-python-client.svg?branch=master + :target: https://travis-ci.org/stackdio/stackdio-python-client + :alt: Build Status + +.. _virtualenvwrapper: https://pypi.python.org/pypi/virtualenvwrapper diff --git a/setup.py b/setup.py index 32376ed..7015fe8 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def test_python_version(): 'platform for everyone.') # Use the README.md as the long description -with open('README.md') as f: +with open('README.rst') as f: LONG_DESCRIPTION = f.read() requirements = [ From 69814b4fd94580f46f79f86ca030504eb08da6ca Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Mon, 11 Apr 2016 17:06:34 -0500 Subject: [PATCH 56/90] Allow 0.8.x servers --- stackdio/client/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index 1c4bc42..faba27f 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -64,9 +64,9 @@ def __init__(self, url=None, username=None, password=None, verify=True, cfg_file except MissingUrlException: self.version = None - if self.version and (self.version[0] != 0 or self.version[1] != 7): + if self.version and (self.version[0] != 0 or self.version[1] != 8): raise IncompatibleVersionException('Server version {0}.{1}.{2} not ' - 'supported.'.format(**self.version)) + 'supported.'.format(*self.version)) @property def url(self): From 960c926d2b1f459c9bdb80763dc98e65f4ef1bb2 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 15 Apr 2016 15:47:26 -0500 Subject: [PATCH 57/90] Support adding title in map files --- setup.py | 2 +- stackdio/cli/mixins/blueprints.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 7015fe8..3c460a0 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def test_python_version(): LONG_DESCRIPTION = f.read() requirements = [ - 'Jinja2==2.7.3', + 'Jinja2>=2.7', 'PyYAML>=3.10', 'click>=6.0,<7.0', 'click-shell>=0.4', diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index 82bdca3..db79ff1 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -74,7 +74,7 @@ def list_templates(client): _recurse_dir(os.path.join(blueprint_dir, 'var_files'), ['yaml', 'yml']) -def _create_single_blueprint(config, template_file, var_files, no_prompt): +def _create_single_blueprint(config, template_file, var_files, no_prompt, extra_vars=None): blueprint_dir = os.path.expanduser(config['blueprint_dir']) gen = BlueprintGenerator([os.path.join(blueprint_dir, 'templates')]) @@ -102,6 +102,7 @@ def _create_single_blueprint(config, template_file, var_files, no_prompt): # Generate the JSON for the blueprint return gen.generate(template_file, final_var_files, # Pass in a list + variables=extra_vars, prompt=no_prompt) @@ -172,7 +173,7 @@ def create_all_blueprints(client): for name, vals in mapping.items(): try: bp_json = _create_single_blueprint(client.config, vals['template'], - vals['var_file'], False) + vals['var_files'], False, {'title': name}) client.create_blueprint(bp_json) click.secho('Created blueprint {0}'.format(name), fg='green') except BlueprintException: From bdd2079c3dcf8f4a04207191e16296576a9e33b7 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 19 Apr 2016 17:18:30 -0500 Subject: [PATCH 58/90] Back ported a bug from master --- stackdio/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index 1c4bc42..fc8bee4 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -66,7 +66,7 @@ def __init__(self, url=None, username=None, password=None, verify=True, cfg_file if self.version and (self.version[0] != 0 or self.version[1] != 7): raise IncompatibleVersionException('Server version {0}.{1}.{2} not ' - 'supported.'.format(**self.version)) + 'supported.'.format(*self.version)) @property def url(self): From f46f928f9290fafb9dee1a8d34f2ac6c313017dd Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 19 Apr 2016 17:29:59 -0500 Subject: [PATCH 59/90] Removed unused pieces --- stackdio/client/__init__.py | 14 ++++---- stackdio/client/stack.py | 1 - stackdio/client/version.py | 72 ------------------------------------- 3 files changed, 8 insertions(+), 79 deletions(-) diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index fc8bee4..6f718ed 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -32,7 +32,6 @@ from .region import RegionMixin from .settings import SettingsMixin from .stack import StackMixin -from .version import _parse_version_string logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -60,13 +59,16 @@ def __init__(self, url=None, username=None, password=None, verify=True, cfg_file if self.usable(): try: - _, self.version = _parse_version_string(self.get_version(raise_for_status=False)) + raw_version = self.get_version(raise_for_status=False) + self.version = raw_version.split('.') except MissingUrlException: - self.version = None + raw_version = None - if self.version and (self.version[0] != 0 or self.version[1] != 7): - raise IncompatibleVersionException('Server version {0}.{1}.{2} not ' - 'supported.'.format(*self.version)) + if raw_version and (self.version[0] != 0 or self.version[1] != 7): + raise IncompatibleVersionException( + 'Server version {0} not supported. Please upgrade ' + 'stackdio-cli to {1}.{2}.0 or higher.'.format(raw_version, *self.version) + ) @property def url(self): diff --git a/stackdio/client/stack.py b/stackdio/client/stack.py index 1d6f7a5..e8cc7aa 100644 --- a/stackdio/client/stack.py +++ b/stackdio/client/stack.py @@ -17,7 +17,6 @@ from .exceptions import StackException from .http import HttpMixin, get, post, put, delete -from .version import deprecated class StackMixin(HttpMixin): diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 4fc39da..2d65da8 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -97,78 +97,6 @@ def get_git_changeset(): __version__ = get_version(VERSION) -def _unsupported_function(func, current_version, accepted_versions): - raise IncompatibleVersionException("%s: %s is not one of %s" % - (func.__name__, - ".".join([str(v) for v in current_version]), - list(accepted_versions))) - - -def _parse_version_string(version_string): - original_version_string = version_string - comparisons = { - "=": operator.eq, - "!=": operator.ne, - "<": operator.lt, - ">": operator.gt, - "<=": operator.le, - ">=": operator.ge - } - - # Determine the comparison function - comp_string = "=" - if version_string[0] in ["<", ">", "=", "!"]: - offset = 1 - if version_string[1] == "=": - offset += 1 - - comp_string = version_string[:offset] - version_string = version_string[offset:] - - # Check if the version appears compatible - try: - int(version_string[0]) - except ValueError: - raise InvalidVersionStringException(original_version_string) - - # String trailing info - version_string = re.split("[a-zA-Z]", version_string)[0] - if version_string[-1] == '.': - version_string = version_string[:-1] - version = version_string.split(".") - - # Pad length to 3 - version += [0] * (3 - len(version)) - - # Convert to ints - version = [int(num) for num in version] - - try: - return comparisons[comp_string], tuple(version) - except KeyError: - raise InvalidVersionStringException(original_version_string) - - -def accepted_versions(*versions): - def decorator(func): - if not versions: - return func - - parsed_versions = [_parse_version_string(version_string) - for version_string in versions] - - @wraps(func) - def wrapper(obj, *args, **kwargs): - for parsed_version in parsed_versions: - comparison, version = parsed_version - if comparison(obj.version, version): - return func(obj, *args, **kwargs) - - return _unsupported_function(func, obj.version, versions) - return wrapper - return decorator - - def deprecated(func): """ This is a decorator which can be used to mark functions From 2a38ac241e846baba1e10d6fbbf2590fa8fea95e Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 19 Apr 2016 17:53:58 -0500 Subject: [PATCH 60/90] Be smarter about version parsing --- stackdio/client/__init__.py | 39 +++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index 6f718ed..763d27f 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -37,6 +37,40 @@ logger.addHandler(logging.NullHandler()) +def _get_server_version_info(version_str): + basic_info = version_str.split('.') + + major = int(basic_info[0]) + minor = int(basic_info[1]) + + version_type = 'final' + extra_id = 0 + + try: + patch_v = int(basic_info[2]) + except ValueError: + for vtype in ('a', 'b', 'rc'): + if vtype in basic_info[2]: + version_type = vtype + idx = basic_info[2].find(vtype) + patch_v = int(basic_info[:idx]) + extra_id = int(basic_info[2][idx + len(vtype):]) + + if version_type == 'final': + raise ValueError('Invalid version: {}'.format(version_str)) + + if len(basic_info) > 3: + for vtype in ('dev', 'post'): + if basic_info[3].startswith(vtype): + version_type = vtype + extra_id = int(basic_info[3][len(vtype):]) + + if version_type == 'final': + raise ValueError('Invalid version: {}'.format(version_str)) + + return major, minor, patch_v, version_type, extra_id + + class StackdioClient(BlueprintMixin, FormulaMixin, AccountMixin, ImageMixin, RegionMixin, StackMixin, SettingsMixin, HttpMixin): @@ -60,11 +94,12 @@ def __init__(self, url=None, username=None, password=None, verify=True, cfg_file if self.usable(): try: raw_version = self.get_version(raise_for_status=False) - self.version = raw_version.split('.') + self.version = _get_server_version_info(raw_version) except MissingUrlException: raw_version = None + self.version = None - if raw_version and (self.version[0] != 0 or self.version[1] != 7): + if self.version and (self.version[0] != 0 or self.version[1] != 7): raise IncompatibleVersionException( 'Server version {0} not supported. Please upgrade ' 'stackdio-cli to {1}.{2}.0 or higher.'.format(raw_version, *self.version) From 8752e81f6050f38343ae892fd8ddee2464d020f5 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 19 Apr 2016 17:59:15 -0500 Subject: [PATCH 61/90] Removed unused imports --- stackdio/client/version.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 2d65da8..2f33555 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -16,9 +16,7 @@ # import datetime -import operator import os -import re import subprocess import warnings from functools import wraps From 67e2cfd59ec9f95bf3814aa6f187d17e99c82582 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 19 Apr 2016 17:59:58 -0500 Subject: [PATCH 62/90] Updating version for 0.7.1 --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 2f33555..0cc38cb 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 7, 1, 'dev', 0) +VERSION = (0, 7, 1, 'final', 0) def get_version(version): From 3b3e2880906cc3c992a2aba7207f0923928d9067 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 19 Apr 2016 18:00:29 -0500 Subject: [PATCH 63/90] Updating version for 0.7.2 development --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 0cc38cb..75e319c 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 7, 1, 'final', 0) +VERSION = (0, 7, 2, 'dev', 0) def get_version(version): From e4dfccee84c77a3ce6a3bf4be3952b64143a139e Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Sun, 24 Apr 2016 22:21:07 -0500 Subject: [PATCH 64/90] Added utilities for labels --- stackdio/cli/mixins/blueprints.py | 16 ++++++++++++++++ stackdio/client/blueprint.py | 30 ++++++++++++++++++++++++++++-- stackdio/client/stack.py | 28 +++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index db79ff1..eac59a8 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -217,3 +217,19 @@ def delete_all_blueprints(client): for blueprint in client.list_blueprints(): client.delete_blueprint(blueprint['id']) click.secho('Deleted blueprint {0}'.format(blueprint['title']), fg='magenta') + + +@blueprints.command(name='create-label') +@pass_client +@click.argument('title') +@click.argument('key') +@click.argument('value') +def create_label(client, title, key, value): + """ + Create a key:value label on a blueprint + """ + blueprint_id = get_blueprint_id(client, title) + + client.add_blueprint_label(blueprint_id, key, value) + + click.echo('Created label on {0}'.format(title)) diff --git a/stackdio/client/blueprint.py b/stackdio/client/blueprint.py index 051ea6b..f684bb6 100644 --- a/stackdio/client/blueprint.py +++ b/stackdio/client/blueprint.py @@ -16,7 +16,7 @@ # from .exceptions import BlueprintException -from .http import HttpMixin, get, post, delete +from .http import HttpMixin, get, post, put, patch, delete class BlueprintMixin(HttpMixin): @@ -67,6 +67,32 @@ def get_blueprint(self, blueprint_id): def search_blueprints(self, **kwargs): pass - @delete('blueprints/{blueprint_id}') + @delete('blueprints/{blueprint_id}/') def delete_blueprint(self, blueprint_id): pass + + @put('blueprints/{blueprint_id}/properties/') + def update_blueprint_properties(self, blueprint_id, properties): + return properties + + @patch('blueprints/{blueprint_id}/properties/') + def partial_update_blueprint_properties(self, blueprint_id, properties): + return properties + + @post('blueprints/{blueprint_id}/labels/') + def add_blueprint_label(self, blueprint_id, key, value): + return { + 'key': key, + 'value': value, + } + + @put('blueprints/{blueprint_id}/labels/{key}/') + def update_blueprint_label(self, blueprint_id, key, value): + return { + 'key': key, + 'value': value, + } + + @delete('blueprints/{blueprint_id}/labels/{key}/') + def delete_blueprint_label(self, blueprint_id, key): + pass diff --git a/stackdio/client/stack.py b/stackdio/client/stack.py index e8cc7aa..0308f25 100644 --- a/stackdio/client/stack.py +++ b/stackdio/client/stack.py @@ -16,7 +16,7 @@ # from .exceptions import StackException -from .http import HttpMixin, get, post, put, delete +from .http import HttpMixin, get, post, put, patch, delete class StackMixin(HttpMixin): @@ -105,6 +105,32 @@ def get_stack_hosts(self, stack_id): """Get a list of all stack hosts""" pass + @put('stacks/{stack_id}/properties/') + def update_stack_properties(self, stack_id, properties): + return properties + + @patch('stacks/{stack_id}/properties/') + def partial_update_stack_properties(self, stack_id, properties): + return properties + + @post('stacks/{stack_id}/labels/') + def add_stack_label(self, stack_id, key, value): + return { + 'key': key, + 'value': value, + } + + @put('stacks/{stack_id}/labels/{key}/') + def update_stack_label(self, stack_id, key, value): + return { + 'key': key, + 'value': value, + } + + @delete('stacks/{stack_id}/labels/{key}/') + def delete_stack_label(self, stack_id, key): + pass + @get('stacks/{stack_id}/logs/') def list_stack_logs(self, stack_id): """Get a list of stack logs""" From 7da7c87b88b0cb0fddbde8b591efe8a40da73f75 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Sun, 24 Apr 2016 22:23:15 -0500 Subject: [PATCH 65/90] Accidentally committed bad version --- stackdio/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index 763d27f..f10bcb8 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -99,7 +99,7 @@ def __init__(self, url=None, username=None, password=None, verify=True, cfg_file raw_version = None self.version = None - if self.version and (self.version[0] != 0 or self.version[1] != 7): + if self.version and (self.version[0] != 0 or self.version[1] != 8): raise IncompatibleVersionException( 'Server version {0} not supported. Please upgrade ' 'stackdio-cli to {1}.{2}.0 or higher.'.format(raw_version, *self.version) From 0b0a9ded57369ba099c93e9835f6fdfa4d6c63ad Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 29 Apr 2016 18:24:25 -0500 Subject: [PATCH 66/90] Fixed bug with creating blueprints from mappings --- stackdio/cli/mixins/blueprints.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index eac59a8..b8c3e88 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -133,13 +133,13 @@ def create_blueprint(client, mapping, template, var_file, no_prompt): raise click.UsageError('Missing \'blueprint_dir\' in config. Please run `configure`.') if mapping: - mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) - if not mapping or mapping not in mapping: + mappings = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) + if not mappings or mapping not in mappings: click.secho('You gave an invalid mapping.', fg='red') return else: - template = mapping[mapping].get('template') - var_file = mapping[mapping].get('var_files', []) + template = mappings[mapping].get('template') + var_file = mappings[mapping].get('var_files', []) if not template: click.secho('Your mapping must specify a template.', fg='red') return From 8d5a79808b6092da231813466bf59998ad5d34e2 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 3 May 2016 13:42:44 -0500 Subject: [PATCH 67/90] Updating version for 0.7.2 --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 75e319c..dd92dd9 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 7, 2, 'dev', 0) +VERSION = (0, 7, 2, 'final', 0) def get_version(version): From 2c9550f114be63c9f3ebaf4980985415f0573801 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 3 May 2016 13:43:14 -0500 Subject: [PATCH 68/90] Updating version for 0.7.3 development --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index dd92dd9..02ccada 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 7, 2, 'final', 0) +VERSION = (0, 7, 3, 'dev', 0) def get_version(version): From 7f81512189c8c36b4aeb4001d8844f74327dce73 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 3 May 2016 13:58:52 -0500 Subject: [PATCH 69/90] Fixed verify issue --- stackdio/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index 763d27f..abe42ae 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -74,7 +74,7 @@ def _get_server_version_info(version_str): class StackdioClient(BlueprintMixin, FormulaMixin, AccountMixin, ImageMixin, RegionMixin, StackMixin, SettingsMixin, HttpMixin): - def __init__(self, url=None, username=None, password=None, verify=True, cfg_file=None): + def __init__(self, url=None, username=None, password=None, verify=None, cfg_file=None): self.config = StackdioConfig(cfg_file) self._password = self.config.get_password() From 62ab72da48a736148a9882e94a6a2db97b8c2a10 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 1 Jun 2016 11:02:24 -0500 Subject: [PATCH 70/90] Updating version for 0.7.3 --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 02ccada..18f7dc7 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 7, 3, 'dev', 0) +VERSION = (0, 7, 3, 'final', 0) def get_version(version): From 048b44716926f4c5fd56f6439722812fa59fb60e Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 1 Jun 2016 11:02:53 -0500 Subject: [PATCH 71/90] Updating version for 0.7.4 development --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 18f7dc7..c98fad7 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 7, 3, 'final', 0) +VERSION = (0, 7, 4, 'dev', 0) def get_version(version): From 8f64bac1ff400f489af9908a4fb83a918e37459a Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Thu, 30 Jun 2016 15:59:57 -0500 Subject: [PATCH 72/90] Add ability to generate all the blueprints at once --- stackdio/cli/blueprints/generator.py | 7 ++++--- stackdio/cli/mixins/blueprints.py | 17 ++++++++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/stackdio/cli/blueprints/generator.py b/stackdio/cli/blueprints/generator.py index 94c0503..efef425 100644 --- a/stackdio/cli/blueprints/generator.py +++ b/stackdio/cli/blueprints/generator.py @@ -184,7 +184,8 @@ def validate(self, template_file): return unset_vars, set_vars - def generate(self, template_file, var_files=(), variables=None, prompt=False, debug=False): + def generate(self, template_file, var_files=(), variables=None, + prompt=False, debug=False, suppress_warnings=False): """ Generate the rendered blueprint and return it as a python dict @@ -255,14 +256,14 @@ def generate(self, template_file, var_files=(), variables=None, prompt=False, de # If it is set elsewhere, it's not an issue optional_vars = optional_vars - set(context) - if null_vars: + if null_vars and not suppress_warnings: warn_str = '\nWARNING: Null variables (replaced with empty string):\n' for var in null_vars: warn_str += ' {0}\n'.format(var) self.warning(warn_str, 0) # Print a warning if there's unset optional variables - if optional_vars: + if optional_vars and not suppress_warnings: warn_str = '\nWARNING: Missing optional variables:\n' for var in sorted(optional_vars): warn_str += ' {0}\n'.format(var) diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index b8c3e88..d9c8138 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -74,7 +74,8 @@ def list_templates(client): _recurse_dir(os.path.join(blueprint_dir, 'var_files'), ['yaml', 'yml']) -def _create_single_blueprint(config, template_file, var_files, no_prompt, extra_vars=None): +def _create_single_blueprint(config, template_file, var_files, no_prompt, + extra_vars=None, suppress_warnings=False): blueprint_dir = os.path.expanduser(config['blueprint_dir']) gen = BlueprintGenerator([os.path.join(blueprint_dir, 'templates')]) @@ -103,7 +104,8 @@ def _create_single_blueprint(config, template_file, var_files, no_prompt, extra_ return gen.generate(template_file, final_var_files, # Pass in a list variables=extra_vars, - prompt=no_prompt) + prompt=no_prompt, + suppress_warnings=suppress_warnings) @blueprints.command(name='create') @@ -170,10 +172,19 @@ def create_all_blueprints(client): raise click.UsageError('Missing \'blueprint_dir\' in config. Please run `configure`.') mapping = yaml.safe_load(open(os.path.join(blueprint_dir, 'mappings.yaml'), 'r')) + blueprints = client.list_blueprints() + + blueprint_titles = [blueprint['title'] for blueprint in blueprints] + for name, vals in mapping.items(): + if name in blueprint_titles: + click.secho('Skipping creation of {0}, it already exists.'.format(name), fg='yellow') + continue + try: bp_json = _create_single_blueprint(client.config, vals['template'], - vals['var_files'], False, {'title': name}) + vals['var_files'], False, {'title': name}, + suppress_warnings=True) client.create_blueprint(bp_json) click.secho('Created blueprint {0}'.format(name), fg='green') except BlueprintException: From 4404f754ae73efdf3290667d12b4aee51d75d7b5 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 1 Jul 2016 16:59:39 -0500 Subject: [PATCH 73/90] Added a couple of blueprint methods and a snapshot mixin --- stackdio/client/__init__.py | 3 ++- stackdio/client/blueprint.py | 8 +++++++ stackdio/client/snapshot.py | 42 ++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 stackdio/client/snapshot.py diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index abe42ae..43daf08 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -32,6 +32,7 @@ from .region import RegionMixin from .settings import SettingsMixin from .stack import StackMixin +from .snapshot import SnapshotMixin logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -72,7 +73,7 @@ def _get_server_version_info(version_str): class StackdioClient(BlueprintMixin, FormulaMixin, AccountMixin, ImageMixin, - RegionMixin, StackMixin, SettingsMixin, HttpMixin): + RegionMixin, StackMixin, SettingsMixin, SnapshotMixin, HttpMixin): def __init__(self, url=None, username=None, password=None, verify=None, cfg_file=None): self.config = StackdioConfig(cfg_file) diff --git a/stackdio/client/blueprint.py b/stackdio/client/blueprint.py index f684bb6..7392918 100644 --- a/stackdio/client/blueprint.py +++ b/stackdio/client/blueprint.py @@ -71,6 +71,14 @@ def search_blueprints(self, **kwargs): def delete_blueprint(self, blueprint_id): pass + @get('blueprints/{blueprint_id}/host_definitions/') + def get_blueprint_host_definitions(self, blueprint_id): + pass + + @get('blueprints/{blueprint_id}/properties/') + def get_blueprint_properties(self, blueprint_id): + pass + @put('blueprints/{blueprint_id}/properties/') def update_blueprint_properties(self, blueprint_id, properties): return properties diff --git a/stackdio/client/snapshot.py b/stackdio/client/snapshot.py new file mode 100644 index 0000000..60af5b5 --- /dev/null +++ b/stackdio/client/snapshot.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014, Digital Reasoning +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .http import HttpMixin, get, post, delete + + +class SnapshotMixin(HttpMixin): + + @post('cloud/snapshots/') + def create_snapshot(self, snapshot): + """Create a snapshot""" + return snapshot + + @get('snapshots/', paginate=True) + def list_snapshots(self): + pass + + @get('snapshots/{snapshot_id}/') + def get_snapshot(self, snapshot_id): + pass + + @get('snapshots/', paginate=True) + def search_snapshots(self, **kwargs): + pass + + @delete('snapshots/{snapshot_id}/') + def delete_snapshot(self, snapshot_id): + pass From 0ca0923681ef8523bec329fdfab406b994e9cca8 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 1 Jul 2016 17:02:22 -0500 Subject: [PATCH 74/90] Missing paginate --- stackdio/client/blueprint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/blueprint.py b/stackdio/client/blueprint.py index 7392918..a48807c 100644 --- a/stackdio/client/blueprint.py +++ b/stackdio/client/blueprint.py @@ -71,7 +71,7 @@ def search_blueprints(self, **kwargs): def delete_blueprint(self, blueprint_id): pass - @get('blueprints/{blueprint_id}/host_definitions/') + @get('blueprints/{blueprint_id}/host_definitions/', paginate=True) def get_blueprint_host_definitions(self, blueprint_id): pass From 82a76ec69a413cf0c1dfe6fe7005d343c2aa8afb Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Fri, 1 Jul 2016 17:03:53 -0500 Subject: [PATCH 75/90] Fixed snapshot urls --- stackdio/client/snapshot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stackdio/client/snapshot.py b/stackdio/client/snapshot.py index 60af5b5..5cda87a 100644 --- a/stackdio/client/snapshot.py +++ b/stackdio/client/snapshot.py @@ -25,18 +25,18 @@ def create_snapshot(self, snapshot): """Create a snapshot""" return snapshot - @get('snapshots/', paginate=True) + @get('cloud/snapshots/', paginate=True) def list_snapshots(self): pass - @get('snapshots/{snapshot_id}/') + @get('cloud/snapshots/{snapshot_id}/') def get_snapshot(self, snapshot_id): pass - @get('snapshots/', paginate=True) + @get('cloud/snapshots/', paginate=True) def search_snapshots(self, **kwargs): pass - @delete('snapshots/{snapshot_id}/') + @delete('cloud/snapshots/{snapshot_id}/') def delete_snapshot(self, snapshot_id): pass From e7168a34d31f1421043e71aa2e9b3e4478b2c328 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 1 Nov 2016 12:12:22 -0500 Subject: [PATCH 76/90] Fixed version parsing --- stackdio/client/__init__.py | 43 ++++++------------------------------- 1 file changed, 6 insertions(+), 37 deletions(-) diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index f10bcb8..877a189 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -15,8 +15,11 @@ # limitations under the License. # +from __future__ import unicode_literals + import logging +from pkg_resources import parse_version from .account import AccountMixin from .blueprint import BlueprintMixin from .config import StackdioConfig @@ -37,44 +40,10 @@ logger.addHandler(logging.NullHandler()) -def _get_server_version_info(version_str): - basic_info = version_str.split('.') - - major = int(basic_info[0]) - minor = int(basic_info[1]) - - version_type = 'final' - extra_id = 0 - - try: - patch_v = int(basic_info[2]) - except ValueError: - for vtype in ('a', 'b', 'rc'): - if vtype in basic_info[2]: - version_type = vtype - idx = basic_info[2].find(vtype) - patch_v = int(basic_info[:idx]) - extra_id = int(basic_info[2][idx + len(vtype):]) - - if version_type == 'final': - raise ValueError('Invalid version: {}'.format(version_str)) - - if len(basic_info) > 3: - for vtype in ('dev', 'post'): - if basic_info[3].startswith(vtype): - version_type = vtype - extra_id = int(basic_info[3][len(vtype):]) - - if version_type == 'final': - raise ValueError('Invalid version: {}'.format(version_str)) - - return major, minor, patch_v, version_type, extra_id - - class StackdioClient(BlueprintMixin, FormulaMixin, AccountMixin, ImageMixin, RegionMixin, StackMixin, SettingsMixin, HttpMixin): - def __init__(self, url=None, username=None, password=None, verify=True, cfg_file=None): + def __init__(self, url=None, username=None, password=None, verify=None, cfg_file=None): self.config = StackdioConfig(cfg_file) self._password = self.config.get_password() @@ -94,12 +63,12 @@ def __init__(self, url=None, username=None, password=None, verify=True, cfg_file if self.usable(): try: raw_version = self.get_version(raise_for_status=False) - self.version = _get_server_version_info(raw_version) + self.version = parse_version(raw_version) except MissingUrlException: raw_version = None self.version = None - if self.version and (self.version[0] != 0 or self.version[1] != 8): + if self.version and not self.version.base_version.startswith('0.8'): raise IncompatibleVersionException( 'Server version {0} not supported. Please upgrade ' 'stackdio-cli to {1}.{2}.0 or higher.'.format(raw_version, *self.version) From 1cc82c42fe0b5875b29aa264ae64373674e6d837 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 1 Nov 2016 12:13:56 -0500 Subject: [PATCH 77/90] parse_version sometimes returns a tuple? --- stackdio/client/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/__init__.py b/stackdio/client/__init__.py index 877a189..4ce2cea 100644 --- a/stackdio/client/__init__.py +++ b/stackdio/client/__init__.py @@ -68,7 +68,7 @@ def __init__(self, url=None, username=None, password=None, verify=None, cfg_file raw_version = None self.version = None - if self.version and not self.version.base_version.startswith('0.8'): + if self.version and (int(self.version[0]) != 0 or int(self.version[1]) != 8): raise IncompatibleVersionException( 'Server version {0} not supported. Please upgrade ' 'stackdio-cli to {1}.{2}.0 or higher.'.format(raw_version, *self.version) From 5e630488cf194a55bd4e5b081d8e0aaebb686a52 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 1 Nov 2016 12:22:30 -0500 Subject: [PATCH 78/90] Removed search_* methods, just pass kwargs to list_* methods instead --- stackdio/cli/mixins/blueprints.py | 2 +- stackdio/cli/mixins/formulas.py | 2 +- stackdio/cli/mixins/stacks.py | 4 ++-- stackdio/client/account.py | 14 ++------------ stackdio/client/blueprint.py | 6 +----- stackdio/client/formula.py | 7 +------ stackdio/client/image.py | 7 +------ stackdio/client/region.py | 12 ++---------- stackdio/client/stack.py | 11 +++++------ 9 files changed, 16 insertions(+), 49 deletions(-) diff --git a/stackdio/cli/mixins/blueprints.py b/stackdio/cli/mixins/blueprints.py index b8c3e88..5d8fab2 100644 --- a/stackdio/cli/mixins/blueprints.py +++ b/stackdio/cli/mixins/blueprints.py @@ -181,7 +181,7 @@ def create_all_blueprints(client): def get_blueprint_id(client, blueprint_title): - found_blueprints = client.search_blueprints(title=blueprint_title) + found_blueprints = client.list_blueprints(title=blueprint_title) if len(found_blueprints) == 0: raise click.Abort('Blueprint "{0}" does not exist'.format(blueprint_title)) diff --git a/stackdio/cli/mixins/formulas.py b/stackdio/cli/mixins/formulas.py index 2fcb748..ec4d47c 100644 --- a/stackdio/cli/mixins/formulas.py +++ b/stackdio/cli/mixins/formulas.py @@ -43,7 +43,7 @@ def import_formula(client, uri, username, password): def get_formula_id(client, formula_uri): - found_formulas = client.search_formulas(uri=formula_uri) + found_formulas = client.list_formulas(uri=formula_uri) if len(found_formulas) == 0: raise click.Abort('Formula "{0}" does not exist'.format(formula_uri)) diff --git a/stackdio/cli/mixins/stacks.py b/stackdio/cli/mixins/stacks.py index a1fa3f1..cea0c30 100644 --- a/stackdio/cli/mixins/stacks.py +++ b/stackdio/cli/mixins/stacks.py @@ -52,7 +52,7 @@ def launch_stack(client, blueprint_title, stack_title): def get_stack_id(client, stack_title): - found_stacks = client.search_stacks(title=stack_title) + found_stacks = client.list_stacks(title=stack_title) if len(found_stacks) == 0: raise click.Abort('Stack "{0}" does not exist'.format(stack_title)) @@ -73,7 +73,7 @@ def stack_history(client, stack_title, length): stack_id = get_stack_id(client, stack_title) history = client.get_stack_history(stack_id) for event in history[0:min(length, len(history))]: - click.echo('[{created}] {level} // {event} // {status}'.format(**event)) + click.echo('[{created}] {message}'.format(**event)) @stacks.command(name='hostnames') diff --git a/stackdio/client/account.py b/stackdio/client/account.py index 928105f..f43b522 100644 --- a/stackdio/client/account.py +++ b/stackdio/client/account.py @@ -21,15 +21,10 @@ class AccountMixin(HttpMixin): @get('cloud/providers/', paginate=True) - def list_providers(self): + def list_providers(self, **kwargs): """List all providers""" pass - @get('cloud/providers/', paginate=True) - def search_providers(self, **kwargs): - """Search for a provider""" - pass - @post('cloud/accounts/') def create_account(self, **kwargs): """Create an account""" @@ -53,7 +48,7 @@ def create_account(self, **kwargs): return form_data @get('cloud/accounts/', paginate=True) - def list_accounts(self): + def list_accounts(self, **kwargs): """List all account""" pass @@ -62,11 +57,6 @@ def get_account(self, account_id): """Return the account that matches the given id""" pass - @get('cloud/accounts/') - def search_accounts(self, **kwargs): - """List all accounts""" - pass - @delete('cloud/accounts/{account_id}/') def delete_account(self, account_id): """List all accounts""" diff --git a/stackdio/client/blueprint.py b/stackdio/client/blueprint.py index f684bb6..e9e9f2e 100644 --- a/stackdio/client/blueprint.py +++ b/stackdio/client/blueprint.py @@ -56,17 +56,13 @@ def create_blueprint(self, blueprint): return blueprint @get('blueprints/', paginate=True) - def list_blueprints(self): + def list_blueprints(self, **kwargs): pass @get('blueprints/{blueprint_id}/') def get_blueprint(self, blueprint_id): pass - @get('blueprints/', paginate=True) - def search_blueprints(self, **kwargs): - pass - @delete('blueprints/{blueprint_id}/') def delete_blueprint(self, blueprint_id): pass diff --git a/stackdio/client/formula.py b/stackdio/client/formula.py index 5e30935..3d819b7 100644 --- a/stackdio/client/formula.py +++ b/stackdio/client/formula.py @@ -37,7 +37,7 @@ def import_formula(self, formula_uri, git_username=None, git_password=None, acce return data @get('formulas/', paginate=True) - def list_formulas(self): + def list_formulas(self, **kwargs): """Return all formulas""" pass @@ -50,11 +50,6 @@ def get_formula(self, formula_id): def list_components_for_version(self, formula_id, version): pass - @get('formulas/', paginate=True) - def search_formulas(self, **kwargs): - """Get a formula with matching id""" - pass - @delete('formulas/{formula_id}/') def delete_formula(self, formula_id): """Delete formula with matching id""" diff --git a/stackdio/client/image.py b/stackdio/client/image.py index 8af4f15..80402f2 100644 --- a/stackdio/client/image.py +++ b/stackdio/client/image.py @@ -32,7 +32,7 @@ def create_image(self, title, image_id, ssh_user, cloud_provider, default_instan } @get('cloud/images/', paginate=True) - def list_images(self): + def list_images(self, **kwargs): """List all images""" pass @@ -41,11 +41,6 @@ def get_image(self, image_id): """Return the image that matches the given id""" pass - @get('cloud/images/', paginate=True) - def search_images(self, **kwargs): - """List all images""" - pass - @delete('cloud/images/{image_id}/') def delete_image(self, image_id): """Delete the image with the given id""" diff --git a/stackdio/client/region.py b/stackdio/client/region.py index 2689d8b..e69237a 100644 --- a/stackdio/client/region.py +++ b/stackdio/client/region.py @@ -20,25 +20,17 @@ class RegionMixin(HttpMixin): @get('cloud/providers/{provider_name}/regions/', paginate=True) - def list_regions(self, provider_name): + def list_regions(self, provider_name, **kwargs): pass @get('cloud/providers/{provider_name}/regions/{region_id}/') def get_region(self, provider_name, region_id): pass - @get('cloud/providers/{provider_name}/regions/', paginate=True) - def search_regions(self, provider_name, **kwargs): - pass - @get('cloud/providers/{provider_name}/zones/', paginate=True) - def list_zones(self): + def list_zones(self, provider_name, **kwargs): pass @get('cloud/providers/{provider_name}/zones/{zone_id}') def get_zone(self, provider_name, zone_id): pass - - @get('cloud/providers/{provider_name}/zones/', paginate=True) - def search_zones(self, provider_name, **kwargs): - pass diff --git a/stackdio/client/stack.py b/stackdio/client/stack.py index 0308f25..1daef54 100644 --- a/stackdio/client/stack.py +++ b/stackdio/client/stack.py @@ -33,7 +33,7 @@ def create_stack(self, stack_data): return stack_data @get('stacks/', paginate=True) - def list_stacks(self): + def list_stacks(self, **kwargs): """Return a list of all stacks""" pass @@ -42,11 +42,6 @@ def get_stack(self, stack_id): """Get stack info""" pass - @get('stacks/', paginate=True) - def search_stacks(self, **kwargs): - """Search for stacks that match the given criteria""" - pass - @delete('stacks/{stack_id}/') def delete_stack(self, stack_id): """Destructively delete a stack forever.""" @@ -100,6 +95,10 @@ def get_stack_history(self, stack_id): """Get stack info""" pass + @get_stack_history.response + def get_stack_history(self, resp): + return list(reversed(resp)) + @get('stacks/{stack_id}/hosts/', paginate=True) def get_stack_hosts(self, stack_id): """Get a list of all stack hosts""" From 57f8e6fbaa309cd9840e411cc639d96c83ef8830 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 1 Nov 2016 12:28:59 -0500 Subject: [PATCH 79/90] Make travis upload to pypi --- .travis.yml | 43 ++++++++++++++++++++++++------------------- setup.py | 8 +++----- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b4d5c0..5dbb743 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,20 +2,13 @@ language: python python: - "2.7" - - "3.3" - "3.4" + - "3.5" cache: directories: - $HOME/.cache/pip -# Set up our environment -env: - NOSE_WITH_XUNIT: 1 - NOSE_WITH_COVERAGE: 1 - NOSE_COVER_BRANCHES: 1 - NOSE_COVER_INCLUSIVE: 1 - # So that we get a docker container sudo: false @@ -37,18 +30,30 @@ script: # Only build artifacts on success after_success: - coveralls - - export STACKDIO_VERSION=`python setup.py --version` - python setup.py sdist - python setup.py bdist_wheel deploy: - provider: releases - api_key: - secure: T4jI1aZQ+wDJBgGxcbdrtLz3zpXA9yZwmrsm8d3GqEGxApMtkKLWq0uqf86C8VkqaY6p4Nm1a/PTApV1isbuSoJbdeMVJA1MlYB/G7QMK7eI8nFqkw7Q4jzuOdEC0D1CPZx7ZWBn0bYxSRTcSeQSnGeGDy2KxekGSZFfIxe4APo= - file: - - dist/stackdio-${STACKDIO_VERSION}.tar.gz - - dist/stackdio-${STACKDIO_VERSION}-py2.py3-none-any.whl - skip_cleanup: true - on: - tags: true - repo: stackdio/stackdio-python-client + - provider: releases + api_key: + secure: T4jI1aZQ+wDJBgGxcbdrtLz3zpXA9yZwmrsm8d3GqEGxApMtkKLWq0uqf86C8VkqaY6p4Nm1a/PTApV1isbuSoJbdeMVJA1MlYB/G7QMK7eI8nFqkw7Q4jzuOdEC0D1CPZx7ZWBn0bYxSRTcSeQSnGeGDy2KxekGSZFfIxe4APo= + file: + - dist/stackdio-${TRAVIS_TAG}.tar.gz + - dist/stackdio-${TRAVIS_TAG}-py2.py3-none-any.whl + skip_cleanup: true + on: + tags: true + repo: stackdio/stackdio-python-client + python: "2.7" + + # Upload to pypi. Do this instead of the pypi provider so that we + # ensure the exact same artifact is uploaded to github and pypi. + # The pypi provider will re-build the 2 artifacts, which is not ideal. + # This requires setting TWINE_USERNAME and TWINE_PASSWORD in travis. + - provider: script + script: twine upload dist/stackdio-${TRAVIS_TAG}.tar.gz dist/stackdio-${TRAVIS_TAG}-py2.py3-none-any.whl + skip_cleanup: true + on: + tags: true + repo: stackdio/stackdio-python-client + python: "2.7" diff --git a/setup.py b/setup.py index 3c460a0..f8869f0 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,6 @@ # limitations under the License. # -import os import sys from setuptools import setup, find_packages @@ -25,9 +24,9 @@ def test_python_version(): major = sys.version_info[0] minor = sys.version_info[1] micro = sys.version_info[2] - if float('%d.%d' % (major, minor)) < 2.6: + if (major, minor) < (2, 7): err_msg = ('Your Python version {0}.{1}.{2} is not supported.\n' - 'stackdio-server requires Python 2.6 or newer.\n'.format(major, minor, micro)) + 'stackdio-server requires Python 2.7 or newer.\n'.format(major, minor, micro)) sys.stderr.write(err_msg) sys.exit(1) @@ -85,7 +84,6 @@ def test_python_version(): 'console_scripts': [ 'stackdio-cli=stackdio.cli:main', 'blueprint-generator=stackdio.cli.blueprints:main', - 'stackdio-config-convert=stackdio.client.config:main', ], }, classifiers=[ @@ -99,8 +97,8 @@ def test_python_version(): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: System :: Clustering', 'Topic :: System :: Distributed Computing', ] From 2de64767b13a3a15b229a571e01052ace059bdd7 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 1 Nov 2016 12:34:37 -0500 Subject: [PATCH 80/90] Removed search method --- stackdio/client/snapshot.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/stackdio/client/snapshot.py b/stackdio/client/snapshot.py index 5cda87a..6264f8b 100644 --- a/stackdio/client/snapshot.py +++ b/stackdio/client/snapshot.py @@ -26,17 +26,13 @@ def create_snapshot(self, snapshot): return snapshot @get('cloud/snapshots/', paginate=True) - def list_snapshots(self): + def list_snapshots(self, **kwargs): pass @get('cloud/snapshots/{snapshot_id}/') def get_snapshot(self, snapshot_id): pass - @get('cloud/snapshots/', paginate=True) - def search_snapshots(self, **kwargs): - pass - @delete('cloud/snapshots/{snapshot_id}/') def delete_snapshot(self, snapshot_id): pass From 674dc7069b8de851e482a5d671808caba06da377 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 1 Nov 2016 15:34:29 -0500 Subject: [PATCH 81/90] Updating version for 0.8.0b1 --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 73cb64f..bcee887 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 8, 0, 'dev', 0) +VERSION = (0, 8, 0, 'b', 1) def get_version(version): From 0099f9d5838c98d287c6e8931e1dd53dc3cc574d Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 1 Nov 2016 15:34:51 -0500 Subject: [PATCH 82/90] Updating version for 0.8.0 development --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index bcee887..73cb64f 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 8, 0, 'b', 1) +VERSION = (0, 8, 0, 'dev', 0) def get_version(version): From 0dcb4cd66e873672aa1a2b539516fbf88592c5f5 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 1 Nov 2016 15:38:26 -0500 Subject: [PATCH 83/90] Install twine --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5dbb743..8b9c997 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ sudo: false install: - pip install -U pip - pip install -U wheel + - pip install -U twine - pip install -U -e .[testing] ## Customize test commands From 2e7d1c1398f86596c4f7d61188d5d4bbd75b2e1c Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Wed, 5 Jul 2017 14:14:59 -0500 Subject: [PATCH 84/90] Support yaml blueprint templates --- stackdio/cli/blueprints/generator.py | 37 +++++++++++++++++----------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/stackdio/cli/blueprints/generator.py b/stackdio/cli/blueprints/generator.py index efef425..b59c935 100644 --- a/stackdio/cli/blueprints/generator.py +++ b/stackdio/cli/blueprints/generator.py @@ -1,15 +1,15 @@ -from __future__ import print_function +from __future__ import print_function, unicode_literals -import sys -import os import json +import os +import sys import click import yaml from jinja2 import Environment, FileSystemLoader, StrictUndefined, meta from jinja2.exceptions import TemplateNotFound, TemplateSyntaxError, UndefinedError -from jinja2.nodes import Assign, Block, Const, If, Not from jinja2.filters import do_replace, evalcontextfilter +from jinja2.nodes import Assign, Block, Const, If, Not class BlueprintException(Exception): @@ -232,12 +232,12 @@ def generate(self, template_file, var_files=(), variables=None, if prompt: # Prompt for missing vars for var in sorted(missing_vars): - context[var] = self.prompt('{0}: '.format(var)) + context[var] = self.prompt('{}: '.format(var)) else: # Print an error error_str = 'Missing variables:\n' for var in sorted(missing_vars): - error_str += ' {0}\n'.format(var) + error_str += ' {}\n'.format(var) self.error_exit(error_str, 0) # Find the set of optional variables (ones inside of a 'if is not undefined' @@ -259,14 +259,14 @@ def generate(self, template_file, var_files=(), variables=None, if null_vars and not suppress_warnings: warn_str = '\nWARNING: Null variables (replaced with empty string):\n' for var in null_vars: - warn_str += ' {0}\n'.format(var) + warn_str += ' {}\n'.format(var) self.warning(warn_str, 0) # Print a warning if there's unset optional variables if optional_vars and not suppress_warnings: warn_str = '\nWARNING: Missing optional variables:\n' for var in sorted(optional_vars): - warn_str += ' {0}\n'.format(var) + warn_str += ' {}\n'.format(var) self.warning(warn_str, 0) # Generate the blueprint @@ -276,24 +276,31 @@ def generate(self, template_file, var_files=(), variables=None, set_vars.update(context) context = set_vars - template_json = template.render(**context) + rendered_template = template.render(**context) if debug: click.echo('\n') - click.echo(template_json) + click.echo(rendered_template) click.echo('\n') - # Return a dict object rather than a string - return json.loads(template_json) + template_extension = template_file.split('.')[-1] + + if template_extension in ('json',): + # Return a dict object rather than a string + return json.loads(rendered_template) + elif template_extension in ('yaml', 'yml'): + return yaml.safe_load(rendered_template) + else: + self.error_exit('Template extension {} is not valid.'.format(template_extension)) except TemplateNotFound: - self.error_exit('Your template file {0} was not found.'.format(template_file)) + self.error_exit('Your template file {} was not found.'.format(template_file)) except TemplateSyntaxError as e: - self.error_exit('Invalid template error at line {0}:\n{1}'.format( + self.error_exit('Invalid template error at line {}:\n{}'.format( e.lineno, str(e) )) except UndefinedError as e: - self.error_exit('Missing variable: {0}'.format(str(e))) + self.error_exit('Missing variable: {}'.format(str(e))) # except ValueError: # self.error_exit('Invalid JSON. Check your template file.') From a0aae18720c0ba553d77b40eab02f9b621a8742e Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 15 May 2018 11:52:03 -0500 Subject: [PATCH 85/90] Readability changes --- stackdio/client/http.py | 5 +---- stackdio/client/stack.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/stackdio/client/http.py b/stackdio/client/http.py index 4eb3d4d..35e2bf7 100644 --- a/stackdio/client/http.py +++ b/stackdio/client/http.py @@ -95,10 +95,7 @@ def __init__(self, dfunc=None, rfunc=None, quiet=False): self.obj = None self.data_func = dfunc - self.response_func = rfunc - - if self.response_func is None: - self.response_func = default_response + self.response_func = rfunc or default_response self.quiet = quiet diff --git a/stackdio/client/stack.py b/stackdio/client/stack.py index 1daef54..396fe3d 100644 --- a/stackdio/client/stack.py +++ b/stackdio/client/stack.py @@ -97,7 +97,7 @@ def get_stack_history(self, stack_id): @get_stack_history.response def get_stack_history(self, resp): - return list(reversed(resp)) + return reversed(resp) @get('stacks/{stack_id}/hosts/', paginate=True) def get_stack_hosts(self, stack_id): From 7889c765218d78c3d398a683228cc2b875c7d61e Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 15 May 2018 11:53:15 -0500 Subject: [PATCH 86/90] Updating version for 0.8.0b2 --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 73cb64f..5687339 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 8, 0, 'dev', 0) +VERSION = (0, 8, 0, 'b', 2) def get_version(version): From 3b4782648670eed7d0e385d4aa6918a297a56451 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 15 May 2018 11:54:11 -0500 Subject: [PATCH 87/90] Updating version for 0.8.0 development --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 5687339..73cb64f 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 8, 0, 'b', 2) +VERSION = (0, 8, 0, 'dev', 0) def get_version(version): From fce41a067c23775f6fca93e4a06faa993d9b5469 Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 15 May 2018 12:05:55 -0500 Subject: [PATCH 88/90] Fixing travis deploy --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8b9c997..619765d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ after_success: deploy: - provider: releases api_key: - secure: T4jI1aZQ+wDJBgGxcbdrtLz3zpXA9yZwmrsm8d3GqEGxApMtkKLWq0uqf86C8VkqaY6p4Nm1a/PTApV1isbuSoJbdeMVJA1MlYB/G7QMK7eI8nFqkw7Q4jzuOdEC0D1CPZx7ZWBn0bYxSRTcSeQSnGeGDy2KxekGSZFfIxe4APo= + secure: WDRJ+QYPfAMuH8sEFPTTEHabaEtfvLWvHiXi69NA3lruIlKr0Id5gpF/Bqr5VfHiz9jdHuBRdVLgYRYVXAVsRkw13N1YlHgR4j4oi61fMugwDTC820Jnf8EDpuvXys8TPiPRh7Xe2XTGc4HMO0moGz6gp9gH4OAsxGgLPNLmiDA= file: - dist/stackdio-${TRAVIS_TAG}.tar.gz - dist/stackdio-${TRAVIS_TAG}-py2.py3-none-any.whl From ac17d07a7b42aea3c753d3543133aaa94385acfd Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 15 May 2018 12:08:00 -0500 Subject: [PATCH 89/90] Updating version for 0.8.0b2 --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 73cb64f..5687339 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 8, 0, 'dev', 0) +VERSION = (0, 8, 0, 'b', 2) def get_version(version): From 9a74959fa19494f1a840920bcaa413ff7c042b6a Mon Sep 17 00:00:00 2001 From: Clark Perkins Date: Tue, 15 May 2018 12:08:32 -0500 Subject: [PATCH 90/90] Updating version for 0.8.0 development --- stackdio/client/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stackdio/client/version.py b/stackdio/client/version.py index 5687339..73cb64f 100644 --- a/stackdio/client/version.py +++ b/stackdio/client/version.py @@ -27,7 +27,7 @@ except Exception: pass -VERSION = (0, 8, 0, 'b', 2) +VERSION = (0, 8, 0, 'dev', 0) def get_version(version):