Add a copy of the nginx builder module

sysadmin/nginx/config/builder/__init__.py

@@ -0,0 +1,210 @@
+"""
+The Builder API defines a pluggable builder framework for manipulating nginx configs from within python.
+
+Building a config
+=================
+
+Every config built using the builder pattern starts off with creating a :class:NginxConfigBuilder::
+
+    from nginx.config.builder import NginxConfigBuilder
+
+    nginx = NginxConfigBuilder()
+
+By default, this comes loaded with a bunch of helpful tools to easily create routes and servers
+in nginx::
+
+    with nginx.add_server() as server:
+        server.add_route('/foo').end()
+        with server.add_route('/bar') as bar:
+             bar.add_route('/baz')
+
+This generates a simple config that looks like this::
+
+    error_log logs/nginx.error.log;
+    worker_processes auto;
+    daemon on;
+    http {
+        include ../conf/mime.types;
+        server {
+            server_name _;
+            location /foo {
+            }
+            location /bar {
+                location /baz {
+                }
+            }
+        }
+    }
+    events {
+        worker_connections 1024;
+    }
+
+Plugins
+=======
+
+A plugin is a class that inherits from :class:`nginx.config.builder.baseplugins.Plugin` that provides
+additional methods which can be chained off of the :class:`NginxConfigBuilder` object. These plugins provide
+convenience methods that manipulate the underlying nginx configuration that gets built by the
+:class:`NginxConfigBuilder`.
+
+A simple plugin only needs to define what methods it's going to export::
+
+    class NoopPlugin(Plugin):
+        name = 'noop'
+
+        @property
+        def exported_methods(self):
+             return {'noop': self.noop}
+
+        def noop(self):
+             pass
+
+This NoopPlugin provides a simple function that can be called off of a :class:`NginxConfigBuilder` that
+does nothing successfully. More complex plugins can be found in :mod:`nginx.config.builder.plugins`
+
+To use this NoopPlugin, we need to create a config builder and then register the plugin with it::
+
+    nginx = NginxConfigBuilder()
+    nginx.noop()  # AttributeError :(
+    nginx.register_plugin(NoopPlugin())
+    nginx.noop()  # it works!
+
+A more complex plugin would actually do something, like a plugin that adds an expiry directive to
+a route::
+
+    class ExpiryPlugin(Plugin):
+        name = 'expiry'
+        @property
+
+"""
+
+from ..api import EmptyBlock, Block, Config
+from .exceptions import ConfigBuilderConflictException, ConfigBuilderException, ConfigBuilderNoSuchMethodException
+from .baseplugins import RoutePlugin, ServerPlugin, Plugin
+
+
+DEFAULT_PLUGINS = (RoutePlugin, ServerPlugin)
+INVALID_PLUGIN_NAMES = ('top,')
+
+
+class NginxConfigBuilder(object):
+    """ Helper that builds a working nginx configuration
+
+    Exposes a plugin-based architecture for generating nginx configurations.
+
+    """
+
+    def __init__(self, worker_processes='auto', worker_connections=512, error_log='logs/error.log', daemon='off'):
+        """
+        :param worker_processes str|int: number of worker processes to start with (default: auto)
+        :param worker_connections int: number of nginx worker connections (default: 512)
+        :param error_log str: path to nginx error log (default: logs/error.log)
+        :param daemon str: whether or not to daemonize nginx (default: on)
+        """
+
+        self.plugins = []
+        self._top = EmptyBlock(
+            worker_processes=worker_processes,
+            error_log=error_log,
+            daemon=daemon,
+        )
+
+        self._http = Block(
+            'http',
+            include='../conf/mime.types'
+        )
+
+        self._cwo = self._http
+        self._events = Block(
+            'events',
+            worker_connections=worker_connections
+        )
+
+        self._methods = {}
+
+        for plugin in DEFAULT_PLUGINS:
+            self.register_plugin(plugin(parent=self._http))
+
+    def _validate_plugin(self, plugin):
+        if not isinstance(plugin, Plugin):
+            raise ConfigBuilderException(
+                "Must be a subclass of {cls}".format(cls=Plugin.__name__),
+                plugin=plugin
+            )
+
+        if plugin.name in INVALID_PLUGIN_NAMES:
+            raise ConfigBuilderException(
+                "{name} is a protected name and cannot be used as the name of"
+                " a plugin".format(name=plugin.name),
+                plugin=plugin
+            )
+
+        if plugin.name in (loaded.name for loaded in self.plugins):
+            raise ConfigBuilderConflictException(plugin=plugin, loaded_plugin=plugin, method_name='name')
+
+        methods = plugin.exported_methods.items()
+
+        # check for conflicts
+        for loaded_plugin in self.plugins:
+            for (name, method) in methods:
+                if name in loaded_plugin.exported_methods:
+                    raise ConfigBuilderConflictException(
+                        plugin=plugin,
+                        loaded_plugin=loaded_plugin,
+                        method_name=name
+                    )
+
+                # Also protect register_plugin, etc.
+                if hasattr(self, name):
+                    raise ConfigBuilderConflictException(
+                        plugin=plugin,
+                        loaded_plugin='top',
+                        method_name=name
+                    )
+
+        # we can only be owned once
+        if plugin._config_builder:
+            raise ConfigBuilderException("Already owned by another NginxConfigBuilder", plugin=plugin)
+        plugin._config_builder = self
+
+    def register_plugin(self, plugin):
+        """ Registers a new nginx builder plugin.
+
+        Plugins must inherit from nginx.builder.baseplugins.Plugin and not expose methods that conflict
+        with already loaded plugins
+
+        :param plugin nginx.builder.baseplugins.Plugin: nginx plugin to add to builder
+        """
+
+        self._validate_plugin(plugin)
+
+        # insert ourselves as the config builder for plugins
+        plugin._config_builder = self
+        self.plugins.append(plugin)
+
+        self._methods.update(plugin.exported_methods)
+
+    @property
+    def top(self):
+        """ Returns the logical top of the config hierarchy.
+
+        This is a convenience method for any plugins that need to quickly access the top of the config tree.
+
+        :returns :class:`nginx.config.Block`: Top of the config block
+        """
+        return self._http
+
+    def __getattr__(self, attr):
+        # Since we want this to be easy to use, we will do method lookup
+        # on the methods that we've gotten from our different config plugins
+        # whenever someone tries to call a method off of the builder.
+        #
+        # This means that plugins can just return a reference to the builder
+        # so that users can just chain methods off of the builder.
+        try:
+            return self._methods[attr]
+        except KeyError:
+            raise ConfigBuilderNoSuchMethodException(attr, builder=self)
+
+    def __repr__(self):
+        return repr(Config(self._top, self._events, self._http))

sysadmin/nginx/config/builder/baseplugins.py

@@ -0,0 +1,206 @@
+import re
+import six
+
+from abc import ABCMeta, abstractproperty
+
+from ..api import Block, Location
+from .exceptions import ConfigBuilderException
+
+
+@six.add_metaclass(ABCMeta)
+class Navigable(object):
+    """ Indicates that a class is navigable.
+
+    This means that it references some type of nginx config machinery,
+    and that this is traversable.
+
+    """
+    _config_builder = None
+
+    def __init__(self, *args, **kwargs):
+        """ Creates a new Navigable class
+
+        :param nginx.builder.ConfigBuilder config_builder: internal ConfigBuilder used to create nginx config objs
+        """
+        # This can sometimes be added by direct access,
+        # in the case of plugins
+        self._config_builder = kwargs.get('config_builder', None)
+        self._parent = kwargs.get('parent', None)
+
+    def chobj(self, obj):
+        """ Changes the current working object to the one provided.
+
+        :param nginx.config.Block obj: object that we're scoping to
+        """
+        self.config_builder._cwo = obj
+
+    @property
+    def current_obj(self):
+        """ Returns the current working object.
+
+        :returns nginx.config.Block: object that we're currently scoped to
+        """
+        return self.config_builder._cwo
+
+    @property
+    def config_builder(self):
+        """ Internal config builder.
+
+        :returns nginx.builder.ConfigBuilder: the internal ConfigBuilder for manipulating the nginx config
+        """
+        return self._config_builder
+
+    def up(self):
+        """ Traverse up the config hierarchy """
+        self.chobj(self.current_obj.parent)
+
+    def add_child(self, child):
+        """ Adds a child to the config object
+
+        :param nginx.config.Builder child: child to insert into config tree
+        """
+        name = re.split(r'\s+', self.current_obj.name)[0]
+        if self.valid_cfg_parents and name not in self.valid_cfg_parents:
+            raise ConfigBuilderException(
+                '{parent} is not a valid parent for this plugin. Call this off of one of these: {valid_parents}'.format(
+                    parent=name if name else 'top',
+                    valid_parents=self.valid_cfg_parents
+                ), plugin=self._get_name()
+            )
+
+        self.current_obj.sections.add(child)
+
+    def __getattr__(self, attr):
+        return getattr(self.config_builder, attr)
+
+    @abstractproperty
+    def valid_cfg_parents(self):
+        return None
+
+    @property
+    def parent(self):
+        return self._parent
+
+    def _get_name(self):
+        return getattr(self, 'name', self.current_obj.name)
+
+
+@six.add_metaclass(ABCMeta)
+class Plugin(Navigable):
+    """ Plugin base class. All plugins must inherit from this
+
+    Defines a few properties that must be defined:
+
+    name - name of the plugin. must be unique per config builder
+    exported_methods - dict of method names -> callables. method names don't need to match
+        callables. exported method names must be unique
+    """
+
+    @abstractproperty
+    def name(self):
+        return ''
+
+    @abstractproperty
+    def exported_methods(self):
+        return {}
+
+    @property
+    def http(self):
+        return self.config_builder._http
+
+    def __str__(self):
+        return self.name
+
+
+@six.add_metaclass(ABCMeta)
+class Endable(Navigable):
+    """ Role that adds an `end` convenience method to close scoped blocks (location, server, et al) """
+
+    def end(self):
+        self.up()
+        return self.config_builder
+
+
+@six.add_metaclass(ABCMeta)
+class Routable(Navigable):
+    """ A thing that can have routes attached to it.
+
+    In nginx, routes can either be attached to server blocks or other routes
+
+    """
+
+    def add_route(self, path, *args, **kwargs):
+        loc = Location(path, *args, **kwargs)
+        self.add_child(loc)
+        self.chobj(loc)
+
+        return RouteWrapper(self.current_obj, self.config_builder)
+
+
+@six.add_metaclass(ABCMeta)
+class Wrapper(object):
+    def __getattr__(self, attr):
+        return getattr(self.config_builder, attr)
+
+
+class RouteWrapper(Routable, Wrapper, Endable):
+    """ This needs to wrap routes emitted by this interface since we can nest routes in nginx configs.
+
+    This also enables users to use the returned routes/servers as a context manager, so that we can
+    sugar-coat syntax even more than we already do
+
+    """
+    valid_cfg_parents = ('server', 'location')
+
+    def __init__(self, wrapped, config_builder):
+        super(RouteWrapper, self).__init__(
+            parent=wrapped,
+            config_builder=config_builder
+        )
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, typ, val, tb):
+        self.end()
+
+
+class RoutePlugin(Plugin, Routable, Endable):
+    """ A plugin that creates routes
+
+    Routes can be nested infinitely.
+
+    Must be called off of either a server or a location block
+    """
+    name = 'route'
+    valid_cfg_parents = ('location', 'server')
+
+    @property
+    def exported_methods(self):
+        return {
+            'add_route': self.add_route,
+        }
+
+
+class ServerPlugin(Plugin, Routable, Endable):
+    """ A plugin that creates server blocks
+
+    Must only be called off of an http block
+    """
+    name = 'server'
+    valid_cfg_parents = ('http',)
+
+    # XXX: add more server options
+    def add_server(self, hostname='_', **kwargs):
+        server = Block('server', server_name=hostname, **kwargs)
+        self.add_child(server)
+        self.chobj(server)
+
+        return RouteWrapper(self.current_obj, self.config_builder)
+
+    @property
+    def exported_methods(self):
+        return {
+            'add_server': self.add_server,
+            'end': self.end
+        }

sysadmin/nginx/config/builder/exceptions.py

@@ -0,0 +1,63 @@
+class ConfigBuilderException(BaseException):
+    """
+    Top-level exception for config builder exceptions.
+
+    :param Plugin plugin: plugin that caused the problem
+    :param NginxConfigBuilder builder: current config builder
+    :param str msg: message describing the problem
+    """
+    def __init__(self, msg, **kwargs):
+        self.plugin = kwargs.get('plugin', None)
+        self.cfg = kwargs.get('builder', None)
+        self.msg = msg
+        super(ConfigBuilderException, self).__init__(msg)
+
+    def __str__(self):
+        return '{plugin}: {msg}'.format(plugin=str(self.plugin), msg=self.msg)
+
+
+class ConfigBuilderConflictException(ConfigBuilderException):
+    """
+    Two plugins have conflicting plugins
+
+    :param Plugin loaded_plugin: plugin that's already been added to config builder
+    :param str method_name: name of the method that exists in both plugins
+    """
+    def __init__(self, **kwargs):
+        self.loaded_plugin = kwargs.get('loaded_plugin', None)
+        self.method_name = kwargs.get('method_name', None)
+        super(ConfigBuilderConflictException, self).__init__('we override __str__', **kwargs)
+
+    def __str__(self):
+        repr_string = (
+            "Method `{method_name}` from `{plugin}` conflicts with a "
+            "method of the same name loaded from `{loaded_plugin}`"
+        )
+
+        kwargs = dict(
+            plugin=str(self.plugin),
+            loaded_plugin=str(self.loaded_plugin),
+            method_name=self.method_name
+        )
+
+        return repr_string.format(**kwargs)
+
+
+class ConfigBuilderNoSuchMethodException(ConfigBuilderException, AttributeError):
+    """
+    Exception raised when a user tries to call a non-existent method
+
+    :param str attr: name of the attribute that the user attempted to call
+    """
+    def __init__(self, attr, **kwargs):
+        self.attr = attr
+        super(ConfigBuilderNoSuchMethodException, self).__init__('', **kwargs)
+
+    def __str__(self):
+        errstr = 'No plugins provide method {method}'.format(method=self.attr)
+
+        if self.builder is not None:
+            methods = '\n\t'.join(self.cfg._methods.keys)
+            errstr = errstr + '\nExported methods:{methods}\n\t'.format(methods=methods)
+
+        return errstr

sysadmin/nginx/config/builder/plugins.py

@@ -0,0 +1,99 @@
+from .baseplugins import Plugin
+from ..api import KeyValueOption, EmptyBlock
+
+from abc import ABCMeta, abstractproperty
+from enum import Enum, unique
+import six
+
+
+class _ValEnum(Enum):
+    """Enum that yields its value on stringification"""
+    def __str__(self):
+        return str(self.value)
+
+
+@unique
+class CacheUseStale(_ValEnum):
+    """
+    enum for possible options to be passed to *_use_stale. enum for safety
+    (preventing invalid options) as well as documentation and reuse
+    """
+
+    error = 'error'
+    timeout = 'timeout'
+    invalid_header = 'invalid_header'
+    updating = 'updating'
+    http_500 = 'http_500'
+    http_503 = 'http_503'
+    http_403 = 'http_403'
+    http_404 = 'http_404'
+    off = 'off'
+
+
+@six.add_metaclass(ABCMeta)
+class CacheRoutePlugin(Plugin):
+    """
+    Route caching superclass
+
+    All caching plugins can inherit off of this, by setting a `cache_prefix` property
+    that gets prepended to each nginx directive that gets set. This automatically sets
+    the top-level cache directives that need to be set, and when called off of a route,
+    it turns caching on for that route for the duration specified in the arguments
+    """
+
+    invalid = ()
+    valid_cfg_parents = ('location',)
+
+    @abstractproperty
+    def cache_prefix(self):
+        pass
+
+    def _set_cache_option(self, opt, val):
+        cp = "{cache_prefix}_".format(cache_prefix=self.cache_prefix)
+        if opt not in self.invalid and val:
+            self.config_builder.top.options[cp + opt] = val
+
+    def cache_route(self, cache_key='$request_uri', ignore_headers=None,
+                    cache_min_uses=1, cache_bypass='$nocache',
+                    cache_use_stale=CacheUseStale.off, cache_valid=None,
+                    cache_convert_head=None):
+        cp = "{cache_prefix}_".format(cache_prefix=self.cache_prefix)
+
+        # set the options directly on the route now
+        self.add_child(EmptyBlock(
+            *tuple(KeyValueOption(cp + 'cache_valid', value='{k} {v}'.format(k=k, v=v))
+                   for (k, v) in cache_valid.items())
+        ))
+
+        # XXX: check if these are set the first time
+        # add options to top
+        self._set_cache_option('cache_key', cache_key)
+        self._set_cache_option('cache_min_uses', cache_min_uses)
+        self._set_cache_option('cache_bypass', cache_bypass)
+        self._set_cache_option('cache_use_stale', cache_use_stale)
+        self._set_cache_option('cache_convert_head', cache_convert_head)
+
+        if 'add_header' not in self.config_builder.top.options:
+            self.config_builder.top.options['add_header'] = []
+        self.config_builder.top.options['add_header'].extend(['X-Cache-Status', '$upstream_cache_status'])
+
+        return self
+
+
+class UWSGICacheRoutePlugin(CacheRoutePlugin):
+    name = 'cache uwsgi route'
+    cache_prefix = 'uwsgi'
+    invalid = ('cache_convert_head',)
+
+    @property
+    def exported_methods(self):
+        return {'cache_uwsgi_route': self.cache_route}
+
+
+class ProxyCacheRoutePlugin(CacheRoutePlugin):
+    name = 'cache proxy route'
+    cache_prefix = 'proxy'
+
+    @property
+    def exported_methods(self):
+        return {'cache_proxy_route': self.cache_route}