Add a copy of the nginx builder module

sysadmin/nginx/__init__.py

@@ -0,0 +1,5 @@
+try:
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

sysadmin/nginx/config/__init__.py

@@ -0,0 +1 @@
+# this is a package

sysadmin/nginx/config/api/__init__.py

@@ -0,0 +1,65 @@
+"""
+The Block API provides objects to programatically generate nginx configurations.
+
+Example::
+
+    >>> from nginx.config.api import Config, Section, Location
+    >>> events = Section('events', worker_connections='1024')
+    >>> http = Section('http', include='../conf/mime.types')
+    >>> http.sections.add(
+    ...     Section(
+    ...         'server',
+    ...         Location(
+    ...             '/foo',
+    ...             proxy_pass='upstream',
+    ...         ),
+    ...         server_name='_',
+    ...     )
+    ... )
+    >>> nginx = Config(
+    ...     events,
+    ...     http,
+    ...     worker_processes='auto',
+    ...     daemon='on',
+    ...     error_log='var/error.log',
+    ... )
+    >>> print(nginx)
+
+    error_log var/error.log;
+    worker_processes auto;
+    daemon on;
+    http {
+        include ../conf/mime.types;
+        server {
+            server_name _;
+            location /foo {
+                proxy_pass upstream;
+            }
+        }
+    }
+    events {
+        worker_connections 1024;
+    }
+
+.. The objects in this submodule are largely inspired by code found in https://github.com/FeroxTL/pynginxconfig-new.
+
+"""
+from .blocks import EmptyBlock, Block, Location
+from .options import Comment, KeyOption, KeyValueOption, KeyMultiValueOption
+
+__all__ = [
+    'EmptyBlock',
+    'Block',
+    'Location',
+    'KeyOption',
+    'KeyValueOption',
+    'KeyValueMultilines',
+    'KeyMultiValueOption',
+    'Comment',
+    'Config',
+    'Section'
+]
+
+# aliases
+Config = EmptyBlock
+Section = Block

sysadmin/nginx/config/api/base.py

@@ -0,0 +1,22 @@
+class Base(object):
+    """ This is the base class for all blocks and options. """
+    _indent_level = 0
+    _indent_char = ' '
+    _indent = 4
+    _parent = None
+
+    def _get_indent(self):
+        return self._indent_char * self._indent * self._indent_level
+
+    def _render(self, name):
+        return '\n{indent}{name}'.format(
+            name=name,
+            indent=self._get_indent()
+        )
+
+    def __str__(self):
+        return str(self.__repr__())
+
+    @property
+    def parent(self):
+        return self._parent

sysadmin/nginx/config/api/blocks.py

@@ -0,0 +1,121 @@
+import six
+
+from .base import Base
+from .options import AttrDict, AttrList, KeyOption, KeyValueOption, KeyMultiValueOption
+
+
+class Block(Base):
+    """ A block represent a named section of an Nginx config, such as 'http', 'server' or 'location'
+
+    Using this object is as simple as providing a name and any sections or options,
+    which can be other Block objects or option objects.
+
+    Example::
+
+        >>> from nginx.config.api import Block
+        >>> http = Block('http', option='value')
+        >>> print(http)
+
+            http {
+                option value;
+            }
+
+
+    """
+    def __init__(self, name, *sections, **options):
+        """ Creates a block.
+
+        Sections should be config objects such as Block or EmptyBlock,
+        Options can be any key/value pair (such as worker_connections=512, etc)
+
+        :param name str: The name of the block.
+        """
+        self.name = name
+        self.sections = AttrList(self)
+        self.options = AttrDict(self)
+
+        self._set_directives(*sections, **options)
+
+    @property
+    def _directives(self):
+        dirs = self._dump_options() + list(self.sections)
+        return [directive for directive in dirs if directive is not self]
+
+    def _set_directives(self, *sections, **options):
+        for section in sections:
+            self.sections.append(section)
+        for key, value in six.iteritems(options):
+            setattr(self.options, key, value)
+
+    def _build_options(self, key, value):
+        if isinstance(value, Block):
+            option = value
+        elif isinstance(value, list):
+            option = KeyMultiValueOption(key, value=value)
+        elif value is None or value == '':
+            option = KeyOption(key)
+        else:
+            if isinstance(value, str) and ' ' in value:
+                option = KeyMultiValueOption(key, value=value.split())
+            else:
+                option = KeyValueOption(key, value=value)
+        return option
+
+    def _dump_options(self):
+        return [self._build_options(key, value) for key, value in six.iteritems(self.options)]
+
+    def __repr__(self):
+        directives = self._directives
+
+        for directive in directives:
+            if directive is not self:
+                directive._indent_level = self._indent_level + 1
+
+        return '\n{indent}{name}{{{directives}\n{indent}}}'.format(
+            name='{0} '.format(self.name),
+            directives=''.join([repr(e) for e in directives]),
+            indent=self._get_indent(),
+        )
+
+
+class EmptyBlock(Block):
+    """ An unnamed block of options and/or sections.
+
+    Empty blocks are useful for representing groups of options.
+
+    For example, you can use them to represent options with non-unique keys:
+
+    Example::
+
+        >>> from nginx.config.helpers import duplicate_options
+        >>> dupes = duplicate_options('key', ['value', 'other_value', 'third_value'])
+        >>> type(dupes)
+        nginx.config.api.blocks.EmptyBlock
+        >>> print(dupes)
+
+        key third_value;
+        key value;
+        key other_value;
+
+
+    """
+    def __init__(self, *sections, **options):
+        """ Create an EmptyBlock. """
+        self.sections = AttrList(self)
+        self.options = AttrDict(self)
+
+        self._set_directives(*sections, **options)
+
+    def __repr__(self):
+        directives = self._directives
+
+        for directive in directives:
+            directive._indent_level = self._indent_level
+
+        return ''.join([repr(o) for o in directives])
+
+
+class Location(Block):
+    """ A Location is just a named block with "location" prefixed """
+    def __init__(self, location, *args, **kwargs):
+        super(Location, self).__init__('location {0}'.format(location), *args, **kwargs)

sysadmin/nginx/config/api/options.py

@@ -0,0 +1,139 @@
+from .base import Base
+
+
+class KeyOption(Base):
+    """ A KeyOption represents a directive with no value.
+
+    For example: http://nginx.org/en/docs/http/ngx_http_core_module.html#internal
+
+    """
+    def __init__(self, name):
+        self.name = self.value = name
+
+    def __repr__(self):
+        return self._render(
+            '{name};'.format(
+                name=self.name,
+            )
+        )
+
+
+class KeyValueOption(Base):
+    """ A key/value directive. This covers most directives available for Nginx """
+    def __init__(self, name, value=''):
+        self.name = name
+        if isinstance(value, bool):
+            self.value = 'off' if value is False else 'on'
+        elif isinstance(value, int):
+            self.value = str(value)
+        elif isinstance(value, list):
+            self.value = [str(e) for e in value]
+        else:
+            self.value = value
+
+    def __repr__(self):
+        return self._render(
+            '{name} {value};'.format(
+                name=self.name,
+                value=self.value
+            )
+        )
+
+
+class KeyMultiValueOption(KeyValueOption):
+    """ A key with multiple space delimited options.
+
+    For example: http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log
+
+    Example::
+
+        >>> from nginx.config.api.options import KeyMultiValueOption
+        >>> a_log = KeyMultiValueOption('access_log', ['/path/to/log.gz', 'combined', 'gzip', 'flush=5m'])
+        >>> print(a_log)
+
+        access_log /path/to/log.gz combined gzip flush=5m;
+
+    """
+    def __repr__(self):
+        return self._render(
+            '{name} {value};'.format(
+                name=self.name,
+                value=' '.join(self.value)
+            )
+        )
+
+
+class KeyValuesMultiLines(Base):
+    def __init__(self, name, values=[]):
+        self.name = name
+        self.lines = []
+        for value in values:
+            if isinstance(value, list):
+                self.lines.append(' '.join([str(v) for v in value]))
+            else:
+                self.lines.append(str(value))
+
+    def __repr__(self):
+        return ''.join(
+            [self._render('{name} {value};'.format(name=self.name, value=line)) for line in self.lines]
+        )
+
+
+class Comment(Base):
+    """ A simple comment object. """
+
+    _offset = ''
+    _comment = ''
+
+    def __init__(self, offset='', comment='', **kwargs):
+        self._offset = offset
+        self._comment = comment
+        super(Comment, self).__init__(**kwargs)
+
+    def __repr__(self):
+        return self._render(
+            '{offset}# {comment}'.format(
+                offset=self._offset,
+                comment=self._comment,
+            )
+        )
+
+
+class AttrDict(dict):
+    """ A dictionary that exposes it's values as attributes. """
+    def __init__(self, owner):
+        self.__dict__ = self
+        self._owner = owner
+
+    def __setitem__(self, key, val):
+        if hasattr(val, '_parent'):
+            val._parent = self._owner
+        return super(AttrDict, self).__setitem__(key, val)
+
+    def __repr__(self):
+        owner = self.pop('_owner')
+        ret = super(AttrDict, self).__repr__()
+        self._owner = owner
+        return ret
+
+
+class AttrList(AttrDict):
+    """ A dictionary/list hybrid that exposes values as attributes. """
+    def __iter__(self):
+        return iter(self.values())
+
+    def append(self, item):
+        if hasattr(item, '_parent'):
+            item._parent = self._owner
+        if hasattr(item, 'name'):
+            self[item.name] = item
+        else:
+            self[hash(item)] = item
+
+    def add(self, *items):
+        for item in items:
+            self.append(item)
+
+
+# alias for backwards compatibility
+KeyValuesMultilines = KeyValuesMultiLines

sysadmin/nginx/config/builder/__init__.py

@@ -0,0 +1,210 @@
+"""
+The Builder API defines a pluggable builder framework for manipulating nginx configs from within python.
+
+Building a config
+=================
+
+Every config built using the builder pattern starts off with creating a :class:NginxConfigBuilder::
+
+    from nginx.config.builder import NginxConfigBuilder
+
+    nginx = NginxConfigBuilder()
+
+By default, this comes loaded with a bunch of helpful tools to easily create routes and servers
+in nginx::
+
+    with nginx.add_server() as server:
+        server.add_route('/foo').end()
+        with server.add_route('/bar') as bar:
+             bar.add_route('/baz')
+
+This generates a simple config that looks like this::
+
+    error_log logs/nginx.error.log;
+    worker_processes auto;
+    daemon on;
+    http {
+        include ../conf/mime.types;
+        server {
+            server_name _;
+            location /foo {
+            }
+            location /bar {
+                location /baz {
+                }
+            }
+        }
+    }
+    events {
+        worker_connections 1024;
+    }
+
+Plugins
+=======
+
+A plugin is a class that inherits from :class:`nginx.config.builder.baseplugins.Plugin` that provides
+additional methods which can be chained off of the :class:`NginxConfigBuilder` object. These plugins provide
+convenience methods that manipulate the underlying nginx configuration that gets built by the
+:class:`NginxConfigBuilder`.
+
+A simple plugin only needs to define what methods it's going to export::
+
+    class NoopPlugin(Plugin):
+        name = 'noop'
+
+        @property
+        def exported_methods(self):
+             return {'noop': self.noop}
+
+        def noop(self):
+             pass
+
+This NoopPlugin provides a simple function that can be called off of a :class:`NginxConfigBuilder` that
+does nothing successfully. More complex plugins can be found in :mod:`nginx.config.builder.plugins`
+
+To use this NoopPlugin, we need to create a config builder and then register the plugin with it::
+
+    nginx = NginxConfigBuilder()
+    nginx.noop()  # AttributeError :(
+    nginx.register_plugin(NoopPlugin())
+    nginx.noop()  # it works!
+
+A more complex plugin would actually do something, like a plugin that adds an expiry directive to
+a route::
+
+    class ExpiryPlugin(Plugin):
+        name = 'expiry'
+        @property
+
+"""
+
+from ..api import EmptyBlock, Block, Config
+from .exceptions import ConfigBuilderConflictException, ConfigBuilderException, ConfigBuilderNoSuchMethodException
+from .baseplugins import RoutePlugin, ServerPlugin, Plugin
+
+
+DEFAULT_PLUGINS = (RoutePlugin, ServerPlugin)
+INVALID_PLUGIN_NAMES = ('top,')
+
+
+class NginxConfigBuilder(object):
+    """ Helper that builds a working nginx configuration
+
+    Exposes a plugin-based architecture for generating nginx configurations.
+
+    """
+
+    def __init__(self, worker_processes='auto', worker_connections=512, error_log='logs/error.log', daemon='off'):
+        """
+        :param worker_processes str|int: number of worker processes to start with (default: auto)
+        :param worker_connections int: number of nginx worker connections (default: 512)
+        :param error_log str: path to nginx error log (default: logs/error.log)
+        :param daemon str: whether or not to daemonize nginx (default: on)
+        """
+
+        self.plugins = []
+        self._top = EmptyBlock(
+            worker_processes=worker_processes,
+            error_log=error_log,
+            daemon=daemon,
+        )
+
+        self._http = Block(
+            'http',
+            include='../conf/mime.types'
+        )
+
+        self._cwo = self._http
+        self._events = Block(
+            'events',
+            worker_connections=worker_connections
+        )
+
+        self._methods = {}
+
+        for plugin in DEFAULT_PLUGINS:
+            self.register_plugin(plugin(parent=self._http))
+
+    def _validate_plugin(self, plugin):
+        if not isinstance(plugin, Plugin):
+            raise ConfigBuilderException(
+                "Must be a subclass of {cls}".format(cls=Plugin.__name__),
+                plugin=plugin
+            )
+
+        if plugin.name in INVALID_PLUGIN_NAMES:
+            raise ConfigBuilderException(
+                "{name} is a protected name and cannot be used as the name of"
+                " a plugin".format(name=plugin.name),
+                plugin=plugin
+            )
+
+        if plugin.name in (loaded.name for loaded in self.plugins):
+            raise ConfigBuilderConflictException(plugin=plugin, loaded_plugin=plugin, method_name='name')
+
+        methods = plugin.exported_methods.items()
+
+        # check for conflicts
+        for loaded_plugin in self.plugins:
+            for (name, method) in methods:
+                if name in loaded_plugin.exported_methods:
+                    raise ConfigBuilderConflictException(
+                        plugin=plugin,
+                        loaded_plugin=loaded_plugin,
+                        method_name=name
+                    )
+
+                # Also protect register_plugin, etc.
+                if hasattr(self, name):
+                    raise ConfigBuilderConflictException(
+                        plugin=plugin,
+                        loaded_plugin='top',
+                        method_name=name
+                    )
+
+        # we can only be owned once
+        if plugin._config_builder:
+            raise ConfigBuilderException("Already owned by another NginxConfigBuilder", plugin=plugin)
+        plugin._config_builder = self
+
+    def register_plugin(self, plugin):
+        """ Registers a new nginx builder plugin.
+
+        Plugins must inherit from nginx.builder.baseplugins.Plugin and not expose methods that conflict
+        with already loaded plugins
+
+        :param plugin nginx.builder.baseplugins.Plugin: nginx plugin to add to builder
+        """
+
+        self._validate_plugin(plugin)
+
+        # insert ourselves as the config builder for plugins
+        plugin._config_builder = self
+        self.plugins.append(plugin)
+
+        self._methods.update(plugin.exported_methods)
+
+    @property
+    def top(self):
+        """ Returns the logical top of the config hierarchy.
+
+        This is a convenience method for any plugins that need to quickly access the top of the config tree.
+
+        :returns :class:`nginx.config.Block`: Top of the config block
+        """
+        return self._http
+
+    def __getattr__(self, attr):
+        # Since we want this to be easy to use, we will do method lookup
+        # on the methods that we've gotten from our different config plugins
+        # whenever someone tries to call a method off of the builder.
+        #
+        # This means that plugins can just return a reference to the builder
+        # so that users can just chain methods off of the builder.
+        try:
+            return self._methods[attr]
+        except KeyError:
+            raise ConfigBuilderNoSuchMethodException(attr, builder=self)
+
+    def __repr__(self):
+        return repr(Config(self._top, self._events, self._http))

sysadmin/nginx/config/builder/baseplugins.py

@@ -0,0 +1,206 @@
+import re
+import six
+
+from abc import ABCMeta, abstractproperty
+
+from ..api import Block, Location
+from .exceptions import ConfigBuilderException
+
+
+@six.add_metaclass(ABCMeta)
+class Navigable(object):
+    """ Indicates that a class is navigable.
+
+    This means that it references some type of nginx config machinery,
+    and that this is traversable.
+
+    """
+    _config_builder = None
+
+    def __init__(self, *args, **kwargs):
+        """ Creates a new Navigable class
+
+        :param nginx.builder.ConfigBuilder config_builder: internal ConfigBuilder used to create nginx config objs
+        """
+        # This can sometimes be added by direct access,
+        # in the case of plugins
+        self._config_builder = kwargs.get('config_builder', None)
+        self._parent = kwargs.get('parent', None)
+
+    def chobj(self, obj):
+        """ Changes the current working object to the one provided.
+
+        :param nginx.config.Block obj: object that we're scoping to
+        """
+        self.config_builder._cwo = obj
+
+    @property
+    def current_obj(self):
+        """ Returns the current working object.
+
+        :returns nginx.config.Block: object that we're currently scoped to
+        """
+        return self.config_builder._cwo
+
+    @property
+    def config_builder(self):
+        """ Internal config builder.
+
+        :returns nginx.builder.ConfigBuilder: the internal ConfigBuilder for manipulating the nginx config
+        """
+        return self._config_builder
+
+    def up(self):
+        """ Traverse up the config hierarchy """
+        self.chobj(self.current_obj.parent)
+
+    def add_child(self, child):
+        """ Adds a child to the config object
+
+        :param nginx.config.Builder child: child to insert into config tree
+        """
+        name = re.split(r'\s+', self.current_obj.name)[0]
+        if self.valid_cfg_parents and name not in self.valid_cfg_parents:
+            raise ConfigBuilderException(
+                '{parent} is not a valid parent for this plugin. Call this off of one of these: {valid_parents}'.format(
+                    parent=name if name else 'top',
+                    valid_parents=self.valid_cfg_parents
+                ), plugin=self._get_name()
+            )
+
+        self.current_obj.sections.add(child)
+
+    def __getattr__(self, attr):
+        return getattr(self.config_builder, attr)
+
+    @abstractproperty
+    def valid_cfg_parents(self):
+        return None
+
+    @property
+    def parent(self):
+        return self._parent
+
+    def _get_name(self):
+        return getattr(self, 'name', self.current_obj.name)
+
+
+@six.add_metaclass(ABCMeta)
+class Plugin(Navigable):
+    """ Plugin base class. All plugins must inherit from this
+
+    Defines a few properties that must be defined:
+
+    name - name of the plugin. must be unique per config builder
+    exported_methods - dict of method names -> callables. method names don't need to match
+        callables. exported method names must be unique
+    """
+
+    @abstractproperty
+    def name(self):
+        return ''
+
+    @abstractproperty
+    def exported_methods(self):
+        return {}
+
+    @property
+    def http(self):
+        return self.config_builder._http
+
+    def __str__(self):
+        return self.name
+
+
+@six.add_metaclass(ABCMeta)
+class Endable(Navigable):
+    """ Role that adds an `end` convenience method to close scoped blocks (location, server, et al) """
+
+    def end(self):
+        self.up()
+        return self.config_builder
+
+
+@six.add_metaclass(ABCMeta)
+class Routable(Navigable):
+    """ A thing that can have routes attached to it.
+
+    In nginx, routes can either be attached to server blocks or other routes
+
+    """
+
+    def add_route(self, path, *args, **kwargs):
+        loc = Location(path, *args, **kwargs)
+        self.add_child(loc)
+        self.chobj(loc)
+
+        return RouteWrapper(self.current_obj, self.config_builder)
+
+
+@six.add_metaclass(ABCMeta)
+class Wrapper(object):
+    def __getattr__(self, attr):
+        return getattr(self.config_builder, attr)
+
+
+class RouteWrapper(Routable, Wrapper, Endable):
+    """ This needs to wrap routes emitted by this interface since we can nest routes in nginx configs.
+
+    This also enables users to use the returned routes/servers as a context manager, so that we can
+    sugar-coat syntax even more than we already do
+
+    """
+    valid_cfg_parents = ('server', 'location')
+
+    def __init__(self, wrapped, config_builder):
+        super(RouteWrapper, self).__init__(
+            parent=wrapped,
+            config_builder=config_builder
+        )
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, typ, val, tb):
+        self.end()
+
+
+class RoutePlugin(Plugin, Routable, Endable):
+    """ A plugin that creates routes
+
+    Routes can be nested infinitely.
+
+    Must be called off of either a server or a location block
+    """
+    name = 'route'
+    valid_cfg_parents = ('location', 'server')
+
+    @property
+    def exported_methods(self):
+        return {
+            'add_route': self.add_route,
+        }
+
+
+class ServerPlugin(Plugin, Routable, Endable):
+    """ A plugin that creates server blocks
+
+    Must only be called off of an http block
+    """
+    name = 'server'
+    valid_cfg_parents = ('http',)
+
+    # XXX: add more server options
+    def add_server(self, hostname='_', **kwargs):
+        server = Block('server', server_name=hostname, **kwargs)
+        self.add_child(server)
+        self.chobj(server)
+
+        return RouteWrapper(self.current_obj, self.config_builder)
+
+    @property
+    def exported_methods(self):
+        return {
+            'add_server': self.add_server,
+            'end': self.end
+        }

sysadmin/nginx/config/builder/exceptions.py

@@ -0,0 +1,63 @@
+class ConfigBuilderException(BaseException):
+    """
+    Top-level exception for config builder exceptions.
+
+    :param Plugin plugin: plugin that caused the problem
+    :param NginxConfigBuilder builder: current config builder
+    :param str msg: message describing the problem
+    """
+    def __init__(self, msg, **kwargs):
+        self.plugin = kwargs.get('plugin', None)
+        self.cfg = kwargs.get('builder', None)
+        self.msg = msg
+        super(ConfigBuilderException, self).__init__(msg)
+
+    def __str__(self):
+        return '{plugin}: {msg}'.format(plugin=str(self.plugin), msg=self.msg)
+
+
+class ConfigBuilderConflictException(ConfigBuilderException):
+    """
+    Two plugins have conflicting plugins
+
+    :param Plugin loaded_plugin: plugin that's already been added to config builder
+    :param str method_name: name of the method that exists in both plugins
+    """
+    def __init__(self, **kwargs):
+        self.loaded_plugin = kwargs.get('loaded_plugin', None)
+        self.method_name = kwargs.get('method_name', None)
+        super(ConfigBuilderConflictException, self).__init__('we override __str__', **kwargs)
+
+    def __str__(self):
+        repr_string = (
+            "Method `{method_name}` from `{plugin}` conflicts with a "
+            "method of the same name loaded from `{loaded_plugin}`"
+        )
+
+        kwargs = dict(
+            plugin=str(self.plugin),
+            loaded_plugin=str(self.loaded_plugin),
+            method_name=self.method_name
+        )
+
+        return repr_string.format(**kwargs)
+
+
+class ConfigBuilderNoSuchMethodException(ConfigBuilderException, AttributeError):
+    """
+    Exception raised when a user tries to call a non-existent method
+
+    :param str attr: name of the attribute that the user attempted to call
+    """
+    def __init__(self, attr, **kwargs):
+        self.attr = attr
+        super(ConfigBuilderNoSuchMethodException, self).__init__('', **kwargs)
+
+    def __str__(self):
+        errstr = 'No plugins provide method {method}'.format(method=self.attr)
+
+        if self.builder is not None:
+            methods = '\n\t'.join(self.cfg._methods.keys)
+            errstr = errstr + '\nExported methods:{methods}\n\t'.format(methods=methods)
+
+        return errstr

sysadmin/nginx/config/builder/plugins.py

@@ -0,0 +1,99 @@
+from .baseplugins import Plugin
+from ..api import KeyValueOption, EmptyBlock
+
+from abc import ABCMeta, abstractproperty
+from enum import Enum, unique
+import six
+
+
+class _ValEnum(Enum):
+    """Enum that yields its value on stringification"""
+    def __str__(self):
+        return str(self.value)
+
+
+@unique
+class CacheUseStale(_ValEnum):
+    """
+    enum for possible options to be passed to *_use_stale. enum for safety
+    (preventing invalid options) as well as documentation and reuse
+    """
+
+    error = 'error'
+    timeout = 'timeout'
+    invalid_header = 'invalid_header'
+    updating = 'updating'
+    http_500 = 'http_500'
+    http_503 = 'http_503'
+    http_403 = 'http_403'
+    http_404 = 'http_404'
+    off = 'off'
+
+
+@six.add_metaclass(ABCMeta)
+class CacheRoutePlugin(Plugin):
+    """
+    Route caching superclass
+
+    All caching plugins can inherit off of this, by setting a `cache_prefix` property
+    that gets prepended to each nginx directive that gets set. This automatically sets
+    the top-level cache directives that need to be set, and when called off of a route,
+    it turns caching on for that route for the duration specified in the arguments
+    """
+
+    invalid = ()
+    valid_cfg_parents = ('location',)
+
+    @abstractproperty
+    def cache_prefix(self):
+        pass
+
+    def _set_cache_option(self, opt, val):
+        cp = "{cache_prefix}_".format(cache_prefix=self.cache_prefix)
+        if opt not in self.invalid and val:
+            self.config_builder.top.options[cp + opt] = val
+
+    def cache_route(self, cache_key='$request_uri', ignore_headers=None,
+                    cache_min_uses=1, cache_bypass='$nocache',
+                    cache_use_stale=CacheUseStale.off, cache_valid=None,
+                    cache_convert_head=None):
+        cp = "{cache_prefix}_".format(cache_prefix=self.cache_prefix)
+
+        # set the options directly on the route now
+        self.add_child(EmptyBlock(
+            *tuple(KeyValueOption(cp + 'cache_valid', value='{k} {v}'.format(k=k, v=v))
+                   for (k, v) in cache_valid.items())
+        ))
+
+        # XXX: check if these are set the first time
+        # add options to top
+        self._set_cache_option('cache_key', cache_key)
+        self._set_cache_option('cache_min_uses', cache_min_uses)
+        self._set_cache_option('cache_bypass', cache_bypass)
+        self._set_cache_option('cache_use_stale', cache_use_stale)
+        self._set_cache_option('cache_convert_head', cache_convert_head)
+
+        if 'add_header' not in self.config_builder.top.options:
+            self.config_builder.top.options['add_header'] = []
+        self.config_builder.top.options['add_header'].extend(['X-Cache-Status', '$upstream_cache_status'])
+
+        return self
+
+
+class UWSGICacheRoutePlugin(CacheRoutePlugin):
+    name = 'cache uwsgi route'
+    cache_prefix = 'uwsgi'
+    invalid = ('cache_convert_head',)
+
+    @property
+    def exported_methods(self):
+        return {'cache_uwsgi_route': self.cache_route}
+
+
+class ProxyCacheRoutePlugin(CacheRoutePlugin):
+    name = 'cache proxy route'
+    cache_prefix = 'proxy'
+
+    @property
+    def exported_methods(self):
+        return {'cache_proxy_route': self.cache_route}

sysadmin/nginx/config/common.py

@@ -0,0 +1,203 @@
+"""
+This module contains functions and variables that provide a variety of commonly used nginx config
+boilerplate.
+"""
+from . import helpers
+from .api import Block, EmptyBlock, KeyMultiValueOption, KeyValueOption
+from .headers import uwsgi_param
+
+
+def listen_options(port, ipv6_enabled=False):
+    if ipv6_enabled:
+        return KeyMultiValueOption(
+            'listen',
+            ['[::]:{}'.format(port), 'ipv6only=off']
+        )
+    else:
+        return KeyValueOption('listen', port)
+
+
+def listen_options_ssl(port, ipv6_enabled=False):
+    if ipv6_enabled:
+        return KeyMultiValueOption(
+            'listen',
+            ['[::]:{}'.format(port), 'ipv6only=off', 'ssl']
+        )
+    else:
+        return KeyMultiValueOption('listen', [port, 'ssl'])
+
+
+def _uwsgi_params():
+    return helpers.duplicate_options(
+        'uwsgi_param',
+        [
+            [uwsgi_param.QUERY_STRING, '$query_string'],
+            [uwsgi_param.REQUEST_METHOD, '$request_method'],
+            [uwsgi_param.CONTENT_TYPE, '$content_type'],
+            [uwsgi_param.CONTENT_LENGTH, '$content_length'],
+            [uwsgi_param.REQUEST_URI, '$request_uri'],
+            [uwsgi_param.PATH_INFO, '$document_uri'],
+            [uwsgi_param.DOCUMENT_ROOT, '$document_root'],
+            [uwsgi_param.SERVER_PROTOCOL, '$server_protocol'],
+            [uwsgi_param.REMOTE_ADDR, '$remote_addr'],
+            [uwsgi_param.REMOTE_PORT, '$remote_port'],
+            [uwsgi_param.SERVER_ADDR, '$server_addr'],
+            [uwsgi_param.SERVER_PORT, '$server_port'],
+            [uwsgi_param.SERVER_NAME, '$server_name'],
+        ]
+    )
+
+
+def _uwsgi_ssl_params():
+    return helpers.duplicate_options(
+        'uwsgi_param',
+        [
+            [uwsgi_param.CLIENT_SSL_CERT, '$ssl_client_raw_cert'],
+        ]
+    )
+
+
+def _gzip_options():
+    """ These are some decent default settings for gzip compression """
+    return EmptyBlock(
+        **dict(
+            gzip='on',
+            gzip_types='application/json',
+            gzip_comp_level=2,
+            gzip_min_length=1024,
+        )
+    )
+
+
+def _uwsgi_cache():
+    """ A set of useful defaults for using nginx's response cache with uWSGI
+
+    This block of options belongs in your HTTP section.
+
+    NB! you must set "set $nocache 0;" in the Location block of your uwsgi backend.
+
+    see: http://nginx.org/en/docs/http/ngx_http_uwsgi_module.html
+    """
+    return EmptyBlock(
+        **dict(
+            uwsgi_cache_path=[
+                'var/nginx_cache',
+                'keys_zone=one:10m',
+                'loader_threshold=300',
+                'loader_files=200',
+                'max_size=200m'
+            ],
+            uwsgi_cache_key='$request_uri',
+            uwsgi_cache_min_uses=1,
+        )
+    )
+
+
+def _uwsgi_cache_location():
+    """ These are some decent defaults for caching uwsgi responses """
+    cache_options = EmptyBlock()
+    # This is a bit of a hack to deal with the Cache-Control header
+    # normally, the uwsgi nginx module doesn't honor the Cache-Control
+    # header at all. For the cases where a user sends `max-age=0` or
+    # `no-cache`, this will do the right thing and bypass the uwsgi
+    # module's cache. This hack does not handle cases where max-age
+    # is set to something else - it will just use the cache in that
+    # case regardless of age
+    cache_options.sections.add(
+        EmptyBlock(set=['$nocache', '0']),
+        Block(
+            'if ($http_cache_control = "max-age=0")',
+            set=['$nocache', '1']
+        ),
+        Block(
+            'if ($http_cache_control = "no-cache")',
+            set=['$nocache', '1']
+        ),
+        EmptyBlock(uwsgi_cache_valid=['404', '5s']),
+        EmptyBlock(uwsgi_cache_valid=['200', '301', '302', '1d']),
+    )
+
+    return cache_options
+
+
+def _large_buffers():
+    """ These are some larger than default buffer settings.
+
+    Use at your own risk!
+
+    """
+    return EmptyBlock(
+        **dict(
+            client_body_buffer_size='128k',
+            client_max_body_size='10m',
+            client_header_buffer_size='1k',
+            large_client_header_buffers=[4, '4k'],
+            output_buffers=[1, '32k'],
+            postpone_output=1460,
+        )
+    )
+
+
+def _statsd_options_location():
+    """ These are some good defaults to supply to Nginx when using the statsd plugin.
+
+    https://github.com/zebrafishlabs/nginx-statsd
+
+    NB! it requires you to include a "statsd_server" directive in your http section.
+    This set of common directives should go in any Location block.
+
+    """
+    statsd = EmptyBlock()
+    statsd.sections.add(
+        EmptyBlock(statsd_count=['"nginx.requests"', '1']),
+        EmptyBlock(statsd_count=['"nginx.responses.$status"', '1', '"$status"']),
+        EmptyBlock(statsd_timing=['"nginx.request_time"', '"$request_time"']),
+        EmptyBlock(statsd_timing=['"nginx.upstream_response_time"', '"$upstream_response_time"']),
+        EmptyBlock(statsd_count=['"nginx.response_length"', '"$request_length"']),
+        EmptyBlock(statsd_count=['"nginx.bytes_sent"', '"$bytes_sent"']),
+    )
+    return statsd
+
+
+# aliases
+uwsgi_params = _uwsgi_params()
+uwsgi_ssl_params = _uwsgi_ssl_params()
+uwsgi_cache = _uwsgi_cache()
+gzip_options = _gzip_options()
+buffer_options = _large_buffers()
+uwsgi_cache_location = _uwsgi_cache_location()
+statsd_options_location = _statsd_options_location()
+
+
+def user_agent_block(blocklist, return_code=403):
+    return Block(
+        'if ($http_user_agent ~* ({}))'.format('|'.join(blocklist)),
+        **{'return': return_code}
+    )
+
+
+def ratelimit_options(qps):
+    """ Rcreate rate limit shared memory zone, used for tracking different connections.
+
+    :param int|str qps: Queries per second to rate limit.
+    """
+    return EmptyBlock(
+        limit_req_zone=[
+            '$binary_remote_addr',
+            'zone=ratelimit_zone:10m',
+            'rate={qps}r/s'.format(qps=qps),
+        ]
+    )
+
+
+def ratelimit_options_location(burst_qps):
+    """ This needs to be added to a location block in order for that location to get rate limiting
+
+    :param int|str burst_qps: Queries per second to allow bursting to.
+    """
+    return EmptyBlock(
+        limit_req_zone=[
+            'zone=ratelimit_zone',
+            'burst={burst_qps}'.format(burst_qps=burst_qps),
+        ]
+    )

sysadmin/nginx/config/headers/uwsgi_param.py

@@ -0,0 +1,18 @@
+# Generic uwsgi_param headers
+
+CONTENT_LENGTH = 'CONTENT_LENGTH'
+CONTENT_TYPE = 'CONTENT_TYPE'
+DOCUMENT_ROOT = 'DOCUMENT_ROOT'
+QUERY_STRING = 'QUERY_STRING'
+PATH_INFO = 'PATH_INFO'
+REMOTE_ADDR = 'REMOTE_ADDR'
+REMOTE_PORT = 'REMOTE_PORT'
+REQUEST_METHOD = 'REQUEST_METHOD'
+REQUEST_URI = 'REQUEST_URI'
+SERVER_ADDR = 'SERVER_ADDR'
+SERVER_NAME = 'SERVER_NAME'
+SERVER_PORT = 'SERVER_PORT'
+SERVER_PROTOCOL = 'SERVER_PROTOCOL'
+
+# SSL uwsgi_param headers
+CLIENT_SSL_CERT = 'CLIENT_SSL_CERT'

sysadmin/nginx/config/helpers.py

@@ -0,0 +1,85 @@
+"""
+Convienence utilities for building nginx configs
+"""
+from multiprocessing import cpu_count
+
+from .api import Config, Location, Section
+from .api.blocks import EmptyBlock
+
+
+def dumps(config_list):
+    """ Dumps a string representation of a config. Accepts a list of config objects.
+
+    :param list config_list: A list of config objects from this module
+    :rtype: str
+    """
+    return ''.join([str(element) for element in config_list])
+
+
+def duplicate_options(key, values):
+    """ There are many cases when building configs that you may have duplicate keys
+
+    This function will produce an EmptyBlock object with duplicate keys but unique values
+
+    Example::
+
+        from nginx.config.helpers import duplicate_options
+        from nginx.config.api import Location
+        loc = Location(
+            '/',
+            duplicate_options('uwsgi_cache_valid', (['404', '5s'], ['200', '60s'])),
+        )
+
+    Which would produce::
+
+        location / {
+            uwsgi_cache_valid 200 60s;
+            uwsgi_cache_valid 404 5s;
+        }
+
+    """
+    duplicates = EmptyBlock()
+
+    for value in values:
+        duplicates.sections.add(EmptyBlock(**{key: value}))
+
+    return duplicates
+
+
+def simple_configuration(port=8080):
+    """ Returns a simple nginx config.
+
+    Also serves as an example of how to build configs using this module.
+
+    :param int port: A port to populate the 'listen' paramter of the server block
+    :rtype str:
+    """
+
+    http = Section(
+        'http',
+        access_log=['logs/access.log', 'combined'],
+    )
+
+    http.sections.server = Section(
+        'server',
+        Location('/'),
+        listen=port,
+    )
+
+    events = Section(
+        'events',
+        worker_connections=4096
+    )
+
+    top = EmptyBlock(
+        worker_processes=cpu_count(),
+        error_log='logs/error.log'
+    )
+
+    config = Config(
+        top,
+        events,
+        http,
+    )
+
+    return config