@classmethod
def setup(
    cls,
    transform={},
    redirect={},
    methods=None,
    auth: Union[None, bool, dict] = None,
    log=None,
    set_xsrf=None,
    error=None,
    xsrf_cookies=None,
    cors: Union[None, bool, dict] = None,
    ratelimit: Optional[dict] = None,
    # If you add any explicit kwargs here, add them to special_keys too.
    **kwargs,
):
    '''
    One-time setup for all request handlers. This is called only when
    gramex.yaml is parsed / changed.
    '''
    cls._on_init_methods = []
    cls._on_finish_methods = []
    cls._set_xsrf = set_xsrf

    cls.kwargs = cls.conf.get('kwargs', AttrDict())

    cls.setup_transform(transform)
    cls.setup_redirect(redirect)
    # Note: call setup_session before setup_auth to ensure that .session is available.
    # This also ensures we override_user before auth
    cls.setup_session(conf.app.get('session'))
    cls.setup_ratelimit(ratelimit, conf.app.get('ratelimit'))
    cls.setup_auth(auth)
    cls.setup_error(error)
    cls.setup_xsrf(xsrf_cookies)
    cls.setup_log()
    cls.setup_httpmethods(methods)
    cls.setup_cors(cors, auth=auth)

    # app.settings.debug enables debugging exceptions using pdb
    if conf.app.settings.get('debug', False):
        cls.log_exception = cls.debug_exception

@classmethod
def clear_special_keys(cls, kwargs, *args):
    '''
    Remove keys handled by BaseHandler that may interfere with setup().
    This should be called explicitly in setup() where required.
    '''
    for special_key in cls.special_keys:
        kwargs.pop(special_key, None)
    for special_key in args:
        kwargs.pop(special_key, None)
    return kwargs

@classmethod
def get_list(
    cls, val: Union[list, tuple, str], key: str = '', eg: str = '', caps: bool = True
) -> set:
    '''Split comma-separated values into a set.

    Process kwargs that can be a comma-separated string or a list,
    like BaseMixin's `methods:`, `cors.origins`, `cors.methods`, `cors.headers`,
    `ratelimit.keys`, etc.

    Examples:
        >>> get_list('GET, PUT') == {'GET', 'PUT'}
        >>> get_list(['GET', ' ,get '], caps=True) == {'GET'}
        >>> get_list([' GET ,  PUT', ' ,POST, ']) == {'GET', 'PUT', 'POST'}

    Parameters:
        val: Input to split. If val is not str/list/tuple, raise `ValueError`
        key: `url:` key to display in error message
        eg: Example values to display in error message
        caps: True to convert values to uppercase

    Returns:
        Unique comma-separated values
    '''
    if isinstance(val, (list, tuple)):
        val = ' '.join(val)
    elif not val:
        val = ''
    if not isinstance(val, str):
        err = f'url:{cls.name}.{key}: {val!r} not a string/list'
        err = err + f', e.g. {eg}' if eg else err
        raise ValueError(err)
    if caps:
        val = val.upper()
    # Return de-duplicated list. Avoid set(), use dict to preserve order
    return list(dict.fromkeys(val.replace(',', ' ').split()))

def check_http_method(self):
    '''If method: [...] is specified, reject all methods not in the allowed methods set'''
    if self.request.method not in self._http_methods:
        raise HTTPError(
            METHOD_NOT_ALLOWED,
            f'{self.name}: method {self.request.method} '
            + f'not in allowed methods {self._http_methods}',
        )

def check_cors(self):
    '''
    For simple CORS requests, send Access-Control-Allow-Origin: <origin>.
    If request needs credentials, allow it.'''
    origin, cred = self.cors_origin()
    if origin:
        self.set_header('Access-Control-Allow-Origin', origin)
        if cred:
            self.set_header('Access-Control-Allow-Credentials', 'true')

def cors_origin(self):
    '''
    Returns the origin to set in Access-Control-Allow-Origin header.
    '''
    # If CORS is not enabled, it fails
    if not self._cors:
        return None, False
    # Assume credentials are passed if handler requires Auth or Cookie is passed
    cred = self._cors['auth'] or self.request.headers.get('Cookie')
    # If origin: *, then allow all origins
    origin = self.request.headers.get('Origin', '').lower()
    if self._cors['origins'] == set('*'):
        return (origin if cred else '*', cred)
    # If it matches any of the wildcards, return specific origin
    for pattern in self._cors['origins']:
        if fnmatch(origin, pattern.lower()):
            return origin, cred
    # If none of the patterns match, it fails
    return None, cred

@classmethod
def setup_default_kwargs(cls):
    '''
    Use default config from handlers.<Class>.* and handlers.BaseHandler.
    Called directly by gramex.services.url().
    NOTE: This updates the kwargs for setup() -- so it must be called BEFORE setup()
    and can't be merged into setup() or called from setup().
    '''
    c = cls.conf.kwargs
    merge(c, objectpath(conf, 'handlers.' + cls.conf.handler, {}), mode='setdefault')
    merge(c, objectpath(conf, 'handlers.BaseHandler', {}), mode='setdefault')

@classmethod
def setup_session(cls, session_conf):
    '''handler.session returns the session object. It is saved on finish.'''
    if session_conf is None:
        return
    cls._session_store = cls._get_store(session_conf)
    cls.session = property(cls.get_session)
    cls._session_expiry = session_conf.get('expiry')
    cls._session_cookie_id = session_conf.get('cookie', 'sid')
    cls._session_cookie = {
        key: session_conf[key]
        for key in ('httponly', 'secure', 'samesite', 'domain')
        if key in session_conf
    }
    # Note: We cannot use path: to specify the Cookie path attribute.
    # session.path is used for the session (JSONStore) file location.
    # So use cookiepath: instead.
    if 'cookiepath' in session_conf:
        cls._session_cookie['path'] = session_conf['cookiepath']
    cls._on_init_methods.append(cls.override_user)
    cls._on_finish_methods.append(cls.set_last_visited)
    # Ensure that session is saved AFTER we set last visited
    cls._on_finish_methods.append(cls.save_session)

@classmethod
def setup_ratelimit(
    cls, ratelimit: Union[Dict, List[Dict], None], ratelimit_app_conf: Union[dict, None]
):
    '''Initialize rate limiting checks'''
    if ratelimit is None:
        return
    if not ratelimit_app_conf:
        raise ValueError(f"url:{cls.name}.ratelimit: no app.ratelimit defined")

    # All ratelimit related info is stored in self._ratelimit
    cls._ratelimit = []
    cls._on_init_methods.append(cls.check_ratelimit)
    cls._on_finish_methods.append(cls.update_ratelimit)

    for ratelimit_conf in ratelimit if isinstance(ratelimit, (list, tuple)) else [ratelimit]:
        cls._setup_ratelimit(ratelimit_conf, ratelimit_app_conf)

@classmethod
def reset_ratelimit(cls, pool: str, keys: List[Any], value: int = 0) -> bool:
    '''Reset the rate limit usage for a specific pool.

    Examples:
        >>> reset_ratelimit('/api', ['2022-01-01', 'x@example.org'])
        >>> reset_ratelimit('/api', ['2022-01-01', 'x@example.org'], 10)

    Parameters:
        pool: Rate limit pool to use. This is the url's `pattern:` unless you specified a
            `kwargs.ratelimit.pool:`
        keys: specific instance to reset. If your `ratelimit.keys` is `[daily, user.id]`,
            keys might look like `['2022-01-01', 'x@example.org']` to clear for that day/user
        value: sets the usage counter to this number (default: `0`)
    '''
    store = cls._get_store(conf.app.get('ratelimit'))
    key = json.dumps([pool] + keys)
    val = store.load(key, None)
    if val is not None and 'n' in val:
        val['n'] = value
        store.dump(key, val)
    else:
        return False

@classmethod
def setup_redirect(cls, redirect):
    '''
    Any handler can have a `redirect:` kwarg that looks like this:

    ```yaml
    redirect:
        query: next         # If the URL has a ?next=..., redirect to that page next
        header: X-Next      # Else if the header has an X-Next=... redirect to that
        url: ...            # Else redirect to this URL
    ```

    Only these 3 keys are allowed. All are optional, and checked in the
    order specified. So, for example:

    ```yaml
    redirect:
        header: X-Next      # Checks the X-Next header first
        query: next         # If it's missing, uses the ?next=
    ```

    You can also specify a string for redirect. `redirect: ...` is the same
    as `redirect: {url: ...}`.

    When any BaseHandler subclass calls `self.save_redirect_page()`, it
    stores the redirect URL in `session['_next_url']`. The URL is
    calculated relative to the handler's URL.

    After that, when the subclass calls `self.redirect_next()`, it
    redirects to `session['_next_url']` and clears the value. (If the
    `_next_url` was not stored, we redirect to the home page `/`.)

    Only some handlers implement redirection. But they all implement it in
    this same consistent way.
    '''
    # Ensure that redirect is a dictionary before proceeding.
    if isinstance(redirect, str):
        redirect = {'url': redirect}
    if not isinstance(redirect, dict):
        app_log.error(f'url:{cls.name}.redirect must be a URL or a dict, not {redirect!r}')
        return

    cls.redirects = []
    add = cls.redirects.append
    for key, value in redirect.items():
        if key == 'query':
            add(lambda h, v=value: h.get_argument(v, None))
        elif key == 'header':
            add(lambda h, v=value: h.request.headers.get(v))
        elif key == 'url':
            add(lambda h, v=value: v)

    # redirect.external=False disallows external URLs
    if not redirect.get('external', False):

        def no_external(method):
            def redirect_method(handler):
                next_uri = method(handler)
                if next_uri is not None:
                    target = urlsplit(next_uri)
                    if not target.scheme and not target.netloc:
                        return next_uri
                    req = handler.request
                    if req.protocol == target.scheme and req.host == target.netloc:
                        return next_uri
                    app_log.error(f'Not redirecting to external url: {next_uri}')

            return redirect_method

        cls.redirects = [no_external(method) for method in cls.redirects]

def authorize(self):
    '''BaseMixin assumes every handler has an authorize() function'''
    pass

@classmethod
def setup_log(cls):
    '''
    Logs access requests to gramex.requests as a CSV file.
    '''
    logger = logging.getLogger('gramex.requests')
    keys = objectpath(conf, 'log.handlers.requests.keys', [])
    log_info = build_log_info(keys)
    cls.log_request = lambda handler: logger.info(log_info(handler))

@classmethod
def setup_error(cls, error):
    '''
    Sample configuration:

    ```yaml
    error:
        404:
            path: template.json         # Use a template
            autoescape: false           # with no autoescape
            whitespace: single          # as a single line
            headers:
                Content-Type: application/json
        500:
            function: module.fn
            args: [=status_code, =kwargs, =handler]
    ```
    '''
    if not error:
        return
    if not isinstance(error, dict):
        return app_log.error(f'url:{cls.name}.error is not a dict')
    # Compile all errors handlers
    cls.error = {}
    for error_code, error_config in error.items():
        try:
            error_code = int(error_code)
            if error_code < 100 or error_code > 1000:
                raise ValueError()
        except ValueError:
            app_log.error(f'url.{cls.name}.error code {error_code} is not a number (100-1000)')
            continue
        if not isinstance(error_config, dict):
            return app_log.error(f'url:{cls.name}.error.{error_code} is not a dict')
        # Make a copy of the original. When we add headers, etc, it shouldn't affect original
        error_config = AttrDict(error_config)
        error_path, error_function = error_config.get('path'), error_config.get('function')
        if error_function:
            if error_path:
                error_config.pop('path')
                app_log.warning(
                    f'url.{cls.name}.error.{error_code} has function:. Ignoring path:'
                )
            cls.error[error_code] = {
                'function': build_transform(
                    error_config,
                    vars={'status_code': None, 'kwargs': None, 'handler': None},
                    filename=f'url:{cls.name}.error.{error_code}',
                )
            }
        elif error_path:
            encoding = error_config.get('encoding', 'utf-8')
            cls.error[error_code] = {'function': cls._error_fn(error_code, error_config)}
            mime_type, encoding = mimetypes.guess_type(error_path, strict=False)
            if mime_type:
                error_config.setdefault('headers', {}).setdefault('Content-Type', mime_type)
        else:
            app_log.error(f'url.{cls.name}.error.{error_code} must have path: or function:')
        # Add the error configuration for reference
        if error_code in cls.error:
            cls.error[error_code]['conf'] = error_config
    cls._write_error, cls.write_error = cls.write_error, cls._write_custom_error

@classmethod
def setup_xsrf(cls, xsrf_cookies):
    '''
    Sample configuration:

    ```yaml
    xsrf_cookies: false         # Disables xsrf_cookies
    xsrf_cookies: true          # or anything other than false keeps it enabled
    ```
    '''
    cls.check_xsrf_cookie = cls.noop if xsrf_cookies is False else cls.xsrf_ajax

def xsrf_check_required(self):
    '''Returns True if the request is (likely) from a browser and needs XSRF check.

    This is used to handle XSRF. If the request is NOT from a browser (e.g. server, AJAX),
    no AJAX checks are required. If any of the following are true, it's not a browser request.

    1. `X-Requested-With: XMLHttpRequest`. XMLHttpRequest sends this
    2. `Sec-Fetch-Mode: cors`. [Fetch sends these][H-Fetch]

    [H-Fetch]: https://developer.mozilla.org/en-US/docs/Glossary/Fetch_metadata_request_header
    [H-Origin]: https://developer.mozilla.org/en-US/docs/Web/API/Request/mode
    '''
    return not (
        self.request.headers.get('X-Requested-With', '').lower() == 'xmlhttprequest'
        or self.request.headers.get('Sec-Fetch-Mode', '').lower() == 'cors'
    )

def xsrf_ajax(self):
    '''Validates XSRF cookies if it's a browser-request (not AJAX)

    Internally, it uses Tornado's check_xsrf_cookie().
    '''
    if self.xsrf_check_required():
        return super(BaseHandler, self).check_xsrf_cookie()

def noop(self):
    '''Does nothing. Used when overriding functions or providing a dummy operation'''
    pass

def save_redirect_page(self):
    '''
    Loop through all redirect: methods and save the first available redirect
    page against the session. Defaults to previously set value, else `/`.

    See [setup_redirect][gramex.handlers.BaseMixin.setup_redirect].
    '''
    for method in self.redirects:
        next_url = method(self)
        if next_url:
            self.session['_next_url'] = urljoin(self.xrequest_uri, next_url)
            return
    self.session.setdefault('_next_url', '/')

def redirect_next(self):
    '''
    Redirect the user `session['_next_url']`. If it does not exist,
    set it up first. Then redirect.

    See [setup_redirect][gramex.handlers.BaseMixin.setup_redirect].
    '''
    if '_next_url' not in self.session:
        self.save_redirect_page()
    self.redirect(self.session.pop('_next_url', '/'))

def get_session(self, expires_days=None, new=False):
    '''
    Return the session object for the cookie "sid" value.
    If no "sid" cookie exists, set up a new one.
    If no session object exists for the sid, create it.
    By default, the session object contains a "id" holding the "sid" value.

    The session is a dict. You must ensure that it is JSON serializable.

    Sessions use these pre-defined timing keys (values are timestamps):

    - `_t` is the expiry time of the session
    - `_l` is the last time the user accessed a page. Updated by
      [setup_redirect][gramex.handlers.BaseMixin.set_last_visited]
    - `_i` is the inactive expiry duration in seconds, i.e. if `now > _l +
      _i`, the session has expired.

    `new=` creates a new session to avoid session fixation.
    https://www.owasp.org/index.php/Session_fixation.
    [`set_user()`][gramex.handlers.AuthHandler.set_user] uses it.
    When the user logs in:

    - If no old session exists, it returns a new session object.
    - If an old session exists, it creates a new "sid" and new session
      object, copying all old contents, but updates the "id" and expiry (_t).
    '''
    if expires_days is None:
        expires_days = self._session_expiry
    # If the expiry time is None, keep in the session store for 1 day
    store_expires = time.time() + (1 if expires_days is None else expires_days) * 24 * 60 * 60
    created_new_sid = False
    if getattr(self, '_session', None) is None:
        # Populate self._session based on the sid. If there's no sid cookie,
        # generate one and create an associated session object
        session_id = self.get_secure_cookie(self._session_cookie_id, max_age_days=9999999)
        # If there's no session id cookie "sid", create a random 32-char cookie
        if session_id is None:
            session_id = self._set_new_session_id(expires_days)
            created_new_sid = True
        # Convert bytes session to unicode before using
        session_id = session_id.decode('ascii')
        # If there's no stored session associated with it, create it
        self._session = self._session_store.load(session_id, {'_t': store_expires})
        # Overwrite id to the session ID even if a handler has changed it
        self._session['id'] = session_id
    # At this point, the "sid" cookie and self._session exist and are synced
    s = self._session
    old_sid = s['id']
    # If session has expiry keys _i and _l defined, check for expiry. Not otherwise
    if '_i' in s and '_l' in s and time.time() > s['_l'] + s['_i']:
        new = True
        s.clear()
    if new and not created_new_sid:
        new_sid = self._set_new_session_id(expires_days).decode('ascii')
        # Update expiry and new SID on session
        s.update(id=new_sid, _t=store_expires)
        # Delete old contents. No _t also means expired
        self._session_store.dump(old_sid, {})

    return s

def save_session(self):
    '''Persist the session object as a JSON'''
    if getattr(self, '_session', None) is not None:
        self._session_store.dump(self._session['id'], self._session)

def otp(
    self,
    expire: float = 60,
    user: Union[str, dict] = None,
    size: int = None,
    type: str = 'OTP',
    **kwargs: Dict[str, Any],
) -> str:
    '''Return one-time password valid for `expire` seconds.

    The OTP is used as the X-Gramex-OTP header or in `?gramex-otp=` on any request.
    This overrides the user with the passed `user` object for that session.

    Parameters:
        expire: Time when this token expires, in seconds (e.g. `60` means 1 minute from now)
        user: User object to store against token. Defaults to current user. Raises HTTP 403
            Unauthorized if there's no user
        size: Length of the OTP in characters. `None` means a full hash string
        type: Identifier for type of OTP. `OTP` for OTPs. Use `Key` for API keys. Auth handlers
            use their class names, e.g. `DBAuth`, `SMSAuth`, `EMailAuth`.
        kwargs: Additional columns to add (if they exist)

    Returns:
        Generated OTP

    Internally, this stores it in `storelocations.otp` database in a table with 4+ keys:

    1. `token`: Generated OTP with `size` characters
    2. `user`: The passed `user` string or dict, JSON-encoded
    3. `type`: The passed `type` string, stored as is
    4. `expire`: The expiry time in seconds since epoch
    5. Any additional columns passed in `kwargs`, if they exist
    '''
    user = self.current_user if user is None else user
    if not user:
        raise HTTPError(UNAUTHORIZED)
    from uuid import uuid4

    otp = uuid4().hex[:size]
    gramex.data.insert(
        **gramex.service.storelocations.otp,
        args={
            'token': [otp],
            'user': [json.dumps(user)],
            'type': [type],
            'expire': [time.time() + expire],
            **{key: [val] for key, val in kwargs.items()},
        },
    )
    return otp

def get_otp(self, key: str, revoke: bool = False) -> Union[str, dict, None]:
    '''Return the user object given the OTP key. Revoke the OTP if requested.

    Parameters:
        key: OTP to return
        revoke: True to revoke the OTP. False to retain it

    Returns:
        `None` if the OTP `key` doesn't exist or has expired.
            Else a dict with keys `user`, `expire`, `type` and `token`.
    '''
    rows = gramex.data.filter(**gramex.service.storelocations.otp, args={'token': [key]})
    if len(rows) == 0:
        return None
    row = rows.iloc[0].to_dict()
    if revoke:
        gramex.data.delete(
            **gramex.service.storelocations.otp, id=['token'], args={'token': [key]}
        )
    # Skip expired tokens
    if row['expire'] <= time.time():
        return None
    # Return the user column parsed as JSON.
    # Add custom keys from the table to the user object.
    row['user'] = json.loads(row['user'])
    custom_keys = [key for key in row if key not in {'user', 'token', 'expire'}]
    if isinstance(row['user'], dict):
        row['user'].update({key: row[key] for key in custom_keys})
    else:
        app_log.warning('Cannot add custom keys to non-dict "user" in: %r', row)
    return row

def revoke_otp(self, key: str) -> Union[str, dict, None]:
    '''Revoke an OTP. Returns the user object from [gramex.handlers.BaseMixin.get_otp][].'''
    return self.get_otp(key, revoke=True)

def apikey(
    self,
    expire: float = 1e9,
    user: Union[str, dict] = None,
    size: int = None,
    **kwargs: Dict[str, Any],
) -> str:
    '''Return API Key. Usage is same as [gramex.handlers.BaseMixin.otp][]

    The API key is used as the X-Gramex-Key header or in `?gramex-key=` on any request.
    This overrides the user with the passed `user` object for that session.
    '''
    return self.otp(expire=expire, user=user, size=size, type='Key', **kwargs)

def revoke_apikey(self, key: str) -> Union[str, dict, None]:
    '''Revoke API Key. Returns the user object from [gramex.handlers.BaseMixin.get_otp][].'''
    return self.revoke_otp(key)

def override_user(self):
    '''Internal method to override the user.

    Use `X-Gramex-User` HTTP header to override current user for the session.
    Use `X-Gramex-OTP` HTTP header to set user based on OTP, or `?gramex-otp=`.
    Use `X-Gramex-Key` HTTP header to set user based on API key, or `?gramex-key=`.
    '''
    headers = self.request.headers
    cipher = headers.get('X-Gramex-User')
    if cipher:
        try:
            user = json.loads(
                decode_signed_value(
                    conf.app.settings['cookie_secret'],
                    'user',
                    cipher,
                    max_age_days=self._session_expiry,
                )
            )
        except Exception:
            raise HTTPError(BAD_REQUEST, f'{self.name}: invalid X-Gramex-User: {cipher}')
        else:
            app_log.debug(f'{self.name}: Overriding user to {user!r}')
            self.session['user'] = user
            return
    # OTP is specified as an X-Gramex-OTP header or ?gramex-otp argument.
    # API Key is specified as an X-Gramex-Key header or ?gramex-key argument.
    # Override the user if either is specified.
    for key in ('OTP', 'Key'):
        token = headers.get(f'X-Gramex-{key}') or self.get_argument(
            f'gramex-{key.lower()}', None
        )
        if token:
            # Revoke OTP keys. Don't revoke API keys
            row = self.get_otp(token, revoke=key == 'OTP')
            if not row:
                raise HTTPError(
                    BAD_REQUEST, f'{self.name}: invalid/expired Gramex {key}: {token}'
                )
            self.session['user'] = row['user']

def set_last_visited(self):
    '''Update session last visited time if we track inactive expiry.

    - `session._l` is the last time the user accessed a page.
    - `session._i` is the seconds of inactivity after which the session expires.
    - If `session._i` is set (we track inactive expiry), we set `session._l` to now.
    '''
    # Called by BaseHandler.prepare() when any user accesses a page.
    # For efficiency reasons, don't call get_session every time. Check
    # session only if there's a valid sid cookie (with possibly long expiry)
    if self.get_secure_cookie(self._session_cookie_id, max_age_days=9999999):
        session = self.get_session()
        if '_i' in session:
            session['_l'] = time.time()

def check_ratelimit(self):
    '''Raise HTTP 429 if usage exceeds rate limit. Set X-Ratelimit-* HTTP headers'''
    for ratelimit in self._ratelimit:
        # If no expiry is specified, store for 100 years
        expiries = [3155760000]
        # Get the rate limit key, limit and expiry
        keys = [ratelimit.pool]
        for key_fn in ratelimit.key_fn:
            if 'key' in key_fn:
                predefined_key = key_fn['key'](self)
                if predefined_key in _PREDEFINED_KEYS:
                    keys.append(_PREDEFINED_KEYS[predefined_key]['function'](self))
                    expiries.append(_PREDEFINED_KEYS[predefined_key]['expiry'](self))
            if 'function' in key_fn:
                keys.append(key_fn['function'](self))
            if 'expiry' in key_fn:
                expiries.append(key_fn['expiry'](self))

        ratelimit.key = json.dumps(keys)
        # Note: if ratelimit_fn() returns a non-int, check_ratelimit will fail. Let it fail.
        ratelimit.limit = ratelimit.limit_fn(self)
        ratelimit.expiry = min(expiries)

        # Ensure usage does not hit limit
        ratelimit.usage = ratelimit.store.load(ratelimit.key, {'n': 0}).get('n', 0)
        if ratelimit.usage >= ratelimit.limit:
            raise HTTPError(
                TOO_MANY_REQUESTS,
                f'{ratelimit.pool}: {ratelimit.key} hit rate limit {ratelimit.limit}',
            )
    self.set_ratelimit_headers()

def update_ratelimit(self):
    '''If request succeeds, increase rate limit usage count by 1'''
    # If response is a HTTP error, don't count towards rate limit
    if self.get_status() >= 400:
        return
    for ratelimit in self._ratelimit:
        # If check_ratelimit failed (e.g. invalid function) and didn't set a key, skip update
        if 'key' not in ratelimit:
            return
        # Increment the rate limit by 1
        usage_obj = ratelimit.store.load(ratelimit.key, {'n': 0})
        usage_obj['n'] += 1
        usage_obj['_t'] = time.time() + ratelimit.expiry
        ratelimit.store.dump(ratelimit.key, usage_obj)

def get_ratelimit(self):
    '''Get the rate limit with the least remaining usage for the current request.

    If there are multiple rate limits, it picks the one with least remaining usage and
    returns an AttrDict with these keys:

    - `limit`: the limit on the rate limit (e.g. 3)
    - `usage`: the usage so far (BEFORE current request, e.g. 0, 1, 2, 3, ...)
    - `remaining`: the remaining requests (AFTER current request, e.g. 2, 1, 0, 0, ...)
    - `expiry`: seconds to expiry for this rate limit
    '''
    ratelimit = min(self._ratelimit, key=lambda r: r.limit - r.usage)
    return AttrDict(
        limit=ratelimit.limit,
        usage=ratelimit.usage,
        # ratelimit.usage goes 0, 1, 2, ...
        # If limit is 3, remaining goes 2, 1, 0, ... -- use (limit - usage - 1)
        # But when usage hits 3, don't show remaining = -1. Show remaining = 0 using max()
        remaining=max(ratelimit.limit - ratelimit.usage - 1, 0),
        expiry=ratelimit.expiry,
    )

def set_ratelimit_headers(self):
    '''Sets the headers from the [Ratelimit HTTP headers draft][1].

    - `X-Ratelimit-Limit` is the rate limit
    - `X-Ratelimit-Remaining` has the remaining requests
    - `X-Ratelimit-Reset` has seconds after which the rate limit resets
    - `Retry-After` is same as X-Ratelimit-Reset, but is set only if the limit is exceeded

    [1]: https://www.ietf.org/archive/id/draft-polli-ratelimit-headers-02.html
    '''
    # Pick the rate limit with the lowest remaining count
    ratelimit = self.get_ratelimit()
    self.set_header('X-Ratelimit-Limit', str(ratelimit.limit))
    self.set_header('X-Ratelimit-Remaining', str(ratelimit.remaining))
    self.set_header('X-RateLimit-Reset', str(ratelimit.expiry))
    if ratelimit.usage >= ratelimit.limit:
        self.set_header('Retry-After', str(ratelimit.expiry))

def initialize_handler(self):
    '''Initialize self.args and other handler attributes'''
    # self.request.arguments does not handle unicode keys well. It returns latin-1 unicode.
    # https://github.com/tornadoweb/tornado/issues/2733
    # Convert this to proper unicode using UTF-8 and store in self.args
    self.args = {}
    for k in self.request.arguments:
        key = k.encode('latin-1').decode('utf-8')
        # Invalid unicode (e.g. ?x=%f4) throws HTTPError. This disrupts even
        # error handlers. So if there's invalid unicode, log & continue.
        try:
            self.args[key] = self.get_arguments(k)
        except HTTPError:
            app_log.exception(f'{self.name}: Invalid unicode ?{k}')
    self._session, self._session_json = None, 'null'
    if self.cache:
        self.cachefile = self.cache()
        self.original_get = self.get
        self.get = self._cached_get
    if self._set_xsrf:
        self.xsrf_token

def update_body_args(self):
    '''Update self.args with JSON body if Content-Type is application/json'''
    if self.request.body:
        content_type = self.request.headers.get('Content-Type', '')
        if content_type == 'application/json':
            self.args.update(json.loads(self.request.body))

def get_arg(self, name, default=..., first=False):
    '''
    Returns the value of the argument with the given name. Similar to
    `.get_argument` but uses `self.args` instead.

    If default is not provided, the argument is considered to be
    required, and we raise a `MissingArgumentError` if it is missing.

    If the argument is repeated, we return the last value. If `first=True`
    is passed, we return the first value.

    `self.args` is always UTF-8 decoded unicode. Whitespaces are stripped.
    '''
    if name not in self.args:
        if default is ...:
            raise MissingArgumentError(name)
        return default
    return self.args[name][0 if first else -1]

def get_current_user(self):
    '''Return the `user` key from the session as an AttrDict if it exists.'''
    result = self.session.get('user')
    return AttrDict(result) if isinstance(result, dict) else result

def log_exception(self, typ, value, tb):
    '''Store the exception value for logging'''
    super(BaseHandler, self).log_exception(typ, value, tb)
    # _exception is stored for use by log_request. Sample error string:
    # ZeroDivisionError: integer division or modulo by zero
    self._exception = traceback.format_exception_only(typ, value)[0].strip()

def argparse(self, *args, **kwargs):
    '''
    Parse URL query parameters and return an AttrDict. For example:

    ```python
    args = handler.argparse('x', 'y')
    args.x      # is the last value of ?x=value
    args.y      # is the last value of ?y=value
    ```

    A missing `?x=` or `?y=` raises a HTTP 400 error mentioning the
    missing key.

    For optional arguments, use:

    ```python
    args = handler.argparse(z={'default': ''})
    args.z      # returns '' if ?z= is missing
    ```

    You can convert the value to a type:

    ```python
    args = handler.argparse(limit={'type': int, 'default': 100})
    args.limit      # returns ?limit= as an integer
    ```

    You can restrict the choice of values. If the query parameter is not in
    choices, we raise a HTTP 400 error mentioning the invalid key & value:

    ```python
    args = handler.argparse(gender={'choices': ['M', 'F']})
    args.gender      # returns ?gender= which will be 'M' or 'F'
    ```

    You can retrieve multiple values as a list:

    ```python
    args = handler.argparse(cols={'nargs': '*', 'default': []})
    args.cols       # returns an array with all ?col= values
    ```

    `type:` conversion and `choices:` apply to each value in the list.

    To return all arguments as a list, pass `list` as the first parameter:

    ```python
    args = handler.argparse(list, 'x', 'y')
    args.x      # ?x=1 sets args.x to ['1'], not '1'
    args.y      # Similarly for ?y=1
    ```

    To handle unicode arguments and return all arguments as `str` or
    `unicode` or `bytes`, pass the type as the first parameter:

    ```python
    args = handler.argparse(str, 'x', 'y')
    args = handler.argparse(bytes, 'x', 'y')
    args = handler.argparse(unicode, 'x', 'y')
    ```

    By default, all arguments are added as str in PY3 and unicode in PY2.

    There are the full list of parameters you can pass to each keyword
    argument:

    - name: Name of the URL query parameter to read. Defaults to the key
    - required: Whether or not the query parameter may be omitted
    - default: The value produced if the argument is missing. Implies required=False
    - nargs: The number of parameters that should be returned. '*' or '+'
      return all values as a list.
    - type: Python type to which the parameter should be converted (e.g. `int`)
    - choices: A container of the allowable values for the argument (after type conversion)

    You can combine all these options. For example:

    ```python
    args = handler.argparse(
        'name',                         # Raise error if ?name= is missing
        department={'name': 'dept'},    # ?dept= is mapped to args.department
        org={'default': 'Gramener'},    # If ?org= is missing, defaults to Gramener
        age={'type': int},              # Convert ?age= to an integer
        married={'type': bool},         # Convert ?married to a boolean
        alias={'nargs': '*'},           # Convert all ?alias= to a list
        gender={'choices': ['M', 'F']}, # Raise error if gender is not M or F
    )
    ```
    '''
    result = AttrDict()

    args_type = str
    if len(args) > 0 and args[0] in (str, bytes, list, None):
        args_type, args = args[0], args[1:]

    for key in args:
        result[key] = self.get_argument(key, None)
        if result[key] is None:
            raise HTTPError(BAD_REQUEST, f'{key}: missing ?{key}=')
    for key, config in kwargs.items():
        name = config.get('name', key)
        val = self.args.get(name, [])

        # default: set if query is missing
        # required: check if query is defined at all
        if len(val) == 0:
            if 'default' in config:
                result[key] = config['default']
                continue
            if config.get('required', False):
                raise HTTPError(BAD_REQUEST, f'{key}: missing ?{name}=')

        # nargs: select the subset of items
        nargs = config.get('nargs', None)
        if isinstance(nargs, int):
            val = val[:nargs]
            if len(val) < nargs:
                val += [''] * (nargs - len(val))
        elif nargs not in ('*', '+', None):
            raise ValueError(f'{key}: invalid nargs {nargs}')

        # convert to specified type
        newtype = config.get('type', None)
        if newtype is not None:
            newval = []
            for v in val:
                try:
                    newval.append(newtype(v))
                except ValueError:
                    raise HTTPError(
                        BAD_REQUEST, f'{key}: type error ?{name}={v} to {newtype!r}'
                    )
            val = newval

        # choices: check valid items
        choices = config.get('choices', None)
        if isinstance(choices, (list, dict, set)):
            choices = set(choices)
            for v in val:
                if v not in choices:
                    raise HTTPError(BAD_REQUEST, f'{key}: invalid choice ?{name}={v}')

        # Set the final value
        if nargs is None:
            if len(val) > 0:
                result[key] = val[-1]
        else:
            result[key] = val

    # Parse remaining keys
    if args_type is list:
        for key, val in self.args.items():
            if key not in args and key not in kwargs:
                result[key] = val
    elif args_type in (str, bytes):
        for key, val in self.args.items():
            if key not in args and key not in kwargs:
                result[key] = args_type(val[0])

    return result

def get_current_user(self):
    '''Return the `user` key from the session as an AttrDict if it exists.'''
    result = self.session.get('user')
    return AttrDict(result) if isinstance(result, dict) else result

def authorize(self):
    '''If a valid user isn't logged in, send a message and close connection'''
    if not self.current_user:
        raise HTTPError(UNAUTHORIZED)
    for permit_generator in self.permissions:
        for result in permit_generator(self):
            if not result:
                raise HTTPError(FORBIDDEN)

@tornado.gen.coroutine
def post(self, *path_args, **path_kwargs):
    '''Saves uploaded files, then updates metadata DB'''
    user = self.current_user or {}
    uploads = self.request.files.get('file', [])
    n = len(uploads)
    # Initialize all DB columns (except ID) to have the same number of rows as uploads.
    # Add `n` rows, and then clip to `n` rows. Effective way to pad AND trim.
    for key in list(self._db_cols.keys())[1:]:
        self.args[key] = self.args.get(key, []) + [''] * n
    for key in self.args:
        self.args[key] = self.args[key][:n]
    for i, upload in enumerate(uploads):
        file = os.path.basename(upload.get('filename', ''))
        ext = os.path.splitext(file)[1]
        path = slug.filename(file)
        # B311:random random() is safe since it's for non-cryptographic use
        while os.path.exists(os.path.join(self.path, path)):
            randomletter = choice(digits + ascii_lowercase)  # nosec B311
            path = os.path.splitext(path)[0] + randomletter + ext
        self.args['file'][i] = file
        self.args['ext'][i] = ext.lower()
        self.args['path'][i] = path
        self.args['size'][i] = len(upload['body'])
        self.args['date'][i] = int(time.time())
        # Guess MIME type from filename if it's unknown
        self.args['mime'][i] = upload['content_type']
        if self.args['mime'][i] == 'application/unknown':
            self.args['mime'][i] = guess_type(file, strict=False)[0]
        # Append user attributes
        for s in self.user_fields:
            self.args[f'user_{s.replace(".", "_")}'][i] = objectpath(user, s)
    self.check_filelimits()
    self.files = self.args
    yield super().post(*path_args, **path_kwargs)

def pre_modify(self, **kwargs):
    '''Called by FormHandler after updating the database, before modify. Save files here.

    This allows the DriveHandler modify: action to access the files.
    '''
    # If POST or PUT, save all files
    if self.request.method in {'POST', 'PUT'}:
        uploads = self.request.files.get('file', [])
        for upload, path in zip(uploads, self.files['path']):
            with open(os.path.join(self.path, path), 'wb') as handle:
                handle.write(upload['body'])
    elif self.request.method == 'DELETE':
        for relpath in self.files['path']:
            path = os.path.join(self.path, relpath)
            if os.path.exists(path):
                os.remove(path)

@tornado.gen.coroutine
def delete(self, *path_args, **path_kwargs):
    '''Deletes files from metadata DB and from file system'''
    conf = self.datasets.data
    files = gramex.data.filter(conf.url, table=conf.table, args=self.args)
    self.files = files.to_dict(orient='list')
    yield super().delete(*path_args, **path_kwargs)

@tornado.gen.coroutine
def put(self, *path_args, **path_kwargs):
    '''Update attributes and files'''
    uploads = self.request.files.get('file', [])[:1]
    id = self.args.get('id', [-1])
    # User cannot change the path, size, date or user attributes
    for s in ('path', 'size', 'date'):
        self.args.pop(s, None)
    for s in self.user_fields:
        self.args.pop(f'user_{s}', None)
    # These are updated only when a file is uploaded
    if len(uploads):
        user = self.current_user or {}
        self.args.setdefault('size', []).append(len(uploads[0]['body']))
        self.args.setdefault('date', []).append(int(time.time()))
        for s in self.user_fields:
            self.args.setdefault(f'user_{s.replace(".", "_")}', []).append(objectpath(user, s))
    conf = self.datasets['data']
    files = gramex.data.filter(conf.url, table=conf.table, args={'id': id})
    self.files = files.to_dict(orient='list')
    self.files.update(self.args)
    yield super().put(*path_args, **path_kwargs)