From cc676a082d75a47d3566e0d5127608103c5fd1a1 Mon Sep 17 00:00:00 2001 From: Luca Beltrame Date: Sun, 30 Jan 2022 11:10:34 +0100 Subject: [PATCH] Paginate status results and implement rate limiting Packages are chunked in groups of 100 to avoid generating too large messages, and each chunk is sent spaced by 100 ms, to avoid rate limiting. In parallel, avoid room flooding by refusing to print statuses with more than 50 repositories or 1000 packages. Fixes issue #1. --- oscbot/__init__.py | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/oscbot/__init__.py b/oscbot/__init__.py index 1469901..979fe56 100644 --- a/oscbot/__init__.py +++ b/oscbot/__init__.py @@ -2,10 +2,12 @@ # SPDX-License-Identifier: AGPL-3.0-or-later from dataclasses import dataclass, field +from time import sleep from typing import Optional, List, Type, Tuple import aiohttp import cryptocode +import more_itertools as mit from lxml import objectify from jinja2 import BaseLoader, Environment @@ -22,10 +24,8 @@ Only showing packages with {{ state }} state. {% endif %}""" -REPO_TEMPLATE = """ -#### {{ repo.name }} - {{ repo.arch }} - -{% for package in repo.packages %} +STATUS_TEMPLATE = """ +{% for package in packages %} {%if package.status != "disabled" %} {% set build_log_url = "{0}/package/live_build_log/{1}/{2}/{3}/{4}/".format( base_url, project, package.name, repo.name, repo.arch) %} @@ -287,8 +287,29 @@ class OSCBot(Plugin): base_url = self.config["instance_url"] + if len(response) > 50: + await evt.respond(f"Too many repositories ({len(response)})" + f"to display in project {project}, " + "please narrow down your search.") + return + for repository in response: - body = self.template.from_string(REPO_TEMPLATE) - message = body.render(repo=repository, project=project, - base_url=base_url) - await evt.respond(message, markdown=True) + await evt.respond(f"#### {repository.name} - {repository.arch}", + markdown=True) + + # Just so that someone doesn't flood by listing the whole + # openSUSE:Factory + if len(repository.packages) > 1000: + await evt.respond( + f"Too many packages ({len(repository.packages)}) " + "to display, please narrow down your search.") + continue + # To avoid creating too large messages, we chunk packages in + # groups of 100 + for packagelist in mit.chunked(repository.packages, 100): + body = self.template.from_string(STATUS_TEMPLATE) + message = body.render(packages=packagelist, base_url=base_url, + project=project, repo=repository) + await evt.respond(message, markdown=True) + # Wait 100 milliseconds to avoid triggering rate limiting + sleep(0.1)