From 6fa4bc7aeabf36e8432c62dcaf056e459d5fc08d Mon Sep 17 00:00:00 2001 From: Luca Beltrame Date: Sat, 17 Sep 2016 01:37:11 +0200 Subject: [PATCH] Scripts to regularly update things off the OBS --- kde_unstable_applications.csv | 200 ++++++++++++++++++++++++++++++++++ kde_unstable_frameworks.csv | 148 +++++++++++++++++++++++++ update_unstable.py | 132 ++++++++++++++++++++++ 3 files changed, 480 insertions(+) create mode 100644 kde_unstable_applications.csv create mode 100644 kde_unstable_frameworks.csv create mode 100644 update_unstable.py diff --git a/kde_unstable_applications.csv b/kde_unstable_applications.csv new file mode 100644 index 0000000..cf442db --- /dev/null +++ b/kde_unstable_applications.csv @@ -0,0 +1,200 @@ +akonadi akonadi-server +akonadi-calendar akonadi-calendar +akonadi-calendar-tools akonadi-calendar-tools +akonadi-contacts akonadi-contacts +akonadi-import-wizard akonadi-import-wizard +akonadi-mime akonadi-mime +akonadi-notes akonadi-notes +akonadi-search akonadi-search +akonadiconsole akonadiconsole +akregator akregator +analitza analitza +ark ark +artikulate artikulate +baloo-widgets baloo5-widgets +blinken blinken +blogilo blogilo +bomber bomber +bovo bovo +calendarsupport calendarsupport +cantor cantor +cervisia cervisia +dolphin dolphin +dolphin-plugins dolphin-plugins +dragon dragonplayer +eventviews eventviews +ffmpegthumbs ffmpegthumbs +filelight filelight +gpgmepp gpgmepp5 +granatier granatier +grantlee-editor grantlee-editor +grantleetheme grantleetheme +gwenview gwenview5 +incidenceeditor incidenceeditor +kaccounts-integration kaccounts-integration +kaccounts-providers kaccounts-providers +kaddressbook kaddressbook +kalarm kalarm +kalarmcal kalarmcal +kalgebra kalgebra +kalzium kalzium +kanagram kanagram +kapman kapman +kapptemplate kapptemplate +kate kate +katomic katomic +kblackbox kblackbox +kblocks kblocks +kblog kblog +kbounce kbounce +kbreakout kbreakout +kbruch kbruch +kcachegrind kcachegrind +kcalc kcalc +kcalcore kcalcore +kcalutils kcalutils +kcharselect kcharselect +kcontacts kcontacts +kcron kcron +kde-baseapps kde-baseapps +kde-dev-scripts kde-dev-scripts +kdebugsettings kdebugsettings +kdeedu-data kdeedu-data +kdegraphics-thumbnailers kdegraphics-thumbnailers +kdenlive kdenlive +kdepim-addons kdepim-addons +kdepim-apps-libs kdepim-apps-libs +kdepim-runtime kdepim-runtime +kdesdk-thumbnailers kdesdk-thumbnailers +kdf kdf +kdgantt2 kdgantt2 +kdiamond kdiamond +kdnssd kdnssd +kfloppy kfloppy +kfourinline kfourinline +kgeography kgeography +kget kget +kgoldrunner kgoldrunner +kgpg kgpg +khangman khangman +khelpcenter khelpcenter5 +kholidays kholidays +kidentitymanagement kidentitymanagement +kig kig +kigo kigo +killbots killbots +kimap kimap +kio-extras kio-extras +kiriki kiriki +kiten kiten +kjumpingcube kjumpingcube +kldap kldap +kleopatra kleopatra5 +klettres klettres +klickety klickety +klines klines +kmag kmag +kmahjongg kmahjongg +kmail kmail +kmail-account-wizard kmail-account-wizard +kmailtransport kmailtransport +kmbox kmbox +kmime kmime +kmines kmines +kmix kmix +kmousetool kmousetool +kmouth kmouth +kmplot kmplot +knavalbattle knavalbattle +knetwalk knetwalk +knotes knotes +kolf kolf +kollision kollision +kolourpaint kolourpaint +kompare kompare +konquest konquest +konsole konsole +kontact kontact +kontactinterface kontactinterface +kopete kopete +korganizer korganizer +kpat kpat +kpimtextedit kpimtextedit +kqtquickcharts kqtquickcharts +krdc krdc +kreversi kreversi +krfb krfb +kross-interpreters kross-interpreters +kruler kruler +kscd kscd +kshisen kshisen +ksirk ksirk +ksnakeduel ksnakeduel +kspaceduel kspaceduel +ksquares ksquares +kstars kstars +ksudoku ksudoku +ksystemlog ksystemlog +kteatime kteatime +ktimer ktimer +ktp-accounts-kcm ktp-accounts-kcm +ktp-approver ktp-approver +ktp-auth-handler ktp-auth-handler +ktp-common-internals ktp-common-internals +ktp-contact-list ktp-contact-list +ktp-contact-runner ktp-contact-runner +ktp-desktop-applets ktp-desktop-applets +ktp-filetransfer-handler ktp-filetransfer-handler +ktp-kded-module ktp-kded-module +ktp-send-file ktp-send-file +ktp-text-ui ktp-text-ui +ktuberling ktuberling +kturtle kturtle +kubrick kubrick +kuser kuser +kwordquiz kwordquiz +libgravatar libgravatar +libgravatar libgravatar +libkcddb libkcddb5 +libkdcraw libkdcraw +libkdegames libkdegames +libkdepim libkdepim +libkeduvocdocument libkeduvocdocument +libkexiv2 libkexiv2 +libkface libkface +libkgapi libkgapi +libkgeomap libkgeomap +libkipi libkipi +libkleo libkleo +libkmahjongg libkmahjongg +libkomparediff2 libkomparediff2 +libksane libksane +libksieve libksieve +ktnef libktnef +prison libprison +lokalize lokalize +lskat lskat +mailcommon mailcommon +mailimporter mailimporter +marble marble +mbox-importer mbox-importer +messagelib messagelib +okteta okteta +okular okular5 frameworks +pairs pairs +palapeli palapeli +parley parley +picmi picmi +pim-data-exporter pim-data-exporter +pim-sieve-editor pim-sieve-editor +pim-storage-service-manager pim-storage-service-manager +pimcommon pimcommon +pimcommon pimcommon +print-manager kde-print-manager +rocs rocs +signon-kwallet-extension signon-kwallet-extension +spectacle spectacle +step step +svgpart svgpart +syndication syndication +umbrello umbrello frameworks diff --git a/kde_unstable_frameworks.csv b/kde_unstable_frameworks.csv new file mode 100644 index 0000000..1d583f8 --- /dev/null +++ b/kde_unstable_frameworks.csv @@ -0,0 +1,148 @@ + attica-qt5 + baloo5 + bluedevil5 + bluez-qt + breeze + breeze4-style + breeze5-icons + breeze-gtk + cmake + cmake-gui + discover + extra-cmake-modules + frameworkintegration + gpgmepp5 + grub2-theme-breeze + hack-fonts + kaccounts-aggregate + kactivities5 + kactivities-stats + kactivitymanagerd + kapidox + karchive + kauth + kbookmarks + kcm_sddm + kcmutils + kcodecs + kcompletion + kconfig + kconfigwidgets + kcoreaddons + kcrash + kdbusaddons + kdeclarative + kde-cli-tools5 + kded + kde-gtk-config5 + kdelibs4support + kde-oxygen-fonts + kdesignerplugin + kdesu + kde-user-manager + kdewebkit + kdnssd-framework + kdoctools + kemoticons + kf5-filesystem + kfilemetadata5 + kgamma5 + kglobalaccel + kguiaddons + khelpcenter5 + khotkeys5 + khtml + ki18n + kiconthemes + kidletime + kimageformats + kinfocenter5 + kinit + kio + kio-extras5 + kirigami + kitemmodels + kitemviews + kjobwidgets + kjs + kjsembed + kmediaplayer + kmenuedit5 + knewstuff + knotifications + knotifyconfig + kpackage + kparts + kpeople5 + kplotting + kpty + kross + krunner + kscreen5 + kscreenlocker + kservice + ksshaskpass5 + ksysguard5 + ktexteditor + ktextwidgets + kunitconversion + kwallet + kwalletmanager5 + kwayland + kwayland-integration + kwidgetsaddons + kwin5 + kwindowsystem + kwrited5 + kxmlgui + kxmlrpcclient5 + libepoxy + libkdecoration2 + libKF5ModemManagerQt + libKF5NetworkManagerQt + libkscreen2 + libksysguard5 + libraw + milou5 + oxygen4 + oxygen5 + oxygen5-icon-theme + PackageKit-Qt5 + phonon4qt5 + phonon4qt5-backend-gstreamer + phonon4qt5-backend-vlc + plasma5-addons + plasma5-desktop + plasma5-integration + plasma5-mediacenter + plasma5-openSUSE + plasma5-pa + plasma5-pk-updates + plasma5-sdk + plasma5-session + plasma5-workspace + plasma5-workspace-wallpapers + plasma-framework + plasma-nm5 + plymouth-theme-breeze + polkit-default-privs + polkit-kde-agent-5 + polkit-qt5-1 + poppler + poppler-qt + poppler-qt5 + powerdevil5 + purpose + rpmlint + rpmlint-Factory + rpmlint-mini + sddm + sni-qt + solid + solid-power + sonnet + sprinter + sprinter-plugins + systemsettings5 + taglib + threadweaver diff --git a/update_unstable.py b/update_unstable.py new file mode 100644 index 0000000..14dca1d --- /dev/null +++ b/update_unstable.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +import argparse +import csv +import logging +from pathlib import Path +import pickle + +import git +import sarge + +logging.basicConfig(format='%(levelname)s:%(message)s', + level=logging.DEBUG) + +class GitHashCache: + + def __init__(self, cache_file): + self.cache = cache_file + self._data = dict() + + def __getitem__(self, key): + + if key not in self._data: + return "" + + return self._data[key] + + def __setitem__(self, key, value): + self._data[key] = value + + def save(self): + logging.debug("Saving pickled data") + with open(self.cache, "wb") as handle: + pickle.dump(self._data, handle, pickle.HIGHEST_PROTOCOL) + + def load(self): + + if not Path(self.cache).exists(): + logging.debug("File cache not found, not loading") + return + + logging.debug("Saving pickled data") + with open(self.cache, "rb") as handle: + self._data = pickle.load(handle) + + +def lsremote(url): + + remote_refs = {} + g = git.cmd.Git() + for ref in g.ls_remote(url).split('\n'): + hash_ref_list = ref.split('\t') + remote_refs[hash_ref_list[1]] = hash_ref_list[0] + + return remote_refs + + +def get_remote_hash(url, branch="master"): + refs = "refs/heads/{}".format(branch) + return lsremote(url)[refs] + + +def run_osc(repository, package_name): + + cmd = "osc service remoterun {0} {1}" + cmd = cmd.format(repository, package_name) + + logging.debug("Running {}".format(cmd)) + + # pid = sarge.run(cmd) + + # if pid.returncode != 0: + # logging.error("Error during service run, package {}".format( + # package_name)) + + logging.debug("Package {} complete".format(package_name)) + + +def update_package(hash_data, package_name, remote_name, obs_repository, + branch): + + repo_name = "kde:{}".format(remote_name) + + remote_hash = get_remote_hash(repo_name, branch) + current_hash = hash_data[remote_name] + + logging.debug("Package {}, theirs {}, ours {}".format(remote_name, + remote_hash, current_hash)) + + if remote_hash != current_hash: + logging.debug("Hash doesn't match, updating") + run_osc(obs_repository, package_name) + # hash_data[remote_name] = remote_hash + + +def update_packages(cache_file, obs_repository, repo_mapping_file): + + hash_data = GitHashCache(cache_file) + hash_data.load() + + logging.info("Updating packages for {}".format(obs_repository)) + with open(repo_mapping_file, "r") as mapping: + reader = csv.reader(mapping, delimiter="\t") + for row in reader: + kde_name, obs_name, branch = row + branch = "master" if not branch else branch + logging.debug("Updating package {} ({})".format(kde_name, + obs_name)) + logging.debug("Using branch {}".format(branch)) + update_package(hash_data, obs_name, kde_name, obs_repository, + branch) + + logging.debug("Saving data") + hash_data.save() + + +def main(): + + parser = argparse.ArgumentParser() + parser.add_argument("repository", help="OBS repository to use") + parser.add_argument("mapping_file", help="KDE:OBS repository mapping file") + + options = parser.parse_args() + + cache_file = Path.home() / ".local/share/obs_{}_cache".format(options.repository) + cache_file = str(cache_file) + + update_packages(cache_file, options.repository, options.mapping_file) + logging.info("Complete") + +if __name__ == "__main__": + main()