diff options
| -rw-r--r-- | .gitignore | 145 | ||||
| -rw-r--r-- | gui.py | 223 | ||||
| -rw-r--r-- | moodle.py | 46 | ||||
| -rwxr-xr-x | muddle | 3 | ||||
| -rw-r--r-- | muddle.ini.example | 3 | ||||
| -rw-r--r-- | muddle.py | 106 | ||||
| -rw-r--r-- | requirements.txt | 7 |
7 files changed, 533 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bfbb67c --- /dev/null +++ b/.gitignore @@ -0,0 +1,145 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# profiling data +.prof + +# End of https://www.toptal.com/developers/gitignore/api/python @@ -0,0 +1,223 @@ +# +# This document *and this document only* uses the Qt naming convention instead +# of PEP because the PyQt5 bindings do not have pythonic names +# + +import sys +import json +import enum +import html +import logging + +from PyQt5.QtGui import QFont +from PyQt5.Qt import QStyle +from PyQt5.QtCore import Qt, QThread, pyqtSlot +from PyQt5.QtWidgets import QApplication, QWidget, QTreeWidget, QTreeWidgetItem, QGridLayout, QHBoxLayout, QPushButton, QProgressBar, QTabWidget, QPlainTextEdit + +import moodle + +log = logging.getLogger("muddle.gui") + +class MoodleItem: + class Type(enum.Enum): + ROOT = 0 + # root + COURSE = 1 + # sections + SECTION = 2 + # modules + MODULE = 3 + FORUM = 3 + RESOURCE = 3 + FOLDER = 3 + ATTENDANCE = 3 + LABEL = 3 + QUIZ = 3 + # contents + CONTENT = 4 + FILE = 4 + URL = 4 + + def __init__(self, nodetype, leaves=[], **kwargs): + self.leaves = leaves + self.type = nodetype + + # TODO: check required attributes + for k, v in kwargs.items(): + setattr(self, k, v) + + self.setupQt() + + # TODO: Qt objects should be on the main thread + # prob cause of the crash + def setupQt(self): + self.qt = QTreeWidgetItem() + + font = QFont("Monospace") + font.setStyleHint(QFont.Monospace) + self.qt.setFont(0, font) + + icons = { + MoodleItem.Type.COURSE : QApplication.style().standardIcon(QStyle.SP_DriveNetIcon), + MoodleItem.Type.FOLDER : QApplication.style().standardIcon(QStyle.SP_DirIcon), + MoodleItem.Type.FILE : QApplication.style().standardIcon(QStyle.SP_FileIcon), + MoodleItem.Type.URL : QApplication.style().standardIcon(QStyle.SP_FileLinkIcon), + } + + if icons.get(self.type): + self.qt.setIcon(0, icons[self.type]) + + if self.type == MoodleItem.Type.FILE: + flags = self.qt.flags() + self.qt.setFlags(flags | Qt.ItemIsUserCheckable) + self.qt.setCheckState(0, Qt.Unchecked) + + self.qt.setChildIndicatorPolicy(QTreeWidgetItem.DontShowIndicatorWhenChildless) + self.qt.setText(0, html.unescape(self.title)) + + def insert(self, node): + self.leaves.append(node) + if not self.type == MoodleItem.Type.ROOT: + self.qt.addChild(node.qt) + + def remove(self, node): + self.leaves.remove(node) + self.qt.removeChild(node.qt) + # TODO: remove from child + +class MoodleFetcher(QThread): + def __init__(self, parent, instance_url, token): + super().__init__() + + self.api = moodle.RestApi(instance_url, token) + self.apihelper = moodle.ApiHelper(self.api) + self.moodleItems = MoodleItem(MoodleItem.Type.ROOT, title="ROOT") + + def run(self): + # This is beyond bad but I don't have access to the moodle documentation, so I had to guess + courses = self.api.core_enrol_get_users_courses(userid=self.apihelper.get_userid()).json() + for course in courses: + courseItem = MoodleItem(MoodleItem.Type.COURSE, + id = course["id"], title = course["shortname"]) + + self.moodleItems.insert(courseItem) + + sections = self.api.core_course_get_contents(courseid=courseItem.id).json() + + for section in sections: + sectionItem = MoodleItem(MoodleItem.Type.SECTION, + id = section["id"], title = section["name"]) + + courseItem.insert(sectionItem) + + modules = section["modules"] if "modules" in section else [] + for module in modules: + moduleType = { + "folder" : MoodleItem.Type.FOLDER, + "resource" : MoodleItem.Type.RESOURCE, + "forum" : MoodleItem.Type.FORUM, + "attendance" : MoodleItem.Type.ATTENDANCE, + "label" : MoodleItem.Type.LABEL, + "quiz" : MoodleItem.Type.QUIZ, + } + + moduleItem = MoodleItem(moduleType.get(module["modname"]) or MoodleItem.Type.MODULE, + id = module["id"], title = module["name"]) + + sectionItem.insert(moduleItem) + + contents = module["contents"] if "contents" in module else [] + for content in contents: + contentType = { + "url" : MoodleItem.Type.URL, + "file" : MoodleItem.Type.FILE, + } + + contentItem = MoodleItem(contentType.get(content["type"]) or MoodleItem.Type.MODULE, + title = content["filename"], url = content["fileurl"]) + + moduleItem.insert(contentItem) + + +class MoodleTreeView(QTreeWidget): + def __init__(self, parent, instance_url, token): + super().__init__(parent) + + self.worker = MoodleFetcher(self, instance_url, token) + self.worker.finished.connect(self.onWorkerDone) + self.worker.start() + + self.initUi() + self.show() + + def initUi(self): + self.setHeaderHidden(True) + self.itemDoubleClicked.connect(self.onItemDoubleClicked, Qt.QueuedConnection) + + @pyqtSlot(QTreeWidgetItem, int) + def onItemDoubleClicked(self, item, col): + log.debug(f"double clicked on item with type {str(item.type)}") + if item.type == MoodleItem.Type.FILE: + pass + + @pyqtSlot() + def onWorkerDone(self): + log.debug("worker done") + for leaf in self.worker.moodleItems.leaves: + self.addTopLevelItem(leaf.qt) + +class QPlainTextEditLogger(logging.Handler): + def __init__(self, parent): + super().__init__() + self.widget = QPlainTextEdit(parent) + self.widget.setReadOnly(True) + + def emit(self, record): + msg = self.format(record) + self.widget.appendPlainText(msg) + + def write(self, m): + pass + +class Muddle(QTabWidget): + def __init__(self, instance_url, token): + super().__init__() + self.instance_url = instance_url + self.token = token + self.initUi() + + def initUi(self): + self.setWindowTitle("Muddle") + + # moodle tab + self.tabmoodle = QWidget() + self.addTab(self.tabmoodle, "Moodle") + + self.tabmoodle.setLayout(QGridLayout()) + self.tabmoodle.layout().addWidget(MoodleTreeView(self, self.instance_url, self.token), 0, 0, 1, -1) + + self.tabmoodle.downloadbtn = QPushButton("Download") + self.tabmoodle.selectallbtn = QPushButton("Select All") + self.tabmoodle.deselectallbtn = QPushButton("Deselect All") + self.tabmoodle.progressbar = QProgressBar() + + self.tabmoodle.layout().addWidget(self.tabmoodle.downloadbtn, 1, 0) + self.tabmoodle.layout().addWidget(self.tabmoodle.selectallbtn, 1, 1) + self.tabmoodle.layout().addWidget(self.tabmoodle.deselectallbtn, 1, 2) + self.tabmoodle.layout().addWidget(self.tabmoodle.progressbar, 2, 0, 1, -1) + + # log tabs + handler = QPlainTextEditLogger(self) + handler.setFormatter(logging.Formatter("%(name)s - %(levelname)s - %(message)s")) + logging.getLogger("muddle").addHandler(handler) + + self.tablogs = handler.widget + self.addTab(self.tablogs, "Logs") + + self.show() + + +def start(instance_url, token): + app = QApplication(sys.argv) + ex = Muddle(instance_url, token) + sys.exit(app.exec_()) diff --git a/moodle.py b/moodle.py new file mode 100644 index 0000000..ee60dc9 --- /dev/null +++ b/moodle.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +import requests +import logging + +log = logging.getLogger("muddle.moodle") + +# +# magic moodle api wrapper +# +def request_token(url, user, password): + token_url = f"{url}/login/token.php" + data = { "username": user, "password": password, "service": "moodle_mobile_app" } + log.debug(f"requesting token with POST to {api_url} with DATA {data}") + return requests.post(token_url, data=data) + +def api_call(url, token, function, **kwargs): + api_url = f"{url}/webservice/rest/server.php?moodlewsrestformat=json" + data = { "wstoken": token, "wsfunction": function } + for k, v in kwargs.items(): + data[str(k)] = v + + log.debug(f"calling api with POST to {api_url} with DATA {data}") + return requests.post(api_url, data=data) + +class RestApi: + def __init__(self, instance_url, token): + self._url = instance_url + self._token = token + + def __getattr__(self, key): + return lambda **kwargs: api_call(self._url, self._token, str(key), **kwargs) + +class ApiHelper: + def __init__(self, api): + self.api = api + + def get_userid(self): + return self.api.core_webservice_get_site_info().json()["userid"] + + def get_file(self, url, local_path): + with requests.post(url, data={"token": self.api._token}, stream=True) as r: + r.raise_for_status() + with open(local_path, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) @@ -0,0 +1,3 @@ +#!/usr/bin/env python3 + +import muddle diff --git a/muddle.ini.example b/muddle.ini.example new file mode 100644 index 0000000..6a1c2f0 --- /dev/null +++ b/muddle.ini.example @@ -0,0 +1,3 @@ +[server] +url = https://moodle.rj.ost.ch +token = <your token here> diff --git a/muddle.py b/muddle.py new file mode 100644 index 0000000..5c96316 --- /dev/null +++ b/muddle.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +import argparse +import configparser +import logging + +import os +import sys +import platform +import pathlib +import json + +import moodle +import gui + +# A R G U M E N T S + +parser = argparse.ArgumentParser(description="Moodle Scraper") +parser.add_argument("-g", "--gui", help="start with graphical interface", action="store_true") +parser.add_argument("-v", "--verbose", help="be more verbose", action="store_true") +parser.add_argument("-c", "--config", help="configuration file", type=str) +parser.add_argument("-l", "--logfile", help="where to save logs", type=str) +args = parser.parse_args() + +# L O G G I N G +logformatter = logging.Formatter("%(name)s - %(levelname)s - %(message)s") +log = logging.getLogger("muddle") +log.setLevel(logging.DEBUG) + +if args.verbose: + cli_handler = logging.StreamHandler() + cli_handler.setLevel(logging.DEBUG) + cli_handler.setFormatter(logformatter) + log.addHandler(cli_handler) + +# C O N F I G S A N D L O G S + +# default location for configuration and log files + +default_config_dir = pathlib.Path.cwd() +default_log_dir = pathlib.Path.cwd() + +if platform.system() == "Linux": + # compliant with XDG + if os.environ.get("XDG_CONFIG_HOME"): + default_config_dir = pathlib.PurePath(os.environ["XDG_CONFIG_HOME"]).joinpath("muddle/") + + elif pathlib.Path("~/.config").expanduser().exists(): + default_config_dir = pathlib.PurePath("~/.config/muddle/").expanduser() + + if os.environ.get("XDG_CACHE_HOME"): + default_log_dir = pathlib.PurePath(os.environ["XDG_CACHE_HOME"]).joinpath("muddle/") + + elif pathlib.Path("~/.cache").expanduser().exists(): + default_log_dir = pathlib.PurePath("~/.cache/muddle/").expanduser() + + +# TODO: implement for other platforms + +default_config_file = default_config_dir.joinpath("muddle.ini") +default_log_file = default_log_dir.joinpath("muddle.log") + +log.debug("set default config path {}".format(default_config_file)) +log.debug("set default log path {}".format(default_log_file)) + +# user parameters + +log_file = pathlib.Path(default_log_file) +if args.logfile: + if os.path.exists(args.logfile): + log_file = pathlib.Path(args.logfile) + log.debug(f"using log file {log_file}") + else: + log.error(f"path is not a file or does not exist {args.logfile}") + log.debug("using default log path") + +# set up logfile +log_file.parent.mkdir(parents=True, exist_ok=True) + +file_handler = logging.FileHandler(log_file) +file_handler.setFormatter(logformatter) +file_handler.setLevel(logging.INFO) + +config_file = pathlib.Path(default_config_file) +if args.config: + if os.path.isfile(args.config): + config_file = pathlib.Path(args.config) + log.debug(f"set config file {config_file}") + else: + log.error(f"path is not a file or does not exist {args.config}") + log.debug("using default config path") + +# parse config +if not config_file.is_file(): + log.error(f"cannot read {config_file}") + sys.exit(1) + +log.debug(f"reading config file {config_file}") +config = configparser.ConfigParser() +config.read(config_file) + + +# G R A P H I C S + +if args.gui: + gui.start(config["server"]["url"], config["server"]["token"]) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0d9c554 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +certifi==2020.6.20 +chardet==3.0.4 +idna==2.10 +PyQt5==5.15.1 +PyQt5-sip==12.8.1 +requests==2.24.0 +urllib3==1.25.10 |