From 266b30b67449ee14a78402f085b2058e7212cc51 Mon Sep 17 00:00:00 2001 From: Nao Pross Date: Wed, 10 Feb 2021 15:45:35 +0100 Subject: Reorganize project directory --- muddle | 3 - muddle/__main__.py | 143 +++++++++++++++ muddle/gui.py | 500 +++++++++++++++++++++++++++++++++++++++++++++++++++++ muddle/moodle.py | 68 ++++++++ muddle/muddle.ui | 259 +++++++++++++++++++++++++++ 5 files changed, 970 insertions(+), 3 deletions(-) delete mode 100755 muddle create mode 100644 muddle/__main__.py create mode 100644 muddle/gui.py create mode 100644 muddle/moodle.py create mode 100644 muddle/muddle.ui (limited to 'muddle') diff --git a/muddle b/muddle deleted file mode 100755 index 2d1b05d..0000000 --- a/muddle +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env python3 - -import muddle diff --git a/muddle/__main__.py b/muddle/__main__.py new file mode 100644 index 0000000..22f2396 --- /dev/null +++ b/muddle/__main__.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 + +import argparse +import configparser +import logging +import colorlog + +import os +import sys +import platform +import pathlib +import json + +import moodle +import gui + + +MUDDLE_VERSION = "0.1.0" + +# A R G U M E N T S + +parser = argparse.ArgumentParser(description="Moodle Scraper") +parser.add_argument("-g", "--gui", help="start with graphical interface", action="store_true") +parser.add_argument("-v", "--verbose", help="be more verbose", action="store_true") +parser.add_argument("-c", "--config", help="configuration file", type=str) +parser.add_argument("-l", "--logfile", help="where to save logs", type=str) +parser.add_argument("-V", "--version", help="version", action="store_true") +args = parser.parse_args() + +# L O G G I N G + +logformatter = logging.Formatter("%(name)s - %(levelname)s - %(message)s") +log = logging.getLogger("muddle") +log.setLevel(logging.DEBUG) + +if args.verbose: + cli_handler = colorlog.StreamHandler() + cli_handler.setLevel(logging.DEBUG) + cli_formatter = colorlog.ColoredFormatter( + "%(name)-13s - %(log_color)s%(levelname)-8s%(reset)s: %(message)s", + datefmt=None, + reset=True, + log_colors={ + 'DEBUG': 'cyan', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'red,bg_white', + } + ) + cli_handler.setFormatter(cli_formatter) + log.addHandler(cli_handler) + +# C O N F I G S A N D L O G S + +# default location for configuration and log files + +default_config_dir = pathlib.Path.cwd() +default_log_dir = pathlib.Path.cwd() + +if platform.system() == "Linux": + # compliant with XDG + if os.environ.get("XDG_CONFIG_HOME"): + default_config_dir = pathlib.PurePath(os.environ["XDG_CONFIG_HOME"]).joinpath("muddle/") + + elif pathlib.Path("~/.config").expanduser().exists(): + default_config_dir = pathlib.Path("~/.config/muddle/").expanduser() + + if os.environ.get("XDG_CACHE_HOME"): + default_log_dir = pathlib.Path(os.environ["XDG_CACHE_HOME"]).joinpath("muddle/") + + elif pathlib.Path("~/.cache").expanduser().exists(): + default_log_dir = pathlib.Path("~/.cache/muddle/").expanduser() + +elif platform.system() == "Windows": + if os.environ.get("APPDATA"): + default_config_dir = pathlib.Path(os.environ["APPDATA"]).joinpath("muddle/") + + if os.environ.get("LOCALAPPDATA"): + default_log_dir = pathlib.Path(os.environ["LOCALAPPDATA"]).joinpath("muddle") + +# TODO: implement for MacOS + +default_config_file = default_config_dir.joinpath("muddle.ini") +default_log_file = default_log_dir.joinpath("muddle.log") + +log.debug("set default config path {}".format(default_config_file)) +log.debug("set default log path {}".format(default_log_file)) + +# user parameters + +log_file = pathlib.Path(default_log_file) +if args.logfile: + if os.path.exists(args.logfile): + log_file = pathlib.Path(args.logfile) + log.debug(f"using log file {log_file}") + else: + log.error(f"path is not a file or does not exist {args.logfile}") + log.debug("using default log path") + +# set up logfile +log_file.parent.mkdir(parents=True, exist_ok=True) + +file_handler = logging.FileHandler(log_file) +file_handler.setFormatter(logformatter) +file_handler.setLevel(logging.INFO) + +config_file = pathlib.Path(default_config_file) +if args.config: + if os.path.isfile(args.config): + config_file = pathlib.Path(args.config) + log.debug(f"set config file {config_file}") + else: + log.error(f"path is not a file or does not exist {args.config}") + log.debug("using default config path") + +# parse config +if not config_file.is_file(): + log.error(f"cannot read {config_file}") + sys.exit(1) + +log.debug(f"reading config file {config_file}") +config = configparser.ConfigParser() +config.read(config_file) + +# runtime data that should NOT be written +config.add_section("runtime_data") +config["runtime_data"]["config_path"] = str(config_file) + + +# S T A R T + +if args.version: + print(f"""Version {MUDDLE_VERSION} +Muddle Copyright (C) 2020-2021 Nao Pross + +This program comes with ABSOLUTELY NO WARRANTY; This is free software, and you +are welcome to redistribute it under certain conditions; see LICENSE.txt for +details. Project repository: https://github.com/NaoPross/Muddle +""") + +if args.gui or config.getboolean("muddle", "always_run_gui"): + gui.start(config) diff --git a/muddle/gui.py b/muddle/gui.py new file mode 100644 index 0000000..3d39cec --- /dev/null +++ b/muddle/gui.py @@ -0,0 +1,500 @@ +# +# This document *and this document only* uses the Qt naming convention instead +# of PEP because the PyQt5 bindings do not have pythonic names +# +import os +import platform +import subprocess +import sys +import json +import enum +import html +import logging +import tempfile + +from PyQt5 import uic +from PyQt5.QtGui import QFont, QIcon, QStandardItemModel, QStandardItem +from PyQt5.Qt import QStyle + +from PyQt5.QtCore import ( + Qt, + QDir, + QThread, + QSignalBlocker, + pyqtSlot, + pyqtSignal, + QObject, + QRegularExpression, + QModelIndex, + QSortFilterProxyModel, +) + +from PyQt5.QtWidgets import ( + QApplication, + QMainWindow, + QWidget, + QTreeView, + QTreeWidget, + QTreeWidgetItem, + QTreeWidgetItemIterator, + QHeaderView, + QGridLayout, + QHBoxLayout, + QPushButton, + QLineEdit, + QProgressBar, + QTabWidget, + QPlainTextEdit, + QFileSystemModel, + QFileDialog, + QCheckBox, +) + +import moodle + +log = logging.getLogger("muddle.gui") + +class MoodleItem(QStandardItem): + class Type(enum.IntEnum): + ROOT = 0 + # root + COURSE = 1 + # sections + SECTION = 2 + # modules + MODULE = 3 + ## specific module types + FORUM = 4 + RESOURCE = 5 + FOLDER = 6 + ATTENDANCE = 7 + LABEL = 8 + QUIZ = 9 + # contents + CONTENT = 10 + ## specific content types + FILE = 11 + URL = 12 + + class Metadata: + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + def __init__(self, parent, nodetype, leaves=[], **kwargs): + super().__init__(parent) + self.metadata = MoodleItem.Metadata(type = nodetype, **kwargs) + + # set icon + icons = { + MoodleItem.Type.COURSE : QStyle.SP_DriveNetIcon, + MoodleItem.Type.FOLDER : QStyle.SP_DirIcon, + MoodleItem.Type.RESOURCE : QStyle.SP_DirLinkIcon, + MoodleItem.Type.FILE : QStyle.SP_FileIcon, + MoodleItem.Type.URL : QStyle.SP_FileLinkIcon, + } + + if self.metadata.type in icons.keys(): + self.setIcon(QApplication.style().standardIcon(icons[self.metadata.type])) + else: + # remove icon, because otherwise it inherits the parent's icon + self.setIcon(QIcon()) + + if self.metadata.type in [ MoodleItem.Type.FILE, MoodleItem.Type.FOLDER, MoodleItem.Type.RESOURCE ]: + # NOTE: because of a Qt Bug setAutoTristate does not work + # the tri-state behavior is implemented below in + # MuddleWindow.onMoodleTreeModelDataChanged() + self.setCheckable(True) + self.setCheckState(Qt.Unchecked) + + self.setEditable(False) + self.setText(html.unescape(self.metadata.title)) + + +class MoodleFetcher(QThread): + loadedItem = pyqtSignal(MoodleItem.Type, object) + + def __init__(self, parent, instanceUrl, token): + super().__init__() + + self.api = moodle.RestApi(instanceUrl, token) + self.apihelper = moodle.ApiHelper(self.api) + + def run(self): + for course in self.getCourses(): + self.loadedItem.emit(MoodleItem.Type.COURSE, course) + for section in self.getSections(course): + self.loadedItem.emit(MoodleItem.Type.SECTION, section) + for module in self.getModules(section): + self.loadedItem.emit(MoodleItem.Type.MODULE, module) + for content in self.getContent(module): + self.loadedItem.emit(MoodleItem.Type.CONTENT, content) + + def getCourses(self): + coursesReq = self.api.core_enrol_get_users_courses(userid = self.apihelper.get_userid()) + if not coursesReq: + return [] + + return coursesReq.json() + + def getSections(self, course): + if not "id" in course: + log.error("cannot get sections from invalid course (no id)") + log.debug(course) + return [] + + sectionsReq = self.api.core_course_get_contents(courseid = str(course["id"])) + if not sectionsReq: + return [] + + sections = sectionsReq.json() + return sections + + def getModules(self, section): + if "modules" in section: + return section["modules"] + else: + return [] + + def getContent(self, module): + if "contents" in module: + return module["contents"] + else: + return [] + + +class MoodleTreeFilterModel(QSortFilterProxyModel): + def __init__(self): + super().__init__() + + +class MoodleTreeModel(QStandardItemModel): + def __init__(self): + super().__init__() + + self.setHorizontalHeaderLabels(["Item", "Size"]) + self.lastInsertedItem = None + self.worker = None + + @pyqtSlot(str, str) + def refresh(self, instanceUrl, token): + if not self.worker or self.worker.isFinished(): + self.setRowCount(0) # instead of clear(), because clear() removes the headers + + self.worker = MoodleFetcher(self, instanceUrl, token) + self.worker.loadedItem.connect(self.onWorkerLoadedItem) + self.worker.finished.connect(self.onWorkerDone) + self.worker.start() + else: + log.debug("A worker is already running, not refreshing") + + @pyqtSlot(MoodleItem.Type, object) + def onWorkerLoadedItem(self, type, item): + # Assume that the items arrive in order + moodleItem = None + parent = None + + # if top level + if type == MoodleItem.Type.COURSE: + moodleItem = MoodleItem( + parent = parent, + nodetype = type, + id = item["id"], + title = item["shortname"]) + + self.invisibleRootItem().insertRow(0, moodleItem) + self.lastInsertedItem = moodleItem + + return + + # otherwise + parent = self.lastInsertedItem + while type <= parent.metadata.type and parent.parent(): + parent = parent.parent() + + if type == MoodleItem.Type.SECTION: + moodleItem = MoodleItem( + parent = parent, + nodetype = type, + id = item["id"], + title = item["name"]) + + elif type == MoodleItem.Type.MODULE: + moduleType = { + "folder" : MoodleItem.Type.FOLDER, + "resource" : MoodleItem.Type.RESOURCE, + "forum" : MoodleItem.Type.FORUM, + "attendance" : MoodleItem.Type.ATTENDANCE, + "label" : MoodleItem.Type.LABEL, + "quiz" : MoodleItem.Type.QUIZ, + } + + moodleItem = MoodleItem( + parent = parent, + nodetype = moduleType.get(item["modname"]) or type, + id = item["id"], + title = item["name"]) + + elif type == MoodleItem.Type.CONTENT: + contentType = { + "url" : MoodleItem.Type.URL, + "file" : MoodleItem.Type.FILE, + } + + moodleItem = MoodleItem( + parent = parent, + nodetype = contentType.get(item["type"]) or type, + title = item["filename"], + url = item["fileurl"]) + + if not moodleItem: + log.error(f"Could not load item of type {type}") + return + + parent.insertRow(0, moodleItem) + self.lastInsertedItem = moodleItem + + @pyqtSlot() + def onWorkerDone(self): + log.debug("worker done") + + +class QLogHandler(QObject, logging.Handler): + newLogMessage = pyqtSignal(str) + + def emit(self, record): + msg = self.format(record) + self.newLogMessage.emit(msg) + + def write(self, m): + pass + + +class MuddleWindow(QMainWindow): + def __init__(self, config): + super(MuddleWindow, self).__init__() + uic.loadUi("muddle/muddle.ui", self) + self.setCentralWidget(self.findChild(QTabWidget, "Muddle")) + + self.instanceUrl = config["server"]["url"] if config.has_option("server", "url") else None + self.token = config["server"]["token"] if config.has_option("server", "token") else None + + # config tab + ## TODO: when any of the settings change, update the values (but not in the config, yet) + + instanceUrlEdit = self.findChild(QLineEdit, "instanceUrlEdit") + if self.instanceUrl: + instanceUrlEdit.setText(self.instanceUrl) + + tokenEdit = self.findChild(QLineEdit, "tokenEdit") + if self.token: + tokenEdit.setText(self.token) + + requestTokenBtn = self.findChild(QPushButton, "requestTokenBtn") + requestTokenBtn.clicked.connect(self.onRequestTokenBtnClicked) + + tokenEdit.textEdited.connect(lambda text: requestTokenBtn.setEnabled(not bool(text))) + + alwaysStartGuiCheckBox = self.findChild(QCheckBox, "alwaysStartGuiCheckBox") + if config.has_option("muddle", "always_run_gui"): + alwaysStartGuiCheckBox.setChecked(config.getboolean("muddle", "always_run_gui")) + + configEdit = self.findChild(QLineEdit, "configEdit") + configEdit.setText(config["runtime_data"]["config_path"]) + + defaultDownloadPathEdit = self.findChild(QLineEdit, "defaultDownloadPathEdit") + if config.has_option("muddle", "default_download_dir"): + defaultDownloadPathEdit.setText(config["muddle"]["default_download_dir"]) + + + # log tab + ## setup logging + self.loghandler = QLogHandler(self) + self.loghandler.setFormatter(logging.Formatter("%(name)s - %(levelname)s - %(message)s")) + self.loghandler.newLogMessage.connect(self.onNewLogMessage) + logging.getLogger("muddle").addHandler(self.loghandler) + + # moodle tab + ## set up proxymodel for moodle treeview + self.moodleTreeModel = MoodleTreeModel() + self.moodleTreeModel.dataChanged.connect(self.onMoodleTreeModelDataChanged) + + self.filterModel = MoodleTreeFilterModel() + self.filterModel.setRecursiveFilteringEnabled(True) + self.filterModel.setDynamicSortFilter(True) + self.filterModel.setSourceModel(self.moodleTreeModel) + + moodleTreeView = self.findChild(QTreeView, "moodleTree") + moodleTreeView.setModel(self.filterModel) + moodleTreeView.setSortingEnabled(True) + moodleTreeView.sortByColumn(0, Qt.AscendingOrder) + ## TODO: change with minimumSize (?) + moodleTreeView.header().setSectionResizeMode(0, QHeaderView.ResizeToContents) + moodleTreeView.doubleClicked.connect(self.onMoodleTreeViewDoubleClicked) + + ## refresh moodle treeview + refreshBtn = self.findChild(QPushButton, "refreshBtn") + refreshBtn.clicked.connect(self.onRefreshBtnClicked) + + if not self.instanceUrl: + refreshBtn.setEnabled(False) + log.warning("no server url configured!") + + if not self.token: + refreshBtn.setEnabled(False) + log.warning("no server token configured!") + + + ## searchbar + searchBar = self.findChild(QLineEdit, "searchBar") + searchBar.textChanged.connect(self.onSearchBarTextChanged) + searchBar.textEdited.connect(self.onSearchBarTextChanged) + + ## select path + selectPathBtn = self.findChild(QPushButton, "selectPathBtn") + selectPathBtn.clicked.connect(self.onSelectPathBtnClicked) + + ## progressbar + self.progressBar = self.findChild(QProgressBar, "downloadProgressBar") + + # self.moodleTreeModel.worker.loaded + # self.moodleTreeModel.worker.loadedItem.connect(lambda t, item:) + + # local filesystem tab + self.downloadPath = QDir.homePath() + + self.fileSystemModel = QFileSystemModel() + self.fileSystemModel.setRootPath(QDir.homePath()) + + localTreeView = self.findChild(QTreeView, "localTab") + localTreeView.setModel(self.fileSystemModel) + localTreeView.setRootIndex(self.fileSystemModel.index(QDir.homePath())) + localTreeView.header().setSectionResizeMode(0, QHeaderView.ResizeToContents) + + downloadPathEdit = self.findChild(QLineEdit, "downloadPathEdit") + downloadPathEdit.setText(self.downloadPath) + downloadPathEdit.editingFinished.connect(self.onDownloadPathEditEditingFinished) + + self.show() + + @pyqtSlot(int) + def setProgressBarTasks(self, nrTasks): + self.progressBar.setMinimum(0) + self.progressBar.setMaximum(tasks) + self.progressBar.reset() + + @pyqtSlot() + def advanceProgressBar(self): + currentValue = self.progressBar.value() + self.progressBar.setValue(currentValue + 1); + + @pyqtSlot(int) + def setProgressBarValue(self, value): + self.progressBar.setValue(value) + + @pyqtSlot() + def onRequestTokenBtnClicked(self): + # TODO: open login dialog + # TODO: test and maybe check if there is already a token + # req = moodle.request_token(self.instance_url, user, password) + pass + + @pyqtSlot(str) + def onSearchBarTextChanged(self, text): + moodleTreeView = self.findChild(QTreeView, "moodleTree") + searchBar = self.findChild(QLineEdit, "searchBar") + + if not text: + self.filterModel.setFilterRegularExpression(".*") + moodleTreeView.collapseAll() + searchBar.setStyleSheet("") + else: + regexp = QRegularExpression(text) + if regexp.isValid(): + self.filterModel.setFilterRegularExpression(regexp) + moodleTreeView.expandAll() + searchBar.setStyleSheet("") + else: + log.debug("invalid search regular expression, not searching") + searchBar.setStyleSheet("QLineEdit { color: red; }") + + @pyqtSlot(str) + def onNewLogMessage(self, msg): + self.findChild(QPlainTextEdit, "logsTab").appendPlainText(msg) + + @pyqtSlot() + def onDownloadPathEditEditingFinished(self): + downloadPathEdit = self.findChild(QLineEdit, "downloadPathEdit") + path = downloadPathEdit.text() + + if not self.updateDownloadPath(path): + downloadPathEdit.setText(self.downloadPath) + + @pyqtSlot() + def onSelectPathBtnClicked(self): + path = QFileDialog.getExistingDirectory( + self, "Select Download Directory", + self.downloadPath, QFileDialog.ShowDirsOnly | QFileDialog.DontResolveSymlinks) + + if not path: + return + + self.updateDownloadPath(path) + + @pyqtSlot() + def onRefreshBtnClicked(self): + if self.instanceUrl and self.token: + self.moodleTreeModel.refresh(self.instanceUrl, self.token) + else: + # TODO: implement error dialog + pass + + @pyqtSlot() + def updateDownloadPath(self, newpath): + if not self.fileSystemModel.index(newpath).isValid(): + return False + + self.downloadPath = newpath + + downloadPathEdit = self.findChild(QLineEdit, "downloadPathEdit") + localTreeView = self.findChild(QTreeView, "localTab") + + downloadPathEdit.setText(self.downloadPath) + localTreeView.setRootIndex(self.fileSystemModel.index(self.downloadPath)) + + @pyqtSlot(QModelIndex) + def onMoodleTreeViewDoubleClicked(self, index): + realIndex = self.filterModel.mapToSource(index) + item = self.moodleTreeModel.itemFromIndex(realIndex) + + if item.metadata.type == MoodleItem.Type.FILE: + log.debug(f"started download from {item.metadata.url}") + + filepath = tempfile.gettempdir()+"/"+item.metadata.title + self.moodleTreeModel.worker.apihelper.get_file(item.metadata.url, filepath) + + if platform.system() == 'Darwin': # macOS + subprocess.Popen(('open', filepath)) + elif platform.system() == 'Windows': # Windows + os.startfile(filepath) + else: # linux variants + subprocess.Popen(('xdg-open', filepath)) + + # this is here to emulate the behavior of setAutoTristate which does not + # work because of a Qt Bug + @pyqtSlot(QModelIndex, QModelIndex) + def onMoodleTreeModelDataChanged(self, topLeft, bottomRight): + # TODO: this can probably be moved in Item.setData() by creating AutoTriStateRole + item = self.moodleTreeModel.itemFromIndex(topLeft) + + if item.hasChildren(): + for i in range(0, item.rowCount()): + # NOTE: this causes the child to emit a signal, which + # automatically causes to recursively set its children + item.child(i).setCheckState(item.checkState()) + + +def start(config): + app = QApplication(sys.argv) + ex = MuddleWindow(config) + sys.exit(app.exec_()) diff --git a/muddle/moodle.py b/muddle/moodle.py new file mode 100644 index 0000000..7f3ccf5 --- /dev/null +++ b/muddle/moodle.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +import requests +import logging + +log = logging.getLogger("muddle.moodle") + +# +# magic moodle api wrapper +# + + +def request_token(url, user, password): + token_url = f"{url}/login/token.php" + data = { + "username": user, + "password": password, + "service": "moodle_mobile_app" + } + log.debug(f"requesting token with POST to {api_url} with DATA {data}") + return requests.post(token_url, data=data) + + +def api_call(url, token, function, **kwargs): + api_url = f"{url}/webservice/rest/server.php?moodlewsrestformat=json" + data = {"wstoken": token, "wsfunction": function} + for k, v in kwargs.items(): + data[str(k)] = v + + log.debug(f"calling api with POST to {api_url} with DATA {data}") + try: + req = requests.post(api_url, data=data) + req.raise_for_status() + return req + except requests.HTTPError: + log.warn(f"Error code returned by HTTP(s) request") + return req + except (requests.ConnectionError, requests.Timeout, requests.ReadTimeout) as e: + log.error(f"Failed to connect for POST request:\n{str(e)}") + return None + + +class RestApi: + def __init__(self, instance_url, token): + self._url = instance_url + self._token = token + + def __getattr__(self, key): + return lambda **kwargs: api_call(self._url, self._token, str(key), **kwargs) + + +class ApiHelper: + def __init__(self, api): + self.api = api + + def get_userid(self): + req = self.api.core_webservice_get_site_info() + if req: + return req.json()["userid"] + else: + return None + + def get_file(self, url, local_path): + with requests.post(url, data={"token": self.api._token}, stream=True) as r: + r.raise_for_status() + with open(local_path, "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + if chunk: + f.write(chunk) diff --git a/muddle/muddle.ui b/muddle/muddle.ui new file mode 100644 index 0000000..f849116 --- /dev/null +++ b/muddle/muddle.ui @@ -0,0 +1,259 @@ + + + MuddleWindow + + + + 0 + 0 + 600 + 750 + + + + Muddle + + + + + 0 + 0 + + + + + 400 + 600 + + + + TabWidget + + + 0 + + + + Moodle + + + + + + true + + + + + + + false + + + Download + + + + + + + true + + + + + + Search (regexp) + + + true + + + + + + + true + + + 0 + + + + + + + Refresh + + + + + + + true + + + + 0 + 0 + + + + Select + + + + + + + false + + + + + + + + Local + + + + + + 0 + 0 + + + + QFrame::Sunken + + + false + + + + + + Qt::TextSelectableByKeyboard|Qt::TextSelectableByMouse + + + Logs + + + + + Settings + + + + + + Moodle + + + + + + Instance URL + + + + + + + true + + + + + + + Token + + + + + + + true + + + + + + + false + + + Request Token + + + + + + + + + + Muddle + + + + + + Config + + + + + + + true + + + + + + + Default download path + + + + + + + Not set + + + true + + + + + + + Always start GUI + + + + + + + + + + + + + 0 + 0 + 600 + 23 + + + + + + + + -- cgit v1.2.1