From 0ee02e4a7e7b771dcbac5ca121be98f91ce45fd2 Mon Sep 17 00:00:00 2001 From: Mattia Mascarello Date: Mon, 1 May 2023 20:20:12 +0200 Subject: [PATCH] Unnamed sections not showing bug fixed, heuristic folder naming --- FSNode.py | 187 +++++++++++++++++++++++++++++++++++++++++------------- main.py | 4 +- moodle.py | 7 +- 3 files changed, 150 insertions(+), 48 deletions(-) diff --git a/FSNode.py b/FSNode.py index adf9cf9..273dec0 100644 --- a/FSNode.py +++ b/FSNode.py @@ -7,16 +7,74 @@ from markdownify import markdownify as md from moodle import Category, Course, Moodle, Section, Folder, File, Label, Url +import unicodedata +import re + + +def slugify(value, allow_unicode=False): + """ + Taken from https://github.com/django/django/blob/master/django/utils/text.py + Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated + dashes to single dashes. Remove characters that aren't alphanumerics, + underscores, or hyphens. Convert to lowercase. Also strip leading and + trailing whitespace, dashes, and underscores. + """ + value = str(value) + s = value.rsplit(".", 1) # split into name and extension + ext = "" + if len(s) == 2: + value = s[0] + ext = "." + slugify(s[1]) + if allow_unicode: + value = unicodedata.normalize('NFKC', value) + else: + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') + value = re.sub(r'[^\w\s-]', '', value.lower()) + res = re.sub(r'[-\s]+', '-', value).strip('-_') + file = res + ext + return file + + +class LinkTo: + pass + + +class UrlLink(LinkTo): + def __init__(self, url: str): + self.url = url + + +class FileLink(LinkTo): + def __init__(self, url: str): + self.url = url + + +class HTMLContentMap(LinkTo): + def __init__(self): + self.htmlMap = {} + + def add(self, html: str, label_id: int): + self.htmlMap[label_id] = md(html) + + def sort(self): + self.htmlMap = dict(sorted(self.htmlMap.items())) + + def markdown_make(self) -> str: + return "\n".join([self.htmlMap[key] for key in self.htmlMap]) + + fileCache = {} + # Virtual Filesystem node class FSNode: - def __init__(self, name: str, is_dir=True): - self.name = name + def __init__(self, name: str, parent, is_dir=True): + self.name = slugify(name) + self.parent = parent self.children = [] self.is_dir = is_dir self.size = 0 - self.url = None + self.linkTo = None def to_stat_struct(self): if self.is_dir: @@ -38,6 +96,12 @@ class FSNode: st_size=self.size ) + def find_child_by_name(self, name: str): + for el in self.children: + if el.name == name: + return el + return None + def resolve_path(self, path: str): if path == "/": return self @@ -53,55 +117,83 @@ class FSNode: return None @staticmethod - def from_category(c: Category, m: Moodle): - f = FSNode(c.name, True) + def from_category(c: Category, parent, m: Moodle): + f = FSNode(c.name, parent, True) for course in c.courses: - f.children.append(FSNode.from_course(course, m)) + f.children.append(FSNode.from_course(course, f, m)) return f @staticmethod - def from_course(c: Course, m: Moodle): - f = FSNode(c.shortname, True) + def from_course(c: Course, parent, m: Moodle): + f = FSNode(c.shortname, parent, True) for s in m.get_course_content(c.id): - f.children.append(FSNode.from_section(s, m)) + f.children.append(FSNode.from_section(s, f, m)) return f @staticmethod - def from_section(s: Section, m: Moodle): - f = FSNode(s.name, True) - for mo in s.modules: - f.children.append(FSNode.from_module(mo, m)) + def from_section(s: Section, parent, m: Moodle): + f = FSNode(s.name, parent, True) if len(s.htmlcontent): - m = FSNode("README.md", False) - m.size = len(s.htmlcontent) - m.url = "html#" + s.htmlcontent + m = FSNode("README.md", f, False) + m.linkTo = HTMLContentMap() + m.linkTo.add(s.htmlcontent, s.id) + m.size = len(m.linkTo.markdown_make()) f.children.append(m) + for mo in s.modules: + el = FSNode.from_module(mo, f, m) + if el is not None: + f.children.append(el) + if s.autoName: # Heuristic name generation + q = f.find_child_by_name(slugify("README.md")) + if q is not None: + md = q.linkTo.markdown_make() + lines = md.split("\n") + line = lines[0] + i = 1 + while not len(re.sub(r'\W+', '', line)) and i < len(lines): + line = lines[i] + i += 1 + if i != len(lines): + title = slugify(line) + # truncate title to 30 chars + title = title[:30] + f.name = title return f @staticmethod - def from_module(mo, m: Moodle): + def from_module(mo, parent, m: Moodle): if isinstance(mo, Folder): - f = FSNode(mo.name, True) + f = FSNode(mo.name, parent, True) for el in mo.files: - r = FSNode.from_module(el, m) + r = FSNode.from_module(el, f, m) if r is not None: f.children.append(r) return f elif isinstance(mo, File): - return FSNode.from_file(mo, m) + return FSNode.from_file(mo, parent, m) elif isinstance(mo, Label): - return FSNode.from_label(mo, m) + FSNode.from_label(mo, parent, m) elif isinstance(mo, Url): - return FSNode.from_url(mo, m) + return FSNode.from_url(mo, parent, m) return None @staticmethod - def from_file(el: File, m: Moodle): - f = FSNode(el.name, False) + def from_file(el: File, parent, m: Moodle): + f = FSNode(el.name, parent, False) f.size = el.filesize - f.url = el.fileurl + f.linkTo = FileLink(el.fileurl) return f + @staticmethod + def hash_link_to(a: LinkTo): + if isinstance(a, UrlLink): + return "url#" + a.url + elif isinstance(a, FileLink): + return a.url + elif isinstance(a, HTMLContentMap): + return "html#" + str(a.htmlMap.keys()) + return None + def read(self, size, offset, mo: Moodle): global fileCache if sys.getsizeof(fileCache) > 3 * 100000000 and len(fileCache) > 2: @@ -109,33 +201,40 @@ class FSNode: for k in list(d.keys())[:len(d) // 2]: del fileCache[k] del d - if self.url not in fileCache.keys(): - if self.url.startswith("html#"): - fileCache[self.url] = {"datetime": time.time(), "content": md(self.url[5:]).encode()} - elif self.url.startswith("url#"): - fileCache[self.url] = {"datetime": time.time(), "content": self.url[4:].encode()} - else: - r = requests.get(self.url + "&token=" + mo.token) - fileCache[self.url] = {"datetime": time.time(), "content": r.content} - return fileCache[self.url]["content"][offset:offset + size] + link_hash = FSNode.hash_link_to(self.linkTo) + if link_hash not in fileCache.keys(): + if isinstance(self.linkTo, HTMLContentMap): + fileCache[self.linkTo] = {"datetime": time.time(), "content": self.linkTo.markdown_make().encode()} + elif isinstance(self.linkTo, UrlLink): + fileCache[self.linkTo] = {"datetime": time.time(), "content": self.linkTo.url.encode()} + elif isinstance(self.linkTo, FileLink): + r = requests.get(self.linkTo.url + "&token=" + mo.token) + fileCache[self.linkTo] = {"datetime": time.time(), "content": r.content} + return fileCache[self.linkTo]["content"][offset:offset + size] @staticmethod def from_moodle(moodle: Moodle): - f = FSNode("/", True) + f = FSNode("/", None, True) for cat in moodle.get_enrolled_categories(): - f.children.append(FSNode.from_category(cat, moodle)) + f.children.append(FSNode.from_category(cat, f, moodle)) return f @staticmethod - def from_label(mo: Label, m): - f = FSNode(mo.name, False) - f.size = len(mo.htmlcontent) - f.url = "html#" + mo.htmlcontent - return f + def from_label(mo: Label, parent, m) -> None: + f = parent.find_child_by_name(slugify("README.md")) + if f is not None: + f.linkTo.add(mo.htmlcontent, mo.id) + else: + f = FSNode(slugify("README.md"), parent, False) + f.linkTo = HTMLContentMap() + f.linkTo.add(mo.htmlcontent, mo.id) + f.size = len(f.linkTo.markdown_make()) + parent.children.append(f) + return None @staticmethod - def from_url(mo: Url, m): - f = FSNode(mo.name, False) + def from_url(mo: Url, parent, m): + f = FSNode(mo.name, parent, False) f.size = len(mo.url) - f.url = "url#" + mo.url + f.linkTo = UrlLink(mo.url) return f diff --git a/main.py b/main.py index 53be22a..1e9704c 100644 --- a/main.py +++ b/main.py @@ -24,8 +24,6 @@ MOUNT = os.getenv('MOUNT') m = Moodle(SITE, MOODLE_USERNAME, PASSWORD) m.login() -print(m.get_user().__dict__) - -logging.basicConfig(level=logging.DEBUG) +#logging.basicConfig(level=logging.DEBUG) if __name__ == '__main__': FUSE(MoodleFS(m), MOUNT, foreground=True, allow_other=True) diff --git a/moodle.py b/moodle.py index 6f74921..94cc76c 100644 --- a/moodle.py +++ b/moodle.py @@ -1,3 +1,5 @@ +import re + import requests # a minimalistic moodle api wrapper, just for the purpose of this project @@ -25,8 +27,10 @@ class Course: class Section: def __init__(self, id_i: int, name: str, htmlcontent: str, modules: list): self.id = id_i - if len(name.strip()) == 0: + self.autoName = False + if len(re.sub(r'\W+', '', name)) == 0: name = "Section " + str(id_i) + self.autoName = True self.name = name self.htmlcontent = htmlcontent self.modules = modules @@ -86,6 +90,7 @@ class Moodle: raise Exception(r["error"]) self.token = r["token"] self.private_token = r["privatetoken"] + self.get_user() def get_user(self) -> MoodleUser: url = self.site_url + "/webservice/rest/server.php?moodlewsrestformat=json"