import glob import os # from collections import deque # import toml # base_url = toml.load('settings.toml')["general"]["base_url"] # def sort_branch( lst ): # ''' # put directories in front of files and # sorts both alphabetically # ''' # files = [] # dirs = [] # for path in lst: # if path.endswith(".md"): # files.append(path) # else: # dirs.append(path) # files.sort() # dirs.sort() # return [*dirs, *files] def sort_branch(lst): """ Puts directories in front of files and sorts both alphabetically. """ files = [path for path in lst if path.endswith(".md")] dirs = [path for path in lst if not path.endswith(".md")] files.sort() dirs.sort() return dirs + files # def make_tree(path="templates/content"): # ''' # creates a dictionary of the directories and files of the doc repository # ''' # if '/' in path: # tree = dict(name=path, children=[], type="directory") # else: # pass # try: lst = os.listdir(path) # except OSError: # pass #ignore errors # else: # lst = sort_branch(lst) # for name in lst: # fn = os.path.join(path, name) # if os.path.isdir(fn) and not fn.endswith(".git"): # if os.listdir(fn): # this line is experimental and needs to be tested # tree['children'].append(make_tree(fn)) # else: # if fn.endswith('.md'): # tree['children'].append(dict(name=fn)) # return tree def make_tree(path="templates/content"): """ Creates a dictionary of the directories and files of the doc repository. """ tree = {'name': path, 'children': [], 'type': 'directory'} try: lst = os.listdir(path) lst = sort_branch(lst) for name in lst: fn = os.path.join(path, name) if os.path.isdir(fn) and not fn.endswith(".git") and os.listdir(fn): tree['children'].append(make_tree(fn)) elif fn.endswith('.md'): tree['children'].append({'name': fn}) except OSError: pass # Ignore errors return tree def rem_readme(path, tree): ''' This functions puts out the path of the README.md and which will be index.html path can be dynamically configured and may have a / at the end nor not ''' index = f"{path.rstrip('/')}/README.md" for item in tree['children']: if isinstance(item, dict): for k, v in item.items(): if k == "name": if v == index: del tree['children'][-1][k] return tree def cut_path_tree(tree, subdir, file_ending): ''' pruning of the tree structure from make_tree() ''' if subdir.endswith('/'): subdir = subdir[:-1] for key, val in tree.items(): if isinstance(val, list): for item in val: item["name"] = item["name"][len(subdir):] if item["name"].endswith(file_ending): item["name"] = item["name"][:-len(file_ending)] if isinstance(item, dict): cut_path_tree(item, subdir, file_ending) return tree def cut_filetype_tree(tree, filetype): ''' removes file type of the links stored in make_tree() ''' for key, val in tree.items(): if isinstance(val, list): for item in val: if item["name"].endswith(filetype): item["name"] = item["name"][:-len(filetype)] if isinstance(item, dict): cut_filetype_tree(item, filetype) return tree # def list_files(path): # ''' # creates a simple, one dimensional list of the doc repository # filters markdown files only # ''' # doc_files = [] # for root, dirs, files in os.walk(path): # for file in files: # if file.endswith(".md") and not ".git" in root: # doc_files.append(os.path.join(root, file)) # return doc_files def list_files(path): """ Creates a simple, one-dimensional list of the doc repository. Filters markdown files only. """ return [ file for file in glob.glob( os.path.join(path, '**/*.md'), recursive=True ) if '.git' not in file ] def build_index(path, file_ending): """ Builds the searchable JSON object containing all markdown files with metadata. """ searchable = {'index': []} file_list = list_files(path) for item in file_list: if item.endswith(file_ending): with open(item, 'r') as file: data = file.readlines() title = data[0].strip('# \n') uri = '/' + item[len(path):-len(file_ending)] + '.html' content = ''.join(data) searchable['index'].append({ 'uri': uri, 'title': title, 'tags': [], 'content': content, 'description': '' }) elif isinstance(item, dict): build_index(item, file_ending) return searchable