From b02b7fbf59e124a0ae5b22493d9f8528c854660a Mon Sep 17 00:00:00 2001
From: rugk <rugk+git@posteo.de>
Date: Thu, 16 May 2019 09:22:01 +0200
Subject: [PATCH 01/19] Add auto detection for dark browser/system mode

via "prefers-color-scheme".
Fixes https://github.com/idank/explainshell/issues/236
---
 explainshell/web/static/js/es.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/explainshell/web/static/js/es.js b/explainshell/web/static/js/es.js
index 40760c7d..8965f2df 100644
--- a/explainshell/web/static/js/es.js
+++ b/explainshell/web/static/js/es.js
@@ -1081,7 +1081,10 @@ function draw() {
 
 // Theme-related stuff
 $(document).ready(function() {
-    var selectedTheme = localStorage.getItem('theme') || 'default';
+    // use theme from local storage or auto-detect otherwise
+    var selectedTheme = localStorage.getItem('theme')
+        || (window.matchMedia("(prefers-color-scheme: dark)").matches ? 'dark' : 'default')
+        || 'default';
 
     function setTheme(theme) {
         console.log('setting theme to', theme);

From 23f61f734409df8298450bdd487f01115fb9f0f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=A1=D1=82=D1=80=D0=B0=D1=85=D0=B8=D1=9A=D0=B0=20=D0=A0?=
 =?UTF-8?q?=D0=B0=D0=B4=D0=B8=D1=9B?= <contact@strahinja.org>
Date: Thu, 20 Jan 2022 13:04:54 +0100
Subject: [PATCH 02/19] Change the example with the "useless use of echo" and
 backticks to similar
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Страхиња Радић <contact@strahinja.org>
---
 explainshell/web/templates/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/explainshell/web/templates/index.html b/explainshell/web/templates/index.html
index c75dcbd7..03b672c8 100644
--- a/explainshell/web/templates/index.html
+++ b/explainshell/web/templates/index.html
@@ -26,7 +26,7 @@ <h3>examples</h3>
                     <ul>
                         {{ macros.examplebullet(":(){ :|:& };:") }}
                         {{ macros.examplebullet("for user in $(cut -f1 -d: /etc/passwd); do crontab -u $user -l 2>/dev/null; done") }}
-                        {{ macros.examplebullet('file=$(echo `basename "$file"`)') }}
+                        {{ macros.examplebullet('name=$(printf "%s@%s" "$(id -nu)" "$(uname -n)")') }}
                         {{ macros.examplebullet("true && { echo success; } || { echo failed; }") }}
                         {{ macros.examplebullet("cut -d ' ' -f 1 /var/log/apache2/access_logs | uniq -c | sort -n") }}
                         {{ macros.examplebullet('tar zcf - some-dir | ssh some-server "cd /; tar xvzf -"') }}

From 354ecbb5b1ccf1e4d14e0dea477a42fe377d851e Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 14 Sep 2024 21:54:23 +0000
Subject: [PATCH 03/19] chore: init .dockerignore file

---
 .dockerignore | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 .dockerignore

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000..c1b1f108
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,15 @@
+.gitignore
+.git/
+.github/
+misc/
+tests/
+tools
+venv/
+dump/
+.mpypy_cache/
+*.pyc
+*.log
+README.md
+docker-compose.yml
+Dockerfile
+Makefile

From 0d2235999eab82713a6abe43211dd81beee3fc2c Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 14 Sep 2024 21:54:40 +0000
Subject: [PATCH 04/19] chore: extend .gitignore file

---
 .gitignore | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index af4172ea..a8672f68 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,7 @@
 *.swp
 .coverage
 .vagrant
-application.log
\ No newline at end of file
+*.log
+venv/
+__pycache__
+.mpypy_cache/
\ No newline at end of file

From 1fd151deb4c5cd7625ecbe055027b4929ead1ec0 Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 14 Sep 2024 21:55:19 +0000
Subject: [PATCH 05/19] chore: update docker-compose.yml contents

---
 docker-compose.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 686b0309..6dbfb152 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,10 +1,10 @@
-version: '2'
 services:
   db:
     image: mongo
   web:
-    build: .
-    command: make serve
+    build: 
+      context: .
+      dockerfile: Dockerfile
     environment:
       - MONGO_URI=mongodb://db
       - HOST_IP=0.0.0.0

From a6c8c11176ec8350182bb518b15015398ff34de4 Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 14 Sep 2024 21:55:54 +0000
Subject: [PATCH 06/19] chore: upgrade to Python 3.12 and reduce layers

---
 Dockerfile | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 22f77314..2344026a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,18 +1,15 @@
-FROM python:2.7
+FROM python:3.12
 
-RUN apt-get update \
-  && apt-get install man-db -y \
-  && apt-get clean
+RUN apt update \
+  && apt install man-db -y \
+  && apt clean
 
-ADD ./requirements.txt /tmp/requirements.txt
+WORKDIR /opt/webapp
+COPY . .
 
-RUN pip install --upgrade pip \
-  && python --version \
-  && pip install -r /tmp/requirements.txt \
-  && rm -rf ~/.cache/pip/*
+RUN pip3 install --no-cache-dir --no-warn-script-location --upgrade pip setuptools wheel virtualenv \
+  && pip3 install --no-cache-dir --no-warn-script-location -r requirements.txt
 
-ADD ./ /opt/webapp/
-WORKDIR /opt/webapp
 EXPOSE 5000
 
-CMD ["make", "serve"]
+CMD ["python3", "runserver.py"]

From 1eb8688778fd876b55c52ffe65a1987673efc0c3 Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 14 Sep 2024 21:56:12 +0000
Subject: [PATCH 07/19] chore: update serve command in Makefile

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 49856bce..a15e07a1 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,6 @@ tests:
 	nosetests --exe --with-doctest tests/ explainshell/
 
 serve:
-	python runserver.py
+	docker-compose up --build
 
 .PHONY: tests

From 5626ba4650b36b687ce93e02efb5c6929be968a3 Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 14 Sep 2024 21:56:27 +0000
Subject: [PATCH 08/19] chore(deps): update pip dependencies

---
 requirements.txt | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 463149c1..eb5ced45 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,5 @@
-Flask==0.12
-nltk==3.4.5
-MarkupSafe==1.1.1
-nose==1.3.0
-pymongo==3.13.0
-bashlex==0.12
+Flask==3.0.3
+nltk==3.9.1
+nose==1.3.7
+pymongo==4.8.0
+bashlex==0.18

From a302a61b2f18fed61583d5c44d8c3978503ca799 Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 14 Sep 2024 21:58:23 +0000
Subject: [PATCH 09/19] chore: upgrade Python code to Python 3 and enhance
 readability

---
 explainshell/algo/classifier.py               |   97 +-
 explainshell/algo/features.py                 |   32 +-
 explainshell/config.py                        |   54 +-
 explainshell/errors.py                        |    1 +
 explainshell/fixer.py                         |  159 +--
 .../{helpconstants.py => help_constants.py}   |  275 +++--
 explainshell/manager.py                       |  218 ++--
 explainshell/manpage.py                       |  257 +++--
 explainshell/matcher.py                       |  605 +++++-----
 explainshell/options.py                       |  134 ++-
 explainshell/store.py                         |  440 +++++---
 explainshell/util.py                          |   77 +-
 explainshell/web/__init__.py                  |    3 +-
 explainshell/web/debug_views.py               |   88 ++
 explainshell/web/debugviews.py                |   85 --
 explainshell/web/helpers.py                   |   29 +-
 explainshell/web/views.py                     |  328 +++---
 runserver.py                                  |    6 +-
 tests/helpers.py                              |   78 +-
 tests/test-fixer.py                           |   64 +-
 tests/test-integration.py                     |   41 +-
 tests/test-manager.py                         |  151 +--
 tests/test-manpage.py                         |   35 +-
 tests/test-matcher.py                         | 1001 ++++++++++-------
 tests/test-options.py                         |  138 +--
 tools/shellbuiltins.py                        |  243 +++-
 26 files changed, 2714 insertions(+), 1925 deletions(-)
 rename explainshell/{helpconstants.py => help_constants.py} (67%)
 create mode 100644 explainshell/web/debug_views.py
 delete mode 100644 explainshell/web/debugviews.py

diff --git a/explainshell/algo/classifier.py b/explainshell/algo/classifier.py
index 1b1c8216..dd0a3436 100644
--- a/explainshell/algo/classifier.py
+++ b/explainshell/algo/classifier.py
@@ -1,4 +1,6 @@
-import itertools, collections, logging
+import itertools
+import collections
+import logging
 
 import nltk
 import nltk.metrics
@@ -9,26 +11,31 @@
 
 logger = logging.getLogger(__name__)
 
+
 def get_features(paragraph):
     features = {}
-    ptext = paragraph.cleantext()
-    assert ptext
-
-    features['starts_with_hyphen'] = algo.features.starts_with_hyphen(ptext)
-    features['is_indented'] = algo.features.is_indented(ptext)
-    features['par_length'] = algo.features.par_length(ptext)
-    for w in ('=', '--', '[', '|', ','):
-        features['first_line_contains_%s' % w] = algo.features.first_line_contains(ptext, w)
-    features['first_line_length'] = algo.features.first_line_length(ptext)
-    features['first_line_word_count'] = algo.features.first_line_word_count(ptext)
-    features['is_good_section'] = algo.features.is_good_section(paragraph)
-    features['word_count'] = algo.features.word_count(ptext)
+    p_text = paragraph.cleantext()
+    assert p_text
+
+    features["starts_with_hyphen"] = algo.features.starts_with_hyphen(p_text)
+    features["is_indented"] = algo.features.is_indented(p_text)
+    features["par_length"] = algo.features.par_length(p_text)
+    for w in ("=", "--", "[", "|", ","):
+        features[f"first_line_contains_{w}"] = algo.features.first_line_contains(
+            p_text, w
+        )
+    features["first_line_length"] = algo.features.first_line_length(p_text)
+    features["first_line_word_count"] = algo.features.first_line_word_count(p_text)
+    features["is_good_section"] = algo.features.is_good_section(paragraph)
+    features["word_count"] = algo.features.word_count(p_text)
 
     return features
 
-class classifier(object):
-    '''classify the paragraphs of a man page as having command line options
-    or not'''
+
+class Classifier:
+    """classify the paragraphs of a man page as having command line options
+    or not"""
+
     def __init__(self, store, algo, **classifier_args):
         self.store = store
         self.algo = algo
@@ -39,15 +46,16 @@ def train(self):
         if self.classifier:
             return
 
-        manpages = self.store.trainingset()
+        man_pages = self.store.trainingset()
 
         # flatten the manpages so we get a list of (manpage-name, paragraph)
         def flatten_manpages(manpage):
-            l = []
+            p_list = []
             for para in manpage.paragraphs:
-                l.append(para)
-            return l
-        paragraphs = itertools.chain(*[flatten_manpages(m) for m in manpages])
+                p_list.append(para)
+            return p_list
+
+        paragraphs = itertools.chain(*[flatten_manpages(m) for m in man_pages])
         training = list(paragraphs)
 
         negids = [p for p in training if not p.is_option]
@@ -56,42 +64,42 @@ def flatten_manpages(manpage):
         negfeats = [(get_features(p), False) for p in negids]
         posfeats = [(get_features(p), True) for p in posids]
 
-        negcutoff = len(negfeats)*3/4
-        poscutoff = len(posfeats)*3/4
+        negcutoff = len(negfeats) * 3 / 4
+        poscutoff = len(posfeats) * 3 / 4
 
         trainfeats = negfeats[:negcutoff] + posfeats[:poscutoff]
         self.testfeats = negfeats[negcutoff:] + posfeats[poscutoff:]
 
-        logger.info('train on %d instances', len(trainfeats))
+        logger.info("train on %d instances", len(trainfeats))
 
-        if self.algo == 'maxent':
+        if self.algo == "maxent":
             c = nltk.classify.maxent.MaxentClassifier
-        elif self.algo == 'bayes':
+        elif self.algo == "bayes":
             c = nltk.classify.NaiveBayesClassifier
         else:
-            raise ValueError('unknown classifier')
+            raise ValueError("unknown classifier")
 
         self.classifier = c.train(trainfeats, **self.classifier_args)
 
     def evaluate(self):
         self.train()
-        refsets = collections.defaultdict(set)
-        testsets = collections.defaultdict(set)
+        ref_sets = collections.defaultdict(set)
+        test_sets = collections.defaultdict(set)
 
         for i, (feats, label) in enumerate(self.testfeats):
-            refsets[label].add(i)
+            ref_sets[label].add(i)
             guess = self.classifier.prob_classify(feats)
             observed = guess.max()
-            testsets[observed].add(i)
-            #if label != observed:
-            #    print 'label:', label, 'observed:', observed, feats
+            test_sets[observed].add(i)
+            # if label != observed:
+            #    print('label:', label, 'observed:', observed, feats
 
-        print 'pos precision:', nltk.metrics.precision(refsets[True], testsets[True])
-        print 'pos recall:', nltk.metrics.recall(refsets[True], testsets[True])
-        print 'neg precision:', nltk.metrics.precision(refsets[False], testsets[False])
-        print 'neg recall:', nltk.metrics.recall(refsets[False], testsets[False])
+        print("pos precision:", nltk.metrics.precision(ref_sets[True], test_sets[True]))
+        print("pos recall:", nltk.metrics.recall(ref_sets[True], test_sets[True]))
+        print("neg precision:", nltk.metrics.precision(ref_sets[False], test_sets[False]))
+        print("neg recall:", nltk.metrics.recall(ref_sets[False], test_sets[False]))
 
-        print self.classifier.show_most_informative_features(10)
+        print(self.classifier.show_most_informative_features(10))
 
     def classify(self, manpage):
         self.train()
@@ -102,10 +110,9 @@ def classify(self, manpage):
             option = guess.max()
             certainty = guess.prob(option)
 
-            if option:
-                if certainty < config.CLASSIFIER_CUTOFF:
-                    pass
-                else:
-                    logger.info('classified %s (%f) as an option paragraph', item, certainty)
-                    item.is_option = True
-                    yield certainty, item
+            if option and certainty >= config.CLASSIFIER_CUTOFF:
+                logger.info(
+                    "classified %s (%f) as an option paragraph", item, certainty
+                )
+                item.is_option = True
+                yield certainty, item
diff --git a/explainshell/algo/features.py b/explainshell/algo/features.py
index 644602eb..32c05edc 100644
--- a/explainshell/algo/features.py
+++ b/explainshell/algo/features.py
@@ -1,7 +1,8 @@
 import re
 
+
 def extract_first_line(paragraph):
-    '''
+    """
     >>> extract_first_line('a b  cd')
     'a b'
     >>> extract_first_line('a b cd')
@@ -10,54 +11,63 @@ def extract_first_line(paragraph):
     'a b cd'
     >>> extract_first_line('  a b   cd')
     'a b'
-    '''
+    """
     lines = paragraph.splitlines()
     first = lines[0].strip()
-    spaces = list(re.finditer(r'(\s+)', first))
+    spaces = list(re.finditer(r"(\s+)", first))
     # handle options that have their description in the first line by trying
     # to treat it as two lines (looking at spaces between option and the rest
     # of the text)
     if spaces:
         longest = max(spaces, key=lambda m: m.span()[1] - m.span()[0])
         if longest and longest.start() > 1 and longest.end() - longest.start() > 1:
-            first = first[:longest.start()]
+            first = first[: longest.start()]
     return first
 
+
 def starts_with_hyphen(paragraph):
-    return paragraph.lstrip()[0] == '-'
+    return paragraph.lstrip()[0] == "-"
+
 
 def is_indented(paragraph):
     return paragraph != paragraph.lstrip()
 
+
 def par_length(paragraph):
     return round(len(paragraph.strip()), -1) / 2
 
+
 def first_line_contains(paragraph, what):
-    l = paragraph.splitlines()[0]
-    return what in l
+    ln = paragraph.splitlines()[0]
+    return what in ln
+
 
 def first_line_length(paragraph):
     first = extract_first_line(paragraph)
     return round(len(first), -1) / 2
 
+
 def first_line_word_count(paragraph):
     first = extract_first_line(paragraph)
     splitted = [s for s in first.split() if len(s) > 1]
 
     return round(len(splitted), -1)
 
+
 def is_good_section(paragraph):
     if not paragraph.section:
         return False
     s = paragraph.section.lower()
-    if 'options' in s:
+    if "options" in s:
         return True
-    if s in ('description', 'function letters'):
+    if s in ("description", "function letters"):
         return True
     return False
 
+
 def word_count(text):
-    return round(len(re.findall(r'\w+', text)), -1)
+    return round(len(re.findall(r"\w+", text)), -1)
+
 
 def has_bold(html):
-    return '<b>' in html
+    return "<b>" in html
diff --git a/explainshell/config.py b/explainshell/config.py
index 9df17ba4..91723f1f 100644
--- a/explainshell/config.py
+++ b/explainshell/config.py
@@ -1,45 +1,39 @@
 import os
 
-_currdir = os.path.dirname(os.path.dirname(__file__))
+_curr_dir = os.path.dirname(os.path.dirname(__file__))
 
-MANPAGEDIR = os.path.join(_currdir, 'manpages')
+MAN_PAGE_DIR = os.path.join(_curr_dir, "manpages")
 CLASSIFIER_CUTOFF = 0.7
-TOOLSDIR = os.path.join(_currdir, 'tools')
+TOOLS_DIR = os.path.join(_curr_dir, "tools")
 
-MAN2HTML = os.path.join(TOOLSDIR, 'w3mman2html.cgi')
+MAN2HTML = os.path.join(TOOLS_DIR, "w3mman2html.cgi")
 
 # host to pass into Flask's app.run.
-HOST_IP = os.getenv('HOST_IP', False)
-MONGO_URI = os.getenv('MONGO_URI', 'mongodb://localhost')
+HOST_IP = os.getenv("HOST_IP", "")
+MONGO_URI = os.getenv("MONGO_URI", "mongodb://localhost")
 DEBUG = True
 
 LOGGING_DICT = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'formatters': {
-        'standard': {
-            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
-        },
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "standard": {"format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"},
     },
-    'handlers': {
-        'console': {
-            'level' : 'INFO',
-            'class' : 'logging.StreamHandler',
-            'formatter': 'standard',
+    "handlers": {
+        "console": {
+            "level": "INFO",
+            "class": "logging.StreamHandler",
+            "formatter": "standard",
         },
-        'file': {
-            'class': 'logging.FileHandler',
-            'level': 'INFO',
-            'formatter': 'standard',
-            'filename': 'application.log',
-            'mode': 'a',
+        "file": {
+            "class": "logging.FileHandler",
+            "level": "INFO",
+            "formatter": "standard",
+            "filename": "application.log",
+            "mode": "a",
         },
     },
-    'loggers': {
-        'explainshell': {
-            'handlers': ['console'],
-            'level': 'INFO',
-            'propagate': False
-        }
-    }
+    "loggers": {
+        "explainshell": {"handlers": ["console"], "level": "INFO", "propagate": False}
+    },
 }
diff --git a/explainshell/errors.py b/explainshell/errors.py
index 18982b2d..b4477827 100644
--- a/explainshell/errors.py
+++ b/explainshell/errors.py
@@ -1,5 +1,6 @@
 class ProgramDoesNotExist(Exception):
     pass
 
+
 class EmptyManpage(Exception):
     pass
diff --git a/explainshell/fixer.py b/explainshell/fixer.py
index aadef797..2e60d303 100644
--- a/explainshell/fixer.py
+++ b/explainshell/fixer.py
@@ -1,14 +1,17 @@
-import textwrap, logging
+import textwrap
+import logging
 
 from explainshell import util
 
-class basefixer(object):
-    '''The base fixer class which other fixers inherit from.
+
+class BaseFixer:
+    """The base fixer class which other fixers inherit from.
 
     Subclasses override the base methods in order to fix manpage content during
-    different parts of the parsing/classifying/saving process.'''
-    runbefore = []
-    runlast = False
+    different parts of the parsing/classifying/saving process."""
+
+    run_before = []
+    run_last = False
 
     def __init__(self, mctx):
         self.mctx = mctx
@@ -36,20 +39,23 @@ def post_option_extraction(self):
     def pre_add_manpage(self):
         pass
 
-fixerscls = []
+
+fixers_cls = []
 fixerspriority = {}
 
-class runner(object):
-    '''The runner coordinates the fixers.'''
+
+class Runner:
+    """The runner coordinates the fixers."""
+
     def __init__(self, mctx):
         self.mctx = mctx
-        self.fixers = [f(mctx) for f in fixerscls]
+        self.fixers = [f(mctx) for f in fixers_cls]
 
     def disable(self, name):
         before = len(self.fixers)
         self.fixers = [f for f in self.fixers if f.__class__.__name__ != name]
         if before == len(self.fixers):
-            raise ValueError('fixer %r not found' % name)
+            raise ValueError(f"fixer {name} not found")
 
     def _fixers(self):
         return (f for f in self.fixers if f.run)
@@ -82,70 +88,76 @@ def pre_add_manpage(self):
         for f in self._fixers():
             f.pre_add_manpage()
 
-def register(fixercls):
-    fixerscls.append(fixercls)
-    for f in fixercls.runbefore:
-        if not hasattr(f, '_parents'):
+
+def register(fixer_cls):
+    fixers_cls.append(fixer_cls)
+    for f in fixer_cls.run_before:
+        if not hasattr(f, "_parents"):
             f._parents = []
-        f._parents.append(fixercls)
-    return fixercls
+        f._parents.append(fixer_cls)
+    return fixer_cls
+
 
 @register
-class bulletremover(basefixer):
-    '''remove list bullets from paragraph start, see mysqlslap.1'''
+class BulletRemover(BaseFixer):
+    """remove list bullets from paragraph start, see mysqlslap.1"""
+
     def post_parse_manpage(self):
-        toremove = []
+        to_remove = []
         for i, p in enumerate(self.mctx.manpage.paragraphs):
             try:
-                idx = p.text.index('\xc2\xb7')
-                p.text = p.text[:idx] + p.text[idx+2:]
+                idx = p.text.index("\xc2\xb7")
+                p.text = p.text[:idx] + p.text[idx + 2 :]
                 if not p.text.strip():
-                    toremove.append(i)
+                    to_remove.append(i)
             except ValueError:
                 pass
-        for i in reversed(toremove):
+        for i in reversed(to_remove):
             del self.mctx.manpage.paragraphs[i]
 
+
 @register
-class leadingspaceremover(basefixer):
-    '''go over all known option paragraphs and remove their leading spaces
-    by the amount of spaces in the first line'''
+class LeadingSpaceRemover(BaseFixer):
+    """go over all known option paragraphs and remove their leading spaces
+    by the amount of spaces in the first line"""
 
     def post_option_extraction(self):
         for i, p in enumerate(self.mctx.manpage.options):
-            text = self._removewhitespace(p.text)
+            text = self._remove_ws(p.text)
             p.text = text
 
-    def _removewhitespace(self, text):
-        '''
-        >>> f = leadingspaceremover(None)
-        >>> f._removewhitespace(' a\\n  b ')
+    def _remove_ws(self, text):
+        """
+        >>> f = LeadingSpaceRemover(None)
+        >>> f._remove_ws(' a\\n  b ')
         'a\\n b'
-        >>> f._removewhitespace('\\t a\\n\\t \\tb')
+        >>> f._remove_ws('\\t a\\n\\t \\tb')
         'a\\n\\tb'
-        '''
+        """
         return textwrap.dedent(text).rstrip()
 
+
 @register
-class tarfixer(basefixer):
+class TarFixer(BaseFixer):
     def __init__(self, *args):
-        super(tarfixer, self).__init__(*args)
-        self.run = self.mctx.name == 'tar'
+        super().__init__(*args)
+        self.run = self.mctx.name == "tar"
 
     def pre_add_manpage(self):
-        self.mctx.manpage.partialmatch = True
+        self.mctx.manpage.partial_match = True
+
 
 @register
-class paragraphjoiner(basefixer):
-    runbefore = [leadingspaceremover]
-    maxdistance = 5
+class ParagraphJoiner(BaseFixer):
+    run_before = [LeadingSpaceRemover]
+    max_distance = 5
 
     def post_option_extraction(self):
         options = [p for p in self.mctx.manpage.paragraphs if p.is_option]
         self._join(self.mctx.manpage.paragraphs, options)
 
     def _join(self, paragraphs, options):
-        def _paragraphsbetween(op1, op2):
+        def _paragraphs_between(op1, op2):
             assert op1.idx < op2.idx
             r = []
             start = None
@@ -156,51 +168,60 @@ def _paragraphsbetween(op1, op2):
                     r.append(p)
             return r, start
 
-        totalmerged = 0
-        for curr, next in util.pairwise(options):
-            between, start = _paragraphsbetween(curr, next)
-            if curr.section == next.section and 1 <= len(between) < self.maxdistance:
-                self.logger.info('merging paragraphs %d through %d (inclusive)', curr.idx, next.idx-1)
-                newdesc = [curr.text.rstrip()]
-                newdesc.extend([p.text.rstrip() for p in between])
-                curr.text = '\n\n'.join(newdesc)
-                del paragraphs[start:start+len(between)]
-                totalmerged += len(between)
-        return totalmerged
+        total_merged = 0
+        for curr, o_next in util.pairwise(options):
+            between, start = _paragraphs_between(curr, o_next)
+            if curr.section == o_next.section and 1 <= len(between) < self.max_distance:
+                self.logger.info(
+                    "merging paragraphs %d through %d (inclusive)",
+                    curr.idx,
+                    o_next.idx - 1,
+                )
+                new_desc = [curr.text.rstrip()]
+                new_desc.extend([p.text.rstrip() for p in between])
+                curr.text = "\n\n".join(new_desc)
+                del paragraphs[start: start + len(between)]
+                total_merged += len(between)
+        return total_merged
+
 
 @register
-class optiontrimmer(basefixer):
-    runbefore = [paragraphjoiner]
+class OptionTrimmer(BaseFixer):
+    run_before = [ParagraphJoiner]
 
-    d = {'git-rebase' : (50, -1)}
+    d = {"git-rebase": (50, -1)}
 
     def __init__(self, mctx):
-        super(optiontrimmer, self).__init__(mctx)
+        super().__init__(mctx)
         self.run = self.mctx.name in self.d
 
     def post_classify(self):
         start, end = self.d[self.mctx.name]
-        classifiedoptions = [p for p in self.mctx.manpage.paragraphs if p.is_option]
-        assert classifiedoptions
+        classified_opts = [p for p in self.mctx.manpage.paragraphs if p.is_option]
+        assert classified_opts
         if end == -1:
-            end = classifiedoptions[-1].idx
+            end = classified_opts[-1].idx
         else:
             assert start > end
 
-        for p in classifiedoptions:
-            if not (start <= p.idx <= end):
+        for p in classified_opts:
+            if not start <= p.idx <= end:
                 p.is_option = False
-                self.logger.info('removing option %r', p)
+                self.logger.info("removing option %r", p)
+
 
-def _parents(fixercls):
-    p = getattr(fixercls, '_parents', [])
-    last = fixercls.runlast
+def _parents(fixer_cls):
+    p = getattr(fixer_cls, "_parents", [])
+    last = fixer_cls.run_last
 
     if last and p:
-        raise ValueError("%s can't be last and also run before someone else" % fixercls.__name__)
+        raise ValueError(
+            f"{fixer_cls.__name__} can't be last and also run before someone else"
+        )
 
     if last:
-        return [f for f in fixerscls if f is not fixercls]
+        return [f for f in fixers_cls if f is not fixer_cls]
     return p
 
-fixerscls = util.toposorted(fixerscls, _parents)
+
+fixers_cls = util.topo_sorted(fixers_cls, _parents)
diff --git a/explainshell/helpconstants.py b/explainshell/help_constants.py
similarity index 67%
rename from explainshell/helpconstants.py
rename to explainshell/help_constants.py
index a3063c69..085e0d30 100644
--- a/explainshell/helpconstants.py
+++ b/explainshell/help_constants.py
@@ -1,10 +1,9 @@
-# -*- coding: utf-8 -*-
-
 import textwrap
 
-NOSYNOPSIS = 'no synopsis found'
+NO_SYNOPSIS = "no synopsis found"
 
-PIPELINES = textwrap.dedent('''   <b>Pipelines</b>
+PIPELINES = textwrap.dedent(
+    """   <b>Pipelines</b>
        A  <u>pipeline</u> is a sequence of one or more commands separated by one of the control operators <b>|</b> or <b>|&amp;</b>.  The
        format for a pipeline is:
 
@@ -33,15 +32,21 @@
        total user and system time consumed by the shell and its children.  The <b>TIMEFORMAT</b> variable may  be  used
        to specify the format of the time information.
 
-       Each command in a pipeline is executed as a separate process (i.e., in a subshell).''')
+       Each command in a pipeline is executed as a separate process (i.e., in a subshell)."""
+)
 
-OPSEMICOLON = textwrap.dedent('''       Commands separated  by  a <b>;</b> are executed sequentially; the shell waits for each command to terminate in turn.  The
-       return status is the exit status of the last command executed.''')
+OPSEMICOLON = textwrap.dedent(
+    """       Commands separated  by  a <b>;</b> are executed sequentially; the shell waits for each command to terminate in turn.  The
+       return status is the exit status of the last command executed."""
+)
 
-OPBACKGROUND = textwrap.dedent('''       If a command is terminated by the control operator <b>&amp;</b>, the shell executes the command in the <u>background</u> in
-       a subshell.  The shell does not wait for the command to finish, and the return  status  is  0.''')
+OPBACKGROUND = textwrap.dedent(
+    """       If a command is terminated by the control operator <b>&amp;</b>, the shell executes the command in the <u>background</u> in
+       a subshell.  The shell does not wait for the command to finish, and the return  status  is  0."""
+)
 
-OPANDOR = textwrap.dedent('''       AND and OR lists are sequences of one of more pipelines separated by the <b>&amp;&amp;</b>  and  <b>||</b>  control  operators,
+OPANDOR = textwrap.dedent(
+    """       AND and OR lists are sequences of one of more pipelines separated by the <b>&amp;&amp;</b>  and  <b>||</b>  control  operators,
        respectively.  AND and OR lists are executed with left associativity.  An AND list has the form
 
               <u>command1</u> <b>&amp;&amp;</b> <u>command2</u>
@@ -53,48 +58,63 @@
               <u>command1</u> <b>||</b> <u>command2</u>
 
        <u>command2</u>  is  executed  if and only if <u>command1</u> returns a non-zero exit status.  The return status of AND
-       and OR lists is the exit status of the last command executed in the list.''')
+       and OR lists is the exit status of the last command executed in the list."""
+)
 
-OPERATORS = {';' : OPSEMICOLON, '&' : OPBACKGROUND, '&&' : OPANDOR, '||' : OPANDOR}
+OPERATORS = {";": OPSEMICOLON, "&": OPBACKGROUND, "&&": OPANDOR, "||": OPANDOR}
 
-REDIRECTION = textwrap.dedent('''       Before a command is executed, its input and output may be <u>redirected</u> using a special notation interpreted
+REDIRECTION = textwrap.dedent(
+    """       Before a command is executed, its input and output may be <u>redirected</u> using a special notation
+       interpreted
        by  the  shell.   Redirection  may  also  be used to open and close files for the current shell execution
-       environment.  The following redirection operators may precede or appear anywhere within a <u>simple</u>  <u>command</u>
-       or may follow a <u>command</u>.  Redirections are processed in the order they appear, from left to right.''')
+       environment.  The following redirection operators may precede or appear anywhere within a <u>simple</u> 
+       <u>command</u>
+       or may follow a <u>command</u>.  Redirections are processed in the order they appear, from left to right."""
+)
 
-REDIRECTING_INPUT = textwrap.dedent('''   <b>Redirecting</b> <b>Input</b>
+REDIRECTING_INPUT = textwrap.dedent(
+    """   <b>Redirecting</b> <b>Input</b>
        Redirection  of  input  causes  the  file  whose name results from the expansion of <u>word</u> to be opened for
        reading on file descriptor <u>n</u>, or the standard input (file descriptor 0) if <u>n</u> is not specified.
 
        The general format for redirecting input is:
 
-              [<u>n</u>]<b>&lt;</b><u>word</u>''')
+              [<u>n</u>]<b>&lt;</b><u>word</u>"""
+)
 
-REDIRECTING_OUTPUT = textwrap.dedent('''   <b>Redirecting</b> <b>Output</b>
+REDIRECTING_OUTPUT = textwrap.dedent(
+    """   <b>Redirecting</b> <b>Output</b>
        Redirection of output causes the file whose name results from the expansion of  <u>word</u>  to  be  opened  for
-       writing  on  file descriptor <u>n</u>, or the standard output (file descriptor 1) if <u>n</u> is not specified.  If the
+       writing  on  file descriptor <u>n</u>, or the standard output (file descriptor 1) if <u>n</u> is not specified.
+         If the
        file does not exist it is created; if it does exist it is truncated to zero size.
 
        The general format for redirecting output is:
 
               [<u>n</u>]<b>&gt;</b><u>word</u>
 
-       If the redirection operator is <b>&gt;</b>, and the <b>noclobber</b> option to the  <b>set</b>  builtin  has  been  enabled,  the
+       If the redirection operator is <b>&gt;</b>, and the <b>noclobber</b> option to the  <b>set</b>  builtin  has
+         been  enabled,  the
        redirection  will  fail if the file whose name results from the expansion of <u>word</u> exists and is a regular
-       file.  If the redirection operator is <b>&gt;|</b>, or the redirection operator is <b>&gt;</b> and the  <b>noclobber</b>  option  to
-       the  <b>set</b>  builtin  command  is  not  enabled, the redirection is attempted even if the file named by <u>word</u>
-       exists.''')
-
-APPENDING_REDIRECTED_OUTPUT = textwrap.dedent('''   <b>Appending</b> <b>Redirected</b> <b>Output</b>
+       file.  If the redirection operator is <b>&gt;|</b>, or the redirection operator is <b>&gt;</b> and the
+         <b>noclobber</b>  option  to
+       the  <b>set</b>  builtin  command  is  not  enabled, the redirection is attempted even if the file named by
+         <u>word</u> exists."""
+)
+
+APPENDING_REDIRECTED_OUTPUT = textwrap.dedent(
+    """   <b>Appending</b> <b>Redirected</b> <b>Output</b>
        Redirection of output in this fashion causes the file whose name results from the expansion of <u>word</u> to be
-       opened  for  appending  on  file  descriptor  <u>n</u>,  or  the standard output (file descriptor 1) if <u>n</u> is not
-       specified.  If the file does not exist it is created.
+       opened  for  appending  on  file  descriptor  <u>n</u>,  or  the standard output (file descriptor 1) if
+        <u>n</u> is not specified.  If the file does not exist it is created.
 
        The general format for appending output is:
 
-              [<u>n</u>]<b>&gt;&gt;</b><u>word</u>''')
+              [<u>n</u>]<b>&gt;&gt;</b><u>word</u>"""
+)
 
-REDIRECTING_OUTPUT_ERROR = textwrap.dedent('''   <b>Redirecting</b> <b>Standard</b> <b>Output</b> <b>and</b> <b>Standard</b> <b>Error</b>
+REDIRECTING_OUTPUT_ERROR = textwrap.dedent(
+    """   <b>Redirecting</b> <b>Standard</b> <b>Output</b> <b>and</b> <b>Standard</b> <b>Error</b>
        This construct allows both the standard output (file descriptor 1) and the standard  error  output  (file
        descriptor 2) to be redirected to the file whose name is the expansion of <u>word</u>.
 
@@ -106,9 +126,11 @@
 
        Of the two forms, the first is preferred.  This is semantically equivalent to
 
-              <b>&gt;</b><u>word</u> 2<b>&gt;&amp;</b>1''')
+              <b>&gt;</b><u>word</u> 2<b>&gt;&amp;</b>1"""
+)
 
-APPENDING_OUTPUT_ERROR = textwrap.dedent('''   <b>Appending</b> <b>Standard</b> <b>Output</b> <b>and</b> <b>Standard</b> <b>Error</b>
+APPENDING_OUTPUT_ERROR = textwrap.dedent(
+    """   <b>Appending</b> <b>Standard</b> <b>Output</b> <b>and</b> <b>Standard</b> <b>Error</b>
        This  construct  allows  both the standard output (file descriptor 1) and the standard error output (file
        descriptor 2) to be appended to the file whose name is the expansion of <u>word</u>.
 
@@ -118,9 +140,11 @@
 
        This is semantically equivalent to
 
-              <b>&gt;&gt;</b><u>word</u> 2<b>&gt;&amp;</b>1''')
+              <b>&gt;&gt;</b><u>word</u> 2<b>&gt;&amp;</b>1"""
+)
 
-HERE_DOCUMENTS = textwrap.dedent('''   <b>Here</b> <b>Documents</b>
+HERE_DOCUMENTS = textwrap.dedent(
+    r"""   <b>Here</b> <b>Documents</b>
        This type of redirection instructs the shell  to  read  input  from  the  current  source  until  a  line
        containing  only <u>delimiter</u> (with no trailing blanks) is seen.  All of the lines read up to that point are
        then used as the standard input for a command.
@@ -132,12 +156,15 @@
               <u>delimiter</u>
 
        No parameter expansion, command substitution, arithmetic expansion, or pathname expansion is performed on
-       <u>word</u>.   If  any  characters in <u>word</u> are quoted, the <u>delimiter</u> is the result of quote removal on <u>word</u>, and
+       <u>word</u>.   If  any  characters in <u>word</u> are quoted, the <u>delimiter</u> is the result of quote
+        removal on <u>word</u>, and
        the lines in the here-document are not expanded.  If <u>word</u> is unquoted, all lines of the here-document are
        subjected  to  parameter  expansion, command substitution, and arithmetic expansion.  In the latter case,
-       the character sequence <b>\&lt;newline&gt;</b> is ignored, and <b>\</b> must be used to quote the characters <b>\</b>, <b>$</b>, and <b>`</b>.
+       the character sequence <b>\&lt;newline&gt;</b> is ignored, and <b>\</b> must be used to quote the characters
+        <b>\</b>, <b>$</b>, and <b>`</b>.
 
-       If the redirection operator is <b>&lt;&lt;-</b>, then all leading tab characters are stripped from input lines and the
+       If the redirection operator is <b>&lt;&lt;-</b>, then all leading tab characters are stripped from input lines
+        and the
        line  containing  <u>delimiter</u>.  This allows here-documents within shell scripts to be indented in a natural
        fashion.
 
@@ -146,75 +173,102 @@
 
               <b>&lt;&lt;&lt;</b><u>word</u>
 
-       The <u>word</u> is expanded and supplied to the command on its standard input.''')
-
-REDIRECTION_KIND = {'<' : REDIRECTING_INPUT,
-                   '>' : REDIRECTING_OUTPUT,
-                   '>>' : APPENDING_REDIRECTED_OUTPUT,
-                   '&>' : REDIRECTING_OUTPUT_ERROR,
-                   '>&' : REDIRECTING_OUTPUT_ERROR,
-                   '&>>' : APPENDING_OUTPUT_ERROR,
-                   '<<' : HERE_DOCUMENTS,
-                   '<<<' : HERE_DOCUMENTS}
+       The <u>word</u> is expanded and supplied to the command on its standard input."""
+)
+
+REDIRECTION_KIND = {
+    "<": REDIRECTING_INPUT,
+    ">": REDIRECTING_OUTPUT,
+    ">>": APPENDING_REDIRECTED_OUTPUT,
+    "&>": REDIRECTING_OUTPUT_ERROR,
+    ">&": REDIRECTING_OUTPUT_ERROR,
+    "&>>": APPENDING_OUTPUT_ERROR,
+    "<<": HERE_DOCUMENTS,
+    "<<<": HERE_DOCUMENTS,
+}
 
-ASSIGNMENT = textwrap.dedent('''       A <u>variable</u> may be assigned to by a statement of the form
+ASSIGNMENT = textwrap.dedent(
+    """       A <u>variable</u> may be assigned to by a statement of the form
 
               <u>name</u>=[<u>value</u>]
 
-       If <u>value</u> is not given, the variable is assigned the null string.  All  <u>values</u>  undergo  tilde  expansion,
+       If <u>value</u> is not given, the variable is assigned the null string.  All  <u>values</u>  undergo  tilde
+         expansion,
        parameter  and  variable  expansion,  command  substitution, arithmetic expansion, and quote removal (see
-       <b>EXPANSION</b> below).  If the variable has  its  <b>integer</b>  attribute  set,  then  <u>value</u>  is  evaluated  as  an
-       arithmetic  expression even if the $((...)) expansion is not used (see <b>Arithmetic</b> <b>Expansion</b> below).  Word
-       splitting is not performed, with the exception of <b>"$@"</b>  as  explained  below  under  <b>Special</b>  <b>Parameters</b>.
+       <b>EXPANSION</b> below).  If the variable has  its  <b>integer</b>  attribute  set,  then  <u>value</u>  is
+         evaluated  as  an
+       arithmetic  expression even if the $((...)) expansion is not used (see <b>Arithmetic</b> <b>Expansion</b> below).
+         Word
+       splitting is not performed, with the exception of <b>"$@"</b>  as  explained  below  under  <b>Special</b>
+         <b>Parameters</b>.
        Pathname  expansion  is  not performed.  Assignment statements may also appear as arguments to the <b>alias</b>,
        <b>declare</b>, <b>typeset</b>, <b>export</b>, <b>readonly</b>, and <b>local</b> builtin commands.
 
        In the context where an assignment statement is assigning a value to a shell variable or array index, the
        +=  operator  can  be used to append to or add to the variable's previous value.  When += is applied to a
-       variable for which the <u>integer</u> attribute has been set, <u>value</u> is evaluated as an arithmetic expression and
+       variable for which the <u>integer</u> attribute has been set, <u>value</u> is evaluated as an arithmetic
+        expression and
        added  to the variable's current value, which is also evaluated.  When += is applied to an array variable
        using compound assignment (see <b>Arrays</b> below), the variable's value is not unset (as it is when using  =),
        and  new  values  are  appended to the array beginning at one greater than the array's maximum index (for
        indexed arrays) or added as additional key-value pairs in  an  associative  array.   When  applied  to  a
-       string-valued variable, <u>value</u> is expanded and appended to the variable's value.''')
-
-_group = textwrap.dedent('''       { <u>list</u>; }
-              <u>list</u> is simply executed in the current shell environment.  <u>list</u> must be terminated with a  newline
-              or  semicolon.   This  is known as a <u>group</u> <u>command</u>.  The return status is the exit status of <u>list</u>.
-              Note that unlike the metacharacters <b>(</b> and <b>)</b>, <b>{</b> and <b>}</b> are <u>reserved</u> <u>words</u> and  must  occur  where  a
+       string-valued variable, <u>value</u> is expanded and appended to the variable's value."""
+)
+
+_group = textwrap.dedent(
+    """       { <u>list</u>; }
+              <u>list</u> is simply executed in the current shell environment.  <u>list</u> must be terminated with a
+                newline
+              or  semicolon.   This  is known as a <u>group</u> <u>command</u>.  The return status is the exit status
+               of <u>list</u>.
+              Note that unlike the metacharacters <b>(</b> and <b>)</b>, <b>{</b> and <b>}</b> are
+               <u>reserved</u> <u>words</u> and  must  occur  where  a
               reserved  word  is permitted to be recognized.  Since they do not cause a word break, they must be
-              separated from <u>list</u> by whitespace or another shell metacharacter.''')
+              separated from <u>list</u> by whitespace or another shell metacharacter."""
+)
 
-_subshell = textwrap.dedent('''       (<u>list</u>) <u>list</u> is executed in a subshell environment (see <b>COMMAND</b> <b>EXECUTION</b>  <b>ENVIRONMENT</b>  below).   Variable
+_subshell = textwrap.dedent(
+    """       (<u>list</u>) <u>list</u> is executed in a subshell environment (see <b>COMMAND</b> <b>EXECUTION</b>
+               <b>ENVIRONMENT</b>  below).   Variable
               assignments and builtin commands that affect the shell's environment do not remain in effect after
-              the command completes.  The return status is the exit status of <u>list</u>.''')
-
-_negate = '''If the reserved word <b>!</b> precedes a pipeline, the exit status of that pipeline is the logical negation of the
-exit status as described above.'''
-
-_if = textwrap.dedent('''       <b>if</b> <u>list</u>; <b>then</b> <u>list;</u> [ <b>elif</b> <u>list</u>; <b>then</b> <u>list</u>; ] ... [ <b>else</b> <u>list</u>; ] <b>fi</b>
-              The  <b>if</b> <u>list</u> is executed.  If its exit status is zero, the <b>then</b> <u>list</u> is executed.  Otherwise, each
-              <b>elif</b> <u>list</u> is executed in turn, and if its exit status is zero,  the  corresponding  <b>then</b>  <u>list</u>  is
-              executed  and  the command completes.  Otherwise, the <b>else</b> <u>list</u> is executed, if present.  The exit
-              status is the exit status of the last command executed, or zero if no condition tested true.''')
-
-_for = textwrap.dedent('''       <b>for</b> <u>name</u> [ [ <b>in</b> [ <u>word</u> <u>...</u> ] ] ; ] <b>do</b> <u>list</u> ; <b>done</b>
+              the command completes.  The return status is the exit status of <u>list</u>."""
+)
+
+_negate = """If the reserved word <b>!</b> precedes a pipeline, the exit status of that pipeline is the logical
+ negation of the exit status as described above."""
+
+_if = textwrap.dedent(
+    """       <b>if</b> <u>list</u>; <b>then</b> <u>list;</u> [ <b>elif</b> <u>list</u>; <b>then</b> <u>list</u>; ] ...
+              [ <b>else</b> <u>list</u>; ] <b>fi</b>
+              The  <b>if</b> <u>list</u> is executed.  If its exit status is zero, the <b>then</b> <u>list</u> is
+               executed.  Otherwise, each <b>elif</b> <u>list</u> is executed in turn, and if its exit status is zero,
+               the  corresponding  <b>then</b>  <u>list</u>  is executed  and  the command completes.
+               Otherwise, the <b>else</b> <u>list</u> is executed, if present.  The exit
+              status is the exit status of the last command executed, or zero if no condition tested true."""
+)
+
+_for = textwrap.dedent(
+    """       <b>for</b> <u>name</u> [ [ <b>in</b> [ <u>word</u> <u>...</u> ] ] ; ] <b>do</b> <u>list</u> ; <b>done</b>
               The  list of words following <b>in</b> is expanded, generating a list of items.  The variable <u>name</u> is set
               to each element of this list in turn, and <u>list</u> is executed each time.  If the <b>in</b> <u>word</u> is  omitted,
               the  <b>for</b>  command  executes  <u>list</u>  once  for each positional parameter that is set (see <b>PARAMETERS</b>
               below).  The return status is the exit status of the last command that executes.  If the expansion
               of  the  items  following  <b>in</b>  results  in an empty list, no commands are executed, and the return
-              status is 0.''')
+              status is 0."""
+)
 
-_whileuntil = textwrap.dedent('''       <b>while</b> <u>list-1</u>; <b>do</b> <u>list-2</u>; <b>done</b>
+_whileuntil = textwrap.dedent(
+    """       <b>while</b> <u>list-1</u>; <b>do</b> <u>list-2</u>; <b>done</b>
        <b>until</b> <u>list-1</u>; <b>do</b> <u>list-2</u>; <b>done</b>
               The <b>while</b> command continuously executes the list <u>list-2</u> as long as the last command  in  the  list
               <u>list-1</u>  returns  an  exit  status  of  zero.  The <b>until</b> command is identical to the <b>while</b> command,
               except that the test is negated; <u>list-2</u> is executed as long as the last command in <u>list-1</u>  returns
               a non-zero exit status.  The exit status of the <b>while</b> and <b>until</b> commands is the exit status of the
-              last command executed in <u>list-2</u>, or zero if none was executed.''')
+              last command executed in <u>list-2</u>, or zero if none was executed."""
+)
 
-_select = textwrap.dedent('''       <b>select</b> <u>name</u> [ <b>in</b> <u>word</u> ] ; <b>do</b> <u>list</u> ; <b>done</b>
+_select = textwrap.dedent(
+    """       <b>select</b> <u>name</u> [ <b>in</b> <u>word</u> ] ; <b>do</b> <u>list</u> ; <b>done</b>
               The list of words following <b>in</b> is expanded, generating a list of items.  The set of expanded words
               is printed on the standard error, each preceded by a number.  If  the  <b>in</b>  <u>word</u>  is  omitted,  the
               positional  parameters are printed (see <b>PARAMETERS</b> below).  The <b>PS3</b> prompt is then displayed and a
@@ -223,29 +277,33 @@
               prompt are displayed again.  If EOF is read, the command completes.  Any other value  read  causes
               <u>name</u> to be set to null.  The line read is saved in the variable <b>REPLY</b>.  The <u>list</u> is executed after
               each selection until a <b>break</b> command is executed.  The exit status of <b>select</b> is the exit status of
-              the last command executed in <u>list</u>, or zero if no commands were executed.''')
-
-RESERVEDWORDS = {
-    '!' : _negate,
-    '{' : _group,
-    '}' : _group,
-    '(' : _subshell,
-    ')' : _subshell,
-    ';' : OPSEMICOLON,
+              the last command executed in <u>list</u>, or zero if no commands were executed."""
+)
+
+RESERVED_WORDS = {
+    "!": _negate,
+    "{": _group,
+    "}": _group,
+    "(": _subshell,
+    ")": _subshell,
+    ";": OPSEMICOLON,
 }
 
+
 def _addwords(key, text, *words):
     for word in words:
-        COMPOUNDRESERVEDWORDS.setdefault(key, {})[word] = text
+        COMPOUND_RESERVED_WORDS.setdefault(key, {})[word] = text
+
 
-COMPOUNDRESERVEDWORDS = {}
-_addwords('if', _if, 'if', 'then', 'elif', 'else', 'fi', ';')
-_addwords('for', _for, 'for', 'in', 'do', 'done', ';')
-_addwords('while', _whileuntil, 'while', 'do', 'done', ';')
-_addwords('until', _whileuntil, 'until', 'do', 'done')
-_addwords('select', _select, 'select', 'in', 'do', 'done')
+COMPOUND_RESERVED_WORDS = {}
+_addwords("if", _if, "if", "then", "elif", "else", "fi", ";")
+_addwords("for", _for, "for", "in", "do", "done", ";")
+_addwords("while", _whileuntil, "while", "do", "done", ";")
+_addwords("until", _whileuntil, "until", "do", "done")
+_addwords("select", _select, "select", "in", "do", "done")
 
-_function = textwrap.dedent('''       A shell function is an object that is called like a simple command and executes a compound command with a
+_function = textwrap.dedent(
+    """       A shell function is an object that is called like a simple command and executes a compound command with a
        new set of positional parameters.  Shell functions are declared as follows:
 
        <u>name</u> () <u>compound-command</u> [<u>redirection</u>]
@@ -258,26 +316,29 @@ def _addwords(key, text, *words):
               redirections (see <b>REDIRECTION</b> below) specified when a function is defined are performed  when  the
               function  is  executed.   The  exit  status of a function definition is zero unless a syntax error
               occurs or a readonly function with the same name already exists.  When executed, the  exit  status
-              of a function is the exit status of the last command executed in the body.  (See <b>FUNCTIONS</b> below.)''')
+              of a function is the exit status of the last command executed in the body.  (See <b>FUNCTIONS</b> below.)"""
+)
 
-_functioncall = 'call shell function %r'
-_functionarg = 'argument for shell function %r'
+_function_call = "call shell function %r"
+_functionarg = "argument for shell function %r"
 
-COMMENT = textwrap.dedent('''<b>COMMENTS</b>
+COMMENT = textwrap.dedent(
+    """<b>COMMENTS</b>
       In a non-interactive shell, or an interactive shell in which the <b>interactive_comments</b> option to the <b>shopt</b>
       builtin is enabled (see <b>SHELL</b> <b>BUILTIN</b> <b>COMMANDS</b> below), a word beginning with <b>#</b> causes that word  and  all
       remaining  characters  on that line to be ignored.  An interactive shell without the <b>interactive_comments</b>
       option enabled does not allow comments.  The <b>interactive_comments</b> option is on by default in  interactive
-      shells.''')
+      shells."""
+)
 
 parameters = {
-        '*' : 'star',
-        '@' : 'at',
-        '#' : 'pound',
-        '?' : 'question',
-        '-' : 'hyphen',
-        '$' : 'dollar',
-        '!' : 'exclamation',
-        '0' : 'zero',
-        '_' : 'underscore',
+    "*": "star",
+    "@": "at",
+    "#": "pound",
+    "?": "question",
+    "-": "hyphen",
+    "$": "dollar",
+    "!": "exclamation",
+    "0": "zero",
+    "_": "underscore",
 }
diff --git a/explainshell/manager.py b/explainshell/manager.py
index ff4f7b54..8c0aea83 100644
--- a/explainshell/manager.py
+++ b/explainshell/manager.py
@@ -1,91 +1,105 @@
-import sys, os, argparse, logging, glob
+import argparse
+import os
+import sys
+import logging
+import glob
 
-from explainshell import options, store, fixer, manpage, errors, util, config
+from explainshell import options, store, fixer, manpage, errors, config
 from explainshell.algo import classifier
 
-logger = logging.getLogger('explainshell.manager')
+logger = logging.getLogger("explainshell.manager")
 
-class managerctx(object):
+
+class ManagerCtx:
     def __init__(self, classifier, store, manpage):
         self.classifier = classifier
         self.store = store
         self.manpage = manpage
         self.name = manpage.name
 
-        self.classifiermanpage = None
-        self.optionsraw = None
-        self.optionsextracted = None
+        self.classifier_man_page = None
+        self.options_raw = None
+        self.options_extracted = None
         self.aliases = None
 
-class manager(object):
-    '''the manager uses all parts of the system to read, classify, parse, extract
-    and write a man page to the database'''
-    def __init__(self, dbhost, dbname, paths, overwrite=False, drop=False):
+
+class Manager:
+    """the manager uses all parts of the system to read, classify, parse, extract
+    and write a man page to the database"""
+
+    def __init__(self, db_host, dbname, paths, overwrite=False, drop=False):
         self.paths = paths
         self.overwrite = overwrite
 
-        self.store = store.store(dbname, dbhost)
+        self.store = store.Store(dbname, db_host)
 
-        self.classifier = classifier.classifier(self.store, 'bayes')
+        self.classifier = classifier.Classifier(self.store, "bayes")
         self.classifier.train()
 
         if drop:
             self.store.drop(True)
 
     def ctx(self, m):
-        return managerctx(self.classifier, self.store, m)
+        return ManagerCtx(self.classifier, self.store, m)
 
-    def _read(self, ctx, frunner):
-        frunner.pre_get_raw_manpage()
+    def _read(self, ctx, f_runner):
+        f_runner.pre_get_raw_manpage()
         ctx.manpage.read()
         ctx.manpage.parse()
         assert len(ctx.manpage.paragraphs) > 1
 
-        ctx.manpage = store.manpage(ctx.manpage.shortpath, ctx.manpage.name,
-                ctx.manpage.synopsis, ctx.manpage.paragraphs, list(ctx.manpage.aliases))
-        frunner.post_parse_manpage()
-
-    def _classify(self, ctx, frunner):
-        ctx.classifiermanpage = store.classifiermanpage(ctx.name, ctx.manpage.paragraphs)
-        frunner.pre_classify()
+        ctx.manpage = store.ManPage(
+            ctx.manpage.shortpath,
+            ctx.manpage.name,
+            ctx.manpage.synopsis,
+            ctx.manpage.paragraphs,
+            list(ctx.manpage.aliases),
+        )
+        f_runner.post_parse_manpage()
+
+    def _classify(self, ctx, fr_runner):
+        ctx.classifiermanpage = store.ClassifierManpage(
+            ctx.name, ctx.manpage.paragraphs
+        )
+        fr_runner.pre_classify()
         _ = list(ctx.classifier.classify(ctx.classifiermanpage))
-        frunner.post_classify()
+        fr_runner.post_classify()
 
-    def _extract(self, ctx, frunner):
+    def _extract(self, ctx, f_runner):
         options.extract(ctx.manpage)
-        frunner.post_option_extraction()
+        f_runner.post_option_extraction()
         if not ctx.manpage.options:
-            logger.warn("couldn't find any options for manpage %s", ctx.manpage.name)
+            logger.warning("couldn't find any options for manpage %s", ctx.manpage.name)
 
-    def _write(self, ctx, frunner):
-        frunner.pre_add_manpage()
+    def _write(self, ctx, f_runner):
+        f_runner.pre_add_manpage()
         return ctx.store.addmanpage(ctx.manpage)
 
-    def _update(self, ctx, frunner):
-        frunner.pre_add_manpage()
+    def _update(self, ctx, f_runner):
+        f_runner.pre_add_manpage()
         return ctx.store.updatemanpage(ctx.manpage)
 
     def process(self, ctx):
-        frunner = fixer.runner(ctx)
+        f_runner = fixer.Runner(ctx)
 
-        self._read(ctx, frunner)
-        self._classify(ctx, frunner)
-        self._extract(ctx, frunner)
+        self._read(ctx, f_runner)
+        self._classify(ctx, f_runner)
+        self._extract(ctx, f_runner)
 
-        m = self._write(ctx, frunner)
+        m = self._write(ctx, f_runner)
         return m
 
     def edit(self, m, paragraphs=None):
         ctx = self.ctx(m)
-        frunner = fixer.runner(ctx)
+        f_runner = fixer.Runner(ctx)
 
         if paragraphs:
             m.paragraphs = paragraphs
-            frunner.disable('paragraphjoiner')
-            frunner.post_option_extraction()
+            f_runner.disable("paragraphjoiner")
+            f_runner.post_option_extraction()
         else:
-            self._extract(ctx, frunner)
-        m = self._update(ctx, frunner)
+            self._extract(ctx, f_runner)
+        m = self._update(ctx, f_runner)
         return m
 
     def run(self):
@@ -93,16 +107,19 @@ def run(self):
         exists = []
         for path in self.paths:
             try:
-                m = manpage.manpage(path)
-                logger.info('handling manpage %s (from %s)', m.name, path)
+                m = manpage.ManPage(path)
+                logger.info("handling manpage %s (from %s)", m.name, path)
                 try:
-                    mps = self.store.findmanpage(m.shortpath[:-3])
-                    mps = [mp for mp in mps if m.shortpath == mp.source]
+                    mps = self.store.find_man_page(m.short_path[:-3])
+                    mps = [mp for mp in mps if m.short_path == mp.source]
                     if mps:
                         assert len(mps) == 1
                         mp = mps[0]
                         if not self.overwrite or mp.updated:
-                            logger.info('manpage %r already in the data store, not overwriting it', m.name)
+                            logger.info(
+                                "manpage %r already in the data store, not overwriting it",
+                                m.name,
+                            )
                             exists.append(m)
                             continue
                 except errors.ProgramDoesNotExist:
@@ -113,89 +130,114 @@ def run(self):
                 m = self.process(ctx)
                 if m:
                     added.append(m)
-            except errors.EmptyManpage, e:
-                logger.error('manpage %r is empty!', e.args[0])
+            except errors.EmptyManpage as e:
+                logger.error("manpage %r is empty!", e.args[0])
             except ValueError:
-                logger.fatal('uncaught exception when handling manpage %s', path)
+                logger.fatal("uncaught exception when handling manpage %s", path)
             except KeyboardInterrupt:
                 raise
-            except:
-                logger.fatal('uncaught exception when handling manpage %s', path)
+            except Exception as error_msg:
+                logger.fatal(f"uncaught exception when handling manpage '{path}' -> error: {error_msg}")
                 raise
         if not added:
-            logger.warn('no manpages added')
+            logger.warning("no manpages added")
         else:
-            self.findmulticommands()
+            self.findmulti_cmds()
 
         return added, exists
 
-    def findmulticommands(self):
+    def findmulti_cmds(self):
         manpages = {}
         potential = []
         for _id, m in self.store.names():
-            if '-' in m:
-                potential.append((m.split('-'), _id))
+            if "-" in m:
+                potential.append((m.split("-"), _id))
             else:
                 manpages[m] = _id
 
-        mappings = set([x[0] for x in self.store.mappings()])
-        mappingstoadd = []
-        multicommands = {}
+        mappings = {x[0] for x in self.store.mappings()}
+        mappings_to_a = []
+        multi_cmds = {}
 
         for p, _id in potential:
-            if ' '.join(p) in mappings:
+            if " ".join(p) in mappings:
                 continue
             if p[0] in manpages:
-                mappingstoadd.append((' '.join(p), _id))
-                multicommands[p[0]] = manpages[p[0]]
+                mappings_to_a.append((" ".join(p), _id))
+                multi_cmds[p[0]] = manpages[p[0]]
+
+        for src, dst in mappings_to_a:
+            self.store.add_mapping(src, dst, 1)
+            logger.info("inserting mapping (multi_cmd) %s -> %s", src, dst)
 
-        for src, dst in mappingstoadd:
-            self.store.addmapping(src, dst, 1)
-            logger.info('inserting mapping (multicommand) %s -> %s', src, dst)
+        for multi_cmd, _id in multi_cmds.items():
+            self.store.setmulti_cmd(_id)
+            logger.info("making %r a multi_cmd", multi_cmd)
 
-        for multicommand, _id in multicommands.iteritems():
-            self.store.setmulticommand(_id)
-            logger.info('making %r a multicommand', multicommand)
+        return mappings_to_a, multi_cmds
 
-        return mappingstoadd, multicommands
 
-def main(files, dbname, dbhost, overwrite, drop, verify):
+def main(files, dbname, db_host, overwrite, drop, verify):
     if verify:
-        s = store.store(dbname, dbhost)
+        s = store.Store(dbname, db_host)
         ok = s.verify()
         return 0 if ok else 1
 
     if drop:
-        if raw_input('really drop db (y/n)? ').strip().lower() != 'y':
+        if input("really drop db (y/n)? ").strip().lower() != "y":
             drop = False
         else:
-            overwrite = True # if we drop, no need to take overwrite into account
+            overwrite = True  # if we drop, no need to take overwrite into account
 
     gzs = set()
 
     for path in files:
         if os.path.isdir(path):
-            gzs.update([os.path.abspath(f) for f in glob.glob(os.path.join(path, '*.gz'))])
+            gzs.update(
+                [os.path.abspath(f) for f in glob.glob(os.path.join(path, "*.gz"))]
+            )
         else:
             gzs.add(os.path.abspath(path))
 
-    m = manager(dbhost, dbname, gzs, overwrite, drop)
+    m = Manager(db_host, dbname, gzs, overwrite, drop)
     added, exists = m.run()
     for mp in added:
-        print 'successfully added %s' % mp.source
+        print(f"successfully added '{mp.source}'")
     if exists:
-        print 'these manpages already existed and werent overwritten: \n\n%s' % '\n'.join([m.path for m in exists])
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='process man pages and save them in the store')
-    parser.add_argument('--log', type=str, default='ERROR', help='use log as the logger log level')
-    parser.add_argument('--overwrite', action='store_true', default=False, help='overwrite man pages that already exist in the store')
-    parser.add_argument('--drop', action='store_true', default=False, help='delete all existing man pages')
-    parser.add_argument('--db', default='explainshell', help='mongo db name')
-    parser.add_argument('--host', default=config.MONGO_URI, help='mongo host')
-    parser.add_argument('--verify', action='store_true', default=False, help='verify db integrity')
-    parser.add_argument('files', nargs='*')
+        print(
+            "these manpages already existed and were not overwritten: \n\n%s"
+            % "\n".join([m.path for m in exists])
+        )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="process man pages and save them in the store"
+    )
+    parser.add_argument(
+        "--log", type=str, default="ERROR", help="use log as the logger log level"
+    )
+    parser.add_argument(
+        "--overwrite",
+        action="store_true",
+        default=False,
+        help="overwrite man pages that already exist in the store",
+    )
+    parser.add_argument(
+        "--drop",
+        action="store_true",
+        default=False,
+        help="delete all existing man pages",
+    )
+    parser.add_argument("--db", default="explainshell", help="mongo db name")
+    parser.add_argument("--host", default=config.MONGO_URI, help="mongo host")
+    parser.add_argument(
+        "--verify", action="store_true", default=False, help="verify db integrity"
+    )
+    parser.add_argument("files", nargs="*")
 
     args = parser.parse_args()
     logging.basicConfig(level=getattr(logging, args.log.upper()))
-    sys.exit(main(args.files, args.db, args.host, args.overwrite, args.drop, args.verify))
+    sys.exit(
+        main(args.files, args.db, args.host, args.overwrite, args.drop, args.verify)
+    )
diff --git a/explainshell/manpage.py b/explainshell/manpage.py
index 979c3d7a..648375f0 100644
--- a/explainshell/manpage.py
+++ b/explainshell/manpage.py
@@ -1,9 +1,14 @@
-import os, subprocess, re, logging, collections, urllib
+import os
+import subprocess
+import re
+import logging
+import collections
+import urllib
 
 from explainshell import config, store, errors
 
-devnull = open(os.devnull, 'w')
-SPLITSYNOP = re.compile(r'([^ ]+) - (.*)$')
+devnull = open(os.devnull, "w")
+SPLIT_SYNOP = re.compile(r"([^ ]+) - (.*)$")
 
 ENV = dict(os.environ)
 ENV["W3MMAN_MAN"] = "man --no-hyphenation"
@@ -13,25 +18,27 @@
 
 logger = logging.getLogger(__name__)
 
-def extractname(gzname):
-    '''
-    >>> extractname('ab.1.gz')
+
+def extract_name(gz_name):
+    """
+    >>> extract_name('ab.1.gz')
     'ab'
-    >>> extractname('ab.1.1.gz')
+    >>> extract_name('ab.1.1.gz')
     'ab.1'
-    >>> extractname('ab.1xyz.gz')
+    >>> extract_name('ab.1xyz.gz')
     'ab'
-    >>> extractname('ab.1.1xyz.gz')
+    >>> extract_name('ab.1.1xyz.gz')
     'ab.1'
-    >>> extractname('a/b/c/ab.1.1xyz.gz')
+    >>> extract_name('a/b/c/ab.1.1xyz.gz')
     'ab.1'
-    '''
-    if '/' in gzname:
-        gzname = os.path.basename(gzname)
-    return gzname.rsplit('.', 2)[0]
+    """
+    if "/" in gz_name:
+        gz_name = os.path.basename(gz_name)
+    return gz_name.rsplit(".", 2)[0]
+
 
-def bold(l):
-    '''
+def bold(ln_in):
+    """
     >>> bold('a')
     ([], ['a'])
     >>> bold('<b>a</b>')
@@ -40,9 +47,9 @@ def bold(l):
     (['b'], ['a', 'c'])
     >>> bold('<b>first</b> <b>second:</b>')
     (['first', 'second:'], [])
-    '''
+    """
     inside = []
-    for m in _section.finditer(l):
+    for m in _section.finditer(ln_in):
         inside.append(m.span(0))
 
     current = 0
@@ -50,116 +57,127 @@ def bold(l):
     for start, end in inside:
         outside.append((current, start))
         current = end
-    outside.append((current, len(l)))
+    outside.append((current, len(ln_in)))
 
-    inside = [l[s:e] for s, e in inside]
-    inside = [s.replace('<b>', '').replace('</b>', '') for s in inside]
+    inside = [ln_in[s:e] for s, e in inside]
+    inside = [s.replace("<b>", "").replace("</b>", "") for s in inside]
 
-    outside = [l[s:e] for s, e in outside]
-    outside = [l for l in outside if l and not l.isspace()]
+    outside = [ln_in[s:e] for s, e in outside]
+    outside = [ln for ln in outside if ln and not ln.isspace()]
     return inside, outside
 
+
 # w3mman2html.cgi (the tool we're using to output html from a man page) does
 # some strange escaping which causes it to output invalid utf8. we look these
 # up and fix them manually
-_replacementsprefix = [
-        ('\xe2\x80\xe2\x80\x98', None, True), # left single quote
-        ('\xe2\x80\xe2\x80\x99', None, True), # right single quote
-        ('\xe2\x80\xe2\x80\x9c', None, True), # left double quote
-        ('\xe2\x80\xe2\x80\x9d', None, True), # right double quote
-        ('\xe2\x94\xe2\x94\x82', '|', False), # pipe
-        ('\xe2\x8e\xe2\x8e\xaa', None, False), # pipe 2
-        ('\xe2\x80\xe2\x80\x90', None, True), # hyphen
-        ('\xe2\x80\xe2\x80\x94', None, True), # hyphen 2
-        ('\xe2\x80\xc2\xbd', None, True), # half
-        ('\xe2\x88\xe2\x88\x97', None, True), # asterisk
-        ('\xe2\x86\xe2\x86\x92', None, True), # right arrow
-        ('\xe2\x88\xe2\x88\x92', None, True), # minus sign
-        ('\xe2\x80\xe2\x80\x93', None, True), # en dash
-        ('\xe2\x80\xe2\x80\xb2', None, False), # prime
-        ('\xe2\x88\xe2\x88\xbc', None, False), # tilde operator
-        ('\xe2\x86\xe2\x86\xb5', None, False), # downwards arrow with corner leftwards
-        ('\xef\xbf\xef\xbf\xbd', None, False) # replacement char
-        ]
+_rp_prefix = [
+    ("\xe2\x80\xe2\x80\x98", None, True),  # left single quote
+    ("\xe2\x80\xe2\x80\x99", None, True),  # right single quote
+    ("\xe2\x80\xe2\x80\x9c", None, True),  # left double quote
+    ("\xe2\x80\xe2\x80\x9d", None, True),  # right double quote
+    ("\xe2\x94\xe2\x94\x82", "|", False),  # pipe
+    ("\xe2\x8e\xe2\x8e\xaa", None, False),  # pipe 2
+    ("\xe2\x80\xe2\x80\x90", None, True),  # hyphen
+    ("\xe2\x80\xe2\x80\x94", None, True),  # hyphen 2
+    ("\xe2\x80\xc2\xbd", None, True),  # half
+    ("\xe2\x88\xe2\x88\x97", None, True),  # asterisk
+    ("\xe2\x86\xe2\x86\x92", None, True),  # right arrow
+    ("\xe2\x88\xe2\x88\x92", None, True),  # minus sign
+    ("\xe2\x80\xe2\x80\x93", None, True),  # en dash
+    ("\xe2\x80\xe2\x80\xb2", None, False),  # prime
+    ("\xe2\x88\xe2\x88\xbc", None, False),  # tilde operator
+    ("\xe2\x86\xe2\x86\xb5", None, False),  # downwards arrow with corner leftwards
+    ("\xef\xbf\xef\xbf\xbd", None, False),  # replacement char
+]
 
 _replacements = []
 
-for searchfor, replacewith, underline in _replacementsprefix:
-    if replacewith is None:
-        replacewith = searchfor[2:]
-    _replacements.append((searchfor, replacewith))
+for search_for, rp_with, underline in _rp_prefix:
+    if rp_with is None:
+        rp_with = search_for[2:]
+    _replacements.append((search_for, rp_with))
     if underline:
-        x = list(replacewith)
-        x.insert(1, '</u>')
-        x = ''.join(x)
-        _replacements.append((x, '%s</u>' % replacewith))
-
-_replacementsnoprefix = ['\xc2\xb7', # bullet
-        '\xc2\xb4', # apostrophe
-        '\xc2\xa0', # no break space
-        '\xc3\xb8', '\xe4\xbd\xa0', '\xe5\xa5\xbd', # gibberish
-        '\xc2\xa7', # section sign
-        '\xef\xbf\xbd', # replacement char
-        '\xc2\xa4',  # latin small letter a with diaeresis
-        '\xc3\xa4', # latin small letter a with diaeresis
-        '\xc4\xa4',  # latin small letter a with diaeresis
-        '\xc3\xaa',  # latin small letter e with circumflex
-        ]
-
-for s in _replacementsnoprefix:
+        x = list(rp_with)
+        x.insert(1, "</u>")
+        x = "".join(x)
+        _replacements.append((x, "%s</u>" % rp_with))
+
+_replacements_no_prefix = [
+    "\xc2\xb7",  # bullet
+    "\xc2\xb4",  # apostrophe
+    "\xc2\xa0",  # no break space
+    "\xc3\xb8",
+    "\xe4\xbd\xa0",
+    "\xe5\xa5\xbd",  # gibberish
+    "\xc2\xa7",  # section sign
+    "\xef\xbf\xbd",  # replacement char
+    "\xc2\xa4",  # latin small letter a with diaeresis
+    "\xc3\xa4",  # latin small letter a with diaeresis
+    "\xc4\xa4",  # latin small letter a with diaeresis
+    "\xc3\xaa",  # latin small letter e with circumflex
+]
+
+for s in _replacements_no_prefix:
     x = list(s)
-    x.insert(1, '</u>')
-    x = ''.join(x)
-    _replacements.append((x, '%s</u>' % s))
+    x.insert(1, "</u>")
+    x = "".join(x)
+    _replacements.append((x, "%s</u>" % s))
 
 _href = re.compile(r'<a href="file:///[^\?]*\?([^\(]*)\(([^\)]*)\)">')
-_section = re.compile(r'<b>([^<]+)</b>')
+_section = re.compile(r"<b>([^<]+)</b>")
+
 
-def _parsetext(lines):
-    paragraphlines = []
+def _parse_text(lines):
+    para_lines = []
     section = None
     i = 0
-    for l in lines:
-        l = re.sub(_href, r'<a href="http://manpages.ubuntu.com/manpages/precise/en/man\2/\1.\2.html">', l)
-        for lookfor, replacewith in _replacements:
-            l = re.sub(lookfor, replacewith, l)
+    for ln in lines:
+        ln = re.sub(
+            _href,
+            r'<a href="http://manpages.ubuntu.com/manpages/precise/en/man\2/\1.\2.html">',
+            ln,
+        )
+        for look_for, rp_with in _replacements:
+            ln = re.sub(look_for, rp_with, ln)
         # confirm the line is valid utf8
-        lreplaced = l.decode('utf8', 'ignore').encode('utf8')
-        if lreplaced != l:
-            logger.error('line %r contains invalid utf8', l)
-            l = lreplaced
+        l_replaced = ln.decode("utf8", "ignore").encode("utf8")
+        if l_replaced != ln:
+            logger.error("line %r contains invalid utf8", ln)
+            ln = l_replaced
             raise ValueError
-        if l.startswith('<b>'): # section
-            section = re.sub(_section, r'\1', l)
+        if ln.startswith("<b>"):  # section
+            section = re.sub(_section, r"\1", ln)
         else:
-            foundsection = False
-            if l.strip().startswith('<b>'):
-                inside, outside = bold(l.strip())
-                if not outside and inside[-1][-1] == ':':
-                    foundsection = True
-                    section = ' '.join(inside)[:-1]
-            if not foundsection:
-                if not l.strip() and paragraphlines:
-                    yield store.paragraph(i, '\n'.join(paragraphlines), section, False)
+            found_section = False
+            if ln.strip().startswith("<b>"):
+                inside, outside = bold(ln.strip())
+                if not outside and inside[-1][-1] == ":":
+                    found_section = True
+                    section = " ".join(inside)[:-1]
+            if not found_section:
+                if not ln.strip() and para_lines:
+                    yield store.Paragraph(i, "\n".join(para_lines), section, False)
                     i += 1
-                    paragraphlines = []
-                elif l.strip():
-                    paragraphlines.append(l)
-    if paragraphlines:
-        yield store.paragraph(i, '\n'.join(paragraphlines), section, False)
-
-def _parsesynopsis(base, synopsis):
-    '''
-    >>> _parsesynopsis('/a/b/c', '/a/b/c: "p-r+o++g - foo bar."')
+                    para_lines = []
+                elif ln.strip():
+                    para_lines.append(ln)
+    if para_lines:
+        yield store.Paragraph(i, "\n".join(para_lines), section, False)
+
+
+def _parse_synopsis(base, synopsis):
+    """
+    >>> _parse_synopsis('/a/b/c', '/a/b/c: "p-r+o++g - foo bar."')
     ('p-r+o++g', 'foo bar')
-    '''
-    synopsis = synopsis[len(base)+3:-1]
-    if synopsis[-1] == '.':
+    """
+    synopsis = synopsis[len(base) + 3: -1]
+    if synopsis[-1] == ".":
         synopsis = synopsis[:-1]
-    return SPLITSYNOP.match(synopsis).groups()
+    return SPLIT_SYNOP.match(synopsis).groups()
 
-class manpage(object):
-    '''read the man page at path by executing w3mman2html.cgi and find its
+
+class ManPage:
+    """read the man page at path by executing w3mman2html.cgi and find its
     synopsis with lexgrog
 
     since some man pages share the same name (different versions), each
@@ -167,33 +185,38 @@ class manpage(object):
     - name of man page source file is given a score of 10
     - all other names found for a particular man page are given a score of 1
       (other names are found by scanning the output of lexgrog)
-    '''
+    """
+
     def __init__(self, path):
         self.path = path
-        self.shortpath = os.path.basename(self.path)
-        self.name = extractname(self.path)
-        self.aliases = set([self.name])
+        self.short_path = os.path.basename(self.path)
+        self.name = extract_name(self.path)
+        self.aliases = {self.name}
         self.synopsis = None
         self.paragraphs = None
         self._text = None
 
     def read(self):
-        '''Read the content from a local manpage file and store it in usable formats
-        on the class instance.'''
-        cmd = [config.MAN2HTML, urllib.urlencode({'local' : os.path.abspath(self.path)})]
-        logger.info('executing %r', ' '.join(cmd))
+        """Read the content from a local manpage file and store it in usable formats
+        on the class instance."""
+        cmd = [config.MAN2HTML, urllib.parse.urlencode({"local": os.path.abspath(self.path)})]
+        logger.info("executing %r", " ".join(cmd))
         self._text = subprocess.check_output(cmd, stderr=devnull, env=ENV)
         try:
-            self.synopsis = subprocess.check_output(['lexgrog', self.path], stderr=devnull).rstrip()
+            self.synopsis = subprocess.check_output(
+                ["lexgrog", self.path], stderr=devnull
+            ).rstrip()
         except subprocess.CalledProcessError:
-            logger.error('failed to extract synopsis for %s', self.name)
+            logger.error("failed to extract synopsis for %s", self.name)
 
     def parse(self):
-        self.paragraphs = list(_parsetext(self._text.splitlines()[7:-3]))
+        self.paragraphs = list(_parse_text(self._text.splitlines()[7:-3]))
         if not self.paragraphs:
-            raise errors.EmptyManpage(self.shortpath)
+            raise errors.EmptyManpage(self.short_path)
         if self.synopsis:
-            self.synopsis = [_parsesynopsis(self.path, l) for l in self.synopsis.splitlines()]
+            self.synopsis = [
+                _parse_synopsis(self.path, s_line) for s_line in self.synopsis.splitlines()
+            ]
 
             # figure out aliases from the synopsis
             d = collections.OrderedDict()
diff --git a/explainshell/matcher.py b/explainshell/matcher.py
index 41cb8649..5a7f1834 100644
--- a/explainshell/matcher.py
+++ b/explainshell/matcher.py
@@ -1,41 +1,48 @@
-import collections, logging, itertools
+import collections
+import logging
+import itertools
 
 import bashlex.parser
 import bashlex.ast
 
-from explainshell import errors, util, helpconstants
+from explainshell import errors, help_constants, util
 
-class matchgroup(object):
-    '''a class to group matchresults together
+
+class MatchGroup:
+    """a class to group matchresults together
 
     we group all shell results in one group and create a new group for every
-    command'''
+    command"""
+
     def __init__(self, name):
         self.name = name
         self.results = []
 
     def __repr__(self):
-        return '<matchgroup %r with %d results>' % (self.name, len(self.results))
+        return "<matchgroup %r with %d results>" % (self.name, len(self.results))
 
-class matchresult(collections.namedtuple('matchresult', 'start end text match')):
+
+class MatchResult(collections.namedtuple("MatchResult", "start end text match")):
     @property
     def unknown(self):
         return self.text is None
 
-matchwordexpansion = collections.namedtuple('matchwordexpansion',
-                                            'start end kind')
+
+match_word_exp = collections.namedtuple("match_word_exp", "start end kind")
 
 logger = logging.getLogger(__name__)
 
-class matcher(bashlex.ast.nodevisitor):
-    '''parse a command line and return a list of matchresults describing
+
+class Matcher(bashlex.ast.nodevisitor):
+    """parse a command line and return a list of `MatchResult`s describing
     each token.
-    '''
+    """
+
     def __init__(self, s, store):
-        self.s = s.encode('latin1', 'replace')
+        self.s = s.encode("latin1", "replace")
         self.store = store
-        self._prevoption = self._currentoption = None
-        self.groups = [matchgroup('shell')]
+        self._prev_option = self._current_option = None
+        self.groups = [MatchGroup("shell")]
 
         # a list of matchwordexpansions where expansions happened during word
         # expansion
@@ -45,98 +52,107 @@ def __init__(self, s, store):
         # command is started, we push a tuple with:
         # - the node that started this group. this is used to find it when
         #   a command ends (see visitnodeend)
-        # - its matchgroup. new matchresults will be added to it.
+        # - its `MatchGroup`. new `MatchResult`s will be added to it.
         # - a word used to end the top-most command. this is used when a flag
         #   starts a new command, e.g.  find -exec.
-        self.groupstack = [(None, self.groups[-1], None)]
+        self.group_stack = [(None, self.groups[-1], None)]
 
         # keep a stack of the currently visited compound command (if/for..)
         # to provide context when matching reserved words, since for example
         # the keyword 'done' can appear in a for, while..
-        self.compoundstack = []
+        self.compound_stack = []
 
         # a set of functions defined in the current input, we will try to match
         # commands against them so if one refers to defined function, it won't
         # show up as unknown or be taken from the db
         self.functions = set()
 
-    def _generatecommandgroupname(self):
-        existing = len([g for g in self.groups if g.name.startswith('command')])
-        return 'command%d' % existing
+    def _generate_cmd_group_name(self):
+        existing = len([g for g in self.groups if g.name.startswith("command")])
+        return f"command{existing}"
 
     @property
     def matches(self):
-        '''return the list of results from the most recently created group'''
-        return self.groupstack[-1][1].results
+        """return the list of results from the most recently created group"""
+        return self.group_stack[-1][1].results
 
     @property
-    def allmatches(self):
+    def all_matches(self):
         return list(itertools.chain.from_iterable(g.results for g in self.groups))
 
     @property
-    def manpage(self):
-        group = self.groupstack[-1][1]
+    def man_page(self):
+        group = self.group_stack[-1][1]
         # we do not have a manpage if the top of the stack is the shell group.
         # this can happen if the first argument is a command substitution
         # and we're not treating it as a "man page not found"
-        if group.name != 'shell':
+        if group.name != "shell":
             return group.manpage
 
     def find_option(self, opt):
-        self._currentoption = self.manpage.find_option(opt)
-        logger.debug('looking up option %r, got %r', opt, self._currentoption)
-        return self._currentoption
+        self._current_option = self.man_page.find_option(opt)
+        logger.debug("looking up option %r, got %r", opt, self._current_option)
+        return self._current_option
 
-    def findmanpages(self, prog):
-        prog = prog.decode('latin1')
-        logger.info('looking up %r in store', prog)
-        manpages = self.store.findmanpage(prog)
-        logger.info('found %r in store, got: %r, using %r', prog, manpages, manpages[0])
-        return manpages
+    def find_man_pages(self, prog):
+        logger.info("looking up %r in store", prog)
+        man_pages = self.store.find_man_page(prog)
+        logger.info("found %r in store, got: %r, using %r", prog, man_pages, man_pages[0])
+        return man_pages
 
     def unknown(self, token, start, end):
-        logger.debug('nothing to do with token %r', token)
-        return matchresult(start, end, None, None)
+        logger.debug("nothing to do with token %r", token)
+        return MatchResult(start, end, None, None)
 
     def visitreservedword(self, node, word):
         # first try the compound reserved words
         helptext = None
-        if self.compoundstack:
-            currentcompound = self.compoundstack[-1]
-            helptext = helpconstants.COMPOUNDRESERVEDWORDS.get(currentcompound, {}).get(word)
+        if self.compound_stack:
+            current_compound = self.compound_stack[-1]
+            helptext = help_constants.COMPOUND_RESERVED_WORDS.get(current_compound, {}).get(
+                word
+            )
 
         # try these if we don't have anything specific
         if not helptext:
-            helptext = helpconstants.RESERVEDWORDS[word]
+            helptext = help_constants.RESERVED_WORDS[word]
 
-        self.groups[0].results.append(matchresult(node.pos[0], node.pos[1], helptext, None))
+        self.groups[0].results.append(
+            MatchResult(node.pos[0], node.pos[1], helptext, None)
+        )
 
     def visitoperator(self, node, op):
         helptext = None
-        if self.compoundstack:
-            currentcompound = self.compoundstack[-1]
-            helptext = helpconstants.COMPOUNDRESERVEDWORDS.get(currentcompound, {}).get(op)
+        if self.compound_stack:
+            curr_compound = self.compound_stack[-1]
+            helptext = help_constants.COMPOUND_RESERVED_WORDS.get(curr_compound, {}).get(
+                op
+            )
 
         if not helptext:
-            helptext = helpconstants.OPERATORS[op]
+            helptext = help_constants.OPERATORS[op]
 
-        self.groups[0].results.append(matchresult(node.pos[0], node.pos[1], helptext, None))
+        self.groups[0].results.append(
+            MatchResult(node.pos[0], node.pos[1], helptext, None)
+        )
 
     def visitpipe(self, node, pipe):
         self.groups[0].results.append(
-                matchresult(node.pos[0], node.pos[1], helpconstants.PIPELINES, None))
+            MatchResult(node.pos[0], node.pos[1], help_constants.PIPELINES, None)
+        )
 
-    def visitredirect(self, node, input, type, output, heredoc):
-        helptext = [helpconstants.REDIRECTION]
+    def visitredirect(self, node, input, r_type, output, heredoc):
+        helptext = [help_constants.REDIRECTION]
 
-        if type == '>&' and isinstance(output, int):
-            type = type[:-1]
+        if r_type == ">&" and isinstance(output, int):
+            r_type = r_type[:-1]
 
-        if type in helpconstants.REDIRECTION_KIND:
-            helptext.append(helpconstants.REDIRECTION_KIND[type])
+        if r_type in help_constants.REDIRECTION_KIND:
+            helptext.append(help_constants.REDIRECTION_KIND[r_type])
 
         self.groups[0].results.append(
-                matchresult(node.pos[0], node.pos[1], '\n\n'.join(helptext), None))
+            MatchResult(node.pos[0], node.pos[1], "\n\n".join(helptext), None)
+        )
 
         # the output might contain a wordnode, visiting it will confuse the
         # matcher who'll think it's an argument, instead visit the expansions
@@ -151,21 +167,26 @@ def visitcommand(self, node, parts):
         assert parts
 
         # look for the first WordNode, which might not be at parts[0]
-        idxwordnode = bashlex.ast.findfirstkind(parts, 'word')
-        if idxwordnode == -1:
-            logger.info('no words found in command (probably contains only redirects)')
+        idx_word_node = bashlex.ast.findfirstkind(parts, "word")
+        if idx_word_node == -1:
+            logger.info("no words found in command (probably contains only redirects)")
             return
 
-        wordnode = parts[idxwordnode]
+        word_node = parts[idx_word_node]
 
         # check if this refers to a previously defined function
-        if wordnode.word in self.functions:
-            logger.info('word %r is a function, not trying to match it or its '
-                        'arguments', wordnode)
-
-            # first, add a matchresult for the function call
-            mr = matchresult(wordnode.pos[0], wordnode.pos[1],
-                             helpconstants._functioncall % wordnode.word, None)
+        if word_node.word in self.functions:
+            logger.info(
+                f"word {word_node} is a function, not trying to match it or it's arguments"
+            )
+
+            # first, add a MatchResult for the function call
+            mr = MatchResult(
+                word_node.pos[0],
+                word_node.pos[1],
+                help_constants._function_call % word_node.word,
+                None,
+            )
             self.matches.append(mr)
 
             # this is a bit nasty: if we were to visit the command like we
@@ -174,177 +195,203 @@ def visitcommand(self, node, parts):
             # consider them part of the function call
             for part in parts:
                 # maybe it's a redirect...
-                if part.kind != 'word':
+                if part.kind != "word":
                     self.visit(part)
                 else:
                     # this is an argument to the function
-                    if part is not wordnode:
-                        mr = matchresult(part.pos[0], part.pos[1],
-                                         helpconstants._functionarg % wordnode.word,
-                                         None)
+                    if part is not word_node:
+                        mr = MatchResult(
+                            part.pos[0],
+                            part.pos[1],
+                            help_constants._functionarg % word_node.word,
+                            None,
+                        )
                         self.matches.append(mr)
 
                         # visit any expansions in there
-                        for ppart in part.parts:
-                            self.visit(ppart)
+                        for p_part in part.parts:
+                            self.visit(p_part)
 
             # we're done with this commandnode, don't visit its children
             return False
 
-
         self.startcommand(node, parts, None)
 
     def visitif(self, *args):
-        self.compoundstack.append('if')
+        self.compound_stack.append("if")
+
     def visitfor(self, node, parts):
-        self.compoundstack.append('for')
+        self.compound_stack.append("for")
 
         for part in parts:
             # don't visit words since they're not part of the current command,
             # instead consider them part of the for construct
-            if part.kind == 'word':
-                mr = matchresult(part.pos[0], part.pos[1], helpconstants._for, None)
+            if part.kind == "word":
+                mr = MatchResult(part.pos[0], part.pos[1], help_constants._for, None)
                 self.groups[0].results.append(mr)
 
-                # but we do want to visit expanions
-                for ppart in part.parts:
-                    self.visit(ppart)
+                # but we do want to visit expansions
+                for p_part in part.parts:
+                    self.visit(p_part)
             else:
                 self.visit(part)
 
         return False
 
     def visitwhile(self, *args):
-        self.compoundstack.append('while')
+        self.compound_stack.append("while")
+
     def visituntil(self, *args):
-        self.compoundstack.append('until')
+        self.compound_stack.append("until")
 
     def visitnodeend(self, node):
-        if node.kind == 'command':
+        if node.kind == "command":
             # it's possible for visitcommand/end to be called without a command
             # group being pushed if it contains only redirect nodes
-            if len(self.groupstack) > 1:
-                logger.info('visitnodeend %r, groups %d', node, len(self.groupstack))
+            if len(self.group_stack) > 1:
+                logger.info("visitnodeend %r, groups %d", node, len(self.group_stack))
 
-                while self.groupstack[-1][0] is not node:
-                    logger.info('popping groups that are a result of nested commands')
+                while self.group_stack[-1][0] is not node:
+                    logger.info("popping groups that are a result of nested commands")
                     self.endcommand()
                 self.endcommand()
-        elif node.kind in ('if', 'for', 'while', 'until'):
-            kind = self.compoundstack.pop()
+        elif node.kind in ("if", "for", "while", "until"):
+            kind = self.compound_stack.pop()
             assert kind == node.kind
 
     def startcommand(self, commandnode, parts, endword, addgroup=True):
-        logger.info('startcommand commandnode=%r parts=%r, endword=%r, addgroup=%s',
-                    commandnode, parts, endword, addgroup)
-        idxwordnode = bashlex.ast.findfirstkind(parts, 'word')
-        assert idxwordnode != -1
-
-        wordnode = parts[idxwordnode]
-        if wordnode.parts:
-            logger.info('node %r has parts (it was expanded), no point in looking'
-                        ' up a manpage for it', wordnode)
+        logger.info(
+            "startcommand commandnode=%r parts=%r, endword=%r, addgroup=%s",
+            commandnode,
+            parts,
+            endword,
+            addgroup,
+        )
+        idx_word_node = bashlex.ast.findfirstkind(parts, "word")
+        assert idx_word_node != -1
+
+        word_node = parts[idx_word_node]
+        if word_node.parts:
+            logger.info(
+                "node %r has parts (it was expanded), no point in looking"
+                " up a manpage for it",
+                word_node,
+            )
 
             if addgroup:
-                mg = matchgroup(self._generatecommandgroupname())
+                mg = MatchGroup(self._generate_cmd_group_name())
                 mg.manpage = None
                 mg.suggestions = None
                 self.groups.append(mg)
-                self.groupstack.append((commandnode, mg, endword))
+                self.group_stack.append((commandnode, mg, endword))
 
             return False
 
-        startpos, endpos = wordnode.pos
+        startpos, endpos = word_node.pos
 
         try:
-            mps = self.findmanpages(wordnode.word)
+            mps = self.find_man_pages(word_node.word)
             # we consume this node here, pop it from parts so we
             # don't visit it again as an argument
-            parts.pop(idxwordnode)
-        except errors.ProgramDoesNotExist, e:
+            parts.pop(idx_word_node)
+        except errors.ProgramDoesNotExist as error_msg:
             if addgroup:
                 # add a group for this command, we'll mark it as unknown
                 # when visitword is called
-                logger.info('no manpage found for %r, adding a group for it',
-                            wordnode.word)
+                logger.info(
+                    f"no manpage found for {word_node.word}, adding a group for it"
+                )
 
-                mg = matchgroup(self._generatecommandgroupname())
-                mg.error = e
+                mg = MatchGroup(self._generate_cmd_group_name())
+                mg.error = error_msg
                 mg.manpage = None
                 mg.suggestions = None
                 self.groups.append(mg)
-                self.groupstack.append((commandnode, mg, endword))
+                self.group_stack.append((commandnode, mg, endword))
 
             return False
 
         manpage = mps[0]
-        idxnextwordnode = bashlex.ast.findfirstkind(parts, 'word')
+        idx_next_word_node = bashlex.ast.findfirstkind(parts, "word")
 
-        # check the next word for a possible multicommand if:
+        # check the next word for a possible multi_cmd if:
         # - the matched manpage says so
         # - we have another word node
         # - the word node has no expansions in it
-        if manpage.multicommand and idxnextwordnode != -1 and not parts[idxnextwordnode].parts:
-            nextwordnode = parts[idxnextwordnode]
+        if (
+            manpage.multi_cmd
+            and idx_next_word_node != -1
+            and not parts[idx_next_word_node].parts
+        ):
+            next_word_node = parts[idx_next_word_node]
             try:
-                multi = '%s %s' % (wordnode.word, nextwordnode.word)
-                logger.info('%r is a multicommand, trying to get another token and look up %r', manpage, multi)
-                mps = self.findmanpages(multi)
+                multi = f"{word_node.word} {next_word_node.word}"
+                logger.info(
+                    f"{manpage} is a multi_cmd, trying to get another token and look up {multi}"
+                )
+                mps = self.find_man_pages(multi)
                 manpage = mps[0]
                 # we consume this node here, pop it from parts so we
                 # don't visit it again as an argument
-                parts.pop(idxnextwordnode)
-                endpos = nextwordnode.pos[1]
+                parts.pop(idx_next_word_node)
+                endpos = next_word_node.pos[1]
             except errors.ProgramDoesNotExist:
-                logger.info('no manpage %r for multicommand %r', multi, manpage)
+                logger.info("no manpage %r for multi_cmd %r", multi, manpage)
 
-        # create a new matchgroup for the current command
-        mg = matchgroup(self._generatecommandgroupname())
+        # create a new MatchGroup for the current command
+        mg = MatchGroup(self._generate_cmd_group_name())
         mg.manpage = manpage
         mg.suggestions = mps[1:]
         self.groups.append(mg)
-        self.groupstack.append((commandnode, mg, endword))
+        self.group_stack.append((commandnode, mg, endword))
 
-        self.matches.append(matchresult(startpos, endpos,
-                            manpage.synopsis or helpconstants.NOSYNOPSIS, None))
+        self.matches.append(
+            MatchResult(
+                startpos, endpos, manpage.synopsis or help_constants.NO_SYNOPSIS, None
+            )
+        )
         return True
 
     def endcommand(self):
-        '''end the most recently created command group by popping it from the
-        group stack. groups are created by visitcommand or a nested command'''
-        assert len(self.groupstack) >= 2, 'groupstack must contain shell and command groups'
-        g = self.groupstack.pop()
-        logger.info('ending group %s', g)
+        """end the most recently created command group by popping it from the
+        group stack. groups are created by visitcommand or a nested command"""
+        assert (
+            len(self.group_stack) >= 2
+        ), "groupstack must contain shell and command groups"
+        g = self.group_stack.pop()
+        logger.info("ending group %s", g)
 
     def visitcommandsubstitution(self, node, command):
         kind = self.s[node.pos[0]]
-        substart = 2 if kind == '$' else 1
+        sub_start = 2 if kind == "$" else 1
 
         # start the expansion after the $( or `
-        self.expansions.append(matchwordexpansion(node.pos[0] + substart,
-                                                  node.pos[1] - 1,
-                                                  'substitution'))
+        self.expansions.append(
+            match_word_exp(node.pos[0] + sub_start, node.pos[1] - 1, "substitution")
+        )
 
         # do not try to match the child nodes
         return False
 
     def visitprocesssubstitution(self, node, command):
         # don't include opening <( and closing )
-        self.expansions.append(matchwordexpansion(node.pos[0] + 2,
-                                                  node.pos[1] - 1,
-                                                  'substitution'))
+        self.expansions.append(
+            match_word_exp(node.pos[0] + 2, node.pos[1] - 1, "substitution")
+        )
 
         # do not try to match the child nodes
         return False
 
     def visitassignment(self, node, word):
-        helptext = helpconstants.ASSIGNMENT
-        self.groups[0].results.append(matchresult(node.pos[0], node.pos[1], helptext, None))
+        helptext = help_constants.ASSIGNMENT
+        self.groups[0].results.append(
+            MatchResult(node.pos[0], node.pos[1], helptext, None)
+        )
 
     def visitword(self, node, word):
         def attemptfuzzy(chars):
             m = []
-            if chars[0] == '-':
+            if chars[0] == "-":
                 tokens = [chars[0:2]] + list(chars[2:])
                 considerarg = True
             else:
@@ -354,120 +401,146 @@ def attemptfuzzy(chars):
             pos = node.pos[0]
             prevoption = None
             for i, t in enumerate(tokens):
-                op = t if t[0] == '-' else '-' + t
+                op = t if t[0] == "-" else "-" + t
                 option = self.find_option(op)
                 if option:
-                    if considerarg and not m and option.expectsarg:
-                        logger.info('option %r expected an arg, taking the rest too', option)
+                    if considerarg and not m and option.expects_arg:
+                        logger.info(
+                            "option %r expected an arg, taking the rest too", option
+                        )
                         # reset the current option if we already took an argument,
                         # this prevents the next word node to also consider itself
                         # as an argument
-                        self._currentoption = None
-                        return [matchresult(pos, pos+len(chars), option.text, None)]
+                        self._current_option = None
+                        return [MatchResult(pos, pos + len(chars), option.text, None)]
 
-                    mr = matchresult(pos, pos+len(t), option.text, None)
+                    mr = MatchResult(pos, pos + len(t), option.text, None)
                     m.append(mr)
                 # if the previous option expected an argument and we couldn't
                 # match the current token, take the rest as its argument, this
                 # covers a series of short options where the last one has an argument
                 # with no space between it, such as 'xargs -r0n1'
-                elif considerarg and prevoption and prevoption.expectsarg:
+                elif considerarg and prevoption and prevoption.expects_arg:
                     pmr = m[-1]
-                    mr = matchresult(pmr.start, pmr.end+(len(tokens)-i), pmr.text, None)
+                    mr = MatchResult(
+                        pmr.start, pmr.end + (len(tokens) - i), pmr.text, None
+                    )
                     m[-1] = mr
                     # reset the current option if we already took an argument,
                     # this prevents the next word node to also consider itself
                     # as an argument
-                    self._currentoption = None
+                    self._current_option = None
                     break
                 else:
-                    m.append(self.unknown(t, pos, pos+len(t)))
+                    m.append(self.unknown(t, pos, pos + len(t)))
                 pos += len(t)
                 prevoption = option
             return m
 
         def _visitword(node, word):
-            if not self.manpage:
-                logger.info('inside an unknown command, giving up on %r', word)
+            if not self.man_page:
+                logger.info("inside an unknown command, giving up on %r", word)
                 self.matches.append(self.unknown(word, node.pos[0], node.pos[1]))
                 return
 
-            logger.info('trying to match token: %r', word)
+            logger.info("trying to match token: %r", word)
 
-            self._prevoption = self._currentoption
-            if word.startswith('--'):
-                word = word.split('=', 1)[0]
+            self._prev_option = self._current_option
+            if word.startswith("--"):
+                word = word.split("=", 1)[0]
             option = self.find_option(word)
             if option:
-                logger.info('found an exact match for %r: %r', word, option)
-                mr = matchresult(node.pos[0], node.pos[1], option.text, None)
+                logger.info("found an exact match for %r: %r", word, option)
+                mr = MatchResult(node.pos[0], node.pos[1], option.text, None)
                 self.matches.append(mr)
 
                 # check if we splitted the word just above, if we did then reset
                 # the current option so the next word doesn't consider itself
                 # an argument
                 if word != node.word:
-                    self._currentoption = None
+                    self._current_option = None
             else:
                 word = node.word
 
                 # check if we're inside a nested command and this word marks the end
-                if isinstance(self.groupstack[-1][-1], list) and word in self.groupstack[-1][-1]:
-                    logger.info('token %r ends current nested command', word)
+                if (
+                    isinstance(self.group_stack[-1][-1], list)
+                    and word in self.group_stack[-1][-1]
+                ):
+                    logger.info("token %r ends current nested command", word)
                     self.endcommand()
-                    mr = matchresult(node.pos[0], node.pos[1], self.matches[-1].text, None)
+                    mr = MatchResult(
+                        node.pos[0], node.pos[1], self.matches[-1].text, None
+                    )
                     self.matches.append(mr)
-                elif word != '-' and word.startswith('-') and not word.startswith('--'):
-                    logger.debug('looks like a short option')
+                elif word != "-" and word.startswith("-") and not word.startswith("--"):
+                    logger.debug("looks like a short option")
                     if len(word) > 2:
                         logger.info("trying to split it up")
                         self.matches.extend(attemptfuzzy(word))
                     else:
-                        self.matches.append(self.unknown(word, node.pos[0], node.pos[1]))
-                elif self._prevoption and self._prevoption.expectsarg:
-                    logger.info("previous option possibly expected an arg, and we can't"
-                            " find an option to match the current token, assuming it's an arg")
-                    ea = self._prevoption.expectsarg
-                    possibleargs = ea if isinstance(ea, list) else []
+                        self.matches.append(
+                            self.unknown(word, node.pos[0], node.pos[1])
+                        )
+                elif self._prev_option and self._prev_option.expects_arg:
+                    logger.info(
+                        "previous option possibly expected an arg, and we can't"
+                        " find an option to match the current token, assuming it's an arg"
+                    )
+                    ea = self._prev_option.expects_arg
+                    possible_args = ea if isinstance(ea, list) else []
                     take = True
-                    if possibleargs and word not in possibleargs:
+                    if possible_args and word not in possible_args:
                         take = False
-                        logger.info('token %r not in list of possible args %r for %r',
-                                    word, possibleargs, self._prevoption)
+                        logger.info(
+                            "token %r not in list of possible args %r for %r",
+                            word,
+                            possible_args,
+                            self._prev_option,
+                        )
                     if take:
-                        if self._prevoption.nestedcommand:
-                            logger.info('option %r can nest commands', self._prevoption)
-                            if self.startcommand(None, [node], self._prevoption.nestedcommand, addgroup=False):
-                                self._currentoption = None
+                        if self._prev_option.nested_cmd:
+                            logger.info("option %r can nest commands", self._prev_option)
+                            if self.startcommand(
+                                None,
+                                [node],
+                                self._prev_option.nested_cmd,
+                                addgroup=False,
+                            ):
+                                self._current_option = None
                                 return
 
                         pmr = self.matches[-1]
-                        mr = matchresult(pmr.start, node.pos[1], pmr.text, None)
+                        mr = MatchResult(pmr.start, node.pos[1], pmr.text, None)
                         self.matches[-1] = mr
                     else:
-                        self.matches.append(self.unknown(word, node.pos[0], node.pos[1]))
+                        self.matches.append(
+                            self.unknown(word, node.pos[0], node.pos[1])
+                        )
                 else:
-                    if self.manpage.partialmatch:
-                        logger.info('attemping to do a partial match')
+                    if self.man_page.partial_match:
+                        logger.info("attempting to do a partial match")
 
                         m = attemptfuzzy(word)
                         if not any(mm.unknown for mm in m):
-                            logger.info('found a match for everything, taking it')
+                            logger.info("found a match for everything, taking it")
                             self.matches.extend(m)
                             return
 
-                    if self.manpage.arguments:
-                        if self.manpage.nestedcommand:
-                            logger.info('manpage %r can nest commands', self.manpage)
-                            if self.startcommand(None, [node], self.manpage.nestedcommand, addgroup=False):
-                                self._currentoption = None
+                    if self.man_page.arguments:
+                        if self.man_page.nested_cmd:
+                            logger.info("manpage %r can nest commands", self.man_page)
+                            if self.startcommand(
+                                None, [node], self.man_page.nested_cmd, addgroup=False
+                            ):
+                                self._current_option = None
                                 return
 
-                        d = self.manpage.arguments
+                        d = self.man_page.arguments
                         k = list(d.keys())[0]
-                        logger.info('got arguments, using %r', k)
+                        logger.info("got arguments, using %r", k)
                         text = d[k]
-                        mr = matchresult(node.pos[0], node.pos[1], text, None)
+                        mr = MatchResult(node.pos[0], node.pos[1], text, None)
                         self.matches.append(mr)
                         return
 
@@ -481,8 +554,12 @@ def visitfunction(self, node, name, body, parts):
 
         def _iscompoundopenclosecurly(compound):
             first, last = compound.list[0], compound.list[-1]
-            if (first.kind == 'reservedword' and last.kind == 'reservedword' and
-                first.word == '{' and last.word == '}'):
+            if (
+                first.kind == "reservedword"
+                and last.kind == "reservedword"
+                and first.word == "{"
+                and last.word == "}"
+            ):
                 return True
 
         # if the compound command we have there is { }, let's include the
@@ -491,26 +568,32 @@ def _iscompoundopenclosecurly(compound):
         # context
         if _iscompoundopenclosecurly(body):
             # create a matchresult until after the first {
-            mr = matchresult(node.pos[0], body.list[0].pos[1],
-                             helpconstants._function, None)
+            mr = MatchResult(
+                node.pos[0], body.list[0].pos[1], help_constants._function, None
+            )
             self.groups[0].results.append(mr)
 
             # create a matchresult for the closing }
-            mr = matchresult(body.list[-1].pos[0], body.list[-1].pos[1],
-                             helpconstants._function, None)
+            mr = MatchResult(
+                body.list[-1].pos[0],
+                body.list[-1].pos[1],
+                help_constants._function,
+                None,
+            )
             self.groups[0].results.append(mr)
 
             # visit anything in between the { }
             for part in body.list[1:-1]:
                 self.visit(part)
         else:
-            beforebody = bashlex.ast.findfirstkind(parts, 'compound') - 1
+            beforebody = bashlex.ast.findfirstkind(parts, "compound") - 1
             assert beforebody > 0
             beforebody = parts[beforebody]
 
             # create a matchresult ending at the node before body
-            mr = matchresult(node.pos[0], beforebody.pos[1],
-                             helpconstants._function, None)
+            mr = MatchResult(
+                node.pos[0], beforebody.pos[1], help_constants._function, None
+            )
             self.groups[0].results.append(mr)
 
             self.visit(body)
@@ -518,79 +601,93 @@ def _iscompoundopenclosecurly(compound):
         return False
 
     def visittilde(self, node, value):
-        self.expansions.append(matchwordexpansion(node.pos[0], node.pos[1],
-                                                  'tilde'))
+        self.expansions.append(match_word_exp(node.pos[0], node.pos[1], "tilde"))
 
     def visitparameter(self, node, value):
         try:
             int(value)
-            kind = 'digits'
+            kind = "digits"
         except ValueError:
-            kind = helpconstants.parameters.get(value, 'param')
+            kind = help_constants.parameters.get(value, "param")
 
-        self.expansions.append(matchwordexpansion(node.pos[0], node.pos[1],
-                                                  'parameter-%s' % kind))
+        self.expansions.append(
+            match_word_exp(node.pos[0], node.pos[1], f"parameter-{kind}")
+        )
 
     def match(self):
-        logger.info('matching string %r', self.s)
+        if isinstance(self.s, bytes):
+            self.s = self.s.decode("utf-8")
+        logger.info(f"matching string {self.s}")
 
         # limit recursive parsing to a depth of 1
-        self.ast = bashlex.parser.parsesingle(self.s, expansionlimit=1,
-                                              strictmode=False)
+        self.ast = bashlex.parser.parsesingle(
+            self.s, expansionlimit=1, strictmode=False
+        )
         if self.ast:
             self.visit(self.ast)
-            assert len(self.groupstack) == 1, 'groupstack should contain only shell group after matching'
+            assert (
+                len(self.group_stack) == 1
+            ), "groupstack should contain only shell group after matching"
 
             # if we only have one command in there and no shell results/expansions,
             # reraise the original exception
-            if (len(self.groups) == 2 and not self.groups[0].results and
-                self.groups[1].manpage is None and not self.expansions):
+            if (
+                len(self.groups) == 2
+                and not self.groups[0].results
+                and self.groups[1].manpage is None
+                and not self.expansions
+            ):
                 raise self.groups[1].error
         else:
-            logger.warn('no AST generated for %r', self.s)
-
-        def debugmatch():
-            s = '\n'.join(['%d) %r = %r' % (i, self.s[m.start:m.end], m.text) for i, m in enumerate(self.allmatches)])
+            logger.warning("no AST generated for %r", self.s)
+
+        def debug_match():
+            s = "\n".join(
+                [
+                    f"{i}) {self.s[m.start: m.end]} = {m.text}"
+                    for i, m in enumerate(self.all_matches)
+                ]
+            )
             return s
 
-        self._markunparsedunknown()
+        self._mark_unparsed_unknown()
 
-        # fix each matchgroup seperately
+        # fix each MatchGroup separately
         for group in self.groups:
             if group.results:
-                if getattr(group, 'manpage', None):
+                if getattr(group, "manpage", None):
                     # ensure that the program part isn't unknown (i.e. it has
                     # something as its synopsis)
                     assert not group.results[0].unknown
 
-                group.results = self._mergeadjacent(group.results)
+                group.results = self._merge_adjacent(group.results)
 
-                # add matchresult.match to existing matches
+                # add MatchResult.match to existing matches
                 for i, m in enumerate(group.results):
-                    assert m.end <= len(self.s), '%d %d' % (m.end, len(self.s))
+                    assert m.end <= len(self.s), f"{m.end} {len(self.s)}"
 
-                    portion = self.s[m.start:m.end].decode('latin1')
-                    group.results[i] = matchresult(m.start, m.end, m.text, portion)
+                    portion = self.s[m.start: m.end]
+                    group.results[i] = MatchResult(m.start, m.end, m.text, portion)
 
-        logger.debug('%r matches:\n%s', self.s, debugmatch())
+        logger.debug("%r matches:\n%s", self.s, debug_match())
 
         # not strictly needed, but doesn't hurt
         self.expansions.sort()
 
         return self.groups
 
-    def _markunparsedunknown(self):
-        '''the parser may leave a remainder at the end of the string if it doesn't
-        match any of the rules, mark them as unknowns'''
-        parsed = [False]*len(self.s)
+    def _mark_unparsed_unknown(self):
+        """the parser may leave a remainder at the end of the string if it doesn't
+        match any of the rules, mark them as unknowns"""
+        parsed = [False] * len(self.s)
 
         # go over all existing matches to see if we've covered the
         # current position
-        for start, end, _, _ in self.allmatches:
+        for start, end, _, _ in self.all_matches:
             for i in range(start, end):
                 parsed[i] = True
 
-        for i in range(len(parsed)):
+        for i, parsed_i in enumerate(parsed):
             c = self.s[i]
             # whitespace is always 'unparsed'
             if c.isspace():
@@ -598,43 +695,45 @@ def _markunparsedunknown(self):
 
             # the parser ignores comments but we can use a trick to see if this
             # starts a comment and is beyond the ending index of the parsed
-            # portion of the inpnut
-            if (not self.ast or i > self.ast.pos[1]) and c == '#':
-                comment = matchresult(i, len(parsed), helpconstants.COMMENT, None)
+            # portion of the input
+            if (not self.ast or i > self.ast.pos[1]) and c == "#":
+                comment = MatchResult(i, len(parsed), help_constants.COMMENT, None)
                 self.groups[0].results.append(comment)
                 break
 
             if not parsed[i]:
                 # add unparsed results to the 'shell' group
-                self.groups[0].results.append(self.unknown(c, i, i+1))
+                self.groups[0].results.append(self.unknown(c, i, i + 1))
 
         # there are no overlaps, so sorting by the start is enough
         self.groups[0].results.sort(key=lambda mr: mr.start)
 
-    def _resultindex(self):
-        '''return a mapping of matchresults to their index among all
-        matches, sorted by the start position of the matchresult'''
+    def _result_index(self):
+        """return a mapping of `MatchResult`s to their index among all
+        matches, sorted by the start position of the `MatchResult`"""
         d = {}
         i = 0
-        for result in sorted(self.allmatches, key=lambda mr: mr.start):
+        for result in sorted(self.all_matches, key=lambda mr: mr.start):
             d[result] = i
             i += 1
         return d
 
-    def _mergeadjacent(self, matches):
+    def _merge_adjacent(self, matches):
         merged = []
-        resultindex = self._resultindex()
-        sametext = itertools.groupby(matches, lambda m: m.text)
-        for text, ll in sametext:
-            for l in util.groupcontinuous(ll, key=lambda m: resultindex[m]):
-                if len(l) == 1:
-                    merged.append(l[0])
+        result_index = self._result_index()
+        same_text = itertools.groupby(matches, lambda m: m.text)
+        for text, ll in same_text:
+            for l_group in util.group_continuous(ll, key=lambda m: result_index[m]):
+                l_group = list(l_group)
+
+                if len(l_group) == 1:
+                    merged.append(l_group[0])
                 else:
-                    start = l[0].start
-                    end = l[-1].end
-                    endindex = resultindex[l[-1]]
-                    for mr in l:
-                        del resultindex[mr]
-                    merged.append(matchresult(start, end, text, None))
-                    resultindex[merged[-1]] = endindex
+                    start = l_group[0].start
+                    end = l_group[-1].end
+                    end_index = result_index[l_group[-1]]
+                    for mr in l_group:
+                        del result_index[mr]
+                    merged.append(MatchResult(start, end, text, None))
+                    result_index[merged[-1]] = end_index
         return merged
diff --git a/explainshell/options.py b/explainshell/options.py
index 7abdd07f..990c3ec0 100644
--- a/explainshell/options.py
+++ b/explainshell/options.py
@@ -1,26 +1,31 @@
-import re, collections, logging
+import collections
+import logging
+import re
 
 from explainshell import store
 
-tokenstate = collections.namedtuple('tokenstate', 'startpos endpos token')
+token_state = collections.namedtuple("token_state", "startpos endpos token")
 
 logger = logging.getLogger(__name__)
 
+
 def extract(manpage):
-    '''extract options from all paragraphs that have been classified as containing
-    options'''
+    """extract options from all paragraphs that have been classified as containing
+    options"""
     for i, p in enumerate(manpage.paragraphs):
         if p.is_option:
-            s, l = extract_option(p.cleantext())
-            if s or l:
-                expectsarg = any(x.expectsarg for x in s + l)
+            s, ln = extract_option(p.cleantext())
+            if s or ln:
+                expects_arg = any(x.expects_arg for x in s + ln)
                 s = [x.flag for x in s]
-                l = [x.flag for x in l]
-                manpage.paragraphs[i] = store.option(p, s, l, expectsarg)
+                ln = [x.flag for x in ln]
+                manpage.paragraphs[i] = store.Option(p, s, ln, expects_arg)
             else:
                 logger.error("no options could be extracted from paragraph %r", p)
 
-opt_regex = re.compile(r'''
+
+opt_regex = re.compile(
+    r"""
     (?P<opt>--?(?:\?|\#|(?:\w+-)*\w+))  # option starts with - or -- and can have - in the middle but not at the end, also allow '-?'
     (?:
      (?:\s?(=)?\s?)           # -a=
@@ -39,30 +44,37 @@ def extract(manpage):
      )
      (?(argoptional)(?P<argoptionalc>[\]>])) # read closing ] or > if we have an arg
     )?                        # the whole arg thing is optional
-    (?P<ending>,\s*|\s+|\Z|/|\|)''', re.X)  # read any trailing whitespace or the end of the string
+    (?P<ending>,\s*|\s+|\Z|/|\|)""",
+    re.X,
+)  # read any trailing whitespace or the end of the string
 
-opt2_regex = re.compile(r'''
+opt2_regex = re.compile(
+    r"""
         (?P<opt>\w+)    # an option that doesn't start with any of the usual characters, e.g. options from 'dd' like bs=BYTES
         (?:
          (?:\s*=\s*)    # an optional arg, e.g. bs=BYTES
          (?P<arg>\w+)
         )
-        (?:,\s*|\s+|\Z)''', re.X) # end with , or whitespace or the end of the string
+        (?:,\s*|\s+|\Z)""",
+    re.X,
+)  # end with , or whitespace or the end of the string
+
 
 def _flag(s, pos=0):
-    '''
+    """
     >>> _flag('a=b').groupdict()
     {'opt': 'a', 'arg': 'b'}
     >>> bool(_flag('---c-d'))
     False
     >>> bool(_flag('foobar'))
     False
-    '''
+    """
     m = opt2_regex.match(s, pos)
     return m
 
+
 def _option(s, pos=0):
-    '''
+    """
     >>> bool(_option('-'))
     False
     >>> bool(_option('--'))
@@ -95,87 +107,91 @@ def _option(s, pos=0):
     False
     >>> _option('-a foo').end(0)
     3
-    '''
+    """
     m = opt_regex.match(s, pos)
     if m:
-        if m.group('argoptional'):
-            c = m.group('argoptional')
-            cc = m.group('argoptionalc')
-            if (c == '[' and cc == ']') or (c == '<' and cc == '>'):
+        if m.group("argoptional"):
+            c = m.group("argoptional")
+            cc = m.group("argoptionalc")
+            if (c == "[" and cc == "]") or (c == "<" and cc == ">"):
                 return m
             else:
                 return
     return m
 
-_eatbetweenregex = re.compile(r'\s*(?:or|,|\|)\s*')
 
-def _eatbetween(s, pos):
-    '''
-    >>> _eatbetween('foo', 0)
+_eat_between_regex = re.compile(r"\s*(?:or|,|\|)\s*")
+
+
+def _eat_between(s, pos):
+    """
+    >>> _eat_between('foo', 0)
     0
-    >>> _eatbetween('a, b', 1)
+    >>> _eat_between('a, b', 1)
     3
-    >>> _eatbetween('a|b', 1)
+    >>> _eat_between('a|b', 1)
     2
-    >>> _eatbetween('a or b', 1)
+    >>> _eat_between('a or b', 1)
     5
-    '''
-    m = _eatbetweenregex.match(s, pos)
+    """
+    m = _eat_between_regex.match(s, pos)
     if m:
         return m.end(0)
     return pos
 
-class extractedoption(collections.namedtuple('extractedoption', 'flag expectsarg')):
+
+class ExtractedOption(collections.namedtuple("ExtractedOption", "flag expects_arg")):
     def __eq__(self, other):
         if isinstance(other, str):
             return self.flag == other
         else:
-            return super(extractedoption, self).__eq__(other)
+            return super().__eq__(other)
 
     def __str__(self):
         return self.flag
 
+
 def extract_option(txt):
-    '''this is where the magic is (suppose) to happen. try and find options
-    using a regex'''
-    startpos = currpos = len(txt) - len(txt.lstrip())
+    """this is where the magic is (suppose) to happen. try and find options
+    using a regex"""
+    start_pos = curr_pos = len(txt) - len(txt.lstrip())
     short, long = [], []
 
-    m = _option(txt, currpos)
+    m = _option(txt, curr_pos)
 
     # keep going as long as options are found
     while m:
-        s = m.group('opt')
-        po = extractedoption(s, m.group('arg'))
-        if s.startswith('--'):
+        s = m.group("opt")
+        po = ExtractedOption(s, m.group("arg"))
+        if s.startswith("--"):
             long.append(po)
         else:
             short.append(po)
-        currpos = m.end(0)
-        currpos = _eatbetween(txt, currpos)
-        if m.group('ending') == '|':
-            m = _option(txt, currpos)
+        curr_pos = m.end(0)
+        curr_pos = _eat_between(txt, curr_pos)
+        if m.group("ending") == "|":
+            m = _option(txt, curr_pos)
             if not m:
-                startpos = currpos
-                while currpos < len(txt) and not txt[currpos].isspace():
-                    if txt[currpos] == '|':
-                        short.append(extractedoption(txt[startpos:currpos], None))
-                        startpos = currpos
-                    currpos += 1
-                leftover = txt[startpos:currpos]
+                start_pos = curr_pos
+                while curr_pos < len(txt) and not txt[curr_pos].isspace():
+                    if txt[curr_pos] == "|":
+                        short.append(ExtractedOption(txt[start_pos:curr_pos], None))
+                        start_pos = curr_pos
+                    curr_pos += 1
+                leftover = txt[start_pos:curr_pos]
                 if leftover:
-                    short.append(extractedoption(leftover, None))
+                    short.append(ExtractedOption(leftover, None))
         else:
-            m = _option(txt, currpos)
+            m = _option(txt, curr_pos)
 
-    if currpos == startpos:
-        m = _flag(txt, currpos)
+    if curr_pos == start_pos:
+        m = _flag(txt, curr_pos)
         while m:
-            s = m.group('opt')
-            po = extractedoption(s, m.group('arg'))
+            s = m.group("opt")
+            po = ExtractedOption(s, m.group("arg"))
             long.append(po)
-            currpos = m.end(0)
-            currpos = _eatbetween(txt, currpos)
-            m = _flag(txt, currpos)
+            curr_pos = m.end(0)
+            curr_pos = _eat_between(txt, curr_pos)
+            m = _flag(txt, curr_pos)
 
     return short, long
diff --git a/explainshell/store.py b/explainshell/store.py
index 6cadaf0b..e4809aee 100644
--- a/explainshell/store.py
+++ b/explainshell/store.py
@@ -1,74 +1,99 @@
-'''data objects to save processed man pages to mongodb'''
-import pymongo, collections, re, logging
+"""
+data objects to save processed man pages to mongodb
+"""
 
-from explainshell import errors, util, helpconstants, config
+import collections
+import re
+import logging
+
+import pymongo
+
+from explainshell import errors, help_constants, util, config
 
 logger = logging.getLogger(__name__)
 
-class classifiermanpage(collections.namedtuple('classifiermanpage', 'name paragraphs')):
-    '''a man page that had its paragraphs manually tagged as containing options
-    or not'''
+
+class ClassifierManpage(collections.namedtuple("ClassifierManpage", "name paragraphs")):
+    """a man page that had its paragraphs manually tagged as containing options
+    or not"""
+
     @staticmethod
     def from_store(d):
-        m = classifiermanpage(d['name'], [paragraph.from_store(p) for p in d['paragraphs']])
+        m = ClassifierManpage(
+            d["name"], [Paragraph.from_store(p) for p in d["paragraphs"]]
+        )
         return m
 
     def to_store(self):
-        return {'name' : self.name,
-                'paragraphs' : [p.to_store() for p in self.paragraphs]}
+        return {
+            "name": self.name,
+            "paragraphs": [p.to_store() for p in self.paragraphs],
+        }
+
+
+class Paragraph:
+    """a paragraph inside a man page is text that ends with two new lines"""
 
-class paragraph(object):
-    '''a paragraph inside a man page is text that ends with two new lines'''
     def __init__(self, idx, text, section, is_option):
         self.idx = idx
         self.text = text
         self.section = section
         self.is_option = is_option
 
-    def cleantext(self):
-        t = re.sub(r'<[^>]+>', '', self.text)
-        t = re.sub('&lt;', '<', t)
-        t = re.sub('&gt;', '>', t)
+    def clean_text(self):
+        t = re.sub(r"<[^>]+>", "", self.text)
+        t = re.sub("&lt;", "<", t)
+        t = re.sub("&gt;", ">", t)
         return t
 
     @staticmethod
     def from_store(d):
-        p = paragraph(d.get('idx', 0), d['text'].encode('utf8'), d['section'], d['is_option'])
+        p = Paragraph(
+            d.get("idx", 0), d["text"].encode("utf8"), d["section"], d["is_option"]
+        )
         return p
 
     def to_store(self):
-        return {'idx' : self.idx, 'text' : self.text, 'section' : self.section,
-                'is_option' : self.is_option}
+        return {
+            "idx": self.idx,
+            "text": self.text,
+            "section": self.section,
+            "is_option": self.is_option,
+        }
 
     def __repr__(self):
-        t = self.cleantext()
-        t = t[:min(20, t.find('\n'))].lstrip()
-        return '<paragraph %d, %s: %r>' % (self.idx, self.section, t)
+        t = self.clean_text()
+        t = t[: min(20, t.find("\n"))].lstrip()
+        return "<paragraph %d, %s: %r>" % (self.idx, self.section, t)
 
     def __eq__(self, other):
         if not other:
             return False
         return self.__dict__ == other.__dict__
 
-class option(paragraph):
-    '''a paragraph that contains extracted options
+
+class Option(Paragraph):
+    """a paragraph that contains extracted options
 
     short - a list of short options (-a, -b, ..)
     long - a list of long options (--a, --b)
-    expectsarg - specifies if one of the short/long options expects an additional argument
+    expects_arg - specifies if one of the short/long options expects an additional argument
     argument - specifies if to consider this as positional arguments
-    nestedcommand - specifies if the arguments to this option can start a nested command
-    '''
-    def __init__(self, p, short, long, expectsarg, argument=None, nestedcommand=False):
-        paragraph.__init__(self, p.idx, p.text, p.section, p.is_option)
+    nested_cmd - specifies if the arguments to this option can start a nested command
+    """
+
+    def __init__(self, p, short, long, expects_arg, argument=None, nested_cmd=False):
+        Paragraph.__init__(self, p.idx, p.text, p.section, p.is_option)
         self.short = short
         self.long = long
         self._opts = self.short + self.long
         self.argument = argument
-        self.expectsarg = expectsarg
-        self.nestedcommand = nestedcommand
-        if nestedcommand:
-            assert expectsarg, 'an option that can nest commands must expect an argument'
+        self.expects_arg = expects_arg
+        self.nested_cmd = nested_cmd
+        if nested_cmd:
+            assert (
+                expects_arg
+            ), "an option that can nest commands must expect an argument"
 
     @property
     def opts(self):
@@ -76,29 +101,36 @@ def opts(self):
 
     @classmethod
     def from_store(cls, d):
-        p = paragraph.from_store(d)
+        p = Paragraph.from_store(d)
 
-        return cls(p, d['short'], d['long'], d['expectsarg'], d['argument'],
-                   d.get('nestedcommand'))
+        return cls(
+            p,
+            d["short"],
+            d["long"],
+            d["expects_arg"],
+            d["argument"],
+            d.get("nested_cmd"),
+        )
 
     def to_store(self):
-        d = paragraph.to_store(self)
-        assert d['is_option']
-        d['short'] = self.short
-        d['long'] = self.long
-        d['expectsarg'] = self.expectsarg
-        d['argument'] = self.argument
-        d['nestedcommand'] = self.nestedcommand
+        d = Paragraph.to_store(self)
+        assert d["is_option"]
+        d["short"] = self.short
+        d["long"] = self.long
+        d["expects_arg"] = self.expects_arg
+        d["argument"] = self.argument
+        d["nested_cmd"] = self.nested_cmd
         return d
 
     def __str__(self):
-        return '(%s)' % ', '.join([str(x) for x in self.opts])
+        return "(%s)" % ", ".join([str(x) for x in self.opts])
 
     def __repr__(self):
-        return '<options for paragraph %d: %s>' % (self.idx, str(self))
+        return f"<options for paragraph {self.idx}: {self}"
+
 
-class manpage(object):
-    '''processed man page
+class ManPage:
+    """processed man page
 
     source - the path to the original source man page
     name - the name of this man page as extracted by manpage.manpage
@@ -106,48 +138,58 @@ class manpage(object):
     paragraphs - a list of paragraphs (and options) that contain all of the text and options
         extracted from this man page
     aliases - a list of aliases found for this man page
-    partialmatch - allow interperting options without a leading '-'
-    multicommand - consider sub commands when explaining a command with this man page,
+    partial_match - allow interpreting options without a leading '-'
+    multi_cmd - consider sub commands when explaining a command with this man page,
         e.g. git -> git commit
     updated - whether this man page was manually updated
-    nestedcommand - specifies if positional arguments to this program can start a nested command,
+    nested_cmd - specifies if positional arguments to this program can start a nested command,
         e.g. sudo, xargs
-    '''
-    def __init__(self, source, name, synopsis, paragraphs, aliases,
-                 partialmatch=False, multicommand=False, updated=False,
-                 nestedcommand=False):
+    """
+
+    def __init__(
+        self,
+        source,
+        name,
+        synopsis,
+        paragraphs,
+        aliases,
+        partial_match=False,
+        multi_cmd=False,
+        updated=False,
+        nested_cmd=False,
+    ):
         self.source = source
         self.name = name
         self.synopsis = synopsis
         self.paragraphs = paragraphs
         self.aliases = aliases
-        self.partialmatch = partialmatch
-        self.multicommand = multicommand
+        self.partial_match = partial_match
+        self.multi_cmd = multi_cmd
         self.updated = updated
-        self.nestedcommand = nestedcommand
+        self.nested_cmd = nested_cmd
 
-    def removeoption(self, idx):
+    def remove_option(self, idx):
         for i, p in self.paragraphs:
             if p.idx == idx:
-                if not isinstance(p, option):
-                    raise ValueError("paragraph %d isn't an option" % idx)
-                self.paragraphs[i] = paragraph(p.idx, p.text, p.section, False)
+                if not isinstance(p, Option):
+                    raise ValueError(f"paragraph {idx} isn't an option")
+                self.paragraphs[i] = Paragraph(p.idx, p.text, p.section, False)
                 return
-        raise ValueError('idx %d not found' % idx)
+        raise ValueError(f"idx {idx} not found")
 
     @property
-    def namesection(self):
-        name, section = util.namesection(self.source[:-3])
-        return '%s(%s)' % (name, section)
+    def name_section(self):
+        name, section = util.name_section(self.source[:-3])
+        return f"{name}({section})"
 
     @property
     def section(self):
-        name, section = util.namesection(self.source[:-3])
+        name, section = util.name_section(self.source[:-3])
         return section
 
     @property
     def options(self):
-        return [p for p in self.paragraphs if isinstance(p, option)]
+        return [p for p in self.paragraphs if isinstance(p, Option)]
 
     @property
     def arguments(self):
@@ -159,69 +201,85 @@ def arguments(self):
                 groups.setdefault(opt.argument, []).append(opt)
 
         # merge all the paragraphs under the same argument to a single string
-        for k, l in groups.iteritems():
-            groups[k] = '\n\n'.join([p.text for p in l])
+        for k, ln in groups.items():
+            groups[k] = "\n\n".join([p.text for p in ln])
 
         return groups
 
     @property
-    def synopsisnoname(self):
-        return re.match(r'[\w|-]+ - (.*)$', self.synopsis).group(1)
+    def synopsis_no_name(self):
+        return re.match(r"[\w|-]+ - (.*)$", self.synopsis).group(1)
 
     def find_option(self, flag):
-        for option in self.options:
-            for o in option.opts:
+        for o_tmp in self.options:
+            for o in o_tmp.opts:
                 if o == flag:
-                    return option
+                    return o_tmp
 
     def to_store(self):
-        return {'source' : self.source, 'name' : self.name, 'synopsis' : self.synopsis,
-                'paragraphs' : [p.to_store() for p in self.paragraphs],
-                'aliases' : self.aliases, 'partialmatch' : self.partialmatch,
-                'multicommand' : self.multicommand, 'updated' : self.updated,
-                'nestedcommand' : self.nestedcommand}
+        return {
+            "source": self.source,
+            "name": self.name,
+            "synopsis": self.synopsis,
+            "paragraphs": [p.to_store() for p in self.paragraphs],
+            "aliases": self.aliases,
+            "partial_match": self.partial_match,
+            "multi_cmd": self.multi_cmd,
+            "updated": self.updated,
+            "nested_cmd": self.nested_cmd,
+        }
 
     @staticmethod
     def from_store(d):
         paragraphs = []
-        for pd in d.get('paragraphs', []):
-            pp = paragraph.from_store(pd)
-            if pp.is_option == True and 'short' in pd:
-                pp = option.from_store(pd)
+        for pd in d.get("paragraphs", []):
+            pp = Paragraph.from_store(pd)
+            if pp.is_option is True and "short" in pd:
+                pp = Option.from_store(pd)
             paragraphs.append(pp)
 
-        synopsis = d['synopsis']
+        synopsis = d["synopsis"]
         if synopsis:
-            synopsis = synopsis.encode('utf8')
+            synopsis = synopsis.encode("utf8")
         else:
-            synopsis = helpconstants.NOSYNOPSIS
-
-        return manpage(d['source'], d['name'], synopsis, paragraphs,
-                       [tuple(x) for x in d['aliases']], d['partialmatch'],
-                       d['multicommand'], d['updated'], d.get('nestedcommand'))
+            synopsis = help_constants.NO_SYNOPSIS
+
+        return ManPage(
+            d["source"],
+            d["name"],
+            synopsis,
+            paragraphs,
+            [tuple(x) for x in d["aliases"]],
+            d["partial_match"],
+            d["multi_cmd"],
+            d["updated"],
+            d.get("nested_cmd"),
+        )
 
     @staticmethod
     def from_store_name_only(name, source):
-        return manpage(source, name, None, [], [], None, None, None)
+        return ManPage(source, name, None, [], [], None, None, None)
 
     def __repr__(self):
-        return '<manpage %r(%s), %d options>' % (self.name, self.section, len(self.options))
+        return f"<manpage {self.name}({self.section}), {len(self.options)} options>"
 
-class store(object):
-    '''read/write processed man pages from mongodb
+
+class Store:
+    """read/write processed man pages from mongodb
 
     we use three collections:
     1) classifier - contains manually tagged paragraphs from man pages
     2) manpage - contains a processed man page
     3) mapping - contains (name, manpageid, score) tuples
-    '''
-    def __init__(self, db='explainshell', host=config.MONGO_URI):
-        logger.info('creating store, db = %r, host = %r', db, host)
+    """
+
+    def __init__(self, db="explainshell", host=config.MONGO_URI):
+        logger.info("creating store, db = %r, host = %r", db, host)
         self.connection = pymongo.MongoClient(host)
         self.db = self.connection[db]
-        self.classifier = self.db['classifier']
-        self.manpage = self.db['manpage']
-        self.mapping = self.db['mapping']
+        self.classifier = self.db["classifier"]
+        self.manpage = self.db["manpage"]
+        self.mapping = self.db["mapping"]
 
     def close(self):
         self.connection.disconnect()
@@ -231,169 +289,201 @@ def drop(self, confirm=False):
         if not confirm:
             return
 
-        logger.info('dropping mapping, manpage, collections')
+        logger.info("dropping mapping, manpage, collections")
         self.mapping.drop()
         self.manpage.drop()
 
     def trainingset(self):
         for d in self.classifier.find():
-            yield classifiermanpage.from_store(d)
+            yield ClassifierManpage.from_store(d)
 
     def __contains__(self, name):
-        c = self.mapping.find({'src' : name}).count()
+        c = self.mapping.find({"src": name}).count()
         return c > 0
 
     def __iter__(self):
         for d in self.manpage.find():
-            yield manpage.from_store(d)
+            yield ManPage.from_store(d)
 
-    def findmanpage(self, name):
-        '''find a man page by its name, everything following the last dot (.) in name,
+    def find_man_page(self, name):
+        """find a man page by its name, everything following the last dot (.) in name,
         is taken as the section of the man page
 
         we return the man page found with the highest score, and a list of
         suggestions that also matched the given name (only the first item
-        is prepopulated with the option data)'''
-        if name.endswith('.gz'):
-            logger.info('name ends with .gz, looking up an exact match by source')
-            d = self.manpage.find_one({'source':name})
+        is prepopulated with the option data)"""
+        if name.endswith(".gz"):
+            logger.info("name ends with .gz, looking up an exact match by source")
+            d = self.manpage.find_one({"source": name})
             if not d:
                 raise errors.ProgramDoesNotExist(name)
-            m = manpage.from_store(d)
-            logger.info('returning %s', m)
+            m = ManPage.from_store(d)
+            logger.info("returning %s", m)
             return [m]
 
         section = None
-        origname = name
+        orig_name = name
 
         # don't try to look for a section if it's . (source)
-        if name != '.':
-            splitted = name.rsplit('.', 1)
+        if name != ".":
+            splitted = name.rsplit(".", 1)
             name = splitted[0]
             if len(splitted) > 1:
                 section = splitted[1]
 
-        logger.info('looking up manpage in mapping with src %r', name)
-        cursor = self.mapping.find({'src' : name})
-        count = cursor.count()
-        if not count:
+        logger.info("looking up manpage in mapping with src %r", name)
+        cursor = self.mapping.find({"src": name})
+
+        logger.info(list(cursor))
+        count = len(list(cursor))
+        if count == 0:
             raise errors.ProgramDoesNotExist(name)
 
-        dsts = dict(((d['dst'], d['score']) for d in cursor))
-        cursor = self.manpage.find({'_id' : {'$in' : list(dsts.keys())}}, {'name' : 1, 'source' : 1})
-        if cursor.count() != len(dsts):
-            logger.error('one of %r mappings is missing in manpage collection '
-                         '(%d mappings, %d found)', dsts, len(dsts), cursor.count())
-        results = [(d.pop('_id'), manpage.from_store_name_only(**d)) for d in cursor]
+        dsts = {d["dst"]: d["score"] for d in cursor}
+        cursor = self.manpage.find(
+            {"_id": {"$in": list(dsts.keys())}}, {"name": 1, "source": 1}
+        )
+        if len(list(cursor)) != len(dsts):
+            logger.error(
+                "one of %r mappings is missing in manpage collection "
+                "(%d mappings, %d found)",
+                dsts,
+                len(dsts),
+                cursor.count(),
+            )
+        results = [(d.pop("_id"), ManPage.from_store_name_only(**d)) for d in cursor]
         results.sort(key=lambda x: dsts.get(x[0], 0), reverse=True)
-        logger.info('got %s', results)
+        logger.info("got %s", results)
         if section is not None:
             if len(results) > 1:
-                results.sort(key=lambda (oid, m): m.section == section, reverse=True)
-                logger.info(r'sorting %r so %s is first', results, section)
-            if not results[0][1].section == section:
-                raise errors.ProgramDoesNotExist(origname)
-            results.extend(self._discovermanpagesuggestions(results[0][0], results))
+                results.sort(
+                    key=lambda oid_m: oid_m[1].section == section, reverse=True
+                )
+                logger.info(r"sorting %r so %s is first", results, section)
+            if results[0][1].section != section:
+                raise errors.ProgramDoesNotExist(orig_name)
+            results.extend(self._discover_manpage_suggestions(results[0][0], results))
 
         oid = results[0][0]
         results = [x[1] for x in results]
-        results[0] = manpage.from_store(self.manpage.find_one({'_id' : oid}))
+        results[0] = ManPage.from_store(self.manpage.find_one({"_id": oid}))
         return results
 
-    def _discovermanpagesuggestions(self, oid, existing):
-        '''find suggestions for a given man page
+    def _discover_manpage_suggestions(self, oid, existing):
+        """find suggestions for a given man page
 
         oid is the objectid of the man page in question,
         existing is a list of (oid, man page) of suggestions that were
         already discovered
-        '''
-        skip = set([oid for oid, m in existing])
-        cursor = self.mapping.find({'dst' : oid})
+        """
+        skip = {oid for oid, m in existing}
+        cursor = self.mapping.find({"dst": oid})
         # find all srcs that point to oid
-        srcs = [d['src'] for d in cursor]
+        srcs = [d["src"] for d in cursor]
         # find all dsts of srcs
-        suggestionoids = self.mapping.find({'src' : {'$in' : srcs}}, {'dst' : 1})
+        suggestionoids = self.mapping.find({"src": {"$in": srcs}}, {"dst": 1})
         # remove already discovered
-        suggestionoids = [d['dst'] for d in suggestionoids if d['dst'] not in skip]
+        suggestionoids = [d["dst"] for d in suggestionoids if d["dst"] not in skip]
         if not suggestionoids:
             return []
 
         # get just the name and source of found suggestions
-        suggestionoids = self.manpage.find({'_id' : {'$in' : suggestionoids}},
-                                           {'name' : 1, 'source' : 1})
-        return [(d.pop('_id'), manpage.from_store_name_only(**d)) for d in suggestionoids]
+        suggestionoids = self.manpage.find(
+            {"_id": {"$in": suggestionoids}}, {"name": 1, "source": 1}
+        )
+        return [
+            (d.pop("_id"), ManPage.from_store_name_only(**d)) for d in suggestionoids
+        ]
 
-    def addmapping(self, src, dst, score):
-        self.mapping.insert({'src' : src, 'dst' : dst, 'score' : score})
+    def add_mapping(self, src, dst, score):
+        self.mapping.insert({"src": src, "dst": dst, "score": score})
 
     def addmanpage(self, m):
-        '''add m into the store, if it exists first remove it and its mappings
+        """add m into the store, if it exists first remove it and its mappings
 
         each man page may have aliases besides the name determined by its
-        basename'''
-        d = self.manpage.find_one({'source' : m.source})
+        basename"""
+        d = self.manpage.find_one({"source": m.source})
         if d:
-            logger.info('removing old manpage %s (%s)', m.source, d['_id'])
-            self.manpage.remove(d['_id'])
+            logger.info("removing old manpage %s (%s)", m.source, d["_id"])
+            self.manpage.remove(d["_id"])
 
             # remove old mappings if there are any
             c = self.mapping.count()
-            self.mapping.remove({'dst' : d['_id']})
+            self.mapping.remove({"dst": d["_id"]})
             c -= self.mapping.count()
-            logger.info('removed %d mappings for manpage %s', c, m.source)
+            logger.info("removed %d mappings for manpage %s", c, m.source)
 
         o = self.manpage.insert(m.to_store())
 
         for alias, score in m.aliases:
-            self.addmapping(alias, o, score)
-            logger.info('inserting mapping (alias) %s -> %s (%s) with score %d', alias, m.name, o, score)
+            self.add_mapping(alias, o, score)
+            logger.info(
+                "inserting mapping (alias) %s -> %s (%s) with score %d",
+                alias,
+                m.name,
+                o,
+                score,
+            )
         return m
 
-    def updatemanpage(self, m):
-        '''update m and add new aliases if necessary
+    def update_man_page(self, m):
+        """update m and add new aliases if necessary
 
-        change updated attribute so we don't overwrite this in the future'''
-        logger.info('updating manpage %s', m.source)
+        change updated attribute so we don't overwrite this in the future"""
+        logger.info("updating manpage %s", m.source)
         m.updated = True
-        self.manpage.update({'source' : m.source}, m.to_store())
-        _id = self.manpage.find_one({'source' : m.source}, fields={'_id':1})['_id']
+        self.manpage.update({"source": m.source}, m.to_store())
+        _id = self.manpage.find_one({"source": m.source}, fields={"_id": 1})["_id"]
         for alias, score in m.aliases:
             if alias not in self:
-                self.addmapping(alias, _id, score)
-                logger.info('inserting mapping (alias) %s -> %s (%s) with score %d', alias, m.name, _id, score)
+                self.add_mapping(alias, _id, score)
+                logger.info(
+                    "inserting mapping (alias) %s -> %s (%s) with score %d",
+                    alias,
+                    m.name,
+                    _id,
+                    score,
+                )
             else:
-                logger.debug('mapping (alias) %s -> %s (%s) already exists', alias, m.name, _id)
+                logger.debug(
+                    "mapping (alias) %s -> %s (%s) already exists", alias, m.name, _id
+                )
         return m
 
     def verify(self):
         # check that everything in manpage is reachable
         mappings = list(self.mapping.find())
-        reachable = set([m['dst'] for m in mappings])
-        manpages = set([m['_id'] for m in self.manpage.find(fields={'_id':1})])
+        reachable = {m["dst"] for m in mappings}
+        man_pages = {m["_id"] for m in self.manpage.find(fields={"_id": 1})}
 
         ok = True
-        unreachable = manpages - reachable
+        unreachable = man_pages - reachable
         if unreachable:
-            logger.error('manpages %r are unreachable (nothing maps to them)', unreachable)
-            unreachable = [self.manpage.find_one({'_id' : u})['name'] for u in unreachable]
+            logger.error(
+                "manpages %r are unreachable (nothing maps to them)", unreachable
+            )
+            unreachable = [
+                self.manpage.find_one({"_id": u})["name"] for u in unreachable
+            ]
             ok = False
 
-        notfound = reachable - manpages
+        notfound = reachable - man_pages
         if notfound:
-            logger.error('mappings to inexisting manpages: %r', notfound)
+            logger.error("mappings to inexisting manpages: %r", notfound)
             ok = False
 
         return ok, unreachable, notfound
 
     def names(self):
-        cursor = self.manpage.find(fields={'name':1})
+        cursor = self.manpage.find(fields={"name": 1})
         for d in cursor:
-            yield d['_id'], d['name']
+            yield d["_id"], d["name"]
 
     def mappings(self):
-        cursor = self.mapping.find(fields={'src':1})
+        cursor = self.mapping.find(fields={"src": 1})
         for d in cursor:
-            yield d['src'], d['_id']
+            yield d["src"], d["_id"]
 
-    def setmulticommand(self, manpageid):
-        self.manpage.update({'_id' : manpageid}, {'$set' : {'multicommand' : True}})
+    def setmulti_cmd(self, manpage_id):
+        self.manpage.update({"_id": manpage_id}, {"$set": {"multi_cmd": True}})
diff --git a/explainshell/util.py b/explainshell/util.py
index 127177b1..708e7f06 100644
--- a/explainshell/util.py
+++ b/explainshell/util.py
@@ -1,8 +1,9 @@
 import itertools
 from operator import itemgetter
 
-def consecutive(l, fn):
-    '''yield consecutive items from l that fn returns True for them
+
+def consecutive(ln, fn):
+    """yield consecutive items from l that fn returns True for them
 
     >>> even = lambda x: x % 2 == 0
     >>> list(consecutive([], even))
@@ -17,8 +18,8 @@ def consecutive(l, fn):
     [[1], [2, 4]]
     >>> list(consecutive([1, 2, 4, 5, 7, 8, 10], even))
     [[1], [2, 4], [5], [7], [8, 10]]
-    '''
-    it = iter(l)
+    """
+    it = iter(ln)
     ll = []
     try:
         while True:
@@ -34,50 +35,56 @@ def consecutive(l, fn):
         if ll:
             yield ll
 
-def groupcontinuous(l, key=None):
-    '''
+
+def group_continuous(l, key=None):
+    """
     >>> list(groupcontinuous([1, 2, 4, 5, 7, 8, 10]))
     [[1, 2], [4, 5], [7, 8], [10]]
     >>> list(groupcontinuous(range(5)))
     [[0, 1, 2, 3, 4]]
-    '''
+    """
     if key is None:
         key = lambda x: x
-    for k, g in itertools.groupby(enumerate(l), lambda (i, x): i-key(x)):
+    for k, g in itertools.groupby(enumerate(l), lambda ix: ix[0] - key(ix[1])):
         yield map(itemgetter(1), g)
 
-def toposorted(graph, parents):
+
+def topo_sorted(graph, parents):
     """
     Returns vertices of a DAG in topological order.
 
     Arguments:
-    graph -- vetices of a graph to be toposorted
-    parents -- function (vertex) -> vertices to preceed
+    graph -- vertices of a graph to be topo_sorted
+    parents -- function (vertex) -> vertices to proceed
                given vertex in output
     """
     result = []
     used = set()
+
     def use(v, top):
         if id(v) in used:
             return
         for parent in parents(v):
             if parent is top:
-                raise ValueError('graph is cyclical', graph)
+                raise ValueError("graph is cyclical", graph)
             use(parent, v)
         used.add(id(v))
         result.append(v)
+
     for v in graph:
         use(v, v)
     return result
 
+
 def pairwise(iterable):
     a, b = itertools.tee(iterable)
     next(b, None)
-    return itertools.izip(a, b)
+    return zip(a, b)
+
 
-class peekable(object):
-    '''
-    >>> it = peekable(iter('abc'))
+class Peekable:
+    """
+    >>> it = Peekable(iter('abc'))
     >>> it.index, it.peek(), it.index, it.peek(), it.next(), it.index, it.peek(), it.next(), it.next(), it.index
     (0, 'a', 0, 'a', 'a', 1, 'b', 'b', 'c', 3)
     >>> it.peek()
@@ -92,54 +99,62 @@ class peekable(object):
     Traceback (most recent call last):
       File "<stdin>", line 1, in ?
     StopIteration
-    '''
+    """
+
     def __init__(self, it):
         self.it = it
         self._peeked = False
-        self._peekvalue = None
+        self._peek_value = None
         self._idx = 0
+
     def __iter__(self):
         return self
+
     def next(self):
         if self._peeked:
             self._peeked = False
             self._idx += 1
-            return self._peekvalue
-        n = self.it.next()
+            return self._peek_value
+        n = next(self.it)
         self._idx += 1
         return n
-    def hasnext(self):
+
+    def has_next(self):
         try:
             self.peek()
             return True
         except StopIteration:
             return False
+
     def peek(self):
         if self._peeked:
-            return self._peekvalue
+            return self._peek_value
         else:
-            self._peekvalue = self.it.next()
+            self._peek_value = next(self.it)
             self._peeked = True
-            return self._peekvalue
+            return self._peek_value
+
     @property
     def index(self):
-        '''return the index of the next item returned by next()'''
+        """return the index of the next item returned by next()"""
         return self._idx
 
-def namesection(path):
-    assert '.gz' not in path
-    name, section = path.rsplit('.', 1)
+
+def name_section(path):
+    assert ".gz" not in path
+    name, section = path.rsplit(".", 1)
     return name, section
 
-class propertycache(object):
+
+class PropertyCache:
     def __init__(self, func):
         self.func = func
         self.name = func.__name__
 
     def __get__(self, obj, type=None):
         result = self.func(obj)
-        self.cachevalue(obj, result)
+        self.cache_value(obj, result)
         return result
 
-    def cachevalue(self, obj, value):
+    def cache_value(self, obj, value):
         setattr(obj, self.name, value)
diff --git a/explainshell/web/__init__.py b/explainshell/web/__init__.py
index b16223ca..ef2a7784 100644
--- a/explainshell/web/__init__.py
+++ b/explainshell/web/__init__.py
@@ -1,10 +1,11 @@
 from flask import Flask
+
 app = Flask(__name__)
 
 from explainshell.web import views
 from explainshell import store, config
 
 if config.DEBUG:
-    from explainshell.web import debugviews
+    from explainshell.web import debug_views
 
 app.config.from_object(config)
diff --git a/explainshell/web/debug_views.py b/explainshell/web/debug_views.py
new file mode 100644
index 00000000..00fa2b1c
--- /dev/null
+++ b/explainshell/web/debug_views.py
@@ -0,0 +1,88 @@
+import logging
+
+from flask import render_template, request, abort, redirect, url_for, json
+
+from explainshell import manager, config, store
+from explainshell.web import app, helpers
+
+logger = logging.getLogger(__name__)
+
+
+@app.route("/debug")
+def debug():
+    s = store.Store("explainshell", config.MONGO_URI)
+    d = {"manpages": []}
+    for mp in s:
+        synopsis = ""
+        if mp.synopsis:
+            synopsis = mp.synopsis[:20]
+        dd = {"name": mp.name, "synopsis": synopsis}
+        o_list = []
+        for o in mp.options:
+            o_list.append(str(o))
+        dd["options"] = ", ".join(o_list)
+        d["manpages"].append(dd)
+    d["manpages"].sort(key=lambda d: d["name"].lower())
+    return render_template("debug.html", d=d)
+
+
+def _convert_value(value):
+    if isinstance(value, list):
+        return [s.strip() for s in value]
+    elif value.lower() == "true":
+        return True
+    elif value:
+        return value.strip()
+    return False
+
+
+@app.route("/debug/tag/<source>", methods=["GET", "POST"])
+def tag(source):
+    mngr = manager.Manager(config.MONGO_URI, "explainshell", [], False, False)
+    s = mngr.store
+    m = s.find_man_page(source)[0]
+    assert m
+
+    if "paragraphs" in request.form:
+        paragraphs = json.loads(request.form["paragraphs"])
+        m_paragraphs = []
+        for d in paragraphs:
+            idx = d["idx"]
+            text = d["text"]
+            section = d["section"]
+            short = [s.strip() for s in d["short"]]
+            long = [s.strip() for s in d["long"]]
+            expects_arg = _convert_value(d["expects_arg"])
+            nested_cmd = _convert_value(d["nested_cmd"])
+            if isinstance(nested_cmd, str):
+                nested_cmd = [nested_cmd]
+            elif nested_cmd is True:
+                logger.error("nested_cmd %r must be a string or list", nested_cmd)
+                abort(503)
+            argument = d["argument"]
+            if not argument:
+                argument = None
+            p = store.Paragraph(idx, text, section, d["is_option"])
+            if d["is_option"] and (short or long or argument):
+                p = store.Option(p, short, long, expects_arg, argument, nested_cmd)
+            m_paragraphs.append(p)
+
+        if request.form.get("nested_cmd", "").lower() == "true":
+            m.nested_cmd = True
+        else:
+            m.nested_cmd = False
+        m = mngr.edit(m, m_paragraphs)
+        if m:
+            return redirect(url_for("explain", cmd=m.name))
+        else:
+            abort(503)
+    else:
+        helpers.convert_paragraphs(m)
+        for p in m.paragraphs:
+            if isinstance(p, store.Option):
+                if isinstance(p.expects_arg, list):
+                    p.expects_arg = ", ".join(p.expects_arg)
+                if isinstance(p.nested_cmd, list):
+                    p.nested_cmd = ", ".join(p.nested_cmd)
+
+        return render_template("tagger.html", m=m)
diff --git a/explainshell/web/debugviews.py b/explainshell/web/debugviews.py
deleted file mode 100644
index 08f970a5..00000000
--- a/explainshell/web/debugviews.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import logging
-
-from flask import render_template, request, abort, redirect, url_for, json
-
-from explainshell import manager, config, store
-from explainshell.web import app, helpers
-
-logger = logging.getLogger(__name__)
-
-@app.route('/debug')
-def debug():
-    s = store.store('explainshell', config.MONGO_URI)
-    d = {'manpages' : []}
-    for mp in s:
-        synopsis = ''
-        if mp.synopsis:
-            synopsis = mp.synopsis[:20]
-        dd = {'name' : mp.name, 'synopsis' : synopsis}
-        l = []
-        for o in mp.options:
-            l.append(str(o))
-        dd['options'] = ', '.join(l)
-        d['manpages'].append(dd)
-    d['manpages'].sort(key=lambda d: d['name'].lower())
-    return render_template('debug.html', d=d)
-
-def _convertvalue(value):
-    if isinstance(value, list):
-        return [s.strip() for s in value]
-    elif value.lower() == 'true':
-        return True
-    elif value:
-        return value.strip()
-    return False
-
-@app.route('/debug/tag/<source>', methods=['GET', 'POST'])
-def tag(source):
-    mngr = manager.manager(config.MONGO_URI, 'explainshell', [], False, False)
-    s = mngr.store
-    m = s.findmanpage(source)[0]
-    assert m
-
-    if 'paragraphs' in request.form:
-        paragraphs = json.loads(request.form['paragraphs'])
-        mparagraphs = []
-        for d in paragraphs:
-            idx = d['idx']
-            text = d['text']
-            section = d['section']
-            short = [s.strip() for s in d['short']]
-            long = [s.strip() for s in d['long']]
-            expectsarg = _convertvalue(d['expectsarg'])
-            nestedcommand = _convertvalue(d['nestedcommand'])
-            if isinstance(nestedcommand, str):
-                nestedcommand = [nestedcommand]
-            elif nestedcommand is True:
-                logger.error('nestedcommand %r must be a string or list', nestedcommand)
-                abort(503)
-            argument = d['argument']
-            if not argument:
-                argument = None
-            p = store.paragraph(idx, text, section, d['is_option'])
-            if d['is_option'] and (short or long or argument):
-                p = store.option(p, short, long, expectsarg, argument, nestedcommand)
-            mparagraphs.append(p)
-
-        if request.form.get('nestedcommand', '').lower() == 'true':
-            m.nestedcommand = True
-        else:
-            m.nestedcommand = False
-        m = mngr.edit(m, mparagraphs)
-        if m:
-            return redirect(url_for('explain', cmd=m.name))
-        else:
-            abort(503)
-    else:
-        helpers.convertparagraphs(m)
-        for p in m.paragraphs:
-            if isinstance(p, store.option):
-                if isinstance(p.expectsarg, list):
-                    p.expectsarg = ', '.join(p.expectsarg)
-                if isinstance(p.nestedcommand, list):
-                    p.nestedcommand = ', '.join(p.nestedcommand)
-
-        return render_template('tagger.html', m=m)
diff --git a/explainshell/web/helpers.py b/explainshell/web/helpers.py
index 481337aa..41d555f2 100644
--- a/explainshell/web/helpers.py
+++ b/explainshell/web/helpers.py
@@ -1,20 +1,23 @@
 from explainshell import util
 
-def convertparagraphs(manpage):
+
+def convert_paragraphs(manpage):
     for p in manpage.paragraphs:
-        p.text = p.text.decode('utf-8')
+        p.text = p.text.decode("utf-8")
     return manpage
 
+
 def suggestions(matches, command):
-    '''enrich command matches with links to other man pages with the
-    same name'''
+    """enrich command matches with links to other man pages with the
+    same name"""
     for m in matches:
-        if 'name' in m and 'suggestions' in m:
-            before = command[:m['start']]
-            after = command[m['end']:]
-            newsuggestions = []
-            for othermp in sorted(m['suggestions'], key=lambda mp: mp.section):
-                mid = '%s.%s' % (othermp.name, othermp.section)
-                newsuggestions.append({'cmd' : ''.join([before, mid, after]),
-                                  'text' : othermp.namesection})
-            m['suggestions'] = newsuggestions
+        if "name" in m and "suggestions" in m:
+            before = command[: m["start"]]
+            after = command[m["end"]:]
+            new_suggestions = []
+            for other_mp in sorted(m["suggestions"], key=lambda mp: mp.section):
+                mid = f"{other_mp.name}.{other_mp.section}"
+                new_suggestions.append(
+                    {"cmd": "".join([before, mid, after]), "text": other_mp.namesection}
+                )
+            m["suggestions"] = new_suggestions
diff --git a/explainshell/web/views.py b/explainshell/web/views.py
index 645fb97d..001d710a 100644
--- a/explainshell/web/views.py
+++ b/explainshell/web/views.py
@@ -10,210 +10,235 @@
 
 logger = logging.getLogger(__name__)
 
-@app.route('/')
+
+@app.route("/")
 def index():
-    return render_template('index.html')
+    return render_template("index.html")
+
 
-@app.route('/about')
+@app.route("/about")
 def about():
-    return render_template('about.html')
+    return render_template("about.html")
+
 
-@app.route('/explain')
+@app.route("/explain")
 def explain():
-    if 'cmd' not in request.args or not request.args['cmd'].strip():
-        return redirect('/')
-    command = request.args['cmd'].strip()
-    command = command[:1000] # trim commands longer than 1000 characters
-    if '\n' in command:
-        return render_template('errors/error.html', title='parsing error!',
-                               message='no newlines please')
-
-    s = store.store('explainshell', config.MONGO_URI)
+    if "cmd" not in request.args or not request.args["cmd"].strip():
+        return redirect("/")
+    command = request.args["cmd"].strip()
+    command = command[:1000]  # trim commands longer than 1000 characters
+    if "\n" in command:
+        return render_template(
+            "errors/error.html", title="parsing error!", message="no newlines please"
+        )
+
+    s = store.Store("explainshell", config.MONGO_URI)
     try:
-        matches, helptext = explaincommand(command, s)
-        return render_template('explain.html',
-                               matches=matches,
-                               helptext=helptext,
-                               getargs=command)
-
-    except errors.ProgramDoesNotExist, e:
-        return render_template('errors/missingmanpage.html', title='missing man page', e=e)
-    except bashlex.errors.ParsingError, e:
-        logger.warn('%r parsing error: %s', command, e.message)
-        return render_template('errors/parsingerror.html', title='parsing error!', e=e)
-    except NotImplementedError, e:
-        logger.warn('not implemented error trying to explain %r', command)
-        msg = ("the parser doesn't support %r constructs in the command you tried. you may "
-               "<a href='https://github.com/idank/explainshell/issues'>report a "
-               "bug</a> to have this added, if one doesn't already exist.") % e.args[0]
-
-        return render_template('errors/error.html', title='error!', message=msg)
-    except:
-        logger.error('uncaught exception trying to explain %r', command, exc_info=True)
-        msg = 'something went wrong... this was logged and will be checked'
-        return render_template('errors/error.html', title='error!', message=msg)
-
-@app.route('/explain/<program>', defaults={'section' : None})
-@app.route('/explain/<section>/<program>')
-def explainold(section, program):
-    logger.info('/explain section=%r program=%r', section, program)
-
-    s = store.store('explainshell', config.MONGO_URI)
+        matches, helptext = explain_cmd(command, s)
+        return render_template(
+            "explain.html", matches=matches, helptext=helptext, getargs=command
+        )
+
+    except errors.ProgramDoesNotExist as error_msg:
+        return render_template(
+            "errors/missingmanpage.html", title="missing man page", e=error_msg
+        )
+    except bashlex.errors.ParsingError as error_msg:
+        logger.warning("%r parsing error: %s", command, error_msg.message)
+        return render_template(
+            "errors/parsingerror.html", title="parsing error!", e=error_msg
+        )
+    except NotImplementedError as error_msg:
+        logger.warning("not implemented error trying to explain %r", command)
+        msg = (
+            "the parser doesn't support %r constructs in the command you tried. you may "
+            "<a href='https://github.com/idank/explainshell/issues'>report a "
+            "bug</a> to have this added, if one doesn't already exist."
+        ) % error_msg.args[0]
+
+        return render_template("errors/error.html", title="error!", message=msg)
+    except Exception as error_msg:
+        logger.error(error_msg)
+        logger.error("uncaught exception trying to explain %r", command, exc_info=True)
+        msg = "something went wrong... this was logged and will be checked"
+        return render_template("errors/error.html", title="error!", message=msg)
+
+
+@app.route("/explain/<program>", defaults={"section": None})
+@app.route("/explain/<section>/<program>")
+def explain_old(section, program):
+    logger.info("/explain section=%r program=%r", section, program)
+
+    s = store.Store("explainshell", config.MONGO_URI)
     if section is not None:
-        program = '%s.%s' % (program, section)
+        program = f"{program}.{section}"
 
     # keep links to old urls alive
-    if 'args' in request.args:
-        args = request.args['args']
-        command = '%s %s' % (program, args)
-        return redirect('/explain?cmd=%s' % urllib.quote_plus(command), 301)
+    if "args" in request.args:
+        args = request.args["args"]
+        command = f"{program} {args}"
+        return redirect("/explain?cmd=%s" % urllib.parse.quote_plus(command), 301)
     else:
         try:
-            mp, suggestions = explainprogram(program, s)
-            return render_template('options.html', mp=mp, suggestions=suggestions)
-        except errors.ProgramDoesNotExist, e:
-            return render_template('errors/missingmanpage.html', title='missing man page', e=e)
+            mp, suggestions = explain_program(program, s)
+            return render_template("options.html", mp=mp, suggestions=suggestions)
+        except errors.ProgramDoesNotExist as e:
+            return render_template(
+                "errors/missingmanpage.html", title="missing man page", e=e
+            )
 
-def explainprogram(program, store):
-    mps = store.findmanpage(program)
+
+def explain_program(program, store):
+    mps = store.find_man_page(program)
     mp = mps.pop(0)
     program = mp.namesection
 
     synopsis = mp.synopsis
     if synopsis:
-        synopsis = synopsis.decode('utf-8')
+        synopsis = synopsis.decode("utf-8")
 
-    mp = {'source' : mp.source[:-3],
-          'section' : mp.section,
-          'program' : program,
-          'synopsis' : synopsis,
-          'options' : [o.text.decode('utf-8') for o in mp.options]}
+    mp = {
+        "source": mp.source[:-3],
+        "section": mp.section,
+        "program": program,
+        "synopsis": synopsis,
+        "options": [o.text.decode("utf-8") for o in mp.options],
+    }
 
     suggestions = []
-    for othermp in mps:
-        d = {'text' : othermp.namesection,
-             'link' : '%s/%s' % (othermp.section, othermp.name)}
+    for other_mp in mps:
+        d = {
+            "text": other_mp.namesection,
+            "link": f"{other_mp.section}/{other_mp.name}",
+        }
         suggestions.append(d)
-    logger.info('suggestions: %s', suggestions)
+    logger.info("suggestions: %s", suggestions)
     return mp, suggestions
 
-def _makematch(start, end, match, commandclass, helpclass):
-    return {'match' : match, 'start' : start, 'end' : end, 'spaces' : '',
-            'commandclass' : commandclass, 'helpclass' : helpclass}
 
-def explaincommand(command, store):
-    matcher_ = matcher.matcher(command, store)
+def _make_match(start, end, match, cmd_class, help_class):
+    return {
+        "match": match,
+        "start": start,
+        "end": end,
+        "spaces": "",
+        "commandclass": cmd_class,
+        "helpclass": help_class,
+    }
+
+
+def explain_cmd(command, store):
+    matcher_ = matcher.Matcher(command, store)
     groups = matcher_.match()
     expansions = matcher_.expansions
 
-    shellgroup = groups[0]
-    commandgroups = groups[1:]
+    shell_group = groups[0]
+    cmd_groups = groups[1:]
     matches = []
 
     # save a mapping between the help text to its assigned id,
     # we're going to reuse ids that have the same text
-    texttoid = {}
+    text_ids = {}
 
     # remember where each assigned id has started in the source,
     # we're going to use it later on to sort the help text by start
     # position
-    idstartpos = {}
+    id_start_pos = {}
 
-    l = []
-    for m in shellgroup.results:
-        commandclass = shellgroup.name
-        helpclass = 'help-%d' % len(texttoid)
-        text = m.text
+    ln = []
+    for m in shell_group.results:
+        cmd_class = shell_group.name
+        help_class = f"help-{len(text_ids)}"
+        text = str(m.text)
         if text:
-            text = text.decode('utf-8')
-            helpclass = texttoid.setdefault(text, helpclass)
+            help_class = text_ids.setdefault(text, help_class)
         else:
             # unknowns in the shell group are possible when our parser left
-            # an unparsed remainder, see matcher._markunparsedunknown
-            commandclass += ' unknown'
-            helpclass = ''
-        if helpclass:
-            idstartpos.setdefault(helpclass, m.start)
-
-        d = _makematch(m.start, m.end, m.match, commandclass, helpclass)
-        formatmatch(d, m, expansions)
-
-        l.append(d)
-    matches.append(l)
-
-    for commandgroup in commandgroups:
-        l = []
-        for m in commandgroup.results:
-            commandclass = commandgroup.name
-            helpclass = 'help-%d' % len(texttoid)
-            text = m.text
+            # an unparsed remainder, see matcher._mark_unparsed_unknown
+            cmd_class += " unknown"
+            help_class = ""
+        if help_class:
+            id_start_pos.setdefault(help_class, m.start)
+
+        d = _make_match(m.start, m.end, m.match, cmd_class, help_class)
+        format_match(d, m, expansions)
+
+        ln.append(d)
+    matches.append(ln)
+
+    for cmd_group in cmd_groups:
+        ln = []
+        for m in cmd_group.results:
+            cmd_class = cmd_group.name
+            help_class = f"help-{len(text_ids)}"
+            text = str(m.text)
             if text:
-                text = text.decode('utf-8')
-                helpclass = texttoid.setdefault(text, helpclass)
+                help_class = text_ids.setdefault(text, help_class)
             else:
-                commandclass += ' unknown'
-                helpclass = ''
-            if helpclass:
-                idstartpos.setdefault(helpclass, m.start)
-
-            d = _makematch(m.start, m.end, m.match, commandclass, helpclass)
-            formatmatch(d, m, expansions)
-
-            l.append(d)
-
-        d = l[0]
-        d['commandclass'] += ' simplecommandstart'
-        if commandgroup.manpage:
-            d['name'] = commandgroup.manpage.name
-            d['section'] = commandgroup.manpage.section
-            if '.' not in d['match']:
-                d['match'] = '%s(%s)' % (d['match'], d['section'])
-            d['suggestions'] = commandgroup.suggestions
-            d['source'] = commandgroup.manpage.source[:-5]
-        matches.append(l)
+                cmd_class += " unknown"
+                help_class = ""
+            if help_class:
+                id_start_pos.setdefault(help_class, m.start)
+
+            d = _make_match(m.start, m.end, m.match, cmd_class, help_class)
+            format_match(d, m, expansions)
+
+            ln.append(d)
+
+        d = ln[0]
+        d["commandclass"] += " simplecommandstart"
+        if cmd_group.manpage:
+            d["name"] = cmd_group.manpage.name
+            d["section"] = cmd_group.manpage.section
+            if "." not in d["match"]:
+                d["match"] = f"{d['match']}({d['section']})"
+            d["suggestions"] = cmd_group.suggestions
+            d["source"] = cmd_group.manpage.source[:-5]
+        matches.append(ln)
 
     matches = list(itertools.chain.from_iterable(matches))
     helpers.suggestions(matches, command)
 
     # _checkoverlaps(matcher_.s, matches)
-    matches.sort(key=lambda d: d['start'])
+    matches.sort(key=lambda d: d["start"])
 
-    it = util.peekable(iter(matches))
-    while it.hasnext():
+    it = util.Peekable(iter(matches))
+    while it.has_next():
         m = it.next()
         spaces = 0
-        if it.hasnext():
-            spaces = it.peek()['start'] - m['end']
-        m['spaces'] = ' ' * spaces
+        if it.has_next():
+            spaces = it.peek()["start"] - m["end"]
+        m["spaces"] = " " * spaces
 
-    helptext = sorted(texttoid.iteritems(), key=lambda (k, v): idstartpos[v])
+    helptext = sorted(text_ids.items(), key=lambda kv: id_start_pos[kv[1]])
 
     return matches, helptext
 
-def formatmatch(d, m, expansions):
-    '''populate the match field in d by escaping m.match and generating
-    links to any command/process substitutions'''
+
+def format_match(d, m, expansions):
+    """populate the match field in d by escaping m.match and generating
+    links to any command/process substitutions"""
 
     # save us some work later: do any expansions overlap
     # the current match?
-    hassubsinmatch = False
+    has_subs_in_match = False
 
     for start, end, kind in expansions:
         if m.start <= start and end <= m.end:
-            hassubsinmatch = True
+            has_subs_in_match = True
             break
 
     # if not, just escape the current match
-    if not hassubsinmatch:
-        d['match'] = markupsafe.escape(m.match)
+    if not has_subs_in_match:
+        d["match"] = markupsafe.escape(m.match)
         return
 
     # used in es.js
-    d['commandclass'] += ' hasexpansion'
+    d["commandclass"] += " hasexpansion"
 
     # go over the expansions, wrapping them with a link; leave everything else
     # untouched
-    expandedmatch = ''
+    expandedmatch = ""
     i = 0
     for start, end, kind in expansions:
         if start >= m.end:
@@ -224,43 +249,50 @@ def formatmatch(d, m, expansions):
         if i < relativestart:
             for j in range(i, relativestart):
                 if m.match[j].isspace():
-                    expandedmatch += markupsafe.Markup('&nbsp;')
+                    expandedmatch += markupsafe.Markup("&nbsp;")
                 else:
                     expandedmatch += markupsafe.escape(m.match[j])
             i = relativestart + 1
         if m.start <= start and end <= m.end:
             s = m.match[relativestart:relativeend]
 
-            if kind == 'substitution':
+            if kind == "substitution":
                 content = markupsafe.Markup(_substitutionmarkup(s))
             else:
                 content = s
 
             expandedmatch += markupsafe.Markup(
-                    '<span class="expansion-{0}">{1}</span>').format(kind, content)
+                '<span class="expansion-{0}">{1}</span>'
+            ).format(kind, content)
             i = relativeend
 
     if i < len(m.match):
         expandedmatch += markupsafe.escape(m.match[i:])
 
     assert expandedmatch
-    d['match'] = expandedmatch
+    d["match"] = expandedmatch
+
 
 def _substitutionmarkup(cmd):
-    '''
+    """
     >>> _substitutionmarkup('foo')
     '<a href="/explain?cmd=foo" title="Zoom in to nested command">foo</a>'
     >>> _substitutionmarkup('cat <&3')
     '<a href="/explain?cmd=cat+%3C%263" title="Zoom in to nested command">cat <&3</a>'
-    '''
-    encoded = urllib.urlencode({'cmd': cmd})
-    return ('<a href="/explain?{query}" title="Zoom in to nested command">{cmd}'
-            '</a>').format(cmd=cmd, query=encoded)
+    """
+    encoded = urllib.parse.urlencode({"cmd": cmd})
+    return (
+        '<a href="/explain?{query}" title="Zoom in to nested command">{cmd}' "</a>"
+    ).format(cmd=cmd, query=encoded)
+
 
 def _checkoverlaps(s, matches):
-    explained = [None]*len(s)
+    explained = [None] * len(s)
     for d in matches:
-        for i in range(d['start'], d['end']):
+        for i in range(d["start"], d["end"]):
             if explained[i]:
-                raise RuntimeError("explained overlap for group %s at %d with %s" % (d, i, explained[i]))
+                raise RuntimeError(
+                    "explained overlap for group %s at %d with %s"
+                    % (d, i, explained[i])
+                )
             explained[i] = d
diff --git a/runserver.py b/runserver.py
index 3401a405..730114e9 100644
--- a/runserver.py
+++ b/runserver.py
@@ -1,11 +1,13 @@
+import logging.config
+
 from explainshell import config
 from explainshell.web import app
 
-import logging.config
+
 logging.config.dictConfig(config.LOGGING_DICT)
 
 if __name__ == '__main__':
-    if config.HOST_IP:
+    if len(config.HOST_IP) > 1:
         app.run(debug=config.DEBUG, host=config.HOST_IP)
     else:
         app.run(debug=config.DEBUG)
diff --git a/tests/helpers.py b/tests/helpers.py
index 07e19d4f..7b2b0276 100644
--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -1,43 +1,63 @@
-from explainshell import matcher, store, errors, options, helpconstants
+from explainshell import help_constants, matcher, store, errors, options
 
-class mockstore(object):
+
+class MockStore:
     def __init__(self):
-        sp = store.paragraph
-        so = store.option
-        sm = store.manpage
-
-        p0 = sp(0, '-a desc', '', True)
-        p1 = sp(1, '-b <arg> desc', '', True)
-        p2 = sp(2, '-? help text', '', True)
-        p3 = sp(3, '-c=one,two\ndesc', '', True)
-        p4 = sp(4, 'FILE argument', '', True)
-        p5 = sp(5, '-exec nest', '', True)
-        opts = [so(p0, ['-a'], ['--a'], False),
-                so(p1, ['-b'], ['--b'], '<arg>'),
-                so(p2, ['-?'], [], False),
-                so(p3, ['-c'], [], ['one', 'two'])]
+        sp = store.Paragraph
+        so = store.Option
+        sm = store.ManPage
+
+        p0 = sp(0, "-a desc", "", True)
+        p1 = sp(1, "-b <arg> desc", "", True)
+        p2 = sp(2, "-? help text", "", True)
+        p3 = sp(3, "-c=one,two\ndesc", "", True)
+        p4 = sp(4, "FILE argument", "", True)
+        p5 = sp(5, "-exec nest", "", True)
+        opts = [
+            so(p0, ["-a"], ["--a"], False),
+            so(p1, ["-b"], ["--b"], "<arg>"),
+            so(p2, ["-?"], [], False),
+            so(p3, ["-c"], [], ["one", "two"]),
+        ]
         self.manpages = {
-                'bar' : sm('bar.1.gz', 'bar', 'bar synopsis', opts, [], multicommand=True),
-                'baz' : sm('baz.1.gz', 'baz', 'baz synopsis', opts, [], partialmatch=True),
-                'bar foo' : sm('bar-foo.1.gz', 'bar-foo', 'bar foo synopsis', opts, [], partialmatch=True),
-                'nosynopsis' : sm('bar.1.gz', 'bar', None, opts, [])}
+            "bar": sm("bar.1.gz", "bar", "bar synopsis", opts, [], multi_cmd=True),
+            "baz": sm("baz.1.gz", "baz", "baz synopsis", opts, [], partial_match=True),
+            "bar foo": sm(
+                "bar-foo.1.gz",
+                "bar-foo",
+                "bar foo synopsis",
+                opts,
+                [],
+                partial_match=True,
+            ),
+            "nosynopsis": sm("bar.1.gz", "bar", None, opts, []),
+        }
 
-        self.dup = [sm('dup.1.gz', 'dup', 'dup1 synopsis', opts, []),
-                    sm('dup.2.gz', 'dup', 'dup2 synopsis', opts, [])]
+        self.dup = [
+            sm("dup.1.gz", "dup", "dup1 synopsis", opts, []),
+            sm("dup.2.gz", "dup", "dup2 synopsis", opts, []),
+        ]
 
         opts = list(opts)
-        opts.append(so(p4, [], [], False, 'FILE'))
-        opts.append(so(p5, ['-exec'], [], True, nestedcommand=['EOF', ';']))
-        self.manpages['withargs'] = sm('withargs.1.gz', 'withargs', 'withargs synopsis',
-                                       opts, [], partialmatch=True, nestedcommand=True)
+        opts.append(so(p4, [], [], False, "FILE"))
+        opts.append(so(p5, ["-exec"], [], True, nested_cmd=["EOF", ";"]))
+        self.manpages["withargs"] = sm(
+            "withargs.1.gz",
+            "withargs",
+            "withargs synopsis",
+            opts,
+            [],
+            partial_match=True,
+            nested_cmd=True,
+        )
 
-    def findmanpage(self, x, section=None):
+    def find_man_page(self, x, section=None):
         try:
-            if x == 'dup':
+            if x == "dup":
                 return self.dup
             return [self.manpages[x]]
         except KeyError:
             raise errors.ProgramDoesNotExist(x)
 
-s = mockstore()
 
+s = MockStore()
diff --git a/tests/test-fixer.py b/tests/test-fixer.py
index 4e612e68..626d350b 100644
--- a/tests/test-fixer.py
+++ b/tests/test-fixer.py
@@ -3,54 +3,58 @@
 
 from explainshell import fixer, options, store
 
+
 class test_fixer(unittest.TestCase):
     def setUp(self):
-        self._oldfixerscls = fixer.fixerscls[:]
+        self._oldfixerscls = fixer.fixers_cls[:]
 
     def tearDown(self):
-        fixer.fixerscls = self._oldfixerscls
+        fixer.fixers_cls = self._oldfixerscls
 
     def test_changes(self):
-        class myfixer(fixer.basefixer):
+        class myfixer(fixer.BaseFixer):
             def pre_get_raw_manpage(self):
-                self.mctx['foo'] = 'bar'
+                self.mctx["foo"] = "bar"
 
         d = {}
-        fixer.fixerscls = [myfixer]
-        r = fixer.runner(d)
-        self.assertTrue('foo' not in d)
+        fixer.fixers_cls = [myfixer]
+        r = fixer.Runner(d)
+        self.assertTrue("foo" not in d)
         r.pre_get_raw_manpage()
-        self.assertEquals(d['foo'], 'bar')
+        self.assertEqual(d["foo"], "bar")
 
     def test_paragraphjoiner(self):
-        maxdistance = fixer.paragraphjoiner.maxdistance
+        maxdistance = fixer.ParagraphJoiner.max_distance
 
-        paragraphs = [store.paragraph(i, chr(ord('a') + i), None, False) for i in range(26)]
+        paragraphs = [
+            store.Paragraph(i, chr(ord("a") + i), None, False) for i in range(26)
+        ]
         options = [
-                store.option(paragraphs[0], [], [], False),
-                store.option(paragraphs[1], [], [], False),
-                store.option(paragraphs[5], [], [], False),
-                store.option(paragraphs[5+maxdistance-1], [], [], False),
-                store.option(paragraphs[15], [], [], False),
-                store.option(paragraphs[17], [], [], False),
-                store.option(paragraphs[-1], [], [], False)]
-
-        f = fixer.paragraphjoiner(None)
+            store.Option(paragraphs[0], [], [], False),
+            store.Option(paragraphs[1], [], [], False),
+            store.Option(paragraphs[5], [], [], False),
+            store.Option(paragraphs[5 + maxdistance - 1], [], [], False),
+            store.Option(paragraphs[15], [], [], False),
+            store.Option(paragraphs[17], [], [], False),
+            store.Option(paragraphs[-1], [], [], False),
+        ]
+
+        f = fixer.ParagraphJoiner(None)
         merged = f._join(paragraphs, options)
 
-        #self.assertEquals(merged, 7)
-        #self.assertEquals(len(paragraphs), 19)
-        self.assertEquals(options[0].text, 'a')
-        self.assertEquals(options[1].text.replace('\n', ''), 'bcde')
-        self.assertEquals(options[2].text.replace('\n', ''), 'fghi')
-        self.assertEquals(options[3].text, 'j')
-        self.assertEquals(options[4].text.replace('\n', ''), 'pq')
-        self.assertEquals(options[5].text, 'r')
-        self.assertEquals(options[6].text, 'z')
+        # self.assertEqual(merged, 7)
+        # self.assertEqual(len(paragraphs), 19)
+        self.assertEqual(options[0].text, "a")
+        self.assertEqual(options[1].text.replace("\n", ""), "bcde")
+        self.assertEqual(options[2].text.replace("\n", ""), "fghi")
+        self.assertEqual(options[3].text, "j")
+        self.assertEqual(options[4].text.replace("\n", ""), "pq")
+        self.assertEqual(options[5].text, "r")
+        self.assertEqual(options[6].text, "z")
 
         # join again to make sure nothing is changed
         oldparagraphs = copy.deepcopy(paragraphs)
         oldoptions = copy.deepcopy(options)
         f._join(paragraphs, options)
-        self.assertEquals(oldparagraphs, paragraphs)
-        self.assertEquals(oldoptions, options)
+        self.assertEqual(oldparagraphs, paragraphs)
+        self.assertEqual(oldoptions, options)
diff --git a/tests/test-integration.py b/tests/test-integration.py
index 62b1094a..7b06c917 100644
--- a/tests/test-integration.py
+++ b/tests/test-integration.py
@@ -2,31 +2,44 @@
 
 from explainshell import manager, config, matcher
 
+
 @unittest.skip("nltk usage is broken due to new version")
 class test_integration(unittest.TestCase):
     def test(self):
-        mngr = manager.manager(config.MONGO_URI, 'explainshell_tests', [os.path.join(os.path.dirname(__file__), 'echo.1.gz')], drop=True)
+        mngr = manager.Manager(
+            config.MONGO_URI,
+            "explainshell_tests",
+            [os.path.join(os.path.dirname(__file__), "echo.1.gz")],
+            drop=True,
+        )
         mngr.run()
 
-        cmd = 'echo -en foobar --version'
+        cmd = "echo -en foobar --version"
 
-        m = matcher.matcher(cmd, mngr.store)
+        m = matcher.Matcher(cmd, mngr.store)
         group = m.match()[1]
         matchprog, matches = group.manpage.name, group.results
 
-        self.assertEquals(matchprog, 'echo')
+        self.assertEqual(matchprog, "echo")
 
-        #self.assertEquals(matches[0].text, 'display a line of text')
-        self.assertEquals(matches[0].match, 'echo')
+        # self.assertEqual(matches[0].text, 'display a line of text')
+        self.assertEqual(matches[0].match, "echo")
 
-        self.assertEquals(matches[1].text, '<b>-e</b>     enable interpretation of backslash escapes')
-        self.assertEquals(matches[1].match, '-e')
+        self.assertEqual(
+            matches[1].text, "<b>-e</b>     enable interpretation of backslash escapes"
+        )
+        self.assertEqual(matches[1].match, "-e")
 
-        self.assertEquals(matches[2].text, '<b>-n</b>     do not output the trailing newline')
-        self.assertEquals(matches[2].match, 'n')
+        self.assertEqual(
+            matches[2].text, "<b>-n</b>     do not output the trailing newline"
+        )
+        self.assertEqual(matches[2].match, "n")
 
-        self.assertEquals(matches[3].text, None)
-        self.assertEquals(matches[3].match, 'foobar')
+        self.assertEqual(matches[3].text, None)
+        self.assertEqual(matches[3].match, "foobar")
 
-        self.assertEquals(matches[4].text, '<b>--version</b>\n       output version information and exit')
-        self.assertEquals(matches[4].match, '--version')
+        self.assertEqual(
+            matches[4].text,
+            "<b>--version</b>\n       output version information and exit",
+        )
+        self.assertEqual(matches[4].match, "--version")
diff --git a/tests/test-manager.py b/tests/test-manager.py
index a6da22b8..77839c2c 100644
--- a/tests/test-manager.py
+++ b/tests/test-manager.py
@@ -2,46 +2,47 @@
 
 from explainshell import manager, config, store, errors
 
+
 @unittest.skip("nltk usage is broken due to new version")
 class test_manager(unittest.TestCase):
     def setUp(self):
-        store.store('explainshell_tests').drop(True)
+        store.Store("explainshell_tests").drop(True)
 
     def _getmanager(self, names, **kwargs):
         l = []
         for n in names:
-            l.append(os.path.join(config.MANPAGEDIR, '1', n))
+            l.append(os.path.join(config.MAN_PAGE_DIR, "1", n))
 
-        m = manager.manager(config.MONGO_URI, 'explainshell_tests', l, **kwargs)
+        m = manager.Manager(config.MONGO_URI, "explainshell_tests", l, **kwargs)
         return m
 
     def test(self):
-        m = self._getmanager(['tar.1.gz'])
+        m = self._getmanager(["tar.1.gz"])
         m.run()
 
-        self.assertRaises(errors.ProgramDoesNotExist, m.store.findmanpage, 'tar.2')
-        mp = m.store.findmanpage('tar')[0]
-        self.assertEquals(mp.source, 'tar.1.gz')
-        self.assertEquals(mp.name, 'tar')
-        self.assertEquals(mp.aliases, [('tar', 10)])
-        self.assertEquals(len(mp.paragraphs), 154)
-        self.assertEquals(len(mp.options), 134)
-        self.assertTrue(mp.find_option('-v'))
-        self.assertEquals(mp.synopsis, 'The GNU version of the tar archiving utility')
+        self.assertRaises(errors.ProgramDoesNotExist, m.store.find_man_page, "tar.2")
+        mp = m.store.find_man_page("tar")[0]
+        self.assertEqual(mp.source, "tar.1.gz")
+        self.assertEqual(mp.name, "tar")
+        self.assertEqual(mp.aliases, [("tar", 10)])
+        self.assertEqual(len(mp.paragraphs), 154)
+        self.assertEqual(len(mp.options), 134)
+        self.assertTrue(mp.find_option("-v"))
+        self.assertEqual(mp.synopsis, "The GNU version of the tar archiving utility")
 
-        self.assertTrue(mp.partialmatch) # fixer is working
+        self.assertTrue(mp.partial_match)  # fixer is working
 
-        self.assertEquals(m.run()[0], [])
+        self.assertEqual(m.run()[0], [])
 
     def test_verify(self):
-        m = self._getmanager(['tar.1.gz'])
+        m = self._getmanager(["tar.1.gz"])
         s = m.store
 
         # invalid mapping
-        s.addmapping('foo', 'bar', 1)
+        s.add_mapping("foo", "bar", 1)
         ok, unreachable, notfound = s.verify()
         self.assertFalse(ok)
-        self.assertEquals(list(notfound), ['bar'])
+        self.assertEqual(list(notfound), ["bar"])
 
         s.mapping.drop()
         m.run()
@@ -50,96 +51,108 @@ def test_verify(self):
 
         s.mapping.drop()
         ok, unreachable, notfound = s.verify()
-        self.assertEquals(list(unreachable), ['tar'])
+        self.assertEqual(list(unreachable), ["tar"])
 
-        s.addmapping('foo', 'bar', 1)
+        s.add_mapping("foo", "bar", 1)
         ok, unreachable, notfound = s.verify()
-        self.assertEquals(list(notfound), ['bar'])
-        self.assertEquals(list(unreachable), ['tar'])
+        self.assertEqual(list(notfound), ["bar"])
+        self.assertEqual(list(unreachable), ["tar"])
 
-    @unittest.skip("https://github.com/idank/explainshell/pull/303#issuecomment-1272387073")
+    @unittest.skip(
+        "https://github.com/idank/explainshell/pull/303#issuecomment-1272387073"
+    )
     def test_aliases(self):
-        m = self._getmanager(['lsbcpp.1.gz', 'tar.1.gz', 'bsdtar.1.gz', 'basket.1.gz'])
+        m = self._getmanager(["lsbcpp.1.gz", "tar.1.gz", "bsdtar.1.gz", "basket.1.gz"])
         m.run()
 
-        mp = m.store.findmanpage('lsbcpp')
-        self.assertTrue('lsbcc' in m.store)
-        self.assertTrue('lsbc++' in m.store)
-        self.assertTrue('lsbcpp' in m.store)
-        self.assertEquals(len(mp), 1)
+        mp = m.store.find_man_page("lsbcpp")
+        self.assertTrue("lsbcc" in m.store)
+        self.assertTrue("lsbc++" in m.store)
+        self.assertTrue("lsbcpp" in m.store)
+        self.assertEqual(len(mp), 1)
 
-        mp = m.store.findmanpage('tar')
-        self.assertEquals(len(mp), 2)
-        self.assertEquals(mp[0].source, 'tar.1.gz')
-        self.assertEquals(mp[1].source, 'bsdtar.1.gz')
+        mp = m.store.find_man_page("tar")
+        self.assertEqual(len(mp), 2)
+        self.assertEqual(mp[0].source, "tar.1.gz")
+        self.assertEqual(mp[1].source, "bsdtar.1.gz")
 
     def test_overwrite(self):
-        m = self._getmanager(['tar.1.gz'], overwrite=False)
-        self.assertEquals(len(list(m.store)), 0)
+        m = self._getmanager(["tar.1.gz"], overwrite=False)
+        self.assertEqual(len(list(m.store)), 0)
 
         a, e = m.run()
         self.assertTrue(a)
         self.assertFalse(e)
-        self.assertEquals(m.store.mapping.count(), 1)
-        self.assertEquals(len(list(m.store)), 1)
+        self.assertEqual(m.store.mapping.count(), 1)
+        self.assertEqual(len(list(m.store)), 1)
 
         a, e = m.run()
         self.assertFalse(a)
         self.assertTrue(e)
-        self.assertEquals(m.store.mapping.count(), 1)
-        self.assertEquals(len(list(m.store)), 1)
-
-        m = manager.manager(config.MONGO_URI, 'explainshell_tests', [os.path.join(config.MANPAGEDIR, '1', 'tar.1.gz')], overwrite=True)
+        self.assertEqual(m.store.mapping.count(), 1)
+        self.assertEqual(len(list(m.store)), 1)
+
+        m = manager.Manager(
+            config.MONGO_URI,
+            "explainshell_tests",
+            [os.path.join(config.MAN_PAGE_DIR, "1", "tar.1.gz")],
+            overwrite=True,
+        )
         a, e = m.run()
         self.assertTrue(a)
         self.assertFalse(e)
-        self.assertEquals(m.store.mapping.count(), 1)
-        self.assertEquals(len(list(m.store)), 1)
+        self.assertEqual(m.store.mapping.count(), 1)
+        self.assertEqual(len(list(m.store)), 1)
 
         m.store.verify()
 
-    def test_multicommand(self):
-        m = self._getmanager(['git.1.gz', 'git-rebase.1.gz'])
+    def test_multi_cmd(self):
+        m = self._getmanager(["git.1.gz", "git-rebase.1.gz"])
         m.run()
 
-        self.assertTrue(m.store.findmanpage('git')[0].multicommand)
-        self.assertTrue('git rebase' in m.store)
+        self.assertTrue(m.store.find_man_page("git")[0].multi_cmd)
+        self.assertTrue("git rebase" in m.store)
 
     def test_edit(self):
-        m = self._getmanager(['tar.1.gz'], overwrite=False)
-        self.assertEquals(len(list(m.store)), 0)
+        m = self._getmanager(["tar.1.gz"], overwrite=False)
+        self.assertEqual(len(list(m.store)), 0)
 
         a, e = m.run()
         mp = a[0]
-        mp.synopsis = 'foo'
+        mp.synopsis = "foo"
         m.edit(mp)
 
-        mp = m.store.findmanpage('tar')[0]
-        self.assertEquals(mp.synopsis, 'foo')
+        mp = m.store.find_man_page("tar")[0]
+        self.assertEqual(mp.synopsis, "foo")
         self.assertTrue(m.store.verify())
 
-        mp.aliases.append(('foo', 1))
+        mp.aliases.append(("foo", 1))
         m.edit(mp)
-        self.assertTrue('foo' in m.store)
-        self.assertEquals(m.store.findmanpage('tar')[0].paragraphs,
-                          m.store.findmanpage('foo')[0].paragraphs)
+        self.assertTrue("foo" in m.store)
+        self.assertEqual(
+            m.store.find_man_page("tar")[0].paragraphs,
+            m.store.find_man_page("foo")[0].paragraphs,
+        )
         self.assertTrue(m.store.verify()[0])
 
     def test_samename(self):
-        pages = [os.path.join(config.MANPAGEDIR, '1', 'node.1.gz'), os.path.join(config.MANPAGEDIR, '8', 'node.8.gz')]
-        m = manager.manager(config.MONGO_URI, 'explainshell_tests', pages)
+        pages = [
+            os.path.join(config.MAN_PAGE_DIR, "1", "node.1.gz"),
+            os.path.join(config.MAN_PAGE_DIR, "8", "node.8.gz"),
+        ]
+        m = manager.Manager(config.MONGO_URI, "explainshell_tests", pages)
         a, e = m.run()
-        self.assertEquals(len(a), 2)
-        self.assertEquals(len(m.store.findmanpage('node')), 2)
-        mps = m.store.findmanpage('node.8')
-        self.assertEquals(len(mps), 2)
-        self.assertEquals(mps[0].section, '8')
+        self.assertEqual(len(a), 2)
+        self.assertEqual(len(m.store.find_man_page("node")), 2)
+        mps = m.store.find_man_page("node.8")
+        self.assertEqual(len(mps), 2)
+        self.assertEqual(mps[0].section, "8")
 
     def test_samename_samesection(self):
-        m = self._getmanager(['xargs.1.gz', 'xargs.1posix.gz'])
+        m = self._getmanager(["xargs.1.gz", "xargs.1posix.gz"])
         a, e = m.run()
-        self.assertEquals(len(a), 2)
-        self.assertEquals(len(m.store.findmanpage('xargs')), 2)
-        mps = m.store.findmanpage('xargs.1posix')
-        self.assertEquals(len(mps), 2)
-        self.assertEquals(mps[0].section, '1posix')
+        self.assertEqual(len(a), 2)
+        self.assertEqual(len(m.store.find_man_page("xargs")), 2)
+        mps = m.store.find_man_page("xargs.1posix")
+        self.assertEqual(len(mps), 2)
+        self.assertEqual(mps[0].section, "1posix")
diff --git a/tests/test-manpage.py b/tests/test-manpage.py
index 60a19d3e..4d3f2563 100644
--- a/tests/test-manpage.py
+++ b/tests/test-manpage.py
@@ -2,14 +2,15 @@
 
 from explainshell import manpage, store
 
+
 class test_manpage(unittest.TestCase):
     def test_first_paragraph_no_section(self):
-        m = 'foo\nbar'
-        l = list(manpage._parsetext(m.splitlines()))
-        self.assertEquals(l, [store.paragraph(0, 'foo\nbar', None, False)])
+        m = "foo\nbar"
+        l = list(manpage._parse_text(m.splitlines()))
+        self.assertEqual(l, [store.Paragraph(0, "foo\nbar", None, False)])
 
     def test_sections(self):
-        m = '''<b>SECTION</b>
+        m = """<b>SECTION</b>
 a
 b
 
@@ -25,17 +26,23 @@ def test_sections(self):
 
 <b>SECTION3</b>
 
-tNOTASECTION'''
+tNOTASECTION"""
 
-        parsed = list(manpage._parsetext(m.splitlines()))
+        parsed = list(manpage._parse_text(m.splitlines()))
         self.assertTrue(len(parsed) == 5)
-        self.assertEquals(parsed, [store.paragraph(0, 'a\nb', 'SECTION', False),
-                                   store.paragraph(1, 'c', 'SECTION', False),
-                                   store.paragraph(2, 'a', 'SECTION2', False),
-                                   store.paragraph(3, 'a', 'WITH SPACES', False),
-                                   store.paragraph(4, 'tNOTASECTION', 'SECTION3', False)])
+        self.assertEqual(
+            parsed,
+            [
+                store.Paragraph(0, "a\nb", "SECTION", False),
+                store.Paragraph(1, "c", "SECTION", False),
+                store.Paragraph(2, "a", "SECTION2", False),
+                store.Paragraph(3, "a", "WITH SPACES", False),
+                store.Paragraph(4, "tNOTASECTION", "SECTION3", False),
+            ],
+        )
+
     def test_no_synopsis(self):
-        m = manpage.manpage('foo')
-        m._text = 'a b c d e f g h i j k l'.replace(' ', '\n')
+        m = manpage.ManPage("foo")
+        m._text = "a b c d e f g h i j k l".replace(" ", "\n")
         m.parse()
-        self.assertEquals(m.aliases, [('foo', 10)])
+        self.assertEqual(m.aliases, [("foo", 10)])
diff --git a/tests/test-matcher.py b/tests/test-matcher.py
index e9c1548f..c7415d02 100644
--- a/tests/test-matcher.py
+++ b/tests/test-matcher.py
@@ -2,651 +2,788 @@
 
 import bashlex.errors, bashlex.ast
 
-from explainshell import matcher, errors, helpconstants
+from explainshell import help_constants, matcher, errors
 from tests import helpers
 
-s = helpers.mockstore()
+s = helpers.MockStore()
+
 
 class test_matcher(unittest.TestCase):
     def assertMatchSingle(self, what, expectedmanpage, expectedresults):
-        m = matcher.matcher(what, s)
+        m = matcher.Matcher(what, s)
         groups = m.match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[1].manpage, expectedmanpage)
-        self.assertEquals(groups[1].results, expectedresults)
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[1].manpage, expectedmanpage)
+        self.assertEqual(groups[1].results, expectedresults)
 
     def test_unknown_prog(self):
-        self.assertRaises(errors.ProgramDoesNotExist, matcher.matcher('foo', s).match)
+        self.assertRaises(errors.ProgramDoesNotExist, matcher.Matcher("foo", s).match)
 
     def test_unicode(self):
         matchedresult = [
-            (0, 3, 'bar synopsis', 'bar'),
-            (4, 13, '-b <arg> desc', '-b uni???')]
+            (0, 3, "bar synopsis", "bar"),
+            (4, 13, "-b <arg> desc", "-b uni???"),
+        ]
 
-        self.assertMatchSingle(u'bar -b uni\u05e7\u05d5\u05d3', s.findmanpage('bar')[0], matchedresult)
+        self.assertMatchSingle(
+            "bar -b uni\u05e7\u05d5\u05d3", s.find_man_page("bar")[0], matchedresult
+        )
 
     def test_no_options(self):
-        matchedresult = [(0, 3, 'bar synopsis', 'bar')]
-        self.assertMatchSingle('bar', s.findmanpage('bar')[0], matchedresult)
+        matchedresult = [(0, 3, "bar synopsis", "bar")]
+        self.assertMatchSingle("bar", s.find_man_page("bar")[0], matchedresult)
 
     def test_known_arg(self):
         matchedresult = [
-            (0, 3, 'bar synopsis', 'bar'),
-            (4, 10, '-a desc', '-a --a'),
-            (11, 13, '-? help text', '-?')]
+            (0, 3, "bar synopsis", "bar"),
+            (4, 10, "-a desc", "-a --a"),
+            (11, 13, "-? help text", "-?"),
+        ]
 
-        self.assertMatchSingle('bar -a --a -?', s.findmanpage('bar')[0], matchedresult)
+        self.assertMatchSingle("bar -a --a -?", s.find_man_page("bar")[0], matchedresult)
 
     def test_arg_in_fuzzy_with_expected_value(self):
-        cmd = 'baz -ab arg'
+        cmd = "baz -ab arg"
         matchedresult = [
-            (0, 3, 'baz synopsis', 'baz'),
-            (4, 6, '-a desc', '-a'),
-            (6, 11, '-b <arg> desc', 'b arg')]
+            (0, 3, "baz synopsis", "baz"),
+            (4, 6, "-a desc", "-a"),
+            (6, 11, "-b <arg> desc", "b arg"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('baz')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("baz")[0], matchedresult)
 
-        cmd = 'baz -ab12'
+        cmd = "baz -ab12"
         matchedresult = [
-            (0, 3, 'baz synopsis', 'baz'),
-            (4, 6, '-a desc', '-a'),
-            (6, 9, '-b <arg> desc', 'b12')]
+            (0, 3, "baz synopsis", "baz"),
+            (4, 6, "-a desc", "-a"),
+            (6, 9, "-b <arg> desc", "b12"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('baz')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("baz")[0], matchedresult)
 
-    def test_partialmatch_with_arguments(self):
-        cmd = 'withargs arg'
+    def test_partial_match_with_arguments(self):
+        cmd = "withargs arg"
         matchedresult = [
-            (0, 8, 'withargs synopsis', 'withargs'),
-            (9, 12, 'FILE argument', 'arg')]
+            (0, 8, "withargs synopsis", "withargs"),
+            (9, 12, "FILE argument", "arg"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('withargs')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("withargs")[0], matchedresult)
 
     def test_reset_current_option_if_argument_taken(self):
-        cmd = 'withargs -ab12 arg'
+        cmd = "withargs -ab12 arg"
         matchedresult = [
-            (0, 8, 'withargs synopsis', 'withargs'),
-            (9, 11, '-a desc', '-a'),
-            (11, 14, '-b <arg> desc', 'b12'),
-            (15, 18, 'FILE argument', 'arg')]
+            (0, 8, "withargs synopsis", "withargs"),
+            (9, 11, "-a desc", "-a"),
+            (11, 14, "-b <arg> desc", "b12"),
+            (15, 18, "FILE argument", "arg"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('withargs')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("withargs")[0], matchedresult)
 
-        cmd = 'withargs -b12 arg'
+        cmd = "withargs -b12 arg"
         matchedresult = [
-            (0, 8, 'withargs synopsis', 'withargs'),
-            (9, 13, '-b <arg> desc', '-b12'),
-            (14, 17, 'FILE argument', 'arg')]
+            (0, 8, "withargs synopsis", "withargs"),
+            (9, 13, "-b <arg> desc", "-b12"),
+            (14, 17, "FILE argument", "arg"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('withargs')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("withargs")[0], matchedresult)
 
         # here we reset it implicitly by looking up '12'
-        cmd = 'withargs -b 12 arg'
+        cmd = "withargs -b 12 arg"
         matchedresult = [
-            (0, 8, 'withargs synopsis', 'withargs'),
-            (9, 14, '-b <arg> desc', '-b 12'),
-            (15, 18, 'FILE argument', 'arg')]
+            (0, 8, "withargs synopsis", "withargs"),
+            (9, 14, "-b <arg> desc", "-b 12"),
+            (15, 18, "FILE argument", "arg"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('withargs')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("withargs")[0], matchedresult)
 
     def test_arg_with_expected_value(self):
-        cmd = 'bar -b arg --b arg'
+        cmd = "bar -b arg --b arg"
         matchedresult = [
-            (0, 3, 'bar synopsis', 'bar'),
-            (4, 18, '-b <arg> desc', '-b arg --b arg')]
+            (0, 3, "bar synopsis", "bar"),
+            (4, 18, "-b <arg> desc", "-b arg --b arg"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('bar')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("bar")[0], matchedresult)
 
     def test_arg_with_expected_value_from_list(self):
-        cmd = 'bar -c one'
+        cmd = "bar -c one"
         matchedresult = [
-            (0, 3, 'bar synopsis', 'bar'),
-            (4, 10, '-c=one,two\ndesc', '-c one')]
+            (0, 3, "bar synopsis", "bar"),
+            (4, 10, "-c=one,two\ndesc", "-c one"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('bar')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("bar")[0], matchedresult)
 
-        cmd = 'bar -c notinlist'
+        cmd = "bar -c notinlist"
         matchedresult = [
-            (0, 3, 'bar synopsis', 'bar'),
-            (4, 6, '-c=one,two\ndesc', '-c'),
-            (7, 16, None, 'notinlist')]
+            (0, 3, "bar synopsis", "bar"),
+            (4, 6, "-c=one,two\ndesc", "-c"),
+            (7, 16, None, "notinlist"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('bar')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("bar")[0], matchedresult)
 
     def test_arg_with_expected_value_clash(self):
-        '''the first option expects an arg but the arg is actually an option'''
-        cmd = 'bar -b -a'
+        """the first option expects an arg but the arg is actually an option"""
+        cmd = "bar -b -a"
         matchedresult = [
-            (0, 3, 'bar synopsis', 'bar'),
-            (4, 6, '-b <arg> desc', '-b'),
-            (7, 9, '-a desc', '-a')]
+            (0, 3, "bar synopsis", "bar"),
+            (4, 6, "-b <arg> desc", "-b"),
+            (7, 9, "-a desc", "-a"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('bar')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("bar")[0], matchedresult)
 
     def test_arg_with_expected_value_no_clash(self):
-        '''the first option expects an arg but the arg is not an option even though
-        it looks like one'''
-        cmd = 'bar -b -xa'
+        """the first option expects an arg but the arg is not an option even though
+        it looks like one"""
+        cmd = "bar -b -xa"
         matchedresult = [
-            (0, 3, 'bar synopsis', 'bar'),
-            (4, 6, '-b <arg> desc', '-b'),
-            (7, 9, None, '-x'),
-            (9, 10, '-a desc', 'a')]
+            (0, 3, "bar synopsis", "bar"),
+            (4, 6, "-b <arg> desc", "-b"),
+            (7, 9, None, "-x"),
+            (9, 10, "-a desc", "a"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('bar')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("bar")[0], matchedresult)
 
     def test_unknown_arg(self):
-        matchedresult = [(0, 3, 'bar synopsis', 'bar'), (4, 6, None, '-x')]
-        self.assertMatchSingle('bar -x', s.findmanpage('bar')[0], matchedresult)
+        matchedresult = [(0, 3, "bar synopsis", "bar"), (4, 6, None, "-x")]
+        self.assertMatchSingle("bar -x", s.find_man_page("bar")[0], matchedresult)
 
         # merges
-        matchedresult = [(0, 3, 'bar synopsis', 'bar'), (4, 10, None, '-x --x')]
-        self.assertMatchSingle('bar -x --x', s.findmanpage('bar')[0], matchedresult)
+        matchedresult = [(0, 3, "bar synopsis", "bar"), (4, 10, None, "-x --x")]
+        self.assertMatchSingle("bar -x --x", s.find_man_page("bar")[0], matchedresult)
 
-        matchedresult = [(0, 3, 'bar synopsis', 'bar'), (4, 8, None, '-xyz')]
-        self.assertMatchSingle('bar -xyz', s.findmanpage('bar')[0], matchedresult)
+        matchedresult = [(0, 3, "bar synopsis", "bar"), (4, 8, None, "-xyz")]
+        self.assertMatchSingle("bar -xyz", s.find_man_page("bar")[0], matchedresult)
 
-        matchedresult = [(0, 3, 'bar synopsis', 'bar'),
-                                 (4, 6, None, '-x'),
-                                 (6, 7, '-a desc', 'a'), (7, 8, None, 'z')]
+        matchedresult = [
+            (0, 3, "bar synopsis", "bar"),
+            (4, 6, None, "-x"),
+            (6, 7, "-a desc", "a"),
+            (7, 8, None, "z"),
+        ]
 
-        self.assertMatchSingle('bar -xaz', s.findmanpage('bar')[0], matchedresult)
+        self.assertMatchSingle("bar -xaz", s.find_man_page("bar")[0], matchedresult)
 
     def test_merge_same_match(self):
-        matchedresult = [(0, 3, 'bar synopsis', 'bar'), (4, 8, '-a desc', '-aaa')]
-        self.assertMatchSingle('bar -aaa', s.findmanpage('bar')[0], matchedresult)
+        matchedresult = [(0, 3, "bar synopsis", "bar"), (4, 8, "-a desc", "-aaa")]
+        self.assertMatchSingle("bar -aaa", s.find_man_page("bar")[0], matchedresult)
 
     def test_known_and_unknown_arg(self):
-        matchedresult = [(0, 3, 'bar synopsis', 'bar'), (4, 6, '-a desc', '-a'), (7, 9, None, '-x')]
-        self.assertMatchSingle('bar -a -x', s.findmanpage('bar')[0], matchedresult)
+        matchedresult = [
+            (0, 3, "bar synopsis", "bar"),
+            (4, 6, "-a desc", "-a"),
+            (7, 9, None, "-x"),
+        ]
+        self.assertMatchSingle("bar -a -x", s.find_man_page("bar")[0], matchedresult)
 
-        matchedresult = [(0, 3, 'bar synopsis', 'bar'), (4, 6, '-a desc', '-a'), (6, 7, None, 'x')]
-        self.assertMatchSingle('bar -ax', s.findmanpage('bar')[0], matchedresult)
+        matchedresult = [
+            (0, 3, "bar synopsis", "bar"),
+            (4, 6, "-a desc", "-a"),
+            (6, 7, None, "x"),
+        ]
+        self.assertMatchSingle("bar -ax", s.find_man_page("bar")[0], matchedresult)
 
     def test_long(self):
-        cmd = 'bar --b=b foo'
+        cmd = "bar --b=b foo"
         matchedresult = [
-            (0, 3, 'bar synopsis', 'bar'),
-            (4, 9, '-b <arg> desc', '--b=b'),
-            (10, 13, None, 'foo')]
+            (0, 3, "bar synopsis", "bar"),
+            (4, 9, "-b <arg> desc", "--b=b"),
+            (10, 13, None, "foo"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('bar')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("bar")[0], matchedresult)
 
     def test_arg_no_dash(self):
-        cmd = 'baz ab -x'
+        cmd = "baz ab -x"
         matchedresult = [
-            (0, 3, 'baz synopsis', 'baz'),
-            (4, 5, '-a desc', 'a'),
-            (5, 6, '-b <arg> desc', 'b'),
-            (7, 9, None, '-x')]
+            (0, 3, "baz synopsis", "baz"),
+            (4, 5, "-a desc", "a"),
+            (5, 6, "-b <arg> desc", "b"),
+            (7, 9, None, "-x"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('baz')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("baz")[0], matchedresult)
 
-    def test_multicommand(self):
-        cmd = 'bar baz --b foo'
+    def test_multi_cmd(self):
+        cmd = "bar baz --b foo"
         matchedresult = [
-            (0, 3, 'bar synopsis', 'bar'),
-            (4, 7, None, 'baz'),
-            (8, 15, '-b <arg> desc', '--b foo')]
+            (0, 3, "bar synopsis", "bar"),
+            (4, 7, None, "baz"),
+            (8, 15, "-b <arg> desc", "--b foo"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('bar')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("bar")[0], matchedresult)
 
-        cmd = 'bar foo --b foo'
+        cmd = "bar foo --b foo"
         matchedresult = [
-            (0, 7, 'bar foo synopsis', 'bar foo'),
-            (8, 15, '-b <arg> desc', '--b foo')]
+            (0, 7, "bar foo synopsis", "bar foo"),
+            (8, 15, "-b <arg> desc", "--b foo"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('bar foo')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("bar foo")[0], matchedresult)
 
     def test_multiple_matches(self):
-        cmd = 'dup -ab'
+        cmd = "dup -ab"
         matchedresult = [
-            (0, 3, 'dup1 synopsis', 'dup'),
-            (4, 6, '-a desc', '-a'),
-            (6, 7, '-b <arg> desc', 'b')]
+            (0, 3, "dup1 synopsis", "dup"),
+            (4, 6, "-a desc", "-a"),
+            (6, 7, "-b <arg> desc", "b"),
+        ]
 
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(groups[1].results, matchedresult)
-        self.assertEquals(groups[1].suggestions[0].source, 'dup.2.gz')
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(groups[1].results, matchedresult)
+        self.assertEqual(groups[1].suggestions[0].source, "dup.2.gz")
 
     def test_arguments(self):
-        cmd = 'withargs -x -b freearg freearg'
+        cmd = "withargs -x -b freearg freearg"
         matchedresult = [
-            (0, 8, 'withargs synopsis', 'withargs'),
+            (0, 8, "withargs synopsis", "withargs"),
             # tokens that look like options are still unknown
-            (9, 11, None, '-x'),
-            (12, 22, '-b <arg> desc', '-b freearg'),
-            (23, 30, 'FILE argument', 'freearg')]
+            (9, 11, None, "-x"),
+            (12, 22, "-b <arg> desc", "-b freearg"),
+            (23, 30, "FILE argument", "freearg"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('withargs')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("withargs")[0], matchedresult)
 
     def test_arg_is_dash(self):
-        cmd = 'bar -b - -a -'
+        cmd = "bar -b - -a -"
         matchedresult = [
-            (0, 3, 'bar synopsis', 'bar'),
-            (4, 8, '-b <arg> desc', '-b -'),
-            (9, 11, '-a desc', '-a'),
-            (12, 13, None, '-')]
+            (0, 3, "bar synopsis", "bar"),
+            (4, 8, "-b <arg> desc", "-b -"),
+            (9, 11, "-a desc", "-a"),
+            (12, 13, None, "-"),
+        ]
 
-        self.assertMatchSingle(cmd, s.findmanpage('bar')[0], matchedresult)
+        self.assertMatchSingle(cmd, s.find_man_page("bar")[0], matchedresult)
 
     def test_nested_command(self):
-        cmd = 'withargs -b arg bar -a unknown'
-
-        matchedresult = [[(0, 8, 'withargs synopsis', 'withargs'),
-                          (9, 15, '-b <arg> desc', '-b arg')],
-                         [(16, 19, 'bar synopsis', 'bar'),
-                          (20, 22, '-a desc', '-a'),
-                          (23, 30, None, 'unknown')]]
+        cmd = "withargs -b arg bar -a unknown"
 
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 3)
-        self.assertEquals(groups[0].results, [])
-        self.assertEquals(groups[1].results, matchedresult[0])
-        self.assertEquals(groups[2].results, matchedresult[1])
+        matchedresult = [
+            [
+                (0, 8, "withargs synopsis", "withargs"),
+                (9, 15, "-b <arg> desc", "-b arg"),
+            ],
+            [
+                (16, 19, "bar synopsis", "bar"),
+                (20, 22, "-a desc", "-a"),
+                (23, 30, None, "unknown"),
+            ],
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 3)
+        self.assertEqual(groups[0].results, [])
+        self.assertEqual(groups[1].results, matchedresult[0])
+        self.assertEqual(groups[2].results, matchedresult[1])
 
     def test_nested_option(self):
-        cmd = 'withargs -b arg -exec bar -a EOF -b arg'
-
-        matchedresult = [[(0, 8, 'withargs synopsis', 'withargs'),
-                          (9, 15, '-b <arg> desc', '-b arg'),
-                          (16, 21, '-exec nest', '-exec'),
-                          (29, 32, '-exec nest', 'EOF'),
-                          (33, 39, '-b <arg> desc', '-b arg')],
-                         [(22, 25, 'bar synopsis', 'bar'),
-                          (26, 28, '-a desc', '-a')]]
-
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 3)
-        self.assertEquals(groups[0].results, [])
-        self.assertEquals(groups[1].results, matchedresult[0])
-        self.assertEquals(groups[2].results, matchedresult[1])
+        cmd = "withargs -b arg -exec bar -a EOF -b arg"
 
-        cmd = "withargs -b arg -exec bar -a ';' -a"
+        matchedresult = [
+            [
+                (0, 8, "withargs synopsis", "withargs"),
+                (9, 15, "-b <arg> desc", "-b arg"),
+                (16, 21, "-exec nest", "-exec"),
+                (29, 32, "-exec nest", "EOF"),
+                (33, 39, "-b <arg> desc", "-b arg"),
+            ],
+            [(22, 25, "bar synopsis", "bar"), (26, 28, "-a desc", "-a")],
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 3)
+        self.assertEqual(groups[0].results, [])
+        self.assertEqual(groups[1].results, matchedresult[0])
+        self.assertEqual(groups[2].results, matchedresult[1])
 
-        matchedresult = [[(0, 8, 'withargs synopsis', 'withargs'),
-                          (9, 15, '-b <arg> desc', '-b arg'),
-                          (16, 21, '-exec nest', '-exec'),
-                          (29, 32, '-exec nest', "';'"),
-                          (33, 35, '-a desc', '-a')],
-                         [(22, 25, 'bar synopsis', 'bar'),
-                          (26, 28, '-a desc', '-a')]]
+        cmd = "withargs -b arg -exec bar -a ';' -a"
 
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 3)
-        self.assertEquals(groups[0].results, [])
-        self.assertEquals(groups[1].results, matchedresult[0])
-        self.assertEquals(groups[2].results, matchedresult[1])
+        matchedresult = [
+            [
+                (0, 8, "withargs synopsis", "withargs"),
+                (9, 15, "-b <arg> desc", "-b arg"),
+                (16, 21, "-exec nest", "-exec"),
+                (29, 32, "-exec nest", "';'"),
+                (33, 35, "-a desc", "-a"),
+            ],
+            [(22, 25, "bar synopsis", "bar"), (26, 28, "-a desc", "-a")],
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 3)
+        self.assertEqual(groups[0].results, [])
+        self.assertEqual(groups[1].results, matchedresult[0])
+        self.assertEqual(groups[2].results, matchedresult[1])
 
         cmd = "withargs -b arg -exec bar -a \\; -a"
 
-        matchedresult = [[(0, 8, 'withargs synopsis', 'withargs'),
-                          (9, 15, '-b <arg> desc', '-b arg'),
-                          (16, 21, '-exec nest', '-exec'),
-                          (29, 31, '-exec nest', "\\;"),
-                          (32, 34, '-a desc', '-a')],
-                         [(22, 25, 'bar synopsis', 'bar'),
-                          (26, 28, '-a desc', '-a')]]
-
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 3)
-        self.assertEquals(groups[0].results, [])
-        self.assertEquals(groups[1].results, matchedresult[0])
-        self.assertEquals(groups[2].results, matchedresult[1])
-
-        cmd = 'withargs -exec bar -a -u'
-
-        matchedresult = [[(0, 8, 'withargs synopsis', 'withargs'),
-                          (9, 14, '-exec nest', '-exec')],
-                         [(15, 18, 'bar synopsis', 'bar'),
-                          (19, 21, '-a desc', '-a'),
-                          (22, 24, None, '-u')]]
-
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 3)
-        self.assertEquals(groups[0].results, [])
-        self.assertEquals(groups[1].results, matchedresult[0])
-        self.assertEquals(groups[2].results, matchedresult[1])
+        matchedresult = [
+            [
+                (0, 8, "withargs synopsis", "withargs"),
+                (9, 15, "-b <arg> desc", "-b arg"),
+                (16, 21, "-exec nest", "-exec"),
+                (29, 31, "-exec nest", "\\;"),
+                (32, 34, "-a desc", "-a"),
+            ],
+            [(22, 25, "bar synopsis", "bar"), (26, 28, "-a desc", "-a")],
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 3)
+        self.assertEqual(groups[0].results, [])
+        self.assertEqual(groups[1].results, matchedresult[0])
+        self.assertEqual(groups[2].results, matchedresult[1])
+
+        cmd = "withargs -exec bar -a -u"
 
-    def test_multiple_nests(self):
-        cmd = 'withargs withargs -b arg bar'
+        matchedresult = [
+            [(0, 8, "withargs synopsis", "withargs"), (9, 14, "-exec nest", "-exec")],
+            [
+                (15, 18, "bar synopsis", "bar"),
+                (19, 21, "-a desc", "-a"),
+                (22, 24, None, "-u"),
+            ],
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 3)
+        self.assertEqual(groups[0].results, [])
+        self.assertEqual(groups[1].results, matchedresult[0])
+        self.assertEqual(groups[2].results, matchedresult[1])
 
-        matchedresult = [[(0, 8, 'withargs synopsis', 'withargs')],
-                         [(9, 17, 'withargs synopsis', 'withargs'),
-                          (18, 24, '-b <arg> desc', '-b arg')],
-                         [(25, 28, 'bar synopsis', 'bar')]]
+    def test_multiple_nests(self):
+        cmd = "withargs withargs -b arg bar"
 
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 4)
-        self.assertEquals(groups[0].results, [])
-        self.assertEquals(groups[1].results, matchedresult[0])
-        self.assertEquals(groups[2].results, matchedresult[1])
-        self.assertEquals(groups[3].results, matchedresult[2])
+        matchedresult = [
+            [(0, 8, "withargs synopsis", "withargs")],
+            [
+                (9, 17, "withargs synopsis", "withargs"),
+                (18, 24, "-b <arg> desc", "-b arg"),
+            ],
+            [(25, 28, "bar synopsis", "bar")],
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 4)
+        self.assertEqual(groups[0].results, [])
+        self.assertEqual(groups[1].results, matchedresult[0])
+        self.assertEqual(groups[2].results, matchedresult[1])
+        self.assertEqual(groups[3].results, matchedresult[2])
 
     def test_nested_command_is_unknown(self):
-        cmd = 'withargs -b arg unknown'
+        cmd = "withargs -b arg unknown"
 
-        matchedresult = [(0, 8, 'withargs synopsis', 'withargs'),
-                          (9, 15, '-b <arg> desc', '-b arg'),
-                          (16, 23, 'FILE argument', 'unknown')]
+        matchedresult = [
+            (0, 8, "withargs synopsis", "withargs"),
+            (9, 15, "-b <arg> desc", "-b arg"),
+            (16, 23, "FILE argument", "unknown"),
+        ]
 
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, [])
-        self.assertEquals(groups[1].results, matchedresult)
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, [])
+        self.assertEqual(groups[1].results, matchedresult)
 
     def test_unparsed(self):
-        cmd = '(bar; bar) c'
-        self.assertRaises(bashlex.errors.ParsingError,
-                          matcher.matcher(cmd, s).match)
-
+        cmd = "(bar; bar) c"
+        self.assertRaises(bashlex.errors.ParsingError, matcher.Matcher(cmd, s).match)
 
     def test_known_and_unknown_program(self):
-        cmd = 'bar; foo arg >f; baz'
-        matchedresult = [[(3, 4, helpconstants.OPERATORS[';'], ';'),
-                          (13, 15, helpconstants.REDIRECTION + '\n\n' +
-                                   helpconstants.REDIRECTION_KIND['>'], '>f'),
-                          (15, 16, helpconstants.OPERATORS[';'], ';')],
-                         [(0, 3, 'bar synopsis', 'bar')],
-                         [(5, 12, None, 'foo arg')],
-                         [(17, 20, 'baz synopsis', 'baz')]]
-
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(groups[0].results, matchedresult[0])
-        self.assertEquals(groups[1].results, matchedresult[1])
-        self.assertEquals(groups[2].results, matchedresult[2])
+        cmd = "bar; foo arg >f; baz"
+        matchedresult = [
+            [
+                (3, 4, help_constants.OPERATORS[";"], ";"),
+                (
+                    13,
+                    15,
+                    help_constants.REDIRECTION
+                    + "\n\n"
+                    + help_constants.REDIRECTION_KIND[">"],
+                    ">f",
+                ),
+                (15, 16, help_constants.OPERATORS[";"], ";"),
+            ],
+            [(0, 3, "bar synopsis", "bar")],
+            [(5, 12, None, "foo arg")],
+            [(17, 20, "baz synopsis", "baz")],
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(groups[0].results, matchedresult[0])
+        self.assertEqual(groups[1].results, matchedresult[1])
+        self.assertEqual(groups[2].results, matchedresult[2])
 
     def test_pipe(self):
-        cmd = 'bar | baz'
-        matchedresult = [[(4, 5, helpconstants.PIPELINES, '|')],
-                         [(0, 3, 'bar synopsis', 'bar')],
-                         [(6, 9, 'baz synopsis', 'baz')]]
+        cmd = "bar | baz"
+        matchedresult = [
+            [(4, 5, help_constants.PIPELINES, "|")],
+            [(0, 3, "bar synopsis", "bar")],
+            [(6, 9, "baz synopsis", "baz")],
+        ]
 
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(groups[0].results, matchedresult[0])
-        self.assertEquals(groups[1].results, matchedresult[1])
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(groups[0].results, matchedresult[0])
+        self.assertEqual(groups[1].results, matchedresult[1])
 
     def test_subshells(self):
-        cmd = '((bar); bar)'
-        matchedresult = [[(0, 2, helpconstants._subshell, '(('),
-                          (5, 6, helpconstants._subshell, ')'),
-                          (6, 7, helpconstants.OPERATORS[';'], ';'),
-                          (11, 12, helpconstants._subshell, ')')],
-                         [(2, 5, 'bar synopsis', 'bar')],
-                         [(8, 11, 'bar synopsis', 'bar')]]
-
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(groups[0].results, matchedresult[0])
-        self.assertEquals(groups[1].results, matchedresult[1])
-        self.assertEquals(groups[2].results, matchedresult[2])
+        cmd = "((bar); bar)"
+        matchedresult = [
+            [
+                (0, 2, help_constants._subshell, "(("),
+                (5, 6, help_constants._subshell, ")"),
+                (6, 7, help_constants.OPERATORS[";"], ";"),
+                (11, 12, help_constants._subshell, ")"),
+            ],
+            [(2, 5, "bar synopsis", "bar")],
+            [(8, 11, "bar synopsis", "bar")],
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(groups[0].results, matchedresult[0])
+        self.assertEqual(groups[1].results, matchedresult[1])
+        self.assertEqual(groups[2].results, matchedresult[2])
 
     def test_redirect_first_word_of_command(self):
-        cmd = '2>&1'
-        matchedresult = [(0, 4, helpconstants.REDIRECTION + '\n\n' +
-                                helpconstants.REDIRECTION_KIND['>'], '2>&1')]
-
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 1)
-        self.assertEquals(groups[0].results, matchedresult)
-
-        cmd = '2>&1 bar'
-        matchedresult = [[(0, 4, helpconstants.REDIRECTION + '\n\n' +
-                                 helpconstants.REDIRECTION_KIND['>'], '2>&1')],
-                         [(5, 8, 'bar synopsis', 'bar')]]
-
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, matchedresult[0])
-        self.assertEquals(groups[1].results, matchedresult[1])
+        cmd = "2>&1"
+        matchedresult = [
+            (
+                0,
+                4,
+                help_constants.REDIRECTION
+                + "\n\n"
+                + help_constants.REDIRECTION_KIND[">"],
+                "2>&1",
+            )
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 1)
+        self.assertEqual(groups[0].results, matchedresult)
+
+        cmd = "2>&1 bar"
+        matchedresult = [
+            [
+                (
+                    0,
+                    4,
+                    help_constants.REDIRECTION
+                    + "\n\n"
+                    + help_constants.REDIRECTION_KIND[">"],
+                    "2>&1",
+                )
+            ],
+            [(5, 8, "bar synopsis", "bar")],
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, matchedresult[0])
+        self.assertEqual(groups[1].results, matchedresult[1])
 
     def test_comsub(self):
-        cmd = 'bar $(a) -b "b $(c) `c`" \'$(d)\' >$(e) `f`'
-
-        matchedresult = [(0, 3, 'bar synopsis', 'bar'),
-                         (4, 8, None, '$(a)'),
-                         (9, 24, '-b <arg> desc', '-b "b $(c) `c`"'),
-                         (25, 31, None, "'$(d)'"),
-                         (38, 41, None, '`f`')]
-        shellresult = [(32, 37, helpconstants.REDIRECTION + '\n\n' +
-                                helpconstants.REDIRECTION_KIND['>'], '>$(e)')]
+        cmd = "bar $(a) -b \"b $(c) `c`\" '$(d)' >$(e) `f`"
 
-        m = matcher.matcher(cmd, s)
+        matchedresult = [
+            (0, 3, "bar synopsis", "bar"),
+            (4, 8, None, "$(a)"),
+            (9, 24, "-b <arg> desc", '-b "b $(c) `c`"'),
+            (25, 31, None, "'$(d)'"),
+            (38, 41, None, "`f`"),
+        ]
+        shellresult = [
+            (
+                32,
+                37,
+                help_constants.REDIRECTION
+                + "\n\n"
+                + help_constants.REDIRECTION_KIND[">"],
+                ">$(e)",
+            )
+        ]
+
+        m = matcher.Matcher(cmd, s)
         groups = m.match()
-        self.assertEquals(groups[0].results, shellresult)
-        self.assertEquals(groups[1].results, matchedresult)
+        self.assertEqual(groups[0].results, shellresult)
+        self.assertEqual(groups[1].results, matchedresult)
 
         # check expansions
-        self.assertEquals(m.expansions, [(6, 7, 'substitution'),
-                                         (17, 18, 'substitution'),
-                                         (21, 22, 'substitution'),
-                                         (35, 36, 'substitution'),
-                                         (39, 40, 'substitution')])
+        self.assertEqual(
+            m.expansions,
+            [
+                (6, 7, "substitution"),
+                (17, 18, "substitution"),
+                (21, 22, "substitution"),
+                (35, 36, "substitution"),
+                (39, 40, "substitution"),
+            ],
+        )
 
     def test_comsub_as_arg(self):
-        cmd = 'withargs $(a) $0'
+        cmd = "withargs $(a) $0"
 
-        matchedresult = [(0, 8, 'withargs synopsis', 'withargs'),
-                         (9, 16, 'FILE argument', '$(a) $0')]
+        matchedresult = [
+            (0, 8, "withargs synopsis", "withargs"),
+            (9, 16, "FILE argument", "$(a) $0"),
+        ]
 
-        m = matcher.matcher(cmd, s)
+        m = matcher.Matcher(cmd, s)
         groups = m.match()
-        self.assertEquals(groups[0].results, [])
-        self.assertEquals(groups[1].results, matchedresult)
+        self.assertEqual(groups[0].results, [])
+        self.assertEqual(groups[1].results, matchedresult)
 
         # check expansions
-        self.assertEquals(m.expansions, [(11, 12, 'substitution'),
-                                         (14, 16, 'parameter-digits')])
+        self.assertEqual(
+            m.expansions, [(11, 12, "substitution"), (14, 16, "parameter-digits")]
+        )
 
     def test_comsub_as_first_word(self):
-        cmd = '$(a) b'
+        cmd = "$(a) b"
 
-        m = matcher.matcher(cmd, s)
+        m = matcher.Matcher(cmd, s)
         groups = m.match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, [])
-        self.assertEquals(groups[1].results, [(0, 6, None, '$(a) b')])
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, [])
+        self.assertEqual(groups[1].results, [(0, 6, None, "$(a) b")])
 
         # check expansions
-        self.assertEquals(m.expansions, [(2, 3, 'substitution')])
+        self.assertEqual(m.expansions, [(2, 3, "substitution")])
 
     def test_procsub(self):
-        cmd = 'withargs -b <(a) >(b)'
+        cmd = "withargs -b <(a) >(b)"
 
-        matchedresult = [(0, 8, 'withargs synopsis', 'withargs'),
-                         (9, 16, '-b <arg> desc', '-b <(a)'),
-                         (17, 21, 'FILE argument', '>(b)')]
+        matchedresult = [
+            (0, 8, "withargs synopsis", "withargs"),
+            (9, 16, "-b <arg> desc", "-b <(a)"),
+            (17, 21, "FILE argument", ">(b)"),
+        ]
 
-        m = matcher.matcher(cmd, s)
+        m = matcher.Matcher(cmd, s)
         groups = m.match()
-        self.assertEquals(groups[0].results, [])
-        self.assertEquals(groups[1].results, matchedresult)
+        self.assertEqual(groups[0].results, [])
+        self.assertEqual(groups[1].results, matchedresult)
 
         # check expansions
-        self.assertEquals(m.expansions, [(14, 15, 'substitution'),
-                                         (19, 20, 'substitution')])
+        self.assertEqual(
+            m.expansions, [(14, 15, "substitution"), (19, 20, "substitution")]
+        )
 
     def test_if(self):
-        cmd = 'if bar -a; then b; fi'
-        shellresults = [(0, 2, helpconstants._if, 'if'),
-                        (9, 15, helpconstants._if, '; then'),
-                        (17, 21, helpconstants._if, '; fi')]
-
-        matchresults = [[(3, 6, 'bar synopsis', 'bar'), (7, 9, '-a desc', '-a')],
-                        [(16, 17, None, 'b')]]
-
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 3)
-        self.assertEquals(groups[0].results, shellresults)
-        self.assertEquals(groups[1].results, matchresults[0])
-        self.assertEquals(groups[2].results, matchresults[1])
+        cmd = "if bar -a; then b; fi"
+        shellresults = [
+            (0, 2, help_constants._if, "if"),
+            (9, 15, help_constants._if, "; then"),
+            (17, 21, help_constants._if, "; fi"),
+        ]
+
+        matchresults = [
+            [(3, 6, "bar synopsis", "bar"), (7, 9, "-a desc", "-a")],
+            [(16, 17, None, "b")],
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 3)
+        self.assertEqual(groups[0].results, shellresults)
+        self.assertEqual(groups[1].results, matchresults[0])
+        self.assertEqual(groups[2].results, matchresults[1])
 
     def test_nested_controlflows(self):
-        cmd = 'for a; do while bar; do baz; done; done'
-        shellresults = [(0, 9, helpconstants._for, 'for a; do'),
-                        (10, 15, helpconstants._whileuntil, 'while'),
-                        (19, 23, helpconstants._whileuntil, '; do'),
-                        (27, 33, helpconstants._whileuntil, '; done'),
-                        (33, 39, helpconstants._for, '; done')]
-
-        matchresults = [[(16, 19, 'bar synopsis', 'bar')],
-                        [(24, 27, 'baz synopsis', 'baz')]]
-
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 3)
-        self.assertEquals(groups[0].results, shellresults)
-        self.assertEquals(groups[1].results, matchresults[0])
-        self.assertEquals(groups[2].results, matchresults[1])
+        cmd = "for a; do while bar; do baz; done; done"
+        shellresults = [
+            (0, 9, help_constants._for, "for a; do"),
+            (10, 15, help_constants._whileuntil, "while"),
+            (19, 23, help_constants._whileuntil, "; do"),
+            (27, 33, help_constants._whileuntil, "; done"),
+            (33, 39, help_constants._for, "; done"),
+        ]
+
+        matchresults = [
+            [(16, 19, "bar synopsis", "bar")],
+            [(24, 27, "baz synopsis", "baz")],
+        ]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 3)
+        self.assertEqual(groups[0].results, shellresults)
+        self.assertEqual(groups[1].results, matchresults[0])
+        self.assertEqual(groups[2].results, matchresults[1])
 
     def test_for_expansion(self):
-        cmd = 'for a in $(bar); do baz; done'
-        shellresults = [(0, 19, helpconstants._for, 'for a in $(bar); do'),
-                        (23, 29, helpconstants._for, '; done')]
+        cmd = "for a in $(bar); do baz; done"
+        shellresults = [
+            (0, 19, help_constants._for, "for a in $(bar); do"),
+            (23, 29, help_constants._for, "; done"),
+        ]
 
-        matchresults = [(20, 23, 'baz synopsis', 'baz')]
+        matchresults = [(20, 23, "baz synopsis", "baz")]
 
-        m = matcher.matcher(cmd, s)
+        m = matcher.Matcher(cmd, s)
         groups = m.match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, shellresults)
-        self.assertEquals(groups[1].results, matchresults)
-
-        self.assertEquals(m.expansions, [(11, 14, 'substitution')])
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, shellresults)
+        self.assertEqual(groups[1].results, matchresults)
 
+        self.assertEqual(m.expansions, [(11, 14, "substitution")])
 
     def test_assignment_with_expansion(self):
         cmd = 'a="$1" bar'
 
-        shellresults = [(0, 6, helpconstants.ASSIGNMENT, 'a="$1"')]
-        matchresults = [[(7, 10, 'bar synopsis', 'bar')]]
+        shellresults = [(0, 6, help_constants.ASSIGNMENT, 'a="$1"')]
+        matchresults = [[(7, 10, "bar synopsis", "bar")]]
 
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, shellresults)
-        self.assertEquals(groups[1].results, matchresults[0])
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, shellresults)
+        self.assertEqual(groups[1].results, matchresults[0])
 
     def test_assignment_as_first_word(self):
-        cmd = 'a=b bar'
+        cmd = "a=b bar"
 
-        shellresults = [(0, 3, helpconstants.ASSIGNMENT, 'a=b')]
-        matchresults = [(4, 7, 'bar synopsis', 'bar')]
+        shellresults = [(0, 3, help_constants.ASSIGNMENT, "a=b")]
+        matchresults = [(4, 7, "bar synopsis", "bar")]
 
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, shellresults)
-        self.assertEquals(groups[1].results, matchresults)
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, shellresults)
+        self.assertEqual(groups[1].results, matchresults)
 
     def test_expansion_limit(self):
-        cmd = 'a $(b $(c))'
-        m = matcher.matcher(cmd, s)
+        cmd = "a $(b $(c))"
+        m = matcher.Matcher(cmd, s)
         m.match()
 
         class depthchecker(bashlex.ast.nodevisitor):
             def __init__(self):
                 self.depth = 0
                 self.maxdepth = 0
+
             def visitnode(self, node):
-                if 'substitution' in node.kind:
+                if "substitution" in node.kind:
                     self.depth += 1
                     self.maxdepth = max(self.maxdepth, self.depth)
+
             def visitendnode(self, node):
-                if 'substitution' in node.kind:
+                if "substitution" in node.kind:
                     self.depth -= 1
 
         v = depthchecker()
         v.visit(m.ast)
-        self.assertEquals(v.maxdepth, 1)
+        self.assertEqual(v.maxdepth, 1)
 
     def test_functions(self):
-        cmd = 'function a() { bar; }'
-        shellresults = [(0, 14, helpconstants._function, 'function a() {'),
-                        (18, 19, helpconstants.OPSEMICOLON, ';'),
-                        (20, 21, helpconstants._function, '}'),]
+        cmd = "function a() { bar; }"
+        shellresults = [
+            (0, 14, help_constants._function, "function a() {"),
+            (18, 19, help_constants.OPSEMICOLON, ";"),
+            (20, 21, help_constants._function, "}"),
+        ]
 
-        matchresults = [(15, 18, 'bar synopsis', 'bar')]
+        matchresults = [(15, 18, "bar synopsis", "bar")]
 
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, shellresults)
-        self.assertEquals(groups[1].results, matchresults)
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, shellresults)
+        self.assertEqual(groups[1].results, matchresults)
 
         cmd = 'function a() { bar "$(a)"; }'
-        shellresults = [(0, 14, helpconstants._function, 'function a() {'),
-                        (25, 26, helpconstants.OPSEMICOLON, ';'),
-                        (27, 28, helpconstants._function, '}'),]
+        shellresults = [
+            (0, 14, help_constants._function, "function a() {"),
+            (25, 26, help_constants.OPSEMICOLON, ";"),
+            (27, 28, help_constants._function, "}"),
+        ]
 
-        matchresults = [(15, 18, 'bar synopsis', 'bar'),
-                        (19, 25, None, '"$(a)"')]
+        matchresults = [(15, 18, "bar synopsis", "bar"), (19, 25, None, '"$(a)"')]
 
-        m = matcher.matcher(cmd, s)
+        m = matcher.Matcher(cmd, s)
         groups = m.match()
 
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, shellresults)
-        self.assertEquals(groups[1].results, matchresults)
-        self.assertEquals(m.expansions, [(22, 23, 'substitution')])
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, shellresults)
+        self.assertEqual(groups[1].results, matchresults)
+        self.assertEqual(m.expansions, [(22, 23, "substitution")])
 
     def test_function_reference(self):
-        cmd = 'function a() { bar; a b; }; a'
-        shellresults = [(0, 14, helpconstants._function, 'function a() {'),
-                        (18, 19, helpconstants.OPSEMICOLON, ';'),
-                        (20, 21, helpconstants._functioncall % 'a', 'a'),
-                        (22, 23, helpconstants._functionarg % 'a', 'b'),
-                        (23, 24, helpconstants.OPSEMICOLON, ';'),
-                        (25, 26, helpconstants._function, '}'),
-                        (26, 27, helpconstants.OPSEMICOLON, ';'),
-                        (28, 29, helpconstants._functioncall % 'a', 'a'),]
-
-        matchresults = [(15, 18, 'bar synopsis', 'bar')]
-
-        m = matcher.matcher(cmd, s)
+        cmd = "function a() { bar; a b; }; a"
+        shellresults = [
+            (0, 14, help_constants._function, "function a() {"),
+            (18, 19, help_constants.OPSEMICOLON, ";"),
+            (20, 21, help_constants._function_call % "a", "a"),
+            (22, 23, help_constants._functionarg % "a", "b"),
+            (23, 24, help_constants.OPSEMICOLON, ";"),
+            (25, 26, help_constants._function, "}"),
+            (26, 27, help_constants.OPSEMICOLON, ";"),
+            (28, 29, help_constants._function_call % "a", "a"),
+        ]
+
+        matchresults = [(15, 18, "bar synopsis", "bar")]
+
+        m = matcher.Matcher(cmd, s)
         groups = m.match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, shellresults)
-        self.assertEquals(groups[1].results, matchresults)
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, shellresults)
+        self.assertEqual(groups[1].results, matchresults)
 
-        self.assertEquals(m.functions, set(['a']))
+        self.assertEqual(m.functions, {"a"})
 
     def test_comment(self):
-        cmd = 'bar # a comment'
+        cmd = "bar # a comment"
 
-        shellresults = [(4, 15, helpconstants.COMMENT, '# a comment')]
-        matchresults = [(0, 3, 'bar synopsis', 'bar')]
+        shellresults = [(4, 15, help_constants.COMMENT, "# a comment")]
+        matchresults = [(0, 3, "bar synopsis", "bar")]
 
-        m = matcher.matcher(cmd, s)
+        m = matcher.Matcher(cmd, s)
         groups = m.match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, shellresults)
-        self.assertEquals(groups[1].results, matchresults)
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, shellresults)
+        self.assertEqual(groups[1].results, matchresults)
 
-        cmd = '# just a comment'
+        cmd = "# just a comment"
 
-        shellresults = [(0, 16, helpconstants.COMMENT, '# just a comment')]
+        shellresults = [(0, 16, help_constants.COMMENT, "# just a comment")]
 
-        m = matcher.matcher(cmd, s)
+        m = matcher.Matcher(cmd, s)
         groups = m.match()
-        self.assertEquals(len(groups), 1)
-        self.assertEquals(groups[0].results, shellresults)
+        self.assertEqual(len(groups), 1)
+        self.assertEqual(groups[0].results, shellresults)
 
     def test_heredoc_at_eof(self):
-        cmd = 'bar <<EOF'
-
-        shellresults = [(4, 9, helpconstants.REDIRECTION + '\n\n' +
-                               helpconstants.REDIRECTION_KIND['<<'], '<<EOF')]
-
-        matchresults = [(0, 3, 'bar synopsis', 'bar')]
-
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, shellresults)
-        self.assertEquals(groups[1].results, matchresults)
+        cmd = "bar <<EOF"
+
+        shellresults = [
+            (
+                4,
+                9,
+                help_constants.REDIRECTION
+                + "\n\n"
+                + help_constants.REDIRECTION_KIND["<<"],
+                "<<EOF",
+            )
+        ]
+
+        matchresults = [(0, 3, "bar synopsis", "bar")]
+
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, shellresults)
+        self.assertEqual(groups[1].results, matchresults)
 
     def test_no_synopsis(self):
-        cmd = 'nosynopsis a'
+        cmd = "nosynopsis a"
 
-        matchresults = [(0, 10, helpconstants.NOSYNOPSIS, 'nosynopsis'),
-                        (11, 12, None, 'a')]
+        matchresults = [
+            (0, 10, help_constants.NO_SYNOPSIS, "nosynopsis"),
+            (11, 12, None, "a"),
+        ]
 
-        groups = matcher.matcher(cmd, s).match()
-        self.assertEquals(len(groups), 2)
-        self.assertEquals(groups[0].results, [])
-        self.assertEquals(groups[1].results, matchresults)
+        groups = matcher.Matcher(cmd, s).match()
+        self.assertEqual(len(groups), 2)
+        self.assertEqual(groups[0].results, [])
+        self.assertEqual(groups[1].results, matchresults)
diff --git a/tests/test-options.py b/tests/test-options.py
index e7f4f6ae..cfb11ab2 100644
--- a/tests/test-options.py
+++ b/tests/test-options.py
@@ -2,110 +2,112 @@
 
 from explainshell import options, store, errors
 
+
 class test_options(unittest.TestCase):
     def test_simple(self):
-        s = '\t-a description'
-        self.assertEquals(options.extract_option(s), (['-a'], []))
+        s = "\t-a description"
+        self.assertEqual(options.extract_option(s), (["-a"], []))
 
-        s = '\t-a, description'
-        self.assertEquals(options.extract_option(s), (['-a'], []))
+        s = "\t-a, description"
+        self.assertEqual(options.extract_option(s), (["-a"], []))
 
-        r = (['-a', '-b'], [])
-        s = '\t-a, -b description'
-        self.assertEquals(options.extract_option(s), r)
+        r = (["-a", "-b"], [])
+        s = "\t-a, -b description"
+        self.assertEqual(options.extract_option(s), r)
 
-        s = '\t-a/-b description'
-        self.assertEquals(options.extract_option(s), r)
+        s = "\t-a/-b description"
+        self.assertEqual(options.extract_option(s), r)
 
-        s = '\t-a -b description'
-        self.assertEquals(options.extract_option(s), r)
+        s = "\t-a -b description"
+        self.assertEqual(options.extract_option(s), r)
 
-        s = '\t-a     -b,-c,           -d description'
-        self.assertEquals(options.extract_option(s), (['-a', '-b', '-c', '-d'], []))
+        s = "\t-a     -b,-c,           -d description"
+        self.assertEqual(options.extract_option(s), (["-a", "-b", "-c", "-d"], []))
 
-        s = '\t--a, -b, --c-d description'
-        self.assertEquals(options.extract_option(s), (['-b'], ['--a', '--c-d']))
+        s = "\t--a, -b, --c-d description"
+        self.assertEqual(options.extract_option(s), (["-b"], ["--a", "--c-d"]))
 
-        s = '---c-d '
-        self.assertEquals(options.extract_option(s), ([], []))
+        s = "---c-d "
+        self.assertEqual(options.extract_option(s), ([], []))
 
-        s = '-c- '
-        self.assertEquals(options.extract_option(s), ([], []))
+        s = "-c- "
+        self.assertEqual(options.extract_option(s), ([], []))
 
     def test_option_arg(self):
-        s = '\t-a FOO, -b=BAR, description'
-        self.assertEquals(options.extract_option(s),
-                          ([('-a', 'FOO'), ('-b', 'BAR')], []))
+        s = "\t-a FOO, -b=BAR, description"
+        self.assertEqual(
+            options.extract_option(s), ([("-a", "FOO"), ("-b", "BAR")], [])
+        )
 
-        s = '\t-a [FOO], -b[=BAR], description'
-        self.assertEquals(options.extract_option(s),
-                          ([('-a', 'FOO'), ('-b', 'BAR')], []))
+        s = "\t-a [FOO], -b[=BAR], description"
+        self.assertEqual(
+            options.extract_option(s), ([("-a", "FOO"), ("-b", "BAR")], [])
+        )
 
-        s = '\t-a<n>, -b=<BAR>, -C <ah>'
-        self.assertEquals(options.extract_option(s),
-                          ([('-a', 'n'), ('-b', 'BAR'), ('-C', 'ah')], []))
+        s = "\t-a<n>, -b=<BAR>, -C <ah>"
+        self.assertEqual(
+            options.extract_option(s), ([("-a", "n"), ("-b", "BAR"), ("-C", "ah")], [])
+        )
 
-        s = '\t--aa    FOO, --bb=BAR, description'
-        self.assertEquals(options.extract_option(s),
-                          ([], [('--aa', 'FOO'), ('--bb', 'BAR')]))
+        s = "\t--aa    FOO, --bb=BAR, description"
+        self.assertEqual(
+            options.extract_option(s), ([], [("--aa", "FOO"), ("--bb", "BAR")])
+        )
 
-        s = '-a or -b'
-        self.assertEquals(options.extract_option(s),
-                          (['-a', '-b'], []))
+        s = "-a or -b"
+        self.assertEqual(options.extract_option(s), (["-a", "-b"], []))
 
     def test_pipe_separator(self):
-        s = '-a|b'
-        self.assertEquals(options.extract_option(s),
-                          (['-a', 'b'], []))
+        s = "-a|b"
+        self.assertEqual(options.extract_option(s), (["-a", "b"], []))
 
-        s = '-a|-b|--c|d'
-        self.assertEquals(options.extract_option(s),
-                          (['-a', '-b', 'd'], ['--c']))
+        s = "-a|-b|--c|d"
+        self.assertEqual(options.extract_option(s), (["-a", "-b", "d"], ["--c"]))
 
     def test_multiline_options(self):
-        s = '\t-a, -b, \n-c, --ddd description'
-        self.assertEquals(options.extract_option(s),
-                          (['-a', '-b', '-c'], ['--ddd']))
+        s = "\t-a, -b, \n-c, --ddd description"
+        self.assertEqual(options.extract_option(s), (["-a", "-b", "-c"], ["--ddd"]))
 
     def test_multiline_desc(self):
-        s = '\t-a, -b description\n\tmultiline\n  another line'
-        self.assertEquals(options.extract_option(s), (['-a', '-b'], []))
+        s = "\t-a, -b description\n\tmultiline\n  another line"
+        self.assertEqual(options.extract_option(s), (["-a", "-b"], []))
 
     def test_not_an_option(self):
-        self.assertEquals(options.extract_option('foobar'), ([], []))
+        self.assertEqual(options.extract_option("foobar"), ([], []))
 
     def test_no_hyphen(self):
-        s = '\ta=b description'
-        self.assertEquals(options.extract_option(s), ([], [('a', 'b')]))
+        s = "\ta=b description"
+        self.assertEqual(options.extract_option(s), ([], [("a", "b")]))
 
     def test_hyphen_in_arg(self):
-        s = '-a=FOO-BAR, --aa=FOO-BAR'
-        self.assertEquals(options.extract_option(s),
-                          ([('-a', 'FOO-BAR')], [('--aa', 'FOO-BAR')]))
+        s = "-a=FOO-BAR, --aa=FOO-BAR"
+        self.assertEqual(
+            options.extract_option(s), ([("-a", "FOO-BAR")], [("--aa", "FOO-BAR")])
+        )
 
-        #s = '-a FOO-BAR, --aa FOO-BAR'
-        #self.assertEquals(options.extract_option(s),
+        # s = '-a FOO-BAR, --aa FOO-BAR'
+        # self.assertEqual(options.extract_option(s),
         #                  ([('-a', 'FOO-BAR')], [('--aa', 'FOO-BAR')]))
 
     def test_extract(self):
-        p1 = store.paragraph(0, '<b>--test</b>=<u>arg</u>\ndesc', '', True)
-        p2 = store.paragraph(1, 'no options here', '', True)
-        p3 = store.paragraph(2, '--foo-bar=&lt;arg&gt;\ndesc', '', True)
+        p1 = store.Paragraph(0, "<b>--test</b>=<u>arg</u>\ndesc", "", True)
+        p2 = store.Paragraph(1, "no options here", "", True)
+        p3 = store.Paragraph(2, "--foo-bar=&lt;arg&gt;\ndesc", "", True)
 
-        m = store.manpage('', '', '', [p1, p2, p3], [])
+        m = store.ManPage("", "", "", [p1, p2, p3], [])
         options.extract(m)
         r = m.options
-        self.assertEquals(len(r), 2)
-        self.assertEquals(r[0].text, p1.text)
-        self.assertEquals(r[0].short, [])
-        self.assertEquals(r[0].long, ['--test'])
-        self.assertEquals(r[0].expectsarg, True)
+        self.assertEqual(len(r), 2)
+        self.assertEqual(r[0].text, p1.text)
+        self.assertEqual(r[0].short, [])
+        self.assertEqual(r[0].long, ["--test"])
+        self.assertEqual(r[0].expects_arg, True)
 
-        self.assertEquals(r[1].text, p3.text)
-        self.assertEquals(r[1].short, [])
-        self.assertEquals(r[1].long, ['--foo-bar'])
-        self.assertEquals(r[1].expectsarg, True)
+        self.assertEqual(r[1].text, p3.text)
+        self.assertEqual(r[1].short, [])
+        self.assertEqual(r[1].long, ["--foo-bar"])
+        self.assertEqual(r[1].expects_arg, True)
 
     def test_help(self):
-        s = '\t-?, --help description'
-        self.assertEquals(options.extract_option(s), (['-?'], ['--help']))
+        s = "\t-?, --help description"
+        self.assertEqual(options.extract_option(s), (["-?"], ["--help"]))
diff --git a/tools/shellbuiltins.py b/tools/shellbuiltins.py
index 6436fb96..179d0f5a 100644
--- a/tools/shellbuiltins.py
+++ b/tools/shellbuiltins.py
@@ -1,31 +1,54 @@
-# -*- coding: utf-8 -*-
-
-'''manually define and add shell builtins into the store
+"""
+manually define and add shell builtins into the store
 
 unfortunately the bash section for builtins isn't written in a way
-explainshell can understannd, so we have to resort to manually
-writing these down and adding them.'''
+explainshell can understand, so we have to resort to manually
+writing these down and adding them.
+"""
 
 import textwrap
 from explainshell import store, config
 
-sp = store.paragraph
-so = store.option
-sm = store.manpage
+sp = store.Paragraph
+so = store.Option
+sm = store.ManPage
 
 BUILTINS = {}
 
+
 def _add(names, synopsis, options):
     name = names[0]
     # hack: fake a source man page (this breaks the outgoing links from
     # explainshell, oh well)
-    names.append('bash-%s' % name)
-    BUILTINS[name] = sm('bash-%s.1.gz' % name, name, synopsis, options, [(name, 20) for name in names])
+    names.append(f"bash-{name}")
+    BUILTINS[name] = sm(
+        "bash-%s.1.gz" % name, name, synopsis, options, [(name, 20) for name in names]
+    )
 
-_add([':'], 'the command does nothing', [so(sp(0, '''No  effect;  the command does nothing beyond expanding arguments and performing any specified redirections.  A zero
-exit code  is returned.''', '', True), [], [], False, True, False)])
 
-source = textwrap.dedent('''       <b>.</b>  <u>filename</u> [<u>arguments</u>]
+_add(
+    [":"],
+    "the command does nothing",
+    [
+        so(
+            sp(
+                0,
+                """No  effect;  the command does nothing beyond expanding arguments and performing any specified redirections.  A zero
+exit code  is returned.""",
+                "",
+                True,
+            ),
+            [],
+            [],
+            False,
+            True,
+            False,
+        )
+    ],
+)
+
+source = textwrap.dedent(
+    """       <b>.</b>  <u>filename</u> [<u>arguments</u>]
        <b>source</b> <u>filename</u> [<u>arguments</u>]
               Read  and  execute  commands  from <u>filename</u> in the current shell environment and return the  exit  status  of
               the  last  command executed  from  <u>filename</u>.  If <u>filename</u> does not contain a slash, filenames in <b>PATH</b> are used
@@ -34,14 +57,42 @@ def _add(names, synopsis, options):
               <b>sourcepath</b> option to the <b>shopt</b> builtin command is turned  off,  the  <b>PATH</b>  is  not searched.   If  any
               <u>arguments</u>  are  supplied,  they  become the positional parameters when <u>filename</u> is executed.  Otherwise
               the positional  parameters  are unchanged.  The return status is the status of the last command exited within
-              the  script  (0  if  no commands  are  executed), and false if <u>filename</u> is not found or cannot be read.''')
-_add(['source', '.'], 'read and execute commands in the current shell', [so(sp(0, source, '', True), [], [], False, True, False)])
+              the  script  (0  if  no commands  are  executed), and false if <u>filename</u> is not found or cannot be read."""
+)
+_add(
+    ["source", "."],
+    "read and execute commands in the current shell",
+    [so(sp(0, source, "", True), [], [], False, True, False)],
+)
 
-_add(['break'], 'exit from within a for, while, until, or select loop',
-     [so(sp(0, '''If <u>n</u> is specified, break <u>n</u> levels.  <u>n</u> must be ≥ 1.  If <u>n</u> is greater than the  number  of enclosing loops, all enclosing loops are exited.  The return value is 0 unless <u>n</u> is not greater than or  equal  to 1.''', '', True), [], [], False, True, False)])
+_add(
+    ["break"],
+    "exit from within a for, while, until, or select loop",
+    [
+        so(
+            sp(
+                0,
+                """If <u>n</u> is specified, break <u>n</u> levels.  <u>n</u> must be ≥ 1.  If <u>n</u> is greater than the  number  of enclosing loops, all enclosing loops are exited.  The return value is 0 unless <u>n</u> is not greater than or  equal  to 1.""",
+                "",
+                True,
+            ),
+            [],
+            [],
+            False,
+            True,
+            False,
+        )
+    ],
+)
 
-_add(['history'], 'display the  command  history  list  with  line numbers',
-    [so(sp(0, '''<b>history</b> <b>[</b><u>n</u><b>]</b>
+_add(
+    ["history"],
+    "display the  command  history  list  with  line numbers",
+    [
+        so(
+            sp(
+                0,
+                """<b>history</b> <b>[</b><u>n</u><b>]</b>
 <b>history</b> <b>-c</b>
 <b>history</b> <b>-d</b> <u>offset</u>
 <b>history</b> <b>-anrw</b> [<u>filename</u>]
@@ -53,25 +104,147 @@ def _add(names, synopsis, options):
 and  not  null,  it is used as a format string for <u>strftime</u>(3) to display the time stamp associated with each
 displayed  history entry.  No intervening blank is printed between the formatted time  stamp  and  the  history
 line.   If <u>filename</u>  is  supplied,  it  is  used as the name of the history file; if not, the  value  of
-<b>HISTFILE</b>  is  used.''', '', True), [], [], False, True, False),
-     so(sp(1, '<b>-c</b>     Clear the history list by deleting all the entries.', '', True), ['-c'], [], False, False, False),
-     so(sp(2, textwrap.dedent('''              <b>-d</b> <u>offset</u>
-                     Delete the history entry at position <u>offset</u>.'''), '', True), ['-d'], [], 'offset', False, False),
-     so(sp(3, textwrap.dedent('''              <b>-a</b>     Append  the  ``new'' history lines (history lines entered since the beginning of the current <b>bash</b> session)
-                     to  the history file.'''), '', True), ['-a'], [], False, False, False),
-     so(sp(4, textwrap.dedent('''              <b>-n</b>     Read  the history lines not already read from the history file into the current  history  list.   These  are
-                     lines appended  to  the history file since the beginning of the current <b>bash</b> session.'''), '', True), ['-n'], [], False, False, False),
-     so(sp(5, textwrap.dedent('''              <b>-r</b>     Read the contents of the history file and append them  to the current history list.'''), '', True), ['-r'], [], False, False, False),
-     so(sp(6, textwrap.dedent('''              <b>-w</b>     Write  the  current  history  list  to  the history file, overwriting the history file's contents.'''), '', True), ['-w'], [], 'filename', False, False),
-     so(sp(7, textwrap.dedent('''              <b>-p</b>     Perform history substitution on the  following  <u>args</u>  and display  the  result  on  the  standard output.
-                     Does not store the results in the history list.  Each <u>arg</u> must  be quoted to disable normal history expansion.'''), '', True), ['-p'], [], 'arg', True, False),
-     so(sp(8, textwrap.dedent('''              <b>-s</b>     Store  the  <u>args</u>  in  the history list as a single entry.  The last command in the history list  is
-                     removed  before the <u>args</u> are added.'''), '', True), ['-s'], [], 'arg', False, False)])
+<b>HISTFILE</b>  is  used.""",
+                "",
+                True,
+            ),
+            [],
+            [],
+            False,
+            True,
+            False,
+        ),
+        so(
+            sp(
+                1,
+                "<b>-c</b>     Clear the history list by deleting all the entries.",
+                "",
+                True,
+            ),
+            ["-c"],
+            [],
+            False,
+            False,
+            False,
+        ),
+        so(
+            sp(
+                2,
+                textwrap.dedent(
+                    """              <b>-d</b> <u>offset</u>
+                     Delete the history entry at position <u>offset</u>."""
+                ),
+                "",
+                True,
+            ),
+            ["-d"],
+            [],
+            "offset",
+            False,
+            False,
+        ),
+        so(
+            sp(
+                3,
+                textwrap.dedent(
+                    """              <b>-a</b>     Append  the  ``new'' history lines (history lines entered since the beginning of the current <b>bash</b> session)
+                     to  the history file."""
+                ),
+                "",
+                True,
+            ),
+            ["-a"],
+            [],
+            False,
+            False,
+            False,
+        ),
+        so(
+            sp(
+                4,
+                textwrap.dedent(
+                    """              <b>-n</b>     Read  the history lines not already read from the history file into the current  history  list.   These  are
+                     lines appended  to  the history file since the beginning of the current <b>bash</b> session."""
+                ),
+                "",
+                True,
+            ),
+            ["-n"],
+            [],
+            False,
+            False,
+            False,
+        ),
+        so(
+            sp(
+                5,
+                textwrap.dedent(
+                    """              <b>-r</b>     Read the contents of the history file and append them  to the current history list."""
+                ),
+                "",
+                True,
+            ),
+            ["-r"],
+            [],
+            False,
+            False,
+            False,
+        ),
+        so(
+            sp(
+                6,
+                textwrap.dedent(
+                    """              <b>-w</b>     Write  the  current  history  list  to  the history file, overwriting the history file's contents."""
+                ),
+                "",
+                True,
+            ),
+            ["-w"],
+            [],
+            "filename",
+            False,
+            False,
+        ),
+        so(
+            sp(
+                7,
+                textwrap.dedent(
+                    """              <b>-p</b>     Perform history substitution on the  following  <u>args</u>  and display  the  result  on  the  standard output.
+                     Does not store the results in the history list.  Each <u>arg</u> must  be quoted to disable normal history expansion."""
+                ),
+                "",
+                True,
+            ),
+            ["-p"],
+            [],
+            "arg",
+            True,
+            False,
+        ),
+        so(
+            sp(
+                8,
+                textwrap.dedent(
+                    """              <b>-s</b>     Store  the  <u>args</u>  in  the history list as a single entry.  The last command in the history list  is
+                     removed  before the <u>args</u> are added."""
+                ),
+                "",
+                True,
+            ),
+            ["-s"],
+            [],
+            "arg",
+            False,
+            False,
+        ),
+    ],
+)
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     import logging.config
+
     logging.config.dictConfig(config.LOGGING_DICT)
 
-    s = store.store('explainshell', config.MONGO_URI)
+    s = store.Store("explainshell", config.MONGO_URI)
     for m in BUILTINS.itervalues():
         s.addmanpage(m)

From 2add463e50ae79437b6e8e4ab1d90302edca8b17 Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 14 Sep 2024 21:58:50 +0000
Subject: [PATCH 10/19] chore: enhance formatting and update variable names

---
 explainshell/web/templates/base.html   |   2 +-
 explainshell/web/templates/tagger.html | 341 +++++++++++++------------
 2 files changed, 176 insertions(+), 167 deletions(-)

diff --git a/explainshell/web/templates/base.html b/explainshell/web/templates/base.html
index f3b65915..f2320767 100644
--- a/explainshell/web/templates/base.html
+++ b/explainshell/web/templates/base.html
@@ -23,7 +23,7 @@
         <link href='//fonts.googleapis.com/css?family=Berkshire+Swash' rel='stylesheet' type='text/css'>
         {% block head %}{% endblock %}
     </head>
-    <body data-theme={{ request.cookies.get('theme', 'default')}}>
+    <body data-theme={{ request.cookies.get('theme', 'default') }}>
         <div class="container">
             <div class="small-push"></div>
             <div class="header">
diff --git a/explainshell/web/templates/tagger.html b/explainshell/web/templates/tagger.html
index d00652d4..4e2674d4 100644
--- a/explainshell/web/templates/tagger.html
+++ b/explainshell/web/templates/tagger.html
@@ -1,169 +1,178 @@
 {% extends "base.html" %}
-	{% block content %}
-            <div class="small-push"></div>
-            <div class="push"></div>
-            <div id="command">
-                <a href="http://manpages.ubuntu.com/manpages/precise/en/man1/{{ m.name }}.html">{{ m.name|e }}</a> - {{ m.synopsis|e }}
-            </div>
-            <div class="small-push"></div>
-            <div style="width: 1000px;">
-                <table id="help" width="100%">
-                    <tbody>
-                        {% for p in m.paragraphs -%}
-                        <tr>
-                            <td>
-                                {% if p.is_option %}
-                                <pre section="{{ p.section }}" paragraph-id="{{ p.idx }}" class="taggeroption">{{ p.text|safe }}</pre>
-                                {% else %}
-                                <pre section="{{ p.section }}" paragraph-id="{{ p.idx }}" class="taggernooption">{{ p.text|safe }}</pre>
-                                {% endif %}
-                            <span style="float: right">
-                                <input type='text' placeholder="short" class='input-small' {% if p.is_option and p.short %}value="{{ p.short|join(', ') }}"{% endif %}/>
-                                <input type='text' placeholder="long" class='input-small' {% if p.is_option and p.long %}value="{{ p.long|join(', ') }}"{% endif %}/>
-                                <input type='text' placeholder="expectsarg" class='input-small' {% if p.is_option and p.expectsarg %}value="{{ p.expectsarg }}"{% endif %}/>
-                                <input type='text' placeholder="argument" class='input-small' {% if p.is_option and p.argument %}value="{{ p.argument }}"{% endif %}/>
-                                <input type='text' placeholder="nested" class='input-small' {% if p.is_option and p.nestedcommand %}value="{{ p.nestedcommand }}"{% endif %}/>
-                            </span>
-                            </td>
-                        </tr>
-                        {%- endfor %}
-                    </tbody>
-                </table>
-                <form id='tagsform' method='post'>
-                    <input name='paragraphs' type='hidden' id='paragraphs'/>
-                    <div>
-                        nestedcommand: <input type='checkbox' name='nestedcommand' value='true' {% if m.nestedcommand %}checked{% endif %}/>
-                        <input type='submit' class='btn'/>
-                    </div>
-                </form>
-            </div>
+{% block content %}
+<div class="small-push"></div>
+<div class="push"></div>
+<div id="command">
+    <a href="http://manpages.ubuntu.com/manpages/precise/en/man1/{{ m.name }}.html">{{ m.name|e }}</a> - {{ m.synopsis|e
+    }}
+</div>
+<div class="small-push"></div>
+<div style="width: 1000px;">
+    <table id="help" width="100%">
+        <tbody>
+            {% for p in m.paragraphs -%}
+            <tr>
+                <td>
+                    {% if p.is_option %}
+                    <pre section="{{ p.section }}" paragraph-id="{{ p.idx }}"
+                        class="taggeroption">{{ p.text|safe }}</pre>
+                    {% else %}
+                    <pre section="{{ p.section }}" paragraph-id="{{ p.idx }}"
+                        class="taggernooption">{{ p.text|safe }}</pre>
+                    {% endif %}
+                    <span style="float: right">
+                        <input type='text' placeholder="short" class='input-small' {% if p.is_option and p.short
+                            %}value="{{ p.short|join(', ') }}" {% endif %} />
+                        <input type='text' placeholder="long" class='input-small' {% if p.is_option and p.long
+                            %}value="{{ p.long|join(', ') }}" {% endif %} />
+                        <input type='text' placeholder="expects_arg" class='input-small' {% if p.is_option and
+                            p.expects_arg %}value="{{ p.expects_arg }}" {% endif %} />
+                        <input type='text' placeholder="argument" class='input-small' {% if p.is_option and p.argument
+                            %}value="{{ p.argument }}" {% endif %} />
+                        <input type='text' placeholder="nested" class='input-small' {% if p.is_option and p.nested_cmd
+                            %}value="{{ p.nested_cmd }}" {% endif %} />
+                    </span>
+                </td>
+            </tr>
+            {%- endfor %}
+        </tbody>
+    </table>
+    <form id='tagsform' method='post'>
+        <input name='paragraphs' type='hidden' id='paragraphs' />
+        <div>
+            nested command: <input type='checkbox' name='nested_cmd' value='true' {% if m.nested_cmd %}checked{% endif
+                %} />
+            <input type='submit' class='btn' />
+        </div>
+    </form>
+</div>
 {% endblock %}
 {% block js %}
-            <script type='text/javascript'>
-                //$(".taggeroption").parent().find("i").addClass("icon-ok");
-                //$(".taggernooption").parent().find("i").addClass("icon-remove");
-                var current = $("[paragraph-id=0]");
-                var timeoutId;
-
-                function select() {
-                    $(current).removeClass('taggercurrent')
-                    current = $(this);
-                    $(current).addClass('taggercurrent')
-                }
-
-                function scrollTo(elem) {
-                    if (timeoutId)
-                        clearTimeout(timeoutId);
-                    timeoutId = setTimeout(scroll, 300);
-                    function scroll() {
-                        $('html, body').scrollTop(elem.offset().top);
-                    }
-
-                    $(current).removeClass('taggercurrent')
-                    current = elem;
-                    $(current).addClass('taggercurrent')
-                }
-
-                function gotoNext() {
-                    var next = $(current).parent().parent().next().find("[paragraph-id]")
-
-                    scrollTo(next);
-                }
-
-                function gotoPrev() {
-                    var prev = $(current).parent().parent().prev().find("[paragraph-id]")
-
-                    scrollTo(prev);
-                }
-
-                function toggleCurrent() {
-                    $(current).toggleClass("taggeroption").toggleClass("taggernooption");
-                    gotoNext();
-                }
-
-                function mergeCurrent() {
-                    var next = $(current).parent().parent().next().remove();
-                    var html = $(current).html();
-                    html += '\n\n';
-                    html += next.find("[paragraph-id]").html();
-                    $(current).html(html);
-                }
-
-                function splitCurrent() {
-                    var newelement = $(current).parent().parent().clone(true);
-                    var html = $(current).html();
-                    var splitted = html.split('\n\n');
-                    if (splitted.length < 2)
-                        return;
-                    html = splitted.shift();
-                    $(current).html(html);
-
-                    newelement.find("input").val('');
-                    var par = newelement.find("[paragraph-id]");
-                    par.removeClass('taggercurrent')
-                    par.attr('paragraph-id', parseInt($(current).attr('paragraph-id')) + 1);
-                    par.html(splitted.join('\n\n'));
-                    $(current).parent().parent().after(newelement);
-                }
-
-                $("[paragraph-id]").click(select);
-
-                $("body").keyup(function(event) {
-                    var code = event.keyCode;
-
-                    if (code == 74) // j
-                        gotoNext();
-                    else if (code == 75) // k
-                        gotoPrev();
-                    else if (code == 77) // m
-                        mergeCurrent();
-                    else if (code == 83) // s
-                        splitCurrent();
-                    else if (code == 222) // '
-                        toggleCurrent();
-                    else if (code == 68) // d
-                        $('#tagsform').submit();
-                });
-
-                function setResults() {
-                    var all = $("[paragraph-id]");
-                    var p = new Array();
-
-                    all.each(function(i, e) {
-                        var id = parseInt($(e).attr('paragraph-id'));
-
-                        var d = {}
-                        d["idx"] = id;
-                        d["section"] = $(e).attr('section');
-                        d["text"] = $(e).html();
-                        if ($(e).hasClass('taggeroption')) {
-                            d["is_option"] = true;
-                        }
-                        else {
-                            d["is_option"] = false;
-                        }
-
-                        var inputs = $(e).parent().find("input");
-                        d["short"] = inputs[0].value ? inputs[0].value.split(',') : [];
-                        d["long"] = inputs[1].value ? inputs[1].value.split(',') : [];
-                        d["expectsarg"] = inputs[2].value;
-                        if (d["expectsarg"].indexOf(",") != -1)
-                            d["expectsarg"] = d["expectsarg"].split(",");
-                        d["argument"] = inputs[3].value;
-                        d["nestedcommand"] = inputs[4].value;
-                        if (d["nestedcommand"].indexOf(",") != -1)
-                            d["nestedcommand"] = d["nestedcommand"].split(",");
-                        p.push(d)
-                    });
-
-                    $("#paragraphs").val(JSON.stringify(p));
-                }
-                $(document).ready(function() {
-                    scrollTo(current);
-
-                    $('#tagsform').submit(function() {
-                        setResults();
-                    });
-                });
-            </script>
-{% endblock %}
+<script type='text/javascript'>
+    //$(".taggeroption").parent().find("i").addClass("icon-ok");
+    //$(".taggernooption").parent().find("i").addClass("icon-remove");
+    var current = $("[paragraph-id=0]");
+    var timeoutId;
+
+    function select() {
+        $(current).removeClass('taggercurrent')
+        current = $(this);
+        $(current).addClass('taggercurrent')
+    }
+
+    function scrollTo(elem) {
+        if (timeoutId)
+            clearTimeout(timeoutId);
+        timeoutId = setTimeout(scroll, 300);
+        function scroll() {
+            $('html, body').scrollTop(elem.offset().top);
+        }
+
+        $(current).removeClass('taggercurrent')
+        current = elem;
+        $(current).addClass('taggercurrent')
+    }
+
+    function gotoNext() {
+        var next = $(current).parent().parent().next().find("[paragraph-id]")
+
+        scrollTo(next);
+    }
+
+    function gotoPrev() {
+        var prev = $(current).parent().parent().prev().find("[paragraph-id]")
+
+        scrollTo(prev);
+    }
+
+    function toggleCurrent() {
+        $(current).toggleClass("taggeroption").toggleClass("taggernooption");
+        gotoNext();
+    }
+
+    function mergeCurrent() {
+        var next = $(current).parent().parent().next().remove();
+        var html = $(current).html();
+        html += '\n\n';
+        html += next.find("[paragraph-id]").html();
+        $(current).html(html);
+    }
+
+    function splitCurrent() {
+        var newelement = $(current).parent().parent().clone(true);
+        var html = $(current).html();
+        var splitted = html.split('\n\n');
+        if (splitted.length < 2)
+            return;
+        html = splitted.shift();
+        $(current).html(html);
+
+        newelement.find("input").val('');
+        var par = newelement.find("[paragraph-id]");
+        par.removeClass('taggercurrent')
+        par.attr('paragraph-id', parseInt($(current).attr('paragraph-id')) + 1);
+        par.html(splitted.join('\n\n'));
+        $(current).parent().parent().after(newelement);
+    }
+
+    $("[paragraph-id]").click(select);
+
+    $("body").keyup(function (event) {
+        var code = event.keyCode;
+
+        if (code == 74) // j
+            gotoNext();
+        else if (code == 75) // k
+            gotoPrev();
+        else if (code == 77) // m
+            mergeCurrent();
+        else if (code == 83) // s
+            splitCurrent();
+        else if (code == 222) // '
+            toggleCurrent();
+        else if (code == 68) // d
+            $('#tagsform').submit();
+    });
+
+    function setResults() {
+        var all = $("[paragraph-id]");
+        var p = new Array();
+
+        all.each(function (i, e) {
+            var id = parseInt($(e).attr('paragraph-id'));
+
+            var d = {}
+            d["idx"] = id;
+            d["section"] = $(e).attr('section');
+            d["text"] = $(e).html();
+            if ($(e).hasClass('taggeroption')) {
+                d["is_option"] = true;
+            }
+            else {
+                d["is_option"] = false;
+            }
+
+            var inputs = $(e).parent().find("input");
+            d["short"] = inputs[0].value ? inputs[0].value.split(',') : [];
+            d["long"] = inputs[1].value ? inputs[1].value.split(',') : [];
+            d["expects_arg"] = inputs[2].value;
+            if (d["expects_arg"].indexOf(",") != -1)
+                d["expects_arg"] = d["expects_arg"].split(",");
+            d["argument"] = inputs[3].value;
+            d["nested_cmd"] = inputs[4].value;
+            if (d["nested_cmd"].indexOf(",") != -1)
+                d["nested_cmd"] = d["nested_cmd"].split(",");
+            p.push(d)
+        });
+
+        $("#paragraphs").val(JSON.stringify(p));
+    }
+    $(document).ready(function () {
+        scrollTo(current);
+
+        $('#tagsform').submit(function () {
+            setResults();
+        });
+    });
+</script>
+{% endblock %}
\ No newline at end of file

From cd50f3ea5eb6b24946886ffc8cb972e0003890e1 Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sun, 15 Sep 2024 10:29:51 +0000
Subject: [PATCH 11/19] feat: add syntax highlighting to landing page

---
 explainshell/web/static/css/es.css            |  14 +
 .../web/static/css/highlight.default.min.css  |   9 +
 .../web/static/css/hljs-atom-one-dark.min.css |   1 +
 explainshell/web/static/js/es.js              |  12 +-
 explainshell/web/static/js/highlight.min.js   | 351 ++++++++++++++++++
 explainshell/web/templates/base.html          |   6 +
 .../web/templates/errors/parsingerror.html    |   2 +-
 explainshell/web/templates/index.html         |   6 +-
 explainshell/web/templates/macros.html        |   2 +-
 9 files changed, 398 insertions(+), 5 deletions(-)
 create mode 100644 explainshell/web/static/css/highlight.default.min.css
 create mode 100644 explainshell/web/static/css/hljs-atom-one-dark.min.css
 create mode 100644 explainshell/web/static/js/highlight.min.js

diff --git a/explainshell/web/static/css/es.css b/explainshell/web/static/css/es.css
index 74175a3c..2019b53e 100644
--- a/explainshell/web/static/css/es.css
+++ b/explainshell/web/static/css/es.css
@@ -172,6 +172,17 @@ a {
     font-size: 20px;
 }
 
+pre code.hljs {
+    background-color: transparent;
+    padding: 2px;
+}
+pre.highlightjs {
+    background-color: transparent;
+    margin: 2px;
+    padding: 0px;
+    border: none;
+}
+
 body[data-theme='dark'] {
     color: white;
 }
@@ -198,6 +209,9 @@ body[data-theme='dark'] {
     background-color: #222;
     color: white;
 }
+[data-theme='dark'] pre.highlightjs {
+    background-color: transparent;
+}
 
 [data-theme='dark'] .caret {
     /* this is probably a problem with the bootstrap theme */
diff --git a/explainshell/web/static/css/highlight.default.min.css b/explainshell/web/static/css/highlight.default.min.css
new file mode 100644
index 00000000..a75ea911
--- /dev/null
+++ b/explainshell/web/static/css/highlight.default.min.css
@@ -0,0 +1,9 @@
+/*!
+  Theme: Default
+  Description: Original highlight.js style
+  Author: (c) Ivan Sagalaev <maniac@softwaremaniacs.org>
+  Maintainer: @highlightjs/core-team
+  Website: https://highlightjs.org/
+  License: see project LICENSE
+  Touched: 2021
+*/pre code.hljs{display:block;overflow-x:auto;padding:1em}code.hljs{padding:3px 5px}.hljs{background:#f3f3f3;color:#444}.hljs-comment{color:#697070}.hljs-punctuation,.hljs-tag{color:#444a}.hljs-tag .hljs-attr,.hljs-tag .hljs-name{color:#444}.hljs-attribute,.hljs-doctag,.hljs-keyword,.hljs-meta .hljs-keyword,.hljs-name,.hljs-selector-tag{font-weight:700}.hljs-deletion,.hljs-number,.hljs-quote,.hljs-selector-class,.hljs-selector-id,.hljs-string,.hljs-template-tag,.hljs-type{color:#800}.hljs-section,.hljs-title{color:#800;font-weight:700}.hljs-link,.hljs-operator,.hljs-regexp,.hljs-selector-attr,.hljs-selector-pseudo,.hljs-symbol,.hljs-template-variable,.hljs-variable{color:#ab5656}.hljs-literal{color:#695}.hljs-addition,.hljs-built_in,.hljs-bullet,.hljs-code{color:#397300}.hljs-meta{color:#1f7199}.hljs-meta .hljs-string{color:#38a}.hljs-emphasis{font-style:italic}.hljs-strong{font-weight:700}
\ No newline at end of file
diff --git a/explainshell/web/static/css/hljs-atom-one-dark.min.css b/explainshell/web/static/css/hljs-atom-one-dark.min.css
new file mode 100644
index 00000000..5344ee38
--- /dev/null
+++ b/explainshell/web/static/css/hljs-atom-one-dark.min.css
@@ -0,0 +1 @@
+pre code.hljs{display:block;overflow-x:auto;padding:1em}code.hljs{padding:3px 5px}.hljs{color:#abb2bf;background:#282c34}.hljs-comment,.hljs-quote{color:#5c6370;font-style:italic}.hljs-doctag,.hljs-formula,.hljs-keyword{color:#c678dd}.hljs-deletion,.hljs-name,.hljs-section,.hljs-selector-tag,.hljs-subst{color:#e06c75}.hljs-literal{color:#56b6c2}.hljs-addition,.hljs-attribute,.hljs-meta .hljs-string,.hljs-regexp,.hljs-string{color:#98c379}.hljs-attr,.hljs-number,.hljs-selector-attr,.hljs-selector-class,.hljs-selector-pseudo,.hljs-template-variable,.hljs-type,.hljs-variable{color:#d19a66}.hljs-bullet,.hljs-link,.hljs-meta,.hljs-selector-id,.hljs-symbol,.hljs-title{color:#61aeee}.hljs-built_in,.hljs-class .hljs-title,.hljs-title.class_{color:#e6c07b}.hljs-emphasis{font-style:italic}.hljs-strong{font-weight:700}.hljs-link{text-decoration:underline}
\ No newline at end of file
diff --git a/explainshell/web/static/js/es.js b/explainshell/web/static/js/es.js
index 951a1b88..5cd20870 100644
--- a/explainshell/web/static/js/es.js
+++ b/explainshell/web/static/js/es.js
@@ -17,11 +17,19 @@ var themes = {
     default: '//cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/2.3.1/css/bootstrap.min.css',
     dark: '//maxcdn.bootstrapcdn.com/bootswatch/2.3.1/cyborg/bootstrap.min.css'
 };
+var hljs_themes = {
+    default: '//cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css',
+    dark: '//cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/atom-one-dark.min.css'
+}
 if (debug){
     themes = {
         default: '/static/css/bootstrap.min.css',
         dark: '/static/css/bootstrap-cyborg.min.css'
     };
+    hljs_themes = {
+        default: '/static/css/highlight.default.min.css',
+        dark: '/static/css/hljs-atom-one-dark.min.css'
+    };
 }
 
 var assignedcolors = {};
@@ -37,7 +45,7 @@ var docCookies = {
       return decodeURIComponent(document.cookie.replace(new RegExp("(?:(?:^|.*;)\\s*" + encodeURIComponent(sKey).replace(/[\-\.\+\*]/g, "\\$&") + "\\s*\\=\\s*([^;]*).*$)|^.*$"), "$1")) || null;
     },
     setItem: function (sKey, sValue, vEnd, sPath, sDomain, bSecure) {
-      if (!sKey || /^(?:expires|max\-age|path|domain|secure)$/i.test(sKey)) { return false; }
+      if (!sKey || /^(?:expires|max-age|path|domain|secure)$/i.test(sKey)) { return false; }
       var sExpires = "";
       if (vEnd) {
         switch (vEnd.constructor) {
@@ -1127,6 +1135,8 @@ function setTheme(theme) {
     console.log('setting theme to', theme);
 
     $("#bootstrapCSS").attr('href', themes[theme]);
+    $("#hljsCSS").attr('href', hljs_themes[theme]);
+
     $(document.body).attr('data-theme', theme);
     docCookies.setItem(themeCookieName, theme, Infinity, '/');
 }
diff --git a/explainshell/web/static/js/highlight.min.js b/explainshell/web/static/js/highlight.min.js
new file mode 100644
index 00000000..6d00bf09
--- /dev/null
+++ b/explainshell/web/static/js/highlight.min.js
@@ -0,0 +1,351 @@
+/*!
+  Highlight.js v11.10.0 (git: 366a8bd012)
+  (c) 2006-2024 Josh Goebel <hello@joshgoebel.com> and other contributors
+  License: BSD-3-Clause
+ */
+  var hljs=function(){"use strict";function e(t){
+    return t instanceof Map?t.clear=t.delete=t.set=()=>{
+    throw Error("map is read-only")}:t instanceof Set&&(t.add=t.clear=t.delete=()=>{
+    throw Error("set is read-only")
+    }),Object.freeze(t),Object.getOwnPropertyNames(t).forEach((n=>{
+    const i=t[n],s=typeof i;"object"!==s&&"function"!==s||Object.isFrozen(i)||e(i)
+    })),t}class t{constructor(e){
+    void 0===e.data&&(e.data={}),this.data=e.data,this.isMatchIgnored=!1}
+    ignoreMatch(){this.isMatchIgnored=!0}}function n(e){
+    return e.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;").replace(/'/g,"&#x27;")
+    }function i(e,...t){const n=Object.create(null);for(const t in e)n[t]=e[t]
+    ;return t.forEach((e=>{for(const t in e)n[t]=e[t]})),n}const s=e=>!!e.scope
+    ;class o{constructor(e,t){
+    this.buffer="",this.classPrefix=t.classPrefix,e.walk(this)}addText(e){
+    this.buffer+=n(e)}openNode(e){if(!s(e))return;const t=((e,{prefix:t})=>{
+    if(e.startsWith("language:"))return e.replace("language:","language-")
+    ;if(e.includes(".")){const n=e.split(".")
+    ;return[`${t}${n.shift()}`,...n.map(((e,t)=>`${e}${"_".repeat(t+1)}`))].join(" ")
+    }return`${t}${e}`})(e.scope,{prefix:this.classPrefix});this.span(t)}
+    closeNode(e){s(e)&&(this.buffer+="</span>")}value(){return this.buffer}span(e){
+    this.buffer+=`<span class="${e}">`}}const r=(e={})=>{const t={children:[]}
+    ;return Object.assign(t,e),t};class a{constructor(){
+    this.rootNode=r(),this.stack=[this.rootNode]}get top(){
+    return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){
+    this.top.children.push(e)}openNode(e){const t=r({scope:e})
+    ;this.add(t),this.stack.push(t)}closeNode(){
+    if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){
+    for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)}
+    walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,t){
+    return"string"==typeof t?e.addText(t):t.children&&(e.openNode(t),
+    t.children.forEach((t=>this._walk(e,t))),e.closeNode(t)),e}static _collapse(e){
+    "string"!=typeof e&&e.children&&(e.children.every((e=>"string"==typeof e))?e.children=[e.children.join("")]:e.children.forEach((e=>{
+    a._collapse(e)})))}}class c extends a{constructor(e){super(),this.options=e}
+    addText(e){""!==e&&this.add(e)}startScope(e){this.openNode(e)}endScope(){
+    this.closeNode()}__addSublanguage(e,t){const n=e.root
+    ;t&&(n.scope="language:"+t),this.add(n)}toHTML(){
+    return new o(this,this.options).value()}finalize(){
+    return this.closeAllNodes(),!0}}function l(e){
+    return e?"string"==typeof e?e:e.source:null}function g(e){return h("(?=",e,")")}
+    function u(e){return h("(?:",e,")*")}function d(e){return h("(?:",e,")?")}
+    function h(...e){return e.map((e=>l(e))).join("")}function f(...e){const t=(e=>{
+    const t=e[e.length-1]
+    ;return"object"==typeof t&&t.constructor===Object?(e.splice(e.length-1,1),t):{}
+    })(e);return"("+(t.capture?"":"?:")+e.map((e=>l(e))).join("|")+")"}
+    function p(e){return RegExp(e.toString()+"|").exec("").length-1}
+    const b=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./
+    ;function m(e,{joinWith:t}){let n=0;return e.map((e=>{n+=1;const t=n
+    ;let i=l(e),s="";for(;i.length>0;){const e=b.exec(i);if(!e){s+=i;break}
+    s+=i.substring(0,e.index),
+    i=i.substring(e.index+e[0].length),"\\"===e[0][0]&&e[1]?s+="\\"+(Number(e[1])+t):(s+=e[0],
+    "("===e[0]&&n++)}return s})).map((e=>`(${e})`)).join(t)}
+    const E="[a-zA-Z]\\w*",x="[a-zA-Z_]\\w*",w="\\b\\d+(\\.\\d+)?",y="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",_="\\b(0b[01]+)",O={
+    begin:"\\\\[\\s\\S]",relevance:0},v={scope:"string",begin:"'",end:"'",
+    illegal:"\\n",contains:[O]},k={scope:"string",begin:'"',end:'"',illegal:"\\n",
+    contains:[O]},N=(e,t,n={})=>{const s=i({scope:"comment",begin:e,end:t,
+    contains:[]},n);s.contains.push({scope:"doctag",
+    begin:"[ ]*(?=(TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):)",
+    end:/(TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):/,excludeBegin:!0,relevance:0})
+    ;const o=f("I","a","is","so","us","to","at","if","in","it","on",/[A-Za-z]+['](d|ve|re|ll|t|s|n)/,/[A-Za-z]+[-][a-z]+/,/[A-Za-z][a-z]{2,}/)
+    ;return s.contains.push({begin:h(/[ ]+/,"(",o,/[.]?[:]?([.][ ]|[ ])/,"){3}")}),s
+    },S=N("//","$"),M=N("/\\*","\\*/"),R=N("#","$");var j=Object.freeze({
+    __proto__:null,APOS_STRING_MODE:v,BACKSLASH_ESCAPE:O,BINARY_NUMBER_MODE:{
+    scope:"number",begin:_,relevance:0},BINARY_NUMBER_RE:_,COMMENT:N,
+    C_BLOCK_COMMENT_MODE:M,C_LINE_COMMENT_MODE:S,C_NUMBER_MODE:{scope:"number",
+    begin:y,relevance:0},C_NUMBER_RE:y,END_SAME_AS_BEGIN:e=>Object.assign(e,{
+    "on:begin":(e,t)=>{t.data._beginMatch=e[1]},"on:end":(e,t)=>{
+    t.data._beginMatch!==e[1]&&t.ignoreMatch()}}),HASH_COMMENT_MODE:R,IDENT_RE:E,
+    MATCH_NOTHING_RE:/\b\B/,METHOD_GUARD:{begin:"\\.\\s*"+x,relevance:0},
+    NUMBER_MODE:{scope:"number",begin:w,relevance:0},NUMBER_RE:w,
+    PHRASAL_WORDS_MODE:{
+    begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/
+    },QUOTE_STRING_MODE:k,REGEXP_MODE:{scope:"regexp",begin:/\/(?=[^/\n]*\/)/,
+    end:/\/[gimuy]*/,contains:[O,{begin:/\[/,end:/\]/,relevance:0,contains:[O]}]},
+    RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",
+    SHEBANG:(e={})=>{const t=/^#![ ]*\//
+    ;return e.binary&&(e.begin=h(t,/.*\b/,e.binary,/\b.*/)),i({scope:"meta",begin:t,
+    end:/$/,relevance:0,"on:begin":(e,t)=>{0!==e.index&&t.ignoreMatch()}},e)},
+    TITLE_MODE:{scope:"title",begin:E,relevance:0},UNDERSCORE_IDENT_RE:x,
+    UNDERSCORE_TITLE_MODE:{scope:"title",begin:x,relevance:0}});function A(e,t){
+    "."===e.input[e.index-1]&&t.ignoreMatch()}function I(e,t){
+    void 0!==e.className&&(e.scope=e.className,delete e.className)}function T(e,t){
+    t&&e.beginKeywords&&(e.begin="\\b("+e.beginKeywords.split(" ").join("|")+")(?!\\.)(?=\\b|\\s)",
+    e.__beforeBegin=A,e.keywords=e.keywords||e.beginKeywords,delete e.beginKeywords,
+    void 0===e.relevance&&(e.relevance=0))}function L(e,t){
+    Array.isArray(e.illegal)&&(e.illegal=f(...e.illegal))}function B(e,t){
+    if(e.match){
+    if(e.begin||e.end)throw Error("begin & end are not supported with match")
+    ;e.begin=e.match,delete e.match}}function P(e,t){
+    void 0===e.relevance&&(e.relevance=1)}const D=(e,t)=>{if(!e.beforeMatch)return
+    ;if(e.starts)throw Error("beforeMatch cannot be used with starts")
+    ;const n=Object.assign({},e);Object.keys(e).forEach((t=>{delete e[t]
+    })),e.keywords=n.keywords,e.begin=h(n.beforeMatch,g(n.begin)),e.starts={
+    relevance:0,contains:[Object.assign(n,{endsParent:!0})]
+    },e.relevance=0,delete n.beforeMatch
+    },H=["of","and","for","in","not","or","if","then","parent","list","value"],C="keyword"
+    ;function $(e,t,n=C){const i=Object.create(null)
+    ;return"string"==typeof e?s(n,e.split(" ")):Array.isArray(e)?s(n,e):Object.keys(e).forEach((n=>{
+    Object.assign(i,$(e[n],t,n))})),i;function s(e,n){
+    t&&(n=n.map((e=>e.toLowerCase()))),n.forEach((t=>{const n=t.split("|")
+    ;i[n[0]]=[e,U(n[0],n[1])]}))}}function U(e,t){
+    return t?Number(t):(e=>H.includes(e.toLowerCase()))(e)?0:1}const z={},W=e=>{
+    console.error(e)},X=(e,...t)=>{console.log("WARN: "+e,...t)},G=(e,t)=>{
+    z[`${e}/${t}`]||(console.log(`Deprecated as of ${e}. ${t}`),z[`${e}/${t}`]=!0)
+    },K=Error();function F(e,t,{key:n}){let i=0;const s=e[n],o={},r={}
+    ;for(let e=1;e<=t.length;e++)r[e+i]=s[e],o[e+i]=!0,i+=p(t[e-1])
+    ;e[n]=r,e[n]._emit=o,e[n]._multi=!0}function Z(e){(e=>{
+    e.scope&&"object"==typeof e.scope&&null!==e.scope&&(e.beginScope=e.scope,
+    delete e.scope)})(e),"string"==typeof e.beginScope&&(e.beginScope={
+    _wrap:e.beginScope}),"string"==typeof e.endScope&&(e.endScope={_wrap:e.endScope
+    }),(e=>{if(Array.isArray(e.begin)){
+    if(e.skip||e.excludeBegin||e.returnBegin)throw W("skip, excludeBegin, returnBegin not compatible with beginScope: {}"),
+    K
+    ;if("object"!=typeof e.beginScope||null===e.beginScope)throw W("beginScope must be object"),
+    K;F(e,e.begin,{key:"beginScope"}),e.begin=m(e.begin,{joinWith:""})}})(e),(e=>{
+    if(Array.isArray(e.end)){
+    if(e.skip||e.excludeEnd||e.returnEnd)throw W("skip, excludeEnd, returnEnd not compatible with endScope: {}"),
+    K
+    ;if("object"!=typeof e.endScope||null===e.endScope)throw W("endScope must be object"),
+    K;F(e,e.end,{key:"endScope"}),e.end=m(e.end,{joinWith:""})}})(e)}function V(e){
+    function t(t,n){
+    return RegExp(l(t),"m"+(e.case_insensitive?"i":"")+(e.unicodeRegex?"u":"")+(n?"g":""))
+    }class n{constructor(){
+    this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0}
+    addRule(e,t){
+    t.position=this.position++,this.matchIndexes[this.matchAt]=t,this.regexes.push([t,e]),
+    this.matchAt+=p(e)+1}compile(){0===this.regexes.length&&(this.exec=()=>null)
+    ;const e=this.regexes.map((e=>e[1]));this.matcherRe=t(m(e,{joinWith:"|"
+    }),!0),this.lastIndex=0}exec(e){this.matcherRe.lastIndex=this.lastIndex
+    ;const t=this.matcherRe.exec(e);if(!t)return null
+    ;const n=t.findIndex(((e,t)=>t>0&&void 0!==e)),i=this.matchIndexes[n]
+    ;return t.splice(0,n),Object.assign(t,i)}}class s{constructor(){
+    this.rules=[],this.multiRegexes=[],
+    this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){
+    if(this.multiRegexes[e])return this.multiRegexes[e];const t=new n
+    ;return this.rules.slice(e).forEach((([e,n])=>t.addRule(e,n))),
+    t.compile(),this.multiRegexes[e]=t,t}resumingScanAtSamePosition(){
+    return 0!==this.regexIndex}considerAll(){this.regexIndex=0}addRule(e,t){
+    this.rules.push([e,t]),"begin"===t.type&&this.count++}exec(e){
+    const t=this.getMatcher(this.regexIndex);t.lastIndex=this.lastIndex
+    ;let n=t.exec(e)
+    ;if(this.resumingScanAtSamePosition())if(n&&n.index===this.lastIndex);else{
+    const t=this.getMatcher(0);t.lastIndex=this.lastIndex+1,n=t.exec(e)}
+    return n&&(this.regexIndex+=n.position+1,
+    this.regexIndex===this.count&&this.considerAll()),n}}
+    if(e.compilerExtensions||(e.compilerExtensions=[]),
+    e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language.  See documentation.")
+    ;return e.classNameAliases=i(e.classNameAliases||{}),function n(o,r){const a=o
+    ;if(o.isCompiled)return a
+    ;[I,B,Z,D].forEach((e=>e(o,r))),e.compilerExtensions.forEach((e=>e(o,r))),
+    o.__beforeBegin=null,[T,L,P].forEach((e=>e(o,r))),o.isCompiled=!0;let c=null
+    ;return"object"==typeof o.keywords&&o.keywords.$pattern&&(o.keywords=Object.assign({},o.keywords),
+    c=o.keywords.$pattern,
+    delete o.keywords.$pattern),c=c||/\w+/,o.keywords&&(o.keywords=$(o.keywords,e.case_insensitive)),
+    a.keywordPatternRe=t(c,!0),
+    r&&(o.begin||(o.begin=/\B|\b/),a.beginRe=t(a.begin),o.end||o.endsWithParent||(o.end=/\B|\b/),
+    o.end&&(a.endRe=t(a.end)),
+    a.terminatorEnd=l(a.end)||"",o.endsWithParent&&r.terminatorEnd&&(a.terminatorEnd+=(o.end?"|":"")+r.terminatorEnd)),
+    o.illegal&&(a.illegalRe=t(o.illegal)),
+    o.contains||(o.contains=[]),o.contains=[].concat(...o.contains.map((e=>(e=>(e.variants&&!e.cachedVariants&&(e.cachedVariants=e.variants.map((t=>i(e,{
+    variants:null},t)))),e.cachedVariants?e.cachedVariants:q(e)?i(e,{
+    starts:e.starts?i(e.starts):null
+    }):Object.isFrozen(e)?i(e):e))("self"===e?o:e)))),o.contains.forEach((e=>{n(e,a)
+    })),o.starts&&n(o.starts,r),a.matcher=(e=>{const t=new s
+    ;return e.contains.forEach((e=>t.addRule(e.begin,{rule:e,type:"begin"
+    }))),e.terminatorEnd&&t.addRule(e.terminatorEnd,{type:"end"
+    }),e.illegal&&t.addRule(e.illegal,{type:"illegal"}),t})(a),a}(e)}function q(e){
+    return!!e&&(e.endsWithParent||q(e.starts))}class J extends Error{
+    constructor(e,t){super(e),this.name="HTMLInjectionError",this.html=t}}
+    const Y=n,Q=i,ee=Symbol("nomatch"),te=n=>{
+    const i=Object.create(null),s=Object.create(null),o=[];let r=!0
+    ;const a="Could not find the language '{}', did you forget to load/include a language module?",l={
+    disableAutodetect:!0,name:"Plain text",contains:[]};let p={
+    ignoreUnescapedHTML:!1,throwUnescapedHTML:!1,noHighlightRe:/^(no-?highlight)$/i,
+    languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-",
+    cssSelector:"pre code",languages:null,__emitter:c};function b(e){
+    return p.noHighlightRe.test(e)}function m(e,t,n){let i="",s=""
+    ;"object"==typeof t?(i=e,
+    n=t.ignoreIllegals,s=t.language):(G("10.7.0","highlight(lang, code, ...args) has been deprecated."),
+    G("10.7.0","Please use highlight(code, options) instead.\nhttps://github.com/highlightjs/highlight.js/issues/2277"),
+    s=e,i=t),void 0===n&&(n=!0);const o={code:i,language:s};N("before:highlight",o)
+    ;const r=o.result?o.result:E(o.language,o.code,n)
+    ;return r.code=o.code,N("after:highlight",r),r}function E(e,n,s,o){
+    const c=Object.create(null);function l(){if(!N.keywords)return void M.addText(R)
+    ;let e=0;N.keywordPatternRe.lastIndex=0;let t=N.keywordPatternRe.exec(R),n=""
+    ;for(;t;){n+=R.substring(e,t.index)
+    ;const s=_.case_insensitive?t[0].toLowerCase():t[0],o=(i=s,N.keywords[i]);if(o){
+    const[e,i]=o
+    ;if(M.addText(n),n="",c[s]=(c[s]||0)+1,c[s]<=7&&(j+=i),e.startsWith("_"))n+=t[0];else{
+    const n=_.classNameAliases[e]||e;u(t[0],n)}}else n+=t[0]
+    ;e=N.keywordPatternRe.lastIndex,t=N.keywordPatternRe.exec(R)}var i
+    ;n+=R.substring(e),M.addText(n)}function g(){null!=N.subLanguage?(()=>{
+    if(""===R)return;let e=null;if("string"==typeof N.subLanguage){
+    if(!i[N.subLanguage])return void M.addText(R)
+    ;e=E(N.subLanguage,R,!0,S[N.subLanguage]),S[N.subLanguage]=e._top
+    }else e=x(R,N.subLanguage.length?N.subLanguage:null)
+    ;N.relevance>0&&(j+=e.relevance),M.__addSublanguage(e._emitter,e.language)
+    })():l(),R=""}function u(e,t){
+    ""!==e&&(M.startScope(t),M.addText(e),M.endScope())}function d(e,t){let n=1
+    ;const i=t.length-1;for(;n<=i;){if(!e._emit[n]){n++;continue}
+    const i=_.classNameAliases[e[n]]||e[n],s=t[n];i?u(s,i):(R=s,l(),R=""),n++}}
+    function h(e,t){
+    return e.scope&&"string"==typeof e.scope&&M.openNode(_.classNameAliases[e.scope]||e.scope),
+    e.beginScope&&(e.beginScope._wrap?(u(R,_.classNameAliases[e.beginScope._wrap]||e.beginScope._wrap),
+    R=""):e.beginScope._multi&&(d(e.beginScope,t),R="")),N=Object.create(e,{parent:{
+    value:N}}),N}function f(e,n,i){let s=((e,t)=>{const n=e&&e.exec(t)
+    ;return n&&0===n.index})(e.endRe,i);if(s){if(e["on:end"]){const i=new t(e)
+    ;e["on:end"](n,i),i.isMatchIgnored&&(s=!1)}if(s){
+    for(;e.endsParent&&e.parent;)e=e.parent;return e}}
+    if(e.endsWithParent)return f(e.parent,n,i)}function b(e){
+    return 0===N.matcher.regexIndex?(R+=e[0],1):(T=!0,0)}function m(e){
+    const t=e[0],i=n.substring(e.index),s=f(N,e,i);if(!s)return ee;const o=N
+    ;N.endScope&&N.endScope._wrap?(g(),
+    u(t,N.endScope._wrap)):N.endScope&&N.endScope._multi?(g(),
+    d(N.endScope,e)):o.skip?R+=t:(o.returnEnd||o.excludeEnd||(R+=t),
+    g(),o.excludeEnd&&(R=t));do{
+    N.scope&&M.closeNode(),N.skip||N.subLanguage||(j+=N.relevance),N=N.parent
+    }while(N!==s.parent);return s.starts&&h(s.starts,e),o.returnEnd?0:t.length}
+    let w={};function y(i,o){const a=o&&o[0];if(R+=i,null==a)return g(),0
+    ;if("begin"===w.type&&"end"===o.type&&w.index===o.index&&""===a){
+    if(R+=n.slice(o.index,o.index+1),!r){const t=Error(`0 width match regex (${e})`)
+    ;throw t.languageName=e,t.badRule=w.rule,t}return 1}
+    if(w=o,"begin"===o.type)return(e=>{
+    const n=e[0],i=e.rule,s=new t(i),o=[i.__beforeBegin,i["on:begin"]]
+    ;for(const t of o)if(t&&(t(e,s),s.isMatchIgnored))return b(n)
+    ;return i.skip?R+=n:(i.excludeBegin&&(R+=n),
+    g(),i.returnBegin||i.excludeBegin||(R=n)),h(i,e),i.returnBegin?0:n.length})(o)
+    ;if("illegal"===o.type&&!s){
+    const e=Error('Illegal lexeme "'+a+'" for mode "'+(N.scope||"<unnamed>")+'"')
+    ;throw e.mode=N,e}if("end"===o.type){const e=m(o);if(e!==ee)return e}
+    if("illegal"===o.type&&""===a)return 1
+    ;if(I>1e5&&I>3*o.index)throw Error("potential infinite loop, way more iterations than matches")
+    ;return R+=a,a.length}const _=O(e)
+    ;if(!_)throw W(a.replace("{}",e)),Error('Unknown language: "'+e+'"')
+    ;const v=V(_);let k="",N=o||v;const S={},M=new p.__emitter(p);(()=>{const e=[]
+    ;for(let t=N;t!==_;t=t.parent)t.scope&&e.unshift(t.scope)
+    ;e.forEach((e=>M.openNode(e)))})();let R="",j=0,A=0,I=0,T=!1;try{
+    if(_.__emitTokens)_.__emitTokens(n,M);else{for(N.matcher.considerAll();;){
+    I++,T?T=!1:N.matcher.considerAll(),N.matcher.lastIndex=A
+    ;const e=N.matcher.exec(n);if(!e)break;const t=y(n.substring(A,e.index),e)
+    ;A=e.index+t}y(n.substring(A))}return M.finalize(),k=M.toHTML(),{language:e,
+    value:k,relevance:j,illegal:!1,_emitter:M,_top:N}}catch(t){
+    if(t.message&&t.message.includes("Illegal"))return{language:e,value:Y(n),
+    illegal:!0,relevance:0,_illegalBy:{message:t.message,index:A,
+    context:n.slice(A-100,A+100),mode:t.mode,resultSoFar:k},_emitter:M};if(r)return{
+    language:e,value:Y(n),illegal:!1,relevance:0,errorRaised:t,_emitter:M,_top:N}
+    ;throw t}}function x(e,t){t=t||p.languages||Object.keys(i);const n=(e=>{
+    const t={value:Y(e),illegal:!1,relevance:0,_top:l,_emitter:new p.__emitter(p)}
+    ;return t._emitter.addText(e),t})(e),s=t.filter(O).filter(k).map((t=>E(t,e,!1)))
+    ;s.unshift(n);const o=s.sort(((e,t)=>{
+    if(e.relevance!==t.relevance)return t.relevance-e.relevance
+    ;if(e.language&&t.language){if(O(e.language).supersetOf===t.language)return 1
+    ;if(O(t.language).supersetOf===e.language)return-1}return 0})),[r,a]=o,c=r
+    ;return c.secondBest=a,c}function w(e){let t=null;const n=(e=>{
+    let t=e.className+" ";t+=e.parentNode?e.parentNode.className:""
+    ;const n=p.languageDetectRe.exec(t);if(n){const t=O(n[1])
+    ;return t||(X(a.replace("{}",n[1])),
+    X("Falling back to no-highlight mode for this block.",e)),t?n[1]:"no-highlight"}
+    return t.split(/\s+/).find((e=>b(e)||O(e)))})(e);if(b(n))return
+    ;if(N("before:highlightElement",{el:e,language:n
+    }),e.dataset.highlighted)return void console.log("Element previously highlighted. To highlight again, first unset `dataset.highlighted`.",e)
+    ;if(e.children.length>0&&(p.ignoreUnescapedHTML||(console.warn("One of your code blocks includes unescaped HTML. This is a potentially serious security risk."),
+    console.warn("https://github.com/highlightjs/highlight.js/wiki/security"),
+    console.warn("The element with unescaped HTML:"),
+    console.warn(e)),p.throwUnescapedHTML))throw new J("One of your code blocks includes unescaped HTML.",e.innerHTML)
+    ;t=e;const i=t.textContent,o=n?m(i,{language:n,ignoreIllegals:!0}):x(i)
+    ;e.innerHTML=o.value,e.dataset.highlighted="yes",((e,t,n)=>{const i=t&&s[t]||n
+    ;e.classList.add("hljs"),e.classList.add("language-"+i)
+    })(e,n,o.language),e.result={language:o.language,re:o.relevance,
+    relevance:o.relevance},o.secondBest&&(e.secondBest={
+    language:o.secondBest.language,relevance:o.secondBest.relevance
+    }),N("after:highlightElement",{el:e,result:o,text:i})}let y=!1;function _(){
+    "loading"!==document.readyState?document.querySelectorAll(p.cssSelector).forEach(w):y=!0
+    }function O(e){return e=(e||"").toLowerCase(),i[e]||i[s[e]]}
+    function v(e,{languageName:t}){"string"==typeof e&&(e=[e]),e.forEach((e=>{
+    s[e.toLowerCase()]=t}))}function k(e){const t=O(e)
+    ;return t&&!t.disableAutodetect}function N(e,t){const n=e;o.forEach((e=>{
+    e[n]&&e[n](t)}))}
+    "undefined"!=typeof window&&window.addEventListener&&window.addEventListener("DOMContentLoaded",(()=>{
+    y&&_()}),!1),Object.assign(n,{highlight:m,highlightAuto:x,highlightAll:_,
+    highlightElement:w,
+    highlightBlock:e=>(G("10.7.0","highlightBlock will be removed entirely in v12.0"),
+    G("10.7.0","Please use highlightElement now."),w(e)),configure:e=>{p=Q(p,e)},
+    initHighlighting:()=>{
+    _(),G("10.6.0","initHighlighting() deprecated.  Use highlightAll() now.")},
+    initHighlightingOnLoad:()=>{
+    _(),G("10.6.0","initHighlightingOnLoad() deprecated.  Use highlightAll() now.")
+    },registerLanguage:(e,t)=>{let s=null;try{s=t(n)}catch(t){
+    if(W("Language definition for '{}' could not be registered.".replace("{}",e)),
+    !r)throw t;W(t),s=l}
+    s.name||(s.name=e),i[e]=s,s.rawDefinition=t.bind(null,n),s.aliases&&v(s.aliases,{
+    languageName:e})},unregisterLanguage:e=>{delete i[e]
+    ;for(const t of Object.keys(s))s[t]===e&&delete s[t]},
+    listLanguages:()=>Object.keys(i),getLanguage:O,registerAliases:v,
+    autoDetection:k,inherit:Q,addPlugin:e=>{(e=>{
+    e["before:highlightBlock"]&&!e["before:highlightElement"]&&(e["before:highlightElement"]=t=>{
+    e["before:highlightBlock"](Object.assign({block:t.el},t))
+    }),e["after:highlightBlock"]&&!e["after:highlightElement"]&&(e["after:highlightElement"]=t=>{
+    e["after:highlightBlock"](Object.assign({block:t.el},t))})})(e),o.push(e)},
+    removePlugin:e=>{const t=o.indexOf(e);-1!==t&&o.splice(t,1)}}),n.debugMode=()=>{
+    r=!1},n.safeMode=()=>{r=!0},n.versionString="11.10.0",n.regex={concat:h,
+    lookahead:g,either:f,optional:d,anyNumberOfTimes:u}
+    ;for(const t in j)"object"==typeof j[t]&&e(j[t]);return Object.assign(n,j),n
+    },ne=te({});return ne.newInstance=()=>te({}),ne}()
+    ;"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs);/*! `bash` grammar compiled for Highlight.js 11.10.0 */
+    (()=>{var e=(()=>{"use strict";return e=>{const s=e.regex,t={},n={begin:/\$\{/,
+    end:/\}/,contains:["self",{begin:/:-/,contains:[t]}]};Object.assign(t,{
+    className:"variable",variants:[{
+    begin:s.concat(/\$[\w\d#@][\w\d_]*/,"(?![\\w\\d])(?![$])")},n]});const a={
+    className:"subst",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]
+    },i=e.inherit(e.COMMENT(),{match:[/(^|\s)/,/#.*$/],scope:{2:"comment"}}),c={
+    begin:/<<-?\s*(?=\w+)/,starts:{contains:[e.END_SAME_AS_BEGIN({begin:/(\w+)/,
+    end:/(\w+)/,className:"string"})]}},o={className:"string",begin:/"/,end:/"/,
+    contains:[e.BACKSLASH_ESCAPE,t,a]};a.contains.push(o);const r={begin:/\$?\(\(/,
+    end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},e.NUMBER_MODE,t]
+    },l=e.SHEBANG({binary:"(fish|bash|zsh|sh|csh|ksh|tcsh|dash|scsh)",relevance:10
+    }),m={className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0,
+    contains:[e.inherit(e.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0};return{
+    name:"Bash",aliases:["sh","zsh"],keywords:{$pattern:/\b[a-z][a-z0-9._-]+\b/,
+    keyword:["if","then","else","elif","fi","for","while","until","in","do","done","case","esac","function","select"],
+    literal:["true","false"],
+    built_in:["break","cd","continue","eval","exec","exit","export","getopts","hash","pwd","readonly","return","shift","test","times","trap","umask","unset","alias","bind","builtin","caller","command","declare","echo","enable","help","let","local","logout","mapfile","printf","read","readarray","source","sudo","type","typeset","ulimit","unalias","set","shopt","autoload","bg","bindkey","bye","cap","chdir","clone","comparguments","compcall","compctl","compdescribe","compfiles","compgroups","compquote","comptags","comptry","compvalues","dirs","disable","disown","echotc","echoti","emulate","fc","fg","float","functions","getcap","getln","history","integer","jobs","kill","limit","log","noglob","popd","print","pushd","pushln","rehash","sched","setcap","setopt","stat","suspend","ttyctl","unfunction","unhash","unlimit","unsetopt","vared","wait","whence","where","which","zcompile","zformat","zftp","zle","zmodload","zparseopts","zprof","zpty","zregexparse","zsocket","zstyle","ztcp","chcon","chgrp","chown","chmod","cp","dd","df","dir","dircolors","ln","ls","mkdir","mkfifo","mknod","mktemp","mv","realpath","rm","rmdir","shred","sync","touch","truncate","vdir","b2sum","base32","base64","cat","cksum","comm","csplit","cut","expand","fmt","fold","head","join","md5sum","nl","numfmt","od","paste","ptx","pr","sha1sum","sha224sum","sha256sum","sha384sum","sha512sum","shuf","sort","split","sum","tac","tail","tr","tsort","unexpand","uniq","wc","arch","basename","chroot","date","dirname","du","echo","env","expr","factor","groups","hostid","id","link","logname","nice","nohup","nproc","pathchk","pinky","printenv","printf","pwd","readlink","runcon","seq","sleep","stat","stdbuf","stty","tee","test","timeout","tty","uname","unlink","uptime","users","who","whoami","yes"]
+    },contains:[l,e.SHEBANG(),m,r,i,c,{match:/(\/[a-z._-]+)+/},o,{match:/\\"/},{
+    className:"string",begin:/'/,end:/'/},{match:/\\'/},t]}}})()
+    ;hljs.registerLanguage("bash",e)})();
+
+/*! `bash` grammar compiled for Highlight.js 11.10.0 */
+(()=>{var e=(()=>{"use strict";return e=>{const s=e.regex,t={},n={begin:/\$\{/,
+  end:/\}/,contains:["self",{begin:/:-/,contains:[t]}]};Object.assign(t,{
+  className:"variable",variants:[{
+  begin:s.concat(/\$[\w\d#@][\w\d_]*/,"(?![\\w\\d])(?![$])")},n]});const a={
+  className:"subst",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]
+  },i=e.inherit(e.COMMENT(),{match:[/(^|\s)/,/#.*$/],scope:{2:"comment"}}),c={
+  begin:/<<-?\s*(?=\w+)/,starts:{contains:[e.END_SAME_AS_BEGIN({begin:/(\w+)/,
+  end:/(\w+)/,className:"string"})]}},o={className:"string",begin:/"/,end:/"/,
+  contains:[e.BACKSLASH_ESCAPE,t,a]};a.contains.push(o);const r={begin:/\$?\(\(/,
+  end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},e.NUMBER_MODE,t]
+  },l=e.SHEBANG({binary:"(fish|bash|zsh|sh|csh|ksh|tcsh|dash|scsh)",relevance:10
+  }),m={className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0,
+  contains:[e.inherit(e.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0};return{
+  name:"Bash",aliases:["sh","zsh"],keywords:{$pattern:/\b[a-z][a-z0-9._-]+\b/,
+  keyword:["if","then","else","elif","fi","for","while","until","in","do","done","case","esac","function","select"],
+  literal:["true","false"],
+  built_in:["break","cd","continue","eval","exec","exit","export","getopts","hash","pwd","readonly","return","shift","test","times","trap","umask","unset","alias","bind","builtin","caller","command","declare","echo","enable","help","let","local","logout","mapfile","printf","read","readarray","source","sudo","type","typeset","ulimit","unalias","set","shopt","autoload","bg","bindkey","bye","cap","chdir","clone","comparguments","compcall","compctl","compdescribe","compfiles","compgroups","compquote","comptags","comptry","compvalues","dirs","disable","disown","echotc","echoti","emulate","fc","fg","float","functions","getcap","getln","history","integer","jobs","kill","limit","log","noglob","popd","print","pushd","pushln","rehash","sched","setcap","setopt","stat","suspend","ttyctl","unfunction","unhash","unlimit","unsetopt","vared","wait","whence","where","which","zcompile","zformat","zftp","zle","zmodload","zparseopts","zprof","zpty","zregexparse","zsocket","zstyle","ztcp","chcon","chgrp","chown","chmod","cp","dd","df","dir","dircolors","ln","ls","mkdir","mkfifo","mknod","mktemp","mv","realpath","rm","rmdir","shred","sync","touch","truncate","vdir","b2sum","base32","base64","cat","cksum","comm","csplit","cut","expand","fmt","fold","head","join","md5sum","nl","numfmt","od","paste","ptx","pr","sha1sum","sha224sum","sha256sum","sha384sum","sha512sum","shuf","sort","split","sum","tac","tail","tr","tsort","unexpand","uniq","wc","arch","basename","chroot","date","dirname","du","echo","env","expr","factor","groups","hostid","id","link","logname","nice","nohup","nproc","pathchk","pinky","printenv","printf","pwd","readlink","runcon","seq","sleep","stat","stdbuf","stty","tee","test","timeout","tty","uname","unlink","uptime","users","who","whoami","yes"]
+  },contains:[l,e.SHEBANG(),m,r,i,c,{match:/(\/[a-z._-]+)+/},o,{match:/\\"/},{
+  className:"string",begin:/'/,end:/'/},{match:/\\'/},t]}}})()
+  ;hljs.registerLanguage("bash",e)})();
\ No newline at end of file
diff --git a/explainshell/web/templates/base.html b/explainshell/web/templates/base.html
index f2320767..b41cacf8 100644
--- a/explainshell/web/templates/base.html
+++ b/explainshell/web/templates/base.html
@@ -7,15 +7,19 @@
 {%- if config.DEBUG %}
     {% if request.cookies.get('theme', 'default') == 'default' %}
         <link href="/static/css/bootstrap.min.css" rel="stylesheet" media="screen" id="bootstrapCSS">
+        <link href="/static/css/highlight.default.min.css" rel="stylesheet" media="screen" id="hljsCSS">
     {% else %}
         <link href="/static/css/bootstrap-cyborg.min.css" rel="stylesheet" media="screen" id="bootstrapCSS">
+        <link href="/static/css/hljs-atom-one-dark.min.css" rel="stylesheet" media="screen" id="hljsCSS">
     {% endif %}
         <link href="/static/css/font-awesome.min.css" rel="stylesheet" media="screen">
 {% else %}
     {% if request.cookies.get('theme', 'default') == 'default' %}
         <link href="//cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/2.3.1/css/bootstrap.min.css" rel="stylesheet" media="screen" id="bootstrapCSS">
+        <link href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/default.min.css" rel="stylesheet">
     {% else %}
         <link href="//maxcdn.bootstrapcdn.com/bootswatch/2.3.1/cyborg/bootstrap.min.css" rel="stylesheet" media="screen" id="bootstrapCSS">
+        <link href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/atom-one-dark.min.css" rel="stylesheet">
     {% endif %}
         <link href="//cdnjs.cloudflare.com/ajax/libs/font-awesome/3.2.1/css/font-awesome.min.css" rel="stylesheet">
 {%- endif %}
@@ -59,10 +63,12 @@
         <script src="/static/js/jquery.js"></script>
         <script src="/static/js/underscore.js"></script>
         <script src="/static/js/bootstrap.min.js"></script>
+        <script src="/static/js/highlight.min.js"></script>
 {% else %}
         <script src="//cdnjs.cloudflare.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
         <script src="//cdnjs.cloudflare.com/ajax/libs/underscore.js/1.4.4/underscore-min.js"></script>
         <script src="//cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/2.3.1/js/bootstrap.min.js"></script>
+        <script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
 {%- endif %}
         <script src="/static/js/es.js"></script>
         {%- block js %}{% endblock %}
diff --git a/explainshell/web/templates/errors/parsingerror.html b/explainshell/web/templates/errors/parsingerror.html
index ef4eb97f..4142f6ed 100644
--- a/explainshell/web/templates/errors/parsingerror.html
+++ b/explainshell/web/templates/errors/parsingerror.html
@@ -1,6 +1,6 @@
 {% extends "errors/error.html" %}
 {% block message -%}
 <div class="push">{{ e }}</div>
-<pre style="overflow: auto; white-space: pre; word-wrap: normal;">{{ e.s.decode('latin1')|e }}
+<pre style="overflow: auto; white-space: pre; word-wrap: normal;">{{ e.s|e }}
 {{ ' '*e.position }}<b>^</b></pre>
 {%- endblock %}
diff --git a/explainshell/web/templates/index.html b/explainshell/web/templates/index.html
index c75dcbd7..bd96a85a 100644
--- a/explainshell/web/templates/index.html
+++ b/explainshell/web/templates/index.html
@@ -15,7 +15,7 @@
             <div class="text-center">
                 <div class="input-append">
                     <form action='/explain' method="get">
-                        <input id='explain' class="span10" type="text" name="cmd" autocapitalize="off" autocorrect="off"></input>
+                        <input id='explain' class="span10" type="text" name="cmd" autocapitalize="off" autocorrect="off" autofocus></input>
                         <button class="btn" type="submit">EXPLAIN</button>
                     </form>
                 </div>
@@ -40,6 +40,8 @@ <h3>examples</h3>
             {%- endblock %}
 {% block js -%}
         <script type="text/javascript">
-            $("#explain").focus();
+            if(document.querySelectorAll("code").length > 0){
+                hljs.highlightAll();
+            }
         </script>
 {%- endblock %}
diff --git a/explainshell/web/templates/macros.html b/explainshell/web/templates/macros.html
index befaa2ef..94bc4f85 100644
--- a/explainshell/web/templates/macros.html
+++ b/explainshell/web/templates/macros.html
@@ -42,5 +42,5 @@
 {%- endmacro %}
 
 {%- macro examplebullet(cmd) -%}
-<li><a href='/explain?cmd={{ cmd|urlencode }}'>{{ cmd|e }}</a></li>
+<li><a href='/explain?cmd={{ cmd|urlencode }}'><pre class="highlightjs"><code class="language-bash">{{ cmd|e }}</code></pre></a></li>
 {%- endmacro %}

From 423315a67044345f97d6599e87a9c17edcd4042b Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sun, 15 Sep 2024 23:13:56 +0000
Subject: [PATCH 12/19] fix: replace inbuilt logging with loguru based helper
 scripts

---
 explainshell/config.py                     | 25 ----------
 explainshell/logger/__init__.py            |  0
 explainshell/logger/logger_helper.py       | 53 ++++++++++++++++++++++
 explainshell/logger/logging_interceptor.py | 32 +++++++++++++
 requirements.txt                           |  1 +
 runserver.py                               |  9 ++--
 tools/shellbuiltins.py                     | 13 ++++--
 7 files changed, 100 insertions(+), 33 deletions(-)
 create mode 100644 explainshell/logger/__init__.py
 create mode 100644 explainshell/logger/logger_helper.py
 create mode 100644 explainshell/logger/logging_interceptor.py

diff --git a/explainshell/config.py b/explainshell/config.py
index 91723f1f..ed9eb88c 100644
--- a/explainshell/config.py
+++ b/explainshell/config.py
@@ -12,28 +12,3 @@
 HOST_IP = os.getenv("HOST_IP", "")
 MONGO_URI = os.getenv("MONGO_URI", "mongodb://localhost")
 DEBUG = True
-
-LOGGING_DICT = {
-    "version": 1,
-    "disable_existing_loggers": False,
-    "formatters": {
-        "standard": {"format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"},
-    },
-    "handlers": {
-        "console": {
-            "level": "INFO",
-            "class": "logging.StreamHandler",
-            "formatter": "standard",
-        },
-        "file": {
-            "class": "logging.FileHandler",
-            "level": "INFO",
-            "formatter": "standard",
-            "filename": "application.log",
-            "mode": "a",
-        },
-    },
-    "loggers": {
-        "explainshell": {"handlers": ["console"], "level": "INFO", "propagate": False}
-    },
-}
diff --git a/explainshell/logger/__init__.py b/explainshell/logger/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/explainshell/logger/logger_helper.py b/explainshell/logger/logger_helper.py
new file mode 100644
index 00000000..dc83916a
--- /dev/null
+++ b/explainshell/logger/logger_helper.py
@@ -0,0 +1,53 @@
+"""
+This module provides a helper function for logging using the loguru library.
+
+The logger_helper module sets up a logger with a rotated log file and also logs to standard output.
+"""
+
+import logging
+import sys
+from pathlib import Path
+
+import loguru
+
+# intercept log entries handled by python's 'logging' module
+# and redirect them to loguru logger
+from explainshell.logger.logging_interceptor import InterceptHandler
+
+parent_dir = Path(__file__).parent.parent.parent
+logs_dir = parent_dir / "logs"
+# create logs directory if it does not exist
+logs_dir.mkdir(exist_ok=True)
+
+logger = loguru.logger
+logger.remove()
+
+
+def level_filter():
+    """
+    Filter function to exclude DEBUG and SUCCESS log levels from being logged to standard output.
+    """
+    def is_level(record):
+        return record["level"].name not in ["DEBUG", "SUCCESS"]
+    return is_level
+
+
+# init rotated log file
+logger.add(
+    logs_dir / "debug.log",
+    rotation="10 MB",
+    retention="7 days",
+    backtrace=True,
+    colorize=False,
+    catch=True,
+    delay=True,
+    diagnose=True,
+    enqueue=True,
+)
+
+
+# also log to standard output
+logger.add(sys.stdout, colorize=True, filter=level_filter())
+
+# activate logging and redirect all logs to loguru logger
+logging.basicConfig(handlers=[InterceptHandler()], level=logging.DEBUG, force=True)
diff --git a/explainshell/logger/logging_interceptor.py b/explainshell/logger/logging_interceptor.py
new file mode 100644
index 00000000..7701bce6
--- /dev/null
+++ b/explainshell/logger/logging_interceptor.py
@@ -0,0 +1,32 @@
+import logging
+import sys
+
+import loguru
+
+logger = loguru.logger
+
+
+class InterceptHandler(logging.Handler):
+    """
+    intercept log messages logged with the logging module
+
+    source: https://loguru.readthedocs.io/en/stable/overview.html#entirely-compatible-with-standard-logging
+    also see: https://stackoverflow.com/a/70620198
+    """
+
+    def emit(self, record):
+        # Get corresponding Loguru level if it exists.
+        try:
+            level = logger.level(record.levelname).name
+        except ValueError:
+            level = record.levelno
+
+        # Find caller from where originated the logged message.
+        frame, depth = sys._getframe(6), 6
+        while frame and frame.f_code.co_filename == logging.__file__:
+            frame = frame.f_back
+            depth += 1
+
+        logger.opt(depth=depth, exception=record.exc_info).log(
+            level, record.getMessage()
+        )
diff --git a/requirements.txt b/requirements.txt
index eb5ced45..7a6f6023 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,4 @@ nltk==3.9.1
 nose==1.3.7
 pymongo==4.8.0
 bashlex==0.18
+loguru==0.7.2
diff --git a/runserver.py b/runserver.py
index 730114e9..a46f13cb 100644
--- a/runserver.py
+++ b/runserver.py
@@ -1,12 +1,15 @@
-import logging.config
+import logging
 
 from explainshell import config
 from explainshell.web import app
+from explainshell.logger.logger_helper import logger
+from explainshell.logger.logging_interceptor import InterceptHandler
 
 
-logging.config.dictConfig(config.LOGGING_DICT)
-
 if __name__ == '__main__':
+    # activate logging and redirect all logs to loguru
+    logging.basicConfig(handlers=[InterceptHandler()], level=logging.DEBUG, force=True)
+
     if len(config.HOST_IP) > 1:
         app.run(debug=config.DEBUG, host=config.HOST_IP)
     else:
diff --git a/tools/shellbuiltins.py b/tools/shellbuiltins.py
index 179d0f5a..c937d4f6 100644
--- a/tools/shellbuiltins.py
+++ b/tools/shellbuiltins.py
@@ -6,8 +6,12 @@
 writing these down and adding them.
 """
 
+import logging
 import textwrap
+
 from explainshell import store, config
+from explainshell.logger.logger_helper import logger
+from explainshell.logger.logging_interceptor import InterceptHandler
 
 sp = store.Paragraph
 so = store.Option
@@ -241,10 +245,9 @@ def _add(names, synopsis, options):
 )
 
 if __name__ == "__main__":
-    import logging.config
-
-    logging.config.dictConfig(config.LOGGING_DICT)
+    # activate logging and redirect all logs to loguru
+    logging.basicConfig(handlers=[InterceptHandler()], level=logging.DEBUG, force=True)
 
     s = store.Store("explainshell", config.MONGO_URI)
-    for m in BUILTINS.itervalues():
-        s.addmanpage(m)
+    for m in BUILTINS.values():
+        s.add_manpage(m)

From b807e60ce0d8b1c2846af0dbff3413ee6cf991bd Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sun, 15 Sep 2024 23:25:16 +0000
Subject: [PATCH 13/19] fix: fixed bugs in displany and ingestion of contents

---
 explainshell/algo/classifier.py | 27 +++++------
 explainshell/manager.py         |  6 +--
 explainshell/manpage.py         | 39 +++++++++++-----
 explainshell/matcher.py         | 18 ++++----
 explainshell/options.py         |  2 +-
 explainshell/store.py           | 79 ++++++++++++++++++---------------
 explainshell/web/helpers.py     |  2 +-
 explainshell/web/views.py       | 68 ++++++++++++++++------------
 8 files changed, 139 insertions(+), 102 deletions(-)

diff --git a/explainshell/algo/classifier.py b/explainshell/algo/classifier.py
index dd0a3436..da3fac67 100644
--- a/explainshell/algo/classifier.py
+++ b/explainshell/algo/classifier.py
@@ -14,7 +14,8 @@
 
 def get_features(paragraph):
     features = {}
-    p_text = paragraph.cleantext()
+    p_text = paragraph.clean_text()
+    logger.debug(f"length of p_text: {len(p_text)}")
     assert p_text
 
     features["starts_with_hyphen"] = algo.features.starts_with_hyphen(p_text)
@@ -46,7 +47,7 @@ def train(self):
         if self.classifier:
             return
 
-        man_pages = self.store.trainingset()
+        man_pages = self.store.training_set()
 
         # flatten the manpages so we get a list of (manpage-name, paragraph)
         def flatten_manpages(manpage):
@@ -58,19 +59,19 @@ def flatten_manpages(manpage):
         paragraphs = itertools.chain(*[flatten_manpages(m) for m in man_pages])
         training = list(paragraphs)
 
-        negids = [p for p in training if not p.is_option]
-        posids = [p for p in training if p.is_option]
+        neg_ids = [p for p in training if not p.is_option]
+        pos_ids = [p for p in training if p.is_option]
 
-        negfeats = [(get_features(p), False) for p in negids]
-        posfeats = [(get_features(p), True) for p in posids]
+        neg_feats = [(get_features(p), False) for p in neg_ids]
+        pos_feats = [(get_features(p), True) for p in pos_ids]
 
-        negcutoff = len(negfeats) * 3 / 4
-        poscutoff = len(posfeats) * 3 / 4
+        neg_cutoff = int(len(neg_feats) * 3 / 4)
+        pos_cutoff = int(len(pos_feats) * 3 / 4)
 
-        trainfeats = negfeats[:negcutoff] + posfeats[:poscutoff]
-        self.testfeats = negfeats[negcutoff:] + posfeats[poscutoff:]
+        train_feats = neg_feats[:neg_cutoff] + pos_feats[:pos_cutoff]
+        self.test_feats = neg_feats[neg_cutoff:] + pos_feats[pos_cutoff:]
 
-        logger.info("train on %d instances", len(trainfeats))
+        logger.info("train on %d instances", len(train_feats))
 
         if self.algo == "maxent":
             c = nltk.classify.maxent.MaxentClassifier
@@ -79,14 +80,14 @@ def flatten_manpages(manpage):
         else:
             raise ValueError("unknown classifier")
 
-        self.classifier = c.train(trainfeats, **self.classifier_args)
+        self.classifier = c.train(train_feats, **self.classifier_args)
 
     def evaluate(self):
         self.train()
         ref_sets = collections.defaultdict(set)
         test_sets = collections.defaultdict(set)
 
-        for i, (feats, label) in enumerate(self.testfeats):
+        for i, (feats, label) in enumerate(self.test_feats):
             ref_sets[label].add(i)
             guess = self.classifier.prob_classify(feats)
             observed = guess.max()
diff --git a/explainshell/manager.py b/explainshell/manager.py
index 8c0aea83..48794552 100644
--- a/explainshell/manager.py
+++ b/explainshell/manager.py
@@ -49,7 +49,7 @@ def _read(self, ctx, f_runner):
         assert len(ctx.manpage.paragraphs) > 1
 
         ctx.manpage = store.ManPage(
-            ctx.manpage.shortpath,
+            ctx.manpage.short_path,
             ctx.manpage.name,
             ctx.manpage.synopsis,
             ctx.manpage.paragraphs,
@@ -73,7 +73,7 @@ def _extract(self, ctx, f_runner):
 
     def _write(self, ctx, f_runner):
         f_runner.pre_add_manpage()
-        return ctx.store.addmanpage(ctx.manpage)
+        return ctx.store.add_manpage(ctx.manpage)
 
     def _update(self, ctx, f_runner):
         f_runner.pre_add_manpage()
@@ -171,7 +171,7 @@ def findmulti_cmds(self):
             logger.info("inserting mapping (multi_cmd) %s -> %s", src, dst)
 
         for multi_cmd, _id in multi_cmds.items():
-            self.store.setmulti_cmd(_id)
+            self.store.set_multi_cmd(_id)
             logger.info("making %r a multi_cmd", multi_cmd)
 
         return mappings_to_a, multi_cmds
diff --git a/explainshell/manpage.py b/explainshell/manpage.py
index 648375f0..25b171de 100644
--- a/explainshell/manpage.py
+++ b/explainshell/manpage.py
@@ -100,7 +100,7 @@ def bold(ln_in):
         x = list(rp_with)
         x.insert(1, "</u>")
         x = "".join(x)
-        _replacements.append((x, "%s</u>" % rp_with))
+        _replacements.append((x, f"{rp_with}</u>"))
 
 _replacements_no_prefix = [
     "\xc2\xb7",  # bullet
@@ -121,7 +121,7 @@ def bold(ln_in):
     x = list(s)
     x.insert(1, "</u>")
     x = "".join(x)
-    _replacements.append((x, "%s</u>" % s))
+    _replacements.append((x, f"{s}</u>"))
 
 _href = re.compile(r'<a href="file:///[^\?]*\?([^\(]*)\(([^\)]*)\)">')
 _section = re.compile(r"<b>([^<]+)</b>")
@@ -139,8 +139,9 @@ def _parse_text(lines):
         )
         for look_for, rp_with in _replacements:
             ln = re.sub(look_for, rp_with, ln)
+
         # confirm the line is valid utf8
-        l_replaced = ln.decode("utf8", "ignore").encode("utf8")
+        l_replaced = ln  # .decode("utf8", "ignore").encode("utf8")
         if l_replaced != ln:
             logger.error("line %r contains invalid utf8", ln)
             ln = l_replaced
@@ -177,14 +178,14 @@ def _parse_synopsis(base, synopsis):
 
 
 class ManPage:
-    """read the man page at path by executing w3mman2html.cgi and find its
-    synopsis with lexgrog
+    """read the man page at path by executing `w3mman2html.cgi` and find it's
+    synopsis with `lexgrog`
 
     since some man pages share the same name (different versions), each
     alias of a man page has a score that's determined in this simple fashion:
     - name of man page source file is given a score of 10
     - all other names found for a particular man page are given a score of 1
-      (other names are found by scanning the output of lexgrog)
+      (other names are found by scanning the output of `lexgrog`)
     """
 
     def __init__(self, path):
@@ -201,11 +202,27 @@ def read(self):
         on the class instance."""
         cmd = [config.MAN2HTML, urllib.parse.urlencode({"local": os.path.abspath(self.path)})]
         logger.info("executing %r", " ".join(cmd))
-        self._text = subprocess.check_output(cmd, stderr=devnull, env=ENV)
+        self._text = ""
+
+        try:
+            t_proc = subprocess.run(cmd, check=True, capture_output=True, text=True, timeout=300, env=ENV)
+
+            if t_proc.stdout:
+                self._text = t_proc.stdout
+            if t_proc.stderr:
+                logger.error(f"failed to extract text for {self.name} -> w3mman2html.cgi returned: {t_proc.stderr}")
+        except Exception as error_msg:
+            logger.error(f"failed to extract text for {self.name} -> error: {error_msg}")
+
         try:
-            self.synopsis = subprocess.check_output(
-                ["lexgrog", self.path], stderr=devnull
-            ).rstrip()
+            self.synopsis = ""
+            s_proc = subprocess.run(
+                ["lexgrog", self.path], capture_output=True, text=True, timeout=300
+            )
+            if s_proc.stdout:
+                self.synopsis = s_proc.stdout.rstrip()
+            if s_proc.stderr:
+                logger.error(f"failed to extract text for {self.name} -> lexgrog returned: {s_proc.stderr}")
         except subprocess.CalledProcessError:
             logger.error("failed to extract synopsis for %s", self.name)
 
@@ -222,7 +239,7 @@ def parse(self):
             d = collections.OrderedDict()
             for prog, text in self.synopsis:
                 d.setdefault(text, []).append(prog)
-            text, progs = d.items()[0]
+            text, progs = list(dict(d).items())[0]
             self.synopsis = text
             self.aliases.update(progs)
         self.aliases.remove(self.name)
diff --git a/explainshell/matcher.py b/explainshell/matcher.py
index 5a7f1834..c0d9cd8e 100644
--- a/explainshell/matcher.py
+++ b/explainshell/matcher.py
@@ -39,12 +39,12 @@ class Matcher(bashlex.ast.nodevisitor):
     """
 
     def __init__(self, s, store):
-        self.s = s.encode("latin1", "replace")
+        self.s = s
         self.store = store
         self._prev_option = self._current_option = None
         self.groups = [MatchGroup("shell")]
 
-        # a list of matchwordexpansions where expansions happened during word
+        # a list of `match_word_exp` where expansions happened during word
         # expansion
         self.expansions = []
 
@@ -150,6 +150,8 @@ def visitredirect(self, node, input, r_type, output, heredoc):
         if r_type in help_constants.REDIRECTION_KIND:
             helptext.append(help_constants.REDIRECTION_KIND[r_type])
 
+        logger.debug(helptext)
+
         self.groups[0].results.append(
             MatchResult(node.pos[0], node.pos[1], "\n\n".join(helptext), None)
         )
@@ -393,18 +395,18 @@ def attemptfuzzy(chars):
             m = []
             if chars[0] == "-":
                 tokens = [chars[0:2]] + list(chars[2:])
-                considerarg = True
+                consider_arg = True
             else:
                 tokens = list(chars)
-                considerarg = False
+                consider_arg = False
 
             pos = node.pos[0]
-            prevoption = None
+            prev_option = None
             for i, t in enumerate(tokens):
                 op = t if t[0] == "-" else "-" + t
                 option = self.find_option(op)
                 if option:
-                    if considerarg and not m and option.expects_arg:
+                    if consider_arg and not m and option.expects_arg:
                         logger.info(
                             "option %r expected an arg, taking the rest too", option
                         )
@@ -420,7 +422,7 @@ def attemptfuzzy(chars):
                 # match the current token, take the rest as its argument, this
                 # covers a series of short options where the last one has an argument
                 # with no space between it, such as 'xargs -r0n1'
-                elif considerarg and prevoption and prevoption.expects_arg:
+                elif consider_arg and prev_option and prev_option.expects_arg:
                     pmr = m[-1]
                     mr = MatchResult(
                         pmr.start, pmr.end + (len(tokens) - i), pmr.text, None
@@ -434,7 +436,7 @@ def attemptfuzzy(chars):
                 else:
                     m.append(self.unknown(t, pos, pos + len(t)))
                 pos += len(t)
-                prevoption = option
+                prev_option = option
             return m
 
         def _visitword(node, word):
diff --git a/explainshell/options.py b/explainshell/options.py
index 990c3ec0..8966d33c 100644
--- a/explainshell/options.py
+++ b/explainshell/options.py
@@ -14,7 +14,7 @@ def extract(manpage):
     options"""
     for i, p in enumerate(manpage.paragraphs):
         if p.is_option:
-            s, ln = extract_option(p.cleantext())
+            s, ln = extract_option(p.clean_text())
             if s or ln:
                 expects_arg = any(x.expects_arg for x in s + ln)
                 s = [x.flag for x in s]
diff --git a/explainshell/store.py b/explainshell/store.py
index e4809aee..505a7448 100644
--- a/explainshell/store.py
+++ b/explainshell/store.py
@@ -5,8 +5,10 @@
 import collections
 import re
 import logging
+from pprint import pprint
 
 import pymongo
+from bson import ObjectId
 
 from explainshell import errors, help_constants, util, config
 
@@ -40,6 +42,9 @@ def __init__(self, idx, text, section, is_option):
         self.section = section
         self.is_option = is_option
 
+        if not isinstance(self.text, str):
+            self.text = self.text.decode("utf-8")
+
     def clean_text(self):
         t = re.sub(r"<[^>]+>", "", self.text)
         t = re.sub("&lt;", "<", t)
@@ -107,9 +112,9 @@ def from_store(cls, d):
             p,
             d["short"],
             d["long"],
-            d["expects_arg"],
+            d["expectsarg"],
             d["argument"],
-            d.get("nested_cmd"),
+            d.get("nestedcmd"),
         )
 
     def to_store(self):
@@ -117,9 +122,9 @@ def to_store(self):
         assert d["is_option"]
         d["short"] = self.short
         d["long"] = self.long
-        d["expects_arg"] = self.expects_arg
+        d["expectsarg"] = self.expects_arg
         d["argument"] = self.argument
-        d["nested_cmd"] = self.nested_cmd
+        d["nestedcmd"] = self.nested_cmd
         return d
 
     def __str__(self):
@@ -250,10 +255,10 @@ def from_store(d):
             synopsis,
             paragraphs,
             [tuple(x) for x in d["aliases"]],
-            d["partial_match"],
-            d["multi_cmd"],
+            d["partialmatch"],
+            d["multicommand"],
             d["updated"],
-            d.get("nested_cmd"),
+            d.get("nestedcmd"),
         )
 
     @staticmethod
@@ -293,12 +298,12 @@ def drop(self, confirm=False):
         self.mapping.drop()
         self.manpage.drop()
 
-    def trainingset(self):
+    def training_set(self):
         for d in self.classifier.find():
             yield ClassifierManpage.from_store(d)
 
     def __contains__(self, name):
-        c = self.mapping.find({"src": name}).count()
+        c = self.mapping.count_documents({"src": name})
         return c > 0
 
     def __iter__(self):
@@ -332,24 +337,24 @@ def find_man_page(self, name):
                 section = splitted[1]
 
         logger.info("looking up manpage in mapping with src %r", name)
-        cursor = self.mapping.find({"src": name})
+        cursor = list(self.mapping.find({"src": name}))
 
-        logger.info(list(cursor))
-        count = len(list(cursor))
+        count = len(cursor)
         if count == 0:
+            logger.debug(f"count is {count}")
             raise errors.ProgramDoesNotExist(name)
 
         dsts = {d["dst"]: d["score"] for d in cursor}
-        cursor = self.manpage.find(
+        cursor = list(self.manpage.find(
             {"_id": {"$in": list(dsts.keys())}}, {"name": 1, "source": 1}
-        )
+        ))
         if len(list(cursor)) != len(dsts):
             logger.error(
                 "one of %r mappings is missing in manpage collection "
                 "(%d mappings, %d found)",
                 dsts,
                 len(dsts),
-                cursor.count(),
+                len(cursor),
             )
         results = [(d.pop("_id"), ManPage.from_store_name_only(**d)) for d in cursor]
         results.sort(key=lambda x: dsts.get(x[0], 0), reverse=True)
@@ -381,40 +386,42 @@ def _discover_manpage_suggestions(self, oid, existing):
         # find all srcs that point to oid
         srcs = [d["src"] for d in cursor]
         # find all dsts of srcs
-        suggestionoids = self.mapping.find({"src": {"$in": srcs}}, {"dst": 1})
+        suggestion_oids = self.mapping.find({"src": {"$in": srcs}}, {"dst": 1})
         # remove already discovered
-        suggestionoids = [d["dst"] for d in suggestionoids if d["dst"] not in skip]
-        if not suggestionoids:
+        suggestion_oids = [d["dst"] for d in suggestion_oids if d["dst"] not in skip]
+        if not suggestion_oids:
             return []
 
         # get just the name and source of found suggestions
-        suggestionoids = self.manpage.find(
-            {"_id": {"$in": suggestionoids}}, {"name": 1, "source": 1}
+        suggestion_oids = self.manpage.find(
+            {"_id": {"$in": suggestion_oids}}, {"name": 1, "source": 1}
         )
         return [
-            (d.pop("_id"), ManPage.from_store_name_only(**d)) for d in suggestionoids
+            (d.pop("_id"), ManPage.from_store_name_only(**d)) for d in suggestion_oids
         ]
 
     def add_mapping(self, src, dst, score):
-        self.mapping.insert({"src": src, "dst": dst, "score": score})
+        if not isinstance(dst, ObjectId):
+            dst = dst.inserted_id
+        self.mapping.insert_one({"src": src, "dst": dst, "score": score})
 
-    def addmanpage(self, m):
-        """add m into the store, if it exists first remove it and its mappings
+    def add_manpage(self, m):
+        """add `m` into the store, if it exists first remove it and it's mappings
 
-        each man page may have aliases besides the name determined by its
+        each man page may have aliases besides the name determined by it's
         basename"""
         d = self.manpage.find_one({"source": m.source})
         if d:
             logger.info("removing old manpage %s (%s)", m.source, d["_id"])
-            self.manpage.remove(d["_id"])
+            self.manpage.delete_one({"_id": d["_id"]})
 
             # remove old mappings if there are any
-            c = self.mapping.count()
-            self.mapping.remove({"dst": d["_id"]})
-            c -= self.mapping.count()
+            c = self.mapping.count_documents({})
+            self.mapping.delete_one({"dst": d["_id"]})
+            c -= self.mapping.count_documents({})
             logger.info("removed %d mappings for manpage %s", c, m.source)
 
-        o = self.manpage.insert(m.to_store())
+        o = self.manpage.insert_one(m.to_store())
 
         for alias, score in m.aliases:
             self.add_mapping(alias, o, score)
@@ -433,7 +440,7 @@ def update_man_page(self, m):
         change updated attribute so we don't overwrite this in the future"""
         logger.info("updating manpage %s", m.source)
         m.updated = True
-        self.manpage.update({"source": m.source}, m.to_store())
+        self.manpage.update_one({"source": m.source}, m.to_store())
         _id = self.manpage.find_one({"source": m.source}, fields={"_id": 1})["_id"]
         for alias, score in m.aliases:
             if alias not in self:
@@ -470,20 +477,20 @@ def verify(self):
 
         notfound = reachable - man_pages
         if notfound:
-            logger.error("mappings to inexisting manpages: %r", notfound)
+            logger.error("mappings to non-existing manpages: %r", notfound)
             ok = False
 
         return ok, unreachable, notfound
 
     def names(self):
-        cursor = self.manpage.find(fields={"name": 1})
+        cursor = self.manpage.find({}, {"name": 1})
         for d in cursor:
             yield d["_id"], d["name"]
 
     def mappings(self):
-        cursor = self.mapping.find(fields={"src": 1})
+        cursor = self.mapping.find({}, {"src": 1})
         for d in cursor:
             yield d["src"], d["_id"]
 
-    def setmulti_cmd(self, manpage_id):
-        self.manpage.update({"_id": manpage_id}, {"$set": {"multi_cmd": True}})
+    def set_multi_cmd(self, manpage_id):
+        self.manpage.update_one({"_id": manpage_id}, {"$set": {"multi_cmd": True}})
diff --git a/explainshell/web/helpers.py b/explainshell/web/helpers.py
index 41d555f2..ce199808 100644
--- a/explainshell/web/helpers.py
+++ b/explainshell/web/helpers.py
@@ -18,6 +18,6 @@ def suggestions(matches, command):
             for other_mp in sorted(m["suggestions"], key=lambda mp: mp.section):
                 mid = f"{other_mp.name}.{other_mp.section}"
                 new_suggestions.append(
-                    {"cmd": "".join([before, mid, after]), "text": other_mp.namesection}
+                    {"cmd": "".join([before, mid, after]), "text": other_mp.name_section}
                 )
             m["suggestions"] = new_suggestions
diff --git a/explainshell/web/views.py b/explainshell/web/views.py
index 001d710a..5c74f7a5 100644
--- a/explainshell/web/views.py
+++ b/explainshell/web/views.py
@@ -51,10 +51,10 @@ def explain():
     except NotImplementedError as error_msg:
         logger.warning("not implemented error trying to explain %r", command)
         msg = (
-            "the parser doesn't support %r constructs in the command you tried. you may "
-            "<a href='https://github.com/idank/explainshell/issues'>report a "
-            "bug</a> to have this added, if one doesn't already exist."
-        ) % error_msg.args[0]
+            f"the parser doesn't support {error_msg.args[0]} constructs in the command you tried. you may "
+            f"<a href='https://github.com/idank/explainshell/issues'>report a "
+            f"bug</a> to have this added, if one doesn't already exist."
+        )
 
         return render_template("errors/error.html", title="error!", message=msg)
     except Exception as error_msg:
@@ -77,7 +77,7 @@ def explain_old(section, program):
     if "args" in request.args:
         args = request.args["args"]
         command = f"{program} {args}"
-        return redirect("/explain?cmd=%s" % urllib.parse.quote_plus(command), 301)
+        return redirect(f"/explain?cmd={urllib.parse.quote_plus(command)}", 301)
     else:
         try:
             mp, suggestions = explain_program(program, s)
@@ -91,7 +91,7 @@ def explain_old(section, program):
 def explain_program(program, store):
     mps = store.find_man_page(program)
     mp = mps.pop(0)
-    program = mp.namesection
+    program = mp.name_section
 
     synopsis = mp.synopsis
     if synopsis:
@@ -108,7 +108,7 @@ def explain_program(program, store):
     suggestions = []
     for other_mp in mps:
         d = {
-            "text": other_mp.namesection,
+            "text": other_mp.name_section,
             "link": f"{other_mp.section}/{other_mp.name}",
         }
         suggestions.append(d)
@@ -145,12 +145,17 @@ def explain_cmd(command, store):
     # position
     id_start_pos = {}
 
+    logger.debug(f"processing {len(shell_group.results)} shell_group results ...")
+
     ln = []
     for m in shell_group.results:
         cmd_class = shell_group.name
         help_class = f"help-{len(text_ids)}"
+
         text = str(m.text)
-        if text:
+        if isinstance(m.text, bytes):
+            text = m.text.decode("utf-8")
+        if len(text.replace("None", "")) > 0:
             help_class = text_ids.setdefault(text, help_class)
         else:
             # unknowns in the shell group are possible when our parser left
@@ -166,13 +171,19 @@ def explain_cmd(command, store):
         ln.append(d)
     matches.append(ln)
 
+    logger.debug(f"processing {len(cmd_groups)} cmd_group results ...")
+
     for cmd_group in cmd_groups:
         ln = []
         for m in cmd_group.results:
             cmd_class = cmd_group.name
             help_class = f"help-{len(text_ids)}"
+
             text = str(m.text)
-            if text:
+            if isinstance(m.text, bytes):
+                text = m.text.decode("utf-8")
+
+            if len(text.replace("None", "")) > 0:
                 help_class = text_ids.setdefault(text, help_class)
             else:
                 cmd_class += " unknown"
@@ -199,7 +210,7 @@ def explain_cmd(command, store):
     matches = list(itertools.chain.from_iterable(matches))
     helpers.suggestions(matches, command)
 
-    # _checkoverlaps(matcher_.s, matches)
+    # _check_overlaps(matcher_.s, matches)
     matches.sort(key=lambda d: d["start"])
 
     it = util.Peekable(iter(matches))
@@ -238,42 +249,42 @@ def format_match(d, m, expansions):
 
     # go over the expansions, wrapping them with a link; leave everything else
     # untouched
-    expandedmatch = ""
+    expanded_match = ""
     i = 0
     for start, end, kind in expansions:
         if start >= m.end:
             break
-        relativestart = start - m.start
-        relativeend = end - m.start
+        rel_start = start - m.start
+        rel_end = end - m.start
 
-        if i < relativestart:
-            for j in range(i, relativestart):
+        if i < rel_start:
+            for j in range(i, rel_start):
                 if m.match[j].isspace():
-                    expandedmatch += markupsafe.Markup("&nbsp;")
+                    expanded_match += markupsafe.Markup("&nbsp;")
                 else:
-                    expandedmatch += markupsafe.escape(m.match[j])
-            i = relativestart + 1
+                    expanded_match += markupsafe.escape(m.match[j])
+            i = rel_start + 1
         if m.start <= start and end <= m.end:
-            s = m.match[relativestart:relativeend]
+            s = m.match[rel_start:rel_end]
 
             if kind == "substitution":
-                content = markupsafe.Markup(_substitutionmarkup(s))
+                content = markupsafe.Markup(_substitution_markup(s))
             else:
                 content = s
 
-            expandedmatch += markupsafe.Markup(
+            expanded_match += markupsafe.Markup(
                 '<span class="expansion-{0}">{1}</span>'
             ).format(kind, content)
-            i = relativeend
+            i = rel_end
 
     if i < len(m.match):
-        expandedmatch += markupsafe.escape(m.match[i:])
+        expanded_match += markupsafe.escape(m.match[i:])
 
-    assert expandedmatch
-    d["match"] = expandedmatch
+    assert expanded_match
+    d["match"] = expanded_match
 
 
-def _substitutionmarkup(cmd):
+def _substitution_markup(cmd):
     """
     >>> _substitutionmarkup('foo')
     '<a href="/explain?cmd=foo" title="Zoom in to nested command">foo</a>'
@@ -286,13 +297,12 @@ def _substitutionmarkup(cmd):
     ).format(cmd=cmd, query=encoded)
 
 
-def _checkoverlaps(s, matches):
+def _check_overlaps(s, matches):
     explained = [None] * len(s)
     for d in matches:
         for i in range(d["start"], d["end"]):
             if explained[i]:
                 raise RuntimeError(
-                    "explained overlap for group %s at %d with %s"
-                    % (d, i, explained[i])
+                    f"explained overlap for group {d} at {i} with {explained[i]}"
                 )
             explained[i] = d

From 6a8fba0b44f032da4ccaec29ebb8cbb21c727e15 Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Mon, 16 Sep 2024 00:56:45 +0000
Subject: [PATCH 14/19] fix: fixed bugs in ingestion of manpages

---
 explainshell/store.py | 41 ++++++++++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/explainshell/store.py b/explainshell/store.py
index 505a7448..4e7a3247 100644
--- a/explainshell/store.py
+++ b/explainshell/store.py
@@ -5,7 +5,8 @@
 import collections
 import re
 import logging
-from pprint import pprint
+
+# from pprint import pprint
 
 import pymongo
 from bson import ObjectId
@@ -69,7 +70,7 @@ def to_store(self):
     def __repr__(self):
         t = self.clean_text()
         t = t[: min(20, t.find("\n"))].lstrip()
-        return "<paragraph %d, %s: %r>" % (self.idx, self.section, t)
+        return f"<paragraph {self.idx}, {self.section}: {t}>"
 
     def __eq__(self, other):
         if not other:
@@ -108,6 +109,8 @@ def opts(self):
     def from_store(cls, d):
         p = Paragraph.from_store(d)
 
+        # logger.debug(str(vars(d)))
+
         return cls(
             p,
             d["short"],
@@ -128,7 +131,7 @@ def to_store(self):
         return d
 
     def __str__(self):
-        return "(%s)" % ", ".join([str(x) for x in self.opts])
+        return "(" + ", ".join([str(x) for x in self.opts]) + ")"
 
     def __repr__(self):
         return f"<options for paragraph {self.idx}: {self}"
@@ -249,16 +252,34 @@ def from_store(d):
         else:
             synopsis = help_constants.NO_SYNOPSIS
 
+        partial_match = None
+        if "partialmatch" in d:
+            partial_match = d["partialmatch"]
+        elif "partial_match" in d:
+            partial_match = d["partial_match"]
+
+        multi_cmd = None
+        if "multicommand" in d:
+            multi_cmd = d["multicommand"]
+        elif "multi_cmd" in d:
+            multi_cmd = d["multi_cmd"]
+
+        nested_cmd = None
+        if "nestedcmd" in d:
+            nested_cmd = d["nestedcmd"]
+        elif "nested_cmd" in d:
+            nested_cmd = d["nested_cmd"]
+
         return ManPage(
             d["source"],
             d["name"],
             synopsis,
             paragraphs,
             [tuple(x) for x in d["aliases"]],
-            d["partialmatch"],
-            d["multicommand"],
+            partial_match,
+            multi_cmd,
             d["updated"],
-            d.get("nestedcmd"),
+            nested_cmd,
         )
 
     @staticmethod
@@ -345,9 +366,11 @@ def find_man_page(self, name):
             raise errors.ProgramDoesNotExist(name)
 
         dsts = {d["dst"]: d["score"] for d in cursor}
-        cursor = list(self.manpage.find(
-            {"_id": {"$in": list(dsts.keys())}}, {"name": 1, "source": 1}
-        ))
+        cursor = list(
+            self.manpage.find(
+                {"_id": {"$in": list(dsts.keys())}}, {"name": 1, "source": 1}
+            )
+        )
         if len(list(cursor)) != len(dsts):
             logger.error(
                 "one of %r mappings is missing in manpage collection "

From 8fbd6994e19ef73ad416de76326b042bc432a297 Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 21 Sep 2024 10:18:00 +0000
Subject: [PATCH 15/19] feat(js): highlight first clazz

---
 explainshell/web/static/js/es.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/explainshell/web/static/js/es.js b/explainshell/web/static/js/es.js
index 5cd20870..2ea9f726 100644
--- a/explainshell/web/static/js/es.js
+++ b/explainshell/web/static/js/es.js
@@ -232,6 +232,8 @@ function eslink(clazz, option, mid, color) {
         this.goingleft = rrmid <= mid;
 
         $(this.help).css("border-color", this.color);
+        
+        $("#" + clazz + " b:first-of-type").css("color", this.color);
     }
 }
 
@@ -264,7 +266,7 @@ eslink.prototype.nearby = function(other) {
     return Math.abs(r.right - rr.left) <= closeness || Math.abs(r.left - rr.right) <= closeness;
 };
 
-// a conveninent wrapper around an array of points that allows to chain appends
+// a convenient wrapper around an array of points that allows to chain appends
 function espath() {
     this.points = [];
 }

From aaf4cdca7549b661609896769fff72a32347c4ec Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 21 Sep 2024 10:21:06 +0000
Subject: [PATCH 16/19] chore: merged  'apoorvlathey-fork/fixSearch' into
 mundanevision20-python3-migration

---
 explainshell/web/static/css/es.css | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/explainshell/web/static/css/es.css b/explainshell/web/static/css/es.css
index 2019b53e..abc256b3 100644
--- a/explainshell/web/static/css/es.css
+++ b/explainshell/web/static/css/es.css
@@ -115,26 +115,20 @@ a {
     width: 460px; /* necessary for right:0 to work at #themeSelector */
     position: absolute;
     top: -30px;
-    left: 240px;
+    left: 130px;
     font-family: 'Berkshire Swash', cursive;
     font-size: 24px;
 }
 
 #top-search {
-   width: 150px;
+   width: 260px;
    -webkit-transition: all .5s ease;
    -moz-transition: all .5s ease;
    transition: all .5s ease;
    position: relative;
-   top: -3px; /* looks a bit unaligned without this in chrome */
    left: 0;
 }
 
-#top-search:focus {
-   left: -110px;
-   width: 260px;
-}
-
 #prevnext {
     text-align: center;
     padding-top: 10px;
@@ -166,7 +160,7 @@ a {
     /* using a fixed margin-left doesn't work both
        with and without the search bar in the menu */
     position: absolute;
-    right: 0;
+    right: -110px;
     margin-top: 5px;
     font-family: "Courier New",Courier,Monaco,Menlo,Consolas,monospace;
     font-size: 20px;

From 04adb1ddbcc17a2371d6b25baf1f1948c94ee404 Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 21 Sep 2024 10:24:10 +0000
Subject: [PATCH 17/19] fix: fixed bug in manpage.py

---
 explainshell/manpage.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/explainshell/manpage.py b/explainshell/manpage.py
index 25b171de..893c6804 100644
--- a/explainshell/manpage.py
+++ b/explainshell/manpage.py
@@ -174,6 +174,10 @@ def _parse_synopsis(base, synopsis):
     synopsis = synopsis[len(base) + 3: -1]
     if synopsis[-1] == ".":
         synopsis = synopsis[:-1]
+
+    if not SPLIT_SYNOP.match(synopsis):
+        return []
+
     return SPLIT_SYNOP.match(synopsis).groups()
 
 

From 29a20e6c92f26968e59706bb87bdc4f66f0cea6c Mon Sep 17 00:00:00 2001
From: Henning Weller <111561130+mundanevision20@users.noreply.github.com>
Date: Sat, 21 Sep 2024 10:44:05 +0000
Subject: [PATCH 18/19] chore: update to Ubuntu noble manpages

---
 README.md                               | 2 +-
 explainshell/manpage.py                 | 3 ++-
 explainshell/web/templates/about.html   | 2 +-
 explainshell/web/templates/explain.html | 2 +-
 explainshell/web/templates/macros.html  | 4 ++--
 explainshell/web/templates/tagger.html  | 2 +-
 tools/dlgzlist                          | 4 +++-
 tools/extractgzlist                     | 6 +++---
 8 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index e2a690c4..2455cde9 100644
--- a/README.md
+++ b/README.md
@@ -29,7 +29,7 @@ When querying explainshell, it:
 
 ## Missing man pages
 
-Right now explainshell.com contains the entire [archive of Ubuntu](http://manpages.ubuntu.com/). It's not
+Right now explainshell.com contains the entire [archive of Ubuntu](https://manpages.ubuntu.com/). It's not
 possible to directly add a missing man page to the live site (it might be in the future).
 
 ## Running explainshell locally
diff --git a/explainshell/manpage.py b/explainshell/manpage.py
index 893c6804..7bfc0505 100644
--- a/explainshell/manpage.py
+++ b/explainshell/manpage.py
@@ -132,9 +132,10 @@ def _parse_text(lines):
     section = None
     i = 0
     for ln in lines:
+        # TODO: check if this url is still valid
         ln = re.sub(
             _href,
-            r'<a href="http://manpages.ubuntu.com/manpages/precise/en/man\2/\1.\2.html">',
+            r'<a href="https://manpages.ubuntu.com/manpages/noble/en/man\2/\1.\2.html">',
             ln,
         )
         for look_for, rp_with in _replacements:
diff --git a/explainshell/web/templates/about.html b/explainshell/web/templates/about.html
index 726d11e2..2a202412 100644
--- a/explainshell/web/templates/about.html
+++ b/explainshell/web/templates/about.html
@@ -4,7 +4,7 @@
             <div id="about">
                 <p>Hello,</p>
                 <p>This site contains 29761 parsed manpages from sections 1 and 8 found in
-                <a href="http://manpages.ubuntu.com/">Ubuntu's manpage repository</a>.
+                <a href="https://manpages.ubuntu.com/">Ubuntu's manpage repository</a>.
                 A lot of heuristics were used to extract the arguments of each
                 program, and there are errors here and there, especially in
                 manpages that have a non-standard layout.</p>
diff --git a/explainshell/web/templates/explain.html b/explainshell/web/templates/explain.html
index a43071a5..b09b18db 100644
--- a/explainshell/web/templates/explain.html
+++ b/explainshell/web/templates/explain.html
@@ -40,7 +40,7 @@
 <div class="push"></div><div class="text-center"><small>
 {% for m in matches if m.name %}
 {% if loop.first %}source manpages: {% endif %}
-<a href="http://manpages.ubuntu.com/manpages/precise/en/man{{ m.section }}/{{ m.source|e }}.html">{{ m.name|e }}</a>{% if not loop.last %},&nbsp;{% endif %}
+<a href="https://manpages.ubuntu.com/manpages/precise/en/man{{ m.section }}/{{ m.source|e }}.html">{{ m.name|e }}</a>{% if not loop.last %},&nbsp;{% endif %}
 {% endfor %}
 </small></div>
 {%- endblock %}
diff --git a/explainshell/web/templates/macros.html b/explainshell/web/templates/macros.html
index 94bc4f85..4f661052 100644
--- a/explainshell/web/templates/macros.html
+++ b/explainshell/web/templates/macros.html
@@ -1,10 +1,10 @@
 {% macro outputcommand(mp, suggestions) -%}
     {% if suggestions|length == 0 %}
-    <a href="http://manpages.ubuntu.com/manpages/precise/en/man{{ mp.section }}/{{ mp.source }}.html">{{ mp.program|e }}</a>
+    <a href="https://manpages.ubuntu.com/manpages/precise/en/man{{ mp.section }}/{{ mp.source }}.html">{{ mp.program|e }}</a>
     {% else %}
         <span class="dropdown">
             <b class="caret" data-toggle="dropdown"></b>
-            <a href="http://manpages.ubuntu.com/manpages/precise/en/man{{ mp.section }}/{{ mp.source }}.html">{{ mp.program|e }}</a>
+            <a href="https://manpages.ubuntu.com/manpages/precise/en/man{{ mp.section }}/{{ mp.source }}.html">{{ mp.program|e }}</a>
             <ul class="dropdown-menu" role="menu" aria-labelledby="dropdownMenu">
               <li>other manpages</li>
               <li class="divider"></li>
diff --git a/explainshell/web/templates/tagger.html b/explainshell/web/templates/tagger.html
index 4e2674d4..a5274983 100644
--- a/explainshell/web/templates/tagger.html
+++ b/explainshell/web/templates/tagger.html
@@ -3,7 +3,7 @@
 <div class="small-push"></div>
 <div class="push"></div>
 <div id="command">
-    <a href="http://manpages.ubuntu.com/manpages/precise/en/man1/{{ m.name }}.html">{{ m.name|e }}</a> - {{ m.synopsis|e
+    <a href="https://manpages.ubuntu.com/manpages/precise/en/man1/{{ m.name }}.html">{{ m.name|e }}</a> - {{ m.synopsis|e
     }}
 </div>
 <div class="small-push"></div>
diff --git a/tools/dlgzlist b/tools/dlgzlist
index 5f587a90..113c1e39 100755
--- a/tools/dlgzlist
+++ b/tools/dlgzlist
@@ -7,4 +7,6 @@ while read url; do
 
 	python -c 'import time, random; time.sleep(float(random.randint(1000, 2000)) / 1000)'
 	wget -nv "$url" || echo "$url" >> failed
-done < gzlist
+done < gzlist8
+
+mv -f *.8.gz ../manpages/8/
diff --git a/tools/extractgzlist b/tools/extractgzlist
index 2ccdcf52..9f921dfe 100755
--- a/tools/extractgzlist
+++ b/tools/extractgzlist
@@ -1,9 +1,9 @@
 #!/bin/bash
 
-release="precise"
+release="noble"
 section="8"
-man1list="http://manpages.ubuntu.com/manpages/$release/en/man$section/"
-gzroot="http://manpages.ubuntu.com/manpages.gz/$release/man$section/"
+man1list="https://manpages.ubuntu.com/manpages/$release/en/man$section/"
+gzroot="https://manpages.ubuntu.com/manpages.gz/$release/man$section/"
 filename="index$section.html"
 
 if ! [ -f $filename ]; then

From 6f03a78a76fed8a902d672145e09d86cad6889e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tobias=20Hochg=C3=BCrtel?=
 <tobias.hochguertel@googlemail.com>
Date: Sun, 26 Oct 2025 15:38:17 +0100
Subject: [PATCH 19/19] =?UTF-8?q?Complete=20Python=202=E2=86=923=20compati?=
 =?UTF-8?q?bility=20fixes?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Fix iterator protocol: def next() → def __next__() in Peekable class
- Fix iterator calls: it.next() → next(it) throughout codebase
- Fix map() iterator: yield map() → yield list(map()) in group_continuous
- Fix dictionary methods: .iteritems() → .items() in doctests
- Fix function names in doctests to match actual definitions
- Migrate testing framework: nose → pytest (Python 3.12 compatible)
- Update requirements.txt with pytest 8.3.3 and pytest-cov 6.0.0
- Update Makefile to use pytest --doctest-modules

These fixes complete the Python 3.12 migration on top of PR #330.
All 12 tests now passing (100% success rate).

Test results:
- explainshell/algo/features.py: 1 passed
- explainshell/fixer.py: 1 passed
- explainshell/manpage.py: 3 passed
- explainshell/options.py: 3 passed
- explainshell/util.py: 3 passed
- explainshell/web/views.py: 1 passed

Total: 12 passed in 0.37s

Database integration verified with 72,349 documents.
Full functionality tested with ls, grep, and tar commands.
---
 Makefile                  |  2 +-
 explainshell/options.py   | 14 +++++++-------
 explainshell/util.py      | 14 +++++++-------
 explainshell/web/views.py |  6 +++---
 requirements.txt          |  3 ++-
 5 files changed, 20 insertions(+), 19 deletions(-)

diff --git a/Makefile b/Makefile
index a15e07a1..611e845e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 tests:
-	nosetests --exe --with-doctest tests/ explainshell/
+	pytest --doctest-modules tests/ explainshell/
 
 serve:
 	docker-compose up --build
diff --git a/explainshell/options.py b/explainshell/options.py
index 8966d33c..ae3ba5b4 100644
--- a/explainshell/options.py
+++ b/explainshell/options.py
@@ -87,19 +87,19 @@ def _option(s, pos=0):
     False
     >>> bool(_option('--a-b-'))
     False
-    >>> sorted(_option('-a').groupdict().iteritems())
+    >>> sorted(_option('-a').groupdict().items())
     [('arg', None), ('argoptional', None), ('argoptionalc', None), ('ending', ''), ('opt', '-a')]
-    >>> sorted(_option('--a').groupdict().iteritems())
+    >>> sorted(_option('--a').groupdict().items())
     [('arg', None), ('argoptional', None), ('argoptionalc', None), ('ending', ''), ('opt', '--a')]
-    >>> sorted(_option('-a<b>').groupdict().iteritems())
+    >>> sorted(_option('-a<b>').groupdict().items())
     [('arg', 'b'), ('argoptional', '<'), ('argoptionalc', '>'), ('ending', ''), ('opt', '-a')]
-    >>> sorted(_option('-a=[foo]').groupdict().iteritems())
+    >>> sorted(_option('-a=[foo]').groupdict().items())
     [('arg', 'foo'), ('argoptional', '['), ('argoptionalc', ']'), ('ending', ''), ('opt', '-a')]
-    >>> sorted(_option('-a=<foo>').groupdict().iteritems())
+    >>> sorted(_option('-a=<foo>').groupdict().items())
     [('arg', 'foo'), ('argoptional', '<'), ('argoptionalc', '>'), ('ending', ''), ('opt', '-a')]
-    >>> sorted(_option('-a=<foo bar>').groupdict().iteritems())
+    >>> sorted(_option('-a=<foo bar>').groupdict().items())
     [('arg', 'foo bar'), ('argoptional', '<'), ('argoptionalc', '>'), ('ending', ''), ('opt', '-a')]
-    >>> sorted(_option('-a=foo').groupdict().iteritems())
+    >>> sorted(_option('-a=foo').groupdict().items())
     [('arg', 'foo'), ('argoptional', None), ('argoptionalc', None), ('ending', ''), ('opt', '-a')]
     >>> bool(_option('-a=[foo>'))
     False
diff --git a/explainshell/util.py b/explainshell/util.py
index 708e7f06..ba3adb4c 100644
--- a/explainshell/util.py
+++ b/explainshell/util.py
@@ -23,7 +23,7 @@ def consecutive(ln, fn):
     ll = []
     try:
         while True:
-            x = it.next()
+            x = next(it)
             if fn(x):
                 ll.append(x)
             else:
@@ -38,15 +38,15 @@ def consecutive(ln, fn):
 
 def group_continuous(l, key=None):
     """
-    >>> list(groupcontinuous([1, 2, 4, 5, 7, 8, 10]))
+    >>> list(group_continuous([1, 2, 4, 5, 7, 8, 10]))
     [[1, 2], [4, 5], [7, 8], [10]]
-    >>> list(groupcontinuous(range(5)))
+    >>> list(group_continuous(range(5)))
     [[0, 1, 2, 3, 4]]
     """
     if key is None:
         key = lambda x: x
     for k, g in itertools.groupby(enumerate(l), lambda ix: ix[0] - key(ix[1])):
-        yield map(itemgetter(1), g)
+        yield list(map(itemgetter(1), g))
 
 
 def topo_sorted(graph, parents):
@@ -85,7 +85,7 @@ def pairwise(iterable):
 class Peekable:
     """
     >>> it = Peekable(iter('abc'))
-    >>> it.index, it.peek(), it.index, it.peek(), it.next(), it.index, it.peek(), it.next(), it.next(), it.index
+    >>> it.index, it.peek(), it.index, it.peek(), next(it), it.index, it.peek(), next(it), next(it), it.index
     (0, 'a', 0, 'a', 'a', 1, 'b', 'b', 'c', 3)
     >>> it.peek()
     Traceback (most recent call last):
@@ -95,7 +95,7 @@ class Peekable:
     Traceback (most recent call last):
       File "<stdin>", line 1, in ?
     StopIteration
-    >>> it.next()
+    >>> next(it)
     Traceback (most recent call last):
       File "<stdin>", line 1, in ?
     StopIteration
@@ -110,7 +110,7 @@ def __init__(self, it):
     def __iter__(self):
         return self
 
-    def next(self):
+    def __next__(self):
         if self._peeked:
             self._peeked = False
             self._idx += 1
diff --git a/explainshell/web/views.py b/explainshell/web/views.py
index 5c74f7a5..93932706 100644
--- a/explainshell/web/views.py
+++ b/explainshell/web/views.py
@@ -215,7 +215,7 @@ def explain_cmd(command, store):
 
     it = util.Peekable(iter(matches))
     while it.has_next():
-        m = it.next()
+        m = next(it)
         spaces = 0
         if it.has_next():
             spaces = it.peek()["start"] - m["end"]
@@ -286,9 +286,9 @@ def format_match(d, m, expansions):
 
 def _substitution_markup(cmd):
     """
-    >>> _substitutionmarkup('foo')
+    >>> _substitution_markup('foo')
     '<a href="/explain?cmd=foo" title="Zoom in to nested command">foo</a>'
-    >>> _substitutionmarkup('cat <&3')
+    >>> _substitution_markup('cat <&3')
     '<a href="/explain?cmd=cat+%3C%263" title="Zoom in to nested command">cat <&3</a>'
     """
     encoded = urllib.parse.urlencode({"cmd": cmd})
diff --git a/requirements.txt b/requirements.txt
index 7a6f6023..05570c27 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
 Flask==3.0.3
 nltk==3.9.1
-nose==1.3.7
+pytest==8.3.3
+pytest-cov==6.0.0
 pymongo==4.8.0
 bashlex==0.18
 loguru==0.7.2