Bring back a devel server for debugging

CHANGELOG.md

@@ -4,15 +4,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [4.0.0] - Unreleased
 ### Added
 - New `transpile` command to replace some functionality of devel server
 
 ### Changed
 - Major rewrite/refactor of `mothd`
+  - There are now providers for State, Puzzles, and Theme. Sqlite, Redis, or S3 should fit in easily.
+  - Server no longer provides unlocked content
+  - Puzzle URLs are now just `/content/${cat}/${points}/`
 - `state/until` is now `state/hours` and can specify multiple begin/end hours
 - `state/disabled` is now `state/enabled`
 - Mothball structure has changed substantially
+  - Mothballs no longer contain `map.txt`
+  - Clients now expect unlocked puzzles to just be `map[string][]int`
 
 ### Deprecated
 

cmd/mothd/httpd.go

@@ -94,8 +94,11 @@ func (h *HTTPServer) StateHandler(w http.ResponseWriter, req *http.Request) {
 	state.TeamNames = export.TeamNames
 	state.PointsLog = export.PointsLog
 
-	// XXX: unstub this
+	state.Puzzles = make(map[string][]int)
-	state.Puzzles = map[string][]int{"sequence": {1}}
+	log.Println("Puzzles")
+	for category := range h.Puzzles.Inventory() {
+		log.Println(category)
+	}
 
 	JSONWrite(w, state)
 }

cmd/mothd/mothballs.go

@@ -30,6 +30,10 @@ func (m *Mothballs) Open(cat string, points int, filename string) (io.ReadCloser
 }
 
 func (m *Mothballs) Inventory() []Category {
+	for cat, zfs := range m.categories {
+		map, err := zfs.Open("map.txt")
+		log.Println("mothballs", cat, zfs)
+	}
 	return []Category{}
 }
 

devel/devel-server.py

@@ -0,0 +1,287 @@
+#!/usr/bin/python3
+
+import asyncio
+import cgitb
+import html
+import cgi
+import http.server
+import io
+import json
+import mimetypes
+import moth
+import logging
+import os
+import pathlib
+import random
+import shutil
+import socketserver
+import sys
+import traceback
+import mothballer
+import parse
+import urllib.parse
+import posixpath
+from http import HTTPStatus
+
+
+sys.dont_write_bytecode = True  # Don't write .pyc files
+
+
+class MothServer(socketserver.ForkingMixIn, http.server.HTTPServer):
+    def __init__(self, server_address, RequestHandlerClass):
+        super().__init__(server_address, RequestHandlerClass)
+        self.args = {}
+
+
+class MothRequestHandler(http.server.SimpleHTTPRequestHandler):
+    endpoints = []
+    
+    def __init__(self, request, client_address, server):
+        self.directory = str(server.args["theme_dir"])
+        try:
+            super().__init__(request, client_address, server, directory=server.args["theme_dir"])
+        except TypeError:
+            super().__init__(request, client_address, server)
+
+
+    # Backport from Python 3.7
+    def translate_path(self, path):
+        # I guess we just hope that some other thread doesn't call getcwd
+        getcwd = os.getcwd
+        os.getcwd = lambda: self.directory
+        ret = super().translate_path(path)
+        os.getcwd = getcwd
+        return ret
+
+
+    def get_puzzle(self):
+        category = self.req.get("cat")
+        points = int(self.req.get("points"))
+        catpath = str(self.server.args["puzzles_dir"].joinpath(category))
+        cat = moth.Category(catpath, self.seed)
+        puzzle = cat.puzzle(points)
+        return puzzle
+        
+        
+    def send_json(self, obj):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+        self.wfile.write(json.dumps(obj).encode("utf-8"))
+
+    
+    def handle_register(self):
+        # Everybody eats when they come to my house
+        ret = {
+            "status": "success",
+            "data": {
+                "short": "You win",
+                "description": "Welcome to the development server, you wily hacker you"
+            }
+        }
+        self.send_json(ret)
+    endpoints.append(('/{seed}/register', handle_register))
+        
+
+    def handle_answer(self):
+        for f in ("cat", "points", "answer"):
+            self.req[f] = self.fields.getfirst(f)
+        puzzle = self.get_puzzle()
+        ret = {
+            "status": "success",
+            "data": {
+               "short": "",
+               "description": "Provided answer was not in list of answers"
+            },
+        }
+
+        if self.req.get("answer") in puzzle.answers:
+            ret["data"]["description"] = "Answer is correct"
+        self.send_json(ret)
+    endpoints.append(('/{seed}/answer', handle_answer))
+
+
+    def puzzlelist(self):
+        puzzles = {}
+        for p in self.server.args["puzzles_dir"].glob("*"):
+            if not p.is_dir() or p.match(".*"):
+                continue
+            catName = p.parts[-1]
+            cat = moth.Category(str(p), self.seed)
+            puzzles[catName] = [[i, str(i)] for i in cat.pointvals()]
+            puzzles[catName].append([0, ""])
+        if len(puzzles) <= 1:
+            logging.warning("No directories found matching {}/*".format(self.server.args["puzzles_dir"]))
+        
+        return puzzles
+
+
+    # XXX: Remove this (redundant) when we've upgraded the bundled theme (probably v3.6 and beyond)
+    def handle_puzzlelist(self):
+        self.send_json(self.puzzlelist())
+    endpoints.append(('/{seed}/puzzles.json', handle_puzzlelist))
+    
+    
+    def handle_state(self):
+        resp = {
+            "Config": {
+                "Devel": True,
+            },
+            "Puzzles": self.puzzlelist(),
+            "Messages": "<p><b>[MOTH Development Server]</b> Participant broadcast messages would go here.</p>",
+        }
+        self.send_json(resp)
+    endpoints.append(('/{seed}/state', handle_state))
+
+    
+    def handle_puzzle(self):
+        puzzle = self.get_puzzle()
+
+        obj = puzzle.package()
+        obj["answers"] = puzzle.answers
+        obj["hint"] = puzzle.hint
+        obj["summary"] = puzzle.summary
+        obj["logs"] = puzzle.logs
+
+        self.send_json(obj)
+    endpoints.append(('/{seed}/content/{cat}/{points}/puzzle.json', handle_puzzle))
+
+
+    def handle_puzzlefile(self):
+        puzzle = self.get_puzzle()
+
+        try:
+            file = puzzle.files[self.req["filename"]]
+        except KeyError:
+            self.send_error(
+                HTTPStatus.NOT_FOUND,
+                "File Not Found: %s" % self.req["filename"],
+            )
+            return
+
+        self.send_response(200)
+        self.send_header("Content-Type", mimetypes.guess_type(file.name))
+        self.end_headers()
+        shutil.copyfileobj(file.stream, self.wfile)
+    endpoints.append(("/{seed}/content/{cat}/{points}/{filename}", handle_puzzlefile))
+    
+
+    def handle_mothballer(self):
+        category = self.req.get("cat")
+        
+        try:
+            catdir = self.server.args["puzzles_dir"].joinpath(category)
+            mb = mothballer.package(category, str(catdir), self.seed)
+        except Exception as ex:
+            logging.exception(ex)
+            self.send_response(500)
+            self.send_header("Content-Type", "text/html; charset=\"utf-8\"")
+            self.end_headers()
+            self.wfile.write(cgitb.html(sys.exc_info()).encode("utf-8"))
+            return
+        
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet_stream")
+        self.end_headers()
+        shutil.copyfileobj(mb, self.wfile)
+    endpoints.append(("/{seed}/mothballer/{cat}.mb", handle_mothballer))
+
+
+    def handle_index(self):
+        seed = random.getrandbits(32)
+        self.send_response(307)
+        self.send_header("Location", "%s/" % seed)
+        self.send_header("Content-Type", "text/html")
+        self.end_headers()
+        self.wfile.write(b"Your browser was supposed to redirect you to <a href=\"%i/\">here</a>." % seed)
+    endpoints.append((r"/", handle_index))
+
+
+    def handle_theme_file(self):
+        self.path = "/" + self.req.get("path", "")
+        super().do_GET()
+    endpoints.append(("/{seed}/", handle_theme_file))
+    endpoints.append(("/{seed}/{path}", handle_theme_file))
+
+
+    def do_GET(self):
+        self.fields = cgi.FieldStorage(
+            fp=self.rfile,
+            headers=self.headers,
+            environ={
+                "REQUEST_METHOD": self.command,
+                "CONTENT_TYPE": self.headers["Content-Type"],
+            },
+        )
+
+        url = urllib.parse.urlparse(self.path)
+        for pattern, function in self.endpoints:
+            result = parse.parse(pattern, url.path)
+            if result:
+                self.req = result.named
+                seed = self.req.get("seed", "random")
+                if seed == "random":
+                    self.seed = random.getrandbits(32)
+                else:
+                    self.seed = int(seed)
+                return function(self)
+        super().do_GET()
+
+    def do_POST(self):
+        self.do_GET()
+
+    def do_HEAD(self):
+        self.send_error(
+            HTTPStatus.NOT_IMPLEMENTED,
+            "Unsupported method (%r)" % self.command,
+        )
+
+# I don't fully understand why you can't do this inside the class definition.
+MothRequestHandler.extensions_map[".mjs"] = "application/ecmascript"
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(description="MOTH puzzle development server")
+    parser.add_argument(
+        '--puzzles', default='puzzles',
+        help="Directory containing your puzzles"
+    )
+    parser.add_argument(
+        '--theme', default='theme',
+        help="Directory containing theme files")
+    parser.add_argument(
+        '--bind', default="127.0.0.1:8080",
+        help="Bind to ip:port"
+    )
+    parser.add_argument(
+        '--base', default="",
+        help="Base URL to this server, for reverse proxy setup"
+    )
+    parser.add_argument(
+        "-v", "--verbose",
+        action="count",
+        default=1,  # Leave at 1, for now, to maintain current default behavior
+        help="Include more verbose logging. Use multiple flags to increase level",
+    )
+    args = parser.parse_args()
+    parts = args.bind.split(":")
+    addr = parts[0] or "0.0.0.0"
+    port = int(parts[1])
+    if args.verbose >= 2:
+        log_level = logging.DEBUG
+    elif args.verbose == 1:
+        log_level = logging.INFO
+    else:
+        log_level = logging.WARNING
+    
+    logging.basicConfig(level=log_level)
+    
+    server = MothServer((addr, port), MothRequestHandler)
+    server.args["base_url"] = args.base
+    server.args["puzzles_dir"] = pathlib.Path(args.puzzles)
+    server.args["theme_dir"] = args.theme
+
+    logging.info("Listening on %s:%d", addr, port)
+    server.serve_forever()

devel/moth.py

@@ -0,0 +1,469 @@
+#!/usr/bin/python3
+
+import argparse
+import contextlib
+import copy
+import glob
+import hashlib
+import html
+import io
+import importlib.machinery
+import logging
+import mistune
+import os
+import random
+import string
+import sys
+import tempfile
+import shlex
+import yaml
+
+messageChars = b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+LOGGER = logging.getLogger(__name__)
+
+def djb2hash(str):
+    h = 5381
+    for c in str.encode("utf-8"):
+        h = ((h * 33) + c) & 0xffffffff
+    return h
+
+@contextlib.contextmanager
+def pushd(newdir):
+    curdir = os.getcwd()
+    LOGGER.debug("Attempting to chdir from %s to %s" % (curdir, newdir))
+    os.chdir(newdir)
+
+    # Force a copy of the old path, instead of just a reference
+    old_path = list(sys.path)
+    old_modules = copy.copy(sys.modules)
+    sys.path.append(newdir)
+
+    try:
+        yield
+    finally:
+        # Restore the old path
+        to_remove = []
+        for module in sys.modules:
+            if module not in old_modules:
+                to_remove.append(module)
+
+        for module in to_remove:
+            del(sys.modules[module])
+
+        sys.path = old_path
+        LOGGER.debug("Changing directory back from %s to %s" % (newdir, curdir))
+        os.chdir(curdir)
+
+
+def loadmod(name, path):
+    abspath = os.path.abspath(path)
+    loader = importlib.machinery.SourceFileLoader(name, abspath)
+    return loader.load_module()
+
+
+# Get a big list of clean words for our answer file.
+ANSWER_WORDS = [w.strip() for w in open(os.path.join(os.path.dirname(__file__),
+                                                     'answer_words.txt'))]
+
+class PuzzleFile:
+    """A file associated with a puzzle.
+
+    path: The path to the original input file. May be None (when this is created from a file handle
+          and there is no original input.
+    handle: A File-like object set to read the file from. You should be able to read straight
+            from it without having to seek to the beginning of the file.
+    name: The name of the output file.
+    visible: A boolean indicating whether this file should visible to the user. If False,
+             the file is still expected to be accessible, but it's path must be known
+             (or figured out) to retrieve it."""
+
+    def __init__(self, stream, name, visible=True):
+        self.stream = stream
+        self.name = name
+        self.visible = visible
+
+class PuzzleSuccess(dict):
+    """Puzzle success objectives
+
+    :param acceptable: Learning outcome from acceptable knowledge of the subject matter
+    :param mastery: Learning outcome from mastery of the subject matter
+    """
+
+    valid_fields = ["acceptable", "mastery"]
+
+    def __init__(self, **kwargs):
+        super(PuzzleSuccess, self).__init__()
+        for key in self.valid_fields:
+            self[key] = None
+        for key, value in kwargs.items():
+            if key in self.valid_fields:
+                self[key] = value
+
+    def __getattr__(self, attr):
+        if attr in self.valid_fields:
+            return self[attr]
+        raise AttributeError("'%s' object has no attribute '%s'" % (type(self).__name__, attr))
+
+    def __setattr__(self, attr, value):
+        if attr in self.valid_fields:
+            self[attr] = value
+        else:
+            raise AttributeError("'%s' object has no attribute '%s'" % (type(self).__name__, attr))
+
+
+class Puzzle:
+    def __init__(self, category_seed, points):
+        """A MOTH Puzzle.
+
+        :param category_seed: A byte string to use as a seed for random numbers for this puzzle.
+                              It is combined with the puzzle points.
+        :param points: The point value of the puzzle.
+        """
+
+        super().__init__()
+
+        self.points = points
+        self.summary = None
+        self.authors = []
+        self.answers = []
+        self.scripts = []
+        self.pattern = None
+        self.hint = None
+        self.files = {}
+        self.body = io.StringIO()
+
+        # NIST NICE objective content
+        self.objective = None  # Text describing the expected learning outcome from solving this puzzle, *why* are you solving this puzzle
+        self.success = PuzzleSuccess()  # Text describing criteria for different levels of success, e.g. {"Acceptable": "Did OK", "Mastery": "Did even better"}
+        self.solution = None  # Text describing how to solve the puzzle
+        self.ksas = []  # A list of references to related NICE KSAs (e.g. K0058, . . .)
+
+        self.logs = []
+        self.randseed = category_seed * self.points
+        self.rand = random.Random(self.randseed)
+
+    def log(self, *vals):
+        """Add a new log message to this puzzle."""
+        msg = ' '.join(str(v) for v in vals)
+        self.logs.append(msg)
+
+    def read_stream(self, stream):
+        header = True
+        line = ""
+        if stream.read(3) == "---":
+            header = "yaml"
+        else:
+            header = "moth"
+
+        stream.seek(0)
+
+        if header == "yaml":
+            LOGGER.info("Puzzle is YAML-formatted")
+            self.read_yaml_header(stream)
+        elif header == "moth":
+            LOGGER.info("Puzzle is MOTH-formatted")
+            self.read_moth_header(stream)
+                
+        for line in stream:
+            self.body.write(line)
+
+    def read_yaml_header(self, stream):
+        contents = ""
+        header = False
+        for line in stream:
+            if line.strip() == "---" and header:  # Handle last line
+                break
+            elif line.strip() == "---":  # Handle first line
+                header = True
+                continue
+            else:
+                contents += line
+
+        config = yaml.safe_load(contents)
+        for key, value in config.items():
+            key = key.lower()
+            self.handle_header_key(key, value)
+
+    def read_moth_header(self, stream):
+        for line in stream:
+            line = line.strip()
+            if not line:
+                break
+
+            key, val = line.split(':', 1)
+            key = key.lower()
+            val = val.strip()
+            self.handle_header_key(key, val)
+
+    def handle_header_key(self, key, val):
+        LOGGER.debug("Handling key: %s, value: %s", key, val)
+        if key == 'author':
+            self.authors.append(val)
+        elif key == 'authors':
+            if not isinstance(val, list):
+                raise ValueError("Authors must be a list, got %s, instead" & (type(val),))
+            self.authors = list(val)
+        elif key == 'summary':
+            self.summary = val
+        elif key == 'answer':
+            if not isinstance(val, str):
+                raise ValueError("Answers must be strings, got %s, instead" % (type(val),))
+            self.answers.append(val)
+        elif key == "answers":
+            for answer in val:
+                if not isinstance(answer, str):
+                    raise ValueError("Answers must be strings, got %s, instead" % (type(answer),))
+                self.answers.append(answer)
+        elif key == 'pattern':
+            self.pattern = val
+        elif key == 'hint':
+            self.hint = val
+        elif key == 'name':
+            pass
+        elif key == 'file':
+            parts = shlex.split(val)
+            name = parts[0]
+            hidden = False
+            LOGGER.debug("Attempting to open %s", os.path.abspath(name))
+            stream = open(name, 'rb')
+            try:
+                name = parts[1]
+                hidden = (parts[2].lower() == "hidden")
+            except IndexError:
+                pass
+            self.files[name] = PuzzleFile(stream, name, not hidden)
+        elif key == 'files':
+            for file in val:
+                path = file["path"]
+                stream = open(path, "rb")
+                name = file.get("name") or path
+                self.files[name] = PuzzleFile(stream, name, not file.get("hidden"))
+        elif key == 'script':
+            stream = open(val, 'rb')
+            self.add_script_stream(stream, val)
+        elif key == "objective":
+            self.objective = val
+        elif key == "success":
+            # Force success dictionary keys to be lower-case
+            self.success = dict((x.lower(), y) for x,y in val.items())
+        elif key == "success.acceptable":
+            self.success.acceptable = val
+        elif key == "success.mastery":
+            self.success.mastery = val
+        elif key == "solution":
+            self.solution = val
+        elif key == "ksas":
+            if not isinstance(val, list):
+                raise ValueError("KSAs must be a list, got %s, instead" & (type(val),))
+            self.ksas = val
+        elif key == "ksa":
+            self.ksas.append(val)
+        else:
+            raise ValueError("Unrecognized header field: {}".format(key))
+
+
+    def read_directory(self, path):
+        try:
+            puzzle_mod = loadmod("puzzle", os.path.join(path, "puzzle.py"))
+        except FileNotFoundError:
+            puzzle_mod = None
+
+        with pushd(path):
+            if puzzle_mod:
+                puzzle_mod.make(self)
+            elif os.path.exists('puzzle.moth'):
+                with open('puzzle.moth') as f:
+                    self.read_stream(f)
+            else:
+                self.authors = ["boggarts"]
+                self.body.write("This puzzle is broken! It has no puzzle.py or puzzle.moth.")
+
+    def random_hash(self):
+        """Create a file basename (no extension) with our number generator."""
+        return ''.join(self.rand.choice(string.ascii_lowercase) for i in range(8))
+
+    def make_temp_file(self, name=None, visible=True):
+        """Get a file object for adding dynamically generated data to the puzzle. When you're
+        done with this file, flush it, but don't close it.
+
+        :param name: The name of the file for links within the puzzle. If this is None, a name
+                     will be generated for you.
+        :param visible: Whether or not the file will be visible to the user.
+        :return: A file object for writing
+        """
+
+        stream = tempfile.TemporaryFile()
+        self.add_stream(stream, name, visible)
+        return stream
+
+    def add_script_stream(self, stream, name):
+        # Make sure this shows up in the header block of the HTML output.
+        self.files[name] = PuzzleFile(stream, name, visible=False)
+        self.scripts.append(name)
+
+    def add_stream(self, stream, name=None, visible=True):
+        if name is None:
+            name = self.random_hash()
+        self.files[name] = PuzzleFile(stream, name, visible)
+
+    def add_file(self, filename, visible=True):
+        fd = open(filename, 'rb')
+        name = os.path.basename(filename)
+        self.add_stream(fd, name=name, visible=visible)
+
+    def randword(self):
+        """Return a randomly-chosen word"""
+
+        return self.rand.choice(ANSWER_WORDS)
+
+    def make_answer(self, word_count=4, sep=' '):
+        """Generate and return a new answer. It's automatically added to the puzzle answer list.
+        :param int word_count: The number of words to include in the answer.
+        :param str|bytes sep: The word separator.
+        :returns: The answer string
+        """
+
+        words = [self.randword() for i in range(word_count)]
+        answer = sep.join(words)
+        self.answers.append(answer)
+        return answer
+
+    hexdump_stdch = stdch = (
+        '················'
+        '················'
+        ' !"#$%&\'()*+,-./'
+        '0123456789:;<=>?'
+        '@ABCDEFGHIJKLMNO'
+        'PQRSTUVWXYZ[\]^_'
+        '`abcdefghijklmno'
+        'pqrstuvwxyz{|}~·'
+        '················'
+        '················'
+        '················'
+        '················'
+        '················'
+        '················'
+        '················'
+        '················'
+    )
+
+    def hexdump(self, buf, charset=hexdump_stdch, gap=('<EFBFBD>', '⌷')):
+        hexes, chars = [], []
+        out = []
+
+        for b in buf:
+            if len(chars) == 16:
+                out.append((hexes, chars))
+                hexes, chars = [], []
+
+            if b is None:
+                h, c = gap
+            else:
+                h = '{:02x}'.format(b)
+                c = charset[b]
+            chars.append(c)
+            hexes.append(h)
+
+        out.append((hexes, chars))
+
+        offset = 0
+        elided = False
+        lastchars = None
+        self.body.write('<pre>')
+        for hexes, chars in out:
+            if chars == lastchars:
+                offset += len(chars)
+                if not elided:
+                    self.body.write('*\n')
+                    elided = True
+                continue
+            lastchars = chars[:]
+            elided = False
+
+            pad = 16 - len(chars)
+            hexes += ['  '] * pad
+
+            self.body.write('{:08x}  '.format(offset))
+            self.body.write(' '.join(hexes[:8]))
+            self.body.write('  ')
+            self.body.write(' '.join(hexes[8:]))
+            self.body.write('  |')
+            self.body.write(html.escape(''.join(chars)))
+            self.body.write('|\n')
+            offset += len(chars)
+        self.body.write('{:08x}\n'.format(offset))
+        self.body.write('</pre>')
+
+    def get_authors(self):
+        return self.authors or [self.author]
+
+    def get_body(self):
+        return self.body.getvalue()
+
+    def html_body(self):
+        """Format and return the markdown for the puzzle body."""
+        return mistune.markdown(self.get_body(), escape=False)
+
+    def package(self, answers=False):
+        """Return a dict packaging of the puzzle."""
+
+        files = [fn for fn,f in self.files.items() if f.visible]
+        hidden = [fn for fn,f in self.files.items() if not f.visible]
+        return {
+            'authors': self.get_authors(),
+            'hashes': self.hashes(),
+            'files': files,
+            'hidden': hidden,
+            'scripts': self.scripts,
+            'pattern': self.pattern,
+            'body': self.html_body(),
+            'objective': self.objective,
+            'success': self.success,
+            'solution': self.solution,
+            'ksas': self.ksas,
+        }
+
+    def hashes(self):
+        "Return a list of answer hashes"
+
+        return [djb2hash(a) for a in self.answers]
+
+
+class Category:
+    def __init__(self, path, seed):
+        self.path = path
+        self.seed = seed
+        self.catmod = None
+
+        try:
+            self.catmod = loadmod('category', os.path.join(path, 'category.py'))
+        except FileNotFoundError:
+            self.catmod = None
+
+    def pointvals(self):
+        if self.catmod:
+            with pushd(self.path):
+                pointvals = self.catmod.pointvals()
+        else:
+            pointvals = []
+            for fpath in glob.glob(os.path.join(self.path, "[0-9]*")):
+                pn = os.path.basename(fpath)
+                points = int(pn)
+                pointvals.append(points)
+        return sorted(pointvals)
+
+    def puzzle(self, points):
+        puzzle = Puzzle(self.seed, points)
+        path = os.path.join(self.path, str(points))
+        if self.catmod:
+            with pushd(self.path):
+                self.catmod.make(points, puzzle)
+        else:
+            with pushd(self.path):
+                puzzle.read_directory(path)
+        return puzzle
+
+    def __iter__(self):
+        for points in self.pointvals():
+            yield self.puzzle(points)

devel/mothballer.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+import argparse
+import binascii
+import hashlib
+import io
+import json
+import logging
+import moth
+import os
+import shutil
+import tempfile
+import zipfile
+import random
+
+SEEDFN = "SEED"
+
+
+def write_kv_pairs(ziphandle, filename, kv):
+    """ Write out a sorted map to file
+    :param ziphandle: a zipfile object
+    :param filename: The filename to write within the zipfile object
+    :param kv:  the map to write out
+    :return:
+    """
+    filehandle = io.StringIO()
+    for key in sorted(kv.keys()):
+        if isinstance(kv[key], list):
+            for val in kv[key]:
+                filehandle.write("%s %s\n" % (key, val))
+        else:
+            filehandle.write("%s %s\n" % (key, kv[key]))
+    filehandle.seek(0)
+    ziphandle.writestr(filename, filehandle.read())
+
+
+def escape(s):
+    return s.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
+
+
+def build_category(categorydir, outdir):
+    category_seed = random.getrandbits(32)
+
+    categoryname = os.path.basename(categorydir.strip(os.sep))
+    zipfilename = os.path.join(outdir, "%s.mb" % categoryname)
+    logging.info("Building {} from {}".format(zipfilename, categorydir))
+
+    if os.path.exists(zipfilename):
+        # open and gather some state
+        existing = zipfile.ZipFile(zipfilename, 'r')
+        try:
+            category_seed = int(existing.open(SEEDFN).read().strip())
+        except Exception:
+            pass
+        existing.close()
+    logging.debug("Using PRNG seed {}".format(category_seed))
+
+    zipfileraw = tempfile.NamedTemporaryFile(delete=False)
+    mothball = package(categoryname, categorydir, category_seed)
+    shutil.copyfileobj(mothball, zipfileraw)
+    zipfileraw.close()
+    shutil.move(zipfileraw.name, zipfilename)
+
+
+# Returns a file-like object containing the contents of the new zip file
+def package(categoryname, categorydir, seed):
+    zfraw = io.BytesIO()
+    zf = zipfile.ZipFile(zfraw, 'x')
+    zf.writestr("category_seed.txt", str(seed))
+
+    cat = moth.Category(categorydir, seed)
+    answers = {}
+    summary = {}
+    for puzzle in cat:
+        logging.info("Processing point value {}".format(puzzle.points))
+
+        answers[puzzle.points] = puzzle.answers
+        summary[puzzle.points] = puzzle.summary
+
+        puzzledir = os.path.join("content", str(puzzle.points))
+        for fn, f in puzzle.files.items():
+            payload = f.stream.read()
+            zf.writestr(os.path.join(puzzledir, fn), payload)
+
+        obj = puzzle.package()
+        zf.writestr(os.path.join(puzzledir, 'puzzle.json'), json.dumps(obj))
+
+    write_kv_pairs(zf, 'answers.txt', answers)
+    write_kv_pairs(zf, 'summaries.txt', summary)
+
+    # clean up
+    zf.close()
+    zfraw.seek(0)
+    return zfraw
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Build a category package')
+    parser.add_argument('outdir', help='Output directory')
+    parser.add_argument('categorydirs', nargs='+', help='Directory of category source')
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG)
+
+    outdir = os.path.abspath(args.outdir)
+    for categorydir in args.categorydirs:
+        categorydir = os.path.abspath(categorydir)
+        build_category(categorydir, outdir)

devel/mothd.service

@@ -0,0 +1,18 @@
+# To install:
+#    sudo cp mothd.service /etc/systemd/system/moth.service
+#    sudo systemctl enable mothd
+#    sudo systemctl start mothd
+
+[Unit]
+Description=Monarch Of The Hill server
+After=network.target auditd.service
+
+[Service]
+WorkingDirectory=/srv/moth
+User=www-data
+ExecStart=/srv/moth/mothd
+KillMode=process
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

devel/setup.cfg

@@ -0,0 +1,8 @@
+[flake8]
+# flake8 is an automated code formatting pedant.
+# Use it, please.
+#
+#     python3 -m flake8 .
+#
+ignore = E501
+exclude = .git

devel/update-words.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+set +e
+
+url='https://rawgit.com/first20hours/google-10000-english/master/google-10000-english-no-swears.txt'
+getter="curl -sL"
+fn="answer_words.txt"
+
+filterer() {
+	grep '......*'
+}
+
+if ! curl -h >/dev/null 2>/dev/null; then
+	getter="wget -q -O -"
+elif ! wget -h >/dev/null 2>/dev/null; then
+	echo "[!] I don't know how to download. I need curl or wget."
+fi
+
+$getter "${url}" | filterer > ${fn}.tmp \
+	&& mv -f ${fn}.tmp ${fn}

devel/validate.py

@@ -0,0 +1,229 @@
+#!/usr/bin/python3
+
+"""A validator for MOTH puzzles"""
+
+import logging
+import os
+import os.path
+import re
+
+import moth
+
+# pylint: disable=len-as-condition, line-too-long
+
+DEFAULT_REQUIRED_FIELDS = ["answers", "authors", "summary"]
+
+LOGGER = logging.getLogger(__name__)
+
+
+class MothValidationError(Exception):
+
+    """An exception for encapsulating MOTH puzzle validation errors"""
+
+
+class MothValidator:
+
+    """A class which validates MOTH categories"""
+
+    def __init__(self, fields):
+        self.required_fields = fields
+        self.results = {"category": {}, "checks": []}
+
+    def validate(self, categorydir, only_errors=False):
+        """Run validation checks against a category"""
+        LOGGER.debug("Loading category from %s", categorydir)
+        try:
+            category = moth.Category(categorydir, 0)
+        except NotADirectoryError:
+            return
+
+        LOGGER.debug("Found %d puzzles in %s", len(category.pointvals()), categorydir)
+
+        self.results["category"][categorydir] = {
+            "puzzles": {},
+            "name": os.path.basename(categorydir.strip(os.sep)),
+        }
+        curr_category = self.results["category"][categorydir]
+
+        for check_function_name in [x for x in dir(self) if x.startswith("check_") and callable(getattr(self, x))]:
+            if check_function_name not in self.results["checks"]:
+                self.results["checks"].append(check_function_name)
+
+        for puzzle in category:
+            LOGGER.info("Processing %s: %s", categorydir, puzzle.points)
+
+            curr_category["puzzles"][puzzle.points] = {}
+            curr_puzzle = curr_category["puzzles"][puzzle.points]
+            curr_puzzle["failures"] = []
+
+            for check_function_name in [x for x in dir(self) if x.startswith("check_") and callable(getattr(self, x))]:
+                check_function = getattr(self, check_function_name)
+                LOGGER.debug("Running %s on %d", check_function_name, puzzle.points)
+
+                try:
+                    check_function(puzzle)
+                except MothValidationError as ex:
+                    curr_puzzle["failures"].append(str(ex))
+
+            if only_errors and len(curr_puzzle["failures"]) == 0:
+                del curr_category["puzzles"][puzzle.points]
+
+    def check_fields(self, puzzle):
+        """Check if the puzzle has the requested fields"""
+        for field in self.required_fields:
+            if not hasattr(puzzle, field) or \
+                    getattr(puzzle,field) is None or \
+                    getattr(puzzle,field) == "":
+                raise MothValidationError("Missing field %s" % (field,))
+
+    @staticmethod
+    def check_has_answers(puzzle):
+        """Check if the puzle has answers defined"""
+        if len(puzzle.answers) == 0:
+            raise MothValidationError("No answers provided")
+
+    @staticmethod
+    def check_unique_answers(puzzle):
+        """Check if puzzle answers are unique"""
+        known_answers = []
+        duplicate_answers = []
+
+        for answer in puzzle.answers:
+            if answer not in known_answers:
+                known_answers.append(answer)
+            else:
+                duplicate_answers.append(answer)
+
+        if len(duplicate_answers) > 0:
+            raise MothValidationError("Duplicate answer(s) %s" % ", ".join(duplicate_answers))
+
+    @staticmethod
+    def check_has_authors(puzzle):
+        """Check if the puzzle has authors defined"""
+        if len(puzzle.authors) == 0:
+            raise MothValidationError("No authors provided")
+
+    @staticmethod
+    def check_unique_authors(puzzle):
+        """Check if puzzle authors are unique"""
+        known_authors = []
+        duplicate_authors = []
+
+        for author in puzzle.authors:
+            if author not in known_authors:
+                known_authors.append(author)
+            else:
+                duplicate_authors.append(author)
+
+        if len(duplicate_authors) > 0:
+            raise MothValidationError("Duplicate author(s) %s" % ", ".join(duplicate_authors))
+
+    @staticmethod
+    def check_has_summary(puzzle):
+        """Check if the puzzle has a summary"""
+        if puzzle.summary is None:
+            raise MothValidationError("Summary has not been provided")
+
+    @staticmethod
+    def check_has_body(puzzle):
+        """Check if the puzzle has a body defined"""
+        old_pos = puzzle.body.tell()
+        puzzle.body.seek(0)
+        if len(puzzle.body.read()) == 0:
+            puzzle.body.seek(old_pos)
+            raise MothValidationError("No body provided")
+
+        puzzle.body.seek(old_pos)
+
+    @staticmethod
+    def check_ksa_format(puzzle):
+        """Check if KSAs are properly formatted"""
+
+        ksa_re = re.compile("^[KSA]\d{4}$")
+        
+        if hasattr(puzzle, "ksa"):
+            for ksa in puzzle.ksa:
+                if ksa_re.match(ksa) is None:
+                    raise MothValidationError("Unrecognized KSA format (%s)" % (ksa,))
+
+    @staticmethod
+    def check_success(puzzle):
+        """Check if success criteria are defined"""
+
+        if not hasattr(puzzle, "success"):
+            raise MothValidationError("Success not defined")
+
+        criteria = ["acceptable", "mastery"]
+        missing_criteria = []
+        for criterion in criteria:
+            if criterion not in puzzle.success.keys() or \
+                    puzzle.success[criterion] is None or \
+                    len(puzzle.success[criterion]) == 0:
+                missing_criteria.append(criterion)
+
+        if len(missing_criteria) > 0:
+            raise MothValidationError("Missing success criteria (%s)" % (", ".join(missing_criteria)))
+
+
+def output_json(data):
+    """Output results in JSON format"""
+    import json
+    print(json.dumps(data))
+
+
+def output_text(data):
+    """Output results in a text-based tabular format"""
+
+    longest_category = max([len(y["name"]) for x, y in data["category"].items()])
+    longest_category = max([longest_category, len("Category")])
+    longest_failure = len("Failures")
+    for category_data in data["category"].values():
+        for points, puzzle_data in category_data["puzzles"].items():
+            longest_failure = max([longest_failure, len(", ".join(puzzle_data["failures"]))])
+
+    formatstr = "| %%%ds | %%6s | %%%ds |" % (longest_category, longest_failure)
+    headerfmt = formatstr % ("Category", "Points", "Failures")
+
+    print(headerfmt)
+    for cat_data in data["category"].values():
+        for points, puzzle_data in sorted(cat_data["puzzles"].items()):
+            print(formatstr % (cat_data["name"], points, ", ".join([str(x) for x in puzzle_data["failures"]])))
+
+
+def main():
+    """Main function"""
+    # pylint: disable=invalid-name
+    import argparse
+
+    LOGGER.addHandler(logging.StreamHandler())
+
+    parser = argparse.ArgumentParser(description="Validate MOTH puzzle field compliance")
+    parser.add_argument("category", nargs="+", help="Categories to validate")
+    parser.add_argument("-f", "--fields", help="Comma-separated list of fields to check for", default=",".join(DEFAULT_REQUIRED_FIELDS))
+
+    parser.add_argument("-o", "--output-format", choices=["text", "json"], default="text", help="Output format (default: text)")
+    parser.add_argument("-e", "--only-errors", action="store_true", default=False, help="Only output errors")
+    parser.add_argument("-v", "--verbose", action="count", default=0, help="Increase verbosity of output, repeat to increase")
+
+    args = parser.parse_args()
+
+    if args.verbose == 1:
+        LOGGER.setLevel("INFO")
+    elif args.verbose > 1:
+        LOGGER.setLevel("DEBUG")
+
+    LOGGER.debug(args)
+    validator = MothValidator(args.fields.split(","))
+
+    for category in args.category:
+        LOGGER.info("Validating %s", category)
+        validator.validate(category, only_errors=args.only_errors)
+
+    if args.output_format == "text":
+        output_text(validator.results)
+    elif args.output_format == "json":
+        output_json(validator.results)
+
+
+if __name__ == "__main__":
+    main()

theme/moth.js

@@ -98,7 +98,8 @@ function renderPuzzles(obj) {
 function renderState(obj) {
   devel = obj.Config.Devel
   if (devel) {
-    sessionStorage.id = "1234"
+    let params = new URLSearchParams(window.location.search)
+    sessionStorage.id = "1"
     sessionStorage.pid = "rodney"
   }
   if (Object.keys(obj.Puzzles).length > 0) {