From 8899927b9fb3bbb5fa53b8bbf3a342ea0b7c1616 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Thu, 5 Jan 2017 16:50:41 -0700 Subject: [PATCH] Productionize puzzle packager --- install | 75 ----------------------- package-puzzles | 127 --------------------------------------- tools/build-puzzles.py | 84 -------------------------- tools/moth.py | 5 +- tools/package-puzzles.py | 112 ++++++++++++++++++++++++++++++++++ 5 files changed, 115 insertions(+), 288 deletions(-) delete mode 100755 install delete mode 100755 package-puzzles delete mode 100755 tools/build-puzzles.py create mode 100755 tools/package-puzzles.py diff --git a/install b/install deleted file mode 100755 index 8b5f77b..0000000 --- a/install +++ /dev/null @@ -1,75 +0,0 @@ -#! /bin/sh - -DESTDIR=$1 - -if [ -z "$DESTDIR" ]; then - echo "Usage: $0 DESTDIR" - exit -fi - -cd $(dirname $0) - -older () { - [ -z "$1" ] && return 1 - target=$1; shift - [ -f $target ] || return 0 - for i in "$@"; do - [ $i -nt $target ] && return 0 - done - return 1 -} - -copy () { - target=$DESTDIR/$1 - if older $target $1; then - echo "COPY $1" - mkdir -p $(dirname $target) - cp $1 $target - fi -} - -setup() { - [ -d $DESTDIR/state ] && return - echo "SETUP" - for i in points.new points.tmp teams; do - dir=$DESTDIR/state/$i - mkdir -p $dir - setfacl -m ${www}:rwx $dir - done - mkdir -p $DESTDIR/packages - >> $DESTDIR/state/points.log - if ! [ -f $DESTDIR/assigned.txt ]; then - hd $DESTDIR/assigned.txt - fi -} - - -echo "Figuring out web user..." -for www in www-data http tc _ _www; do - id $www && break -done -if [ $www = _ ]; then - echo "Unable to determine httpd user on this system. Dying." - exit 1 -fi - -mkdir -p $DESTDIR || exit 1 - -setup -git ls-files | while read fn; do - case "$fn" in - install|.*) - ;; - doc/*) - ;; - www/*) - copy $fn - ;; - bin/*) - copy $fn - ;; - *) - echo "??? $fn" - ;; - esac -done diff --git a/package-puzzles b/package-puzzles deleted file mode 100755 index f730a17..0000000 --- a/package-puzzles +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import binascii -import glob -import hashlib -import io -import json -import os -import shutil -import sys -import zipfile - -import puzzles - -TMPFILE = "%s.tmp" - -def write_kv_pairs(ziphandle, filename, kv): - """ Write out a sorted map to file - :param ziphandle: a zipfile object - :param filename: The filename to write within the zipfile object - :param kv: the map to write out - :return: - """ - filehandle = io.StringIO() - for key in sorted(kv.keys()): - if type(kv[key]) == type([]): - for val in kv[key]: - filehandle.write("%s: %s%s" % (key, val, os.linesep)) - else: - filehandle.write("%s: %s%s" % (key, kv[key], os.linesep)) - filehandle.seek(0) - ziphandle.writestr(filename, filehandle.read()) - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Build a category package') - parser.add_argument('categorydirs', nargs='+', help='Directory of category source') - parser.add_argument('outdir', help='Output directory') - args = parser.parse_args() - - for categorydir in args.categorydirs: - puzzles_dict = {} - secrets = {} - - categoryname = os.path.basename(categorydir.strip(os.sep)) - - # create zipfile - zipfilename = os.path.join(args.outdir, "%s.zip" % categoryname) - if os.path.exists(zipfilename): - # open and gather some state - zf = zipfile.ZipFile(zipfilename, 'r') - try: - category_seed = zf.open(os.path.join(categoryname, "category_seed.txt")).read().strip() - except: - pass - zf.close() - - zf = zipfile.ZipFile(TMPFILE % zipfilename, 'w') - if 'category_seed' not in locals(): - category_seed = binascii.b2a_hex(os.urandom(20)) - - # read in category details (will be pflarr in future) - for categorypath in glob.glob(os.path.join(categorydir, "*", "puzzle.moth")): - points = categorypath.split(os.sep)[-2] # directory before '/puzzle.moth' - categorypath = os.path.dirname(categorypath) - print(categorypath) - try: - points = int(points) - except: - print("Failed to identify points on: %s" % categorypath, file=sys.stderr) - continue - puzzle = puzzles.Puzzle(category_seed, categorypath) - puzzles_dict[points] = puzzle - - # build mapping, answers, and summary - mapping = {} - answers = {} - summary = {} - for points in sorted(puzzles_dict): - puzzle = puzzles_dict[points] - hashmap = hashlib.sha1(category_seed) - hashmap.update(str(points).encode('utf-8')) - mapping[points] = hashmap.hexdigest() - answers[points] = puzzle['answer'] - if len(puzzle['summary']) > 0: - summary[points] = puzzle['summary'] - - # write mapping, answers, summary, category_seed - write_kv_pairs(zf, os.path.join(categoryname, 'map.txt'), mapping) - write_kv_pairs(zf, os.path.join(categoryname, 'answers.txt'), answers) - write_kv_pairs(zf, os.path.join(categoryname, 'summary.txt'), summary) - zf.writestr(os.path.join(categoryname, "category_seed.txt"), category_seed) - - # write out puzzles - for points in sorted(puzzles_dict): - puzzle = puzzles_dict[points] - puzzledir = os.path.join(categoryname, 'content', mapping[points]) - puzzledict = { - 'author': puzzle.author, - 'hashes': puzzle.hashes(), - 'files': [f.name for f in puzzle.files if f.visible], - 'body': puzzle.html_body(), - } - secretsdict = { - 'summary': puzzle.summary, - 'answers': puzzle.answers, - } - - # write associated files - assoc_files = [] - for fobj in puzzle['files']: - if fobj.visible == True: - assoc_files.append(fobj.name) - zf.writestr(os.path.join(puzzledir, fobj.name), \ - fobj.handle.read()) - - if len(assoc_files) > 0: - puzzlejson["associated_files"] = assoc_files - - # non-optimal writing of file-like objects, but it works - zf.writestr(os.path.join(puzzledir, 'puzzle.json'), \ - json.dumps(puzzlejson)) - - # clean up - zf.close() - shutil.move(TMPFILE % zipfilename, zipfilename) -#vim:py diff --git a/tools/build-puzzles.py b/tools/build-puzzles.py deleted file mode 100755 index 8138e83..0000000 --- a/tools/build-puzzles.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/python3 - -import hmac -import base64 -import argparse -import glob -import json -import os -import markdown -import random - -messageChars = b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' - -def djb2hash(buf): - h = 5381 - for c in buf: - h = ((h * 33) + c) & 0xffffffff - return h - -class Puzzle: - def __init__(self, stream): - self.message = bytes(random.choice(messageChars) for i in range(20)) - self.fields = {} - self.answers = [] - self.hashes = [] - - body = [] - header = True - for line in stream: - if header: - line = line.strip() - if not line.strip(): - header = False - continue - key, val = line.split(':', 1) - key = key.lower() - val = val.strip() - self._add_field(key, val) - else: - body.append(line) - self.body = ''.join(body) - - def _add_field(self, key, val): - if key == 'answer': - h = djb2hash(val.encode('utf8')) - self.answers.append(val) - self.hashes.append(h) - else: - self.fields[key] = val - - def publish(self): - obj = { - 'author': self.fields['author'], - 'hashes': self.hashes, - 'body': markdown.markdown(self.body), - } - return obj - - def secrets(self): - obj = { - 'answers': self.answers, - 'summary': self.fields['summary'], - } - return obj - - -parser = argparse.ArgumentParser(description='Build a puzzle category') -parser.add_argument('puzzledir', nargs='+', help='Directory of puzzle source') -args = parser.parse_args() - -for puzzledir in args.puzzledir: - puzzles = {} - secrets = {} - for puzzlePath in glob.glob(os.path.join(puzzledir, "*.moth")): - filename = os.path.basename(puzzlePath) - points, ext = os.path.splitext(filename) - points = int(points) - puzzle = Puzzle(open(puzzlePath)) - puzzles[points] = puzzle - - for points in sorted(puzzles): - puzzle = puzzles[points] - print(puzzle.secrets()) - diff --git a/tools/moth.py b/tools/moth.py index 017f882..9e842d4 100644 --- a/tools/moth.py +++ b/tools/moth.py @@ -86,6 +86,7 @@ class Puzzle: continue key, val = line.split(':', 1) key = key.lower() + val = val.strip() if key == 'author': self.author = val elif key == 'summary': @@ -178,7 +179,7 @@ class Puzzle: def hashes(self): "Return a list of answer hashes" - return [djbhash(a) for a in self.answers] + return [djb2hash(a.encode('utf-8')) for a in self.answers] class Category: @@ -212,6 +213,6 @@ class Category: puzzle.read_directory(path) return puzzle - def puzzles(self): + def __iter__(self): for points in self.pointvals: yield self.puzzle(points) diff --git a/tools/package-puzzles.py b/tools/package-puzzles.py new file mode 100755 index 0000000..90b248b --- /dev/null +++ b/tools/package-puzzles.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +import argparse +import binascii +import glob +import hashlib +import io +import json +import logging +import moth +import os +import shutil +import string +import sys +import tempfile +import zipfile + +def write_kv_pairs(ziphandle, filename, kv): + """ Write out a sorted map to file + :param ziphandle: a zipfile object + :param filename: The filename to write within the zipfile object + :param kv: the map to write out + :return: + """ + filehandle = io.StringIO() + for key in sorted(kv.keys()): + if type(kv[key]) == type([]): + for val in kv[key]: + filehandle.write("%s: %s%s" % (key, val, os.linesep)) + else: + filehandle.write("%s: %s%s" % (key, kv[key], os.linesep)) + filehandle.seek(0) + ziphandle.writestr(filename, filehandle.read()) + +def build_category(categorydir, outdir): + zipfileraw = tempfile.NamedTemporaryFile(delete=False) + zf = zipfile.ZipFile(zipfileraw, 'x') + + category_seed = binascii.b2a_hex(os.urandom(20)) + puzzles_dict = {} + secrets = {} + + categoryname = os.path.basename(categorydir.strip(os.sep)) + seedfn = os.path.join("category_seed.txt") + zipfilename = os.path.join(outdir, "%s.zip" % categoryname) + logging.info("Building {} from {}".format(zipfilename, categorydir)) + + if os.path.exists(zipfilename): + # open and gather some state + existing = zipfile.ZipFile(zipfilename, 'r') + try: + category_seed = existing.open(seedfn).read().strip() + except: + pass + existing.close() + logging.debug("Using PRNG seed {}".format(category_seed)) + + zf.writestr(seedfn, category_seed) + + cat = moth.Category(categorydir, category_seed) + mapping = {} + answers = {} + summary = {} + for puzzle in cat: + logging.info("Processing point value {}".format(puzzle.points)) + + hashmap = hashlib.sha1(category_seed) + hashmap.update(str(puzzle.points).encode('utf-8')) + puzzlehash = hashmap.hexdigest() + + mapping[puzzle.points] = puzzlehash + answers[puzzle.points] = puzzle.answers + summary[puzzle.points] = puzzle.summary + + puzzledir = os.path.join('content', puzzlehash) + files = [] + for fn, f in puzzle.files.items(): + if f.visible: + files.append(fn) + payload = f.stream.read() + zf.writestr(os.path.join(puzzledir, fn), payload) + + puzzledict = { + 'author': puzzle.author, + 'hashes': puzzle.hashes(), + 'files': files, + 'body': puzzle.html_body(), + } + puzzlejson = json.dumps(puzzledict) + zf.writestr(os.path.join(puzzledir, 'puzzle.json'), puzzlejson) + + write_kv_pairs(zf, 'map.txt', mapping) + write_kv_pairs(zf, 'answers.txt', answers) + write_kv_pairs(zf, 'summaries.txt', summary) + + # clean up + zf.close() + + shutil.move(zipfileraw.name, zipfilename) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Build a category package') + parser.add_argument('categorydirs', nargs='+', help='Directory of category source') + parser.add_argument('outdir', help='Output directory') + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG) + + for categorydir in args.categorydirs: + build_category(categorydir, args.outdir) +