From e460f5b6b3e70a669da982bd2e2e1f15d431710f Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Thu, 20 Sep 2018 03:44:34 +0000 Subject: [PATCH] Cache points.json and puzzles.json --- Dockerfile.moth-compile | 7 -- Dockerfile.moth => Dockerfile.mothd | 0 bin/httpd | 16 ---- bin/install-category | 26 ------ bin/new | 37 -------- bin/once | 95 --------------------- bin/points | 45 ---------- bin/puzzles | 53 ------------ bin/server-start | 5 -- {bin => contrib}/award | 0 {bin => contrib}/mktokens | 0 {tools => devel}/answer_words.txt | 0 {tools => devel}/devel-server.py | 0 {tools => devel}/mistune.py | 0 {tools => devel}/moth.py | 0 {tools => devel}/mothd.service | 0 {tools => devel}/package-puzzles.py | 0 setup.cfg => devel/setup.cfg | 0 src/handlers.go | 127 ++++------------------------ src/instance.go | 2 + src/maintenance.go | 124 +++++++++++++++++++++++++-- src/mothball.go | 16 ++-- tools/mothd | 14 --- 23 files changed, 140 insertions(+), 427 deletions(-) delete mode 100644 Dockerfile.moth-compile rename Dockerfile.moth => Dockerfile.mothd (100%) delete mode 100755 bin/httpd delete mode 100755 bin/install-category delete mode 100755 bin/new delete mode 100755 bin/once delete mode 100755 bin/points delete mode 100755 bin/puzzles delete mode 100755 bin/server-start rename {bin => contrib}/award (100%) rename {bin => contrib}/mktokens (100%) rename {tools => devel}/answer_words.txt (100%) rename {tools => devel}/devel-server.py (100%) rename {tools => devel}/mistune.py (100%) rename {tools => devel}/moth.py (100%) rename {tools => devel}/mothd.service (100%) rename {tools => devel}/package-puzzles.py (100%) rename setup.cfg => devel/setup.cfg (100%) delete mode 100755 tools/mothd diff --git a/Dockerfile.moth-compile b/Dockerfile.moth-compile deleted file mode 100644 index 5680e4f..0000000 --- a/Dockerfile.moth-compile +++ /dev/null @@ -1,7 +0,0 @@ -FROM alpine - -RUN apk --no-cache add python3 py3-pillow - -COPY tools/package-puzzles.py tools/moth.py tools/mistune.py tools/answer_words.txt /moth/ - -ENTRYPOINT ["python3", "/moth/package-puzzles.py"] diff --git a/Dockerfile.moth b/Dockerfile.mothd similarity index 100% rename from Dockerfile.moth rename to Dockerfile.mothd diff --git a/bin/httpd b/bin/httpd deleted file mode 100755 index 854aa5b..0000000 --- a/bin/httpd +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -# Starts a standalone server using tcpsvd and eris - -echo "Figuring out web user..." -for www in www-data http tc _ _www; do - id $www && break -done -if [ $www = _ ]; then - echo "Unable to determine httpd user on this system. Dying." - exit 1 -fi - -cd $(dirname $0)/../www -tcpserver -RHI localhost -u $www -g $www 0 80 eris -c -. - diff --git a/bin/install-category b/bin/install-category deleted file mode 100755 index 060524e..0000000 --- a/bin/install-category +++ /dev/null @@ -1,26 +0,0 @@ -#! /bin/sh -e - -package=$1 -if ! [ -n "$package" -a -f $package ]; then - echo "Usage: $0 PACKAGE" - exit 1 -fi -shift - - -cat=$(basename $package .zip) -outdir=$(dirname $(dirname $0))/packages/$cat - -echo "Extracting to $outdir..." -mkdir -p $outdir -unzip -o -d $outdir $package - -echo "Fixing permissions..." -chmod a+rx $outdir/*/ $outdir/*/* -chmod -R a+r $outdir -find $outdir/content -name \*.cgi -exec chmod a+rx {} \; - -if [ ! -h $outdir/../../www/$cat ]; then - echo "Linking into web space..." - ln -sf ../packages/$cat/content $outdir/../../www/$cat -fi diff --git a/bin/new b/bin/new deleted file mode 100755 index e765e88..0000000 --- a/bin/new +++ /dev/null @@ -1,37 +0,0 @@ -#! /bin/sh - -newdir=$1 -if [ -z "$newdir" ]; then - echo "Usage: $0 NEWDIR" - exit 1 -fi - -KOTH_BASE=$(cd $(dirname $0)/.. && pwd) - -echo "Figuring out web user..." -for www in www-data http _; do - id $www && break -done - -if [ $www = _ ]; then - echo "Unable to determine httpd user on this system. Dying." - exit 1 -fi - -mkdir -p $newdir -cd $newdir - -for i in points.new points.tmp teams; do - mkdir -p state/$i - setfacl -m ${www}:rwx state/$i -done - ->> state/points.log - -if ! [ -f assigned.txt ]; then - hd < /dev/urandom | awk '{print $3 $4 $5 $6;}' | head -n 100 > assigned.txt -fi - -mkdir -p www -cp -r $KOTH_BASE/html/* www/ -cp $KOTH_BASE/bin/*.cgi www/ diff --git a/bin/once b/bin/once deleted file mode 100755 index 388ba21..0000000 --- a/bin/once +++ /dev/null @@ -1,95 +0,0 @@ -#! /bin/sh - -if [ -n "$1" ]; then - cd $1 -else - cd $(dirname $0)/.. -fi -basedir=$(pwd) - -log () { - echo "moth: $@" 1>&2 -} - -# Do nothing if `disabled` is present -if [ -f state/disabled ]; then - log "Instance disabled; doing nothing" - exit -fi - -# Are we stopping at a certain time? -if [ -f state/until ]; then - read -r until < state/until - when=$(date -d "$until" +%s) - now=$(date +%s) - if [ $now -ge $when ]; then - log "End time reached; doing nothing" - exit - fi -fi - -# Reset to initial state? -if [ ! -f state/initialized ]; then - log "Resetting contest state" - - rm -rf state/teams state/points.new state/points.tmp - mkdir -p state/teams state/points.new state/points.tmp - chown www:www state/teams state/points.new state/points.tmp # Needs root. Use Docker. - : > state/points.log - echo 'Remove this file to obliterate teams and points' > state/initialized -fi - -# Create some team names if needed -if [ ! -f state/assigned.txt ]; then - log "Generating team names" - hd state/assigned.txt -fi - -# Install new categories -for pkg in puzzles/*; do - cat=$(basename $pkg .zip) - if [ ! -f packages/$cat/installed ] || [ $pkg -nt packages/$cat/installed ]; then - log "Installing $pkg" - bin/install-category $pkg - : >packages/$cat/installed - fi -done - -# Helpful error message -if [ $(ls packages | wc -l) -eq 0 ]; then - log "error: No packages installed" - exit -fi - -# Create a list of currently-active categories -: > state/categories.txt.new -for dn in packages/*; do - cat=${dn##packages/} - echo "$cat" >> state/categories.txt.new -done -mv state/categories.txt.new state/categories.txt - -# Collect new points -find state/points.new -type f | while read fn; do - # Skip files opened by another process - lsof $fn | grep -q $fn && continue - - # Skip partially written files - [ $(wc -l < $fn) -gt 0 ] || continue - - # filter the file for unique awards - sort -k 4 $fn | uniq -f 1 | sort -n >> state/points.log - - # Now kill the file - rm -f $fn -done - -# Generate new puzzles.json -if bin/puzzles $basedir > state/puzzles.json.new; then - mv state/puzzles.json.new state/puzzles.json -fi - -# Generate new points.json -if bin/points $basedir > state/points.json.new; then - mv state/points.json.new state/points.json -fi diff --git a/bin/points b/bin/points deleted file mode 100755 index 79a4e5d..0000000 --- a/bin/points +++ /dev/null @@ -1,45 +0,0 @@ -#! /usr/bin/lua5.3 - -local basedir = arg[1] -local statedir = basedir .. "/state" - -io.write('{\n "points": [\n') -local teams = {} -local teamnames = {} -local nteams = 0 -local NR = 0 - -for line in io.lines(statedir .. "/points.log") do - local ts, hash, cat, points = line:match("(%d+) (%g+) (%g+) (%d+)") - local teamno = teams[hash] - - if not teamno then - teamno = nteams - teams[hash] = teamno - nteams = nteams + 1 - - teamnames[hash] = io.lines(statedir .. "/teams/" .. hash)() - end - - if NR > 0 then - -- JSON sucks, barfs if you have a comma with nothing after it - io.write(",\n") - end - NR = NR + 1 - - io.write(' [' .. ts .. ', "' .. teamno .. '", "' .. cat .. '", ' .. points .. ']') -end - -io.write('\n],\n "teams": {\n') - -NR = 0 -for hash,teamname in pairs(teamnames) do - if NR > 0 then - io.write(",\n") - end - NR = NR + 1 - - teamno = teams[hash] - io.write(' "' .. teamno .. '": "' .. teamname .. '"') -end -io.write('\n }\n}\n') diff --git a/bin/puzzles b/bin/puzzles deleted file mode 100755 index 0259174..0000000 --- a/bin/puzzles +++ /dev/null @@ -1,53 +0,0 @@ -#! /usr/bin/lua5.3 - -local basedir = arg[1] -local statedir = basedir .. "/state" - -local max_by_cat = {} -for cat in io.lines(statedir .. "/categories.txt") do - max_by_cat[cat] = 0 -end - -for line in io.lines(statedir .. "/points.log") do - local ts, team, cat, points = line:match("^(%d+) (%g+) (%g+) (%d+)") - points = tonumber(points) or 0 - - -- Skip scores for removed categories - if (max_by_cat[cat] ~= nil) then - max_by_cat[cat] = math.max(max_by_cat[cat], points) - end -end - - -local i = 0 -io.write('{\n') -for cat, biggest in pairs(max_by_cat) do - local points, dirname - local j = 0 - - if i > 0 then - io.write(',\n') - end - i = i + 1 - - io.write(' "' .. cat .. '": [\n') - for line in io.lines(basedir .. "/packages/" .. cat .. "/map.txt") do - points, dirname = line:match("^(%d+) (.*)") - points = tonumber(points) - - if j > 0 then - io.write(',\n') - end - j = j + 1 - io.write(' [' .. points .. ', "' .. dirname .. '"]') - if (points > biggest) then - break - end - end - if (points == biggest) then - io.write(',\n') - io.write(' [0, ""]') - end - io.write('\n ]') -end -io.write('\n}\n') diff --git a/bin/server-start b/bin/server-start deleted file mode 100755 index a60fc73..0000000 --- a/bin/server-start +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -# Starts a standalone server using tcpsvd and eris - -tcpserver diff --git a/bin/award b/contrib/award similarity index 100% rename from bin/award rename to contrib/award diff --git a/bin/mktokens b/contrib/mktokens similarity index 100% rename from bin/mktokens rename to contrib/mktokens diff --git a/tools/answer_words.txt b/devel/answer_words.txt similarity index 100% rename from tools/answer_words.txt rename to devel/answer_words.txt diff --git a/tools/devel-server.py b/devel/devel-server.py similarity index 100% rename from tools/devel-server.py rename to devel/devel-server.py diff --git a/tools/mistune.py b/devel/mistune.py similarity index 100% rename from tools/mistune.py rename to devel/mistune.py diff --git a/tools/moth.py b/devel/moth.py similarity index 100% rename from tools/moth.py rename to devel/moth.py diff --git a/tools/mothd.service b/devel/mothd.service similarity index 100% rename from tools/mothd.service rename to devel/mothd.service diff --git a/tools/package-puzzles.py b/devel/package-puzzles.py similarity index 100% rename from tools/package-puzzles.py rename to devel/package-puzzles.py diff --git a/setup.cfg b/devel/setup.cfg similarity index 100% rename from setup.cfg rename to devel/setup.cfg diff --git a/src/handlers.go b/src/handlers.go index 4750c70..3b0ec91 100644 --- a/src/handlers.go +++ b/src/handlers.go @@ -2,7 +2,6 @@ package main import ( "bufio" - "encoding/json" "fmt" "io" "log" @@ -35,7 +34,7 @@ func hasLine(r io.Reader, line string) bool { return false } -func (ctx Instance) registerHandler(w http.ResponseWriter, req *http.Request) { +func (ctx *Instance) registerHandler(w http.ResponseWriter, req *http.Request) { teamname := req.FormValue("name") teamid := req.FormValue("id") @@ -93,7 +92,7 @@ func (ctx Instance) registerHandler(w http.ResponseWriter, req *http.Request) { ) } -func (ctx Instance) tokenHandler(w http.ResponseWriter, req *http.Request) { +func (ctx *Instance) tokenHandler(w http.ResponseWriter, req *http.Request) { teamid := req.FormValue("id") token := req.FormValue("token") @@ -157,7 +156,7 @@ func (ctx Instance) tokenHandler(w http.ResponseWriter, req *http.Request) { ) } -func (ctx Instance) answerHandler(w http.ResponseWriter, req *http.Request) { +func (ctx *Instance) answerHandler(w http.ResponseWriter, req *http.Request) { teamid := req.FormValue("id") category := req.FormValue("cat") pointstr := req.FormValue("points") @@ -210,138 +209,42 @@ func (ctx Instance) answerHandler(w http.ResponseWriter, req *http.Request) { ) } -type PuzzleMap struct { - Points int - Path string -} - -func (pm *PuzzleMap) MarshalJSON() ([]byte, error) { - if pm == nil { - return []byte("null"), nil - } - - jPath, err := json.Marshal(pm.Path) - if err != nil { - return nil, err - } - - ret := fmt.Sprintf("[%d,%s]", pm.Points, string(jPath)) - return []byte(ret), nil -} - -func (ctx Instance) puzzlesHandler(w http.ResponseWriter, req *http.Request) { +func (ctx *Instance) puzzlesHandler(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - - maxByCategory := map[string]int{} - for _, a := range ctx.PointsLog() { - if a.Points > maxByCategory[a.Category] { - maxByCategory[a.Category] = a.Points - } - } - - res := map[string][]PuzzleMap{} - for catName, mb := range ctx.Categories { - mf, err := mb.Open("map.txt") - if err != nil { - log.Print(err) - continue - } - defer mf.Close() - - pm := make([]PuzzleMap, 0, 30) - completed := true - scanner := bufio.NewScanner(mf) - for scanner.Scan() { - line := scanner.Text() - - var pointval int - var dir string - - n, err := fmt.Sscanf(line, "%d %s", &pointval, &dir) - if err != nil { - log.Printf("Parsing map for %s: %v", catName, err) - continue - } else if n != 2 { - log.Printf("Parsing map for %s: short read", catName) - continue - } - - pm = append(pm, PuzzleMap{pointval, dir}) - - if pointval > maxByCategory[catName] { - completed = false - break - } - } - if completed { - pm = append(pm, PuzzleMap{0, ""}) - } - - res[catName] = pm - } - jres, _ := json.Marshal(res) - w.Write(jres) + w.Write(ctx.jPuzzleList) } -func (ctx Instance) pointsHandler(w http.ResponseWriter, req *http.Request) { - var ret struct { - Teams map[string]string `json:"teams"` - Points []*Award `json:"points"` - } - ret.Teams = map[string]string{} - ret.Points = ctx.PointsLog() - - teamNumbersById := map[string]int{} - for nr, a := range ret.Points { - teamNumber, ok := teamNumbersById[a.TeamId] - if !ok { - teamName, err := ctx.TeamName(a.TeamId) - if err != nil { - teamName = "[unregistered]" - } - teamNumber = nr - teamNumbersById[a.TeamId] = teamNumber - ret.Teams[strconv.FormatInt(int64(teamNumber), 16)] = teamName - } - a.TeamId = strconv.FormatInt(int64(teamNumber), 16) - } - - jret, err := json.Marshal(ret) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - +func (ctx *Instance) pointsHandler(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - w.Write(jret) + w.Write(ctx.jPointsLog) } -func (ctx Instance) contentHandler(w http.ResponseWriter, req *http.Request) { +func (ctx *Instance) contentHandler(w http.ResponseWriter, req *http.Request) { // Prevent directory traversal if strings.Contains(req.URL.Path, "/.") { http.Error(w, "Not Found", http.StatusNotFound) return } - + // Be clever: use only the last three parts of the path. This may prove to be a bad idea. parts := strings.Split(req.URL.Path, "/") if len(parts) < 3 { http.Error(w, "Not Found", http.StatusNotFound) return } - + fileName := parts[len(parts)-1] puzzleId := parts[len(parts)-2] categoryName := parts[len(parts)-3] - + mb, ok := ctx.Categories[categoryName] if !ok { http.Error(w, "Not Found", http.StatusNotFound) return } - + mbFilename := fmt.Sprintf("content/%s/%s", puzzleId, fileName) mf, err := mb.Open(mbFilename) if err != nil { @@ -350,15 +253,15 @@ func (ctx Instance) contentHandler(w http.ResponseWriter, req *http.Request) { return } defer mf.Close() - + http.ServeContent(w, req, fileName, mf.ModTime(), mf) } -func (ctx Instance) staticHandler(w http.ResponseWriter, req *http.Request) { +func (ctx *Instance) staticHandler(w http.ResponseWriter, req *http.Request) { ServeStatic(w, req, ctx.ResourcesDir) } -func (ctx Instance) BindHandlers(mux *http.ServeMux) { +func (ctx *Instance) BindHandlers(mux *http.ServeMux) { mux.HandleFunc(ctx.Base+"/", ctx.staticHandler) mux.HandleFunc(ctx.Base+"/register", ctx.registerHandler) mux.HandleFunc(ctx.Base+"/token", ctx.tokenHandler) diff --git a/src/instance.go b/src/instance.go index 267a85e..1c18de1 100644 --- a/src/instance.go +++ b/src/instance.go @@ -19,6 +19,8 @@ type Instance struct { ResourcesDir string Categories map[string]*Mothball update chan bool + jPuzzleList []byte + jPointsLog []byte } func NewInstance(base, mothballDir, stateDir, resourcesDir string) (*Instance, error) { diff --git a/src/maintenance.go b/src/maintenance.go index 77a282a..cba6a04 100644 --- a/src/maintenance.go +++ b/src/maintenance.go @@ -1,16 +1,121 @@ package main import ( + "bufio" + "encoding/json" "fmt" "io/ioutil" "log" "os" + "strconv" "strings" "time" ) +type PuzzleMap struct { + Points int + Path string +} + +func (pm *PuzzleMap) MarshalJSON() ([]byte, error) { + if pm == nil { + return []byte("null"), nil + } + + jPath, err := json.Marshal(pm.Path) + if err != nil { + return nil, err + } + + ret := fmt.Sprintf("[%d,%s]", pm.Points, string(jPath)) + return []byte(ret), nil +} + +func (ctx *Instance) generatePuzzleList() error { + maxByCategory := map[string]int{} + for _, a := range ctx.PointsLog() { + if a.Points > maxByCategory[a.Category] { + maxByCategory[a.Category] = a.Points + } + } + + ret := map[string][]PuzzleMap{} + for catName, mb := range ctx.Categories { + mf, err := mb.Open("map.txt") + if err != nil { + return err + } + defer mf.Close() + + pm := make([]PuzzleMap, 0, 30) + completed := true + scanner := bufio.NewScanner(mf) + for scanner.Scan() { + line := scanner.Text() + + var pointval int + var dir string + + n, err := fmt.Sscanf(line, "%d %s", &pointval, &dir) + if err != nil { + return err + } else if n != 2 { + return fmt.Errorf("Parsing map for %s: short read", catName) + } + + pm = append(pm, PuzzleMap{pointval, dir}) + + if pointval > maxByCategory[catName] { + completed = false + break + } + } + if completed { + pm = append(pm, PuzzleMap{0, ""}) + } + + ret[catName] = pm + } + + jpl, err := json.Marshal(ret) + if err == nil { + ctx.jPuzzleList = jpl + } + return err +} + +func (ctx *Instance) generatePointsLog() error { + var ret struct { + Teams map[string]string `json:"teams"` + Points []*Award `json:"points"` + } + ret.Teams = map[string]string{} + ret.Points = ctx.PointsLog() + + teamNumbersById := map[string]int{} + for nr, a := range ret.Points { + teamNumber, ok := teamNumbersById[a.TeamId] + if !ok { + teamName, err := ctx.TeamName(a.TeamId) + if err != nil { + teamName = "[unregistered]" + } + teamNumber = nr + teamNumbersById[a.TeamId] = teamNumber + ret.Teams[strconv.FormatInt(int64(teamNumber), 16)] = teamName + } + a.TeamId = strconv.FormatInt(int64(teamNumber), 16) + } + + jpl, err := json.Marshal(ret) + if err == nil { + ctx.jPointsLog = jpl + } + return err +} + // maintenance runs -func (ctx *Instance) Tidy() { +func (ctx *Instance) tidy() { // Do they want to reset everything? ctx.MaybeInitialize() @@ -67,13 +172,11 @@ func (ctx *Instance) Tidy() { ctx.Categories[categoryName] = mb } } - - ctx.CollectPoints() } // collectPoints gathers up files in points.new/ and appends their contents to points.log, // removing each points.new/ file as it goes. -func (ctx *Instance) CollectPoints() { +func (ctx *Instance) collectPoints() { logf, err := os.OpenFile(ctx.StatePath("points.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { log.Printf("Can't append to points log: %s", err) @@ -122,12 +225,15 @@ func (ctx *Instance) CollectPoints() { // maintenance is the goroutine that runs a periodic maintenance task func (ctx *Instance) Maintenance(maintenanceInterval time.Duration) { for { - ctx.Tidy() + ctx.tidy() + ctx.collectPoints() + ctx.generatePuzzleList() + ctx.generatePointsLog() select { - case <-ctx.update: - // log.Print("Forced update") - case <-time.After(maintenanceInterval): - // log.Print("Housekeeping...") + case <-ctx.update: + // log.Print("Forced update") + case <-time.After(maintenanceInterval): + // log.Print("Housekeeping...") } } } diff --git a/src/mothball.go b/src/mothball.go index 4e2a962..149dbf5 100644 --- a/src/mothball.go +++ b/src/mothball.go @@ -17,9 +17,9 @@ type Mothball struct { } type MothballFile struct { - f io.ReadCloser + f io.ReadCloser pos int64 - zf *zip.File + zf *zip.File io.Reader io.Seeker io.Closer @@ -27,9 +27,9 @@ type MothballFile struct { func NewMothballFile(zf *zip.File) (*MothballFile, error) { mf := &MothballFile{ - zf: zf, + zf: zf, pos: 0, - f: nil, + f: nil, } if err := mf.reopen(); err != nil { return nil, err @@ -72,7 +72,7 @@ func (mf *MothballFile) Seek(offset int64, whence int) (int64, error) { case io.SeekEnd: pos = int64(mf.zf.UncompressedSize64) - int64(offset) } - + if pos < 0 { return mf.pos, fmt.Errorf("Tried to seek %d before start of file", pos) } @@ -88,8 +88,8 @@ func (mf *MothballFile) Seek(offset int64, whence int) (int64, error) { return mf.pos, err } } - - buf := make([]byte, 32 * 1024) + + buf := make([]byte, 32*1024) for pos > mf.pos { l := pos - mf.pos if l > int64(cap(buf)) { @@ -103,7 +103,7 @@ func (mf *MothballFile) Seek(offset int64, whence int) (int64, error) { return mf.pos, fmt.Errorf("Short read (%d bytes)", n) } } - + return mf.pos, nil } diff --git a/tools/mothd b/tools/mothd deleted file mode 100755 index ffad247..0000000 --- a/tools/mothd +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/sh - -cd ${1:-$(dirname $0)} -KOTH_BASE=$(pwd) - -echo "Running koth instances in $KOTH_BASE" - -while true; do - for i in $KOTH_BASE/*/assigned.txt; do - dir=${i%/*} - $dir/bin/once - done - sleep 5 -done