Cache points.json and puzzles.json

This commit is contained in:
Neale Pickett 2018-09-20 03:44:34 +00:00
parent bc6b373659
commit e460f5b6b3
23 changed files with 140 additions and 427 deletions

View File

@ -1,7 +0,0 @@
FROM alpine
RUN apk --no-cache add python3 py3-pillow
COPY tools/package-puzzles.py tools/moth.py tools/mistune.py tools/answer_words.txt /moth/
ENTRYPOINT ["python3", "/moth/package-puzzles.py"]

View File

@ -1,16 +0,0 @@
#!/bin/sh -e
# Starts a standalone server using tcpsvd and eris
echo "Figuring out web user..."
for www in www-data http tc _ _www; do
id $www && break
done
if [ $www = _ ]; then
echo "Unable to determine httpd user on this system. Dying."
exit 1
fi
cd $(dirname $0)/../www
tcpserver -RHI localhost -u $www -g $www 0 80 eris -c -.

View File

@ -1,26 +0,0 @@
#! /bin/sh -e
package=$1
if ! [ -n "$package" -a -f $package ]; then
echo "Usage: $0 PACKAGE"
exit 1
fi
shift
cat=$(basename $package .zip)
outdir=$(dirname $(dirname $0))/packages/$cat
echo "Extracting to $outdir..."
mkdir -p $outdir
unzip -o -d $outdir $package
echo "Fixing permissions..."
chmod a+rx $outdir/*/ $outdir/*/*
chmod -R a+r $outdir
find $outdir/content -name \*.cgi -exec chmod a+rx {} \;
if [ ! -h $outdir/../../www/$cat ]; then
echo "Linking into web space..."
ln -sf ../packages/$cat/content $outdir/../../www/$cat
fi

37
bin/new
View File

@ -1,37 +0,0 @@
#! /bin/sh
newdir=$1
if [ -z "$newdir" ]; then
echo "Usage: $0 NEWDIR"
exit 1
fi
KOTH_BASE=$(cd $(dirname $0)/.. && pwd)
echo "Figuring out web user..."
for www in www-data http _; do
id $www && break
done
if [ $www = _ ]; then
echo "Unable to determine httpd user on this system. Dying."
exit 1
fi
mkdir -p $newdir
cd $newdir
for i in points.new points.tmp teams; do
mkdir -p state/$i
setfacl -m ${www}:rwx state/$i
done
>> state/points.log
if ! [ -f assigned.txt ]; then
hd < /dev/urandom | awk '{print $3 $4 $5 $6;}' | head -n 100 > assigned.txt
fi
mkdir -p www
cp -r $KOTH_BASE/html/* www/
cp $KOTH_BASE/bin/*.cgi www/

View File

@ -1,95 +0,0 @@
#! /bin/sh
if [ -n "$1" ]; then
cd $1
else
cd $(dirname $0)/..
fi
basedir=$(pwd)
log () {
echo "moth: $@" 1>&2
}
# Do nothing if `disabled` is present
if [ -f state/disabled ]; then
log "Instance disabled; doing nothing"
exit
fi
# Are we stopping at a certain time?
if [ -f state/until ]; then
read -r until < state/until
when=$(date -d "$until" +%s)
now=$(date +%s)
if [ $now -ge $when ]; then
log "End time reached; doing nothing"
exit
fi
fi
# Reset to initial state?
if [ ! -f state/initialized ]; then
log "Resetting contest state"
rm -rf state/teams state/points.new state/points.tmp
mkdir -p state/teams state/points.new state/points.tmp
chown www:www state/teams state/points.new state/points.tmp # Needs root. Use Docker.
: > state/points.log
echo 'Remove this file to obliterate teams and points' > state/initialized
fi
# Create some team names if needed
if [ ! -f state/assigned.txt ]; then
log "Generating team names"
hd </dev/urandom | awk '{print $3 $4 $5 $6;}' | head -n 100 > state/assigned.txt
fi
# Install new categories
for pkg in puzzles/*; do
cat=$(basename $pkg .zip)
if [ ! -f packages/$cat/installed ] || [ $pkg -nt packages/$cat/installed ]; then
log "Installing $pkg"
bin/install-category $pkg
: >packages/$cat/installed
fi
done
# Helpful error message
if [ $(ls packages | wc -l) -eq 0 ]; then
log "error: No packages installed"
exit
fi
# Create a list of currently-active categories
: > state/categories.txt.new
for dn in packages/*; do
cat=${dn##packages/}
echo "$cat" >> state/categories.txt.new
done
mv state/categories.txt.new state/categories.txt
# Collect new points
find state/points.new -type f | while read fn; do
# Skip files opened by another process
lsof $fn | grep -q $fn && continue
# Skip partially written files
[ $(wc -l < $fn) -gt 0 ] || continue
# filter the file for unique awards
sort -k 4 $fn | uniq -f 1 | sort -n >> state/points.log
# Now kill the file
rm -f $fn
done
# Generate new puzzles.json
if bin/puzzles $basedir > state/puzzles.json.new; then
mv state/puzzles.json.new state/puzzles.json
fi
# Generate new points.json
if bin/points $basedir > state/points.json.new; then
mv state/points.json.new state/points.json
fi

View File

@ -1,45 +0,0 @@
#! /usr/bin/lua5.3
local basedir = arg[1]
local statedir = basedir .. "/state"
io.write('{\n "points": [\n')
local teams = {}
local teamnames = {}
local nteams = 0
local NR = 0
for line in io.lines(statedir .. "/points.log") do
local ts, hash, cat, points = line:match("(%d+) (%g+) (%g+) (%d+)")
local teamno = teams[hash]
if not teamno then
teamno = nteams
teams[hash] = teamno
nteams = nteams + 1
teamnames[hash] = io.lines(statedir .. "/teams/" .. hash)()
end
if NR > 0 then
-- JSON sucks, barfs if you have a comma with nothing after it
io.write(",\n")
end
NR = NR + 1
io.write(' [' .. ts .. ', "' .. teamno .. '", "' .. cat .. '", ' .. points .. ']')
end
io.write('\n],\n "teams": {\n')
NR = 0
for hash,teamname in pairs(teamnames) do
if NR > 0 then
io.write(",\n")
end
NR = NR + 1
teamno = teams[hash]
io.write(' "' .. teamno .. '": "' .. teamname .. '"')
end
io.write('\n }\n}\n')

View File

@ -1,53 +0,0 @@
#! /usr/bin/lua5.3
local basedir = arg[1]
local statedir = basedir .. "/state"
local max_by_cat = {}
for cat in io.lines(statedir .. "/categories.txt") do
max_by_cat[cat] = 0
end
for line in io.lines(statedir .. "/points.log") do
local ts, team, cat, points = line:match("^(%d+) (%g+) (%g+) (%d+)")
points = tonumber(points) or 0
-- Skip scores for removed categories
if (max_by_cat[cat] ~= nil) then
max_by_cat[cat] = math.max(max_by_cat[cat], points)
end
end
local i = 0
io.write('{\n')
for cat, biggest in pairs(max_by_cat) do
local points, dirname
local j = 0
if i > 0 then
io.write(',\n')
end
i = i + 1
io.write(' "' .. cat .. '": [\n')
for line in io.lines(basedir .. "/packages/" .. cat .. "/map.txt") do
points, dirname = line:match("^(%d+) (.*)")
points = tonumber(points)
if j > 0 then
io.write(',\n')
end
j = j + 1
io.write(' [' .. points .. ', "' .. dirname .. '"]')
if (points > biggest) then
break
end
end
if (points == biggest) then
io.write(',\n')
io.write(' [0, ""]')
end
io.write('\n ]')
end
io.write('\n}\n')

View File

@ -1,5 +0,0 @@
#!/bin/sh
# Starts a standalone server using tcpsvd and eris
tcpserver

View File

@ -2,7 +2,6 @@ package main
import (
"bufio"
"encoding/json"
"fmt"
"io"
"log"
@ -35,7 +34,7 @@ func hasLine(r io.Reader, line string) bool {
return false
}
func (ctx Instance) registerHandler(w http.ResponseWriter, req *http.Request) {
func (ctx *Instance) registerHandler(w http.ResponseWriter, req *http.Request) {
teamname := req.FormValue("name")
teamid := req.FormValue("id")
@ -93,7 +92,7 @@ func (ctx Instance) registerHandler(w http.ResponseWriter, req *http.Request) {
)
}
func (ctx Instance) tokenHandler(w http.ResponseWriter, req *http.Request) {
func (ctx *Instance) tokenHandler(w http.ResponseWriter, req *http.Request) {
teamid := req.FormValue("id")
token := req.FormValue("token")
@ -157,7 +156,7 @@ func (ctx Instance) tokenHandler(w http.ResponseWriter, req *http.Request) {
)
}
func (ctx Instance) answerHandler(w http.ResponseWriter, req *http.Request) {
func (ctx *Instance) answerHandler(w http.ResponseWriter, req *http.Request) {
teamid := req.FormValue("id")
category := req.FormValue("cat")
pointstr := req.FormValue("points")
@ -210,115 +209,19 @@ func (ctx Instance) answerHandler(w http.ResponseWriter, req *http.Request) {
)
}
type PuzzleMap struct {
Points int
Path string
}
func (pm *PuzzleMap) MarshalJSON() ([]byte, error) {
if pm == nil {
return []byte("null"), nil
}
jPath, err := json.Marshal(pm.Path)
if err != nil {
return nil, err
}
ret := fmt.Sprintf("[%d,%s]", pm.Points, string(jPath))
return []byte(ret), nil
}
func (ctx Instance) puzzlesHandler(w http.ResponseWriter, req *http.Request) {
func (ctx *Instance) puzzlesHandler(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
maxByCategory := map[string]int{}
for _, a := range ctx.PointsLog() {
if a.Points > maxByCategory[a.Category] {
maxByCategory[a.Category] = a.Points
}
}
res := map[string][]PuzzleMap{}
for catName, mb := range ctx.Categories {
mf, err := mb.Open("map.txt")
if err != nil {
log.Print(err)
continue
}
defer mf.Close()
pm := make([]PuzzleMap, 0, 30)
completed := true
scanner := bufio.NewScanner(mf)
for scanner.Scan() {
line := scanner.Text()
var pointval int
var dir string
n, err := fmt.Sscanf(line, "%d %s", &pointval, &dir)
if err != nil {
log.Printf("Parsing map for %s: %v", catName, err)
continue
} else if n != 2 {
log.Printf("Parsing map for %s: short read", catName)
continue
}
pm = append(pm, PuzzleMap{pointval, dir})
if pointval > maxByCategory[catName] {
completed = false
break
}
}
if completed {
pm = append(pm, PuzzleMap{0, ""})
}
res[catName] = pm
}
jres, _ := json.Marshal(res)
w.Write(jres)
w.Write(ctx.jPuzzleList)
}
func (ctx Instance) pointsHandler(w http.ResponseWriter, req *http.Request) {
var ret struct {
Teams map[string]string `json:"teams"`
Points []*Award `json:"points"`
}
ret.Teams = map[string]string{}
ret.Points = ctx.PointsLog()
teamNumbersById := map[string]int{}
for nr, a := range ret.Points {
teamNumber, ok := teamNumbersById[a.TeamId]
if !ok {
teamName, err := ctx.TeamName(a.TeamId)
if err != nil {
teamName = "[unregistered]"
}
teamNumber = nr
teamNumbersById[a.TeamId] = teamNumber
ret.Teams[strconv.FormatInt(int64(teamNumber), 16)] = teamName
}
a.TeamId = strconv.FormatInt(int64(teamNumber), 16)
}
jret, err := json.Marshal(ret)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
func (ctx *Instance) pointsHandler(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(jret)
w.Write(ctx.jPointsLog)
}
func (ctx Instance) contentHandler(w http.ResponseWriter, req *http.Request) {
func (ctx *Instance) contentHandler(w http.ResponseWriter, req *http.Request) {
// Prevent directory traversal
if strings.Contains(req.URL.Path, "/.") {
http.Error(w, "Not Found", http.StatusNotFound)
@ -354,11 +257,11 @@ func (ctx Instance) contentHandler(w http.ResponseWriter, req *http.Request) {
http.ServeContent(w, req, fileName, mf.ModTime(), mf)
}
func (ctx Instance) staticHandler(w http.ResponseWriter, req *http.Request) {
func (ctx *Instance) staticHandler(w http.ResponseWriter, req *http.Request) {
ServeStatic(w, req, ctx.ResourcesDir)
}
func (ctx Instance) BindHandlers(mux *http.ServeMux) {
func (ctx *Instance) BindHandlers(mux *http.ServeMux) {
mux.HandleFunc(ctx.Base+"/", ctx.staticHandler)
mux.HandleFunc(ctx.Base+"/register", ctx.registerHandler)
mux.HandleFunc(ctx.Base+"/token", ctx.tokenHandler)

View File

@ -19,6 +19,8 @@ type Instance struct {
ResourcesDir string
Categories map[string]*Mothball
update chan bool
jPuzzleList []byte
jPointsLog []byte
}
func NewInstance(base, mothballDir, stateDir, resourcesDir string) (*Instance, error) {

View File

@ -1,16 +1,121 @@
package main
import (
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"strconv"
"strings"
"time"
)
type PuzzleMap struct {
Points int
Path string
}
func (pm *PuzzleMap) MarshalJSON() ([]byte, error) {
if pm == nil {
return []byte("null"), nil
}
jPath, err := json.Marshal(pm.Path)
if err != nil {
return nil, err
}
ret := fmt.Sprintf("[%d,%s]", pm.Points, string(jPath))
return []byte(ret), nil
}
func (ctx *Instance) generatePuzzleList() error {
maxByCategory := map[string]int{}
for _, a := range ctx.PointsLog() {
if a.Points > maxByCategory[a.Category] {
maxByCategory[a.Category] = a.Points
}
}
ret := map[string][]PuzzleMap{}
for catName, mb := range ctx.Categories {
mf, err := mb.Open("map.txt")
if err != nil {
return err
}
defer mf.Close()
pm := make([]PuzzleMap, 0, 30)
completed := true
scanner := bufio.NewScanner(mf)
for scanner.Scan() {
line := scanner.Text()
var pointval int
var dir string
n, err := fmt.Sscanf(line, "%d %s", &pointval, &dir)
if err != nil {
return err
} else if n != 2 {
return fmt.Errorf("Parsing map for %s: short read", catName)
}
pm = append(pm, PuzzleMap{pointval, dir})
if pointval > maxByCategory[catName] {
completed = false
break
}
}
if completed {
pm = append(pm, PuzzleMap{0, ""})
}
ret[catName] = pm
}
jpl, err := json.Marshal(ret)
if err == nil {
ctx.jPuzzleList = jpl
}
return err
}
func (ctx *Instance) generatePointsLog() error {
var ret struct {
Teams map[string]string `json:"teams"`
Points []*Award `json:"points"`
}
ret.Teams = map[string]string{}
ret.Points = ctx.PointsLog()
teamNumbersById := map[string]int{}
for nr, a := range ret.Points {
teamNumber, ok := teamNumbersById[a.TeamId]
if !ok {
teamName, err := ctx.TeamName(a.TeamId)
if err != nil {
teamName = "[unregistered]"
}
teamNumber = nr
teamNumbersById[a.TeamId] = teamNumber
ret.Teams[strconv.FormatInt(int64(teamNumber), 16)] = teamName
}
a.TeamId = strconv.FormatInt(int64(teamNumber), 16)
}
jpl, err := json.Marshal(ret)
if err == nil {
ctx.jPointsLog = jpl
}
return err
}
// maintenance runs
func (ctx *Instance) Tidy() {
func (ctx *Instance) tidy() {
// Do they want to reset everything?
ctx.MaybeInitialize()
@ -67,13 +172,11 @@ func (ctx *Instance) Tidy() {
ctx.Categories[categoryName] = mb
}
}
ctx.CollectPoints()
}
// collectPoints gathers up files in points.new/ and appends their contents to points.log,
// removing each points.new/ file as it goes.
func (ctx *Instance) CollectPoints() {
func (ctx *Instance) collectPoints() {
logf, err := os.OpenFile(ctx.StatePath("points.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
log.Printf("Can't append to points log: %s", err)
@ -122,12 +225,15 @@ func (ctx *Instance) CollectPoints() {
// maintenance is the goroutine that runs a periodic maintenance task
func (ctx *Instance) Maintenance(maintenanceInterval time.Duration) {
for {
ctx.Tidy()
ctx.tidy()
ctx.collectPoints()
ctx.generatePuzzleList()
ctx.generatePointsLog()
select {
case <-ctx.update:
// log.Print("Forced update")
case <-time.After(maintenanceInterval):
// log.Print("Housekeeping...")
case <-ctx.update:
// log.Print("Forced update")
case <-time.After(maintenanceInterval):
// log.Print("Housekeeping...")
}
}
}

View File

@ -17,9 +17,9 @@ type Mothball struct {
}
type MothballFile struct {
f io.ReadCloser
f io.ReadCloser
pos int64
zf *zip.File
zf *zip.File
io.Reader
io.Seeker
io.Closer
@ -27,9 +27,9 @@ type MothballFile struct {
func NewMothballFile(zf *zip.File) (*MothballFile, error) {
mf := &MothballFile{
zf: zf,
zf: zf,
pos: 0,
f: nil,
f: nil,
}
if err := mf.reopen(); err != nil {
return nil, err
@ -89,7 +89,7 @@ func (mf *MothballFile) Seek(offset int64, whence int) (int64, error) {
}
}
buf := make([]byte, 32 * 1024)
buf := make([]byte, 32*1024)
for pos > mf.pos {
l := pos - mf.pos
if l > int64(cap(buf)) {

View File

@ -1,14 +0,0 @@
#! /bin/sh
cd ${1:-$(dirname $0)}
KOTH_BASE=$(pwd)
echo "Running koth instances in $KOTH_BASE"
while true; do
for i in $KOTH_BASE/*/assigned.txt; do
dir=${i%/*}
$dir/bin/once
done
sleep 5
done