Merge branch 'main' into 179-intermittent-test-failures-on-gitlab-ci

This commit is contained in:
Neale Pickett 2022-05-10 19:21:25 -06:00
commit dfc31eb9f3
17 changed files with 246 additions and 143 deletions

View File

@ -1,70 +0,0 @@
name: Build/Test/Push
on:
push:
branches:
- v3
- devel
- main
tags:
- 'v*.*.*'
jobs:
test-mothd:
name: Test mothd
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v2
with:
go-version: 1.13
- name: Retrieve code
uses: actions/checkout@v2
- name: Test
run: go test ./...
publish:
name: Publish container images
runs-on: ubuntu-latest
steps:
- name: Retrieve code
uses: actions/checkout@v2
- name: Gitlab variables
id: vars
run: build/ci/gitlab-vars
- name: Login to GitHub Packages Docker Registry
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.CR_PAT }}
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: neale
password: ${{ secrets.DOCKER_TOKEN }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
# Currently required, because buildx doesn't support auto-push from docker
- name: Set up builder
uses: docker/setup-buildx-action@v1
id: buildx
- name: Build and push moth image
uses: docker/build-push-action@v2
with:
builder: ${{ steps.buildx.outputs.name }}
target: moth
file: build/package/Containerfile
push: true
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64
tags: |
dirtbags/moth:${{ steps.vars.outputs.tag }}
ghcr.io/dirtbags/moth:${{ steps.vars.outputs.tag }}

View File

@ -4,7 +4,12 @@ stages:
test:
stage: test
image: golang:1.17
image: golang:1.18
only:
refs:
- main
- merge_requests
script:
- go test -race ./...
@ -15,10 +20,5 @@ push:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
script:
- mkdir ~/.docker
- echo "$DOCKER_AUTH_CONFIG" > ~/.docker/config.json
- echo "$DOCKER_AUTH_CONFIG" | tee ~/.docker/config.json | md5sum
- sh build/ci/ci.sh publish
- >
docker build
--tag ghcr.io/dirtbags/moth:$CI_COMMIT_REF_SLUG
--file build/package/Containerfile
.

View File

@ -4,16 +4,42 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [unreleased]
## [v4.4.8] - 2022-05-10
### Changed
- You can now join with a team ID not appearing in `teamids.txt`,
as long as it is registered (in the `teams/` directory)
## [v4.4.7] - 2022-05-10
### Changed
- Initializing an instance now truncates `events.csv`
## [v4.4.6] - 2021-10-26
### Added
- State is now cached in memory, in an attempt to reduce filesystem metadata operations,
which kill NFS.
## [v4.4.5] - 2021-10-26
### Added
- Images deploying to docker hub too. We're now at capacity for our Docker Hub team.
## [v4.4.4] - 2021-10-20
### Changed
- Trying to get CI push of built images. I expect this to fail, too. But in a way that can help me debug the issue.
## [v4.3.3] - 2021-10-20
### Fixed
- Points awarded while scoring is paused are now correctly sorted (#168)
- Writing a new mothball with the same name is now detected and the new mothball loaded (#172)
- Regression test for issue where URL path leading directories were ignored (#144)
- A few other very minor bugs were closed when I couldn't reproduce them or decided they weren't actually bugs.
### Changed
- Many error messages were changed to start with a lower-case letter,
in order to satisfy a new linter check.
- CI/CD moved to our Cyber Fire Gitlab instance
- I attempted to have the build thingy automatically build moth:v4 and moth:v4.3 and moth:v4.3.3 images,
but I can't test it without tagging a release.
So v4.3.4 might come out very soon after this ;)
## [v4.2.2] - 2021-09-30
### Added

View File

@ -33,7 +33,7 @@ You can read more about why we made these decisions in [philosophy](docs/philoso
Run in demonstration mode
===========
docker run --rm -it -p 8080:8080 dirtbags/moth-devel
docker run --rm -it -p 8080:8080 ghcr.io/dirtbags/moth-devel
Then open http://localhost:8080/ and check out the example puzzles.

View File

@ -2,7 +2,7 @@
set -e
images="ghcr.io/dirtbags/moth"
images="ghcr.io/dirtbags/moth dirtbags/moth"
ACTION=$1
if [ -z "$ACTION" ]; then
@ -27,9 +27,9 @@ run () {
tags () {
pfx=$1
for base in $images; do
echo $pfx $base:${CI_COMMIT_REF_SLUG}
echo $pfx $base:${CI_COMMIT_REF_SLUG%.*}
echo $pfx $base:${CI_COMMIT_REF_SLUG%.*.*}
echo $pfx $base:${CI_COMMIT_REF_NAME}
echo $pfx $base:${CI_COMMIT_REF_NAME%.*}
echo $pfx $base:${CI_COMMIT_REF_NAME%.*.*}
done | uniq
}

View File

@ -37,6 +37,7 @@ func (hs *HTTPServer) TestRequest(path string, args map[string]string) *httptest
func TestHttpd(t *testing.T) {
server := NewTestServer()
hs := NewHTTPServer("/", server)
stateProvider := server.State.(*State)
if r := hs.TestRequest("/", nil); r.Result().StatusCode != 200 {
t.Error(r.Result())
@ -73,6 +74,8 @@ func TestHttpd(t *testing.T) {
t.Error("Register failed", r.Body.String())
}
time.Sleep(TestMaintenanceInterval)
if r := hs.TestRequest("/state", nil); r.Result().StatusCode != 200 {
t.Error(r.Result())
} else if r.Body.String() != `{"Config":{"Devel":false},"Messages":"messages.html","TeamNames":{"self":"GoTeam"},"PointsLog":[],"Puzzles":{"pategory":[1]}}` {
@ -128,7 +131,8 @@ func TestHttpd(t *testing.T) {
t.Error("Unexpected body", r.Body.String())
}
server.State.refresh()
time.Sleep(TestMaintenanceInterval)
stateProvider.refresh()
if r := hs.TestRequest("/content/pategory/2/puzzle.json", nil); r.Result().StatusCode != 200 {
t.Error(r.Result())

28
cmd/mothd/issues_test.go Normal file
View File

@ -0,0 +1,28 @@
package main
import (
"testing"
"github.com/spf13/afero"
)
func TestIssue156(t *testing.T) {
puzzles := NewTestMothballs()
state := NewTestState()
theme := NewTestTheme()
server := NewMothServer(Configuration{}, theme, state, puzzles)
afero.WriteFile(state, "teams/bloop", []byte("bloop: the team"), 0644)
state.refresh()
handler := server.NewHandler("", "bloop")
es := handler.ExportState()
if _, ok := es.TeamNames["self"]; !ok {
t.Fail()
}
err := handler.Register("bloop: the other team")
if err != ErrAlreadyRegistered {
t.Fail()
}
}

View File

@ -175,6 +175,7 @@ func (m *Mothballs) refresh() {
m.categories[categoryName] = zipCategory{
Fs: zipfs.New(zrc),
Closer: f,
mtime: fi.ModTime(),
}
log.Println("Adding category:", categoryName)

View File

@ -187,12 +187,15 @@ func (mh *MothRequestHandler) ExportState() *StateExport {
return mh.exportStateIfRegistered(false)
}
func (mh *MothRequestHandler) exportStateIfRegistered(override bool) *StateExport {
// Export state, replacing the team ID with "self" if the team is registered.
//
// If forceRegistered is true, go ahead and export it anyway
func (mh *MothRequestHandler) exportStateIfRegistered(forceRegistered bool) *StateExport {
export := StateExport{}
export.Config = mh.Config
teamName, err := mh.State.TeamName(mh.teamID)
registered := override || mh.Config.Devel || (err == nil)
registered := forceRegistered || mh.Config.Devel || (err == nil)
export.Messages = mh.State.Messages()
export.TeamNames = make(map[string]string)

View File

@ -16,11 +16,13 @@ const TestTeamID = "teamID"
// See function definition for details.
func NewTestServer() *MothServer {
puzzles := NewTestMothballs()
puzzles.refresh()
go puzzles.Maintain(TestMaintenanceInterval)
state := NewTestState()
afero.WriteFile(state, "teamids.txt", []byte("teamID\n"), 0644)
afero.WriteFile(state, "messages.html", []byte("messages.html"), 0644)
state.refresh()
go state.Maintain(TestMaintenanceInterval)
theme := NewTestTheme()
@ -83,13 +85,16 @@ func TestProdServer(t *testing.T) {
t.Error("index.html wrong contents", contents)
}
// Wait for refresh to pick everything up
time.Sleep(TestMaintenanceInterval)
{
es := handler.ExportState()
if es.Config.Devel {
t.Error("Marked as development server", es.Config)
}
if len(es.Puzzles) != 1 {
t.Error("Puzzle categories wrong length")
t.Error("Puzzle categories wrong length", len(es.Puzzles))
}
if es.Messages != "messages.html" {
t.Error("Messages has wrong contents")

View File

@ -11,6 +11,7 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/dirtbags/moth/pkg/award"
@ -42,6 +43,12 @@ type State struct {
eventStream chan []string
eventWriter *csv.Writer
eventWriterFile afero.File
// Caches, so we're not hammering NFS with metadata operations
teamNames map[string]string
pointsLog award.List
messages string
lock sync.RWMutex
}
// NewState returns a new State struct backed by the given Fs
@ -51,6 +58,8 @@ func NewState(fs afero.Fs) *State {
Enabled: true,
refreshNow: make(chan bool, 5),
eventStream: make(chan []string, 80),
teamNames: make(map[string]string),
}
if err := s.reopenEventLog(); err != nil {
log.Fatal(err)
@ -120,21 +129,25 @@ func (s *State) updateEnabled() {
// TeamName returns team name given a team ID.
func (s *State) TeamName(teamID string) (string, error) {
teamFs := afero.NewBasePathFs(s.Fs, "teams")
teamNameBytes, err := afero.ReadFile(teamFs, teamID)
if os.IsNotExist(err) {
s.lock.RLock()
name, ok := s.teamNames[teamID]
s.lock.RUnlock()
if !ok {
return "", fmt.Errorf("unregistered team ID: %s", teamID)
} else if err != nil {
return "", fmt.Errorf("unregistered team ID: %s (%s)", teamID, err)
}
teamName := strings.TrimSpace(string(teamNameBytes))
return teamName, nil
return name, nil
}
// SetTeamName writes out team name.
// This can only be done once per team.
func (s *State) SetTeamName(teamID, teamName string) error {
s.lock.RLock()
_, ok := s.teamNames[teamID]
s.lock.RUnlock()
if ok {
return ErrAlreadyRegistered
}
idsFile, err := s.Open("teamids.txt")
if err != nil {
return fmt.Errorf("team IDs file does not exist")
@ -163,36 +176,26 @@ func (s *State) SetTeamName(teamID, teamName string) error {
log.Printf("Setting team name [%s] in file %s", teamName, teamFilename)
fmt.Fprintln(teamFile, teamName)
teamFile.Close()
s.refreshNow <- true
return nil
}
// PointsLog retrieves the current points log.
func (s *State) PointsLog() award.List {
f, err := s.Open("points.log")
if err != nil {
log.Println(err)
return nil
}
defer f.Close()
pointsLog := make(award.List, 0, 200)
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
cur, err := award.Parse(line)
if err != nil {
log.Printf("Skipping malformed award line %s: %s", line, err)
continue
}
pointsLog = append(pointsLog, cur)
}
return pointsLog
s.lock.RLock()
ret := make(award.List, len(s.pointsLog))
copy(ret, s.pointsLog)
s.lock.RUnlock()
return ret
}
// Messages retrieves the current messages.
func (s *State) Messages() string {
bMessages, _ := afero.ReadFile(s, "messages.html")
return string(bMessages)
s.lock.RLock() // It's not clear to me that this actually needs to happen
defer s.lock.RUnlock()
return s.messages
}
// AwardPoints gives points to teamID in category.
@ -260,12 +263,14 @@ func (s *State) collectPoints() {
}
duplicate := false
for _, e := range s.PointsLog() {
s.lock.RLock()
for _, e := range s.pointsLog {
if awd.Equal(e) {
duplicate = true
break
}
}
s.lock.RUnlock()
if duplicate {
log.Print("Skipping duplicate points: ", awd.String())
@ -279,6 +284,11 @@ func (s *State) collectPoints() {
}
fmt.Fprintln(logf, awd.String())
logf.Close()
// Stick this on the cache too
s.lock.Lock()
s.pointsLog = append(s.pointsLog, awd)
s.lock.Unlock()
}
if err := s.Remove(filename); err != nil {
@ -300,6 +310,7 @@ func (s *State) maybeInitialize() {
s.Remove("enabled")
s.Remove("hours.txt")
s.Remove("points.log")
s.Remove("events.csv")
s.Remove("messages.html")
s.Remove("mothd.log")
s.RemoveAll("points.tmp")
@ -402,12 +413,64 @@ func (s *State) reopenEventLog() error {
return nil
}
func (s *State) updateCaches() {
s.lock.Lock()
defer s.lock.Unlock()
if f, err := s.Open("points.log"); err != nil {
log.Println(err)
} else {
defer f.Close()
pointsLog := make(award.List, 0, 200)
scanner := bufio.NewScanner(f)
for scanner.Scan() {
line := scanner.Text()
cur, err := award.Parse(line)
if err != nil {
log.Printf("Skipping malformed award line %s: %s", line, err)
continue
}
pointsLog = append(pointsLog, cur)
}
s.pointsLog = pointsLog
}
{
// The compiler recognizes this as an optimization case
for k := range s.teamNames {
delete(s.teamNames, k)
}
teamsFs := afero.NewBasePathFs(s.Fs, "teams")
if dirents, err := afero.ReadDir(teamsFs, "."); err != nil {
log.Printf("Reading team ids: %v", err)
} else {
for _, dirent := range dirents {
teamID := dirent.Name()
if teamNameBytes, err := afero.ReadFile(teamsFs, teamID); err != nil {
log.Printf("Reading team %s: %v", teamID, err)
} else {
teamName := strings.TrimSpace(string(teamNameBytes))
s.teamNames[teamID] = teamName
}
}
}
}
if bMessages, err := afero.ReadFile(s, "messages.html"); err == nil {
s.messages = string(bMessages)
}
}
func (s *State) refresh() {
s.maybeInitialize()
s.updateEnabled()
if s.Enabled {
s.collectPoints()
}
s.updateCaches()
}
// Maintain performs housekeeping on a State struct.

View File

@ -62,6 +62,7 @@ func TestState(t *testing.T) {
if err := s.SetTeamName(teamID, "wat"); err == nil {
t.Errorf("Registering team a second time didn't fail")
}
s.refresh()
if name, err := s.TeamName(teamID); err != nil {
t.Error(err)
} else if name != teamName {
@ -73,9 +74,6 @@ func TestState(t *testing.T) {
if err := s.AwardPoints(teamID, category, points); err != nil {
t.Error(err)
}
if err := s.AwardPoints(teamID, category, points); err != nil {
t.Error("Two awards before refresh:", err)
}
// Flex duplicate detection with different timestamp
if f, err := s.Create("points.new/moo"); err != nil {
t.Error("Creating duplicate points file:", err)
@ -83,24 +81,34 @@ func TestState(t *testing.T) {
fmt.Fprintln(f, time.Now().Unix()+1, teamID, category, points)
f.Close()
}
s.AwardPoints(teamID, category, points)
s.refresh()
pl = s.PointsLog()
if len(pl) != 1 {
for i, award := range pl {
t.Logf("pl[%d] == %s", i, award.String())
}
t.Errorf("After awarding duplicate points, points log has length %d", len(pl))
} else if (pl[0].TeamID != teamID) || (pl[0].Category != category) || (pl[0].Points != points) {
t.Errorf("Incorrect logged award %v", pl)
}
if err := s.AwardPoints(teamID, category, points); err == nil {
t.Error("Duplicate points award didn't fail")
t.Error("Duplicate points award after refresh didn't fail")
}
if err := s.AwardPoints(teamID, category, points+1); err != nil {
t.Error("Awarding more points:", err)
}
pl = s.PointsLog()
if len(pl) != 1 {
t.Errorf("After awarding points, points log has length %d", len(pl))
} else if (pl[0].TeamID != teamID) || (pl[0].Category != category) || (pl[0].Points != points) {
t.Errorf("Incorrect logged award %v", pl)
s.refresh()
if len(s.PointsLog()) != 2 {
t.Errorf("There should be two awards")
}
afero.WriteFile(s, "points.log", []byte("intentional parse error\n"), 0644)
s.refresh()
if len(s.PointsLog()) != 0 {
t.Errorf("Intentional parse error breaks pointslog")
}
@ -108,7 +116,8 @@ func TestState(t *testing.T) {
t.Error(err)
}
s.refresh()
if len(s.PointsLog()) != 2 {
if len(s.PointsLog()) != 1 {
t.Log(s.PointsLog())
t.Error("Intentional parse error screws up all parsing")
}

View File

@ -81,7 +81,7 @@ func (t *T) ParseArgs() (Command, error) {
default:
fmt.Fprintln(t.Stderr, "ERROR:", t.Args[1], "is not a valid command")
usage(t.Stderr)
return nothing, fmt.Errorf("Invalid command")
return nothing, fmt.Errorf("invalid command")
}
if err := flags.Parse(t.Args[2:]); err != nil {

View File

@ -4,7 +4,9 @@ import (
"archive/zip"
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"strings"
"testing"
@ -202,3 +204,32 @@ func TestFilesystem(t *testing.T) {
t.Error("Wrong file pulled", stdout.String())
}
}
func TestCwd(t *testing.T) {
testwd, err := os.Getwd()
if err != nil {
t.Error("Can't get current working directory!")
return
}
defer os.Chdir(testwd)
stdin := new(bytes.Buffer)
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
tp := T{
Stdin: stdin,
Stdout: stdout,
Stderr: stderr,
BaseFs: afero.NewOsFs(),
}
stdout.Reset()
os.Chdir("/")
if err := tp.Run(
"file",
fmt.Sprintf("-dir=%s/testdata/cat1/1", testwd),
"moo.txt",
); err != nil {
t.Error(err)
}
}

View File

@ -25,12 +25,12 @@ so you can watch the access log and any error messages.
### Podman
podman run --rm -it -p 8080:8080 -v /srv/moth/puzzles:/puzzles:ro dirtbags/moth-devel
podman run --rm -it -p 8080:8080 -v /srv/moth/puzzles:/puzzles:ro ghcr.io/dirtbags/moth-devel
### Docker
docker run --rm -it -p 8080:8080 -v /srv/moth/puzzles:/puzzles:ro dirtbags/moth-devel
docker run --rm -it -p 8080:8080 -v /srv/moth/puzzles:/puzzles:ro ghcr.io/dirtbags/moth-devel
### Native

View File

@ -33,11 +33,11 @@ We're going to assume you put everything in `/srv/moth`, like we suggested.
### Podman
podman run --name=moth -d -v /srv/moth/mothballs:/mothballs:ro -v /srv/moth/state:/state dirtbags/moth
podman run --name=moth -d -v /srv/moth/mothballs:/mothballs:ro -v /srv/moth/state:/state ghcr.io/dirtbags/moth
### Docker
docker run --name=moth -d -v /srv/moth/mothballs:/mothballs:ro -v /srv/moth/state:/state dirtbags/moth
docker run --name=moth -d -v /srv/moth/mothballs:/mothballs:ro -v /srv/moth/state:/state ghcr.io/dirtbags/moth
### Native

View File

@ -41,16 +41,18 @@ Each line has four fields:
1602702913 2255 sequence 16
```
`events.log` format
`events.csv` format
----------------------
The events log is a space-separated file.
The events log is a comma-separated variable (CSV) file.
It ought to import into any spreadsheet program painlessly.
Each line has six fields minimum:
| `timestamp` | `event` | `participantID` | `teamID` | `category` | `points` | `extra`... |
| --- | --- | --- | --- | --- | --- | --- |
| int | string | string | string | string | int | string... |
| Unix epoch | Event type | Team's unique ID| Participant's (hopefully) unique ID | Name of category, if any | Points awarded, if any | Additional fields, if any |
| Unix epoch | Event type | Participant's (hopefully) unique ID | Team's unique ID | Name of category, if any | Points awarded, if any | Additional fields, if any |
Fields after `points` contain extra fields associated with the event.
@ -61,6 +63,7 @@ These may change in the future.
* init: startup of server
* disabled: points accumulation disabled
* enabled: points accumulation re-enabled
* register: team registration
* load: puzzle load
* wrong: wrong answer submitted
* correct: correct answer submitted
@ -68,14 +71,14 @@ These may change in the future.
### Example
```
1602716345 init - - - - 0
1602716349 load 2255 player5 sequence 1
1602716450 load 4824 player3 sequence 1
1602716359 correct 2255 player5 sequence 1
1602716423 wrong 4824 player3 sequence 1
1602716428 correct 4824 player3 sequence 1
1602716530 correct 4824 player3 sequence 1
1602716546 abduction 4824 player3 - 0 alien FM1490
1602716345,init,-,-,-,-,0
1602716349,load,2255,player5,sequence,1
1602716450,load,4824,player3,sequence,1
1602716359,correct,2255,player5,sequence,1
1602716423,wrong,4824,player3,sequence,1
1602716428,correct,4824,player3,sequence,1
1602716530,correct,4824,player3,sequence,1
1602716546,abduction,4824,player3,-,0,alien,FM1490
```
The final entry is a made-up "alien abduction" entry,