From ce037ebca300e08d01029de50c444149cc977087 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Wed, 20 Oct 2021 11:43:48 -0600 Subject: [PATCH 01/20] Stop pushing images from Github --- .github/workflows/build+test.yml | 57 ++------------------------------ 1 file changed, 2 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build+test.yml b/.github/workflows/build+test.yml index e25dda5..b24e26f 100644 --- a/.github/workflows/build+test.yml +++ b/.github/workflows/build+test.yml @@ -1,13 +1,4 @@ -name: Build/Test/Push - -on: - push: - branches: - - v3 - - devel - - main - tags: - - 'v*.*.*' +name: Test jobs: test-mothd: @@ -17,54 +8,10 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.13 + go-version: 1.17 - name: Retrieve code uses: actions/checkout@v2 - name: Test run: go test ./... - - publish: - name: Publish container images - runs-on: ubuntu-latest - steps: - - name: Retrieve code - uses: actions/checkout@v2 - - - name: Gitlab variables - id: vars - run: build/ci/gitlab-vars - - - name: Login to GitHub Packages Docker Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.CR_PAT }} - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: neale - password: ${{ secrets.DOCKER_TOKEN }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - # Currently required, because buildx doesn't support auto-push from docker - - name: Set up builder - uses: docker/setup-buildx-action@v1 - id: buildx - - - name: Build and push moth image - uses: docker/build-push-action@v2 - with: - builder: ${{ steps.buildx.outputs.name }} - target: moth - file: build/package/Containerfile - push: true - platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64 - tags: | - dirtbags/moth:${{ steps.vars.outputs.tag }} - ghcr.io/dirtbags/moth:${{ steps.vars.outputs.tag }} From 6f1f889be730867b014b3d3156797e1537ea9f08 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Wed, 20 Oct 2021 13:10:24 -0600 Subject: [PATCH 02/20] Attempt to reproduce #154 --- cmd/transpile/main.go | 2 +- cmd/transpile/main_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/cmd/transpile/main.go b/cmd/transpile/main.go index 58a0bdb..f225464 100644 --- a/cmd/transpile/main.go +++ b/cmd/transpile/main.go @@ -81,7 +81,7 @@ func (t *T) ParseArgs() (Command, error) { default: fmt.Fprintln(t.Stderr, "ERROR:", t.Args[1], "is not a valid command") usage(t.Stderr) - return nothing, fmt.Errorf("Invalid command") + return nothing, fmt.Errorf("invalid command") } if err := flags.Parse(t.Args[2:]); err != nil { diff --git a/cmd/transpile/main_test.go b/cmd/transpile/main_test.go index e501337..042795a 100644 --- a/cmd/transpile/main_test.go +++ b/cmd/transpile/main_test.go @@ -4,7 +4,9 @@ import ( "archive/zip" "bytes" "encoding/json" + "fmt" "io/ioutil" + "os" "strings" "testing" @@ -202,3 +204,32 @@ func TestFilesystem(t *testing.T) { t.Error("Wrong file pulled", stdout.String()) } } + +func TestCwd(t *testing.T) { + testwd, err := os.Getwd() + if err != nil { + t.Error("Can't get current working directory!") + return + } + defer os.Chdir(testwd) + + stdin := new(bytes.Buffer) + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + tp := T{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + BaseFs: afero.NewOsFs(), + } + + stdout.Reset() + os.Chdir("/") + if err := tp.Run( + "file", + fmt.Sprintf("-dir=%s/testdata/cat1/1", testwd), + "moo.txt", + ); err != nil { + t.Error(err) + } +} From 6e5e2c3adf382a2da102b5c902bb8ef3983869fd Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Wed, 20 Oct 2021 14:22:21 -0600 Subject: [PATCH 03/20] v4.3.3 release --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c87828..b8d207b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [v4.3.3] - 2021-10-20 ### Fixed - Points awarded while scoring is paused are now correctly sorted (#168) - Writing a new mothball with the same name is now detected and the new mothball loaded (#172) @@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Many error messages were changed to start with a lower-case letter, in order to satisfy a new linter check. - CI/CD moved to our Cyber Fire Gitlab instance +- I attempted to have the build thingy automatically build moth:v4 and moth:v4.3 and moth:v4.3.3 images, + but I can't test it without tagging a release. + So v4.3.4 might come out very soon after this ;) ## [v4.2.2] - 2021-09-30 ### Added From e1e9157841680eab025313ba90dd17674460b37e Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Wed, 20 Oct 2021 14:22:30 -0600 Subject: [PATCH 04/20] v4.3.3? --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8d207b..fda815b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Points awarded while scoring is paused are now correctly sorted (#168) - Writing a new mothball with the same name is now detected and the new mothball loaded (#172) - Regression test for issue where URL path leading directories were ignored (#144) +- A few other very minor bugs were closed when I couldn't reproduce them or decided they weren't actually bugs. ### Changed - Many error messages were changed to start with a lower-case letter, From e15a505d7be080ea9f8c871ee3457b339d8e993d Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Wed, 20 Oct 2021 14:27:32 -0600 Subject: [PATCH 05/20] v4.4.4. Sigh. --- .gitlab-ci.yml | 2 +- CHANGELOG.md | 4 ++++ build/ci/ci.sh | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b6d291d..a1e5b42 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,7 @@ push: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' script: - mkdir ~/.docker - - echo "$DOCKER_AUTH_CONFIG" > ~/.docker/config.json + - echo "$DOCKER_AUTH_CONFIG" | tee ~/.docker/config.json | md5sum - sh build/ci/ci.sh publish - > docker build diff --git a/CHANGELOG.md b/CHANGELOG.md index fda815b..7913147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [v4.4.4] - 2021-10-20 +### Changed +- Trying to get CI push of built images. I expect this to fail, too. But in a way that can help me debug the issue. + ## [v4.3.3] - 2021-10-20 ### Fixed - Points awarded while scoring is paused are now correctly sorted (#168) diff --git a/build/ci/ci.sh b/build/ci/ci.sh index 7bdb4e3..8e87f52 100755 --- a/build/ci/ci.sh +++ b/build/ci/ci.sh @@ -27,9 +27,9 @@ run () { tags () { pfx=$1 for base in $images; do - echo $pfx $base:${CI_COMMIT_REF_SLUG} - echo $pfx $base:${CI_COMMIT_REF_SLUG%.*} - echo $pfx $base:${CI_COMMIT_REF_SLUG%.*.*} + echo $pfx $base:${CI_COMMIT_REF_NAME} + echo $pfx $base:${CI_COMMIT_REF_NAME%.*} + echo $pfx $base:${CI_COMMIT_REF_NAME%.*.*} done | uniq } From 127beca1fc84c3432e67085ca441ef33e28b8553 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Wed, 20 Oct 2021 14:35:21 -0600 Subject: [PATCH 06/20] Remove superfluous CI build script line in --- .gitlab-ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1e5b42..fa43a60 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,8 +21,3 @@ push: - mkdir ~/.docker - echo "$DOCKER_AUTH_CONFIG" | tee ~/.docker/config.json | md5sum - sh build/ci/ci.sh publish - - > - docker build - --tag ghcr.io/dirtbags/moth:$CI_COMMIT_REF_SLUG - --file build/package/Containerfile - . From a2ce3682ab1770a6e77510173a68dccd4b64d606 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Wed, 20 Oct 2021 14:47:32 -0600 Subject: [PATCH 07/20] Push images to docker hub, but say to use ghcr --- CHANGELOG.md | 4 ++++ README.md | 2 +- docs/development.md | 4 ++-- docs/getting-started.md | 4 ++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7913147..21ba822 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] - 2021-10-20 +### Added +- Images deploying to docker hub too. We're now at capacity for our Docker Hub team. + ## [v4.4.4] - 2021-10-20 ### Changed - Trying to get CI push of built images. I expect this to fail, too. But in a way that can help me debug the issue. diff --git a/README.md b/README.md index c6709cb..b78bab7 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ You can read more about why we made these decisions in [philosophy](docs/philoso Run in demonstration mode =========== - docker run --rm -it -p 8080:8080 dirtbags/moth-devel + docker run --rm -it -p 8080:8080 ghcr.io/dirtbags/moth-devel Then open http://localhost:8080/ and check out the example puzzles. diff --git a/docs/development.md b/docs/development.md index 1a14b32..ccc789a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -25,12 +25,12 @@ so you can watch the access log and any error messages. ### Podman - podman run --rm -it -p 8080:8080 -v /srv/moth/puzzles:/puzzles:ro dirtbags/moth-devel + podman run --rm -it -p 8080:8080 -v /srv/moth/puzzles:/puzzles:ro ghcr.io/dirtbags/moth-devel ### Docker - docker run --rm -it -p 8080:8080 -v /srv/moth/puzzles:/puzzles:ro dirtbags/moth-devel + docker run --rm -it -p 8080:8080 -v /srv/moth/puzzles:/puzzles:ro ghcr.io/dirtbags/moth-devel ### Native diff --git a/docs/getting-started.md b/docs/getting-started.md index a1a9258..9cc6185 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -33,11 +33,11 @@ We're going to assume you put everything in `/srv/moth`, like we suggested. ### Podman - podman run --name=moth -d -v /srv/moth/mothballs:/mothballs:ro -v /srv/moth/state:/state dirtbags/moth + podman run --name=moth -d -v /srv/moth/mothballs:/mothballs:ro -v /srv/moth/state:/state ghcr.io/dirtbags/moth ### Docker - docker run --name=moth -d -v /srv/moth/mothballs:/mothballs:ro -v /srv/moth/state:/state dirtbags/moth + docker run --name=moth -d -v /srv/moth/mothballs:/mothballs:ro -v /srv/moth/state:/state ghcr.io/dirtbags/moth ### Native From 40f8f717785a36a63315cd4bc7ce17f432c3f9b1 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Wed, 20 Oct 2021 14:49:40 -0600 Subject: [PATCH 08/20] oops, add in dockerhub repo to ci.sh --- build/ci/ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci/ci.sh b/build/ci/ci.sh index 8e87f52..773dba4 100755 --- a/build/ci/ci.sh +++ b/build/ci/ci.sh @@ -2,7 +2,7 @@ set -e -images="ghcr.io/dirtbags/moth" +images="ghcr.io/dirtbags/moth dirtbags/moth" ACTION=$1 if [ -z "$ACTION" ]; then From 2003b20cc411cd7a48708a50cd02a36d119c4031 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Thu, 21 Oct 2021 17:40:40 -0600 Subject: [PATCH 09/20] fix documentation error --- docs/logs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/logs.md b/docs/logs.md index 24e4704..2217f04 100644 --- a/docs/logs.md +++ b/docs/logs.md @@ -50,7 +50,7 @@ Each line has six fields minimum: | `timestamp` | `event` | `participantID` | `teamID` | `category` | `points` | `extra`... | | --- | --- | --- | --- | --- | --- | --- | | int | string | string | string | string | int | string... | -| Unix epoch | Event type | Team's unique ID| Participant's (hopefully) unique ID | Name of category, if any | Points awarded, if any | Additional fields, if any | +| Unix epoch | Event type | Participant's (hopefully) unique ID | Team's unique ID | Name of category, if any | Points awarded, if any | Additional fields, if any | Fields after `points` contain extra fields associated with the event. From ace940ba12f1e0a99823bd270322580d3f5ac8e2 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Thu, 21 Oct 2021 18:25:49 -0600 Subject: [PATCH 10/20] further update events.csv description --- docs/logs.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/logs.md b/docs/logs.md index 2217f04..a5b5422 100644 --- a/docs/logs.md +++ b/docs/logs.md @@ -41,10 +41,12 @@ Each line has four fields: 1602702913 2255 sequence 16 ``` -`events.log` format +`events.csv` format ---------------------- -The events log is a space-separated file. +The events log is a comma-separated variable (CSV) file. +It ought to import into any spreadsheet program painlessly. + Each line has six fields minimum: | `timestamp` | `event` | `participantID` | `teamID` | `category` | `points` | `extra`... | @@ -61,6 +63,7 @@ These may change in the future. * init: startup of server * disabled: points accumulation disabled * enabled: points accumulation re-enabled +* register: team registration * load: puzzle load * wrong: wrong answer submitted * correct: correct answer submitted @@ -68,14 +71,14 @@ These may change in the future. ### Example ``` -1602716345 init - - - - 0 -1602716349 load 2255 player5 sequence 1 -1602716450 load 4824 player3 sequence 1 -1602716359 correct 2255 player5 sequence 1 -1602716423 wrong 4824 player3 sequence 1 -1602716428 correct 4824 player3 sequence 1 -1602716530 correct 4824 player3 sequence 1 -1602716546 abduction 4824 player3 - 0 alien FM1490 +1602716345,init,-,-,-,-,0 +1602716349,load,2255,player5,sequence,1 +1602716450,load,4824,player3,sequence,1 +1602716359,correct,2255,player5,sequence,1 +1602716423,wrong,4824,player3,sequence,1 +1602716428,correct,4824,player3,sequence,1 +1602716530,correct,4824,player3,sequence,1 +1602716546,abduction,4824,player3,-,0,alien,FM1490 ``` The final entry is a made-up "alien abduction" entry, From 4bb68193199baa65732ff8fd4a52b5a487b35178 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Mon, 25 Oct 2021 13:37:56 -0600 Subject: [PATCH 11/20] Fix to reopening all mothballs every 2s #180 --- cmd/mothd/mothballs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/mothd/mothballs.go b/cmd/mothd/mothballs.go index 67b7679..f204975 100644 --- a/cmd/mothd/mothballs.go +++ b/cmd/mothd/mothballs.go @@ -168,6 +168,7 @@ func (m *Mothballs) refresh() { m.categories[categoryName] = zipCategory{ Fs: zipfs.New(zrc), Closer: f, + mtime: fi.ModTime(), } log.Println("Adding category:", categoryName) From 471ded7303fe40a0ab04f1e9ecd7047e13bd881e Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Tue, 26 Oct 2021 12:48:23 -0600 Subject: [PATCH 12/20] Cache state --- CHANGELOG.md | 7 ++- cmd/mothd/httpd_test.go | 2 + cmd/mothd/server_test.go | 5 +- cmd/mothd/state.go | 115 +++++++++++++++++++++++++++++---------- cmd/mothd/state_test.go | 29 ++++++---- 5 files changed, 116 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21ba822..6173708 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] - 2021-10-20 +## [unreleased] - 2021-10-26 +### Added +- State is now cached in memory, in an attempt to reduce filesystem metadata operations, + which kill NFS. + +## [v4.4.5] - 2021-10-26 ### Added - Images deploying to docker hub too. We're now at capacity for our Docker Hub team. diff --git a/cmd/mothd/httpd_test.go b/cmd/mothd/httpd_test.go index 6c623c3..fc4ae9a 100644 --- a/cmd/mothd/httpd_test.go +++ b/cmd/mothd/httpd_test.go @@ -70,6 +70,8 @@ func TestHttpd(t *testing.T) { t.Error("Register failed", r.Body.String()) } + time.Sleep(TestMaintenanceInterval) + if r := hs.TestRequest("/state", nil); r.Result().StatusCode != 200 { t.Error(r.Result()) } else if r.Body.String() != `{"Config":{"Devel":false},"Messages":"messages.html","TeamNames":{"self":"GoTeam"},"PointsLog":[],"Puzzles":{"pategory":[1]}}` { diff --git a/cmd/mothd/server_test.go b/cmd/mothd/server_test.go index 2434e22..6b6e621 100644 --- a/cmd/mothd/server_test.go +++ b/cmd/mothd/server_test.go @@ -80,13 +80,16 @@ func TestProdServer(t *testing.T) { t.Error("index.html wrong contents", contents) } + // Wait for refresh to pick everything up + time.Sleep(TestMaintenanceInterval) + { es := handler.ExportState() if es.Config.Devel { t.Error("Marked as development server", es.Config) } if len(es.Puzzles) != 1 { - t.Error("Puzzle categories wrong length") + t.Error("Puzzle categories wrong length", len(es.Puzzles)) } if es.Messages != "messages.html" { t.Error("Messages has wrong contents") diff --git a/cmd/mothd/state.go b/cmd/mothd/state.go index b8cb231..9d11621 100644 --- a/cmd/mothd/state.go +++ b/cmd/mothd/state.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" "github.com/dirtbags/moth/pkg/award" @@ -42,6 +43,12 @@ type State struct { eventStream chan []string eventWriter *csv.Writer eventWriterFile afero.File + + // Caches, so we're not hammering NFS with metadata operations + teamNames map[string]string + pointsLog award.List + messages string + lock sync.RWMutex } // NewState returns a new State struct backed by the given Fs @@ -51,6 +58,8 @@ func NewState(fs afero.Fs) *State { Enabled: true, refreshNow: make(chan bool, 5), eventStream: make(chan []string, 80), + + teamNames: make(map[string]string), } if err := s.reopenEventLog(); err != nil { log.Fatal(err) @@ -120,16 +129,13 @@ func (s *State) updateEnabled() { // TeamName returns team name given a team ID. func (s *State) TeamName(teamID string) (string, error) { - teamFs := afero.NewBasePathFs(s.Fs, "teams") - teamNameBytes, err := afero.ReadFile(teamFs, teamID) - if os.IsNotExist(err) { + s.lock.RLock() + name, ok := s.teamNames[teamID] + s.lock.RUnlock() + if !ok { return "", fmt.Errorf("unregistered team ID: %s", teamID) - } else if err != nil { - return "", fmt.Errorf("unregistered team ID: %s (%s)", teamID, err) } - - teamName := strings.TrimSpace(string(teamNameBytes)) - return teamName, nil + return name, nil } // SetTeamName writes out team name. @@ -163,36 +169,26 @@ func (s *State) SetTeamName(teamID, teamName string) error { log.Printf("Setting team name [%s] in file %s", teamName, teamFilename) fmt.Fprintln(teamFile, teamName) teamFile.Close() + + s.refreshNow <- true + return nil } // PointsLog retrieves the current points log. func (s *State) PointsLog() award.List { - f, err := s.Open("points.log") - if err != nil { - log.Println(err) - return nil - } - defer f.Close() - - pointsLog := make(award.List, 0, 200) - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - cur, err := award.Parse(line) - if err != nil { - log.Printf("Skipping malformed award line %s: %s", line, err) - continue - } - pointsLog = append(pointsLog, cur) - } - return pointsLog + s.lock.RLock() + ret := make(award.List, len(s.pointsLog)) + copy(ret, s.pointsLog) + s.lock.RUnlock() + return ret } // Messages retrieves the current messages. func (s *State) Messages() string { - bMessages, _ := afero.ReadFile(s, "messages.html") - return string(bMessages) + s.lock.RLock() // It's not clear to me that this actually needs to happen + defer s.lock.RUnlock() + return s.messages } // AwardPoints gives points to teamID in category. @@ -260,12 +256,14 @@ func (s *State) collectPoints() { } duplicate := false - for _, e := range s.PointsLog() { + s.lock.RLock() + for _, e := range s.pointsLog { if awd.Equal(e) { duplicate = true break } } + s.lock.RUnlock() if duplicate { log.Print("Skipping duplicate points: ", awd.String()) @@ -279,6 +277,11 @@ func (s *State) collectPoints() { } fmt.Fprintln(logf, awd.String()) logf.Close() + + // Stick this on the cache too + s.lock.Lock() + s.pointsLog = append(s.pointsLog, awd) + s.lock.Unlock() } if err := s.Remove(filename); err != nil { @@ -402,12 +405,64 @@ func (s *State) reopenEventLog() error { return nil } +func (s *State) updateCaches() { + s.lock.Lock() + defer s.lock.Unlock() + + if f, err := s.Open("points.log"); err != nil { + log.Println(err) + } else { + defer f.Close() + + pointsLog := make(award.List, 0, 200) + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + cur, err := award.Parse(line) + if err != nil { + log.Printf("Skipping malformed award line %s: %s", line, err) + continue + } + pointsLog = append(pointsLog, cur) + } + s.pointsLog = pointsLog + } + + { + // The compiler recognizes this as an optimization case + for k := range s.teamNames { + delete(s.teamNames, k) + } + + teamsFs := afero.NewBasePathFs(s.Fs, "teams") + if dirents, err := afero.ReadDir(teamsFs, "."); err != nil { + log.Printf("Reading team ids: %v", err) + } else { + for _, dirent := range dirents { + teamID := dirent.Name() + if teamNameBytes, err := afero.ReadFile(teamsFs, teamID); err != nil { + log.Printf("Reading team %s: %v", teamID, err) + } else { + teamName := strings.TrimSpace(string(teamNameBytes)) + s.teamNames[teamID] = teamName + } + } + } + + } + + if bMessages, err := afero.ReadFile(s, "messages.html"); err == nil { + s.messages = string(bMessages) + } +} + func (s *State) refresh() { s.maybeInitialize() s.updateEnabled() if s.Enabled { s.collectPoints() } + s.updateCaches() } // Maintain performs housekeeping on a State struct. diff --git a/cmd/mothd/state_test.go b/cmd/mothd/state_test.go index 5cb1097..21f1ea2 100644 --- a/cmd/mothd/state_test.go +++ b/cmd/mothd/state_test.go @@ -62,6 +62,7 @@ func TestState(t *testing.T) { if err := s.SetTeamName(teamID, "wat"); err == nil { t.Errorf("Registering team a second time didn't fail") } + s.refresh() if name, err := s.TeamName(teamID); err != nil { t.Error(err) } else if name != teamName { @@ -73,9 +74,6 @@ func TestState(t *testing.T) { if err := s.AwardPoints(teamID, category, points); err != nil { t.Error(err) } - if err := s.AwardPoints(teamID, category, points); err != nil { - t.Error("Two awards before refresh:", err) - } // Flex duplicate detection with different timestamp if f, err := s.Create("points.new/moo"); err != nil { t.Error("Creating duplicate points file:", err) @@ -83,24 +81,34 @@ func TestState(t *testing.T) { fmt.Fprintln(f, time.Now().Unix()+1, teamID, category, points) f.Close() } + + s.AwardPoints(teamID, category, points) s.refresh() + pl = s.PointsLog() + if len(pl) != 1 { + for i, award := range pl { + t.Logf("pl[%d] == %s", i, award.String()) + } + t.Errorf("After awarding duplicate points, points log has length %d", len(pl)) + } else if (pl[0].TeamID != teamID) || (pl[0].Category != category) || (pl[0].Points != points) { + t.Errorf("Incorrect logged award %v", pl) + } if err := s.AwardPoints(teamID, category, points); err == nil { - t.Error("Duplicate points award didn't fail") + t.Error("Duplicate points award after refresh didn't fail") } if err := s.AwardPoints(teamID, category, points+1); err != nil { t.Error("Awarding more points:", err) } - pl = s.PointsLog() - if len(pl) != 1 { - t.Errorf("After awarding points, points log has length %d", len(pl)) - } else if (pl[0].TeamID != teamID) || (pl[0].Category != category) || (pl[0].Points != points) { - t.Errorf("Incorrect logged award %v", pl) + s.refresh() + if len(s.PointsLog()) != 2 { + t.Errorf("There should be two awards") } afero.WriteFile(s, "points.log", []byte("intentional parse error\n"), 0644) + s.refresh() if len(s.PointsLog()) != 0 { t.Errorf("Intentional parse error breaks pointslog") } @@ -108,7 +116,8 @@ func TestState(t *testing.T) { t.Error(err) } s.refresh() - if len(s.PointsLog()) != 2 { + if len(s.PointsLog()) != 1 { + t.Log(s.PointsLog()) t.Error("Intentional parse error screws up all parsing") } From bb41697ba6b13d1896ba9526666a55c92034315e Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Tue, 26 Oct 2021 13:33:57 -0600 Subject: [PATCH 13/20] v4.4.6 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6173708..1d0aa54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] - 2021-10-26 +## [v4.4.6] - 2021-10-26 ### Added - State is now cached in memory, in an attempt to reduce filesystem metadata operations, which kill NFS. From b6eea388d9d5a6affd0a6eafa48db091158e697e Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Fri, 5 Nov 2021 14:30:06 -0600 Subject: [PATCH 14/20] Remove github workflow --- .github/workflows/build+test.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .github/workflows/build+test.yml diff --git a/.github/workflows/build+test.yml b/.github/workflows/build+test.yml deleted file mode 100644 index b24e26f..0000000 --- a/.github/workflows/build+test.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Test - -jobs: - test-mothd: - name: Test mothd - runs-on: ubuntu-latest - steps: - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: 1.17 - - - name: Retrieve code - uses: actions/checkout@v2 - - - name: Test - run: go test ./... From eea674b1a45e1bde43a21d66d983ba1f8505fba2 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Tue, 10 May 2022 13:30:44 -0600 Subject: [PATCH 15/20] Remove `events.csv` on init. Fixes #177 --- cmd/mothd/state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/mothd/state.go b/cmd/mothd/state.go index 9d11621..d8f09d1 100644 --- a/cmd/mothd/state.go +++ b/cmd/mothd/state.go @@ -303,6 +303,7 @@ func (s *State) maybeInitialize() { s.Remove("enabled") s.Remove("hours.txt") s.Remove("points.log") + s.Remove("events.csv") s.Remove("messages.html") s.Remove("mothd.log") s.RemoveAll("points.tmp") From e5a3b26c934e77798166490dfab1bba174931a3c Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Tue, 10 May 2022 15:26:01 -0600 Subject: [PATCH 16/20] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d0aa54..65d2a40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [v4.4.7] - 2022-05-10 +### Changed +- Initializing an instance now truncates `events.csv` + ## [v4.4.6] - 2021-10-26 ### Added - State is now cached in memory, in an attempt to reduce filesystem metadata operations, From 5b6555cd9adafb7d57aaa7b079e41f2ebc4185e5 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Tue, 10 May 2022 17:47:26 -0600 Subject: [PATCH 17/20] Check team existence before registering. Fixes #156 --- cmd/mothd/issues_test.go | 28 ++++++++++++++++++++++++++++ cmd/mothd/server.go | 7 +++++-- cmd/mothd/state.go | 7 +++++++ 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 cmd/mothd/issues_test.go diff --git a/cmd/mothd/issues_test.go b/cmd/mothd/issues_test.go new file mode 100644 index 0000000..26a6f6d --- /dev/null +++ b/cmd/mothd/issues_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "testing" + + "github.com/spf13/afero" +) + +func TestIssue156(t *testing.T) { + puzzles := NewTestMothballs() + state := NewTestState() + theme := NewTestTheme() + server := NewMothServer(Configuration{}, theme, state, puzzles) + + afero.WriteFile(state, "teams/bloop", []byte("bloop: the team"), 0644) + state.refresh() + + handler := server.NewHandler("", "bloop") + es := handler.ExportState() + if _, ok := es.TeamNames["self"]; !ok { + t.Fail() + } + + err := handler.Register("bloop: the other team") + if err != ErrAlreadyRegistered { + t.Fail() + } +} diff --git a/cmd/mothd/server.go b/cmd/mothd/server.go index 608e6d6..f53ecdf 100644 --- a/cmd/mothd/server.go +++ b/cmd/mothd/server.go @@ -184,12 +184,15 @@ func (mh *MothRequestHandler) ExportState() *StateExport { return mh.exportStateIfRegistered(false) } -func (mh *MothRequestHandler) exportStateIfRegistered(override bool) *StateExport { +// Export state, replacing the team ID with "self" if the team is registered. +// +// If forceRegistered is true, go ahead and export it anyway +func (mh *MothRequestHandler) exportStateIfRegistered(forceRegistered bool) *StateExport { export := StateExport{} export.Config = mh.Config teamName, err := mh.State.TeamName(mh.teamID) - registered := override || mh.Config.Devel || (err == nil) + registered := forceRegistered || mh.Config.Devel || (err == nil) export.Messages = mh.State.Messages() export.TeamNames = make(map[string]string) diff --git a/cmd/mothd/state.go b/cmd/mothd/state.go index d8f09d1..eca4d5b 100644 --- a/cmd/mothd/state.go +++ b/cmd/mothd/state.go @@ -141,6 +141,13 @@ func (s *State) TeamName(teamID string) (string, error) { // SetTeamName writes out team name. // This can only be done once per team. func (s *State) SetTeamName(teamID, teamName string) error { + s.lock.RLock() + _, ok := s.teamNames[teamID] + s.lock.RUnlock() + if ok { + return ErrAlreadyRegistered + } + idsFile, err := s.Open("teamids.txt") if err != nil { return fmt.Errorf("team IDs file does not exist") From 6d7fb9ebf51eb45f573e8e82b4670f97bf5cb4cc Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Tue, 10 May 2022 17:53:31 -0600 Subject: [PATCH 18/20] update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65d2a40..215c54a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [v4.4.8] - 2022-05-10 +### Changed +- You can now join with a team ID not appearing in `teamids.txt`, + as long as it is registered (in the `teams/` directory) + ## [v4.4.7] - 2022-05-10 ### Changed - Initializing an instance now truncates `events.csv` From d014384b05ec181ecf9a9bbb75455e5326bb4cc0 Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Tue, 10 May 2022 17:59:08 -0600 Subject: [PATCH 19/20] A possible fix for #179 --- cmd/mothd/server_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/mothd/server_test.go b/cmd/mothd/server_test.go index 6b6e621..ab4f986 100644 --- a/cmd/mothd/server_test.go +++ b/cmd/mothd/server_test.go @@ -13,11 +13,13 @@ const TestTeamID = "teamID" func NewTestServer() *MothServer { puzzles := NewTestMothballs() + puzzles.refresh() go puzzles.Maintain(TestMaintenanceInterval) state := NewTestState() afero.WriteFile(state, "teamids.txt", []byte("teamID\n"), 0644) afero.WriteFile(state, "messages.html", []byte("messages.html"), 0644) + state.refresh() go state.Maintain(TestMaintenanceInterval) theme := NewTestTheme() From be74961e942a9c57e92f98f1114c3a6b28d8ad0f Mon Sep 17 00:00:00 2001 From: Neale Pickett Date: Tue, 10 May 2022 19:11:47 -0600 Subject: [PATCH 20/20] still trying to fix race condition --- .gitlab-ci.yml | 2 +- cmd/mothd/httpd_test.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fa43a60..65fc796 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ stages: test: stage: test - image: golang:1.17 + image: golang:1.18 only: refs: - main diff --git a/cmd/mothd/httpd_test.go b/cmd/mothd/httpd_test.go index fc4ae9a..f5482d2 100644 --- a/cmd/mothd/httpd_test.go +++ b/cmd/mothd/httpd_test.go @@ -33,7 +33,9 @@ func (hs *HTTPServer) TestRequest(path string, args map[string]string) *httptest } func TestHttpd(t *testing.T) { - hs := NewHTTPServer("/", NewTestServer()) + server := NewTestServer() + hs := NewHTTPServer("/", server) + stateProvider := server.State.(*State) if r := hs.TestRequest("/", nil); r.Result().StatusCode != 200 { t.Error(r.Result()) @@ -113,6 +115,7 @@ func TestHttpd(t *testing.T) { } time.Sleep(TestMaintenanceInterval) + stateProvider.refresh() if r := hs.TestRequest("/content/pategory/2/puzzle.json", nil); r.Result().StatusCode != 200 { t.Error(r.Result())