transpiler work

This commit is contained in:
Neale Pickett 2020-08-28 17:41:17 -06:00
parent 7d5e215b75
commit 0418877db0
7 changed files with 367 additions and 239 deletions

View File

@ -1,138 +1,47 @@
package main
import (
"os"
"fmt"
"log"
"path/filepath"
"hash/fnv"
"encoding/binary"
"encoding/json"
"encoding/hex"
"strconv"
"math/rand"
"context"
"time"
"os/exec"
"bytes"
"github.com/spf13/afero"
)
type PuzzleEntry struct {
Id string
Points int
Puzzle Puzzle
// NewCategory returns a new category for the given path in the given fs.
func NewCategory(fs afero.Fs, cat string) Category {
return Category{
Fs: afero.NewBasePathFs(fs, cat),
}
}
func PrngOfStrings(input ...string) (*rand.Rand) {
hasher := fnv.New64()
for _, s := range input {
fmt.Fprint(hasher, s, "\n")
}
seed := binary.BigEndian.Uint64(hasher.Sum(nil))
source := rand.NewSource(int64(seed))
return rand.New(source)
// Category represents an on-disk category.
type Category struct {
afero.Fs
}
func runPuzzleGen(puzzlePath string, seed string) (*Puzzle, error) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, puzzlePath)
cmd.Env = append(
os.Environ(),
fmt.Sprintf("MOTH_PUZZLE_SEED=%s", seed),
)
stdout, err := cmd.Output()
// Puzzles returns a list of puzzle values.
func (c Category) Puzzles() ([]int, error) {
puzzleEntries, err := afero.ReadDir(c, ".")
if err != nil {
return nil, err
}
jsdec := json.NewDecoder(bytes.NewReader(stdout))
jsdec.DisallowUnknownFields()
puzzle := new(Puzzle)
err = jsdec.Decode(puzzle)
if err != nil {
return nil, err
}
return puzzle, nil
}
func ParsePuzzle(puzzlePath string, puzzleSeed string) (*Puzzle, error) {
var puzzle *Puzzle
// Try the .moth file first
puzzleMothPath := filepath.Join(puzzlePath, "puzzle.moth")
puzzleFd, err := os.Open(puzzleMothPath)
if err == nil {
defer puzzleFd.Close()
puzzle, err = ParseMoth(puzzleFd)
if err != nil {
return nil, err
}
} else if os.IsNotExist(err) {
var genErr error
puzzleGenPath := filepath.Join(puzzlePath, "mkpuzzle")
puzzle, genErr = runPuzzleGen(puzzleGenPath, puzzlePath)
if genErr != nil {
bigErr := fmt.Errorf(
"%v; (%s: %v)",
genErr,
filepath.Base(puzzleMothPath), err,
)
return nil, bigErr
}
} else {
return nil, err
}
return puzzle, nil
}
func ParseCategory(categoryPath string, seed string) ([]PuzzleEntry, error) {
categoryFd, err := os.Open(categoryPath)
if err != nil {
return nil, err
}
defer categoryFd.Close()
puzzleDirs, err := categoryFd.Readdirnames(0)
if err != nil {
return nil, err
}
puzzleEntries := make([]PuzzleEntry, 0, len(puzzleDirs))
for _, puzzleDir := range puzzleDirs {
puzzlePath := filepath.Join(categoryPath, puzzleDir)
puzzleSeed := fmt.Sprintf("%s/%s", seed, puzzleDir)
puzzle, err := ParsePuzzle(puzzlePath, puzzleSeed)
if err != nil {
log.Printf("Skipping %s: %v", puzzlePath, err)
puzzles := make([]int, 0, len(puzzleEntries))
for _, ent := range puzzleEntries {
if !ent.IsDir() {
continue
}
// Determine point value from directory name
points, err := strconv.Atoi(puzzleDir)
if err != nil {
return nil, err
if points, err := strconv.Atoi(ent.Name()); err != nil {
log.Println("Skipping non-numeric directory", ent.Name())
continue
} else {
puzzles = append(puzzles, points)
}
// Create a category entry for this
prng := PrngOfStrings(puzzlePath)
idBytes := make([]byte, 16)
prng.Read(idBytes)
id := hex.EncodeToString(idBytes)
puzzleEntry := PuzzleEntry{
Id: id,
Puzzle: *puzzle,
Points: points,
}
puzzleEntries = append(puzzleEntries, puzzleEntry)
}
return puzzleEntries, nil
return puzzles, nil
}
// Puzzle returns the Puzzle associated with points.
func (c Category) Puzzle(points int) (*Puzzle, error) {
return NewPuzzle(c.Fs, points)
}

View File

@ -1,9 +0,0 @@
module github.com/russross/blackfriday/v2
go 1.13
require (
github.com/russross/blackfriday v2.0.0+incompatible
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
gopkg.in/yaml.v2 v2.3.0
)

View File

@ -1,7 +0,0 @@
github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

View File

@ -1,74 +1,125 @@
package main
import (
"flag"
"encoding/json"
"path/filepath"
"strconv"
"strings"
"os"
"log"
"flag"
"fmt"
"io"
"log"
"os"
"sort"
"github.com/GoBike/envflag"
"github.com/spf13/afero"
)
func seedJoin(parts ...string) string {
return strings.Join(parts, "::")
// T contains everything required for a transpilation invocation (across the nation).
type T struct {
// What action to take
w io.Writer
Cat string
Points int
Answer string
Filename string
Fs afero.Fs
}
func usage() {
out := flag.CommandLine.Output()
name := flag.CommandLine.Name()
fmt.Fprintf(out, "Usage: %s [OPTION]... CATEGORY [PUZZLE [FILENAME]]\n", name)
fmt.Fprintf(out, "\n")
fmt.Fprintf(out, "Transpile CATEGORY, or provide individual category components.\n")
fmt.Fprintf(out, "If PUZZLE is provided, only transpile the given puzzle.\n")
fmt.Fprintf(out, "If FILENAME is provided, output provided file.\n")
flag.PrintDefaults()
// NewCategory returns a new Category as specified by cat.
func (t *T) NewCategory(cat string) Category {
return NewCategory(t.Fs, cat)
}
// ParseArgs parses command-line arguments into T, returning the action to take
func (t *T) ParseArgs() string {
action := flag.String("action", "inventory", "Action to take: must be 'inventory', 'open', 'answer', or 'mothball'")
flag.StringVar(&t.Cat, "cat", "", "Puzzle category")
flag.IntVar(&t.Points, "points", 0, "Puzzle point value")
flag.StringVar(&t.Answer, "answer", "", "Answer to check for correctness, for 'answer' action")
flag.StringVar(&t.Filename, "filename", "", "Filename, for 'open' action")
basedir := flag.String("basedir", ".", "Base directory containing all puzzles")
envflag.Parse()
osfs := afero.NewOsFs()
t.Fs = afero.NewBasePathFs(osfs, *basedir)
return *action
}
// Handle performs the requested action
func (t *T) Handle(action string) error {
switch action {
case "inventory":
return t.PrintInventory()
case "open":
return t.Open()
default:
return fmt.Errorf("Unimplemented action: %s", action)
}
}
// PrintInventory prints a puzzle inventory to stdout
func (t *T) PrintInventory() error {
dirEnts, err := afero.ReadDir(t.Fs, ".")
if err != nil {
return err
}
for _, ent := range dirEnts {
if ent.IsDir() {
c := t.NewCategory(ent.Name())
if puzzles, err := c.Puzzles(); err != nil {
log.Print(err)
continue
} else {
fmt.Fprint(t.w, ent.Name())
sort.Ints(puzzles)
for _, points := range puzzles {
fmt.Fprint(t.w, " ")
fmt.Fprint(t.w, points)
}
fmt.Fprintln(t.w)
}
}
}
return nil
}
// Open writes a file to the writer.
func (t *T) Open() error {
c := t.NewCategory(t.Cat)
p, err := c.Puzzle(t.Points)
if err != nil {
return err
}
switch t.Filename {
case "puzzle.json", "":
jp, err := json.Marshal(p)
if err != nil {
return err
}
t.w.Write(jp)
default:
f, err := p.Open(t.Filename)
if err != nil {
return err
}
defer f.Close()
if _, err := io.Copy(t.w, f); err != nil {
return err
}
}
return nil
}
func main() {
// XXX: Convert puzzle.py to standalone thingies
flag.Usage = usage
points := flag.Int("points", 0, "Transpile only this point value puzzle")
mothball := flag.Bool("mothball", false, "Generate a mothball")
flag.Parse()
baseSeedString := os.Getenv("MOTH_SEED")
jsenc := json.NewEncoder(os.Stdout)
jsenc.SetEscapeHTML(false)
jsenc.SetIndent("", " ")
for _, categoryPath := range flag.Args() {
categoryName := filepath.Base(categoryPath)
categorySeed := seedJoin(baseSeedString, categoryName)
if *points > 0 {
puzzleDir := strconv.Itoa(*points)
puzzleSeed := seedJoin(categorySeed, puzzleDir)
puzzlePath := filepath.Join(categoryPath, puzzleDir)
puzzle, err := ParsePuzzle(puzzlePath, puzzleSeed)
if err != nil {
log.Print(err)
continue
t := &T{
w: os.Stdout,
}
if err := jsenc.Encode(puzzle); err != nil {
action := t.ParseArgs()
if err := t.Handle(action); err != nil {
log.Fatal(err)
}
} else {
puzzles, err := ParseCategory(categoryPath, categorySeed)
if err != nil {
log.Print(err)
continue
}
if err := jsenc.Encode(puzzles); err != nil {
log.Print(err)
continue
}
}
}
}

View File

@ -0,0 +1,88 @@
package main
import (
"bytes"
"encoding/json"
"testing"
"github.com/spf13/afero"
)
var testMothYaml = []byte(`---
answers:
- YAML answer
pre:
authors:
- Arthur
- Buster
- DW
---
YAML body
`)
var testMothRfc822 = []byte(`author: test
Author: Arthur
author: Fred Flintstone
answer: RFC822 answer
RFC822 body
`)
func newTestFs() afero.Fs {
fs := afero.NewMemMapFs()
afero.WriteFile(fs, "cat0/1/puzzle.moth", testMothYaml, 0644)
afero.WriteFile(fs, "cat0/1/moo.txt", []byte("Moo."), 0644)
afero.WriteFile(fs, "cat0/2/puzzle.moth", testMothRfc822, 0644)
afero.WriteFile(fs, "cat0/3/puzzle.moth", testMothYaml, 0644)
afero.WriteFile(fs, "cat0/4/puzzle.moth", testMothYaml, 0644)
afero.WriteFile(fs, "cat0/5/puzzle.moth", testMothYaml, 0644)
afero.WriteFile(fs, "cat0/10/puzzle.moth", []byte(`---
Answers:
- moo
Authors:
- bad field
---
body
`), 0644)
afero.WriteFile(fs, "cat0/20/puzzle.moth", []byte("Answer: no\nBadField: yes\n\nbody\n"), 0644)
afero.WriteFile(fs, "cat1/93/puzzle.moth", []byte("Answer: no\n\nbody"), 0644)
return fs
}
func TestThings(t *testing.T) {
stdout := new(bytes.Buffer)
tp := T{
w: stdout,
Fs: newTestFs(),
}
if err := tp.Handle("inventory"); err != nil {
t.Error(err)
}
if stdout.String() != "cat0 1 2 3 4 5 10 20\ncat1 93\n" {
t.Errorf("Bad inventory: %#v", stdout.String())
}
stdout.Reset()
tp.Cat = "cat0"
tp.Points = 1
if err := tp.Handle("open"); err != nil {
t.Error(err)
}
p := Puzzle{}
if err := json.Unmarshal(stdout.Bytes(), &p); err != nil {
t.Error(err)
}
if p.Answers[0] != "YAML answer" {
t.Error("Didn't return the right object")
}
stdout.Reset()
tp.Filename = "moo.txt"
if err := tp.Handle("open"); err != nil {
t.Error(err)
}
if stdout.String() != "Moo." {
t.Error("Wrong file pulled")
}
}

View File

@ -3,22 +3,36 @@ package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log"
"net/mail"
"os/exec"
"strconv"
"strings"
"time"
"github.com/russross/blackfriday"
"github.com/spf13/afero"
"gopkg.in/yaml.v2"
)
type Attachment struct {
Filename string // Filename presented as part of puzzle
FilesystemPath string // Filename in backing FS (URL, mothball, or local FS)
Listed bool // Whether this file is listed as an attachment
// NewPuzzle returns a new Puzzle for points.
func NewPuzzle(fs afero.Fs, points int) (*Puzzle, error) {
p := &Puzzle{
fs: afero.NewBasePathFs(fs, strconv.Itoa(points)),
}
if err := p.parseMoth(); err != nil {
return p, err
}
// BUG(neale): Doesn't yet handle "puzzle.py" or "mkpuzzle"
return p, nil
}
// Puzzle contains everything about a puzzle.
type Puzzle struct {
Pre struct {
Authors []string
@ -42,21 +56,17 @@ type Puzzle struct {
Summary string
}
Answers []string
fs afero.Fs
}
type HeaderParser func([]byte) (*Puzzle, error)
func YamlParser(input []byte) (*Puzzle, error) {
puzzle := new(Puzzle)
err := yaml.Unmarshal(input, puzzle)
if err != nil {
return nil, err
}
return puzzle, nil
// Attachment carries information about an attached file.
type Attachment struct {
Filename string // Filename presented as part of puzzle
FilesystemPath string // Filename in backing FS (URL, mothball, or local FS)
Listed bool // Whether this file is listed as an attachment
}
func AttachmentParser(val []string) []Attachment {
func legacyAttachmentParser(val []string) []Attachment {
ret := make([]Attachment, len(val))
for idx, txt := range val {
parts := strings.SplitN(txt, " ", 3)
@ -77,62 +87,70 @@ func AttachmentParser(val []string) []Attachment {
return ret
}
func Rfc822Parser(input []byte) (*Puzzle, error) {
msgBytes := append(input, '\n')
r := bytes.NewReader(msgBytes)
func (p *Puzzle) yamlHeaderParser(r io.Reader) error {
decoder := yaml.NewDecoder(r)
decoder.SetStrict(true)
return decoder.Decode(p)
}
func (p *Puzzle) rfc822HeaderParser(r io.Reader) error {
m, err := mail.ReadMessage(r)
if err != nil {
return nil, err
return fmt.Errorf("Parsing RFC822 headers: %v", err)
}
puzzle := new(Puzzle)
for key, val := range m.Header {
key = strings.ToLower(key)
switch key {
case "author":
puzzle.Pre.Authors = val
p.Pre.Authors = val
case "pattern":
puzzle.Pre.AnswerPattern = val[0]
p.Pre.AnswerPattern = val[0]
case "script":
puzzle.Pre.Scripts = AttachmentParser(val)
p.Pre.Scripts = legacyAttachmentParser(val)
case "file":
puzzle.Pre.Attachments = AttachmentParser(val)
p.Pre.Attachments = legacyAttachmentParser(val)
case "answer":
puzzle.Answers = val
p.Answers = val
case "summary":
puzzle.Debug.Summary = val[0]
p.Debug.Summary = val[0]
case "hint":
puzzle.Debug.Hints = val
p.Debug.Hints = val
case "ksa":
puzzle.Post.KSAs = val
p.Post.KSAs = val
default:
return nil, fmt.Errorf("Unknown header field: %s", key)
return fmt.Errorf("Unknown header field: %s", key)
}
}
return puzzle, nil
return nil
}
func ParseMoth(r io.Reader) (*Puzzle, error) {
headerEnd := ""
func (p *Puzzle) parseMoth() error {
r, err := p.fs.Open("puzzle.moth")
if err != nil {
return err
}
defer r.Close()
headerBuf := new(bytes.Buffer)
headerParser := Rfc822Parser
headerParser := p.rfc822HeaderParser
headerEnd := ""
scanner := bufio.NewScanner(r)
lineNo := 0
for scanner.Scan() {
line := scanner.Text()
lineNo += 1
lineNo++
if lineNo == 1 {
if line == "---" {
headerParser = YamlParser
headerParser = p.yamlHeaderParser
headerEnd = "---"
continue
} else {
headerParser = Rfc822Parser
}
}
if line == headerEnd {
headerBuf.WriteRune('\n')
break
}
headerBuf.WriteString(line)
@ -142,22 +160,55 @@ func ParseMoth(r io.Reader) (*Puzzle, error) {
bodyBuf := new(bytes.Buffer)
for scanner.Scan() {
line := scanner.Text()
lineNo += 1
lineNo++
bodyBuf.WriteString(line)
bodyBuf.WriteRune('\n')
}
puzzle, err := headerParser(headerBuf.Bytes())
if err != nil {
return nil, err
if err := headerParser(headerBuf); err != nil {
return err
}
// Markdownify the body
bodyB := blackfriday.Run(bodyBuf.Bytes())
if (puzzle.Pre.Body != "") && (len(bodyB) > 0) {
log.Print("Body specified in header; overwriting...")
if (p.Pre.Body != "") && (bodyBuf.Len() > 0) {
return fmt.Errorf("Puzzle body present in header and in moth body")
}
puzzle.Pre.Body = string(bodyB)
p.Pre.Body = string(blackfriday.Run(bodyBuf.Bytes()))
return puzzle, nil
return nil
}
func (p *Puzzle) mkpuzzle() error {
bfs, ok := p.fs.(*afero.BasePathFs)
if !ok {
return fmt.Errorf("Fs won't resolve real paths for %v", p)
}
mkpuzzlePath, err := bfs.RealPath("mkpuzzle")
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, mkpuzzlePath)
stdout, err := cmd.Output()
if err != nil {
return err
}
jsdec := json.NewDecoder(bytes.NewReader(stdout))
jsdec.DisallowUnknownFields()
puzzle := new(Puzzle)
if err := jsdec.Decode(puzzle); err != nil {
return err
}
return nil
}
// Open returns a newly-opened file.
func (p *Puzzle) Open(name string) (io.ReadCloser, error) {
// BUG(neale): You cannot open generated files in puzzles, only files actually on the disk
return p.fs.Open(name)
}

View File

@ -0,0 +1,45 @@
package main
import (
"testing"
"github.com/spf13/afero"
)
func TestPuzzle(t *testing.T) {
puzzleFs := newTestFs()
catFs := afero.NewBasePathFs(puzzleFs, "cat0")
p1, err := NewPuzzle(catFs, 1)
if err != nil {
t.Error(err)
}
t.Log(p1)
if (len(p1.Answers) == 0) || (p1.Answers[0] != "YAML answer") {
t.Error("Answers are wrong", p1.Answers)
}
if (len(p1.Pre.Authors) != 3) || (p1.Pre.Authors[1] != "Buster") {
t.Error("Authors are wrong", p1.Pre.Authors)
}
if p1.Pre.Body != "<p>YAML body</p>\n" {
t.Errorf("Body parsed wrong: %#v", p1.Pre.Body)
}
p2, err := NewPuzzle(catFs, 2)
if err != nil {
t.Error(err)
}
if (len(p2.Answers) == 0) || (p2.Answers[0] != "RFC822 answer") {
t.Error("Answers are wrong", p2.Answers)
}
if (len(p2.Pre.Authors) != 3) || (p2.Pre.Authors[1] != "Arthur") {
t.Error("Authors are wrong", p2.Pre.Authors)
}
if p2.Pre.Body != "<p>RFC822 body</p>\n" {
t.Errorf("Body parsed wrong: %#v", p2.Pre.Body)
}
if _, err := NewPuzzle(catFs, 10); err == nil {
t.Error("Broken YAML didn't trigger an error")
}
}